| File: | root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c |
| Warning: | line 1179, column 29 Division by zero |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
| 1 | #define _CRT_SECURE_NO_DEPRECATE // Disables "unsafe" warnings on Windows | |||
| 2 | #define _USE_MATH_DEFINES // For M_PI on MSVC | |||
| 3 | ||||
| 4 | #include "ggml-backend-impl.h" | |||
| 5 | #include "ggml-backend.h" | |||
| 6 | #include "traits.h" | |||
| 7 | #include "ggml-cpu-impl.h" | |||
| 8 | #include "ggml-impl.h" | |||
| 9 | #include "quants.h" | |||
| 10 | #include "ggml-threading.h" | |||
| 11 | #include "unary-ops.h" | |||
| 12 | #include "binary-ops.h" | |||
| 13 | #include "vec.h" | |||
| 14 | #include "ops.h" | |||
| 15 | #include "ggml.h" | |||
| 16 | #include "common.h" | |||
| 17 | ||||
| 18 | #if defined(_MSC_VER) || defined(__MINGW32__) | |||
| 19 | #include <malloc.h> // using malloc.h with MSC/MINGW | |||
| 20 | #elif !defined(__FreeBSD__) && !defined(__NetBSD__) && !defined(__OpenBSD__) | |||
| 21 | #include <alloca.h> | |||
| 22 | #endif | |||
| 23 | ||||
| 24 | #include <assert.h> | |||
| 25 | #include <errno(*__errno_location ()).h> | |||
| 26 | #include <time.h> | |||
| 27 | #include <math.h> | |||
| 28 | #include <stdlib.h> | |||
| 29 | #include <string.h> | |||
| 30 | #include <stdint.h> | |||
| 31 | #include <inttypes.h> | |||
| 32 | #include <stdio.h> | |||
| 33 | #include <float.h> | |||
| 34 | #include <limits.h> | |||
| 35 | #include <stdarg.h> | |||
| 36 | #include <signal.h> | |||
| 37 | #if defined(__gnu_linux__1) | |||
| 38 | #include <syscall.h> | |||
| 39 | #endif | |||
| 40 | ||||
| 41 | #ifdef GGML_USE_OPENMP | |||
| 42 | #include <omp.h> | |||
| 43 | #endif | |||
| 44 | ||||
| 45 | #if defined(__ARM_FEATURE_SVE) || defined(__ARM_FEATURE_MATMUL_INT8) | |||
| 46 | #undef GGML_USE_LLAMAFILE | |||
| 47 | #endif | |||
| 48 | ||||
| 49 | #ifdef GGML_USE_LLAMAFILE | |||
| 50 | #include "llamafile/sgemm.h" | |||
| 51 | #endif | |||
| 52 | ||||
| 53 | #ifdef GGML_USE_CPU_RISCV64_SPACEMIT | |||
| 54 | # include "spacemit/ime.h" | |||
| 55 | #endif | |||
| 56 | ||||
| 57 | // Note: once we move threading into a separate C++ file | |||
| 58 | // will use std::hardware_destructive_interference_size instead of hardcoding it here | |||
| 59 | // and we'll use C++ attribute syntax. | |||
| 60 | #define GGML_CACHE_LINE64 64 | |||
| 61 | ||||
| 62 | #if defined(__clang__1) || defined(__GNUC__4) | |||
| 63 | #define GGML_CACHE_ALIGN__attribute__((aligned(64))) __attribute__((aligned(GGML_CACHE_LINE64))) | |||
| 64 | #endif | |||
| 65 | ||||
| 66 | #if defined(__has_feature)0 | |||
| 67 | #if __has_feature(thread_sanitizer)0 | |||
| 68 | #define GGML_TSAN_ENABLED 1 | |||
| 69 | #endif | |||
| 70 | #else // __has_feature | |||
| 71 | #if defined(__SANITIZE_THREAD__) | |||
| 72 | #define GGML_TSAN_ENABLED 1 | |||
| 73 | #endif | |||
| 74 | #endif // __has_feature | |||
| 75 | ||||
| 76 | #define UNUSEDGGML_UNUSED GGML_UNUSED | |||
| 77 | #define SWAP(x, y, T)do { T SWAP = x; (x) = y; (y) = SWAP; } while (0) do { T SWAP = x; (x) = y; (y) = SWAP; } while (0) | |||
| 78 | ||||
| 79 | // precomputed f32 table for f16 (256 KB) (simd-mappings.h) | |||
| 80 | float ggml_table_f32_f16[1 << 16]; | |||
| 81 | ||||
| 82 | // precomputed f32 table for e8m0 half (1 KB) (simd-mappings.h) | |||
| 83 | float ggml_table_f32_e8m0_half[1 << 8]; | |||
| 84 | ||||
| 85 | #if defined(__ARM_ARCH) | |||
| 86 | struct ggml_arm_arch_features_type { | |||
| 87 | int sve_cnt; | |||
| 88 | } ggml_arm_arch_features = { 0 }; | |||
| 89 | #endif | |||
| 90 | ||||
| 91 | #if defined(__riscv) | |||
| 92 | struct ggml_riscv_arch_features_type { | |||
| 93 | int rvv_vlen; | |||
| 94 | } ggml_riscv_arch_features = { 0 }; | |||
| 95 | #endif | |||
| 96 | ||||
| 97 | #if defined(_WIN32) | |||
| 98 | ||||
| 99 | #define WIN32_LEAN_AND_MEAN | |||
| 100 | #ifndef NOMINMAX | |||
| 101 | #define NOMINMAX | |||
| 102 | #endif | |||
| 103 | #include <windows.h> | |||
| 104 | ||||
| 105 | #if defined(_MSC_VER) && !defined(__clang__1) | |||
| 106 | #define GGML_CACHE_ALIGN__attribute__((aligned(64))) __declspec(align(GGML_CACHE_LINE64)) | |||
| 107 | ||||
| 108 | typedef volatile LONG atomic_int; | |||
| 109 | typedef atomic_int atomic_bool; | |||
| 110 | typedef atomic_int atomic_flag; | |||
| 111 | ||||
| 112 | #define ATOMIC_FLAG_INIT{ 0 } 0 | |||
| 113 | ||||
| 114 | typedef enum { | |||
| 115 | memory_order_relaxed, | |||
| 116 | memory_order_consume, | |||
| 117 | memory_order_acquire, | |||
| 118 | memory_order_release, | |||
| 119 | memory_order_acq_rel, | |||
| 120 | memory_order_seq_cst | |||
| 121 | } memory_order; | |||
| 122 | ||||
| 123 | static void atomic_store(atomic_int * ptr, LONG val)__c11_atomic_store(atomic_int * ptr, LONG val, 5) { | |||
| 124 | InterlockedExchange(ptr, val); | |||
| 125 | } | |||
| 126 | static void atomic_store_explicit__c11_atomic_store(atomic_int * ptr, LONG val, memory_order mo) { | |||
| 127 | // TODO: add support for explicit memory order | |||
| 128 | InterlockedExchange(ptr, val); | |||
| 129 | } | |||
| 130 | static LONG atomic_load(atomic_int * ptr)__c11_atomic_load(atomic_int * ptr, 5) { | |||
| 131 | return InterlockedCompareExchange(ptr, 0, 0); | |||
| 132 | } | |||
| 133 | static LONG atomic_load_explicit__c11_atomic_load(atomic_int * ptr, memory_order mo) { | |||
| 134 | // TODO: add support for explicit memory order | |||
| 135 | return InterlockedCompareExchange(ptr, 0, 0); | |||
| 136 | } | |||
| 137 | static LONG atomic_fetch_add(atomic_int * ptr, LONG inc)__c11_atomic_fetch_add(atomic_int * ptr, LONG inc, 5) { | |||
| 138 | return InterlockedExchangeAdd(ptr, inc); | |||
| 139 | } | |||
| 140 | static LONG atomic_fetch_add_explicit__c11_atomic_fetch_add(atomic_int * ptr, LONG inc, memory_order mo) { | |||
| 141 | // TODO: add support for explicit memory order | |||
| 142 | return InterlockedExchangeAdd(ptr, inc); | |||
| 143 | } | |||
| 144 | static atomic_bool atomic_flag_test_and_set(atomic_flag * ptr)__c11_atomic_exchange(&(atomic_flag * ptr)->_Value, 1, 5) { | |||
| 145 | return InterlockedExchange(ptr, 1); | |||
| 146 | } | |||
| 147 | static void atomic_flag_clear(atomic_flag * ptr)__c11_atomic_store(&(atomic_flag * ptr)->_Value, 0, 5) { | |||
| 148 | InterlockedExchange(ptr, 0); | |||
| 149 | } | |||
| 150 | static void atomic_thread_fence(memory_order mo)__c11_atomic_thread_fence(memory_order mo) { | |||
| 151 | MemoryBarrier(); | |||
| 152 | } | |||
| 153 | #else // clang | |||
| 154 | #include <stdatomic.h> | |||
| 155 | #endif | |||
| 156 | ||||
| 157 | typedef HANDLE pthread_t; | |||
| 158 | ||||
| 159 | typedef DWORD thread_ret_t; | |||
| 160 | static int pthread_create(pthread_t * out, void * unused, thread_ret_t(*func)(void *), void * arg) { | |||
| 161 | (void) unused; | |||
| 162 | HANDLE handle = CreateThread(NULL((void*)0), 0, (LPTHREAD_START_ROUTINE) func, arg, 0, NULL((void*)0)); | |||
| 163 | if (handle == NULL((void*)0)) | |||
| 164 | { | |||
| 165 | return EAGAIN11; | |||
| 166 | } | |||
| 167 | ||||
| 168 | *out = handle; | |||
| 169 | return 0; | |||
| 170 | } | |||
| 171 | ||||
| 172 | static int pthread_join(pthread_t thread, void * unused) { | |||
| 173 | (void) unused; | |||
| 174 | int ret = (int) WaitForSingleObject(thread, INFINITE); | |||
| 175 | CloseHandle(thread); | |||
| 176 | return ret; | |||
| 177 | } | |||
| 178 | ||||
| 179 | static int sched_yield (void) { | |||
| 180 | Sleep (0); | |||
| 181 | return 0; | |||
| 182 | } | |||
| 183 | #else | |||
| 184 | ||||
| 185 | #include <pthread.h> | |||
| 186 | #include <stdatomic.h> | |||
| 187 | #include <sched.h> | |||
| 188 | #if defined(__FreeBSD__) | |||
| 189 | #include <pthread_np.h> | |||
| 190 | #endif | |||
| 191 | ||||
| 192 | typedef void * thread_ret_t; | |||
| 193 | ||||
| 194 | #include <sys/types.h> | |||
| 195 | #include <sys/stat.h> | |||
| 196 | #include <unistd.h> | |||
| 197 | ||||
| 198 | #endif | |||
| 199 | ||||
| 200 | typedef pthread_t ggml_thread_t; | |||
| 201 | ||||
| 202 | #define GGML_THREADPOOL_N_THREADS_MASK(0xffffU) (0xffffU) | |||
| 203 | #define GGML_THREADPOOL_N_THREADS_BITS(16) (16) | |||
| 204 | ||||
| 205 | #if defined(__APPLE__) | |||
| 206 | #include <unistd.h> | |||
| 207 | #include <mach/mach.h> | |||
| 208 | #include <TargetConditionals.h> | |||
| 209 | #endif | |||
| 210 | ||||
| 211 | static const struct ggml_type_traits_cpu type_traits_cpu[GGML_TYPE_COUNT] = { | |||
| 212 | [GGML_TYPE_F32] = { | |||
| 213 | .from_float = (ggml_from_float_t) ggml_cpu_fp32_to_fp32, | |||
| 214 | .vec_dot = (ggml_vec_dot_t) ggml_vec_dot_f32, | |||
| 215 | .vec_dot_type = GGML_TYPE_F32, | |||
| 216 | .nrows = 1, | |||
| 217 | }, | |||
| 218 | [GGML_TYPE_F16] = { | |||
| 219 | .from_float = (ggml_from_float_t) ggml_cpu_fp32_to_fp16, | |||
| 220 | .vec_dot = (ggml_vec_dot_t) ggml_vec_dot_f16, | |||
| 221 | .vec_dot_type = GGML_TYPE_F16, | |||
| 222 | .nrows = 1, | |||
| 223 | }, | |||
| 224 | [GGML_TYPE_Q1_0] = { | |||
| 225 | .from_float = quantize_row_q1_0, | |||
| 226 | .vec_dot = ggml_vec_dot_q1_0_q8_0, | |||
| 227 | .vec_dot_type = GGML_TYPE_Q8_0, | |||
| 228 | .nrows = 1, | |||
| 229 | }, | |||
| 230 | [GGML_TYPE_Q4_0] = { | |||
| 231 | .from_float = quantize_row_q4_0, | |||
| 232 | .vec_dot = ggml_vec_dot_q4_0_q8_0, | |||
| 233 | .vec_dot_type = GGML_TYPE_Q8_0, | |||
| 234 | #if defined (__ARM_FEATURE_MATMUL_INT8) | |||
| 235 | .nrows = 2, | |||
| 236 | #else | |||
| 237 | .nrows = 1, | |||
| 238 | #endif | |||
| 239 | }, | |||
| 240 | [GGML_TYPE_Q4_1] = { | |||
| 241 | .from_float = quantize_row_q4_1, | |||
| 242 | .vec_dot = ggml_vec_dot_q4_1_q8_1, | |||
| 243 | .vec_dot_type = GGML_TYPE_Q8_1, | |||
| 244 | #if defined (__ARM_FEATURE_MATMUL_INT8) | |||
| 245 | .nrows = 2, | |||
| 246 | #else | |||
| 247 | .nrows = 1, | |||
| 248 | #endif | |||
| 249 | }, | |||
| 250 | [GGML_TYPE_Q5_0] = { | |||
| 251 | .from_float = quantize_row_q5_0, | |||
| 252 | .vec_dot = ggml_vec_dot_q5_0_q8_0, | |||
| 253 | .vec_dot_type = GGML_TYPE_Q8_0, | |||
| 254 | .nrows = 1, | |||
| 255 | }, | |||
| 256 | [GGML_TYPE_Q5_1] = { | |||
| 257 | .from_float = quantize_row_q5_1, | |||
| 258 | .vec_dot = ggml_vec_dot_q5_1_q8_1, | |||
| 259 | .vec_dot_type = GGML_TYPE_Q8_1, | |||
| 260 | .nrows = 1, | |||
| 261 | }, | |||
| 262 | [GGML_TYPE_Q8_0] = { | |||
| 263 | .from_float = quantize_row_q8_0, | |||
| 264 | .vec_dot = ggml_vec_dot_q8_0_q8_0, | |||
| 265 | .vec_dot_type = GGML_TYPE_Q8_0, | |||
| 266 | #if defined (__ARM_FEATURE_MATMUL_INT8) | |||
| 267 | .nrows = 2, | |||
| 268 | #else | |||
| 269 | .nrows = 1, | |||
| 270 | #endif | |||
| 271 | }, | |||
| 272 | [GGML_TYPE_Q8_1] = { | |||
| 273 | .from_float = quantize_row_q8_1, | |||
| 274 | .vec_dot_type = GGML_TYPE_Q8_1, | |||
| 275 | .nrows = 1, | |||
| 276 | }, | |||
| 277 | [GGML_TYPE_MXFP4] = { | |||
| 278 | .from_float = quantize_row_mxfp4, | |||
| 279 | .vec_dot = ggml_vec_dot_mxfp4_q8_0, | |||
| 280 | .vec_dot_type = GGML_TYPE_Q8_0, | |||
| 281 | .nrows = 1, | |||
| 282 | }, | |||
| 283 | [GGML_TYPE_NVFP4] = { | |||
| 284 | .from_float = quantize_row_nvfp4, | |||
| 285 | .vec_dot = ggml_vec_dot_nvfp4_q8_0, | |||
| 286 | .vec_dot_type = GGML_TYPE_Q8_0, | |||
| 287 | .nrows = 1, | |||
| 288 | }, | |||
| 289 | [GGML_TYPE_Q2_K] = { | |||
| 290 | .from_float = quantize_row_q2_K, | |||
| 291 | .vec_dot = ggml_vec_dot_q2_K_q8_K, | |||
| 292 | .vec_dot_type = GGML_TYPE_Q8_K, | |||
| 293 | .nrows = 1, | |||
| 294 | }, | |||
| 295 | [GGML_TYPE_Q3_K] = { | |||
| 296 | .from_float = quantize_row_q3_K, | |||
| 297 | .vec_dot = ggml_vec_dot_q3_K_q8_K, | |||
| 298 | .vec_dot_type = GGML_TYPE_Q8_K, | |||
| 299 | .nrows = 1, | |||
| 300 | }, | |||
| 301 | [GGML_TYPE_Q4_K] = { | |||
| 302 | .from_float = quantize_row_q4_K, | |||
| 303 | .vec_dot = ggml_vec_dot_q4_K_q8_K, | |||
| 304 | .vec_dot_type = GGML_TYPE_Q8_K, | |||
| 305 | #if defined (__ARM_FEATURE_MATMUL_INT8) | |||
| 306 | .nrows = 2, | |||
| 307 | #else | |||
| 308 | .nrows = 1, | |||
| 309 | #endif | |||
| 310 | }, | |||
| 311 | [GGML_TYPE_Q5_K] = { | |||
| 312 | .from_float = quantize_row_q5_K, | |||
| 313 | .vec_dot = ggml_vec_dot_q5_K_q8_K, | |||
| 314 | .vec_dot_type = GGML_TYPE_Q8_K, | |||
| 315 | .nrows = 1, | |||
| 316 | }, | |||
| 317 | [GGML_TYPE_Q6_K] = { | |||
| 318 | .from_float = quantize_row_q6_K, | |||
| 319 | .vec_dot = ggml_vec_dot_q6_K_q8_K, | |||
| 320 | .vec_dot_type = GGML_TYPE_Q8_K, | |||
| 321 | #if defined (__ARM_FEATURE_MATMUL_INT8) | |||
| 322 | .nrows = 2, | |||
| 323 | #else | |||
| 324 | .nrows = 1, | |||
| 325 | #endif | |||
| 326 | }, | |||
| 327 | [GGML_TYPE_IQ2_XXS] = { | |||
| 328 | .from_float = NULL((void*)0), | |||
| 329 | .vec_dot = ggml_vec_dot_iq2_xxs_q8_K, | |||
| 330 | .vec_dot_type = GGML_TYPE_Q8_K, | |||
| 331 | .nrows = 1, | |||
| 332 | }, | |||
| 333 | [GGML_TYPE_IQ2_XS] = { | |||
| 334 | .from_float = NULL((void*)0), | |||
| 335 | .vec_dot = ggml_vec_dot_iq2_xs_q8_K, | |||
| 336 | .vec_dot_type = GGML_TYPE_Q8_K, | |||
| 337 | .nrows = 1, | |||
| 338 | }, | |||
| 339 | [GGML_TYPE_IQ3_XXS] = { | |||
| 340 | // NOTE: from_float for iq3 and iq2_s was removed because these quants require initialization in ggml_quantize_init | |||
| 341 | //.from_float = quantize_row_iq3_xxs, | |||
| 342 | .vec_dot = ggml_vec_dot_iq3_xxs_q8_K, | |||
| 343 | .vec_dot_type = GGML_TYPE_Q8_K, | |||
| 344 | .nrows = 1, | |||
| 345 | }, | |||
| 346 | [GGML_TYPE_IQ3_S] = { | |||
| 347 | //.from_float = quantize_row_iq3_s, | |||
| 348 | .vec_dot = ggml_vec_dot_iq3_s_q8_K, | |||
| 349 | .vec_dot_type = GGML_TYPE_Q8_K, | |||
| 350 | .nrows = 1, | |||
| 351 | }, | |||
| 352 | [GGML_TYPE_IQ2_S] = { | |||
| 353 | //.from_float = quantize_row_iq2_s, | |||
| 354 | .vec_dot = ggml_vec_dot_iq2_s_q8_K, | |||
| 355 | .vec_dot_type = GGML_TYPE_Q8_K, | |||
| 356 | .nrows = 1, | |||
| 357 | }, | |||
| 358 | [GGML_TYPE_IQ1_S] = { | |||
| 359 | .from_float = NULL((void*)0), | |||
| 360 | .vec_dot = ggml_vec_dot_iq1_s_q8_K, | |||
| 361 | .vec_dot_type = GGML_TYPE_Q8_K, | |||
| 362 | .nrows = 1, | |||
| 363 | }, | |||
| 364 | [GGML_TYPE_IQ1_M] = { | |||
| 365 | .from_float = NULL((void*)0), | |||
| 366 | .vec_dot = ggml_vec_dot_iq1_m_q8_K, | |||
| 367 | .vec_dot_type = GGML_TYPE_Q8_K, | |||
| 368 | .nrows = 1, | |||
| 369 | }, | |||
| 370 | [GGML_TYPE_IQ4_NL] = { | |||
| 371 | .from_float = quantize_row_iq4_nl, | |||
| 372 | .vec_dot = ggml_vec_dot_iq4_nl_q8_0, | |||
| 373 | .vec_dot_type = GGML_TYPE_Q8_0, | |||
| 374 | .nrows = 1, | |||
| 375 | }, | |||
| 376 | [GGML_TYPE_IQ4_XS] = { | |||
| 377 | .from_float = quantize_row_iq4_xs, | |||
| 378 | .vec_dot = ggml_vec_dot_iq4_xs_q8_K, | |||
| 379 | .vec_dot_type = GGML_TYPE_Q8_K, | |||
| 380 | .nrows = 1, | |||
| 381 | }, | |||
| 382 | [GGML_TYPE_Q8_K] = { | |||
| 383 | .from_float = quantize_row_q8_K, | |||
| 384 | }, | |||
| 385 | [GGML_TYPE_BF16] = { | |||
| 386 | .from_float = (ggml_from_float_t) ggml_cpu_fp32_to_bf16, | |||
| 387 | .vec_dot = (ggml_vec_dot_t) ggml_vec_dot_bf16, | |||
| 388 | .vec_dot_type = GGML_TYPE_BF16, | |||
| 389 | .nrows = 1, | |||
| 390 | }, | |||
| 391 | [GGML_TYPE_TQ1_0] = { | |||
| 392 | .from_float = quantize_row_tq1_0, | |||
| 393 | .vec_dot = ggml_vec_dot_tq1_0_q8_K, | |||
| 394 | .vec_dot_type = GGML_TYPE_Q8_K, | |||
| 395 | .nrows = 1, | |||
| 396 | }, | |||
| 397 | [GGML_TYPE_TQ2_0] = { | |||
| 398 | .from_float = quantize_row_tq2_0, | |||
| 399 | .vec_dot = ggml_vec_dot_tq2_0_q8_K, | |||
| 400 | .vec_dot_type = GGML_TYPE_Q8_K, | |||
| 401 | .nrows = 1, | |||
| 402 | }, | |||
| 403 | [GGML_TYPE_I32] = { | |||
| 404 | .from_float = (ggml_from_float_t) ggml_cpu_fp32_to_i32, | |||
| 405 | }, | |||
| 406 | }; | |||
| 407 | ||||
| 408 | const struct ggml_type_traits_cpu * ggml_get_type_traits_cpu(enum ggml_type type) { | |||
| 409 | return &type_traits_cpu[type]; | |||
| 410 | } | |||
| 411 | ||||
| 412 | // | |||
| 413 | // Threading defs | |||
| 414 | // | |||
| 415 | ||||
| 416 | typedef pthread_t ggml_thread_t; | |||
| 417 | ||||
| 418 | #if defined(_WIN32) | |||
| 419 | ||||
| 420 | typedef CONDITION_VARIABLE ggml_cond_t; | |||
| 421 | typedef SRWLOCK ggml_mutex_t; | |||
| 422 | ||||
| 423 | #define ggml_mutex_init(m)pthread_mutex_init(m, ((void*)0)) InitializeSRWLock(m) | |||
| 424 | #define ggml_mutex_destroy(m)pthread_mutex_destroy(m) | |||
| 425 | #define ggml_mutex_lock(m)pthread_mutex_lock(m) AcquireSRWLockExclusive(m) | |||
| 426 | #define ggml_mutex_unlock(m)pthread_mutex_unlock(m) ReleaseSRWLockExclusive(m) | |||
| 427 | #define ggml_mutex_lock_shared(m)pthread_mutex_lock(m) AcquireSRWLockShared(m) | |||
| 428 | #define ggml_mutex_unlock_shared(m)pthread_mutex_unlock(m) ReleaseSRWLockShared(m) | |||
| 429 | ||||
| 430 | #define ggml_cond_init(c)pthread_cond_init(c, ((void*)0)) InitializeConditionVariable(c) | |||
| 431 | #define ggml_cond_destroy(c)pthread_cond_destroy(c) | |||
| 432 | #define ggml_cond_wait(c, m)pthread_cond_wait(c, m) SleepConditionVariableSRW(c, m, INFINITE, CONDITION_VARIABLE_LOCKMODE_SHARED) | |||
| 433 | #define ggml_cond_broadcast(c)pthread_cond_broadcast(c) WakeAllConditionVariable(c) | |||
| 434 | ||||
| 435 | #define ggml_thread_createpthread_create pthread_create | |||
| 436 | #define ggml_thread_joinpthread_join pthread_join | |||
| 437 | ||||
| 438 | #else | |||
| 439 | ||||
| 440 | typedef pthread_cond_t ggml_cond_t; | |||
| 441 | typedef pthread_mutex_t ggml_mutex_t; | |||
| 442 | ||||
| 443 | #define ggml_mutex_init(m)pthread_mutex_init(m, ((void*)0)) pthread_mutex_init(m, NULL((void*)0)) | |||
| 444 | #define ggml_mutex_destroy(m)pthread_mutex_destroy(m) pthread_mutex_destroy(m) | |||
| 445 | #define ggml_mutex_lock(m)pthread_mutex_lock(m) pthread_mutex_lock(m) | |||
| 446 | #define ggml_mutex_unlock(m)pthread_mutex_unlock(m) pthread_mutex_unlock(m) | |||
| 447 | #define ggml_mutex_lock_shared(m)pthread_mutex_lock(m) pthread_mutex_lock(m) | |||
| 448 | #define ggml_mutex_unlock_shared(m)pthread_mutex_unlock(m) pthread_mutex_unlock(m) | |||
| 449 | ||||
| 450 | #define ggml_lock_init(x)(void)(x) UNUSED(x)(void)(x) | |||
| 451 | #define ggml_lock_destroy(x)(void)(x) UNUSED(x)(void)(x) | |||
| 452 | #if defined(__x86_64__1) || (defined(_MSC_VER) && defined(_M_AMD64)) | |||
| 453 | #define ggml_lock_lock(x)_mm_pause() _mm_pause() | |||
| 454 | #else | |||
| 455 | #define ggml_lock_lock(x)_mm_pause() UNUSED(x)(void)(x) | |||
| 456 | #endif | |||
| 457 | #define ggml_lock_unlock(x)(void)(x) UNUSED(x)(void)(x) | |||
| 458 | ||||
| 459 | #define GGML_LOCK_INITIALIZER0 0 | |||
| 460 | #define ggml_cond_init(c)pthread_cond_init(c, ((void*)0)) pthread_cond_init(c, NULL((void*)0)) | |||
| 461 | #define ggml_cond_destroy(c)pthread_cond_destroy(c) pthread_cond_destroy(c) | |||
| 462 | #define ggml_cond_wait(c, m)pthread_cond_wait(c, m) pthread_cond_wait(c, m) | |||
| 463 | #define ggml_cond_broadcast(c)pthread_cond_broadcast(c) pthread_cond_broadcast(c) | |||
| 464 | ||||
| 465 | #define ggml_thread_createpthread_create pthread_create | |||
| 466 | #define ggml_thread_joinpthread_join pthread_join | |||
| 467 | ||||
| 468 | #endif | |||
| 469 | ||||
| 470 | // Threadpool def | |||
| 471 | struct ggml_threadpool { | |||
| 472 | ggml_mutex_t mutex; // mutex for cond.var | |||
| 473 | ggml_cond_t cond; // cond.var for waiting for new work | |||
| 474 | ||||
| 475 | struct ggml_cgraph * cgraph; | |||
| 476 | struct ggml_cplan * cplan; | |||
| 477 | ||||
| 478 | // synchronization primitives | |||
| 479 | atomic_int n_graph; // updated when there is work to be done (i.e each graph) holds graph and active thread counts. | |||
| 480 | atomic_int GGML_CACHE_ALIGN__attribute__((aligned(64))) n_barrier; | |||
| 481 | atomic_int GGML_CACHE_ALIGN__attribute__((aligned(64))) n_barrier_passed; | |||
| 482 | atomic_int GGML_CACHE_ALIGN__attribute__((aligned(64))) current_chunk; // currently processing chunk during Mat_Mul, shared between all the threads. | |||
| 483 | ||||
| 484 | // these are atomic as an annotation for thread-sanitizer | |||
| 485 | atomic_bool stop; // Used for stopping the threadpool altogether | |||
| 486 | atomic_bool pause; // Used for pausing the threadpool or individual threads | |||
| 487 | atomic_int abort; // Used for aborting processing of a graph | |||
| 488 | ||||
| 489 | struct ggml_compute_state * workers; // per thread state | |||
| 490 | int n_threads; // Number of threads in the pool | |||
| 491 | int32_t prio; // Scheduling priority | |||
| 492 | uint32_t poll; // Polling level (0 - no polling) | |||
| 493 | ||||
| 494 | void (*thread_create_callback)(void); | |||
| 495 | void (*thread_destroy_callback)(void); | |||
| 496 | ||||
| 497 | enum ggml_status ec; | |||
| 498 | }; | |||
| 499 | ||||
| 500 | // Per-thread state | |||
| 501 | struct ggml_compute_state { | |||
| 502 | #ifndef GGML_USE_OPENMP | |||
| 503 | ggml_thread_t thrd; | |||
| 504 | int last_graph; | |||
| 505 | bool_Bool pending; | |||
| 506 | #endif | |||
| 507 | bool_Bool cpumask[GGML_MAX_N_THREADS512]; | |||
| 508 | struct ggml_threadpool * threadpool; | |||
| 509 | int ith; | |||
| 510 | }; | |||
| 511 | ||||
| 512 | // Helpers for polling loops | |||
| 513 | #if defined(__aarch64__) && ( defined(__clang__1) || defined(__GNUC__4) ) | |||
| 514 | static inline void ggml_thread_cpu_relax(void) { | |||
| 515 | __asm__ volatile("yield" ::: "memory"); | |||
| 516 | } | |||
| 517 | #elif defined(__x86_64__1) | |||
| 518 | static inline void ggml_thread_cpu_relax(void) { | |||
| 519 | _mm_pause(); | |||
| 520 | } | |||
| 521 | #elif defined(__riscv) | |||
| 522 | static inline void ggml_thread_cpu_relax(void) { | |||
| 523 | #ifdef __riscv_zihintpause | |||
| 524 | __asm__ __volatile__ ("pause"); | |||
| 525 | #else | |||
| 526 | /* Encoding of the pause instruction */ | |||
| 527 | __asm__ __volatile__ (".4byte 0x100000F"); | |||
| 528 | #endif | |||
| 529 | } | |||
| 530 | #else | |||
| 531 | static inline void ggml_thread_cpu_relax(void) {;} | |||
| 532 | #endif | |||
| 533 | ||||
| 534 | // | |||
| 535 | // NUMA support | |||
| 536 | // | |||
| 537 | ||||
| 538 | #define GGML_NUMA_MAX_NODES8 8 | |||
| 539 | #define GGML_NUMA_MAX_CPUS512 512 | |||
| 540 | ||||
| 541 | struct ggml_numa_node { | |||
| 542 | uint32_t cpus[GGML_NUMA_MAX_CPUS512]; // hardware threads on this node | |||
| 543 | uint32_t n_cpus; | |||
| 544 | }; | |||
| 545 | ||||
| 546 | struct ggml_numa_nodes { | |||
| 547 | enum ggml_numa_strategy numa_strategy; | |||
| 548 | struct ggml_numa_node nodes[GGML_NUMA_MAX_NODES8]; | |||
| 549 | uint32_t n_nodes; | |||
| 550 | uint32_t total_cpus; // hardware threads on system | |||
| 551 | uint32_t current_node; // node on which main process is execting | |||
| 552 | #if defined(__gnu_linux__1) | |||
| 553 | cpu_set_t cpuset; // cpuset from numactl | |||
| 554 | #else | |||
| 555 | uint32_t cpuset; // no NUMA support outside of Linux at this time. Use a portable datatype | |||
| 556 | #endif | |||
| 557 | }; | |||
| 558 | ||||
| 559 | // | |||
| 560 | // ggml state | |||
| 561 | // | |||
| 562 | ||||
| 563 | struct ggml_state { | |||
| 564 | struct ggml_numa_nodes numa; | |||
| 565 | }; | |||
| 566 | ||||
| 567 | static struct ggml_state g_state = {0}; | |||
| 568 | ||||
| 569 | void ggml_barrier(struct ggml_threadpool * tp) { | |||
| 570 | int n_threads = atomic_load_explicit__c11_atomic_load(&tp->n_graph, memory_order_relaxed) & GGML_THREADPOOL_N_THREADS_MASK(0xffffU); | |||
| 571 | if (n_threads == 1) { | |||
| 572 | return; | |||
| 573 | } | |||
| 574 | ||||
| 575 | #ifdef GGML_USE_OPENMP | |||
| 576 | #pragma omp barrier | |||
| 577 | #else | |||
| 578 | int n_passed = atomic_load_explicit__c11_atomic_load(&tp->n_barrier_passed, memory_order_relaxed); | |||
| 579 | ||||
| 580 | // enter barrier (full seq-cst fence) | |||
| 581 | int n_barrier = atomic_fetch_add_explicit__c11_atomic_fetch_add(&tp->n_barrier, 1, memory_order_seq_cst); | |||
| 582 | ||||
| 583 | if (n_barrier == (n_threads - 1)) { | |||
| 584 | // last thread | |||
| 585 | atomic_store_explicit__c11_atomic_store(&tp->n_barrier, 0, memory_order_relaxed); | |||
| 586 | ||||
| 587 | // exit barrier (full seq-cst fence) | |||
| 588 | atomic_fetch_add_explicit__c11_atomic_fetch_add(&tp->n_barrier_passed, 1, memory_order_seq_cst); | |||
| 589 | return; | |||
| 590 | } | |||
| 591 | ||||
| 592 | // wait for other threads | |||
| 593 | while (atomic_load_explicit__c11_atomic_load(&tp->n_barrier_passed, memory_order_relaxed) == n_passed) { | |||
| 594 | ggml_thread_cpu_relax(); | |||
| 595 | } | |||
| 596 | ||||
| 597 | // exit barrier (full seq-cst fence) | |||
| 598 | // TSAN doesn't support standalone fence yet, we use a dummy read-modify-write instead | |||
| 599 | #ifdef GGML_TSAN_ENABLED | |||
| 600 | atomic_fetch_add_explicit__c11_atomic_fetch_add(&tp->n_barrier_passed, 0, memory_order_seq_cst); | |||
| 601 | #else | |||
| 602 | atomic_thread_fence(memory_order_seq_cst)__c11_atomic_thread_fence(memory_order_seq_cst); | |||
| 603 | #endif | |||
| 604 | #endif | |||
| 605 | } | |||
| 606 | ||||
| 607 | void ggml_threadpool_chunk_set(struct ggml_threadpool * tp, int value) { | |||
| 608 | atomic_store_explicit__c11_atomic_store(&tp->current_chunk, value, memory_order_relaxed); | |||
| 609 | } | |||
| 610 | ||||
| 611 | int ggml_threadpool_chunk_add(struct ggml_threadpool * tp, int value) { | |||
| 612 | return atomic_fetch_add_explicit__c11_atomic_fetch_add(&tp->current_chunk, value, memory_order_relaxed); | |||
| 613 | } | |||
| 614 | ||||
| 615 | #if defined(__gnu_linux__1) | |||
| 616 | static cpu_set_t ggml_get_numa_affinity(void) { | |||
| 617 | cpu_set_t cpuset; | |||
| 618 | pthread_t thread; | |||
| 619 | thread = pthread_self(); | |||
| 620 | CPU_ZERO(&cpuset)do __builtin_memset (&cpuset, '\0', sizeof (cpu_set_t)); while (0); | |||
| 621 | pthread_getaffinity_np(thread, sizeof(cpu_set_t), &cpuset); | |||
| 622 | return cpuset; | |||
| 623 | } | |||
| 624 | #else | |||
| 625 | static uint32_t ggml_get_numa_affinity(void) { | |||
| 626 | return 0; // no NUMA support | |||
| 627 | } | |||
| 628 | #endif | |||
| 629 | ||||
| 630 | void ggml_numa_init(enum ggml_numa_strategy numa_flag) { | |||
| 631 | if (g_state.numa.n_nodes > 0) { | |||
| 632 | fprintf(stderrstderr, "ggml_numa_init: NUMA already initialized\n"); | |||
| 633 | ||||
| 634 | return; | |||
| 635 | } | |||
| 636 | ||||
| 637 | #if defined(__gnu_linux__1) | |||
| 638 | struct stat st; | |||
| 639 | char path[256]; | |||
| 640 | int rv; | |||
| 641 | ||||
| 642 | // set numa scheme | |||
| 643 | g_state.numa.numa_strategy = numa_flag; | |||
| 644 | ||||
| 645 | GGML_PRINT_DEBUG("numa strategy %u\n",g_state.numa.numa_strategy); | |||
| 646 | ||||
| 647 | g_state.numa.cpuset = ggml_get_numa_affinity(); | |||
| 648 | ||||
| 649 | // enumerate nodes | |||
| 650 | while (g_state.numa.n_nodes < GGML_NUMA_MAX_NODES8) { | |||
| 651 | rv = snprintf(path, sizeof(path), "/sys/devices/system/node/node%u", g_state.numa.n_nodes); | |||
| 652 | GGML_ASSERT(rv > 0 && (unsigned)rv < sizeof(path))if (!(rv > 0 && (unsigned)rv < sizeof(path))) ggml_abort ("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 652, "GGML_ASSERT(%s) failed", "rv > 0 && (unsigned)rv < sizeof(path)" ); | |||
| 653 | if (stat(path, &st) != 0) { break; } | |||
| 654 | ++g_state.numa.n_nodes; | |||
| 655 | } | |||
| 656 | ||||
| 657 | // enumerate CPUs | |||
| 658 | while (g_state.numa.total_cpus < GGML_NUMA_MAX_CPUS512) { | |||
| 659 | rv = snprintf(path, sizeof(path), "/sys/devices/system/cpu/cpu%u", g_state.numa.total_cpus); | |||
| 660 | GGML_ASSERT(rv > 0 && (unsigned)rv < sizeof(path))if (!(rv > 0 && (unsigned)rv < sizeof(path))) ggml_abort ("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 660, "GGML_ASSERT(%s) failed", "rv > 0 && (unsigned)rv < sizeof(path)" ); | |||
| 661 | if (stat(path, &st) != 0) { break; } | |||
| 662 | ++g_state.numa.total_cpus; | |||
| 663 | } | |||
| 664 | ||||
| 665 | GGML_PRINT_DEBUG("found %u numa nodes, %u CPUs\n", g_state.numa.n_nodes, g_state.numa.total_cpus); | |||
| 666 | ||||
| 667 | // figure out which node we're on | |||
| 668 | uint current_cpu; | |||
| 669 | int getcpu_ret = 0; | |||
| 670 | #if __GLIBC__2 > 2 || (__GLIBC__2 == 2 && __GLIBC_MINOR__43 > 33) || defined(__COSMOPOLITAN__) | |||
| 671 | getcpu_ret = getcpu(¤t_cpu, &g_state.numa.current_node); | |||
| 672 | #else | |||
| 673 | // old glibc doesn't have a wrapper for this call. Fall back on direct syscall | |||
| 674 | # if !defined(SYS_getcpu309) && defined(SYS_get_cpu) | |||
| 675 | # define SYS_getcpu309 SYS_get_cpu // some older glibc versions use this name | |||
| 676 | # endif | |||
| 677 | getcpu_ret = syscall(SYS_getcpu309, ¤t_cpu, &g_state.numa.current_node); | |||
| 678 | #endif | |||
| 679 | ||||
| 680 | if (g_state.numa.n_nodes < 1 || g_state.numa.total_cpus < 1 || getcpu_ret != 0) { | |||
| 681 | g_state.numa.n_nodes = 0; | |||
| 682 | return; | |||
| 683 | } | |||
| 684 | ||||
| 685 | GGML_PRINT_DEBUG("found our process on numa node %u, CPU %u\n", g_state.numa.current_node, current_cpu); | |||
| 686 | ||||
| 687 | for (uint32_t n = 0; n < g_state.numa.n_nodes; ++n) { | |||
| 688 | struct ggml_numa_node * node = &g_state.numa.nodes[n]; | |||
| 689 | GGML_PRINT_DEBUG("CPUs on node %u:", n); | |||
| 690 | node->n_cpus = 0; | |||
| 691 | for (uint32_t c = 0; c < g_state.numa.total_cpus; ++c) { | |||
| 692 | rv = snprintf(path, sizeof(path), "/sys/devices/system/node/node%u/cpu%u", n, c); | |||
| 693 | GGML_ASSERT(rv > 0 && (unsigned)rv < sizeof(path))if (!(rv > 0 && (unsigned)rv < sizeof(path))) ggml_abort ("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 693, "GGML_ASSERT(%s) failed", "rv > 0 && (unsigned)rv < sizeof(path)" ); | |||
| 694 | if (stat(path, &st) == 0) { | |||
| 695 | node->cpus[node->n_cpus++] = c; | |||
| 696 | GGML_PRINT_DEBUG(" %u", c); | |||
| 697 | } | |||
| 698 | } | |||
| 699 | GGML_PRINT_DEBUG("\n"); | |||
| 700 | } | |||
| 701 | ||||
| 702 | if (ggml_is_numa()) { | |||
| 703 | FILE *fptr = fopen("/proc/sys/kernel/numa_balancing", "r"); | |||
| 704 | if (fptr != NULL((void*)0)) { | |||
| 705 | char buf[42]; | |||
| 706 | if (fgets(buf, sizeof(buf), fptr) && strncmp(buf, "0\n", sizeof(buf)) != 0) { | |||
| 707 | GGML_LOG_WARN("/proc/sys/kernel/numa_balancing is enabled, this has been observed to impair performance\n")ggml_log_internal(GGML_LOG_LEVEL_WARN , "/proc/sys/kernel/numa_balancing is enabled, this has been observed to impair performance\n" ); | |||
| 708 | } | |||
| 709 | fclose(fptr); | |||
| 710 | } | |||
| 711 | } | |||
| 712 | #else | |||
| 713 | UNUSED(numa_flag)(void)(numa_flag); | |||
| 714 | // TODO | |||
| 715 | #endif | |||
| 716 | } | |||
| 717 | ||||
| 718 | bool_Bool ggml_is_numa(void) { | |||
| 719 | return g_state.numa.n_nodes > 1; | |||
| 720 | } | |||
| 721 | ||||
| 722 | #if defined(__ARM_ARCH) | |||
| 723 | #if defined(__aarch64__) && defined(__ARM_FEATURE_SVE) | |||
| 724 | #include <arm_sve.h> | |||
| 725 | static void ggml_init_arm_arch_features(void) { | |||
| 726 | ggml_arm_arch_features.sve_cnt = svcntb(); | |||
| 727 | } | |||
| 728 | #else | |||
| 729 | static void ggml_init_arm_arch_features(void) {} | |||
| 730 | #endif | |||
| 731 | #endif // __ARM_ARCH | |||
| 732 | ||||
| 733 | #if defined(__riscv) && defined(__riscv_v_intrinsic) | |||
| 734 | #include <riscv_vector.h> | |||
| 735 | static void ggml_init_riscv_arch_features(void) { | |||
| 736 | ggml_riscv_arch_features.rvv_vlen = __riscv_vlenb(); | |||
| 737 | } | |||
| 738 | #else | |||
| 739 | static void ggml_init_riscv_arch_features(void) {} | |||
| 740 | #endif | |||
| 741 | ||||
| 742 | struct ggml_tensor * ggml_new_i32(struct ggml_context * ctx, int32_t value) { | |||
| 743 | GGML_ASSERT(!ggml_get_no_alloc(ctx))if (!(!ggml_get_no_alloc(ctx))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 743, "GGML_ASSERT(%s) failed", "!ggml_get_no_alloc(ctx)"); | |||
| 744 | ||||
| 745 | struct ggml_tensor * result = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 1); | |||
| 746 | ||||
| 747 | ggml_set_i32(result, value); | |||
| 748 | ||||
| 749 | return result; | |||
| 750 | } | |||
| 751 | ||||
| 752 | struct ggml_tensor * ggml_new_f32(struct ggml_context * ctx, float value) { | |||
| 753 | GGML_ASSERT(!ggml_get_no_alloc(ctx))if (!(!ggml_get_no_alloc(ctx))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 753, "GGML_ASSERT(%s) failed", "!ggml_get_no_alloc(ctx)"); | |||
| 754 | ||||
| 755 | struct ggml_tensor * result = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1); | |||
| 756 | ||||
| 757 | ggml_set_f32(result, value); | |||
| 758 | ||||
| 759 | return result; | |||
| 760 | } | |||
| 761 | ||||
| 762 | struct ggml_tensor * ggml_set_i32 (struct ggml_tensor * tensor, int32_t value) { | |||
| 763 | const int n = ggml_nrows(tensor); | |||
| 764 | const int nc = tensor->ne[0]; | |||
| 765 | const size_t n1 = tensor->nb[1]; | |||
| 766 | ||||
| 767 | char * const data = tensor->data; | |||
| 768 | ||||
| 769 | switch (tensor->type) { | |||
| 770 | case GGML_TYPE_I8: | |||
| 771 | { | |||
| 772 | assert(tensor->nb[0] == sizeof(int8_t))((void) sizeof (__assert_single_arg (tensor->nb[0] == sizeof (int8_t))), __extension__ ({ if (tensor->nb[0] == sizeof(int8_t )) ; else __assert_fail ("tensor->nb[0] == sizeof(int8_t)" , "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 772, __extension__ __PRETTY_FUNCTION__); })); | |||
| 773 | for (int i = 0; i < n; i++) { | |||
| 774 | ggml_vec_set_i8(nc, (int8_t *)(data + i*n1), value); | |||
| 775 | } | |||
| 776 | } break; | |||
| 777 | case GGML_TYPE_I16: | |||
| 778 | { | |||
| 779 | assert(tensor->nb[0] == sizeof(int16_t))((void) sizeof (__assert_single_arg (tensor->nb[0] == sizeof (int16_t))), __extension__ ({ if (tensor->nb[0] == sizeof( int16_t)) ; else __assert_fail ("tensor->nb[0] == sizeof(int16_t)" , "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 779, __extension__ __PRETTY_FUNCTION__); })); | |||
| 780 | for (int i = 0; i < n; i++) { | |||
| 781 | ggml_vec_set_i16(nc, (int16_t *)(data + i*n1), value); | |||
| 782 | } | |||
| 783 | } break; | |||
| 784 | case GGML_TYPE_I32: | |||
| 785 | { | |||
| 786 | assert(tensor->nb[0] == sizeof(int32_t))((void) sizeof (__assert_single_arg (tensor->nb[0] == sizeof (int32_t))), __extension__ ({ if (tensor->nb[0] == sizeof( int32_t)) ; else __assert_fail ("tensor->nb[0] == sizeof(int32_t)" , "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 786, __extension__ __PRETTY_FUNCTION__); })); | |||
| 787 | for (int i = 0; i < n; i++) { | |||
| 788 | ggml_vec_set_i32(nc, (int32_t *)(data + i*n1), value); | |||
| 789 | } | |||
| 790 | } break; | |||
| 791 | case GGML_TYPE_F16: | |||
| 792 | { | |||
| 793 | assert(tensor->nb[0] == sizeof(ggml_fp16_t))((void) sizeof (__assert_single_arg (tensor->nb[0] == sizeof (ggml_fp16_t))), __extension__ ({ if (tensor->nb[0] == sizeof (ggml_fp16_t)) ; else __assert_fail ("tensor->nb[0] == sizeof(ggml_fp16_t)" , "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 793, __extension__ __PRETTY_FUNCTION__); })); | |||
| 794 | for (int i = 0; i < n; i++) { | |||
| 795 | ggml_vec_set_f16(nc, (ggml_fp16_t *)(data + i*n1), GGML_CPU_FP32_TO_FP16(value)ggml_compute_fp32_to_fp16(value)); | |||
| 796 | } | |||
| 797 | } break; | |||
| 798 | case GGML_TYPE_BF16: | |||
| 799 | { | |||
| 800 | assert(tensor->nb[0] == sizeof(ggml_fp16_t))((void) sizeof (__assert_single_arg (tensor->nb[0] == sizeof (ggml_fp16_t))), __extension__ ({ if (tensor->nb[0] == sizeof (ggml_fp16_t)) ; else __assert_fail ("tensor->nb[0] == sizeof(ggml_fp16_t)" , "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 800, __extension__ __PRETTY_FUNCTION__); })); | |||
| 801 | for (int i = 0; i < n; i++) { | |||
| 802 | ggml_vec_set_bf16(nc, (ggml_bf16_t *)(data + i*n1), GGML_FP32_TO_BF16(value)ggml_compute_fp32_to_bf16(value)); | |||
| 803 | } | |||
| 804 | } break; | |||
| 805 | case GGML_TYPE_F32: | |||
| 806 | { | |||
| 807 | assert(tensor->nb[0] == sizeof(float))((void) sizeof (__assert_single_arg (tensor->nb[0] == sizeof (float))), __extension__ ({ if (tensor->nb[0] == sizeof(float )) ; else __assert_fail ("tensor->nb[0] == sizeof(float)", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 807, __extension__ __PRETTY_FUNCTION__); })); | |||
| 808 | for (int i = 0; i < n; i++) { | |||
| 809 | ggml_vec_set_f32(nc, (float *)(data + i*n1), value); | |||
| 810 | } | |||
| 811 | } break; | |||
| 812 | default: | |||
| 813 | { | |||
| 814 | GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 814, "fatal error"); | |||
| 815 | } | |||
| 816 | } | |||
| 817 | ||||
| 818 | return tensor; | |||
| 819 | } | |||
| 820 | ||||
| 821 | struct ggml_tensor * ggml_set_f32(struct ggml_tensor * tensor, float value) { | |||
| 822 | const int n = ggml_nrows(tensor); | |||
| 823 | const int nc = tensor->ne[0]; | |||
| 824 | const size_t n1 = tensor->nb[1]; | |||
| 825 | ||||
| 826 | char * const data = tensor->data; | |||
| 827 | ||||
| 828 | switch (tensor->type) { | |||
| 829 | case GGML_TYPE_I8: | |||
| 830 | { | |||
| 831 | assert(tensor->nb[0] == sizeof(int8_t))((void) sizeof (__assert_single_arg (tensor->nb[0] == sizeof (int8_t))), __extension__ ({ if (tensor->nb[0] == sizeof(int8_t )) ; else __assert_fail ("tensor->nb[0] == sizeof(int8_t)" , "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 831, __extension__ __PRETTY_FUNCTION__); })); | |||
| 832 | for (int i = 0; i < n; i++) { | |||
| 833 | ggml_vec_set_i8(nc, (int8_t *)(data + i*n1), value); | |||
| 834 | } | |||
| 835 | } break; | |||
| 836 | case GGML_TYPE_I16: | |||
| 837 | { | |||
| 838 | assert(tensor->nb[0] == sizeof(int16_t))((void) sizeof (__assert_single_arg (tensor->nb[0] == sizeof (int16_t))), __extension__ ({ if (tensor->nb[0] == sizeof( int16_t)) ; else __assert_fail ("tensor->nb[0] == sizeof(int16_t)" , "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 838, __extension__ __PRETTY_FUNCTION__); })); | |||
| 839 | for (int i = 0; i < n; i++) { | |||
| 840 | ggml_vec_set_i16(nc, (int16_t *)(data + i*n1), value); | |||
| 841 | } | |||
| 842 | } break; | |||
| 843 | case GGML_TYPE_I32: | |||
| 844 | { | |||
| 845 | assert(tensor->nb[0] == sizeof(int32_t))((void) sizeof (__assert_single_arg (tensor->nb[0] == sizeof (int32_t))), __extension__ ({ if (tensor->nb[0] == sizeof( int32_t)) ; else __assert_fail ("tensor->nb[0] == sizeof(int32_t)" , "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 845, __extension__ __PRETTY_FUNCTION__); })); | |||
| 846 | for (int i = 0; i < n; i++) { | |||
| 847 | ggml_vec_set_i32(nc, (int32_t *)(data + i*n1), value); | |||
| 848 | } | |||
| 849 | } break; | |||
| 850 | case GGML_TYPE_F16: | |||
| 851 | { | |||
| 852 | assert(tensor->nb[0] == sizeof(ggml_fp16_t))((void) sizeof (__assert_single_arg (tensor->nb[0] == sizeof (ggml_fp16_t))), __extension__ ({ if (tensor->nb[0] == sizeof (ggml_fp16_t)) ; else __assert_fail ("tensor->nb[0] == sizeof(ggml_fp16_t)" , "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 852, __extension__ __PRETTY_FUNCTION__); })); | |||
| 853 | for (int i = 0; i < n; i++) { | |||
| 854 | ggml_vec_set_f16(nc, (ggml_fp16_t *)(data + i*n1), GGML_CPU_FP32_TO_FP16(value)ggml_compute_fp32_to_fp16(value)); | |||
| 855 | } | |||
| 856 | } break; | |||
| 857 | case GGML_TYPE_BF16: | |||
| 858 | { | |||
| 859 | assert(tensor->nb[0] == sizeof(ggml_bf16_t))((void) sizeof (__assert_single_arg (tensor->nb[0] == sizeof (ggml_bf16_t))), __extension__ ({ if (tensor->nb[0] == sizeof (ggml_bf16_t)) ; else __assert_fail ("tensor->nb[0] == sizeof(ggml_bf16_t)" , "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 859, __extension__ __PRETTY_FUNCTION__); })); | |||
| 860 | for (int i = 0; i < n; i++) { | |||
| 861 | ggml_vec_set_bf16(nc, (ggml_bf16_t *)(data + i*n1), GGML_FP32_TO_BF16(value)ggml_compute_fp32_to_bf16(value)); | |||
| 862 | } | |||
| 863 | } break; | |||
| 864 | case GGML_TYPE_F32: | |||
| 865 | { | |||
| 866 | assert(tensor->nb[0] == sizeof(float))((void) sizeof (__assert_single_arg (tensor->nb[0] == sizeof (float))), __extension__ ({ if (tensor->nb[0] == sizeof(float )) ; else __assert_fail ("tensor->nb[0] == sizeof(float)", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 866, __extension__ __PRETTY_FUNCTION__); })); | |||
| 867 | for (int i = 0; i < n; i++) { | |||
| 868 | ggml_vec_set_f32(nc, (float *)(data + i*n1), value); | |||
| 869 | } | |||
| 870 | } break; | |||
| 871 | default: | |||
| 872 | { | |||
| 873 | GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 873, "fatal error"); | |||
| 874 | } | |||
| 875 | } | |||
| 876 | ||||
| 877 | return tensor; | |||
| 878 | } | |||
| 879 | ||||
| 880 | int32_t ggml_get_i32_1d(const struct ggml_tensor * tensor, int i) { | |||
| 881 | if (!ggml_is_contiguous(tensor)) { | |||
| 882 | int64_t id[4] = { 0, 0, 0, 0 }; | |||
| 883 | ggml_unravel_index(tensor, i, &id[0], &id[1], &id[2], &id[3]); | |||
| 884 | return ggml_get_i32_nd(tensor, id[0], id[1], id[2], id[3]); | |||
| 885 | } | |||
| 886 | switch (tensor->type) { | |||
| 887 | case GGML_TYPE_I8: | |||
| 888 | { | |||
| 889 | GGML_ASSERT(tensor->nb[0] == sizeof(int8_t))if (!(tensor->nb[0] == sizeof(int8_t))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 889, "GGML_ASSERT(%s) failed", "tensor->nb[0] == sizeof(int8_t)" ); | |||
| 890 | return ((int8_t *)(tensor->data))[i]; | |||
| 891 | } | |||
| 892 | case GGML_TYPE_I16: | |||
| 893 | { | |||
| 894 | GGML_ASSERT(tensor->nb[0] == sizeof(int16_t))if (!(tensor->nb[0] == sizeof(int16_t))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 894, "GGML_ASSERT(%s) failed", "tensor->nb[0] == sizeof(int16_t)" ); | |||
| 895 | return ((int16_t *)(tensor->data))[i]; | |||
| 896 | } | |||
| 897 | case GGML_TYPE_I32: | |||
| 898 | { | |||
| 899 | GGML_ASSERT(tensor->nb[0] == sizeof(int32_t))if (!(tensor->nb[0] == sizeof(int32_t))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 899, "GGML_ASSERT(%s) failed", "tensor->nb[0] == sizeof(int32_t)" ); | |||
| 900 | return ((int32_t *)(tensor->data))[i]; | |||
| 901 | } | |||
| 902 | case GGML_TYPE_F16: | |||
| 903 | { | |||
| 904 | GGML_ASSERT(tensor->nb[0] == sizeof(ggml_fp16_t))if (!(tensor->nb[0] == sizeof(ggml_fp16_t))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 904, "GGML_ASSERT(%s) failed", "tensor->nb[0] == sizeof(ggml_fp16_t)" ); | |||
| 905 | return GGML_CPU_FP16_TO_FP32(((ggml_fp16_t *)(tensor->data))[i])ggml_lookup_fp16_to_fp32(((ggml_fp16_t *)(tensor->data))[i ]); | |||
| 906 | } | |||
| 907 | case GGML_TYPE_BF16: | |||
| 908 | { | |||
| 909 | GGML_ASSERT(tensor->nb[0] == sizeof(ggml_bf16_t))if (!(tensor->nb[0] == sizeof(ggml_bf16_t))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 909, "GGML_ASSERT(%s) failed", "tensor->nb[0] == sizeof(ggml_bf16_t)" ); | |||
| 910 | return GGML_BF16_TO_FP32(((ggml_bf16_t *)(tensor->data))[i])ggml_compute_bf16_to_fp32(((ggml_bf16_t *)(tensor->data))[ i]); | |||
| 911 | } | |||
| 912 | case GGML_TYPE_F32: | |||
| 913 | { | |||
| 914 | GGML_ASSERT(tensor->nb[0] == sizeof(float))if (!(tensor->nb[0] == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 914, "GGML_ASSERT(%s) failed", "tensor->nb[0] == sizeof(float)" ); | |||
| 915 | return ((float *)(tensor->data))[i]; | |||
| 916 | } | |||
| 917 | default: | |||
| 918 | { | |||
| 919 | GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 919, "fatal error"); | |||
| 920 | } | |||
| 921 | } | |||
| 922 | } | |||
| 923 | ||||
| 924 | void ggml_set_i32_1d(const struct ggml_tensor * tensor, int i, int32_t value) { | |||
| 925 | if (!ggml_is_contiguous(tensor)) { | |||
| 926 | int64_t id[4] = { 0, 0, 0, 0 }; | |||
| 927 | ggml_unravel_index(tensor, i, &id[0], &id[1], &id[2], &id[3]); | |||
| 928 | ggml_set_i32_nd(tensor, id[0], id[1], id[2], id[3], value); | |||
| 929 | return; | |||
| 930 | } | |||
| 931 | switch (tensor->type) { | |||
| 932 | case GGML_TYPE_I8: | |||
| 933 | { | |||
| 934 | GGML_ASSERT(tensor->nb[0] == sizeof(int8_t))if (!(tensor->nb[0] == sizeof(int8_t))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 934, "GGML_ASSERT(%s) failed", "tensor->nb[0] == sizeof(int8_t)" ); | |||
| 935 | ((int8_t *)(tensor->data))[i] = value; | |||
| 936 | } break; | |||
| 937 | case GGML_TYPE_I16: | |||
| 938 | { | |||
| 939 | GGML_ASSERT(tensor->nb[0] == sizeof(int16_t))if (!(tensor->nb[0] == sizeof(int16_t))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 939, "GGML_ASSERT(%s) failed", "tensor->nb[0] == sizeof(int16_t)" ); | |||
| 940 | ((int16_t *)(tensor->data))[i] = value; | |||
| 941 | } break; | |||
| 942 | case GGML_TYPE_I32: | |||
| 943 | { | |||
| 944 | GGML_ASSERT(tensor->nb[0] == sizeof(int32_t))if (!(tensor->nb[0] == sizeof(int32_t))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 944, "GGML_ASSERT(%s) failed", "tensor->nb[0] == sizeof(int32_t)" ); | |||
| 945 | ((int32_t *)(tensor->data))[i] = value; | |||
| 946 | } break; | |||
| 947 | case GGML_TYPE_F16: | |||
| 948 | { | |||
| 949 | GGML_ASSERT(tensor->nb[0] == sizeof(ggml_fp16_t))if (!(tensor->nb[0] == sizeof(ggml_fp16_t))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 949, "GGML_ASSERT(%s) failed", "tensor->nb[0] == sizeof(ggml_fp16_t)" ); | |||
| 950 | ((ggml_fp16_t *)(tensor->data))[i] = GGML_CPU_FP32_TO_FP16(value)ggml_compute_fp32_to_fp16(value); | |||
| 951 | } break; | |||
| 952 | case GGML_TYPE_BF16: | |||
| 953 | { | |||
| 954 | GGML_ASSERT(tensor->nb[0] == sizeof(ggml_bf16_t))if (!(tensor->nb[0] == sizeof(ggml_bf16_t))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 954, "GGML_ASSERT(%s) failed", "tensor->nb[0] == sizeof(ggml_bf16_t)" ); | |||
| 955 | ((ggml_bf16_t *)(tensor->data))[i] = GGML_FP32_TO_BF16(value)ggml_compute_fp32_to_bf16(value); | |||
| 956 | } break; | |||
| 957 | case GGML_TYPE_F32: | |||
| 958 | { | |||
| 959 | GGML_ASSERT(tensor->nb[0] == sizeof(float))if (!(tensor->nb[0] == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 959, "GGML_ASSERT(%s) failed", "tensor->nb[0] == sizeof(float)" ); | |||
| 960 | ((float *)(tensor->data))[i] = value; | |||
| 961 | } break; | |||
| 962 | default: | |||
| 963 | { | |||
| 964 | GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 964, "fatal error"); | |||
| 965 | } | |||
| 966 | } | |||
| 967 | } | |||
| 968 | ||||
| 969 | int32_t ggml_get_i32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3) { | |||
| 970 | void * data = (char *) tensor->data + i0*tensor->nb[0] + i1*tensor->nb[1] + i2*tensor->nb[2] + i3*tensor->nb[3]; | |||
| 971 | switch (tensor->type) { | |||
| 972 | case GGML_TYPE_I8: | |||
| 973 | return ((int8_t *) data)[0]; | |||
| 974 | case GGML_TYPE_I16: | |||
| 975 | return ((int16_t *) data)[0]; | |||
| 976 | case GGML_TYPE_I32: | |||
| 977 | return ((int32_t *) data)[0]; | |||
| 978 | case GGML_TYPE_F16: | |||
| 979 | return GGML_CPU_FP16_TO_FP32(((ggml_fp16_t *) data)[0])ggml_lookup_fp16_to_fp32(((ggml_fp16_t *) data)[0]); | |||
| 980 | case GGML_TYPE_BF16: | |||
| 981 | return GGML_BF16_TO_FP32(((ggml_bf16_t *) data)[0])ggml_compute_bf16_to_fp32(((ggml_bf16_t *) data)[0]); | |||
| 982 | case GGML_TYPE_F32: | |||
| 983 | return ((float *) data)[0]; | |||
| 984 | default: | |||
| 985 | GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 985, "fatal error"); | |||
| 986 | } | |||
| 987 | } | |||
| 988 | ||||
| 989 | void ggml_set_i32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3, int32_t value) { | |||
| 990 | void * data = (char *) tensor->data + i0*tensor->nb[0] + i1*tensor->nb[1] + i2*tensor->nb[2] + i3*tensor->nb[3]; | |||
| 991 | switch (tensor->type) { | |||
| 992 | case GGML_TYPE_I8: | |||
| 993 | { | |||
| 994 | ((int8_t *)(data))[0] = value; | |||
| 995 | } break; | |||
| 996 | case GGML_TYPE_I16: | |||
| 997 | { | |||
| 998 | ((int16_t *)(data))[0] = value; | |||
| 999 | } break; | |||
| 1000 | case GGML_TYPE_I32: | |||
| 1001 | { | |||
| 1002 | ((int32_t *)(data))[0] = value; | |||
| 1003 | } break; | |||
| 1004 | case GGML_TYPE_F16: | |||
| 1005 | { | |||
| 1006 | ((ggml_fp16_t *)(data))[0] = GGML_CPU_FP32_TO_FP16(value)ggml_compute_fp32_to_fp16(value); | |||
| 1007 | } break; | |||
| 1008 | case GGML_TYPE_BF16: | |||
| 1009 | { | |||
| 1010 | ((ggml_bf16_t *)(data))[0] = GGML_FP32_TO_BF16(value)ggml_compute_fp32_to_bf16(value); | |||
| 1011 | } break; | |||
| 1012 | case GGML_TYPE_F32: | |||
| 1013 | { | |||
| 1014 | ((float *)(data))[0] = value; | |||
| 1015 | } break; | |||
| 1016 | default: | |||
| 1017 | { | |||
| 1018 | GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 1018, "fatal error"); | |||
| 1019 | } | |||
| 1020 | } | |||
| 1021 | } | |||
| 1022 | ||||
| 1023 | float ggml_get_f32_1d(const struct ggml_tensor * tensor, int i) { | |||
| 1024 | if (!ggml_is_contiguous(tensor)) { | |||
| 1025 | int64_t id[4] = { 0, 0, 0, 0 }; | |||
| 1026 | ggml_unravel_index(tensor, i, &id[0], &id[1], &id[2], &id[3]); | |||
| 1027 | return ggml_get_f32_nd(tensor, id[0], id[1], id[2], id[3]); | |||
| 1028 | } | |||
| 1029 | switch (tensor->type) { | |||
| 1030 | case GGML_TYPE_I8: | |||
| 1031 | { | |||
| 1032 | return ((int8_t *)(tensor->data))[i]; | |||
| 1033 | } | |||
| 1034 | case GGML_TYPE_I16: | |||
| 1035 | { | |||
| 1036 | return ((int16_t *)(tensor->data))[i]; | |||
| 1037 | } | |||
| 1038 | case GGML_TYPE_I32: | |||
| 1039 | { | |||
| 1040 | return ((int32_t *)(tensor->data))[i]; | |||
| 1041 | } | |||
| 1042 | case GGML_TYPE_F16: | |||
| 1043 | { | |||
| 1044 | return GGML_CPU_FP16_TO_FP32(((ggml_fp16_t *)(tensor->data))[i])ggml_lookup_fp16_to_fp32(((ggml_fp16_t *)(tensor->data))[i ]); | |||
| 1045 | } | |||
| 1046 | case GGML_TYPE_BF16: | |||
| 1047 | { | |||
| 1048 | return GGML_BF16_TO_FP32(((ggml_bf16_t *)(tensor->data))[i])ggml_compute_bf16_to_fp32(((ggml_bf16_t *)(tensor->data))[ i]); | |||
| 1049 | } | |||
| 1050 | case GGML_TYPE_F32: | |||
| 1051 | { | |||
| 1052 | return ((float *)(tensor->data))[i]; | |||
| 1053 | } | |||
| 1054 | default: | |||
| 1055 | { | |||
| 1056 | GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 1056, "fatal error"); | |||
| 1057 | } | |||
| 1058 | } | |||
| 1059 | } | |||
| 1060 | ||||
| 1061 | void ggml_set_f32_1d(const struct ggml_tensor * tensor, int i, float value) { | |||
| 1062 | if (!ggml_is_contiguous(tensor)) { | |||
| 1063 | int64_t id[4] = { 0, 0, 0, 0 }; | |||
| 1064 | ggml_unravel_index(tensor, i, &id[0], &id[1], &id[2], &id[3]); | |||
| 1065 | ggml_set_f32_nd(tensor, id[0], id[1], id[2], id[3], value); | |||
| 1066 | return; | |||
| 1067 | } | |||
| 1068 | switch (tensor->type) { | |||
| 1069 | case GGML_TYPE_I8: | |||
| 1070 | { | |||
| 1071 | ((int8_t *)(tensor->data))[i] = value; | |||
| 1072 | } break; | |||
| 1073 | case GGML_TYPE_I16: | |||
| 1074 | { | |||
| 1075 | ((int16_t *)(tensor->data))[i] = value; | |||
| 1076 | } break; | |||
| 1077 | case GGML_TYPE_I32: | |||
| 1078 | { | |||
| 1079 | ((int32_t *)(tensor->data))[i] = value; | |||
| 1080 | } break; | |||
| 1081 | case GGML_TYPE_F16: | |||
| 1082 | { | |||
| 1083 | ((ggml_fp16_t *)(tensor->data))[i] = GGML_CPU_FP32_TO_FP16(value)ggml_compute_fp32_to_fp16(value); | |||
| 1084 | } break; | |||
| 1085 | case GGML_TYPE_BF16: | |||
| 1086 | { | |||
| 1087 | ((ggml_bf16_t *)(tensor->data))[i] = GGML_FP32_TO_BF16(value)ggml_compute_fp32_to_bf16(value); | |||
| 1088 | } break; | |||
| 1089 | case GGML_TYPE_F32: | |||
| 1090 | { | |||
| 1091 | ((float *)(tensor->data))[i] = value; | |||
| 1092 | } break; | |||
| 1093 | default: | |||
| 1094 | { | |||
| 1095 | GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 1095, "fatal error"); | |||
| 1096 | } | |||
| 1097 | } | |||
| 1098 | } | |||
| 1099 | ||||
| 1100 | float ggml_get_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3) { | |||
| 1101 | void * data = (char *) tensor->data + i0*tensor->nb[0] + i1*tensor->nb[1] + i2*tensor->nb[2] + i3*tensor->nb[3]; | |||
| 1102 | switch (tensor->type) { | |||
| 1103 | case GGML_TYPE_I8: | |||
| 1104 | return ((int8_t *) data)[0]; | |||
| 1105 | case GGML_TYPE_I16: | |||
| 1106 | return ((int16_t *) data)[0]; | |||
| 1107 | case GGML_TYPE_I32: | |||
| 1108 | return ((int32_t *) data)[0]; | |||
| 1109 | case GGML_TYPE_F16: | |||
| 1110 | return GGML_CPU_FP16_TO_FP32(((ggml_fp16_t *) data)[0])ggml_lookup_fp16_to_fp32(((ggml_fp16_t *) data)[0]); | |||
| 1111 | case GGML_TYPE_BF16: | |||
| 1112 | return GGML_BF16_TO_FP32(((ggml_bf16_t *) data)[0])ggml_compute_bf16_to_fp32(((ggml_bf16_t *) data)[0]); | |||
| 1113 | case GGML_TYPE_F32: | |||
| 1114 | return ((float *) data)[0]; | |||
| 1115 | default: | |||
| 1116 | GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 1116, "fatal error"); | |||
| 1117 | } | |||
| 1118 | } | |||
| 1119 | ||||
| 1120 | void ggml_set_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3, float value) { | |||
| 1121 | void * data = (char *) tensor->data + i0*tensor->nb[0] + i1*tensor->nb[1] + i2*tensor->nb[2] + i3*tensor->nb[3]; | |||
| 1122 | switch (tensor->type) { | |||
| 1123 | case GGML_TYPE_I8: | |||
| 1124 | { | |||
| 1125 | ((int8_t *)(data))[0] = value; | |||
| 1126 | } break; | |||
| 1127 | case GGML_TYPE_I16: | |||
| 1128 | { | |||
| 1129 | ((int16_t *)(data))[0] = value; | |||
| 1130 | } break; | |||
| 1131 | case GGML_TYPE_I32: | |||
| 1132 | { | |||
| 1133 | ((int32_t *)(data))[0] = value; | |||
| 1134 | } break; | |||
| 1135 | case GGML_TYPE_F16: | |||
| 1136 | { | |||
| 1137 | ((ggml_fp16_t *)(data))[0] = GGML_CPU_FP32_TO_FP16(value)ggml_compute_fp32_to_fp16(value); | |||
| 1138 | } break; | |||
| 1139 | case GGML_TYPE_BF16: | |||
| 1140 | { | |||
| 1141 | ((ggml_bf16_t *)(data))[0] = GGML_FP32_TO_BF16(value)ggml_compute_fp32_to_bf16(value); | |||
| 1142 | } break; | |||
| 1143 | case GGML_TYPE_F32: | |||
| 1144 | { | |||
| 1145 | ((float *)(data))[0] = value; | |||
| 1146 | } break; | |||
| 1147 | default: | |||
| 1148 | { | |||
| 1149 | GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 1149, "fatal error"); | |||
| 1150 | } | |||
| 1151 | } | |||
| 1152 | } | |||
| 1153 | ||||
| 1154 | //////////////////////////////////////////////////////////////////////////////// | |||
| 1155 | ||||
| 1156 | // ggml_compute_forward_mul_mat | |||
| 1157 | ||||
| 1158 | static void ggml_compute_forward_mul_mat_one_chunk( | |||
| 1159 | const struct ggml_compute_params * params, | |||
| 1160 | struct ggml_tensor * dst, | |||
| 1161 | const enum ggml_type type, | |||
| 1162 | const int64_t num_rows_per_vec_dot, | |||
| 1163 | const int64_t ir0_start, | |||
| 1164 | const int64_t ir0_end, | |||
| 1165 | const int64_t ir1_start, | |||
| 1166 | const int64_t ir1_end) { | |||
| 1167 | ||||
| 1168 | const struct ggml_tensor * src0 = dst->src[0]; | |||
| 1169 | const struct ggml_tensor * src1 = dst->src[1]; | |||
| 1170 | ||||
| 1171 | GGML_TENSOR_BINARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00 ); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void) (ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; ( void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] : 0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[ 0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)-> nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)-> nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)-> nb[3] : 0; (void)(nb03); const int64_t ne10 = (src1) ? (src1) ->ne[0] : 0; (void)(ne10); const int64_t ne11 = (src1) ? ( src1)->ne[1] : 0; (void)(ne11); const int64_t ne12 = (src1 ) ? (src1)->ne[2] : 0; (void)(ne12); const int64_t ne13 = ( src1) ? (src1)->ne[3] : 0; (void)(ne13); const size_t nb10 = (src1) ? (src1)->nb[0] : 0; (void)(nb10); const size_t nb11 = (src1) ? (src1)->nb[1] : 0; (void)(nb11); const size_t nb12 = (src1) ? (src1)->nb[2] : 0; (void)(nb12); const size_t nb13 = (src1) ? (src1)->nb[3] : 0; (void)(nb13); const int64_t ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0); const int64_t ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1); const int64_t ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2); const int64_t ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3); const size_t nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const size_t nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const size_t nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const size_t nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3); | |||
| ||||
| 1172 | ||||
| 1173 | const bool_Bool src1_cont = ggml_is_contiguous(src1); | |||
| 1174 | ||||
| 1175 | ggml_vec_dot_t const vec_dot = type_traits_cpu[type].vec_dot; | |||
| 1176 | enum ggml_type const vec_dot_type = type_traits_cpu[type].vec_dot_type; | |||
| 1177 | ||||
| 1178 | // broadcast factors | |||
| 1179 | const int64_t r2 = ne12 / ne02; | |||
| ||||
| 1180 | const int64_t r3 = ne13 / ne03; | |||
| 1181 | ||||
| 1182 | //printf("ir0_start = %6lld, ir0_end = %6lld, ir1_start = %6lld, ir1_end = %6lld\n", ir0_start, ir0_end, ir1_start, ir1_end); | |||
| 1183 | ||||
| 1184 | // threads with no work simply yield (not sure if it helps) | |||
| 1185 | if (ir0_start >= ir0_end || ir1_start >= ir1_end) { | |||
| 1186 | return; | |||
| 1187 | } | |||
| 1188 | ||||
| 1189 | const void * wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata; | |||
| 1190 | const size_t row_size = ggml_row_size(vec_dot_type, ne10); | |||
| 1191 | ||||
| 1192 | assert(ne12 % ne02 == 0)((void) sizeof (__assert_single_arg (ne12 % ne02 == 0)), __extension__ ({ if (ne12 % ne02 == 0) ; else __assert_fail ("ne12 % ne02 == 0" , "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 1192, __extension__ __PRETTY_FUNCTION__); })); | |||
| 1193 | assert(ne13 % ne03 == 0)((void) sizeof (__assert_single_arg (ne13 % ne03 == 0)), __extension__ ({ if (ne13 % ne03 == 0) ; else __assert_fail ("ne13 % ne03 == 0" , "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 1193, __extension__ __PRETTY_FUNCTION__); })); | |||
| 1194 | ||||
| 1195 | // block-tiling attempt | |||
| 1196 | const int64_t blck_0 = 16; | |||
| 1197 | const int64_t blck_1 = 16; | |||
| 1198 | ||||
| 1199 | const size_t src1_col_stride = src1_cont || src1->type != vec_dot_type ? row_size : nb11; | |||
| 1200 | ||||
| 1201 | // attempt to reduce false-sharing (does not seem to make a difference) | |||
| 1202 | // 16 * 2, accounting for mmla kernels | |||
| 1203 | float tmp[32]; | |||
| 1204 | ||||
| 1205 | for (int64_t iir1 = ir1_start; iir1 < ir1_end; iir1 += blck_1) { | |||
| 1206 | for (int64_t iir0 = ir0_start; iir0 < ir0_end; iir0 += blck_0) { | |||
| 1207 | for (int64_t ir1 = iir1; ir1 < iir1 + blck_1 && ir1 < ir1_end; ir1 += num_rows_per_vec_dot) { | |||
| 1208 | const int64_t i13 = (ir1 / (ne12 * ne1)); | |||
| 1209 | const int64_t i12 = (ir1 - i13 * ne12 * ne1) / ne1; | |||
| 1210 | const int64_t i11 = (ir1 - i13 * ne12 * ne1 - i12 * ne1); | |||
| 1211 | ||||
| 1212 | // broadcast src0 into src1 | |||
| 1213 | const int64_t i03 = i13 / r3; | |||
| 1214 | const int64_t i02 = i12 / r2; | |||
| 1215 | ||||
| 1216 | const int64_t i1 = i11; | |||
| 1217 | const int64_t i2 = i12; | |||
| 1218 | const int64_t i3 = i13; | |||
| 1219 | ||||
| 1220 | const char * src0_row = (const char*)src0->data + (0 + i02 * nb02 + i03 * nb03); | |||
| 1221 | ||||
| 1222 | // desc: when src1 is not a contiguous memory block we have to calculate the offset using the strides | |||
| 1223 | // if it is, then we have either copied the data to params->wdata and made it contiguous or we are using | |||
| 1224 | // the original src1 data pointer, so we should index using the indices directly | |||
| 1225 | // TODO: this is a bit of a hack, we should probably have a better way to handle this | |||
| 1226 | const char * src1_col = (const char*)wdata + | |||
| 1227 | (src1_cont || src1->type != vec_dot_type | |||
| 1228 | ? (i11 + i12 * ne11 + i13 * ne12 * ne11) * row_size | |||
| 1229 | : (i11 * nb11 + i12 * nb12 + i13 * nb13)); | |||
| 1230 | float * dst_col = (float*)((char*)dst->data + (i1 * nb1 + i2 * nb2 + i3 * nb3)); | |||
| 1231 | ||||
| 1232 | //for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir0_end; ++ir0) { | |||
| 1233 | // vec_dot(ne00, &dst_col[ir0], src0_row + ir0*nb01, src1_col); | |||
| 1234 | //} | |||
| 1235 | ||||
| 1236 | for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir0_end; ir0 += num_rows_per_vec_dot) { | |||
| 1237 | vec_dot(ne00, &tmp[ir0 - iir0], (num_rows_per_vec_dot > 1 ? 16 : 0), src0_row + ir0 * nb01, (num_rows_per_vec_dot > 1 ? nb01 : 0), src1_col, (num_rows_per_vec_dot > 1 ? src1_col_stride : 0), num_rows_per_vec_dot); | |||
| 1238 | } | |||
| 1239 | ||||
| 1240 | for (int cn = 0; cn < num_rows_per_vec_dot; ++cn) { | |||
| 1241 | memcpy(&dst_col[iir0 + cn * nb1 / nb0], tmp + (cn * 16), (MIN(iir0 + blck_0, ir0_end)((iir0 + blck_0) < (ir0_end) ? (iir0 + blck_0) : (ir0_end) ) - iir0) * sizeof(float)); | |||
| 1242 | } | |||
| 1243 | } | |||
| 1244 | } | |||
| 1245 | } | |||
| 1246 | } | |||
| 1247 | ||||
| 1248 | void ggml_compute_forward_mul_mat( | |||
| 1249 | const struct ggml_compute_params * params, | |||
| 1250 | struct ggml_tensor * dst) { | |||
| 1251 | ||||
| 1252 | const struct ggml_tensor * src0 = dst->src[0]; | |||
| 1253 | const struct ggml_tensor * src1 = dst->src[1]; | |||
| 1254 | ||||
| 1255 | const int32_t hint = ggml_get_op_params_i32(dst, 1); | |||
| 1256 | if (hint == GGML_HINT_SRC0_IS_HADAMARD && !params->use_ref) { | |||
| 1257 | ggml_compute_forward_fwht(params, dst); | |||
| 1258 | return; | |||
| 1259 | } | |||
| 1260 | ||||
| 1261 | GGML_TENSOR_BINARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00 ); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void) (ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; ( void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] : 0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[ 0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)-> nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)-> nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)-> nb[3] : 0; (void)(nb03); const int64_t ne10 = (src1) ? (src1) ->ne[0] : 0; (void)(ne10); const int64_t ne11 = (src1) ? ( src1)->ne[1] : 0; (void)(ne11); const int64_t ne12 = (src1 ) ? (src1)->ne[2] : 0; (void)(ne12); const int64_t ne13 = ( src1) ? (src1)->ne[3] : 0; (void)(ne13); const size_t nb10 = (src1) ? (src1)->nb[0] : 0; (void)(nb10); const size_t nb11 = (src1) ? (src1)->nb[1] : 0; (void)(nb11); const size_t nb12 = (src1) ? (src1)->nb[2] : 0; (void)(nb12); const size_t nb13 = (src1) ? (src1)->nb[3] : 0; (void)(nb13); const int64_t ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0); const int64_t ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1); const int64_t ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2); const int64_t ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3); const size_t nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const size_t nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const size_t nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const size_t nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3); | |||
| 1262 | ||||
| 1263 | const int ith = params->ith; | |||
| 1264 | const int nth = params->nth; | |||
| 1265 | ||||
| 1266 | enum ggml_type const vec_dot_type = type_traits_cpu[src0->type].vec_dot_type; | |||
| 1267 | ggml_from_float_t const from_float = type_traits_cpu[vec_dot_type].from_float; | |||
| 1268 | int64_t const vec_dot_num_rows = type_traits_cpu[src0->type].nrows; | |||
| 1269 | ||||
| 1270 | GGML_ASSERT(ne0 == ne01)if (!(ne0 == ne01)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 1270, "GGML_ASSERT(%s) failed", "ne0 == ne01"); | |||
| 1271 | GGML_ASSERT(ne1 == ne11)if (!(ne1 == ne11)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 1271, "GGML_ASSERT(%s) failed", "ne1 == ne11"); | |||
| 1272 | GGML_ASSERT(ne2 == ne12)if (!(ne2 == ne12)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 1272, "GGML_ASSERT(%s) failed", "ne2 == ne12"); | |||
| 1273 | GGML_ASSERT(ne3 == ne13)if (!(ne3 == ne13)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 1273, "GGML_ASSERT(%s) failed", "ne3 == ne13"); | |||
| 1274 | ||||
| 1275 | // we don't support permuted src0 or src1 | |||
| 1276 | GGML_ASSERT(nb00 == ggml_type_size(src0->type))if (!(nb00 == ggml_type_size(src0->type))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 1276, "GGML_ASSERT(%s) failed", "nb00 == ggml_type_size(src0->type)" ); | |||
| 1277 | GGML_ASSERT(nb10 == ggml_type_size(src1->type))if (!(nb10 == ggml_type_size(src1->type))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 1277, "GGML_ASSERT(%s) failed", "nb10 == ggml_type_size(src1->type)" ); | |||
| 1278 | ||||
| 1279 | // dst cannot be transposed or permuted | |||
| 1280 | GGML_ASSERT(nb0 == sizeof(float))if (!(nb0 == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 1280, "GGML_ASSERT(%s) failed", "nb0 == sizeof(float)"); | |||
| 1281 | GGML_ASSERT(nb0 <= nb1)if (!(nb0 <= nb1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 1281, "GGML_ASSERT(%s) failed", "nb0 <= nb1"); | |||
| 1282 | GGML_ASSERT(nb1 <= nb2)if (!(nb1 <= nb2)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 1282, "GGML_ASSERT(%s) failed", "nb1 <= nb2"); | |||
| 1283 | GGML_ASSERT(nb2 <= nb3)if (!(nb2 <= nb3)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 1283, "GGML_ASSERT(%s) failed", "nb2 <= nb3"); | |||
| 1284 | ||||
| 1285 | // nb01 >= nb00 - src0 is not transposed | |||
| 1286 | // compute by src0 rows | |||
| 1287 | ||||
| 1288 | // TODO: extract to "extra_op" | |||
| 1289 | #if GGML_USE_LLAMAFILE | |||
| 1290 | // broadcast factors | |||
| 1291 | const int64_t r2 = ne12 / ne02; | |||
| 1292 | const int64_t r3 = ne13 / ne03; | |||
| 1293 | ||||
| 1294 | const bool_Bool src1_cont = ggml_is_contiguous(src1); | |||
| 1295 | ||||
| 1296 | if (src1_cont) { | |||
| 1297 | for (int64_t i13 = 0; i13 < ne13; i13++) | |||
| 1298 | for (int64_t i12 = 0; i12 < ne12; i12++) | |||
| 1299 | if (!llamafile_sgemm(params, | |||
| 1300 | ne01, ne11, ne00/ggml_blck_size(src0->type), | |||
| 1301 | (const char *)src0->data + i12/r2*nb02 + i13/r3*nb03, | |||
| 1302 | nb01/ggml_type_size(src0->type), | |||
| 1303 | (const char *)src1->data + i12*nb12 + i13*nb13, | |||
| 1304 | nb11/ggml_type_size(src1->type), | |||
| 1305 | (char *)dst->data + i12*nb2 + i13*nb3, | |||
| 1306 | nb1/ggml_type_size(dst->type), | |||
| 1307 | src0->type, | |||
| 1308 | src1->type, | |||
| 1309 | dst->type)) | |||
| 1310 | goto UseGgmlGemm1; | |||
| 1311 | return; | |||
| 1312 | } | |||
| 1313 | UseGgmlGemm1:; | |||
| 1314 | #endif | |||
| 1315 | ||||
| 1316 | if (src1->type != vec_dot_type) { | |||
| 1317 | char * wdata = params->wdata; | |||
| 1318 | ||||
| 1319 | const size_t nbw0 = ggml_type_size(vec_dot_type); | |||
| 1320 | const size_t nbw1 = ggml_row_size(vec_dot_type, ne10); | |||
| 1321 | const size_t nbw2 = nbw1*ne11; | |||
| 1322 | const size_t nbw3 = nbw2*ne12; | |||
| 1323 | ||||
| 1324 | assert(params->wsize >= ne13*nbw3)((void) sizeof (__assert_single_arg (params->wsize >= ne13 *nbw3)), __extension__ ({ if (params->wsize >= ne13*nbw3 ) ; else __assert_fail ("params->wsize >= ne13*nbw3", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 1324, __extension__ __PRETTY_FUNCTION__); })); | |||
| 1325 | GGML_ASSERT(src1->type == GGML_TYPE_F32)if (!(src1->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 1325, "GGML_ASSERT(%s) failed", "src1->type == GGML_TYPE_F32" ); | |||
| 1326 | ||||
| 1327 | #if 0 | |||
| 1328 | for (int64_t i13 = 0; i13 < ne13; ++i13) { | |||
| 1329 | for (int64_t i12 = 0; i12 < ne12; ++i12) { | |||
| 1330 | for (int64_t i11 = ith; i11 < ne11; i11 += nth) { | |||
| 1331 | from_float((float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11), | |||
| 1332 | (void *) (wdata + i13*nbw3 + i12*nbw2 + i11*nbw1), | |||
| 1333 | ne10); | |||
| 1334 | } | |||
| 1335 | } | |||
| 1336 | } | |||
| 1337 | #else | |||
| 1338 | for (int64_t i13 = 0; i13 < ne13; ++i13) { | |||
| 1339 | for (int64_t i12 = 0; i12 < ne12; ++i12) { | |||
| 1340 | for (int64_t i11 = 0; i11 < ne11; ++i11) { | |||
| 1341 | size_t bs = ggml_blck_size(vec_dot_type); | |||
| 1342 | int64_t ne10_block_start = (ith * ne10/bs) / nth; | |||
| 1343 | int64_t ne10_block_end = ((ith + 1) * ne10/bs) / nth; | |||
| 1344 | from_float((float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11 + ne10_block_start*bs*nb10), | |||
| 1345 | (void *) (wdata + i13*nbw3 + i12*nbw2 + i11*nbw1 + ne10_block_start*nbw0), | |||
| 1346 | (ne10_block_end - ne10_block_start) * bs); | |||
| 1347 | } | |||
| 1348 | } | |||
| 1349 | } | |||
| 1350 | #endif | |||
| 1351 | } | |||
| 1352 | ||||
| 1353 | if (ith == 0) { | |||
| 1354 | // Every thread starts at ith, so the first unprocessed chunk is nth. This save a bit of coordination right at the start. | |||
| 1355 | atomic_store_explicit__c11_atomic_store(¶ms->threadpool->current_chunk, nth, memory_order_relaxed); | |||
| 1356 | } | |||
| 1357 | ||||
| 1358 | ggml_barrier(params->threadpool); | |||
| 1359 | ||||
| 1360 | #if GGML_USE_LLAMAFILE | |||
| 1361 | if (src1->type != vec_dot_type) { | |||
| 1362 | const void* wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata; | |||
| 1363 | const size_t row_size = ggml_row_size(vec_dot_type, ne10); | |||
| 1364 | ||||
| 1365 | for (int64_t i13 = 0; i13 < ne13; i13++) | |||
| 1366 | for (int64_t i12 = 0; i12 < ne12; i12++) | |||
| 1367 | if (!llamafile_sgemm(params, | |||
| 1368 | ne01, ne11, ne00/ggml_blck_size(src0->type), | |||
| 1369 | (const char *)src0->data + i12/r2*nb02 + i13/r3*nb03, | |||
| 1370 | nb01/ggml_type_size(src0->type), | |||
| 1371 | (const char *)wdata + (i12*ne11 + i13*ne12*ne11)*row_size, | |||
| 1372 | row_size/ggml_type_size(vec_dot_type), | |||
| 1373 | (char *)dst->data + i12*nb2 + i13*nb3, | |||
| 1374 | nb1/ggml_type_size(dst->type), | |||
| 1375 | src0->type, | |||
| 1376 | vec_dot_type, | |||
| 1377 | dst->type)) | |||
| 1378 | goto UseGgmlGemm2; | |||
| 1379 | return; | |||
| 1380 | } | |||
| 1381 | UseGgmlGemm2:; | |||
| 1382 | #endif | |||
| 1383 | ||||
| 1384 | // This is the size of the first dimension of the result, so we can iterate that way. (see the ASSERT above, these are the same numbers) | |||
| 1385 | const int64_t nr0 = ne0; | |||
| 1386 | ||||
| 1387 | // This is the size of the rest of the dimensions of the result | |||
| 1388 | const int64_t nr1 = ne1 * ne2 * ne3; | |||
| 1389 | ||||
| 1390 | // Now select a reasonable chunk size. | |||
| 1391 | int chunk_size = 16; | |||
| 1392 | ||||
| 1393 | // We need to step up the size if it's small | |||
| 1394 | if (nr0 == 1 || nr1 == 1) { | |||
| 1395 | chunk_size = 64; | |||
| 1396 | } | |||
| 1397 | ||||
| 1398 | // distribute the work across the inner or outer loop based on which one is larger | |||
| 1399 | // The number of chunks in the 0/1 dim. | |||
| 1400 | // CEIL(nr0/chunk_size) | |||
| 1401 | int64_t nchunk0 = (nr0 + chunk_size - 1) / chunk_size; | |||
| 1402 | int64_t nchunk1 = (nr1 + chunk_size - 1) / chunk_size; | |||
| 1403 | ||||
| 1404 | // If the chunking is poor for the number of threads on this setup, scrap the whole plan. Re-chunk it by thread. | |||
| 1405 | // Also, chunking by thread was measured to have perform better on NUMA systems. See https://github.com/ggml-org/llama.cpp/pull/6915 | |||
| 1406 | // In theory, chunking should be just as useful on NUMA and non NUMA systems, but testing disagreed with that. | |||
| 1407 | if (nchunk0 * nchunk1 < nth * 4 || ggml_is_numa()) { | |||
| 1408 | // distribute the thread work across the inner or outer loop based on which one is larger | |||
| 1409 | nchunk0 = nr0 > nr1 ? nth : 1; // parallelize by src0 rows | |||
| 1410 | nchunk1 = nr0 > nr1 ? 1 : nth; // parallelize by src1 rows | |||
| 1411 | } | |||
| 1412 | ||||
| 1413 | // The number of elements in each chunk | |||
| 1414 | const int64_t dr0 = (nr0 + nchunk0 - 1) / nchunk0; | |||
| 1415 | const int64_t dr1 = (nr1 + nchunk1 - 1) / nchunk1; | |||
| 1416 | ||||
| 1417 | // The first chunk comes from our thread_id, the rest will get auto-assigned. | |||
| 1418 | int current_chunk = ith; | |||
| 1419 | ||||
| 1420 | while (current_chunk < nchunk0 * nchunk1) { | |||
| 1421 | const int64_t ith0 = current_chunk % nchunk0; | |||
| 1422 | const int64_t ith1 = current_chunk / nchunk0; | |||
| 1423 | ||||
| 1424 | const int64_t ir0_start = dr0 * ith0; | |||
| 1425 | const int64_t ir0_end = MIN(ir0_start + dr0, nr0)((ir0_start + dr0) < (nr0) ? (ir0_start + dr0) : (nr0)); | |||
| 1426 | ||||
| 1427 | const int64_t ir1_start = dr1 * ith1; | |||
| 1428 | const int64_t ir1_end = MIN(ir1_start + dr1, nr1)((ir1_start + dr1) < (nr1) ? (ir1_start + dr1) : (nr1)); | |||
| 1429 | ||||
| 1430 | // dot kernels can handle 1 row and col at a time, but mmla kernels can process 2 rows and cols | |||
| 1431 | int64_t num_rows_per_vec_dot = vec_dot_num_rows; | |||
| 1432 | ||||
| 1433 | // these checks are needed to avoid crossing dim1 boundaries | |||
| 1434 | // can be optimized, but the logic would become more complicated, so keeping it like this for simplicity | |||
| 1435 | if ((nr0 % 2 != 0) || (ne11 % 2 != 0) || ((ir0_end - ir0_start) % 2 != 0) || ((ir1_end - ir1_start) % 2 != 0)) { | |||
| 1436 | num_rows_per_vec_dot = 1; | |||
| 1437 | } | |||
| 1438 | ggml_compute_forward_mul_mat_one_chunk(params, dst, src0->type, num_rows_per_vec_dot, ir0_start, ir0_end, ir1_start, ir1_end); | |||
| 1439 | ||||
| 1440 | if (nth >= nchunk0 * nchunk1) { | |||
| 1441 | break; | |||
| 1442 | } | |||
| 1443 | ||||
| 1444 | current_chunk = atomic_fetch_add_explicit__c11_atomic_fetch_add(¶ms->threadpool->current_chunk, 1, memory_order_relaxed); | |||
| 1445 | } | |||
| 1446 | } | |||
| 1447 | ||||
| 1448 | // ggml_compute_forward_mul_mat_id | |||
| 1449 | ||||
| 1450 | #define MMID_MATRIX_ROW(row_id, i1)matrix_rows[(row_id)*ids->ne[0]*ids->ne[1] + (i1)] matrix_rows[(row_id)*ids->ne[0]*ids->ne[1] + (i1)] | |||
| 1451 | ||||
| 1452 | struct mmid_row_mapping { | |||
| 1453 | int32_t i1; | |||
| 1454 | int32_t i2; | |||
| 1455 | }; | |||
| 1456 | ||||
| 1457 | static void ggml_compute_forward_mul_mat_id_one_chunk( | |||
| 1458 | struct ggml_tensor * dst, | |||
| 1459 | const struct ggml_tensor * src0, | |||
| 1460 | const struct ggml_tensor * src1, | |||
| 1461 | const struct ggml_tensor * ids, | |||
| 1462 | const int64_t cur_a, | |||
| 1463 | const int64_t ir0_start, | |||
| 1464 | const int64_t ir0_end, | |||
| 1465 | const int64_t ir1_start, | |||
| 1466 | const int64_t ir1_end, | |||
| 1467 | const char * src0_cur, | |||
| 1468 | const struct mmid_row_mapping * matrix_rows, | |||
| 1469 | const size_t row_size, | |||
| 1470 | const bool_Bool src1_cont, | |||
| 1471 | const void * wdata) { | |||
| 1472 | ||||
| 1473 | GGML_TENSOR_BINARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00 ); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void) (ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; ( void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] : 0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[ 0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)-> nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)-> nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)-> nb[3] : 0; (void)(nb03); const int64_t ne10 = (src1) ? (src1) ->ne[0] : 0; (void)(ne10); const int64_t ne11 = (src1) ? ( src1)->ne[1] : 0; (void)(ne11); const int64_t ne12 = (src1 ) ? (src1)->ne[2] : 0; (void)(ne12); const int64_t ne13 = ( src1) ? (src1)->ne[3] : 0; (void)(ne13); const size_t nb10 = (src1) ? (src1)->nb[0] : 0; (void)(nb10); const size_t nb11 = (src1) ? (src1)->nb[1] : 0; (void)(nb11); const size_t nb12 = (src1) ? (src1)->nb[2] : 0; (void)(nb12); const size_t nb13 = (src1) ? (src1)->nb[3] : 0; (void)(nb13); const int64_t ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0); const int64_t ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1); const int64_t ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2); const int64_t ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3); const size_t nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const size_t nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const size_t nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const size_t nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3); | |||
| 1474 | ||||
| 1475 | const enum ggml_type type = src0->type; | |||
| 1476 | ||||
| 1477 | ggml_vec_dot_t const vec_dot = type_traits_cpu[type].vec_dot; | |||
| 1478 | enum ggml_type const vec_dot_type = type_traits_cpu[type].vec_dot_type; | |||
| 1479 | ||||
| 1480 | const int64_t blck_0 = 16; | |||
| 1481 | const int64_t blck_1 = 16; | |||
| 1482 | ||||
| 1483 | float tmp[16]; | |||
| 1484 | ||||
| 1485 | for (int64_t iir1 = ir1_start; iir1 < ir1_end; iir1 += blck_1) { | |||
| 1486 | for (int64_t iir0 = ir0_start; iir0 < ir0_end; iir0 += blck_0) { | |||
| 1487 | for (int64_t ir1 = iir1; ir1 < iir1 + blck_1 && ir1 < ir1_end; ++ir1) { | |||
| 1488 | const int64_t _i12 = ir1; // logical row index for this expert | |||
| 1489 | ||||
| 1490 | struct mmid_row_mapping row_mapping = MMID_MATRIX_ROW(cur_a, _i12)matrix_rows[(cur_a)*ids->ne[0]*ids->ne[1] + (_i12)]; | |||
| 1491 | const int id = row_mapping.i1; // selected expert index | |||
| 1492 | ||||
| 1493 | const int64_t i11 = id % ne11; | |||
| 1494 | const int64_t i12 = row_mapping.i2; // row index in src1 | |||
| 1495 | ||||
| 1496 | const int64_t i1 = id; // selected expert index | |||
| 1497 | const int64_t i2 = i12; // row | |||
| 1498 | ||||
| 1499 | // desc: when src1 is not a contiguous memory block we have to calculate the offset using the strides | |||
| 1500 | // if it is, then we have either copied the data to params->wdata and made it contiguous or we are using | |||
| 1501 | // the original src1 data pointer, so we should index using the indices directly | |||
| 1502 | // TODO: this is a bit of a hack, we should probably have a better way to handle this | |||
| 1503 | const char * src1_col = (const char *) wdata + | |||
| 1504 | (src1_cont || src1->type != vec_dot_type | |||
| 1505 | ? (i11 + i12*ne11)*row_size | |||
| 1506 | : (i11*nb11 + i12*nb12)); | |||
| 1507 | ||||
| 1508 | float * dst_col = (float *) ((char *) dst->data + (i1*nb1 + i2*nb2)); | |||
| 1509 | ||||
| 1510 | for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir0_end; ++ir0) { | |||
| 1511 | vec_dot(ne00, &tmp[ir0 - iir0], 0, src0_cur + ir0*nb01, 0, src1_col, 0, 1); | |||
| 1512 | } | |||
| 1513 | ||||
| 1514 | memcpy(&dst_col[iir0], tmp, (MIN(iir0 + blck_0, ir0_end)((iir0 + blck_0) < (ir0_end) ? (iir0 + blck_0) : (ir0_end) ) - iir0)*sizeof(float)); | |||
| 1515 | } | |||
| 1516 | } | |||
| 1517 | } | |||
| 1518 | } | |||
| 1519 | ||||
| 1520 | static void * incr_ptr_aligned(void ** p, size_t size, size_t align) { | |||
| 1521 | ||||
| 1522 | void * ptr = *p; | |||
| 1523 | ptr = (void *) GGML_PAD((uintptr_t) ptr, align)((((uintptr_t) ptr) + (align) - 1) & ~((align) - 1)); | |||
| 1524 | *p = (void *) ((char *) ptr + size); | |||
| 1525 | return ptr; | |||
| 1526 | } | |||
| 1527 | ||||
| 1528 | static void ggml_compute_forward_mul_mat_id( | |||
| 1529 | const struct ggml_compute_params * params, | |||
| 1530 | struct ggml_tensor * dst) { | |||
| 1531 | ||||
| 1532 | const struct ggml_tensor * src0 = dst->src[0]; | |||
| 1533 | const struct ggml_tensor * src1 = dst->src[1]; | |||
| 1534 | const struct ggml_tensor * ids = dst->src[2]; | |||
| 1535 | ||||
| 1536 | GGML_TENSOR_BINARY_OP_LOCALSconst int64_t ne00 = (src0) ? (src0)->ne[0] : 0; (void)(ne00 ); const int64_t ne01 = (src0) ? (src0)->ne[1] : 0; (void) (ne01); const int64_t ne02 = (src0) ? (src0)->ne[2] : 0; ( void)(ne02); const int64_t ne03 = (src0) ? (src0)->ne[3] : 0; (void)(ne03); const size_t nb00 = (src0) ? (src0)->nb[ 0] : 0; (void)(nb00); const size_t nb01 = (src0) ? (src0)-> nb[1] : 0; (void)(nb01); const size_t nb02 = (src0) ? (src0)-> nb[2] : 0; (void)(nb02); const size_t nb03 = (src0) ? (src0)-> nb[3] : 0; (void)(nb03); const int64_t ne10 = (src1) ? (src1) ->ne[0] : 0; (void)(ne10); const int64_t ne11 = (src1) ? ( src1)->ne[1] : 0; (void)(ne11); const int64_t ne12 = (src1 ) ? (src1)->ne[2] : 0; (void)(ne12); const int64_t ne13 = ( src1) ? (src1)->ne[3] : 0; (void)(ne13); const size_t nb10 = (src1) ? (src1)->nb[0] : 0; (void)(nb10); const size_t nb11 = (src1) ? (src1)->nb[1] : 0; (void)(nb11); const size_t nb12 = (src1) ? (src1)->nb[2] : 0; (void)(nb12); const size_t nb13 = (src1) ? (src1)->nb[3] : 0; (void)(nb13); const int64_t ne0 = (dst) ? (dst)->ne[0] : 0; (void)(ne0); const int64_t ne1 = (dst) ? (dst)->ne[1] : 0; (void)(ne1); const int64_t ne2 = (dst) ? (dst)->ne[2] : 0; (void)(ne2); const int64_t ne3 = (dst) ? (dst)->ne[3] : 0; (void)(ne3); const size_t nb0 = (dst) ? (dst)->nb[0] : 0; (void)(nb0); const size_t nb1 = (dst) ? (dst)->nb[1] : 0; (void)(nb1); const size_t nb2 = (dst) ? (dst)->nb[2] : 0; (void)(nb2); const size_t nb3 = (dst) ? (dst)->nb[3] : 0; (void)(nb3); | |||
| 1537 | ||||
| 1538 | const int ith = params->ith; | |||
| 1539 | const int nth = params->nth; | |||
| 1540 | ||||
| 1541 | const enum ggml_type type = src0->type; | |||
| 1542 | ||||
| 1543 | const bool_Bool src1_cont = ggml_is_contiguous(src1); | |||
| 1544 | ||||
| 1545 | enum ggml_type const vec_dot_type = type_traits_cpu[type].vec_dot_type; | |||
| 1546 | ggml_from_float_t const from_float = type_traits_cpu[vec_dot_type].from_float; | |||
| 1547 | ||||
| 1548 | // we don't support permuted src0 or src1 | |||
| 1549 | GGML_ASSERT(nb00 == ggml_type_size(type))if (!(nb00 == ggml_type_size(type))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 1549, "GGML_ASSERT(%s) failed", "nb00 == ggml_type_size(type)" ); | |||
| 1550 | GGML_ASSERT(nb10 == ggml_type_size(src1->type))if (!(nb10 == ggml_type_size(src1->type))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 1550, "GGML_ASSERT(%s) failed", "nb10 == ggml_type_size(src1->type)" ); | |||
| 1551 | ||||
| 1552 | // dst cannot be transposed or permuted | |||
| 1553 | GGML_ASSERT(nb0 == sizeof(float))if (!(nb0 == sizeof(float))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 1553, "GGML_ASSERT(%s) failed", "nb0 == sizeof(float)"); | |||
| 1554 | GGML_ASSERT(nb0 <= nb1)if (!(nb0 <= nb1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 1554, "GGML_ASSERT(%s) failed", "nb0 <= nb1"); | |||
| 1555 | GGML_ASSERT(nb1 <= nb2)if (!(nb1 <= nb2)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 1555, "GGML_ASSERT(%s) failed", "nb1 <= nb2"); | |||
| 1556 | GGML_ASSERT(nb2 <= nb3)if (!(nb2 <= nb3)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 1556, "GGML_ASSERT(%s) failed", "nb2 <= nb3"); | |||
| 1557 | ||||
| 1558 | // row groups | |||
| 1559 | const int n_ids = ids->ne[0]; // n_expert_used | |||
| 1560 | const int n_as = ne02; // n_expert | |||
| 1561 | ||||
| 1562 | void * wdata_cur = params->wdata; | |||
| 1563 | ||||
| 1564 | if (src1->type != vec_dot_type) { | |||
| 1565 | incr_ptr_aligned(&wdata_cur, ggml_row_size(vec_dot_type, ggml_nelements(src1)), sizeof(int64_t)); | |||
| 1566 | } | |||
| 1567 | ||||
| 1568 | int64_t * matrix_row_counts = // [n_as] | |||
| 1569 | incr_ptr_aligned(&wdata_cur, n_as*sizeof(int64_t), sizeof(int64_t)); | |||
| 1570 | ||||
| 1571 | struct mmid_row_mapping * matrix_rows = // [n_as][ids->ne[0]*ids->ne[1]] | |||
| 1572 | incr_ptr_aligned(&wdata_cur, n_as*ids->ne[0]*ids->ne[1]*sizeof(struct mmid_row_mapping), sizeof(int64_t)); | |||
| 1573 | ||||
| 1574 | char (*atomic_current_chunk)[CACHE_LINE_SIZE64] = // [n_as] | |||
| 1575 | incr_ptr_aligned(&wdata_cur, CACHE_LINE_SIZE64 * n_as, CACHE_LINE_SIZE64); | |||
| 1576 | ||||
| 1577 | GGML_ASSERT(params->wsize >= (size_t)((char *) wdata_cur - (char *) params->wdata))if (!(params->wsize >= (size_t)((char *) wdata_cur - (char *) params->wdata))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 1577, "GGML_ASSERT(%s) failed", "params->wsize >= (size_t)((char *) wdata_cur - (char *) params->wdata)" ); | |||
| 1578 | ||||
| 1579 | if (src1->type != vec_dot_type) { | |||
| 1580 | char * wdata = params->wdata; | |||
| 1581 | ||||
| 1582 | const size_t nbw0 = ggml_type_size(vec_dot_type); | |||
| 1583 | const size_t nbw1 = ggml_row_size(vec_dot_type, ne10); | |||
| 1584 | const size_t nbw2 = nbw1*ne11; | |||
| 1585 | const size_t nbw3 = nbw2*ne12; | |||
| 1586 | ||||
| 1587 | assert(params->wsize >= ne13*nbw3)((void) sizeof (__assert_single_arg (params->wsize >= ne13 *nbw3)), __extension__ ({ if (params->wsize >= ne13*nbw3 ) ; else __assert_fail ("params->wsize >= ne13*nbw3", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 1587, __extension__ __PRETTY_FUNCTION__); })); | |||
| 1588 | GGML_ASSERT(src1->type == GGML_TYPE_F32)if (!(src1->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 1588, "GGML_ASSERT(%s) failed", "src1->type == GGML_TYPE_F32" ); | |||
| 1589 | ||||
| 1590 | #if 0 | |||
| 1591 | for (int64_t i13 = 0; i13 < ne13; ++i13) { | |||
| 1592 | for (int64_t i12 = ith; i12 < ne12; i12 += nth) { | |||
| 1593 | for (int64_t i11 = 0; i11 < ne11; ++i11) { | |||
| 1594 | from_float((float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11), | |||
| 1595 | (void *) (wdata + i13*nbw3 + i12*nbw2 + i11*nbw1), | |||
| 1596 | ne10); | |||
| 1597 | } | |||
| 1598 | } | |||
| 1599 | } | |||
| 1600 | #else | |||
| 1601 | for (int64_t i13 = 0; i13 < ne13; ++i13) { | |||
| 1602 | for (int64_t i12 = 0; i12 < ne12; ++i12) { | |||
| 1603 | for (int64_t i11 = 0; i11 < ne11; ++i11) { | |||
| 1604 | size_t bs = ggml_blck_size(vec_dot_type); | |||
| 1605 | int64_t ne10_block_start = (ith * ne10/bs) / nth; | |||
| 1606 | int64_t ne10_block_end = ((ith + 1) * ne10/bs) / nth; | |||
| 1607 | from_float((float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11 + ne10_block_start*bs*nb10), | |||
| 1608 | (void *) (wdata + i13*nbw3 + i12*nbw2 + i11*nbw1 + ne10_block_start*nbw0), | |||
| 1609 | (ne10_block_end - ne10_block_start) * bs); | |||
| 1610 | } | |||
| 1611 | } | |||
| 1612 | } | |||
| 1613 | #endif | |||
| 1614 | } | |||
| 1615 | ||||
| 1616 | if (ith == 0) { | |||
| 1617 | // initialize matrix_row_counts | |||
| 1618 | memset(matrix_row_counts, 0, n_as*sizeof(int64_t)); | |||
| 1619 | ||||
| 1620 | // group rows by src0 matrix | |||
| 1621 | for (int64_t iid1 = 0; iid1 < ids->ne[1]; ++iid1) { | |||
| 1622 | for (int id = 0; id < n_ids; ++id) { | |||
| 1623 | const int32_t i02 = *(const int32_t *) ((const char *) ids->data + iid1*ids->nb[1] + id*ids->nb[0]); | |||
| 1624 | ||||
| 1625 | assert(i02 >= 0 && i02 < n_as)((void) sizeof (__assert_single_arg (i02 >= 0 && i02 < n_as)), __extension__ ({ if (i02 >= 0 && i02 < n_as) ; else __assert_fail ("i02 >= 0 && i02 < n_as" , "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 1625, __extension__ __PRETTY_FUNCTION__); })); | |||
| 1626 | ||||
| 1627 | MMID_MATRIX_ROW(i02, matrix_row_counts[i02])matrix_rows[(i02)*ids->ne[0]*ids->ne[1] + (matrix_row_counts [i02])] = (struct mmid_row_mapping) {id, iid1}; | |||
| 1628 | matrix_row_counts[i02] += 1; | |||
| 1629 | } | |||
| 1630 | } | |||
| 1631 | } | |||
| 1632 | ||||
| 1633 | // reset current_chunk | |||
| 1634 | for (int cur_a = ith; cur_a < n_as; cur_a += nth) { | |||
| 1635 | atomic_int * current_chunk_ctr = (atomic_int *)(atomic_current_chunk + cur_a); | |||
| 1636 | *current_chunk_ctr = nth; | |||
| 1637 | } | |||
| 1638 | ||||
| 1639 | ggml_barrier(params->threadpool); | |||
| 1640 | ||||
| 1641 | for (int cur_a = 0; cur_a < n_as; ++cur_a) { | |||
| 1642 | const int64_t cne1 = matrix_row_counts[cur_a]; | |||
| 1643 | ||||
| 1644 | if (cne1 == 0) { | |||
| 1645 | continue; | |||
| 1646 | } | |||
| 1647 | ||||
| 1648 | const char * src0_cur = (const char *) src0->data + cur_a * nb02; | |||
| 1649 | const void * wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata; | |||
| 1650 | const size_t row_size = ggml_row_size(vec_dot_type, ne10); | |||
| 1651 | ||||
| 1652 | const int64_t nr0 = ne01; | |||
| 1653 | const int64_t nr1 = cne1; | |||
| 1654 | ||||
| 1655 | int chunk_size = 16; | |||
| 1656 | if (nr0 == 1 || nr1 == 1) { | |||
| 1657 | chunk_size = 64; | |||
| 1658 | } | |||
| 1659 | ||||
| 1660 | // disable for NUMA | |||
| 1661 | const bool_Bool disable_chunking = ggml_is_numa(); | |||
| 1662 | ||||
| 1663 | int64_t nchunk0 = (nr0 + chunk_size - 1) / chunk_size; | |||
| 1664 | int64_t nchunk1 = (nr1 + chunk_size - 1) / chunk_size; | |||
| 1665 | ||||
| 1666 | if (nchunk0 * nchunk1 < nth * 4 || disable_chunking) { | |||
| 1667 | nchunk0 = nr0 > nr1 ? nth : 1; | |||
| 1668 | nchunk1 = nr0 > nr1 ? 1 : nth; | |||
| 1669 | } | |||
| 1670 | ||||
| 1671 | const int64_t dr0 = (nr0 + nchunk0 - 1) / nchunk0; | |||
| 1672 | const int64_t dr1 = (nr1 + nchunk1 - 1) / nchunk1; | |||
| 1673 | ||||
| 1674 | int current_chunk = ith; | |||
| 1675 | ||||
| 1676 | atomic_int * current_chunk_ctr = (atomic_int *)(atomic_current_chunk + cur_a); | |||
| 1677 | ||||
| 1678 | while (current_chunk < nchunk0 * nchunk1) { | |||
| 1679 | const int64_t ith0 = current_chunk % nchunk0; | |||
| 1680 | const int64_t ith1 = current_chunk / nchunk0; | |||
| 1681 | ||||
| 1682 | const int64_t ir0_start = dr0 * ith0; | |||
| 1683 | const int64_t ir0_end = MIN(ir0_start + dr0, nr0)((ir0_start + dr0) < (nr0) ? (ir0_start + dr0) : (nr0)); | |||
| 1684 | ||||
| 1685 | const int64_t ir1_start = dr1 * ith1; | |||
| 1686 | const int64_t ir1_end = MIN(ir1_start + dr1, nr1)((ir1_start + dr1) < (nr1) ? (ir1_start + dr1) : (nr1)); | |||
| 1687 | ||||
| 1688 | ggml_compute_forward_mul_mat_id_one_chunk( | |||
| 1689 | dst, src0, src1, ids, cur_a, | |||
| 1690 | ir0_start, ir0_end, ir1_start, ir1_end, | |||
| 1691 | src0_cur, matrix_rows, row_size, src1_cont, wdata | |||
| 1692 | ); | |||
| 1693 | ||||
| 1694 | if (nth >= nchunk0 * nchunk1) { | |||
| 1695 | break; | |||
| 1696 | } | |||
| 1697 | ||||
| 1698 | current_chunk = atomic_fetch_add_explicit__c11_atomic_fetch_add(current_chunk_ctr, 1, memory_order_relaxed); | |||
| 1699 | } | |||
| 1700 | } | |||
| 1701 | } | |||
| 1702 | ||||
| 1703 | ///////////////////////////////// | |||
| 1704 | ||||
| 1705 | static void ggml_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * tensor) { | |||
| 1706 | GGML_ASSERT(params)if (!(params)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 1706, "GGML_ASSERT(%s) failed", "params"); | |||
| 1707 | ||||
| 1708 | if (tensor->op == GGML_OP_NONE || ggml_is_empty(tensor)) { | |||
| 1709 | return; | |||
| 1710 | } | |||
| 1711 | ||||
| 1712 | // extra_buffer op? | |||
| 1713 | if (ggml_cpu_extra_compute_forward(params, tensor)) { | |||
| 1714 | return; | |||
| 1715 | } | |||
| 1716 | ||||
| 1717 | switch (tensor->op) { | |||
| 1718 | case GGML_OP_DUP: | |||
| 1719 | { | |||
| 1720 | ggml_compute_forward_dup(params, tensor); | |||
| 1721 | } break; | |||
| 1722 | case GGML_OP_ADD: | |||
| 1723 | { | |||
| 1724 | ggml_compute_forward_add(params, tensor); | |||
| 1725 | } break; | |||
| 1726 | case GGML_OP_ADD_ID: | |||
| 1727 | { | |||
| 1728 | ggml_compute_forward_add_id(params, tensor); | |||
| 1729 | } break; | |||
| 1730 | case GGML_OP_ADD1: | |||
| 1731 | { | |||
| 1732 | ggml_compute_forward_add1(params, tensor); | |||
| 1733 | } break; | |||
| 1734 | case GGML_OP_ACC: | |||
| 1735 | { | |||
| 1736 | ggml_compute_forward_acc(params, tensor); | |||
| 1737 | } break; | |||
| 1738 | case GGML_OP_SUB: | |||
| 1739 | { | |||
| 1740 | ggml_compute_forward_sub(params, tensor); | |||
| 1741 | } break; | |||
| 1742 | case GGML_OP_MUL: | |||
| 1743 | { | |||
| 1744 | ggml_compute_forward_mul(params, tensor); | |||
| 1745 | } break; | |||
| 1746 | case GGML_OP_DIV: | |||
| 1747 | { | |||
| 1748 | ggml_compute_forward_div(params, tensor); | |||
| 1749 | } break; | |||
| 1750 | case GGML_OP_SQR: | |||
| 1751 | { | |||
| 1752 | ggml_compute_forward_sqr(params, tensor); | |||
| 1753 | } break; | |||
| 1754 | case GGML_OP_SQRT: | |||
| 1755 | { | |||
| 1756 | ggml_compute_forward_sqrt(params, tensor); | |||
| 1757 | } break; | |||
| 1758 | case GGML_OP_LOG: | |||
| 1759 | { | |||
| 1760 | ggml_compute_forward_log(params, tensor); | |||
| 1761 | } break; | |||
| 1762 | case GGML_OP_SIN: | |||
| 1763 | { | |||
| 1764 | ggml_compute_forward_sin(params, tensor); | |||
| 1765 | } break; | |||
| 1766 | case GGML_OP_COS: | |||
| 1767 | { | |||
| 1768 | ggml_compute_forward_cos(params, tensor); | |||
| 1769 | } break; | |||
| 1770 | case GGML_OP_SUM: | |||
| 1771 | { | |||
| 1772 | ggml_compute_forward_sum(params, tensor); | |||
| 1773 | } break; | |||
| 1774 | case GGML_OP_SUM_ROWS: | |||
| 1775 | { | |||
| 1776 | ggml_compute_forward_sum_rows(params, tensor); | |||
| 1777 | } break; | |||
| 1778 | case GGML_OP_CUMSUM: | |||
| 1779 | { | |||
| 1780 | ggml_compute_forward_cumsum(params, tensor); | |||
| 1781 | } break; | |||
| 1782 | case GGML_OP_MEAN: | |||
| 1783 | { | |||
| 1784 | ggml_compute_forward_mean(params, tensor); | |||
| 1785 | } break; | |||
| 1786 | case GGML_OP_ARGMAX: | |||
| 1787 | { | |||
| 1788 | ggml_compute_forward_argmax(params, tensor); | |||
| 1789 | } break; | |||
| 1790 | case GGML_OP_COUNT_EQUAL: | |||
| 1791 | { | |||
| 1792 | ggml_compute_forward_count_equal(params, tensor); | |||
| 1793 | } break; | |||
| 1794 | case GGML_OP_REPEAT: | |||
| 1795 | { | |||
| 1796 | ggml_compute_forward_repeat(params, tensor); | |||
| 1797 | } break; | |||
| 1798 | case GGML_OP_REPEAT_BACK: | |||
| 1799 | { | |||
| 1800 | ggml_compute_forward_repeat_back(params, tensor); | |||
| 1801 | } break; | |||
| 1802 | case GGML_OP_CONCAT: | |||
| 1803 | { | |||
| 1804 | ggml_compute_forward_concat(params, tensor); | |||
| 1805 | } break; | |||
| 1806 | case GGML_OP_SILU_BACK: | |||
| 1807 | { | |||
| 1808 | ggml_compute_forward_silu_back(params, tensor); | |||
| 1809 | } break; | |||
| 1810 | case GGML_OP_NORM: | |||
| 1811 | { | |||
| 1812 | ggml_compute_forward_norm(params, tensor); | |||
| 1813 | } break; | |||
| 1814 | case GGML_OP_RMS_NORM: | |||
| 1815 | { | |||
| 1816 | ggml_compute_forward_rms_norm(params, tensor); | |||
| 1817 | } break; | |||
| 1818 | case GGML_OP_RMS_NORM_BACK: | |||
| 1819 | { | |||
| 1820 | ggml_compute_forward_rms_norm_back(params, tensor); | |||
| 1821 | } break; | |||
| 1822 | case GGML_OP_GROUP_NORM: | |||
| 1823 | { | |||
| 1824 | ggml_compute_forward_group_norm(params, tensor); | |||
| 1825 | } break; | |||
| 1826 | case GGML_OP_L2_NORM: | |||
| 1827 | { | |||
| 1828 | ggml_compute_forward_l2_norm(params, tensor); | |||
| 1829 | } break; | |||
| 1830 | case GGML_OP_MUL_MAT: | |||
| 1831 | { | |||
| 1832 | ggml_compute_forward_mul_mat(params, tensor); | |||
| 1833 | } break; | |||
| 1834 | case GGML_OP_MUL_MAT_ID: | |||
| 1835 | { | |||
| 1836 | ggml_compute_forward_mul_mat_id(params, tensor); | |||
| 1837 | } break; | |||
| 1838 | case GGML_OP_OUT_PROD: | |||
| 1839 | { | |||
| 1840 | ggml_compute_forward_out_prod(params, tensor); | |||
| 1841 | } break; | |||
| 1842 | case GGML_OP_SCALE: | |||
| 1843 | { | |||
| 1844 | ggml_compute_forward_scale(params, tensor); | |||
| 1845 | } break; | |||
| 1846 | case GGML_OP_SET: | |||
| 1847 | { | |||
| 1848 | ggml_compute_forward_set(params, tensor); | |||
| 1849 | } break; | |||
| 1850 | case GGML_OP_CPY: | |||
| 1851 | { | |||
| 1852 | ggml_compute_forward_cpy(params, tensor); | |||
| 1853 | } break; | |||
| 1854 | case GGML_OP_CONT: | |||
| 1855 | { | |||
| 1856 | ggml_compute_forward_cont(params, tensor); | |||
| 1857 | } break; | |||
| 1858 | case GGML_OP_GET_ROWS: | |||
| 1859 | { | |||
| 1860 | ggml_compute_forward_get_rows(params, tensor); | |||
| 1861 | } break; | |||
| 1862 | case GGML_OP_GET_ROWS_BACK: | |||
| 1863 | { | |||
| 1864 | ggml_compute_forward_get_rows_back(params, tensor); | |||
| 1865 | } break; | |||
| 1866 | case GGML_OP_SET_ROWS: | |||
| 1867 | { | |||
| 1868 | ggml_compute_forward_set_rows(params, tensor); | |||
| 1869 | } break; | |||
| 1870 | case GGML_OP_DIAG: | |||
| 1871 | { | |||
| 1872 | ggml_compute_forward_diag(params, tensor); | |||
| 1873 | } break; | |||
| 1874 | case GGML_OP_DIAG_MASK_INF: | |||
| 1875 | { | |||
| 1876 | ggml_compute_forward_diag_mask_inf(params, tensor); | |||
| 1877 | } break; | |||
| 1878 | case GGML_OP_DIAG_MASK_ZERO: | |||
| 1879 | { | |||
| 1880 | ggml_compute_forward_diag_mask_zero(params, tensor); | |||
| 1881 | } break; | |||
| 1882 | case GGML_OP_SOFT_MAX: | |||
| 1883 | { | |||
| 1884 | ggml_compute_forward_soft_max(params, tensor); | |||
| 1885 | } break; | |||
| 1886 | case GGML_OP_SOFT_MAX_BACK: | |||
| 1887 | { | |||
| 1888 | ggml_compute_forward_soft_max_ext_back(params, tensor); | |||
| 1889 | } break; | |||
| 1890 | case GGML_OP_ROPE: | |||
| 1891 | { | |||
| 1892 | ggml_compute_forward_rope(params, tensor); | |||
| 1893 | } break; | |||
| 1894 | case GGML_OP_ROPE_BACK: | |||
| 1895 | { | |||
| 1896 | ggml_compute_forward_rope_back(params, tensor); | |||
| 1897 | } break; | |||
| 1898 | case GGML_OP_CLAMP: | |||
| 1899 | { | |||
| 1900 | ggml_compute_forward_clamp(params, tensor); | |||
| 1901 | } break; | |||
| 1902 | case GGML_OP_CONV_TRANSPOSE_1D: | |||
| 1903 | { | |||
| 1904 | ggml_compute_forward_conv_transpose_1d(params, tensor); | |||
| 1905 | } break; | |||
| 1906 | case GGML_OP_IM2COL: | |||
| 1907 | { | |||
| 1908 | ggml_compute_forward_im2col(params, tensor); | |||
| 1909 | } break; | |||
| 1910 | case GGML_OP_IM2COL_BACK: | |||
| 1911 | { | |||
| 1912 | ggml_compute_forward_im2col_back_f32(params, tensor); | |||
| 1913 | } break; | |||
| 1914 | case GGML_OP_IM2COL_3D: | |||
| 1915 | { | |||
| 1916 | ggml_compute_forward_im2col_3d(params, tensor); | |||
| 1917 | } break; | |||
| 1918 | case GGML_OP_COL2IM_1D: | |||
| 1919 | { | |||
| 1920 | ggml_compute_forward_col2im_1d(params, tensor); | |||
| 1921 | } break; | |||
| 1922 | case GGML_OP_CONV_2D: | |||
| 1923 | { | |||
| 1924 | ggml_compute_forward_conv_2d(params, tensor); | |||
| 1925 | } break; | |||
| 1926 | case GGML_OP_CONV_3D: | |||
| 1927 | { | |||
| 1928 | ggml_compute_forward_conv_3d(params, tensor); | |||
| 1929 | } break; | |||
| 1930 | case GGML_OP_CONV_2D_DW: | |||
| 1931 | { | |||
| 1932 | ggml_compute_forward_conv_2d_dw(params, tensor); | |||
| 1933 | } break; | |||
| 1934 | case GGML_OP_CONV_TRANSPOSE_2D: | |||
| 1935 | { | |||
| 1936 | ggml_compute_forward_conv_transpose_2d(params, tensor); | |||
| 1937 | } break; | |||
| 1938 | case GGML_OP_POOL_1D: | |||
| 1939 | { | |||
| 1940 | ggml_compute_forward_pool_1d(params, tensor); | |||
| 1941 | } break; | |||
| 1942 | case GGML_OP_POOL_2D: | |||
| 1943 | { | |||
| 1944 | ggml_compute_forward_pool_2d(params, tensor); | |||
| 1945 | } break; | |||
| 1946 | case GGML_OP_POOL_2D_BACK: | |||
| 1947 | { | |||
| 1948 | ggml_compute_forward_pool_2d_back(params, tensor); | |||
| 1949 | } break; | |||
| 1950 | case GGML_OP_UPSCALE: | |||
| 1951 | { | |||
| 1952 | ggml_compute_forward_upscale(params, tensor); | |||
| 1953 | } break; | |||
| 1954 | case GGML_OP_PAD: | |||
| 1955 | { | |||
| 1956 | ggml_compute_forward_pad(params, tensor); | |||
| 1957 | } break; | |||
| 1958 | case GGML_OP_PAD_REFLECT_1D: | |||
| 1959 | { | |||
| 1960 | ggml_compute_forward_pad_reflect_1d(params, tensor); | |||
| 1961 | } break; | |||
| 1962 | case GGML_OP_ROLL: | |||
| 1963 | { | |||
| 1964 | ggml_compute_forward_roll(params, tensor); | |||
| 1965 | } break; | |||
| 1966 | case GGML_OP_ARANGE: | |||
| 1967 | { | |||
| 1968 | ggml_compute_forward_arange(params, tensor); | |||
| 1969 | } break; | |||
| 1970 | case GGML_OP_TIMESTEP_EMBEDDING: | |||
| 1971 | { | |||
| 1972 | ggml_compute_forward_timestep_embedding(params, tensor); | |||
| 1973 | } break; | |||
| 1974 | case GGML_OP_ARGSORT: | |||
| 1975 | { | |||
| 1976 | ggml_compute_forward_argsort(params, tensor); | |||
| 1977 | } break; | |||
| 1978 | case GGML_OP_TOP_K: | |||
| 1979 | { | |||
| 1980 | ggml_compute_forward_top_k(params, tensor); | |||
| 1981 | } break; | |||
| 1982 | case GGML_OP_LEAKY_RELU: | |||
| 1983 | { | |||
| 1984 | ggml_compute_forward_leaky_relu(params, tensor); | |||
| 1985 | } break; | |||
| 1986 | case GGML_OP_TRI: | |||
| 1987 | { | |||
| 1988 | ggml_compute_forward_tri(params, tensor); | |||
| 1989 | } break; | |||
| 1990 | case GGML_OP_FILL: | |||
| 1991 | { | |||
| 1992 | ggml_compute_forward_fill(params, tensor); | |||
| 1993 | } break; | |||
| 1994 | case GGML_OP_FLASH_ATTN_EXT: | |||
| 1995 | { | |||
| 1996 | ggml_compute_forward_flash_attn_ext(params, tensor); | |||
| 1997 | } break; | |||
| 1998 | case GGML_OP_FLASH_ATTN_BACK: | |||
| 1999 | { | |||
| 2000 | int32_t t = ggml_get_op_params_i32(tensor, 0); | |||
| 2001 | GGML_ASSERT(t == 0 || t == 1)if (!(t == 0 || t == 1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 2001, "GGML_ASSERT(%s) failed", "t == 0 || t == 1"); | |||
| 2002 | bool_Bool masked = t != 0; | |||
| 2003 | ggml_compute_forward_flash_attn_back(params, masked, tensor); | |||
| 2004 | } break; | |||
| 2005 | case GGML_OP_SSM_CONV: | |||
| 2006 | { | |||
| 2007 | ggml_compute_forward_ssm_conv(params, tensor); | |||
| 2008 | } break; | |||
| 2009 | case GGML_OP_SSM_SCAN: | |||
| 2010 | { | |||
| 2011 | ggml_compute_forward_ssm_scan(params, tensor); | |||
| 2012 | } break; | |||
| 2013 | case GGML_OP_WIN_PART: | |||
| 2014 | { | |||
| 2015 | ggml_compute_forward_win_part(params, tensor); | |||
| 2016 | } break; | |||
| 2017 | case GGML_OP_WIN_UNPART: | |||
| 2018 | { | |||
| 2019 | ggml_compute_forward_win_unpart(params, tensor); | |||
| 2020 | } break; | |||
| 2021 | case GGML_OP_UNARY: | |||
| 2022 | { | |||
| 2023 | ggml_compute_forward_unary(params, tensor); | |||
| 2024 | } break; | |||
| 2025 | case GGML_OP_GLU: | |||
| 2026 | { | |||
| 2027 | ggml_compute_forward_glu(params, tensor); | |||
| 2028 | } break; | |||
| 2029 | case GGML_OP_GET_REL_POS: | |||
| 2030 | { | |||
| 2031 | ggml_compute_forward_get_rel_pos(params, tensor); | |||
| 2032 | } break; | |||
| 2033 | case GGML_OP_ADD_REL_POS: | |||
| 2034 | { | |||
| 2035 | ggml_compute_forward_add_rel_pos(params, tensor); | |||
| 2036 | } break; | |||
| 2037 | case GGML_OP_RWKV_WKV6: | |||
| 2038 | { | |||
| 2039 | ggml_compute_forward_rwkv_wkv6(params, tensor); | |||
| 2040 | } break; | |||
| 2041 | case GGML_OP_GATED_LINEAR_ATTN: | |||
| 2042 | { | |||
| 2043 | ggml_compute_forward_gla(params, tensor); | |||
| 2044 | } break; | |||
| 2045 | case GGML_OP_RWKV_WKV7: | |||
| 2046 | { | |||
| 2047 | ggml_compute_forward_rwkv_wkv7(params, tensor); | |||
| 2048 | } break; | |||
| 2049 | case GGML_OP_SOLVE_TRI: | |||
| 2050 | { | |||
| 2051 | ggml_compute_forward_solve_tri(params, tensor); | |||
| 2052 | } break; | |||
| 2053 | case GGML_OP_GATED_DELTA_NET: | |||
| 2054 | { | |||
| 2055 | ggml_compute_forward_gated_delta_net(params, tensor); | |||
| 2056 | } break; | |||
| 2057 | case GGML_OP_MAP_CUSTOM1: | |||
| 2058 | { | |||
| 2059 | ggml_compute_forward_map_custom1(params, tensor); | |||
| 2060 | } | |||
| 2061 | break; | |||
| 2062 | case GGML_OP_MAP_CUSTOM2: | |||
| 2063 | { | |||
| 2064 | ggml_compute_forward_map_custom2(params, tensor); | |||
| 2065 | } | |||
| 2066 | break; | |||
| 2067 | case GGML_OP_MAP_CUSTOM3: | |||
| 2068 | { | |||
| 2069 | ggml_compute_forward_map_custom3(params, tensor); | |||
| 2070 | } | |||
| 2071 | break; | |||
| 2072 | case GGML_OP_CUSTOM: | |||
| 2073 | { | |||
| 2074 | ggml_compute_forward_custom(params, tensor); | |||
| 2075 | } | |||
| 2076 | break; | |||
| 2077 | case GGML_OP_CROSS_ENTROPY_LOSS: | |||
| 2078 | { | |||
| 2079 | ggml_compute_forward_cross_entropy_loss(params, tensor); | |||
| 2080 | } | |||
| 2081 | break; | |||
| 2082 | case GGML_OP_CROSS_ENTROPY_LOSS_BACK: | |||
| 2083 | { | |||
| 2084 | ggml_compute_forward_cross_entropy_loss_back(params, tensor); | |||
| 2085 | } | |||
| 2086 | break; | |||
| 2087 | case GGML_OP_OPT_STEP_ADAMW: | |||
| 2088 | { | |||
| 2089 | ggml_compute_forward_opt_step_adamw(params, tensor); | |||
| 2090 | } | |||
| 2091 | break; | |||
| 2092 | case GGML_OP_OPT_STEP_SGD: | |||
| 2093 | { | |||
| 2094 | ggml_compute_forward_opt_step_sgd(params, tensor); | |||
| 2095 | } | |||
| 2096 | break; | |||
| 2097 | case GGML_OP_NONE: | |||
| 2098 | { | |||
| 2099 | // nop | |||
| 2100 | } break; | |||
| 2101 | case GGML_OP_RESHAPE: | |||
| 2102 | { | |||
| 2103 | // nop | |||
| 2104 | } break; | |||
| 2105 | case GGML_OP_PERMUTE: | |||
| 2106 | { | |||
| 2107 | // nop | |||
| 2108 | } break; | |||
| 2109 | case GGML_OP_VIEW: | |||
| 2110 | { | |||
| 2111 | // nop | |||
| 2112 | } break; | |||
| 2113 | case GGML_OP_TRANSPOSE: | |||
| 2114 | { | |||
| 2115 | // nop | |||
| 2116 | } break; | |||
| 2117 | case GGML_OP_COUNT: | |||
| 2118 | { | |||
| 2119 | GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 2119, "fatal error"); | |||
| 2120 | } | |||
| 2121 | } | |||
| 2122 | } | |||
| 2123 | ||||
| 2124 | // Android's libc implementation "bionic" does not support setting affinity | |||
| 2125 | #if defined(__gnu_linux__1) | |||
| 2126 | static void set_numa_thread_affinity(int thread_n) { | |||
| 2127 | if (!ggml_is_numa()) { | |||
| 2128 | return; | |||
| 2129 | } | |||
| 2130 | ||||
| 2131 | int node_num; | |||
| 2132 | int rv; | |||
| 2133 | size_t setsize = CPU_ALLOC_SIZE(g_state.numa.total_cpus)((((g_state.numa.total_cpus) + (8 * sizeof (__cpu_mask)) - 1) / (8 * sizeof (__cpu_mask))) * sizeof (__cpu_mask)); | |||
| 2134 | ||||
| 2135 | switch(g_state.numa.numa_strategy) { | |||
| 2136 | case GGML_NUMA_STRATEGY_DISTRIBUTE: | |||
| 2137 | // run thread on node_num thread_n / (threads per node) | |||
| 2138 | node_num = thread_n % g_state.numa.n_nodes; | |||
| 2139 | break; | |||
| 2140 | case GGML_NUMA_STRATEGY_ISOLATE: | |||
| 2141 | // run thread on current_node | |||
| 2142 | node_num = g_state.numa.current_node; | |||
| 2143 | break; | |||
| 2144 | case GGML_NUMA_STRATEGY_NUMACTL: | |||
| 2145 | // use the cpuset that numactl gave us | |||
| 2146 | rv = pthread_setaffinity_np(pthread_self(), setsize, &g_state.numa.cpuset); | |||
| 2147 | if (rv) { | |||
| 2148 | fprintf(stderrstderr, "warning: pthread_setaffinity_np() failed: %s\n",strerror(rv)); | |||
| 2149 | } | |||
| 2150 | return; | |||
| 2151 | default: | |||
| 2152 | return; | |||
| 2153 | } | |||
| 2154 | ||||
| 2155 | struct ggml_numa_node * node = &g_state.numa.nodes[node_num]; | |||
| 2156 | ||||
| 2157 | cpu_set_t * cpus = CPU_ALLOC(g_state.numa.total_cpus)__sched_cpualloc (g_state.numa.total_cpus); | |||
| 2158 | CPU_ZERO_S(setsize, cpus)do __builtin_memset (cpus, '\0', setsize); while (0); | |||
| 2159 | for (size_t i = 0; i < node->n_cpus; ++i) { | |||
| 2160 | CPU_SET_S(node->cpus[i], setsize, cpus)(__extension__ ({ size_t __cpu = (node->cpus[i]); __cpu / 8 < (setsize) ? (((__cpu_mask *) ((cpus)->__bits))[((__cpu ) / (8 * sizeof (__cpu_mask)))] |= ((__cpu_mask) 1 << ( (__cpu) % (8 * sizeof (__cpu_mask))))) : 0; })); | |||
| 2161 | } | |||
| 2162 | ||||
| 2163 | rv = pthread_setaffinity_np(pthread_self(), setsize, cpus); | |||
| 2164 | if (rv) { | |||
| 2165 | fprintf(stderrstderr, "warning: pthread_setaffinity_np() failed: %s\n", strerror(rv)); | |||
| 2166 | } | |||
| 2167 | ||||
| 2168 | CPU_FREE(cpus)__sched_cpufree (cpus); | |||
| 2169 | } | |||
| 2170 | ||||
| 2171 | static void clear_numa_thread_affinity(void) { | |||
| 2172 | if (!ggml_is_numa()) { | |||
| 2173 | return; | |||
| 2174 | } | |||
| 2175 | ||||
| 2176 | size_t setsize = CPU_ALLOC_SIZE(g_state.numa.total_cpus)((((g_state.numa.total_cpus) + (8 * sizeof (__cpu_mask)) - 1) / (8 * sizeof (__cpu_mask))) * sizeof (__cpu_mask)); | |||
| 2177 | ||||
| 2178 | cpu_set_t * cpus = CPU_ALLOC(g_state.numa.total_cpus)__sched_cpualloc (g_state.numa.total_cpus); | |||
| 2179 | CPU_ZERO_S(setsize, cpus)do __builtin_memset (cpus, '\0', setsize); while (0); | |||
| 2180 | for (unsigned i = 0; i < g_state.numa.total_cpus; ++i) { | |||
| 2181 | CPU_SET_S(i, setsize, cpus)(__extension__ ({ size_t __cpu = (i); __cpu / 8 < (setsize ) ? (((__cpu_mask *) ((cpus)->__bits))[((__cpu) / (8 * sizeof (__cpu_mask)))] |= ((__cpu_mask) 1 << ((__cpu) % (8 * sizeof (__cpu_mask))))) : 0; })); | |||
| 2182 | } | |||
| 2183 | ||||
| 2184 | int rv = pthread_setaffinity_np(pthread_self(), setsize, cpus); | |||
| 2185 | if (rv) { | |||
| 2186 | fprintf(stderrstderr, "warning: pthread_setaffinity_np() failed: %s\n", strerror(rv)); | |||
| 2187 | } | |||
| 2188 | ||||
| 2189 | CPU_FREE(cpus)__sched_cpufree (cpus); | |||
| 2190 | } | |||
| 2191 | #else | |||
| 2192 | // TODO: Windows etc. | |||
| 2193 | // (the linux implementation may also work on BSD, someone should test) | |||
| 2194 | static void set_numa_thread_affinity(int thread_n) { UNUSED(thread_n)(void)(thread_n); } | |||
| 2195 | static void clear_numa_thread_affinity(void) {} | |||
| 2196 | #endif | |||
| 2197 | ||||
| 2198 | static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) { | |||
| 2199 | int n_tasks = 0; | |||
| 2200 | ||||
| 2201 | if (ggml_is_empty(node)) { | |||
| 2202 | // no need to multi-thread a no-op | |||
| 2203 | n_tasks = 1; | |||
| 2204 | return n_tasks; | |||
| 2205 | } | |||
| 2206 | ||||
| 2207 | switch (node->op) { | |||
| 2208 | case GGML_OP_CPY: | |||
| 2209 | case GGML_OP_DUP: | |||
| 2210 | case GGML_OP_CONT: | |||
| 2211 | case GGML_OP_ADD: | |||
| 2212 | case GGML_OP_ADD_ID: | |||
| 2213 | case GGML_OP_ADD1: | |||
| 2214 | case GGML_OP_ACC: | |||
| 2215 | case GGML_OP_CUMSUM: | |||
| 2216 | case GGML_OP_TRI: | |||
| 2217 | case GGML_OP_FILL: | |||
| 2218 | { | |||
| 2219 | n_tasks = n_threads; | |||
| 2220 | } break; | |||
| 2221 | case GGML_OP_SUB: | |||
| 2222 | case GGML_OP_SQR: | |||
| 2223 | case GGML_OP_SQRT: | |||
| 2224 | case GGML_OP_LOG: | |||
| 2225 | case GGML_OP_SIN: | |||
| 2226 | case GGML_OP_COS: | |||
| 2227 | case GGML_OP_SUM: | |||
| 2228 | case GGML_OP_SUM_ROWS: | |||
| 2229 | case GGML_OP_MEAN: | |||
| 2230 | case GGML_OP_ARGMAX: | |||
| 2231 | { | |||
| 2232 | n_tasks = 1; | |||
| 2233 | } break; | |||
| 2234 | case GGML_OP_COUNT_EQUAL: | |||
| 2235 | case GGML_OP_SOLVE_TRI: | |||
| 2236 | case GGML_OP_GATED_DELTA_NET: | |||
| 2237 | { | |||
| 2238 | n_tasks = n_threads; | |||
| 2239 | } break; | |||
| 2240 | case GGML_OP_REPEAT: | |||
| 2241 | case GGML_OP_REPEAT_BACK: | |||
| 2242 | case GGML_OP_LEAKY_RELU: | |||
| 2243 | { | |||
| 2244 | n_tasks = 1; | |||
| 2245 | } break; | |||
| 2246 | case GGML_OP_UNARY: | |||
| 2247 | switch (ggml_get_unary_op(node)) { | |||
| 2248 | case GGML_UNARY_OP_ABS: | |||
| 2249 | case GGML_UNARY_OP_SGN: | |||
| 2250 | case GGML_UNARY_OP_NEG: | |||
| 2251 | case GGML_UNARY_OP_STEP: | |||
| 2252 | case GGML_UNARY_OP_TANH: | |||
| 2253 | case GGML_UNARY_OP_ELU: | |||
| 2254 | case GGML_UNARY_OP_RELU: | |||
| 2255 | case GGML_UNARY_OP_SIGMOID: | |||
| 2256 | case GGML_UNARY_OP_HARDSWISH: | |||
| 2257 | case GGML_UNARY_OP_HARDSIGMOID: | |||
| 2258 | case GGML_UNARY_OP_EXP: | |||
| 2259 | case GGML_UNARY_OP_SOFTPLUS: | |||
| 2260 | case GGML_UNARY_OP_EXPM1: | |||
| 2261 | case GGML_UNARY_OP_FLOOR: | |||
| 2262 | case GGML_UNARY_OP_CEIL: | |||
| 2263 | case GGML_UNARY_OP_ROUND: | |||
| 2264 | case GGML_UNARY_OP_TRUNC: | |||
| 2265 | { | |||
| 2266 | n_tasks = 1; | |||
| 2267 | } break; | |||
| 2268 | ||||
| 2269 | case GGML_UNARY_OP_GELU: | |||
| 2270 | case GGML_UNARY_OP_GELU_ERF: | |||
| 2271 | case GGML_UNARY_OP_GELU_QUICK: | |||
| 2272 | case GGML_UNARY_OP_SILU: | |||
| 2273 | case GGML_UNARY_OP_XIELU: | |||
| 2274 | { | |||
| 2275 | n_tasks = n_threads; | |||
| 2276 | } break; | |||
| 2277 | default: | |||
| 2278 | GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 2278, "fatal error"); | |||
| 2279 | } | |||
| 2280 | break; | |||
| 2281 | case GGML_OP_GLU: | |||
| 2282 | switch (ggml_get_glu_op(node)) { | |||
| 2283 | case GGML_GLU_OP_REGLU: | |||
| 2284 | case GGML_GLU_OP_GEGLU: | |||
| 2285 | case GGML_GLU_OP_SWIGLU: | |||
| 2286 | case GGML_GLU_OP_SWIGLU_OAI: | |||
| 2287 | case GGML_GLU_OP_GEGLU_ERF: | |||
| 2288 | case GGML_GLU_OP_GEGLU_QUICK: | |||
| 2289 | { | |||
| 2290 | n_tasks = n_threads; | |||
| 2291 | } break; | |||
| 2292 | default: | |||
| 2293 | GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 2293, "fatal error"); | |||
| 2294 | } | |||
| 2295 | break; | |||
| 2296 | case GGML_OP_SILU_BACK: | |||
| 2297 | case GGML_OP_MUL: | |||
| 2298 | case GGML_OP_DIV: | |||
| 2299 | case GGML_OP_NORM: | |||
| 2300 | case GGML_OP_RMS_NORM: | |||
| 2301 | case GGML_OP_RMS_NORM_BACK: | |||
| 2302 | case GGML_OP_L2_NORM: | |||
| 2303 | case GGML_OP_GROUP_NORM: | |||
| 2304 | case GGML_OP_CONCAT: | |||
| 2305 | case GGML_OP_MUL_MAT: | |||
| 2306 | case GGML_OP_MUL_MAT_ID: | |||
| 2307 | case GGML_OP_OUT_PROD: | |||
| 2308 | { | |||
| 2309 | n_tasks = n_threads; | |||
| 2310 | } break; | |||
| 2311 | case GGML_OP_GET_ROWS: | |||
| 2312 | case GGML_OP_SET_ROWS: | |||
| 2313 | { | |||
| 2314 | // FIXME: get_rows can use additional threads, but the cost of launching additional threads | |||
| 2315 | // decreases performance with GPU offloading | |||
| 2316 | //n_tasks = n_threads; | |||
| 2317 | n_tasks = 1; | |||
| 2318 | } break; | |||
| 2319 | case GGML_OP_SCALE: | |||
| 2320 | case GGML_OP_SET: | |||
| 2321 | case GGML_OP_RESHAPE: | |||
| 2322 | case GGML_OP_VIEW: | |||
| 2323 | case GGML_OP_PERMUTE: | |||
| 2324 | case GGML_OP_TRANSPOSE: | |||
| 2325 | case GGML_OP_GET_ROWS_BACK: | |||
| 2326 | case GGML_OP_DIAG: | |||
| 2327 | { | |||
| 2328 | n_tasks = 1; | |||
| 2329 | } break; | |||
| 2330 | case GGML_OP_DIAG_MASK_ZERO: | |||
| 2331 | case GGML_OP_DIAG_MASK_INF: | |||
| 2332 | case GGML_OP_SOFT_MAX_BACK: | |||
| 2333 | case GGML_OP_ROPE: | |||
| 2334 | case GGML_OP_ROPE_BACK: | |||
| 2335 | case GGML_OP_ADD_REL_POS: | |||
| 2336 | { | |||
| 2337 | n_tasks = n_threads; | |||
| 2338 | } break; | |||
| 2339 | case GGML_OP_CLAMP: | |||
| 2340 | { | |||
| 2341 | n_tasks = 1; //TODO | |||
| 2342 | } break; | |||
| 2343 | case GGML_OP_SOFT_MAX: | |||
| 2344 | { | |||
| 2345 | n_tasks = MIN(n_threads, ggml_nrows(node->src[0]))((n_threads) < (ggml_nrows(node->src[0])) ? (n_threads) : (ggml_nrows(node->src[0]))); | |||
| 2346 | } break; | |||
| 2347 | case GGML_OP_IM2COL: | |||
| 2348 | case GGML_OP_IM2COL_BACK: | |||
| 2349 | case GGML_OP_IM2COL_3D: | |||
| 2350 | case GGML_OP_CONV_2D: | |||
| 2351 | case GGML_OP_CONV_3D: | |||
| 2352 | case GGML_OP_CONV_2D_DW: | |||
| 2353 | case GGML_OP_COL2IM_1D: | |||
| 2354 | case GGML_OP_CONV_TRANSPOSE_1D: | |||
| 2355 | case GGML_OP_CONV_TRANSPOSE_2D: | |||
| 2356 | { | |||
| 2357 | n_tasks = n_threads; | |||
| 2358 | } break; | |||
| 2359 | case GGML_OP_POOL_1D: | |||
| 2360 | case GGML_OP_POOL_2D: | |||
| 2361 | case GGML_OP_POOL_2D_BACK: | |||
| 2362 | { | |||
| 2363 | n_tasks = 1; | |||
| 2364 | } break; | |||
| 2365 | case GGML_OP_UPSCALE: | |||
| 2366 | case GGML_OP_PAD: | |||
| 2367 | case GGML_OP_PAD_REFLECT_1D: | |||
| 2368 | case GGML_OP_ROLL: | |||
| 2369 | case GGML_OP_ARANGE: | |||
| 2370 | case GGML_OP_TIMESTEP_EMBEDDING: | |||
| 2371 | case GGML_OP_ARGSORT: | |||
| 2372 | case GGML_OP_TOP_K: | |||
| 2373 | case GGML_OP_FLASH_ATTN_EXT: | |||
| 2374 | case GGML_OP_FLASH_ATTN_BACK: | |||
| 2375 | case GGML_OP_SSM_CONV: | |||
| 2376 | case GGML_OP_SSM_SCAN: | |||
| 2377 | { | |||
| 2378 | n_tasks = n_threads; | |||
| 2379 | } break; | |||
| 2380 | case GGML_OP_RWKV_WKV6: | |||
| 2381 | case GGML_OP_GATED_LINEAR_ATTN: | |||
| 2382 | case GGML_OP_RWKV_WKV7: | |||
| 2383 | { | |||
| 2384 | const int64_t n_heads = node->src[1]->ne[1]; | |||
| 2385 | n_tasks = MIN(n_threads, n_heads)((n_threads) < (n_heads) ? (n_threads) : (n_heads)); | |||
| 2386 | } break; | |||
| 2387 | case GGML_OP_WIN_PART: | |||
| 2388 | case GGML_OP_WIN_UNPART: | |||
| 2389 | case GGML_OP_GET_REL_POS: | |||
| 2390 | { | |||
| 2391 | n_tasks = 1; | |||
| 2392 | } break; | |||
| 2393 | case GGML_OP_MAP_CUSTOM1: | |||
| 2394 | { | |||
| 2395 | struct ggml_map_custom1_op_params p; | |||
| 2396 | memcpy(&p, node->op_params, sizeof(p)); | |||
| 2397 | if (p.n_tasks == GGML_N_TASKS_MAX(-1)) { | |||
| 2398 | n_tasks = n_threads; | |||
| 2399 | } else { | |||
| 2400 | n_tasks = MIN(p.n_tasks, n_threads)((p.n_tasks) < (n_threads) ? (p.n_tasks) : (n_threads)); | |||
| 2401 | } | |||
| 2402 | } break; | |||
| 2403 | case GGML_OP_MAP_CUSTOM2: | |||
| 2404 | { | |||
| 2405 | struct ggml_map_custom2_op_params p; | |||
| 2406 | memcpy(&p, node->op_params, sizeof(p)); | |||
| 2407 | if (p.n_tasks == GGML_N_TASKS_MAX(-1)) { | |||
| 2408 | n_tasks = n_threads; | |||
| 2409 | } else { | |||
| 2410 | n_tasks = MIN(p.n_tasks, n_threads)((p.n_tasks) < (n_threads) ? (p.n_tasks) : (n_threads)); | |||
| 2411 | } | |||
| 2412 | } break; | |||
| 2413 | case GGML_OP_MAP_CUSTOM3: | |||
| 2414 | { | |||
| 2415 | struct ggml_map_custom3_op_params p; | |||
| 2416 | memcpy(&p, node->op_params, sizeof(p)); | |||
| 2417 | if (p.n_tasks == GGML_N_TASKS_MAX(-1)) { | |||
| 2418 | n_tasks = n_threads; | |||
| 2419 | } else { | |||
| 2420 | n_tasks = MIN(p.n_tasks, n_threads)((p.n_tasks) < (n_threads) ? (p.n_tasks) : (n_threads)); | |||
| 2421 | } | |||
| 2422 | } break; | |||
| 2423 | case GGML_OP_CUSTOM: | |||
| 2424 | { | |||
| 2425 | struct ggml_custom_op_params p; | |||
| 2426 | memcpy(&p, node->op_params, sizeof(p)); | |||
| 2427 | if (p.n_tasks == GGML_N_TASKS_MAX(-1)) { | |||
| 2428 | n_tasks = n_threads; | |||
| 2429 | } else { | |||
| 2430 | n_tasks = MIN(p.n_tasks, n_threads)((p.n_tasks) < (n_threads) ? (p.n_tasks) : (n_threads)); | |||
| 2431 | } | |||
| 2432 | } break; | |||
| 2433 | case GGML_OP_CROSS_ENTROPY_LOSS: | |||
| 2434 | case GGML_OP_CROSS_ENTROPY_LOSS_BACK: | |||
| 2435 | case GGML_OP_OPT_STEP_ADAMW: | |||
| 2436 | case GGML_OP_OPT_STEP_SGD: | |||
| 2437 | { | |||
| 2438 | n_tasks = n_threads; | |||
| 2439 | } break; | |||
| 2440 | case GGML_OP_NONE: | |||
| 2441 | { | |||
| 2442 | n_tasks = 1; | |||
| 2443 | } break; | |||
| 2444 | case GGML_OP_COUNT: | |||
| 2445 | { | |||
| 2446 | GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 2446, "fatal error"); | |||
| 2447 | } | |||
| 2448 | default: | |||
| 2449 | { | |||
| 2450 | fprintf(stderrstderr, "%s: op not implemented: ", __func__); | |||
| 2451 | if (node->op < GGML_OP_COUNT) { | |||
| 2452 | fprintf(stderrstderr, "%s\n", ggml_op_name(node->op)); | |||
| 2453 | } else { | |||
| 2454 | fprintf(stderrstderr, "%d\n", node->op); | |||
| 2455 | } | |||
| 2456 | GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 2456, "fatal error"); | |||
| 2457 | } | |||
| 2458 | } | |||
| 2459 | ||||
| 2460 | assert(n_tasks > 0)((void) sizeof (__assert_single_arg (n_tasks > 0)), __extension__ ({ if (n_tasks > 0) ; else __assert_fail ("n_tasks > 0" , "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 2460, __extension__ __PRETTY_FUNCTION__); })); | |||
| 2461 | ||||
| 2462 | return n_tasks; | |||
| 2463 | } | |||
| 2464 | ||||
| 2465 | static thread_ret_t ggml_graph_compute_secondary_thread(void* data); | |||
| 2466 | ||||
| 2467 | #if defined(_WIN32) | |||
| 2468 | #include "windows.h" | |||
| 2469 | ||||
| 2470 | // TODO: support > 64 CPUs | |||
| 2471 | static bool_Bool ggml_thread_apply_affinity(bool_Bool * mask) { | |||
| 2472 | HANDLE h = GetCurrentThread(); | |||
| 2473 | uint64_t bitmask = 0ULL; | |||
| 2474 | ||||
| 2475 | assert(GGML_MAX_N_THREADS >= 64)((void) sizeof (__assert_single_arg (512 >= 64)), __extension__ ({ if (512 >= 64) ; else __assert_fail ("GGML_MAX_N_THREADS >= 64" , "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 2475, __extension__ __PRETTY_FUNCTION__); })); | |||
| 2476 | ||||
| 2477 | for (int32_t i = 0; i < 8; i++) { | |||
| 2478 | int32_t idx = i * 8; | |||
| 2479 | uint8_t val = 0; | |||
| 2480 | val |= mask[idx + 0] << 0; | |||
| 2481 | val |= mask[idx + 1] << 1; | |||
| 2482 | val |= mask[idx + 2] << 2; | |||
| 2483 | val |= mask[idx + 3] << 3; | |||
| 2484 | val |= mask[idx + 4] << 4; | |||
| 2485 | val |= mask[idx + 5] << 5; | |||
| 2486 | val |= mask[idx + 6] << 6; | |||
| 2487 | val |= mask[idx + 7] << 7; | |||
| 2488 | bitmask |= (uint64_t)val << idx; | |||
| 2489 | } | |||
| 2490 | ||||
| 2491 | for (int32_t i = 64; i < GGML_MAX_N_THREADS512; i++) { | |||
| 2492 | if (mask[i]) { | |||
| 2493 | fprintf(stderrstderr, "warn: setting thread-affinity for > 64 CPUs isn't supported on windows!\n"); | |||
| 2494 | break; | |||
| 2495 | } | |||
| 2496 | } | |||
| 2497 | ||||
| 2498 | DWORD_PTR m = (DWORD_PTR)bitmask; | |||
| 2499 | ||||
| 2500 | m = SetThreadAffinityMask(h, m); | |||
| 2501 | ||||
| 2502 | return m != 0; | |||
| 2503 | } | |||
| 2504 | ||||
| 2505 | static bool_Bool ggml_thread_apply_priority(int32_t prio) { | |||
| 2506 | // Note that on Windows the Process Priority Class must be updated in order to set Thread priority. | |||
| 2507 | // This is up to the applications. | |||
| 2508 | DWORD p = THREAD_PRIORITY_NORMAL; | |||
| 2509 | switch (prio) { | |||
| 2510 | case GGML_SCHED_PRIO_LOW: p = THREAD_PRIORITY_BELOW_NORMAL; break; | |||
| 2511 | case GGML_SCHED_PRIO_NORMAL: p = THREAD_PRIORITY_NORMAL; break; | |||
| 2512 | case GGML_SCHED_PRIO_MEDIUM: p = THREAD_PRIORITY_ABOVE_NORMAL; break; | |||
| 2513 | case GGML_SCHED_PRIO_HIGH: p = THREAD_PRIORITY_HIGHEST; break; | |||
| 2514 | case GGML_SCHED_PRIO_REALTIME: p = THREAD_PRIORITY_TIME_CRITICAL; break; | |||
| 2515 | } | |||
| 2516 | ||||
| 2517 | if (prio != GGML_SCHED_PRIO_LOW) { | |||
| 2518 | // Tell Windows that this thread should not be throttled (needs its own CPU core). | |||
| 2519 | // Newer Windows 11 versions aggressively park (offline) CPU cores and often place | |||
| 2520 | // all our threads onto the first 4 cores which results in terrible performance with | |||
| 2521 | // n_threads > 4 | |||
| 2522 | #if _WIN32_WINNT >= 0x0602 | |||
| 2523 | THREAD_POWER_THROTTLING_STATE t; | |||
| 2524 | ZeroMemory(&t, sizeof(t)); | |||
| 2525 | t.Version = THREAD_POWER_THROTTLING_CURRENT_VERSION; | |||
| 2526 | t.ControlMask = THREAD_POWER_THROTTLING_EXECUTION_SPEED; | |||
| 2527 | t.StateMask = 0; | |||
| 2528 | ||||
| 2529 | if (!SetThreadInformation(GetCurrentThread(), ThreadPowerThrottling, &t, sizeof(t))) { | |||
| 2530 | GGML_LOG_DEBUG("failed to disable thread power throttling %d : (%d)\n", prio, (int) GetLastError())ggml_log_internal(GGML_LOG_LEVEL_DEBUG, "failed to disable thread power throttling %d : (%d)\n" , prio, (int) GetLastError()); | |||
| 2531 | return false0; | |||
| 2532 | } | |||
| 2533 | #endif | |||
| 2534 | } | |||
| 2535 | ||||
| 2536 | if (prio == GGML_SCHED_PRIO_NORMAL) { | |||
| 2537 | // Keep inherited policy/priority | |||
| 2538 | return true1; | |||
| 2539 | } | |||
| 2540 | ||||
| 2541 | if (!SetThreadPriority(GetCurrentThread(), p)) { | |||
| 2542 | fprintf(stderrstderr, "warn: failed to set thread priority %d : (%d)\n", prio, (int) GetLastError()); | |||
| 2543 | return false0; | |||
| 2544 | } | |||
| 2545 | ||||
| 2546 | return true1; | |||
| 2547 | } | |||
| 2548 | ||||
| 2549 | #elif defined(__APPLE__) | |||
| 2550 | #include <sys/types.h> | |||
| 2551 | #include <sys/resource.h> | |||
| 2552 | ||||
| 2553 | static bool_Bool ggml_thread_apply_affinity(const bool_Bool * mask) { | |||
| 2554 | // Not supported on Apple platforms | |||
| 2555 | UNUSED(mask)(void)(mask); | |||
| 2556 | return true1; | |||
| 2557 | } | |||
| 2558 | ||||
| 2559 | static bool_Bool ggml_thread_apply_priority(int32_t prio) { | |||
| 2560 | struct sched_param p; | |||
| 2561 | int32_t policy = SCHED_OTHER0; | |||
| 2562 | switch (prio) { | |||
| 2563 | // TODO: there seems to be no way to set lower prio on Apple platforms | |||
| 2564 | case GGML_SCHED_PRIO_LOW: policy = SCHED_OTHER0; p.sched_prioritysched_priority = 0; break; | |||
| 2565 | case GGML_SCHED_PRIO_NORMAL: policy = SCHED_OTHER0; p.sched_prioritysched_priority = 0; break; | |||
| 2566 | case GGML_SCHED_PRIO_MEDIUM: policy = SCHED_FIFO1; p.sched_prioritysched_priority = 40; break; | |||
| 2567 | case GGML_SCHED_PRIO_HIGH: policy = SCHED_FIFO1; p.sched_prioritysched_priority = 80; break; | |||
| 2568 | case GGML_SCHED_PRIO_REALTIME: policy = SCHED_FIFO1; p.sched_prioritysched_priority = 90; break; | |||
| 2569 | } | |||
| 2570 | ||||
| 2571 | if (prio == GGML_SCHED_PRIO_NORMAL) { | |||
| 2572 | // Keep inherited policy/priority | |||
| 2573 | return true1; | |||
| 2574 | } | |||
| 2575 | ||||
| 2576 | int32_t err = pthread_setschedparam(pthread_self(), policy, &p); | |||
| 2577 | if (err != 0) { | |||
| 2578 | fprintf(stderrstderr, "warn: failed to set thread priority %d : %s (%d)\n", prio, strerror(err), err); | |||
| 2579 | return false0; | |||
| 2580 | } | |||
| 2581 | ||||
| 2582 | return true1; | |||
| 2583 | } | |||
| 2584 | ||||
| 2585 | #elif defined(__gnu_linux__1) | |||
| 2586 | // TODO: this may not work on BSD, to be verified | |||
| 2587 | ||||
| 2588 | static bool_Bool ggml_thread_apply_affinity(const bool_Bool * mask) { | |||
| 2589 | cpu_set_t cpuset; | |||
| 2590 | int err; | |||
| 2591 | ||||
| 2592 | CPU_ZERO(&cpuset)do __builtin_memset (&cpuset, '\0', sizeof (cpu_set_t)); while (0); | |||
| 2593 | ||||
| 2594 | for (uint32_t i = 0; i < GGML_MAX_N_THREADS512; i++) { | |||
| 2595 | if (mask[i]) { | |||
| 2596 | GGML_PRINT_DEBUG("Thread %lx: adding %d to cpuset\n", pthread_self(), i); | |||
| 2597 | CPU_SET(i, &cpuset)(__extension__ ({ size_t __cpu = (i); __cpu / 8 < (sizeof ( cpu_set_t)) ? (((__cpu_mask *) ((&cpuset)->__bits))[(( __cpu) / (8 * sizeof (__cpu_mask)))] |= ((__cpu_mask) 1 << ((__cpu) % (8 * sizeof (__cpu_mask))))) : 0; })); | |||
| 2598 | } | |||
| 2599 | } | |||
| 2600 | ||||
| 2601 | #ifdef __ANDROID__ | |||
| 2602 | err = sched_setaffinity(0, sizeof(cpuset), &cpuset); | |||
| 2603 | if (err < 0) { | |||
| 2604 | err = errno(*__errno_location ()); | |||
| 2605 | } | |||
| 2606 | #else | |||
| 2607 | err = pthread_setaffinity_np(pthread_self(), sizeof(cpuset), &cpuset); | |||
| 2608 | #endif | |||
| 2609 | if (err != 0) { | |||
| 2610 | fprintf(stderrstderr, "warn: failed to set affinity mask 0x%llx : %s (%d)\n", (unsigned long long)mask, strerror(err), err); | |||
| 2611 | return false0; | |||
| 2612 | } | |||
| 2613 | ||||
| 2614 | return true1; | |||
| 2615 | } | |||
| 2616 | ||||
| 2617 | static bool_Bool ggml_thread_apply_priority(int32_t prio) { | |||
| 2618 | struct sched_param p; | |||
| 2619 | int32_t policy = SCHED_OTHER0; | |||
| 2620 | switch (prio) { | |||
| 2621 | case GGML_SCHED_PRIO_LOW: policy = SCHED_BATCH3; p.sched_prioritysched_priority = 0; break; | |||
| 2622 | case GGML_SCHED_PRIO_NORMAL: policy = SCHED_OTHER0; p.sched_prioritysched_priority = 0; break; | |||
| 2623 | case GGML_SCHED_PRIO_MEDIUM: policy = SCHED_FIFO1; p.sched_prioritysched_priority = 40; break; | |||
| 2624 | case GGML_SCHED_PRIO_HIGH: policy = SCHED_FIFO1; p.sched_prioritysched_priority = 80; break; | |||
| 2625 | case GGML_SCHED_PRIO_REALTIME: policy = SCHED_FIFO1; p.sched_prioritysched_priority = 90; break; | |||
| 2626 | } | |||
| 2627 | ||||
| 2628 | if (prio == GGML_SCHED_PRIO_NORMAL) { | |||
| 2629 | // Keep inherited policy/priority | |||
| 2630 | return true1; | |||
| 2631 | } | |||
| 2632 | ||||
| 2633 | int32_t err = pthread_setschedparam(pthread_self(), policy, &p); | |||
| 2634 | if (err != 0) { | |||
| 2635 | fprintf(stderrstderr, "warn: failed to set thread priority %d : %s (%d)\n", prio, strerror(err), err); | |||
| 2636 | return false0; | |||
| 2637 | } | |||
| 2638 | ||||
| 2639 | return true1; | |||
| 2640 | } | |||
| 2641 | ||||
| 2642 | #else // unsupported platforms | |||
| 2643 | ||||
| 2644 | static bool_Bool ggml_thread_apply_affinity(const bool_Bool * mask) { | |||
| 2645 | UNUSED(mask)(void)(mask); | |||
| 2646 | return true1; | |||
| 2647 | } | |||
| 2648 | ||||
| 2649 | static bool_Bool ggml_thread_apply_priority(int32_t prio) { | |||
| 2650 | UNUSED(prio)(void)(prio); | |||
| 2651 | return true1; | |||
| 2652 | } | |||
| 2653 | ||||
| 2654 | #endif | |||
| 2655 | ||||
| 2656 | static bool_Bool ggml_thread_cpumask_is_valid(const bool_Bool * mask) { | |||
| 2657 | for (int i = 0; i < GGML_MAX_N_THREADS512; i++) { | |||
| 2658 | if (mask[i]) { return true1; } | |||
| 2659 | } | |||
| 2660 | return false0; | |||
| 2661 | } | |||
| 2662 | ||||
| 2663 | static void ggml_thread_cpumask_next(const bool_Bool * global_mask, bool_Bool * local_mask, bool_Bool strict, int32_t* iter) { | |||
| 2664 | if (!strict) { | |||
| 2665 | memcpy(local_mask, global_mask, GGML_MAX_N_THREADS512); | |||
| 2666 | return; | |||
| 2667 | } else { | |||
| 2668 | memset(local_mask, 0, GGML_MAX_N_THREADS512); | |||
| 2669 | int32_t base_idx = *iter; | |||
| 2670 | for (int32_t i = 0; i < GGML_MAX_N_THREADS512; i++) { | |||
| 2671 | int32_t idx = base_idx + i; | |||
| 2672 | if (idx >= GGML_MAX_N_THREADS512) { | |||
| 2673 | // Just a cheaper modulo | |||
| 2674 | idx -= GGML_MAX_N_THREADS512; | |||
| 2675 | } | |||
| 2676 | if (global_mask[idx]) { | |||
| 2677 | local_mask[idx] = 1; | |||
| 2678 | *iter = idx + 1; | |||
| 2679 | return; | |||
| 2680 | } | |||
| 2681 | } | |||
| 2682 | } | |||
| 2683 | } | |||
| 2684 | ||||
| 2685 | void ggml_threadpool_free(struct ggml_threadpool* threadpool) { | |||
| 2686 | if (!threadpool) return; | |||
| 2687 | ||||
| 2688 | const int n_threads = threadpool->n_threads; | |||
| 2689 | ||||
| 2690 | #ifndef GGML_USE_OPENMP | |||
| 2691 | struct ggml_compute_state* workers = threadpool->workers; | |||
| 2692 | ||||
| 2693 | ggml_mutex_lock(&threadpool->mutex)pthread_mutex_lock(&threadpool->mutex); | |||
| 2694 | ||||
| 2695 | threadpool->stop = true1; | |||
| 2696 | threadpool->pause = false0; | |||
| 2697 | ||||
| 2698 | ggml_cond_broadcast(&threadpool->cond)pthread_cond_broadcast(&threadpool->cond); | |||
| 2699 | ggml_mutex_unlock(&threadpool->mutex)pthread_mutex_unlock(&threadpool->mutex); | |||
| 2700 | ||||
| 2701 | for (int j = 1; j < n_threads; j++) { | |||
| 2702 | int32_t rc = ggml_thread_joinpthread_join(workers[j].thrd, NULL((void*)0)); | |||
| 2703 | GGML_ASSERT(rc == GGML_EXIT_SUCCESS || rc == GGML_EXIT_ABORTED)if (!(rc == 0 || rc == 1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 2703, "GGML_ASSERT(%s) failed", "rc == GGML_EXIT_SUCCESS || rc == GGML_EXIT_ABORTED" ); | |||
| 2704 | UNUSED(rc)(void)(rc); | |||
| 2705 | } | |||
| 2706 | ||||
| 2707 | ggml_mutex_destroy(&threadpool->mutex)pthread_mutex_destroy(&threadpool->mutex); | |||
| 2708 | ggml_cond_destroy(&threadpool->cond)pthread_cond_destroy(&threadpool->cond); | |||
| 2709 | #endif // GGML_USE_OPENMP | |||
| 2710 | ||||
| 2711 | const size_t workers_size = sizeof(struct ggml_compute_state) * n_threads; | |||
| 2712 | ggml_aligned_free(threadpool->workers, workers_size); | |||
| 2713 | ggml_aligned_free(threadpool, sizeof(struct ggml_threadpool)); | |||
| 2714 | } | |||
| 2715 | ||||
| 2716 | #ifndef GGML_USE_OPENMP | |||
| 2717 | // pause/resume must be called under mutex | |||
| 2718 | static void ggml_threadpool_pause_locked(struct ggml_threadpool * threadpool) { | |||
| 2719 | GGML_PRINT_DEBUG("Pausing threadpool\n"); | |||
| 2720 | threadpool->pause = true1; | |||
| 2721 | ggml_cond_broadcast(&threadpool->cond)pthread_cond_broadcast(&threadpool->cond); | |||
| 2722 | } | |||
| 2723 | ||||
| 2724 | static void ggml_threadpool_resume_locked(struct ggml_threadpool * threadpool) { | |||
| 2725 | GGML_PRINT_DEBUG("Resuming threadpool\n"); | |||
| 2726 | threadpool->pause = false0; | |||
| 2727 | ggml_cond_broadcast(&threadpool->cond)pthread_cond_broadcast(&threadpool->cond); | |||
| 2728 | } | |||
| 2729 | #endif | |||
| 2730 | ||||
| 2731 | void ggml_threadpool_pause(struct ggml_threadpool * threadpool) { | |||
| 2732 | #ifndef GGML_USE_OPENMP | |||
| 2733 | ggml_mutex_lock(&threadpool->mutex)pthread_mutex_lock(&threadpool->mutex); | |||
| 2734 | if (!threadpool->pause) { | |||
| 2735 | ggml_threadpool_pause_locked(threadpool); | |||
| 2736 | } | |||
| 2737 | ggml_mutex_unlock(&threadpool->mutex)pthread_mutex_unlock(&threadpool->mutex); | |||
| 2738 | #else | |||
| 2739 | UNUSED(threadpool)(void)(threadpool); | |||
| 2740 | #endif | |||
| 2741 | } | |||
| 2742 | ||||
| 2743 | void ggml_threadpool_resume(struct ggml_threadpool * threadpool) { | |||
| 2744 | #ifndef GGML_USE_OPENMP | |||
| 2745 | ggml_mutex_lock(&threadpool->mutex)pthread_mutex_lock(&threadpool->mutex); | |||
| 2746 | if (threadpool->pause) { | |||
| 2747 | ggml_threadpool_resume_locked(threadpool); | |||
| 2748 | } | |||
| 2749 | ggml_mutex_unlock(&threadpool->mutex)pthread_mutex_unlock(&threadpool->mutex); | |||
| 2750 | #else | |||
| 2751 | UNUSED(threadpool)(void)(threadpool); | |||
| 2752 | #endif | |||
| 2753 | } | |||
| 2754 | ||||
| 2755 | struct ggml_cplan ggml_graph_plan( | |||
| 2756 | const struct ggml_cgraph * cgraph, | |||
| 2757 | int n_threads, | |||
| 2758 | struct ggml_threadpool * threadpool) { | |||
| 2759 | ||||
| 2760 | if (threadpool == NULL((void*)0)) { | |||
| 2761 | //GGML_PRINT_DEBUG("Threadpool is not specified. Will create a disposable threadpool : n_threads %d\n", n_threads); | |||
| 2762 | } | |||
| 2763 | if (n_threads <= 0) { | |||
| 2764 | n_threads = threadpool ? threadpool->n_threads : GGML_DEFAULT_N_THREADS4; | |||
| 2765 | } | |||
| 2766 | ||||
| 2767 | #if defined(__EMSCRIPTEN__) && !defined(__EMSCRIPTEN_PTHREADS__) | |||
| 2768 | // Emscripten without pthreads support can only use a single thread | |||
| 2769 | n_threads = 1; | |||
| 2770 | #endif | |||
| 2771 | ||||
| 2772 | size_t work_size = 0; | |||
| 2773 | ||||
| 2774 | struct ggml_cplan cplan; | |||
| 2775 | memset(&cplan, 0, sizeof(struct ggml_cplan)); | |||
| 2776 | ||||
| 2777 | int max_tasks = 1; | |||
| 2778 | ||||
| 2779 | // thread scheduling for the different operations + work buffer size estimation | |||
| 2780 | for (int i = 0; i < cgraph->n_nodes; i++) { | |||
| 2781 | struct ggml_tensor * node = cgraph->nodes[i]; | |||
| 2782 | ||||
| 2783 | const int n_tasks = ggml_get_n_tasks(node, n_threads); | |||
| 2784 | ||||
| 2785 | max_tasks = MAX(max_tasks, n_tasks)((max_tasks) > (n_tasks) ? (max_tasks) : (n_tasks)); | |||
| 2786 | ||||
| 2787 | size_t cur = 0; | |||
| 2788 | ||||
| 2789 | if (!ggml_cpu_extra_work_size(n_threads, node, &cur)) { | |||
| 2790 | switch (node->op) { | |||
| 2791 | case GGML_OP_CPY: | |||
| 2792 | case GGML_OP_DUP: | |||
| 2793 | { | |||
| 2794 | if (ggml_is_quantized(node->type) || | |||
| 2795 | // F16 -> BF16 and BF16 -> F16 copies go through intermediate F32 | |||
| 2796 | (node->src[0]->type == GGML_TYPE_F16 && node->src[1] && node->src[1]->type == GGML_TYPE_BF16) || | |||
| 2797 | (node->src[0]->type == GGML_TYPE_BF16 && node->src[1] && node->src[1]->type == GGML_TYPE_F16) || | |||
| 2798 | // conversion between F32 and I32 | |||
| 2799 | (node->src[0]->type == GGML_TYPE_F32 && node->src[1] && node->src[1]->type == GGML_TYPE_I32) || | |||
| 2800 | (node->src[0]->type == GGML_TYPE_I32 && node->src[1] && node->src[1]->type == GGML_TYPE_F32)) { | |||
| 2801 | cur = ggml_type_size(GGML_TYPE_F32) * node->ne[0] * n_tasks; | |||
| 2802 | } | |||
| 2803 | } break; | |||
| 2804 | case GGML_OP_ADD: | |||
| 2805 | case GGML_OP_ADD_ID: | |||
| 2806 | case GGML_OP_ADD1: | |||
| 2807 | { | |||
| 2808 | if (ggml_is_quantized(node->src[0]->type)) { | |||
| 2809 | cur = ggml_type_size(GGML_TYPE_F32) * node->src[0]->ne[0] * n_tasks; | |||
| 2810 | } | |||
| 2811 | } break; | |||
| 2812 | case GGML_OP_ACC: | |||
| 2813 | { | |||
| 2814 | if (ggml_is_quantized(node->src[0]->type)) { | |||
| 2815 | cur = ggml_type_size(GGML_TYPE_F32) * node->src[1]->ne[0] * n_tasks; | |||
| 2816 | } | |||
| 2817 | } break; | |||
| 2818 | case GGML_OP_COUNT_EQUAL: | |||
| 2819 | { | |||
| 2820 | cur = ggml_type_size(node->type)*n_tasks; | |||
| 2821 | } break; | |||
| 2822 | case GGML_OP_MUL_MAT: | |||
| 2823 | { | |||
| 2824 | const enum ggml_type vec_dot_type = type_traits_cpu[node->src[0]->type].vec_dot_type; | |||
| 2825 | ||||
| 2826 | if (node->src[1]->type != vec_dot_type) { | |||
| 2827 | cur = ggml_row_size(vec_dot_type, ggml_nelements(node->src[1])); | |||
| 2828 | } | |||
| 2829 | } break; | |||
| 2830 | case GGML_OP_MUL_MAT_ID: | |||
| 2831 | { | |||
| 2832 | cur = 0; | |||
| 2833 | const struct ggml_tensor * src0 = node->src[0]; | |||
| 2834 | const struct ggml_tensor * src1 = node->src[1]; | |||
| 2835 | const struct ggml_tensor * ids = node->src[2]; | |||
| 2836 | const enum ggml_type vec_dot_type = type_traits_cpu[src0->type].vec_dot_type; | |||
| 2837 | const int n_as = src0->ne[2]; | |||
| 2838 | // src1 | |||
| 2839 | if (src1->type != vec_dot_type) { | |||
| 2840 | cur += ggml_row_size(vec_dot_type, ggml_nelements(src1)) + sizeof(int64_t); | |||
| 2841 | } | |||
| 2842 | // matrix_row_counts | |||
| 2843 | cur += n_as * sizeof(int64_t) + sizeof(int64_t); | |||
| 2844 | // matrix_rows | |||
| 2845 | cur += n_as*ids->ne[0]*ids->ne[1]*sizeof(struct mmid_row_mapping) + sizeof(int64_t); | |||
| 2846 | // atomic_current_chunk | |||
| 2847 | cur += CACHE_LINE_SIZE64*n_as + CACHE_LINE_SIZE64; | |||
| 2848 | } break; | |||
| 2849 | case GGML_OP_OUT_PROD: | |||
| 2850 | { | |||
| 2851 | if (ggml_is_quantized(node->src[0]->type)) { | |||
| 2852 | cur = ggml_type_size(GGML_TYPE_F32) * node->src[0]->ne[0] * n_tasks; | |||
| 2853 | } | |||
| 2854 | } break; | |||
| 2855 | case GGML_OP_SOFT_MAX: | |||
| 2856 | case GGML_OP_ROPE: | |||
| 2857 | case GGML_OP_ROPE_BACK: | |||
| 2858 | { | |||
| 2859 | cur = ggml_type_size(GGML_TYPE_F32) * node->ne[0] * n_tasks; | |||
| 2860 | } break; | |||
| 2861 | case GGML_OP_CONV_TRANSPOSE_1D: | |||
| 2862 | { | |||
| 2863 | GGML_ASSERT(node->src[0]->ne[3] == 1)if (!(node->src[0]->ne[3] == 1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 2863, "GGML_ASSERT(%s) failed", "node->src[0]->ne[3] == 1" ); | |||
| 2864 | GGML_ASSERT(node->src[1]->ne[2] == 1)if (!(node->src[1]->ne[2] == 1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 2864, "GGML_ASSERT(%s) failed", "node->src[1]->ne[2] == 1" ); | |||
| 2865 | GGML_ASSERT(node->src[1]->ne[3] == 1)if (!(node->src[1]->ne[3] == 1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 2865, "GGML_ASSERT(%s) failed", "node->src[1]->ne[3] == 1" ); | |||
| 2866 | ||||
| 2867 | const int64_t ne00 = node->src[0]->ne[0]; // K | |||
| 2868 | const int64_t ne01 = node->src[0]->ne[1]; // Cout | |||
| 2869 | const int64_t ne02 = node->src[0]->ne[2]; // Cin | |||
| 2870 | const int64_t ne10 = node->src[1]->ne[0]; // L | |||
| 2871 | const int64_t ne11 = node->src[1]->ne[1]; // Cin | |||
| 2872 | ||||
| 2873 | if ((node->src[0]->type == GGML_TYPE_F16 || | |||
| 2874 | node->src[0]->type == GGML_TYPE_BF16) && | |||
| 2875 | node->src[1]->type == GGML_TYPE_F32) { | |||
| 2876 | cur += sizeof(ggml_fp16_t)*ne00*ne01*ne02; | |||
| 2877 | cur += sizeof(ggml_fp16_t)*ne10*ne11; | |||
| 2878 | } else if (node->src[0]->type == GGML_TYPE_F32 && | |||
| 2879 | node->src[1]->type == GGML_TYPE_F32) { | |||
| 2880 | cur += sizeof(float)*ne00*ne01*ne02; | |||
| 2881 | cur += sizeof(float)*ne10*ne11; | |||
| 2882 | } else { | |||
| 2883 | GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 2883, "fatal error"); | |||
| 2884 | } | |||
| 2885 | } break; | |||
| 2886 | case GGML_OP_CONV_2D: | |||
| 2887 | case GGML_OP_CONV_3D: | |||
| 2888 | { | |||
| 2889 | cur = GGML_IM2COL_WORK_SIZE(16 * 1024 * 1024); | |||
| 2890 | } break; | |||
| 2891 | case GGML_OP_CONV_TRANSPOSE_2D: | |||
| 2892 | { | |||
| 2893 | const int64_t ne00 = node->src[0]->ne[0]; // W | |||
| 2894 | const int64_t ne01 = node->src[0]->ne[1]; // H | |||
| 2895 | const int64_t ne02 = node->src[0]->ne[2]; // Channels Out | |||
| 2896 | const int64_t ne03 = node->src[0]->ne[3]; // Channels In | |||
| 2897 | ||||
| 2898 | const int64_t ne10 = node->src[1]->ne[0]; // W | |||
| 2899 | const int64_t ne11 = node->src[1]->ne[1]; // H | |||
| 2900 | const int64_t ne12 = node->src[1]->ne[2]; // Channels In | |||
| 2901 | ||||
| 2902 | GGML_ASSERT(node->src[0]->type == GGML_TYPE_F16 || node->src[0]->type == GGML_TYPE_F32)if (!(node->src[0]->type == GGML_TYPE_F16 || node->src [0]->type == GGML_TYPE_F32)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 2902, "GGML_ASSERT(%s) failed", "node->src[0]->type == GGML_TYPE_F16 || node->src[0]->type == GGML_TYPE_F32" ); | |||
| 2903 | GGML_ASSERT(node->src[1]->type == GGML_TYPE_F32)if (!(node->src[1]->type == GGML_TYPE_F32)) ggml_abort( "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 2903, "GGML_ASSERT(%s) failed", "node->src[1]->type == GGML_TYPE_F32" ); | |||
| 2904 | ||||
| 2905 | cur += ggml_type_size(node->src[0]->type) * ne00 * ne01 * ne02 * ne03; | |||
| 2906 | cur += ggml_type_size(node->src[0]->type) * ne10 * ne11 * ne12; | |||
| 2907 | ||||
| 2908 | } break; | |||
| 2909 | case GGML_OP_TOP_K: | |||
| 2910 | { | |||
| 2911 | cur += sizeof(int32_t)*node->src[0]->ne[0]*n_tasks; | |||
| 2912 | } break; | |||
| 2913 | case GGML_OP_FLASH_ATTN_EXT: | |||
| 2914 | { | |||
| 2915 | const int64_t neq2 = node->src[0]->ne[2]; // number of query heads | |||
| 2916 | const int64_t DK = node->src[1]->ne[0]; | |||
| 2917 | const int64_t DV = node->src[2]->ne[0]; | |||
| 2918 | ||||
| 2919 | // Tiled flash attention scratch (tile sizes defined in common.h) | |||
| 2920 | // Per-thread: Q_q + KQ + mask + VKQ32 + V32 + K_f32 + padding | |||
| 2921 | size_t prefill = sizeof(float)*(GGML_FA_TILE_Q64*DK + 2*GGML_FA_TILE_Q64*GGML_FA_TILE_KV64 + GGML_FA_TILE_Q64*DV + GGML_FA_TILE_KV64*DV + GGML_FA_TILE_KV64*DK)*n_tasks; | |||
| 2922 | ||||
| 2923 | // Decode path: n_kv_chunks = n_tasks (one chunk per thread) | |||
| 2924 | // Per-thread: VKQ accmulator (DV), partial M, partial S + intra-thread scratch for V, Q and VKQ | |||
| 2925 | size_t n_chunks = n_tasks; | |||
| 2926 | size_t decode = sizeof(float)*(neq2*n_chunks*(2+DV) + n_tasks*(DK + 2*DV)); | |||
| 2927 | ||||
| 2928 | cur += MAX(prefill, decode)((prefill) > (decode) ? (prefill) : (decode)); | |||
| 2929 | } break; | |||
| 2930 | case GGML_OP_FLASH_ATTN_BACK: | |||
| 2931 | { | |||
| 2932 | const int64_t D = node->src[0]->ne[0]; | |||
| 2933 | const int64_t ne11 = ggml_up(node->src[1]->ne[1], GGML_SOFT_MAX_UNROLL4); | |||
| 2934 | const int64_t mxDn = MAX(D, ne11)((D) > (ne11) ? (D) : (ne11)) * 2; // *2 because of S and SM in ggml_compute_forward_flash_attn_back | |||
| 2935 | if (node->src[1]->type == GGML_TYPE_F32) { | |||
| 2936 | cur = sizeof(float)*mxDn*n_tasks; // TODO: this can become (n_tasks-1) | |||
| 2937 | cur += sizeof(float)*mxDn*n_tasks; // this is overestimated by x2 | |||
| 2938 | } else if (node->src[1]->type == GGML_TYPE_F16) { | |||
| 2939 | cur = sizeof(float)*mxDn*n_tasks; // TODO: this can become (n_tasks-1) | |||
| 2940 | cur += sizeof(float)*mxDn*n_tasks; // this is overestimated by x2 | |||
| 2941 | } else if (node->src[1]->type == GGML_TYPE_BF16) { | |||
| 2942 | cur = sizeof(float)*mxDn*n_tasks; // TODO: this can become (n_tasks-1) | |||
| 2943 | cur += sizeof(float)*mxDn*n_tasks; // this is overestimated by x2 | |||
| 2944 | } | |||
| 2945 | } break; | |||
| 2946 | ||||
| 2947 | case GGML_OP_CROSS_ENTROPY_LOSS: | |||
| 2948 | { | |||
| 2949 | cur = ggml_type_size(node->type)*(n_tasks + node->src[0]->ne[0]*n_tasks); | |||
| 2950 | } break; | |||
| 2951 | case GGML_OP_GATED_DELTA_NET: | |||
| 2952 | { | |||
| 2953 | const int64_t S_v = node->src[2]->ne[0]; | |||
| 2954 | const int64_t K = ggml_get_op_params_i32(node, 0); | |||
| 2955 | const int64_t per_thread = S_v + (K > 1 ? S_v * S_v : 0); | |||
| 2956 | cur = per_thread * sizeof(float) * n_tasks; | |||
| 2957 | } break; | |||
| 2958 | case GGML_OP_COUNT: | |||
| 2959 | { | |||
| 2960 | GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 2960, "fatal error"); | |||
| 2961 | } | |||
| 2962 | default: | |||
| 2963 | break; | |||
| 2964 | } | |||
| 2965 | } | |||
| 2966 | ||||
| 2967 | work_size = MAX(work_size, cur)((work_size) > (cur) ? (work_size) : (cur)); | |||
| 2968 | } | |||
| 2969 | ||||
| 2970 | if (work_size > 0) { | |||
| 2971 | work_size += CACHE_LINE_SIZE64*(n_threads); | |||
| 2972 | } | |||
| 2973 | ||||
| 2974 | cplan.threadpool = threadpool; | |||
| 2975 | cplan.n_threads = MIN(max_tasks, n_threads)((max_tasks) < (n_threads) ? (max_tasks) : (n_threads)); | |||
| 2976 | cplan.work_size = work_size; | |||
| 2977 | cplan.work_data = NULL((void*)0); | |||
| 2978 | ||||
| 2979 | return cplan; | |||
| 2980 | } | |||
| 2981 | ||||
| 2982 | ||||
| 2983 | // Try to fuse the current node with subsequent nodes for better performance. | |||
| 2984 | // Returns the number of nodes skipped by fusion (>=1), or 0 if no fusion was applied. | |||
| 2985 | static bool_Bool ggml_cpu_disable_fusion = false0; // initialized once in ggml_cpu_init(), read-only afterwards | |||
| 2986 | ||||
| 2987 | static int ggml_cpu_try_fuse_ops( | |||
| 2988 | const struct ggml_cgraph * cgraph, | |||
| 2989 | const int node_n, | |||
| 2990 | const struct ggml_compute_params * params, | |||
| 2991 | const struct ggml_cplan * cplan) { | |||
| 2992 | ||||
| 2993 | if (ggml_cpu_disable_fusion || cplan->use_ref) { | |||
| 2994 | return 0; | |||
| 2995 | } | |||
| 2996 | ||||
| 2997 | struct ggml_tensor * node = cgraph->nodes[node_n]; | |||
| 2998 | ||||
| 2999 | if (node->op == GGML_OP_RMS_NORM) { | |||
| 3000 | // RMS_NORM + MUL fusion | |||
| 3001 | const enum ggml_op fuse_ops[] = { GGML_OP_RMS_NORM, GGML_OP_MUL }; | |||
| 3002 | if (ggml_can_fuse(cgraph, node_n, fuse_ops, 2)) { | |||
| 3003 | struct ggml_tensor * mul_node = cgraph->nodes[node_n + 1]; | |||
| 3004 | const struct ggml_tensor * mul_w = (mul_node->src[0] == node) | |||
| 3005 | ? mul_node->src[1] : mul_node->src[0]; | |||
| 3006 | if (node->src[0]->type == GGML_TYPE_F32 && | |||
| 3007 | mul_node->type == GGML_TYPE_F32 && | |||
| 3008 | mul_w->type == GGML_TYPE_F32 && | |||
| 3009 | mul_w->ne[0] == node->ne[0] && | |||
| 3010 | mul_w->nb[0] == sizeof(float)) { | |||
| 3011 | ||||
| 3012 | ggml_compute_forward_rms_norm_mul_fused(params, node, mul_node); | |||
| 3013 | return 1; | |||
| 3014 | } | |||
| 3015 | } | |||
| 3016 | } | |||
| 3017 | ||||
| 3018 | return 0; | |||
| 3019 | } | |||
| 3020 | ||||
| 3021 | static thread_ret_t ggml_graph_compute_thread(void * data) { | |||
| 3022 | struct ggml_compute_state * state = (struct ggml_compute_state *) data; | |||
| 3023 | struct ggml_threadpool * tp = state->threadpool; | |||
| 3024 | ||||
| 3025 | const struct ggml_cgraph * cgraph = tp->cgraph; | |||
| 3026 | const struct ggml_cplan * cplan = tp->cplan; | |||
| 3027 | ||||
| 3028 | #ifdef GGML_USE_CPU_RISCV64_SPACEMIT | |||
| 3029 | ggml_backend_cpu_riscv64_spacemit_set_numa_thread_affinity(state->ith); | |||
| 3030 | #else | |||
| 3031 | set_numa_thread_affinity(state->ith); | |||
| 3032 | #endif | |||
| 3033 | ||||
| 3034 | struct ggml_compute_params params = { | |||
| 3035 | /*.ith =*/ state->ith, | |||
| 3036 | /*.nth =*/ atomic_load_explicit__c11_atomic_load(&tp->n_graph, memory_order_relaxed) & GGML_THREADPOOL_N_THREADS_MASK(0xffffU), | |||
| 3037 | /*.wsize =*/ cplan->work_size, | |||
| 3038 | /*.wdata =*/ cplan->work_data, | |||
| 3039 | /*.threadpool =*/ tp, | |||
| 3040 | /*.use_ref =*/ cplan->use_ref, | |||
| 3041 | }; | |||
| 3042 | ||||
| 3043 | #ifdef GGML_USE_OPENMP | |||
| 3044 | GGML_PRINT_DEBUG("thread #%d compute-start cplan %p\n", state->ith, (const void *)cplan); | |||
| 3045 | #else | |||
| 3046 | GGML_PRINT_DEBUG("thread #%d compute-start cplan %p last-graph %d\n", state->ith, (const void *)cplan, state->last_graph); | |||
| 3047 | #endif | |||
| 3048 | ||||
| 3049 | for (int node_n = 0; node_n < cgraph->n_nodes && atomic_load_explicit__c11_atomic_load(&tp->abort, memory_order_relaxed) != node_n; node_n++) { | |||
| 3050 | struct ggml_tensor * node = cgraph->nodes[node_n]; | |||
| 3051 | ||||
| 3052 | if (ggml_op_is_empty(node->op)) { | |||
| 3053 | // skip NOPs | |||
| 3054 | continue; | |||
| 3055 | } | |||
| 3056 | ||||
| 3057 | if ((node->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) { | |||
| 3058 | continue; | |||
| 3059 | } | |||
| 3060 | ||||
| 3061 | // TODO: move fused-op detection into ggml_graph_plan so fusion decisions are made once at planning time | |||
| 3062 | // Try fused ops, fall back to normal compute | |||
| 3063 | const int n_fused = ggml_cpu_try_fuse_ops(cgraph, node_n, ¶ms, cplan); | |||
| 3064 | if (n_fused > 0) { | |||
| 3065 | node_n += n_fused; | |||
| 3066 | } else { | |||
| 3067 | ggml_compute_forward(¶ms, node); | |||
| 3068 | } | |||
| 3069 | ||||
| 3070 | if (state->ith == 0 && cplan->abort_callback && | |||
| 3071 | cplan->abort_callback(cplan->abort_callback_data)) { | |||
| 3072 | atomic_store_explicit__c11_atomic_store(&tp->abort, node_n + 1, memory_order_relaxed); | |||
| 3073 | tp->ec = GGML_STATUS_ABORTED; | |||
| 3074 | } | |||
| 3075 | ||||
| 3076 | if (node_n + 1 < cgraph->n_nodes) { | |||
| 3077 | ggml_barrier(state->threadpool); | |||
| 3078 | } | |||
| 3079 | } | |||
| 3080 | ||||
| 3081 | #ifdef GGML_USE_OPENMP | |||
| 3082 | GGML_PRINT_DEBUG("thread #%d compute-done cplan %p\n", state->ith, (const void *)cplan); | |||
| 3083 | #else | |||
| 3084 | GGML_PRINT_DEBUG("thread #%d compute-done cplan %p last-graph %d\n", state->ith, (const void *)cplan, state->last_graph); | |||
| 3085 | #endif | |||
| 3086 | ||||
| 3087 | ggml_barrier(state->threadpool); | |||
| 3088 | ||||
| 3089 | #ifdef GGML_USE_CPU_RISCV64_SPACEMIT | |||
| 3090 | ggml_backend_cpu_riscv64_spacemit_clear_numa_thread_affinity_threaded(state->ith); | |||
| 3091 | #endif | |||
| 3092 | ||||
| 3093 | return 0; | |||
| 3094 | } | |||
| 3095 | ||||
| 3096 | #ifndef GGML_USE_OPENMP | |||
| 3097 | ||||
| 3098 | // check if thread is ready to proceed (exit from polling or sleeping) | |||
| 3099 | // returns true if loops should exit, sets state->pending to indicate new work | |||
| 3100 | static inline bool_Bool ggml_graph_compute_thread_ready(struct ggml_compute_state * state) { | |||
| 3101 | struct ggml_threadpool * threadpool = state->threadpool; | |||
| 3102 | ||||
| 3103 | if (state->pending || threadpool->stop || threadpool->pause) { return true1; } | |||
| 3104 | ||||
| 3105 | // check for new graph/work | |||
| 3106 | int n_graph = atomic_load_explicit__c11_atomic_load(&threadpool->n_graph, memory_order_relaxed); | |||
| 3107 | int n_threads = n_graph & GGML_THREADPOOL_N_THREADS_MASK(0xffffU); | |||
| 3108 | if (n_graph != state->last_graph) { | |||
| 3109 | state->pending = (state->ith < n_threads); | |||
| 3110 | state->last_graph = n_graph; | |||
| 3111 | return true1; | |||
| 3112 | } | |||
| 3113 | ||||
| 3114 | return false0; | |||
| 3115 | } | |||
| 3116 | ||||
| 3117 | // sync thread state after polling | |||
| 3118 | static inline void ggml_graph_compute_thread_sync(struct ggml_compute_state * state) { | |||
| 3119 | // TSAN doesn't support standalone fence yet, we use a dummy read-modify-write instead | |||
| 3120 | #ifdef GGML_TSAN_ENABLED | |||
| 3121 | atomic_fetch_add_explicit__c11_atomic_fetch_add(&state->threadpool->n_graph, 0, memory_order_seq_cst); | |||
| 3122 | #else | |||
| 3123 | atomic_thread_fence(memory_order_seq_cst)__c11_atomic_thread_fence(memory_order_seq_cst); | |||
| 3124 | #endif | |||
| 3125 | UNUSED(state)(void)(state); | |||
| 3126 | } | |||
| 3127 | ||||
| 3128 | static inline bool_Bool ggml_graph_compute_poll_for_work(struct ggml_compute_state * state) { | |||
| 3129 | struct ggml_threadpool * threadpool = state->threadpool; | |||
| 3130 | ||||
| 3131 | // This seems to make 0 ... 100 a decent range for polling level across modern processors. | |||
| 3132 | // Perhaps, we can adjust it dynamically based on load and things. | |||
| 3133 | const uint64_t n_rounds = 1024UL * 128 * threadpool->poll; | |||
| 3134 | ||||
| 3135 | for (uint64_t i=0; !ggml_graph_compute_thread_ready(state) && i < n_rounds; i++) { | |||
| 3136 | // No new work. Keep polling. | |||
| 3137 | ggml_thread_cpu_relax(); | |||
| 3138 | } | |||
| 3139 | ||||
| 3140 | return state->pending; | |||
| 3141 | } | |||
| 3142 | ||||
| 3143 | static inline bool_Bool ggml_graph_compute_check_for_work(struct ggml_compute_state * state) { | |||
| 3144 | struct ggml_threadpool * threadpool = state->threadpool; | |||
| 3145 | ||||
| 3146 | if (ggml_graph_compute_poll_for_work(state)) { | |||
| 3147 | ggml_graph_compute_thread_sync(state); | |||
| 3148 | return state->pending; | |||
| 3149 | } | |||
| 3150 | ||||
| 3151 | ggml_mutex_lock_shared(&threadpool->mutex)pthread_mutex_lock(&threadpool->mutex); | |||
| 3152 | while (!ggml_graph_compute_thread_ready(state)) { | |||
| 3153 | // No new work. Wait for the signal. | |||
| 3154 | GGML_PRINT_DEBUG("thread #%d waiting for work (sleeping)\n", state->ith); | |||
| 3155 | ggml_cond_wait(&threadpool->cond, &threadpool->mutex)pthread_cond_wait(&threadpool->cond, &threadpool-> mutex); | |||
| 3156 | } | |||
| 3157 | ggml_mutex_unlock_shared(&threadpool->mutex)pthread_mutex_unlock(&threadpool->mutex); | |||
| 3158 | ||||
| 3159 | return state->pending; | |||
| 3160 | } | |||
| 3161 | ||||
| 3162 | static thread_ret_t ggml_graph_compute_secondary_thread(void* data) { | |||
| 3163 | struct ggml_compute_state * state = (struct ggml_compute_state *) data; | |||
| 3164 | struct ggml_threadpool * threadpool = state->threadpool; | |||
| 3165 | ||||
| 3166 | if (threadpool->thread_create_callback) { | |||
| 3167 | threadpool->thread_create_callback(); | |||
| 3168 | } | |||
| 3169 | ||||
| 3170 | ggml_thread_apply_priority(threadpool->prio); | |||
| 3171 | if (ggml_thread_cpumask_is_valid(state->cpumask)) { | |||
| 3172 | ggml_thread_apply_affinity(state->cpumask); | |||
| 3173 | } | |||
| 3174 | ||||
| 3175 | while (true1) { | |||
| 3176 | // Check if we need to sleep | |||
| 3177 | while (threadpool->pause) { | |||
| 3178 | GGML_PRINT_DEBUG("thread #%d inside pause loop\n", state->ith); | |||
| 3179 | ggml_mutex_lock_shared(&threadpool->mutex)pthread_mutex_lock(&threadpool->mutex); | |||
| 3180 | if (threadpool->pause) { | |||
| 3181 | ggml_cond_wait(&threadpool->cond, &threadpool->mutex)pthread_cond_wait(&threadpool->cond, &threadpool-> mutex); | |||
| 3182 | } | |||
| 3183 | GGML_PRINT_DEBUG("thread #%d resuming after wait\n", state->ith); | |||
| 3184 | ggml_mutex_unlock_shared(&threadpool->mutex)pthread_mutex_unlock(&threadpool->mutex); | |||
| 3185 | } | |||
| 3186 | ||||
| 3187 | // This needs to be checked for after the cond_wait | |||
| 3188 | if (threadpool->stop) break; | |||
| 3189 | ||||
| 3190 | // Check if there is new work | |||
| 3191 | // The main thread is the only one that can dispatch new work | |||
| 3192 | ||||
| 3193 | ggml_graph_compute_check_for_work(state); | |||
| 3194 | if (state->pending) { | |||
| 3195 | state->pending = false0; | |||
| 3196 | ggml_graph_compute_thread(state); | |||
| 3197 | } | |||
| 3198 | } | |||
| 3199 | ||||
| 3200 | if (threadpool->thread_destroy_callback) { | |||
| 3201 | threadpool->thread_destroy_callback(); | |||
| 3202 | } | |||
| 3203 | ||||
| 3204 | return (thread_ret_t) 0; | |||
| 3205 | } | |||
| 3206 | ||||
| 3207 | // Start processing new graph | |||
| 3208 | static void ggml_graph_compute_kickoff(struct ggml_threadpool * threadpool, int n_threads) | |||
| 3209 | { | |||
| 3210 | // Always take the mutex here because the worker threads are doing hybrid poll/wait | |||
| 3211 | ||||
| 3212 | ggml_mutex_lock(&threadpool->mutex)pthread_mutex_lock(&threadpool->mutex); | |||
| 3213 | ||||
| 3214 | // Update the number of active threads and the graph count | |||
| 3215 | int n_graph = atomic_load_explicit__c11_atomic_load(&threadpool->n_graph, memory_order_relaxed) >> GGML_THREADPOOL_N_THREADS_BITS(16); | |||
| 3216 | n_graph = ((n_graph + 1) << GGML_THREADPOOL_N_THREADS_BITS(16)) | (n_threads & GGML_THREADPOOL_N_THREADS_MASK(0xffffU)); | |||
| 3217 | ||||
| 3218 | GGML_PRINT_DEBUG("compute-kickoff: n_threads %d n_graph %d\n", n_threads, n_graph); | |||
| 3219 | ||||
| 3220 | // Indicate the graph is ready to be processed | |||
| 3221 | // We need the full seq-cst fence here because of the polling threads (used in thread_sync) | |||
| 3222 | atomic_store_explicit__c11_atomic_store(&threadpool->n_graph, n_graph, memory_order_seq_cst); | |||
| 3223 | ||||
| 3224 | if (threadpool->pause) { | |||
| 3225 | // Update main thread prio and affinity to match the threadpool settings | |||
| 3226 | ggml_thread_apply_priority(threadpool->prio); | |||
| 3227 | if (ggml_thread_cpumask_is_valid(threadpool->workers[0].cpumask)) { | |||
| 3228 | ggml_thread_apply_affinity(threadpool->workers[0].cpumask); | |||
| 3229 | } | |||
| 3230 | ||||
| 3231 | // resume does cond broadcast | |||
| 3232 | ggml_threadpool_resume_locked(threadpool); | |||
| 3233 | } else { | |||
| 3234 | ggml_cond_broadcast(&threadpool->cond)pthread_cond_broadcast(&threadpool->cond); | |||
| 3235 | } | |||
| 3236 | ||||
| 3237 | ggml_mutex_unlock(&threadpool->mutex)pthread_mutex_unlock(&threadpool->mutex); | |||
| 3238 | } | |||
| 3239 | ||||
| 3240 | #endif // GGML_USE_OPENMP | |||
| 3241 | ||||
| 3242 | static struct ggml_threadpool * ggml_threadpool_new_impl( | |||
| 3243 | struct ggml_threadpool_params * tpp, | |||
| 3244 | struct ggml_cgraph * cgraph, | |||
| 3245 | struct ggml_cplan * cplan) { | |||
| 3246 | ||||
| 3247 | struct ggml_threadpool * threadpool = | |||
| 3248 | ggml_aligned_malloc(sizeof(struct ggml_threadpool)); | |||
| 3249 | { | |||
| 3250 | threadpool->cgraph = cgraph; | |||
| 3251 | threadpool->cplan = cplan; | |||
| 3252 | threadpool->n_graph = 0; | |||
| 3253 | threadpool->n_barrier = 0; | |||
| 3254 | threadpool->n_barrier_passed = 0; | |||
| 3255 | threadpool->current_chunk = 0; | |||
| 3256 | threadpool->stop = false0; | |||
| 3257 | threadpool->pause = tpp->paused; | |||
| 3258 | threadpool->abort = -1; | |||
| 3259 | threadpool->workers = NULL((void*)0); | |||
| 3260 | threadpool->n_threads = tpp->n_threads; | |||
| 3261 | threadpool->poll = tpp->poll; | |||
| 3262 | threadpool->prio = tpp->prio; | |||
| 3263 | threadpool->thread_create_callback = tpp->thread_create_callback; | |||
| 3264 | threadpool->thread_destroy_callback = tpp->thread_destroy_callback; | |||
| 3265 | threadpool->ec = GGML_STATUS_SUCCESS; | |||
| 3266 | } | |||
| 3267 | ||||
| 3268 | // Allocate and init workers state | |||
| 3269 | const size_t workers_size = sizeof(struct ggml_compute_state) * tpp->n_threads; | |||
| 3270 | struct ggml_compute_state * workers = ggml_aligned_malloc(workers_size); | |||
| 3271 | ||||
| 3272 | memset(workers, 0, workers_size); | |||
| 3273 | for (int j = 0; j < tpp->n_threads; j++) { | |||
| 3274 | workers[j].threadpool = threadpool; | |||
| 3275 | workers[j].ith = j; | |||
| 3276 | } | |||
| 3277 | ||||
| 3278 | threadpool->workers = workers; | |||
| 3279 | ||||
| 3280 | #ifdef GGML_USE_OPENMP | |||
| 3281 | int32_t cpumask_iter = 0; | |||
| 3282 | ||||
| 3283 | // Compute CPU masks for each thread | |||
| 3284 | for (int j = 0; j < tpp->n_threads; j++) { | |||
| 3285 | ggml_thread_cpumask_next(tpp->cpumask, workers[j].cpumask, tpp->strict_cpu, &cpumask_iter); | |||
| 3286 | } | |||
| 3287 | #else // GGML_USE_OPENMP | |||
| 3288 | ggml_mutex_init(&threadpool->mutex)pthread_mutex_init(&threadpool->mutex, ((void*)0)); | |||
| 3289 | ggml_cond_init(&threadpool->cond)pthread_cond_init(&threadpool->cond, ((void*)0)); | |||
| 3290 | ||||
| 3291 | // Spin the threads for all workers, and update CPU placements. | |||
| 3292 | // Place the main thread last (towards the higher numbered CPU cores). | |||
| 3293 | ||||
| 3294 | int32_t cpumask_iter = 0; | |||
| 3295 | ||||
| 3296 | for (int j = 1; j < tpp->n_threads; j++) { | |||
| 3297 | ggml_thread_cpumask_next(tpp->cpumask, workers[j].cpumask, tpp->strict_cpu, &cpumask_iter); | |||
| 3298 | ||||
| 3299 | int32_t rc = ggml_thread_createpthread_create(&workers[j].thrd, NULL((void*)0), ggml_graph_compute_secondary_thread, &workers[j]); | |||
| 3300 | GGML_ASSERT(rc == 0)if (!(rc == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 3300, "GGML_ASSERT(%s) failed", "rc == 0"); | |||
| 3301 | } | |||
| 3302 | ||||
| 3303 | ggml_thread_cpumask_next(tpp->cpumask, workers[0].cpumask, tpp->strict_cpu, &cpumask_iter); | |||
| 3304 | ||||
| 3305 | if (!threadpool->pause) { | |||
| 3306 | // Update main thread prio and affinity at the start, otherwise we'll do it in resume | |||
| 3307 | ggml_thread_apply_priority(threadpool->prio); | |||
| 3308 | if (ggml_thread_cpumask_is_valid(threadpool->workers[0].cpumask)) { | |||
| 3309 | ggml_thread_apply_affinity(threadpool->workers[0].cpumask); | |||
| 3310 | } | |||
| 3311 | } | |||
| 3312 | #endif // GGML_USE_OPENMP | |||
| 3313 | ||||
| 3314 | return threadpool; | |||
| 3315 | } | |||
| 3316 | ||||
| 3317 | struct ggml_threadpool * ggml_threadpool_new(struct ggml_threadpool_params * tpp) { | |||
| 3318 | return ggml_threadpool_new_impl(tpp, NULL((void*)0), NULL((void*)0)); | |||
| 3319 | } | |||
| 3320 | ||||
| 3321 | enum ggml_status ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan) { | |||
| 3322 | ggml_cpu_init(); | |||
| 3323 | ||||
| 3324 | GGML_ASSERT(cplan)if (!(cplan)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 3324, "GGML_ASSERT(%s) failed", "cplan"); | |||
| 3325 | GGML_ASSERT(cplan->n_threads > 0)if (!(cplan->n_threads > 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 3325, "GGML_ASSERT(%s) failed", "cplan->n_threads > 0" ); | |||
| 3326 | GGML_ASSERT(cplan->work_size == 0 || cplan->work_data != NULL)if (!(cplan->work_size == 0 || cplan->work_data != ((void *)0))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-c.c" , 3326, "GGML_ASSERT(%s) failed", "cplan->work_size == 0 || cplan->work_data != NULL" ); | |||
| 3327 | ||||
| 3328 | int n_threads = cplan->n_threads; | |||
| 3329 | struct ggml_threadpool * threadpool = cplan->threadpool; | |||
| 3330 | ||||
| 3331 | bool_Bool disposable_threadpool = false0; | |||
| 3332 | ||||
| 3333 | if (threadpool == NULL((void*)0)) { | |||
| 3334 | //GGML_PRINT_DEBUG("Threadpool is not specified. Will create a disposable threadpool : n_threads %d\n", n_threads); | |||
| 3335 | disposable_threadpool = true1; | |||
| 3336 | ||||
| 3337 | struct ggml_threadpool_params ttp = ggml_threadpool_params_default(n_threads); | |||
| 3338 | threadpool = ggml_threadpool_new_impl(&ttp, cgraph, cplan); | |||
| 3339 | } else { | |||
| 3340 | // Reset some of the parameters that need resetting | |||
| 3341 | // No worker threads should be accessing the parameters below at this stage | |||
| 3342 | threadpool->cgraph = cgraph; | |||
| 3343 | threadpool->cplan = cplan; | |||
| 3344 | threadpool->current_chunk = 0; | |||
| 3345 | threadpool->abort = -1; | |||
| 3346 | threadpool->ec = GGML_STATUS_SUCCESS; | |||
| 3347 | } | |||
| 3348 | ||||
| 3349 | #ifdef GGML_USE_OPENMP | |||
| 3350 | if (n_threads > 1) { | |||
| 3351 | #pragma omp parallel num_threads(n_threads) | |||
| 3352 | { | |||
| 3353 | #pragma omp single | |||
| 3354 | { | |||
| 3355 | // update the number of threads from the actual number of threads that we got from OpenMP | |||
| 3356 | n_threads = omp_get_num_threads(); | |||
| 3357 | atomic_store_explicit__c11_atomic_store(&threadpool->n_graph, n_threads, memory_order_relaxed); | |||
| 3358 | } | |||
| 3359 | ||||
| 3360 | // Apply thread CPU mask and priority | |||
| 3361 | int ith = omp_get_thread_num(); | |||
| 3362 | ||||
| 3363 | ggml_thread_apply_priority(threadpool->prio); | |||
| 3364 | if (ggml_thread_cpumask_is_valid(threadpool->workers[ith].cpumask)) { | |||
| 3365 | ggml_thread_apply_affinity(threadpool->workers[ith].cpumask); | |||
| 3366 | } | |||
| 3367 | ggml_graph_compute_thread(&threadpool->workers[ith]); | |||
| 3368 | } | |||
| 3369 | } else { | |||
| 3370 | atomic_store_explicit__c11_atomic_store(&threadpool->n_graph, 1, memory_order_relaxed); | |||
| 3371 | ggml_graph_compute_thread(&threadpool->workers[0]); | |||
| 3372 | } | |||
| 3373 | #else | |||
| 3374 | if (n_threads > threadpool->n_threads) { | |||
| 3375 | GGML_LOG_WARN("cplan requested more threads (%d) than available (%d)\n", n_threads, threadpool->n_threads)ggml_log_internal(GGML_LOG_LEVEL_WARN , "cplan requested more threads (%d) than available (%d)\n" , n_threads, threadpool->n_threads); | |||
| 3376 | n_threads = threadpool->n_threads; | |||
| 3377 | } | |||
| 3378 | ||||
| 3379 | // Kick all threads to start the new graph | |||
| 3380 | ggml_graph_compute_kickoff(threadpool, n_threads); | |||
| 3381 | ||||
| 3382 | // This is a work thread too | |||
| 3383 | ggml_graph_compute_thread(&threadpool->workers[0]); | |||
| 3384 | #endif | |||
| 3385 | ||||
| 3386 | // don't leave affinity set on the main thread | |||
| 3387 | clear_numa_thread_affinity(); | |||
| 3388 | ||||
| 3389 | enum ggml_status ret = threadpool->ec; | |||
| 3390 | ||||
| 3391 | if (disposable_threadpool) { | |||
| 3392 | ggml_threadpool_free(threadpool); | |||
| 3393 | } | |||
| 3394 | ||||
| 3395 | return ret; | |||
| 3396 | } | |||
| 3397 | ||||
| 3398 | enum ggml_status ggml_graph_compute_with_ctx(struct ggml_context * ctx, struct ggml_cgraph * cgraph, int n_threads) { | |||
| 3399 | struct ggml_cplan cplan = ggml_graph_plan(cgraph, n_threads, NULL((void*)0)); | |||
| 3400 | ||||
| 3401 | cplan.work_data = (uint8_t *)ggml_new_buffer(ctx, cplan.work_size); | |||
| 3402 | ||||
| 3403 | return ggml_graph_compute(cgraph, &cplan); | |||
| 3404 | } | |||
| 3405 | ||||
| 3406 | void ggml_cpu_fp32_to_fp32(const float * x, float * y, int64_t n) { | |||
| 3407 | memcpy(y, x, n * sizeof(float)); | |||
| 3408 | } | |||
| 3409 | ||||
| 3410 | void ggml_cpu_fp32_to_fp16(const float * x, ggml_fp16_t * y, int64_t n) { | |||
| 3411 | int64_t i = 0; | |||
| 3412 | #if defined(__F16C__1) | |||
| 3413 | #if defined(__AVX512F__) | |||
| 3414 | for (; i + 15 < n; i += 16) { | |||
| 3415 | __m512 x_vec = _mm512_loadu_ps(x + i); | |||
| 3416 | __m256i y_vec = _mm512_cvtps_ph(x_vec, _MM_FROUND_TO_NEAREST_INT)((__m256i)__builtin_ia32_vcvtps2ph512_mask((__v16sf)(__m512)( x_vec), (int)(0x00), (__v16hi)_mm256_undefined_si256(), (__mmask16 )-1)); | |||
| 3417 | _mm256_storeu_si256((__m256i *)(y + i), y_vec); | |||
| 3418 | } | |||
| 3419 | #endif | |||
| 3420 | for (; i + 7 < n; i += 8) { | |||
| 3421 | __m256 x_vec = _mm256_loadu_ps(x + i); | |||
| 3422 | __m128i y_vec = _mm256_cvtps_ph(x_vec, _MM_FROUND_TO_NEAREST_INT)((__m128i)__builtin_ia32_vcvtps2ph256((__v8sf)(__m256)(x_vec) , (0x00))); | |||
| 3423 | _mm_storeu_si128((__m128i *)(y + i), y_vec); | |||
| 3424 | } | |||
| 3425 | for (; i + 3 < n; i += 4) { | |||
| 3426 | __m128 x_vec = _mm_loadu_ps(x + i); | |||
| 3427 | __m128i y_vec = _mm_cvtps_ph(x_vec, _MM_FROUND_TO_NEAREST_INT)((__m128i)__builtin_ia32_vcvtps2ph((__v4sf)(__m128)(x_vec), ( 0x00))); | |||
| 3428 | _mm_storel_epi64((__m128i *)(y + i), y_vec); | |||
| 3429 | } | |||
| 3430 | #elif defined(__riscv_zvfh) | |||
| 3431 | for (int vl; i < n; i += vl) { | |||
| 3432 | vl = __riscv_vsetvl_e32m2(n - i); | |||
| 3433 | vfloat32m2_t vx = __riscv_vle32_v_f32m2(&x[i], vl); | |||
| 3434 | vfloat16m1_t vy = __riscv_vfncvt_f_f_w_f16m1(vx, vl); | |||
| 3435 | __riscv_vse16_v_f16m1((_Float16 *)&y[i], vy, vl); | |||
| 3436 | } | |||
| 3437 | #endif | |||
| 3438 | for (; i < n; ++i) { | |||
| 3439 | y[i] = GGML_CPU_FP32_TO_FP16(x[i])ggml_compute_fp32_to_fp16(x[i]); | |||
| 3440 | } | |||
| 3441 | } | |||
| 3442 | ||||
| 3443 | void ggml_cpu_fp16_to_fp32(const ggml_fp16_t * x, float * y, int64_t n) { | |||
| 3444 | int64_t i = 0; | |||
| 3445 | #if defined(__F16C__1) | |||
| 3446 | #if defined(__AVX512F__) | |||
| 3447 | for (; i + 15 < n; i += 16) { | |||
| 3448 | __m256i x_vec = _mm256_loadu_si256((const __m256i *)(x + i)); | |||
| 3449 | __m512 y_vec = _mm512_cvtph_ps(x_vec); | |||
| 3450 | _mm512_storeu_ps(y + i, y_vec); | |||
| 3451 | } | |||
| 3452 | #endif | |||
| 3453 | for (; i + 7 < n; i += 8) { | |||
| 3454 | __m128i x_vec = _mm_loadu_si128((const __m128i *)(x + i)); | |||
| 3455 | __m256 y_vec = _mm256_cvtph_ps(x_vec); | |||
| 3456 | _mm256_storeu_ps(y + i, y_vec); | |||
| 3457 | } | |||
| 3458 | for (; i + 3 < n; i += 4) { | |||
| 3459 | __m128i x_vec = _mm_loadl_epi64((const __m128i *)(x + i)); | |||
| 3460 | __m128 y_vec = _mm_cvtph_ps(x_vec); | |||
| 3461 | _mm_storeu_ps(y + i, y_vec); | |||
| 3462 | } | |||
| 3463 | ||||
| 3464 | #elif defined(__riscv_v_intrinsic) && defined(__riscv_zvfhmin) | |||
| 3465 | // calculate step size | |||
| 3466 | const int epr = __riscv_vsetvlmax_e16m2(); | |||
| 3467 | const int step = epr * 2; | |||
| 3468 | const int np = (n & ~(step - 1)); | |||
| 3469 | ||||
| 3470 | // unroll by 2 | |||
| 3471 | for (; i < np; i += step) { | |||
| 3472 | vfloat16m2_t ax0 = __riscv_vle16_v_f16m2((const _Float16*)x + i, epr); | |||
| 3473 | vfloat32m4_t ay0 = __riscv_vfwcvt_f_f_v_f32m4(ax0, epr); | |||
| 3474 | __riscv_vse32_v_f32m4(y + i, ay0, epr); | |||
| 3475 | ||||
| 3476 | vfloat16m2_t ax1 = __riscv_vle16_v_f16m2((const _Float16*)x + i + epr, epr); | |||
| 3477 | vfloat32m4_t ay1 = __riscv_vfwcvt_f_f_v_f32m4(ax1, epr); | |||
| 3478 | __riscv_vse32_v_f32m4(y + i + epr, ay1, epr); | |||
| 3479 | } | |||
| 3480 | ||||
| 3481 | // leftovers | |||
| 3482 | int vl; | |||
| 3483 | for (i = np; i < n; i += vl) { | |||
| 3484 | vl = __riscv_vsetvl_e16m2(n - i); | |||
| 3485 | vfloat16m2_t ax0 = __riscv_vle16_v_f16m2((const _Float16*)x + i, vl); | |||
| 3486 | vfloat32m4_t ay0 = __riscv_vfwcvt_f_f_v_f32m4(ax0, vl); | |||
| 3487 | __riscv_vse32_v_f32m4(y + i, ay0, vl); | |||
| 3488 | } | |||
| 3489 | ||||
| 3490 | #endif | |||
| 3491 | ||||
| 3492 | for (; i < n; ++i) { | |||
| 3493 | y[i] = GGML_CPU_FP16_TO_FP32(x[i])ggml_lookup_fp16_to_fp32(x[i]); | |||
| 3494 | } | |||
| 3495 | } | |||
| 3496 | ||||
| 3497 | void ggml_cpu_fp32_to_bf16(const float * x, ggml_bf16_t * y, int64_t n) { | |||
| 3498 | int64_t i = 0; | |||
| 3499 | for (; i < n; ++i) { | |||
| 3500 | y[i] = GGML_FP32_TO_BF16(x[i])ggml_compute_fp32_to_bf16(x[i]); | |||
| 3501 | } | |||
| 3502 | } | |||
| 3503 | ||||
| 3504 | void ggml_cpu_fp32_to_i32(const float * x, int32_t * y, int64_t n) { | |||
| 3505 | int64_t i = 0; | |||
| 3506 | for (; i < n; ++i) { | |||
| 3507 | y[i] = x[i]; | |||
| 3508 | } | |||
| 3509 | } | |||
| 3510 | ||||
| 3511 | void ggml_cpu_bf16_to_fp32(const ggml_bf16_t * x, float * y, int64_t n) { | |||
| 3512 | int64_t i = 0; | |||
| 3513 | #if defined(__AVX2__1) | |||
| 3514 | #if defined(__AVX512F__) | |||
| 3515 | for (; i + 15 < n; i += 16) { | |||
| 3516 | _mm512_storeu_ps(y + i, | |||
| 3517 | _mm512_castsi512_ps( | |||
| 3518 | _mm512_slli_epi32( | |||
| 3519 | _mm512_cvtepu16_epi32( | |||
| 3520 | _mm256_loadu_si256( | |||
| 3521 | (const __m256i *)(x + i))), | |||
| 3522 | 16))); | |||
| 3523 | } | |||
| 3524 | #endif | |||
| 3525 | for (; i + 7 < n; i += 8) { | |||
| 3526 | _mm256_storeu_ps(y + i, | |||
| 3527 | _mm256_castsi256_ps( | |||
| 3528 | _mm256_slli_epi32( | |||
| 3529 | _mm256_cvtepu16_epi32( | |||
| 3530 | _mm_loadu_si128( | |||
| 3531 | (const __m128i *)(x + i))), | |||
| 3532 | 16))); | |||
| 3533 | } | |||
| 3534 | #elif defined(__riscv_v_intrinsic) && defined(__riscv_zvfbfmin) | |||
| 3535 | // calculate step size | |||
| 3536 | const int epr = __riscv_vsetvlmax_e16m2(); | |||
| 3537 | const int step = epr * 2; | |||
| 3538 | const int np = (n & ~(step - 1)); | |||
| 3539 | ||||
| 3540 | // unroll by 2 | |||
| 3541 | for (; i < np; i += step) { | |||
| 3542 | vbfloat16m2_t ax0 = __riscv_vle16_v_bf16m2((const __bf16*)x + i, epr); | |||
| 3543 | vfloat32m4_t ay0 = __riscv_vfwcvtbf16_f_f_v_f32m4(ax0, epr); | |||
| 3544 | __riscv_vse32_v_f32m4(y + i, ay0, epr); | |||
| 3545 | ||||
| 3546 | vbfloat16m2_t ax1 = __riscv_vle16_v_bf16m2((const __bf16*)x + i + epr, epr); | |||
| 3547 | vfloat32m4_t ay1 = __riscv_vfwcvtbf16_f_f_v_f32m4(ax1, epr); | |||
| 3548 | __riscv_vse32_v_f32m4(y + i + epr, ay1, epr); | |||
| 3549 | } | |||
| 3550 | ||||
| 3551 | // leftovers | |||
| 3552 | int vl; | |||
| 3553 | for (i = np; i < n; i += vl) { | |||
| 3554 | vl = __riscv_vsetvl_e16m2(n - i); | |||
| 3555 | vbfloat16m2_t ax0 = __riscv_vle16_v_bf16m2((const __bf16*)x + i, vl); | |||
| 3556 | vfloat32m4_t ay0 = __riscv_vfwcvtbf16_f_f_v_f32m4(ax0, vl); | |||
| 3557 | __riscv_vse32_v_f32m4(y + i, ay0, vl); | |||
| 3558 | } | |||
| 3559 | #endif | |||
| 3560 | for (; i < n; i++) { | |||
| 3561 | y[i] = GGML_BF16_TO_FP32(x[i])ggml_compute_bf16_to_fp32(x[i]); | |||
| 3562 | } | |||
| 3563 | } | |||
| 3564 | ||||
| 3565 | int ggml_cpu_has_avx(void) { | |||
| 3566 | #if defined(__AVX__1) | |||
| 3567 | return 1; | |||
| 3568 | #else | |||
| 3569 | return 0; | |||
| 3570 | #endif | |||
| 3571 | } | |||
| 3572 | ||||
| 3573 | int ggml_cpu_has_avx_vnni(void) { | |||
| 3574 | #if defined(__AVXVNNI__) | |||
| 3575 | return 1; | |||
| 3576 | #else | |||
| 3577 | return 0; | |||
| 3578 | #endif | |||
| 3579 | } | |||
| 3580 | ||||
| 3581 | int ggml_cpu_has_avx2(void) { | |||
| 3582 | #if defined(__AVX2__1) | |||
| 3583 | return 1; | |||
| 3584 | #else | |||
| 3585 | return 0; | |||
| 3586 | #endif | |||
| 3587 | } | |||
| 3588 | ||||
| 3589 | int ggml_cpu_has_avx512(void) { | |||
| 3590 | #if defined(__AVX512F__) | |||
| 3591 | return 1; | |||
| 3592 | #else | |||
| 3593 | return 0; | |||
| 3594 | #endif | |||
| 3595 | } | |||
| 3596 | ||||
| 3597 | int ggml_cpu_has_avx512_vbmi(void) { | |||
| 3598 | #if defined(__AVX512VBMI__) | |||
| 3599 | return 1; | |||
| 3600 | #else | |||
| 3601 | return 0; | |||
| 3602 | #endif | |||
| 3603 | } | |||
| 3604 | ||||
| 3605 | int ggml_cpu_has_avx512_vnni(void) { | |||
| 3606 | #if defined(__AVX512VNNI__) | |||
| 3607 | return 1; | |||
| 3608 | #else | |||
| 3609 | return 0; | |||
| 3610 | #endif | |||
| 3611 | } | |||
| 3612 | ||||
| 3613 | int ggml_cpu_has_avx512_bf16(void) { | |||
| 3614 | #if defined(__AVX512BF16__) | |||
| 3615 | return 1; | |||
| 3616 | #else | |||
| 3617 | return 0; | |||
| 3618 | #endif | |||
| 3619 | } | |||
| 3620 | ||||
| 3621 | int ggml_cpu_has_amx_int8(void) { | |||
| 3622 | #if defined(__AMX_INT8__) | |||
| 3623 | return 1; | |||
| 3624 | #else | |||
| 3625 | return 0; | |||
| 3626 | #endif | |||
| 3627 | } | |||
| 3628 | ||||
| 3629 | int ggml_cpu_has_bmi2(void) { | |||
| 3630 | #if defined(__BMI2__1) | |||
| 3631 | return 1; | |||
| 3632 | #else | |||
| 3633 | return 0; | |||
| 3634 | #endif | |||
| 3635 | } | |||
| 3636 | ||||
| 3637 | int ggml_cpu_has_fma(void) { | |||
| 3638 | #if defined(__FMA__1) | |||
| 3639 | return 1; | |||
| 3640 | #else | |||
| 3641 | return 0; | |||
| 3642 | #endif | |||
| 3643 | } | |||
| 3644 | ||||
| 3645 | int ggml_cpu_has_arm_fma(void) { | |||
| 3646 | #if defined(__ARM_FEATURE_FMA) | |||
| 3647 | return 1; | |||
| 3648 | #else | |||
| 3649 | return 0; | |||
| 3650 | #endif | |||
| 3651 | } | |||
| 3652 | ||||
| 3653 | int ggml_cpu_has_riscv_v(void) { | |||
| 3654 | #if defined(__riscv_v_intrinsic) | |||
| 3655 | return 1; | |||
| 3656 | #else | |||
| 3657 | return 0; | |||
| 3658 | #endif | |||
| 3659 | } | |||
| 3660 | ||||
| 3661 | int ggml_cpu_get_rvv_vlen(void) { | |||
| 3662 | #if defined(__riscv) && defined(__riscv_v_intrinsic) | |||
| 3663 | return ggml_riscv_arch_features.rvv_vlen; | |||
| 3664 | #else | |||
| 3665 | return 0; | |||
| 3666 | #endif | |||
| 3667 | } | |||
| 3668 | ||||
| 3669 | int ggml_cpu_has_f16c(void) { | |||
| 3670 | #if defined(__F16C__1) | |||
| 3671 | return 1; | |||
| 3672 | #else | |||
| 3673 | return 0; | |||
| 3674 | #endif | |||
| 3675 | } | |||
| 3676 | ||||
| 3677 | int ggml_cpu_has_fp16_va(void) { | |||
| 3678 | #if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) | |||
| 3679 | return 1; | |||
| 3680 | #else | |||
| 3681 | return 0; | |||
| 3682 | #endif | |||
| 3683 | } | |||
| 3684 | ||||
| 3685 | int ggml_cpu_has_wasm_simd(void) { | |||
| 3686 | #if defined(__wasm_simd128__) | |||
| 3687 | return 1; | |||
| 3688 | #else | |||
| 3689 | return 0; | |||
| 3690 | #endif | |||
| 3691 | } | |||
| 3692 | ||||
| 3693 | int ggml_cpu_has_llamafile(void) { | |||
| 3694 | #if defined(GGML_USE_LLAMAFILE) | |||
| 3695 | return 1; | |||
| 3696 | #else | |||
| 3697 | return 0; | |||
| 3698 | #endif | |||
| 3699 | } | |||
| 3700 | ||||
| 3701 | int ggml_cpu_has_sse3(void) { | |||
| 3702 | #if defined(__SSE3__1) | |||
| 3703 | return 1; | |||
| 3704 | #else | |||
| 3705 | return 0; | |||
| 3706 | #endif | |||
| 3707 | } | |||
| 3708 | ||||
| 3709 | int ggml_cpu_has_ssse3(void) { | |||
| 3710 | #if defined(__SSSE3__1) | |||
| 3711 | return 1; | |||
| 3712 | #else | |||
| 3713 | return 0; | |||
| 3714 | #endif | |||
| 3715 | } | |||
| 3716 | ||||
| 3717 | int ggml_cpu_has_vsx(void) { | |||
| 3718 | #if defined(__POWER9_VECTOR__) | |||
| 3719 | return 1; | |||
| 3720 | #else | |||
| 3721 | return 0; | |||
| 3722 | #endif | |||
| 3723 | } | |||
| 3724 | ||||
| 3725 | int ggml_cpu_has_vxe(void) { | |||
| 3726 | #if defined(__VXE__) || defined(__VXE2__) | |||
| 3727 | return 1; | |||
| 3728 | #else | |||
| 3729 | return 0; | |||
| 3730 | #endif | |||
| 3731 | } | |||
| 3732 | ||||
| 3733 | int ggml_cpu_has_neon(void) { | |||
| 3734 | #if defined(__ARM_ARCH) && defined(__ARM_NEON) | |||
| 3735 | return 1; | |||
| 3736 | #else | |||
| 3737 | return 0; | |||
| 3738 | #endif | |||
| 3739 | } | |||
| 3740 | ||||
| 3741 | int ggml_cpu_has_dotprod(void) { | |||
| 3742 | #if defined(__ARM_ARCH) && defined(__ARM_FEATURE_DOTPROD) | |||
| 3743 | return 1; | |||
| 3744 | #else | |||
| 3745 | return 0; | |||
| 3746 | #endif | |||
| 3747 | } | |||
| 3748 | ||||
| 3749 | int ggml_cpu_has_sve(void) { | |||
| 3750 | #if defined(__ARM_ARCH) && defined(__ARM_FEATURE_SVE) | |||
| 3751 | return 1; | |||
| 3752 | #else | |||
| 3753 | return 0; | |||
| 3754 | #endif | |||
| 3755 | } | |||
| 3756 | ||||
| 3757 | int ggml_cpu_has_matmul_int8(void) { | |||
| 3758 | #if defined(__ARM_ARCH) && defined(__ARM_FEATURE_MATMUL_INT8) | |||
| 3759 | return 1; | |||
| 3760 | #else | |||
| 3761 | return 0; | |||
| 3762 | #endif | |||
| 3763 | } | |||
| 3764 | ||||
| 3765 | int ggml_cpu_get_sve_cnt(void) { | |||
| 3766 | #if defined(__ARM_ARCH) && defined(__ARM_FEATURE_SVE) | |||
| 3767 | return ggml_arm_arch_features.sve_cnt; | |||
| 3768 | #else | |||
| 3769 | return 0; | |||
| 3770 | #endif | |||
| 3771 | } | |||
| 3772 | ||||
| 3773 | int ggml_cpu_has_sme(void) { | |||
| 3774 | #if defined(__ARM_ARCH) && defined(__ARM_FEATURE_SME) | |||
| 3775 | return 1; | |||
| 3776 | #else | |||
| 3777 | return 0; | |||
| 3778 | #endif | |||
| 3779 | } | |||
| 3780 | ||||
| 3781 | void ggml_cpu_init(void) { | |||
| 3782 | // needed to initialize ggml_time | |||
| 3783 | { | |||
| 3784 | struct ggml_init_params params = { 0, NULL((void*)0), false0 }; | |||
| 3785 | struct ggml_context * ctx = ggml_init(params); | |||
| 3786 | ggml_free(ctx); | |||
| 3787 | } | |||
| 3788 | ||||
| 3789 | ggml_critical_section_start(); | |||
| 3790 | ||||
| 3791 | static bool_Bool is_first_call = true1; | |||
| 3792 | ||||
| 3793 | if (is_first_call) { | |||
| 3794 | // initialize GELU, Quick GELU, SILU and EXP F32 tables | |||
| 3795 | { | |||
| 3796 | const uint64_t t_start = ggml_time_us(); UNUSED(t_start)(void)(t_start); | |||
| 3797 | ||||
| 3798 | for (int i = 0; i < (1 << 16); ++i) { | |||
| 3799 | union { | |||
| 3800 | uint16_t u16; | |||
| 3801 | ggml_fp16_t fp16; | |||
| 3802 | } u = {i}; | |||
| 3803 | float f = GGML_COMPUTE_FP16_TO_FP32(u.fp16)ggml_compute_fp16_to_fp32(u.fp16); | |||
| 3804 | ggml_table_f32_f16[i] = f; | |||
| 3805 | ggml_table_gelu_f16[i] = GGML_CPU_FP32_TO_FP16(ggml_gelu_f32(f))ggml_compute_fp32_to_fp16(ggml_gelu_f32(f)); | |||
| 3806 | ggml_table_gelu_quick_f16[i] = GGML_CPU_FP32_TO_FP16(ggml_gelu_quick_f32(f))ggml_compute_fp32_to_fp16(ggml_gelu_quick_f32(f)); | |||
| 3807 | } | |||
| 3808 | ||||
| 3809 | // initialize E8M0 half table (256 entries) | |||
| 3810 | for (int i = 0; i < (1 << 8); ++i) { | |||
| 3811 | ggml_table_f32_e8m0_half[i] = GGML_E8M0_TO_FP32_HALF(i)ggml_e8m0_to_fp32_half(i); | |||
| 3812 | } | |||
| 3813 | ||||
| 3814 | const uint64_t t_end = ggml_time_us(); UNUSED(t_end)(void)(t_end); | |||
| 3815 | ||||
| 3816 | GGML_PRINT_DEBUG("%s: GELU, Quick GELU, SILU and EXP tables initialized in %f ms\n", __func__, (t_end - t_start)/1000.0); | |||
| 3817 | ||||
| 3818 | #ifdef GGML_USE_OPENMP | |||
| 3819 | //if (!getenv("OMP_WAIT_POLICY")) { | |||
| 3820 | // // set the wait policy to active, so that OpenMP threads don't sleep | |||
| 3821 | // setenv("OMP_WAIT_POLICY", "active", 0) | |||
| 3822 | //} | |||
| 3823 | ||||
| 3824 | if (!getenv("KMP_BLOCKTIME")) { | |||
| 3825 | // set the time to wait before sleeping a thread | |||
| 3826 | // this is less aggressive than setting the wait policy to active, but should achieve similar results in most cases | |||
| 3827 | #ifdef _WIN32 | |||
| 3828 | _putenv_s("KMP_BLOCKTIME", "200"); // 200ms | |||
| 3829 | #else | |||
| 3830 | setenv("KMP_BLOCKTIME", "200", 0); // 200ms | |||
| 3831 | #endif | |||
| 3832 | } | |||
| 3833 | #endif | |||
| 3834 | } | |||
| 3835 | ||||
| 3836 | #if defined(__ARM_ARCH) | |||
| 3837 | ggml_init_arm_arch_features(); | |||
| 3838 | #endif | |||
| 3839 | ||||
| 3840 | #if defined(__riscv) | |||
| 3841 | ggml_init_riscv_arch_features(); | |||
| 3842 | #endif | |||
| 3843 | ||||
| 3844 | { | |||
| 3845 | const char * env = getenv("GGML_CPU_DISABLE_FUSION"); | |||
| 3846 | ggml_cpu_disable_fusion = (env != NULL((void*)0) && atoi(env) == 1); | |||
| 3847 | } | |||
| 3848 | ||||
| 3849 | is_first_call = false0; | |||
| 3850 | } | |||
| 3851 | ||||
| 3852 | ggml_critical_section_end(); | |||
| 3853 | } |