Bug Summary

File:root/firefox-clang/third_party/llama.cpp/src/llama-model-loader.cpp
Warning:line 227, column 26
Dereference of null pointer

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -O2 -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name llama-model-loader.cpp -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -fhalf-no-semantic-interposition -mframe-pointer=all -relaxed-aliasing -ffp-contract=off -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -target-feature +avx -target-feature +avx2 -target-feature +bmi2 -target-feature +f16c -target-feature +fma -target-feature +sse4.2 -tune-cpu generic -debugger-tuning=gdb -fdebug-compilation-dir=/root/firefox-clang/obj-x86_64-pc-linux-gnu/third_party/llama.cpp -fcoverage-compilation-dir=/root/firefox-clang/obj-x86_64-pc-linux-gnu/third_party/llama.cpp -resource-dir /usr/lib/llvm-23/lib/clang/23 -include /root/firefox-clang/config/gcc_hidden.h -include /root/firefox-clang/obj-x86_64-pc-linux-gnu/mozilla-config.h -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/stl_wrappers -D _GLIBCXX_ASSERTIONS=1 -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/system_wrappers -U _FORTIFY_SOURCE -D _FORTIFY_SOURCE=2 -D DEBUG=1 -D _GNU_SOURCE=1 -D GGML_USE_CPU=1 -D GGML_VERSION="GGML_VERSION" -D GGML_COMMIT="GGML_COMMIT" -D GGML_SHARED=1 -D LLAMA_SHARED=1 -D GGML_BUILD=1 -D LLAMA_BUILD=1 -D GGML_BACKEND_SHARED=1 -D GGML_BACKEND_BUILD=1 -D MOZ_HAS_MOZGLUE -I /root/firefox-clang/third_party/llama.cpp -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/third_party/llama.cpp -I /root/firefox-clang/third_party/llama.cpp/ggml -I /root/firefox-clang/third_party/llama.cpp/ggml/include -I /root/firefox-clang/third_party/llama.cpp/ggml/src -I /root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu -I /root/firefox-clang/third_party/llama.cpp/include -I /root/firefox-clang/third_party/llama.cpp/src -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/include -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/include/nspr -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/include/nss -D MOZILLA_CLIENT -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/16/../../../../include/c++/16 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/16/../../../../include/x86_64-linux-gnu/c++/16 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/16/../../../../include/c++/16/backward -internal-isystem /usr/lib/llvm-23/lib/clang/23/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/16/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -Wno-error=pessimizing-move -Wno-error=large-by-value-copy=128 -Wno-error=implicit-int-float-conversion -Wno-error=thread-safety-analysis -Wno-error=tautological-type-limit-compare -Wno-invalid-offsetof -Wno-range-loop-analysis -Wno-deprecated-anon-enum-enum-conversion -Wno-deprecated-enum-enum-conversion -Wno-inline-new-delete -Wno-error=deprecated-declarations -Wno-error=array-bounds -Wno-error=free-nonheap-object -Wno-error=atomic-alignment -Wno-error=deprecated-builtins -Wno-psabi -Wno-error=builtin-macro-redefined -Wno-vla-cxx-extension -Wno-unknown-warning-option -Wno-character-conversion -Wno-sign-compare -Wno-unused-function -Wno-tautological-unsigned-enum-zero-compare -Wno-implicit-fallthrough -Wno-unreachable-code -std=gnu++20 -fdeprecated-macro -ferror-limit 19 -fstrict-flex-arrays=1 -stack-protector 2 -fstack-clash-protection -ftrivial-auto-var-init=pattern -fno-rtti -fgnuc-version=4.2.1 -fno-implicit-modules -fskip-odr-check-in-gmf -fno-sized-deallocation -fno-aligned-allocation -fdiagnostics-absolute-paths -vectorize-loops -vectorize-slp -analyzer-checker optin.performance.Padding -analyzer-output=html -analyzer-config stable-report-filename=true -mllvm -dwarf-linkage-names=Abstract -faddrsig -fdwarf2-cfi-asm -o /tmp/scan-build-2026-09-01-224014-2642839-1 -x c++ /root/firefox-clang/third_party/llama.cpp/src/llama-model-loader.cpp
1#include "llama-model-loader.h"
2
3#include "ggml-alloc.h"
4#include "ggml.h"
5#include "gguf.h"
6#include "llama-hparams.h"
7
8#include <algorithm>
9#include <array>
10#include <cinttypes>
11#include <cstdint>
12#include <cstring>
13#include <regex>
14#include <mutex>
15
16#include "moz-overrides.h"
17
18static const size_t kiB = 1024;
19static const size_t MiB = 1024*kiB;
20static const size_t GiB = 1024*MiB;
21
22const char * llama_file_version_name(llama_fver version) {
23 switch (version) {
24 case GGUF_FILE_VERSION_V1: return "GGUF V1 (support until nov 2023)";
25 case GGUF_FILE_VERSION_V2: return "GGUF V2";
26 case GGUF_FILE_VERSION_V3: return "GGUF V3 (latest)";
27 }
28
29 return "unknown";
30}
31
32static std::string llama_model_ftype_name(llama_ftype ftype) {
33 if (ftype & LLAMA_FTYPE_GUESSED) {
34 return llama_model_ftype_name((enum llama_ftype) (ftype & ~LLAMA_FTYPE_GUESSED)) + " (guessed)";
35 }
36
37 switch (ftype) {
38 case LLAMA_FTYPE_ALL_F32: return "all F32";
39 case LLAMA_FTYPE_MOSTLY_F16: return "F16";
40 case LLAMA_FTYPE_MOSTLY_BF16: return "BF16";
41 case LLAMA_FTYPE_MOSTLY_Q1_0: return "Q1_0";
42 case LLAMA_FTYPE_MOSTLY_Q4_0: return "Q4_0";
43 case LLAMA_FTYPE_MOSTLY_Q4_1: return "Q4_1";
44 case LLAMA_FTYPE_MOSTLY_Q5_0: return "Q5_0";
45 case LLAMA_FTYPE_MOSTLY_Q5_1: return "Q5_1";
46 case LLAMA_FTYPE_MOSTLY_Q8_0: return "Q8_0";
47 case LLAMA_FTYPE_MOSTLY_MXFP4_MOE: return "MXFP4 MoE";
48 case LLAMA_FTYPE_MOSTLY_NVFP4: return "NVFP4";
49 case LLAMA_FTYPE_MOSTLY_Q2_K: return "Q2_K - Medium";
50 case LLAMA_FTYPE_MOSTLY_Q2_K_S: return "Q2_K - Small";
51 case LLAMA_FTYPE_MOSTLY_Q3_K_S: return "Q3_K - Small";
52 case LLAMA_FTYPE_MOSTLY_Q3_K_M: return "Q3_K - Medium";
53 case LLAMA_FTYPE_MOSTLY_Q3_K_L: return "Q3_K - Large";
54 case LLAMA_FTYPE_MOSTLY_Q4_K_S: return "Q4_K - Small";
55 case LLAMA_FTYPE_MOSTLY_Q4_K_M: return "Q4_K - Medium";
56 case LLAMA_FTYPE_MOSTLY_Q5_K_S: return "Q5_K - Small";
57 case LLAMA_FTYPE_MOSTLY_Q5_K_M: return "Q5_K - Medium";
58 case LLAMA_FTYPE_MOSTLY_Q6_K: return "Q6_K";
59 case LLAMA_FTYPE_MOSTLY_TQ1_0: return "TQ1_0 - 1.69 bpw ternary";
60 case LLAMA_FTYPE_MOSTLY_TQ2_0: return "TQ2_0 - 2.06 bpw ternary";
61 case LLAMA_FTYPE_MOSTLY_IQ2_XXS: return "IQ2_XXS - 2.0625 bpw";
62 case LLAMA_FTYPE_MOSTLY_IQ2_XS: return "IQ2_XS - 2.3125 bpw";
63 case LLAMA_FTYPE_MOSTLY_IQ2_S: return "IQ2_S - 2.5 bpw";
64 case LLAMA_FTYPE_MOSTLY_IQ2_M: return "IQ2_M - 2.7 bpw";
65 case LLAMA_FTYPE_MOSTLY_IQ3_XS: return "IQ3_XS - 3.3 bpw";
66 case LLAMA_FTYPE_MOSTLY_IQ3_XXS: return "IQ3_XXS - 3.0625 bpw";
67 case LLAMA_FTYPE_MOSTLY_IQ1_S: return "IQ1_S - 1.5625 bpw";
68 case LLAMA_FTYPE_MOSTLY_IQ1_M: return "IQ1_M - 1.75 bpw";
69 case LLAMA_FTYPE_MOSTLY_IQ4_NL: return "IQ4_NL - 4.5 bpw";
70 case LLAMA_FTYPE_MOSTLY_IQ4_XS: return "IQ4_XS - 4.25 bpw";
71 case LLAMA_FTYPE_MOSTLY_IQ3_S: return "IQ3_S - 3.4375 bpw";
72 case LLAMA_FTYPE_MOSTLY_IQ3_M: return "IQ3_S mix - 3.66 bpw";
73
74 default: return "unknown, may not work";
75 }
76}
77
78// return a list of splits for a given path
79// for example, given "<name>-00002-of-00004.gguf", returns list of all 4 splits
80static std::vector<std::string> llama_get_list_splits(const std::string & path, const int idx, const int n_split) {
81 std::vector<std::string> paths;
82 std::string split_prefix;
83 std::vector<char> buf(llama_path_max(), 0);
84
85 {
86 int ret = llama_split_prefix(buf.data(), buf.size(), path.c_str(), idx, n_split);
87 if (!ret) {
88 throwabort_with_suppression(); if (false) std::runtime_error(format("invalid split file name: %s", path.c_str()));
89 }
90 split_prefix = std::string(buf.data(), ret);
91 }
92
93 if (split_prefix.empty()) {
94 throwabort_with_suppression(); if (false) std::runtime_error(format("invalid split file: %s", path.c_str()));
95 }
96
97 for (int idx = 0; idx < n_split; ++idx) {
98 int ret = llama_split_path(buf.data(), buf.size(), split_prefix.c_str(), idx, n_split);
99 paths.push_back(std::string(buf.data(), ret));
100 }
101
102 return paths;
103}
104
105namespace GGUFMeta {
106 template <typename T, gguf_type gt_, T (*gfun)(const gguf_context *, const int64_t)>
107 struct GKV_Base_Type {
108 static constexpr gguf_type gt = gt_;
109
110 static T getter(const gguf_context * ctx, const int kid) {
111 return gfun(ctx, kid);
112 }
113 };
114
115 template<typename T> struct GKV_Base;
116
117 template<> struct GKV_Base<bool >: GKV_Base_Type<bool, GGUF_TYPE_BOOL, gguf_get_val_bool> {};
118 template<> struct GKV_Base<uint8_t >: GKV_Base_Type<uint8_t, GGUF_TYPE_UINT8, gguf_get_val_u8 > {};
119 template<> struct GKV_Base<uint16_t >: GKV_Base_Type<uint16_t, GGUF_TYPE_UINT16, gguf_get_val_u16 > {};
120 template<> struct GKV_Base<uint32_t >: GKV_Base_Type<uint32_t, GGUF_TYPE_UINT32, gguf_get_val_u32 > {};
121 template<> struct GKV_Base<uint64_t >: GKV_Base_Type<uint64_t, GGUF_TYPE_UINT64, gguf_get_val_u64 > {};
122 template<> struct GKV_Base<int8_t >: GKV_Base_Type<int8_t, GGUF_TYPE_INT8, gguf_get_val_i8 > {};
123 template<> struct GKV_Base<int16_t >: GKV_Base_Type<int16_t, GGUF_TYPE_INT16, gguf_get_val_i16 > {};
124 template<> struct GKV_Base<int32_t >: GKV_Base_Type<int32_t, GGUF_TYPE_INT32, gguf_get_val_i32 > {};
125 template<> struct GKV_Base<int64_t >: GKV_Base_Type<int64_t, GGUF_TYPE_INT64, gguf_get_val_i64 > {};
126 template<> struct GKV_Base<float >: GKV_Base_Type<float, GGUF_TYPE_FLOAT32, gguf_get_val_f32 > {};
127 template<> struct GKV_Base<double >: GKV_Base_Type<double, GGUF_TYPE_FLOAT64, gguf_get_val_f64 > {};
128 template<> struct GKV_Base<const char *>: GKV_Base_Type<const char *, GGUF_TYPE_STRING, gguf_get_val_str > {};
129
130 template<> struct GKV_Base<std::string> {
131 static constexpr gguf_type gt = GGUF_TYPE_STRING;
132
133 static std::string getter(const gguf_context * ctx, const int kid) {
134 return gguf_get_val_str(ctx, kid);
135 }
136 };
137
138 struct ArrayInfo {
139 const gguf_type gt;
140 const size_t length;
141 const void * data;
142 };
143
144 template<> struct GKV_Base<ArrayInfo> {
145 public:
146 static constexpr gguf_type gt = GGUF_TYPE_ARRAY;
147 static ArrayInfo getter(const gguf_context *ctx, const int k) {
148 const enum gguf_type arr_type = gguf_get_arr_type(ctx, k);
149 return ArrayInfo {
150 arr_type,
151 gguf_get_arr_n(ctx, k),
152 arr_type == GGUF_TYPE_STRING ? nullptr : gguf_get_arr_data(ctx, k),
153 };
154 }
155 };
156
157 template<typename T>
158 class GKV : public GKV_Base<T> {
159 GKV() = delete;
160
161 public:
162 static T get_kv(const gguf_context * ctx, const int k) {
163 const enum gguf_type kt = gguf_get_kv_type(ctx, k);
164
165 if (kt != GKV::gt) {
166 throwabort_with_suppression(); if (false) std::runtime_error(format("key %s has wrong type %s but expected type %s",
167 gguf_get_key(ctx, k), gguf_type_name(kt), gguf_type_name(GKV::gt)));
168 }
169 return GKV::getter(ctx, k);
170 }
171
172 static const char * override_type_to_str(const llama_model_kv_override_type ty) {
173 switch (ty) {
174 case LLAMA_KV_OVERRIDE_TYPE_BOOL: return "bool";
175 case LLAMA_KV_OVERRIDE_TYPE_INT: return "int";
176 case LLAMA_KV_OVERRIDE_TYPE_FLOAT: return "float";
177 case LLAMA_KV_OVERRIDE_TYPE_STR: return "str";
178 }
179 return "unknown";
180 }
181
182 static bool validate_override(const llama_model_kv_override_type expected_type, const struct llama_model_kv_override * ovrd) {
183 if (!ovrd) { return false; }
184 if (ovrd->tag == expected_type) {
185 LLAMA_LOG_INFO("%s: Using metadata override (%5s) '%s' = ",llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: Using metadata override (%5s) '%s' = "
, __func__, override_type_to_str(ovrd->tag), ovrd->key)
186 __func__, override_type_to_str(ovrd->tag), ovrd->key)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: Using metadata override (%5s) '%s' = "
, __func__, override_type_to_str(ovrd->tag), ovrd->key);
187 switch (ovrd->tag) {
188 case LLAMA_KV_OVERRIDE_TYPE_BOOL: {
189 LLAMA_LOG_INFO("%s\n", ovrd->val_bool ? "true" : "false")llama_log_internal(GGML_LOG_LEVEL_INFO , "%s\n", ovrd->val_bool
? "true" : "false");
190 } break;
191 case LLAMA_KV_OVERRIDE_TYPE_INT: {
192 LLAMA_LOG_INFO("%" PRId64 "\n", ovrd->val_i64)llama_log_internal(GGML_LOG_LEVEL_INFO , "%" "l" "d" "\n", ovrd
->val_i64);
193 } break;
194 case LLAMA_KV_OVERRIDE_TYPE_FLOAT: {
195 LLAMA_LOG_INFO("%.6f\n", ovrd->val_f64)llama_log_internal(GGML_LOG_LEVEL_INFO , "%.6f\n", ovrd->val_f64
);
196 } break;
197 case LLAMA_KV_OVERRIDE_TYPE_STR: {
198 LLAMA_LOG_INFO("%s\n", ovrd->val_str)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s\n", ovrd->val_str
);
199 } break;
200 default:
201 // Shouldn't be possible to end up here, but just in case...
202 throwabort_with_suppression(); if (false) std::runtime_error(
203 format("Unsupported attempt to override %s type for metadata key %s\n",
204 override_type_to_str(ovrd->tag), ovrd->key));
205 }
206 return true;
207 }
208 LLAMA_LOG_WARN("%s: Warning: Bad metadata override type for key '%s', expected %s but got %s\n",llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: Warning: Bad metadata override type for key '%s', expected %s but got %s\n"
, __func__, ovrd->key, override_type_to_str(expected_type)
, override_type_to_str(ovrd->tag))
209 __func__, ovrd->key, override_type_to_str(expected_type), override_type_to_str(ovrd->tag))llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: Warning: Bad metadata override type for key '%s', expected %s but got %s\n"
, __func__, ovrd->key, override_type_to_str(expected_type)
, override_type_to_str(ovrd->tag));
210 return false;
211 }
212
213 template<typename OT>
214 static typename std::enable_if<std::is_same<OT, bool>::value, bool>::type
215 try_override(OT & target, const struct llama_model_kv_override * ovrd) {
216 if (validate_override(LLAMA_KV_OVERRIDE_TYPE_BOOL, ovrd)) {
217 target = ovrd->val_bool;
218 return true;
219 }
220 return false;
221 }
222
223 template<typename OT>
224 static typename std::enable_if<!std::is_same<OT, bool>::value && std::is_integral<OT>::value, bool>::type
225 try_override(OT & target, const struct llama_model_kv_override * ovrd) {
226 if (validate_override(LLAMA_KV_OVERRIDE_TYPE_INT, ovrd)) {
14
←
Assuming the condition is true
→
15
←
Taking true branch
→
227 target = ovrd->val_i64;
16
←
Dereference of null pointer
228 return true;
229 }
230 return false;
231 }
232
233 template<typename OT>
234 static typename std::enable_if<std::is_floating_point<OT>::value, bool>::type
235 try_override(T & target, const struct llama_model_kv_override * ovrd) {
236 if (validate_override(LLAMA_KV_OVERRIDE_TYPE_FLOAT, ovrd)) {
237 target = ovrd->val_f64;
238 return true;
239 }
240 return false;
241 }
242
243 template<typename OT>
244 static typename std::enable_if<std::is_same<OT, std::string>::value, bool>::type
245 try_override(T & target, const struct llama_model_kv_override * ovrd) {
246 if (validate_override(LLAMA_KV_OVERRIDE_TYPE_STR, ovrd)) {
247 target = ovrd->val_str;
248 return true;
249 }
250 return false;
251 }
252
253 static bool set(const gguf_context * ctx, const int k, T & target, const struct llama_model_kv_override * ovrd = nullptr) {
254 if (try_override<T>(target, ovrd)) {
12
←
Passing null pointer value via 2nd parameter 'ovrd'
→
13
←
Calling 'GKV::try_override'
→
255 return true;
256 }
257 if (k < 0) { return false; }
258 target = get_kv(ctx, k);
259 return true;
260 }
261
262 static bool set(const gguf_context * ctx, const char * key, T & target, const struct llama_model_kv_override * ovrd = nullptr) {
263 return set(ctx, gguf_find_key(ctx, key), target, ovrd);
10
←
Passing null pointer value via 4th parameter 'ovrd'
→
11
←
Calling 'GKV::set'
→
264 }
265
266 static bool set(const gguf_context * ctx, const std::string & key, T & target, const struct llama_model_kv_override * ovrd = nullptr) {
267 return set(ctx, key.c_str(), target, ovrd);
8
←
Passing null pointer value via 4th parameter 'ovrd'
→
9
←
Calling 'GKV::set'
→
268 }
269 };
270}
271
272 template<typename T>
273 typename std::enable_if<std::is_integral<T>::value, bool>::type
274 llama_model_loader::get_arr_n(const std::string & key, T & result, bool required) {
275 const int kid = gguf_find_key(metadata, key.c_str());
276
277 if (kid < 0) {
278 if (required) {
279 throwabort_with_suppression(); if (false) std::runtime_error(format("key not found in model: %s", key.c_str()));
280 }
281 return false;
282 }
283
284 struct GGUFMeta::ArrayInfo arr_info =
285 GGUFMeta::GKV<GGUFMeta::ArrayInfo>::get_kv(metadata, kid);
286
287
288 result = arr_info.length;
289 return true;
290 }
291
292 template<typename T>
293 typename std::enable_if<std::is_integral<T>::value, bool>::type
294 llama_model_loader::get_arr_n(enum llm_kv kid, T & result, bool required) {
295 return get_arr_n(llm_kv(kid), result, required);
296 }
297
298 template bool llama_model_loader::get_arr_n(enum llm_kv kid, uint32_t & result, bool required);
299
300 template<typename T>
301 bool llama_model_loader::get_arr(const std::string & key, std::vector<T> & result, bool required) {
302 const gguf_context * ctx = metadata;
303 const int kid = gguf_find_key(ctx, key.c_str());
304
305 if (kid < 0 || gguf_get_kv_type(ctx, kid) != GGUF_TYPE_ARRAY) {
306 if (required) {
307 throwabort_with_suppression(); if (false) std::runtime_error(format("array key not found in model: %s", key.c_str()));
308 }
309 return false;
310 }
311
312 struct GGUFMeta::ArrayInfo arr_info =
313 GGUFMeta::GKV<GGUFMeta::ArrayInfo>::get_kv(ctx, kid);
314
315 switch (arr_info.gt) {
316 case GGUF_TYPE_UINT32:
317 case GGUF_TYPE_INT32: GGML_ASSERT((std::is_same<T, int32_t>::value) ||if (!((std::is_same<T, int32_t>::value) || (std::is_same
<T, uint32_t>::value))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-model-loader.cpp"
, 318, "GGML_ASSERT(%s) failed", "(std::is_same<T, int32_t>::value) || (std::is_same<T, uint32_t>::value)"
)
318 (std::is_same<T, uint32_t>::value))if (!((std::is_same<T, int32_t>::value) || (std::is_same
<T, uint32_t>::value))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-model-loader.cpp"
, 318, "GGML_ASSERT(%s) failed", "(std::is_same<T, int32_t>::value) || (std::is_same<T, uint32_t>::value)"
); break;
319 case GGUF_TYPE_FLOAT32: GGML_ASSERT((std::is_same<T, float>::value))if (!((std::is_same<T, float>::value))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-model-loader.cpp"
, 319, "GGML_ASSERT(%s) failed", "(std::is_same<T, float>::value)"
); break;
320 case GGUF_TYPE_STRING: GGML_ASSERT((std::is_same<T, std::string>::value))if (!((std::is_same<T, std::string>::value))) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/src/llama-model-loader.cpp"
, 320, "GGML_ASSERT(%s) failed", "(std::is_same<T, std::string>::value)"
); break;
321 default:
322 throwabort_with_suppression(); if (false) std::runtime_error(format("%s is not a string/float32/uint32/int32 array", key.c_str()));
323 }
324
325 if constexpr (std::is_same<T, std::string>::value) {
326 const size_t n_items = gguf_get_arr_n(ctx, kid);
327 result.clear();
328
329 for (size_t i = 0; i < n_items; i++) {
330 const T value = gguf_get_arr_str(ctx, kid, i);
331 result.emplace_back(value);
332 }
333 } else {
334 result.resize(arr_info.length);
335 result.assign((const T*)arr_info.data, (const T *)arr_info.data + arr_info.length);
336 }
337
338 return true;
339 }
340
341 template<typename T, size_t N_MAX>
342 bool llama_model_loader::get_arr(const std::string & key, std::array<T, N_MAX> & result, bool required) {
343 const gguf_context * ctx = metadata;
344 const int kid = gguf_find_key(ctx, key.c_str());
345
346 if (kid < 0 || gguf_get_kv_type(ctx, kid) != GGUF_TYPE_ARRAY) {
347 if (required) {
348 throwabort_with_suppression(); if (false) std::runtime_error(format("array key not found in model: %s", key.c_str()));
349 }
350 return false;
351 }
352
353 struct GGUFMeta::ArrayInfo arr_info =
354 GGUFMeta::GKV<GGUFMeta::ArrayInfo>::get_kv(ctx, kid);
355
356 switch (arr_info.gt) {
357 case GGUF_TYPE_BOOL:
358 case GGUF_TYPE_UINT32:
359 case GGUF_TYPE_INT32: GGML_ASSERT((std::is_same<T, int32_t>::value) ||if (!((std::is_same<T, int32_t>::value) || (std::is_same
<T, uint32_t>::value))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-model-loader.cpp"
, 360, "GGML_ASSERT(%s) failed", "(std::is_same<T, int32_t>::value) || (std::is_same<T, uint32_t>::value)"
)
360 (std::is_same<T, uint32_t>::value))if (!((std::is_same<T, int32_t>::value) || (std::is_same
<T, uint32_t>::value))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-model-loader.cpp"
, 360, "GGML_ASSERT(%s) failed", "(std::is_same<T, int32_t>::value) || (std::is_same<T, uint32_t>::value)"
); break;
361 case GGUF_TYPE_FLOAT32: GGML_ASSERT((std::is_same<T, float>::value))if (!((std::is_same<T, float>::value))) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-model-loader.cpp"
, 361, "GGML_ASSERT(%s) failed", "(std::is_same<T, float>::value)"
); break;
362 case GGUF_TYPE_STRING: GGML_ASSERT((std::is_same<T, std::string>::value))if (!((std::is_same<T, std::string>::value))) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/src/llama-model-loader.cpp"
, 362, "GGML_ASSERT(%s) failed", "(std::is_same<T, std::string>::value)"
); break;
363 default:
364 throwabort_with_suppression(); if (false) std::runtime_error(format("%s is not a string/float32/uint32/int32 array", key.c_str()));
365 }
366
367 if (arr_info.length > N_MAX) {
368 throwabort_with_suppression(); if (false) std::runtime_error(format("array length %u for key %s exceeds max %u", (uint32_t) arr_info.length, key.c_str(), (uint32_t) N_MAX));
369 }
370
371 if constexpr (std::is_same<T, std::string>::value) {
372 const size_t n_items = gguf_get_arr_n(ctx, kid);
373
374 for (size_t i = 0; i < n_items; i++) {
375 const T value = gguf_get_arr_str(ctx, kid, i);
376 result[i] = value;
377 }
378 } else {
379 if (arr_info.gt == GGUF_TYPE_BOOL) {
380 const int8_t * values = (const int8_t *) arr_info.data;
381 std::transform(values, values + arr_info.length, result.begin(), [](int8_t x) {
382 return static_cast<T>(x != 0);
383 });
384 } else {
385 std::copy((const T*)arr_info.data, (const T *)arr_info.data + arr_info.length, result.begin());
386 }
387 }
388
389 return true;
390 }
391
392 template<typename T>
393 bool llama_model_loader::get_arr(enum llm_kv kid, T & result, bool required) {
394 return get_arr(llm_kv(kid), result, required);
395 }
396
397 template bool llama_model_loader::get_arr<std::vector<std::string>>(enum llm_kv kid, std::vector<std::string> & result, bool required);
398 template bool llama_model_loader::get_arr<std::array<int32_t, 512>>(enum llm_kv kid, std::array<int32_t, 512> & result, bool required);
399 template bool llama_model_loader::get_arr<std::vector<int32_t>>(enum llm_kv kid, std::vector<int32_t> & result, bool required);
400
401 template<typename T>
402 bool llama_model_loader::get_key(const std::string & key, T & result, bool required) {
403 auto it = kv_overrides.find(key);
404
405 const struct llama_model_kv_override * override =
5
←
'override' initialized to a null pointer value
→
406 it != kv_overrides.end() ? &it->second : nullptr;
3
←
Assuming the condition is false
→
4
←
'?' condition is false
→
407
408 const bool found = GGUFMeta::GKV<T>::set(metadata, key, result, override);
6
←
Passing null pointer value via 4th parameter 'ovrd'
→
7
←
Calling 'GKV::set'
→
409
410 if (required && !found) {
411 throwabort_with_suppression(); if (false) std::runtime_error(format("key not found in model: %s", key.c_str()));
412 }
413
414 return found;
415 }
416
417 template<typename T>
418 bool llama_model_loader::get_key(enum llm_kv kid, T & result, bool required) {
419 return get_key(llm_kv(kid), result, required);
2
←
Calling 'llama_model_loader::get_key'
→
420 }
421
422 template bool llama_model_loader::get_key<bool> (enum llm_kv kid, bool & result, bool required);
423 template bool llama_model_loader::get_key<float> (enum llm_kv kid, float & result, bool required);
424 template bool llama_model_loader::get_key<uint32_t> (enum llm_kv kid, uint32_t & result, bool required);
425 template bool llama_model_loader::get_key<std::string>(enum llm_kv kid, std::string & result, bool required);
426
427 template<>
428 bool llama_model_loader::get_key(enum llm_kv kid, enum llama_pooling_type & result, bool required) {
429 uint32_t tmp;
430 const bool found = get_key(kid, tmp, required);
1
Calling 'llama_model_loader::get_key'
→
431 if (found) {
432 result = (enum llama_pooling_type) tmp;
433 } else {
434 result = LLAMA_POOLING_TYPE_UNSPECIFIED;
435 }
436 return found;
437 }
438
439 // get array of n <= N_MAX elements, or a single element repeated n times
440 template<typename T, size_t N_MAX>
441 bool llama_model_loader::get_key_or_arr(const std::string & key, std::array<T, N_MAX> & result, uint32_t n, bool required) {
442 const int kid = gguf_find_key(metadata, key.c_str());
443
444 if (kid < 0) {
445 if (required) {
446 throwabort_with_suppression(); if (false) std::runtime_error(format("key not found in model: %s", key.c_str()));
447 }
448 return false;
449 }
450
451 if (n > N_MAX) {
452 throwabort_with_suppression(); if (false) std::runtime_error(format("n > N_MAX: %u > %u for key %s", n, (uint32_t) N_MAX, key.c_str()));
453 }
454
455 if (gguf_get_kv_type(metadata, kid) == GGUF_TYPE_ARRAY) {
456 struct GGUFMeta::ArrayInfo arr_info =
457 GGUFMeta::GKV<GGUFMeta::ArrayInfo>::get_kv(metadata, kid);
458
459 if (n != arr_info.length) {
460 throwabort_with_suppression(); if (false) std::runtime_error(format("key %s has wrong array length; expected %u, got %u", key.c_str(), n, (uint32_t) arr_info.length));
461 }
462
463 return get_arr(key, result, required);
464 }
465
466 T value;
467
468 bool ok = get_key(key, value, required);
469 if (!ok) {
470 return false;
471 }
472
473 for (uint32_t i = 0; i < n; i++) {
474 result[i] = value;
475 }
476
477 return true;
478 }
479
480 template<typename T>
481 bool llama_model_loader::get_key_or_arr(enum llm_kv kid, T & result, uint32_t n, bool required) {
482 return get_key_or_arr(llm_kv(kid), result, n, required);
483 }
484
485 bool llama_model_loader::get_key_or_arr(enum llm_kv kid, uint32_t & result, bool required) {
486 const std::string key = llm_kv(kid);
487
488 const int id = gguf_find_key(metadata, key.c_str());
489
490 if (id < 0) {
491 if (required) {
492 throwabort_with_suppression(); if (false) std::runtime_error(format("key not found in model: %s", key.c_str()));
493 }
494 return false;
495 }
496
497 // throw and error if type is an array
498 if (gguf_get_kv_type(metadata, id) == GGUF_TYPE_ARRAY) {
499 if (required) {
500 throwabort_with_suppression(); if (false) std::runtime_error(format("expected scalar, found array for key: %s", key.c_str()));
501 }
502 return false;
503 }
504
505 return get_key(key, result, required);
506 }
507
508 // TODO: this is not very clever - figure out something better
509 template bool llama_model_loader::get_key_or_arr<std::array<int, 4>> (enum llm_kv kid, std::array<int, 4> & result, uint32_t n, bool required);
510 template bool llama_model_loader::get_key_or_arr<std::array<uint32_t, 512>>(enum llm_kv kid, std::array<uint32_t, 512> & result, uint32_t n, bool required);
511 template bool llama_model_loader::get_key_or_arr<std::array<float, 512>>(enum llm_kv kid, std::array<float, 512> & result, uint32_t n, bool required);
512
513
514llama_model_loader::llama_model_loader(
515 struct gguf_context * meta,
516 llama_model_set_tensor_data_t set_tensor_data,
517 void * set_tensor_data_ud,
518 const std::string & fname,
519 std::vector<std::string> & splits,
520 FILE * file,
521 const void * buffer,
522 size_t buffer_size,
523 bool use_mmap,
524 bool use_direct_io,
525 bool check_tensors,
526 bool no_alloc,
527 const llama_model_kv_override * param_overrides_p,
528 const llama_model_tensor_buft_override * param_tensor_buft_overrides_p)
529 : metadata(meta), set_tensor_data(set_tensor_data), set_tensor_data_ud(set_tensor_data_ud) {
530 int trace = 0;
531 if (getenv("LLAMA_TRACE")) {
532 trace = atoi(getenv("LLAMA_TRACE"));
533 }
534
535 if (param_overrides_p != nullptr) {
536 for (const struct llama_model_kv_override * p = param_overrides_p; p->key[0] != 0; p++) {
537 kv_overrides.insert({std::string(p->key), *p});
538 }
539 }
540
541 tensor_buft_overrides = param_tensor_buft_overrides_p;
542
543 if (!fname.empty()) {
544 // Load the main GGUF
545 struct ggml_context * ctx = NULL__null;
546 struct gguf_init_params params = {
547 /*.no_alloc = */ true,
548 /*.ctx = */ &ctx,
549 };
550
551 metadata_ptr.reset(gguf_init_from_file(fname.c_str(), params));
552 metadata = metadata_ptr.get();
553 if (metadata == nullptr) {
554 throwabort_with_suppression(); if (false) std::runtime_error(format("%s: failed to load model from %s", __func__, fname.c_str()));
555 }
556
557 get_key(llm_kv(LLM_KV_GENERAL_ARCHITECTURE), arch_name, false);
558 llm_kv = LLM_KV(llm_arch_from_string(arch_name));
559
560 files.emplace_back(new llama_file(fname.c_str(), "rb", use_direct_io));
561 contexts.emplace_back(ctx);
562
563 if (use_mmap && use_direct_io) {
564 if (files.back()->has_direct_io()) {
565 LLAMA_LOG_WARN("%s: direct I/O is enabled, disabling mmap\n", __func__)llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: direct I/O is enabled, disabling mmap\n"
, __func__);
566 use_mmap = false;
567 } else {
568 LLAMA_LOG_WARN("%s: direct I/O is not available, using mmap\n", __func__)llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: direct I/O is not available, using mmap\n"
, __func__);
569 use_direct_io = false;
570
571 // reopen file using std::fopen for mmap
572 files.pop_back();
573 files.emplace_back(new llama_file(fname.c_str(), "rb", false));
574 }
575 }
576
577 // Save tensors data offset of the main file.
578 // For subsidiary files, `meta` tensor data offset must not be used,
579 // so we build a unified tensors index for weights.
580 for (ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) {
581 std::string tensor_name = std::string(cur->name);
582 // make sure there is no duplicated tensor names
583 if (weights_map.find(tensor_name) != weights_map.end()) {
584 throwabort_with_suppression(); if (false) std::runtime_error(format("invalid model: tensor '%s' is duplicated", ggml_get_name(cur)));
585 }
586 n_elements += ggml_nelements(cur);
587 n_bytes += ggml_nbytes(cur);
588 weights_map.emplace(tensor_name, llama_tensor_weight(files.back().get(), 0, metadata, cur));
589 }
590 uint16_t n_split = 0;
591 get_key(llm_kv(LLM_KV_SPLIT_COUNT), n_split, false);
592
593 // Load additional GGML contexts
594 if (n_split > 1) {
595 // make sure the main file is loaded first
596 uint16_t idx = 0;
597 const std::string kv_split_no = llm_kv(LLM_KV_SPLIT_NO);
598 get_key(kv_split_no, idx);
599 if (idx != 0) {
600 throwabort_with_suppression(); if (false) std::runtime_error(format("illegal split file idx: %d (file: %s), model must be loaded with the first split", idx, fname.c_str()));
601 }
602
603 // generate list of splits if needed
604 if (splits.empty()) {
605 splits = llama_get_list_splits(fname, idx, n_split);
606 }
607
608 // in case user give a custom list of splits, check if it matches the expected number
609 if (n_split != (uint16_t)splits.size()) {
610 throwabort_with_suppression(); if (false) std::runtime_error(format("invalid split count, given: %zu splits, but expected %d", splits.size(), n_split));
611 }
612
613 if (trace > 0) {
614 LLAMA_LOG_INFO("%s: loading additional %d GGUFs\n", __func__, n_split)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: loading additional %d GGUFs\n"
, __func__, n_split);
615 }
616
617 // load other splits
618 for (idx = 1; idx < n_split; idx++) {
619 const char * fname_split = splits[idx].c_str();
620
621 struct gguf_init_params split_params = {
622 /*.no_alloc = */ true,
623 /*.ctx = */ &ctx,
624 };
625 gguf_context_ptr ctx_gguf { gguf_init_from_file(fname_split, split_params) };
626 if (!ctx_gguf) {
627 throwabort_with_suppression(); if (false) std::runtime_error(format("%s: failed to load GGUF split from %s", __func__, fname_split));
628 }
629
630 // check idx
631 {
632 const int kid = gguf_find_key(ctx_gguf.get(), kv_split_no.c_str());
633 if (kid < 0) {
634 throwabort_with_suppression(); if (false) std::runtime_error(format("missing key %s in GGUF split %s", kv_split_no.c_str(), fname_split));
635 }
636 int idx_gguf = gguf_get_val_u16(ctx_gguf.get(), kid);
637 if (idx_gguf != idx) {
638 throwabort_with_suppression(); if (false) std::runtime_error(format("invalid split file idx: %d (file: %s), expected %d", idx_gguf, fname_split, idx));
639 }
640 }
641
642 files.emplace_back(new llama_file(fname_split, "rb", use_direct_io));
643 contexts.emplace_back(ctx);
644
645 // Save tensors data offset info of the shard.
646 for (ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) {
647 std::string tensor_name = std::string(cur->name);
648 // make sure there is no duplicated tensor names
649 if (weights_map.find(tensor_name) != weights_map.end()) {
650 throwabort_with_suppression(); if (false) std::runtime_error(format("invalid model: tensor '%s' is duplicated", ggml_get_name(cur)));
651 }
652 n_elements += ggml_nelements(cur);
653 n_bytes += ggml_nbytes(cur);
654 weights_map.emplace(tensor_name, llama_tensor_weight(files.back().get(), idx, ctx_gguf.get(), cur));
655 }
656 }
657
658 get_key(llm_kv(LLM_KV_SPLIT_TENSORS_COUNT), n_tensors);
659
660 // sanity check
661 {
662 const int n_tensors_loaded = (int) weights_map.size();
663 if (n_tensors != n_tensors_loaded) {
664 throwabort_with_suppression(); if (false) std::runtime_error(format("corrupted model: %d tensors expected but %d found", n_tensors, n_tensors_loaded));
665 }
666 }
667
668 LLAMA_LOG_INFO("%s: additional %d GGUFs metadata loaded.\n", __func__, n_split - 1)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: additional %d GGUFs metadata loaded.\n"
, __func__, n_split - 1);
669 }
670 } else if (file != nullptr) {
671 struct ggml_context * ctx = NULL__null;
672 struct gguf_init_params params = {
673 /*.no_alloc = */ true,
674 /*.ctx = */ &ctx,
675 };
676
677 metadata_ptr.reset(gguf_init_from_file_ptr(file, params));
678 metadata = metadata_ptr.get();
679 if (metadata == nullptr) {
680 throwabort_with_suppression(); if (false) std::runtime_error(format("%s: failed to load model from file pointer", __func__));
681 }
682
683 get_key(llm_kv(LLM_KV_GENERAL_ARCHITECTURE), arch_name, false);
684 llm_kv = LLM_KV(llm_arch_from_string(arch_name));
685
686 files.emplace_back(new llama_file(file));
687 contexts.emplace_back(ctx);
688
689 // Save tensors data offset info of the main file.
690 for (ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) {
691 std::string tensor_name = std::string(cur->name);
692 // make sure there is no duplicated tensor names
693 if (weights_map.find(tensor_name) != weights_map.end()) {
694 throwabort_with_suppression(); if (false) std::runtime_error(format("invalid model: tensor '%s' is duplicated", ggml_get_name(cur)));
695 }
696 n_elements += ggml_nelements(cur);
697 n_bytes += ggml_nbytes(cur);
698 weights_map.emplace(tensor_name, llama_tensor_weight(files.back().get(), 0, metadata, cur));
699 }
700 } else if (buffer != nullptr) {
701 // Firefox: load the model from an in-memory buffer (no file/mmap)
702 struct ggml_context * ctx = NULL__null;
703 struct gguf_init_params params = {
704 /*.no_alloc = */ true,
705 /*.ctx = */ &ctx,
706 };
707
708 metadata_ptr.reset(gguf_init_from_buffer(buffer, buffer_size, params));
709 metadata = metadata_ptr.get();
710 if (metadata == nullptr) {
711 throwabort_with_suppression(); if (false) std::runtime_error(format("%s: failed to load model from buffer", __func__));
712 }
713
714 this->buffer_data = buffer;
715 this->buffer_size = buffer_size;
716
717 get_key(llm_kv(LLM_KV_GENERAL_ARCHITECTURE), arch_name, false);
718 llm_kv = LLM_KV(llm_arch_from_string(arch_name));
719
720 contexts.emplace_back(ctx);
721
722 // Save tensors data offset info, bounds-checked against the buffer.
723 for (ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) {
724 std::string tensor_name = std::string(cur->name);
725 if (weights_map.find(tensor_name) != weights_map.end()) {
726 throwabort_with_suppression(); if (false) std::runtime_error(format("invalid model: tensor '%s' is duplicated", ggml_get_name(cur)));
727 }
728 n_elements += ggml_nelements(cur);
729 n_bytes += ggml_nbytes(cur);
730 weights_map.emplace(tensor_name, llama_tensor_weight(buffer_size, 0, metadata, cur));
731 }
732 } else {
733 get_key(llm_kv(LLM_KV_GENERAL_ARCHITECTURE), arch_name, false);
734 llm_kv = LLM_KV(llm_arch_from_string(arch_name));
735 }
736
737 n_kv = gguf_get_n_kv(metadata);
738 n_tensors = weights_map.size();
739
740 fver = (enum llama_fver) gguf_get_version(metadata);
741
742 LLAMA_LOG_INFO("%s: loaded meta data with %d key-value pairs and %d tensors from %s (version %s)\n",llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: loaded meta data with %d key-value pairs and %d tensors from %s (version %s)\n"
, __func__, n_kv, n_tensors, fname.empty() ? "(file*)" : fname
.c_str(), llama_file_version_name(fver))
743 __func__, n_kv, n_tensors, fname.empty() ? "(file*)" : fname.c_str(), llama_file_version_name(fver))llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: loaded meta data with %d key-value pairs and %d tensors from %s (version %s)\n"
, __func__, n_kv, n_tensors, fname.empty() ? "(file*)" : fname
.c_str(), llama_file_version_name(fver));
744
745 // determine file type based on the number of tensors for each quantization and print meta data
746 // TODO: make optional
747 {
748 std::map<enum ggml_type, uint32_t> n_type;
749
750 uint32_t n_type_max = 0;
751 enum ggml_type type_max = GGML_TYPE_F32;
752
753 for (const auto & it : weights_map) {
754 const llama_tensor_weight & w = it.second;
755 const ggml_tensor * tensor = w.tensor;
756
757 enum ggml_type type = tensor->type;
758
759 n_type[type]++;
760
761 if (n_type_max < n_type[type]) {
762 n_type_max = n_type[type];
763 type_max = type;
764 }
765
766 if (trace > 0) {
767 const uint16_t sid = w.idx;
768 LLAMA_LOG_INFO("%s: - tensor split %2d: %32s %-8s [ %s ] %8.2f MiB\n", __func__,llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: - tensor split %2d: %32s %-8s [ %s ] %8.2f MiB\n"
, __func__, sid, ggml_get_name(tensor), ggml_type_name(type),
llama_format_tensor_shape(tensor).c_str(), ggml_nbytes(tensor
)/1024.0f/1024.0f)
769 sid, ggml_get_name(tensor), ggml_type_name(type), llama_format_tensor_shape(tensor).c_str(),llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: - tensor split %2d: %32s %-8s [ %s ] %8.2f MiB\n"
, __func__, sid, ggml_get_name(tensor), ggml_type_name(type),
llama_format_tensor_shape(tensor).c_str(), ggml_nbytes(tensor
)/1024.0f/1024.0f)
770 ggml_nbytes(tensor)/1024.0f/1024.0f)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: - tensor split %2d: %32s %-8s [ %s ] %8.2f MiB\n"
, __func__, sid, ggml_get_name(tensor), ggml_type_name(type),
llama_format_tensor_shape(tensor).c_str(), ggml_nbytes(tensor
)/1024.0f/1024.0f);
771 }
772 }
773
774 switch (type_max) {
775 case GGML_TYPE_F32: ftype = LLAMA_FTYPE_ALL_F32; break;
776 case GGML_TYPE_F16: ftype = LLAMA_FTYPE_MOSTLY_F16; break;
777 case GGML_TYPE_BF16: ftype = LLAMA_FTYPE_MOSTLY_BF16; break;
778 case GGML_TYPE_Q4_0: ftype = LLAMA_FTYPE_MOSTLY_Q4_0; break;
779 case GGML_TYPE_Q4_1: ftype = LLAMA_FTYPE_MOSTLY_Q4_1; break;
780 case GGML_TYPE_Q5_0: ftype = LLAMA_FTYPE_MOSTLY_Q5_0; break;
781 case GGML_TYPE_Q5_1: ftype = LLAMA_FTYPE_MOSTLY_Q5_1; break;
782 case GGML_TYPE_Q8_0: ftype = LLAMA_FTYPE_MOSTLY_Q8_0; break;
783 case GGML_TYPE_Q2_K: ftype = LLAMA_FTYPE_MOSTLY_Q2_K; break;
784 case GGML_TYPE_Q3_K: ftype = LLAMA_FTYPE_MOSTLY_Q3_K_M; break;
785 case GGML_TYPE_Q4_K: ftype = LLAMA_FTYPE_MOSTLY_Q4_K_M; break;
786 case GGML_TYPE_Q5_K: ftype = LLAMA_FTYPE_MOSTLY_Q5_K_M; break;
787 case GGML_TYPE_Q6_K: ftype = LLAMA_FTYPE_MOSTLY_Q6_K; break;
788 case GGML_TYPE_TQ1_0: ftype = LLAMA_FTYPE_MOSTLY_TQ1_0; break;
789 case GGML_TYPE_TQ2_0: ftype = LLAMA_FTYPE_MOSTLY_TQ2_0; break;
790 case GGML_TYPE_IQ2_XXS: ftype = LLAMA_FTYPE_MOSTLY_IQ2_XXS; break;
791 case GGML_TYPE_IQ2_XS: ftype = LLAMA_FTYPE_MOSTLY_IQ2_XS; break;
792 case GGML_TYPE_IQ2_S: ftype = LLAMA_FTYPE_MOSTLY_IQ2_S; break;
793 case GGML_TYPE_IQ3_XXS: ftype = LLAMA_FTYPE_MOSTLY_IQ3_XXS; break;
794 case GGML_TYPE_IQ1_S: ftype = LLAMA_FTYPE_MOSTLY_IQ1_S; break;
795 case GGML_TYPE_IQ1_M: ftype = LLAMA_FTYPE_MOSTLY_IQ1_M; break;
796 case GGML_TYPE_IQ4_NL: ftype = LLAMA_FTYPE_MOSTLY_IQ4_NL; break;
797 case GGML_TYPE_IQ4_XS: ftype = LLAMA_FTYPE_MOSTLY_IQ4_XS; break;
798 case GGML_TYPE_IQ3_S: ftype = LLAMA_FTYPE_MOSTLY_IQ3_S; break;
799 case GGML_TYPE_NVFP4: ftype = LLAMA_FTYPE_MOSTLY_NVFP4; break;
800 case GGML_TYPE_Q1_0: ftype = LLAMA_FTYPE_MOSTLY_Q1_0; break;
801 default:
802 {
803 LLAMA_LOG_WARN("%s: unknown type %s\n", __func__, ggml_type_name(type_max))llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: unknown type %s\n"
, __func__, ggml_type_name(type_max));
804 ftype = LLAMA_FTYPE_ALL_F32;
805 } break;
806 }
807
808 // this is a way to mark that we have "guessed" the file type
809 ftype = (llama_ftype) (ftype | LLAMA_FTYPE_GUESSED);
810
811 {
812 uint32_t ftype_val = 0;
813 if (get_key(LLM_KV_GENERAL_FILE_TYPE, ftype_val, false)) {
814 ftype = (llama_ftype) ftype_val;
815 }
816 }
817 // Firefox: this dump proved quite expensive for nothing in profiling.
818 // Gated away in default builds.
819 if (trace > 0) {
820 LLAMA_LOG_INFO("%s: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n", __func__)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n"
, __func__);
821
822 for (int i = 0; i < n_kv; i++) {
823 const char * name = gguf_get_key(metadata, i);
824 const enum gguf_type type = gguf_get_kv_type(metadata, i);
825 const std::string type_name =
826 type == GGUF_TYPE_ARRAY
827 ? format("%s[%s,%zu]", gguf_type_name(type), gguf_type_name(gguf_get_arr_type(metadata, i)), gguf_get_arr_n(metadata, i))
828 : gguf_type_name(type);
829
830 std::string value = gguf_kv_to_str(metadata, i);
831 const size_t MAX_VALUE_LEN = 40;
832 if (value.size() > MAX_VALUE_LEN) {
833 value = format("%s...", value.substr(0, MAX_VALUE_LEN - 3).c_str());
834 }
835 replace_all(value, "\n", "\\n");
836
837 LLAMA_LOG_INFO("%s: - kv %3d: %42s %-16s = %s\n", __func__, i, name, type_name.c_str(), value.c_str())llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: - kv %3d: %42s %-16s = %s\n"
, __func__, i, name, type_name.c_str(), value.c_str());
838 }
839 }
840 // print type counts
841 for (auto & kv : n_type) {
842 if (kv.second == 0) {
843 continue;
844 }
845
846 LLAMA_LOG_INFO("%s: - type %4s: %4d tensors\n", __func__, ggml_type_name(kv.first), kv.second)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: - type %4s: %4d tensors\n"
, __func__, ggml_type_name(kv.first), kv.second);
847 }
848 }
849
850 if (!llama_mmap::SUPPORTED) {
851 LLAMA_LOG_WARN("%s: mmap is not supported on this platform\n", __func__)llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: mmap is not supported on this platform\n"
, __func__);
852 use_mmap = false;
853 }
854
855 if (buffer_data != nullptr) {
856 // buffer-based loading reads tensor data directly from memory
857 use_mmap = false;
858 }
859
860 this->use_mmap = use_mmap;
861 this->use_direct_io = use_direct_io;
862 this->check_tensors = check_tensors;
863 this->no_alloc = no_alloc;
864}
865
866std::string llama_model_loader::get_arch_name() const {
867 return arch_name;
868}
869
870enum llm_arch llama_model_loader::get_arch() const {
871 return llm_kv.arch;
872}
873
874const llama_model_loader::llama_tensor_weight * llama_model_loader::get_weight(const char * name) const {
875 auto pos = weights_map.find(name);
876 if (pos != weights_map.end()) {
877 return &pos->second;
878 }
879
880 return nullptr;
881}
882
883const llama_model_loader::llama_tensor_weight & llama_model_loader::require_weight(const char * name) const {
884 const llama_tensor_weight * weight = get_weight(name);
885 if (!weight) {
886 throwabort_with_suppression(); if (false) std::runtime_error(format("%s: tensor '%s' not found", __func__, name));
887 }
888 return *weight;
889}
890
891struct ggml_tensor * llama_model_loader::get_tensor_meta(const char * name) const {
892 const auto * weight = get_weight(name);
893 if (!weight) {
894 return nullptr;
895 }
896 return weight->tensor;
897}
898
899struct ggml_tensor * llama_model_loader::require_tensor_meta(const std::string & name) const {
900 struct ggml_tensor * tensor = get_tensor_meta(name.c_str());
901 if (!tensor) {
902 throwabort_with_suppression(); if (false) std::runtime_error(format("%s: tensor '%s' not found", __func__, name.c_str()));
903 }
904 return tensor;
905}
906
907const struct ggml_tensor * llama_model_loader::check_tensor_dims(const std::string & name, const std::vector<int64_t> & ne, bool required) const {
908 const struct ggml_tensor * cur = get_tensor_meta(name.c_str());
909
910 if (cur == NULL__null) {
911 if (!required) {
912 return NULL__null;
913 }
914 throwabort_with_suppression(); if (false) std::runtime_error(format("%s: tensor '%s' not found", __func__, name.c_str()));
915 }
916
917 {
918 bool is_ok = true;
919 for (size_t i = 0; i < GGML_MAX_DIMS4; ++i) {
920 if ((i < ne.size() && ne[i] != cur->ne[i]) || (i >= ne.size() && cur->ne[i] != 1)) {
921 is_ok = false;
922 break;
923 }
924 }
925 if (!is_ok) {
926 throwabort_with_suppression(); if (false) std::runtime_error(
927 format("%s: tensor '%s' has wrong shape; expected %s, got %s",
928 __func__, name.c_str(),
929 llama_format_tensor_shape(ne).c_str(),
930 llama_format_tensor_shape(cur).c_str()));
931 }
932 }
933
934 return cur;
935}
936
937// checks if the weight tensor can be used with the specified buffer type and device
938static bool weight_buft_supported(const llama_hparams & hparams, ggml_tensor * w, ggml_op op, ggml_backend_buffer_type_t buft, ggml_backend_dev_t dev) {
939 GGML_ASSERT(w != nullptr)if (!(w != nullptr)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-model-loader.cpp"
, 939, "GGML_ASSERT(%s) failed", "w != nullptr");
940
941 if (op == GGML_OP_NONE) {
942 return true;
943 }
944
945 ggml_init_params params = {
946 /*.mem_size =*/ ggml_tensor_overhead()*8,
947 /*.mem_buffer =*/ NULL__null,
948 /*.no_alloc =*/ true,
949 };
950 ggml_context_ptr ctx_ptr { ggml_init(params) };
951 if (!ctx_ptr) {
952 throwabort_with_suppression(); if (false) std::runtime_error(format("failed to create ggml context"));
953 }
954 ggml_context * ctx = ctx_ptr.get();
955
956 ggml_tensor * op_tensor = nullptr;
957
958 switch (op) {
959 case GGML_OP_GET_ROWS:
960 {
961 ggml_tensor * b = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 512);
962 op_tensor = ggml_get_rows(ctx, w, b);
963 } break;
964 case GGML_OP_MUL_MAT:
965 {
966 ggml_tensor * b = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, w->ne[0], 512, w->ne[2], w->ne[3]);
967 op_tensor = ggml_mul_mat(ctx, w, b);
968 } break;
969 case GGML_OP_MUL_MAT_ID:
970 {
971 const int n_expert_used = hparams.n_expert_used;
972 GGML_ASSERT(n_expert_used > 0)if (!(n_expert_used > 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-model-loader.cpp"
, 972, "GGML_ASSERT(%s) failed", "n_expert_used > 0");
973 ggml_tensor * b = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, w->ne[0], n_expert_used, 512);
974 ggml_tensor * ids = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_expert_used, 512);
975 op_tensor = ggml_mul_mat_id(ctx, w, b, ids);
976 } break;
977 case GGML_OP_ADD:
978 {
979 ggml_tensor * a = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, w->ne[0], w->ne[1], w->ne[2], w->ne[3]);
980 op_tensor = ggml_add(ctx, a, w);
981 } break;
982 case GGML_OP_ADD_ID:
983 {
984 const int n_expert_used = hparams.n_expert_used;
985 GGML_ASSERT(n_expert_used > 0)if (!(n_expert_used > 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-model-loader.cpp"
, 985, "GGML_ASSERT(%s) failed", "n_expert_used > 0");
986 ggml_tensor * a = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, w->ne[0], n_expert_used, 512);
987 ggml_tensor * c = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_expert_used, 512);
988 op_tensor = ggml_add_id(ctx, a, w, c);
989 } break;
990 case GGML_OP_MUL:
991 {
992 ggml_tensor * a = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, w->ne[0], w->ne[1], w->ne[2], w->ne[3]);
993 op_tensor = ggml_mul(ctx, a, w);
994 } break;
995 case GGML_OP_DIV:
996 {
997 ggml_tensor * a = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, w->ne[0]);
998 op_tensor = ggml_div(ctx, a, w);
999 } break;
1000 case GGML_OP_ROPE:
1001 {
1002 const int n_embd_head = hparams.n_embd_head_v();
1003 const int n_head = hparams.n_head();
1004 ggml_tensor * a = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, n_embd_head, n_head, 512);
1005 ggml_tensor * b = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 512);
1006 op_tensor = ggml_rope_ext(
1007 ctx, a, b, w,
1008 0, 0, 0, 0, 0,
1009 0, 0, 0, 0
1010 );
1011
1012 } break;
1013 case GGML_OP_SSM_CONV:
1014 {
1015 const int64_t n_seq_tokens = 512;
1016 const int64_t n_seqs = 3;
1017 ggml_tensor * conv_x = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, w->ne[0] - 1 + n_seq_tokens, w->ne[1], n_seqs);
1018 op_tensor = ggml_ssm_conv(ctx, conv_x, w);
1019 } break;
1020 case GGML_OP_SSM_SCAN:
1021 {
1022 // w is ssm_a, which is used to distinguish Mamba-1 and Mamba-2
1023 const int64_t d_state = w->ne[0] == 1 ? hparams.ssm_d_state : w->ne[0];
1024 const int64_t n_head = w->ne[1];
1025 const int64_t head_dim = hparams.ssm_d_inner / n_head;
1026 const int64_t n_group = hparams.ssm_n_group ? hparams.ssm_n_group : 1;
1027 const int64_t n_seq_tokens = 512;
1028 const int64_t n_seqs = 3;
1029 ggml_tensor * s = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, d_state, head_dim, n_head, n_seqs);
1030 ggml_tensor * x = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, head_dim, n_head, n_seq_tokens, n_seqs);
1031 ggml_tensor * dt = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, n_head, n_seq_tokens, n_seqs);
1032 ggml_tensor * B = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, d_state, n_group, n_seq_tokens, n_seqs);
1033 ggml_tensor * C = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, d_state, n_group, n_seq_tokens, n_seqs);
1034 ggml_tensor * ids = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, n_seqs);
1035 op_tensor = ggml_ssm_scan(ctx, s, x, dt, w, B, C, ids);
1036 } break;
1037 case GGML_OP_RWKV_WKV6:
1038 {
1039 // FIXME
1040 const int64_t S = 123;
1041 const int64_t H = 123;
1042 const int64_t n_tokens = 123;
1043 const int64_t n_seqs = 123;
1044 ggml_tensor * k = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, S, H, n_tokens);
1045 ggml_tensor * v = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, S, H, n_tokens);
1046 ggml_tensor * r = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, S, H, n_tokens);
1047 ggml_tensor * tf = w;
1048 ggml_tensor * td = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, S, H, n_tokens);
1049 ggml_tensor * state = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, S, n_seqs, S, H);
1050 op_tensor = ggml_rwkv_wkv6(ctx, k, v, r, tf, td, state);
1051 } break;
1052 case GGML_OP_IM2COL:
1053 {
1054 const int n_embd_inp = hparams.n_embd_inp();
1055 ggml_tensor * b = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, n_embd_inp, w->ne[1], 1, 1);
1056 op_tensor = ggml_im2col(ctx, w, b, 1, 0, 0, 0, 1, 0, false, GGML_TYPE_F16);
1057 } break;
1058 case GGML_OP_SCALE:
1059 {
1060 op_tensor = ggml_scale(ctx, w, 1.0f);
1061 } break;
1062 default:
1063 GGML_ABORT("%s: missing test for op %s for tensor %s", __func__, ggml_op_name(op), w->name)ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-model-loader.cpp"
, 1063, "%s: missing test for op %s for tensor %s", __func__,
ggml_op_name(op), w->name);
1064 }
1065
1066 // create a temporary dummy buffer for the weight so that supports_op can check the buffer type
1067 GGML_ASSERT(w->buffer == nullptr)if (!(w->buffer == nullptr)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-model-loader.cpp"
, 1067, "GGML_ASSERT(%s) failed", "w->buffer == nullptr");
1068 w->buffer = ggml_backend_buft_alloc_buffer(buft, 0);
1069 bool op_supported = ggml_backend_dev_supports_op(dev, op_tensor);
1070 ggml_backend_buffer_free(w->buffer);
1071 w->buffer = nullptr;
1072
1073 return op_supported;
1074}
1075
1076// find the first buffer type in the list that can use the tensor
1077static ggml_backend_buffer_type_t select_weight_buft(const llama_hparams & hparams, ggml_tensor * tensor, ggml_op op, const buft_list_t * buft_list) {
1078 GGML_ASSERT(!buft_list->empty())if (!(!buft_list->empty())) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-model-loader.cpp"
, 1078, "GGML_ASSERT(%s) failed", "!buft_list->empty()");
1079 for (const auto & cur : *buft_list) {
1080 ggml_backend_dev_t cur_dev = cur.first;
1081 ggml_backend_buffer_type_t cur_buft = cur.second;
1082 if (weight_buft_supported(hparams, tensor, op, cur_buft, cur_dev)) {
1083 return cur_buft;
1084 }
1085 }
1086
1087 return nullptr;
1088}
1089
1090struct ggml_tensor * llama_model_loader::create_tensor(
1091 const llama_hparams & hparams, const buft_list_t * buft_list_cpu, const buft_list_t * buft_list_input, const buft_list_t * buft_list_output,
1092 const buft_list_t * buft_list_layer, const LLM_TN_IMPL & tn, const std::initializer_list<int64_t> & ne, int flags) {
1093 auto ctx_for_buft = [&](ggml_backend_buffer_type_t buft) -> ggml_context * {
1094 auto it = ctx_map.find(buft);
1095 if (it == ctx_map.end()) {
1096 // one ggml context per buffer type
1097 int max_n_tensors = n_tensors;
1098 max_n_tensors += 1; // duplicated output tensor
1099 max_n_tensors += hparams.n_layer()*2; // duplicated rope freq tensors
1100 if (files.empty()) {
1101 max_n_tensors += hparams.n_layer()*256; // this should be well above what any model actually uses
1102 }
1103 const size_t ctx_size = ggml_tensor_overhead()*max_n_tensors;
1104
1105 ggml_init_params params = {
1106 /*.mem_size =*/ ctx_size,
1107 /*.mem_buffer =*/ NULL__null,
1108 /*.no_alloc =*/ true,
1109 };
1110
1111 ggml_context * ctx = ggml_init(params);
1112 if (!ctx) {
1113 throwabort_with_suppression(); if (false) std::runtime_error(format("failed to create ggml context"));
1114 }
1115
1116 ctx_map.emplace(buft, ctx);
1117
1118 return ctx;
1119 }
1120 return it->second.get();
1121 };
1122
1123 auto buft_for_tensor = [&](ggml_tensor * t_meta) -> ggml_backend_buffer_type_t {
1124 if (!t_meta) {
1125 if (flags & TENSOR_NOT_REQUIRED) {
1126 return nullptr;
1127 }
1128 throwabort_with_suppression(); if (false) std::runtime_error(format("missing tensor '%s'", tn.str().c_str()));
1129 }
1130
1131 // some models use the token embedding tensor as the output, but since these are used in different layers and with different ops
1132 // the tensor is duplicated
1133 // to handle this, we check if the tensor is duplicated, and if so, we assume that it is being loaded as the output tensor
1134 llm_tensor tn_tensor = tn.tensor;
1135 if (tn.tensor == LLM_TENSOR_TOKEN_EMBD && (flags & TENSOR_DUPLICATED)) {
1136 tn_tensor = LLM_TENSOR_OUTPUT;
1137 }
1138
1139 llm_tensor_info info;
1140 tryif (true) {
1141 info = llm_tensor_info_for(tn_tensor);
1142 } catch (const std::out_of_range & e)if (static const std::exception e, err, error, ex; false) {
1143 throwabort_with_suppression(); if (false) std::runtime_error(format("missing tensor info mapping for %s", tn.str().c_str()));
1144 }
1145
1146 // skip unused tensors
1147 if (info.op == GGML_OP_NONE || (flags & TENSOR_SKIP)) {
1148 const size_t nbytes = ggml_nbytes(t_meta);
1149 LLAMA_LOG_WARN("model has unused tensor %s (size = %zu bytes) -- ignoring\n", tn.str().c_str(), nbytes)llama_log_internal(GGML_LOG_LEVEL_WARN , "model has unused tensor %s (size = %zu bytes) -- ignoring\n"
, tn.str().c_str(), nbytes);
1150
1151 size_data -= nbytes;
1152 n_created++;
1153
1154 return nullptr;
1155 }
1156
1157 // tensors with "bias" suffix are always used with GGML_OP_ADD or GGML_OP_ADD_ID
1158 ggml_op op;
1159 bool bias = tn.suffix != nullptr && strcmp(tn.suffix, "bias") == 0;
1160 if (bias) {
1161 if (info.op == GGML_OP_MUL_MAT_ID) {
1162 op = GGML_OP_ADD_ID;
1163 } else {
1164 op = GGML_OP_ADD;
1165 }
1166 } else {
1167 op = info.op;
1168 }
1169
1170 // sanity checks
1171 if (info.layer == LLM_TENSOR_LAYER_INPUT || info.layer == LLM_TENSOR_LAYER_OUTPUT) {
1172 if (tn.bid != -1) {
1173 GGML_ABORT("input/output layer tensor %s used with a layer number", tn.str().c_str())ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-model-loader.cpp"
, 1173, "input/output layer tensor %s used with a layer number"
, tn.str().c_str());
1174 }
1175 } else {
1176 if (tn.bid == -1) {
1177 GGML_ABORT("repeating layer tensor %s used without a layer number", tn.str().c_str())ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-model-loader.cpp"
, 1177, "repeating layer tensor %s used without a layer number"
, tn.str().c_str());
1178 }
1179 }
1180
1181 // select the buffer type for this tensor
1182 const buft_list_t * buft_list;
1183 switch (info.layer) {
1184 case LLM_TENSOR_LAYER_INPUT:
1185 buft_list = buft_list_input;
1186 break;
1187 case LLM_TENSOR_LAYER_OUTPUT:
1188 buft_list = buft_list_output;
1189 break;
1190 case LLM_TENSOR_LAYER_REPEATING:
1191 GGML_ASSERT(buft_list_layer != nullptr)if (!(buft_list_layer != nullptr)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-model-loader.cpp"
, 1191, "GGML_ASSERT(%s) failed", "buft_list_layer != nullptr"
);
1192 buft_list = buft_list_layer;
1193 break;
1194 default:
1195 GGML_ABORT("invalid layer %d for tensor %s", info.layer, tn.str().c_str())ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-model-loader.cpp"
, 1195, "invalid layer %d for tensor %s", info.layer, tn.str(
).c_str());
1196 }
1197
1198 ggml_backend_buffer_type_t buft = nullptr;
1199
1200 // check overrides
1201 if (tensor_buft_overrides) {
1202 std::string tensor_name = tn.str();
1203 for (const auto * overrides = tensor_buft_overrides; overrides->pattern != nullptr; ++overrides) {
1204 std::regex pattern(overrides->pattern);
1205 if (std::regex_search(tensor_name, pattern)) {
1206 if (overrides->buft == ggml_backend_cpu_buffer_type()) {
1207 // when overriding to a CPU buffer, consider the extra buffer types
1208 buft = select_weight_buft(hparams, t_meta, op, buft_list_cpu);
1209 if (use_mmap) {
1210 static std::once_flag once;
1211 std::call_once(once, [] {
1212 LLAMA_LOG_WARN("llama_model_loader: tensor overrides to CPU are used with mmap enabled - consider using --no-mmap for better performance\n")llama_log_internal(GGML_LOG_LEVEL_WARN , "llama_model_loader: tensor overrides to CPU are used with mmap enabled - consider using --no-mmap for better performance\n"
);
1213 });
1214 }
1215 } else {
1216 buft = overrides->buft;
1217 }
1218
1219 LLAMA_LOG_DEBUG("tensor %s (%zu MiB %s) buffer type overridden to %s\n",llama_log_internal(GGML_LOG_LEVEL_DEBUG, "tensor %s (%zu MiB %s) buffer type overridden to %s\n"
, tensor_name.c_str(), ggml_nbytes(t_meta) / 1024 / 1024, ggml_type_name
(t_meta->type), ggml_backend_buft_name(buft))
1220 tensor_name.c_str(),llama_log_internal(GGML_LOG_LEVEL_DEBUG, "tensor %s (%zu MiB %s) buffer type overridden to %s\n"
, tensor_name.c_str(), ggml_nbytes(t_meta) / 1024 / 1024, ggml_type_name
(t_meta->type), ggml_backend_buft_name(buft))
1221 ggml_nbytes(t_meta) / 1024 / 1024, ggml_type_name(t_meta->type),llama_log_internal(GGML_LOG_LEVEL_DEBUG, "tensor %s (%zu MiB %s) buffer type overridden to %s\n"
, tensor_name.c_str(), ggml_nbytes(t_meta) / 1024 / 1024, ggml_type_name
(t_meta->type), ggml_backend_buft_name(buft))
1222 ggml_backend_buft_name(buft))llama_log_internal(GGML_LOG_LEVEL_DEBUG, "tensor %s (%zu MiB %s) buffer type overridden to %s\n"
, tensor_name.c_str(), ggml_nbytes(t_meta) / 1024 / 1024, ggml_type_name
(t_meta->type), ggml_backend_buft_name(buft));
1223 break;
1224 }
1225 }
1226 }
1227
1228 if (!buft) {
1229 buft = select_weight_buft(hparams, t_meta, op, buft_list);
1230 if (!buft) {
1231 throwabort_with_suppression(); if (false) std::runtime_error(format("failed to find a compatible buffer type for tensor %s", tn.str().c_str()));
1232 }
1233 }
1234
1235 // avoid using a host buffer when using mmap
1236 auto * buft_dev = ggml_backend_buft_get_device(buft);
1237 if (use_mmap && buft_dev && buft == ggml_backend_dev_host_buffer_type(buft_dev)) {
1238 auto * cpu_dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);
1239 if (!cpu_dev) {
1240 throwabort_with_suppression(); if (false) std::runtime_error("no CPU backend found");
1241 }
1242 buft = ggml_backend_dev_buffer_type(cpu_dev);
1243 }
1244
1245 if (buft != buft_list->front().second) {
1246 if (n_tensors_moved == 0) {
1247 first_tensor_moved_name = t_meta->name;
1248 first_tensor_moved_type_name = ggml_type_name(t_meta->type);
1249 first_moved_from_buft = buft_list->front().second;
1250 first_moved_to_buft = buft;
1251 }
1252 n_tensors_moved++;
1253 }
1254
1255 return buft;
1256 };
1257
1258 if (files.empty()) {
1259 if (flags & TENSOR_SKIP_IF_VIRTUAL) {
1260 return nullptr;
1261 }
1262 ggml_type type = GGML_TYPE_F32;
1263 const int64_t tid = gguf_find_tensor(metadata, tn.str().c_str());
1264 if (tid != -1) {
1265 type = gguf_get_tensor_type(metadata, tid);
1266 }
1267
1268 // for tensors that are not required some of the dimensions can be invalid:
1269 if (flags & TENSOR_NOT_REQUIRED) {
1270 for (size_t dim = 0; dim < ne.size(); dim++) {
1271 if (ne.begin()[dim] <= 0) {
1272 return nullptr;
1273 }
1274 }
1275 }
1276
1277 ggml_tensor t_meta;
1278 memset(&t_meta, 0, sizeof(ggml_tensor));
1279 t_meta.type = type;
1280 for (size_t dim = 0; dim < GGML_MAX_DIMS4; dim++) {
1281 t_meta.ne[dim] = dim < ne.size() ? ne.begin()[dim] : 1;
1282 GGML_ASSERT(t_meta.ne[dim] >= 1)if (!(t_meta.ne[dim] >= 1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-model-loader.cpp"
, 1282, "GGML_ASSERT(%s) failed", "t_meta.ne[dim] >= 1");
1283 t_meta.nb[dim] = dim == 0 ? ggml_type_size(type) : t_meta.ne[dim-1]*t_meta.nb[dim-1];
1284 GGML_ASSERT(t_meta.nb[dim] >= 1)if (!(t_meta.nb[dim] >= 1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-model-loader.cpp"
, 1284, "GGML_ASSERT(%s) failed", "t_meta.nb[dim] >= 1");
1285 }
1286 ggml_set_name(&t_meta, tn.str().c_str());
1287
1288 ggml_backend_buffer_type_t buft = buft_for_tensor(&t_meta);
1289 GGML_ASSERT(buft != nullptr)if (!(buft != nullptr)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-model-loader.cpp"
, 1289, "GGML_ASSERT(%s) failed", "buft != nullptr");
1290 ggml_context * ctx = ctx_for_buft(buft);
1291 ggml_tensor * ret = ggml_dup_tensor(ctx, &t_meta);
1292 ggml_set_name(ret, tn.str().c_str());
1293 return ret;
1294 }
1295
1296 ggml_tensor * t_meta = get_tensor_meta(tn.str().c_str());
1297 ggml_backend_buffer_type_t buft = buft_for_tensor(t_meta);
1298 if (buft == nullptr) {
1299 return nullptr; // return type is ggml_tensor *
1300 }
1301 ggml_context * ctx = ctx_for_buft(buft);
1302
1303 // if duplicated, check if the original tensor was allocated in the same buffer type context and avoid creating a new one
1304 if (flags & TENSOR_DUPLICATED) {
1305 ggml_tensor * t = ggml_get_tensor(ctx, tn.str().c_str());
1306 if (t) {
1307 return t;
1308 }
1309 }
1310
1311 LLAMA_LOG_DEBUG("%s: loading tensor %s\n", __func__, tn.str().c_str())llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: loading tensor %s\n"
, __func__, tn.str().c_str());
1312 const struct ggml_tensor * cur = check_tensor_dims(tn.str(), ne, !(flags & TENSOR_NOT_REQUIRED));
1313
1314 if (cur == NULL__null) {
1315 return NULL__null;
1316 }
1317
1318 const bool duplicated = flags & TENSOR_DUPLICATED;
1319
1320 struct ggml_tensor * tensor = ggml_dup_tensor(ctx, cur);
1321 ggml_set_name(tensor, ggml_get_name(cur));
1322
1323 if (duplicated) {
1324 size_data += ggml_nbytes(cur);
1325 } else {
1326 n_created++;
1327 }
1328
1329 return tensor;
1330}
1331
1332struct ggml_tensor * llama_model_loader::create_tensor_as_view(struct ggml_context * ctx, struct ggml_tensor * base, const std::string & name, const std::initializer_list<int64_t> & ne, size_t offset, bool required) {
1333 const struct ggml_tensor * cur = check_tensor_dims(name, ne, required);
1334
1335 if (cur == NULL__null) {
1336 return NULL__null;
1337 }
1338
1339 if (cur->type != base->type) {
1340 throwabort_with_suppression(); if (false) std::runtime_error(format("%s: tensor '%s' has wrong type; expected %s, got %s", __func__, name.c_str(), ggml_type_name(base->type), ggml_type_name(cur->type)));
1341 }
1342
1343 std::array<int64_t, GGML_MAX_DIMS4> dims;
1344 for (size_t i = 0; i < GGML_MAX_DIMS4; ++i) {
1345 dims[i] = i < ne.size() ? ne.begin()[i] : 1;
1346 }
1347
1348 struct ggml_tensor * tensor = ggml_view_4d(ctx, base,
1349 dims[0], dims[1], dims[2], dims[3],
1350 cur->nb[1], cur->nb[2], cur->nb[3],
1351 offset);
1352
1353 ggml_set_name(tensor, name.c_str());
1354
1355 n_created++;
1356
1357 return tensor;
1358}
1359
1360void llama_model_loader::done_getting_tensors(bool partial) const {
1361 if (n_created > n_tensors) {
1362 throwabort_with_suppression(); if (false) std::runtime_error(format("%s: too many tensors created; expected %d, got %d", __func__, n_tensors, n_created));
1363 }
1364 if (n_created < n_tensors) {
1365 if (!partial) {
1366 throwabort_with_suppression(); if (false) std::runtime_error(format("%s: wrong number of tensors; expected %d, got %d", __func__, n_tensors, n_created));
1367 }
1368 LLAMA_LOG_INFO("%s: partial load — used %d of %d tensors in the file (rest belong to a sibling model on the same .gguf)\n",llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: partial load — used %d of %d tensors in the file (rest belong to a sibling model on the same .gguf)\n"
, __func__, n_created, n_tensors)
1369 __func__, n_created, n_tensors)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: partial load — used %d of %d tensors in the file (rest belong to a sibling model on the same .gguf)\n"
, __func__, n_created, n_tensors);
1370 }
1371 if (n_tensors_moved > 0) {
1372 LLAMA_LOG_DEBUG("%s: tensor '%s' (%s) (and %zu others) cannot be used with preferred buffer type %s, using %s instead\n",llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: tensor '%s' (%s) (and %zu others) cannot be used with preferred buffer type %s, using %s instead\n"
, __func__, first_tensor_moved_name.c_str(), first_tensor_moved_type_name
.c_str(), n_tensors_moved - 1, ggml_backend_buft_name(first_moved_from_buft
), ggml_backend_buft_name(first_moved_to_buft))
1373 __func__, first_tensor_moved_name.c_str(), first_tensor_moved_type_name.c_str(), n_tensors_moved - 1,llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: tensor '%s' (%s) (and %zu others) cannot be used with preferred buffer type %s, using %s instead\n"
, __func__, first_tensor_moved_name.c_str(), first_tensor_moved_type_name
.c_str(), n_tensors_moved - 1, ggml_backend_buft_name(first_moved_from_buft
), ggml_backend_buft_name(first_moved_to_buft))
1374 ggml_backend_buft_name(first_moved_from_buft), ggml_backend_buft_name(first_moved_to_buft))llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: tensor '%s' (%s) (and %zu others) cannot be used with preferred buffer type %s, using %s instead\n"
, __func__, first_tensor_moved_name.c_str(), first_tensor_moved_type_name
.c_str(), n_tensors_moved - 1, ggml_backend_buft_name(first_moved_from_buft
), ggml_backend_buft_name(first_moved_to_buft));
1375 }
1376}
1377
1378void llama_model_loader::init_mappings(bool prefetch, llama_mlocks * mlock_mmaps) {
1379 if (use_mmap) {
1380 mappings.reserve(files.size());
1381 mmaps_used.reserve(files.size());
1382 for (const auto & file : files) {
1383 bool is_numa = false;
1384
1385 auto * dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);
1386 if (dev) {
1387 auto * reg = ggml_backend_dev_backend_reg(dev);
1388 auto * is_numa_fn = (decltype(ggml_is_numa) *) ggml_backend_reg_get_proc_address(reg, "ggml_backend_cpu_is_numa");
1389 if (is_numa_fn) {
1390 is_numa = is_numa_fn();
1391 }
1392 }
1393
1394 std::unique_ptr<llama_mmap> mapping = std::make_unique<llama_mmap>(file.get(), prefetch ? -1 : 0, is_numa);
1395 mmaps_used.emplace_back(mapping->size(), 0);
1396 if (mlock_mmaps) {
1397 std::unique_ptr<llama_mlock> mlock_mmap(new llama_mlock());
1398 mlock_mmap->init(mapping->addr());
1399 mlock_mmaps->emplace_back(std::move(mlock_mmap));
1400 }
1401 mappings.emplace_back(std::move(mapping));
1402 }
1403 }
1404
1405 // compute the total size of all tensors for progress reporting
1406 for (const auto & it : weights_map) {
1407 size_data += ggml_nbytes(it.second.tensor);
1408 }
1409}
1410
1411void llama_model_loader::get_mapping_range(size_t * first, size_t * last, void ** addr, int idx, ggml_context * ctx) const {
1412 GGML_ASSERT(!mappings.empty())if (!(!mappings.empty())) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-model-loader.cpp"
, 1412, "GGML_ASSERT(%s) failed", "!mappings.empty()");
1413 const auto & mapping = mappings.at(idx);
1414
1415 *first = mapping->size();
1416 *last = 0;
1417 *addr = mapping->addr();
1418 for (ggml_tensor * tensor = ggml_get_first_tensor(ctx); tensor; tensor = ggml_get_next_tensor(ctx, tensor)) {
1419 const auto * weight = get_weight(ggml_get_name(tensor));
1420 if (!weight || weight->idx != idx) {
1421 continue;
1422 }
1423 *first = std::min(*first, weight->offs);
1424 *last = std::max(*last, weight->offs + ggml_nbytes(tensor));
1425 }
1426}
1427
1428void llama_model_loader::load_data_for(struct ggml_tensor * cur) const {
1429 const auto & w = require_weight(ggml_get_name(cur));
1430
1431 if (buffer_data != nullptr) {
1432 GGML_ASSERT(cur->data != nullptr)if (!(cur->data != nullptr)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-model-loader.cpp"
, 1432, "GGML_ASSERT(%s) failed", "cur->data != nullptr");
1433 GGML_ASSERT(w.offs + ggml_nbytes(cur) <= buffer_size)if (!(w.offs + ggml_nbytes(cur) <= buffer_size)) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/src/llama-model-loader.cpp"
, 1433, "GGML_ASSERT(%s) failed", "w.offs + ggml_nbytes(cur) <= buffer_size"
);
1434 memcpy(cur->data, (const uint8_t *) buffer_data + w.offs, ggml_nbytes(cur));
1435 } else if (use_mmap) {
1436 const auto & mapping = mappings.at(w.idx);
1437 if (cur->data == nullptr) {
1438 cur->data = (uint8_t *)mapping->addr() + w.offs;
1439 } else {
1440 memcpy(cur->data, (uint8_t *)mapping->addr() + w.offs, ggml_nbytes(cur));
1441 }
1442 } else {
1443 GGML_ASSERT(cur->data != nullptr)if (!(cur->data != nullptr)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-model-loader.cpp"
, 1443, "GGML_ASSERT(%s) failed", "cur->data != nullptr");
1444 GGML_ASSERT(w.idx < files.size())if (!(w.idx < files.size())) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-model-loader.cpp"
, 1444, "GGML_ASSERT(%s) failed", "w.idx < files.size()");
1445 const auto & file = files.at(w.idx);
1446 file->seek(w.offs, SEEK_SET0);
1447 file->read_raw(cur->data, ggml_nbytes(cur));
1448 }
1449
1450 if (check_tensors && !ggml_validate_row_data(cur->type, cur->data, ggml_nbytes(cur))) {
1451 throwabort_with_suppression(); if (false) std::runtime_error(format("tensor '%s' has invalid data", ggml_get_name(cur)));
1452 }
1453}
1454
1455bool llama_model_loader::load_all_data(
1456 struct ggml_context * ctx,
1457 llama_buf_map & bufs,
1458 llama_mlocks * lmlocks,
1459 llama_progress_callback progress_callback,
1460 void * progress_callback_user_data) {
1461 if (files.empty()) {
1462 for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
1463 set_tensor_data(t, set_tensor_data_ud);
1464 }
1465 return true;
1466 }
1467 GGML_ASSERT(size_data != 0 && "call init_mappings() first")if (!(size_data != 0 && "call init_mappings() first")
) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-model-loader.cpp"
, 1467, "GGML_ASSERT(%s) failed", "size_data != 0 && \"call init_mappings() first\""
);
1468
1469 std::vector<no_init<uint8_t>> read_buf;
1470 std::vector<std::pair<ggml_tensor *, bool>> validation_result;
1471
1472 // 4 staging buffers for async uploads, each sized 1MB seems to be a good default for single NVMe drives.
1473 // NVMe raid configurations might require more / larger buffers.
1474 constexpr size_t n_buffers = 4;
1475
1476 size_t alignment = 1;
1477 for (const auto & file : files) {
1478 alignment = std::max(file->read_alignment(), alignment);
1479 }
1480
1481 // Buffer size: balance between memory usage and I/O efficiency
1482 // 64MB works well for NVMe drives
1483 const size_t buffer_size = alignment != 1 ? 64 * 1024 * 1024 + 2 * alignment : 1 * 1024 * 1024;
1484
1485 std::vector<ggml_backend_buffer_t> host_buffers;
1486 std::vector<ggml_backend_event_t> events;
1487 std::vector<void *> host_ptrs;
1488 size_t buffer_idx = 0; // buffer to use for async loads
1489 ggml_backend_t upload_backend = [&](const char * func) -> ggml_backend_t {
1490 if (use_mmap || check_tensors) {
1491 return nullptr;
1492 }
1493 // When not using mmaped io use async uploads from pinned memory to GPU memory.
1494 // First determine if the backend supports the necessary features for async uploads.
1495 auto * buf = bufs.count(0) ? bufs.at(0) : nullptr;
1496 if (!buf) {
1497 LLAMA_LOG_DEBUG("%s: no buffer found for async uploads\n", func)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: no buffer found for async uploads\n"
, func);
1498 return nullptr;
1499 }
1500
1501 auto * buft = ggml_backend_buffer_get_type(buf);
1502 auto * dev = ggml_backend_buft_get_device(buft);
1503 if (!dev) {
1504 LLAMA_LOG_DEBUG("%s: no device found for buffer type %s for async uploads\n", func,llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: no device found for buffer type %s for async uploads\n"
, func, ggml_backend_buft_name(buft))
1505 ggml_backend_buft_name(buft))llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: no device found for buffer type %s for async uploads\n"
, func, ggml_backend_buft_name(buft));
1506 return nullptr;
1507 }
1508
1509 if (buft != ggml_backend_dev_buffer_type(dev)) {
1510 LLAMA_LOG_DEBUG("%s: buffer type %s is not the default buffer type for device %s for async uploads\n", func,llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: buffer type %s is not the default buffer type for device %s for async uploads\n"
, func, ggml_backend_buft_name(buft), ggml_backend_dev_name(dev
))
1511 ggml_backend_buft_name(buft), ggml_backend_dev_name(dev))llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: buffer type %s is not the default buffer type for device %s for async uploads\n"
, func, ggml_backend_buft_name(buft), ggml_backend_dev_name(dev
));
1512 return nullptr;
1513 }
1514
1515 ggml_backend_dev_props props;
1516 ggml_backend_dev_get_props(dev, &props);
1517 if (!props.caps.async || !props.caps.host_buffer || !props.caps.events) {
1518 LLAMA_LOG_DEBUG("%s: device %s does not support async, host buffers or events\n", func,llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: device %s does not support async, host buffers or events\n"
, func, ggml_backend_dev_name(dev))
1519 ggml_backend_dev_name(dev))llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: device %s does not support async, host buffers or events\n"
, func, ggml_backend_dev_name(dev));
1520 return nullptr;
1521 }
1522
1523 auto * host_buft = ggml_backend_dev_host_buffer_type(dev);
1524 if (!host_buft) {
1525 LLAMA_LOG_DEBUG("%s: no host buffer type found for device %s\n", func,llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: no host buffer type found for device %s\n"
, func, ggml_backend_dev_name(dev))
1526 ggml_backend_dev_name(dev))llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: no host buffer type found for device %s\n"
, func, ggml_backend_dev_name(dev));
1527 return nullptr;
1528 }
1529
1530 // If the backend is supported, create pinned memory buffers and events for synchronisation.
1531 for (size_t idx = 0; idx < n_buffers; ++idx) {
1532 auto * buf = ggml_backend_buft_alloc_buffer(host_buft, buffer_size);
1533
1534 if (!buf) {
1535 LLAMA_LOG_DEBUG("%s: failed to allocate host buffer for async uploads for device %s\n", func,llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: failed to allocate host buffer for async uploads for device %s\n"
, func, ggml_backend_dev_name(dev))
1536 ggml_backend_dev_name(dev))llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: failed to allocate host buffer for async uploads for device %s\n"
, func, ggml_backend_dev_name(dev));
1537 return nullptr;
1538 }
1539
1540 host_buffers.emplace_back(buf);
1541 host_ptrs.emplace_back(ggml_backend_buffer_get_base(buf));
1542
1543 auto * event = ggml_backend_event_new(dev);
1544 if (!event) {
1545 LLAMA_LOG_DEBUG("%s: failed to create event for async uploads for device %s\n", func,llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: failed to create event for async uploads for device %s\n"
, func, ggml_backend_dev_name(dev))
1546 ggml_backend_dev_name(dev))llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: failed to create event for async uploads for device %s\n"
, func, ggml_backend_dev_name(dev));
1547 return nullptr;
1548 }
1549
1550 events.emplace_back(event);
1551 }
1552
1553 ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr);
1554 if (!backend) {
1555 LLAMA_LOG_DEBUG("%s: failed to initialize backend for device %s for async uploads\n", func,llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: failed to initialize backend for device %s for async uploads\n"
, func, ggml_backend_dev_name(dev))
1556 ggml_backend_dev_name(dev))llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: failed to initialize backend for device %s for async uploads\n"
, func, ggml_backend_dev_name(dev));
1557 return nullptr;
1558 }
1559
1560 return backend;
1561 }(__func__);
1562
1563 if (upload_backend) {
1564 LLAMA_LOG_DEBUG("%s: using async uploads for device %s, buffer type %s, backend %s\n", __func__,llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: using async uploads for device %s, buffer type %s, backend %s\n"
, __func__, ggml_backend_dev_name(ggml_backend_get_device(upload_backend
)), ggml_backend_buft_name(ggml_backend_buffer_get_type(bufs.
at(0))), ggml_backend_name(upload_backend))
1565 ggml_backend_dev_name(ggml_backend_get_device(upload_backend)),llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: using async uploads for device %s, buffer type %s, backend %s\n"
, __func__, ggml_backend_dev_name(ggml_backend_get_device(upload_backend
)), ggml_backend_buft_name(ggml_backend_buffer_get_type(bufs.
at(0))), ggml_backend_name(upload_backend))
1566 ggml_backend_buft_name(ggml_backend_buffer_get_type(bufs.at(0))),llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: using async uploads for device %s, buffer type %s, backend %s\n"
, __func__, ggml_backend_dev_name(ggml_backend_get_device(upload_backend
)), ggml_backend_buft_name(ggml_backend_buffer_get_type(bufs.
at(0))), ggml_backend_name(upload_backend))
1567 ggml_backend_name(upload_backend))llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: using async uploads for device %s, buffer type %s, backend %s\n"
, __func__, ggml_backend_dev_name(ggml_backend_get_device(upload_backend
)), ggml_backend_buft_name(ggml_backend_buffer_get_type(bufs.
at(0))), ggml_backend_name(upload_backend));
1568 }
1569
1570 for (struct ggml_tensor * cur = ggml_get_first_tensor(ctx); cur != NULL__null; cur = ggml_get_next_tensor(ctx, cur)) {
1571 const auto * weight = get_weight(ggml_get_name(cur));
1572 if (weight == nullptr) {
1573 // this can happen with split experts models
1574 continue;
1575 }
1576
1577 if (progress_callback) {
1578 if (!progress_callback((float) size_done / size_data, progress_callback_user_data)) {
1579 return false;
1580 }
1581 }
1582
1583 size_t n_size = ggml_nbytes(cur);
1584
1585 if (buffer_data != nullptr) {
1586 // Firefox: tensor data lives in the in-memory buffer
1587 GGML_ASSERT(weight->offs + n_size <= buffer_size)if (!(weight->offs + n_size <= buffer_size)) ggml_abort
("/root/firefox-clang/third_party/llama.cpp/src/llama-model-loader.cpp"
, 1587, "GGML_ASSERT(%s) failed", "weight->offs + n_size <= buffer_size"
);
1588 const uint8_t * data = (const uint8_t *) buffer_data + weight->offs;
1589 if (check_tensors) {
1590 validation_result.push_back(std::make_pair(cur, ggml_validate_row_data(cur->type, data, n_size)));
1591 }
1592 ggml_backend_tensor_set(cur, data, 0, n_size);
1593 } else if (use_mmap) {
1594 const auto & mapping = mappings.at(weight->idx);
1595 ggml_backend_buffer_t buf_mmap = nullptr;
1596 if (bufs.count(weight->idx)) {
1597 buf_mmap = bufs.at(weight->idx);
1598 }
1599 uint8_t * data = (uint8_t *) mapping->addr() + weight->offs;
1600
1601 if (check_tensors) {
1602 validation_result.push_back(std::make_pair(cur, ggml_validate_row_data(cur->type, data, n_size)));
1603 }
1604
1605 GGML_ASSERT(buf_mmap || cur->data)if (!(buf_mmap || cur->data)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-model-loader.cpp"
, 1605, "GGML_ASSERT(%s) failed", "buf_mmap || cur->data"); // either we have a buffer to allocate the tensor in, or it is already allocated
1606 if (buf_mmap && cur->data == nullptr) {
1607 ggml_backend_tensor_alloc(buf_mmap, cur, data);
1608 if (lmlocks) {
1609 const auto & lmlock = lmlocks->at(weight->idx);
1610 lmlock->grow_to(weight->offs + n_size);
1611 }
1612
1613 auto & mmap_used = mmaps_used[weight->idx];
1614 mmap_used.first = std::min(mmap_used.first, weight->offs);
1615 mmap_used.second = std::max(mmap_used.second, weight->offs + n_size);
1616 } else {
1617 ggml_backend_tensor_set(cur, data, 0, n_size);
1618 }
1619 } else {
1620 const auto & file = files.at(weight->idx);
1621
1622 if (ggml_backend_buffer_is_host(cur->buffer)) {
1623 file->seek(weight->offs, SEEK_SET0);
1624 file->read_raw(cur->data, n_size);
1625 if (check_tensors) {
1626 validation_result.push_back(std::make_pair(cur, ggml_validate_row_data(cur->type, cur->data, n_size)));
1627 }
1628 } else {
1629 // If upload_backend is valid load the tensor in chunks to pinned memory and upload the buffers asynchronously to the GPU.
1630 if (upload_backend) {
1631 size_t offset = weight->offs;
1632 alignment = file->read_alignment();
1633 size_t aligned_offset = offset & ~(alignment - 1);
1634 size_t offset_from_alignment = offset - aligned_offset;
1635 file->seek(aligned_offset, SEEK_SET0);
1636
1637 // Calculate aligned read boundaries
1638 size_t read_start = aligned_offset;
1639 size_t read_end = (offset + n_size + alignment - 1) & ~(alignment - 1);
1640
1641 size_t bytes_read = 0;
1642 size_t data_read = 0; // Actual tensor data copied (excluding padding)
1643
1644 while (bytes_read < read_end - read_start) {
1645 size_t read_size = std::min<size_t>(buffer_size, read_end - read_start - bytes_read);
1646
1647 // Align the destination pointer within the pinned buffer
1648 uintptr_t ptr_dest_aligned = (reinterpret_cast<uintptr_t>(host_ptrs[buffer_idx]) + alignment - 1) & ~(alignment - 1);
1649
1650 // Wait for previous upload to complete before reusing buffer
1651 ggml_backend_event_synchronize(events[buffer_idx]);
1652
1653 // Read aligned chunk from file
1654 file->read_raw_unsafe(reinterpret_cast<void *>(ptr_dest_aligned), read_size);
1655
1656 // Calculate actual data portion (excluding alignment padding)
1657 uintptr_t ptr_data = ptr_dest_aligned;
1658 size_t data_to_copy = read_size;
1659
1660 // Skip alignment padding at start of first chunk
1661 if (bytes_read == 0) {
1662 ptr_data += offset_from_alignment;
1663 data_to_copy -= offset_from_alignment;
1664 }
1665
1666 // Trim alignment padding at end of last chunk
1667 if (aligned_offset + bytes_read + read_size > offset + n_size) {
1668 data_to_copy -= (read_end - (offset + n_size));
1669 }
1670
1671 // Async upload actual data to GPU
1672 ggml_backend_tensor_set_async(upload_backend, cur,
1673 reinterpret_cast<void *>(ptr_data), data_read, data_to_copy);
1674 ggml_backend_event_record(events[buffer_idx], upload_backend);
1675
1676 data_read += data_to_copy;
1677 bytes_read += read_size;
1678
1679 ++buffer_idx;
1680 buffer_idx %= n_buffers;
1681 }
1682 } else {
1683 read_buf.resize(n_size);
1684 file->seek(weight->offs, SEEK_SET0);
1685 file->read_raw(read_buf.data(), n_size);
1686 ggml_backend_tensor_set(cur, read_buf.data(), 0, n_size);
1687 if (check_tensors && !ggml_validate_row_data(cur->type, read_buf.data(), n_size)) {
1688 throwabort_with_suppression(); if (false) std::runtime_error(format("tensor '%s' has invalid data", ggml_get_name(cur)));
1689 }
1690 }
1691 }
1692 }
1693
1694 size_done += n_size;
1695 }
1696
1697 // free temporary resources used for async uploads
1698 for (auto * event : events) {
1699 ggml_backend_event_synchronize(event);
1700 ggml_backend_event_free(event);
1701 }
1702 for (auto * buf : host_buffers) {
1703 ggml_backend_buffer_free(buf);
1704 }
1705 ggml_backend_free(upload_backend);
1706
1707 // check validation results
1708 bool validation_failed = false;
1709 for (const auto & result : validation_result) {
1710 if (!result.second) {
1711 LLAMA_LOG_ERROR("%s: tensor '%s' has invalid data\n", __func__, ggml_get_name(result.first))llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: tensor '%s' has invalid data\n"
, __func__, ggml_get_name(result.first));
1712 validation_failed = true;
1713 }
1714 }
1715 if (validation_failed) {
1716 throwabort_with_suppression(); if (false) std::runtime_error("found tensors with invalid data");
1717 }
1718
1719 // check if this is the last call and do final cleanup
1720 if (size_done >= size_data) {
1721 // unmap offloaded tensors and metadata
1722 if (use_mmap) {
1723 for (uint32_t idx = 0; idx < mappings.size(); idx++) {
1724 const auto & mmap_used = mmaps_used.at(idx);
1725 auto & mapping = mappings.at(idx);
1726 mapping->unmap_fragment(0, mmap_used.first);
1727 if (mmap_used.second != 0) {
1728 mapping->unmap_fragment(mmap_used.second, mapping->size());
1729 }
1730 }
1731 }
1732 if (progress_callback) {
1733 // Even though the model is done loading, we still honor
1734 // cancellation since we need to free allocations.
1735 return progress_callback(1.0f, progress_callback_user_data);
1736 }
1737 }
1738
1739 return true;
1740}
1741
1742std::string llama_model_loader::ftype_name() const {
1743 return llama_model_ftype_name(ftype);
1744}
1745
1746void llama_model_loader::print_info() const {
1747 LLAMA_LOG_INFO("%s: file format = %s\n", __func__, llama_file_version_name(fver))llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: file format = %s\n"
, __func__, llama_file_version_name(fver));
1748 LLAMA_LOG_INFO("%s: file type = %s\n", __func__, llama_model_ftype_name(ftype).c_str())llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: file type = %s\n"
, __func__, llama_model_ftype_name(ftype).c_str());
1749 if (n_bytes < GiB) {
1750 LLAMA_LOG_INFO("%s: file size = %.2f MiB (%.2f BPW) \n", __func__, n_bytes/1024.0/1024.0, n_bytes*8.0/n_elements)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: file size = %.2f MiB (%.2f BPW) \n"
, __func__, n_bytes/1024.0/1024.0, n_bytes*8.0/n_elements);
1751 } else {
1752 LLAMA_LOG_INFO("%s: file size = %.2f GiB (%.2f BPW) \n", __func__, n_bytes/1024.0/1024.0/1024.0, n_bytes*8.0/n_elements)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: file size = %.2f GiB (%.2f BPW) \n"
, __func__, n_bytes/1024.0/1024.0/1024.0, n_bytes*8.0/n_elements
);
1753 }
1754}