Bug Summary

File:root/firefox-clang/security/nss/lib/smime/cmsudf.c
Warning:line 82, column 39
Subtraction of a probably non-null pointer and a null pointer may result in undefined behavior

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -O2 -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name cmsudf.c -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -fhalf-no-semantic-interposition -mframe-pointer=all -relaxed-aliasing -ffp-contract=off -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -fdebug-compilation-dir=/root/firefox-clang/obj-x86_64-pc-linux-gnu/security/nss/lib/smime/smime_smime -fcoverage-compilation-dir=/root/firefox-clang/obj-x86_64-pc-linux-gnu/security/nss/lib/smime/smime_smime -resource-dir /usr/lib/llvm-23/lib/clang/23 -include /root/firefox-clang/obj-x86_64-pc-linux-gnu/mozilla-config.h -U _FORTIFY_SOURCE -D _FORTIFY_SOURCE=2 -D DEBUG -D NSS_FIPS_DISABLED -D NSS_NO_INIT_SUPPORT -D NSS_X86_OR_X64 -D NSS_X64 -D NSS_USE_64 -D USE_UTIL_DIRECTLY -D NO_NSPR_10_SUPPORT -D SSL_DISABLE_DEPRECATED_CIPHER_SUITE_NAMES -D LINUX2_1 -D LINUX -D linux -D _DEFAULT_SOURCE -D _BSD_SOURCE -D _POSIX_SOURCE -D SDB_MEASURE_USE_TEMP_DIR -D HAVE_STRERROR -D XP_UNIX -D _REENTRANT -D NSS_DISABLE_DBM -D NSS_DISABLE_LIBPKIX -D NSS_USE_PKCS5_PBKD2_PARAMS2_ONLY -D SOFTOKEN_USE_PKCS5_PBKD2_PARAMS2_ONLY -I /root/firefox-clang/security/nss/lib/smime -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/security/nss/lib/smime/smime_smime -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/include/nspr -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/include/private/nss -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/include/nss -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/include -D MOZILLA_CLIENT -internal-isystem /usr/lib/llvm-23/lib/clang/23/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/16/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -Wno-error=tautological-type-limit-compare -Wno-range-loop-analysis -Wno-error=deprecated-declarations -Wno-error=array-bounds -Wno-error=free-nonheap-object -Wno-error=atomic-alignment -Wno-error=deprecated-builtins -Wno-psabi -Wno-error=builtin-macro-redefined -Wno-unknown-warning-option -Wno-character-conversion -ferror-limit 19 -fstrict-flex-arrays=1 -stack-protector 2 -fstack-clash-protection -ftrivial-auto-var-init=pattern -fgnuc-version=4.2.1 -fskip-odr-check-in-gmf -fdiagnostics-absolute-paths -vectorize-loops -vectorize-slp -analyzer-checker optin.performance.Padding -analyzer-output=html -analyzer-config stable-report-filename=true -mllvm -dwarf-linkage-names=Abstract -faddrsig -fdwarf2-cfi-asm -o /tmp/scan-build-2026-09-01-224014-2642839-1 -x c /root/firefox-clang/security/nss/lib/smime/cmsudf.c
1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4
5/*
6 * CMS User Define Types
7 */
8
9#include "cmslocal.h"
10
11#include "prinit.h"
12#include "pk11func.h"
13#include "secitem.h"
14#include "secoid.h"
15#include "secerr.h"
16#include "nss.h"
17
18typedef struct nsscmstypeInfoStr nsscmstypeInfo;
19struct nsscmstypeInfoStr {
20 SECOidTag type;
21 SEC_ASN1Template *template;
22 size_t size;
23 PRBool isData;
24 NSSCMSGenericWrapperDataDestroy destroy;
25 NSSCMSGenericWrapperDataCallback decode_before;
26 NSSCMSGenericWrapperDataCallback decode_after;
27 NSSCMSGenericWrapperDataCallback decode_end;
28 NSSCMSGenericWrapperDataCallback encode_start;
29 NSSCMSGenericWrapperDataCallback encode_before;
30 NSSCMSGenericWrapperDataCallback encode_after;
31};
32
33/* make sure the global tables are only initialized once */
34static PRCallOnceType nsscmstypeOnce;
35static PRCallOnceType nsscmstypeClearOnce;
36/* lock for adding a new entry */
37static PRLock *nsscmstypeAddLock;
38/* lock for the hash table */
39static PRLock *nsscmstypeHashLock;
40/* the hash table itself */
41static PLHashTable *nsscmstypeHash;
42/* arena to hold all the hash table data */
43static PLArenaPool *nsscmstypeArena;
44
45/*
46 * clean up our global tables
47 */
48SECStatus
49nss_cmstype_shutdown(void *appData, void *reserved)
50{
51 if (nsscmstypeHashLock) {
52 PR_Lock(nsscmstypeHashLock);
53 }
54 if (nsscmstypeHash) {
55 PL_HashTableDestroy(nsscmstypeHash);
56 nsscmstypeHash = NULL((void*)0);
57 }
58 if (nsscmstypeArena) {
59 PORT_FreeArenaPORT_FreeArena_Util(nsscmstypeArena, PR_FALSE0);
60 nsscmstypeArena = NULL((void*)0);
61 }
62 if (nsscmstypeAddLock) {
63 PR_DestroyLock(nsscmstypeAddLock);
64 }
65 if (nsscmstypeHashLock) {
66 PRLock *oldLock = nsscmstypeHashLock;
67 nsscmstypeHashLock = NULL((void*)0);
68 PR_Unlock(oldLock);
69 PR_DestroyLock(oldLock);
70 }
71
72 /* don't clear out the PR_ONCE data if we failed our inital call */
73 if (appData == NULL((void*)0)) {
74 nsscmstypeOnce = nsscmstypeClearOnce;
75 }
76 return SECSuccess;
77}
78
79static PLHashNumber
80nss_cmstype_hash_key(const void *key)
81{
82 return (PLHashNumber)((char *)key - (char *)NULL((void*)0));
2
←
Subtraction of a probably non-null pointer and a null pointer may result in undefined behavior
83}
84
85static PRIntn
86nss_cmstype_compare_keys(const void *v1, const void *v2)
87{
88 PLHashNumber value1 = nss_cmstype_hash_key(v1);
1
Calling 'nss_cmstype_hash_key'
→
89 PLHashNumber value2 = nss_cmstype_hash_key(v2);
90
91 return (value1 == value2);
92}
93
94/*
95 * initialize our hash tables, called once on the first attemat to register
96 * a new SMIME type.
97 */
98static PRStatus
99nss_cmstype_init(void)
100{
101 SECStatus rv;
102
103 nsscmstypeHashLock = PR_NewLock();
104 if (nsscmstypeHashLock == NULL((void*)0)) {
105 return PR_FAILURE;
106 }
107 nsscmstypeAddLock = PR_NewLock();
108 if (nsscmstypeHashLock == NULL((void*)0)) {
109 goto fail;
110 }
111 nsscmstypeHash = PL_NewHashTable(64, nss_cmstype_hash_key,
112 nss_cmstype_compare_keys,
113 PL_CompareValues, NULL((void*)0), NULL((void*)0));
114 if (nsscmstypeHash == NULL((void*)0)) {
115 goto fail;
116 }
117 nsscmstypeArena = PORT_NewArenaPORT_NewArena_Util(2048);
118 if (nsscmstypeArena == NULL((void*)0)) {
119 goto fail;
120 }
121 rv = NSS_RegisterShutdown(nss_cmstype_shutdown, NULL((void*)0));
122 if (rv != SECSuccess) {
123 goto fail;
124 }
125 return PR_SUCCESS;
126
127fail:
128 nss_cmstype_shutdown(&nsscmstypeOnce, NULL((void*)0));
129 return PR_FAILURE;
130}
131
132/*
133 * look up and registered SIME type
134 */
135static const nsscmstypeInfo *
136nss_cmstype_lookup(SECOidTag type)
137{
138 nsscmstypeInfo *typeInfo = NULL((void*)0);
139 ;
140 if (!nsscmstypeHash) {
141 return NULL((void*)0);
142 }
143 PR_Lock(nsscmstypeHashLock);
144 if (nsscmstypeHash) {
145 typeInfo = PL_HashTableLookupConst(nsscmstypeHash, (void *)type);
146 }
147 PR_Unlock(nsscmstypeHashLock);
148 return typeInfo;
149}
150
151/*
152 * add a new type to the SMIME type table
153 */
154static SECStatus
155nss_cmstype_add(SECOidTag type, nsscmstypeInfo *typeinfo)
156{
157 PLHashEntry *entry;
158
159 if (!nsscmstypeHash) {
160 /* assert? this shouldn't happen */
161 return SECFailure;
162 }
163 PR_Lock(nsscmstypeHashLock);
164 /* this is really paranoia. If we really are racing nsscmstypeHash, we'll
165 * also be racing nsscmstypeHashLock... */
166 if (!nsscmstypeHash) {
167 PR_Unlock(nsscmstypeHashLock);
168 return SECFailure;
169 }
170 entry = PL_HashTableAdd(nsscmstypeHash, (void *)type, typeinfo);
171 PR_Unlock(nsscmstypeHashLock);
172 return entry ? SECSuccess : SECFailure;
173}
174
175/* helper functions to manage new content types
176 */
177
178PRBool
179NSS_CMSType_IsWrapper(SECOidTag type)
180{
181 const nsscmstypeInfo *typeInfo = NULL((void*)0);
182
183 switch (type) {
184 case SEC_OID_PKCS7_SIGNED_DATA:
185 case SEC_OID_PKCS7_ENVELOPED_DATA:
186 case SEC_OID_CMS_AUTH_ENVELOPED_DATA:
187 case SEC_OID_PKCS7_DIGESTED_DATA:
188 case SEC_OID_PKCS7_ENCRYPTED_DATA:
189 return PR_TRUE1;
190 default:
191 typeInfo = nss_cmstype_lookup(type);
192 if (typeInfo && !typeInfo->isData) {
193 return PR_TRUE1;
194 }
195 }
196 return PR_FALSE0;
197}
198
199PRBool
200NSS_CMSType_IsData(SECOidTag type)
201{
202 const nsscmstypeInfo *typeInfo = NULL((void*)0);
203
204 switch (type) {
205 case SEC_OID_PKCS7_DATA:
206 return PR_TRUE1;
207 default:
208 typeInfo = nss_cmstype_lookup(type);
209 if (typeInfo && typeInfo->isData) {
210 return PR_TRUE1;
211 }
212 }
213 return PR_FALSE0;
214}
215
216const SEC_ASN1Template *
217NSS_CMSType_GetTemplate(SECOidTag type)
218{
219 const nsscmstypeInfo *typeInfo = nss_cmstype_lookup(type);
220
221 if (typeInfo && typeInfo->template) {
222 return typeInfo->template;
223 }
224 return SEC_ASN1_GET(SEC_PointerToOctetStringTemplate)SEC_PointerToOctetStringTemplate_Util;
225}
226
227size_t
228NSS_CMSType_GetContentSize(SECOidTag type)
229{
230 const nsscmstypeInfo *typeInfo = nss_cmstype_lookup(type);
231
232 if (typeInfo) {
233 return typeInfo->size;
234 }
235 return sizeof(SECItem *);
236}
237
238void
239NSS_CMSGenericWrapperData_Destroy(SECOidTag type, NSSCMSGenericWrapperData *gd)
240{
241 const nsscmstypeInfo *typeInfo = nss_cmstype_lookup(type);
242
243 if (typeInfo && (typeInfo->destroy) && (gd != NULL((void*)0))) {
244 (*typeInfo->destroy)(gd);
245 }
246}
247
248SECStatus
249NSS_CMSGenericWrapperData_Decode_BeforeData(SECOidTag type,
250 NSSCMSGenericWrapperData *gd)
251{
252 const nsscmstypeInfo *typeInfo;
253
254 /* short cut common case */
255 if (type == SEC_OID_PKCS7_DATA) {
256 return SECSuccess;
257 }
258
259 typeInfo = nss_cmstype_lookup(type);
260 if (typeInfo) {
261 if (typeInfo->decode_before) {
262 return (*typeInfo->decode_before)(gd);
263 }
264 /* decoder ops optional for data tags */
265 if (typeInfo->isData) {
266 return SECSuccess;
267 }
268 }
269 /* expected a function, but none existed */
270 return SECFailure;
271}
272
273SECStatus
274NSS_CMSGenericWrapperData_Decode_AfterData(SECOidTag type,
275 NSSCMSGenericWrapperData *gd)
276{
277 const nsscmstypeInfo *typeInfo;
278
279 /* short cut common case */
280 if (type == SEC_OID_PKCS7_DATA) {
281 return SECSuccess;
282 }
283
284 typeInfo = nss_cmstype_lookup(type);
285 if (typeInfo) {
286 if (typeInfo->decode_after) {
287 return (*typeInfo->decode_after)(gd);
288 }
289 /* decoder ops optional for data tags */
290 if (typeInfo->isData) {
291 return SECSuccess;
292 }
293 }
294 /* expected a function, but none existed */
295 return SECFailure;
296}
297
298SECStatus
299NSS_CMSGenericWrapperData_Decode_AfterEnd(SECOidTag type,
300 NSSCMSGenericWrapperData *gd)
301{
302 const nsscmstypeInfo *typeInfo;
303
304 /* short cut common case */
305 if (type == SEC_OID_PKCS7_DATA) {
306 return SECSuccess;
307 }
308
309 typeInfo = nss_cmstype_lookup(type);
310 if (typeInfo) {
311 if (typeInfo->decode_end) {
312 return (*typeInfo->decode_end)(gd);
313 }
314 /* decoder ops optional for data tags */
315 if (typeInfo->isData) {
316 return SECSuccess;
317 }
318 }
319 /* expected a function, but none existed */
320 return SECFailure;
321}
322
323SECStatus
324NSS_CMSGenericWrapperData_Encode_BeforeStart(SECOidTag type,
325 NSSCMSGenericWrapperData *gd)
326{
327 const nsscmstypeInfo *typeInfo;
328
329 /* short cut common case */
330 if (type == SEC_OID_PKCS7_DATA) {
331 return SECSuccess;
332 }
333
334 typeInfo = nss_cmstype_lookup(type);
335 if (typeInfo) {
336 if (typeInfo->encode_start) {
337 return (*typeInfo->encode_start)(gd);
338 }
339 /* decoder ops optional for data tags */
340 if (typeInfo->isData) {
341 return SECSuccess;
342 }
343 }
344 /* expected a function, but none existed */
345 return SECFailure;
346}
347
348SECStatus
349NSS_CMSGenericWrapperData_Encode_BeforeData(SECOidTag type,
350 NSSCMSGenericWrapperData *gd)
351{
352 const nsscmstypeInfo *typeInfo;
353
354 /* short cut common case */
355 if (type == SEC_OID_PKCS7_DATA) {
356 return SECSuccess;
357 }
358
359 typeInfo = nss_cmstype_lookup(type);
360 if (typeInfo) {
361 if (typeInfo->encode_before) {
362 return (*typeInfo->encode_before)(gd);
363 }
364 /* decoder ops optional for data tags */
365 if (typeInfo->isData) {
366 return SECSuccess;
367 }
368 }
369 /* expected a function, but none existed */
370 return SECFailure;
371}
372
373SECStatus
374NSS_CMSGenericWrapperData_Encode_AfterData(SECOidTag type,
375 NSSCMSGenericWrapperData *gd)
376{
377 const nsscmstypeInfo *typeInfo;
378
379 /* short cut common case */
380 if (type == SEC_OID_PKCS7_DATA) {
381 return SECSuccess;
382 }
383
384 typeInfo = nss_cmstype_lookup(type);
385 if (typeInfo) {
386 if (typeInfo->encode_after) {
387 return (*typeInfo->encode_after)(gd);
388 }
389 /* decoder ops optional for data tags */
390 if (typeInfo->isData) {
391 return SECSuccess;
392 }
393 }
394 /* expected a function, but none existed */
395 return SECFailure;
396}
397
398SECStatus
399NSS_CMSType_RegisterContentType(SECOidTag type,
400 SEC_ASN1Template *asn1Template, size_t size,
401 NSSCMSGenericWrapperDataDestroy destroy,
402 NSSCMSGenericWrapperDataCallback decode_before,
403 NSSCMSGenericWrapperDataCallback decode_after,
404 NSSCMSGenericWrapperDataCallback decode_end,
405 NSSCMSGenericWrapperDataCallback encode_start,
406 NSSCMSGenericWrapperDataCallback encode_before,
407 NSSCMSGenericWrapperDataCallback encode_after,
408 PRBool isData)
409{
410 PRStatus rc;
411 SECStatus rv;
412 nsscmstypeInfo *typeInfo;
413 const nsscmstypeInfo *exists;
414
415 rc = PR_CallOnce(&nsscmstypeOnce, nss_cmstype_init);
416 if (rc == PR_FAILURE) {
417 return SECFailure;
418 }
419 PR_Lock(nsscmstypeAddLock);
420 exists = nss_cmstype_lookup(type);
421 if (exists) {
422 PR_Unlock(nsscmstypeAddLock);
423 /* already added */
424 return SECSuccess;
425 }
426 typeInfo = PORT_ArenaNew(nsscmstypeArena, nsscmstypeInfo)(nsscmstypeInfo *)PORT_ArenaAlloc_Util(nsscmstypeArena, sizeof
(nsscmstypeInfo));
427 typeInfo->type = type;
428 typeInfo->size = size;
429 typeInfo->isData = isData;
430 typeInfo->template = asn1Template;
431 typeInfo->destroy = destroy;
432 typeInfo->decode_before = decode_before;
433 typeInfo->decode_after = decode_after;
434 typeInfo->decode_end = decode_end;
435 typeInfo->encode_start = encode_start;
436 typeInfo->encode_before = encode_before;
437 typeInfo->encode_after = encode_after;
438 rv = nss_cmstype_add(type, typeInfo);
439 PR_Unlock(nsscmstypeAddLock);
440 return rv;
441}