Bug Summary

File:s/lib/smime/cmsencode.c
Warning:line 663, column 9
Value stored to 'rv' is never read

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name cmsencode.c -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -fhalf-no-semantic-interposition -mframe-pointer=all -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -fdebug-compilation-dir=/var/lib/jenkins/workspace/nss-scan-build/nss/lib/smime -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/var/lib/jenkins/workspace/nss-scan-build/nss/lib/smime -resource-dir /usr/lib/llvm-18/lib/clang/18 -D HAVE_STRERROR -D LINUX -D linux -D XP_UNIX -D XP_UNIX -D DEBUG -U NDEBUG -D _DEFAULT_SOURCE -D _BSD_SOURCE -D _POSIX_SOURCE -D SDB_MEASURE_USE_TEMP_DIR -D _REENTRANT -D DEBUG -U NDEBUG -D _DEFAULT_SOURCE -D _BSD_SOURCE -D _POSIX_SOURCE -D SDB_MEASURE_USE_TEMP_DIR -D _REENTRANT -D NSS_DISABLE_SSE3 -D NSS_NO_INIT_SUPPORT -D USE_UTIL_DIRECTLY -D NO_NSPR_10_SUPPORT -D SSL_DISABLE_DEPRECATED_CIPHER_SUITE_NAMES -I ../../../dist/Linux4.19_x86_64_gcc_glibc_PTH_64_DBG.OBJ/include -I ../../../dist/public/nss -I ../../../dist/private/nss -internal-isystem /usr/lib/llvm-18/lib/clang/18/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/14/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -std=c99 -ferror-limit 19 -fgnuc-version=4.2.1 -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2024-05-18-082241-28900-1 -x c cmsencode.c
1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4
5/*
6 * CMS encoding.
7 */
8
9#include "cmslocal.h"
10
11#include "cert.h"
12#include "keyhi.h"
13#include "secasn1.h"
14#include "secoid.h"
15#include "secitem.h"
16#include "pk11func.h"
17#include "secerr.h"
18
19struct nss_cms_encoder_output {
20 NSSCMSContentCallback outputfn;
21 void *outputarg;
22 PLArenaPool *destpoolp;
23 SECItem *dest;
24};
25
26struct NSSCMSEncoderContextStr {
27 SEC_ASN1EncoderContext *ecx; /* ASN.1 encoder context */
28 PRBool ecxupdated; /* true if data was handed in */
29 NSSCMSMessage *cmsg; /* pointer to the root message */
30 SECOidTag type; /* type tag of the current content */
31 NSSCMSContent content; /* pointer to current content */
32 struct nss_cms_encoder_output output; /* output function */
33 int error; /* error code */
34 NSSCMSEncoderContext *childp7ecx; /* link to child encoder context */
35};
36
37static SECStatus nss_cms_before_data(NSSCMSEncoderContext *p7ecx);
38static SECStatus nss_cms_after_data(NSSCMSEncoderContext *p7ecx);
39static SECStatus nss_cms_encoder_update(NSSCMSEncoderContext *p7ecx,
40 const char *data, unsigned long len);
41static SECStatus nss_cms_encoder_work_data(NSSCMSEncoderContext *p7ecx, SECItem *dest,
42 const unsigned char *data, unsigned long len,
43 PRBool final, PRBool innermost);
44
45extern const SEC_ASN1Template NSSCMSMessageTemplate[];
46
47/*
48 * The little output function that the ASN.1 encoder calls to hand
49 * us bytes which we in turn hand back to our caller (via the callback
50 * they gave us).
51 */
52static void
53nss_cms_encoder_out(void *arg, const char *buf, unsigned long len,
54 int depth, SEC_ASN1EncodingPart data_kind)
55{
56 struct nss_cms_encoder_output *output = (struct nss_cms_encoder_output *)arg;
57 unsigned char *dest;
58 unsigned long offset;
59
60#ifdef CMSDEBUG
61 int i;
62 const char *data_name = "unknown";
63
64 switch (data_kind) {
65 case SEC_ASN1_Identifier:
66 data_name = "identifier";
67 break;
68 case SEC_ASN1_Length:
69 data_name = "length";
70 break;
71 case SEC_ASN1_Contents:
72 data_name = "contents";
73 break;
74 case SEC_ASN1_EndOfContents:
75 data_name = "end-of-contents";
76 break;
77 }
78 fprintf(stderrstderr, "kind = %s, depth = %d, len = %d\n", data_name, depth, len);
79 for (i = 0; i < len; i++) {
80 fprintf(stderrstderr, " %02x%s", (unsigned int)buf[i] & 0xff, ((i % 16) == 15) ? "\n" : "");
81 }
82 if ((i % 16) != 0)
83 fprintf(stderrstderr, "\n");
84#endif
85
86 if (output->outputfn != NULL((void*)0))
87 /* call output callback with DER data */
88 output->outputfn(output->outputarg, buf, len);
89
90 if (output->dest != NULL((void*)0)) {
91 /* store DER data in SECItem */
92 offset = output->dest->len;
93 if (offset == 0) {
94 dest = (unsigned char *)PORT_ArenaAllocPORT_ArenaAlloc_Util(output->destpoolp, len);
95 } else {
96 dest = (unsigned char *)PORT_ArenaGrowPORT_ArenaGrow_Util(output->destpoolp,
97 output->dest->data,
98 output->dest->len,
99 output->dest->len + len);
100 }
101 if (dest == NULL((void*)0))
102 /* oops */
103 return;
104
105 output->dest->data = dest;
106 output->dest->len += len;
107
108 /* copy it in */
109 if (len) {
110 PORT_Memcpymemcpy(output->dest->data + offset, buf, len);
111 }
112 }
113}
114
115/*
116 * nss_cms_encoder_notify - ASN.1 encoder callback
117 *
118 * this function is called by the ASN.1 encoder before and after the encoding of
119 * every object. here, it is used to keep track of data structures, set up
120 * encryption and/or digesting and possibly set up child encoders.
121 */
122static void
123nss_cms_encoder_notify(void *arg, PRBool before, void *dest, int depth)
124{
125 NSSCMSEncoderContext *p7ecx;
126 NSSCMSContentInfo *rootcinfo, *cinfo;
127 PRBool after = !before;
128 SECOidTag childtype;
129 SECItem *item;
130
131 p7ecx = (NSSCMSEncoderContext *)arg;
132 PORT_Assert(p7ecx != NULL)((p7ecx != ((void*)0))?((void)0):PR_Assert("p7ecx != NULL","cmsencode.c"
,132));
133
134 rootcinfo = &(p7ecx->cmsg->contentInfo);
135
136#ifdef CMSDEBUG
137 fprintf(stderrstderr, "%6.6s, dest = 0x%08x, depth = %d\n", before ? "before" : "after",
138 dest, depth);
139#endif
140
141 /*
142 * Watch for the content field, at which point we want to instruct
143 * the ASN.1 encoder to start taking bytes from the buffer.
144 */
145 if (NSS_CMSType_IsData(p7ecx->type)) {
146 cinfo = NSS_CMSContent_GetContentInfo(p7ecx->content.pointer, p7ecx->type);
147 if (before && dest == &(cinfo->rawContent)) {
148 /* just set up encoder to grab from user - no encryption or digesting */
149 if ((item = cinfo->content.data) != NULL((void*)0))
150 (void)nss_cms_encoder_work_data(p7ecx, NULL((void*)0), item->data,
151 item->len, PR_TRUE1, PR_TRUE1);
152 else
153 SEC_ASN1EncoderSetTakeFromBufSEC_ASN1EncoderSetTakeFromBuf_Util(p7ecx->ecx);
154 SEC_ASN1EncoderClearNotifyProcSEC_ASN1EncoderClearNotifyProc_Util(p7ecx->ecx); /* no need to get notified anymore */
155 }
156 } else if (NSS_CMSType_IsWrapper(p7ecx->type)) {
157 /* when we know what the content is, we encode happily until we reach the inner content */
158 cinfo = NSS_CMSContent_GetContentInfo(p7ecx->content.pointer, p7ecx->type);
159 childtype = NSS_CMSContentInfo_GetContentTypeTag(cinfo);
160
161 if (after && dest == &(cinfo->contentType)) {
162 /* we're right before encoding the data (if we have some or not) */
163 /* (for encrypted data, we're right before the contentEncAlg which may change */
164 /* in nss_cms_before_data because of IV calculation when setting up encryption) */
165 if (nss_cms_before_data(p7ecx) != SECSuccess)
166 p7ecx->error = PORT_GetErrorPORT_GetError_Util();
167 }
168 if (before && dest == &(cinfo->rawContent)) {
169 if (p7ecx->childp7ecx == NULL((void*)0)) {
170 if ((NSS_CMSType_IsData(childtype) && (item = cinfo->content.data) != NULL((void*)0))) {
171 /* we are the innermost non-data and we have data - feed it in */
172 (void)nss_cms_encoder_work_data(p7ecx, NULL((void*)0), item->data,
173 item->len, PR_TRUE1, PR_TRUE1);
174 } else {
175 /* else we'll have to get data from user */
176 SEC_ASN1EncoderSetTakeFromBufSEC_ASN1EncoderSetTakeFromBuf_Util(p7ecx->ecx);
177 }
178 } else {
179 /* if we have a nested encoder, wait for its data */
180 SEC_ASN1EncoderSetTakeFromBufSEC_ASN1EncoderSetTakeFromBuf_Util(p7ecx->ecx);
181 }
182 }
183 if (after && dest == &(cinfo->rawContent)) {
184 if (nss_cms_after_data(p7ecx) != SECSuccess)
185 p7ecx->error = PORT_GetErrorPORT_GetError_Util();
186 SEC_ASN1EncoderClearNotifyProcSEC_ASN1EncoderClearNotifyProc_Util(p7ecx->ecx); /* no need to get notified anymore */
187 }
188 } else {
189 /* we're still in the root message */
190 if (after && dest == &(rootcinfo->contentType)) {
191 /* got the content type OID now - so find out the type tag */
192 p7ecx->type = NSS_CMSContentInfo_GetContentTypeTag(rootcinfo);
193 /* set up a pointer to our current content */
194 p7ecx->content = rootcinfo->content;
195 }
196 }
197}
198
199/*
200 * nss_cms_before_data - setup the current encoder to receive data
201 */
202static SECStatus
203nss_cms_before_data(NSSCMSEncoderContext *p7ecx)
204{
205 SECStatus rv;
206 SECOidTag childtype;
207 NSSCMSContentInfo *cinfo;
208 NSSCMSEncoderContext *childp7ecx;
209 const SEC_ASN1Template *template;
210
211 /* call _Encode_BeforeData handlers */
212 switch (p7ecx->type) {
213 case SEC_OID_PKCS7_SIGNED_DATA:
214 /* we're encoding a signedData, so set up the digests */
215 rv = NSS_CMSSignedData_Encode_BeforeData(p7ecx->content.signedData);
216 break;
217 case SEC_OID_PKCS7_DIGESTED_DATA:
218 /* we're encoding a digestedData, so set up the digest */
219 rv = NSS_CMSDigestedData_Encode_BeforeData(p7ecx->content.digestedData);
220 break;
221 case SEC_OID_PKCS7_ENVELOPED_DATA:
222 rv = NSS_CMSEnvelopedData_Encode_BeforeData(p7ecx->content.envelopedData);
223 break;
224 case SEC_OID_PKCS7_ENCRYPTED_DATA:
225 rv = NSS_CMSEncryptedData_Encode_BeforeData(p7ecx->content.encryptedData);
226 break;
227 default:
228 if (NSS_CMSType_IsWrapper(p7ecx->type)) {
229 rv = NSS_CMSGenericWrapperData_Encode_BeforeData(p7ecx->type,
230 p7ecx->content.genericData);
231 } else {
232 rv = SECFailure;
233 }
234 }
235 if (rv != SECSuccess)
236 return SECFailure;
237
238 /* ok, now we have a pointer to cinfo */
239 /* find out what kind of data is encapsulated */
240
241 cinfo = NSS_CMSContent_GetContentInfo(p7ecx->content.pointer, p7ecx->type);
242 childtype = NSS_CMSContentInfo_GetContentTypeTag(cinfo);
243
244 if (NSS_CMSType_IsWrapper(childtype)) {
245 /* in these cases, we need to set up a child encoder! */
246 /* create new encoder context */
247 childp7ecx = PORT_ZAllocPORT_ZAlloc_Util(sizeof(NSSCMSEncoderContext));
248 if (childp7ecx == NULL((void*)0))
249 return SECFailure;
250
251 /* the CHILD encoder needs to hand its encoded data to the CURRENT encoder
252 * (which will encrypt and/or digest it)
253 * this needs to route back into our update function
254 * which finds the lowest encoding context & encrypts and computes digests */
255 childp7ecx->type = childtype;
256 childp7ecx->content = cinfo->content;
257 /* use the non-recursive update function here, of course */
258 childp7ecx->output.outputfn = (NSSCMSContentCallback)nss_cms_encoder_update;
259 childp7ecx->output.outputarg = p7ecx;
260 childp7ecx->output.destpoolp = NULL((void*)0);
261 childp7ecx->output.dest = NULL((void*)0);
262 childp7ecx->cmsg = p7ecx->cmsg;
263 childp7ecx->ecxupdated = PR_FALSE0;
264 childp7ecx->childp7ecx = NULL((void*)0);
265
266 template = NSS_CMSUtil_GetTemplateByTypeTag(childtype);
267 if (template == NULL((void*)0))
268 goto loser; /* cannot happen */
269
270 /* now initialize the data for encoding the first third */
271 switch (childp7ecx->type) {
272 case SEC_OID_PKCS7_SIGNED_DATA:
273 rv = NSS_CMSSignedData_Encode_BeforeStart(cinfo->content.signedData);
274 break;
275 case SEC_OID_PKCS7_ENVELOPED_DATA:
276 rv = NSS_CMSEnvelopedData_Encode_BeforeStart(cinfo->content.envelopedData);
277 break;
278 case SEC_OID_PKCS7_DIGESTED_DATA:
279 rv = NSS_CMSDigestedData_Encode_BeforeStart(cinfo->content.digestedData);
280 break;
281 case SEC_OID_PKCS7_ENCRYPTED_DATA:
282 rv = NSS_CMSEncryptedData_Encode_BeforeStart(cinfo->content.encryptedData);
283 break;
284 default:
285 rv = NSS_CMSGenericWrapperData_Encode_BeforeStart(childp7ecx->type,
286 cinfo->content.genericData);
287 break;
288 }
289 if (rv != SECSuccess)
290 goto loser;
291
292 /*
293 * Initialize the BER encoder.
294 */
295 childp7ecx->ecx = SEC_ASN1EncoderStartSEC_ASN1EncoderStart_Util(cinfo->content.pointer, template,
296 nss_cms_encoder_out, &(childp7ecx->output));
297 if (childp7ecx->ecx == NULL((void*)0))
298 goto loser;
299
300 /*
301 * Indicate that we are streaming. We will be streaming until we
302 * get past the contents bytes.
303 */
304 if (!cinfo->privateInfo || !cinfo->privateInfo->dontStream)
305 SEC_ASN1EncoderSetStreamingSEC_ASN1EncoderSetStreaming_Util(childp7ecx->ecx);
306
307 /*
308 * The notify function will watch for the contents field.
309 */
310 p7ecx->childp7ecx = childp7ecx;
311 SEC_ASN1EncoderSetNotifyProcSEC_ASN1EncoderSetNotifyProc_Util(childp7ecx->ecx, nss_cms_encoder_notify,
312 childp7ecx);
313
314 /* please note that we are NOT calling SEC_ASN1EncoderUpdate here to kick off the */
315 /* encoding process - we'll do that from the update function instead */
316 /* otherwise we'd be encoding data from a call of the notify function of the */
317 /* parent encoder (which would not work) */
318
319 } else if (NSS_CMSType_IsData(childtype)) {
320 p7ecx->childp7ecx = NULL((void*)0);
321 } else {
322 /* we do not know this type */
323 p7ecx->error = SEC_ERROR_BAD_DER;
324 }
325
326 return SECSuccess;
327
328loser:
329 if (childp7ecx) {
330 if (childp7ecx->ecx)
331 SEC_ASN1EncoderFinishSEC_ASN1EncoderFinish_Util(childp7ecx->ecx);
332 PORT_FreePORT_Free_Util(childp7ecx);
333 p7ecx->childp7ecx = NULL((void*)0);
334 }
335 return SECFailure;
336}
337
338static SECStatus
339nss_cms_after_data(NSSCMSEncoderContext *p7ecx)
340{
341 SECStatus rv = SECFailure;
342
343 switch (p7ecx->type) {
344 case SEC_OID_PKCS7_SIGNED_DATA:
345 /* this will finish the digests and sign */
346 rv = NSS_CMSSignedData_Encode_AfterData(p7ecx->content.signedData);
347 break;
348 case SEC_OID_PKCS7_ENVELOPED_DATA:
349 rv = NSS_CMSEnvelopedData_Encode_AfterData(p7ecx->content.envelopedData);
350 break;
351 case SEC_OID_PKCS7_DIGESTED_DATA:
352 rv = NSS_CMSDigestedData_Encode_AfterData(p7ecx->content.digestedData);
353 break;
354 case SEC_OID_PKCS7_ENCRYPTED_DATA:
355 rv = NSS_CMSEncryptedData_Encode_AfterData(p7ecx->content.encryptedData);
356 break;
357 default:
358 if (NSS_CMSType_IsWrapper(p7ecx->type)) {
359 rv = NSS_CMSGenericWrapperData_Encode_AfterData(p7ecx->type,
360 p7ecx->content.genericData);
361 } else {
362 rv = SECFailure;
363 }
364 break;
365 }
366 return rv;
367}
368
369/*
370 * nss_cms_encoder_work_data - process incoming data
371 *
372 * (from the user or the next encoding layer)
373 * Here, we need to digest and/or encrypt, then pass it on
374 */
375static SECStatus
376nss_cms_encoder_work_data(NSSCMSEncoderContext *p7ecx, SECItem *dest,
377 const unsigned char *data, unsigned long len,
378 PRBool final, PRBool innermost)
379{
380 unsigned char *buf = NULL((void*)0);
381 SECStatus rv;
382 NSSCMSContentInfo *cinfo;
383
384 rv = SECSuccess; /* may as well be optimistic */
385
386 /*
387 * We should really have data to process, or we should be trying
388 * to finish/flush the last block. (This is an overly paranoid
389 * check since all callers are in this file and simple inspection
390 * proves they do it right. But it could find a bug in future
391 * modifications/development, that is why it is here.)
392 */
393 PORT_Assert((data != NULL && len) || final)(((data != ((void*)0) && len) || final)?((void)0):PR_Assert
("(data != NULL && len) || final","cmsencode.c",393));
394
395 /* we got data (either from the caller, or from a lower level encoder) */
396 cinfo = NSS_CMSContent_GetContentInfo(p7ecx->content.pointer, p7ecx->type);
397 if (!cinfo) {
398 /* The original programmer didn't expect this to happen */
399 p7ecx->error = SEC_ERROR_LIBRARY_FAILURE;
400 return SECFailure;
401 }
402
403 /* Update the running digest. */
404 if (len && cinfo->privateInfo && cinfo->privateInfo->digcx != NULL((void*)0))
405 NSS_CMSDigestContext_Update(cinfo->privateInfo->digcx, data, len);
406
407 /* Encrypt this chunk. */
408 if (cinfo->privateInfo && cinfo->privateInfo->ciphcx != NULL((void*)0)) {
409 unsigned int inlen; /* length of data being encrypted */
410 unsigned int outlen; /* length of encrypted data */
411 unsigned int buflen; /* length available for encrypted data */
412
413 inlen = len;
414 buflen = NSS_CMSCipherContext_EncryptLength(cinfo->privateInfo->ciphcx,
415 inlen, final);
416 if (buflen == 0) {
417 /*
418 * No output is expected, but the input data may be buffered
419 * so we still have to call Encrypt.
420 */
421 rv = NSS_CMSCipherContext_Encrypt(cinfo->privateInfo->ciphcx, NULL((void*)0), NULL((void*)0), 0,
422 data, inlen, final);
423 if (final) {
424 len = 0;
425 goto done;
426 }
427 return rv;
428 }
429
430 if (dest != NULL((void*)0))
431 buf = (unsigned char *)PORT_ArenaAllocPORT_ArenaAlloc_Util(p7ecx->cmsg->poolp, buflen);
432 else
433 buf = (unsigned char *)PORT_AllocPORT_Alloc_Util(buflen);
434
435 if (buf == NULL((void*)0)) {
436 rv = SECFailure;
437 } else {
438 rv = NSS_CMSCipherContext_Encrypt(cinfo->privateInfo->ciphcx, buf,
439 &outlen, buflen,
440 data, inlen, final);
441 data = buf;
442 len = outlen;
443 }
444 if (rv != SECSuccess)
445 /* encryption or malloc failed? */
446 return rv;
447 }
448
449 /*
450 * at this point (data,len) has everything we'd like to give to the CURRENT encoder
451 * (which will encode it, then hand it back to the user or the parent encoder)
452 * We don't encode the data if we're innermost and we're told not to include the data
453 */
454 if (p7ecx->ecx != NULL((void*)0) && len &&
455 (!innermost || cinfo->rawContent != cinfo->content.pointer))
456 rv = SEC_ASN1EncoderUpdateSEC_ASN1EncoderUpdate_Util(p7ecx->ecx, (const char *)data, len);
457
458done:
459
460 if (cinfo->privateInfo && cinfo->privateInfo->ciphcx != NULL((void*)0)) {
461 if (dest != NULL((void*)0)) {
462 dest->data = buf;
463 dest->len = len;
464 } else if (buf != NULL((void*)0)) {
465 PORT_FreePORT_Free_Util(buf);
466 }
467 }
468 return rv;
469}
470
471/*
472 * nss_cms_encoder_update - deliver encoded data to the next higher level
473 *
474 * no recursion here because we REALLY want to end up at the next higher encoder!
475 */
476static SECStatus
477nss_cms_encoder_update(NSSCMSEncoderContext *p7ecx, const char *data, unsigned long len)
478{
479 /* XXX Error handling needs help. Return what? Do "Finish" on failure? */
480 return nss_cms_encoder_work_data(p7ecx, NULL((void*)0), (const unsigned char *)data,
481 len, PR_FALSE0, PR_FALSE0);
482}
483
484/*
485 * NSS_CMSEncoder_Start - set up encoding of a CMS message
486 *
487 * "cmsg" - message to encode
488 * "outputfn", "outputarg" - callback function for delivery of DER-encoded output
489 * will not be called if NULL.
490 * "dest" - if non-NULL, pointer to SECItem that will hold the DER-encoded output
491 * "destpoolp" - pool to allocate DER-encoded output in
492 * "pwfn", pwfn_arg" - callback function for getting token password
493 * "decrypt_key_cb", "decrypt_key_cb_arg" - callback function for getting bulk key for encryptedData
494 * "detached_digestalgs", "detached_digests" - digests from detached content
495 */
496NSSCMSEncoderContext *
497NSS_CMSEncoder_Start(NSSCMSMessage *cmsg,
498 NSSCMSContentCallback outputfn, void *outputarg,
499 SECItem *dest, PLArenaPool *destpoolp,
500 PK11PasswordFunc pwfn, void *pwfn_arg,
501 NSSCMSGetDecryptKeyCallback decrypt_key_cb, void *decrypt_key_cb_arg,
502 SECAlgorithmID **detached_digestalgs, SECItem **detached_digests)
503{
504 NSSCMSEncoderContext *p7ecx;
505 SECStatus rv;
506 NSSCMSContentInfo *cinfo;
507 SECOidTag tag;
508
509 NSS_CMSMessage_SetEncodingParams(cmsg, pwfn, pwfn_arg, decrypt_key_cb, decrypt_key_cb_arg,
510 detached_digestalgs, detached_digests);
511
512 p7ecx = (NSSCMSEncoderContext *)PORT_ZAllocPORT_ZAlloc_Util(sizeof(NSSCMSEncoderContext));
513 if (p7ecx == NULL((void*)0)) {
514 PORT_SetErrorPORT_SetError_Util(SEC_ERROR_NO_MEMORY);
515 return NULL((void*)0);
516 }
517
518 p7ecx->cmsg = cmsg;
519 p7ecx->output.outputfn = outputfn;
520 p7ecx->output.outputarg = outputarg;
521 p7ecx->output.dest = dest;
522 p7ecx->output.destpoolp = destpoolp;
523 p7ecx->type = SEC_OID_UNKNOWN;
524
525 cinfo = NSS_CMSMessage_GetContentInfo(cmsg);
526
527 tag = NSS_CMSContentInfo_GetContentTypeTag(cinfo);
528 switch (tag) {
529 case SEC_OID_PKCS7_SIGNED_DATA:
530 rv = NSS_CMSSignedData_Encode_BeforeStart(cinfo->content.signedData);
531 break;
532 case SEC_OID_PKCS7_ENVELOPED_DATA:
533 rv = NSS_CMSEnvelopedData_Encode_BeforeStart(cinfo->content.envelopedData);
534 break;
535 case SEC_OID_PKCS7_DIGESTED_DATA:
536 rv = NSS_CMSDigestedData_Encode_BeforeStart(cinfo->content.digestedData);
537 break;
538 case SEC_OID_PKCS7_ENCRYPTED_DATA:
539 rv = NSS_CMSEncryptedData_Encode_BeforeStart(cinfo->content.encryptedData);
540 break;
541 default:
542 if (NSS_CMSType_IsWrapper(tag)) {
543 rv = NSS_CMSGenericWrapperData_Encode_BeforeStart(tag,
544 p7ecx->content.genericData);
545 } else {
546 rv = SECFailure;
547 }
548 break;
549 }
550 if (rv != SECSuccess) {
551 PORT_FreePORT_Free_Util(p7ecx);
552 return NULL((void*)0);
553 }
554
555 /* Initialize the BER encoder.
556 * Note that this will not encode anything until the first call to SEC_ASN1EncoderUpdate */
557 p7ecx->ecx = SEC_ASN1EncoderStartSEC_ASN1EncoderStart_Util(cmsg, NSSCMSMessageTemplate,
558 nss_cms_encoder_out, &(p7ecx->output));
559 if (p7ecx->ecx == NULL((void*)0)) {
560 PORT_FreePORT_Free_Util(p7ecx);
561 return NULL((void*)0);
562 }
563 p7ecx->ecxupdated = PR_FALSE0;
564
565 /*
566 * Indicate that we are streaming. We will be streaming until we
567 * get past the contents bytes.
568 */
569 if (!cinfo->privateInfo || !cinfo->privateInfo->dontStream)
570 SEC_ASN1EncoderSetStreamingSEC_ASN1EncoderSetStreaming_Util(p7ecx->ecx);
571
572 /*
573 * The notify function will watch for the contents field.
574 */
575 SEC_ASN1EncoderSetNotifyProcSEC_ASN1EncoderSetNotifyProc_Util(p7ecx->ecx, nss_cms_encoder_notify, p7ecx);
576
577 /* this will kick off the encoding process & encode everything up to the content bytes,
578 * at which point the notify function sets streaming mode (and possibly creates
579 * a child encoder). */
580 p7ecx->ecxupdated = PR_TRUE1;
581 if (SEC_ASN1EncoderUpdateSEC_ASN1EncoderUpdate_Util(p7ecx->ecx, NULL((void*)0), 0) != SECSuccess) {
582 PORT_FreePORT_Free_Util(p7ecx);
583 return NULL((void*)0);
584 }
585
586 return p7ecx;
587}
588
589/*
590 * NSS_CMSEncoder_Update - take content data delivery from the user
591 *
592 * "p7ecx" - encoder context
593 * "data" - content data
594 * "len" - length of content data
595 *
596 * need to find the lowest level (and call SEC_ASN1EncoderUpdate on the way down),
597 * then hand the data to the work_data fn
598 */
599SECStatus
600NSS_CMSEncoder_Update(NSSCMSEncoderContext *p7ecx, const char *data, unsigned long len)
601{
602 SECStatus rv;
603 NSSCMSContentInfo *cinfo;
604 SECOidTag childtype;
605
606 if (p7ecx->error)
607 return SECFailure;
608
609 /* hand data to the innermost decoder */
610 if (p7ecx->childp7ecx) {
611 /* tell the child to start encoding, up to its first data byte, if it
612 * hasn't started yet */
613 if (!p7ecx->childp7ecx->ecxupdated) {
614 p7ecx->childp7ecx->ecxupdated = PR_TRUE1;
615 if (SEC_ASN1EncoderUpdateSEC_ASN1EncoderUpdate_Util(p7ecx->childp7ecx->ecx, NULL((void*)0), 0) != SECSuccess)
616 return SECFailure;
617 }
618 /* recursion here */
619 rv = NSS_CMSEncoder_Update(p7ecx->childp7ecx, data, len);
620 } else {
621 /* we are at innermost decoder */
622 /* find out about our inner content type - must be data */
623 cinfo = NSS_CMSContent_GetContentInfo(p7ecx->content.pointer, p7ecx->type);
624 if (!cinfo) {
625 /* The original programmer didn't expect this to happen */
626 p7ecx->error = SEC_ERROR_LIBRARY_FAILURE;
627 return SECFailure;
628 }
629
630 childtype = NSS_CMSContentInfo_GetContentTypeTag(cinfo);
631 if (!NSS_CMSType_IsData(childtype))
632 return SECFailure;
633 /* and we must not have preset data */
634 if (cinfo->content.data != NULL((void*)0))
635 return SECFailure;
636
637 /* hand it the data so it can encode it (let DER trickle up the chain) */
638 rv = nss_cms_encoder_work_data(p7ecx, NULL((void*)0), (const unsigned char *)data,
639 len, PR_FALSE0, PR_TRUE1);
640 }
641 return rv;
642}
643
644/*
645 * NSS_CMSEncoder_Cancel - stop all encoding
646 *
647 * we need to walk down the chain of encoders and the finish them from the innermost out
648 */
649SECStatus
650NSS_CMSEncoder_Cancel(NSSCMSEncoderContext *p7ecx)
651{
652 SECStatus rv = SECFailure;
653
654 /* XXX do this right! */
655
656 /*
657 * Finish any inner decoders before us so that all the encoded data is flushed
658 * This basically finishes all the decoders from the innermost to the outermost.
659 * Finishing an inner decoder may result in data being updated to the outer decoder
660 * while we are already in NSS_CMSEncoder_Finish, but that's allright.
661 */
662 if (p7ecx->childp7ecx) {
663 rv = NSS_CMSEncoder_Cancel(p7ecx->childp7ecx); /* frees p7ecx->childp7ecx */
Value stored to 'rv' is never read
664 /* remember rv for now */
665#ifdef CMSDEBUG
666 if (rv != SECSuccess) {
667 fprintf(stderrstderr, "Fail to cancel inner encoder\n");
668 }
669#endif
670 }
671
672 /*
673 * On the way back up, there will be no more data (if we had an
674 * inner encoder, it is done now!)
675 * Flush out any remaining data and/or finish digests.
676 */
677 rv = nss_cms_encoder_work_data(p7ecx, NULL((void*)0), NULL((void*)0), 0, PR_TRUE1, (p7ecx->childp7ecx == NULL((void*)0)));
678 if (rv != SECSuccess)
679 goto loser;
680
681 p7ecx->childp7ecx = NULL((void*)0);
682
683 /* kick the encoder back into working mode again.
684 * We turn off streaming stuff (which will cause the encoder to continue
685 * encoding happily, now that we have all the data (like digests) ready for it).
686 */
687 SEC_ASN1EncoderClearTakeFromBufSEC_ASN1EncoderClearTakeFromBuf_Util(p7ecx->ecx);
688 SEC_ASN1EncoderClearStreamingSEC_ASN1EncoderClearStreaming_Util(p7ecx->ecx);
689
690 /* now that TakeFromBuf is off, this will kick this encoder to finish encoding */
691 rv = SEC_ASN1EncoderUpdateSEC_ASN1EncoderUpdate_Util(p7ecx->ecx, NULL((void*)0), 0);
692
693loser:
694 SEC_ASN1EncoderFinishSEC_ASN1EncoderFinish_Util(p7ecx->ecx);
695 PORT_FreePORT_Free_Util(p7ecx);
696 return rv;
697}
698
699/*
700 * NSS_CMSEncoder_Finish - signal the end of data
701 *
702 * we need to walk down the chain of encoders and the finish them from the innermost out
703 */
704SECStatus
705NSS_CMSEncoder_Finish(NSSCMSEncoderContext *p7ecx)
706{
707 SECStatus rv = SECFailure;
708 NSSCMSContentInfo *cinfo;
709
710 /*
711 * Finish any inner decoders before us so that all the encoded data is flushed
712 * This basically finishes all the decoders from the innermost to the outermost.
713 * Finishing an inner decoder may result in data being updated to the outer decoder
714 * while we are already in NSS_CMSEncoder_Finish, but that's allright.
715 */
716 if (p7ecx->childp7ecx) {
717 /* tell the child to start encoding, up to its first data byte, if it
718 * hasn't yet */
719 if (!p7ecx->childp7ecx->ecxupdated) {
720 p7ecx->childp7ecx->ecxupdated = PR_TRUE1;
721 rv = SEC_ASN1EncoderUpdateSEC_ASN1EncoderUpdate_Util(p7ecx->childp7ecx->ecx, NULL((void*)0), 0);
722 if (rv != SECSuccess) {
723 NSS_CMSEncoder_Finish(p7ecx->childp7ecx); /* frees p7ecx->childp7ecx */
724 goto loser;
725 }
726 }
727 rv = NSS_CMSEncoder_Finish(p7ecx->childp7ecx); /* frees p7ecx->childp7ecx */
728 if (rv != SECSuccess)
729 goto loser;
730 }
731
732 /*
733 * On the way back up, there will be no more data (if we had an
734 * inner encoder, it is done now!)
735 * Flush out any remaining data and/or finish digests.
736 */
737 rv = nss_cms_encoder_work_data(p7ecx, NULL((void*)0), NULL((void*)0), 0, PR_TRUE1, (p7ecx->childp7ecx == NULL((void*)0)));
738 if (rv != SECSuccess)
739 goto loser;
740
741 p7ecx->childp7ecx = NULL((void*)0);
742
743 cinfo = NSS_CMSContent_GetContentInfo(p7ecx->content.pointer, p7ecx->type);
744 if (!cinfo) {
745 /* The original programmer didn't expect this to happen */
746 p7ecx->error = SEC_ERROR_LIBRARY_FAILURE;
747 rv = SECFailure;
748 goto loser;
749 }
750 SEC_ASN1EncoderClearTakeFromBufSEC_ASN1EncoderClearTakeFromBuf_Util(p7ecx->ecx);
751 SEC_ASN1EncoderClearStreamingSEC_ASN1EncoderClearStreaming_Util(p7ecx->ecx);
752 /* now that TakeFromBuf is off, this will kick this encoder to finish encoding */
753 rv = SEC_ASN1EncoderUpdateSEC_ASN1EncoderUpdate_Util(p7ecx->ecx, NULL((void*)0), 0);
754
755 if (p7ecx->error)
756 rv = SECFailure;
757
758loser:
759 SEC_ASN1EncoderFinishSEC_ASN1EncoderFinish_Util(p7ecx->ecx);
760 PORT_FreePORT_Free_Util(p7ecx);
761 return rv;
762}