Bug Summary

File:root/firefox-clang/third_party/aom/aom/src/aom_encoder.c
Warning:line 236, column 47
Addition of a null pointer (from variable 'dst_buf') and a probably nonzero integer value (via field 'sz') may result in undefined behavior

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -O2 -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name aom_encoder.c -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -fhalf-no-semantic-interposition -mframe-pointer=all -relaxed-aliasing -ffp-contract=off -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -fdebug-compilation-dir=/root/firefox-clang/obj-x86_64-pc-linux-gnu/media/libaom -fcoverage-compilation-dir=/root/firefox-clang/obj-x86_64-pc-linux-gnu/media/libaom -resource-dir /usr/lib/llvm-22/lib/clang/22 -include /root/firefox-clang/obj-x86_64-pc-linux-gnu/mozilla-config.h -U _FORTIFY_SOURCE -D _FORTIFY_SOURCE=2 -D _GLIBCXX_ASSERTIONS -D DEBUG=1 -D MOZ_HAS_MOZGLUE -I /root/firefox-clang/media/libaom -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/media/libaom -I /root/firefox-clang/media/libaom/config/linux/x64 -I /root/firefox-clang/media/libaom/config -I /root/firefox-clang/third_party/aom -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/include -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/include/nspr -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/include/nss -D MOZILLA_CLIENT -internal-isystem /usr/lib/llvm-22/lib/clang/22/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/15/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -Wno-error=tautological-type-limit-compare -Wno-range-loop-analysis -Wno-error=deprecated-declarations -Wno-error=array-bounds -Wno-error=free-nonheap-object -Wno-error=atomic-alignment -Wno-error=deprecated-builtins -Wno-psabi -Wno-error=builtin-macro-redefined -Wno-unknown-warning-option -Wno-character-conversion -Wno-sign-compare -Wno-unused-function -Wno-unreachable-code -Wno-unneeded-internal-declaration -ferror-limit 19 -fstrict-flex-arrays=1 -stack-protector 2 -fstack-clash-protection -ftrivial-auto-var-init=pattern -fgnuc-version=4.2.1 -fskip-odr-check-in-gmf -vectorize-loops -vectorize-slp -analyzer-checker optin.performance.Padding -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -fdwarf2-cfi-asm -o /tmp/scan-build-2026-01-17-100050-2808198-1 -x c /root/firefox-clang/third_party/aom/aom/src/aom_encoder.c
1/*
2 * Copyright (c) 2016, Alliance for Open Media. All rights reserved.
3 *
4 * This source code is subject to the terms of the BSD 2 Clause License and
5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6 * was not distributed with this source code in the LICENSE file, you can
7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8 * Media Patent License 1.0 was not distributed with this source code in the
9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10 */
11
12/*!\file
13 * \brief Provides the high level interface to wrap encoder algorithms.
14 *
15 */
16#include "config/aom_config.h"
17
18#if HAVE_FEXCEPT1
19#ifndef _GNU_SOURCE
20#define _GNU_SOURCE
21#endif
22#include <fenv.h>
23#endif
24
25#include <limits.h>
26#include <stdint.h>
27#include <string.h>
28
29#include "aom/aom_encoder.h"
30#include "aom/internal/aom_codec_internal.h"
31
32#define SAVE_STATUS(ctx, var)(ctx ? (ctx->err = var) : var) (ctx ? (ctx->err = var) : var)
33
34static aom_codec_alg_priv_t *get_alg_priv(aom_codec_ctx_t *ctx) {
35 return (aom_codec_alg_priv_t *)ctx->priv;
36}
37
38aom_codec_err_t aom_codec_enc_init_ver(aom_codec_ctx_t *ctx,
39 aom_codec_iface_t *iface,
40 const aom_codec_enc_cfg_t *cfg,
41 aom_codec_flags_t flags, int ver) {
42 aom_codec_err_t res;
43 // The value of AOM_ENCODER_ABI_VERSION in libaom v3.0.0 and v3.1.0 - v3.1.3.
44 //
45 // We are compatible with these older libaom releases. AOM_ENCODER_ABI_VERSION
46 // was incremented after these releases for two reasons:
47 // 1. AOM_ENCODER_ABI_VERSION takes contribution from
48 // AOM_EXT_PART_ABI_VERSION. The external partition API is still
49 // experimental, so it should not be considered as part of the stable ABI.
50 // fd9ed8366 External partition: Define APIs
51 // https://aomedia-review.googlesource.com/c/aom/+/135663
52 // 2. As a way to detect the presence of speeds 7-9 in all-intra mode. I (wtc)
53 // suggested this change because I misunderstood how
54 // AOM_ENCODER_ABI_VERSION was used.
55 // bbdfa68d1 AllIntra: Redefine all-intra mode speed features for speed 7+
56 // https://aomedia-review.googlesource.com/c/aom/+/140624
57 const int aom_encoder_abi_version_25 = 25;
58
59 // TODO(bug aomedia:3228): Remove the check for aom_encoder_abi_version_25 in
60 // libaom v4.0.0.
61 if (ver != AOM_ENCODER_ABI_VERSION(10 + (7 + (9)) + 3) && ver != aom_encoder_abi_version_25)
62 res = AOM_CODEC_ABI_MISMATCH;
63 else if (!ctx || !iface || !cfg)
64 res = AOM_CODEC_INVALID_PARAM;
65 else if (iface->abi_version != AOM_CODEC_INTERNAL_ABI_VERSION(7))
66 res = AOM_CODEC_ABI_MISMATCH;
67 else if (!(iface->caps & AOM_CODEC_CAP_ENCODER0x2))
68 res = AOM_CODEC_INCAPABLE;
69 else if ((flags & AOM_CODEC_USE_PSNR0x10000) && !(iface->caps & AOM_CODEC_CAP_PSNR0x10000))
70 res = AOM_CODEC_INCAPABLE;
71 else if ((flags & AOM_CODEC_USE_HIGHBITDEPTH0x40000) &&
72 !(iface->caps & AOM_CODEC_CAP_HIGHBITDEPTH0x40000)) {
73 res = AOM_CODEC_INCAPABLE;
74 } else if (cfg->g_bit_depth > 8 &&
75 (flags & AOM_CODEC_USE_HIGHBITDEPTH0x40000) == 0) {
76 res = AOM_CODEC_INVALID_PARAM;
77 ctx->err_detail =
78 "High bit-depth used without the AOM_CODEC_USE_HIGHBITDEPTH flag.";
79 } else {
80 ctx->iface = iface;
81 ctx->name = iface->name;
82 ctx->priv = NULL((void*)0);
83 ctx->init_flags = flags;
84 ctx->config.enc = cfg;
85 res = ctx->iface->init(ctx);
86
87 if (res) {
88 // IMPORTANT: ctx->priv->err_detail must be null or point to a string
89 // that remains valid after ctx->priv is destroyed, such as a C string
90 // literal. This makes it safe to call aom_codec_error_detail() after
91 // aom_codec_enc_init_ver() failed.
92 ctx->err_detail = ctx->priv ? ctx->priv->err_detail : NULL((void*)0);
93 aom_codec_destroy(ctx);
94 }
95 }
96
97 return SAVE_STATUS(ctx, res)(ctx ? (ctx->err = res) : res);
98}
99
100aom_codec_err_t aom_codec_enc_config_default(aom_codec_iface_t *iface,
101 aom_codec_enc_cfg_t *cfg,
102 unsigned int usage) {
103 aom_codec_err_t res;
104
105 if (!iface || !cfg)
106 res = AOM_CODEC_INVALID_PARAM;
107 else if (!(iface->caps & AOM_CODEC_CAP_ENCODER0x2))
108 res = AOM_CODEC_INCAPABLE;
109 else {
110 res = AOM_CODEC_INVALID_PARAM;
111
112 for (int i = 0; i < iface->enc.cfg_count; ++i) {
113 if (iface->enc.cfgs[i].g_usage == usage) {
114 *cfg = iface->enc.cfgs[i];
115 res = AOM_CODEC_OK;
116 /* default values */
117 memset(&cfg->encoder_cfg, 0, sizeof(cfg->encoder_cfg));
118 cfg->encoder_cfg.super_block_size = 0; // Dynamic
119 cfg->encoder_cfg.max_partition_size = 128;
120 cfg->encoder_cfg.min_partition_size = 4;
121 cfg->encoder_cfg.disable_trellis_quant = 3;
122 break;
123 }
124 }
125 }
126 return res;
127}
128
129#if AOM_ARCH_X860 || AOM_ARCH_X86_641
130/* On X86, disable the x87 unit's internal 80 bit precision for better
131 * consistency with the SSE unit's 64 bit precision.
132 */
133#include "aom_ports/x86.h"
134#define FLOATING_POINT_SET_PRECISIONunsigned short x87_orig_mode = x87_set_double_precision(); \
135 unsigned short x87_orig_mode = x87_set_double_precision();
136#define FLOATING_POINT_RESTORE_PRECISIONx87_set_control_word(x87_orig_mode); x87_set_control_word(x87_orig_mode);
137#else
138#define FLOATING_POINT_SET_PRECISIONunsigned short x87_orig_mode = x87_set_double_precision();
139#define FLOATING_POINT_RESTORE_PRECISIONx87_set_control_word(x87_orig_mode);
140#endif // AOM_ARCH_X86 || AOM_ARCH_X86_64
141
142#if HAVE_FEXCEPT1 && CONFIG_DEBUG0
143#define FLOATING_POINT_SET_EXCEPTIONS \
144 const int float_excepts = \
145 feenableexcept(FE_DIVBYZERO0x04 | FE_UNDERFLOW0x10 | FE_OVERFLOW0x08);
146#define FLOATING_POINT_RESTORE_EXCEPTIONS \
147 if (float_excepts != -1) { \
148 fedisableexcept(FE_ALL_EXCEPT(0x20 | 0x04 | 0x10 | 0x08 | 0x01)); \
149 feenableexcept(float_excepts); \
150 }
151#else
152#define FLOATING_POINT_SET_EXCEPTIONS
153#define FLOATING_POINT_RESTORE_EXCEPTIONS
154#endif // HAVE_FEXCEPT && CONFIG_DEBUG
155
156/* clang-format off */
157#define FLOATING_POINT_INITdo { unsigned short x87_orig_mode = x87_set_double_precision(
); \
158 do { \
159 FLOATING_POINT_SET_PRECISIONunsigned short x87_orig_mode = x87_set_double_precision(); \
160 FLOATING_POINT_SET_EXCEPTIONS
161
162#define FLOATING_POINT_RESTOREx87_set_control_word(x87_orig_mode); } while (0); \
163 FLOATING_POINT_RESTORE_EXCEPTIONS \
164 FLOATING_POINT_RESTORE_PRECISIONx87_set_control_word(x87_orig_mode); \
165 } while (0);
166/* clang-format on */
167
168aom_codec_err_t aom_codec_encode(aom_codec_ctx_t *ctx, const aom_image_t *img,
169 aom_codec_pts_t pts, unsigned long duration,
170 aom_enc_frame_flags_t flags) {
171 aom_codec_err_t res = AOM_CODEC_OK;
172
173 if (!ctx || (img && !duration))
174 res = AOM_CODEC_INVALID_PARAM;
175 else if (!ctx->iface || !ctx->priv)
176 res = AOM_CODEC_ERROR;
177 else if (!(ctx->iface->caps & AOM_CODEC_CAP_ENCODER0x2))
178 res = AOM_CODEC_INCAPABLE;
179 else if (img && ((img->fmt & AOM_IMG_FMT_HIGHBITDEPTH0x800) != 0) !=
180 ((ctx->init_flags & AOM_CODEC_USE_HIGHBITDEPTH0x40000) != 0)) {
181 res = AOM_CODEC_INVALID_PARAM;
182#if ULONG_MAX(9223372036854775807L *2UL+1UL) > UINT32_MAX(4294967295U)
183 } else if (duration > UINT32_MAX(4294967295U)) {
184 res = AOM_CODEC_INVALID_PARAM;
185#endif
186 } else {
187 /* Execute in a normalized floating point environment, if the platform
188 * requires it.
189 */
190 FLOATING_POINT_INITdo { unsigned short x87_orig_mode = x87_set_double_precision(
);
191 res = ctx->iface->enc.encode(get_alg_priv(ctx), img, pts, duration, flags);
192 FLOATING_POINT_RESTOREx87_set_control_word(x87_orig_mode); } while (0);
193 }
194
195 return SAVE_STATUS(ctx, res)(ctx ? (ctx->err = res) : res);
196}
197
198const aom_codec_cx_pkt_t *aom_codec_get_cx_data(aom_codec_ctx_t *ctx,
199 aom_codec_iter_t *iter) {
200 const aom_codec_cx_pkt_t *pkt = NULL((void*)0);
201
202 if (ctx) {
1
Assuming 'ctx' is non-null
→
2
←
Taking true branch
→
203 if (!iter)
3
←
Assuming 'iter' is non-null
→
204 ctx->err = AOM_CODEC_INVALID_PARAM;
205 else if (!ctx->iface || !ctx->priv)
4
←
Assuming field 'iface' is non-null
→
5
←
Assuming field 'priv' is non-null
→
6
←
Taking false branch
→
206 ctx->err = AOM_CODEC_ERROR;
207 else if (!(ctx->iface->caps & AOM_CODEC_CAP_ENCODER0x2))
7
←
Assuming the condition is false
→
8
←
Taking false branch
→
208 ctx->err = AOM_CODEC_INCAPABLE;
209 else
210 pkt = ctx->iface->enc.get_cx_data(get_alg_priv(ctx), iter);
9
←
Value assigned to field 'buf'
→
211 }
212
213 if (pkt && pkt->kind == AOM_CODEC_CX_FRAME_PKT) {
10
←
Assuming 'pkt' is non-null
→
11
←
Assuming field 'kind' is equal to AOM_CODEC_CX_FRAME_PKT
→
12
←
Taking true branch
→
214 // If the application has specified a destination area for the
215 // compressed data, and the codec has not placed the data there,
216 // and it fits, copy it.
217 aom_codec_priv_t *const priv = ctx->priv;
218 char *const dst_buf = (char *)priv->enc.cx_data_dst_buf.buf;
13
←
'dst_buf' initialized here
→
219
220 if (dst_buf && pkt->data.raw.buf != dst_buf &&
14
←
Assuming 'dst_buf' is null
→
221 pkt->data.raw.sz + priv->enc.cx_data_pad_before +
222 priv->enc.cx_data_pad_after <=
223 priv->enc.cx_data_dst_buf.sz) {
224 aom_codec_cx_pkt_t *modified_pkt = &priv->enc.cx_data_pkt;
225
226 memcpy(dst_buf + priv->enc.cx_data_pad_before, pkt->data.raw.buf,
227 pkt->data.raw.sz);
228 *modified_pkt = *pkt;
229 modified_pkt->data.raw.buf = dst_buf;
230 modified_pkt->data.raw.sz +=
231 priv->enc.cx_data_pad_before + priv->enc.cx_data_pad_after;
232 pkt = modified_pkt;
233 }
234
235 if (dst_buf == pkt->data.raw.buf) {
15
←
Assuming 'dst_buf' is equal to field 'buf'
→
16
←
Taking true branch
→
236 priv->enc.cx_data_dst_buf.buf = dst_buf + pkt->data.raw.sz;
17
←
Addition of a null pointer (from variable 'dst_buf') and a probably nonzero integer value (via field 'sz') may result in undefined behavior
237 priv->enc.cx_data_dst_buf.sz -= pkt->data.raw.sz;
238 }
239 }
240
241 return pkt;
242}
243
244aom_codec_err_t aom_codec_set_cx_data_buf(aom_codec_ctx_t *ctx,
245 const aom_fixed_buf_t *buf,
246 unsigned int pad_before,
247 unsigned int pad_after) {
248 if (!ctx || !ctx->priv) return AOM_CODEC_INVALID_PARAM;
249
250 if (buf) {
251 ctx->priv->enc.cx_data_dst_buf = *buf;
252 ctx->priv->enc.cx_data_pad_before = pad_before;
253 ctx->priv->enc.cx_data_pad_after = pad_after;
254 } else {
255 ctx->priv->enc.cx_data_dst_buf.buf = NULL((void*)0);
256 ctx->priv->enc.cx_data_dst_buf.sz = 0;
257 ctx->priv->enc.cx_data_pad_before = 0;
258 ctx->priv->enc.cx_data_pad_after = 0;
259 }
260
261 return AOM_CODEC_OK;
262}
263
264const aom_image_t *aom_codec_get_preview_frame(aom_codec_ctx_t *ctx) {
265 aom_image_t *img = NULL((void*)0);
266
267 if (ctx) {
268 if (!ctx->iface || !ctx->priv)
269 ctx->err = AOM_CODEC_ERROR;
270 else if (!(ctx->iface->caps & AOM_CODEC_CAP_ENCODER0x2))
271 ctx->err = AOM_CODEC_INCAPABLE;
272 else if (!ctx->iface->enc.get_preview)
273 ctx->err = AOM_CODEC_INCAPABLE;
274 else
275 img = ctx->iface->enc.get_preview(get_alg_priv(ctx));
276 }
277
278 return img;
279}
280
281aom_fixed_buf_t *aom_codec_get_global_headers(aom_codec_ctx_t *ctx) {
282 aom_fixed_buf_t *buf = NULL((void*)0);
283
284 if (ctx) {
285 if (!ctx->iface || !ctx->priv)
286 ctx->err = AOM_CODEC_ERROR;
287 else if (!(ctx->iface->caps & AOM_CODEC_CAP_ENCODER0x2))
288 ctx->err = AOM_CODEC_INCAPABLE;
289 else if (!ctx->iface->enc.get_glob_hdrs)
290 ctx->err = AOM_CODEC_INCAPABLE;
291 else
292 buf = ctx->iface->enc.get_glob_hdrs(get_alg_priv(ctx));
293 }
294
295 return buf;
296}
297
298aom_codec_err_t aom_codec_enc_config_set(aom_codec_ctx_t *ctx,
299 const aom_codec_enc_cfg_t *cfg) {
300 aom_codec_err_t res;
301
302 if (!ctx || !ctx->iface || !ctx->priv || !cfg)
303 res = AOM_CODEC_INVALID_PARAM;
304 else if (!(ctx->iface->caps & AOM_CODEC_CAP_ENCODER0x2))
305 res = AOM_CODEC_INCAPABLE;
306 else
307 res = ctx->iface->enc.cfg_set(get_alg_priv(ctx), cfg);
308
309 return SAVE_STATUS(ctx, res)(ctx ? (ctx->err = res) : res);
310}
311
312int aom_codec_pkt_list_add(struct aom_codec_pkt_list *list,
313 const struct aom_codec_cx_pkt *pkt) {
314 if (list->cnt < list->max) {
315 list->pkts[list->cnt++] = *pkt;
316 return 0;
317 }
318
319 return 1;
320}
321
322const aom_codec_cx_pkt_t *aom_codec_pkt_list_get(
323 struct aom_codec_pkt_list *list, aom_codec_iter_t *iter) {
324 const aom_codec_cx_pkt_t *pkt;
325
326 if (!(*iter)) {
327 *iter = list->pkts;
328 }
329
330 pkt = (const aom_codec_cx_pkt_t *)*iter;
331
332 if ((size_t)(pkt - list->pkts) < list->cnt)
333 *iter = pkt + 1;
334 else
335 pkt = NULL((void*)0);
336
337 return pkt;
338}