Bug Summary

File:root/firefox-clang/third_party/dav1d/src/lf_mask.c
Warning:line 100, column 23
1st function call argument is an uninitialized value

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -O2 -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name lf_mask.c -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -fhalf-no-semantic-interposition -mframe-pointer=all -relaxed-aliasing -ffp-contract=off -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -fdebug-compilation-dir=/root/firefox-clang/obj-x86_64-pc-linux-gnu/media/libdav1d -fcoverage-compilation-dir=/root/firefox-clang/obj-x86_64-pc-linux-gnu/media/libdav1d -resource-dir /usr/lib/llvm-23/lib/clang/23 -include /root/firefox-clang/config/gcc_hidden.h -include /root/firefox-clang/obj-x86_64-pc-linux-gnu/mozilla-config.h -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/system_wrappers -U _FORTIFY_SOURCE -D _FORTIFY_SOURCE=2 -D DEBUG=1 -D DAV1D_API= -D STACK_ALIGNMENT=16 -D MOZ_HAS_MOZGLUE -D MOZILLA_INTERNAL_API -D IMPL_LIBXUL -D MOZ_SUPPORT_LEAKCHECKING -D STATIC_EXPORTABLE_JS_API -I /root/firefox-clang/media/libdav1d -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/media/libdav1d -I /root/firefox-clang/third_party/dav1d -I /root/firefox-clang/third_party/dav1d/include -I /root/firefox-clang/third_party/dav1d/include/dav1d -I /root/firefox-clang/third_party/dav1d/src -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/include -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/include/nspr -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/include/nss -D MOZILLA_CLIENT -internal-isystem /usr/lib/llvm-23/lib/clang/23/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/16/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -Wno-error=tautological-type-limit-compare -Wno-range-loop-analysis -Wno-error=deprecated-declarations -Wno-error=array-bounds -Wno-error=free-nonheap-object -Wno-error=atomic-alignment -Wno-error=deprecated-builtins -Wno-psabi -Wno-error=builtin-macro-redefined -Wno-unknown-warning-option -Wno-character-conversion -ferror-limit 19 -fstrict-flex-arrays=1 -stack-protector 2 -fstack-clash-protection -ftrivial-auto-var-init=pattern -fgnuc-version=4.2.1 -fskip-odr-check-in-gmf -fdiagnostics-absolute-paths -vectorize-loops -vectorize-slp -analyzer-checker optin.performance.Padding -analyzer-output=html -analyzer-config stable-report-filename=true -mllvm -dwarf-linkage-names=Abstract -faddrsig -fdwarf2-cfi-asm -o /tmp/scan-build-2026-11-12-001017-1011516-1 -x c /root/firefox-clang/third_party/dav1d/src/lf_mask.c
1/*
2 * Copyright © 2018, VideoLAN and dav1d authors
3 * Copyright © 2018, Two Orioles, LLC
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright notice, this
10 * list of conditions and the following disclaimer.
11 *
12 * 2. Redistributions in binary form must reproduce the above copyright notice,
13 * this list of conditions and the following disclaimer in the documentation
14 * and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
20 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28#include "config.h"
29
30#include <string.h>
31
32#include "common/intops.h"
33
34#include "src/ctx.h"
35#include "src/levels.h"
36#include "src/lf_mask.h"
37#include "src/tables.h"
38
39static void decomp_tx(uint8_t (*const txa)[2 /* txsz, step */][32 /* y */][32 /* x */],
40 const enum RectTxfmSize from,
41 const int depth,
42 const int y_off, const int x_off,
43 const uint16_t *const tx_masks)
44{
45 const TxfmInfo *const t_dim = &dav1d_txfm_dimensions[from];
46 const int is_split = (from == (int) TX_4X4 || depth
11.1
'depth' is <= 1
16.1
'depth' is <= 1
 > 1) ? 0 :
11
←
Assuming 'from' is not equal to TX_4X4
→
12
←
'?' condition is false
→
16
←
Assuming 'from' is not equal to TX_4X4
→
17
←
'?' condition is false
→
47 (tx_masks[depth] >> (y_off * 4 + x_off)) & 1;
48
49 if (is_split) {
13
←
Assuming 'is_split' is not equal to 0
→
14
←
Taking true branch
→
18
←
Assuming 'is_split' is 0
→
19
←
Taking false branch
→
50 const enum RectTxfmSize sub = t_dim->sub;
51 const int htw4 = t_dim->w >> 1, hth4 = t_dim->h >> 1;
52
53 decomp_tx(txa, sub, depth + 1, y_off * 2 + 0, x_off * 2 + 0, tx_masks);
15
←
Calling 'decomp_tx'
→
24
←
Returning from 'decomp_tx'
→
54 if (t_dim->w >= t_dim->h)
25
←
Assuming field 'w' is < field 'h'
→
26
←
Taking false branch
→
55 decomp_tx((uint8_t(*)[2][32][32]) &txa[0][0][0][htw4],
56 sub, depth + 1, y_off * 2 + 0, x_off * 2 + 1, tx_masks);
57 if (t_dim->h >= t_dim->w) {
27
←
Assuming field 'h' is >= field 'w'
→
28
←
Taking true branch
→
58 decomp_tx((uint8_t(*)[2][32][32]) &txa[0][0][hth4][0],
59 sub, depth + 1, y_off * 2 + 1, x_off * 2 + 0, tx_masks);
60 if (t_dim->w
28.1
Field 'w' is < field 'h'
 >= t_dim->h)
29
←
Taking false branch
→
61 decomp_tx((uint8_t(*)[2][32][32]) &txa[0][0][hth4][htw4],
62 sub, depth + 1, y_off * 2 + 1, x_off * 2 + 1, tx_masks);
63 }
64 } else {
65 const int lw = imin(2, t_dim->lw), lh = imin(2, t_dim->lh);
66
67#define set_ctx(rep_macro) \
68 for (int y = 0; y < t_dim->h; y++) { \
69 rep_macro(txa[0][0][y], 0, lw); \
70 rep_macro(txa[1][0][y], 0, lh); \
71 txa[0][1][y][0] = t_dim->w; \
72 } \
73 rep_macro(txa[1][1][0], 0, t_dim->h);
74 case_set_upto16(t_dim->lw)switch (t_dim->lw) { case 0: set_ctx(set_ctx1); break; case
1: set_ctx(set_ctx2); break; case 2: set_ctx(set_ctx4); break
; case 3: set_ctx(set_ctx8); break; case 4: set_ctx(set_ctx16
); break; default: ((void) sizeof ((0) ? 1 : 0), __extension__
({ if (0) ; else __assert_fail ("0", "/root/firefox-clang/third_party/dav1d/src/lf_mask.c"
, 74, __extension__ __PRETTY_FUNCTION__); })); };
20
←
Control jumps to 'case 0:' at line 74
→
21
←
Assuming 'y' is >= field 'h'
→
22
←
Loop condition is false. Execution continues on line 74
→
23
←
 Execution continues on line 53
→
75#undef set_ctx
76 }
77}
78
79static inline void mask_edges_inter(uint16_t (*const masks)[32][3][2],
80 const int by4, const int bx4,
81 const int w4, const int h4, const int skip,
82 const enum RectTxfmSize max_tx,
83 const uint16_t *const tx_masks,
84 uint8_t *const a, uint8_t *const l)
85{
86 const TxfmInfo *const t_dim = &dav1d_txfm_dimensions[max_tx];
87 int y, x;
88
89 ALIGN_STK_16(uint8_t, txa, 2 /* edge */, [2 /* txsz, step */][32 /* y */][32 /* x */])uint8_t txa[2][2 ][32 ][32 ] __attribute__((aligned(16)));
90 for (int y_off = 0, y = 0; y < h4; y += t_dim->h, y_off++)
8
←
Loop condition is true. Entering loop body
→
33
←
Assuming 'y' is >= 'h4'
→
34
←
Loop condition is false. Execution continues on line 96
→
91 for (int x_off = 0, x = 0; x < w4; x += t_dim->w, x_off++)
9
←
Loop condition is true. Entering loop body
→
31
←
Assuming 'x' is >= 'w4'
→
32
←
Loop condition is false. Execution continues on line 90
→
92 decomp_tx((uint8_t(*)[2][32][32]) &txa[0][0][y][x],
10
←
Calling 'decomp_tx'
→
30
←
Returning from 'decomp_tx'
→
93 max_tx, 0, y_off, x_off, tx_masks);
94
95 // left block edge
96 unsigned mask = 1U << by4;
97 for (y = 0; y < h4; y++, mask <<= 1) {
35
←
Loop condition is true. Entering loop body
→
98 const int sidx = mask >= 0x10000;
36
←
Assuming 'mask' is < 65536
→
99 const unsigned smask = mask >> (sidx << 4);
100 masks[0][bx4][imin(txa[0][0][y][0], l[y])][sidx] |= smask;
37
←
1st function call argument is an uninitialized value
101 }
102
103 // top block edge
104 for (x = 0, mask = 1U << bx4; x < w4; x++, mask <<= 1) {
105 const int sidx = mask >= 0x10000;
106 const unsigned smask = mask >> (sidx << 4);
107 masks[1][by4][imin(txa[1][0][0][x], a[x])][sidx] |= smask;
108 }
109
110 if (!skip) {
111 // inner (tx) left|right edges
112 for (y = 0, mask = 1U << by4; y < h4; y++, mask <<= 1) {
113 const int sidx = mask >= 0x10000U;
114 const unsigned smask = mask >> (sidx << 4);
115 int ltx = txa[0][0][y][0];
116 int step = txa[0][1][y][0];
117 for (x = step; x < w4; x += step) {
118 const int rtx = txa[0][0][y][x];
119 masks[0][bx4 + x][imin(rtx, ltx)][sidx] |= smask;
120 ltx = rtx;
121 step = txa[0][1][y][x];
122 }
123 }
124
125 // top
126 // inner (tx) --- edges
127 // bottom
128 for (x = 0, mask = 1U << bx4; x < w4; x++, mask <<= 1) {
129 const int sidx = mask >= 0x10000U;
130 const unsigned smask = mask >> (sidx << 4);
131 int ttx = txa[1][0][0][x];
132 int step = txa[1][1][0][x];
133 for (y = step; y < h4; y += step) {
134 const int btx = txa[1][0][y][x];
135 masks[1][by4 + y][imin(ttx, btx)][sidx] |= smask;
136 ttx = btx;
137 step = txa[1][1][y][x];
138 }
139 }
140 }
141
142 for (y = 0; y < h4; y++)
143 l[y] = txa[0][0][y][w4 - 1];
144 memcpy(a, txa[1][0][h4 - 1], w4);
145}
146
147static inline void mask_edges_intra(uint16_t (*const masks)[32][3][2],
148 const int by4, const int bx4,
149 const int w4, const int h4,
150 const enum RectTxfmSize tx,
151 uint8_t *const a, uint8_t *const l)
152{
153 const TxfmInfo *const t_dim = &dav1d_txfm_dimensions[tx];
154 const int twl4 = t_dim->lw, thl4 = t_dim->lh;
155 const int twl4c = imin(2, twl4), thl4c = imin(2, thl4);
156 int y, x;
157
158 // left block edge
159 unsigned mask = 1U << by4;
160 for (y = 0; y < h4; y++, mask <<= 1) {
161 const int sidx = mask >= 0x10000;
162 const unsigned smask = mask >> (sidx << 4);
163 masks[0][bx4][imin(twl4c, l[y])][sidx] |= smask;
164 }
165
166 // top block edge
167 for (x = 0, mask = 1U << bx4; x < w4; x++, mask <<= 1) {
168 const int sidx = mask >= 0x10000;
169 const unsigned smask = mask >> (sidx << 4);
170 masks[1][by4][imin(thl4c, a[x])][sidx] |= smask;
171 }
172
173 // inner (tx) left|right edges
174 const int hstep = t_dim->w;
175 unsigned t = 1U << by4;
176 unsigned inner = (unsigned) ((((uint64_t) t) << h4) - t);
177 unsigned inner1 = inner & 0xffff, inner2 = inner >> 16;
178 for (x = hstep; x < w4; x += hstep) {
179 if (inner1) masks[0][bx4 + x][twl4c][0] |= inner1;
180 if (inner2) masks[0][bx4 + x][twl4c][1] |= inner2;
181 }
182
183 // top
184 // inner (tx) --- edges
185 // bottom
186 const int vstep = t_dim->h;
187 t = 1U << bx4;
188 inner = (unsigned) ((((uint64_t) t) << w4) - t);
189 inner1 = inner & 0xffff;
190 inner2 = inner >> 16;
191 for (y = vstep; y < h4; y += vstep) {
192 if (inner1) masks[1][by4 + y][thl4c][0] |= inner1;
193 if (inner2) masks[1][by4 + y][thl4c][1] |= inner2;
194 }
195
196 dav1d_memset_likely_pow2(a, thl4c, w4);
197 dav1d_memset_likely_pow2(l, twl4c, h4);
198}
199
200static void mask_edges_chroma(uint16_t (*const masks)[32][2][2],
201 const int cby4, const int cbx4,
202 const int cw4, const int ch4,
203 const int skip_inter,
204 const enum RectTxfmSize tx,
205 uint8_t *const a, uint8_t *const l,
206 const int ss_hor, const int ss_ver)
207{
208 const TxfmInfo *const t_dim = &dav1d_txfm_dimensions[tx];
209 const int twl4 = t_dim->lw, thl4 = t_dim->lh;
210 const int twl4c = !!twl4, thl4c = !!thl4;
211 int y, x;
212 const int vbits = 4 - ss_ver, hbits = 4 - ss_hor;
213 const int vmask = 16 >> ss_ver, hmask = 16 >> ss_hor;
214 const unsigned vmax = 1 << vmask, hmax = 1 << hmask;
215
216 // left block edge
217 unsigned mask = 1U << cby4;
218 for (y = 0; y < ch4; y++, mask <<= 1) {
219 const int sidx = mask >= vmax;
220 const unsigned smask = mask >> (sidx << vbits);
221 masks[0][cbx4][imin(twl4c, l[y])][sidx] |= smask;
222 }
223
224 // top block edge
225 for (x = 0, mask = 1U << cbx4; x < cw4; x++, mask <<= 1) {
226 const int sidx = mask >= hmax;
227 const unsigned smask = mask >> (sidx << hbits);
228 masks[1][cby4][imin(thl4c, a[x])][sidx] |= smask;
229 }
230
231 if (!skip_inter) {
232 // inner (tx) left|right edges
233 const int hstep = t_dim->w;
234 unsigned t = 1U << cby4;
235 unsigned inner = (unsigned) ((((uint64_t) t) << ch4) - t);
236 unsigned inner1 = inner & ((1 << vmask) - 1), inner2 = inner >> vmask;
237 for (x = hstep; x < cw4; x += hstep) {
238 if (inner1) masks[0][cbx4 + x][twl4c][0] |= inner1;
239 if (inner2) masks[0][cbx4 + x][twl4c][1] |= inner2;
240 }
241
242 // top
243 // inner (tx) --- edges
244 // bottom
245 const int vstep = t_dim->h;
246 t = 1U << cbx4;
247 inner = (unsigned) ((((uint64_t) t) << cw4) - t);
248 inner1 = inner & ((1 << hmask) - 1), inner2 = inner >> hmask;
249 for (y = vstep; y < ch4; y += vstep) {
250 if (inner1) masks[1][cby4 + y][thl4c][0] |= inner1;
251 if (inner2) masks[1][cby4 + y][thl4c][1] |= inner2;
252 }
253 }
254
255 dav1d_memset_likely_pow2(a, thl4c, cw4);
256 dav1d_memset_likely_pow2(l, twl4c, ch4);
257}
258
259static void fill_levelcache_block(uint8_t (*level_cache_ptr)[4],
260 const ptrdiff_t b_stride,
261 const int bw, const int bh,
262 const unsigned v0,
263 const unsigned v1)
264{
265 union alias16 v = (union alias16) { .u8 = { v0, v1 }};
266
267 for (int y = 0; y < bh; y++) {
268 for (int x = 0; x < bw; x++)
269 ((union alias16 *) &level_cache_ptr[x][0])->u16 = v.u16;
270 level_cache_ptr += b_stride;
271 }
272}
273
274void dav1d_create_lf_mask_intra(Av1Filter *const lflvl,
275 uint8_t (*const level_cache)[4],
276 const ptrdiff_t b4_stride,
277 const uint8_t (*filter_level)[8][2],
278 const int bx, const int by,
279 const int iw, const int ih,
280 const enum BlockSize bs,
281 const enum RectTxfmSize ytx,
282 const enum RectTxfmSize uvtx,
283 const enum Dav1dPixelLayout layout,
284 uint8_t *const ay, uint8_t *const ly,
285 uint8_t *const auv, uint8_t *const luv)
286{
287 const uint8_t *const b_dim = dav1d_block_dimensions[bs];
288 const int bw4 = imin(iw - bx, b_dim[0]);
289 const int bh4 = imin(ih - by, b_dim[1]);
290 const int bx4 = bx & 31;
291 const int by4 = by & 31;
292 assert(bw4 >= 0 && bh4 >= 0)((void) sizeof ((bw4 >= 0 && bh4 >= 0) ? 1 : 0)
, __extension__ ({ if (bw4 >= 0 && bh4 >= 0) ; else
__assert_fail ("bw4 >= 0 && bh4 >= 0", "/root/firefox-clang/third_party/dav1d/src/lf_mask.c"
, 292, __extension__ __PRETTY_FUNCTION__); }));
293
294 if (bw4 && bh4) {
295 uint8_t (*level_cache_ptr)[4] = level_cache + by * b4_stride + bx;
296
297 fill_levelcache_block(level_cache_ptr, b4_stride, bw4, bh4,
298 filter_level[0][0][0], filter_level[1][0][0]);
299 mask_edges_intra(lflvl->filter_y, by4, bx4, bw4, bh4, ytx, ay, ly);
300 }
301
302 if (!auv) return;
303
304 const int ss_ver = layout == DAV1D_PIXEL_LAYOUT_I420;
305 const int ss_hor = layout != DAV1D_PIXEL_LAYOUT_I444;
306 const int cbw4 = imin(((iw + ss_hor) >> ss_hor) - (bx >> ss_hor),
307 (b_dim[0] + ss_hor) >> ss_hor);
308 const int cbh4 = imin(((ih + ss_ver) >> ss_ver) - (by >> ss_ver),
309 (b_dim[1] + ss_ver) >> ss_ver);
310 assert(cbw4 >= 0 && cbh4 >= 0)((void) sizeof ((cbw4 >= 0 && cbh4 >= 0) ? 1 : 0
), __extension__ ({ if (cbw4 >= 0 && cbh4 >= 0)
; else __assert_fail ("cbw4 >= 0 && cbh4 >= 0"
, "/root/firefox-clang/third_party/dav1d/src/lf_mask.c", 310,
__extension__ __PRETTY_FUNCTION__); }));
311
312 if (!cbw4 || !cbh4) return;
313
314 const int cbx4 = bx4 >> ss_hor;
315 const int cby4 = by4 >> ss_ver;
316
317 uint8_t (*level_cache_ptr)[4] =
318 level_cache + (by >> ss_ver) * b4_stride + (bx >> ss_hor);
319
320 fill_levelcache_block((uint8_t(*)[4]) &level_cache_ptr[0][2], b4_stride,
321 cbw4, cbh4, filter_level[2][0][0],
322 filter_level[3][0][0]);
323 mask_edges_chroma(lflvl->filter_uv, cby4, cbx4, cbw4, cbh4, 0, uvtx,
324 auv, luv, ss_hor, ss_ver);
325}
326
327void dav1d_create_lf_mask_inter(Av1Filter *const lflvl,
328 uint8_t (*const level_cache)[4],
329 const ptrdiff_t b4_stride,
330 const uint8_t (*filter_level)[8][2],
331 const int bx, const int by,
332 const int iw, const int ih,
333 const int skip, const enum BlockSize bs,
334 const enum RectTxfmSize max_ytx,
335 const uint16_t *const tx_masks,
336 const enum RectTxfmSize uvtx,
337 const enum Dav1dPixelLayout layout,
338 uint8_t *const ay, uint8_t *const ly,
339 uint8_t *const auv, uint8_t *const luv)
340{
341 const uint8_t *const b_dim = dav1d_block_dimensions[bs];
342 const int bw4 = imin(iw - bx, b_dim[0]);
343 const int bh4 = imin(ih - by, b_dim[1]);
344 const int bx4 = bx & 31;
345 const int by4 = by & 31;
346 assert(bw4 >= 0 && bh4 >= 0)((void) sizeof ((bw4 >= 0 && bh4 >= 0) ? 1 : 0)
, __extension__ ({ if (bw4 >= 0 && bh4 >= 0) ; else
__assert_fail ("bw4 >= 0 && bh4 >= 0", "/root/firefox-clang/third_party/dav1d/src/lf_mask.c"
, 346, __extension__ __PRETTY_FUNCTION__); }));
1
Assuming 'bw4' is >= 0
→
2
←
Assuming 'bh4' is >= 0
→
3
←
Taking true branch
→
347
348 if (bw4 && bh4) {
4
←
Assuming 'bw4' is not equal to 0
→
5
←
Assuming 'bh4' is not equal to 0
→
6
←
Taking true branch
→
349 uint8_t (*level_cache_ptr)[4] = level_cache + by * b4_stride + bx;
350
351 fill_levelcache_block(level_cache_ptr, b4_stride, bw4, bh4,
352 filter_level[0][0][0], filter_level[1][0][0]);
353 mask_edges_inter(lflvl->filter_y, by4, bx4, bw4, bh4, skip,
7
←
Calling 'mask_edges_inter'
→
354 max_ytx, tx_masks, ay, ly);
355 }
356
357 if (!auv) return;
358
359 const int ss_ver = layout == DAV1D_PIXEL_LAYOUT_I420;
360 const int ss_hor = layout != DAV1D_PIXEL_LAYOUT_I444;
361 const int cbw4 = imin(((iw + ss_hor) >> ss_hor) - (bx >> ss_hor),
362 (b_dim[0] + ss_hor) >> ss_hor);
363 const int cbh4 = imin(((ih + ss_ver) >> ss_ver) - (by >> ss_ver),
364 (b_dim[1] + ss_ver) >> ss_ver);
365 assert(cbw4 >= 0 && cbh4 >= 0)((void) sizeof ((cbw4 >= 0 && cbh4 >= 0) ? 1 : 0
), __extension__ ({ if (cbw4 >= 0 && cbh4 >= 0)
; else __assert_fail ("cbw4 >= 0 && cbh4 >= 0"
, "/root/firefox-clang/third_party/dav1d/src/lf_mask.c", 365,
__extension__ __PRETTY_FUNCTION__); }));
366
367 if (!cbw4 || !cbh4) return;
368
369 const int cbx4 = bx4 >> ss_hor;
370 const int cby4 = by4 >> ss_ver;
371
372 uint8_t (*level_cache_ptr)[4] =
373 level_cache + (by >> ss_ver) * b4_stride + (bx >> ss_hor);
374
375 fill_levelcache_block((uint8_t(*)[4]) &level_cache_ptr[0][2], b4_stride,
376 cbw4, cbh4, filter_level[2][0][0],
377 filter_level[3][0][0]);
378 mask_edges_chroma(lflvl->filter_uv, cby4, cbx4, cbw4, cbh4, skip, uvtx,
379 auv, luv, ss_hor, ss_ver);
380}
381
382void dav1d_calc_eih(Av1FilterLUT *const lim_lut, const int filter_sharpness) {
383 // set E/I/H values from loopfilter level
384 const int sharp = filter_sharpness;
385 for (int level = 0; level < 64; level++) {
386 int limit = level;
387
388 if (sharp > 0) {
389 limit >>= (sharp + 3) >> 2;
390 limit = imin(limit, 9 - sharp);
391 }
392 limit = imax(limit, 1);
393
394 lim_lut->i[level] = limit;
395 lim_lut->e[level] = 2 * (level + 2) + limit;
396 }
397 lim_lut->sharp[0] = (sharp + 3) >> 2;
398 lim_lut->sharp[1] = sharp ? 9 - sharp : 0xff;
399}
400
401static void calc_lf_value(uint8_t (*const lflvl_values)[2],
402 const int base_lvl, const int lf_delta,
403 const int seg_delta,
404 const Dav1dLoopfilterModeRefDeltas *const mr_delta)
405{
406 const int base = iclip(iclip(base_lvl + lf_delta, 0, 63) + seg_delta, 0, 63);
407
408 if (!mr_delta) {
409 memset(lflvl_values, base, sizeof(*lflvl_values) * 8);
410 } else {
411 const int sh = base >= 32;
412 lflvl_values[0][0] = lflvl_values[0][1] =
413 iclip(base + (mr_delta->ref_delta[0] * (1 << sh)), 0, 63);
414 for (int r = 1; r < 8; r++) {
415 for (int m = 0; m < 2; m++) {
416 const int delta =
417 mr_delta->mode_delta[m] + mr_delta->ref_delta[r];
418 lflvl_values[r][m] = iclip(base + (delta * (1 << sh)), 0, 63);
419 }
420 }
421 }
422}
423
424static inline void calc_lf_value_chroma(uint8_t (*const lflvl_values)[2],
425 const int base_lvl, const int lf_delta,
426 const int seg_delta,
427 const Dav1dLoopfilterModeRefDeltas *const mr_delta)
428{
429 if (!base_lvl)
430 memset(lflvl_values, 0, sizeof(*lflvl_values) * 8);
431 else
432 calc_lf_value(lflvl_values, base_lvl, lf_delta, seg_delta, mr_delta);
433}
434
435void dav1d_calc_lf_values(uint8_t (*const lflvl_values)[4][8][2],
436 const Dav1dFrameHeader *const hdr,
437 const int8_t lf_delta[4])
438{
439 const int n_seg = hdr->segmentation.enabled ? 8 : 1;
440
441 if (!hdr->loopfilter.level_y[0] && !hdr->loopfilter.level_y[1]) {
442 memset(lflvl_values, 0, sizeof(*lflvl_values) * n_seg);
443 return;
444 }
445
446 const Dav1dLoopfilterModeRefDeltas *const mr_deltas =
447 hdr->loopfilter.mode_ref_delta_enabled ?
448 &hdr->loopfilter.mode_ref_deltas : NULL((void*)0);
449 for (int s = 0; s < n_seg; s++) {
450 const Dav1dSegmentationData *const segd =
451 hdr->segmentation.enabled ? &hdr->segmentation.seg_data.d[s] : NULL((void*)0);
452
453 calc_lf_value(lflvl_values[s][0], hdr->loopfilter.level_y[0],
454 lf_delta[0], segd ? segd->delta_lf_y_v : 0, mr_deltas);
455 calc_lf_value(lflvl_values[s][1], hdr->loopfilter.level_y[1],
456 lf_delta[hdr->delta.lf.multi ? 1 : 0],
457 segd ? segd->delta_lf_y_h : 0, mr_deltas);
458 calc_lf_value_chroma(lflvl_values[s][2], hdr->loopfilter.level_u,
459 lf_delta[hdr->delta.lf.multi ? 2 : 0],
460 segd ? segd->delta_lf_u : 0, mr_deltas);
461 calc_lf_value_chroma(lflvl_values[s][3], hdr->loopfilter.level_v,
462 lf_delta[hdr->delta.lf.multi ? 3 : 0],
463 segd ? segd->delta_lf_v : 0, mr_deltas);
464 }
465}