Bug Summary

File:var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp
Warning:line 12711, column 8
Value stored to 'extractObject' during its initialization is never read

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name Unified_cpp_js_src_jit3.cpp -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -fhalf-no-semantic-interposition -mframe-pointer=all -relaxed-aliasing -ffp-contract=off -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -fdebug-compilation-dir=/var/lib/jenkins/workspace/firefox-scan-build/obj-x86_64-pc-linux-gnu/js/src/jit -fcoverage-compilation-dir=/var/lib/jenkins/workspace/firefox-scan-build/obj-x86_64-pc-linux-gnu/js/src/jit -resource-dir /usr/lib/llvm-19/lib/clang/19 -include /var/lib/jenkins/workspace/firefox-scan-build/config/gcc_hidden.h -include /var/lib/jenkins/workspace/firefox-scan-build/obj-x86_64-pc-linux-gnu/mozilla-config.h -I /var/lib/jenkins/workspace/firefox-scan-build/obj-x86_64-pc-linux-gnu/dist/stl_wrappers -I /var/lib/jenkins/workspace/firefox-scan-build/obj-x86_64-pc-linux-gnu/dist/system_wrappers -U _FORTIFY_SOURCE -D _FORTIFY_SOURCE=2 -D DEBUG=1 -D WASM_SUPPORTS_HUGE_MEMORY -D JS_CACHEIR_SPEW -D JS_STRUCTURED_SPEW -D JS_HAS_CTYPES -D FFI_BUILDING -D EXPORT_JS_API -D MOZ_HAS_MOZGLUE -D MOZ_SUPPORT_LEAKCHECKING -I /var/lib/jenkins/workspace/firefox-scan-build/js/src/jit -I /var/lib/jenkins/workspace/firefox-scan-build/obj-x86_64-pc-linux-gnu/js/src/jit -I /var/lib/jenkins/workspace/firefox-scan-build/obj-x86_64-pc-linux-gnu/js/src -I /var/lib/jenkins/workspace/firefox-scan-build/js/src -I /var/lib/jenkins/workspace/firefox-scan-build/obj-x86_64-pc-linux-gnu/dist/include -I /var/lib/jenkins/workspace/firefox-scan-build/obj-x86_64-pc-linux-gnu/dist/include/nspr -I /var/lib/jenkins/workspace/firefox-scan-build/obj-x86_64-pc-linux-gnu/dist/include/nss -D MOZILLA_CLIENT -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/14/../../../../include/c++/14 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/14/../../../../include/x86_64-linux-gnu/c++/14 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/14/../../../../include/c++/14/backward -internal-isystem /usr/lib/llvm-19/lib/clang/19/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/14/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O3 -Wno-error=tautological-type-limit-compare -Wno-invalid-offsetof -Wno-range-loop-analysis -Wno-deprecated-anon-enum-enum-conversion -Wno-deprecated-enum-enum-conversion -Wno-deprecated-this-capture -Wno-inline-new-delete -Wno-error=deprecated-declarations -Wno-error=array-bounds -Wno-error=free-nonheap-object -Wno-error=atomic-alignment -Wno-error=deprecated-builtins -Wno-psabi -Wno-error=builtin-macro-redefined -Wno-vla-cxx-extension -Wno-unknown-warning-option -fdeprecated-macro -ferror-limit 19 -fstrict-flex-arrays=1 -stack-protector 2 -fstack-clash-protection -ftrivial-auto-var-init=pattern -fno-rtti -fgnuc-version=4.2.1 -fskip-odr-check-in-gmf -fno-sized-deallocation -fno-aligned-allocation -vectorize-loops -vectorize-slp -analyzer-checker optin.performance.Padding -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2024-09-22-115206-3586786-1 -x c++ Unified_cpp_js_src_jit3.cpp
1/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2 * vim: set ts=8 sts=2 et sw=2 tw=80:
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7#include "jit/CodeGenerator.h"
8
9#include "mozilla/Assertions.h"
10#include "mozilla/Casting.h"
11#include "mozilla/DebugOnly.h"
12#include "mozilla/EndianUtils.h"
13#include "mozilla/EnumeratedArray.h"
14#include "mozilla/EnumeratedRange.h"
15#include "mozilla/EnumSet.h"
16#include "mozilla/IntegerTypeTraits.h"
17#include "mozilla/Latin1.h"
18#include "mozilla/MathAlgorithms.h"
19#include "mozilla/ScopeExit.h"
20#include "mozilla/SIMD.h"
21
22#include <limits>
23#include <type_traits>
24#include <utility>
25
26#include "jslibmath.h"
27#include "jsmath.h"
28#include "jsnum.h"
29
30#include "builtin/MapObject.h"
31#include "builtin/RegExp.h"
32#include "builtin/String.h"
33#include "irregexp/RegExpTypes.h"
34#include "jit/ABIArgGenerator.h"
35#include "jit/CompileInfo.h"
36#include "jit/InlineScriptTree.h"
37#include "jit/Invalidation.h"
38#include "jit/IonGenericCallStub.h"
39#include "jit/IonIC.h"
40#include "jit/IonScript.h"
41#include "jit/JitcodeMap.h"
42#include "jit/JitFrames.h"
43#include "jit/JitRuntime.h"
44#include "jit/JitSpewer.h"
45#include "jit/JitZone.h"
46#include "jit/Linker.h"
47#include "jit/MIRGenerator.h"
48#include "jit/MoveEmitter.h"
49#include "jit/RangeAnalysis.h"
50#include "jit/RegExpStubConstants.h"
51#include "jit/SafepointIndex.h"
52#include "jit/SharedICHelpers.h"
53#include "jit/SharedICRegisters.h"
54#include "jit/VMFunctions.h"
55#include "jit/WarpSnapshot.h"
56#include "js/ColumnNumber.h" // JS::LimitedColumnNumberOneOrigin
57#include "js/experimental/JitInfo.h" // JSJit{Getter,Setter}CallArgs, JSJitMethodCallArgsTraits, JSJitInfo
58#include "js/friend/DOMProxy.h" // JS::ExpandoAndGeneration
59#include "js/RegExpFlags.h" // JS::RegExpFlag
60#include "js/ScalarType.h" // js::Scalar::Type
61#include "proxy/DOMProxy.h"
62#include "proxy/ScriptedProxyHandler.h"
63#include "util/CheckedArithmetic.h"
64#include "util/Unicode.h"
65#include "vm/ArrayBufferViewObject.h"
66#include "vm/AsyncFunction.h"
67#include "vm/AsyncIteration.h"
68#include "vm/BuiltinObjectKind.h"
69#include "vm/FunctionFlags.h" // js::FunctionFlags
70#include "vm/Interpreter.h"
71#include "vm/JSAtomUtils.h" // AtomizeString
72#include "vm/MatchPairs.h"
73#include "vm/RegExpObject.h"
74#include "vm/RegExpStatics.h"
75#include "vm/StaticStrings.h"
76#include "vm/StringObject.h"
77#include "vm/StringType.h"
78#include "vm/TypedArrayObject.h"
79#include "wasm/WasmCodegenConstants.h"
80#include "wasm/WasmPI.h"
81#include "wasm/WasmValType.h"
82#ifdef MOZ_VTUNE1
83# include "vtune/VTuneWrapper.h"
84#endif
85#include "wasm/WasmBinary.h"
86#include "wasm/WasmGC.h"
87#include "wasm/WasmGcObject.h"
88#include "wasm/WasmStubs.h"
89
90#include "builtin/Boolean-inl.h"
91#include "jit/MacroAssembler-inl.h"
92#include "jit/shared/CodeGenerator-shared-inl.h"
93#include "jit/TemplateObject-inl.h"
94#include "jit/VMFunctionList-inl.h"
95#include "vm/BytecodeUtil-inl.h"
96#include "vm/JSScript-inl.h"
97#include "wasm/WasmInstance-inl.h"
98
99using namespace js;
100using namespace js::jit;
101
102using mozilla::CheckedUint32;
103using mozilla::DebugOnly;
104using mozilla::FloatingPoint;
105using mozilla::NegativeInfinity;
106using mozilla::PositiveInfinity;
107
108using JS::ExpandoAndGeneration;
109
110namespace js {
111namespace jit {
112
113#ifdef CHECK_OSIPOINT_REGISTERS1
114template <class Op>
115static void HandleRegisterDump(Op op, MacroAssembler& masm,
116 LiveRegisterSet liveRegs, Register activation,
117 Register scratch) {
118 const size_t baseOffset = JitActivation::offsetOfRegs();
119
120 // Handle live GPRs.
121 for (GeneralRegisterIterator iter(liveRegs.gprs()); iter.more(); ++iter) {
122 Register reg = *iter;
123 Address dump(activation, baseOffset + RegisterDump::offsetOfRegister(reg));
124
125 if (reg == activation) {
126 // To use the original value of the activation register (that's
127 // now on top of the stack), we need the scratch register.
128 masm.push(scratch);
129 masm.loadPtr(Address(masm.getStackPointer(), sizeof(uintptr_t)), scratch);
130 op(scratch, dump);
131 masm.pop(scratch);
132 } else {
133 op(reg, dump);
134 }
135 }
136
137 // Handle live FPRs.
138 for (FloatRegisterIterator iter(liveRegs.fpus()); iter.more(); ++iter) {
139 FloatRegister reg = *iter;
140 Address dump(activation, baseOffset + RegisterDump::offsetOfRegister(reg));
141 op(reg, dump);
142 }
143}
144
145class StoreOp {
146 MacroAssembler& masm;
147
148 public:
149 explicit StoreOp(MacroAssembler& masm) : masm(masm) {}
150
151 void operator()(Register reg, Address dump) { masm.storePtr(reg, dump); }
152 void operator()(FloatRegister reg, Address dump) {
153 if (reg.isDouble()) {
154 masm.storeDouble(reg, dump);
155 } else if (reg.isSingle()) {
156 masm.storeFloat32(reg, dump);
157 } else if (reg.isSimd128()) {
158 MOZ_CRASH("Unexpected case for SIMD")do { do { } while (false); MOZ_ReportCrash("" "Unexpected case for SIMD"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 158); AnnotateMozCrashReason("MOZ_CRASH(" "Unexpected case for SIMD"
")"); do { *((volatile int*)__null) = 158; __attribute__((nomerge
)) ::abort(); } while (false); } while (false)
;
159 } else {
160 MOZ_CRASH("Unexpected register type.")do { do { } while (false); MOZ_ReportCrash("" "Unexpected register type."
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 160); AnnotateMozCrashReason("MOZ_CRASH(" "Unexpected register type."
")"); do { *((volatile int*)__null) = 160; __attribute__((nomerge
)) ::abort(); } while (false); } while (false)
;
161 }
162 }
163};
164
165class VerifyOp {
166 MacroAssembler& masm;
167 Label* failure_;
168
169 public:
170 VerifyOp(MacroAssembler& masm, Label* failure)
171 : masm(masm), failure_(failure) {}
172
173 void operator()(Register reg, Address dump) {
174 masm.branchPtr(Assembler::NotEqual, dump, reg, failure_);
175 }
176 void operator()(FloatRegister reg, Address dump) {
177 if (reg.isDouble()) {
178 ScratchDoubleScope scratch(masm);
179 masm.loadDouble(dump, scratch);
180 masm.branchDouble(Assembler::DoubleNotEqual, scratch, reg, failure_);
181 } else if (reg.isSingle()) {
182 ScratchFloat32Scope scratch(masm);
183 masm.loadFloat32(dump, scratch);
184 masm.branchFloat(Assembler::DoubleNotEqual, scratch, reg, failure_);
185 } else if (reg.isSimd128()) {
186 MOZ_CRASH("Unexpected case for SIMD")do { do { } while (false); MOZ_ReportCrash("" "Unexpected case for SIMD"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 186); AnnotateMozCrashReason("MOZ_CRASH(" "Unexpected case for SIMD"
")"); do { *((volatile int*)__null) = 186; __attribute__((nomerge
)) ::abort(); } while (false); } while (false)
;
187 } else {
188 MOZ_CRASH("Unexpected register type.")do { do { } while (false); MOZ_ReportCrash("" "Unexpected register type."
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 188); AnnotateMozCrashReason("MOZ_CRASH(" "Unexpected register type."
")"); do { *((volatile int*)__null) = 188; __attribute__((nomerge
)) ::abort(); } while (false); } while (false)
;
189 }
190 }
191};
192
193void CodeGenerator::verifyOsiPointRegs(LSafepoint* safepoint) {
194 // Ensure the live registers stored by callVM did not change between
195 // the call and this OsiPoint. Try-catch relies on this invariant.
196
197 // Load pointer to the JitActivation in a scratch register.
198 AllocatableGeneralRegisterSet allRegs(GeneralRegisterSet::All());
199 Register scratch = allRegs.takeAny();
200 masm.push(scratch);
201 masm.loadJitActivation(scratch);
202
203 // If we should not check registers (because the instruction did not call
204 // into the VM, or a GC happened), we're done.
205 Label failure, done;
206 Address checkRegs(scratch, JitActivation::offsetOfCheckRegs());
207 masm.branch32(Assembler::Equal, checkRegs, Imm32(0), &done);
208
209 // Having more than one VM function call made in one visit function at
210 // runtime is a sec-ciritcal error, because if we conservatively assume that
211 // one of the function call can re-enter Ion, then the invalidation process
212 // will potentially add a call at a random location, by patching the code
213 // before the return address.
214 masm.branch32(Assembler::NotEqual, checkRegs, Imm32(1), &failure);
215
216 // Set checkRegs to 0, so that we don't try to verify registers after we
217 // return from this script to the caller.
218 masm.store32(Imm32(0), checkRegs);
219
220 // Ignore clobbered registers. Some instructions (like LValueToInt32) modify
221 // temps after calling into the VM. This is fine because no other
222 // instructions (including this OsiPoint) will depend on them. Also
223 // backtracking can also use the same register for an input and an output.
224 // These are marked as clobbered and shouldn't get checked.
225 LiveRegisterSet liveRegs;
226 liveRegs.set() = RegisterSet::Intersect(
227 safepoint->liveRegs().set(),
228 RegisterSet::Not(safepoint->clobberedRegs().set()));
229
230 VerifyOp op(masm, &failure);
231 HandleRegisterDump<VerifyOp>(op, masm, liveRegs, scratch, allRegs.getAny());
232
233 masm.jump(&done);
234
235 // Do not profile the callWithABI that occurs below. This is to avoid a
236 // rare corner case that occurs when profiling interacts with itself:
237 //
238 // When slow profiling assertions are turned on, FunctionBoundary ops
239 // (which update the profiler pseudo-stack) may emit a callVM, which
240 // forces them to have an osi point associated with them. The
241 // FunctionBoundary for inline function entry is added to the caller's
242 // graph with a PC from the caller's code, but during codegen it modifies
243 // Gecko Profiler instrumentation to add the callee as the current top-most
244 // script. When codegen gets to the OSIPoint, and the callWithABI below is
245 // emitted, the codegen thinks that the current frame is the callee, but
246 // the PC it's using from the OSIPoint refers to the caller. This causes
247 // the profiler instrumentation of the callWithABI below to ASSERT, since
248 // the script and pc are mismatched. To avoid this, we simply omit
249 // instrumentation for these callWithABIs.
250
251 // Any live register captured by a safepoint (other than temp registers)
252 // must remain unchanged between the call and the OsiPoint instruction.
253 masm.bind(&failure);
254 masm.assumeUnreachable("Modified registers between VM call and OsiPoint");
255
256 masm.bind(&done);
257 masm.pop(scratch);
258}
259
260bool CodeGenerator::shouldVerifyOsiPointRegs(LSafepoint* safepoint) {
261 if (!checkOsiPointRegisters) {
262 return false;
263 }
264
265 if (safepoint->liveRegs().emptyGeneral() &&
266 safepoint->liveRegs().emptyFloat()) {
267 return false; // No registers to check.
268 }
269
270 return true;
271}
272
273void CodeGenerator::resetOsiPointRegs(LSafepoint* safepoint) {
274 if (!shouldVerifyOsiPointRegs(safepoint)) {
275 return;
276 }
277
278 // Set checkRegs to 0. If we perform a VM call, the instruction
279 // will set it to 1.
280 AllocatableGeneralRegisterSet allRegs(GeneralRegisterSet::All());
281 Register scratch = allRegs.takeAny();
282 masm.push(scratch);
283 masm.loadJitActivation(scratch);
284 Address checkRegs(scratch, JitActivation::offsetOfCheckRegs());
285 masm.store32(Imm32(0), checkRegs);
286 masm.pop(scratch);
287}
288
289static void StoreAllLiveRegs(MacroAssembler& masm, LiveRegisterSet liveRegs) {
290 // Store a copy of all live registers before performing the call.
291 // When we reach the OsiPoint, we can use this to check nothing
292 // modified them in the meantime.
293
294 // Load pointer to the JitActivation in a scratch register.
295 AllocatableGeneralRegisterSet allRegs(GeneralRegisterSet::All());
296 Register scratch = allRegs.takeAny();
297 masm.push(scratch);
298 masm.loadJitActivation(scratch);
299
300 Address checkRegs(scratch, JitActivation::offsetOfCheckRegs());
301 masm.add32(Imm32(1), checkRegs);
302
303 StoreOp op(masm);
304 HandleRegisterDump<StoreOp>(op, masm, liveRegs, scratch, allRegs.getAny());
305
306 masm.pop(scratch);
307}
308#endif // CHECK_OSIPOINT_REGISTERS
309
310// Before doing any call to Cpp, you should ensure that volatile
311// registers are evicted by the register allocator.
312void CodeGenerator::callVMInternal(VMFunctionId id, LInstruction* ins) {
313 TrampolinePtr code = gen->jitRuntime()->getVMWrapper(id);
314 const VMFunctionData& fun = GetVMFunction(id);
315
316 // Stack is:
317 // ... frame ...
318 // [args]
319#ifdef DEBUG1
320 MOZ_ASSERT(pushedArgs_ == fun.explicitArgs)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(pushedArgs_ == fun.explicitArgs)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(pushedArgs_ == fun.explicitArgs
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"pushedArgs_ == fun.explicitArgs", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 320); AnnotateMozCrashReason("MOZ_ASSERT" "(" "pushedArgs_ == fun.explicitArgs"
")"); do { *((volatile int*)__null) = 320; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
321 pushedArgs_ = 0;
322#endif
323
324#ifdef CHECK_OSIPOINT_REGISTERS1
325 if (shouldVerifyOsiPointRegs(ins->safepoint())) {
326 StoreAllLiveRegs(masm, ins->safepoint()->liveRegs());
327 }
328#endif
329
330#ifdef DEBUG1
331 if (ins->mirRaw()) {
332 MOZ_ASSERT(ins->mirRaw()->isInstruction())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ins->mirRaw()->isInstruction())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(ins->mirRaw()->isInstruction
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("ins->mirRaw()->isInstruction()", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 332); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ins->mirRaw()->isInstruction()"
")"); do { *((volatile int*)__null) = 332; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
333 MInstruction* mir = ins->mirRaw()->toInstruction();
334 MOZ_ASSERT_IF(mir->needsResumePoint(), mir->resumePoint())do { if (mir->needsResumePoint()) { do { static_assert( mozilla
::detail::AssertionConditionType<decltype(mir->resumePoint
())>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(mir->resumePoint()))), 0))) { do { } while (false
); MOZ_ReportAssertionFailure("mir->resumePoint()", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 334); AnnotateMozCrashReason("MOZ_ASSERT" "(" "mir->resumePoint()"
")"); do { *((volatile int*)__null) = 334; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false)
;
335
336 // If this MIR instruction has an overridden AliasSet, set the JitRuntime's
337 // disallowArbitraryCode_ flag so we can assert this VMFunction doesn't call
338 // RunScript. Whitelist MInterruptCheck and MCheckOverRecursed because
339 // interrupt callbacks can call JS (chrome JS or shell testing functions).
340 bool isWhitelisted = mir->isInterruptCheck() || mir->isCheckOverRecursed();
341 if (!mir->hasDefaultAliasSet() && !isWhitelisted) {
342 const void* addr = gen->jitRuntime()->addressOfDisallowArbitraryCode();
343 masm.move32(Imm32(1), ReturnReg);
344 masm.store32(ReturnReg, AbsoluteAddress(addr));
345 }
346 }
347#endif
348
349 // Push an exit frame descriptor.
350 masm.PushFrameDescriptor(FrameType::IonJS);
351
352 // Call the wrapper function. The wrapper is in charge to unwind the stack
353 // when returning from the call. Failures are handled with exceptions based
354 // on the return value of the C functions. To guard the outcome of the
355 // returned value, use another LIR instruction.
356 ensureOsiSpace();
357 uint32_t callOffset = masm.callJit(code);
358 markSafepointAt(callOffset, ins);
359
360#ifdef DEBUG1
361 // Reset the disallowArbitraryCode flag after the call.
362 {
363 const void* addr = gen->jitRuntime()->addressOfDisallowArbitraryCode();
364 masm.push(ReturnReg);
365 masm.move32(Imm32(0), ReturnReg);
366 masm.store32(ReturnReg, AbsoluteAddress(addr));
367 masm.pop(ReturnReg);
368 }
369#endif
370
371 // Pop rest of the exit frame and the arguments left on the stack.
372 int framePop =
373 sizeof(ExitFrameLayout) - ExitFrameLayout::bytesPoppedAfterCall();
374 masm.implicitPop(fun.explicitStackSlots() * sizeof(void*) + framePop);
375
376 // Stack is:
377 // ... frame ...
378}
379
380template <typename Fn, Fn fn>
381void CodeGenerator::callVM(LInstruction* ins) {
382 VMFunctionId id = VMFunctionToId<Fn, fn>::id;
383 callVMInternal(id, ins);
384}
385
386// ArgSeq store arguments for OutOfLineCallVM.
387//
388// OutOfLineCallVM are created with "oolCallVM" function. The third argument of
389// this function is an instance of a class which provides a "generate" in charge
390// of pushing the argument, with "pushArg", for a VMFunction.
391//
392// Such list of arguments can be created by using the "ArgList" function which
393// creates one instance of "ArgSeq", where the type of the arguments are
394// inferred from the type of the arguments.
395//
396// The list of arguments must be written in the same order as if you were
397// calling the function in C++.
398//
399// Example:
400// ArgList(ToRegister(lir->lhs()), ToRegister(lir->rhs()))
401
402template <typename... ArgTypes>
403class ArgSeq {
404 std::tuple<std::remove_reference_t<ArgTypes>...> args_;
405
406 template <std::size_t... ISeq>
407 inline void generate(CodeGenerator* codegen,
408 std::index_sequence<ISeq...>) const {
409 // Arguments are pushed in reverse order, from last argument to first
410 // argument.
411 (codegen->pushArg(std::get<sizeof...(ISeq) - 1 - ISeq>(args_)), ...);
412 }
413
414 public:
415 explicit ArgSeq(ArgTypes&&... args)
416 : args_(std::forward<ArgTypes>(args)...) {}
417
418 inline void generate(CodeGenerator* codegen) const {
419 generate(codegen, std::index_sequence_for<ArgTypes...>{});
420 }
421
422#ifdef DEBUG1
423 static constexpr size_t numArgs = sizeof...(ArgTypes);
424#endif
425};
426
427template <typename... ArgTypes>
428inline ArgSeq<ArgTypes...> ArgList(ArgTypes&&... args) {
429 return ArgSeq<ArgTypes...>(std::forward<ArgTypes>(args)...);
430}
431
432// Store wrappers, to generate the right move of data after the VM call.
433
434struct StoreNothing {
435 inline void generate(CodeGenerator* codegen) const {}
436 inline LiveRegisterSet clobbered() const {
437 return LiveRegisterSet(); // No register gets clobbered
438 }
439};
440
441class StoreRegisterTo {
442 private:
443 Register out_;
444
445 public:
446 explicit StoreRegisterTo(Register out) : out_(out) {}
447
448 inline void generate(CodeGenerator* codegen) const {
449 // It's okay to use storePointerResultTo here - the VMFunction wrapper
450 // ensures the upper bytes are zero for bool/int32 return values.
451 codegen->storePointerResultTo(out_);
452 }
453 inline LiveRegisterSet clobbered() const {
454 LiveRegisterSet set;
455 set.add(out_);
456 return set;
457 }
458};
459
460class StoreFloatRegisterTo {
461 private:
462 FloatRegister out_;
463
464 public:
465 explicit StoreFloatRegisterTo(FloatRegister out) : out_(out) {}
466
467 inline void generate(CodeGenerator* codegen) const {
468 codegen->storeFloatResultTo(out_);
469 }
470 inline LiveRegisterSet clobbered() const {
471 LiveRegisterSet set;
472 set.add(out_);
473 return set;
474 }
475};
476
477template <typename Output>
478class StoreValueTo_ {
479 private:
480 Output out_;
481
482 public:
483 explicit StoreValueTo_(const Output& out) : out_(out) {}
484
485 inline void generate(CodeGenerator* codegen) const {
486 codegen->storeResultValueTo(out_);
487 }
488 inline LiveRegisterSet clobbered() const {
489 LiveRegisterSet set;
490 set.add(out_);
491 return set;
492 }
493};
494
495template <typename Output>
496StoreValueTo_<Output> StoreValueTo(const Output& out) {
497 return StoreValueTo_<Output>(out);
498}
499
500template <typename Fn, Fn fn, class ArgSeq, class StoreOutputTo>
501class OutOfLineCallVM : public OutOfLineCodeBase<CodeGenerator> {
502 private:
503 LInstruction* lir_;
504 ArgSeq args_;
505 StoreOutputTo out_;
506
507 public:
508 OutOfLineCallVM(LInstruction* lir, const ArgSeq& args,
509 const StoreOutputTo& out)
510 : lir_(lir), args_(args), out_(out) {}
511
512 void accept(CodeGenerator* codegen) override {
513 codegen->visitOutOfLineCallVM(this);
514 }
515
516 LInstruction* lir() const { return lir_; }
517 const ArgSeq& args() const { return args_; }
518 const StoreOutputTo& out() const { return out_; }
519};
520
521template <typename Fn, Fn fn, class ArgSeq, class StoreOutputTo>
522OutOfLineCode* CodeGenerator::oolCallVM(LInstruction* lir, const ArgSeq& args,
523 const StoreOutputTo& out) {
524 MOZ_ASSERT(lir->mirRaw())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(lir->mirRaw())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(lir->mirRaw()))), 0))) { do
{ } while (false); MOZ_ReportAssertionFailure("lir->mirRaw()"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 524); AnnotateMozCrashReason("MOZ_ASSERT" "(" "lir->mirRaw()"
")"); do { *((volatile int*)__null) = 524; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
525 MOZ_ASSERT(lir->mirRaw()->isInstruction())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(lir->mirRaw()->isInstruction())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(lir->mirRaw()->isInstruction
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("lir->mirRaw()->isInstruction()", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 525); AnnotateMozCrashReason("MOZ_ASSERT" "(" "lir->mirRaw()->isInstruction()"
")"); do { *((volatile int*)__null) = 525; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
526
527#ifdef DEBUG1
528 VMFunctionId id = VMFunctionToId<Fn, fn>::id;
529 const VMFunctionData& fun = GetVMFunction(id);
530 MOZ_ASSERT(fun.explicitArgs == args.numArgs)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(fun.explicitArgs == args.numArgs)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(fun.explicitArgs == args.numArgs
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"fun.explicitArgs == args.numArgs", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 530); AnnotateMozCrashReason("MOZ_ASSERT" "(" "fun.explicitArgs == args.numArgs"
")"); do { *((volatile int*)__null) = 530; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
531 MOZ_ASSERT(fun.returnsData() !=do { static_assert( mozilla::detail::AssertionConditionType<
decltype(fun.returnsData() != (std::is_same_v<StoreOutputTo
, StoreNothing>))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(fun.returnsData() != (std::is_same_v
<StoreOutputTo, StoreNothing>)))), 0))) { do { } while (
false); MOZ_ReportAssertionFailure("fun.returnsData() != (std::is_same_v<StoreOutputTo, StoreNothing>)"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 532); AnnotateMozCrashReason("MOZ_ASSERT" "(" "fun.returnsData() != (std::is_same_v<StoreOutputTo, StoreNothing>)"
")"); do { *((volatile int*)__null) = 532; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
532 (std::is_same_v<StoreOutputTo, StoreNothing>))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(fun.returnsData() != (std::is_same_v<StoreOutputTo
, StoreNothing>))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(fun.returnsData() != (std::is_same_v
<StoreOutputTo, StoreNothing>)))), 0))) { do { } while (
false); MOZ_ReportAssertionFailure("fun.returnsData() != (std::is_same_v<StoreOutputTo, StoreNothing>)"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 532); AnnotateMozCrashReason("MOZ_ASSERT" "(" "fun.returnsData() != (std::is_same_v<StoreOutputTo, StoreNothing>)"
")"); do { *((volatile int*)__null) = 532; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
533#endif
534
535 OutOfLineCode* ool = new (alloc())
536 OutOfLineCallVM<Fn, fn, ArgSeq, StoreOutputTo>(lir, args, out);
537 addOutOfLineCode(ool, lir->mirRaw()->toInstruction());
538 return ool;
539}
540
541template <typename Fn, Fn fn, class ArgSeq, class StoreOutputTo>
542void CodeGenerator::visitOutOfLineCallVM(
543 OutOfLineCallVM<Fn, fn, ArgSeq, StoreOutputTo>* ool) {
544 LInstruction* lir = ool->lir();
545
546#ifdef JS_JITSPEW1
547 JitSpewStart(JitSpew_Codegen, " # LIR=%s",
548 lir->opName());
549 if (const char* extra = lir->getExtraName()) {
550 JitSpewCont(JitSpew_Codegen, ":%s", extra);
551 }
552 JitSpewFin(JitSpew_Codegen);
553#endif
554 perfSpewer_.recordInstruction(masm, lir);
555 saveLive(lir);
556 ool->args().generate(this);
557 callVM<Fn, fn>(lir);
558 ool->out().generate(this);
559 restoreLiveIgnore(lir, ool->out().clobbered());
560 masm.jump(ool->rejoin());
561}
562
563class OutOfLineICFallback : public OutOfLineCodeBase<CodeGenerator> {
564 private:
565 LInstruction* lir_;
566 size_t cacheIndex_;
567 size_t cacheInfoIndex_;
568
569 public:
570 OutOfLineICFallback(LInstruction* lir, size_t cacheIndex,
571 size_t cacheInfoIndex)
572 : lir_(lir), cacheIndex_(cacheIndex), cacheInfoIndex_(cacheInfoIndex) {}
573
574 void bind(MacroAssembler* masm) override {
575 // The binding of the initial jump is done in
576 // CodeGenerator::visitOutOfLineICFallback.
577 }
578
579 size_t cacheIndex() const { return cacheIndex_; }
580 size_t cacheInfoIndex() const { return cacheInfoIndex_; }
581 LInstruction* lir() const { return lir_; }
582
583 void accept(CodeGenerator* codegen) override {
584 codegen->visitOutOfLineICFallback(this);
585 }
586};
587
588void CodeGeneratorShared::addIC(LInstruction* lir, size_t cacheIndex) {
589 if (cacheIndex == SIZE_MAX(18446744073709551615UL)) {
590 masm.setOOM();
591 return;
592 }
593
594 DataPtr<IonIC> cache(this, cacheIndex);
595 MInstruction* mir = lir->mirRaw()->toInstruction();
596 cache->setScriptedLocation(mir->block()->info().script(),
597 mir->resumePoint()->pc());
598
599 Register temp = cache->scratchRegisterForEntryJump();
600 icInfo_.back().icOffsetForJump = masm.movWithPatch(ImmWord(-1), temp);
601 masm.jump(Address(temp, 0));
602
603 MOZ_ASSERT(!icInfo_.empty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!icInfo_.empty())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!icInfo_.empty()))), 0))) { do
{ } while (false); MOZ_ReportAssertionFailure("!icInfo_.empty()"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 603); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!icInfo_.empty()"
")"); do { *((volatile int*)__null) = 603; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
604
605 OutOfLineICFallback* ool =
606 new (alloc()) OutOfLineICFallback(lir, cacheIndex, icInfo_.length() - 1);
607 addOutOfLineCode(ool, mir);
608
609 masm.bind(ool->rejoin());
610 cache->setRejoinOffset(CodeOffset(ool->rejoin()->offset()));
611}
612
613void CodeGenerator::visitOutOfLineICFallback(OutOfLineICFallback* ool) {
614 LInstruction* lir = ool->lir();
615 size_t cacheIndex = ool->cacheIndex();
616 size_t cacheInfoIndex = ool->cacheInfoIndex();
617
618 DataPtr<IonIC> ic(this, cacheIndex);
619
620 // Register the location of the OOL path in the IC.
621 ic->setFallbackOffset(CodeOffset(masm.currentOffset()));
622
623 switch (ic->kind()) {
624 case CacheKind::GetProp:
625 case CacheKind::GetElem: {
626 IonGetPropertyIC* getPropIC = ic->asGetPropertyIC();
627
628 saveLive(lir);
629
630 pushArg(getPropIC->id());
631 pushArg(getPropIC->value());
632 icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
633 pushArg(ImmGCPtr(gen->outerInfo().script()));
634
635 using Fn = bool (*)(JSContext*, HandleScript, IonGetPropertyIC*,
636 HandleValue, HandleValue, MutableHandleValue);
637 callVM<Fn, IonGetPropertyIC::update>(lir);
638
639 StoreValueTo(getPropIC->output()).generate(this);
640 restoreLiveIgnore(lir, StoreValueTo(getPropIC->output()).clobbered());
641
642 masm.jump(ool->rejoin());
643 return;
644 }
645 case CacheKind::GetPropSuper:
646 case CacheKind::GetElemSuper: {
647 IonGetPropSuperIC* getPropSuperIC = ic->asGetPropSuperIC();
648
649 saveLive(lir);
650
651 pushArg(getPropSuperIC->id());
652 pushArg(getPropSuperIC->receiver());
653 pushArg(getPropSuperIC->object());
654 icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
655 pushArg(ImmGCPtr(gen->outerInfo().script()));
656
657 using Fn =
658 bool (*)(JSContext*, HandleScript, IonGetPropSuperIC*, HandleObject,
659 HandleValue, HandleValue, MutableHandleValue);
660 callVM<Fn, IonGetPropSuperIC::update>(lir);
661
662 StoreValueTo(getPropSuperIC->output()).generate(this);
663 restoreLiveIgnore(lir,
664 StoreValueTo(getPropSuperIC->output()).clobbered());
665
666 masm.jump(ool->rejoin());
667 return;
668 }
669 case CacheKind::SetProp:
670 case CacheKind::SetElem: {
671 IonSetPropertyIC* setPropIC = ic->asSetPropertyIC();
672
673 saveLive(lir);
674
675 pushArg(setPropIC->rhs());
676 pushArg(setPropIC->id());
677 pushArg(setPropIC->object());
678 icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
679 pushArg(ImmGCPtr(gen->outerInfo().script()));
680
681 using Fn = bool (*)(JSContext*, HandleScript, IonSetPropertyIC*,
682 HandleObject, HandleValue, HandleValue);
683 callVM<Fn, IonSetPropertyIC::update>(lir);
684
685 restoreLive(lir);
686
687 masm.jump(ool->rejoin());
688 return;
689 }
690 case CacheKind::GetName: {
691 IonGetNameIC* getNameIC = ic->asGetNameIC();
692
693 saveLive(lir);
694
695 pushArg(getNameIC->environment());
696 icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
697 pushArg(ImmGCPtr(gen->outerInfo().script()));
698
699 using Fn = bool (*)(JSContext*, HandleScript, IonGetNameIC*, HandleObject,
700 MutableHandleValue);
701 callVM<Fn, IonGetNameIC::update>(lir);
702
703 StoreValueTo(getNameIC->output()).generate(this);
704 restoreLiveIgnore(lir, StoreValueTo(getNameIC->output()).clobbered());
705
706 masm.jump(ool->rejoin());
707 return;
708 }
709 case CacheKind::BindName: {
710 IonBindNameIC* bindNameIC = ic->asBindNameIC();
711
712 saveLive(lir);
713
714 pushArg(bindNameIC->environment());
715 icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
716 pushArg(ImmGCPtr(gen->outerInfo().script()));
717
718 using Fn =
719 JSObject* (*)(JSContext*, HandleScript, IonBindNameIC*, HandleObject);
720 callVM<Fn, IonBindNameIC::update>(lir);
721
722 StoreRegisterTo(bindNameIC->output()).generate(this);
723 restoreLiveIgnore(lir, StoreRegisterTo(bindNameIC->output()).clobbered());
724
725 masm.jump(ool->rejoin());
726 return;
727 }
728 case CacheKind::GetIterator: {
729 IonGetIteratorIC* getIteratorIC = ic->asGetIteratorIC();
730
731 saveLive(lir);
732
733 pushArg(getIteratorIC->value());
734 icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
735 pushArg(ImmGCPtr(gen->outerInfo().script()));
736
737 using Fn = JSObject* (*)(JSContext*, HandleScript, IonGetIteratorIC*,
738 HandleValue);
739 callVM<Fn, IonGetIteratorIC::update>(lir);
740
741 StoreRegisterTo(getIteratorIC->output()).generate(this);
742 restoreLiveIgnore(lir,
743 StoreRegisterTo(getIteratorIC->output()).clobbered());
744
745 masm.jump(ool->rejoin());
746 return;
747 }
748 case CacheKind::OptimizeSpreadCall: {
749 auto* optimizeSpreadCallIC = ic->asOptimizeSpreadCallIC();
750
751 saveLive(lir);
752
753 pushArg(optimizeSpreadCallIC->value());
754 icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
755 pushArg(ImmGCPtr(gen->outerInfo().script()));
756
757 using Fn = bool (*)(JSContext*, HandleScript, IonOptimizeSpreadCallIC*,
758 HandleValue, MutableHandleValue);
759 callVM<Fn, IonOptimizeSpreadCallIC::update>(lir);
760
761 StoreValueTo(optimizeSpreadCallIC->output()).generate(this);
762 restoreLiveIgnore(
763 lir, StoreValueTo(optimizeSpreadCallIC->output()).clobbered());
764
765 masm.jump(ool->rejoin());
766 return;
767 }
768 case CacheKind::In: {
769 IonInIC* inIC = ic->asInIC();
770
771 saveLive(lir);
772
773 pushArg(inIC->object());
774 pushArg(inIC->key());
775 icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
776 pushArg(ImmGCPtr(gen->outerInfo().script()));
777
778 using Fn = bool (*)(JSContext*, HandleScript, IonInIC*, HandleValue,
779 HandleObject, bool*);
780 callVM<Fn, IonInIC::update>(lir);
781
782 StoreRegisterTo(inIC->output()).generate(this);
783 restoreLiveIgnore(lir, StoreRegisterTo(inIC->output()).clobbered());
784
785 masm.jump(ool->rejoin());
786 return;
787 }
788 case CacheKind::HasOwn: {
789 IonHasOwnIC* hasOwnIC = ic->asHasOwnIC();
790
791 saveLive(lir);
792
793 pushArg(hasOwnIC->id());
794 pushArg(hasOwnIC->value());
795 icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
796 pushArg(ImmGCPtr(gen->outerInfo().script()));
797
798 using Fn = bool (*)(JSContext*, HandleScript, IonHasOwnIC*, HandleValue,
799 HandleValue, int32_t*);
800 callVM<Fn, IonHasOwnIC::update>(lir);
801
802 StoreRegisterTo(hasOwnIC->output()).generate(this);
803 restoreLiveIgnore(lir, StoreRegisterTo(hasOwnIC->output()).clobbered());
804
805 masm.jump(ool->rejoin());
806 return;
807 }
808 case CacheKind::CheckPrivateField: {
809 IonCheckPrivateFieldIC* checkPrivateFieldIC = ic->asCheckPrivateFieldIC();
810
811 saveLive(lir);
812
813 pushArg(checkPrivateFieldIC->id());
814 pushArg(checkPrivateFieldIC->value());
815
816 icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
817 pushArg(ImmGCPtr(gen->outerInfo().script()));
818
819 using Fn = bool (*)(JSContext*, HandleScript, IonCheckPrivateFieldIC*,
820 HandleValue, HandleValue, bool*);
821 callVM<Fn, IonCheckPrivateFieldIC::update>(lir);
822
823 StoreRegisterTo(checkPrivateFieldIC->output()).generate(this);
824 restoreLiveIgnore(
825 lir, StoreRegisterTo(checkPrivateFieldIC->output()).clobbered());
826
827 masm.jump(ool->rejoin());
828 return;
829 }
830 case CacheKind::InstanceOf: {
831 IonInstanceOfIC* hasInstanceOfIC = ic->asInstanceOfIC();
832
833 saveLive(lir);
834
835 pushArg(hasInstanceOfIC->rhs());
836 pushArg(hasInstanceOfIC->lhs());
837 icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
838 pushArg(ImmGCPtr(gen->outerInfo().script()));
839
840 using Fn = bool (*)(JSContext*, HandleScript, IonInstanceOfIC*,
841 HandleValue lhs, HandleObject rhs, bool* res);
842 callVM<Fn, IonInstanceOfIC::update>(lir);
843
844 StoreRegisterTo(hasInstanceOfIC->output()).generate(this);
845 restoreLiveIgnore(lir,
846 StoreRegisterTo(hasInstanceOfIC->output()).clobbered());
847
848 masm.jump(ool->rejoin());
849 return;
850 }
851 case CacheKind::UnaryArith: {
852 IonUnaryArithIC* unaryArithIC = ic->asUnaryArithIC();
853
854 saveLive(lir);
855
856 pushArg(unaryArithIC->input());
857 icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
858 pushArg(ImmGCPtr(gen->outerInfo().script()));
859
860 using Fn = bool (*)(JSContext* cx, HandleScript outerScript,
861 IonUnaryArithIC* stub, HandleValue val,
862 MutableHandleValue res);
863 callVM<Fn, IonUnaryArithIC::update>(lir);
864
865 StoreValueTo(unaryArithIC->output()).generate(this);
866 restoreLiveIgnore(lir, StoreValueTo(unaryArithIC->output()).clobbered());
867
868 masm.jump(ool->rejoin());
869 return;
870 }
871 case CacheKind::ToPropertyKey: {
872 IonToPropertyKeyIC* toPropertyKeyIC = ic->asToPropertyKeyIC();
873
874 saveLive(lir);
875
876 pushArg(toPropertyKeyIC->input());
877 icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
878 pushArg(ImmGCPtr(gen->outerInfo().script()));
879
880 using Fn = bool (*)(JSContext* cx, HandleScript outerScript,
881 IonToPropertyKeyIC* ic, HandleValue val,
882 MutableHandleValue res);
883 callVM<Fn, IonToPropertyKeyIC::update>(lir);
884
885 StoreValueTo(toPropertyKeyIC->output()).generate(this);
886 restoreLiveIgnore(lir,
887 StoreValueTo(toPropertyKeyIC->output()).clobbered());
888
889 masm.jump(ool->rejoin());
890 return;
891 }
892 case CacheKind::BinaryArith: {
893 IonBinaryArithIC* binaryArithIC = ic->asBinaryArithIC();
894
895 saveLive(lir);
896
897 pushArg(binaryArithIC->rhs());
898 pushArg(binaryArithIC->lhs());
899 icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
900 pushArg(ImmGCPtr(gen->outerInfo().script()));
901
902 using Fn = bool (*)(JSContext* cx, HandleScript outerScript,
903 IonBinaryArithIC* stub, HandleValue lhs,
904 HandleValue rhs, MutableHandleValue res);
905 callVM<Fn, IonBinaryArithIC::update>(lir);
906
907 StoreValueTo(binaryArithIC->output()).generate(this);
908 restoreLiveIgnore(lir, StoreValueTo(binaryArithIC->output()).clobbered());
909
910 masm.jump(ool->rejoin());
911 return;
912 }
913 case CacheKind::Compare: {
914 IonCompareIC* compareIC = ic->asCompareIC();
915
916 saveLive(lir);
917
918 pushArg(compareIC->rhs());
919 pushArg(compareIC->lhs());
920 icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
921 pushArg(ImmGCPtr(gen->outerInfo().script()));
922
923 using Fn =
924 bool (*)(JSContext* cx, HandleScript outerScript, IonCompareIC* stub,
925 HandleValue lhs, HandleValue rhs, bool* res);
926 callVM<Fn, IonCompareIC::update>(lir);
927
928 StoreRegisterTo(compareIC->output()).generate(this);
929 restoreLiveIgnore(lir, StoreRegisterTo(compareIC->output()).clobbered());
930
931 masm.jump(ool->rejoin());
932 return;
933 }
934 case CacheKind::CloseIter: {
935 IonCloseIterIC* closeIterIC = ic->asCloseIterIC();
936
937 saveLive(lir);
938
939 pushArg(closeIterIC->iter());
940 icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
941 pushArg(ImmGCPtr(gen->outerInfo().script()));
942
943 using Fn =
944 bool (*)(JSContext*, HandleScript, IonCloseIterIC*, HandleObject);
945 callVM<Fn, IonCloseIterIC::update>(lir);
946
947 restoreLive(lir);
948
949 masm.jump(ool->rejoin());
950 return;
951 }
952 case CacheKind::OptimizeGetIterator: {
953 auto* optimizeGetIteratorIC = ic->asOptimizeGetIteratorIC();
954
955 saveLive(lir);
956
957 pushArg(optimizeGetIteratorIC->value());
958 icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
959 pushArg(ImmGCPtr(gen->outerInfo().script()));
960
961 using Fn = bool (*)(JSContext*, HandleScript, IonOptimizeGetIteratorIC*,
962 HandleValue, bool* res);
963 callVM<Fn, IonOptimizeGetIteratorIC::update>(lir);
964
965 StoreRegisterTo(optimizeGetIteratorIC->output()).generate(this);
966 restoreLiveIgnore(
967 lir, StoreRegisterTo(optimizeGetIteratorIC->output()).clobbered());
968
969 masm.jump(ool->rejoin());
970 return;
971 }
972 case CacheKind::Call:
973 case CacheKind::TypeOf:
974 case CacheKind::TypeOfEq:
975 case CacheKind::ToBool:
976 case CacheKind::GetIntrinsic:
977 case CacheKind::NewArray:
978 case CacheKind::NewObject:
979 MOZ_CRASH("Unsupported IC")do { do { } while (false); MOZ_ReportCrash("" "Unsupported IC"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 979); AnnotateMozCrashReason("MOZ_CRASH(" "Unsupported IC" ")"
); do { *((volatile int*)__null) = 979; __attribute__((nomerge
)) ::abort(); } while (false); } while (false)
;
980 }
981 MOZ_CRASH()do { do { } while (false); MOZ_ReportCrash("" , "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 981); AnnotateMozCrashReason("MOZ_CRASH(" ")"); do { *((volatile
int*)__null) = 981; __attribute__((nomerge)) ::abort(); } while
(false); } while (false)
;
982}
983
984StringObject* MNewStringObject::templateObj() const {
985 return &templateObj_->as<StringObject>();
986}
987
988CodeGenerator::CodeGenerator(MIRGenerator* gen, LIRGraph* graph,
989 MacroAssembler* masm)
990 : CodeGeneratorSpecific(gen, graph, masm),
991 ionScriptLabels_(gen->alloc()),
992 ionNurseryObjectLabels_(gen->alloc()),
993 scriptCounts_(nullptr),
994 zoneStubsToReadBarrier_(0) {}
995
996CodeGenerator::~CodeGenerator() { js_delete(scriptCounts_); }
997
998void CodeGenerator::visitValueToInt32(LValueToInt32* lir) {
999 ValueOperand operand = ToValue(lir, LValueToInt32::Input);
1000 Register output = ToRegister(lir->output());
1001 FloatRegister temp = ToFloatRegister(lir->tempFloat());
1002
1003 Label fails;
1004 if (lir->mode() == LValueToInt32::TRUNCATE) {
1005 OutOfLineCode* oolDouble = oolTruncateDouble(temp, output, lir->mir());
1006
1007 // We can only handle strings in truncation contexts, like bitwise
1008 // operations.
1009 Register stringReg = ToRegister(lir->temp());
1010 using Fn = bool (*)(JSContext*, JSString*, double*);
1011 auto* oolString = oolCallVM<Fn, StringToNumber>(lir, ArgList(stringReg),
1012 StoreFloatRegisterTo(temp));
1013 Label* stringEntry = oolString->entry();
1014 Label* stringRejoin = oolString->rejoin();
1015
1016 masm.truncateValueToInt32(operand, stringEntry, stringRejoin,
1017 oolDouble->entry(), stringReg, temp, output,
1018 &fails);
1019 masm.bind(oolDouble->rejoin());
1020 } else {
1021 MOZ_ASSERT(lir->mode() == LValueToInt32::NORMAL)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(lir->mode() == LValueToInt32::NORMAL)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(lir->mode() == LValueToInt32::NORMAL))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("lir->mode() == LValueToInt32::NORMAL"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 1021); AnnotateMozCrashReason("MOZ_ASSERT" "(" "lir->mode() == LValueToInt32::NORMAL"
")"); do { *((volatile int*)__null) = 1021; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
1022 masm.convertValueToInt32(operand, temp, output, &fails,
1023 lir->mirNormal()->needsNegativeZeroCheck(),
1024 lir->mirNormal()->conversion());
1025 }
1026
1027 bailoutFrom(&fails, lir->snapshot());
1028}
1029
1030void CodeGenerator::visitValueToDouble(LValueToDouble* lir) {
1031 ValueOperand operand = ToValue(lir, LValueToDouble::InputIndex);
1032 FloatRegister output = ToFloatRegister(lir->output());
1033
1034 Label fail;
1035 masm.convertValueToDouble(operand, output, &fail);
1036 bailoutFrom(&fail, lir->snapshot());
1037}
1038
1039void CodeGenerator::visitValueToFloat32(LValueToFloat32* lir) {
1040 ValueOperand operand = ToValue(lir, LValueToFloat32::InputIndex);
1041 FloatRegister output = ToFloatRegister(lir->output());
1042
1043 Label fail;
1044 masm.convertValueToFloat32(operand, output, &fail);
1045 bailoutFrom(&fail, lir->snapshot());
1046}
1047
1048void CodeGenerator::visitValueToFloat16(LValueToFloat16* lir) {
1049 ValueOperand operand = ToValue(lir, LValueToFloat16::InputIndex);
1050 Register temp = ToTempRegisterOrInvalid(lir->temp0());
1051 FloatRegister output = ToFloatRegister(lir->output());
1052
1053 LiveRegisterSet volatileRegs;
1054 if (!MacroAssembler::SupportsFloat64To16()) {
1055 volatileRegs = liveVolatileRegs(lir);
1056 }
1057
1058 Label fail;
1059 masm.convertValueToFloat16(operand, output, temp, volatileRegs, &fail);
1060 bailoutFrom(&fail, lir->snapshot());
1061}
1062
1063void CodeGenerator::visitValueToBigInt(LValueToBigInt* lir) {
1064 ValueOperand operand = ToValue(lir, LValueToBigInt::InputIndex);
1065 Register output = ToRegister(lir->output());
1066
1067 using Fn = BigInt* (*)(JSContext*, HandleValue);
1068 auto* ool =
1069 oolCallVM<Fn, ToBigInt>(lir, ArgList(operand), StoreRegisterTo(output));
1070
1071 Register tag = masm.extractTag(operand, output);
1072
1073 Label notBigInt, done;
1074 masm.branchTestBigInt(Assembler::NotEqual, tag, &notBigInt);
1075 masm.unboxBigInt(operand, output);
1076 masm.jump(&done);
1077 masm.bind(&notBigInt);
1078
1079 masm.branchTestBoolean(Assembler::Equal, tag, ool->entry());
1080 masm.branchTestString(Assembler::Equal, tag, ool->entry());
1081
1082 // ToBigInt(object) can have side-effects; all other types throw a TypeError.
1083 bailout(lir->snapshot());
1084
1085 masm.bind(ool->rejoin());
1086 masm.bind(&done);
1087}
1088
1089void CodeGenerator::visitInt32ToDouble(LInt32ToDouble* lir) {
1090 masm.convertInt32ToDouble(ToRegister(lir->input()),
1091 ToFloatRegister(lir->output()));
1092}
1093
1094void CodeGenerator::visitFloat32ToDouble(LFloat32ToDouble* lir) {
1095 masm.convertFloat32ToDouble(ToFloatRegister(lir->input()),
1096 ToFloatRegister(lir->output()));
1097}
1098
1099void CodeGenerator::visitDoubleToFloat32(LDoubleToFloat32* lir) {
1100 masm.convertDoubleToFloat32(ToFloatRegister(lir->input()),
1101 ToFloatRegister(lir->output()));
1102}
1103
1104void CodeGenerator::visitInt32ToFloat32(LInt32ToFloat32* lir) {
1105 masm.convertInt32ToFloat32(ToRegister(lir->input()),
1106 ToFloatRegister(lir->output()));
1107}
1108
1109void CodeGenerator::visitDoubleToFloat16(LDoubleToFloat16* lir) {
1110 LiveRegisterSet volatileRegs;
1111 if (!MacroAssembler::SupportsFloat64To16()) {
1112 volatileRegs = liveVolatileRegs(lir);
1113 }
1114 masm.convertDoubleToFloat16(
1115 ToFloatRegister(lir->input()), ToFloatRegister(lir->output()),
1116 ToTempRegisterOrInvalid(lir->temp0()), volatileRegs);
1117}
1118
1119void CodeGenerator::visitDoubleToFloat32ToFloat16(
1120 LDoubleToFloat32ToFloat16* lir) {
1121 masm.convertDoubleToFloat16(
1122 ToFloatRegister(lir->input()), ToFloatRegister(lir->output()),
1123 ToRegister(lir->temp0()), ToRegister(lir->temp1()));
1124}
1125
1126void CodeGenerator::visitFloat32ToFloat16(LFloat32ToFloat16* lir) {
1127 LiveRegisterSet volatileRegs;
1128 if (!MacroAssembler::SupportsFloat32To16()) {
1129 volatileRegs = liveVolatileRegs(lir);
1130 }
1131 masm.convertFloat32ToFloat16(
1132 ToFloatRegister(lir->input()), ToFloatRegister(lir->output()),
1133 ToTempRegisterOrInvalid(lir->temp0()), volatileRegs);
1134}
1135
1136void CodeGenerator::visitInt32ToFloat16(LInt32ToFloat16* lir) {
1137 LiveRegisterSet volatileRegs;
1138 if (!MacroAssembler::SupportsFloat32To16()) {
1139 volatileRegs = liveVolatileRegs(lir);
1140 }
1141 masm.convertInt32ToFloat16(
1142 ToRegister(lir->input()), ToFloatRegister(lir->output()),
1143 ToTempRegisterOrInvalid(lir->temp0()), volatileRegs);
1144}
1145
1146void CodeGenerator::visitDoubleToInt32(LDoubleToInt32* lir) {
1147 Label fail;
1148 FloatRegister input = ToFloatRegister(lir->input());
1149 Register output = ToRegister(lir->output());
1150 masm.convertDoubleToInt32(input, output, &fail,
1151 lir->mir()->needsNegativeZeroCheck());
1152 bailoutFrom(&fail, lir->snapshot());
1153}
1154
1155void CodeGenerator::visitFloat32ToInt32(LFloat32ToInt32* lir) {
1156 Label fail;
1157 FloatRegister input = ToFloatRegister(lir->input());
1158 Register output = ToRegister(lir->output());
1159 masm.convertFloat32ToInt32(input, output, &fail,
1160 lir->mir()->needsNegativeZeroCheck());
1161 bailoutFrom(&fail, lir->snapshot());
1162}
1163
1164void CodeGenerator::visitInt32ToIntPtr(LInt32ToIntPtr* lir) {
1165#ifdef JS_64BIT1
1166 // This LIR instruction is only used if the input can be negative.
1167 MOZ_ASSERT(lir->mir()->canBeNegative())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(lir->mir()->canBeNegative())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(lir->mir()->canBeNegative
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("lir->mir()->canBeNegative()", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 1167); AnnotateMozCrashReason("MOZ_ASSERT" "(" "lir->mir()->canBeNegative()"
")"); do { *((volatile int*)__null) = 1167; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
1168
1169 Register output = ToRegister(lir->output());
1170 const LAllocation* input = lir->input();
1171 if (input->isRegister()) {
1172 masm.move32SignExtendToPtr(ToRegister(input), output);
1173 } else {
1174 masm.load32SignExtendToPtr(ToAddress(input), output);
1175 }
1176#else
1177 MOZ_CRASH("Not used on 32-bit platforms")do { do { } while (false); MOZ_ReportCrash("" "Not used on 32-bit platforms"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 1177); AnnotateMozCrashReason("MOZ_CRASH(" "Not used on 32-bit platforms"
")"); do { *((volatile int*)__null) = 1177; __attribute__((nomerge
)) ::abort(); } while (false); } while (false)
;
1178#endif
1179}
1180
1181void CodeGenerator::visitNonNegativeIntPtrToInt32(
1182 LNonNegativeIntPtrToInt32* lir) {
1183#ifdef JS_64BIT1
1184 Register output = ToRegister(lir->output());
1185 MOZ_ASSERT(ToRegister(lir->input()) == output)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ToRegister(lir->input()) == output)>::isValid,
"invalid assertion condition"); if ((__builtin_expect(!!(!(!
!(ToRegister(lir->input()) == output))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("ToRegister(lir->input()) == output"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 1185); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ToRegister(lir->input()) == output"
")"); do { *((volatile int*)__null) = 1185; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
1186
1187 Label bail;
1188 masm.guardNonNegativeIntPtrToInt32(output, &bail);
1189 bailoutFrom(&bail, lir->snapshot());
1190#else
1191 MOZ_CRASH("Not used on 32-bit platforms")do { do { } while (false); MOZ_ReportCrash("" "Not used on 32-bit platforms"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 1191); AnnotateMozCrashReason("MOZ_CRASH(" "Not used on 32-bit platforms"
")"); do { *((volatile int*)__null) = 1191; __attribute__((nomerge
)) ::abort(); } while (false); } while (false)
;
1192#endif
1193}
1194
1195void CodeGenerator::visitIntPtrToDouble(LIntPtrToDouble* lir) {
1196 Register input = ToRegister(lir->input());
1197 FloatRegister output = ToFloatRegister(lir->output());
1198 masm.convertIntPtrToDouble(input, output);
1199}
1200
1201void CodeGenerator::visitAdjustDataViewLength(LAdjustDataViewLength* lir) {
1202 Register output = ToRegister(lir->output());
1203 MOZ_ASSERT(ToRegister(lir->input()) == output)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ToRegister(lir->input()) == output)>::isValid,
"invalid assertion condition"); if ((__builtin_expect(!!(!(!
!(ToRegister(lir->input()) == output))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("ToRegister(lir->input()) == output"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 1203); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ToRegister(lir->input()) == output"
")"); do { *((volatile int*)__null) = 1203; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
1204
1205 uint32_t byteSize = lir->mir()->byteSize();
1206
1207#ifdef DEBUG1
1208 Label ok;
1209 masm.branchTestPtr(Assembler::NotSigned, output, output, &ok);
1210 masm.assumeUnreachable("Unexpected negative value in LAdjustDataViewLength");
1211 masm.bind(&ok);
1212#endif
1213
1214 Label bail;
1215 masm.branchSubPtr(Assembler::Signed, Imm32(byteSize - 1), output, &bail);
1216 bailoutFrom(&bail, lir->snapshot());
1217}
1218
1219void CodeGenerator::emitOOLTestObject(Register objreg,
1220 Label* ifEmulatesUndefined,
1221 Label* ifDoesntEmulateUndefined,
1222 Register scratch) {
1223 saveVolatile(scratch);
1224#if defined(DEBUG1) || defined(FUZZING)
1225 masm.loadPtr(AbsoluteAddress(
1226 gen->runtime->addressOfHasSeenObjectEmulateUndefinedFuse()),
1227 scratch);
1228 using Fn = bool (*)(JSObject* obj, size_t fuseValue);
1229 masm.setupAlignedABICall();
1230 masm.passABIArg(objreg);
1231 masm.passABIArg(scratch);
1232 masm.callWithABI<Fn, js::EmulatesUndefinedCheckFuse>();
1233#else
1234 using Fn = bool (*)(JSObject* obj);
1235 masm.setupAlignedABICall();
1236 masm.passABIArg(objreg);
1237 masm.callWithABI<Fn, js::EmulatesUndefined>();
1238#endif
1239 masm.storeCallPointerResult(scratch);
1240 restoreVolatile(scratch);
1241
1242 masm.branchIfTrueBool(scratch, ifEmulatesUndefined);
1243 masm.jump(ifDoesntEmulateUndefined);
1244}
1245
1246// Base out-of-line code generator for all tests of the truthiness of an
1247// object, where the object might not be truthy. (Recall that per spec all
1248// objects are truthy, but we implement the JSCLASS_EMULATES_UNDEFINED class
1249// flag to permit objects to look like |undefined| in certain contexts,
1250// including in object truthiness testing.) We check truthiness inline except
1251// when we're testing it on a proxy, in which case out-of-line code will call
1252// EmulatesUndefined for a conclusive answer.
1253class OutOfLineTestObject : public OutOfLineCodeBase<CodeGenerator> {
1254 Register objreg_;
1255 Register scratch_;
1256
1257 Label* ifEmulatesUndefined_;
1258 Label* ifDoesntEmulateUndefined_;
1259
1260#ifdef DEBUG1
1261 bool initialized() { return ifEmulatesUndefined_ != nullptr; }
1262#endif
1263
1264 public:
1265 OutOfLineTestObject()
1266 : ifEmulatesUndefined_(nullptr), ifDoesntEmulateUndefined_(nullptr) {}
1267
1268 void accept(CodeGenerator* codegen) final {
1269 MOZ_ASSERT(initialized())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(initialized())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(initialized()))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("initialized()",
"/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 1269); AnnotateMozCrashReason("MOZ_ASSERT" "(" "initialized()"
")"); do { *((volatile int*)__null) = 1269; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
1270 codegen->emitOOLTestObject(objreg_, ifEmulatesUndefined_,
1271 ifDoesntEmulateUndefined_, scratch_);
1272 }
1273
1274 // Specify the register where the object to be tested is found, labels to
1275 // jump to if the object is truthy or falsy, and a scratch register for
1276 // use in the out-of-line path.
1277 void setInputAndTargets(Register objreg, Label* ifEmulatesUndefined,
1278 Label* ifDoesntEmulateUndefined, Register scratch) {
1279 MOZ_ASSERT(!initialized())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!initialized())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!initialized()))), 0))) { do
{ } while (false); MOZ_ReportAssertionFailure("!initialized()"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 1279); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!initialized()"
")"); do { *((volatile int*)__null) = 1279; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
1280 MOZ_ASSERT(ifEmulatesUndefined)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ifEmulatesUndefined)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(ifEmulatesUndefined))), 0)))
{ do { } while (false); MOZ_ReportAssertionFailure("ifEmulatesUndefined"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 1280); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ifEmulatesUndefined"
")"); do { *((volatile int*)__null) = 1280; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
1281 objreg_ = objreg;
1282 scratch_ = scratch;
1283 ifEmulatesUndefined_ = ifEmulatesUndefined;
1284 ifDoesntEmulateUndefined_ = ifDoesntEmulateUndefined;
1285 }
1286};
1287
1288// A subclass of OutOfLineTestObject containing two extra labels, for use when
1289// the ifTruthy/ifFalsy labels are needed in inline code as well as out-of-line
1290// code. The user should bind these labels in inline code, and specify them as
1291// targets via setInputAndTargets, as appropriate.
1292class OutOfLineTestObjectWithLabels : public OutOfLineTestObject {
1293 Label label1_;
1294 Label label2_;
1295
1296 public:
1297 OutOfLineTestObjectWithLabels() = default;
1298
1299 Label* label1() { return &label1_; }
1300 Label* label2() { return &label2_; }
1301};
1302
1303void CodeGenerator::testObjectEmulatesUndefinedKernel(
1304 Register objreg, Label* ifEmulatesUndefined,
1305 Label* ifDoesntEmulateUndefined, Register scratch,
1306 OutOfLineTestObject* ool) {
1307 ool->setInputAndTargets(objreg, ifEmulatesUndefined, ifDoesntEmulateUndefined,
1308 scratch);
1309
1310 // Perform a fast-path check of the object's class flags if the object's
1311 // not a proxy. Let out-of-line code handle the slow cases that require
1312 // saving registers, making a function call, and restoring registers.
1313 masm.branchIfObjectEmulatesUndefined(objreg, scratch, ool->entry(),
1314 ifEmulatesUndefined);
1315}
1316
1317void CodeGenerator::branchTestObjectEmulatesUndefined(
1318 Register objreg, Label* ifEmulatesUndefined,
1319 Label* ifDoesntEmulateUndefined, Register scratch,
1320 OutOfLineTestObject* ool) {
1321 MOZ_ASSERT(!ifDoesntEmulateUndefined->bound(),do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!ifDoesntEmulateUndefined->bound())>::isValid,
"invalid assertion condition"); if ((__builtin_expect(!!(!(!
!(!ifDoesntEmulateUndefined->bound()))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("!ifDoesntEmulateUndefined->bound()"
" (" "ifDoesntEmulateUndefined will be bound to the fallthrough path"
")", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 1322); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!ifDoesntEmulateUndefined->bound()"
") (" "ifDoesntEmulateUndefined will be bound to the fallthrough path"
")"); do { *((volatile int*)__null) = 1322; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
1322 "ifDoesntEmulateUndefined will be bound to the fallthrough path")do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!ifDoesntEmulateUndefined->bound())>::isValid,
"invalid assertion condition"); if ((__builtin_expect(!!(!(!
!(!ifDoesntEmulateUndefined->bound()))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("!ifDoesntEmulateUndefined->bound()"
" (" "ifDoesntEmulateUndefined will be bound to the fallthrough path"
")", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 1322); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!ifDoesntEmulateUndefined->bound()"
") (" "ifDoesntEmulateUndefined will be bound to the fallthrough path"
")"); do { *((volatile int*)__null) = 1322; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
1323
1324 testObjectEmulatesUndefinedKernel(objreg, ifEmulatesUndefined,
1325 ifDoesntEmulateUndefined, scratch, ool);
1326 masm.bind(ifDoesntEmulateUndefined);
1327}
1328
1329void CodeGenerator::testObjectEmulatesUndefined(Register objreg,
1330 Label* ifEmulatesUndefined,
1331 Label* ifDoesntEmulateUndefined,
1332 Register scratch,
1333 OutOfLineTestObject* ool) {
1334 testObjectEmulatesUndefinedKernel(objreg, ifEmulatesUndefined,
1335 ifDoesntEmulateUndefined, scratch, ool);
1336 masm.jump(ifDoesntEmulateUndefined);
1337}
1338
1339void CodeGenerator::testValueTruthyForType(
1340 JSValueType type, ScratchTagScope& tag, const ValueOperand& value,
1341 Register tempToUnbox, Register temp, FloatRegister floatTemp,
1342 Label* ifTruthy, Label* ifFalsy, OutOfLineTestObject* ool,
1343 bool skipTypeTest) {
1344#ifdef DEBUG1
1345 if (skipTypeTest) {
1346 Label expected;
1347 masm.branchTestType(Assembler::Equal, tag, type, &expected);
1348 masm.assumeUnreachable("Unexpected Value type in testValueTruthyForType");
1349 masm.bind(&expected);
1350 }
1351#endif
1352
1353 // Handle irregular types first.
1354 switch (type) {
1355 case JSVAL_TYPE_UNDEFINED:
1356 case JSVAL_TYPE_NULL:
1357 // Undefined and null are falsy.
1358 if (!skipTypeTest) {
1359 masm.branchTestType(Assembler::Equal, tag, type, ifFalsy);
1360 } else {
1361 masm.jump(ifFalsy);
1362 }
1363 return;
1364 case JSVAL_TYPE_SYMBOL:
1365 // Symbols are truthy.
1366 if (!skipTypeTest) {
1367 masm.branchTestSymbol(Assembler::Equal, tag, ifTruthy);
1368 } else {
1369 masm.jump(ifTruthy);
1370 }
1371 return;
1372 case JSVAL_TYPE_OBJECT: {
1373 Label notObject;
1374 if (!skipTypeTest) {
1375 masm.branchTestObject(Assembler::NotEqual, tag, &notObject);
1376 }
1377 ScratchTagScopeRelease _(&tag);
1378 Register objreg = masm.extractObject(value, tempToUnbox);
1379 testObjectEmulatesUndefined(objreg, ifFalsy, ifTruthy, temp, ool);
1380 masm.bind(&notObject);
1381 return;
1382 }
1383 default:
1384 break;
1385 }
1386
1387 // Check the type of the value (unless this is the last possible type).
1388 Label differentType;
1389 if (!skipTypeTest) {
1390 masm.branchTestType(Assembler::NotEqual, tag, type, &differentType);
1391 }
1392
1393 // Branch if the value is falsy.
1394 ScratchTagScopeRelease _(&tag);
1395 switch (type) {
1396 case JSVAL_TYPE_BOOLEAN: {
1397 masm.branchTestBooleanTruthy(false, value, ifFalsy);
1398 break;
1399 }
1400 case JSVAL_TYPE_INT32: {
1401 masm.branchTestInt32Truthy(false, value, ifFalsy);
1402 break;
1403 }
1404 case JSVAL_TYPE_STRING: {
1405 masm.branchTestStringTruthy(false, value, ifFalsy);
1406 break;
1407 }
1408 case JSVAL_TYPE_BIGINT: {
1409 masm.branchTestBigIntTruthy(false, value, ifFalsy);
1410 break;
1411 }
1412 case JSVAL_TYPE_DOUBLE: {
1413 masm.unboxDouble(value, floatTemp);
1414 masm.branchTestDoubleTruthy(false, floatTemp, ifFalsy);
1415 break;
1416 }
1417 default:
1418 MOZ_CRASH("Unexpected value type")do { do { } while (false); MOZ_ReportCrash("" "Unexpected value type"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 1418); AnnotateMozCrashReason("MOZ_CRASH(" "Unexpected value type"
")"); do { *((volatile int*)__null) = 1418; __attribute__((nomerge
)) ::abort(); } while (false); } while (false)
;
1419 }
1420
1421 // If we reach this point, the value is truthy. We fall through for
1422 // truthy on the last test; otherwise, branch.
1423 if (!skipTypeTest) {
1424 masm.jump(ifTruthy);
1425 }
1426
1427 masm.bind(&differentType);
1428}
1429
1430void CodeGenerator::testValueTruthy(const ValueOperand& value,
1431 Register tempToUnbox, Register temp,
1432 FloatRegister floatTemp,
1433 const TypeDataList& observedTypes,
1434 Label* ifTruthy, Label* ifFalsy,
1435 OutOfLineTestObject* ool) {
1436 ScratchTagScope tag(masm, value);
1437 masm.splitTagForTest(value, tag);
1438
1439 const std::initializer_list<JSValueType> defaultOrder = {
1440 JSVAL_TYPE_UNDEFINED, JSVAL_TYPE_NULL, JSVAL_TYPE_BOOLEAN,
1441 JSVAL_TYPE_INT32, JSVAL_TYPE_OBJECT, JSVAL_TYPE_STRING,
1442 JSVAL_TYPE_DOUBLE, JSVAL_TYPE_SYMBOL, JSVAL_TYPE_BIGINT};
1443
1444 mozilla::EnumSet<JSValueType, uint32_t> remaining(defaultOrder);
1445
1446 // Generate tests for previously observed types first.
1447 // The TypeDataList is sorted by descending frequency.
1448 for (auto& observed : observedTypes) {
1449 JSValueType type = observed.type();
1450 remaining -= type;
1451
1452 testValueTruthyForType(type, tag, value, tempToUnbox, temp, floatTemp,
1453 ifTruthy, ifFalsy, ool, /*skipTypeTest*/ false);
1454 }
1455
1456 // Generate tests for remaining types.
1457 for (auto type : defaultOrder) {
1458 if (!remaining.contains(type)) {
1459 continue;
1460 }
1461 remaining -= type;
1462
1463 // We don't need a type test for the last possible type.
1464 bool skipTypeTest = remaining.isEmpty();
1465 testValueTruthyForType(type, tag, value, tempToUnbox, temp, floatTemp,
1466 ifTruthy, ifFalsy, ool, skipTypeTest);
1467 }
1468 MOZ_ASSERT(remaining.isEmpty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(remaining.isEmpty())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(remaining.isEmpty()))), 0)))
{ do { } while (false); MOZ_ReportAssertionFailure("remaining.isEmpty()"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 1468); AnnotateMozCrashReason("MOZ_ASSERT" "(" "remaining.isEmpty()"
")"); do { *((volatile int*)__null) = 1468; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
1469
1470 // We fall through if the final test is truthy.
1471}
1472
1473void CodeGenerator::visitTestIAndBranch(LTestIAndBranch* test) {
1474 Register input = ToRegister(test->input());
1475 MBasicBlock* ifTrue = test->ifTrue();
1476 MBasicBlock* ifFalse = test->ifFalse();
1477
1478 if (isNextBlock(ifFalse->lir())) {
1479 masm.branchTest32(Assembler::NonZero, input, input,
1480 getJumpLabelForBranch(ifTrue));
1481 } else {
1482 masm.branchTest32(Assembler::Zero, input, input,
1483 getJumpLabelForBranch(ifFalse));
1484 jumpToBlock(ifTrue);
1485 }
1486}
1487
1488void CodeGenerator::visitTestIPtrAndBranch(LTestIPtrAndBranch* test) {
1489 Register input = ToRegister(test->input());
1490 MBasicBlock* ifTrue = test->ifTrue();
1491 MBasicBlock* ifFalse = test->ifFalse();
1492
1493 if (isNextBlock(ifFalse->lir())) {
1494 masm.branchTestPtr(Assembler::NonZero, input, input,
1495 getJumpLabelForBranch(ifTrue));
1496 } else {
1497 masm.branchTestPtr(Assembler::Zero, input, input,
1498 getJumpLabelForBranch(ifFalse));
1499 jumpToBlock(ifTrue);
1500 }
1501}
1502
1503void CodeGenerator::visitTestI64AndBranch(LTestI64AndBranch* test) {
1504 Register64 input = ToRegister64(test->input());
1505 MBasicBlock* ifTrue = test->ifTrue();
1506 MBasicBlock* ifFalse = test->ifFalse();
1507
1508 if (isNextBlock(ifFalse->lir())) {
1509 masm.branchTest64(Assembler::NonZero, input, input,
1510 getJumpLabelForBranch(ifTrue));
1511 } else if (isNextBlock(ifTrue->lir())) {
1512 masm.branchTest64(Assembler::Zero, input, input,
1513 getJumpLabelForBranch(ifFalse));
1514 } else {
1515 masm.branchTest64(Assembler::NonZero, input, input,
1516 getJumpLabelForBranch(ifTrue),
1517 getJumpLabelForBranch(ifFalse));
1518 }
1519}
1520
1521void CodeGenerator::visitTestBIAndBranch(LTestBIAndBranch* lir) {
1522 Register input = ToRegister(lir->input());
1523 MBasicBlock* ifTrue = lir->ifTrue();
1524 MBasicBlock* ifFalse = lir->ifFalse();
1525
1526 if (isNextBlock(ifFalse->lir())) {
1527 masm.branchIfBigIntIsNonZero(input, getJumpLabelForBranch(ifTrue));
1528 } else {
1529 masm.branchIfBigIntIsZero(input, getJumpLabelForBranch(ifFalse));
1530 jumpToBlock(ifTrue);
1531 }
1532}
1533
1534static Assembler::Condition ReverseCondition(Assembler::Condition condition) {
1535 switch (condition) {
1536 case Assembler::Equal:
1537 case Assembler::NotEqual:
1538 return condition;
1539 case Assembler::Above:
1540 return Assembler::Below;
1541 case Assembler::AboveOrEqual:
1542 return Assembler::BelowOrEqual;
1543 case Assembler::Below:
1544 return Assembler::Above;
1545 case Assembler::BelowOrEqual:
1546 return Assembler::AboveOrEqual;
1547 case Assembler::GreaterThan:
1548 return Assembler::LessThan;
1549 case Assembler::GreaterThanOrEqual:
1550 return Assembler::LessThanOrEqual;
1551 case Assembler::LessThan:
1552 return Assembler::GreaterThan;
1553 case Assembler::LessThanOrEqual:
1554 return Assembler::GreaterThanOrEqual;
1555 default:
1556 break;
1557 }
1558 MOZ_CRASH("unhandled condition")do { do { } while (false); MOZ_ReportCrash("" "unhandled condition"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 1558); AnnotateMozCrashReason("MOZ_CRASH(" "unhandled condition"
")"); do { *((volatile int*)__null) = 1558; __attribute__((nomerge
)) ::abort(); } while (false); } while (false)
;
1559}
1560
1561void CodeGenerator::visitCompare(LCompare* comp) {
1562 MCompare::CompareType compareType = comp->mir()->compareType();
1563 Assembler::Condition cond = JSOpToCondition(compareType, comp->jsop());
1564 Register left = ToRegister(comp->left());
1565 const LAllocation* right = comp->right();
1566 Register output = ToRegister(comp->output());
1567
1568 if (compareType == MCompare::Compare_Object ||
1569 compareType == MCompare::Compare_Symbol ||
1570 compareType == MCompare::Compare_IntPtr ||
1571 compareType == MCompare::Compare_UIntPtr ||
1572 compareType == MCompare::Compare_WasmAnyRef) {
1573 if (right->isConstant()) {
1574 MOZ_ASSERT(compareType == MCompare::Compare_IntPtr ||do { static_assert( mozilla::detail::AssertionConditionType<
decltype(compareType == MCompare::Compare_IntPtr || compareType
== MCompare::Compare_UIntPtr)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(compareType == MCompare::Compare_IntPtr
|| compareType == MCompare::Compare_UIntPtr))), 0))) { do { }
while (false); MOZ_ReportAssertionFailure("compareType == MCompare::Compare_IntPtr || compareType == MCompare::Compare_UIntPtr"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 1575); AnnotateMozCrashReason("MOZ_ASSERT" "(" "compareType == MCompare::Compare_IntPtr || compareType == MCompare::Compare_UIntPtr"
")"); do { *((volatile int*)__null) = 1575; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
1575 compareType == MCompare::Compare_UIntPtr)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(compareType == MCompare::Compare_IntPtr || compareType
== MCompare::Compare_UIntPtr)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(compareType == MCompare::Compare_IntPtr
|| compareType == MCompare::Compare_UIntPtr))), 0))) { do { }
while (false); MOZ_ReportAssertionFailure("compareType == MCompare::Compare_IntPtr || compareType == MCompare::Compare_UIntPtr"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 1575); AnnotateMozCrashReason("MOZ_ASSERT" "(" "compareType == MCompare::Compare_IntPtr || compareType == MCompare::Compare_UIntPtr"
")"); do { *((volatile int*)__null) = 1575; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
1576 masm.cmpPtrSet(cond, left, ImmWord(ToInt32(right)), output);
1577 } else if (right->isRegister()) {
1578 masm.cmpPtrSet(cond, left, ToRegister(right), output);
1579 } else {
1580 masm.cmpPtrSet(ReverseCondition(cond), ToAddress(right), left, output);
1581 }
1582 return;
1583 }
1584
1585 MOZ_ASSERT(compareType == MCompare::Compare_Int32 ||do { static_assert( mozilla::detail::AssertionConditionType<
decltype(compareType == MCompare::Compare_Int32 || compareType
== MCompare::Compare_UInt32)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(compareType == MCompare::Compare_Int32
|| compareType == MCompare::Compare_UInt32))), 0))) { do { }
while (false); MOZ_ReportAssertionFailure("compareType == MCompare::Compare_Int32 || compareType == MCompare::Compare_UInt32"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 1586); AnnotateMozCrashReason("MOZ_ASSERT" "(" "compareType == MCompare::Compare_Int32 || compareType == MCompare::Compare_UInt32"
")"); do { *((volatile int*)__null) = 1586; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
1586 compareType == MCompare::Compare_UInt32)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(compareType == MCompare::Compare_Int32 || compareType
== MCompare::Compare_UInt32)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(compareType == MCompare::Compare_Int32
|| compareType == MCompare::Compare_UInt32))), 0))) { do { }
while (false); MOZ_ReportAssertionFailure("compareType == MCompare::Compare_Int32 || compareType == MCompare::Compare_UInt32"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 1586); AnnotateMozCrashReason("MOZ_ASSERT" "(" "compareType == MCompare::Compare_Int32 || compareType == MCompare::Compare_UInt32"
")"); do { *((volatile int*)__null) = 1586; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
1587
1588 if (right->isConstant()) {
1589 masm.cmp32Set(cond, left, Imm32(ToInt32(right)), output);
1590 } else if (right->isRegister()) {
1591 masm.cmp32Set(cond, left, ToRegister(right), output);
1592 } else {
1593 masm.cmp32Set(ReverseCondition(cond), ToAddress(right), left, output);
1594 }
1595}
1596
1597void CodeGenerator::visitCompareAndBranch(LCompareAndBranch* comp) {
1598 MCompare::CompareType compareType = comp->cmpMir()->compareType();
1599 Assembler::Condition cond = JSOpToCondition(compareType, comp->jsop());
1600 Register left = ToRegister(comp->left());
1601 const LAllocation* right = comp->right();
1602
1603 MBasicBlock* ifTrue = comp->ifTrue();
1604 MBasicBlock* ifFalse = comp->ifFalse();
1605
1606 // If the next block is the true case, invert the condition to fall through.
1607 Label* label;
1608 if (isNextBlock(ifTrue->lir())) {
1609 cond = Assembler::InvertCondition(cond);
1610 label = getJumpLabelForBranch(ifFalse);
1611 } else {
1612 label = getJumpLabelForBranch(ifTrue);
1613 }
1614
1615 if (compareType == MCompare::Compare_Object ||
1616 compareType == MCompare::Compare_Symbol ||
1617 compareType == MCompare::Compare_IntPtr ||
1618 compareType == MCompare::Compare_UIntPtr ||
1619 compareType == MCompare::Compare_WasmAnyRef) {
1620 if (right->isConstant()) {
1621 MOZ_ASSERT(compareType == MCompare::Compare_IntPtr ||do { static_assert( mozilla::detail::AssertionConditionType<
decltype(compareType == MCompare::Compare_IntPtr || compareType
== MCompare::Compare_UIntPtr)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(compareType == MCompare::Compare_IntPtr
|| compareType == MCompare::Compare_UIntPtr))), 0))) { do { }
while (false); MOZ_ReportAssertionFailure("compareType == MCompare::Compare_IntPtr || compareType == MCompare::Compare_UIntPtr"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 1622); AnnotateMozCrashReason("MOZ_ASSERT" "(" "compareType == MCompare::Compare_IntPtr || compareType == MCompare::Compare_UIntPtr"
")"); do { *((volatile int*)__null) = 1622; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
1622 compareType == MCompare::Compare_UIntPtr)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(compareType == MCompare::Compare_IntPtr || compareType
== MCompare::Compare_UIntPtr)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(compareType == MCompare::Compare_IntPtr
|| compareType == MCompare::Compare_UIntPtr))), 0))) { do { }
while (false); MOZ_ReportAssertionFailure("compareType == MCompare::Compare_IntPtr || compareType == MCompare::Compare_UIntPtr"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 1622); AnnotateMozCrashReason("MOZ_ASSERT" "(" "compareType == MCompare::Compare_IntPtr || compareType == MCompare::Compare_UIntPtr"
")"); do { *((volatile int*)__null) = 1622; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
1623 masm.branchPtr(cond, left, ImmWord(ToInt32(right)), label);
1624 } else if (right->isRegister()) {
1625 masm.branchPtr(cond, left, ToRegister(right), label);
1626 } else {
1627 masm.branchPtr(ReverseCondition(cond), ToAddress(right), left, label);
1628 }
1629 } else {
1630 MOZ_ASSERT(compareType == MCompare::Compare_Int32 ||do { static_assert( mozilla::detail::AssertionConditionType<
decltype(compareType == MCompare::Compare_Int32 || compareType
== MCompare::Compare_UInt32)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(compareType == MCompare::Compare_Int32
|| compareType == MCompare::Compare_UInt32))), 0))) { do { }
while (false); MOZ_ReportAssertionFailure("compareType == MCompare::Compare_Int32 || compareType == MCompare::Compare_UInt32"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 1631); AnnotateMozCrashReason("MOZ_ASSERT" "(" "compareType == MCompare::Compare_Int32 || compareType == MCompare::Compare_UInt32"
")"); do { *((volatile int*)__null) = 1631; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
1631 compareType == MCompare::Compare_UInt32)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(compareType == MCompare::Compare_Int32 || compareType
== MCompare::Compare_UInt32)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(compareType == MCompare::Compare_Int32
|| compareType == MCompare::Compare_UInt32))), 0))) { do { }
while (false); MOZ_ReportAssertionFailure("compareType == MCompare::Compare_Int32 || compareType == MCompare::Compare_UInt32"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 1631); AnnotateMozCrashReason("MOZ_ASSERT" "(" "compareType == MCompare::Compare_Int32 || compareType == MCompare::Compare_UInt32"
")"); do { *((volatile int*)__null) = 1631; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
1632
1633 if (right->isConstant()) {
1634 masm.branch32(cond, left, Imm32(ToInt32(right)), label);
1635 } else if (right->isRegister()) {
1636 masm.branch32(cond, left, ToRegister(right), label);
1637 } else {
1638 masm.branch32(ReverseCondition(cond), ToAddress(right), left, label);
1639 }
1640 }
1641
1642 if (!isNextBlock(ifTrue->lir())) {
1643 jumpToBlock(ifFalse);
1644 }
1645}
1646
1647void CodeGenerator::visitCompareI64(LCompareI64* lir) {
1648 MCompare::CompareType compareType = lir->mir()->compareType();
1649 MOZ_ASSERT(compareType == MCompare::Compare_Int64 ||do { static_assert( mozilla::detail::AssertionConditionType<
decltype(compareType == MCompare::Compare_Int64 || compareType
== MCompare::Compare_UInt64)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(compareType == MCompare::Compare_Int64
|| compareType == MCompare::Compare_UInt64))), 0))) { do { }
while (false); MOZ_ReportAssertionFailure("compareType == MCompare::Compare_Int64 || compareType == MCompare::Compare_UInt64"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 1650); AnnotateMozCrashReason("MOZ_ASSERT" "(" "compareType == MCompare::Compare_Int64 || compareType == MCompare::Compare_UInt64"
")"); do { *((volatile int*)__null) = 1650; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
1650 compareType == MCompare::Compare_UInt64)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(compareType == MCompare::Compare_Int64 || compareType
== MCompare::Compare_UInt64)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(compareType == MCompare::Compare_Int64
|| compareType == MCompare::Compare_UInt64))), 0))) { do { }
while (false); MOZ_ReportAssertionFailure("compareType == MCompare::Compare_Int64 || compareType == MCompare::Compare_UInt64"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 1650); AnnotateMozCrashReason("MOZ_ASSERT" "(" "compareType == MCompare::Compare_Int64 || compareType == MCompare::Compare_UInt64"
")"); do { *((volatile int*)__null) = 1650; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
1651 bool isSigned = compareType == MCompare::Compare_Int64;
1652 Assembler::Condition cond = JSOpToCondition(lir->jsop(), isSigned);
1653 Register64 left = ToRegister64(lir->left());
1654 LInt64Allocation right = lir->right();
1655 Register output = ToRegister(lir->output());
1656
1657 if (IsConstant(right)) {
1658 masm.cmp64Set(cond, left, Imm64(ToInt64(right)), output);
1659 } else if (IsRegister64(right)) {
1660 masm.cmp64Set(cond, left, ToRegister64(right), output);
1661 } else {
1662 masm.cmp64Set(ReverseCondition(cond), ToAddress(right), left, output);
1663 }
1664}
1665
1666void CodeGenerator::visitCompareI64AndBranch(LCompareI64AndBranch* lir) {
1667 MCompare::CompareType compareType = lir->cmpMir()->compareType();
1668 MOZ_ASSERT(compareType == MCompare::Compare_Int64 ||do { static_assert( mozilla::detail::AssertionConditionType<
decltype(compareType == MCompare::Compare_Int64 || compareType
== MCompare::Compare_UInt64)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(compareType == MCompare::Compare_Int64
|| compareType == MCompare::Compare_UInt64))), 0))) { do { }
while (false); MOZ_ReportAssertionFailure("compareType == MCompare::Compare_Int64 || compareType == MCompare::Compare_UInt64"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 1669); AnnotateMozCrashReason("MOZ_ASSERT" "(" "compareType == MCompare::Compare_Int64 || compareType == MCompare::Compare_UInt64"
")"); do { *((volatile int*)__null) = 1669; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
1669 compareType == MCompare::Compare_UInt64)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(compareType == MCompare::Compare_Int64 || compareType
== MCompare::Compare_UInt64)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(compareType == MCompare::Compare_Int64
|| compareType == MCompare::Compare_UInt64))), 0))) { do { }
while (false); MOZ_ReportAssertionFailure("compareType == MCompare::Compare_Int64 || compareType == MCompare::Compare_UInt64"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 1669); AnnotateMozCrashReason("MOZ_ASSERT" "(" "compareType == MCompare::Compare_Int64 || compareType == MCompare::Compare_UInt64"
")"); do { *((volatile int*)__null) = 1669; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
1670 bool isSigned = compareType == MCompare::Compare_Int64;
1671 Assembler::Condition cond = JSOpToCondition(lir->jsop(), isSigned);
1672 Register64 left = ToRegister64(lir->left());
1673 LInt64Allocation right = lir->right();
1674
1675 MBasicBlock* ifTrue = lir->ifTrue();
1676 MBasicBlock* ifFalse = lir->ifFalse();
1677
1678 Label* trueLabel = getJumpLabelForBranch(ifTrue);
1679 Label* falseLabel = getJumpLabelForBranch(ifFalse);
1680
1681 // If the next block is the true case, invert the condition to fall through.
1682 if (isNextBlock(ifTrue->lir())) {
1683 cond = Assembler::InvertCondition(cond);
1684 trueLabel = falseLabel;
1685 falseLabel = nullptr;
1686 } else if (isNextBlock(ifFalse->lir())) {
1687 falseLabel = nullptr;
1688 }
1689
1690 if (IsConstant(right)) {
1691 masm.branch64(cond, left, Imm64(ToInt64(right)), trueLabel, falseLabel);
1692 } else if (IsRegister64(right)) {
1693 masm.branch64(cond, left, ToRegister64(right), trueLabel, falseLabel);
1694 } else {
1695 masm.branch64(ReverseCondition(cond), ToAddress(right), left, trueLabel,
1696 falseLabel);
1697 }
1698}
1699
1700void CodeGenerator::visitBitAndAndBranch(LBitAndAndBranch* baab) {
1701 Assembler::Condition cond = baab->cond();
1702 Register left = ToRegister(baab->left());
1703 const LAllocation* right = baab->right();
1704
1705 MBasicBlock* ifTrue = baab->ifTrue();
1706 MBasicBlock* ifFalse = baab->ifFalse();
1707
1708 // If the next block is the true case, invert the condition to fall through.
1709 Label* label;
1710 if (isNextBlock(ifTrue->lir())) {
1711 cond = Assembler::InvertCondition(cond);
1712 label = getJumpLabelForBranch(ifFalse);
1713 } else {
1714 label = getJumpLabelForBranch(ifTrue);
1715 }
1716
1717 if (right->isConstant()) {
1718 masm.branchTest32(cond, left, Imm32(ToInt32(right)), label);
1719 } else {
1720 masm.branchTest32(cond, left, ToRegister(right), label);
1721 }
1722
1723 if (!isNextBlock(ifTrue->lir())) {
1724 jumpToBlock(ifFalse);
1725 }
1726}
1727
1728void CodeGenerator::visitBitAnd64AndBranch(LBitAnd64AndBranch* baab) {
1729 Assembler::Condition cond = baab->cond();
1730 Register64 left = ToRegister64(baab->left());
1731 LInt64Allocation right = baab->right();
1732
1733 MBasicBlock* ifTrue = baab->ifTrue();
1734 MBasicBlock* ifFalse = baab->ifFalse();
1735
1736 Label* trueLabel = getJumpLabelForBranch(ifTrue);
1737 Label* falseLabel = getJumpLabelForBranch(ifFalse);
1738
1739 // If the next block is the true case, invert the condition to fall through.
1740 if (isNextBlock(ifTrue->lir())) {
1741 cond = Assembler::InvertCondition(cond);
1742 trueLabel = falseLabel;
1743 falseLabel = nullptr;
1744 } else if (isNextBlock(ifFalse->lir())) {
1745 falseLabel = nullptr;
1746 }
1747
1748 if (IsConstant(right)) {
1749 masm.branchTest64(cond, left, Imm64(ToInt64(right)), trueLabel, falseLabel);
1750 } else {
1751 masm.branchTest64(cond, left, ToRegister64(right), trueLabel, falseLabel);
1752 }
1753}
1754
1755void CodeGenerator::assertObjectDoesNotEmulateUndefined(
1756 Register input, Register temp, const MInstruction* mir) {
1757#if defined(DEBUG1) || defined(FUZZING)
1758 // Validate that the object indeed doesn't have the emulates undefined flag.
1759 auto* ool = new (alloc()) OutOfLineTestObjectWithLabels();
1760 addOutOfLineCode(ool, mir);
1761
1762 Label* doesNotEmulateUndefined = ool->label1();
1763 Label* emulatesUndefined = ool->label2();
1764
1765 testObjectEmulatesUndefined(input, emulatesUndefined, doesNotEmulateUndefined,
1766 temp, ool);
1767 masm.bind(emulatesUndefined);
1768 masm.assumeUnreachable(
1769 "Found an object emulating undefined while the fuse is intact");
1770 masm.bind(doesNotEmulateUndefined);
1771#endif
1772}
1773
1774void CodeGenerator::visitTestOAndBranch(LTestOAndBranch* lir) {
1775 Label* truthy = getJumpLabelForBranch(lir->ifTruthy());
1776 Label* falsy = getJumpLabelForBranch(lir->ifFalsy());
1777 Register input = ToRegister(lir->input());
1778
1779 bool intact = hasSeenObjectEmulateUndefinedFuseIntactAndDependencyNoted();
1780 if (intact) {
1781 assertObjectDoesNotEmulateUndefined(input, ToRegister(lir->temp()),
1782 lir->mir());
1783 // Bug 1874905: It would be fantastic if this could be optimized out
1784 masm.jump(truthy);
1785 } else {
1786 auto* ool = new (alloc()) OutOfLineTestObject();
1787 addOutOfLineCode(ool, lir->mir());
1788
1789 testObjectEmulatesUndefined(input, falsy, truthy, ToRegister(lir->temp()),
1790 ool);
1791 }
1792}
1793
1794void CodeGenerator::visitTestVAndBranch(LTestVAndBranch* lir) {
1795 auto* ool = new (alloc()) OutOfLineTestObject();
1796 addOutOfLineCode(ool, lir->mir());
1797
1798 Label* truthy = getJumpLabelForBranch(lir->ifTruthy());
1799 Label* falsy = getJumpLabelForBranch(lir->ifFalsy());
1800
1801 ValueOperand input = ToValue(lir, LTestVAndBranch::Input);
1802 Register tempToUnbox = ToTempUnboxRegister(lir->temp1());
1803 Register temp = ToRegister(lir->temp2());
1804 FloatRegister floatTemp = ToFloatRegister(lir->tempFloat());
1805 const TypeDataList& observedTypes = lir->mir()->observedTypes();
1806
1807 testValueTruthy(input, tempToUnbox, temp, floatTemp, observedTypes, truthy,
1808 falsy, ool);
1809 masm.jump(truthy);
1810}
1811
1812void CodeGenerator::visitBooleanToString(LBooleanToString* lir) {
1813 Register input = ToRegister(lir->input());
1814 Register output = ToRegister(lir->output());
1815 const JSAtomState& names = gen->runtime->names();
1816 Label true_, done;
1817
1818 masm.branchTest32(Assembler::NonZero, input, input, &true_);
1819 masm.movePtr(ImmGCPtr(names.false_), output);
1820 masm.jump(&done);
1821
1822 masm.bind(&true_);
1823 masm.movePtr(ImmGCPtr(names.true_), output);
1824
1825 masm.bind(&done);
1826}
1827
1828void CodeGenerator::visitIntToString(LIntToString* lir) {
1829 Register input = ToRegister(lir->input());
1830 Register output = ToRegister(lir->output());
1831
1832 using Fn = JSLinearString* (*)(JSContext*, int);
1833 OutOfLineCode* ool = oolCallVM<Fn, Int32ToString<CanGC>>(
1834 lir, ArgList(input), StoreRegisterTo(output));
1835
1836 masm.lookupStaticIntString(input, output, gen->runtime->staticStrings(),
1837 ool->entry());
1838
1839 masm.bind(ool->rejoin());
1840}
1841
1842void CodeGenerator::visitDoubleToString(LDoubleToString* lir) {
1843 FloatRegister input = ToFloatRegister(lir->input());
1844 Register temp = ToRegister(lir->temp0());
1845 Register output = ToRegister(lir->output());
1846
1847 using Fn = JSString* (*)(JSContext*, double);
1848 OutOfLineCode* ool = oolCallVM<Fn, NumberToString<CanGC>>(
1849 lir, ArgList(input), StoreRegisterTo(output));
1850
1851 // Try double to integer conversion and run integer to string code.
1852 masm.convertDoubleToInt32(input, temp, ool->entry(), false);
1853 masm.lookupStaticIntString(temp, output, gen->runtime->staticStrings(),
1854 ool->entry());
1855
1856 masm.bind(ool->rejoin());
1857}
1858
1859void CodeGenerator::visitValueToString(LValueToString* lir) {
1860 ValueOperand input = ToValue(lir, LValueToString::InputIndex);
1861 Register output = ToRegister(lir->output());
1862
1863 using Fn = JSString* (*)(JSContext*, HandleValue);
1864 OutOfLineCode* ool = oolCallVM<Fn, ToStringSlow<CanGC>>(
1865 lir, ArgList(input), StoreRegisterTo(output));
1866
1867 Label done;
1868 Register tag = masm.extractTag(input, output);
1869 const JSAtomState& names = gen->runtime->names();
1870
1871 // String
1872 {
1873 Label notString;
1874 masm.branchTestString(Assembler::NotEqual, tag, &notString);
1875 masm.unboxString(input, output);
1876 masm.jump(&done);
1877 masm.bind(&notString);
1878 }
1879
1880 // Integer
1881 {
1882 Label notInteger;
1883 masm.branchTestInt32(Assembler::NotEqual, tag, &notInteger);
1884 Register unboxed = ToTempUnboxRegister(lir->temp0());
1885 unboxed = masm.extractInt32(input, unboxed);
1886 masm.lookupStaticIntString(unboxed, output, gen->runtime->staticStrings(),
1887 ool->entry());
1888 masm.jump(&done);
1889 masm.bind(&notInteger);
1890 }
1891
1892 // Double
1893 {
1894 // Note: no fastpath. Need two extra registers and can only convert doubles
1895 // that fit integers and are smaller than StaticStrings::INT_STATIC_LIMIT.
1896 masm.branchTestDouble(Assembler::Equal, tag, ool->entry());
1897 }
1898
1899 // Undefined
1900 {
1901 Label notUndefined;
1902 masm.branchTestUndefined(Assembler::NotEqual, tag, &notUndefined);
1903 masm.movePtr(ImmGCPtr(names.undefined), output);
1904 masm.jump(&done);
1905 masm.bind(&notUndefined);
1906 }
1907
1908 // Null
1909 {
1910 Label notNull;
1911 masm.branchTestNull(Assembler::NotEqual, tag, &notNull);
1912 masm.movePtr(ImmGCPtr(names.null), output);
1913 masm.jump(&done);
1914 masm.bind(&notNull);
1915 }
1916
1917 // Boolean
1918 {
1919 Label notBoolean, true_;
1920 masm.branchTestBoolean(Assembler::NotEqual, tag, &notBoolean);
1921 masm.branchTestBooleanTruthy(true, input, &true_);
1922 masm.movePtr(ImmGCPtr(names.false_), output);
1923 masm.jump(&done);
1924 masm.bind(&true_);
1925 masm.movePtr(ImmGCPtr(names.true_), output);
1926 masm.jump(&done);
1927 masm.bind(&notBoolean);
1928 }
1929
1930 // Objects/symbols are only possible when |mir->mightHaveSideEffects()|.
1931 if (lir->mir()->mightHaveSideEffects()) {
1932 // Object
1933 if (lir->mir()->supportSideEffects()) {
1934 masm.branchTestObject(Assembler::Equal, tag, ool->entry());
1935 } else {
1936 // Bail.
1937 MOZ_ASSERT(lir->mir()->needsSnapshot())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(lir->mir()->needsSnapshot())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(lir->mir()->needsSnapshot
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("lir->mir()->needsSnapshot()", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 1937); AnnotateMozCrashReason("MOZ_ASSERT" "(" "lir->mir()->needsSnapshot()"
")"); do { *((volatile int*)__null) = 1937; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
1938 Label bail;
1939 masm.branchTestObject(Assembler::Equal, tag, &bail);
1940 bailoutFrom(&bail, lir->snapshot());
1941 }
1942
1943 // Symbol
1944 if (lir->mir()->supportSideEffects()) {
1945 masm.branchTestSymbol(Assembler::Equal, tag, ool->entry());
1946 } else {
1947 // Bail.
1948 MOZ_ASSERT(lir->mir()->needsSnapshot())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(lir->mir()->needsSnapshot())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(lir->mir()->needsSnapshot
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("lir->mir()->needsSnapshot()", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 1948); AnnotateMozCrashReason("MOZ_ASSERT" "(" "lir->mir()->needsSnapshot()"
")"); do { *((volatile int*)__null) = 1948; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
1949 Label bail;
1950 masm.branchTestSymbol(Assembler::Equal, tag, &bail);
1951 bailoutFrom(&bail, lir->snapshot());
1952 }
1953 }
1954
1955 // BigInt
1956 {
1957 // No fastpath currently implemented.
1958 masm.branchTestBigInt(Assembler::Equal, tag, ool->entry());
1959 }
1960
1961 masm.assumeUnreachable("Unexpected type for LValueToString.");
1962
1963 masm.bind(&done);
1964 masm.bind(ool->rejoin());
1965}
1966
1967using StoreBufferMutationFn = void (*)(js::gc::StoreBuffer*, js::gc::Cell**);
1968
1969static void EmitStoreBufferMutation(MacroAssembler& masm, Register holder,
1970 size_t offset, Register buffer,
1971 LiveGeneralRegisterSet& liveVolatiles,
1972 StoreBufferMutationFn fun) {
1973 Label callVM;
1974 Label exit;
1975
1976 // Call into the VM to barrier the write. The only registers that need to
1977 // be preserved are those in liveVolatiles, so once they are saved on the
1978 // stack all volatile registers are available for use.
1979 masm.bind(&callVM);
1980 masm.PushRegsInMask(liveVolatiles);
1981
1982 AllocatableGeneralRegisterSet regs(GeneralRegisterSet::Volatile());
1983 regs.takeUnchecked(buffer);
1984 regs.takeUnchecked(holder);
1985 Register addrReg = regs.takeAny();
1986
1987 masm.computeEffectiveAddress(Address(holder, offset), addrReg);
1988
1989 bool needExtraReg = !regs.hasAny<GeneralRegisterSet::DefaultType>();
1990 if (needExtraReg) {
1991 masm.push(holder);
1992 masm.setupUnalignedABICall(holder);
1993 } else {
1994 masm.setupUnalignedABICall(regs.takeAny());
1995 }
1996 masm.passABIArg(buffer);
1997 masm.passABIArg(addrReg);
1998 masm.callWithABI(DynamicFunction<StoreBufferMutationFn>(fun),
1999 ABIType::General, CheckUnsafeCallWithABI::DontCheckOther);
2000
2001 if (needExtraReg) {
2002 masm.pop(holder);
2003 }
2004 masm.PopRegsInMask(liveVolatiles);
2005 masm.bind(&exit);
2006}
2007
2008// Warning: this function modifies prev and next.
2009static void EmitPostWriteBarrierS(MacroAssembler& masm, Register holder,
2010 size_t offset, Register prev, Register next,
2011 LiveGeneralRegisterSet& liveVolatiles) {
2012 Label exit;
2013 Label checkRemove, putCell;
2014
2015 // if (next && (buffer = next->storeBuffer()))
2016 // but we never pass in nullptr for next.
2017 Register storebuffer = next;
2018 masm.loadStoreBuffer(next, storebuffer);
2019 masm.branchPtr(Assembler::Equal, storebuffer, ImmWord(0), &checkRemove);
2020
2021 // if (prev && prev->storeBuffer())
2022 masm.branchPtr(Assembler::Equal, prev, ImmWord(0), &putCell);
2023 masm.loadStoreBuffer(prev, prev);
2024 masm.branchPtr(Assembler::NotEqual, prev, ImmWord(0), &exit);
2025
2026 // buffer->putCell(cellp)
2027 masm.bind(&putCell);
2028 EmitStoreBufferMutation(masm, holder, offset, storebuffer, liveVolatiles,
2029 JSString::addCellAddressToStoreBuffer);
2030 masm.jump(&exit);
2031
2032 // if (prev && (buffer = prev->storeBuffer()))
2033 masm.bind(&checkRemove);
2034 masm.branchPtr(Assembler::Equal, prev, ImmWord(0), &exit);
2035 masm.loadStoreBuffer(prev, storebuffer);
2036 masm.branchPtr(Assembler::Equal, storebuffer, ImmWord(0), &exit);
2037 EmitStoreBufferMutation(masm, holder, offset, storebuffer, liveVolatiles,
2038 JSString::removeCellAddressFromStoreBuffer);
2039
2040 masm.bind(&exit);
2041}
2042
2043void CodeGenerator::visitRegExp(LRegExp* lir) {
2044 Register output = ToRegister(lir->output());
2045 Register temp = ToRegister(lir->temp0());
2046 JSObject* source = lir->mir()->source();
2047
2048 using Fn = JSObject* (*)(JSContext*, Handle<RegExpObject*>);
2049 OutOfLineCode* ool = oolCallVM<Fn, CloneRegExpObject>(
2050 lir, ArgList(ImmGCPtr(source)), StoreRegisterTo(output));
2051 if (lir->mir()->hasShared()) {
2052 TemplateObject templateObject(source);
2053 masm.createGCObject(output, temp, templateObject, gc::Heap::Default,
2054 ool->entry());
2055 } else {
2056 masm.jump(ool->entry());
2057 }
2058 masm.bind(ool->rejoin());
2059}
2060
2061static constexpr int32_t RegExpPairsVectorStartOffset(
2062 int32_t inputOutputDataStartOffset) {
2063 return inputOutputDataStartOffset + int32_t(InputOutputDataSize) +
2064 int32_t(sizeof(MatchPairs));
2065}
2066
2067static Address RegExpPairCountAddress(MacroAssembler& masm,
2068 int32_t inputOutputDataStartOffset) {
2069 return Address(FramePointer, inputOutputDataStartOffset +
2070 int32_t(InputOutputDataSize) +
2071 MatchPairs::offsetOfPairCount());
2072}
2073
2074static void UpdateRegExpStatics(MacroAssembler& masm, Register regexp,
2075 Register input, Register lastIndex,
2076 Register staticsReg, Register temp1,
2077 Register temp2, gc::Heap initialStringHeap,
2078 LiveGeneralRegisterSet& volatileRegs) {
2079 Address pendingInputAddress(staticsReg,
2080 RegExpStatics::offsetOfPendingInput());
2081 Address matchesInputAddress(staticsReg,
2082 RegExpStatics::offsetOfMatchesInput());
2083 Address lazySourceAddress(staticsReg, RegExpStatics::offsetOfLazySource());
2084 Address lazyIndexAddress(staticsReg, RegExpStatics::offsetOfLazyIndex());
2085
2086 masm.guardedCallPreBarrier(pendingInputAddress, MIRType::String);
2087 masm.guardedCallPreBarrier(matchesInputAddress, MIRType::String);
2088 masm.guardedCallPreBarrier(lazySourceAddress, MIRType::String);
2089
2090 if (initialStringHeap == gc::Heap::Default) {
2091 // Writing into RegExpStatics tenured memory; must post-barrier.
2092 if (staticsReg.volatile_()) {
2093 volatileRegs.add(staticsReg);
2094 }
2095
2096 masm.loadPtr(pendingInputAddress, temp1);
2097 masm.storePtr(input, pendingInputAddress);
2098 masm.movePtr(input, temp2);
2099 EmitPostWriteBarrierS(masm, staticsReg,
2100 RegExpStatics::offsetOfPendingInput(),
2101 temp1 /* prev */, temp2 /* next */, volatileRegs);
2102
2103 masm.loadPtr(matchesInputAddress, temp1);
2104 masm.storePtr(input, matchesInputAddress);
2105 masm.movePtr(input, temp2);
2106 EmitPostWriteBarrierS(masm, staticsReg,
2107 RegExpStatics::offsetOfMatchesInput(),
2108 temp1 /* prev */, temp2 /* next */, volatileRegs);
2109 } else {
2110 masm.debugAssertGCThingIsTenured(input, temp1);
2111 masm.storePtr(input, pendingInputAddress);
2112 masm.storePtr(input, matchesInputAddress);
2113 }
2114
2115 masm.storePtr(lastIndex,
2116 Address(staticsReg, RegExpStatics::offsetOfLazyIndex()));
2117 masm.store32(
2118 Imm32(1),
2119 Address(staticsReg, RegExpStatics::offsetOfPendingLazyEvaluation()));
2120
2121 masm.unboxNonDouble(Address(regexp, NativeObject::getFixedSlotOffset(
2122 RegExpObject::SHARED_SLOT)),
2123 temp1, JSVAL_TYPE_PRIVATE_GCTHING);
2124 masm.loadPtr(Address(temp1, RegExpShared::offsetOfSource()), temp2);
2125 masm.storePtr(temp2, lazySourceAddress);
2126 static_assert(sizeof(JS::RegExpFlags) == 1, "load size must match flag size");
2127 masm.load8ZeroExtend(Address(temp1, RegExpShared::offsetOfFlags()), temp2);
2128 masm.store8(temp2, Address(staticsReg, RegExpStatics::offsetOfLazyFlags()));
2129}
2130
2131// Prepare an InputOutputData and optional MatchPairs which space has been
2132// allocated for on the stack, and try to execute a RegExp on a string input.
2133// If the RegExp was successfully executed and matched the input, fallthrough.
2134// Otherwise, jump to notFound or failure.
2135//
2136// inputOutputDataStartOffset is the offset relative to the frame pointer
2137// register. This offset is negative for the RegExpExecTest stub.
2138static bool PrepareAndExecuteRegExp(MacroAssembler& masm, Register regexp,
2139 Register input, Register lastIndex,
2140 Register temp1, Register temp2,
2141 Register temp3,
2142 int32_t inputOutputDataStartOffset,
2143 gc::Heap initialStringHeap, Label* notFound,
2144 Label* failure) {
2145 JitSpew(JitSpew_Codegen, "# Emitting PrepareAndExecuteRegExp");
2146
2147 using irregexp::InputOutputData;
2148
2149 /*
2150 * [SMDOC] Stack layout for PrepareAndExecuteRegExp
2151 *
2152 * Before this function is called, the caller is responsible for
2153 * allocating enough stack space for the following data:
2154 *
2155 * inputOutputDataStartOffset +-----> +---------------+
2156 * |InputOutputData|
2157 * inputStartAddress +----------> inputStart|
2158 * inputEndAddress +----------> inputEnd|
2159 * startIndexAddress +----------> startIndex|
2160 * matchesAddress +----------> matches|-----+
2161 * +---------------+ |
2162 * matchPairs(Address|Offset) +-----> +---------------+ <--+
2163 * | MatchPairs |
2164 * pairCountAddress +----------> count |
2165 * pairsPointerAddress +----------> pairs |-----+
2166 * +---------------+ |
2167 * pairsArray(Address|Offset) +-----> +---------------+ <--+
2168 * | MatchPair |
2169 * firstMatchStartAddress +----------> start | <--+
2170 * | limit | |
2171 * +---------------+ |
2172 * . |
2173 * . Reserved space for
2174 * . RegExpObject::MaxPairCount
2175 * . MatchPair objects
2176 * . |
2177 * +---------------+ |
2178 * | MatchPair | |
2179 * | start | |
2180 * | limit | <--+
2181 * +---------------+
2182 */
2183
2184 int32_t ioOffset = inputOutputDataStartOffset;
2185 int32_t matchPairsOffset = ioOffset + int32_t(sizeof(InputOutputData));
2186 int32_t pairsArrayOffset = matchPairsOffset + int32_t(sizeof(MatchPairs));
2187
2188 Address inputStartAddress(FramePointer,
2189 ioOffset + InputOutputData::offsetOfInputStart());
2190 Address inputEndAddress(FramePointer,
2191 ioOffset + InputOutputData::offsetOfInputEnd());
2192 Address startIndexAddress(FramePointer,
2193 ioOffset + InputOutputData::offsetOfStartIndex());
2194 Address matchesAddress(FramePointer,
2195 ioOffset + InputOutputData::offsetOfMatches());
2196
2197 Address matchPairsAddress(FramePointer, matchPairsOffset);
2198 Address pairCountAddress(FramePointer,
2199 matchPairsOffset + MatchPairs::offsetOfPairCount());
2200 Address pairsPointerAddress(FramePointer,
2201 matchPairsOffset + MatchPairs::offsetOfPairs());
2202
2203 Address pairsArrayAddress(FramePointer, pairsArrayOffset);
2204 Address firstMatchStartAddress(FramePointer,
2205 pairsArrayOffset + MatchPair::offsetOfStart());
2206
2207 // First, fill in a skeletal MatchPairs instance on the stack. This will be
2208 // passed to the OOL stub in the caller if we aren't able to execute the
2209 // RegExp inline, and that stub needs to be able to determine whether the
2210 // execution finished successfully.
2211
2212 // Initialize MatchPairs::pairCount to 1. The correct value can only
2213 // be determined after loading the RegExpShared. If the RegExpShared
2214 // has Kind::Atom, this is the correct pairCount.
2215 masm.store32(Imm32(1), pairCountAddress);
2216
2217 // Initialize MatchPairs::pairs pointer
2218 masm.computeEffectiveAddress(pairsArrayAddress, temp1);
2219 masm.storePtr(temp1, pairsPointerAddress);
2220
2221 // Initialize MatchPairs::pairs[0]::start to MatchPair::NoMatch
2222 masm.store32(Imm32(MatchPair::NoMatch), firstMatchStartAddress);
2223
2224 // Determine the set of volatile inputs to save when calling into C++ or
2225 // regexp code.
2226 LiveGeneralRegisterSet volatileRegs;
2227 if (lastIndex.volatile_()) {
2228 volatileRegs.add(lastIndex);
2229 }
2230 if (input.volatile_()) {
2231 volatileRegs.add(input);
2232 }
2233 if (regexp.volatile_()) {
2234 volatileRegs.add(regexp);
2235 }
2236
2237 // Ensure the input string is not a rope.
2238 Label isLinear;
2239 masm.branchIfNotRope(input, &isLinear);
2240 {
2241 masm.PushRegsInMask(volatileRegs);
2242
2243 using Fn = JSLinearString* (*)(JSString*);
2244 masm.setupUnalignedABICall(temp1);
2245 masm.passABIArg(input);
2246 masm.callWithABI<Fn, js::jit::LinearizeForCharAccessPure>();
2247
2248 MOZ_ASSERT(!volatileRegs.has(temp1))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!volatileRegs.has(temp1))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!volatileRegs.has(temp1)))),
0))) { do { } while (false); MOZ_ReportAssertionFailure("!volatileRegs.has(temp1)"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 2248); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!volatileRegs.has(temp1)"
")"); do { *((volatile int*)__null) = 2248; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
2249 masm.storeCallPointerResult(temp1);
2250 masm.PopRegsInMask(volatileRegs);
2251
2252 masm.branchTestPtr(Assembler::Zero, temp1, temp1, failure);
2253 }
2254 masm.bind(&isLinear);
2255
2256 // Load the RegExpShared.
2257 Register regexpReg = temp1;
2258 Address sharedSlot = Address(
2259 regexp, NativeObject::getFixedSlotOffset(RegExpObject::SHARED_SLOT));
2260 masm.branchTestUndefined(Assembler::Equal, sharedSlot, failure);
2261 masm.unboxNonDouble(sharedSlot, regexpReg, JSVAL_TYPE_PRIVATE_GCTHING);
2262
2263 // Handle Atom matches
2264 Label notAtom, checkSuccess;
2265 masm.branchPtr(Assembler::Equal,
2266 Address(regexpReg, RegExpShared::offsetOfPatternAtom()),
2267 ImmWord(0), &notAtom);
2268 {
2269 masm.computeEffectiveAddress(matchPairsAddress, temp3);
2270
2271 masm.PushRegsInMask(volatileRegs);
2272 using Fn =
2273 RegExpRunStatus (*)(RegExpShared* re, const JSLinearString* input,
2274 size_t start, MatchPairs* matchPairs);
2275 masm.setupUnalignedABICall(temp2);
2276 masm.passABIArg(regexpReg);
2277 masm.passABIArg(input);
2278 masm.passABIArg(lastIndex);
2279 masm.passABIArg(temp3);
2280 masm.callWithABI<Fn, js::ExecuteRegExpAtomRaw>();
2281
2282 MOZ_ASSERT(!volatileRegs.has(temp1))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!volatileRegs.has(temp1))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!volatileRegs.has(temp1)))),
0))) { do { } while (false); MOZ_ReportAssertionFailure("!volatileRegs.has(temp1)"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 2282); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!volatileRegs.has(temp1)"
")"); do { *((volatile int*)__null) = 2282; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
2283 masm.storeCallInt32Result(temp1);
2284 masm.PopRegsInMask(volatileRegs);
2285
2286 masm.jump(&checkSuccess);
2287 }
2288 masm.bind(&notAtom);
2289
2290 // Don't handle regexps with too many capture pairs.
2291 masm.load32(Address(regexpReg, RegExpShared::offsetOfPairCount()), temp2);
2292 masm.branch32(Assembler::Above, temp2, Imm32(RegExpObject::MaxPairCount),
2293 failure);
2294
2295 // Fill in the pair count in the MatchPairs on the stack.
2296 masm.store32(temp2, pairCountAddress);
2297
2298 // Load code pointer and length of input (in bytes).
2299 // Store the input start in the InputOutputData.
2300 Register codePointer = temp1; // Note: temp1 was previously regexpReg.
2301 Register byteLength = temp3;
2302 {
2303 Label isLatin1, done;
2304 masm.loadStringLength(input, byteLength);
2305
2306 masm.branchLatin1String(input, &isLatin1);
2307
2308 // Two-byte input
2309 masm.loadStringChars(input, temp2, CharEncoding::TwoByte);
2310 masm.storePtr(temp2, inputStartAddress);
2311 masm.loadPtr(
2312 Address(regexpReg, RegExpShared::offsetOfJitCode(/*latin1 =*/false)),
2313 codePointer);
2314 masm.lshiftPtr(Imm32(1), byteLength);
2315 masm.jump(&done);
2316
2317 // Latin1 input
2318 masm.bind(&isLatin1);
2319 masm.loadStringChars(input, temp2, CharEncoding::Latin1);
2320 masm.storePtr(temp2, inputStartAddress);
2321 masm.loadPtr(
2322 Address(regexpReg, RegExpShared::offsetOfJitCode(/*latin1 =*/true)),
2323 codePointer);
2324
2325 masm.bind(&done);
2326
2327 // Store end pointer
2328 masm.addPtr(byteLength, temp2);
2329 masm.storePtr(temp2, inputEndAddress);
2330 }
2331
2332 // Guard that the RegExpShared has been compiled for this type of input.
2333 // If it has not been compiled, we fall back to the OOL case, which will
2334 // do a VM call into the interpreter.
2335 // TODO: add an interpreter trampoline?
2336 masm.branchPtr(Assembler::Equal, codePointer, ImmWord(0), failure);
2337 masm.loadPtr(Address(codePointer, JitCode::offsetOfCode()), codePointer);
2338
2339 // Finish filling in the InputOutputData instance on the stack
2340 masm.computeEffectiveAddress(matchPairsAddress, temp2);
2341 masm.storePtr(temp2, matchesAddress);
2342 masm.storePtr(lastIndex, startIndexAddress);
2343
2344 // Execute the RegExp.
2345 masm.computeEffectiveAddress(
2346 Address(FramePointer, inputOutputDataStartOffset), temp2);
2347 masm.PushRegsInMask(volatileRegs);
2348 masm.setupUnalignedABICall(temp3);
2349 masm.passABIArg(temp2);
2350 masm.callWithABI(codePointer);
2351 masm.storeCallInt32Result(temp1);
2352 masm.PopRegsInMask(volatileRegs);
2353
2354 masm.bind(&checkSuccess);
2355 masm.branch32(Assembler::Equal, temp1,
2356 Imm32(int32_t(RegExpRunStatus::Success_NotFound)), notFound);
2357 masm.branch32(Assembler::Equal, temp1, Imm32(int32_t(RegExpRunStatus::Error)),
2358 failure);
2359
2360 // Lazily update the RegExpStatics.
2361 size_t offset = GlobalObjectData::offsetOfRegExpRealm() +
2362 RegExpRealm::offsetOfRegExpStatics();
2363 masm.loadGlobalObjectData(temp1);
2364 masm.loadPtr(Address(temp1, offset), temp1);
2365 UpdateRegExpStatics(masm, regexp, input, lastIndex, temp1, temp2, temp3,
2366 initialStringHeap, volatileRegs);
2367
2368 return true;
2369}
2370
2371static void EmitInitDependentStringBase(MacroAssembler& masm,
2372 Register dependent, Register base,
2373 Register temp1, Register temp2,
2374 bool needsPostBarrier) {
2375 // Determine the base string to use and store it in temp2.
2376 Label notDependent, markedDependedOn;
2377 masm.load32(Address(base, JSString::offsetOfFlags()), temp1);
2378 masm.branchTest32(Assembler::Zero, temp1, Imm32(JSString::DEPENDENT_BIT),
2379 &notDependent);
2380 {
2381 // The base is also a dependent string. Load its base to prevent chains of
2382 // dependent strings in most cases. This must either be an atom or already
2383 // have the DEPENDED_ON_BIT set.
2384 masm.loadDependentStringBase(base, temp2);
2385 masm.jump(&markedDependedOn);
2386 }
2387 masm.bind(&notDependent);
2388 {
2389 // The base is not a dependent string. Set the DEPENDED_ON_BIT if it's not
2390 // an atom.
2391 masm.movePtr(base, temp2);
2392 masm.branchTest32(Assembler::NonZero, temp1, Imm32(JSString::ATOM_BIT),
2393 &markedDependedOn);
2394 masm.or32(Imm32(JSString::DEPENDED_ON_BIT), temp1);
2395 masm.store32(temp1, Address(temp2, JSString::offsetOfFlags()));
2396 }
2397 masm.bind(&markedDependedOn);
2398
2399#ifdef DEBUG1
2400 // Assert the base has the DEPENDED_ON_BIT set or is an atom.
2401 Label isAppropriatelyMarked;
2402 masm.branchTest32(Assembler::NonZero,
2403 Address(temp2, JSString::offsetOfFlags()),
2404 Imm32(JSString::ATOM_BIT | JSString::DEPENDED_ON_BIT),
2405 &isAppropriatelyMarked);
2406 masm.assumeUnreachable("Base string is missing DEPENDED_ON_BIT");
2407 masm.bind(&isAppropriatelyMarked);
2408#endif
2409 masm.storeDependentStringBase(temp2, dependent);
2410
2411 // Post-barrier the base store. The base is still in temp2.
2412 if (needsPostBarrier) {
2413 Label done;
2414 masm.branchPtrInNurseryChunk(Assembler::Equal, dependent, temp1, &done);
2415 masm.branchPtrInNurseryChunk(Assembler::NotEqual, temp2, temp1, &done);
2416
2417 LiveRegisterSet regsToSave(RegisterSet::Volatile());
2418 regsToSave.takeUnchecked(temp1);
2419 regsToSave.takeUnchecked(temp2);
2420
2421 masm.PushRegsInMask(regsToSave);
2422
2423 masm.mov(ImmPtr(masm.runtime()), temp1);
2424
2425 using Fn = void (*)(JSRuntime* rt, js::gc::Cell* cell);
2426 masm.setupUnalignedABICall(temp2);
2427 masm.passABIArg(temp1);
2428 masm.passABIArg(dependent);
2429 masm.callWithABI<Fn, PostWriteBarrier>();
2430
2431 masm.PopRegsInMask(regsToSave);
2432
2433 masm.bind(&done);
2434 } else {
2435#ifdef DEBUG1
2436 Label done;
2437 masm.branchPtrInNurseryChunk(Assembler::Equal, dependent, temp1, &done);
2438 masm.branchPtrInNurseryChunk(Assembler::NotEqual, temp2, temp1, &done);
2439 masm.assumeUnreachable("Missing post barrier for dependent string base");
2440 masm.bind(&done);
2441#endif
2442 }
2443}
2444
2445static void CopyStringChars(MacroAssembler& masm, Register to, Register from,
2446 Register len, Register byteOpScratch,
2447 CharEncoding encoding,
2448 size_t maximumLength = SIZE_MAX(18446744073709551615UL));
2449
2450class CreateDependentString {
2451 CharEncoding encoding_;
2452 Register string_;
2453 Register temp1_;
2454 Register temp2_;
2455 Label* failure_;
2456
2457 enum class FallbackKind : uint8_t {
2458 InlineString,
2459 FatInlineString,
2460 NotInlineString,
2461 Count
2462 };
2463 mozilla::EnumeratedArray<FallbackKind, Label, size_t(FallbackKind::Count)>
2464 fallbacks_, joins_;
2465
2466 public:
2467 CreateDependentString(CharEncoding encoding, Register string, Register temp1,
2468 Register temp2, Label* failure)
2469 : encoding_(encoding),
2470 string_(string),
2471 temp1_(temp1),
2472 temp2_(temp2),
2473 failure_(failure) {}
2474
2475 Register string() const { return string_; }
2476 CharEncoding encoding() const { return encoding_; }
2477
2478 // Generate code that creates DependentString.
2479 // Caller should call generateFallback after masm.ret(), to generate
2480 // fallback path.
2481 void generate(MacroAssembler& masm, const JSAtomState& names,
2482 CompileRuntime* runtime, Register base,
2483 BaseIndex startIndexAddress, BaseIndex limitIndexAddress,
2484 gc::Heap initialStringHeap);
2485
2486 // Generate fallback path for creating DependentString.
2487 void generateFallback(MacroAssembler& masm);
2488};
2489
2490void CreateDependentString::generate(MacroAssembler& masm,
2491 const JSAtomState& names,
2492 CompileRuntime* runtime, Register base,
2493 BaseIndex startIndexAddress,
2494 BaseIndex limitIndexAddress,
2495 gc::Heap initialStringHeap) {
2496 JitSpew(JitSpew_Codegen, "# Emitting CreateDependentString (encoding=%s)",
2497 (encoding_ == CharEncoding::Latin1 ? "Latin-1" : "Two-Byte"));
2498
2499 auto newGCString = [&](FallbackKind kind) {
2500 uint32_t flags = kind == FallbackKind::InlineString
2501 ? JSString::INIT_THIN_INLINE_FLAGS
2502 : kind == FallbackKind::FatInlineString
2503 ? JSString::INIT_FAT_INLINE_FLAGS
2504 : JSString::INIT_DEPENDENT_FLAGS;
2505 if (encoding_ == CharEncoding::Latin1) {
2506 flags |= JSString::LATIN1_CHARS_BIT;
2507 }
2508
2509 if (kind != FallbackKind::FatInlineString) {
2510 masm.newGCString(string_, temp2_, initialStringHeap, &fallbacks_[kind]);
2511 } else {
2512 masm.newGCFatInlineString(string_, temp2_, initialStringHeap,
2513 &fallbacks_[kind]);
2514 }
2515 masm.bind(&joins_[kind]);
2516 masm.store32(Imm32(flags), Address(string_, JSString::offsetOfFlags()));
2517 };
2518
2519 // Compute the string length.
2520 masm.load32(startIndexAddress, temp2_);
2521 masm.load32(limitIndexAddress, temp1_);
2522 masm.sub32(temp2_, temp1_);
2523
2524 Label done, nonEmpty;
2525
2526 // Zero length matches use the empty string.
2527 masm.branchTest32(Assembler::NonZero, temp1_, temp1_, &nonEmpty);
2528 masm.movePtr(ImmGCPtr(names.empty_), string_);
2529 masm.jump(&done);
2530
2531 masm.bind(&nonEmpty);
2532
2533 // Complete matches use the base string.
2534 Label nonBaseStringMatch;
2535 masm.branchTest32(Assembler::NonZero, temp2_, temp2_, &nonBaseStringMatch);
2536 masm.branch32(Assembler::NotEqual, Address(base, JSString::offsetOfLength()),
2537 temp1_, &nonBaseStringMatch);
2538 masm.movePtr(base, string_);
2539 masm.jump(&done);
2540
2541 masm.bind(&nonBaseStringMatch);
2542
2543 Label notInline;
2544
2545 int32_t maxInlineLength = encoding_ == CharEncoding::Latin1
2546 ? JSFatInlineString::MAX_LENGTH_LATIN1
2547 : JSFatInlineString::MAX_LENGTH_TWO_BYTE;
2548 masm.branch32(Assembler::Above, temp1_, Imm32(maxInlineLength), &notInline);
2549 {
2550 // Make a thin or fat inline string.
2551 Label stringAllocated, fatInline;
2552
2553 int32_t maxThinInlineLength = encoding_ == CharEncoding::Latin1
2554 ? JSThinInlineString::MAX_LENGTH_LATIN1
2555 : JSThinInlineString::MAX_LENGTH_TWO_BYTE;
2556 masm.branch32(Assembler::Above, temp1_, Imm32(maxThinInlineLength),
2557 &fatInline);
2558 if (encoding_ == CharEncoding::Latin1) {
2559 // One character Latin-1 strings can be loaded directly from the
2560 // static strings table.
2561 Label thinInline;
2562 masm.branch32(Assembler::Above, temp1_, Imm32(1), &thinInline);
2563 {
2564 static_assert(
2565 StaticStrings::UNIT_STATIC_LIMIT - 1 == JSString::MAX_LATIN1_CHAR,
2566 "Latin-1 strings can be loaded from static strings");
2567
2568 masm.loadStringChars(base, temp1_, encoding_);
2569 masm.loadChar(temp1_, temp2_, temp1_, encoding_);
2570
2571 masm.lookupStaticString(temp1_, string_, runtime->staticStrings());
2572
2573 masm.jump(&done);
2574 }
2575 masm.bind(&thinInline);
2576 }
2577 {
2578 newGCString(FallbackKind::InlineString);
2579 masm.jump(&stringAllocated);
2580 }
2581 masm.bind(&fatInline);
2582 { newGCString(FallbackKind::FatInlineString); }
2583 masm.bind(&stringAllocated);
2584
2585 masm.store32(temp1_, Address(string_, JSString::offsetOfLength()));
2586
2587 masm.push(string_);
2588 masm.push(base);
2589
2590 MOZ_ASSERT(startIndexAddress.base == FramePointer,do { static_assert( mozilla::detail::AssertionConditionType<
decltype(startIndexAddress.base == FramePointer)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(startIndexAddress.base == FramePointer))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("startIndexAddress.base == FramePointer"
" (" "startIndexAddress is still valid after stack pushes" ")"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 2591); AnnotateMozCrashReason("MOZ_ASSERT" "(" "startIndexAddress.base == FramePointer"
") (" "startIndexAddress is still valid after stack pushes" ")"
); do { *((volatile int*)__null) = 2591; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
2591 "startIndexAddress is still valid after stack pushes")do { static_assert( mozilla::detail::AssertionConditionType<
decltype(startIndexAddress.base == FramePointer)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(startIndexAddress.base == FramePointer))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("startIndexAddress.base == FramePointer"
" (" "startIndexAddress is still valid after stack pushes" ")"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 2591); AnnotateMozCrashReason("MOZ_ASSERT" "(" "startIndexAddress.base == FramePointer"
") (" "startIndexAddress is still valid after stack pushes" ")"
); do { *((volatile int*)__null) = 2591; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
2592
2593 // Load chars pointer for the new string.
2594 masm.loadInlineStringCharsForStore(string_, string_);
2595
2596 // Load the source characters pointer.
2597 masm.loadStringChars(base, temp2_, encoding_);
2598 masm.load32(startIndexAddress, base);
2599 masm.addToCharPtr(temp2_, base, encoding_);
2600
2601 CopyStringChars(masm, string_, temp2_, temp1_, base, encoding_);
2602
2603 masm.pop(base);
2604 masm.pop(string_);
2605
2606 masm.jump(&done);
2607 }
2608
2609 masm.bind(&notInline);
2610
2611 {
2612 // Make a dependent string.
2613 // Warning: string may be tenured (if the fallback case is hit), so
2614 // stores into it must be post barriered.
2615 newGCString(FallbackKind::NotInlineString);
2616
2617 masm.store32(temp1_, Address(string_, JSString::offsetOfLength()));
2618
2619 masm.loadNonInlineStringChars(base, temp1_, encoding_);
2620 masm.load32(startIndexAddress, temp2_);
2621 masm.addToCharPtr(temp1_, temp2_, encoding_);
2622 masm.storeNonInlineStringChars(temp1_, string_);
2623
2624 EmitInitDependentStringBase(masm, string_, base, temp1_, temp2_,
2625 /* needsPostBarrier = */ true);
2626 }
2627
2628 masm.bind(&done);
2629}
2630
2631void CreateDependentString::generateFallback(MacroAssembler& masm) {
2632 JitSpew(JitSpew_Codegen,
2633 "# Emitting CreateDependentString fallback (encoding=%s)",
2634 (encoding_ == CharEncoding::Latin1 ? "Latin-1" : "Two-Byte"));
2635
2636 LiveRegisterSet regsToSave(RegisterSet::Volatile());
2637 regsToSave.takeUnchecked(string_);
2638 regsToSave.takeUnchecked(temp2_);
2639
2640 for (FallbackKind kind : mozilla::MakeEnumeratedRange(FallbackKind::Count)) {
2641 masm.bind(&fallbacks_[kind]);
2642
2643 masm.PushRegsInMask(regsToSave);
2644
2645 using Fn = void* (*)(JSContext* cx);
2646 masm.setupUnalignedABICall(string_);
2647 masm.loadJSContext(string_);
2648 masm.passABIArg(string_);
2649 if (kind == FallbackKind::FatInlineString) {
2650 masm.callWithABI<Fn, AllocateFatInlineString>();
2651 } else {
2652 masm.callWithABI<Fn, AllocateDependentString>();
2653 }
2654 masm.storeCallPointerResult(string_);
2655
2656 masm.PopRegsInMask(regsToSave);
2657
2658 masm.branchPtr(Assembler::Equal, string_, ImmWord(0), failure_);
2659
2660 masm.jump(&joins_[kind]);
2661 }
2662}
2663
2664// Generate the RegExpMatcher and RegExpExecMatch stubs. These are very similar,
2665// but RegExpExecMatch also has to load and update .lastIndex for global/sticky
2666// regular expressions.
2667static JitCode* GenerateRegExpMatchStubShared(JSContext* cx,
2668 gc::Heap initialStringHeap,
2669 bool isExecMatch) {
2670 if (isExecMatch) {
2671 JitSpew(JitSpew_Codegen, "# Emitting RegExpExecMatch stub");
2672 } else {
2673 JitSpew(JitSpew_Codegen, "# Emitting RegExpMatcher stub");
2674 }
2675
2676 // |initialStringHeap| could be stale after a GC.
2677 JS::AutoCheckCannotGC nogc(cx);
2678
2679 Register regexp = RegExpMatcherRegExpReg;
2680 Register input = RegExpMatcherStringReg;
2681 Register lastIndex = RegExpMatcherLastIndexReg;
2682 ValueOperand result = JSReturnOperand;
2683
2684 // We are free to clobber all registers, as LRegExpMatcher is a call
2685 // instruction.
2686 AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
2687 regs.take(input);
2688 regs.take(regexp);
2689 regs.take(lastIndex);
2690
2691 Register temp1 = regs.takeAny();
2692 Register temp2 = regs.takeAny();
2693 Register temp3 = regs.takeAny();
2694 Register maybeTemp4 = InvalidReg;
2695 if (!regs.empty()) {
2696 // There are not enough registers on x86.
2697 maybeTemp4 = regs.takeAny();
2698 }
2699 Register maybeTemp5 = InvalidReg;
2700 if (!regs.empty()) {
2701 // There are not enough registers on x86.
2702 maybeTemp5 = regs.takeAny();
2703 }
2704
2705 Address flagsSlot(regexp, RegExpObject::offsetOfFlags());
2706 Address lastIndexSlot(regexp, RegExpObject::offsetOfLastIndex());
2707
2708 TempAllocator temp(&cx->tempLifoAlloc());
2709 JitContext jcx(cx);
2710 StackMacroAssembler masm(cx, temp);
2711 AutoCreatedBy acb(masm, "GenerateRegExpMatchStubShared");
2712
2713#ifdef JS_USE_LINK_REGISTER
2714 masm.pushReturnAddress();
2715#endif
2716 masm.push(FramePointer);
2717 masm.moveStackPtrTo(FramePointer);
2718
2719 Label notFoundZeroLastIndex;
2720 if (isExecMatch) {
2721 masm.loadRegExpLastIndex(regexp, input, lastIndex, &notFoundZeroLastIndex);
2722 }
2723
2724 // The InputOutputData is placed above the frame pointer and return address on
2725 // the stack.
2726 int32_t inputOutputDataStartOffset = 2 * sizeof(void*);
2727
2728 Label notFound, oolEntry;
2729 if (!PrepareAndExecuteRegExp(masm, regexp, input, lastIndex, temp1, temp2,
2730 temp3, inputOutputDataStartOffset,
2731 initialStringHeap, &notFound, &oolEntry)) {
2732 return nullptr;
2733 }
2734
2735 // If a regexp has named captures, fall back to the OOL stub, which
2736 // will end up calling CreateRegExpMatchResults.
2737 Register shared = temp2;
2738 masm.unboxNonDouble(Address(regexp, NativeObject::getFixedSlotOffset(
2739 RegExpObject::SHARED_SLOT)),
2740 shared, JSVAL_TYPE_PRIVATE_GCTHING);
2741 masm.branchPtr(Assembler::NotEqual,
2742 Address(shared, RegExpShared::offsetOfGroupsTemplate()),
2743 ImmWord(0), &oolEntry);
2744
2745 // Similarly, if the |hasIndices| flag is set, fall back to the OOL stub.
2746 masm.branchTest32(Assembler::NonZero,
2747 Address(shared, RegExpShared::offsetOfFlags()),
2748 Imm32(int32_t(JS::RegExpFlag::HasIndices)), &oolEntry);
2749
2750 Address pairCountAddress =
2751 RegExpPairCountAddress(masm, inputOutputDataStartOffset);
2752
2753 // Construct the result.
2754 Register object = temp1;
2755 {
2756 // In most cases, the array will have just 1-2 elements, so we optimize for
2757 // that by emitting separate code paths for capacity 2/6/14 (= 4/8/16 slots
2758 // because two slots are used for the elements header).
2759
2760 // Load the array length in temp2 and the shape in temp3.
2761 Label allocated;
2762 masm.load32(pairCountAddress, temp2);
2763 size_t offset = GlobalObjectData::offsetOfRegExpRealm() +
2764 RegExpRealm::offsetOfNormalMatchResultShape();
2765 masm.loadGlobalObjectData(temp3);
2766 masm.loadPtr(Address(temp3, offset), temp3);
2767
2768 auto emitAllocObject = [&](size_t elementCapacity) {
2769 gc::AllocKind kind = GuessArrayGCKind(elementCapacity);
2770 MOZ_ASSERT(CanChangeToBackgroundAllocKind(kind, &ArrayObject::class_))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(CanChangeToBackgroundAllocKind(kind, &ArrayObject
::class_))>::isValid, "invalid assertion condition"); if (
(__builtin_expect(!!(!(!!(CanChangeToBackgroundAllocKind(kind
, &ArrayObject::class_)))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("CanChangeToBackgroundAllocKind(kind, &ArrayObject::class_)"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 2770); AnnotateMozCrashReason("MOZ_ASSERT" "(" "CanChangeToBackgroundAllocKind(kind, &ArrayObject::class_)"
")"); do { *((volatile int*)__null) = 2770; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
2771 kind = ForegroundToBackgroundAllocKind(kind);
2772
2773#ifdef DEBUG1
2774 // Assert all of the available slots are used for |elementCapacity|
2775 // elements.
2776 size_t usedSlots = ObjectElements::VALUES_PER_HEADER + elementCapacity;
2777 MOZ_ASSERT(usedSlots == GetGCKindSlots(kind))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(usedSlots == GetGCKindSlots(kind))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(usedSlots == GetGCKindSlots(
kind)))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("usedSlots == GetGCKindSlots(kind)", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 2777); AnnotateMozCrashReason("MOZ_ASSERT" "(" "usedSlots == GetGCKindSlots(kind)"
")"); do { *((volatile int*)__null) = 2777; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
2778#endif
2779
2780 constexpr size_t numUsedDynamicSlots =
2781 RegExpRealm::MatchResultObjectSlotSpan;
2782 constexpr size_t numDynamicSlots =
2783 RegExpRealm::MatchResultObjectNumDynamicSlots;
2784 constexpr size_t arrayLength = 1;
2785 masm.createArrayWithFixedElements(object, temp3, temp2, temp3,
2786 arrayLength, elementCapacity,
2787 numUsedDynamicSlots, numDynamicSlots,
2788 kind, gc::Heap::Default, &oolEntry);
2789 };
2790
2791 Label moreThan2;
2792 masm.branch32(Assembler::Above, temp2, Imm32(2), &moreThan2);
2793 emitAllocObject(2);
2794 masm.jump(&allocated);
2795
2796 Label moreThan6;
2797 masm.bind(&moreThan2);
2798 masm.branch32(Assembler::Above, temp2, Imm32(6), &moreThan6);
2799 emitAllocObject(6);
2800 masm.jump(&allocated);
2801
2802 masm.bind(&moreThan6);
2803 static_assert(RegExpObject::MaxPairCount == 14);
2804 emitAllocObject(RegExpObject::MaxPairCount);
2805
2806 masm.bind(&allocated);
2807 }
2808
2809 // clang-format off
2810 /*
2811 * [SMDOC] Stack layout for the RegExpMatcher stub
2812 *
2813 * +---------------+
2814 * FramePointer +-----> |Caller-FramePtr|
2815 * +---------------+
2816 * |Return-Address |
2817 * +---------------+
2818 * inputOutputDataStartOffset +-----> +---------------+
2819 * |InputOutputData|
2820 * +---------------+
2821 * +---------------+
2822 * | MatchPairs |
2823 * pairsCountAddress +-----------> count |
2824 * | pairs |
2825 * | |
2826 * +---------------+
2827 * pairsVectorStartOffset +-----> +---------------+
2828 * | MatchPair |
2829 * matchPairStart +------------> start | <-------+
2830 * matchPairLimit +------------> limit | | Reserved space for
2831 * +---------------+ | `RegExpObject::MaxPairCount`
2832 * . | MatchPair objects.
2833 * . |
2834 * . | `count` objects will be
2835 * +---------------+ | initialized and can be
2836 * | MatchPair | | accessed below.
2837 * | start | <-------+
2838 * | limit |
2839 * +---------------+
2840 */
2841 // clang-format on
2842
2843 static_assert(sizeof(MatchPair) == 2 * sizeof(int32_t),
2844 "MatchPair consists of two int32 values representing the start"
2845 "and the end offset of the match");
2846
2847 int32_t pairsVectorStartOffset =
2848 RegExpPairsVectorStartOffset(inputOutputDataStartOffset);
2849
2850 // Incremented by one below for each match pair.
2851 Register matchIndex = temp2;
2852 masm.move32(Imm32(0), matchIndex);
2853
2854 // The element in which to store the result of the current match.
2855 size_t elementsOffset = NativeObject::offsetOfFixedElements();
2856 BaseObjectElementIndex objectMatchElement(object, matchIndex, elementsOffset);
2857
2858 // The current match pair's "start" and "limit" member.
2859 BaseIndex matchPairStart(FramePointer, matchIndex, TimesEight,
2860 pairsVectorStartOffset + MatchPair::offsetOfStart());
2861 BaseIndex matchPairLimit(FramePointer, matchIndex, TimesEight,
2862 pairsVectorStartOffset + MatchPair::offsetOfLimit());
2863
2864 Label* depStrFailure = &oolEntry;
2865 Label restoreRegExpAndLastIndex;
2866
2867 Register temp4;
2868 if (maybeTemp4 == InvalidReg) {
2869 depStrFailure = &restoreRegExpAndLastIndex;
2870
2871 // We don't have enough registers for a fourth temporary. Reuse |regexp|
2872 // as a temporary. We restore its value at |restoreRegExpAndLastIndex|.
2873 masm.push(regexp);
2874 temp4 = regexp;
2875 } else {
2876 temp4 = maybeTemp4;
2877 }
2878
2879 Register temp5;
2880 if (maybeTemp5 == InvalidReg) {
2881 depStrFailure = &restoreRegExpAndLastIndex;
2882
2883 // We don't have enough registers for a fifth temporary. Reuse |lastIndex|
2884 // as a temporary. We restore its value at |restoreRegExpAndLastIndex|.
2885 masm.push(lastIndex);
2886 temp5 = lastIndex;
2887 } else {
2888 temp5 = maybeTemp5;
2889 }
2890
2891 auto maybeRestoreRegExpAndLastIndex = [&]() {
2892 if (maybeTemp5 == InvalidReg) {
2893 masm.pop(lastIndex);
2894 }
2895 if (maybeTemp4 == InvalidReg) {
2896 masm.pop(regexp);
2897 }
2898 };
2899
2900 // Loop to construct the match strings. There are two different loops,
2901 // depending on whether the input is a Two-Byte or a Latin-1 string.
2902 CreateDependentString depStrs[]{
2903 {CharEncoding::TwoByte, temp3, temp4, temp5, depStrFailure},
2904 {CharEncoding::Latin1, temp3, temp4, temp5, depStrFailure},
2905 };
2906
2907 {
2908 Label isLatin1, done;
2909 masm.branchLatin1String(input, &isLatin1);
2910
2911 for (auto& depStr : depStrs) {
2912 if (depStr.encoding() == CharEncoding::Latin1) {
2913 masm.bind(&isLatin1);
2914 }
2915
2916 Label matchLoop;
2917 masm.bind(&matchLoop);
2918
2919 static_assert(MatchPair::NoMatch == -1,
2920 "MatchPair::start is negative if no match was found");
2921
2922 Label isUndefined, storeDone;
2923 masm.branch32(Assembler::LessThan, matchPairStart, Imm32(0),
2924 &isUndefined);
2925 {
2926 depStr.generate(masm, cx->names(), CompileRuntime::get(cx->runtime()),
2927 input, matchPairStart, matchPairLimit,
2928 initialStringHeap);
2929
2930 // Storing into nursery-allocated results object's elements; no post
2931 // barrier.
2932 masm.storeValue(JSVAL_TYPE_STRING, depStr.string(), objectMatchElement);
2933 masm.jump(&storeDone);
2934 }
2935 masm.bind(&isUndefined);
2936 { masm.storeValue(UndefinedValue(), objectMatchElement); }
2937 masm.bind(&storeDone);
2938
2939 masm.add32(Imm32(1), matchIndex);
2940 masm.branch32(Assembler::LessThanOrEqual, pairCountAddress, matchIndex,
2941 &done);
2942 masm.jump(&matchLoop);
2943 }
2944
2945#ifdef DEBUG1
2946 masm.assumeUnreachable("The match string loop doesn't fall through.");
2947#endif
2948
2949 masm.bind(&done);
2950 }
2951
2952 maybeRestoreRegExpAndLastIndex();
2953
2954 // Fill in the rest of the output object.
2955 masm.store32(
2956 matchIndex,
2957 Address(object,
2958 elementsOffset + ObjectElements::offsetOfInitializedLength()));
2959 masm.store32(
2960 matchIndex,
2961 Address(object, elementsOffset + ObjectElements::offsetOfLength()));
2962
2963 Address firstMatchPairStartAddress(
2964 FramePointer, pairsVectorStartOffset + MatchPair::offsetOfStart());
2965 Address firstMatchPairLimitAddress(
2966 FramePointer, pairsVectorStartOffset + MatchPair::offsetOfLimit());
2967
2968 static_assert(RegExpRealm::MatchResultObjectIndexSlot == 0,
2969 "First slot holds the 'index' property");
2970 static_assert(RegExpRealm::MatchResultObjectInputSlot == 1,
2971 "Second slot holds the 'input' property");
2972
2973 masm.loadPtr(Address(object, NativeObject::offsetOfSlots()), temp2);
2974
2975 masm.load32(firstMatchPairStartAddress, temp3);
2976 masm.storeValue(JSVAL_TYPE_INT32, temp3, Address(temp2, 0));
2977
2978 // No post barrier needed (address is within nursery object.)
2979 masm.storeValue(JSVAL_TYPE_STRING, input, Address(temp2, sizeof(Value)));
2980
2981 // For the ExecMatch stub, if the regular expression is global or sticky, we
2982 // have to update its .lastIndex slot.
2983 if (isExecMatch) {
2984 MOZ_ASSERT(object != lastIndex)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(object != lastIndex)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(object != lastIndex))), 0)))
{ do { } while (false); MOZ_ReportAssertionFailure("object != lastIndex"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 2984); AnnotateMozCrashReason("MOZ_ASSERT" "(" "object != lastIndex"
")"); do { *((volatile int*)__null) = 2984; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
2985 Label notGlobalOrSticky;
2986 masm.branchTest32(Assembler::Zero, flagsSlot,
2987 Imm32(JS::RegExpFlag::Global | JS::RegExpFlag::Sticky),
2988 &notGlobalOrSticky);
2989 masm.load32(firstMatchPairLimitAddress, lastIndex);
2990 masm.storeValue(JSVAL_TYPE_INT32, lastIndex, lastIndexSlot);
2991 masm.bind(&notGlobalOrSticky);
2992 }
2993
2994 // All done!
2995 masm.tagValue(JSVAL_TYPE_OBJECT, object, result);
2996 masm.pop(FramePointer);
2997 masm.ret();
2998
2999 masm.bind(&notFound);
3000 if (isExecMatch) {
3001 Label notGlobalOrSticky;
3002 masm.branchTest32(Assembler::Zero, flagsSlot,
3003 Imm32(JS::RegExpFlag::Global | JS::RegExpFlag::Sticky),
3004 &notGlobalOrSticky);
3005 masm.bind(&notFoundZeroLastIndex);
3006 masm.storeValue(Int32Value(0), lastIndexSlot);
3007 masm.bind(&notGlobalOrSticky);
3008 }
3009 masm.moveValue(NullValue(), result);
3010 masm.pop(FramePointer);
3011 masm.ret();
3012
3013 // Fallback paths for CreateDependentString.
3014 for (auto& depStr : depStrs) {
3015 depStr.generateFallback(masm);
3016 }
3017
3018 // Fall-through to the ool entry after restoring the registers.
3019 masm.bind(&restoreRegExpAndLastIndex);
3020 maybeRestoreRegExpAndLastIndex();
3021
3022 // Use an undefined value to signal to the caller that the OOL stub needs to
3023 // be called.
3024 masm.bind(&oolEntry);
3025 masm.moveValue(UndefinedValue(), result);
3026 masm.pop(FramePointer);
3027 masm.ret();
3028
3029 Linker linker(masm);
3030 JitCode* code = linker.newCode(cx, CodeKind::Other);
3031 if (!code) {
3032 return nullptr;
3033 }
3034
3035 const char* name = isExecMatch ? "RegExpExecMatchStub" : "RegExpMatcherStub";
3036 CollectPerfSpewerJitCodeProfile(code, name);
3037#ifdef MOZ_VTUNE1
3038 vtune::MarkStub(code, name);
3039#endif
3040
3041 return code;
3042}
3043
3044JitCode* JitZone::generateRegExpMatcherStub(JSContext* cx) {
3045 return GenerateRegExpMatchStubShared(cx, initialStringHeap,
3046 /* isExecMatch = */ false);
3047}
3048
3049JitCode* JitZone::generateRegExpExecMatchStub(JSContext* cx) {
3050 return GenerateRegExpMatchStubShared(cx, initialStringHeap,
3051 /* isExecMatch = */ true);
3052}
3053
3054class OutOfLineRegExpMatcher : public OutOfLineCodeBase<CodeGenerator> {
3055 LRegExpMatcher* lir_;
3056
3057 public:
3058 explicit OutOfLineRegExpMatcher(LRegExpMatcher* lir) : lir_(lir) {}
3059
3060 void accept(CodeGenerator* codegen) override {
3061 codegen->visitOutOfLineRegExpMatcher(this);
3062 }
3063
3064 LRegExpMatcher* lir() const { return lir_; }
3065};
3066
3067void CodeGenerator::visitOutOfLineRegExpMatcher(OutOfLineRegExpMatcher* ool) {
3068 LRegExpMatcher* lir = ool->lir();
3069 Register lastIndex = ToRegister(lir->lastIndex());
3070 Register input = ToRegister(lir->string());
3071 Register regexp = ToRegister(lir->regexp());
3072
3073 AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
3074 regs.take(lastIndex);
3075 regs.take(input);
3076 regs.take(regexp);
3077 Register temp = regs.takeAny();
3078
3079 masm.computeEffectiveAddress(
3080 Address(masm.getStackPointer(), InputOutputDataSize), temp);
3081
3082 pushArg(temp);
3083 pushArg(lastIndex);
3084 pushArg(input);
3085 pushArg(regexp);
3086
3087 // We are not using oolCallVM because we are in a Call, and that live
3088 // registers are already saved by the the register allocator.
3089 using Fn =
3090 bool (*)(JSContext*, HandleObject regexp, HandleString input,
3091 int32_t lastIndex, MatchPairs* pairs, MutableHandleValue output);
3092 callVM<Fn, RegExpMatcherRaw>(lir);
3093
3094 masm.jump(ool->rejoin());
3095}
3096
3097void CodeGenerator::visitRegExpMatcher(LRegExpMatcher* lir) {
3098 MOZ_ASSERT(ToRegister(lir->regexp()) == RegExpMatcherRegExpReg)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ToRegister(lir->regexp()) == RegExpMatcherRegExpReg
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(ToRegister(lir->regexp()) == RegExpMatcherRegExpReg
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"ToRegister(lir->regexp()) == RegExpMatcherRegExpReg", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 3098); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ToRegister(lir->regexp()) == RegExpMatcherRegExpReg"
")"); do { *((volatile int*)__null) = 3098; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
3099 MOZ_ASSERT(ToRegister(lir->string()) == RegExpMatcherStringReg)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ToRegister(lir->string()) == RegExpMatcherStringReg
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(ToRegister(lir->string()) == RegExpMatcherStringReg
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"ToRegister(lir->string()) == RegExpMatcherStringReg", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 3099); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ToRegister(lir->string()) == RegExpMatcherStringReg"
")"); do { *((volatile int*)__null) = 3099; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
3100 MOZ_ASSERT(ToRegister(lir->lastIndex()) == RegExpMatcherLastIndexReg)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ToRegister(lir->lastIndex()) == RegExpMatcherLastIndexReg
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(ToRegister(lir->lastIndex()) == RegExpMatcherLastIndexReg
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"ToRegister(lir->lastIndex()) == RegExpMatcherLastIndexReg"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 3100); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ToRegister(lir->lastIndex()) == RegExpMatcherLastIndexReg"
")"); do { *((volatile int*)__null) = 3100; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
3101 MOZ_ASSERT(ToOutValue(lir) == JSReturnOperand)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ToOutValue(lir) == JSReturnOperand)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(ToOutValue(lir) == JSReturnOperand
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"ToOutValue(lir) == JSReturnOperand", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 3101); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ToOutValue(lir) == JSReturnOperand"
")"); do { *((volatile int*)__null) = 3101; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
3102
3103#if defined(JS_NUNBOX32)
3104 static_assert(RegExpMatcherRegExpReg != JSReturnReg_Type);
3105 static_assert(RegExpMatcherRegExpReg != JSReturnReg_Data);
3106 static_assert(RegExpMatcherStringReg != JSReturnReg_Type);
3107 static_assert(RegExpMatcherStringReg != JSReturnReg_Data);
3108 static_assert(RegExpMatcherLastIndexReg != JSReturnReg_Type);
3109 static_assert(RegExpMatcherLastIndexReg != JSReturnReg_Data);
3110#elif defined(JS_PUNBOX641)
3111 static_assert(RegExpMatcherRegExpReg != JSReturnReg);
3112 static_assert(RegExpMatcherStringReg != JSReturnReg);
3113 static_assert(RegExpMatcherLastIndexReg != JSReturnReg);
3114#endif
3115
3116 masm.reserveStack(RegExpReservedStack);
3117
3118 OutOfLineRegExpMatcher* ool = new (alloc()) OutOfLineRegExpMatcher(lir);
3119 addOutOfLineCode(ool, lir->mir());
3120
3121 const JitZone* jitZone = gen->realm->zone()->jitZone();
3122 JitCode* regExpMatcherStub =
3123 jitZone->regExpMatcherStubNoBarrier(&zoneStubsToReadBarrier_);
3124 masm.call(regExpMatcherStub);
3125 masm.branchTestUndefined(Assembler::Equal, JSReturnOperand, ool->entry());
3126 masm.bind(ool->rejoin());
3127
3128 masm.freeStack(RegExpReservedStack);
3129}
3130
3131class OutOfLineRegExpExecMatch : public OutOfLineCodeBase<CodeGenerator> {
3132 LRegExpExecMatch* lir_;
3133
3134 public:
3135 explicit OutOfLineRegExpExecMatch(LRegExpExecMatch* lir) : lir_(lir) {}
3136
3137 void accept(CodeGenerator* codegen) override {
3138 codegen->visitOutOfLineRegExpExecMatch(this);
3139 }
3140
3141 LRegExpExecMatch* lir() const { return lir_; }
3142};
3143
3144void CodeGenerator::visitOutOfLineRegExpExecMatch(
3145 OutOfLineRegExpExecMatch* ool) {
3146 LRegExpExecMatch* lir = ool->lir();
3147 Register input = ToRegister(lir->string());
3148 Register regexp = ToRegister(lir->regexp());
3149
3150 AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
3151 regs.take(input);
3152 regs.take(regexp);
3153 Register temp = regs.takeAny();
3154
3155 masm.computeEffectiveAddress(
3156 Address(masm.getStackPointer(), InputOutputDataSize), temp);
3157
3158 pushArg(temp);
3159 pushArg(input);
3160 pushArg(regexp);
3161
3162 // We are not using oolCallVM because we are in a Call and live registers have
3163 // already been saved by the register allocator.
3164 using Fn =
3165 bool (*)(JSContext*, Handle<RegExpObject*> regexp, HandleString input,
3166 MatchPairs* pairs, MutableHandleValue output);
3167 callVM<Fn, RegExpBuiltinExecMatchFromJit>(lir);
3168 masm.jump(ool->rejoin());
3169}
3170
3171void CodeGenerator::visitRegExpExecMatch(LRegExpExecMatch* lir) {
3172 MOZ_ASSERT(ToRegister(lir->regexp()) == RegExpMatcherRegExpReg)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ToRegister(lir->regexp()) == RegExpMatcherRegExpReg
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(ToRegister(lir->regexp()) == RegExpMatcherRegExpReg
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"ToRegister(lir->regexp()) == RegExpMatcherRegExpReg", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 3172); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ToRegister(lir->regexp()) == RegExpMatcherRegExpReg"
")"); do { *((volatile int*)__null) = 3172; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
3173 MOZ_ASSERT(ToRegister(lir->string()) == RegExpMatcherStringReg)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ToRegister(lir->string()) == RegExpMatcherStringReg
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(ToRegister(lir->string()) == RegExpMatcherStringReg
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"ToRegister(lir->string()) == RegExpMatcherStringReg", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 3173); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ToRegister(lir->string()) == RegExpMatcherStringReg"
")"); do { *((volatile int*)__null) = 3173; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
3174 MOZ_ASSERT(ToOutValue(lir) == JSReturnOperand)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ToOutValue(lir) == JSReturnOperand)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(ToOutValue(lir) == JSReturnOperand
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"ToOutValue(lir) == JSReturnOperand", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 3174); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ToOutValue(lir) == JSReturnOperand"
")"); do { *((volatile int*)__null) = 3174; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
3175
3176#if defined(JS_NUNBOX32)
3177 static_assert(RegExpMatcherRegExpReg != JSReturnReg_Type);
3178 static_assert(RegExpMatcherRegExpReg != JSReturnReg_Data);
3179 static_assert(RegExpMatcherStringReg != JSReturnReg_Type);
3180 static_assert(RegExpMatcherStringReg != JSReturnReg_Data);
3181#elif defined(JS_PUNBOX641)
3182 static_assert(RegExpMatcherRegExpReg != JSReturnReg);
3183 static_assert(RegExpMatcherStringReg != JSReturnReg);
3184#endif
3185
3186 masm.reserveStack(RegExpReservedStack);
3187
3188 auto* ool = new (alloc()) OutOfLineRegExpExecMatch(lir);
3189 addOutOfLineCode(ool, lir->mir());
3190
3191 const JitZone* jitZone = gen->realm->zone()->jitZone();
3192 JitCode* regExpExecMatchStub =
3193 jitZone->regExpExecMatchStubNoBarrier(&zoneStubsToReadBarrier_);
3194 masm.call(regExpExecMatchStub);
3195 masm.branchTestUndefined(Assembler::Equal, JSReturnOperand, ool->entry());
3196
3197 masm.bind(ool->rejoin());
3198 masm.freeStack(RegExpReservedStack);
3199}
3200
3201JitCode* JitZone::generateRegExpSearcherStub(JSContext* cx) {
3202 JitSpew(JitSpew_Codegen, "# Emitting RegExpSearcher stub");
3203
3204 Register regexp = RegExpSearcherRegExpReg;
3205 Register input = RegExpSearcherStringReg;
3206 Register lastIndex = RegExpSearcherLastIndexReg;
3207 Register result = ReturnReg;
3208
3209 // We are free to clobber all registers, as LRegExpSearcher is a call
3210 // instruction.
3211 AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
3212 regs.take(input);
3213 regs.take(regexp);
3214 regs.take(lastIndex);
3215
3216 Register temp1 = regs.takeAny();
3217 Register temp2 = regs.takeAny();
3218 Register temp3 = regs.takeAny();
3219
3220 TempAllocator temp(&cx->tempLifoAlloc());
3221 JitContext jcx(cx);
3222 StackMacroAssembler masm(cx, temp);
3223 AutoCreatedBy acb(masm, "JitZone::generateRegExpSearcherStub");
3224
3225#ifdef JS_USE_LINK_REGISTER
3226 masm.pushReturnAddress();
3227#endif
3228 masm.push(FramePointer);
3229 masm.moveStackPtrTo(FramePointer);
3230
3231#ifdef DEBUG1
3232 // Store sentinel value to cx->regExpSearcherLastLimit.
3233 // See comment in RegExpSearcherImpl.
3234 masm.loadJSContext(temp1);
3235 masm.store32(Imm32(RegExpSearcherLastLimitSentinel),
3236 Address(temp1, JSContext::offsetOfRegExpSearcherLastLimit()));
3237#endif
3238
3239 // The InputOutputData is placed above the frame pointer and return address on
3240 // the stack.
3241 int32_t inputOutputDataStartOffset = 2 * sizeof(void*);
3242
3243 Label notFound, oolEntry;
3244 if (!PrepareAndExecuteRegExp(masm, regexp, input, lastIndex, temp1, temp2,
3245 temp3, inputOutputDataStartOffset,
3246 initialStringHeap, &notFound, &oolEntry)) {
3247 return nullptr;
3248 }
3249
3250 // clang-format off
3251 /*
3252 * [SMDOC] Stack layout for the RegExpSearcher stub
3253 *
3254 * +---------------+
3255 * FramePointer +-----> |Caller-FramePtr|
3256 * +---------------+
3257 * |Return-Address |
3258 * +---------------+
3259 * inputOutputDataStartOffset +-----> +---------------+
3260 * |InputOutputData|
3261 * +---------------+
3262 * +---------------+
3263 * | MatchPairs |
3264 * | count |
3265 * | pairs |
3266 * | |
3267 * +---------------+
3268 * pairsVectorStartOffset +-----> +---------------+
3269 * | MatchPair |
3270 * matchPairStart +------------> start | <-------+
3271 * matchPairLimit +------------> limit | | Reserved space for
3272 * +---------------+ | `RegExpObject::MaxPairCount`
3273 * . | MatchPair objects.
3274 * . |
3275 * . | Only a single object will
3276 * +---------------+ | be initialized and can be
3277 * | MatchPair | | accessed below.
3278 * | start | <-------+
3279 * | limit |
3280 * +---------------+
3281 */
3282 // clang-format on
3283
3284 int32_t pairsVectorStartOffset =
3285 RegExpPairsVectorStartOffset(inputOutputDataStartOffset);
3286 Address matchPairStart(FramePointer,
3287 pairsVectorStartOffset + MatchPair::offsetOfStart());
3288 Address matchPairLimit(FramePointer,
3289 pairsVectorStartOffset + MatchPair::offsetOfLimit());
3290
3291 // Store match limit to cx->regExpSearcherLastLimit and return the index.
3292 masm.load32(matchPairLimit, result);
3293 masm.loadJSContext(input);
3294 masm.store32(result,
3295 Address(input, JSContext::offsetOfRegExpSearcherLastLimit()));
3296 masm.load32(matchPairStart, result);
3297 masm.pop(FramePointer);
3298 masm.ret();
3299
3300 masm.bind(&notFound);
3301 masm.move32(Imm32(RegExpSearcherResultNotFound), result);
3302 masm.pop(FramePointer);
3303 masm.ret();
3304
3305 masm.bind(&oolEntry);
3306 masm.move32(Imm32(RegExpSearcherResultFailed), result);
3307 masm.pop(FramePointer);
3308 masm.ret();
3309
3310 Linker linker(masm);
3311 JitCode* code = linker.newCode(cx, CodeKind::Other);
3312 if (!code) {
3313 return nullptr;
3314 }
3315
3316 CollectPerfSpewerJitCodeProfile(code, "RegExpSearcherStub");
3317#ifdef MOZ_VTUNE1
3318 vtune::MarkStub(code, "RegExpSearcherStub");
3319#endif
3320
3321 return code;
3322}
3323
3324class OutOfLineRegExpSearcher : public OutOfLineCodeBase<CodeGenerator> {
3325 LRegExpSearcher* lir_;
3326
3327 public:
3328 explicit OutOfLineRegExpSearcher(LRegExpSearcher* lir) : lir_(lir) {}
3329
3330 void accept(CodeGenerator* codegen) override {
3331 codegen->visitOutOfLineRegExpSearcher(this);
3332 }
3333
3334 LRegExpSearcher* lir() const { return lir_; }
3335};
3336
3337void CodeGenerator::visitOutOfLineRegExpSearcher(OutOfLineRegExpSearcher* ool) {
3338 LRegExpSearcher* lir = ool->lir();
3339 Register lastIndex = ToRegister(lir->lastIndex());
3340 Register input = ToRegister(lir->string());
3341 Register regexp = ToRegister(lir->regexp());
3342
3343 AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
3344 regs.take(lastIndex);
3345 regs.take(input);
3346 regs.take(regexp);
3347 Register temp = regs.takeAny();
3348
3349 masm.computeEffectiveAddress(
3350 Address(masm.getStackPointer(), InputOutputDataSize), temp);
3351
3352 pushArg(temp);
3353 pushArg(lastIndex);
3354 pushArg(input);
3355 pushArg(regexp);
3356
3357 // We are not using oolCallVM because we are in a Call, and that live
3358 // registers are already saved by the the register allocator.
3359 using Fn = bool (*)(JSContext* cx, HandleObject regexp, HandleString input,
3360 int32_t lastIndex, MatchPairs* pairs, int32_t* result);
3361 callVM<Fn, RegExpSearcherRaw>(lir);
3362
3363 masm.jump(ool->rejoin());
3364}
3365
3366void CodeGenerator::visitRegExpSearcher(LRegExpSearcher* lir) {
3367 MOZ_ASSERT(ToRegister(lir->regexp()) == RegExpSearcherRegExpReg)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ToRegister(lir->regexp()) == RegExpSearcherRegExpReg
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(ToRegister(lir->regexp()) == RegExpSearcherRegExpReg
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"ToRegister(lir->regexp()) == RegExpSearcherRegExpReg", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 3367); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ToRegister(lir->regexp()) == RegExpSearcherRegExpReg"
")"); do { *((volatile int*)__null) = 3367; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
3368 MOZ_ASSERT(ToRegister(lir->string()) == RegExpSearcherStringReg)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ToRegister(lir->string()) == RegExpSearcherStringReg
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(ToRegister(lir->string()) == RegExpSearcherStringReg
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"ToRegister(lir->string()) == RegExpSearcherStringReg", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 3368); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ToRegister(lir->string()) == RegExpSearcherStringReg"
")"); do { *((volatile int*)__null) = 3368; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
3369 MOZ_ASSERT(ToRegister(lir->lastIndex()) == RegExpSearcherLastIndexReg)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ToRegister(lir->lastIndex()) == RegExpSearcherLastIndexReg
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(ToRegister(lir->lastIndex()) == RegExpSearcherLastIndexReg
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"ToRegister(lir->lastIndex()) == RegExpSearcherLastIndexReg"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 3369); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ToRegister(lir->lastIndex()) == RegExpSearcherLastIndexReg"
")"); do { *((volatile int*)__null) = 3369; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
3370 MOZ_ASSERT(ToRegister(lir->output()) == ReturnReg)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ToRegister(lir->output()) == ReturnReg)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(ToRegister(lir->output()) == ReturnReg))), 0))) { do { }
while (false); MOZ_ReportAssertionFailure("ToRegister(lir->output()) == ReturnReg"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 3370); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ToRegister(lir->output()) == ReturnReg"
")"); do { *((volatile int*)__null) = 3370; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
3371
3372 static_assert(RegExpSearcherRegExpReg != ReturnReg);
3373 static_assert(RegExpSearcherStringReg != ReturnReg);
3374 static_assert(RegExpSearcherLastIndexReg != ReturnReg);
3375
3376 masm.reserveStack(RegExpReservedStack);
3377
3378 OutOfLineRegExpSearcher* ool = new (alloc()) OutOfLineRegExpSearcher(lir);
3379 addOutOfLineCode(ool, lir->mir());
3380
3381 const JitZone* jitZone = gen->realm->zone()->jitZone();
3382 JitCode* regExpSearcherStub =
3383 jitZone->regExpSearcherStubNoBarrier(&zoneStubsToReadBarrier_);
3384 masm.call(regExpSearcherStub);
3385 masm.branch32(Assembler::Equal, ReturnReg, Imm32(RegExpSearcherResultFailed),
3386 ool->entry());
3387 masm.bind(ool->rejoin());
3388
3389 masm.freeStack(RegExpReservedStack);
3390}
3391
3392void CodeGenerator::visitRegExpSearcherLastLimit(
3393 LRegExpSearcherLastLimit* lir) {
3394 Register result = ToRegister(lir->output());
3395 Register scratch = ToRegister(lir->temp0());
3396
3397 masm.loadAndClearRegExpSearcherLastLimit(result, scratch);
3398}
3399
3400JitCode* JitZone::generateRegExpExecTestStub(JSContext* cx) {
3401 JitSpew(JitSpew_Codegen, "# Emitting RegExpExecTest stub");
3402
3403 Register regexp = RegExpExecTestRegExpReg;
3404 Register input = RegExpExecTestStringReg;
3405 Register result = ReturnReg;
3406
3407 TempAllocator temp(&cx->tempLifoAlloc());
3408 JitContext jcx(cx);
3409 StackMacroAssembler masm(cx, temp);
3410 AutoCreatedBy acb(masm, "JitZone::generateRegExpExecTestStub");
3411
3412#ifdef JS_USE_LINK_REGISTER
3413 masm.pushReturnAddress();
3414#endif
3415 masm.push(FramePointer);
3416 masm.moveStackPtrTo(FramePointer);
3417
3418 // We are free to clobber all registers, as LRegExpExecTest is a call
3419 // instruction.
3420 AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
3421 regs.take(input);
3422 regs.take(regexp);
3423
3424 // Ensure lastIndex != result.
3425 regs.take(result);
3426 Register lastIndex = regs.takeAny();
3427 regs.add(result);
3428 Register temp1 = regs.takeAny();
3429 Register temp2 = regs.takeAny();
3430 Register temp3 = regs.takeAny();
3431
3432 Address flagsSlot(regexp, RegExpObject::offsetOfFlags());
3433 Address lastIndexSlot(regexp, RegExpObject::offsetOfLastIndex());
3434
3435 masm.reserveStack(RegExpReservedStack);
3436
3437 // Load lastIndex and skip RegExp execution if needed.
3438 Label notFoundZeroLastIndex;
3439 masm.loadRegExpLastIndex(regexp, input, lastIndex, &notFoundZeroLastIndex);
3440
3441 // In visitRegExpMatcher and visitRegExpSearcher, we reserve stack space
3442 // before calling the stub. For RegExpExecTest we call the stub before
3443 // reserving stack space, so the offset of the InputOutputData relative to the
3444 // frame pointer is negative.
3445 constexpr int32_t inputOutputDataStartOffset = -int32_t(RegExpReservedStack);
3446
3447 // On ARM64, load/store instructions can encode an immediate offset in the
3448 // range [-256, 4095]. If we ever fail this assertion, it would be more
3449 // efficient to store the data above the frame pointer similar to
3450 // RegExpMatcher and RegExpSearcher.
3451 static_assert(inputOutputDataStartOffset >= -256);
3452
3453 Label notFound, oolEntry;
3454 if (!PrepareAndExecuteRegExp(masm, regexp, input, lastIndex, temp1, temp2,
3455 temp3, inputOutputDataStartOffset,
3456 initialStringHeap, &notFound, &oolEntry)) {
3457 return nullptr;
3458 }
3459
3460 // Set `result` to true/false to indicate found/not-found, or to
3461 // RegExpExecTestResultFailed if we have to retry in C++. If the regular
3462 // expression is global or sticky, we also have to update its .lastIndex slot.
3463
3464 Label done;
3465 int32_t pairsVectorStartOffset =
3466 RegExpPairsVectorStartOffset(inputOutputDataStartOffset);
3467 Address matchPairLimit(FramePointer,
3468 pairsVectorStartOffset + MatchPair::offsetOfLimit());
3469
3470 masm.move32(Imm32(1), result);
3471 masm.branchTest32(Assembler::Zero, flagsSlot,
3472 Imm32(JS::RegExpFlag::Global | JS::RegExpFlag::Sticky),
3473 &done);
3474 masm.load32(matchPairLimit, lastIndex);
3475 masm.storeValue(JSVAL_TYPE_INT32, lastIndex, lastIndexSlot);
3476 masm.jump(&done);
3477
3478 masm.bind(&notFound);
3479 masm.move32(Imm32(0), result);
3480 masm.branchTest32(Assembler::Zero, flagsSlot,
3481 Imm32(JS::RegExpFlag::Global | JS::RegExpFlag::Sticky),
3482 &done);
3483 masm.storeValue(Int32Value(0), lastIndexSlot);
3484 masm.jump(&done);
3485
3486 masm.bind(&notFoundZeroLastIndex);
3487 masm.move32(Imm32(0), result);
3488 masm.storeValue(Int32Value(0), lastIndexSlot);
3489 masm.jump(&done);
3490
3491 masm.bind(&oolEntry);
3492 masm.move32(Imm32(RegExpExecTestResultFailed), result);
3493
3494 masm.bind(&done);
3495 masm.freeStack(RegExpReservedStack);
3496 masm.pop(FramePointer);
3497 masm.ret();
3498
3499 Linker linker(masm);
3500 JitCode* code = linker.newCode(cx, CodeKind::Other);
3501 if (!code) {
3502 return nullptr;
3503 }
3504
3505 CollectPerfSpewerJitCodeProfile(code, "RegExpExecTestStub");
3506#ifdef MOZ_VTUNE1
3507 vtune::MarkStub(code, "RegExpExecTestStub");
3508#endif
3509
3510 return code;
3511}
3512
3513class OutOfLineRegExpExecTest : public OutOfLineCodeBase<CodeGenerator> {
3514 LRegExpExecTest* lir_;
3515
3516 public:
3517 explicit OutOfLineRegExpExecTest(LRegExpExecTest* lir) : lir_(lir) {}
3518
3519 void accept(CodeGenerator* codegen) override {
3520 codegen->visitOutOfLineRegExpExecTest(this);
3521 }
3522
3523 LRegExpExecTest* lir() const { return lir_; }
3524};
3525
3526void CodeGenerator::visitOutOfLineRegExpExecTest(OutOfLineRegExpExecTest* ool) {
3527 LRegExpExecTest* lir = ool->lir();
3528 Register input = ToRegister(lir->string());
3529 Register regexp = ToRegister(lir->regexp());
3530
3531 pushArg(input);
3532 pushArg(regexp);
3533
3534 // We are not using oolCallVM because we are in a Call and live registers have
3535 // already been saved by the register allocator.
3536 using Fn = bool (*)(JSContext* cx, Handle<RegExpObject*> regexp,
3537 HandleString input, bool* result);
3538 callVM<Fn, RegExpBuiltinExecTestFromJit>(lir);
3539
3540 masm.jump(ool->rejoin());
3541}
3542
3543void CodeGenerator::visitRegExpExecTest(LRegExpExecTest* lir) {
3544 MOZ_ASSERT(ToRegister(lir->regexp()) == RegExpExecTestRegExpReg)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ToRegister(lir->regexp()) == RegExpExecTestRegExpReg
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(ToRegister(lir->regexp()) == RegExpExecTestRegExpReg
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"ToRegister(lir->regexp()) == RegExpExecTestRegExpReg", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 3544); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ToRegister(lir->regexp()) == RegExpExecTestRegExpReg"
")"); do { *((volatile int*)__null) = 3544; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
3545 MOZ_ASSERT(ToRegister(lir->string()) == RegExpExecTestStringReg)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ToRegister(lir->string()) == RegExpExecTestStringReg
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(ToRegister(lir->string()) == RegExpExecTestStringReg
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"ToRegister(lir->string()) == RegExpExecTestStringReg", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 3545); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ToRegister(lir->string()) == RegExpExecTestStringReg"
")"); do { *((volatile int*)__null) = 3545; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
3546 MOZ_ASSERT(ToRegister(lir->output()) == ReturnReg)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ToRegister(lir->output()) == ReturnReg)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(ToRegister(lir->output()) == ReturnReg))), 0))) { do { }
while (false); MOZ_ReportAssertionFailure("ToRegister(lir->output()) == ReturnReg"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 3546); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ToRegister(lir->output()) == ReturnReg"
")"); do { *((volatile int*)__null) = 3546; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
3547
3548 static_assert(RegExpExecTestRegExpReg != ReturnReg);
3549 static_assert(RegExpExecTestStringReg != ReturnReg);
3550
3551 auto* ool = new (alloc()) OutOfLineRegExpExecTest(lir);
3552 addOutOfLineCode(ool, lir->mir());
3553
3554 const JitZone* jitZone = gen->realm->zone()->jitZone();
3555 JitCode* regExpExecTestStub =
3556 jitZone->regExpExecTestStubNoBarrier(&zoneStubsToReadBarrier_);
3557 masm.call(regExpExecTestStub);
3558
3559 masm.branch32(Assembler::Equal, ReturnReg, Imm32(RegExpExecTestResultFailed),
3560 ool->entry());
3561
3562 masm.bind(ool->rejoin());
3563}
3564
3565void CodeGenerator::visitRegExpHasCaptureGroups(LRegExpHasCaptureGroups* ins) {
3566 Register regexp = ToRegister(ins->regexp());
3567 Register input = ToRegister(ins->input());
3568 Register output = ToRegister(ins->output());
3569
3570 using Fn =
3571 bool (*)(JSContext*, Handle<RegExpObject*>, Handle<JSString*>, bool*);
3572 auto* ool = oolCallVM<Fn, js::RegExpHasCaptureGroups>(
3573 ins, ArgList(regexp, input), StoreRegisterTo(output));
3574
3575 // Load RegExpShared in |output|.
3576 Label vmCall;
3577 masm.loadParsedRegExpShared(regexp, output, ool->entry());
3578
3579 // Return true iff pairCount > 1.
3580 Label returnTrue;
3581 masm.branch32(Assembler::Above,
3582 Address(output, RegExpShared::offsetOfPairCount()), Imm32(1),
3583 &returnTrue);
3584 masm.move32(Imm32(0), output);
3585 masm.jump(ool->rejoin());
3586
3587 masm.bind(&returnTrue);
3588 masm.move32(Imm32(1), output);
3589
3590 masm.bind(ool->rejoin());
3591}
3592
3593class OutOfLineRegExpPrototypeOptimizable
3594 : public OutOfLineCodeBase<CodeGenerator> {
3595 LRegExpPrototypeOptimizable* ins_;
3596
3597 public:
3598 explicit OutOfLineRegExpPrototypeOptimizable(LRegExpPrototypeOptimizable* ins)
3599 : ins_(ins) {}
3600
3601 void accept(CodeGenerator* codegen) override {
3602 codegen->visitOutOfLineRegExpPrototypeOptimizable(this);
3603 }
3604 LRegExpPrototypeOptimizable* ins() const { return ins_; }
3605};
3606
3607void CodeGenerator::visitRegExpPrototypeOptimizable(
3608 LRegExpPrototypeOptimizable* ins) {
3609 Register object = ToRegister(ins->object());
3610 Register output = ToRegister(ins->output());
3611 Register temp = ToRegister(ins->temp0());
3612
3613 OutOfLineRegExpPrototypeOptimizable* ool =
3614 new (alloc()) OutOfLineRegExpPrototypeOptimizable(ins);
3615 addOutOfLineCode(ool, ins->mir());
3616
3617 const GlobalObject* global = gen->realm->maybeGlobal();
3618 MOZ_ASSERT(global)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(global)>::isValid, "invalid assertion condition")
; if ((__builtin_expect(!!(!(!!(global))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("global", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 3618); AnnotateMozCrashReason("MOZ_ASSERT" "(" "global" ")"
); do { *((volatile int*)__null) = 3618; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
3619 masm.branchIfNotRegExpPrototypeOptimizable(object, temp, global,
3620 ool->entry());
3621 masm.move32(Imm32(0x1), output);
3622
3623 masm.bind(ool->rejoin());
3624}
3625
3626void CodeGenerator::visitOutOfLineRegExpPrototypeOptimizable(
3627 OutOfLineRegExpPrototypeOptimizable* ool) {
3628 LRegExpPrototypeOptimizable* ins = ool->ins();
3629 Register object = ToRegister(ins->object());
3630 Register output = ToRegister(ins->output());
3631
3632 saveVolatile(output);
3633
3634 using Fn = bool (*)(JSContext* cx, JSObject* proto);
3635 masm.setupAlignedABICall();
3636 masm.loadJSContext(output);
3637 masm.passABIArg(output);
3638 masm.passABIArg(object);
3639 masm.callWithABI<Fn, RegExpPrototypeOptimizableRaw>();
3640 masm.storeCallBoolResult(output);
3641
3642 restoreVolatile(output);
3643
3644 masm.jump(ool->rejoin());
3645}
3646
3647class OutOfLineRegExpInstanceOptimizable
3648 : public OutOfLineCodeBase<CodeGenerator> {
3649 LRegExpInstanceOptimizable* ins_;
3650
3651 public:
3652 explicit OutOfLineRegExpInstanceOptimizable(LRegExpInstanceOptimizable* ins)
3653 : ins_(ins) {}
3654
3655 void accept(CodeGenerator* codegen) override {
3656 codegen->visitOutOfLineRegExpInstanceOptimizable(this);
3657 }
3658 LRegExpInstanceOptimizable* ins() const { return ins_; }
3659};
3660
3661void CodeGenerator::visitRegExpInstanceOptimizable(
3662 LRegExpInstanceOptimizable* ins) {
3663 Register object = ToRegister(ins->object());
3664 Register output = ToRegister(ins->output());
3665 Register temp = ToRegister(ins->temp0());
3666
3667 OutOfLineRegExpInstanceOptimizable* ool =
3668 new (alloc()) OutOfLineRegExpInstanceOptimizable(ins);
3669 addOutOfLineCode(ool, ins->mir());
3670
3671 const GlobalObject* global = gen->realm->maybeGlobal();
3672 MOZ_ASSERT(global)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(global)>::isValid, "invalid assertion condition")
; if ((__builtin_expect(!!(!(!!(global))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("global", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 3672); AnnotateMozCrashReason("MOZ_ASSERT" "(" "global" ")"
); do { *((volatile int*)__null) = 3672; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
3673 masm.branchIfNotRegExpInstanceOptimizable(object, temp, global, ool->entry());
3674 masm.move32(Imm32(0x1), output);
3675
3676 masm.bind(ool->rejoin());
3677}
3678
3679void CodeGenerator::visitOutOfLineRegExpInstanceOptimizable(
3680 OutOfLineRegExpInstanceOptimizable* ool) {
3681 LRegExpInstanceOptimizable* ins = ool->ins();
3682 Register object = ToRegister(ins->object());
3683 Register proto = ToRegister(ins->proto());
3684 Register output = ToRegister(ins->output());
3685
3686 saveVolatile(output);
3687
3688 using Fn = bool (*)(JSContext* cx, JSObject* obj, JSObject* proto);
3689 masm.setupAlignedABICall();
3690 masm.loadJSContext(output);
3691 masm.passABIArg(output);
3692 masm.passABIArg(object);
3693 masm.passABIArg(proto);
3694 masm.callWithABI<Fn, RegExpInstanceOptimizableRaw>();
3695 masm.storeCallBoolResult(output);
3696
3697 restoreVolatile(output);
3698
3699 masm.jump(ool->rejoin());
3700}
3701
3702static void FindFirstDollarIndex(MacroAssembler& masm, Register str,
3703 Register len, Register temp0, Register temp1,
3704 Register output, CharEncoding encoding) {
3705#ifdef DEBUG1
3706 Label ok;
3707 masm.branch32(Assembler::GreaterThan, len, Imm32(0), &ok);
3708 masm.assumeUnreachable("Length should be greater than 0.");
3709 masm.bind(&ok);
3710#endif
3711
3712 Register chars = temp0;
3713 masm.loadStringChars(str, chars, encoding);
3714
3715 masm.move32(Imm32(0), output);
3716
3717 Label start, done;
3718 masm.bind(&start);
3719
3720 Register currentChar = temp1;
3721 masm.loadChar(chars, output, currentChar, encoding);
3722 masm.branch32(Assembler::Equal, currentChar, Imm32('$'), &done);
3723
3724 masm.add32(Imm32(1), output);
3725 masm.branch32(Assembler::NotEqual, output, len, &start);
3726
3727 masm.move32(Imm32(-1), output);
3728
3729 masm.bind(&done);
3730}
3731
3732void CodeGenerator::visitGetFirstDollarIndex(LGetFirstDollarIndex* ins) {
3733 Register str = ToRegister(ins->str());
3734 Register output = ToRegister(ins->output());
3735 Register temp0 = ToRegister(ins->temp0());
3736 Register temp1 = ToRegister(ins->temp1());
3737 Register len = ToRegister(ins->temp2());
3738
3739 using Fn = bool (*)(JSContext*, JSString*, int32_t*);
3740 OutOfLineCode* ool = oolCallVM<Fn, GetFirstDollarIndexRaw>(
3741 ins, ArgList(str), StoreRegisterTo(output));
3742
3743 masm.branchIfRope(str, ool->entry());
3744 masm.loadStringLength(str, len);
3745
3746 Label isLatin1, done;
3747 masm.branchLatin1String(str, &isLatin1);
3748 {
3749 FindFirstDollarIndex(masm, str, len, temp0, temp1, output,
3750 CharEncoding::TwoByte);
3751 masm.jump(&done);
3752 }
3753 masm.bind(&isLatin1);
3754 {
3755 FindFirstDollarIndex(masm, str, len, temp0, temp1, output,
3756 CharEncoding::Latin1);
3757 }
3758 masm.bind(&done);
3759 masm.bind(ool->rejoin());
3760}
3761
3762void CodeGenerator::visitStringReplace(LStringReplace* lir) {
3763 if (lir->replacement()->isConstant()) {
3764 pushArg(ImmGCPtr(lir->replacement()->toConstant()->toString()));
3765 } else {
3766 pushArg(ToRegister(lir->replacement()));
3767 }
3768
3769 if (lir->pattern()->isConstant()) {
3770 pushArg(ImmGCPtr(lir->pattern()->toConstant()->toString()));
3771 } else {
3772 pushArg(ToRegister(lir->pattern()));
3773 }
3774
3775 if (lir->string()->isConstant()) {
3776 pushArg(ImmGCPtr(lir->string()->toConstant()->toString()));
3777 } else {
3778 pushArg(ToRegister(lir->string()));
3779 }
3780
3781 using Fn =
3782 JSString* (*)(JSContext*, HandleString, HandleString, HandleString);
3783 if (lir->mir()->isFlatReplacement()) {
3784 callVM<Fn, StringFlatReplaceString>(lir);
3785 } else {
3786 callVM<Fn, StringReplace>(lir);
3787 }
3788}
3789
3790void CodeGenerator::visitBinaryValueCache(LBinaryValueCache* lir) {
3791 LiveRegisterSet liveRegs = lir->safepoint()->liveRegs();
3792 TypedOrValueRegister lhs =
3793 TypedOrValueRegister(ToValue(lir, LBinaryValueCache::LhsIndex));
3794 TypedOrValueRegister rhs =
3795 TypedOrValueRegister(ToValue(lir, LBinaryValueCache::RhsIndex));
3796 ValueOperand output = ToOutValue(lir);
3797
3798 JSOp jsop = JSOp(*lir->mirRaw()->toInstruction()->resumePoint()->pc());
3799
3800 switch (jsop) {
3801 case JSOp::Add:
3802 case JSOp::Sub:
3803 case JSOp::Mul:
3804 case JSOp::Div:
3805 case JSOp::Mod:
3806 case JSOp::Pow:
3807 case JSOp::BitAnd:
3808 case JSOp::BitOr:
3809 case JSOp::BitXor:
3810 case JSOp::Lsh:
3811 case JSOp::Rsh:
3812 case JSOp::Ursh: {
3813 IonBinaryArithIC ic(liveRegs, lhs, rhs, output);
3814 addIC(lir, allocateIC(ic));
3815 return;
3816 }
3817 default:
3818 MOZ_CRASH("Unsupported jsop in MBinaryValueCache")do { do { } while (false); MOZ_ReportCrash("" "Unsupported jsop in MBinaryValueCache"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 3818); AnnotateMozCrashReason("MOZ_CRASH(" "Unsupported jsop in MBinaryValueCache"
")"); do { *((volatile int*)__null) = 3818; __attribute__((nomerge
)) ::abort(); } while (false); } while (false)
;
3819 }
3820}
3821
3822void CodeGenerator::visitBinaryBoolCache(LBinaryBoolCache* lir) {
3823 LiveRegisterSet liveRegs = lir->safepoint()->liveRegs();
3824 TypedOrValueRegister lhs =
3825 TypedOrValueRegister(ToValue(lir, LBinaryBoolCache::LhsIndex));
3826 TypedOrValueRegister rhs =
3827 TypedOrValueRegister(ToValue(lir, LBinaryBoolCache::RhsIndex));
3828 Register output = ToRegister(lir->output());
3829
3830 JSOp jsop = JSOp(*lir->mirRaw()->toInstruction()->resumePoint()->pc());
3831
3832 switch (jsop) {
3833 case JSOp::Lt:
3834 case JSOp::Le:
3835 case JSOp::Gt:
3836 case JSOp::Ge:
3837 case JSOp::Eq:
3838 case JSOp::Ne:
3839 case JSOp::StrictEq:
3840 case JSOp::StrictNe: {
3841 IonCompareIC ic(liveRegs, lhs, rhs, output);
3842 addIC(lir, allocateIC(ic));
3843 return;
3844 }
3845 default:
3846 MOZ_CRASH("Unsupported jsop in MBinaryBoolCache")do { do { } while (false); MOZ_ReportCrash("" "Unsupported jsop in MBinaryBoolCache"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 3846); AnnotateMozCrashReason("MOZ_CRASH(" "Unsupported jsop in MBinaryBoolCache"
")"); do { *((volatile int*)__null) = 3846; __attribute__((nomerge
)) ::abort(); } while (false); } while (false)
;
3847 }
3848}
3849
3850void CodeGenerator::visitUnaryCache(LUnaryCache* lir) {
3851 LiveRegisterSet liveRegs = lir->safepoint()->liveRegs();
3852 TypedOrValueRegister input =
3853 TypedOrValueRegister(ToValue(lir, LUnaryCache::InputIndex));
3854 ValueOperand output = ToOutValue(lir);
3855
3856 IonUnaryArithIC ic(liveRegs, input, output);
3857 addIC(lir, allocateIC(ic));
3858}
3859
3860void CodeGenerator::visitModuleMetadata(LModuleMetadata* lir) {
3861 pushArg(ImmPtr(lir->mir()->module()));
3862
3863 using Fn = JSObject* (*)(JSContext*, HandleObject);
3864 callVM<Fn, js::GetOrCreateModuleMetaObject>(lir);
3865}
3866
3867void CodeGenerator::visitDynamicImport(LDynamicImport* lir) {
3868 pushArg(ToValue(lir, LDynamicImport::OptionsIndex));
3869 pushArg(ToValue(lir, LDynamicImport::SpecifierIndex));
3870 pushArg(ImmGCPtr(current->mir()->info().script()));
3871
3872 using Fn = JSObject* (*)(JSContext*, HandleScript, HandleValue, HandleValue);
3873 callVM<Fn, js::StartDynamicModuleImport>(lir);
3874}
3875
3876void CodeGenerator::visitLambda(LLambda* lir) {
3877 Register envChain = ToRegister(lir->environmentChain());
3878 Register output = ToRegister(lir->output());
3879 Register tempReg = ToRegister(lir->temp0());
3880
3881 JSFunction* fun = lir->mir()->templateFunction();
3882
3883 using Fn = JSObject* (*)(JSContext*, HandleFunction, HandleObject);
3884 OutOfLineCode* ool = oolCallVM<Fn, js::Lambda>(
3885 lir, ArgList(ImmGCPtr(fun), envChain), StoreRegisterTo(output));
3886
3887 TemplateObject templateObject(fun);
3888 masm.createGCObject(output, tempReg, templateObject, gc::Heap::Default,
3889 ool->entry());
3890
3891 masm.storeValue(JSVAL_TYPE_OBJECT, envChain,
3892 Address(output, JSFunction::offsetOfEnvironment()));
3893 // No post barrier needed because output is guaranteed to be allocated in
3894 // the nursery.
3895
3896 masm.bind(ool->rejoin());
3897}
3898
3899void CodeGenerator::visitFunctionWithProto(LFunctionWithProto* lir) {
3900 Register envChain = ToRegister(lir->envChain());
3901 Register prototype = ToRegister(lir->prototype());
3902
3903 pushArg(prototype);
3904 pushArg(envChain);
3905 pushArg(ImmGCPtr(lir->mir()->function()));
3906
3907 using Fn =
3908 JSObject* (*)(JSContext*, HandleFunction, HandleObject, HandleObject);
3909 callVM<Fn, js::FunWithProtoOperation>(lir);
3910}
3911
3912void CodeGenerator::visitSetFunName(LSetFunName* lir) {
3913 pushArg(Imm32(lir->mir()->prefixKind()));
3914 pushArg(ToValue(lir, LSetFunName::NameIndex));
3915 pushArg(ToRegister(lir->fun()));
3916
3917 using Fn =
3918 bool (*)(JSContext*, HandleFunction, HandleValue, FunctionPrefixKind);
3919 callVM<Fn, js::SetFunctionName>(lir);
3920}
3921
3922void CodeGenerator::visitOsiPoint(LOsiPoint* lir) {
3923 // Note: markOsiPoint ensures enough space exists between the last
3924 // LOsiPoint and this one to patch adjacent call instructions.
3925
3926 MOZ_ASSERT(masm.framePushed() == frameSize())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(masm.framePushed() == frameSize())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(masm.framePushed() == frameSize
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("masm.framePushed() == frameSize()", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 3926); AnnotateMozCrashReason("MOZ_ASSERT" "(" "masm.framePushed() == frameSize()"
")"); do { *((volatile int*)__null) = 3926; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
3927
3928 uint32_t osiCallPointOffset = markOsiPoint(lir);
3929
3930 LSafepoint* safepoint = lir->associatedSafepoint();
3931 MOZ_ASSERT(!safepoint->osiCallPointOffset())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!safepoint->osiCallPointOffset())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!safepoint->osiCallPointOffset
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("!safepoint->osiCallPointOffset()", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 3931); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!safepoint->osiCallPointOffset()"
")"); do { *((volatile int*)__null) = 3931; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
3932 safepoint->setOsiCallPointOffset(osiCallPointOffset);
3933
3934#ifdef DEBUG1
3935 // There should be no movegroups or other instructions between
3936 // an instruction and its OsiPoint. This is necessary because
3937 // we use the OsiPoint's snapshot from within VM calls.
3938 for (LInstructionReverseIterator iter(current->rbegin(lir));
3939 iter != current->rend(); iter++) {
3940 if (*iter == lir) {
3941 continue;
3942 }
3943 MOZ_ASSERT(!iter->isMoveGroup())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!iter->isMoveGroup())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!iter->isMoveGroup()))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("!iter->isMoveGroup()"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 3943); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!iter->isMoveGroup()"
")"); do { *((volatile int*)__null) = 3943; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
3944 MOZ_ASSERT(iter->safepoint() == safepoint)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(iter->safepoint() == safepoint)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(iter->safepoint() == safepoint
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"iter->safepoint() == safepoint", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 3944); AnnotateMozCrashReason("MOZ_ASSERT" "(" "iter->safepoint() == safepoint"
")"); do { *((volatile int*)__null) = 3944; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
3945 break;
3946 }
3947#endif
3948
3949#ifdef CHECK_OSIPOINT_REGISTERS1
3950 if (shouldVerifyOsiPointRegs(safepoint)) {
3951 verifyOsiPointRegs(safepoint);
3952 }
3953#endif
3954}
3955
3956void CodeGenerator::visitPhi(LPhi* lir) {
3957 MOZ_CRASH("Unexpected LPhi in CodeGenerator")do { do { } while (false); MOZ_ReportCrash("" "Unexpected LPhi in CodeGenerator"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 3957); AnnotateMozCrashReason("MOZ_CRASH(" "Unexpected LPhi in CodeGenerator"
")"); do { *((volatile int*)__null) = 3957; __attribute__((nomerge
)) ::abort(); } while (false); } while (false)
;
3958}
3959
3960void CodeGenerator::visitGoto(LGoto* lir) { jumpToBlock(lir->target()); }
3961
3962void CodeGenerator::visitTableSwitch(LTableSwitch* ins) {
3963 MTableSwitch* mir = ins->mir();
3964 Label* defaultcase = skipTrivialBlocks(mir->getDefault())->lir()->label();
3965 const LAllocation* temp;
3966
3967 if (mir->getOperand(0)->type() != MIRType::Int32) {
3968 temp = ins->tempInt()->output();
3969
3970 // The input is a double, so try and convert it to an integer.
3971 // If it does not fit in an integer, take the default case.
3972 masm.convertDoubleToInt32(ToFloatRegister(ins->index()), ToRegister(temp),
3973 defaultcase, false);
3974 } else {
3975 temp = ins->index();
3976 }
3977
3978 emitTableSwitchDispatch(mir, ToRegister(temp),
3979 ToRegisterOrInvalid(ins->tempPointer()));
3980}
3981
3982void CodeGenerator::visitTableSwitchV(LTableSwitchV* ins) {
3983 MTableSwitch* mir = ins->mir();
3984 Label* defaultcase = skipTrivialBlocks(mir->getDefault())->lir()->label();
3985
3986 Register index = ToRegister(ins->tempInt());
3987 ValueOperand value = ToValue(ins, LTableSwitchV::InputValue);
3988 Register tag = masm.extractTag(value, index);
3989 masm.branchTestNumber(Assembler::NotEqual, tag, defaultcase);
3990
3991 Label unboxInt, isInt;
3992 masm.branchTestInt32(Assembler::Equal, tag, &unboxInt);
3993 {
3994 FloatRegister floatIndex = ToFloatRegister(ins->tempFloat());
3995 masm.unboxDouble(value, floatIndex);
3996 masm.convertDoubleToInt32(floatIndex, index, defaultcase, false);
3997 masm.jump(&isInt);
3998 }
3999
4000 masm.bind(&unboxInt);
4001 masm.unboxInt32(value, index);
4002
4003 masm.bind(&isInt);
4004
4005 emitTableSwitchDispatch(mir, index, ToRegisterOrInvalid(ins->tempPointer()));
4006}
4007
4008void CodeGenerator::visitParameter(LParameter* lir) {}
4009
4010void CodeGenerator::visitCallee(LCallee* lir) {
4011 Register callee = ToRegister(lir->output());
4012 Address ptr(FramePointer, JitFrameLayout::offsetOfCalleeToken());
4013
4014 masm.loadFunctionFromCalleeToken(ptr, callee);
4015}
4016
4017void CodeGenerator::visitIsConstructing(LIsConstructing* lir) {
4018 Register output = ToRegister(lir->output());
4019 Address calleeToken(FramePointer, JitFrameLayout::offsetOfCalleeToken());
4020 masm.loadPtr(calleeToken, output);
4021
4022 // We must be inside a function.
4023 MOZ_ASSERT(current->mir()->info().script()->function())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(current->mir()->info().script()->function()
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(current->mir()->info().script()->function()
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"current->mir()->info().script()->function()", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 4023); AnnotateMozCrashReason("MOZ_ASSERT" "(" "current->mir()->info().script()->function()"
")"); do { *((volatile int*)__null) = 4023; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
4024
4025 // The low bit indicates whether this call is constructing, just clear the
4026 // other bits.
4027 static_assert(CalleeToken_Function == 0x0,
4028 "CalleeTokenTag value should match");
4029 static_assert(CalleeToken_FunctionConstructing == 0x1,
4030 "CalleeTokenTag value should match");
4031 masm.andPtr(Imm32(0x1), output);
4032}
4033
4034void CodeGenerator::visitReturn(LReturn* lir) {
4035#if defined(JS_NUNBOX32)
4036 DebugOnly<LAllocation*> type = lir->getOperand(TYPE_INDEX);
4037 DebugOnly<LAllocation*> payload = lir->getOperand(PAYLOAD_INDEX);
4038 MOZ_ASSERT(ToRegister(type) == JSReturnReg_Type)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ToRegister(type) == JSReturnReg_Type)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(ToRegister(type) == JSReturnReg_Type
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"ToRegister(type) == JSReturnReg_Type", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 4038); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ToRegister(type) == JSReturnReg_Type"
")"); do { *((volatile int*)__null) = 4038; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
4039 MOZ_ASSERT(ToRegister(payload) == JSReturnReg_Data)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ToRegister(payload) == JSReturnReg_Data)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(ToRegister(payload) == JSReturnReg_Data))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("ToRegister(payload) == JSReturnReg_Data"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 4039); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ToRegister(payload) == JSReturnReg_Data"
")"); do { *((volatile int*)__null) = 4039; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
4040#elif defined(JS_PUNBOX641)
4041 DebugOnly<LAllocation*> result = lir->getOperand(0);
4042 MOZ_ASSERT(ToRegister(result) == JSReturnReg)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ToRegister(result) == JSReturnReg)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(ToRegister(result) == JSReturnReg
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"ToRegister(result) == JSReturnReg", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 4042); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ToRegister(result) == JSReturnReg"
")"); do { *((volatile int*)__null) = 4042; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
4043#endif
4044 // Don't emit a jump to the return label if this is the last block, as
4045 // it'll fall through to the epilogue.
4046 //
4047 // This is -not- true however for a Generator-return, which may appear in the
4048 // middle of the last block, so we should always emit the jump there.
4049 if (current->mir() != *gen->graph().poBegin() || lir->isGenerator()) {
4050 masm.jump(&returnLabel_);
4051 }
4052}
4053
4054void CodeGenerator::visitOsrEntry(LOsrEntry* lir) {
4055 Register temp = ToRegister(lir->temp());
4056
4057 // Remember the OSR entry offset into the code buffer.
4058 masm.flushBuffer();
4059 setOsrEntryOffset(masm.size());
4060
4061 // Allocate the full frame for this function
4062 // Note we have a new entry here. So we reset MacroAssembler::framePushed()
4063 // to 0, before reserving the stack.
4064 MOZ_ASSERT(masm.framePushed() == frameSize())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(masm.framePushed() == frameSize())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(masm.framePushed() == frameSize
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("masm.framePushed() == frameSize()", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 4064); AnnotateMozCrashReason("MOZ_ASSERT" "(" "masm.framePushed() == frameSize()"
")"); do { *((volatile int*)__null) = 4064; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
4065 masm.setFramePushed(0);
4066
4067 // The Baseline code ensured both the frame pointer and stack pointer point to
4068 // the JitFrameLayout on the stack.
4069
4070 // If profiling, save the current frame pointer to a per-thread global field.
4071 if (isProfilerInstrumentationEnabled()) {
4072 masm.profilerEnterFrame(FramePointer, temp);
4073 }
4074
4075 masm.reserveStack(frameSize());
4076 MOZ_ASSERT(masm.framePushed() == frameSize())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(masm.framePushed() == frameSize())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(masm.framePushed() == frameSize
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("masm.framePushed() == frameSize()", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 4076); AnnotateMozCrashReason("MOZ_ASSERT" "(" "masm.framePushed() == frameSize()"
")"); do { *((volatile int*)__null) = 4076; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
4077
4078 // Ensure that the Ion frames is properly aligned.
4079 masm.assertStackAlignment(JitStackAlignment, 0);
4080}
4081
4082void CodeGenerator::visitOsrEnvironmentChain(LOsrEnvironmentChain* lir) {
4083 const LAllocation* frame = lir->getOperand(0);
4084 const LDefinition* object = lir->getDef(0);
4085
4086 const ptrdiff_t frameOffset =
4087 BaselineFrame::reverseOffsetOfEnvironmentChain();
4088
4089 masm.loadPtr(Address(ToRegister(frame), frameOffset), ToRegister(object));
4090}
4091
4092void CodeGenerator::visitOsrArgumentsObject(LOsrArgumentsObject* lir) {
4093 const LAllocation* frame = lir->getOperand(0);
4094 const LDefinition* object = lir->getDef(0);
4095
4096 const ptrdiff_t frameOffset = BaselineFrame::reverseOffsetOfArgsObj();
4097
4098 masm.loadPtr(Address(ToRegister(frame), frameOffset), ToRegister(object));
4099}
4100
4101void CodeGenerator::visitOsrValue(LOsrValue* value) {
4102 const LAllocation* frame = value->getOperand(0);
4103 const ValueOperand out = ToOutValue(value);
4104
4105 const ptrdiff_t frameOffset = value->mir()->frameOffset();
4106
4107 masm.loadValue(Address(ToRegister(frame), frameOffset), out);
4108}
4109
4110void CodeGenerator::visitOsrReturnValue(LOsrReturnValue* lir) {
4111 const LAllocation* frame = lir->getOperand(0);
4112 const ValueOperand out = ToOutValue(lir);
4113
4114 Address flags =
4115 Address(ToRegister(frame), BaselineFrame::reverseOffsetOfFlags());
4116 Address retval =
4117 Address(ToRegister(frame), BaselineFrame::reverseOffsetOfReturnValue());
4118
4119 masm.moveValue(UndefinedValue(), out);
4120
4121 Label done;
4122 masm.branchTest32(Assembler::Zero, flags, Imm32(BaselineFrame::HAS_RVAL),
4123 &done);
4124 masm.loadValue(retval, out);
4125 masm.bind(&done);
4126}
4127
4128void CodeGenerator::visitStackArgT(LStackArgT* lir) {
4129 const LAllocation* arg = lir->arg();
4130 MIRType argType = lir->type();
4131 uint32_t argslot = lir->argslot();
4132 MOZ_ASSERT(argslot - 1u < graph.argumentSlotCount())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(argslot - 1u < graph.argumentSlotCount())>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(argslot - 1u < graph.argumentSlotCount()))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("argslot - 1u < graph.argumentSlotCount()"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 4132); AnnotateMozCrashReason("MOZ_ASSERT" "(" "argslot - 1u < graph.argumentSlotCount()"
")"); do { *((volatile int*)__null) = 4132; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
4133
4134 Address dest = AddressOfPassedArg(argslot);
4135
4136 if (arg->isFloatReg()) {
4137 masm.boxDouble(ToFloatRegister(arg), dest);
4138 } else if (arg->isRegister()) {
4139 masm.storeValue(ValueTypeFromMIRType(argType), ToRegister(arg), dest);
4140 } else {
4141 masm.storeValue(arg->toConstant()->toJSValue(), dest);
4142 }
4143}
4144
4145void CodeGenerator::visitStackArgV(LStackArgV* lir) {
4146 ValueOperand val = ToValue(lir, 0);
4147 uint32_t argslot = lir->argslot();
4148 MOZ_ASSERT(argslot - 1u < graph.argumentSlotCount())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(argslot - 1u < graph.argumentSlotCount())>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(argslot - 1u < graph.argumentSlotCount()))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("argslot - 1u < graph.argumentSlotCount()"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 4148); AnnotateMozCrashReason("MOZ_ASSERT" "(" "argslot - 1u < graph.argumentSlotCount()"
")"); do { *((volatile int*)__null) = 4148; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
4149
4150 masm.storeValue(val, AddressOfPassedArg(argslot));
4151}
4152
4153void CodeGenerator::visitMoveGroup(LMoveGroup* group) {
4154 if (!group->numMoves()) {
4155 return;
4156 }
4157
4158 MoveResolver& resolver = masm.moveResolver();
4159
4160 for (size_t i = 0; i < group->numMoves(); i++) {
4161 const LMove& move = group->getMove(i);
4162
4163 LAllocation from = move.from();
4164 LAllocation to = move.to();
4165 LDefinition::Type type = move.type();
4166
4167 // No bogus moves.
4168 MOZ_ASSERT(from != to)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(from != to)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(from != to))), 0))) { do { }
while (false); MOZ_ReportAssertionFailure("from != to", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 4168); AnnotateMozCrashReason("MOZ_ASSERT" "(" "from != to"
")"); do { *((volatile int*)__null) = 4168; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
4169 MOZ_ASSERT(!from.isConstant())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!from.isConstant())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!from.isConstant()))), 0))) {
do { } while (false); MOZ_ReportAssertionFailure("!from.isConstant()"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 4169); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!from.isConstant()"
")"); do { *((volatile int*)__null) = 4169; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
4170 MoveOp::Type moveType;
4171 switch (type) {
4172 case LDefinition::OBJECT:
4173 case LDefinition::SLOTS:
4174 case LDefinition::WASM_ANYREF:
4175#ifdef JS_NUNBOX32
4176 case LDefinition::TYPE:
4177 case LDefinition::PAYLOAD:
4178#else
4179 case LDefinition::BOX:
4180#endif
4181 case LDefinition::GENERAL:
4182 case LDefinition::STACKRESULTS:
4183 moveType = MoveOp::GENERAL;
4184 break;
4185 case LDefinition::INT32:
4186 moveType = MoveOp::INT32;
4187 break;
4188 case LDefinition::FLOAT32:
4189 moveType = MoveOp::FLOAT32;
4190 break;
4191 case LDefinition::DOUBLE:
4192 moveType = MoveOp::DOUBLE;
4193 break;
4194 case LDefinition::SIMD128:
4195 moveType = MoveOp::SIMD128;
4196 break;
4197 default:
4198 MOZ_CRASH("Unexpected move type")do { do { } while (false); MOZ_ReportCrash("" "Unexpected move type"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 4198); AnnotateMozCrashReason("MOZ_CRASH(" "Unexpected move type"
")"); do { *((volatile int*)__null) = 4198; __attribute__((nomerge
)) ::abort(); } while (false); } while (false)
;
4199 }
4200
4201 masm.propagateOOM(
4202 resolver.addMove(toMoveOperand(from), toMoveOperand(to), moveType));
4203 }
4204
4205 masm.propagateOOM(resolver.resolve());
4206 if (masm.oom()) {
4207 return;
4208 }
4209
4210 MoveEmitter emitter(masm);
4211
4212#ifdef JS_CODEGEN_X86
4213 if (group->maybeScratchRegister().isGeneralReg()) {
4214 emitter.setScratchRegister(
4215 group->maybeScratchRegister().toGeneralReg()->reg());
4216 } else {
4217 resolver.sortMemoryToMemoryMoves();
4218 }
4219#endif
4220
4221 emitter.emit(resolver);
4222 emitter.finish();
4223}
4224
4225void CodeGenerator::visitInteger(LInteger* lir) {
4226 masm.move32(Imm32(lir->i32()), ToRegister(lir->output()));
4227}
4228
4229void CodeGenerator::visitInteger64(LInteger64* lir) {
4230 masm.move64(Imm64(lir->i64()), ToOutRegister64(lir));
4231}
4232
4233void CodeGenerator::visitPointer(LPointer* lir) {
4234 masm.movePtr(ImmGCPtr(lir->gcptr()), ToRegister(lir->output()));
4235}
4236
4237void CodeGenerator::visitDouble(LDouble* ins) {
4238 masm.loadConstantDouble(ins->value(), ToFloatRegister(ins->output()));
4239}
4240
4241void CodeGenerator::visitFloat32(LFloat32* ins) {
4242 masm.loadConstantFloat32(ins->value(), ToFloatRegister(ins->output()));
4243}
4244
4245void CodeGenerator::visitValue(LValue* value) {
4246 ValueOperand result = ToOutValue(value);
4247 masm.moveValue(value->value(), result);
4248}
4249
4250void CodeGenerator::visitNurseryObject(LNurseryObject* lir) {
4251 Register output = ToRegister(lir->output());
4252 uint32_t nurseryIndex = lir->mir()->nurseryIndex();
4253
4254 // Load a pointer to the entry in IonScript's nursery objects list.
4255 CodeOffset label = masm.movWithPatch(ImmWord(uintptr_t(-1)), output);
4256 masm.propagateOOM(ionNurseryObjectLabels_.emplaceBack(label, nurseryIndex));
4257
4258 // Load the JSObject*.
4259 masm.loadPtr(Address(output, 0), output);
4260}
4261
4262void CodeGenerator::visitKeepAliveObject(LKeepAliveObject* lir) {
4263 // No-op.
4264}
4265
4266void CodeGenerator::visitDebugEnterGCUnsafeRegion(
4267 LDebugEnterGCUnsafeRegion* lir) {
4268 Register temp = ToRegister(lir->temp0());
4269
4270 masm.loadJSContext(temp);
4271
4272 Address inUnsafeRegion(temp, JSContext::offsetOfInUnsafeRegion());
4273 masm.add32(Imm32(1), inUnsafeRegion);
4274
4275 Label ok;
4276 masm.branch32(Assembler::GreaterThan, inUnsafeRegion, Imm32(0), &ok);
4277 masm.assumeUnreachable("unbalanced enter/leave GC unsafe region");
4278 masm.bind(&ok);
4279}
4280
4281void CodeGenerator::visitDebugLeaveGCUnsafeRegion(
4282 LDebugLeaveGCUnsafeRegion* lir) {
4283 Register temp = ToRegister(lir->temp0());
4284
4285 masm.loadJSContext(temp);
4286
4287 Address inUnsafeRegion(temp, JSContext::offsetOfInUnsafeRegion());
4288 masm.add32(Imm32(-1), inUnsafeRegion);
4289
4290 Label ok;
4291 masm.branch32(Assembler::GreaterThanOrEqual, inUnsafeRegion, Imm32(0), &ok);
4292 masm.assumeUnreachable("unbalanced enter/leave GC unsafe region");
4293 masm.bind(&ok);
4294}
4295
4296void CodeGenerator::visitSlots(LSlots* lir) {
4297 Address slots(ToRegister(lir->object()), NativeObject::offsetOfSlots());
4298 masm.loadPtr(slots, ToRegister(lir->output()));
4299}
4300
4301void CodeGenerator::visitLoadDynamicSlotV(LLoadDynamicSlotV* lir) {
4302 ValueOperand dest = ToOutValue(lir);
4303 Register base = ToRegister(lir->input());
4304 int32_t offset = lir->mir()->slot() * sizeof(js::Value);
4305
4306 masm.loadValue(Address(base, offset), dest);
4307}
4308
4309static ConstantOrRegister ToConstantOrRegister(const LAllocation* value,
4310 MIRType valueType) {
4311 if (value->isConstant()) {
4312 return ConstantOrRegister(value->toConstant()->toJSValue());
4313 }
4314 return TypedOrValueRegister(valueType, ToAnyRegister(value));
4315}
4316
4317void CodeGenerator::visitStoreDynamicSlotT(LStoreDynamicSlotT* lir) {
4318 Register base = ToRegister(lir->slots());
4319 int32_t offset = lir->mir()->slot() * sizeof(js::Value);
4320 Address dest(base, offset);
4321
4322 if (lir->mir()->needsBarrier()) {
4323 emitPreBarrier(dest);
4324 }
4325
4326 MIRType valueType = lir->mir()->value()->type();
4327 ConstantOrRegister value = ToConstantOrRegister(lir->value(), valueType);
4328 masm.storeUnboxedValue(value, valueType, dest);
4329}
4330
4331void CodeGenerator::visitStoreDynamicSlotV(LStoreDynamicSlotV* lir) {
4332 Register base = ToRegister(lir->slots());
4333 int32_t offset = lir->mir()->slot() * sizeof(Value);
4334
4335 const ValueOperand value = ToValue(lir, LStoreDynamicSlotV::ValueIndex);
4336
4337 if (lir->mir()->needsBarrier()) {
4338 emitPreBarrier(Address(base, offset));
4339 }
4340
4341 masm.storeValue(value, Address(base, offset));
4342}
4343
4344void CodeGenerator::visitElements(LElements* lir) {
4345 Address elements(ToRegister(lir->object()), NativeObject::offsetOfElements());
4346 masm.loadPtr(elements, ToRegister(lir->output()));
4347}
4348
4349void CodeGenerator::visitFunctionEnvironment(LFunctionEnvironment* lir) {
4350 Address environment(ToRegister(lir->function()),
4351 JSFunction::offsetOfEnvironment());
4352 masm.unboxObject(environment, ToRegister(lir->output()));
4353}
4354
4355void CodeGenerator::visitHomeObject(LHomeObject* lir) {
4356 Register func = ToRegister(lir->function());
4357 Address homeObject(func, FunctionExtended::offsetOfMethodHomeObjectSlot());
4358
4359 masm.assertFunctionIsExtended(func);
4360#ifdef DEBUG1
4361 Label isObject;
4362 masm.branchTestObject(Assembler::Equal, homeObject, &isObject);
4363 masm.assumeUnreachable("[[HomeObject]] must be Object");
4364 masm.bind(&isObject);
4365#endif
4366
4367 masm.unboxObject(homeObject, ToRegister(lir->output()));
4368}
4369
4370void CodeGenerator::visitHomeObjectSuperBase(LHomeObjectSuperBase* lir) {
4371 Register homeObject = ToRegister(lir->homeObject());
4372 ValueOperand output = ToOutValue(lir);
4373 Register temp = output.scratchReg();
4374
4375 masm.loadObjProto(homeObject, temp);
4376
4377#ifdef DEBUG1
4378 // We won't encounter a lazy proto, because the prototype is guaranteed to
4379 // either be a JSFunction or a PlainObject, and only proxy objects can have a
4380 // lazy proto.
4381 MOZ_ASSERT(uintptr_t(TaggedProto::LazyProto) == 1)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(uintptr_t(TaggedProto::LazyProto) == 1)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(uintptr_t(TaggedProto::LazyProto) == 1))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("uintptr_t(TaggedProto::LazyProto) == 1"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 4381); AnnotateMozCrashReason("MOZ_ASSERT" "(" "uintptr_t(TaggedProto::LazyProto) == 1"
")"); do { *((volatile int*)__null) = 4381; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
4382
4383 Label proxyCheckDone;
4384 masm.branchPtr(Assembler::NotEqual, temp, ImmWord(1), &proxyCheckDone);
4385 masm.assumeUnreachable("Unexpected lazy proto in JSOp::SuperBase");
4386 masm.bind(&proxyCheckDone);
4387#endif
4388
4389 Label nullProto, done;
4390 masm.branchPtr(Assembler::Equal, temp, ImmWord(0), &nullProto);
4391
4392 // Box prototype and return
4393 masm.tagValue(JSVAL_TYPE_OBJECT, temp, output);
4394 masm.jump(&done);
4395
4396 masm.bind(&nullProto);
4397 masm.moveValue(NullValue(), output);
4398
4399 masm.bind(&done);
4400}
4401
4402template <class T>
4403static T* ToConstantObject(MDefinition* def) {
4404 MOZ_ASSERT(def->isConstant())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(def->isConstant())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(def->isConstant()))), 0))
) { do { } while (false); MOZ_ReportAssertionFailure("def->isConstant()"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 4404); AnnotateMozCrashReason("MOZ_ASSERT" "(" "def->isConstant()"
")"); do { *((volatile int*)__null) = 4404; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
4405 return &def->toConstant()->toObject().as<T>();
4406}
4407
4408void CodeGenerator::visitNewLexicalEnvironmentObject(
4409 LNewLexicalEnvironmentObject* lir) {
4410 Register output = ToRegister(lir->output());
4411 Register temp = ToRegister(lir->temp0());
4412
4413 auto* templateObj = ToConstantObject<BlockLexicalEnvironmentObject>(
4414 lir->mir()->templateObj());
4415 auto* scope = &templateObj->scope();
4416 gc::Heap initialHeap = gc::Heap::Default;
4417
4418 using Fn =
4419 BlockLexicalEnvironmentObject* (*)(JSContext*, Handle<LexicalScope*>);
4420 auto* ool =
4421 oolCallVM<Fn, BlockLexicalEnvironmentObject::createWithoutEnclosing>(
4422 lir, ArgList(ImmGCPtr(scope)), StoreRegisterTo(output));
4423
4424 TemplateObject templateObject(templateObj);
4425 masm.createGCObject(output, temp, templateObject, initialHeap, ool->entry());
4426
4427 masm.bind(ool->rejoin());
4428}
4429
4430void CodeGenerator::visitNewClassBodyEnvironmentObject(
4431 LNewClassBodyEnvironmentObject* lir) {
4432 Register output = ToRegister(lir->output());
4433 Register temp = ToRegister(lir->temp0());
4434
4435 auto* templateObj = ToConstantObject<ClassBodyLexicalEnvironmentObject>(
4436 lir->mir()->templateObj());
4437 auto* scope = &templateObj->scope();
4438 gc::Heap initialHeap = gc::Heap::Default;
4439
4440 using Fn = ClassBodyLexicalEnvironmentObject* (*)(JSContext*,
4441 Handle<ClassBodyScope*>);
4442 auto* ool =
4443 oolCallVM<Fn, ClassBodyLexicalEnvironmentObject::createWithoutEnclosing>(
4444 lir, ArgList(ImmGCPtr(scope)), StoreRegisterTo(output));
4445
4446 TemplateObject templateObject(templateObj);
4447 masm.createGCObject(output, temp, templateObject, initialHeap, ool->entry());
4448
4449 masm.bind(ool->rejoin());
4450}
4451
4452void CodeGenerator::visitNewVarEnvironmentObject(
4453 LNewVarEnvironmentObject* lir) {
4454 Register output = ToRegister(lir->output());
4455 Register temp = ToRegister(lir->temp0());
4456
4457 auto* templateObj =
4458 ToConstantObject<VarEnvironmentObject>(lir->mir()->templateObj());
4459 auto* scope = &templateObj->scope().as<VarScope>();
4460 gc::Heap initialHeap = gc::Heap::Default;
4461
4462 using Fn = VarEnvironmentObject* (*)(JSContext*, Handle<VarScope*>);
4463 auto* ool = oolCallVM<Fn, VarEnvironmentObject::createWithoutEnclosing>(
4464 lir, ArgList(ImmGCPtr(scope)), StoreRegisterTo(output));
4465
4466 TemplateObject templateObject(templateObj);
4467 masm.createGCObject(output, temp, templateObject, initialHeap, ool->entry());
4468
4469 masm.bind(ool->rejoin());
4470}
4471
4472void CodeGenerator::visitGuardShape(LGuardShape* guard) {
4473 Register obj = ToRegister(guard->input());
4474 Register temp = ToTempRegisterOrInvalid(guard->temp0());
4475 Label bail;
4476 masm.branchTestObjShape(Assembler::NotEqual, obj, guard->mir()->shape(), temp,
4477 obj, &bail);
4478 bailoutFrom(&bail, guard->snapshot());
4479}
4480
4481void CodeGenerator::visitGuardFuse(LGuardFuse* guard) {
4482 auto fuseIndex = guard->mir()->fuseIndex();
4483
4484 Register temp = ToRegister(guard->temp0());
4485 Label bail;
4486
4487 // Bake specific fuse address for Ion code, because we won't share this code
4488 // across realms.
4489 GuardFuse* fuse = mirGen().realm->realmFuses().getFuseByIndex(fuseIndex);
4490 masm.loadPtr(AbsoluteAddress(fuse->fuseRef()), temp);
4491 masm.branchPtr(Assembler::NotEqual, temp, ImmPtr(nullptr), &bail);
4492
4493 bailoutFrom(&bail, guard->snapshot());
4494}
4495
4496void CodeGenerator::visitGuardMultipleShapes(LGuardMultipleShapes* guard) {
4497 Register obj = ToRegister(guard->object());
4498 Register shapeList = ToRegister(guard->shapeList());
4499 Register temp = ToRegister(guard->temp0());
4500 Register temp2 = ToRegister(guard->temp1());
4501 Register temp3 = ToRegister(guard->temp2());
4502 Register spectre = ToTempRegisterOrInvalid(guard->temp3());
4503
4504 Label bail;
4505 masm.loadPtr(Address(shapeList, NativeObject::offsetOfElements()), temp);
4506 masm.branchTestObjShapeList(Assembler::NotEqual, obj, temp, temp2, temp3,
4507 spectre, &bail);
4508 bailoutFrom(&bail, guard->snapshot());
4509}
4510
4511void CodeGenerator::visitGuardProto(LGuardProto* guard) {
4512 Register obj = ToRegister(guard->object());
4513 Register expected = ToRegister(guard->expected());
4514 Register temp = ToRegister(guard->temp0());
4515
4516 masm.loadObjProto(obj, temp);
4517
4518 Label bail;
4519 masm.branchPtr(Assembler::NotEqual, temp, expected, &bail);
4520 bailoutFrom(&bail, guard->snapshot());
4521}
4522
4523void CodeGenerator::visitGuardNullProto(LGuardNullProto* guard) {
4524 Register obj = ToRegister(guard->input());
4525 Register temp = ToRegister(guard->temp0());
4526
4527 masm.loadObjProto(obj, temp);
4528
4529 Label bail;
4530 masm.branchTestPtr(Assembler::NonZero, temp, temp, &bail);
4531 bailoutFrom(&bail, guard->snapshot());
4532}
4533
4534void CodeGenerator::visitGuardIsNativeObject(LGuardIsNativeObject* guard) {
4535 Register obj = ToRegister(guard->input());
4536 Register temp = ToRegister(guard->temp0());
4537
4538 Label bail;
4539 masm.branchIfNonNativeObj(obj, temp, &bail);
4540 bailoutFrom(&bail, guard->snapshot());
4541}
4542
4543void CodeGenerator::visitGuardGlobalGeneration(LGuardGlobalGeneration* guard) {
4544 Register temp = ToRegister(guard->temp0());
4545 Label bail;
4546
4547 masm.load32(AbsoluteAddress(guard->mir()->generationAddr()), temp);
4548 masm.branch32(Assembler::NotEqual, temp, Imm32(guard->mir()->expected()),
4549 &bail);
4550 bailoutFrom(&bail, guard->snapshot());
4551}
4552
4553void CodeGenerator::visitGuardIsProxy(LGuardIsProxy* guard) {
4554 Register obj = ToRegister(guard->input());
4555 Register temp = ToRegister(guard->temp0());
4556
4557 Label bail;
4558 masm.branchTestObjectIsProxy(false, obj, temp, &bail);
4559 bailoutFrom(&bail, guard->snapshot());
4560}
4561
4562void CodeGenerator::visitGuardIsNotProxy(LGuardIsNotProxy* guard) {
4563 Register obj = ToRegister(guard->input());
4564 Register temp = ToRegister(guard->temp0());
4565
4566 Label bail;
4567 masm.branchTestObjectIsProxy(true, obj, temp, &bail);
4568 bailoutFrom(&bail, guard->snapshot());
4569}
4570
4571void CodeGenerator::visitGuardIsNotDOMProxy(LGuardIsNotDOMProxy* guard) {
4572 Register proxy = ToRegister(guard->proxy());
4573 Register temp = ToRegister(guard->temp0());
4574
4575 Label bail;
4576 masm.branchTestProxyHandlerFamily(Assembler::Equal, proxy, temp,
4577 GetDOMProxyHandlerFamily(), &bail);
4578 bailoutFrom(&bail, guard->snapshot());
4579}
4580
4581void CodeGenerator::visitProxyGet(LProxyGet* lir) {
4582 Register proxy = ToRegister(lir->proxy());
4583 Register temp = ToRegister(lir->temp0());
4584
4585 pushArg(lir->mir()->id(), temp);
4586 pushArg(proxy);
4587
4588 using Fn = bool (*)(JSContext*, HandleObject, HandleId, MutableHandleValue);
4589 callVM<Fn, ProxyGetProperty>(lir);
4590}
4591
4592void CodeGenerator::visitProxyGetByValue(LProxyGetByValue* lir) {
4593 Register proxy = ToRegister(lir->proxy());
4594 ValueOperand idVal = ToValue(lir, LProxyGetByValue::IdIndex);
4595
4596 pushArg(idVal);
4597 pushArg(proxy);
4598
4599 using Fn =
4600 bool (*)(JSContext*, HandleObject, HandleValue, MutableHandleValue);
4601 callVM<Fn, ProxyGetPropertyByValue>(lir);
4602}
4603
4604void CodeGenerator::visitProxyHasProp(LProxyHasProp* lir) {
4605 Register proxy = ToRegister(lir->proxy());
4606 ValueOperand idVal = ToValue(lir, LProxyHasProp::IdIndex);
4607
4608 pushArg(idVal);
4609 pushArg(proxy);
4610
4611 using Fn = bool (*)(JSContext*, HandleObject, HandleValue, bool*);
4612 if (lir->mir()->hasOwn()) {
4613 callVM<Fn, ProxyHasOwn>(lir);
4614 } else {
4615 callVM<Fn, ProxyHas>(lir);
4616 }
4617}
4618
4619void CodeGenerator::visitProxySet(LProxySet* lir) {
4620 Register proxy = ToRegister(lir->proxy());
4621 ValueOperand rhs = ToValue(lir, LProxySet::RhsIndex);
4622 Register temp = ToRegister(lir->temp0());
4623
4624 pushArg(Imm32(lir->mir()->strict()));
4625 pushArg(rhs);
4626 pushArg(lir->mir()->id(), temp);
4627 pushArg(proxy);
4628
4629 using Fn = bool (*)(JSContext*, HandleObject, HandleId, HandleValue, bool);
4630 callVM<Fn, ProxySetProperty>(lir);
4631}
4632
4633void CodeGenerator::visitProxySetByValue(LProxySetByValue* lir) {
4634 Register proxy = ToRegister(lir->proxy());
4635 ValueOperand idVal = ToValue(lir, LProxySetByValue::IdIndex);
4636 ValueOperand rhs = ToValue(lir, LProxySetByValue::RhsIndex);
4637
4638 pushArg(Imm32(lir->mir()->strict()));
4639 pushArg(rhs);
4640 pushArg(idVal);
4641 pushArg(proxy);
4642
4643 using Fn = bool (*)(JSContext*, HandleObject, HandleValue, HandleValue, bool);
4644 callVM<Fn, ProxySetPropertyByValue>(lir);
4645}
4646
4647void CodeGenerator::visitCallSetArrayLength(LCallSetArrayLength* lir) {
4648 Register obj = ToRegister(lir->obj());
4649 ValueOperand rhs = ToValue(lir, LCallSetArrayLength::RhsIndex);
4650
4651 pushArg(Imm32(lir->mir()->strict()));
4652 pushArg(rhs);
4653 pushArg(obj);
4654
4655 using Fn = bool (*)(JSContext*, HandleObject, HandleValue, bool);
4656 callVM<Fn, jit::SetArrayLength>(lir);
4657}
4658
4659void CodeGenerator::visitMegamorphicLoadSlot(LMegamorphicLoadSlot* lir) {
4660 Register obj = ToRegister(lir->object());
4661 Register temp0 = ToRegister(lir->temp0());
4662 Register temp1 = ToRegister(lir->temp1());
4663 Register temp2 = ToRegister(lir->temp2());
4664 Register temp3 = ToRegister(lir->temp3());
4665 ValueOperand output = ToOutValue(lir);
4666
4667 Label cacheHit;
4668 masm.emitMegamorphicCacheLookup(lir->mir()->name(), obj, temp0, temp1, temp2,
4669 output, &cacheHit);
4670
4671 Label bail;
4672 masm.branchIfNonNativeObj(obj, temp0, &bail);
4673
4674 masm.Push(UndefinedValue());
4675 masm.moveStackPtrTo(temp3);
4676
4677 using Fn = bool (*)(JSContext* cx, JSObject* obj, PropertyKey id,
4678 MegamorphicCache::Entry* cacheEntry, Value* vp);
4679 masm.setupAlignedABICall();
4680 masm.loadJSContext(temp0);
4681 masm.passABIArg(temp0);
4682 masm.passABIArg(obj);
4683 masm.movePropertyKey(lir->mir()->name(), temp1);
4684 masm.passABIArg(temp1);
4685 masm.passABIArg(temp2);
4686 masm.passABIArg(temp3);
4687
4688 masm.callWithABI<Fn, GetNativeDataPropertyPure>();
4689
4690 MOZ_ASSERT(!output.aliases(ReturnReg))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!output.aliases(ReturnReg))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!output.aliases(ReturnReg)))
), 0))) { do { } while (false); MOZ_ReportAssertionFailure("!output.aliases(ReturnReg)"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 4690); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!output.aliases(ReturnReg)"
")"); do { *((volatile int*)__null) = 4690; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
4691 masm.Pop(output);
4692
4693 masm.branchIfFalseBool(ReturnReg, &bail);
4694 masm.bind(&cacheHit);
4695
4696 bailoutFrom(&bail, lir->snapshot());
4697}
4698
4699void CodeGenerator::visitMegamorphicLoadSlotPermissive(
4700 LMegamorphicLoadSlotPermissive* lir) {
4701 Register obj = ToRegister(lir->object());
4702 Register temp0 = ToRegister(lir->temp0());
4703 Register temp1 = ToRegister(lir->temp1());
4704 Register temp2 = ToRegister(lir->temp2());
4705 ValueOperand output = ToOutValue(lir);
4706
4707 Label cacheHit;
4708 masm.emitMegamorphicCacheLookup(lir->mir()->name(), obj, temp0, temp1, temp2,
4709 output, &cacheHit);
4710
4711 masm.movePropertyKey(lir->mir()->name(), temp1);
4712 pushArg(temp2);
4713 pushArg(temp1);
4714 pushArg(obj);
4715
4716 using Fn = bool (*)(JSContext*, HandleObject, HandleId,
4717 MegamorphicCacheEntry*, MutableHandleValue);
4718 callVM<Fn, GetPropMaybeCached>(lir);
4719
4720 masm.bind(&cacheHit);
4721}
4722
4723void CodeGenerator::visitMegamorphicLoadSlotByValue(
4724 LMegamorphicLoadSlotByValue* lir) {
4725 Register obj = ToRegister(lir->object());
4726 ValueOperand idVal = ToValue(lir, LMegamorphicLoadSlotByValue::IdIndex);
4727 Register temp0 = ToRegister(lir->temp0());
4728 Register temp1 = ToRegister(lir->temp1());
4729 Register temp2 = ToRegister(lir->temp2());
4730 ValueOperand output = ToOutValue(lir);
4731
4732 Label cacheHit, bail;
4733 masm.emitMegamorphicCacheLookupByValue(idVal, obj, temp0, temp1, temp2,
4734 output, &cacheHit);
4735
4736 masm.branchIfNonNativeObj(obj, temp0, &bail);
4737
4738 // idVal will be in vp[0], result will be stored in vp[1].
4739 masm.reserveStack(sizeof(Value));
4740 masm.Push(idVal);
4741 masm.moveStackPtrTo(temp0);
4742
4743 using Fn = bool (*)(JSContext* cx, JSObject* obj,
4744 MegamorphicCache::Entry* cacheEntry, Value* vp);
4745 masm.setupAlignedABICall();
4746 masm.loadJSContext(temp1);
4747 masm.passABIArg(temp1);
4748 masm.passABIArg(obj);
4749 masm.passABIArg(temp2);
4750 masm.passABIArg(temp0);
4751 masm.callWithABI<Fn, GetNativeDataPropertyByValuePure>();
4752
4753 MOZ_ASSERT(!idVal.aliases(temp0))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!idVal.aliases(temp0))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!idVal.aliases(temp0)))), 0)
)) { do { } while (false); MOZ_ReportAssertionFailure("!idVal.aliases(temp0)"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 4753); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!idVal.aliases(temp0)"
")"); do { *((volatile int*)__null) = 4753; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
4754 masm.storeCallPointerResult(temp0);
4755 masm.Pop(idVal);
4756
4757 uint32_t framePushed = masm.framePushed();
4758 Label ok;
4759 masm.branchIfTrueBool(temp0, &ok);
4760 masm.freeStack(sizeof(Value)); // Discard result Value.
4761 masm.jump(&bail);
4762
4763 masm.bind(&ok);
4764 masm.setFramePushed(framePushed);
4765 masm.Pop(output);
4766
4767 masm.bind(&cacheHit);
4768
4769 bailoutFrom(&bail, lir->snapshot());
4770}
4771
4772void CodeGenerator::visitMegamorphicLoadSlotByValuePermissive(
4773 LMegamorphicLoadSlotByValuePermissive* lir) {
4774 Register obj = ToRegister(lir->object());
4775 ValueOperand idVal = ToValue(lir, LMegamorphicLoadSlotByValue::IdIndex);
4776 Register temp0 = ToRegister(lir->temp0());
4777 Register temp1 = ToRegister(lir->temp1());
4778 Register temp2 = ToRegister(lir->temp2());
4779 ValueOperand output = ToOutValue(lir);
4780
4781 Label cacheHit;
4782 masm.emitMegamorphicCacheLookupByValue(idVal, obj, temp0, temp1, temp2,
4783 output, &cacheHit);
4784
4785 pushArg(temp2);
4786 pushArg(idVal);
4787 pushArg(obj);
4788
4789 using Fn = bool (*)(JSContext*, HandleObject, HandleValue,
4790 MegamorphicCacheEntry*, MutableHandleValue);
4791 callVM<Fn, GetElemMaybeCached>(lir);
4792
4793 masm.bind(&cacheHit);
4794}
4795
4796void CodeGenerator::visitMegamorphicStoreSlot(LMegamorphicStoreSlot* lir) {
4797 Register obj = ToRegister(lir->object());
4798 ValueOperand value = ToValue(lir, LMegamorphicStoreSlot::RhsIndex);
4799
4800 Register temp0 = ToRegister(lir->temp0());
4801#ifndef JS_CODEGEN_X86
4802 Register temp1 = ToRegister(lir->temp1());
4803 Register temp2 = ToRegister(lir->temp2());
4804#endif
4805
4806 // The instruction is marked as call-instruction so only these registers are
4807 // live.
4808 LiveRegisterSet liveRegs;
4809 liveRegs.addUnchecked(obj);
4810 liveRegs.addUnchecked(value);
4811 liveRegs.addUnchecked(temp0);
4812#ifndef JS_CODEGEN_X86
4813 liveRegs.addUnchecked(temp1);
4814 liveRegs.addUnchecked(temp2);
4815#endif
4816
4817 Label cacheHit, done;
4818#ifdef JS_CODEGEN_X86
4819 masm.emitMegamorphicCachedSetSlot(
4820 lir->mir()->name(), obj, temp0, value, liveRegs, &cacheHit,
4821 [](MacroAssembler& masm, const Address& addr, MIRType mirType) {
4822 EmitPreBarrier(masm, addr, mirType);
4823 });
4824#else
4825 masm.emitMegamorphicCachedSetSlot(
4826 lir->mir()->name(), obj, temp0, temp1, temp2, value, liveRegs, &cacheHit,
4827 [](MacroAssembler& masm, const Address& addr, MIRType mirType) {
4828 EmitPreBarrier(masm, addr, mirType);
4829 });
4830#endif
4831
4832 pushArg(Imm32(lir->mir()->strict()));
4833 pushArg(value);
4834 pushArg(lir->mir()->name(), temp0);
4835 pushArg(obj);
4836
4837 using Fn = bool (*)(JSContext*, HandleObject, HandleId, HandleValue, bool);
4838 callVM<Fn, SetPropertyMegamorphic<true>>(lir);
4839
4840 masm.jump(&done);
4841 masm.bind(&cacheHit);
4842
4843 masm.branchPtrInNurseryChunk(Assembler::Equal, obj, temp0, &done);
4844 masm.branchValueIsNurseryCell(Assembler::NotEqual, value, temp0, &done);
4845
4846 // Note: because this is a call-instruction, no registers need to be saved.
4847 MOZ_ASSERT(lir->isCall())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(lir->isCall())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(lir->isCall()))), 0))) { do
{ } while (false); MOZ_ReportAssertionFailure("lir->isCall()"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 4847); AnnotateMozCrashReason("MOZ_ASSERT" "(" "lir->isCall()"
")"); do { *((volatile int*)__null) = 4847; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
4848 emitPostWriteBarrier(obj);
4849
4850 masm.bind(&done);
4851}
4852
4853void CodeGenerator::visitMegamorphicHasProp(LMegamorphicHasProp* lir) {
4854 Register obj = ToRegister(lir->object());
4855 ValueOperand idVal = ToValue(lir, LMegamorphicHasProp::IdIndex);
4856 Register temp0 = ToRegister(lir->temp0());
4857 Register temp1 = ToRegister(lir->temp1());
4858 Register temp2 = ToRegister(lir->temp2());
4859 Register output = ToRegister(lir->output());
4860
4861 Label bail, cacheHit;
4862 masm.emitMegamorphicCacheLookupExists(idVal, obj, temp0, temp1, temp2, output,
4863 &cacheHit, lir->mir()->hasOwn());
4864
4865 masm.branchIfNonNativeObj(obj, temp0, &bail);
4866
4867 // idVal will be in vp[0], result will be stored in vp[1].
4868 masm.reserveStack(sizeof(Value));
4869 masm.Push(idVal);
4870 masm.moveStackPtrTo(temp0);
4871
4872 using Fn = bool (*)(JSContext* cx, JSObject* obj,
4873 MegamorphicCache::Entry* cacheEntry, Value* vp);
4874 masm.setupAlignedABICall();
4875 masm.loadJSContext(temp1);
4876 masm.passABIArg(temp1);
4877 masm.passABIArg(obj);
4878 masm.passABIArg(temp2);
4879 masm.passABIArg(temp0);
4880 if (lir->mir()->hasOwn()) {
4881 masm.callWithABI<Fn, HasNativeDataPropertyPure<true>>();
4882 } else {
4883 masm.callWithABI<Fn, HasNativeDataPropertyPure<false>>();
4884 }
4885
4886 MOZ_ASSERT(!idVal.aliases(temp0))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!idVal.aliases(temp0))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!idVal.aliases(temp0)))), 0)
)) { do { } while (false); MOZ_ReportAssertionFailure("!idVal.aliases(temp0)"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 4886); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!idVal.aliases(temp0)"
")"); do { *((volatile int*)__null) = 4886; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
4887 masm.storeCallPointerResult(temp0);
4888 masm.Pop(idVal);
4889
4890 uint32_t framePushed = masm.framePushed();
4891 Label ok;
4892 masm.branchIfTrueBool(temp0, &ok);
4893 masm.freeStack(sizeof(Value)); // Discard result Value.
4894 masm.jump(&bail);
4895
4896 masm.bind(&ok);
4897 masm.setFramePushed(framePushed);
4898 masm.unboxBoolean(Address(masm.getStackPointer(), 0), output);
4899 masm.freeStack(sizeof(Value));
4900 masm.bind(&cacheHit);
4901
4902 bailoutFrom(&bail, lir->snapshot());
4903}
4904
4905void CodeGenerator::visitSmallObjectVariableKeyHasProp(
4906 LSmallObjectVariableKeyHasProp* lir) {
4907 Register id = ToRegister(lir->id());
4908 Register output = ToRegister(lir->output());
4909
4910#ifdef DEBUG1
4911 Label isAtom;
4912 masm.branchTest32(Assembler::NonZero, Address(id, JSString::offsetOfFlags()),
4913 Imm32(JSString::ATOM_BIT), &isAtom);
4914 masm.assumeUnreachable("Expected atom input");
4915 masm.bind(&isAtom);
4916#endif
4917
4918 SharedShape* shape = &lir->mir()->shape()->asShared();
4919
4920 Label done, success;
4921 for (SharedShapePropertyIter<NoGC> iter(shape); !iter.done(); iter++) {
4922 masm.branchPtr(Assembler::Equal, id, ImmGCPtr(iter->key().toAtom()),
4923 &success);
4924 }
4925 masm.move32(Imm32(0), output);
4926 masm.jump(&done);
4927 masm.bind(&success);
4928 masm.move32(Imm32(1), output);
4929 masm.bind(&done);
4930}
4931
4932void CodeGenerator::visitGuardIsNotArrayBufferMaybeShared(
4933 LGuardIsNotArrayBufferMaybeShared* guard) {
4934 Register obj = ToRegister(guard->input());
4935 Register temp = ToRegister(guard->temp0());
4936
4937 Label bail;
4938 masm.loadObjClassUnsafe(obj, temp);
4939 masm.branchPtr(Assembler::Equal, temp,
4940 ImmPtr(&FixedLengthArrayBufferObject::class_), &bail);
4941 masm.branchPtr(Assembler::Equal, temp,
4942 ImmPtr(&FixedLengthSharedArrayBufferObject::class_), &bail);
4943 masm.branchPtr(Assembler::Equal, temp,
4944 ImmPtr(&ResizableArrayBufferObject::class_), &bail);
4945 masm.branchPtr(Assembler::Equal, temp,
4946 ImmPtr(&GrowableSharedArrayBufferObject::class_), &bail);
4947 bailoutFrom(&bail, guard->snapshot());
4948}
4949
4950void CodeGenerator::visitGuardIsTypedArray(LGuardIsTypedArray* guard) {
4951 Register obj = ToRegister(guard->input());
4952 Register temp = ToRegister(guard->temp0());
4953
4954 Label bail;
4955 masm.loadObjClassUnsafe(obj, temp);
4956 masm.branchIfClassIsNotTypedArray(temp, &bail);
4957 bailoutFrom(&bail, guard->snapshot());
4958}
4959
4960void CodeGenerator::visitGuardIsFixedLengthTypedArray(
4961 LGuardIsFixedLengthTypedArray* guard) {
4962 Register obj = ToRegister(guard->input());
4963 Register temp = ToRegister(guard->temp0());
4964
4965 Label bail;
4966 masm.loadObjClassUnsafe(obj, temp);
4967 masm.branchIfClassIsNotFixedLengthTypedArray(temp, &bail);
4968 bailoutFrom(&bail, guard->snapshot());
4969}
4970
4971void CodeGenerator::visitGuardIsResizableTypedArray(
4972 LGuardIsResizableTypedArray* guard) {
4973 Register obj = ToRegister(guard->input());
4974 Register temp = ToRegister(guard->temp0());
4975
4976 Label bail;
4977 masm.loadObjClassUnsafe(obj, temp);
4978 masm.branchIfClassIsNotResizableTypedArray(temp, &bail);
4979 bailoutFrom(&bail, guard->snapshot());
4980}
4981
4982void CodeGenerator::visitGuardHasProxyHandler(LGuardHasProxyHandler* guard) {
4983 Register obj = ToRegister(guard->input());
4984
4985 Label bail;
4986
4987 Address handlerAddr(obj, ProxyObject::offsetOfHandler());
4988 masm.branchPtr(Assembler::NotEqual, handlerAddr,
4989 ImmPtr(guard->mir()->handler()), &bail);
4990
4991 bailoutFrom(&bail, guard->snapshot());
4992}
4993
4994void CodeGenerator::visitGuardObjectIdentity(LGuardObjectIdentity* guard) {
4995 Register input = ToRegister(guard->input());
4996 Register expected = ToRegister(guard->expected());
4997
4998 Assembler::Condition cond =
4999 guard->mir()->bailOnEquality() ? Assembler::Equal : Assembler::NotEqual;
5000 bailoutCmpPtr(cond, input, expected, guard->snapshot());
5001}
5002
5003void CodeGenerator::visitGuardSpecificFunction(LGuardSpecificFunction* guard) {
5004 Register input = ToRegister(guard->input());
5005 Register expected = ToRegister(guard->expected());
5006
5007 bailoutCmpPtr(Assembler::NotEqual, input, expected, guard->snapshot());
5008}
5009
5010void CodeGenerator::visitGuardSpecificAtom(LGuardSpecificAtom* guard) {
5011 Register str = ToRegister(guard->str());
5012 Register scratch = ToRegister(guard->temp0());
5013
5014 LiveRegisterSet volatileRegs = liveVolatileRegs(guard);
5015 volatileRegs.takeUnchecked(scratch);
5016
5017 Label bail;
5018 masm.guardSpecificAtom(str, guard->mir()->atom(), scratch, volatileRegs,
5019 &bail);
5020 bailoutFrom(&bail, guard->snapshot());
5021}
5022
5023void CodeGenerator::visitGuardSpecificSymbol(LGuardSpecificSymbol* guard) {
5024 Register symbol = ToRegister(guard->symbol());
5025
5026 bailoutCmpPtr(Assembler::NotEqual, symbol, ImmGCPtr(guard->mir()->expected()),
5027 guard->snapshot());
5028}
5029
5030void CodeGenerator::visitGuardSpecificInt32(LGuardSpecificInt32* guard) {
5031 Register num = ToRegister(guard->num());
5032
5033 bailoutCmp32(Assembler::NotEqual, num, Imm32(guard->mir()->expected()),
5034 guard->snapshot());
5035}
5036
5037void CodeGenerator::visitGuardStringToIndex(LGuardStringToIndex* lir) {
5038 Register str = ToRegister(lir->string());
5039 Register output = ToRegister(lir->output());
5040
5041 Label vmCall, done;
5042 masm.loadStringIndexValue(str, output, &vmCall);
5043 masm.jump(&done);
5044
5045 {
5046 masm.bind(&vmCall);
5047
5048 LiveRegisterSet volatileRegs = liveVolatileRegs(lir);
5049 volatileRegs.takeUnchecked(output);
5050 masm.PushRegsInMask(volatileRegs);
5051
5052 using Fn = int32_t (*)(JSString* str);
5053 masm.setupAlignedABICall();
5054 masm.passABIArg(str);
5055 masm.callWithABI<Fn, GetIndexFromString>();
5056 masm.storeCallInt32Result(output);
5057
5058 masm.PopRegsInMask(volatileRegs);
5059
5060 // GetIndexFromString returns a negative value on failure.
5061 bailoutTest32(Assembler::Signed, output, output, lir->snapshot());
5062 }
5063
5064 masm.bind(&done);
5065}
5066
5067void CodeGenerator::visitGuardStringToInt32(LGuardStringToInt32* lir) {
5068 Register str = ToRegister(lir->string());
5069 Register output = ToRegister(lir->output());
5070 Register temp = ToRegister(lir->temp0());
5071
5072 LiveRegisterSet volatileRegs = liveVolatileRegs(lir);
5073
5074 Label bail;
5075 masm.guardStringToInt32(str, output, temp, volatileRegs, &bail);
5076 bailoutFrom(&bail, lir->snapshot());
5077}
5078
5079void CodeGenerator::visitGuardStringToDouble(LGuardStringToDouble* lir) {
5080 Register str = ToRegister(lir->string());
5081 FloatRegister output = ToFloatRegister(lir->output());
5082 Register temp0 = ToRegister(lir->temp0());
5083 Register temp1 = ToRegister(lir->temp1());
5084
5085 Label vmCall, done;
5086 // Use indexed value as fast path if possible.
5087 masm.loadStringIndexValue(str, temp0, &vmCall);
5088 masm.convertInt32ToDouble(temp0, output);
5089 masm.jump(&done);
5090 {
5091 masm.bind(&vmCall);
5092
5093 // Reserve stack for holding the result value of the call.
5094 masm.reserveStack(sizeof(double));
5095 masm.moveStackPtrTo(temp0);
5096
5097 LiveRegisterSet volatileRegs = liveVolatileRegs(lir);
5098 volatileRegs.takeUnchecked(temp0);
5099 volatileRegs.takeUnchecked(temp1);
5100 masm.PushRegsInMask(volatileRegs);
5101
5102 using Fn = bool (*)(JSContext* cx, JSString* str, double* result);
5103 masm.setupAlignedABICall();
5104 masm.loadJSContext(temp1);
5105 masm.passABIArg(temp1);
5106 masm.passABIArg(str);
5107 masm.passABIArg(temp0);
5108 masm.callWithABI<Fn, StringToNumberPure>();
5109 masm.storeCallPointerResult(temp0);
5110
5111 masm.PopRegsInMask(volatileRegs);
5112
5113 Label ok;
5114 masm.branchIfTrueBool(temp0, &ok);
5115 {
5116 // OOM path, recovered by StringToNumberPure.
5117 //
5118 // Use addToStackPtr instead of freeStack as freeStack tracks stack height
5119 // flow-insensitively, and using it here would confuse the stack height
5120 // tracking.
5121 masm.addToStackPtr(Imm32(sizeof(double)));
5122 bailout(lir->snapshot());
5123 }
5124 masm.bind(&ok);
5125 masm.Pop(output);
5126 }
5127 masm.bind(&done);
5128}
5129
5130void CodeGenerator::visitGuardNoDenseElements(LGuardNoDenseElements* guard) {
5131 Register obj = ToRegister(guard->input());
5132 Register temp = ToRegister(guard->temp0());
5133
5134 // Load obj->elements.
5135 masm.loadPtr(Address(obj, NativeObject::offsetOfElements()), temp);
5136
5137 // Make sure there are no dense elements.
5138 Address initLength(temp, ObjectElements::offsetOfInitializedLength());
5139 bailoutCmp32(Assembler::NotEqual, initLength, Imm32(0), guard->snapshot());
5140}
5141
5142void CodeGenerator::visitBooleanToInt64(LBooleanToInt64* lir) {
5143 Register input = ToRegister(lir->input());
5144 Register64 output = ToOutRegister64(lir);
5145
5146 masm.move32To64ZeroExtend(input, output);
5147}
5148
5149void CodeGenerator::emitStringToInt64(LInstruction* lir, Register input,
5150 Register64 output) {
5151 Register temp = output.scratchReg();
5152
5153 saveLive(lir);
5154
5155 masm.reserveStack(sizeof(uint64_t));
5156 masm.moveStackPtrTo(temp);
5157 pushArg(temp);
5158 pushArg(input);
5159
5160 using Fn = bool (*)(JSContext*, HandleString, uint64_t*);
5161 callVM<Fn, DoStringToInt64>(lir);
5162
5163 masm.load64(Address(masm.getStackPointer(), 0), output);
5164 masm.freeStack(sizeof(uint64_t));
5165
5166 restoreLiveIgnore(lir, StoreValueTo(output).clobbered());
5167}
5168
5169void CodeGenerator::visitStringToInt64(LStringToInt64* lir) {
5170 Register input = ToRegister(lir->input());
5171 Register64 output = ToOutRegister64(lir);
5172
5173 emitStringToInt64(lir, input, output);
5174}
5175
5176void CodeGenerator::visitValueToInt64(LValueToInt64* lir) {
5177 ValueOperand input = ToValue(lir, LValueToInt64::InputIndex);
5178 Register temp = ToRegister(lir->temp0());
5179 Register64 output = ToOutRegister64(lir);
5180
5181 int checks = 3;
5182
5183 Label fail, done;
5184 // Jump to fail if this is the last check and we fail it,
5185 // otherwise to the next test.
5186 auto emitTestAndUnbox = [&](auto testAndUnbox) {
5187 MOZ_ASSERT(checks > 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(checks > 0)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(checks > 0))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("checks > 0",
"/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 5187); AnnotateMozCrashReason("MOZ_ASSERT" "(" "checks > 0"
")"); do { *((volatile int*)__null) = 5187; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
5188
5189 checks--;
5190 Label notType;
5191 Label* target = checks ? &notType : &fail;
5192
5193 testAndUnbox(target);
5194
5195 if (checks) {
5196 masm.jump(&done);
5197 masm.bind(&notType);
5198 }
5199 };
5200
5201 Register tag = masm.extractTag(input, temp);
5202
5203 // BigInt.
5204 emitTestAndUnbox([&](Label* target) {
5205 masm.branchTestBigInt(Assembler::NotEqual, tag, target);
5206 masm.unboxBigInt(input, temp);
5207 masm.loadBigInt64(temp, output);
5208 });
5209
5210 // Boolean
5211 emitTestAndUnbox([&](Label* target) {
5212 masm.branchTestBoolean(Assembler::NotEqual, tag, target);
5213 masm.unboxBoolean(input, temp);
5214 masm.move32To64ZeroExtend(temp, output);
5215 });
5216
5217 // String
5218 emitTestAndUnbox([&](Label* target) {
5219 masm.branchTestString(Assembler::NotEqual, tag, target);
5220 masm.unboxString(input, temp);
5221 emitStringToInt64(lir, temp, output);
5222 });
5223
5224 MOZ_ASSERT(checks == 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(checks == 0)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(checks == 0))), 0))) { do { }
while (false); MOZ_ReportAssertionFailure("checks == 0", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 5224); AnnotateMozCrashReason("MOZ_ASSERT" "(" "checks == 0"
")"); do { *((volatile int*)__null) = 5224; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
5225
5226 bailoutFrom(&fail, lir->snapshot());
5227 masm.bind(&done);
5228}
5229
5230void CodeGenerator::visitTruncateBigIntToInt64(LTruncateBigIntToInt64* lir) {
5231 Register operand = ToRegister(lir->input());
5232 Register64 output = ToOutRegister64(lir);
5233
5234 masm.loadBigInt64(operand, output);
5235}
5236
5237OutOfLineCode* CodeGenerator::createBigIntOutOfLine(LInstruction* lir,
5238 Scalar::Type type,
5239 Register64 input,
5240 Register output) {
5241#if JS_BITS_PER_WORD64 == 32
5242 using Fn = BigInt* (*)(JSContext*, uint32_t, uint32_t);
5243 auto args = ArgList(input.low, input.high);
5244#else
5245 using Fn = BigInt* (*)(JSContext*, uint64_t);
5246 auto args = ArgList(input);
5247#endif
5248
5249 if (type == Scalar::BigInt64) {
5250 return oolCallVM<Fn, jit::CreateBigIntFromInt64>(lir, args,
5251 StoreRegisterTo(output));
5252 }
5253 MOZ_ASSERT(type == Scalar::BigUint64)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(type == Scalar::BigUint64)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(type == Scalar::BigUint64)))
, 0))) { do { } while (false); MOZ_ReportAssertionFailure("type == Scalar::BigUint64"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 5253); AnnotateMozCrashReason("MOZ_ASSERT" "(" "type == Scalar::BigUint64"
")"); do { *((volatile int*)__null) = 5253; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
5254 return oolCallVM<Fn, jit::CreateBigIntFromUint64>(lir, args,
5255 StoreRegisterTo(output));
5256}
5257
5258void CodeGenerator::emitCreateBigInt(LInstruction* lir, Scalar::Type type,
5259 Register64 input, Register output,
5260 Register maybeTemp,
5261 Register64 maybeTemp64) {
5262 OutOfLineCode* ool = createBigIntOutOfLine(lir, type, input, output);
5263
5264 if (maybeTemp != InvalidReg) {
5265 masm.newGCBigInt(output, maybeTemp, initialBigIntHeap(), ool->entry());
5266 } else {
5267 AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
5268 regs.take(input);
5269 regs.take(output);
5270
5271 Register temp = regs.takeAny();
5272
5273 masm.push(temp);
5274
5275 Label fail, ok;
5276 masm.newGCBigInt(output, temp, initialBigIntHeap(), &fail);
5277 masm.pop(temp);
5278 masm.jump(&ok);
5279 masm.bind(&fail);
5280 masm.pop(temp);
5281 masm.jump(ool->entry());
5282 masm.bind(&ok);
5283 }
5284 masm.initializeBigInt64(type, output, input, maybeTemp64);
5285 masm.bind(ool->rejoin());
5286}
5287
5288void CodeGenerator::visitInt32ToBigInt(LInt32ToBigInt* lir) {
5289 Register input = ToRegister(lir->input());
5290 Register temp = ToRegister(lir->temp0());
5291 Register output = ToRegister(lir->output());
5292
5293 using Fn = BigInt* (*)(JSContext*, int32_t);
5294 auto* ool = oolCallVM<Fn, jit::CreateBigIntFromInt32>(
5295 lir, ArgList(input), StoreRegisterTo(output));
5296
5297 masm.newGCBigInt(output, temp, initialBigIntHeap(), ool->entry());
5298 masm.move32SignExtendToPtr(input, temp);
5299 masm.initializeBigIntPtr(output, temp);
5300 masm.bind(ool->rejoin());
5301}
5302
5303void CodeGenerator::visitInt64ToBigInt(LInt64ToBigInt* lir) {
5304 Register64 input = ToRegister64(lir->input());
5305 Register64 temp = ToRegister64(lir->temp());
5306 Register output = ToRegister(lir->output());
5307
5308 emitCreateBigInt(lir, Scalar::BigInt64, input, output, temp.scratchReg(),
5309 temp);
5310}
5311
5312void CodeGenerator::visitUint64ToBigInt(LUint64ToBigInt* lir) {
5313 Register64 input = ToRegister64(lir->input());
5314 Register temp = ToRegister(lir->temp0());
5315 Register output = ToRegister(lir->output());
5316
5317 emitCreateBigInt(lir, Scalar::BigUint64, input, output, temp);
5318}
5319
5320void CodeGenerator::visitInt64ToIntPtr(LInt64ToIntPtr* lir) {
5321 Register64 input = ToRegister64(lir->input());
5322#ifdef JS_64BIT1
5323 MOZ_ASSERT(input.reg == ToRegister(lir->output()))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(input.reg == ToRegister(lir->output()))>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(input.reg == ToRegister(lir->output())))), 0))) { do { }
while (false); MOZ_ReportAssertionFailure("input.reg == ToRegister(lir->output())"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 5323); AnnotateMozCrashReason("MOZ_ASSERT" "(" "input.reg == ToRegister(lir->output())"
")"); do { *((volatile int*)__null) = 5323; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
5324#else
5325 Register output = ToRegister(lir->output());
5326#endif
5327
5328 Label bail;
5329 if (lir->mir()->elementType() == Scalar::BigInt64) {
5330 masm.branchInt64NotInPtrRange(input, &bail);
5331 } else {
5332 masm.branchUInt64NotInPtrRange(input, &bail);
5333 }
5334 bailoutFrom(&bail, lir->snapshot());
5335
5336#ifndef JS_64BIT1
5337 masm.move64To32(input, output);
5338#endif
5339}
5340
5341void CodeGenerator::visitIntPtrToInt64(LIntPtrToInt64* lir) {
5342#ifdef JS_64BIT1
5343 MOZ_CRASH("Not used on 64-bit platforms")do { do { } while (false); MOZ_ReportCrash("" "Not used on 64-bit platforms"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 5343); AnnotateMozCrashReason("MOZ_CRASH(" "Not used on 64-bit platforms"
")"); do { *((volatile int*)__null) = 5343; __attribute__((nomerge
)) ::abort(); } while (false); } while (false)
;
5344#else
5345 Register input = ToRegister(lir->input());
5346 Register64 output = ToOutRegister64(lir);
5347
5348 masm.move32To64SignExtend(input, output);
5349#endif
5350}
5351
5352void CodeGenerator::visitGuardValue(LGuardValue* lir) {
5353 ValueOperand input = ToValue(lir, LGuardValue::InputIndex);
5354 Value expected = lir->mir()->expected();
5355 Label bail;
5356 masm.branchTestValue(Assembler::NotEqual, input, expected, &bail);
5357 bailoutFrom(&bail, lir->snapshot());
5358}
5359
5360void CodeGenerator::visitGuardNullOrUndefined(LGuardNullOrUndefined* lir) {
5361 ValueOperand input = ToValue(lir, LGuardNullOrUndefined::InputIndex);
5362
5363 ScratchTagScope tag(masm, input);
5364 masm.splitTagForTest(input, tag);
5365
5366 Label done;
5367 masm.branchTestNull(Assembler::Equal, tag, &done);
5368
5369 Label bail;
5370 masm.branchTestUndefined(Assembler::NotEqual, tag, &bail);
5371 bailoutFrom(&bail, lir->snapshot());
5372
5373 masm.bind(&done);
5374}
5375
5376void CodeGenerator::visitGuardIsNotObject(LGuardIsNotObject* lir) {
5377 ValueOperand input = ToValue(lir, LGuardIsNotObject::InputIndex);
5378
5379 Label bail;
5380 masm.branchTestObject(Assembler::Equal, input, &bail);
5381 bailoutFrom(&bail, lir->snapshot());
5382}
5383
5384void CodeGenerator::visitGuardFunctionFlags(LGuardFunctionFlags* lir) {
5385 Register function = ToRegister(lir->function());
5386
5387 Label bail;
5388 if (uint16_t flags = lir->mir()->expectedFlags()) {
5389 masm.branchTestFunctionFlags(function, flags, Assembler::Zero, &bail);
5390 }
5391 if (uint16_t flags = lir->mir()->unexpectedFlags()) {
5392 masm.branchTestFunctionFlags(function, flags, Assembler::NonZero, &bail);
5393 }
5394 bailoutFrom(&bail, lir->snapshot());
5395}
5396
5397void CodeGenerator::visitGuardFunctionIsNonBuiltinCtor(
5398 LGuardFunctionIsNonBuiltinCtor* lir) {
5399 Register function = ToRegister(lir->function());
5400 Register temp = ToRegister(lir->temp0());
5401
5402 Label bail;
5403 masm.branchIfNotFunctionIsNonBuiltinCtor(function, temp, &bail);
5404 bailoutFrom(&bail, lir->snapshot());
5405}
5406
5407void CodeGenerator::visitGuardFunctionKind(LGuardFunctionKind* lir) {
5408 Register function = ToRegister(lir->function());
5409 Register temp = ToRegister(lir->temp0());
5410
5411 Assembler::Condition cond =
5412 lir->mir()->bailOnEquality() ? Assembler::Equal : Assembler::NotEqual;
5413
5414 Label bail;
5415 masm.branchFunctionKind(cond, lir->mir()->expected(), function, temp, &bail);
5416 bailoutFrom(&bail, lir->snapshot());
5417}
5418
5419void CodeGenerator::visitGuardFunctionScript(LGuardFunctionScript* lir) {
5420 Register function = ToRegister(lir->function());
5421
5422 Address scriptAddr(function, JSFunction::offsetOfJitInfoOrScript());
5423 bailoutCmpPtr(Assembler::NotEqual, scriptAddr,
5424 ImmGCPtr(lir->mir()->expected()), lir->snapshot());
5425}
5426
5427// Out-of-line path to update the store buffer.
5428class OutOfLineCallPostWriteBarrier : public OutOfLineCodeBase<CodeGenerator> {
5429 LInstruction* lir_;
5430 const LAllocation* object_;
5431
5432 public:
5433 OutOfLineCallPostWriteBarrier(LInstruction* lir, const LAllocation* object)
5434 : lir_(lir), object_(object) {}
5435
5436 void accept(CodeGenerator* codegen) override {
5437 codegen->visitOutOfLineCallPostWriteBarrier(this);
5438 }
5439
5440 LInstruction* lir() const { return lir_; }
5441 const LAllocation* object() const { return object_; }
5442};
5443
5444static void EmitStoreBufferCheckForConstant(MacroAssembler& masm,
5445 const gc::TenuredCell* cell,
5446 AllocatableGeneralRegisterSet& regs,
5447 Label* exit, Label* callVM) {
5448 Register temp = regs.takeAny();
5449
5450 gc::Arena* arena = cell->arena();
5451
5452 Register cells = temp;
5453 masm.loadPtr(AbsoluteAddress(&arena->bufferedCells()), cells);
5454
5455 size_t index = gc::ArenaCellSet::getCellIndex(cell);
5456 size_t word;
5457 uint32_t mask;
5458 gc::ArenaCellSet::getWordIndexAndMask(index, &word, &mask);
5459 size_t offset = gc::ArenaCellSet::offsetOfBits() + word * sizeof(uint32_t);
5460
5461 masm.branchTest32(Assembler::NonZero, Address(cells, offset), Imm32(mask),
5462 exit);
5463
5464 // Check whether this is the sentinel set and if so call the VM to allocate
5465 // one for this arena.
5466 masm.branchPtr(Assembler::Equal,
5467 Address(cells, gc::ArenaCellSet::offsetOfArena()),
5468 ImmPtr(nullptr), callVM);
5469
5470 // Add the cell to the set.
5471 masm.or32(Imm32(mask), Address(cells, offset));
5472 masm.jump(exit);
5473
5474 regs.add(temp);
5475}
5476
5477static void EmitPostWriteBarrier(MacroAssembler& masm, CompileRuntime* runtime,
5478 Register objreg, JSObject* maybeConstant,
5479 bool isGlobal,
5480 AllocatableGeneralRegisterSet& regs) {
5481 MOZ_ASSERT_IF(isGlobal, maybeConstant)do { if (isGlobal) { do { static_assert( mozilla::detail::AssertionConditionType
<decltype(maybeConstant)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(maybeConstant))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("maybeConstant",
"/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 5481); AnnotateMozCrashReason("MOZ_ASSERT" "(" "maybeConstant"
")"); do { *((volatile int*)__null) = 5481; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false)
;
5482
5483 Label callVM;
5484 Label exit;
5485
5486 Register temp = regs.takeAny();
5487
5488 // We already have a fast path to check whether a global is in the store
5489 // buffer.
5490 if (!isGlobal) {
5491 if (maybeConstant) {
5492 // Check store buffer bitmap directly for known object.
5493 EmitStoreBufferCheckForConstant(masm, &maybeConstant->asTenured(), regs,
5494 &exit, &callVM);
5495 } else {
5496 // Check one element cache to avoid VM call.
5497 masm.branchPtr(Assembler::Equal,
5498 AbsoluteAddress(runtime->addressOfLastBufferedWholeCell()),
5499 objreg, &exit);
5500 }
5501 }
5502
5503 // Call into the VM to barrier the write.
5504 masm.bind(&callVM);
5505
5506 Register runtimereg = temp;
5507 masm.mov(ImmPtr(runtime), runtimereg);
5508
5509 masm.setupAlignedABICall();
5510 masm.passABIArg(runtimereg);
5511 masm.passABIArg(objreg);
5512 if (isGlobal) {
5513 using Fn = void (*)(JSRuntime* rt, GlobalObject* obj);
5514 masm.callWithABI<Fn, PostGlobalWriteBarrier>();
5515 } else {
5516 using Fn = void (*)(JSRuntime* rt, js::gc::Cell* obj);
5517 masm.callWithABI<Fn, PostWriteBarrier>();
5518 }
5519
5520 masm.bind(&exit);
5521}
5522
5523void CodeGenerator::emitPostWriteBarrier(const LAllocation* obj) {
5524 AllocatableGeneralRegisterSet regs(GeneralRegisterSet::Volatile());
5525
5526 Register objreg;
5527 JSObject* object = nullptr;
5528 bool isGlobal = false;
5529 if (obj->isConstant()) {
5530 object = &obj->toConstant()->toObject();
5531 isGlobal = isGlobalObject(object);
5532 objreg = regs.takeAny();
5533 masm.movePtr(ImmGCPtr(object), objreg);
5534 } else {
5535 objreg = ToRegister(obj);
5536 regs.takeUnchecked(objreg);
5537 }
5538
5539 EmitPostWriteBarrier(masm, gen->runtime, objreg, object, isGlobal, regs);
5540}
5541
5542// Returns true if `def` might be allocated in the nursery.
5543static bool ValueNeedsPostBarrier(MDefinition* def) {
5544 if (def->isBox()) {
5545 def = def->toBox()->input();
5546 }
5547 if (def->type() == MIRType::Value) {
5548 return true;
5549 }
5550 return NeedsPostBarrier(def->type());
5551}
5552
5553class OutOfLineElementPostWriteBarrier
5554 : public OutOfLineCodeBase<CodeGenerator> {
5555 LiveRegisterSet liveVolatileRegs_;
5556 const LAllocation* index_;
5557 int32_t indexDiff_;
5558 Register obj_;
5559 Register scratch_;
5560
5561 public:
5562 OutOfLineElementPostWriteBarrier(const LiveRegisterSet& liveVolatileRegs,
5563 Register obj, const LAllocation* index,
5564 Register scratch, int32_t indexDiff)
5565 : liveVolatileRegs_(liveVolatileRegs),
5566 index_(index),
5567 indexDiff_(indexDiff),
5568 obj_(obj),
5569 scratch_(scratch) {}
5570
5571 void accept(CodeGenerator* codegen) override {
5572 codegen->visitOutOfLineElementPostWriteBarrier(this);
5573 }
5574
5575 const LiveRegisterSet& liveVolatileRegs() const { return liveVolatileRegs_; }
5576 const LAllocation* index() const { return index_; }
5577 int32_t indexDiff() const { return indexDiff_; }
5578
5579 Register object() const { return obj_; }
5580 Register scratch() const { return scratch_; }
5581};
5582
5583void CodeGenerator::emitElementPostWriteBarrier(
5584 MInstruction* mir, const LiveRegisterSet& liveVolatileRegs, Register obj,
5585 const LAllocation* index, Register scratch, const ConstantOrRegister& val,
5586 int32_t indexDiff) {
5587 if (val.constant()) {
5588 MOZ_ASSERT_IF(val.value().isGCThing(),do { if (val.value().isGCThing()) { do { static_assert( mozilla
::detail::AssertionConditionType<decltype(!IsInsideNursery
(val.value().toGCThing()))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!IsInsideNursery(val.value()
.toGCThing())))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("!IsInsideNursery(val.value().toGCThing())", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 5589); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!IsInsideNursery(val.value().toGCThing())"
")"); do { *((volatile int*)__null) = 5589; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false)
5589 !IsInsideNursery(val.value().toGCThing()))do { if (val.value().isGCThing()) { do { static_assert( mozilla
::detail::AssertionConditionType<decltype(!IsInsideNursery
(val.value().toGCThing()))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!IsInsideNursery(val.value()
.toGCThing())))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("!IsInsideNursery(val.value().toGCThing())", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 5589); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!IsInsideNursery(val.value().toGCThing())"
")"); do { *((volatile int*)__null) = 5589; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false)
;
5590 return;
5591 }
5592
5593 TypedOrValueRegister reg = val.reg();
5594 if (reg.hasTyped() && !NeedsPostBarrier(reg.type())) {
5595 return;
5596 }
5597
5598 auto* ool = new (alloc()) OutOfLineElementPostWriteBarrier(
5599 liveVolatileRegs, obj, index, scratch, indexDiff);
5600 addOutOfLineCode(ool, mir);
5601
5602 masm.branchPtrInNurseryChunk(Assembler::Equal, obj, scratch, ool->rejoin());
5603
5604 if (reg.hasValue()) {
5605 masm.branchValueIsNurseryCell(Assembler::Equal, reg.valueReg(), scratch,
5606 ool->entry());
5607 } else {
5608 masm.branchPtrInNurseryChunk(Assembler::Equal, reg.typedReg().gpr(),
5609 scratch, ool->entry());
5610 }
5611
5612 masm.bind(ool->rejoin());
5613}
5614
5615void CodeGenerator::visitOutOfLineElementPostWriteBarrier(
5616 OutOfLineElementPostWriteBarrier* ool) {
5617 Register obj = ool->object();
5618 Register scratch = ool->scratch();
5619 const LAllocation* index = ool->index();
5620 int32_t indexDiff = ool->indexDiff();
5621
5622 masm.PushRegsInMask(ool->liveVolatileRegs());
5623
5624 AllocatableGeneralRegisterSet regs(GeneralRegisterSet::Volatile());
5625 regs.takeUnchecked(obj);
5626 regs.takeUnchecked(scratch);
5627
5628 Register indexReg;
5629 if (index->isConstant()) {
5630 indexReg = regs.takeAny();
5631 masm.move32(Imm32(ToInt32(index) + indexDiff), indexReg);
5632 } else {
5633 indexReg = ToRegister(index);
5634 regs.takeUnchecked(indexReg);
5635 if (indexDiff != 0) {
5636 masm.add32(Imm32(indexDiff), indexReg);
5637 }
5638 }
5639
5640 masm.setupUnalignedABICall(scratch);
5641 masm.movePtr(ImmPtr(gen->runtime), scratch);
5642 masm.passABIArg(scratch);
5643 masm.passABIArg(obj);
5644 masm.passABIArg(indexReg);
5645 using Fn = void (*)(JSRuntime* rt, JSObject* obj, int32_t index);
5646 masm.callWithABI<Fn, PostWriteElementBarrier>();
5647
5648 // We don't need a sub32 here because indexReg must be in liveVolatileRegs
5649 // if indexDiff is not zero, so it will be restored below.
5650 MOZ_ASSERT_IF(indexDiff != 0, ool->liveVolatileRegs().has(indexReg))do { if (indexDiff != 0) { do { static_assert( mozilla::detail
::AssertionConditionType<decltype(ool->liveVolatileRegs
().has(indexReg))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(ool->liveVolatileRegs().has
(indexReg)))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("ool->liveVolatileRegs().has(indexReg)", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 5650); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ool->liveVolatileRegs().has(indexReg)"
")"); do { *((volatile int*)__null) = 5650; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false)
;
5651
5652 masm.PopRegsInMask(ool->liveVolatileRegs());
5653
5654 masm.jump(ool->rejoin());
5655}
5656
5657void CodeGenerator::emitPostWriteBarrier(Register objreg) {
5658 AllocatableGeneralRegisterSet regs(GeneralRegisterSet::Volatile());
5659 regs.takeUnchecked(objreg);
5660 EmitPostWriteBarrier(masm, gen->runtime, objreg, nullptr, false, regs);
5661}
5662
5663void CodeGenerator::visitOutOfLineCallPostWriteBarrier(
5664 OutOfLineCallPostWriteBarrier* ool) {
5665 saveLiveVolatile(ool->lir());
5666 const LAllocation* obj = ool->object();
5667 emitPostWriteBarrier(obj);
5668 restoreLiveVolatile(ool->lir());
5669
5670 masm.jump(ool->rejoin());
5671}
5672
5673void CodeGenerator::maybeEmitGlobalBarrierCheck(const LAllocation* maybeGlobal,
5674 OutOfLineCode* ool) {
5675 // Check whether an object is a global that we have already barriered before
5676 // calling into the VM.
5677 //
5678 // We only check for the script's global, not other globals within the same
5679 // compartment, because we bake in a pointer to realm->globalWriteBarriered
5680 // and doing that would be invalid for other realms because they could be
5681 // collected before the Ion code is discarded.
5682
5683 if (!maybeGlobal->isConstant()) {
5684 return;
5685 }
5686
5687 JSObject* obj = &maybeGlobal->toConstant()->toObject();
5688 if (gen->realm->maybeGlobal() != obj) {
5689 return;
5690 }
5691
5692 const uint32_t* addr = gen->realm->addressOfGlobalWriteBarriered();
5693 masm.branch32(Assembler::NotEqual, AbsoluteAddress(addr), Imm32(0),
5694 ool->rejoin());
5695}
5696
5697template <class LPostBarrierType, MIRType nurseryType>
5698void CodeGenerator::visitPostWriteBarrierCommon(LPostBarrierType* lir,
5699 OutOfLineCode* ool) {
5700 static_assert(NeedsPostBarrier(nurseryType));
5701
5702 addOutOfLineCode(ool, lir->mir());
5703
5704 Register temp = ToTempRegisterOrInvalid(lir->temp0());
5705
5706 if (lir->object()->isConstant()) {
5707 // Constant nursery objects cannot appear here, see
5708 // LIRGenerator::visitPostWriteElementBarrier.
5709 MOZ_ASSERT(!IsInsideNursery(&lir->object()->toConstant()->toObject()))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!IsInsideNursery(&lir->object()->toConstant
()->toObject()))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!IsInsideNursery(&lir->
object()->toConstant()->toObject())))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("!IsInsideNursery(&lir->object()->toConstant()->toObject())"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 5709); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!IsInsideNursery(&lir->object()->toConstant()->toObject())"
")"); do { *((volatile int*)__null) = 5709; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
5710 } else {
5711 masm.branchPtrInNurseryChunk(Assembler::Equal, ToRegister(lir->object()),
5712 temp, ool->rejoin());
5713 }
5714
5715 maybeEmitGlobalBarrierCheck(lir->object(), ool);
5716
5717 Register value = ToRegister(lir->value());
5718 if constexpr (nurseryType == MIRType::Object) {
5719 MOZ_ASSERT(lir->mir()->value()->type() == MIRType::Object)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(lir->mir()->value()->type() == MIRType::Object
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(lir->mir()->value()->type() == MIRType::Object
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"lir->mir()->value()->type() == MIRType::Object", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 5719); AnnotateMozCrashReason("MOZ_ASSERT" "(" "lir->mir()->value()->type() == MIRType::Object"
")"); do { *((volatile int*)__null) = 5719; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
5720 } else if constexpr (nurseryType == MIRType::String) {
5721 MOZ_ASSERT(lir->mir()->value()->type() == MIRType::String)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(lir->mir()->value()->type() == MIRType::String
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(lir->mir()->value()->type() == MIRType::String
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"lir->mir()->value()->type() == MIRType::String", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 5721); AnnotateMozCrashReason("MOZ_ASSERT" "(" "lir->mir()->value()->type() == MIRType::String"
")"); do { *((volatile int*)__null) = 5721; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
5722 } else {
5723 static_assert(nurseryType == MIRType::BigInt);
5724 MOZ_ASSERT(lir->mir()->value()->type() == MIRType::BigInt)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(lir->mir()->value()->type() == MIRType::BigInt
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(lir->mir()->value()->type() == MIRType::BigInt
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"lir->mir()->value()->type() == MIRType::BigInt", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 5724); AnnotateMozCrashReason("MOZ_ASSERT" "(" "lir->mir()->value()->type() == MIRType::BigInt"
")"); do { *((volatile int*)__null) = 5724; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
5725 }
5726 masm.branchPtrInNurseryChunk(Assembler::Equal, value, temp, ool->entry());
5727
5728 masm.bind(ool->rejoin());
5729}
5730
5731template <class LPostBarrierType>
5732void CodeGenerator::visitPostWriteBarrierCommonV(LPostBarrierType* lir,
5733 OutOfLineCode* ool) {
5734 addOutOfLineCode(ool, lir->mir());
5735
5736 Register temp = ToTempRegisterOrInvalid(lir->temp0());
5737
5738 if (lir->object()->isConstant()) {
5739 // Constant nursery objects cannot appear here, see
5740 // LIRGenerator::visitPostWriteElementBarrier.
5741 MOZ_ASSERT(!IsInsideNursery(&lir->object()->toConstant()->toObject()))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!IsInsideNursery(&lir->object()->toConstant
()->toObject()))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!IsInsideNursery(&lir->
object()->toConstant()->toObject())))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("!IsInsideNursery(&lir->object()->toConstant()->toObject())"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 5741); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!IsInsideNursery(&lir->object()->toConstant()->toObject())"
")"); do { *((volatile int*)__null) = 5741; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
5742 } else {
5743 masm.branchPtrInNurseryChunk(Assembler::Equal, ToRegister(lir->object()),
5744 temp, ool->rejoin());
5745 }
5746
5747 maybeEmitGlobalBarrierCheck(lir->object(), ool);
5748
5749 ValueOperand value = ToValue(lir, LPostBarrierType::ValueIndex);
5750 masm.branchValueIsNurseryCell(Assembler::Equal, value, temp, ool->entry());
5751
5752 masm.bind(ool->rejoin());
5753}
5754
5755void CodeGenerator::visitPostWriteBarrierO(LPostWriteBarrierO* lir) {
5756 auto ool = new (alloc()) OutOfLineCallPostWriteBarrier(lir, lir->object());
5757 visitPostWriteBarrierCommon<LPostWriteBarrierO, MIRType::Object>(lir, ool);
5758}
5759
5760void CodeGenerator::visitPostWriteBarrierS(LPostWriteBarrierS* lir) {
5761 auto ool = new (alloc()) OutOfLineCallPostWriteBarrier(lir, lir->object());
5762 visitPostWriteBarrierCommon<LPostWriteBarrierS, MIRType::String>(lir, ool);
5763}
5764
5765void CodeGenerator::visitPostWriteBarrierBI(LPostWriteBarrierBI* lir) {
5766 auto ool = new (alloc()) OutOfLineCallPostWriteBarrier(lir, lir->object());
5767 visitPostWriteBarrierCommon<LPostWriteBarrierBI, MIRType::BigInt>(lir, ool);
5768}
5769
5770void CodeGenerator::visitPostWriteBarrierV(LPostWriteBarrierV* lir) {
5771 auto ool = new (alloc()) OutOfLineCallPostWriteBarrier(lir, lir->object());
5772 visitPostWriteBarrierCommonV(lir, ool);
5773}
5774
5775// Out-of-line path to update the store buffer.
5776class OutOfLineCallPostWriteElementBarrier
5777 : public OutOfLineCodeBase<CodeGenerator> {
5778 LInstruction* lir_;
5779 const LAllocation* object_;
5780 const LAllocation* index_;
5781
5782 public:
5783 OutOfLineCallPostWriteElementBarrier(LInstruction* lir,
5784 const LAllocation* object,
5785 const LAllocation* index)
5786 : lir_(lir), object_(object), index_(index) {}
5787
5788 void accept(CodeGenerator* codegen) override {
5789 codegen->visitOutOfLineCallPostWriteElementBarrier(this);
5790 }
5791
5792 LInstruction* lir() const { return lir_; }
5793
5794 const LAllocation* object() const { return object_; }
5795
5796 const LAllocation* index() const { return index_; }
5797};
5798
5799void CodeGenerator::visitOutOfLineCallPostWriteElementBarrier(
5800 OutOfLineCallPostWriteElementBarrier* ool) {
5801 saveLiveVolatile(ool->lir());
5802
5803 const LAllocation* obj = ool->object();
5804 const LAllocation* index = ool->index();
5805
5806 Register objreg = obj->isConstant() ? InvalidReg : ToRegister(obj);
5807 Register indexreg = ToRegister(index);
5808
5809 AllocatableGeneralRegisterSet regs(GeneralRegisterSet::Volatile());
5810 regs.takeUnchecked(indexreg);
5811
5812 if (obj->isConstant()) {
5813 objreg = regs.takeAny();
5814 masm.movePtr(ImmGCPtr(&obj->toConstant()->toObject()), objreg);
5815 } else {
5816 regs.takeUnchecked(objreg);
5817 }
5818
5819 Register runtimereg = regs.takeAny();
5820 using Fn = void (*)(JSRuntime* rt, JSObject* obj, int32_t index);
5821 masm.setupAlignedABICall();
5822 masm.mov(ImmPtr(gen->runtime), runtimereg);
5823 masm.passABIArg(runtimereg);
5824 masm.passABIArg(objreg);
5825 masm.passABIArg(indexreg);
5826 masm.callWithABI<Fn, PostWriteElementBarrier>();
5827
5828 restoreLiveVolatile(ool->lir());
5829
5830 masm.jump(ool->rejoin());
5831}
5832
5833void CodeGenerator::visitPostWriteElementBarrierO(
5834 LPostWriteElementBarrierO* lir) {
5835 auto ool = new (alloc())
5836 OutOfLineCallPostWriteElementBarrier(lir, lir->object(), lir->index());
5837 visitPostWriteBarrierCommon<LPostWriteElementBarrierO, MIRType::Object>(lir,
5838 ool);
5839}
5840
5841void CodeGenerator::visitPostWriteElementBarrierS(
5842 LPostWriteElementBarrierS* lir) {
5843 auto ool = new (alloc())
5844 OutOfLineCallPostWriteElementBarrier(lir, lir->object(), lir->index());
5845 visitPostWriteBarrierCommon<LPostWriteElementBarrierS, MIRType::String>(lir,
5846 ool);
5847}
5848
5849void CodeGenerator::visitPostWriteElementBarrierBI(
5850 LPostWriteElementBarrierBI* lir) {
5851 auto ool = new (alloc())
5852 OutOfLineCallPostWriteElementBarrier(lir, lir->object(), lir->index());
5853 visitPostWriteBarrierCommon<LPostWriteElementBarrierBI, MIRType::BigInt>(lir,
5854 ool);
5855}
5856
5857void CodeGenerator::visitPostWriteElementBarrierV(
5858 LPostWriteElementBarrierV* lir) {
5859 auto ool = new (alloc())
5860 OutOfLineCallPostWriteElementBarrier(lir, lir->object(), lir->index());
5861 visitPostWriteBarrierCommonV(lir, ool);
5862}
5863
5864void CodeGenerator::visitAssertCanElidePostWriteBarrier(
5865 LAssertCanElidePostWriteBarrier* lir) {
5866 Register object = ToRegister(lir->object());
5867 ValueOperand value =
5868 ToValue(lir, LAssertCanElidePostWriteBarrier::ValueIndex);
5869 Register temp = ToRegister(lir->temp0());
5870
5871 Label ok;
5872 masm.branchPtrInNurseryChunk(Assembler::Equal, object, temp, &ok);
5873 masm.branchValueIsNurseryCell(Assembler::NotEqual, value, temp, &ok);
5874
5875 masm.assumeUnreachable("Unexpected missing post write barrier");
5876
5877 masm.bind(&ok);
5878}
5879
5880template <typename LCallIns>
5881void CodeGenerator::emitCallNative(LCallIns* call, JSNative native,
5882 Register argContextReg, Register argUintNReg,
5883 Register argVpReg, Register tempReg,
5884 uint32_t unusedStack) {
5885 masm.checkStackAlignment();
5886
5887 // Native functions have the signature:
5888 // bool (*)(JSContext*, unsigned, Value* vp)
5889 // Where vp[0] is space for an outparam, vp[1] is |this|, and vp[2] onward
5890 // are the function arguments.
5891
5892 // Allocate space for the outparam, moving the StackPointer to what will be
5893 // &vp[1].
5894 masm.adjustStack(unusedStack);
5895
5896 // Push a Value containing the callee object: natives are allowed to access
5897 // their callee before setting the return value. The StackPointer is moved
5898 // to &vp[0].
5899 //
5900 // Also reserves the space for |NativeExitFrameLayout::{lo,hi}CalleeResult_|.
5901 if constexpr (std::is_same_v<LCallIns, LCallClassHook>) {
5902 Register calleeReg = ToRegister(call->getCallee());
5903 masm.Push(TypedOrValueRegister(MIRType::Object, AnyRegister(calleeReg)));
5904
5905 // Enter the callee realm.
5906 if (call->mir()->maybeCrossRealm()) {
5907 masm.switchToObjectRealm(calleeReg, tempReg);
5908 }
5909 } else {
5910 WrappedFunction* target = call->mir()->getSingleTarget();
5911 masm.Push(ObjectValue(*target->rawNativeJSFunction()));
5912
5913 // Enter the callee realm.
5914 if (call->mir()->maybeCrossRealm()) {
5915 masm.movePtr(ImmGCPtr(target->rawNativeJSFunction()), tempReg);
5916 masm.switchToObjectRealm(tempReg, tempReg);
5917 }
5918 }
5919
5920 // Preload arguments into registers.
5921 masm.loadJSContext(argContextReg);
5922 masm.moveStackPtrTo(argVpReg);
5923
5924 // Initialize |NativeExitFrameLayout::argc_|.
5925 masm.Push(argUintNReg);
5926
5927 // Construct native exit frame.
5928 //
5929 // |buildFakeExitFrame| initializes |NativeExitFrameLayout::exit_| and
5930 // |enterFakeExitFrameForNative| initializes |NativeExitFrameLayout::footer_|.
5931 //
5932 // The NativeExitFrameLayout is now fully initialized.
5933 uint32_t safepointOffset = masm.buildFakeExitFrame(tempReg);
5934 masm.enterFakeExitFrameForNative(argContextReg, tempReg,
5935 call->mir()->isConstructing());
5936
5937 markSafepointAt(safepointOffset, call);
5938
5939 // Construct and execute call.
5940 masm.setupAlignedABICall();
5941 masm.passABIArg(argContextReg);
5942 masm.passABIArg(argUintNReg);
5943 masm.passABIArg(argVpReg);
5944
5945 ensureOsiSpace();
5946 // If we're using a simulator build, `native` will already point to the
5947 // simulator's call-redirection code for LCallClassHook. Load the address in
5948 // a register first so that we don't try to redirect it a second time.
5949 bool emittedCall = false;
5950#ifdef JS_SIMULATOR
5951 if constexpr (std::is_same_v<LCallIns, LCallClassHook>) {
5952 masm.movePtr(ImmPtr(native), tempReg);
5953 masm.callWithABI(tempReg);
5954 emittedCall = true;
5955 }
5956#endif
5957 if (!emittedCall) {
5958 masm.callWithABI(DynamicFunction<JSNative>(native), ABIType::General,
5959 CheckUnsafeCallWithABI::DontCheckHasExitFrame);
5960 }
5961
5962 // Test for failure.
5963 masm.branchIfFalseBool(ReturnReg, masm.failureLabel());
5964
5965 // Exit the callee realm.
5966 if (call->mir()->maybeCrossRealm()) {
5967 masm.switchToRealm(gen->realm->realmPtr(), ReturnReg);
5968 }
5969
5970 // Load the outparam vp[0] into output register(s).
5971 masm.loadValue(
5972 Address(masm.getStackPointer(), NativeExitFrameLayout::offsetOfResult()),
5973 JSReturnOperand);
5974
5975 // Until C++ code is instrumented against Spectre, prevent speculative
5976 // execution from returning any private data.
5977 if (JitOptions.spectreJitToCxxCalls && !call->mir()->ignoresReturnValue() &&
5978 call->mir()->hasLiveDefUses()) {
5979 masm.speculationBarrier();
5980 }
5981
5982#ifdef DEBUG1
5983 // Native constructors are guaranteed to return an Object value.
5984 if (call->mir()->isConstructing()) {
5985 Label notPrimitive;
5986 masm.branchTestPrimitive(Assembler::NotEqual, JSReturnOperand,
5987 &notPrimitive);
5988 masm.assumeUnreachable("native constructors don't return primitives");
5989 masm.bind(&notPrimitive);
5990 }
5991#endif
5992}
5993
5994template <typename LCallIns>
5995void CodeGenerator::emitCallNative(LCallIns* call, JSNative native) {
5996 uint32_t unusedStack =
5997 UnusedStackBytesForCall(call->mir()->paddedNumStackArgs());
5998
5999 // Registers used for callWithABI() argument-passing.
6000 const Register argContextReg = ToRegister(call->getArgContextReg());
6001 const Register argUintNReg = ToRegister(call->getArgUintNReg());
6002 const Register argVpReg = ToRegister(call->getArgVpReg());
6003
6004 // Misc. temporary registers.
6005 const Register tempReg = ToRegister(call->getTempReg());
6006
6007 DebugOnly<uint32_t> initialStack = masm.framePushed();
6008
6009 // Initialize the argc register.
6010 masm.move32(Imm32(call->mir()->numActualArgs()), argUintNReg);
6011
6012 // Create the exit frame and call the native.
6013 emitCallNative(call, native, argContextReg, argUintNReg, argVpReg, tempReg,
6014 unusedStack);
6015
6016 // The next instruction is removing the footer of the exit frame, so there
6017 // is no need for leaveFakeExitFrame.
6018
6019 // Move the StackPointer back to its original location, unwinding the native
6020 // exit frame.
6021 masm.adjustStack(NativeExitFrameLayout::Size() - unusedStack);
6022 MOZ_ASSERT(masm.framePushed() == initialStack)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(masm.framePushed() == initialStack)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(masm.framePushed() == initialStack
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"masm.framePushed() == initialStack", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 6022); AnnotateMozCrashReason("MOZ_ASSERT" "(" "masm.framePushed() == initialStack"
")"); do { *((volatile int*)__null) = 6022; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
6023}
6024
6025void CodeGenerator::visitCallNative(LCallNative* call) {
6026 WrappedFunction* target = call->getSingleTarget();
6027 MOZ_ASSERT(target)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(target)>::isValid, "invalid assertion condition")
; if ((__builtin_expect(!!(!(!!(target))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("target", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 6027); AnnotateMozCrashReason("MOZ_ASSERT" "(" "target" ")"
); do { *((volatile int*)__null) = 6027; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
6028 MOZ_ASSERT(target->isNativeWithoutJitEntry())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(target->isNativeWithoutJitEntry())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(target->isNativeWithoutJitEntry
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("target->isNativeWithoutJitEntry()", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 6028); AnnotateMozCrashReason("MOZ_ASSERT" "(" "target->isNativeWithoutJitEntry()"
")"); do { *((volatile int*)__null) = 6028; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
6029
6030 JSNative native = target->native();
6031 if (call->ignoresReturnValue() && target->hasJitInfo()) {
6032 const JSJitInfo* jitInfo = target->jitInfo();
6033 if (jitInfo->type() == JSJitInfo::IgnoresReturnValueNative) {
6034 native = jitInfo->ignoresReturnValueMethod;
6035 }
6036 }
6037 emitCallNative(call, native);
6038}
6039
6040void CodeGenerator::visitCallClassHook(LCallClassHook* call) {
6041 emitCallNative(call, call->mir()->target());
6042}
6043
6044static void LoadDOMPrivate(MacroAssembler& masm, Register obj, Register priv,
6045 DOMObjectKind kind) {
6046 // Load the value in DOM_OBJECT_SLOT for a native or proxy DOM object. This
6047 // will be in the first slot but may be fixed or non-fixed.
6048 MOZ_ASSERT(obj != priv)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(obj != priv)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(obj != priv))), 0))) { do { }
while (false); MOZ_ReportAssertionFailure("obj != priv", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 6048); AnnotateMozCrashReason("MOZ_ASSERT" "(" "obj != priv"
")"); do { *((volatile int*)__null) = 6048; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
6049
6050 switch (kind) {
6051 case DOMObjectKind::Native:
6052 // If it's a native object, the value must be in a fixed slot.
6053 // See CanAttachDOMCall in CacheIR.cpp.
6054 masm.debugAssertObjHasFixedSlots(obj, priv);
6055 masm.loadPrivate(Address(obj, NativeObject::getFixedSlotOffset(0)), priv);
6056 break;
6057 case DOMObjectKind::Proxy: {
6058#ifdef DEBUG1
6059 // Sanity check: it must be a DOM proxy.
6060 Label isDOMProxy;
6061 masm.branchTestProxyHandlerFamily(
6062 Assembler::Equal, obj, priv, GetDOMProxyHandlerFamily(), &isDOMProxy);
6063 masm.assumeUnreachable("Expected a DOM proxy");
6064 masm.bind(&isDOMProxy);
6065#endif
6066 masm.loadPtr(Address(obj, ProxyObject::offsetOfReservedSlots()), priv);
6067 masm.loadPrivate(
6068 Address(priv, js::detail::ProxyReservedSlots::offsetOfSlot(0)), priv);
6069 break;
6070 }
6071 }
6072}
6073
6074void CodeGenerator::visitCallDOMNative(LCallDOMNative* call) {
6075 WrappedFunction* target = call->getSingleTarget();
6076 MOZ_ASSERT(target)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(target)>::isValid, "invalid assertion condition")
; if ((__builtin_expect(!!(!(!!(target))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("target", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 6076); AnnotateMozCrashReason("MOZ_ASSERT" "(" "target" ")"
); do { *((volatile int*)__null) = 6076; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
6077 MOZ_ASSERT(target->isNativeWithoutJitEntry())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(target->isNativeWithoutJitEntry())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(target->isNativeWithoutJitEntry
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("target->isNativeWithoutJitEntry()", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 6077); AnnotateMozCrashReason("MOZ_ASSERT" "(" "target->isNativeWithoutJitEntry()"
")"); do { *((volatile int*)__null) = 6077; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
6078 MOZ_ASSERT(target->hasJitInfo())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(target->hasJitInfo())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(target->hasJitInfo()))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("target->hasJitInfo()"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 6078); AnnotateMozCrashReason("MOZ_ASSERT" "(" "target->hasJitInfo()"
")"); do { *((volatile int*)__null) = 6078; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
6079 MOZ_ASSERT(call->mir()->isCallDOMNative())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(call->mir()->isCallDOMNative())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(call->mir()->isCallDOMNative
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("call->mir()->isCallDOMNative()", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 6079); AnnotateMozCrashReason("MOZ_ASSERT" "(" "call->mir()->isCallDOMNative()"
")"); do { *((volatile int*)__null) = 6079; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
6080
6081 int unusedStack = UnusedStackBytesForCall(call->mir()->paddedNumStackArgs());
6082
6083 // Registers used for callWithABI() argument-passing.
6084 const Register argJSContext = ToRegister(call->getArgJSContext());
6085 const Register argObj = ToRegister(call->getArgObj());
6086 const Register argPrivate = ToRegister(call->getArgPrivate());
6087 const Register argArgs = ToRegister(call->getArgArgs());
6088
6089 DebugOnly<uint32_t> initialStack = masm.framePushed();
6090
6091 masm.checkStackAlignment();
6092
6093 // DOM methods have the signature:
6094 // bool (*)(JSContext*, HandleObject, void* private, const
6095 // JSJitMethodCallArgs& args)
6096 // Where args is initialized from an argc and a vp, vp[0] is space for an
6097 // outparam and the callee, vp[1] is |this|, and vp[2] onward are the
6098 // function arguments. Note that args stores the argv, not the vp, and
6099 // argv == vp + 2.
6100
6101 // Nestle the stack up against the pushed arguments, leaving StackPointer at
6102 // &vp[1]
6103 masm.adjustStack(unusedStack);
6104 // argObj is filled with the extracted object, then returned.
6105 Register obj = masm.extractObject(Address(masm.getStackPointer(), 0), argObj);
6106 MOZ_ASSERT(obj == argObj)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(obj == argObj)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(obj == argObj))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("obj == argObj",
"/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 6106); AnnotateMozCrashReason("MOZ_ASSERT" "(" "obj == argObj"
")"); do { *((volatile int*)__null) = 6106; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
6107
6108 // Push a Value containing the callee object: natives are allowed to access
6109 // their callee before setting the return value. After this the StackPointer
6110 // points to &vp[0].
6111 masm.Push(ObjectValue(*target->rawNativeJSFunction()));
6112
6113 // Now compute the argv value. Since StackPointer is pointing to &vp[0] and
6114 // argv is &vp[2] we just need to add 2*sizeof(Value) to the current
6115 // StackPointer.
6116 static_assert(JSJitMethodCallArgsTraits::offsetOfArgv == 0);
6117 static_assert(JSJitMethodCallArgsTraits::offsetOfArgc ==
6118 IonDOMMethodExitFrameLayoutTraits::offsetOfArgcFromArgv);
6119 masm.computeEffectiveAddress(
6120 Address(masm.getStackPointer(), 2 * sizeof(Value)), argArgs);
6121
6122 LoadDOMPrivate(masm, obj, argPrivate,
6123 static_cast<MCallDOMNative*>(call->mir())->objectKind());
6124
6125 // Push argc from the call instruction into what will become the IonExitFrame
6126 masm.Push(Imm32(call->numActualArgs()));
6127
6128 // Push our argv onto the stack
6129 masm.Push(argArgs);
6130 // And store our JSJitMethodCallArgs* in argArgs.
6131 masm.moveStackPtrTo(argArgs);
6132
6133 // Push |this| object for passing HandleObject. We push after argc to
6134 // maintain the same sp-relative location of the object pointer with other
6135 // DOMExitFrames.
6136 masm.Push(argObj);
6137 masm.moveStackPtrTo(argObj);
6138
6139 if (call->mir()->maybeCrossRealm()) {
6140 // We use argJSContext as scratch register here.
6141 masm.movePtr(ImmGCPtr(target->rawNativeJSFunction()), argJSContext);
6142 masm.switchToObjectRealm(argJSContext, argJSContext);
6143 }
6144
6145 // Construct native exit frame.
6146 uint32_t safepointOffset = masm.buildFakeExitFrame(argJSContext);
6147 masm.loadJSContext(argJSContext);
6148 masm.enterFakeExitFrame(argJSContext, argJSContext,
6149 ExitFrameType::IonDOMMethod);
6150
6151 markSafepointAt(safepointOffset, call);
6152
6153 // Construct and execute call.
6154 masm.setupAlignedABICall();
6155 masm.loadJSContext(argJSContext);
6156 masm.passABIArg(argJSContext);
6157 masm.passABIArg(argObj);
6158 masm.passABIArg(argPrivate);
6159 masm.passABIArg(argArgs);
6160 ensureOsiSpace();
6161 masm.callWithABI(DynamicFunction<JSJitMethodOp>(target->jitInfo()->method),
6162 ABIType::General,
6163 CheckUnsafeCallWithABI::DontCheckHasExitFrame);
6164
6165 if (target->jitInfo()->isInfallible) {
6166 masm.loadValue(Address(masm.getStackPointer(),
6167 IonDOMMethodExitFrameLayout::offsetOfResult()),
6168 JSReturnOperand);
6169 } else {
6170 // Test for failure.
6171 masm.branchIfFalseBool(ReturnReg, masm.exceptionLabel());
6172
6173 // Load the outparam vp[0] into output register(s).
6174 masm.loadValue(Address(masm.getStackPointer(),
6175 IonDOMMethodExitFrameLayout::offsetOfResult()),
6176 JSReturnOperand);
6177 }
6178
6179 // Switch back to the current realm if needed. Note: if the DOM method threw
6180 // an exception, the exception handler will do this.
6181 if (call->mir()->maybeCrossRealm()) {
6182 static_assert(!JSReturnOperand.aliases(ReturnReg),
6183 "Clobbering ReturnReg should not affect the return value");
6184 masm.switchToRealm(gen->realm->realmPtr(), ReturnReg);
6185 }
6186
6187 // Until C++ code is instrumented against Spectre, prevent speculative
6188 // execution from returning any private data.
6189 if (JitOptions.spectreJitToCxxCalls && call->mir()->hasLiveDefUses()) {
6190 masm.speculationBarrier();
6191 }
6192
6193 // The next instruction is removing the footer of the exit frame, so there
6194 // is no need for leaveFakeExitFrame.
6195
6196 // Move the StackPointer back to its original location, unwinding the native
6197 // exit frame.
6198 masm.adjustStack(IonDOMMethodExitFrameLayout::Size() - unusedStack);
6199 MOZ_ASSERT(masm.framePushed() == initialStack)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(masm.framePushed() == initialStack)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(masm.framePushed() == initialStack
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"masm.framePushed() == initialStack", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 6199); AnnotateMozCrashReason("MOZ_ASSERT" "(" "masm.framePushed() == initialStack"
")"); do { *((volatile int*)__null) = 6199; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
6200}
6201
6202void CodeGenerator::visitCallGetIntrinsicValue(LCallGetIntrinsicValue* lir) {
6203 pushArg(ImmGCPtr(lir->mir()->name()));
6204
6205 using Fn = bool (*)(JSContext* cx, Handle<PropertyName*>, MutableHandleValue);
6206 callVM<Fn, GetIntrinsicValue>(lir);
6207}
6208
6209void CodeGenerator::emitCallInvokeFunction(
6210 LInstruction* call, Register calleereg, bool constructing,
6211 bool ignoresReturnValue, uint32_t argc, uint32_t unusedStack) {
6212 // Nestle %esp up to the argument vector.
6213 // Each path must account for framePushed_ separately, for callVM to be valid.
6214 masm.freeStack(unusedStack);
6215
6216 pushArg(masm.getStackPointer()); // argv.
6217 pushArg(Imm32(argc)); // argc.
6218 pushArg(Imm32(ignoresReturnValue));
6219 pushArg(Imm32(constructing)); // constructing.
6220 pushArg(calleereg); // JSFunction*.
6221
6222 using Fn = bool (*)(JSContext*, HandleObject, bool, bool, uint32_t, Value*,
6223 MutableHandleValue);
6224 callVM<Fn, jit::InvokeFunction>(call);
6225
6226 // Un-nestle %esp from the argument vector. No prefix was pushed.
6227 masm.reserveStack(unusedStack);
6228}
6229
6230void CodeGenerator::visitCallGeneric(LCallGeneric* call) {
6231 // The callee is passed straight through to the trampoline.
6232 MOZ_ASSERT(ToRegister(call->getCallee()) == IonGenericCallCalleeReg)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ToRegister(call->getCallee()) == IonGenericCallCalleeReg
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(ToRegister(call->getCallee()) == IonGenericCallCalleeReg
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"ToRegister(call->getCallee()) == IonGenericCallCalleeReg"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 6232); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ToRegister(call->getCallee()) == IonGenericCallCalleeReg"
")"); do { *((volatile int*)__null) = 6232; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
6233
6234 Register argcReg = ToRegister(call->getArgc());
6235 uint32_t unusedStack =
6236 UnusedStackBytesForCall(call->mir()->paddedNumStackArgs());
6237
6238 // Known-target case is handled by LCallKnown.
6239 MOZ_ASSERT(!call->hasSingleTarget())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!call->hasSingleTarget())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!call->hasSingleTarget())
)), 0))) { do { } while (false); MOZ_ReportAssertionFailure("!call->hasSingleTarget()"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 6239); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!call->hasSingleTarget()"
")"); do { *((volatile int*)__null) = 6239; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
6240
6241 masm.checkStackAlignment();
6242
6243 masm.move32(Imm32(call->numActualArgs()), argcReg);
6244
6245 // Nestle the StackPointer up to the argument vector.
6246 masm.freeStack(unusedStack);
6247 ensureOsiSpace();
6248
6249 auto kind = call->mir()->isConstructing() ? IonGenericCallKind::Construct
6250 : IonGenericCallKind::Call;
6251
6252 TrampolinePtr genericCallStub =
6253 gen->jitRuntime()->getIonGenericCallStub(kind);
6254 uint32_t callOffset = masm.callJit(genericCallStub);
6255 markSafepointAt(callOffset, call);
6256
6257 if (call->mir()->maybeCrossRealm()) {
6258 static_assert(!JSReturnOperand.aliases(ReturnReg),
6259 "ReturnReg available as scratch after scripted calls");
6260 masm.switchToRealm(gen->realm->realmPtr(), ReturnReg);
6261 }
6262
6263 // Restore stack pointer.
6264 masm.setFramePushed(frameSize());
6265 emitRestoreStackPointerFromFP();
6266
6267 // If the return value of the constructing function is Primitive,
6268 // replace the return value with the Object from CreateThis.
6269 if (call->mir()->isConstructing()) {
6270 Label notPrimitive;
6271 masm.branchTestPrimitive(Assembler::NotEqual, JSReturnOperand,
6272 &notPrimitive);
6273 masm.loadValue(Address(masm.getStackPointer(), unusedStack),
6274 JSReturnOperand);
6275#ifdef DEBUG1
6276 masm.branchTestPrimitive(Assembler::NotEqual, JSReturnOperand,
6277 &notPrimitive);
6278 masm.assumeUnreachable("CreateThis creates an object");
6279#endif
6280 masm.bind(&notPrimitive);
6281 }
6282}
6283
6284void JitRuntime::generateIonGenericCallArgumentsShift(
6285 MacroAssembler& masm, Register argc, Register curr, Register end,
6286 Register scratch, Label* done) {
6287 static_assert(sizeof(Value) == 8);
6288 // There are |argc| Values on the stack. Shift them all down by 8 bytes,
6289 // overwriting the first value.
6290
6291 // Initialize `curr` to the destination of the first copy, and `end` to the
6292 // final value of curr.
6293 masm.moveStackPtrTo(curr);
6294 masm.computeEffectiveAddress(BaseValueIndex(curr, argc), end);
6295
6296 Label loop;
6297 masm.bind(&loop);
6298 masm.branchPtr(Assembler::Equal, curr, end, done);
6299 masm.loadPtr(Address(curr, 8), scratch);
6300 masm.storePtr(scratch, Address(curr, 0));
6301 masm.addPtr(Imm32(sizeof(uintptr_t)), curr);
6302 masm.jump(&loop);
6303}
6304
6305void JitRuntime::generateIonGenericCallStub(MacroAssembler& masm,
6306 IonGenericCallKind kind) {
6307 AutoCreatedBy acb(masm, "JitRuntime::generateIonGenericCallStub");
6308 ionGenericCallStubOffset_[kind] = startTrampolineCode(masm);
6309
6310 // This code is tightly coupled with visitCallGeneric.
6311 //
6312 // Upon entry:
6313 // IonGenericCallCalleeReg contains a pointer to the callee object.
6314 // IonGenericCallArgcReg contains the number of actual args.
6315 // The arguments have been pushed onto the stack:
6316 // [newTarget] (iff isConstructing)
6317 // [argN]
6318 // ...
6319 // [arg1]
6320 // [arg0]
6321 // [this]
6322 // <return address> (if not JS_USE_LINK_REGISTER)
6323 //
6324 // This trampoline is responsible for entering the callee's realm,
6325 // massaging the stack into the right shape, and then performing a
6326 // tail call. We will return directly to the Ion code from the
6327 // callee.
6328 //
6329 // To do a tail call, we keep the return address in a register, even
6330 // on platforms that don't normally use a link register, and push it
6331 // just before jumping to the callee, after we are done setting up
6332 // the stack.
6333 //
6334 // The caller is responsible for switching back to the caller's
6335 // realm and cleaning up the stack.
6336
6337 Register calleeReg = IonGenericCallCalleeReg;
6338 Register argcReg = IonGenericCallArgcReg;
6339 Register scratch = IonGenericCallScratch;
6340 Register scratch2 = IonGenericCallScratch2;
6341
6342#ifndef JS_USE_LINK_REGISTER
6343 Register returnAddrReg = IonGenericCallReturnAddrReg;
6344 masm.pop(returnAddrReg);
6345#endif
6346
6347#ifdef JS_CODEGEN_ARM
6348 // The default second scratch register on arm is lr, which we need
6349 // preserved for tail calls.
6350 AutoNonDefaultSecondScratchRegister andssr(masm, IonGenericSecondScratchReg);
6351#endif
6352
6353 bool isConstructing = kind == IonGenericCallKind::Construct;
6354
6355 Label entry, notFunction, noJitEntry, vmCall;
6356 masm.bind(&entry);
6357
6358 // Guard that the callee is actually a function.
6359 masm.branchTestObjIsFunction(Assembler::NotEqual, calleeReg, scratch,
6360 calleeReg, &notFunction);
6361
6362 // Guard that the callee supports the [[Call]] or [[Construct]] operation.
6363 // If these tests fail, we will call into the VM to throw an exception.
6364 if (isConstructing) {
6365 masm.branchTestFunctionFlags(calleeReg, FunctionFlags::CONSTRUCTOR,
6366 Assembler::Zero, &vmCall);
6367 } else {
6368 masm.branchFunctionKind(Assembler::Equal, FunctionFlags::ClassConstructor,
6369 calleeReg, scratch, &vmCall);
6370 }
6371
6372 if (isConstructing) {
6373 // Use the slow path if CreateThis was unable to create the |this| object.
6374 Address thisAddr(masm.getStackPointer(), 0);
6375 masm.branchTestNull(Assembler::Equal, thisAddr, &vmCall);
6376 }
6377
6378 masm.switchToObjectRealm(calleeReg, scratch);
6379
6380 // Load jitCodeRaw for callee if it exists.
6381 masm.branchIfFunctionHasNoJitEntry(calleeReg, &noJitEntry);
6382
6383 // ****************************
6384 // * Functions with jit entry *
6385 // ****************************
6386 masm.loadJitCodeRaw(calleeReg, scratch2);
6387
6388 // Construct the JitFrameLayout.
6389 masm.PushCalleeToken(calleeReg, isConstructing);
6390 masm.PushFrameDescriptorForJitCall(FrameType::IonJS, argcReg, scratch);
6391#ifndef JS_USE_LINK_REGISTER
6392 masm.push(returnAddrReg);
6393#endif
6394
6395 // Check whether we need a rectifier frame.
6396 Label noRectifier;
6397 masm.loadFunctionArgCount(calleeReg, scratch);
6398 masm.branch32(Assembler::BelowOrEqual, scratch, argcReg, &noRectifier);
6399 {
6400 // Tail-call the arguments rectifier.
6401 // Because all trampolines are created at the same time,
6402 // we can't create a TrampolinePtr for the arguments rectifier,
6403 // because it hasn't been linked yet. We can, however, directly
6404 // encode its offset.
6405 Label rectifier;
6406 bindLabelToOffset(&rectifier, argumentsRectifierOffset_);
6407
6408 masm.jump(&rectifier);
6409 }
6410
6411 // Tail call the jit entry.
6412 masm.bind(&noRectifier);
6413 masm.jump(scratch2);
6414
6415 // ********************
6416 // * Native functions *
6417 // ********************
6418 masm.bind(&noJitEntry);
6419 if (!isConstructing) {
6420 generateIonGenericCallFunCall(masm, &entry, &vmCall);
6421 }
6422 generateIonGenericCallNativeFunction(masm, isConstructing);
6423
6424 // *******************
6425 // * Bound functions *
6426 // *******************
6427 // TODO: support class hooks?
6428 masm.bind(&notFunction);
6429 if (!isConstructing) {
6430 // TODO: support generic bound constructors?
6431 generateIonGenericCallBoundFunction(masm, &entry, &vmCall);
6432 }
6433
6434 // ********************
6435 // * Fallback VM call *
6436 // ********************
6437 masm.bind(&vmCall);
6438
6439 masm.push(masm.getStackPointer()); // argv
6440 masm.push(argcReg); // argc
6441 masm.push(Imm32(false)); // ignores return value
6442 masm.push(Imm32(isConstructing)); // constructing
6443 masm.push(calleeReg); // callee
6444
6445 using Fn = bool (*)(JSContext*, HandleObject, bool, bool, uint32_t, Value*,
6446 MutableHandleValue);
6447 VMFunctionId id = VMFunctionToId<Fn, jit::InvokeFunction>::id;
6448 uint32_t invokeFunctionOffset = functionWrapperOffsets_[size_t(id)];
6449 Label invokeFunctionVMEntry;
6450 bindLabelToOffset(&invokeFunctionVMEntry, invokeFunctionOffset);
6451
6452 masm.pushFrameDescriptor(FrameType::IonJS);
6453#ifndef JS_USE_LINK_REGISTER
6454 masm.push(returnAddrReg);
6455#endif
6456 masm.jump(&invokeFunctionVMEntry);
6457}
6458
6459void JitRuntime::generateIonGenericCallNativeFunction(MacroAssembler& masm,
6460 bool isConstructing) {
6461 Register calleeReg = IonGenericCallCalleeReg;
6462 Register argcReg = IonGenericCallArgcReg;
6463 Register scratch = IonGenericCallScratch;
6464 Register scratch2 = IonGenericCallScratch2;
6465 Register contextReg = IonGenericCallScratch3;
6466#ifndef JS_USE_LINK_REGISTER
6467 Register returnAddrReg = IonGenericCallReturnAddrReg;
6468#endif
6469
6470 // Push a value containing the callee, which will become argv[0].
6471 masm.pushValue(JSVAL_TYPE_OBJECT, calleeReg);
6472
6473 // Load the callee address into calleeReg.
6474#ifdef JS_SIMULATOR
6475 masm.movePtr(ImmPtr(RedirectedCallAnyNative()), calleeReg);
6476#else
6477 masm.loadPrivate(Address(calleeReg, JSFunction::offsetOfNativeOrEnv()),
6478 calleeReg);
6479#endif
6480
6481 // Load argv into scratch2.
6482 masm.moveStackPtrTo(scratch2);
6483
6484 // Push argc.
6485 masm.push(argcReg);
6486
6487 masm.loadJSContext(contextReg);
6488
6489 // Construct native exit frame. Note that unlike other cases in this
6490 // trampoline, this code does not use a tail call.
6491 masm.pushFrameDescriptor(FrameType::IonJS);
6492#ifdef JS_USE_LINK_REGISTER
6493 masm.pushReturnAddress();
6494#else
6495 masm.push(returnAddrReg);
6496#endif
6497
6498 masm.push(FramePointer);
6499 masm.moveStackPtrTo(FramePointer);
6500 masm.enterFakeExitFrameForNative(contextReg, scratch, isConstructing);
6501
6502 masm.setupUnalignedABICall(scratch);
6503 masm.passABIArg(contextReg); // cx
6504 masm.passABIArg(argcReg); // argc
6505 masm.passABIArg(scratch2); // argv
6506
6507 masm.callWithABI(calleeReg);
6508
6509 // Test for failure.
6510 masm.branchIfFalseBool(ReturnReg, masm.exceptionLabel());
6511
6512 masm.loadValue(
6513 Address(masm.getStackPointer(), NativeExitFrameLayout::offsetOfResult()),
6514 JSReturnOperand);
6515
6516 // Leave the exit frame.
6517 masm.moveToStackPtr(FramePointer);
6518 masm.pop(FramePointer);
6519
6520 // Return.
6521 masm.ret();
6522}
6523
6524void JitRuntime::generateIonGenericCallFunCall(MacroAssembler& masm,
6525 Label* entry, Label* vmCall) {
6526 Register calleeReg = IonGenericCallCalleeReg;
6527 Register argcReg = IonGenericCallArgcReg;
6528 Register scratch = IonGenericCallScratch;
6529 Register scratch2 = IonGenericCallScratch2;
6530 Register scratch3 = IonGenericCallScratch3;
6531
6532 Label notFunCall;
6533 masm.branchPtr(Assembler::NotEqual,
6534 Address(calleeReg, JSFunction::offsetOfNativeOrEnv()),
6535 ImmPtr(js::fun_call), &notFunCall);
6536
6537 // In general, we can implement fun_call by replacing calleeReg with
6538 // |this|, sliding all the other arguments down, and decrementing argc.
6539 //
6540 // *BEFORE* *AFTER*
6541 // [argN] argc = N+1 <padding>
6542 // ... [argN] argc = N
6543 // [arg1] ...
6544 // [arg0] [arg1] <- now arg0
6545 // [this] <- top of stack (aligned) [arg0] <- now this
6546 //
6547 // The only exception is when argc is already 0, in which case instead
6548 // of shifting arguments down we replace [this] with UndefinedValue():
6549 //
6550 // *BEFORE* *AFTER*
6551 // [this] argc = 0 [undef] argc = 0
6552 //
6553 // After making this transformation, we can jump back to the beginning
6554 // of this trampoline to handle the inner call.
6555
6556 // Guard that |this| is an object. If it is, replace calleeReg.
6557 masm.fallibleUnboxObject(Address(masm.getStackPointer(), 0), scratch, vmCall);
6558 masm.movePtr(scratch, calleeReg);
6559
6560 Label hasArgs;
6561 masm.branch32(Assembler::NotEqual, argcReg, Imm32(0), &hasArgs);
6562
6563 // No arguments. Replace |this| with |undefined| and start from the top.
6564 masm.storeValue(UndefinedValue(), Address(masm.getStackPointer(), 0));
6565 masm.jump(entry);
6566
6567 masm.bind(&hasArgs);
6568
6569 Label doneSliding;
6570 generateIonGenericCallArgumentsShift(masm, argcReg, scratch, scratch2,
6571 scratch3, &doneSliding);
6572 masm.bind(&doneSliding);
6573 masm.sub32(Imm32(1), argcReg);
6574
6575 masm.jump(entry);
6576
6577 masm.bind(&notFunCall);
6578}
6579
6580void JitRuntime::generateIonGenericCallBoundFunction(MacroAssembler& masm,
6581 Label* entry,
6582 Label* vmCall) {
6583 Register calleeReg = IonGenericCallCalleeReg;
6584 Register argcReg = IonGenericCallArgcReg;
6585 Register scratch = IonGenericCallScratch;
6586 Register scratch2 = IonGenericCallScratch2;
6587 Register scratch3 = IonGenericCallScratch3;
6588
6589 masm.branchTestObjClass(Assembler::NotEqual, calleeReg,
6590 &BoundFunctionObject::class_, scratch, calleeReg,
6591 vmCall);
6592
6593 Address targetSlot(calleeReg, BoundFunctionObject::offsetOfTargetSlot());
6594 Address flagsSlot(calleeReg, BoundFunctionObject::offsetOfFlagsSlot());
6595 Address thisSlot(calleeReg, BoundFunctionObject::offsetOfBoundThisSlot());
6596 Address firstInlineArgSlot(
6597 calleeReg, BoundFunctionObject::offsetOfFirstInlineBoundArg());
6598
6599 // Check that we won't be pushing too many arguments.
6600 masm.load32(flagsSlot, scratch);
6601 masm.rshift32(Imm32(BoundFunctionObject::NumBoundArgsShift), scratch);
6602 masm.add32(argcReg, scratch);
6603 masm.branch32(Assembler::Above, scratch, Imm32(JIT_ARGS_LENGTH_MAX), vmCall);
6604
6605 // The stack is currently correctly aligned for a jit call. We will
6606 // be updating the `this` value and potentially adding additional
6607 // arguments. On platforms with 16-byte alignment, if the number of
6608 // bound arguments is odd, we have to move the arguments that are
6609 // currently on the stack. For example, with one bound argument:
6610 //
6611 // *BEFORE* *AFTER*
6612 // [argN] <padding>
6613 // ... [argN] |
6614 // [arg1] ... | These arguments have been
6615 // [arg0] [arg1] | shifted down 8 bytes.
6616 // [this] <- top of stack (aligned) [arg0] v
6617 // [bound0] <- one bound argument (odd)
6618 // [boundThis] <- top of stack (aligned)
6619 //
6620 Label poppedThis;
6621 if (JitStackValueAlignment > 1) {
6622 Label alreadyAligned;
6623 masm.branchTest32(Assembler::Zero, flagsSlot,
6624 Imm32(1 << BoundFunctionObject::NumBoundArgsShift),
6625 &alreadyAligned);
6626
6627 // We have an odd number of bound arguments. Shift the existing arguments
6628 // down by 8 bytes.
6629 generateIonGenericCallArgumentsShift(masm, argcReg, scratch, scratch2,
6630 scratch3, &poppedThis);
6631 masm.bind(&alreadyAligned);
6632 }
6633
6634 // Pop the current `this`. It will be replaced with the bound `this`.
6635 masm.freeStack(sizeof(Value));
6636 masm.bind(&poppedThis);
6637
6638 // Load the number of bound arguments in scratch
6639 masm.load32(flagsSlot, scratch);
6640 masm.rshift32(Imm32(BoundFunctionObject::NumBoundArgsShift), scratch);
6641
6642 Label donePushingBoundArguments;
6643 masm.branch32(Assembler::Equal, scratch, Imm32(0),
6644 &donePushingBoundArguments);
6645
6646 // Update argc to include bound arguments.
6647 masm.add32(scratch, argcReg);
6648
6649 // Load &boundArgs[0] in scratch2.
6650 Label outOfLineBoundArguments, haveBoundArguments;
6651 masm.branch32(Assembler::Above, scratch,
6652 Imm32(BoundFunctionObject::MaxInlineBoundArgs),
6653 &outOfLineBoundArguments);
6654 masm.computeEffectiveAddress(firstInlineArgSlot, scratch2);
6655 masm.jump(&haveBoundArguments);
6656
6657 masm.bind(&outOfLineBoundArguments);
6658 masm.unboxObject(firstInlineArgSlot, scratch2);
6659 masm.loadPtr(Address(scratch2, NativeObject::offsetOfElements()), scratch2);
6660
6661 masm.bind(&haveBoundArguments);
6662
6663 // Load &boundArgs[numBoundArgs] in scratch.
6664 BaseObjectElementIndex lastBoundArg(scratch2, scratch);
6665 masm.computeEffectiveAddress(lastBoundArg, scratch);
6666
6667 // Push the bound arguments, starting with the last one.
6668 // Copying pre-decrements scratch until scratch2 is reached.
6669 Label boundArgumentsLoop;
6670 masm.bind(&boundArgumentsLoop);
6671 masm.subPtr(Imm32(sizeof(Value)), scratch);
6672 masm.pushValue(Address(scratch, 0));
6673 masm.branchPtr(Assembler::Above, scratch, scratch2, &boundArgumentsLoop);
6674 masm.bind(&donePushingBoundArguments);
6675
6676 // Push the bound `this`.
6677 masm.pushValue(thisSlot);
6678
6679 // Load the target in calleeReg.
6680 masm.unboxObject(targetSlot, calleeReg);
6681
6682 // At this point, all preconditions for entering the trampoline are met:
6683 // - calleeReg contains a pointer to the callee object
6684 // - argcReg contains the number of actual args (now including bound args)
6685 // - the arguments are on the stack with the correct alignment.
6686 // Instead of generating more code, we can jump back to the entry point
6687 // of the trampoline to call the bound target.
6688 masm.jump(entry);
6689}
6690
6691void CodeGenerator::visitCallKnown(LCallKnown* call) {
6692 Register calleereg = ToRegister(call->getFunction());
6693 Register objreg = ToRegister(call->getTempObject());
6694 uint32_t unusedStack =
6695 UnusedStackBytesForCall(call->mir()->paddedNumStackArgs());
6696 WrappedFunction* target = call->getSingleTarget();
6697
6698 // Native single targets (except Wasm and TrampolineNative functions) are
6699 // handled by LCallNative.
6700 MOZ_ASSERT(target->hasJitEntry())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(target->hasJitEntry())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(target->hasJitEntry()))),
0))) { do { } while (false); MOZ_ReportAssertionFailure("target->hasJitEntry()"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 6700); AnnotateMozCrashReason("MOZ_ASSERT" "(" "target->hasJitEntry()"
")"); do { *((volatile int*)__null) = 6700; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
6701
6702 // Missing arguments must have been explicitly appended by WarpBuilder.
6703 DebugOnly<unsigned> numNonArgsOnStack = 1 + call->isConstructing();
6704 MOZ_ASSERT(target->nargs() <=do { static_assert( mozilla::detail::AssertionConditionType<
decltype(target->nargs() <= call->mir()->numStackArgs
() - numNonArgsOnStack)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(target->nargs() <= call
->mir()->numStackArgs() - numNonArgsOnStack))), 0))) { do
{ } while (false); MOZ_ReportAssertionFailure("target->nargs() <= call->mir()->numStackArgs() - numNonArgsOnStack"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 6705); AnnotateMozCrashReason("MOZ_ASSERT" "(" "target->nargs() <= call->mir()->numStackArgs() - numNonArgsOnStack"
")"); do { *((volatile int*)__null) = 6705; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
6705 call->mir()->numStackArgs() - numNonArgsOnStack)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(target->nargs() <= call->mir()->numStackArgs
() - numNonArgsOnStack)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(target->nargs() <= call
->mir()->numStackArgs() - numNonArgsOnStack))), 0))) { do
{ } while (false); MOZ_ReportAssertionFailure("target->nargs() <= call->mir()->numStackArgs() - numNonArgsOnStack"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 6705); AnnotateMozCrashReason("MOZ_ASSERT" "(" "target->nargs() <= call->mir()->numStackArgs() - numNonArgsOnStack"
")"); do { *((volatile int*)__null) = 6705; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
6706
6707 MOZ_ASSERT_IF(call->isConstructing(), target->isConstructor())do { if (call->isConstructing()) { do { static_assert( mozilla
::detail::AssertionConditionType<decltype(target->isConstructor
())>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(target->isConstructor()))), 0))) { do { } while (
false); MOZ_ReportAssertionFailure("target->isConstructor()"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 6707); AnnotateMozCrashReason("MOZ_ASSERT" "(" "target->isConstructor()"
")"); do { *((volatile int*)__null) = 6707; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false)
;
6708
6709 masm.checkStackAlignment();
6710
6711 if (target->isClassConstructor() && !call->isConstructing()) {
6712 emitCallInvokeFunction(call, calleereg, call->isConstructing(),
6713 call->ignoresReturnValue(), call->numActualArgs(),
6714 unusedStack);
6715 return;
6716 }
6717
6718 MOZ_ASSERT_IF(target->isClassConstructor(), call->isConstructing())do { if (target->isClassConstructor()) { do { static_assert
( mozilla::detail::AssertionConditionType<decltype(call->
isConstructing())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(call->isConstructing())))
, 0))) { do { } while (false); MOZ_ReportAssertionFailure("call->isConstructing()"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 6718); AnnotateMozCrashReason("MOZ_ASSERT" "(" "call->isConstructing()"
")"); do { *((volatile int*)__null) = 6718; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false)
;
6719
6720 MOZ_ASSERT(!call->mir()->needsThisCheck())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!call->mir()->needsThisCheck())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!call->mir()->needsThisCheck
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("!call->mir()->needsThisCheck()", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 6720); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!call->mir()->needsThisCheck()"
")"); do { *((volatile int*)__null) = 6720; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
6721
6722 if (call->mir()->maybeCrossRealm()) {
6723 masm.switchToObjectRealm(calleereg, objreg);
6724 }
6725
6726 masm.loadJitCodeRaw(calleereg, objreg);
6727
6728 // Nestle the StackPointer up to the argument vector.
6729 masm.freeStack(unusedStack);
6730
6731 // Construct the JitFrameLayout.
6732 masm.PushCalleeToken(calleereg, call->mir()->isConstructing());
6733 masm.PushFrameDescriptorForJitCall(FrameType::IonJS, call->numActualArgs());
6734
6735 // Finally call the function in objreg.
6736 ensureOsiSpace();
6737 uint32_t callOffset = masm.callJit(objreg);
6738 markSafepointAt(callOffset, call);
6739
6740 if (call->mir()->maybeCrossRealm()) {
6741 static_assert(!JSReturnOperand.aliases(ReturnReg),
6742 "ReturnReg available as scratch after scripted calls");
6743 masm.switchToRealm(gen->realm->realmPtr(), ReturnReg);
6744 }
6745
6746 // Restore stack pointer: pop JitFrameLayout fields still left on the stack
6747 // and undo the earlier |freeStack(unusedStack)|.
6748 int prefixGarbage =
6749 sizeof(JitFrameLayout) - JitFrameLayout::bytesPoppedAfterCall();
6750 masm.adjustStack(prefixGarbage - unusedStack);
6751
6752 // If the return value of the constructing function is Primitive,
6753 // replace the return value with the Object from CreateThis.
6754 if (call->mir()->isConstructing()) {
6755 Label notPrimitive;
6756 masm.branchTestPrimitive(Assembler::NotEqual, JSReturnOperand,
6757 &notPrimitive);
6758 masm.loadValue(Address(masm.getStackPointer(), unusedStack),
6759 JSReturnOperand);
6760#ifdef DEBUG1
6761 masm.branchTestPrimitive(Assembler::NotEqual, JSReturnOperand,
6762 &notPrimitive);
6763 masm.assumeUnreachable("CreateThis creates an object");
6764#endif
6765 masm.bind(&notPrimitive);
6766 }
6767}
6768
6769template <typename T>
6770void CodeGenerator::emitCallInvokeFunction(T* apply) {
6771 pushArg(masm.getStackPointer()); // argv.
6772 pushArg(ToRegister(apply->getArgc())); // argc.
6773 pushArg(Imm32(apply->mir()->ignoresReturnValue())); // ignoresReturnValue.
6774 pushArg(Imm32(apply->mir()->isConstructing())); // isConstructing.
6775 pushArg(ToRegister(apply->getFunction())); // JSFunction*.
6776
6777 using Fn = bool (*)(JSContext*, HandleObject, bool, bool, uint32_t, Value*,
6778 MutableHandleValue);
6779 callVM<Fn, jit::InvokeFunction>(apply);
6780}
6781
6782// Do not bailout after the execution of this function since the stack no longer
6783// correspond to what is expected by the snapshots.
6784void CodeGenerator::emitAllocateSpaceForApply(Register argcreg,
6785 Register scratch) {
6786 // Use scratch register to calculate stack space (including padding).
6787 masm.movePtr(argcreg, scratch);
6788
6789 // Align the JitFrameLayout on the JitStackAlignment.
6790 if (JitStackValueAlignment > 1) {
6791 MOZ_ASSERT(frameSize() % JitStackAlignment == 0,do { static_assert( mozilla::detail::AssertionConditionType<
decltype(frameSize() % JitStackAlignment == 0)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(frameSize() % JitStackAlignment
== 0))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("frameSize() % JitStackAlignment == 0" " (" "Stack padding assumes that the frameSize is correct"
")", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 6792); AnnotateMozCrashReason("MOZ_ASSERT" "(" "frameSize() % JitStackAlignment == 0"
") (" "Stack padding assumes that the frameSize is correct" ")"
); do { *((volatile int*)__null) = 6792; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
6792 "Stack padding assumes that the frameSize is correct")do { static_assert( mozilla::detail::AssertionConditionType<
decltype(frameSize() % JitStackAlignment == 0)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(frameSize() % JitStackAlignment
== 0))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("frameSize() % JitStackAlignment == 0" " (" "Stack padding assumes that the frameSize is correct"
")", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 6792); AnnotateMozCrashReason("MOZ_ASSERT" "(" "frameSize() % JitStackAlignment == 0"
") (" "Stack padding assumes that the frameSize is correct" ")"
); do { *((volatile int*)__null) = 6792; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
6793 MOZ_ASSERT(JitStackValueAlignment == 2)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(JitStackValueAlignment == 2)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(JitStackValueAlignment == 2)
)), 0))) { do { } while (false); MOZ_ReportAssertionFailure("JitStackValueAlignment == 2"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 6793); AnnotateMozCrashReason("MOZ_ASSERT" "(" "JitStackValueAlignment == 2"
")"); do { *((volatile int*)__null) = 6793; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
6794 Label noPaddingNeeded;
6795 // If the number of arguments is odd, then we do not need any padding.
6796 //
6797 // Note: The |JitStackValueAlignment == 2| condition requires that the
6798 // overall number of values on the stack is even. When we have an odd number
6799 // of arguments, we don't need any padding, because the |thisValue| is
6800 // pushed after the arguments, so the overall number of values on the stack
6801 // is even.
6802 masm.branchTestPtr(Assembler::NonZero, argcreg, Imm32(1), &noPaddingNeeded);
6803 masm.addPtr(Imm32(1), scratch);
6804 masm.bind(&noPaddingNeeded);
6805 }
6806
6807 // Reserve space for copying the arguments.
6808 NativeObject::elementsSizeMustNotOverflow();
6809 masm.lshiftPtr(Imm32(ValueShift), scratch);
6810 masm.subFromStackPtr(scratch);
6811
6812#ifdef DEBUG1
6813 // Put a magic value in the space reserved for padding. Note, this code cannot
6814 // be merged with the previous test, as not all architectures can write below
6815 // their stack pointers.
6816 if (JitStackValueAlignment > 1) {
6817 MOZ_ASSERT(JitStackValueAlignment == 2)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(JitStackValueAlignment == 2)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(JitStackValueAlignment == 2)
)), 0))) { do { } while (false); MOZ_ReportAssertionFailure("JitStackValueAlignment == 2"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 6817); AnnotateMozCrashReason("MOZ_ASSERT" "(" "JitStackValueAlignment == 2"
")"); do { *((volatile int*)__null) = 6817; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
6818 Label noPaddingNeeded;
6819 // If the number of arguments is odd, then we do not need any padding.
6820 masm.branchTestPtr(Assembler::NonZero, argcreg, Imm32(1), &noPaddingNeeded);
6821 BaseValueIndex dstPtr(masm.getStackPointer(), argcreg);
6822 masm.storeValue(MagicValue(JS_ARG_POISON), dstPtr);
6823 masm.bind(&noPaddingNeeded);
6824 }
6825#endif
6826}
6827
6828// Do not bailout after the execution of this function since the stack no longer
6829// correspond to what is expected by the snapshots.
6830void CodeGenerator::emitAllocateSpaceForConstructAndPushNewTarget(
6831 Register argcreg, Register newTargetAndScratch) {
6832 // Align the JitFrameLayout on the JitStackAlignment. Contrary to
6833 // |emitAllocateSpaceForApply()|, we're always pushing a magic value, because
6834 // we can't write to |newTargetAndScratch| before |new.target| has been pushed
6835 // onto the stack.
6836 if (JitStackValueAlignment > 1) {
6837 MOZ_ASSERT(frameSize() % JitStackAlignment == 0,do { static_assert( mozilla::detail::AssertionConditionType<
decltype(frameSize() % JitStackAlignment == 0)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(frameSize() % JitStackAlignment
== 0))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("frameSize() % JitStackAlignment == 0" " (" "Stack padding assumes that the frameSize is correct"
")", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 6838); AnnotateMozCrashReason("MOZ_ASSERT" "(" "frameSize() % JitStackAlignment == 0"
") (" "Stack padding assumes that the frameSize is correct" ")"
); do { *((volatile int*)__null) = 6838; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
6838 "Stack padding assumes that the frameSize is correct")do { static_assert( mozilla::detail::AssertionConditionType<
decltype(frameSize() % JitStackAlignment == 0)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(frameSize() % JitStackAlignment
== 0))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("frameSize() % JitStackAlignment == 0" " (" "Stack padding assumes that the frameSize is correct"
")", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 6838); AnnotateMozCrashReason("MOZ_ASSERT" "(" "frameSize() % JitStackAlignment == 0"
") (" "Stack padding assumes that the frameSize is correct" ")"
); do { *((volatile int*)__null) = 6838; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
6839 MOZ_ASSERT(JitStackValueAlignment == 2)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(JitStackValueAlignment == 2)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(JitStackValueAlignment == 2)
)), 0))) { do { } while (false); MOZ_ReportAssertionFailure("JitStackValueAlignment == 2"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 6839); AnnotateMozCrashReason("MOZ_ASSERT" "(" "JitStackValueAlignment == 2"
")"); do { *((volatile int*)__null) = 6839; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
6840
6841 Label noPaddingNeeded;
6842 // If the number of arguments is even, then we do not need any padding.
6843 //
6844 // Note: The |JitStackValueAlignment == 2| condition requires that the
6845 // overall number of values on the stack is even. When we have an even
6846 // number of arguments, we don't need any padding, because |new.target| is
6847 // is pushed before the arguments and |thisValue| is pushed after all
6848 // arguments, so the overall number of values on the stack is even.
6849 masm.branchTestPtr(Assembler::Zero, argcreg, Imm32(1), &noPaddingNeeded);
6850 masm.pushValue(MagicValue(JS_ARG_POISON));
6851 masm.bind(&noPaddingNeeded);
6852 }
6853
6854 // Push |new.target| after the padding value, but before any arguments.
6855 masm.pushValue(JSVAL_TYPE_OBJECT, newTargetAndScratch);
6856
6857 // Use newTargetAndScratch to calculate stack space (including padding).
6858 masm.movePtr(argcreg, newTargetAndScratch);
6859
6860 // Reserve space for copying the arguments.
6861 NativeObject::elementsSizeMustNotOverflow();
6862 masm.lshiftPtr(Imm32(ValueShift), newTargetAndScratch);
6863 masm.subFromStackPtr(newTargetAndScratch);
6864}
6865
6866// Destroys argvIndex and copyreg.
6867void CodeGenerator::emitCopyValuesForApply(Register argvSrcBase,
6868 Register argvIndex, Register copyreg,
6869 size_t argvSrcOffset,
6870 size_t argvDstOffset) {
6871 Label loop;
6872 masm.bind(&loop);
6873
6874 // As argvIndex is off by 1, and we use the decBranchPtr instruction to loop
6875 // back, we have to substract the size of the word which are copied.
6876 BaseValueIndex srcPtr(argvSrcBase, argvIndex,
6877 int32_t(argvSrcOffset) - sizeof(void*));
6878 BaseValueIndex dstPtr(masm.getStackPointer(), argvIndex,
6879 int32_t(argvDstOffset) - sizeof(void*));
6880 masm.loadPtr(srcPtr, copyreg);
6881 masm.storePtr(copyreg, dstPtr);
6882
6883 // Handle 32 bits architectures.
6884 if (sizeof(Value) == 2 * sizeof(void*)) {
6885 BaseValueIndex srcPtrLow(argvSrcBase, argvIndex,
6886 int32_t(argvSrcOffset) - 2 * sizeof(void*));
6887 BaseValueIndex dstPtrLow(masm.getStackPointer(), argvIndex,
6888 int32_t(argvDstOffset) - 2 * sizeof(void*));
6889 masm.loadPtr(srcPtrLow, copyreg);
6890 masm.storePtr(copyreg, dstPtrLow);
6891 }
6892
6893 masm.decBranchPtr(Assembler::NonZero, argvIndex, Imm32(1), &loop);
6894}
6895
6896void CodeGenerator::emitRestoreStackPointerFromFP() {
6897 // This is used to restore the stack pointer after a call with a dynamic
6898 // number of arguments.
6899
6900 MOZ_ASSERT(masm.framePushed() == frameSize())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(masm.framePushed() == frameSize())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(masm.framePushed() == frameSize
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("masm.framePushed() == frameSize()", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 6900); AnnotateMozCrashReason("MOZ_ASSERT" "(" "masm.framePushed() == frameSize()"
")"); do { *((volatile int*)__null) = 6900; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
6901
6902 int32_t offset = -int32_t(frameSize());
6903 masm.computeEffectiveAddress(Address(FramePointer, offset),
6904 masm.getStackPointer());
6905}
6906
6907void CodeGenerator::emitPushArguments(Register argcreg, Register scratch,
6908 Register copyreg, uint32_t extraFormals) {
6909 Label end;
6910
6911 // Skip the copy of arguments if there are none.
6912 masm.branchTestPtr(Assembler::Zero, argcreg, argcreg, &end);
6913
6914 // clang-format off
6915 //
6916 // We are making a copy of the arguments which are above the JitFrameLayout
6917 // of the current Ion frame.
6918 //
6919 // [arg1] [arg0] <- src [this] [JitFrameLayout] [.. frameSize ..] [pad] [arg1] [arg0] <- dst
6920 //
6921 // clang-format on
6922
6923 // Compute the source and destination offsets into the stack.
6924 //
6925 // The |extraFormals| parameter is used when copying rest-parameters and
6926 // allows to skip the initial parameters before the actual rest-parameters.
6927 Register argvSrcBase = FramePointer;
6928 size_t argvSrcOffset =
6929 JitFrameLayout::offsetOfActualArgs() + extraFormals * sizeof(JS::Value);
6930 size_t argvDstOffset = 0;
6931
6932 Register argvIndex = scratch;
6933 masm.move32(argcreg, argvIndex);
6934
6935 // Copy arguments.
6936 emitCopyValuesForApply(argvSrcBase, argvIndex, copyreg, argvSrcOffset,
6937 argvDstOffset);
6938
6939 // Join with all arguments copied.
6940 masm.bind(&end);
6941}
6942
6943void CodeGenerator::emitPushArguments(LApplyArgsGeneric* apply) {
6944 // Holds the function nargs.
6945 Register argcreg = ToRegister(apply->getArgc());
6946 Register copyreg = ToRegister(apply->getTempObject());
6947 Register scratch = ToRegister(apply->getTempForArgCopy());
6948 uint32_t extraFormals = apply->numExtraFormals();
6949
6950 // Allocate space on the stack for arguments.
6951 emitAllocateSpaceForApply(argcreg, scratch);
6952
6953 emitPushArguments(argcreg, scratch, copyreg, extraFormals);
6954
6955 // Push |this|.
6956 masm.pushValue(ToValue(apply, LApplyArgsGeneric::ThisIndex));
6957}
6958
6959void CodeGenerator::emitPushArguments(LApplyArgsObj* apply) {
6960 Register argsObj = ToRegister(apply->getArgsObj());
6961 Register tmpArgc = ToRegister(apply->getTempObject());
6962 Register scratch = ToRegister(apply->getTempForArgCopy());
6963
6964 // argc and argsObj are mapped to the same calltemp register.
6965 MOZ_ASSERT(argsObj == ToRegister(apply->getArgc()))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(argsObj == ToRegister(apply->getArgc()))>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(argsObj == ToRegister(apply->getArgc())))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("argsObj == ToRegister(apply->getArgc())"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 6965); AnnotateMozCrashReason("MOZ_ASSERT" "(" "argsObj == ToRegister(apply->getArgc())"
")"); do { *((volatile int*)__null) = 6965; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
6966
6967 // Load argc into tmpArgc.
6968 masm.loadArgumentsObjectLength(argsObj, tmpArgc);
6969
6970 // Allocate space on the stack for arguments.
6971 emitAllocateSpaceForApply(tmpArgc, scratch);
6972
6973 // Load arguments data.
6974 masm.loadPrivate(Address(argsObj, ArgumentsObject::getDataSlotOffset()),
6975 argsObj);
6976 size_t argsSrcOffset = ArgumentsData::offsetOfArgs();
6977
6978 // This is the end of the lifetime of argsObj.
6979 // After this call, the argsObj register holds the argument count instead.
6980 emitPushArrayAsArguments(tmpArgc, argsObj, scratch, argsSrcOffset);
6981
6982 // Push |this|.
6983 masm.pushValue(ToValue(apply, LApplyArgsObj::ThisIndex));
6984}
6985
6986void CodeGenerator::emitPushArrayAsArguments(Register tmpArgc,
6987 Register srcBaseAndArgc,
6988 Register scratch,
6989 size_t argvSrcOffset) {
6990 // Preconditions:
6991 // 1. |tmpArgc| * sizeof(Value) bytes have been allocated at the top of
6992 // the stack to hold arguments.
6993 // 2. |srcBaseAndArgc| + |srcOffset| points to an array of |tmpArgc| values.
6994 //
6995 // Postconditions:
6996 // 1. The arguments at |srcBaseAndArgc| + |srcOffset| have been copied into
6997 // the allocated space.
6998 // 2. |srcBaseAndArgc| now contains the original value of |tmpArgc|.
6999 //
7000 // |scratch| is used as a temp register within this function and clobbered.
7001
7002 Label noCopy, epilogue;
7003
7004 // Skip the copy of arguments if there are none.
7005 masm.branchTestPtr(Assembler::Zero, tmpArgc, tmpArgc, &noCopy);
7006 {
7007 // Copy the values. This code is skipped entirely if there are no values.
7008 size_t argvDstOffset = 0;
7009
7010 Register argvSrcBase = srcBaseAndArgc;
7011
7012 // Stash away |tmpArgc| and adjust argvDstOffset accordingly.
7013 masm.push(tmpArgc);
7014 Register argvIndex = tmpArgc;
7015 argvDstOffset += sizeof(void*);
7016
7017 // Copy
7018 emitCopyValuesForApply(argvSrcBase, argvIndex, scratch, argvSrcOffset,
7019 argvDstOffset);
7020
7021 // Restore.
7022 masm.pop(srcBaseAndArgc); // srcBaseAndArgc now contains argc.
7023 masm.jump(&epilogue);
7024 }
7025 masm.bind(&noCopy);
7026 {
7027 // Clear argc if we skipped the copy step.
7028 masm.movePtr(ImmWord(0), srcBaseAndArgc);
7029 }
7030
7031 // Join with all arguments copied.
7032 // Note, "srcBase" has become "argc".
7033 masm.bind(&epilogue);
7034}
7035
7036void CodeGenerator::emitPushArguments(LApplyArrayGeneric* apply) {
7037 Register elements = ToRegister(apply->getElements());
7038 Register tmpArgc = ToRegister(apply->getTempObject());
7039 Register scratch = ToRegister(apply->getTempForArgCopy());
7040
7041 // argc and elements are mapped to the same calltemp register.
7042 MOZ_ASSERT(elements == ToRegister(apply->getArgc()))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(elements == ToRegister(apply->getArgc()))>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(elements == ToRegister(apply->getArgc())))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("elements == ToRegister(apply->getArgc())"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 7042); AnnotateMozCrashReason("MOZ_ASSERT" "(" "elements == ToRegister(apply->getArgc())"
")"); do { *((volatile int*)__null) = 7042; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
7043
7044 // Invariants guarded in the caller:
7045 // - the array is not too long
7046 // - the array length equals its initialized length
7047
7048 // The array length is our argc for the purposes of allocating space.
7049 masm.load32(Address(elements, ObjectElements::offsetOfLength()), tmpArgc);
7050
7051 // Allocate space for the values.
7052 emitAllocateSpaceForApply(tmpArgc, scratch);
7053
7054 // After this call "elements" has become "argc".
7055 size_t elementsOffset = 0;
7056 emitPushArrayAsArguments(tmpArgc, elements, scratch, elementsOffset);
7057
7058 // Push |this|.
7059 masm.pushValue(ToValue(apply, LApplyArrayGeneric::ThisIndex));
7060}
7061
7062void CodeGenerator::emitPushArguments(LConstructArgsGeneric* construct) {
7063 // Holds the function nargs.
7064 Register argcreg = ToRegister(construct->getArgc());
7065 Register copyreg = ToRegister(construct->getTempObject());
7066 Register scratch = ToRegister(construct->getTempForArgCopy());
7067 uint32_t extraFormals = construct->numExtraFormals();
7068
7069 // newTarget and scratch are mapped to the same calltemp register.
7070 MOZ_ASSERT(scratch == ToRegister(construct->getNewTarget()))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(scratch == ToRegister(construct->getNewTarget()))
>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(scratch == ToRegister(construct->getNewTarget()))
)), 0))) { do { } while (false); MOZ_ReportAssertionFailure("scratch == ToRegister(construct->getNewTarget())"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 7070); AnnotateMozCrashReason("MOZ_ASSERT" "(" "scratch == ToRegister(construct->getNewTarget())"
")"); do { *((volatile int*)__null) = 7070; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
7071
7072 // Allocate space for the values.
7073 // After this call "newTarget" has become "scratch".
7074 emitAllocateSpaceForConstructAndPushNewTarget(argcreg, scratch);
7075
7076 emitPushArguments(argcreg, scratch, copyreg, extraFormals);
7077
7078 // Push |this|.
7079 masm.pushValue(ToValue(construct, LConstructArgsGeneric::ThisIndex));
7080}
7081
7082void CodeGenerator::emitPushArguments(LConstructArrayGeneric* construct) {
7083 Register elements = ToRegister(construct->getElements());
7084 Register tmpArgc = ToRegister(construct->getTempObject());
7085 Register scratch = ToRegister(construct->getTempForArgCopy());
7086
7087 // argc and elements are mapped to the same calltemp register.
7088 MOZ_ASSERT(elements == ToRegister(construct->getArgc()))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(elements == ToRegister(construct->getArgc()))>
::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(elements == ToRegister(construct->getArgc())))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("elements == ToRegister(construct->getArgc())"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 7088); AnnotateMozCrashReason("MOZ_ASSERT" "(" "elements == ToRegister(construct->getArgc())"
")"); do { *((volatile int*)__null) = 7088; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
7089
7090 // newTarget and scratch are mapped to the same calltemp register.
7091 MOZ_ASSERT(scratch == ToRegister(construct->getNewTarget()))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(scratch == ToRegister(construct->getNewTarget()))
>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(scratch == ToRegister(construct->getNewTarget()))
)), 0))) { do { } while (false); MOZ_ReportAssertionFailure("scratch == ToRegister(construct->getNewTarget())"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 7091); AnnotateMozCrashReason("MOZ_ASSERT" "(" "scratch == ToRegister(construct->getNewTarget())"
")"); do { *((volatile int*)__null) = 7091; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
7092
7093 // Invariants guarded in the caller:
7094 // - the array is not too long
7095 // - the array length equals its initialized length
7096
7097 // The array length is our argc for the purposes of allocating space.
7098 masm.load32(Address(elements, ObjectElements::offsetOfLength()), tmpArgc);
7099
7100 // Allocate space for the values.
7101 // After this call "newTarget" has become "scratch".
7102 emitAllocateSpaceForConstructAndPushNewTarget(tmpArgc, scratch);
7103
7104 // After this call "elements" has become "argc".
7105 size_t elementsOffset = 0;
7106 emitPushArrayAsArguments(tmpArgc, elements, scratch, elementsOffset);
7107
7108 // Push |this|.
7109 masm.pushValue(ToValue(construct, LConstructArrayGeneric::ThisIndex));
7110}
7111
7112template <typename T>
7113void CodeGenerator::emitApplyGeneric(T* apply) {
7114 // Holds the function object.
7115 Register calleereg = ToRegister(apply->getFunction());
7116
7117 // Temporary register for modifying the function object.
7118 Register objreg = ToRegister(apply->getTempObject());
7119 Register scratch = ToRegister(apply->getTempForArgCopy());
7120
7121 // Holds the function nargs, computed in the invoker or (for ApplyArray,
7122 // ConstructArray, or ApplyArgsObj) in the argument pusher.
7123 Register argcreg = ToRegister(apply->getArgc());
7124
7125 // Copy the arguments of the current function.
7126 //
7127 // In the case of ApplyArray, ConstructArray, or ApplyArgsObj, also compute
7128 // argc. The argc register and the elements/argsObj register are the same;
7129 // argc must not be referenced before the call to emitPushArguments() and
7130 // elements/argsObj must not be referenced after it returns.
7131 //
7132 // In the case of ConstructArray or ConstructArgs, also overwrite newTarget;
7133 // newTarget must not be referenced after this point.
7134 //
7135 // objreg is dead across this call.
7136 emitPushArguments(apply);
7137
7138 masm.checkStackAlignment();
7139
7140 bool constructing = apply->mir()->isConstructing();
7141
7142 // If the function is native, the call is compiled through emitApplyNative.
7143 MOZ_ASSERT_IF(apply->hasSingleTarget(),do { if (apply->hasSingleTarget()) { do { static_assert( mozilla
::detail::AssertionConditionType<decltype(!apply->getSingleTarget
()->isNativeWithoutJitEntry())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!apply->getSingleTarget()
->isNativeWithoutJitEntry()))), 0))) { do { } while (false
); MOZ_ReportAssertionFailure("!apply->getSingleTarget()->isNativeWithoutJitEntry()"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 7144); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!apply->getSingleTarget()->isNativeWithoutJitEntry()"
")"); do { *((volatile int*)__null) = 7144; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false)
7144 !apply->getSingleTarget()->isNativeWithoutJitEntry())do { if (apply->hasSingleTarget()) { do { static_assert( mozilla
::detail::AssertionConditionType<decltype(!apply->getSingleTarget
()->isNativeWithoutJitEntry())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!apply->getSingleTarget()
->isNativeWithoutJitEntry()))), 0))) { do { } while (false
); MOZ_ReportAssertionFailure("!apply->getSingleTarget()->isNativeWithoutJitEntry()"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 7144); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!apply->getSingleTarget()->isNativeWithoutJitEntry()"
")"); do { *((volatile int*)__null) = 7144; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false)
;
7145
7146 Label end, invoke;
7147
7148 // Unless already known, guard that calleereg is actually a function object.
7149 if (!apply->hasSingleTarget()) {
7150 masm.branchTestObjIsFunction(Assembler::NotEqual, calleereg, objreg,
7151 calleereg, &invoke);
7152 }
7153
7154 // Guard that calleereg is an interpreted function with a JSScript.
7155 masm.branchIfFunctionHasNoJitEntry(calleereg, &invoke);
7156
7157 // Guard that callee allows the [[Call]] or [[Construct]] operation required.
7158 if (constructing) {
7159 masm.branchTestFunctionFlags(calleereg, FunctionFlags::CONSTRUCTOR,
7160 Assembler::Zero, &invoke);
7161 } else {
7162 masm.branchFunctionKind(Assembler::Equal, FunctionFlags::ClassConstructor,
7163 calleereg, objreg, &invoke);
7164 }
7165
7166 // Use the slow path if CreateThis was unable to create the |this| object.
7167 if (constructing) {
7168 Address thisAddr(masm.getStackPointer(), 0);
7169 masm.branchTestNull(Assembler::Equal, thisAddr, &invoke);
7170 }
7171
7172 // Call with an Ion frame or a rectifier frame.
7173 {
7174 if (apply->mir()->maybeCrossRealm()) {
7175 masm.switchToObjectRealm(calleereg, objreg);
7176 }
7177
7178 // Knowing that calleereg is a non-native function, load jitcode.
7179 masm.loadJitCodeRaw(calleereg, objreg);
7180
7181 masm.PushCalleeToken(calleereg, constructing);
7182 masm.PushFrameDescriptorForJitCall(FrameType::IonJS, argcreg, scratch);
7183
7184 Label underflow, rejoin;
7185
7186 // Check whether the provided arguments satisfy target argc.
7187 if (!apply->hasSingleTarget()) {
7188 Register nformals = scratch;
7189 masm.loadFunctionArgCount(calleereg, nformals);
7190 masm.branch32(Assembler::Below, argcreg, nformals, &underflow);
7191 } else {
7192 masm.branch32(Assembler::Below, argcreg,
7193 Imm32(apply->getSingleTarget()->nargs()), &underflow);
7194 }
7195
7196 // Skip the construction of the rectifier frame because we have no
7197 // underflow.
7198 masm.jump(&rejoin);
7199
7200 // Argument fixup needed. Get ready to call the argumentsRectifier.
7201 {
7202 masm.bind(&underflow);
7203
7204 // Hardcode the address of the argumentsRectifier code.
7205 TrampolinePtr argumentsRectifier =
7206 gen->jitRuntime()->getArgumentsRectifier();
7207 masm.movePtr(argumentsRectifier, objreg);
7208 }
7209
7210 masm.bind(&rejoin);
7211
7212 // Finally call the function in objreg, as assigned by one of the paths
7213 // above.
7214 ensureOsiSpace();
7215 uint32_t callOffset = masm.callJit(objreg);
7216 markSafepointAt(callOffset, apply);
7217
7218 if (apply->mir()->maybeCrossRealm()) {
7219 static_assert(!JSReturnOperand.aliases(ReturnReg),
7220 "ReturnReg available as scratch after scripted calls");
7221 masm.switchToRealm(gen->realm->realmPtr(), ReturnReg);
7222 }
7223
7224 // Discard JitFrameLayout fields still left on the stack.
7225 masm.freeStack(sizeof(JitFrameLayout) -
7226 JitFrameLayout::bytesPoppedAfterCall());
7227 masm.jump(&end);
7228 }
7229
7230 // Handle uncompiled or native functions.
7231 {
7232 masm.bind(&invoke);
7233 emitCallInvokeFunction(apply);
7234 }
7235
7236 masm.bind(&end);
7237
7238 // If the return value of the constructing function is Primitive, replace the
7239 // return value with the Object from CreateThis.
7240 if (constructing) {
7241 Label notPrimitive;
7242 masm.branchTestPrimitive(Assembler::NotEqual, JSReturnOperand,
7243 &notPrimitive);
7244 masm.loadValue(Address(masm.getStackPointer(), 0), JSReturnOperand);
7245
7246#ifdef DEBUG1
7247 masm.branchTestPrimitive(Assembler::NotEqual, JSReturnOperand,
7248 &notPrimitive);
7249 masm.assumeUnreachable("CreateThis creates an object");
7250#endif
7251
7252 masm.bind(&notPrimitive);
7253 }
7254
7255 // Pop arguments and continue.
7256 emitRestoreStackPointerFromFP();
7257}
7258
7259template <typename T>
7260void CodeGenerator::emitAlignStackForApplyNative(T* apply, Register argc) {
7261 static_assert(JitStackAlignment % ABIStackAlignment == 0,
7262 "aligning on JIT stack subsumes ABI alignment");
7263
7264 // Align the arguments on the JitStackAlignment.
7265 if (JitStackValueAlignment > 1) {
7266 MOZ_ASSERT(JitStackValueAlignment == 2,do { static_assert( mozilla::detail::AssertionConditionType<
decltype(JitStackValueAlignment == 2)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(JitStackValueAlignment == 2)
)), 0))) { do { } while (false); MOZ_ReportAssertionFailure("JitStackValueAlignment == 2"
" (" "Stack padding adds exactly one Value" ")", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 7267); AnnotateMozCrashReason("MOZ_ASSERT" "(" "JitStackValueAlignment == 2"
") (" "Stack padding adds exactly one Value" ")"); do { *((volatile
int*)__null) = 7267; __attribute__((nomerge)) ::abort(); } while
(false); } } while (false)
7267 "Stack padding adds exactly one Value")do { static_assert( mozilla::detail::AssertionConditionType<
decltype(JitStackValueAlignment == 2)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(JitStackValueAlignment == 2)
)), 0))) { do { } while (false); MOZ_ReportAssertionFailure("JitStackValueAlignment == 2"
" (" "Stack padding adds exactly one Value" ")", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 7267); AnnotateMozCrashReason("MOZ_ASSERT" "(" "JitStackValueAlignment == 2"
") (" "Stack padding adds exactly one Value" ")"); do { *((volatile
int*)__null) = 7267; __attribute__((nomerge)) ::abort(); } while
(false); } } while (false)
;
7268 MOZ_ASSERT(frameSize() % JitStackValueAlignment == 0,do { static_assert( mozilla::detail::AssertionConditionType<
decltype(frameSize() % JitStackValueAlignment == 0)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(frameSize() % JitStackValueAlignment == 0))), 0))) { do { }
while (false); MOZ_ReportAssertionFailure("frameSize() % JitStackValueAlignment == 0"
" (" "Stack padding assumes that the frameSize is correct" ")"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 7269); AnnotateMozCrashReason("MOZ_ASSERT" "(" "frameSize() % JitStackValueAlignment == 0"
") (" "Stack padding assumes that the frameSize is correct" ")"
); do { *((volatile int*)__null) = 7269; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
7269 "Stack padding assumes that the frameSize is correct")do { static_assert( mozilla::detail::AssertionConditionType<
decltype(frameSize() % JitStackValueAlignment == 0)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(frameSize() % JitStackValueAlignment == 0))), 0))) { do { }
while (false); MOZ_ReportAssertionFailure("frameSize() % JitStackValueAlignment == 0"
" (" "Stack padding assumes that the frameSize is correct" ")"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 7269); AnnotateMozCrashReason("MOZ_ASSERT" "(" "frameSize() % JitStackValueAlignment == 0"
") (" "Stack padding assumes that the frameSize is correct" ")"
); do { *((volatile int*)__null) = 7269; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
7270
7271 Assembler::Condition cond;
7272 if constexpr (T::isConstructing()) {
7273 // If the number of arguments is even, then we do not need any padding.
7274 //
7275 // Also see emitAllocateSpaceForApply().
7276 cond = Assembler::Zero;
7277 } else {
7278 // If the number of arguments is odd, then we do not need any padding.
7279 //
7280 // Also see emitAllocateSpaceForConstructAndPushNewTarget().
7281 cond = Assembler::NonZero;
7282 }
7283
7284 Label noPaddingNeeded;
7285 masm.branchTestPtr(cond, argc, Imm32(1), &noPaddingNeeded);
7286 masm.pushValue(MagicValue(JS_ARG_POISON));
7287 masm.bind(&noPaddingNeeded);
7288 }
7289}
7290
7291template <typename T>
7292void CodeGenerator::emitPushNativeArguments(T* apply) {
7293 Register argc = ToRegister(apply->getArgc());
7294 Register tmpArgc = ToRegister(apply->getTempObject());
7295 Register scratch = ToRegister(apply->getTempForArgCopy());
7296 uint32_t extraFormals = apply->numExtraFormals();
7297
7298 // Align stack.
7299 emitAlignStackForApplyNative(apply, argc);
7300
7301 // Push newTarget.
7302 if constexpr (T::isConstructing()) {
7303 masm.pushValue(JSVAL_TYPE_OBJECT, ToRegister(apply->getNewTarget()));
7304 }
7305
7306 // Push arguments.
7307 Label noCopy;
7308 masm.branchTestPtr(Assembler::Zero, argc, argc, &noCopy);
7309 {
7310 // Use scratch register to calculate stack space.
7311 masm.movePtr(argc, scratch);
7312
7313 // Reserve space for copying the arguments.
7314 NativeObject::elementsSizeMustNotOverflow();
7315 masm.lshiftPtr(Imm32(ValueShift), scratch);
7316 masm.subFromStackPtr(scratch);
7317
7318 // Compute the source and destination offsets into the stack.
7319 Register argvSrcBase = FramePointer;
7320 size_t argvSrcOffset =
7321 JitFrameLayout::offsetOfActualArgs() + extraFormals * sizeof(JS::Value);
7322 size_t argvDstOffset = 0;
7323
7324 Register argvIndex = tmpArgc;
7325 masm.move32(argc, argvIndex);
7326
7327 // Copy arguments.
7328 emitCopyValuesForApply(argvSrcBase, argvIndex, scratch, argvSrcOffset,
7329 argvDstOffset);
7330 }
7331 masm.bind(&noCopy);
7332
7333 // Push |this|.
7334 if constexpr (T::isConstructing()) {
7335 masm.pushValue(MagicValue(JS_IS_CONSTRUCTING));
7336 } else {
7337 masm.pushValue(ToValue(apply, T::ThisIndex));
7338 }
7339}
7340
7341template <typename T>
7342void CodeGenerator::emitPushArrayAsNativeArguments(T* apply) {
7343 Register argc = ToRegister(apply->getArgc());
7344 Register elements = ToRegister(apply->getElements());
7345 Register tmpArgc = ToRegister(apply->getTempObject());
7346 Register scratch = ToRegister(apply->getTempForArgCopy());
7347
7348 // NB: argc and elements are mapped to the same register.
7349 MOZ_ASSERT(argc == elements)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(argc == elements)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(argc == elements))), 0))) { do
{ } while (false); MOZ_ReportAssertionFailure("argc == elements"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 7349); AnnotateMozCrashReason("MOZ_ASSERT" "(" "argc == elements"
")"); do { *((volatile int*)__null) = 7349; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
7350
7351 // Invariants guarded in the caller:
7352 // - the array is not too long
7353 // - the array length equals its initialized length
7354
7355 // The array length is our argc.
7356 masm.load32(Address(elements, ObjectElements::offsetOfLength()), tmpArgc);
7357
7358 // Align stack.
7359 emitAlignStackForApplyNative(apply, tmpArgc);
7360
7361 // Push newTarget.
7362 if constexpr (T::isConstructing()) {
7363 masm.pushValue(JSVAL_TYPE_OBJECT, ToRegister(apply->getNewTarget()));
7364 }
7365
7366 // Skip the copy of arguments if there are none.
7367 Label noCopy;
7368 masm.branchTestPtr(Assembler::Zero, tmpArgc, tmpArgc, &noCopy);
7369 {
7370 // |tmpArgc| is off-by-one, so adjust the offset accordingly.
7371 BaseObjectElementIndex srcPtr(elements, tmpArgc,
7372 -int32_t(sizeof(JS::Value)));
7373
7374 Label loop;
7375 masm.bind(&loop);
7376 masm.pushValue(srcPtr, scratch);
7377 masm.decBranchPtr(Assembler::NonZero, tmpArgc, Imm32(1), &loop);
7378 }
7379 masm.bind(&noCopy);
7380
7381 // Set argc in preparation for calling the native function.
7382 masm.load32(Address(elements, ObjectElements::offsetOfLength()), argc);
7383
7384 // Push |this|.
7385 if constexpr (T::isConstructing()) {
7386 masm.pushValue(MagicValue(JS_IS_CONSTRUCTING));
7387 } else {
7388 masm.pushValue(ToValue(apply, T::ThisIndex));
7389 }
7390}
7391
7392void CodeGenerator::emitPushArguments(LApplyArgsNative* apply) {
7393 emitPushNativeArguments(apply);
7394}
7395
7396void CodeGenerator::emitPushArguments(LApplyArrayNative* apply) {
7397 emitPushArrayAsNativeArguments(apply);
7398}
7399
7400void CodeGenerator::emitPushArguments(LConstructArgsNative* construct) {
7401 emitPushNativeArguments(construct);
7402}
7403
7404void CodeGenerator::emitPushArguments(LConstructArrayNative* construct) {
7405 emitPushArrayAsNativeArguments(construct);
7406}
7407
7408void CodeGenerator::emitPushArguments(LApplyArgsObjNative* apply) {
7409 Register argc = ToRegister(apply->getArgc());
7410 Register argsObj = ToRegister(apply->getArgsObj());
7411 Register tmpArgc = ToRegister(apply->getTempObject());
7412 Register scratch = ToRegister(apply->getTempForArgCopy());
7413 Register scratch2 = ToRegister(apply->getTempExtra());
7414
7415 // NB: argc and argsObj are mapped to the same register.
7416 MOZ_ASSERT(argc == argsObj)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(argc == argsObj)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(argc == argsObj))), 0))) { do
{ } while (false); MOZ_ReportAssertionFailure("argc == argsObj"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 7416); AnnotateMozCrashReason("MOZ_ASSERT" "(" "argc == argsObj"
")"); do { *((volatile int*)__null) = 7416; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
7417
7418 // Load argc into tmpArgc.
7419 masm.loadArgumentsObjectLength(argsObj, tmpArgc);
7420
7421 // Align stack.
7422 emitAlignStackForApplyNative(apply, tmpArgc);
7423
7424 // Push arguments.
7425 Label noCopy, epilogue;
7426 masm.branchTestPtr(Assembler::Zero, tmpArgc, tmpArgc, &noCopy);
7427 {
7428 // Use scratch register to calculate stack space.
7429 masm.movePtr(tmpArgc, scratch);
7430
7431 // Reserve space for copying the arguments.
7432 NativeObject::elementsSizeMustNotOverflow();
7433 masm.lshiftPtr(Imm32(ValueShift), scratch);
7434 masm.subFromStackPtr(scratch);
7435
7436 // Load arguments data.
7437 Register argvSrcBase = argsObj;
7438 masm.loadPrivate(Address(argsObj, ArgumentsObject::getDataSlotOffset()),
7439 argvSrcBase);
7440 size_t argvSrcOffset = ArgumentsData::offsetOfArgs();
7441 size_t argvDstOffset = 0;
7442
7443 Register argvIndex = scratch2;
7444 masm.move32(tmpArgc, argvIndex);
7445
7446 // Copy the values.
7447 emitCopyValuesForApply(argvSrcBase, argvIndex, scratch, argvSrcOffset,
7448 argvDstOffset);
7449 }
7450 masm.bind(&noCopy);
7451
7452 // Set argc in preparation for calling the native function.
7453 masm.movePtr(tmpArgc, argc);
7454
7455 // Push |this|.
7456 masm.pushValue(ToValue(apply, LApplyArgsObjNative::ThisIndex));
7457}
7458
7459template <typename T>
7460void CodeGenerator::emitApplyNative(T* apply) {
7461 MOZ_ASSERT(T::isConstructing() == apply->mir()->isConstructing(),do { static_assert( mozilla::detail::AssertionConditionType<
decltype(T::isConstructing() == apply->mir()->isConstructing
())>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(T::isConstructing() == apply->mir()->isConstructing
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("T::isConstructing() == apply->mir()->isConstructing()"
" (" "isConstructing condition must be consistent" ")", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 7462); AnnotateMozCrashReason("MOZ_ASSERT" "(" "T::isConstructing() == apply->mir()->isConstructing()"
") (" "isConstructing condition must be consistent" ")"); do
{ *((volatile int*)__null) = 7462; __attribute__((nomerge)) ::
abort(); } while (false); } } while (false)
7462 "isConstructing condition must be consistent")do { static_assert( mozilla::detail::AssertionConditionType<
decltype(T::isConstructing() == apply->mir()->isConstructing
())>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(T::isConstructing() == apply->mir()->isConstructing
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("T::isConstructing() == apply->mir()->isConstructing()"
" (" "isConstructing condition must be consistent" ")", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 7462); AnnotateMozCrashReason("MOZ_ASSERT" "(" "T::isConstructing() == apply->mir()->isConstructing()"
") (" "isConstructing condition must be consistent" ")"); do
{ *((volatile int*)__null) = 7462; __attribute__((nomerge)) ::
abort(); } while (false); } } while (false)
;
7463
7464 WrappedFunction* target = apply->mir()->getSingleTarget();
7465 MOZ_ASSERT(target->isNativeWithoutJitEntry())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(target->isNativeWithoutJitEntry())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(target->isNativeWithoutJitEntry
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("target->isNativeWithoutJitEntry()", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 7465); AnnotateMozCrashReason("MOZ_ASSERT" "(" "target->isNativeWithoutJitEntry()"
")"); do { *((volatile int*)__null) = 7465; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
7466
7467 JSNative native = target->native();
7468 if (apply->mir()->ignoresReturnValue() && target->hasJitInfo()) {
7469 const JSJitInfo* jitInfo = target->jitInfo();
7470 if (jitInfo->type() == JSJitInfo::IgnoresReturnValueNative) {
7471 native = jitInfo->ignoresReturnValueMethod;
7472 }
7473 }
7474
7475 // Push arguments, including newTarget and |this|.
7476 emitPushArguments(apply);
7477
7478 // Registers used for callWithABI() argument-passing.
7479 Register argContextReg = ToRegister(apply->getTempObject());
7480 Register argUintNReg = ToRegister(apply->getArgc());
7481 Register argVpReg = ToRegister(apply->getTempForArgCopy());
7482 Register tempReg = ToRegister(apply->getTempExtra());
7483
7484 // No unused stack for variadic calls.
7485 uint32_t unusedStack = 0;
7486
7487 // Pushed arguments don't change the pushed frames amount.
7488 MOZ_ASSERT(masm.framePushed() == frameSize())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(masm.framePushed() == frameSize())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(masm.framePushed() == frameSize
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("masm.framePushed() == frameSize()", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 7488); AnnotateMozCrashReason("MOZ_ASSERT" "(" "masm.framePushed() == frameSize()"
")"); do { *((volatile int*)__null) = 7488; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
7489
7490 // Create the exit frame and call the native.
7491 emitCallNative(apply, native, argContextReg, argUintNReg, argVpReg, tempReg,
7492 unusedStack);
7493
7494 // The exit frame is still on the stack.
7495 MOZ_ASSERT(masm.framePushed() == frameSize() + NativeExitFrameLayout::Size())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(masm.framePushed() == frameSize() + NativeExitFrameLayout
::Size())>::isValid, "invalid assertion condition"); if ((
__builtin_expect(!!(!(!!(masm.framePushed() == frameSize() + NativeExitFrameLayout
::Size()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("masm.framePushed() == frameSize() + NativeExitFrameLayout::Size()"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 7495); AnnotateMozCrashReason("MOZ_ASSERT" "(" "masm.framePushed() == frameSize() + NativeExitFrameLayout::Size()"
")"); do { *((volatile int*)__null) = 7495; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
7496
7497 // The next instruction is removing the exit frame, so there is no need for
7498 // leaveFakeExitFrame.
7499
7500 // Pop arguments and continue.
7501 masm.setFramePushed(frameSize());
7502 emitRestoreStackPointerFromFP();
7503}
7504
7505template <typename T>
7506void CodeGenerator::emitApplyArgsGuard(T* apply) {
7507 LSnapshot* snapshot = apply->snapshot();
7508 Register argcreg = ToRegister(apply->getArgc());
7509
7510 // Ensure that we have a reasonable number of arguments.
7511 bailoutCmp32(Assembler::Above, argcreg, Imm32(JIT_ARGS_LENGTH_MAX), snapshot);
7512}
7513
7514template <typename T>
7515void CodeGenerator::emitApplyArgsObjGuard(T* apply) {
7516 Register argsObj = ToRegister(apply->getArgsObj());
7517 Register temp = ToRegister(apply->getTempObject());
7518
7519 Label bail;
7520 masm.loadArgumentsObjectLength(argsObj, temp, &bail);
7521 masm.branch32(Assembler::Above, temp, Imm32(JIT_ARGS_LENGTH_MAX), &bail);
7522 bailoutFrom(&bail, apply->snapshot());
7523}
7524
7525template <typename T>
7526void CodeGenerator::emitApplyArrayGuard(T* apply) {
7527 LSnapshot* snapshot = apply->snapshot();
7528 Register elements = ToRegister(apply->getElements());
7529 Register tmp = ToRegister(apply->getTempObject());
7530
7531 Address length(elements, ObjectElements::offsetOfLength());
7532 masm.load32(length, tmp);
7533
7534 // Ensure that we have a reasonable number of arguments.
7535 bailoutCmp32(Assembler::Above, tmp, Imm32(JIT_ARGS_LENGTH_MAX), snapshot);
7536
7537 // Ensure that the array does not contain an uninitialized tail.
7538
7539 Address initializedLength(elements,
7540 ObjectElements::offsetOfInitializedLength());
7541 masm.sub32(initializedLength, tmp);
7542 bailoutCmp32(Assembler::NotEqual, tmp, Imm32(0), snapshot);
7543}
7544
7545void CodeGenerator::visitApplyArgsGeneric(LApplyArgsGeneric* apply) {
7546 emitApplyArgsGuard(apply);
7547 emitApplyGeneric(apply);
7548}
7549
7550void CodeGenerator::visitApplyArgsObj(LApplyArgsObj* apply) {
7551 emitApplyArgsObjGuard(apply);
7552 emitApplyGeneric(apply);
7553}
7554
7555void CodeGenerator::visitApplyArrayGeneric(LApplyArrayGeneric* apply) {
7556 emitApplyArrayGuard(apply);
7557 emitApplyGeneric(apply);
7558}
7559
7560void CodeGenerator::visitConstructArgsGeneric(LConstructArgsGeneric* lir) {
7561 emitApplyArgsGuard(lir);
7562 emitApplyGeneric(lir);
7563}
7564
7565void CodeGenerator::visitConstructArrayGeneric(LConstructArrayGeneric* lir) {
7566 emitApplyArrayGuard(lir);
7567 emitApplyGeneric(lir);
7568}
7569
7570void CodeGenerator::visitApplyArgsNative(LApplyArgsNative* lir) {
7571 emitApplyArgsGuard(lir);
7572 emitApplyNative(lir);
7573}
7574
7575void CodeGenerator::visitApplyArgsObjNative(LApplyArgsObjNative* lir) {
7576 emitApplyArgsObjGuard(lir);
7577 emitApplyNative(lir);
7578}
7579
7580void CodeGenerator::visitApplyArrayNative(LApplyArrayNative* lir) {
7581 emitApplyArrayGuard(lir);
7582 emitApplyNative(lir);
7583}
7584
7585void CodeGenerator::visitConstructArgsNative(LConstructArgsNative* lir) {
7586 emitApplyArgsGuard(lir);
7587 emitApplyNative(lir);
7588}
7589
7590void CodeGenerator::visitConstructArrayNative(LConstructArrayNative* lir) {
7591 emitApplyArrayGuard(lir);
7592 emitApplyNative(lir);
7593}
7594
7595void CodeGenerator::visitBail(LBail* lir) { bailout(lir->snapshot()); }
7596
7597void CodeGenerator::visitUnreachable(LUnreachable* lir) {
7598 masm.assumeUnreachable("end-of-block assumed unreachable");
7599}
7600
7601void CodeGenerator::visitEncodeSnapshot(LEncodeSnapshot* lir) {
7602 encode(lir->snapshot());
7603}
7604
7605void CodeGenerator::visitUnreachableResultV(LUnreachableResultV* lir) {
7606 masm.assumeUnreachable("must be unreachable");
7607}
7608
7609void CodeGenerator::visitUnreachableResultT(LUnreachableResultT* lir) {
7610 masm.assumeUnreachable("must be unreachable");
7611}
7612
7613// Out-of-line path to report over-recursed error and fail.
7614class CheckOverRecursedFailure : public OutOfLineCodeBase<CodeGenerator> {
7615 LInstruction* lir_;
7616
7617 public:
7618 explicit CheckOverRecursedFailure(LInstruction* lir) : lir_(lir) {}
7619
7620 void accept(CodeGenerator* codegen) override {
7621 codegen->visitCheckOverRecursedFailure(this);
7622 }
7623
7624 LInstruction* lir() const { return lir_; }
7625};
7626
7627void CodeGenerator::visitCheckOverRecursed(LCheckOverRecursed* lir) {
7628 // If we don't push anything on the stack, skip the check.
7629 if (omitOverRecursedCheck()) {
7630 return;
7631 }
7632
7633 // Ensure that this frame will not cross the stack limit.
7634 // This is a weak check, justified by Ion using the C stack: we must always
7635 // be some distance away from the actual limit, since if the limit is
7636 // crossed, an error must be thrown, which requires more frames.
7637 //
7638 // It must always be possible to trespass past the stack limit.
7639 // Ion may legally place frames very close to the limit. Calling additional
7640 // C functions may then violate the limit without any checking.
7641 //
7642 // Since Ion frames exist on the C stack, the stack limit may be
7643 // dynamically set by JS_SetThreadStackLimit() and JS_SetNativeStackQuota().
7644
7645 CheckOverRecursedFailure* ool = new (alloc()) CheckOverRecursedFailure(lir);
7646 addOutOfLineCode(ool, lir->mir());
7647
7648 // Conditional forward (unlikely) branch to failure.
7649 const void* limitAddr = gen->runtime->addressOfJitStackLimit();
7650 masm.branchStackPtrRhs(Assembler::AboveOrEqual, AbsoluteAddress(limitAddr),
7651 ool->entry());
7652 masm.bind(ool->rejoin());
7653}
7654
7655void CodeGenerator::visitCheckOverRecursedFailure(
7656 CheckOverRecursedFailure* ool) {
7657 // The OOL path is hit if the recursion depth has been exceeded.
7658 // Throw an InternalError for over-recursion.
7659
7660 // LFunctionEnvironment can appear before LCheckOverRecursed, so we have
7661 // to save all live registers to avoid crashes if CheckOverRecursed triggers
7662 // a GC.
7663 saveLive(ool->lir());
7664
7665 using Fn = bool (*)(JSContext*);
7666 callVM<Fn, CheckOverRecursed>(ool->lir());
7667
7668 restoreLive(ool->lir());
7669 masm.jump(ool->rejoin());
7670}
7671
7672IonScriptCounts* CodeGenerator::maybeCreateScriptCounts() {
7673 // If scripts are being profiled, create a new IonScriptCounts for the
7674 // profiling data, which will be attached to the associated JSScript or
7675 // wasm module after code generation finishes.
7676 if (!gen->hasProfilingScripts()) {
7677 return nullptr;
7678 }
7679
7680 // This test inhibits IonScriptCount creation for wasm code which is
7681 // currently incompatible with wasm codegen for two reasons: (1) wasm code
7682 // must be serializable and script count codegen bakes in absolute
7683 // addresses, (2) wasm code does not have a JSScript with which to associate
7684 // code coverage data.
7685 JSScript* script = gen->outerInfo().script();
7686 if (!script) {
7687 return nullptr;
7688 }
7689
7690 auto counts = MakeUnique<IonScriptCounts>();
7691 if (!counts || !counts->init(graph.numBlocks())) {
7692 return nullptr;
7693 }
7694
7695 for (size_t i = 0; i < graph.numBlocks(); i++) {
7696 MBasicBlock* block = graph.getBlock(i)->mir();
7697
7698 uint32_t offset = 0;
7699 char* description = nullptr;
7700 if (MResumePoint* resume = block->entryResumePoint()) {
7701 // Find a PC offset in the outermost script to use. If this
7702 // block is from an inlined script, find a location in the
7703 // outer script to associate information about the inlining
7704 // with.
7705 while (resume->caller()) {
7706 resume = resume->caller();
7707 }
7708 offset = script->pcToOffset(resume->pc());
7709
7710 if (block->entryResumePoint()->caller()) {
7711 // Get the filename and line number of the inner script.
7712 JSScript* innerScript = block->info().script();
7713 description = js_pod_calloc<char>(200);
7714 if (description) {
7715 snprintf(description, 200, "%s:%u", innerScript->filename(),
7716 innerScript->lineno());
7717 }
7718 }
7719 }
7720
7721 if (!counts->block(i).init(block->id(), offset, description,
7722 block->numSuccessors())) {
7723 return nullptr;
7724 }
7725
7726 for (size_t j = 0; j < block->numSuccessors(); j++) {
7727 counts->block(i).setSuccessor(
7728 j, skipTrivialBlocks(block->getSuccessor(j))->id());
7729 }
7730 }
7731
7732 scriptCounts_ = counts.release();
7733 return scriptCounts_;
7734}
7735
7736// Structure for managing the state tracked for a block by script counters.
7737struct ScriptCountBlockState {
7738 IonBlockCounts& block;
7739 MacroAssembler& masm;
7740
7741 Sprinter printer;
7742
7743 public:
7744 ScriptCountBlockState(IonBlockCounts* block, MacroAssembler* masm)
7745 : block(*block), masm(*masm), printer(GetJitContext()->cx, false) {}
7746
7747 bool init() {
7748 if (!printer.init()) {
7749 return false;
7750 }
7751
7752 // Bump the hit count for the block at the start. This code is not
7753 // included in either the text for the block or the instruction byte
7754 // counts.
7755 masm.inc64(AbsoluteAddress(block.addressOfHitCount()));
7756
7757 // Collect human readable assembly for the code generated in the block.
7758 masm.setPrinter(&printer);
7759
7760 return true;
7761 }
7762
7763 void visitInstruction(LInstruction* ins) {
7764#ifdef JS_JITSPEW1
7765 // Prefix stream of assembly instructions with their LIR instruction
7766 // name and any associated high level info.
7767 if (const char* extra = ins->getExtraName()) {
7768 printer.printf("[%s:%s]\n", ins->opName(), extra);
7769 } else {
7770 printer.printf("[%s]\n", ins->opName());
7771 }
7772#endif
7773 }
7774
7775 ~ScriptCountBlockState() {
7776 masm.setPrinter(nullptr);
7777
7778 if (JS::UniqueChars str = printer.release()) {
7779 block.setCode(str.get());
7780 }
7781 }
7782};
7783
7784void CodeGenerator::branchIfInvalidated(Register temp, Label* invalidated) {
7785 CodeOffset label = masm.movWithPatch(ImmWord(uintptr_t(-1)), temp);
7786 masm.propagateOOM(ionScriptLabels_.append(label));
7787
7788 // If IonScript::invalidationCount_ != 0, the script has been invalidated.
7789 masm.branch32(Assembler::NotEqual,
7790 Address(temp, IonScript::offsetOfInvalidationCount()), Imm32(0),
7791 invalidated);
7792}
7793
7794#ifdef DEBUG1
7795void CodeGenerator::emitAssertGCThingResult(Register input,
7796 const MDefinition* mir) {
7797 MIRType type = mir->type();
7798 MOZ_ASSERT(type == MIRType::Object || type == MIRType::String ||do { static_assert( mozilla::detail::AssertionConditionType<
decltype(type == MIRType::Object || type == MIRType::String ||
type == MIRType::Symbol || type == MIRType::BigInt)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(type == MIRType::Object || type == MIRType::String || type
== MIRType::Symbol || type == MIRType::BigInt))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("type == MIRType::Object || type == MIRType::String || type == MIRType::Symbol || type == MIRType::BigInt"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 7799); AnnotateMozCrashReason("MOZ_ASSERT" "(" "type == MIRType::Object || type == MIRType::String || type == MIRType::Symbol || type == MIRType::BigInt"
")"); do { *((volatile int*)__null) = 7799; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
7799 type == MIRType::Symbol || type == MIRType::BigInt)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(type == MIRType::Object || type == MIRType::String ||
type == MIRType::Symbol || type == MIRType::BigInt)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(type == MIRType::Object || type == MIRType::String || type
== MIRType::Symbol || type == MIRType::BigInt))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("type == MIRType::Object || type == MIRType::String || type == MIRType::Symbol || type == MIRType::BigInt"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 7799); AnnotateMozCrashReason("MOZ_ASSERT" "(" "type == MIRType::Object || type == MIRType::String || type == MIRType::Symbol || type == MIRType::BigInt"
")"); do { *((volatile int*)__null) = 7799; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
7800
7801 AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
7802 regs.take(input);
7803
7804 Register temp = regs.takeAny();
7805 masm.push(temp);
7806
7807 // Don't check if the script has been invalidated. In that case invalid
7808 // types are expected (until we reach the OsiPoint and bailout).
7809 Label done;
7810 branchIfInvalidated(temp, &done);
7811
7812# ifndef JS_SIMULATOR
7813 // Check that we have a valid GC pointer.
7814 // Disable for wasm because we don't have a context on wasm compilation
7815 // threads and this needs a context.
7816 // Also disable for simulator builds because the C++ call is a lot slower
7817 // there than on actual hardware.
7818 if (JitOptions.fullDebugChecks && !IsCompilingWasm()) {
7819 saveVolatile();
7820 masm.setupUnalignedABICall(temp);
7821 masm.loadJSContext(temp);
7822 masm.passABIArg(temp);
7823 masm.passABIArg(input);
7824
7825 switch (type) {
7826 case MIRType::Object: {
7827 using Fn = void (*)(JSContext* cx, JSObject* obj);
7828 masm.callWithABI<Fn, AssertValidObjectPtr>();
7829 break;
7830 }
7831 case MIRType::String: {
7832 using Fn = void (*)(JSContext* cx, JSString* str);
7833 masm.callWithABI<Fn, AssertValidStringPtr>();
7834 break;
7835 }
7836 case MIRType::Symbol: {
7837 using Fn = void (*)(JSContext* cx, JS::Symbol* sym);
7838 masm.callWithABI<Fn, AssertValidSymbolPtr>();
7839 break;
7840 }
7841 case MIRType::BigInt: {
7842 using Fn = void (*)(JSContext* cx, JS::BigInt* bi);
7843 masm.callWithABI<Fn, AssertValidBigIntPtr>();
7844 break;
7845 }
7846 default:
7847 MOZ_CRASH()do { do { } while (false); MOZ_ReportCrash("" , "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 7847); AnnotateMozCrashReason("MOZ_CRASH(" ")"); do { *((volatile
int*)__null) = 7847; __attribute__((nomerge)) ::abort(); } while
(false); } while (false)
;
7848 }
7849
7850 restoreVolatile();
7851 }
7852# endif
7853
7854 masm.bind(&done);
7855 masm.pop(temp);
7856}
7857
7858void CodeGenerator::emitAssertResultV(const ValueOperand input,
7859 const MDefinition* mir) {
7860 AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
7861 regs.take(input);
7862
7863 Register temp1 = regs.takeAny();
7864 Register temp2 = regs.takeAny();
7865 masm.push(temp1);
7866 masm.push(temp2);
7867
7868 // Don't check if the script has been invalidated. In that case invalid
7869 // types are expected (until we reach the OsiPoint and bailout).
7870 Label done;
7871 branchIfInvalidated(temp1, &done);
7872
7873 // Check that we have a valid GC pointer.
7874 if (JitOptions.fullDebugChecks) {
7875 saveVolatile();
7876
7877 masm.pushValue(input);
7878 masm.moveStackPtrTo(temp1);
7879
7880 using Fn = void (*)(JSContext* cx, Value* v);
7881 masm.setupUnalignedABICall(temp2);
7882 masm.loadJSContext(temp2);
7883 masm.passABIArg(temp2);
7884 masm.passABIArg(temp1);
7885 masm.callWithABI<Fn, AssertValidValue>();
7886 masm.popValue(input);
7887 restoreVolatile();
7888 }
7889
7890 masm.bind(&done);
7891 masm.pop(temp2);
7892 masm.pop(temp1);
7893}
7894
7895void CodeGenerator::emitGCThingResultChecks(LInstruction* lir,
7896 MDefinition* mir) {
7897 if (lir->numDefs() == 0) {
7898 return;
7899 }
7900
7901 MOZ_ASSERT(lir->numDefs() == 1)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(lir->numDefs() == 1)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(lir->numDefs() == 1))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("lir->numDefs() == 1"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 7901); AnnotateMozCrashReason("MOZ_ASSERT" "(" "lir->numDefs() == 1"
")"); do { *((volatile int*)__null) = 7901; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
7902 if (lir->getDef(0)->isBogusTemp()) {
7903 return;
7904 }
7905
7906 Register output = ToRegister(lir->getDef(0));
7907 emitAssertGCThingResult(output, mir);
7908}
7909
7910void CodeGenerator::emitValueResultChecks(LInstruction* lir, MDefinition* mir) {
7911 if (lir->numDefs() == 0) {
7912 return;
7913 }
7914
7915 MOZ_ASSERT(lir->numDefs() == BOX_PIECES)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(lir->numDefs() == 1)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(lir->numDefs() == 1))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("lir->numDefs() == 1"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 7915); AnnotateMozCrashReason("MOZ_ASSERT" "(" "lir->numDefs() == 1"
")"); do { *((volatile int*)__null) = 7915; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
7916 if (!lir->getDef(0)->output()->isRegister()) {
7917 return;
7918 }
7919
7920 ValueOperand output = ToOutValue(lir);
7921
7922 emitAssertResultV(output, mir);
7923}
7924
7925void CodeGenerator::emitDebugResultChecks(LInstruction* ins) {
7926 // In debug builds, check that LIR instructions return valid values.
7927
7928 MDefinition* mir = ins->mirRaw();
7929 if (!mir) {
7930 return;
7931 }
7932
7933 switch (mir->type()) {
7934 case MIRType::Object:
7935 case MIRType::String:
7936 case MIRType::Symbol:
7937 case MIRType::BigInt:
7938 emitGCThingResultChecks(ins, mir);
7939 break;
7940 case MIRType::Value:
7941 emitValueResultChecks(ins, mir);
7942 break;
7943 default:
7944 break;
7945 }
7946}
7947
7948void CodeGenerator::emitDebugForceBailing(LInstruction* lir) {
7949 if (MOZ_LIKELY(!gen->options.ionBailAfterEnabled())(__builtin_expect(!!(!gen->options.ionBailAfterEnabled()),
1))
) {
7950 return;
7951 }
7952 if (!lir->snapshot()) {
7953 return;
7954 }
7955 if (lir->isOsiPoint()) {
7956 return;
7957 }
7958
7959 masm.comment("emitDebugForceBailing");
7960 const void* bailAfterCounterAddr =
7961 gen->runtime->addressOfIonBailAfterCounter();
7962
7963 AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
7964
7965 Label done, notBail;
7966 masm.branch32(Assembler::Equal, AbsoluteAddress(bailAfterCounterAddr),
7967 Imm32(0), &done);
7968 {
7969 Register temp = regs.takeAny();
7970
7971 masm.push(temp);
7972 masm.load32(AbsoluteAddress(bailAfterCounterAddr), temp);
7973 masm.sub32(Imm32(1), temp);
7974 masm.store32(temp, AbsoluteAddress(bailAfterCounterAddr));
7975
7976 masm.branch32(Assembler::NotEqual, temp, Imm32(0), &notBail);
7977 {
7978 masm.pop(temp);
7979 bailout(lir->snapshot());
7980 }
7981 masm.bind(&notBail);
7982 masm.pop(temp);
7983 }
7984 masm.bind(&done);
7985}
7986#endif
7987
7988bool CodeGenerator::generateBody() {
7989 JitSpewCont(JitSpew_Codegen, "\n");
7990 AutoCreatedBy acb(masm, "CodeGenerator::generateBody");
7991
7992 JitSpew(JitSpew_Codegen, "==== BEGIN CodeGenerator::generateBody ====");
7993 IonScriptCounts* counts = maybeCreateScriptCounts();
7994
7995 const bool compilingWasm = gen->compilingWasm();
7996
7997 for (size_t i = 0; i < graph.numBlocks(); i++) {
7998 current = graph.getBlock(i);
7999
8000 // Don't emit any code for trivial blocks, containing just a goto. Such
8001 // blocks are created to split critical edges, and if we didn't end up
8002 // putting any instructions in them, we can skip them.
8003 if (current->isTrivial()) {
8004 continue;
8005 }
8006
8007#ifdef JS_JITSPEW1
8008 const char* filename = nullptr;
8009 size_t lineNumber = 0;
8010 JS::LimitedColumnNumberOneOrigin columnNumber;
8011 if (current->mir()->info().script()) {
8012 filename = current->mir()->info().script()->filename();
8013 if (current->mir()->pc()) {
8014 lineNumber = PCToLineNumber(current->mir()->info().script(),
8015 current->mir()->pc(), &columnNumber);
8016 }
8017 }
8018 JitSpew(JitSpew_Codegen, "--------------------------------");
8019 JitSpew(JitSpew_Codegen, "# block%zu %s:%zu:%u%s:", i,
8020 filename ? filename : "?", lineNumber,
8021 columnNumber.oneOriginValue(),
8022 current->mir()->isLoopHeader() ? " (loop header)" : "");
8023#endif
8024
8025 if (current->mir()->isLoopHeader() && compilingWasm) {
8026 masm.nopAlign(CodeAlignment);
8027 }
8028
8029 masm.bind(current->label());
8030
8031 mozilla::Maybe<ScriptCountBlockState> blockCounts;
8032 if (counts) {
8033 blockCounts.emplace(&counts->block(i), &masm);
8034 if (!blockCounts->init()) {
8035 return false;
8036 }
8037 }
8038
8039 for (LInstructionIterator iter = current->begin(); iter != current->end();
8040 iter++) {
8041 if (!alloc().ensureBallast()) {
8042 return false;
8043 }
8044
8045 perfSpewer_.recordInstruction(masm, *iter);
8046#ifdef JS_JITSPEW1
8047 JitSpewStart(JitSpew_Codegen, " # LIR=%s",
8048 iter->opName());
8049 if (const char* extra = iter->getExtraName()) {
8050 JitSpewCont(JitSpew_Codegen, ":%s", extra);
8051 }
8052 JitSpewFin(JitSpew_Codegen);
8053#endif
8054
8055 if (counts) {
8056 blockCounts->visitInstruction(*iter);
8057 }
8058
8059#ifdef CHECK_OSIPOINT_REGISTERS1
8060 if (iter->safepoint() && !compilingWasm) {
8061 resetOsiPointRegs(iter->safepoint());
8062 }
8063#endif
8064
8065 if (!compilingWasm) {
8066 if (MDefinition* mir = iter->mirRaw()) {
8067 if (!addNativeToBytecodeEntry(mir->trackedSite())) {
8068 return false;
8069 }
8070 }
8071 }
8072
8073 setElement(*iter); // needed to encode correct snapshot location.
8074
8075#ifdef DEBUG1
8076 emitDebugForceBailing(*iter);
8077#endif
8078
8079 switch (iter->op()) {
8080#ifndef JS_CODEGEN_NONE
8081# define LIROP(op) \
8082 case LNode::Opcode::op: \
8083 visit##op(iter->to##op()); \
8084 break;
8085 LIR_OPCODE_LIST(LIROP)LIROP(Phi)LIROP(Box)LIROP(OsiPoint)LIROP(MoveGroup)LIROP(Integer
)LIROP(Integer64)LIROP(Pointer)LIROP(Double)LIROP(Float32)LIROP
(Value)LIROP(NurseryObject)LIROP(Parameter)LIROP(Callee)LIROP
(IsConstructing)LIROP(Goto)LIROP(NewArray)LIROP(NewArrayDynamicLength
)LIROP(NewIterator)LIROP(NewTypedArray)LIROP(NewTypedArrayDynamicLength
)LIROP(NewTypedArrayFromArray)LIROP(NewTypedArrayFromArrayBuffer
)LIROP(BindFunction)LIROP(NewBoundFunction)LIROP(NewObject)LIROP
(NewPlainObject)LIROP(NewArrayObject)LIROP(NewNamedLambdaObject
)LIROP(NewCallObject)LIROP(NewStringObject)LIROP(InitElemGetterSetter
)LIROP(MutateProto)LIROP(InitPropGetterSetter)LIROP(CheckOverRecursed
)LIROP(WasmTrap)LIROP(WasmTrapIfNull)LIROP(WasmRefIsSubtypeOfConcrete
)LIROP(WasmRefIsSubtypeOfAbstract)LIROP(WasmRefIsSubtypeOfConcreteAndBranch
)LIROP(WasmRefIsSubtypeOfAbstractAndBranch)LIROP(WasmNewStructObject
)LIROP(WasmNewArrayObject)LIROP(WasmReinterpret)LIROP(WasmReinterpretFromI64
)LIROP(WasmReinterpretToI64)LIROP(Rotate)LIROP(RotateI64)LIROP
(InterruptCheck)LIROP(WasmStackSwitchToMain)LIROP(WasmStackSwitchToSuspendable
)LIROP(WasmStackContinueOnSuspendable)LIROP(WasmInterruptCheck
)LIROP(TypeOfV)LIROP(TypeOfO)LIROP(TypeOfName)LIROP(TypeOfIsNonPrimitiveV
)LIROP(TypeOfIsNonPrimitiveO)LIROP(TypeOfIsPrimitive)LIROP(ToAsyncIter
)LIROP(ToPropertyKeyCache)LIROP(CreateThis)LIROP(CreateArgumentsObject
)LIROP(CreateInlinedArgumentsObject)LIROP(GetInlinedArgument)
LIROP(GetInlinedArgumentHole)LIROP(GetArgumentsObjectArg)LIROP
(SetArgumentsObjectArg)LIROP(LoadArgumentsObjectArg)LIROP(LoadArgumentsObjectArgHole
)LIROP(InArgumentsObjectArg)LIROP(ArgumentsObjectLength)LIROP
(ArrayFromArgumentsObject)LIROP(GuardArgumentsObjectFlags)LIROP
(BoundFunctionNumArgs)LIROP(GuardBoundFunctionIsConstructor)LIROP
(ReturnFromCtor)LIROP(BoxNonStrictThis)LIROP(ImplicitThis)LIROP
(StackArgT)LIROP(StackArgV)LIROP(CallGeneric)LIROP(CallKnown)
LIROP(CallNative)LIROP(CallDOMNative)LIROP(CallClassHook)LIROP
(Bail)LIROP(Unreachable)LIROP(EncodeSnapshot)LIROP(UnreachableResultV
)LIROP(UnreachableResultT)LIROP(GetDOMProperty)LIROP(GetDOMMemberV
)LIROP(GetDOMMemberT)LIROP(SetDOMProperty)LIROP(LoadDOMExpandoValue
)LIROP(LoadDOMExpandoValueGuardGeneration)LIROP(LoadDOMExpandoValueIgnoreGeneration
)LIROP(GuardDOMExpandoMissingOrGuardShape)LIROP(ApplyArgsGeneric
)LIROP(ApplyArgsObj)LIROP(ApplyArrayGeneric)LIROP(ConstructArgsGeneric
)LIROP(ConstructArrayGeneric)LIROP(ApplyArgsNative)LIROP(ApplyArgsObjNative
)LIROP(ApplyArrayNative)LIROP(ConstructArgsNative)LIROP(ConstructArrayNative
)LIROP(TestIAndBranch)LIROP(TestIPtrAndBranch)LIROP(TestI64AndBranch
)LIROP(TestDAndBranch)LIROP(TestFAndBranch)LIROP(TestBIAndBranch
)LIROP(TestOAndBranch)LIROP(TestVAndBranch)LIROP(Compare)LIROP
(CompareI64)LIROP(CompareI64AndBranch)LIROP(CompareAndBranch)
LIROP(CompareD)LIROP(CompareF)LIROP(CompareDAndBranch)LIROP(CompareFAndBranch
)LIROP(CompareS)LIROP(CompareSInline)LIROP(CompareSSingle)LIROP
(CompareBigInt)LIROP(CompareBigIntInt32)LIROP(CompareBigIntDouble
)LIROP(CompareBigIntString)LIROP(CompareBigIntInt32AndBranch)
LIROP(BitAndAndBranch)LIROP(BitAnd64AndBranch)LIROP(IsNullOrLikeUndefinedV
)LIROP(IsNullOrLikeUndefinedT)LIROP(IsNull)LIROP(IsUndefined)
LIROP(IsNullOrLikeUndefinedAndBranchV)LIROP(IsNullOrLikeUndefinedAndBranchT
)LIROP(IsNullAndBranch)LIROP(IsUndefinedAndBranch)LIROP(SameValueDouble
)LIROP(SameValue)LIROP(NotI)LIROP(NotIPtr)LIROP(NotI64)LIROP(
NotD)LIROP(NotF)LIROP(NotBI)LIROP(NotO)LIROP(NotV)LIROP(BitNotI
)LIROP(BitNotI64)LIROP(BitOpI)LIROP(BitOpI64)LIROP(ShiftI)LIROP
(ShiftI64)LIROP(SignExtendInt32)LIROP(SignExtendInt64)LIROP(UrshD
)LIROP(Return)LIROP(Throw)LIROP(ThrowWithStack)LIROP(MinMaxI)
LIROP(MinMaxD)LIROP(MinMaxF)LIROP(MinMaxArrayI)LIROP(MinMaxArrayD
)LIROP(NegI)LIROP(NegI64)LIROP(NegD)LIROP(NegF)LIROP(AbsI)LIROP
(AbsD)LIROP(AbsF)LIROP(CopySignD)LIROP(CopySignF)LIROP(ClzI)LIROP
(ClzI64)LIROP(CtzI)LIROP(CtzI64)LIROP(PopcntI)LIROP(PopcntI64
)LIROP(SqrtD)LIROP(SqrtF)LIROP(Atan2D)LIROP(Hypot)LIROP(PowI)
LIROP(PowII)LIROP(PowD)LIROP(PowOfTwoI)LIROP(SignI)LIROP(SignD
)LIROP(SignDI)LIROP(MathFunctionD)LIROP(MathFunctionF)LIROP(AddI
)LIROP(AddI64)LIROP(SubI)LIROP(SubI64)LIROP(MulI64)LIROP(MathD
)LIROP(MathF)LIROP(ModD)LIROP(ModPowTwoD)LIROP(WasmBuiltinModD
)LIROP(BigIntAdd)LIROP(BigIntSub)LIROP(BigIntMul)LIROP(BigIntDiv
)LIROP(BigIntMod)LIROP(BigIntPow)LIROP(BigIntBitAnd)LIROP(BigIntBitOr
)LIROP(BigIntBitXor)LIROP(BigIntLsh)LIROP(BigIntRsh)LIROP(BigIntIncrement
)LIROP(BigIntDecrement)LIROP(BigIntNegate)LIROP(BigIntBitNot)
LIROP(BigIntToIntPtr)LIROP(IntPtrToBigInt)LIROP(BigIntPtrAdd)
LIROP(BigIntPtrSub)LIROP(BigIntPtrMul)LIROP(BigIntPtrDiv)LIROP
(BigIntPtrDivPowTwo)LIROP(BigIntPtrMod)LIROP(BigIntPtrModPowTwo
)LIROP(BigIntPtrPow)LIROP(BigIntPtrBitAnd)LIROP(BigIntPtrBitOr
)LIROP(BigIntPtrBitXor)LIROP(BigIntPtrLsh)LIROP(BigIntPtrRsh)
LIROP(BigIntPtrBitNot)LIROP(Int32ToStringWithBase)LIROP(NumberParseInt
)LIROP(DoubleParseInt)LIROP(Concat)LIROP(LinearizeString)LIROP
(LinearizeForCharAccess)LIROP(LinearizeForCodePointAccess)LIROP
(ToRelativeStringIndex)LIROP(CharCodeAt)LIROP(CharCodeAtOrNegative
)LIROP(CodePointAt)LIROP(CodePointAtOrNegative)LIROP(NegativeToNaN
)LIROP(NegativeToUndefined)LIROP(FromCharCode)LIROP(FromCharCodeEmptyIfNegative
)LIROP(FromCharCodeUndefinedIfNegative)LIROP(FromCodePoint)LIROP
(StringIncludes)LIROP(StringIncludesSIMD)LIROP(StringIndexOf)
LIROP(StringIndexOfSIMD)LIROP(StringLastIndexOf)LIROP(StringStartsWith
)LIROP(StringStartsWithInline)LIROP(StringEndsWith)LIROP(StringEndsWithInline
)LIROP(StringToLowerCase)LIROP(CharCodeToLowerCase)LIROP(StringToUpperCase
)LIROP(CharCodeToUpperCase)LIROP(StringTrimStartIndex)LIROP(StringTrimEndIndex
)LIROP(StringSplit)LIROP(Substr)LIROP(Int32ToDouble)LIROP(Float32ToDouble
)LIROP(DoubleToFloat32)LIROP(Int32ToFloat32)LIROP(DoubleToFloat16
)LIROP(DoubleToFloat32ToFloat16)LIROP(Float32ToFloat16)LIROP(
Int32ToFloat16)LIROP(ValueToDouble)LIROP(ValueToFloat32)LIROP
(ValueToFloat16)LIROP(ValueToInt32)LIROP(ValueToBigInt)LIROP(
DoubleToInt32)LIROP(Float32ToInt32)LIROP(TruncateDToInt32)LIROP
(WasmBuiltinTruncateDToInt32)LIROP(TruncateFToInt32)LIROP(WasmBuiltinTruncateFToInt32
)LIROP(WasmTruncateToInt32)LIROP(WrapInt64ToInt32)LIROP(ExtendInt32ToInt64
)LIROP(BooleanToString)LIROP(IntToString)LIROP(DoubleToString
)LIROP(ValueToString)LIROP(PowHalfD)LIROP(NaNToZero)LIROP(OsrEntry
)LIROP(OsrValue)LIROP(OsrEnvironmentChain)LIROP(OsrReturnValue
)LIROP(OsrArgumentsObject)LIROP(RegExp)LIROP(RegExpMatcher)LIROP
(RegExpSearcher)LIROP(RegExpSearcherLastLimit)LIROP(RegExpExecMatch
)LIROP(RegExpExecTest)LIROP(RegExpHasCaptureGroups)LIROP(RegExpPrototypeOptimizable
)LIROP(RegExpInstanceOptimizable)LIROP(GetFirstDollarIndex)LIROP
(StringReplace)LIROP(BinaryValueCache)LIROP(BinaryBoolCache)LIROP
(UnaryCache)LIROP(ModuleMetadata)LIROP(DynamicImport)LIROP(Lambda
)LIROP(FunctionWithProto)LIROP(SetFunName)LIROP(KeepAliveObject
)LIROP(DebugEnterGCUnsafeRegion)LIROP(DebugLeaveGCUnsafeRegion
)LIROP(Slots)LIROP(Elements)LIROP(InitializedLength)LIROP(SetInitializedLength
)LIROP(ArrayLength)LIROP(SetArrayLength)LIROP(FunctionLength)
LIROP(FunctionName)LIROP(GetNextEntryForIterator)LIROP(ArrayBufferByteLength
)LIROP(ArrayBufferViewLength)LIROP(ArrayBufferViewByteOffset)
LIROP(ArrayBufferViewElements)LIROP(TypedArrayElementSize)LIROP
(ResizableTypedArrayLength)LIROP(ResizableTypedArrayByteOffsetMaybeOutOfBounds
)LIROP(ResizableDataViewByteLength)LIROP(GrowableSharedArrayBufferByteLength
)LIROP(GuardResizableArrayBufferViewInBounds)LIROP(GuardResizableArrayBufferViewInBoundsOrDetached
)LIROP(GuardHasAttachedArrayBuffer)LIROP(GuardNumberToIntPtrIndex
)LIROP(BoundsCheck)LIROP(BoundsCheckRange)LIROP(BoundsCheckLower
)LIROP(SpectreMaskIndex)LIROP(LoadElementV)LIROP(InArray)LIROP
(GuardElementNotHole)LIROP(LoadElementHole)LIROP(StoreElementV
)LIROP(StoreElementT)LIROP(StoreHoleValueElement)LIROP(StoreElementHoleV
)LIROP(StoreElementHoleT)LIROP(ArrayPopShift)LIROP(ArrayPush)
LIROP(ArraySlice)LIROP(ArgumentsSlice)LIROP(FrameArgumentsSlice
)LIROP(InlineArgumentsSlice)LIROP(NormalizeSliceTerm)LIROP(ArrayJoin
)LIROP(ObjectKeys)LIROP(ObjectKeysLength)LIROP(LoadUnboxedScalar
)LIROP(LoadUnboxedInt64)LIROP(LoadDataViewElement)LIROP(LoadDataViewElement64
)LIROP(LoadTypedArrayElementHole)LIROP(LoadTypedArrayElementHoleBigInt
)LIROP(StoreUnboxedScalar)LIROP(StoreUnboxedInt64)LIROP(StoreDataViewElement
)LIROP(StoreDataViewElement64)LIROP(StoreTypedArrayElementHole
)LIROP(StoreTypedArrayElementHoleInt64)LIROP(AtomicIsLockFree
)LIROP(CompareExchangeTypedArrayElement)LIROP(AtomicExchangeTypedArrayElement
)LIROP(AtomicTypedArrayElementBinop)LIROP(AtomicTypedArrayElementBinopForEffect
)LIROP(AtomicLoad64)LIROP(AtomicStore64)LIROP(CompareExchangeTypedArrayElement64
)LIROP(AtomicExchangeTypedArrayElement64)LIROP(AtomicTypedArrayElementBinop64
)LIROP(AtomicTypedArrayElementBinopForEffect64)LIROP(EffectiveAddress
)LIROP(ClampIToUint8)LIROP(ClampDToUint8)LIROP(ClampVToUint8)
LIROP(LoadScriptedProxyHandler)LIROP(CheckScriptedProxyGetResult
)LIROP(IdToStringOrSymbol)LIROP(LoadFixedSlotV)LIROP(LoadFixedSlotAndAtomize
)LIROP(LoadFixedSlotT)LIROP(LoadFixedSlotAndUnbox)LIROP(LoadDynamicSlotAndUnbox
)LIROP(LoadElementAndUnbox)LIROP(LoadFixedSlotUnboxAndAtomize
)LIROP(LoadDynamicSlotUnboxAndAtomize)LIROP(AddAndStoreSlot)LIROP
(AllocateAndStoreSlot)LIROP(AddSlotAndCallAddPropHook)LIROP(StoreFixedSlotV
)LIROP(StoreFixedSlotT)LIROP(GetNameCache)LIROP(CallGetIntrinsicValue
)LIROP(GetPropSuperCache)LIROP(GetPropertyCache)LIROP(BindNameCache
)LIROP(CallBindVar)LIROP(LoadDynamicSlotV)LIROP(LoadDynamicSlotAndAtomize
)LIROP(StoreDynamicSlotV)LIROP(StoreDynamicSlotT)LIROP(StringLength
)LIROP(Floor)LIROP(FloorF)LIROP(Ceil)LIROP(CeilF)LIROP(Round)
LIROP(RoundF)LIROP(Trunc)LIROP(TruncF)LIROP(NearbyInt)LIROP(NearbyIntF
)LIROP(FunctionEnvironment)LIROP(HomeObject)LIROP(HomeObjectSuperBase
)LIROP(NewLexicalEnvironmentObject)LIROP(NewClassBodyEnvironmentObject
)LIROP(NewVarEnvironmentObject)LIROP(MegamorphicSetElement)LIROP
(CallDeleteProperty)LIROP(CallDeleteElement)LIROP(ObjectToIterator
)LIROP(ValueToIterator)LIROP(IteratorHasIndicesAndBranch)LIROP
(LoadSlotByIteratorIndex)LIROP(StoreSlotByIteratorIndex)LIROP
(SetPropertyCache)LIROP(GetIteratorCache)LIROP(OptimizeSpreadCallCache
)LIROP(IteratorMore)LIROP(IsNoIterAndBranch)LIROP(IteratorEnd
)LIROP(CloseIterCache)LIROP(OptimizeGetIteratorCache)LIROP(ArgumentsLength
)LIROP(GetFrameArgument)LIROP(GetFrameArgumentHole)LIROP(Rest
)LIROP(Int32ToIntPtr)LIROP(NonNegativeIntPtrToInt32)LIROP(IntPtrToDouble
)LIROP(AdjustDataViewLength)LIROP(BooleanToInt64)LIROP(StringToInt64
)LIROP(ValueToInt64)LIROP(TruncateBigIntToInt64)LIROP(Int32ToBigInt
)LIROP(Int64ToBigInt)LIROP(Uint64ToBigInt)LIROP(Int64ToIntPtr
)LIROP(IntPtrToInt64)LIROP(PostWriteBarrierO)LIROP(PostWriteBarrierS
)LIROP(PostWriteBarrierBI)LIROP(PostWriteBarrierV)LIROP(PostWriteElementBarrierO
)LIROP(PostWriteElementBarrierS)LIROP(PostWriteElementBarrierBI
)LIROP(PostWriteElementBarrierV)LIROP(AssertCanElidePostWriteBarrier
)LIROP(GuardObjectIdentity)LIROP(GuardSpecificFunction)LIROP(
GuardSpecificAtom)LIROP(GuardSpecificSymbol)LIROP(GuardSpecificInt32
)LIROP(GuardStringToIndex)LIROP(GuardStringToInt32)LIROP(GuardStringToDouble
)LIROP(GuardShape)LIROP(GuardMultipleShapes)LIROP(GuardProto)
LIROP(GuardNullProto)LIROP(GuardIsNativeObject)LIROP(GuardGlobalGeneration
)LIROP(GuardFuse)LIROP(GuardIsProxy)LIROP(GuardIsNotProxy)LIROP
(GuardIsNotDOMProxy)LIROP(ProxyGet)LIROP(ProxyGetByValue)LIROP
(ProxyHasProp)LIROP(ProxySet)LIROP(ProxySetByValue)LIROP(CallSetArrayLength
)LIROP(MegamorphicLoadSlot)LIROP(MegamorphicLoadSlotByValue)LIROP
(MegamorphicLoadSlotPermissive)LIROP(MegamorphicLoadSlotByValuePermissive
)LIROP(MegamorphicStoreSlot)LIROP(MegamorphicHasProp)LIROP(SmallObjectVariableKeyHasProp
)LIROP(GuardIsNotArrayBufferMaybeShared)LIROP(GuardIsTypedArray
)LIROP(GuardIsFixedLengthTypedArray)LIROP(GuardIsResizableTypedArray
)LIROP(GuardHasProxyHandler)LIROP(GuardNoDenseElements)LIROP(
InCache)LIROP(HasOwnCache)LIROP(CheckPrivateFieldCache)LIROP(
NewPrivateName)LIROP(InstanceOfO)LIROP(InstanceOfV)LIROP(InstanceOfCache
)LIROP(IsCallableO)LIROP(IsCallableV)LIROP(IsConstructor)LIROP
(IsCrossRealmArrayConstructor)LIROP(IsArrayO)LIROP(IsArrayV)LIROP
(IsTypedArray)LIROP(IsObject)LIROP(IsObjectAndBranch)LIROP(IsNullOrUndefined
)LIROP(IsNullOrUndefinedAndBranch)LIROP(HasClass)LIROP(GuardToClass
)LIROP(GuardToEitherClass)LIROP(GuardToFunction)LIROP(ObjectClassToString
)LIROP(WasmSelect)LIROP(WasmSelectI64)LIROP(WasmCompareAndSelect
)LIROP(WasmAddOffset)LIROP(WasmAddOffset64)LIROP(WasmBoundsCheck
)LIROP(WasmBoundsCheck64)LIROP(WasmBoundsCheckRange32)LIROP(WasmExtendU32Index
)LIROP(WasmWrapU32Index)LIROP(WasmClampTable64Index)LIROP(WasmAlignmentCheck
)LIROP(WasmAlignmentCheck64)LIROP(WasmLoadInstance)LIROP(WasmLoadInstance64
)LIROP(WasmHeapReg)LIROP(WasmLoad)LIROP(WasmLoadI64)LIROP(WasmStore
)LIROP(WasmStoreI64)LIROP(AsmJSLoadHeap)LIROP(AsmJSStoreHeap)
LIROP(WasmCompareExchangeHeap)LIROP(WasmFence)LIROP(WasmAtomicExchangeHeap
)LIROP(WasmAtomicBinopHeap)LIROP(WasmAtomicBinopHeapForEffect
)LIROP(WasmLoadSlot)LIROP(WasmLoadElement)LIROP(WasmLoadSlotI64
)LIROP(WasmLoadElementI64)LIROP(WasmStoreSlot)LIROP(WasmStoreSlotI64
)LIROP(WasmStoreElement)LIROP(WasmStoreElementI64)LIROP(WasmStoreElementRef
)LIROP(WasmLoadTableElement)LIROP(WasmDerivedPointer)LIROP(WasmDerivedIndexPointer
)LIROP(WasmStoreRef)LIROP(WasmPostWriteBarrierImmediate)LIROP
(WasmPostWriteBarrierIndex)LIROP(WasmParameter)LIROP(WasmParameterI64
)LIROP(WasmReturn)LIROP(WasmReturnI64)LIROP(WasmReturnVoid)LIROP
(WasmStackArg)LIROP(WasmStackArgI64)LIROP(WasmNullConstant)LIROP
(WasmCallIndirectAdjunctSafepoint)LIROP(WasmCall)LIROP(WasmCallLandingPrePad
)LIROP(WasmRegisterResult)LIROP(WasmRegisterPairResult)LIROP(
WasmStackResultArea)LIROP(WasmStackResult)LIROP(WasmStackResult64
)LIROP(AssertRangeI)LIROP(AssertRangeD)LIROP(AssertRangeF)LIROP
(AssertRangeV)LIROP(AssertClass)LIROP(AssertShape)LIROP(GuardValue
)LIROP(GuardNullOrUndefined)LIROP(GuardIsNotObject)LIROP(GuardFunctionFlags
)LIROP(GuardFunctionIsNonBuiltinCtor)LIROP(GuardFunctionKind)
LIROP(GuardFunctionScript)LIROP(IncrementWarmUpCounter)LIROP(
LexicalCheck)LIROP(ThrowRuntimeLexicalError)LIROP(ThrowMsg)LIROP
(GlobalDeclInstantiation)LIROP(MemoryBarrier)LIROP(Debugger)LIROP
(NewTarget)LIROP(Random)LIROP(CheckReturn)LIROP(CheckIsObj)LIROP
(CheckObjCoercible)LIROP(CheckClassHeritage)LIROP(CheckThis)LIROP
(CheckThisReinit)LIROP(Generator)LIROP(AsyncResolve)LIROP(AsyncReject
)LIROP(AsyncAwait)LIROP(CanSkipAwait)LIROP(MaybeExtractAwaitValue
)LIROP(DebugCheckSelfHosted)LIROP(IsPackedArray)LIROP(GuardArrayIsPacked
)LIROP(GetPrototypeOf)LIROP(ObjectWithProto)LIROP(ObjectStaticProto
)LIROP(BuiltinObject)LIROP(SuperFunction)LIROP(InitHomeObject
)LIROP(IsTypedArrayConstructor)LIROP(LoadValueTag)LIROP(GuardTagNotEqual
)LIROP(LoadWrapperTarget)LIROP(GuardHasGetterSetter)LIROP(GuardIsExtensible
)LIROP(GuardInt32IsNonNegative)LIROP(GuardInt32Range)LIROP(GuardIndexIsNotDenseElement
)LIROP(GuardIndexIsValidUpdateOrAdd)LIROP(CallAddOrUpdateSparseElement
)LIROP(CallGetSparseElement)LIROP(CallNativeGetElement)LIROP(
CallNativeGetElementSuper)LIROP(CallObjectHasSparseElement)LIROP
(BigIntAsIntN)LIROP(BigIntAsIntN64)LIROP(BigIntAsIntN32)LIROP
(GuardNonGCThing)LIROP(ToHashableNonGCThing)LIROP(ToHashableString
)LIROP(ToHashableValue)LIROP(HashNonGCThing)LIROP(HashString)
LIROP(HashSymbol)LIROP(HashBigInt)LIROP(HashObject)LIROP(HashValue
)LIROP(SetObjectHasNonBigInt)LIROP(SetObjectHasBigInt)LIROP(SetObjectHasValue
)LIROP(SetObjectHasValueVMCall)LIROP(SetObjectSize)LIROP(MapObjectHasNonBigInt
)LIROP(MapObjectHasBigInt)LIROP(MapObjectHasValue)LIROP(MapObjectHasValueVMCall
)LIROP(MapObjectGetNonBigInt)LIROP(MapObjectGetBigInt)LIROP(MapObjectGetValueVMCall
)LIROP(MapObjectSize)LIROP(BigIntAsUintN)LIROP(BigIntAsUintN64
)LIROP(BigIntAsUintN32)LIROP(IonToWasmCall)LIROP(IonToWasmCallV
)LIROP(IonToWasmCallI64)LIROP(WasmAnyRefFromJSValue)LIROP(WasmAnyRefFromJSObject
)LIROP(WasmAnyRefFromJSString)LIROP(WasmAnyRefIsJSString)LIROP
(WasmTrapIfAnyRefIsNotJSString)LIROP(WasmNewI31Ref)LIROP(WasmI31RefGet
)LIROP(Simd128)LIROP(WasmTernarySimd128)LIROP(WasmBinarySimd128
)LIROP(WasmBinarySimd128WithConstant)LIROP(WasmVariableShiftSimd128
)LIROP(WasmConstantShiftSimd128)LIROP(WasmSignReplicationSimd128
)LIROP(WasmShuffleSimd128)LIROP(WasmPermuteSimd128)LIROP(WasmReplaceLaneSimd128
)LIROP(WasmReplaceInt64LaneSimd128)LIROP(WasmScalarToSimd128)
LIROP(WasmInt64ToSimd128)LIROP(WasmUnarySimd128)LIROP(WasmReduceSimd128
)LIROP(WasmReduceAndBranchSimd128)LIROP(WasmReduceSimd128ToInt64
)LIROP(WasmLoadLaneSimd128)LIROP(WasmStoreLaneSimd128)LIROP(Unbox
)LIROP(UnboxFloatingPoint)LIROP(WasmUint32ToDouble)LIROP(WasmUint32ToFloat32
)LIROP(DivI)LIROP(ModI)LIROP(DivPowTwoI)LIROP(ModPowTwoI)LIROP
(TableSwitch)LIROP(TableSwitchV)LIROP(MulI)LIROP(DivOrModI64)
LIROP(UDivOrModI64)LIROP(DivOrModConstantI)LIROP(UDivOrMod)LIROP
(UDivOrModConstant)LIROP(WasmTruncateToInt64)LIROP(Int64ToFloatingPoint
)LIROP(MapObjectGetValue)
8086# undef LIROP
8087#endif
8088 case LNode::Opcode::Invalid:
8089 default:
8090 MOZ_CRASH("Invalid LIR op")do { do { } while (false); MOZ_ReportCrash("" "Invalid LIR op"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 8090); AnnotateMozCrashReason("MOZ_CRASH(" "Invalid LIR op"
")"); do { *((volatile int*)__null) = 8090; __attribute__((nomerge
)) ::abort(); } while (false); } while (false)
;
8091 }
8092
8093#ifdef DEBUG1
8094 if (!counts) {
8095 emitDebugResultChecks(*iter);
8096 }
8097#endif
8098 }
8099 if (masm.oom()) {
8100 return false;
8101 }
8102 }
8103
8104 JitSpew(JitSpew_Codegen, "==== END CodeGenerator::generateBody ====\n");
8105 return true;
8106}
8107
8108// Out-of-line object allocation for LNewArray.
8109class OutOfLineNewArray : public OutOfLineCodeBase<CodeGenerator> {
8110 LNewArray* lir_;
8111
8112 public:
8113 explicit OutOfLineNewArray(LNewArray* lir) : lir_(lir) {}
8114
8115 void accept(CodeGenerator* codegen) override {
8116 codegen->visitOutOfLineNewArray(this);
8117 }
8118
8119 LNewArray* lir() const { return lir_; }
8120};
8121
8122void CodeGenerator::visitNewArrayCallVM(LNewArray* lir) {
8123 Register objReg = ToRegister(lir->output());
8124
8125 MOZ_ASSERT(!lir->isCall())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!lir->isCall())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!lir->isCall()))), 0))) {
do { } while (false); MOZ_ReportAssertionFailure("!lir->isCall()"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 8125); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!lir->isCall()"
")"); do { *((volatile int*)__null) = 8125; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
8126 saveLive(lir);
8127
8128 JSObject* templateObject = lir->mir()->templateObject();
8129
8130 if (templateObject) {
8131 pushArg(ImmGCPtr(templateObject->shape()));
8132 pushArg(Imm32(lir->mir()->length()));
8133
8134 using Fn = ArrayObject* (*)(JSContext*, uint32_t, Handle<Shape*>);
8135 callVM<Fn, NewArrayWithShape>(lir);
8136 } else {
8137 pushArg(Imm32(GenericObject));
8138 pushArg(Imm32(lir->mir()->length()));
8139
8140 using Fn = ArrayObject* (*)(JSContext*, uint32_t, NewObjectKind);
8141 callVM<Fn, NewArrayOperation>(lir);
8142 }
8143
8144 masm.storeCallPointerResult(objReg);
8145
8146 MOZ_ASSERT(!lir->safepoint()->liveRegs().has(objReg))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!lir->safepoint()->liveRegs().has(objReg))>
::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(!lir->safepoint()->liveRegs().has(objReg)))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("!lir->safepoint()->liveRegs().has(objReg)"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 8146); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!lir->safepoint()->liveRegs().has(objReg)"
")"); do { *((volatile int*)__null) = 8146; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
8147 restoreLive(lir);
8148}
8149
8150void CodeGenerator::visitAtan2D(LAtan2D* lir) {
8151 FloatRegister y = ToFloatRegister(lir->y());
8152 FloatRegister x = ToFloatRegister(lir->x());
8153
8154 using Fn = double (*)(double x, double y);
8155 masm.setupAlignedABICall();
8156 masm.passABIArg(y, ABIType::Float64);
8157 masm.passABIArg(x, ABIType::Float64);
8158 masm.callWithABI<Fn, ecmaAtan2>(ABIType::Float64);
8159
8160 MOZ_ASSERT(ToFloatRegister(lir->output()) == ReturnDoubleReg)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ToFloatRegister(lir->output()) == ReturnDoubleReg
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(ToFloatRegister(lir->output()) == ReturnDoubleReg
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"ToFloatRegister(lir->output()) == ReturnDoubleReg", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 8160); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ToFloatRegister(lir->output()) == ReturnDoubleReg"
")"); do { *((volatile int*)__null) = 8160; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
8161}
8162
8163void CodeGenerator::visitHypot(LHypot* lir) {
8164 uint32_t numArgs = lir->numArgs();
8165 masm.setupAlignedABICall();
8166
8167 for (uint32_t i = 0; i < numArgs; ++i) {
8168 masm.passABIArg(ToFloatRegister(lir->getOperand(i)), ABIType::Float64);
8169 }
8170
8171 switch (numArgs) {
8172 case 2: {
8173 using Fn = double (*)(double x, double y);
8174 masm.callWithABI<Fn, ecmaHypot>(ABIType::Float64);
8175 break;
8176 }
8177 case 3: {
8178 using Fn = double (*)(double x, double y, double z);
8179 masm.callWithABI<Fn, hypot3>(ABIType::Float64);
8180 break;
8181 }
8182 case 4: {
8183 using Fn = double (*)(double x, double y, double z, double w);
8184 masm.callWithABI<Fn, hypot4>(ABIType::Float64);
8185 break;
8186 }
8187 default:
8188 MOZ_CRASH("Unexpected number of arguments to hypot function.")do { do { } while (false); MOZ_ReportCrash("" "Unexpected number of arguments to hypot function."
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 8188); AnnotateMozCrashReason("MOZ_CRASH(" "Unexpected number of arguments to hypot function."
")"); do { *((volatile int*)__null) = 8188; __attribute__((nomerge
)) ::abort(); } while (false); } while (false)
;
8189 }
8190 MOZ_ASSERT(ToFloatRegister(lir->output()) == ReturnDoubleReg)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ToFloatRegister(lir->output()) == ReturnDoubleReg
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(ToFloatRegister(lir->output()) == ReturnDoubleReg
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"ToFloatRegister(lir->output()) == ReturnDoubleReg", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 8190); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ToFloatRegister(lir->output()) == ReturnDoubleReg"
")"); do { *((volatile int*)__null) = 8190; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
8191}
8192
8193void CodeGenerator::visitNewArray(LNewArray* lir) {
8194 Register objReg = ToRegister(lir->output());
8195 Register tempReg = ToRegister(lir->temp());
8196 DebugOnly<uint32_t> length = lir->mir()->length();
8197
8198 MOZ_ASSERT(length <= NativeObject::MAX_DENSE_ELEMENTS_COUNT)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(length <= NativeObject::MAX_DENSE_ELEMENTS_COUNT)
>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(length <= NativeObject::MAX_DENSE_ELEMENTS_COUNT)
)), 0))) { do { } while (false); MOZ_ReportAssertionFailure("length <= NativeObject::MAX_DENSE_ELEMENTS_COUNT"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 8198); AnnotateMozCrashReason("MOZ_ASSERT" "(" "length <= NativeObject::MAX_DENSE_ELEMENTS_COUNT"
")"); do { *((volatile int*)__null) = 8198; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
8199
8200 if (lir->mir()->isVMCall()) {
8201 visitNewArrayCallVM(lir);
8202 return;
8203 }
8204
8205 OutOfLineNewArray* ool = new (alloc()) OutOfLineNewArray(lir);
8206 addOutOfLineCode(ool, lir->mir());
8207 TemplateObject templateObject(lir->mir()->templateObject());
8208#ifdef DEBUG1
8209 size_t numInlineElements = gc::GetGCKindSlots(templateObject.getAllocKind()) -
8210 ObjectElements::VALUES_PER_HEADER;
8211 MOZ_ASSERT(length <= numInlineElements,do { static_assert( mozilla::detail::AssertionConditionType<
decltype(length <= numInlineElements)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(length <= numInlineElements
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"length <= numInlineElements" " (" "Inline allocation only supports inline elements"
")", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 8212); AnnotateMozCrashReason("MOZ_ASSERT" "(" "length <= numInlineElements"
") (" "Inline allocation only supports inline elements" ")")
; do { *((volatile int*)__null) = 8212; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
8212 "Inline allocation only supports inline elements")do { static_assert( mozilla::detail::AssertionConditionType<
decltype(length <= numInlineElements)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(length <= numInlineElements
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"length <= numInlineElements" " (" "Inline allocation only supports inline elements"
")", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 8212); AnnotateMozCrashReason("MOZ_ASSERT" "(" "length <= numInlineElements"
") (" "Inline allocation only supports inline elements" ")")
; do { *((volatile int*)__null) = 8212; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
8213#endif
8214 masm.createGCObject(objReg, tempReg, templateObject,
8215 lir->mir()->initialHeap(), ool->entry());
8216
8217 masm.bind(ool->rejoin());
8218}
8219
8220void CodeGenerator::visitOutOfLineNewArray(OutOfLineNewArray* ool) {
8221 visitNewArrayCallVM(ool->lir());
8222 masm.jump(ool->rejoin());
8223}
8224
8225void CodeGenerator::visitNewArrayDynamicLength(LNewArrayDynamicLength* lir) {
8226 Register lengthReg = ToRegister(lir->length());
8227 Register objReg = ToRegister(lir->output());
8228 Register tempReg = ToRegister(lir->temp0());
8229
8230 JSObject* templateObject = lir->mir()->templateObject();
8231 gc::Heap initialHeap = lir->mir()->initialHeap();
8232
8233 using Fn = ArrayObject* (*)(JSContext*, Handle<ArrayObject*>, int32_t length,
8234 gc::AllocSite*);
8235 OutOfLineCode* ool = oolCallVM<Fn, ArrayConstructorOneArg>(
8236 lir, ArgList(ImmGCPtr(templateObject), lengthReg, ImmPtr(nullptr)),
8237 StoreRegisterTo(objReg));
8238
8239 bool canInline = true;
8240 size_t inlineLength = 0;
8241 if (templateObject->as<ArrayObject>().hasFixedElements()) {
8242 size_t numSlots =
8243 gc::GetGCKindSlots(templateObject->asTenured().getAllocKind());
8244 inlineLength = numSlots - ObjectElements::VALUES_PER_HEADER;
8245 } else {
8246 canInline = false;
8247 }
8248
8249 if (canInline) {
8250 // Try to do the allocation inline if the template object is big enough
8251 // for the length in lengthReg. If the length is bigger we could still
8252 // use the template object and not allocate the elements, but it's more
8253 // efficient to do a single big allocation than (repeatedly) reallocating
8254 // the array later on when filling it.
8255 masm.branch32(Assembler::Above, lengthReg, Imm32(inlineLength),
8256 ool->entry());
8257
8258 TemplateObject templateObj(templateObject);
8259 masm.createGCObject(objReg, tempReg, templateObj, initialHeap,
8260 ool->entry());
8261
8262 size_t lengthOffset = NativeObject::offsetOfFixedElements() +
8263 ObjectElements::offsetOfLength();
8264 masm.store32(lengthReg, Address(objReg, lengthOffset));
8265 } else {
8266 masm.jump(ool->entry());
8267 }
8268
8269 masm.bind(ool->rejoin());
8270}
8271
8272void CodeGenerator::visitNewIterator(LNewIterator* lir) {
8273 Register objReg = ToRegister(lir->output());
8274 Register tempReg = ToRegister(lir->temp0());
8275
8276 OutOfLineCode* ool;
8277 switch (lir->mir()->type()) {
8278 case MNewIterator::ArrayIterator: {
8279 using Fn = ArrayIteratorObject* (*)(JSContext*);
8280 ool = oolCallVM<Fn, NewArrayIterator>(lir, ArgList(),
8281 StoreRegisterTo(objReg));
8282 break;
8283 }
8284 case MNewIterator::StringIterator: {
8285 using Fn = StringIteratorObject* (*)(JSContext*);
8286 ool = oolCallVM<Fn, NewStringIterator>(lir, ArgList(),
8287 StoreRegisterTo(objReg));
8288 break;
8289 }
8290 case MNewIterator::RegExpStringIterator: {
8291 using Fn = RegExpStringIteratorObject* (*)(JSContext*);
8292 ool = oolCallVM<Fn, NewRegExpStringIterator>(lir, ArgList(),
8293 StoreRegisterTo(objReg));
8294 break;
8295 }
8296 default:
8297 MOZ_CRASH("unexpected iterator type")do { do { } while (false); MOZ_ReportCrash("" "unexpected iterator type"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 8297); AnnotateMozCrashReason("MOZ_CRASH(" "unexpected iterator type"
")"); do { *((volatile int*)__null) = 8297; __attribute__((nomerge
)) ::abort(); } while (false); } while (false)
;
8298 }
8299
8300 TemplateObject templateObject(lir->mir()->templateObject());
8301 masm.createGCObject(objReg, tempReg, templateObject, gc::Heap::Default,
8302 ool->entry());
8303
8304 masm.bind(ool->rejoin());
8305}
8306
8307void CodeGenerator::visitNewTypedArray(LNewTypedArray* lir) {
8308 Register objReg = ToRegister(lir->output());
8309 Register tempReg = ToRegister(lir->temp0());
8310 Register lengthReg = ToRegister(lir->temp1());
8311 LiveRegisterSet liveRegs = liveVolatileRegs(lir);
8312
8313 JSObject* templateObject = lir->mir()->templateObject();
8314 gc::Heap initialHeap = lir->mir()->initialHeap();
8315
8316 auto* ttemplate = &templateObject->as<FixedLengthTypedArrayObject>();
8317
8318 size_t n = ttemplate->length();
8319 MOZ_ASSERT(n <= INT32_MAX,do { static_assert( mozilla::detail::AssertionConditionType<
decltype(n <= (2147483647))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(n <= (2147483647)))), 0))
) { do { } while (false); MOZ_ReportAssertionFailure("n <= (2147483647)"
" (" "Template objects are only created for int32 lengths" ")"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 8320); AnnotateMozCrashReason("MOZ_ASSERT" "(" "n <= (2147483647)"
") (" "Template objects are only created for int32 lengths" ")"
); do { *((volatile int*)__null) = 8320; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
8320 "Template objects are only created for int32 lengths")do { static_assert( mozilla::detail::AssertionConditionType<
decltype(n <= (2147483647))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(n <= (2147483647)))), 0))
) { do { } while (false); MOZ_ReportAssertionFailure("n <= (2147483647)"
" (" "Template objects are only created for int32 lengths" ")"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 8320); AnnotateMozCrashReason("MOZ_ASSERT" "(" "n <= (2147483647)"
") (" "Template objects are only created for int32 lengths" ")"
); do { *((volatile int*)__null) = 8320; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
8321
8322 using Fn = TypedArrayObject* (*)(JSContext*, HandleObject, int32_t length);
8323 OutOfLineCode* ool = oolCallVM<Fn, NewTypedArrayWithTemplateAndLength>(
8324 lir, ArgList(ImmGCPtr(templateObject), Imm32(n)),
8325 StoreRegisterTo(objReg));
8326
8327 TemplateObject templateObj(templateObject);
8328 masm.createGCObject(objReg, tempReg, templateObj, initialHeap, ool->entry());
8329
8330 masm.initTypedArraySlots(objReg, tempReg, lengthReg, liveRegs, ool->entry(),
8331 ttemplate, MacroAssembler::TypedArrayLength::Fixed);
8332
8333 masm.bind(ool->rejoin());
8334}
8335
8336void CodeGenerator::visitNewTypedArrayDynamicLength(
8337 LNewTypedArrayDynamicLength* lir) {
8338 Register lengthReg = ToRegister(lir->length());
8339 Register objReg = ToRegister(lir->output());
8340 Register tempReg = ToRegister(lir->temp0());
8341 LiveRegisterSet liveRegs = liveVolatileRegs(lir);
8342
8343 JSObject* templateObject = lir->mir()->templateObject();
8344 gc::Heap initialHeap = lir->mir()->initialHeap();
8345
8346 auto* ttemplate = &templateObject->as<FixedLengthTypedArrayObject>();
8347
8348 using Fn = TypedArrayObject* (*)(JSContext*, HandleObject, int32_t length);
8349 OutOfLineCode* ool = oolCallVM<Fn, NewTypedArrayWithTemplateAndLength>(
8350 lir, ArgList(ImmGCPtr(templateObject), lengthReg),
8351 StoreRegisterTo(objReg));
8352
8353 // Volatile |lengthReg| is saved across the ABI call in |initTypedArraySlots|.
8354 MOZ_ASSERT_IF(lengthReg.volatile_(), liveRegs.has(lengthReg))do { if (lengthReg.volatile_()) { do { static_assert( mozilla
::detail::AssertionConditionType<decltype(liveRegs.has(lengthReg
))>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(liveRegs.has(lengthReg)))), 0))) { do { } while (false
); MOZ_ReportAssertionFailure("liveRegs.has(lengthReg)", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 8354); AnnotateMozCrashReason("MOZ_ASSERT" "(" "liveRegs.has(lengthReg)"
")"); do { *((volatile int*)__null) = 8354; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false)
;
8355
8356 TemplateObject templateObj(templateObject);
8357 masm.createGCObject(objReg, tempReg, templateObj, initialHeap, ool->entry());
8358
8359 masm.initTypedArraySlots(objReg, tempReg, lengthReg, liveRegs, ool->entry(),
8360 ttemplate,
8361 MacroAssembler::TypedArrayLength::Dynamic);
8362
8363 masm.bind(ool->rejoin());
8364}
8365
8366void CodeGenerator::visitNewTypedArrayFromArray(LNewTypedArrayFromArray* lir) {
8367 pushArg(ToRegister(lir->array()));
8368 pushArg(ImmGCPtr(lir->mir()->templateObject()));
8369
8370 using Fn = TypedArrayObject* (*)(JSContext*, HandleObject, HandleObject);
8371 callVM<Fn, js::NewTypedArrayWithTemplateAndArray>(lir);
8372}
8373
8374void CodeGenerator::visitNewTypedArrayFromArrayBuffer(
8375 LNewTypedArrayFromArrayBuffer* lir) {
8376 pushArg(ToValue(lir, LNewTypedArrayFromArrayBuffer::LengthIndex));
8377 pushArg(ToValue(lir, LNewTypedArrayFromArrayBuffer::ByteOffsetIndex));
8378 pushArg(ToRegister(lir->arrayBuffer()));
8379 pushArg(ImmGCPtr(lir->mir()->templateObject()));
8380
8381 using Fn = TypedArrayObject* (*)(JSContext*, HandleObject, HandleObject,
8382 HandleValue, HandleValue);
8383 callVM<Fn, js::NewTypedArrayWithTemplateAndBuffer>(lir);
8384}
8385
8386void CodeGenerator::visitBindFunction(LBindFunction* lir) {
8387 Register target = ToRegister(lir->target());
8388 Register temp1 = ToRegister(lir->temp0());
8389 Register temp2 = ToRegister(lir->temp1());
8390
8391 // Try to allocate a new BoundFunctionObject we can pass to the VM function.
8392 // If this fails, we set temp1 to nullptr so we do the allocation in C++.
8393 TemplateObject templateObject(lir->mir()->templateObject());
8394 Label allocOk, allocFailed;
8395 masm.createGCObject(temp1, temp2, templateObject, gc::Heap::Default,
8396 &allocFailed);
8397 masm.jump(&allocOk);
8398
8399 masm.bind(&allocFailed);
8400 masm.movePtr(ImmWord(0), temp1);
8401
8402 masm.bind(&allocOk);
8403
8404 // Set temp2 to the address of the first argument on the stack.
8405 // Note that the Value slots used for arguments are currently aligned for a
8406 // JIT call, even though that's not strictly necessary for calling into C++.
8407 uint32_t argc = lir->mir()->numStackArgs();
8408 if (JitStackValueAlignment > 1) {
8409 argc = AlignBytes(argc, JitStackValueAlignment);
8410 }
8411 uint32_t unusedStack = UnusedStackBytesForCall(argc);
8412 masm.computeEffectiveAddress(Address(masm.getStackPointer(), unusedStack),
8413 temp2);
8414
8415 pushArg(temp1);
8416 pushArg(Imm32(lir->mir()->numStackArgs()));
8417 pushArg(temp2);
8418 pushArg(target);
8419
8420 using Fn = BoundFunctionObject* (*)(JSContext*, Handle<JSObject*>, Value*,
8421 uint32_t, Handle<BoundFunctionObject*>);
8422 callVM<Fn, js::BoundFunctionObject::functionBindImpl>(lir);
8423}
8424
8425void CodeGenerator::visitNewBoundFunction(LNewBoundFunction* lir) {
8426 Register output = ToRegister(lir->output());
8427 Register temp = ToRegister(lir->temp0());
8428
8429 JSObject* templateObj = lir->mir()->templateObj();
8430
8431 using Fn = BoundFunctionObject* (*)(JSContext*, Handle<BoundFunctionObject*>);
8432 OutOfLineCode* ool = oolCallVM<Fn, BoundFunctionObject::createWithTemplate>(
8433 lir, ArgList(ImmGCPtr(templateObj)), StoreRegisterTo(output));
8434
8435 TemplateObject templateObject(templateObj);
8436 masm.createGCObject(output, temp, templateObject, gc::Heap::Default,
8437 ool->entry());
8438
8439 masm.bind(ool->rejoin());
8440}
8441
8442// Out-of-line object allocation for JSOp::NewObject.
8443class OutOfLineNewObject : public OutOfLineCodeBase<CodeGenerator> {
8444 LNewObject* lir_;
8445
8446 public:
8447 explicit OutOfLineNewObject(LNewObject* lir) : lir_(lir) {}
8448
8449 void accept(CodeGenerator* codegen) override {
8450 codegen->visitOutOfLineNewObject(this);
8451 }
8452
8453 LNewObject* lir() const { return lir_; }
8454};
8455
8456void CodeGenerator::visitNewObjectVMCall(LNewObject* lir) {
8457 Register objReg = ToRegister(lir->output());
8458
8459 MOZ_ASSERT(!lir->isCall())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!lir->isCall())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!lir->isCall()))), 0))) {
do { } while (false); MOZ_ReportAssertionFailure("!lir->isCall()"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 8459); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!lir->isCall()"
")"); do { *((volatile int*)__null) = 8459; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
8460 saveLive(lir);
8461
8462 JSObject* templateObject = lir->mir()->templateObject();
8463
8464 // If we're making a new object with a class prototype (that is, an object
8465 // that derives its class from its prototype instead of being
8466 // PlainObject::class_'d) from self-hosted code, we need a different init
8467 // function.
8468 switch (lir->mir()->mode()) {
8469 case MNewObject::ObjectLiteral: {
8470 MOZ_ASSERT(!templateObject)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!templateObject)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!templateObject))), 0))) { do
{ } while (false); MOZ_ReportAssertionFailure("!templateObject"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 8470); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!templateObject"
")"); do { *((volatile int*)__null) = 8470; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
8471 pushArg(ImmPtr(lir->mir()->resumePoint()->pc()));
8472 pushArg(ImmGCPtr(lir->mir()->block()->info().script()));
8473
8474 using Fn = JSObject* (*)(JSContext*, HandleScript, const jsbytecode* pc);
8475 callVM<Fn, NewObjectOperation>(lir);
8476 break;
8477 }
8478 case MNewObject::ObjectCreate: {
8479 pushArg(ImmGCPtr(templateObject));
8480
8481 using Fn = PlainObject* (*)(JSContext*, Handle<PlainObject*>);
8482 callVM<Fn, ObjectCreateWithTemplate>(lir);
8483 break;
8484 }
8485 }
8486
8487 masm.storeCallPointerResult(objReg);
8488
8489 MOZ_ASSERT(!lir->safepoint()->liveRegs().has(objReg))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!lir->safepoint()->liveRegs().has(objReg))>
::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(!lir->safepoint()->liveRegs().has(objReg)))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("!lir->safepoint()->liveRegs().has(objReg)"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 8489); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!lir->safepoint()->liveRegs().has(objReg)"
")"); do { *((volatile int*)__null) = 8489; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
8490 restoreLive(lir);
8491}
8492
8493static bool ShouldInitFixedSlots(MIRGenerator* gen, LNewPlainObject* lir,
8494 const Shape* shape, uint32_t nfixed) {
8495 // Look for StoreFixedSlot instructions following an object allocation
8496 // that write to this object before a GC is triggered or this object is
8497 // passed to a VM call. If all fixed slots will be initialized, the
8498 // allocation code doesn't need to set the slots to |undefined|.
8499
8500 if (nfixed == 0) {
8501 return false;
8502 }
8503
8504#ifdef DEBUG1
8505 // The bailAfter testing function can trigger a bailout between allocating the
8506 // object and initializing the slots.
8507 if (gen->options.ionBailAfterEnabled()) {
8508 return true;
8509 }
8510#endif
8511
8512 // Keep track of the fixed slots that are initialized. initializedSlots is
8513 // a bit mask with a bit for each slot.
8514 MOZ_ASSERT(nfixed <= NativeObject::MAX_FIXED_SLOTS)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(nfixed <= NativeObject::MAX_FIXED_SLOTS)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(nfixed <= NativeObject::MAX_FIXED_SLOTS))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("nfixed <= NativeObject::MAX_FIXED_SLOTS"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 8514); AnnotateMozCrashReason("MOZ_ASSERT" "(" "nfixed <= NativeObject::MAX_FIXED_SLOTS"
")"); do { *((volatile int*)__null) = 8514; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
8515 static_assert(NativeObject::MAX_FIXED_SLOTS <= 32,
8516 "Slot bits must fit in 32 bits");
8517 uint32_t initializedSlots = 0;
8518 uint32_t numInitialized = 0;
8519
8520 MInstruction* allocMir = lir->mir();
8521 MBasicBlock* block = allocMir->block();
8522
8523 // Skip the allocation instruction.
8524 MInstructionIterator iter = block->begin(allocMir);
8525 MOZ_ASSERT(*iter == allocMir)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(*iter == allocMir)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(*iter == allocMir))), 0))) {
do { } while (false); MOZ_ReportAssertionFailure("*iter == allocMir"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 8525); AnnotateMozCrashReason("MOZ_ASSERT" "(" "*iter == allocMir"
")"); do { *((volatile int*)__null) = 8525; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
8526 iter++;
8527
8528 // Handle the leading shape guard, if present.
8529 for (; iter != block->end(); iter++) {
8530 if (iter->isConstant()) {
8531 // This instruction won't trigger a GC or read object slots.
8532 continue;
8533 }
8534 if (iter->isGuardShape()) {
8535 auto* guard = iter->toGuardShape();
8536 if (guard->object() != allocMir || guard->shape() != shape) {
8537 return true;
8538 }
8539 allocMir = guard;
8540 iter++;
8541 }
8542 break;
8543 }
8544
8545 for (; iter != block->end(); iter++) {
8546 if (iter->isConstant() || iter->isPostWriteBarrier()) {
8547 // These instructions won't trigger a GC or read object slots.
8548 continue;
8549 }
8550
8551 if (iter->isStoreFixedSlot()) {
8552 MStoreFixedSlot* store = iter->toStoreFixedSlot();
8553 if (store->object() != allocMir) {
8554 return true;
8555 }
8556
8557 // We may not initialize this object slot on allocation, so the
8558 // pre-barrier could read uninitialized memory. Simply disable
8559 // the barrier for this store: the object was just initialized
8560 // so the barrier is not necessary.
8561 store->setNeedsBarrier(false);
8562
8563 uint32_t slot = store->slot();
8564 MOZ_ASSERT(slot < nfixed)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(slot < nfixed)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(slot < nfixed))), 0))) { do
{ } while (false); MOZ_ReportAssertionFailure("slot < nfixed"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 8564); AnnotateMozCrashReason("MOZ_ASSERT" "(" "slot < nfixed"
")"); do { *((volatile int*)__null) = 8564; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
8565 if ((initializedSlots & (1 << slot)) == 0) {
8566 numInitialized++;
8567 initializedSlots |= (1 << slot);
8568
8569 if (numInitialized == nfixed) {
8570 // All fixed slots will be initialized.
8571 MOZ_ASSERT(mozilla::CountPopulation32(initializedSlots) == nfixed)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(mozilla::CountPopulation32(initializedSlots) == nfixed
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(mozilla::CountPopulation32(initializedSlots) == nfixed
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"mozilla::CountPopulation32(initializedSlots) == nfixed", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 8571); AnnotateMozCrashReason("MOZ_ASSERT" "(" "mozilla::CountPopulation32(initializedSlots) == nfixed"
")"); do { *((volatile int*)__null) = 8571; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
8572 return false;
8573 }
8574 }
8575 continue;
8576 }
8577
8578 // Unhandled instruction, assume it bails or reads object slots.
8579 return true;
8580 }
8581
8582 MOZ_CRASH("Shouldn't get here")do { do { } while (false); MOZ_ReportCrash("" "Shouldn't get here"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 8582); AnnotateMozCrashReason("MOZ_CRASH(" "Shouldn't get here"
")"); do { *((volatile int*)__null) = 8582; __attribute__((nomerge
)) ::abort(); } while (false); } while (false)
;
8583}
8584
8585void CodeGenerator::visitNewObject(LNewObject* lir) {
8586 Register objReg = ToRegister(lir->output());
8587 Register tempReg = ToRegister(lir->temp());
8588
8589 if (lir->mir()->isVMCall()) {
8590 visitNewObjectVMCall(lir);
8591 return;
8592 }
8593
8594 OutOfLineNewObject* ool = new (alloc()) OutOfLineNewObject(lir);
8595 addOutOfLineCode(ool, lir->mir());
8596
8597 TemplateObject templateObject(lir->mir()->templateObject());
8598
8599 masm.createGCObject(objReg, tempReg, templateObject,
8600 lir->mir()->initialHeap(), ool->entry());
8601
8602 masm.bind(ool->rejoin());
8603}
8604
8605void CodeGenerator::visitOutOfLineNewObject(OutOfLineNewObject* ool) {
8606 visitNewObjectVMCall(ool->lir());
8607 masm.jump(ool->rejoin());
8608}
8609
8610void CodeGenerator::visitNewPlainObject(LNewPlainObject* lir) {
8611 Register objReg = ToRegister(lir->output());
8612 Register temp0Reg = ToRegister(lir->temp0());
8613 Register temp1Reg = ToRegister(lir->temp1());
8614 Register shapeReg = ToRegister(lir->temp2());
8615
8616 auto* mir = lir->mir();
8617 const Shape* shape = mir->shape();
8618 gc::Heap initialHeap = mir->initialHeap();
8619 gc::AllocKind allocKind = mir->allocKind();
8620
8621 using Fn =
8622 JSObject* (*)(JSContext*, Handle<SharedShape*>, gc::AllocKind, gc::Heap);
8623 OutOfLineCode* ool = oolCallVM<Fn, NewPlainObjectOptimizedFallback>(
8624 lir,
8625 ArgList(ImmGCPtr(shape), Imm32(int32_t(allocKind)),
8626 Imm32(int32_t(initialHeap))),
8627 StoreRegisterTo(objReg));
8628
8629 bool initContents =
8630 ShouldInitFixedSlots(gen, lir, shape, mir->numFixedSlots());
8631
8632 masm.movePtr(ImmGCPtr(shape), shapeReg);
8633 masm.createPlainGCObject(
8634 objReg, shapeReg, temp0Reg, temp1Reg, mir->numFixedSlots(),
8635 mir->numDynamicSlots(), allocKind, initialHeap, ool->entry(),
8636 AllocSiteInput(gc::CatchAllAllocSite::Optimized), initContents);
8637
8638#ifdef DEBUG1
8639 // ShouldInitFixedSlots expects that the leading GuardShape will never fail,
8640 // so ensure the newly created object has the correct shape. Should the guard
8641 // ever fail, we may end up with uninitialized fixed slots, which can confuse
8642 // the GC.
8643 Label ok;
8644 masm.branchTestObjShape(Assembler::Equal, objReg, shape, temp0Reg, objReg,
8645 &ok);
8646 masm.assumeUnreachable("Newly created object has the correct shape");
8647 masm.bind(&ok);
8648#endif
8649
8650 masm.bind(ool->rejoin());
8651}
8652
8653void CodeGenerator::visitNewArrayObject(LNewArrayObject* lir) {
8654 Register objReg = ToRegister(lir->output());
8655 Register temp0Reg = ToRegister(lir->temp0());
8656 Register shapeReg = ToRegister(lir->temp1());
8657
8658 auto* mir = lir->mir();
8659 uint32_t arrayLength = mir->length();
8660
8661 gc::AllocKind allocKind = GuessArrayGCKind(arrayLength);
8662 MOZ_ASSERT(CanChangeToBackgroundAllocKind(allocKind, &ArrayObject::class_))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(CanChangeToBackgroundAllocKind(allocKind, &ArrayObject
::class_))>::isValid, "invalid assertion condition"); if (
(__builtin_expect(!!(!(!!(CanChangeToBackgroundAllocKind(allocKind
, &ArrayObject::class_)))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("CanChangeToBackgroundAllocKind(allocKind, &ArrayObject::class_)"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 8662); AnnotateMozCrashReason("MOZ_ASSERT" "(" "CanChangeToBackgroundAllocKind(allocKind, &ArrayObject::class_)"
")"); do { *((volatile int*)__null) = 8662; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
8663 allocKind = ForegroundToBackgroundAllocKind(allocKind);
8664
8665 uint32_t slotCount = GetGCKindSlots(allocKind);
8666 MOZ_ASSERT(slotCount >= ObjectElements::VALUES_PER_HEADER)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(slotCount >= ObjectElements::VALUES_PER_HEADER)>
::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(slotCount >= ObjectElements::VALUES_PER_HEADER)))
, 0))) { do { } while (false); MOZ_ReportAssertionFailure("slotCount >= ObjectElements::VALUES_PER_HEADER"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 8666); AnnotateMozCrashReason("MOZ_ASSERT" "(" "slotCount >= ObjectElements::VALUES_PER_HEADER"
")"); do { *((volatile int*)__null) = 8666; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
8667 uint32_t arrayCapacity = slotCount - ObjectElements::VALUES_PER_HEADER;
8668
8669 const Shape* shape = mir->shape();
8670
8671 NewObjectKind objectKind =
8672 mir->initialHeap() == gc::Heap::Tenured ? TenuredObject : GenericObject;
8673
8674 using Fn =
8675 ArrayObject* (*)(JSContext*, uint32_t, gc::AllocKind, NewObjectKind);
8676 OutOfLineCode* ool = oolCallVM<Fn, NewArrayObjectOptimizedFallback>(
8677 lir,
8678 ArgList(Imm32(arrayLength), Imm32(int32_t(allocKind)), Imm32(objectKind)),
8679 StoreRegisterTo(objReg));
8680
8681 masm.movePtr(ImmPtr(shape), shapeReg);
8682 masm.createArrayWithFixedElements(
8683 objReg, shapeReg, temp0Reg, InvalidReg, arrayLength, arrayCapacity, 0, 0,
8684 allocKind, mir->initialHeap(), ool->entry(),
8685 AllocSiteInput(gc::CatchAllAllocSite::Optimized));
8686 masm.bind(ool->rejoin());
8687}
8688
8689void CodeGenerator::visitNewNamedLambdaObject(LNewNamedLambdaObject* lir) {
8690 Register objReg = ToRegister(lir->output());
8691 Register tempReg = ToRegister(lir->temp0());
8692 const CompileInfo& info = lir->mir()->block()->info();
8693
8694 using Fn = js::NamedLambdaObject* (*)(JSContext*, HandleFunction);
8695 OutOfLineCode* ool = oolCallVM<Fn, NamedLambdaObject::createWithoutEnclosing>(
8696 lir, ArgList(info.funMaybeLazy()), StoreRegisterTo(objReg));
8697
8698 TemplateObject templateObject(lir->mir()->templateObj());
8699
8700 masm.createGCObject(objReg, tempReg, templateObject, gc::Heap::Default,
8701 ool->entry());
8702
8703 masm.bind(ool->rejoin());
8704}
8705
8706void CodeGenerator::visitNewCallObject(LNewCallObject* lir) {
8707 Register objReg = ToRegister(lir->output());
8708 Register tempReg = ToRegister(lir->temp0());
8709
8710 CallObject* templateObj = lir->mir()->templateObject();
8711
8712 using Fn = CallObject* (*)(JSContext*, Handle<SharedShape*>);
8713 OutOfLineCode* ool = oolCallVM<Fn, CallObject::createWithShape>(
8714 lir, ArgList(ImmGCPtr(templateObj->sharedShape())),
8715 StoreRegisterTo(objReg));
8716
8717 // Inline call object creation, using the OOL path only for tricky cases.
8718 TemplateObject templateObject(templateObj);
8719 masm.createGCObject(objReg, tempReg, templateObject, gc::Heap::Default,
8720 ool->entry());
8721
8722 masm.bind(ool->rejoin());
8723}
8724
8725void CodeGenerator::visitNewStringObject(LNewStringObject* lir) {
8726 Register input = ToRegister(lir->input());
8727 Register output = ToRegister(lir->output());
8728 Register temp = ToRegister(lir->temp0());
8729
8730 StringObject* templateObj = lir->mir()->templateObj();
8731
8732 using Fn = JSObject* (*)(JSContext*, HandleString);
8733 OutOfLineCode* ool = oolCallVM<Fn, NewStringObject>(lir, ArgList(input),
8734 StoreRegisterTo(output));
8735
8736 TemplateObject templateObject(templateObj);
8737 masm.createGCObject(output, temp, templateObject, gc::Heap::Default,
8738 ool->entry());
8739
8740 masm.loadStringLength(input, temp);
8741
8742 masm.storeValue(JSVAL_TYPE_STRING, input,
8743 Address(output, StringObject::offsetOfPrimitiveValue()));
8744 masm.storeValue(JSVAL_TYPE_INT32, temp,
8745 Address(output, StringObject::offsetOfLength()));
8746
8747 masm.bind(ool->rejoin());
8748}
8749
8750void CodeGenerator::visitInitElemGetterSetter(LInitElemGetterSetter* lir) {
8751 Register obj = ToRegister(lir->object());
8752 Register value = ToRegister(lir->value());
8753
8754 pushArg(value);
8755 pushArg(ToValue(lir, LInitElemGetterSetter::IdIndex));
8756 pushArg(obj);
8757 pushArg(ImmPtr(lir->mir()->resumePoint()->pc()));
8758
8759 using Fn = bool (*)(JSContext*, jsbytecode*, HandleObject, HandleValue,
8760 HandleObject);
8761 callVM<Fn, InitElemGetterSetterOperation>(lir);
8762}
8763
8764void CodeGenerator::visitMutateProto(LMutateProto* lir) {
8765 Register objReg = ToRegister(lir->object());
8766
8767 pushArg(ToValue(lir, LMutateProto::ValueIndex));
8768 pushArg(objReg);
8769
8770 using Fn =
8771 bool (*)(JSContext* cx, Handle<PlainObject*> obj, HandleValue value);
8772 callVM<Fn, MutatePrototype>(lir);
8773}
8774
8775void CodeGenerator::visitInitPropGetterSetter(LInitPropGetterSetter* lir) {
8776 Register obj = ToRegister(lir->object());
8777 Register value = ToRegister(lir->value());
8778
8779 pushArg(value);
8780 pushArg(ImmGCPtr(lir->mir()->name()));
8781 pushArg(obj);
8782 pushArg(ImmPtr(lir->mir()->resumePoint()->pc()));
8783
8784 using Fn = bool (*)(JSContext*, jsbytecode*, HandleObject,
8785 Handle<PropertyName*>, HandleObject);
8786 callVM<Fn, InitPropGetterSetterOperation>(lir);
8787}
8788
8789void CodeGenerator::visitCreateThis(LCreateThis* lir) {
8790 const LAllocation* callee = lir->callee();
8791 const LAllocation* newTarget = lir->newTarget();
8792
8793 if (newTarget->isConstant()) {
8794 pushArg(ImmGCPtr(&newTarget->toConstant()->toObject()));
8795 } else {
8796 pushArg(ToRegister(newTarget));
8797 }
8798
8799 if (callee->isConstant()) {
8800 pushArg(ImmGCPtr(&callee->toConstant()->toObject()));
8801 } else {
8802 pushArg(ToRegister(callee));
8803 }
8804
8805 using Fn = bool (*)(JSContext* cx, HandleObject callee,
8806 HandleObject newTarget, MutableHandleValue rval);
8807 callVM<Fn, jit::CreateThisFromIon>(lir);
8808}
8809
8810void CodeGenerator::visitCreateArgumentsObject(LCreateArgumentsObject* lir) {
8811 // This should be getting constructed in the first block only, and not any OSR
8812 // entry blocks.
8813 MOZ_ASSERT(lir->mir()->block()->id() == 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(lir->mir()->block()->id() == 0)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(lir->mir()->block()->id() == 0))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("lir->mir()->block()->id() == 0"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 8813); AnnotateMozCrashReason("MOZ_ASSERT" "(" "lir->mir()->block()->id() == 0"
")"); do { *((volatile int*)__null) = 8813; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
8814
8815 Register callObj = ToRegister(lir->callObject());
8816 Register temp0 = ToRegister(lir->temp0());
8817 Label done;
8818
8819 if (ArgumentsObject* templateObj = lir->mir()->templateObject()) {
8820 Register objTemp = ToRegister(lir->temp1());
8821 Register cxTemp = ToRegister(lir->temp2());
8822
8823 masm.Push(callObj);
8824
8825 // Try to allocate an arguments object. This will leave the reserved
8826 // slots uninitialized, so it's important we don't GC until we
8827 // initialize these slots in ArgumentsObject::finishForIonPure.
8828 Label failure;
8829 TemplateObject templateObject(templateObj);
8830 masm.createGCObject(objTemp, temp0, templateObject, gc::Heap::Default,
8831 &failure,
8832 /* initContents = */ false);
8833
8834 masm.moveStackPtrTo(temp0);
8835 masm.addPtr(Imm32(masm.framePushed()), temp0);
8836
8837 using Fn = ArgumentsObject* (*)(JSContext* cx, jit::JitFrameLayout* frame,
8838 JSObject* scopeChain, ArgumentsObject* obj);
8839 masm.setupAlignedABICall();
8840 masm.loadJSContext(cxTemp);
8841 masm.passABIArg(cxTemp);
8842 masm.passABIArg(temp0);
8843 masm.passABIArg(callObj);
8844 masm.passABIArg(objTemp);
8845
8846 masm.callWithABI<Fn, ArgumentsObject::finishForIonPure>();
8847 masm.branchTestPtr(Assembler::Zero, ReturnReg, ReturnReg, &failure);
8848
8849 // Discard saved callObj on the stack.
8850 masm.addToStackPtr(Imm32(sizeof(uintptr_t)));
8851 masm.jump(&done);
8852
8853 masm.bind(&failure);
8854 masm.Pop(callObj);
8855 }
8856
8857 masm.moveStackPtrTo(temp0);
8858 masm.addPtr(Imm32(frameSize()), temp0);
8859
8860 pushArg(callObj);
8861 pushArg(temp0);
8862
8863 using Fn = ArgumentsObject* (*)(JSContext*, JitFrameLayout*, HandleObject);
8864 callVM<Fn, ArgumentsObject::createForIon>(lir);
8865
8866 masm.bind(&done);
8867}
8868
8869void CodeGenerator::visitCreateInlinedArgumentsObject(
8870 LCreateInlinedArgumentsObject* lir) {
8871 Register callObj = ToRegister(lir->getCallObject());
8872 Register callee = ToRegister(lir->getCallee());
8873 Register argsAddress = ToRegister(lir->temp1());
8874 Register argsObj = ToRegister(lir->temp2());
8875
8876 // TODO: Do we have to worry about alignment here?
8877
8878 // Create a contiguous array of values for ArgumentsObject::create
8879 // by pushing the arguments onto the stack in reverse order.
8880 uint32_t argc = lir->mir()->numActuals();
8881 for (uint32_t i = 0; i < argc; i++) {
8882 uint32_t argNum = argc - i - 1;
8883 uint32_t index = LCreateInlinedArgumentsObject::ArgIndex(argNum);
8884 ConstantOrRegister arg =
8885 toConstantOrRegister(lir, index, lir->mir()->getArg(argNum)->type());
8886 masm.Push(arg);
8887 }
8888 masm.moveStackPtrTo(argsAddress);
8889
8890 Label done;
8891 if (ArgumentsObject* templateObj = lir->mir()->templateObject()) {
8892 LiveRegisterSet liveRegs;
8893 liveRegs.add(callObj);
8894 liveRegs.add(callee);
8895
8896 masm.PushRegsInMask(liveRegs);
8897
8898 // We are free to clobber all registers, as LCreateInlinedArgumentsObject is
8899 // a call instruction.
8900 AllocatableGeneralRegisterSet allRegs(GeneralRegisterSet::All());
8901 allRegs.take(callObj);
8902 allRegs.take(callee);
8903 allRegs.take(argsObj);
8904 allRegs.take(argsAddress);
8905
8906 Register temp3 = allRegs.takeAny();
8907 Register temp4 = allRegs.takeAny();
8908
8909 // Try to allocate an arguments object. This will leave the reserved slots
8910 // uninitialized, so it's important we don't GC until we initialize these
8911 // slots in ArgumentsObject::finishForIonPure.
8912 Label failure;
8913 TemplateObject templateObject(templateObj);
8914 masm.createGCObject(argsObj, temp3, templateObject, gc::Heap::Default,
8915 &failure,
8916 /* initContents = */ false);
8917
8918 Register numActuals = temp3;
8919 masm.move32(Imm32(argc), numActuals);
8920
8921 using Fn = ArgumentsObject* (*)(JSContext*, JSObject*, JSFunction*, Value*,
8922 uint32_t, ArgumentsObject*);
8923 masm.setupAlignedABICall();
8924 masm.loadJSContext(temp4);
8925 masm.passABIArg(temp4);
8926 masm.passABIArg(callObj);
8927 masm.passABIArg(callee);
8928 masm.passABIArg(argsAddress);
8929 masm.passABIArg(numActuals);
8930 masm.passABIArg(argsObj);
8931
8932 masm.callWithABI<Fn, ArgumentsObject::finishInlineForIonPure>();
8933 masm.branchTestPtr(Assembler::Zero, ReturnReg, ReturnReg, &failure);
8934
8935 // Discard saved callObj, callee, and values array on the stack.
8936 masm.addToStackPtr(
8937 Imm32(MacroAssembler::PushRegsInMaskSizeInBytes(liveRegs) +
8938 argc * sizeof(Value)));
8939 masm.jump(&done);
8940
8941 masm.bind(&failure);
8942 masm.PopRegsInMask(liveRegs);
8943
8944 // Reload argsAddress because it may have been overridden.
8945 masm.moveStackPtrTo(argsAddress);
8946 }
8947
8948 pushArg(Imm32(argc));
8949 pushArg(callObj);
8950 pushArg(callee);
8951 pushArg(argsAddress);
8952
8953 using Fn = ArgumentsObject* (*)(JSContext*, Value*, HandleFunction,
8954 HandleObject, uint32_t);
8955 callVM<Fn, ArgumentsObject::createForInlinedIon>(lir);
8956
8957 // Discard the array of values.
8958 masm.freeStack(argc * sizeof(Value));
8959
8960 masm.bind(&done);
8961}
8962
8963template <class GetInlinedArgument>
8964void CodeGenerator::emitGetInlinedArgument(GetInlinedArgument* lir,
8965 Register index,
8966 ValueOperand output) {
8967 uint32_t numActuals = lir->mir()->numActuals();
8968 MOZ_ASSERT(numActuals <= ArgumentsObject::MaxInlinedArgs)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(numActuals <= ArgumentsObject::MaxInlinedArgs)>
::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(numActuals <= ArgumentsObject::MaxInlinedArgs))),
0))) { do { } while (false); MOZ_ReportAssertionFailure("numActuals <= ArgumentsObject::MaxInlinedArgs"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 8968); AnnotateMozCrashReason("MOZ_ASSERT" "(" "numActuals <= ArgumentsObject::MaxInlinedArgs"
")"); do { *((volatile int*)__null) = 8968; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
8969
8970 // The index has already been bounds-checked, so the code we
8971 // generate here should be unreachable. We can end up in this
8972 // situation in self-hosted code using GetArgument(), or in a
8973 // monomorphically inlined function if we've inlined some CacheIR
8974 // that was created for a different caller.
8975 if (numActuals == 0) {
8976 masm.assumeUnreachable("LGetInlinedArgument: invalid index");
8977 return;
8978 }
8979
8980 // Check the first n-1 possible indices.
8981 Label done;
8982 for (uint32_t i = 0; i < numActuals - 1; i++) {
8983 Label skip;
8984 ConstantOrRegister arg = toConstantOrRegister(
8985 lir, GetInlinedArgument::ArgIndex(i), lir->mir()->getArg(i)->type());
8986 masm.branch32(Assembler::NotEqual, index, Imm32(i), &skip);
8987 masm.moveValue(arg, output);
8988
8989 masm.jump(&done);
8990 masm.bind(&skip);
8991 }
8992
8993#ifdef DEBUG1
8994 Label skip;
8995 masm.branch32(Assembler::Equal, index, Imm32(numActuals - 1), &skip);
8996 masm.assumeUnreachable("LGetInlinedArgument: invalid index");
8997 masm.bind(&skip);
8998#endif
8999
9000 // The index has already been bounds-checked, so load the last argument.
9001 uint32_t lastIdx = numActuals - 1;
9002 ConstantOrRegister arg =
9003 toConstantOrRegister(lir, GetInlinedArgument::ArgIndex(lastIdx),
9004 lir->mir()->getArg(lastIdx)->type());
9005 masm.moveValue(arg, output);
9006 masm.bind(&done);
9007}
9008
9009void CodeGenerator::visitGetInlinedArgument(LGetInlinedArgument* lir) {
9010 Register index = ToRegister(lir->getIndex());
9011 ValueOperand output = ToOutValue(lir);
9012
9013 emitGetInlinedArgument(lir, index, output);
9014}
9015
9016void CodeGenerator::visitGetInlinedArgumentHole(LGetInlinedArgumentHole* lir) {
9017 Register index = ToRegister(lir->getIndex());
9018 ValueOperand output = ToOutValue(lir);
9019
9020 uint32_t numActuals = lir->mir()->numActuals();
9021
9022 if (numActuals == 0) {
9023 bailoutCmp32(Assembler::LessThan, index, Imm32(0), lir->snapshot());
9024 masm.moveValue(UndefinedValue(), output);
9025 return;
9026 }
9027
9028 Label outOfBounds, done;
9029 masm.branch32(Assembler::AboveOrEqual, index, Imm32(numActuals),
9030 &outOfBounds);
9031
9032 emitGetInlinedArgument(lir, index, output);
9033 masm.jump(&done);
9034
9035 masm.bind(&outOfBounds);
9036 bailoutCmp32(Assembler::LessThan, index, Imm32(0), lir->snapshot());
9037 masm.moveValue(UndefinedValue(), output);
9038
9039 masm.bind(&done);
9040}
9041
9042void CodeGenerator::visitGetArgumentsObjectArg(LGetArgumentsObjectArg* lir) {
9043 Register temp = ToRegister(lir->temp0());
9044 Register argsObj = ToRegister(lir->argsObject());
9045 ValueOperand out = ToOutValue(lir);
9046
9047 masm.loadPrivate(Address(argsObj, ArgumentsObject::getDataSlotOffset()),
9048 temp);
9049 Address argAddr(temp, ArgumentsData::offsetOfArgs() +
9050 lir->mir()->argno() * sizeof(Value));
9051 masm.loadValue(argAddr, out);
9052#ifdef DEBUG1
9053 Label success;
9054 masm.branchTestMagic(Assembler::NotEqual, out, &success);
9055 masm.assumeUnreachable(
9056 "Result from ArgumentObject shouldn't be JSVAL_TYPE_MAGIC.");
9057 masm.bind(&success);
9058#endif
9059}
9060
9061void CodeGenerator::visitSetArgumentsObjectArg(LSetArgumentsObjectArg* lir) {
9062 Register temp = ToRegister(lir->getTemp(0));
9063 Register argsObj = ToRegister(lir->argsObject());
9064 ValueOperand value = ToValue(lir, LSetArgumentsObjectArg::ValueIndex);
9065
9066 masm.loadPrivate(Address(argsObj, ArgumentsObject::getDataSlotOffset()),
9067 temp);
9068 Address argAddr(temp, ArgumentsData::offsetOfArgs() +
9069 lir->mir()->argno() * sizeof(Value));
9070 emitPreBarrier(argAddr);
9071#ifdef DEBUG1
9072 Label success;
9073 masm.branchTestMagic(Assembler::NotEqual, argAddr, &success);
9074 masm.assumeUnreachable(
9075 "Result in ArgumentObject shouldn't be JSVAL_TYPE_MAGIC.");
9076 masm.bind(&success);
9077#endif
9078 masm.storeValue(value, argAddr);
9079}
9080
9081void CodeGenerator::visitLoadArgumentsObjectArg(LLoadArgumentsObjectArg* lir) {
9082 Register temp = ToRegister(lir->temp0());
9083 Register argsObj = ToRegister(lir->argsObject());
9084 Register index = ToRegister(lir->index());
9085 ValueOperand out = ToOutValue(lir);
9086
9087 Label bail;
9088 masm.loadArgumentsObjectElement(argsObj, index, out, temp, &bail);
9089 bailoutFrom(&bail, lir->snapshot());
9090}
9091
9092void CodeGenerator::visitLoadArgumentsObjectArgHole(
9093 LLoadArgumentsObjectArgHole* lir) {
9094 Register temp = ToRegister(lir->temp0());
9095 Register argsObj = ToRegister(lir->argsObject());
9096 Register index = ToRegister(lir->index());
9097 ValueOperand out = ToOutValue(lir);
9098
9099 Label bail;
9100 masm.loadArgumentsObjectElementHole(argsObj, index, out, temp, &bail);
9101 bailoutFrom(&bail, lir->snapshot());
9102}
9103
9104void CodeGenerator::visitInArgumentsObjectArg(LInArgumentsObjectArg* lir) {
9105 Register temp = ToRegister(lir->temp0());
9106 Register argsObj = ToRegister(lir->argsObject());
9107 Register index = ToRegister(lir->index());
9108 Register out = ToRegister(lir->output());
9109
9110 Label bail;
9111 masm.loadArgumentsObjectElementExists(argsObj, index, out, temp, &bail);
9112 bailoutFrom(&bail, lir->snapshot());
9113}
9114
9115void CodeGenerator::visitArgumentsObjectLength(LArgumentsObjectLength* lir) {
9116 Register argsObj = ToRegister(lir->argsObject());
9117 Register out = ToRegister(lir->output());
9118
9119 Label bail;
9120 masm.loadArgumentsObjectLength(argsObj, out, &bail);
9121 bailoutFrom(&bail, lir->snapshot());
9122}
9123
9124void CodeGenerator::visitArrayFromArgumentsObject(
9125 LArrayFromArgumentsObject* lir) {
9126 pushArg(ToRegister(lir->argsObject()));
9127
9128 using Fn = ArrayObject* (*)(JSContext*, Handle<ArgumentsObject*>);
9129 callVM<Fn, js::ArrayFromArgumentsObject>(lir);
9130}
9131
9132void CodeGenerator::visitGuardArgumentsObjectFlags(
9133 LGuardArgumentsObjectFlags* lir) {
9134 Register argsObj = ToRegister(lir->argsObject());
9135 Register temp = ToRegister(lir->temp0());
9136
9137 Label bail;
9138 masm.branchTestArgumentsObjectFlags(argsObj, temp, lir->mir()->flags(),
9139 Assembler::NonZero, &bail);
9140 bailoutFrom(&bail, lir->snapshot());
9141}
9142
9143void CodeGenerator::visitBoundFunctionNumArgs(LBoundFunctionNumArgs* lir) {
9144 Register obj = ToRegister(lir->object());
9145 Register output = ToRegister(lir->output());
9146
9147 masm.unboxInt32(Address(obj, BoundFunctionObject::offsetOfFlagsSlot()),
9148 output);
9149 masm.rshift32(Imm32(BoundFunctionObject::NumBoundArgsShift), output);
9150}
9151
9152void CodeGenerator::visitGuardBoundFunctionIsConstructor(
9153 LGuardBoundFunctionIsConstructor* lir) {
9154 Register obj = ToRegister(lir->object());
9155
9156 Label bail;
9157 Address flagsSlot(obj, BoundFunctionObject::offsetOfFlagsSlot());
9158 masm.branchTest32(Assembler::Zero, flagsSlot,
9159 Imm32(BoundFunctionObject::IsConstructorFlag), &bail);
9160 bailoutFrom(&bail, lir->snapshot());
9161}
9162
9163void CodeGenerator::visitReturnFromCtor(LReturnFromCtor* lir) {
9164 ValueOperand value = ToValue(lir, LReturnFromCtor::ValueIndex);
9165 Register obj = ToRegister(lir->object());
9166 Register output = ToRegister(lir->output());
9167
9168 Label valueIsObject, end;
9169
9170 masm.branchTestObject(Assembler::Equal, value, &valueIsObject);
9171
9172 // Value is not an object. Return that other object.
9173 masm.movePtr(obj, output);
9174 masm.jump(&end);
9175
9176 // Value is an object. Return unbox(Value).
9177 masm.bind(&valueIsObject);
9178 Register payload = masm.extractObject(value, output);
9179 if (payload != output) {
9180 masm.movePtr(payload, output);
9181 }
9182
9183 masm.bind(&end);
9184}
9185
9186class OutOfLineBoxNonStrictThis : public OutOfLineCodeBase<CodeGenerator> {
9187 LBoxNonStrictThis* ins_;
9188
9189 public:
9190 explicit OutOfLineBoxNonStrictThis(LBoxNonStrictThis* ins) : ins_(ins) {}
9191 void accept(CodeGenerator* codegen) override {
9192 codegen->visitOutOfLineBoxNonStrictThis(this);
9193 }
9194 LBoxNonStrictThis* ins() const { return ins_; }
9195};
9196
9197void CodeGenerator::visitBoxNonStrictThis(LBoxNonStrictThis* lir) {
9198 ValueOperand value = ToValue(lir, LBoxNonStrictThis::ValueIndex);
9199 Register output = ToRegister(lir->output());
9200
9201 auto* ool = new (alloc()) OutOfLineBoxNonStrictThis(lir);
9202 addOutOfLineCode(ool, lir->mir());
9203
9204 masm.fallibleUnboxObject(value, output, ool->entry());
9205 masm.bind(ool->rejoin());
9206}
9207
9208void CodeGenerator::visitOutOfLineBoxNonStrictThis(
9209 OutOfLineBoxNonStrictThis* ool) {
9210 LBoxNonStrictThis* lir = ool->ins();
9211
9212 ValueOperand value = ToValue(lir, LBoxNonStrictThis::ValueIndex);
9213 Register output = ToRegister(lir->output());
9214
9215 Label notNullOrUndefined;
9216 {
9217 Label isNullOrUndefined;
9218 ScratchTagScope tag(masm, value);
9219 masm.splitTagForTest(value, tag);
9220 masm.branchTestUndefined(Assembler::Equal, tag, &isNullOrUndefined);
9221 masm.branchTestNull(Assembler::NotEqual, tag, &notNullOrUndefined);
9222 masm.bind(&isNullOrUndefined);
9223 masm.movePtr(ImmGCPtr(lir->mir()->globalThis()), output);
9224 masm.jump(ool->rejoin());
9225 }
9226
9227 masm.bind(&notNullOrUndefined);
9228
9229 saveLive(lir);
9230
9231 pushArg(value);
9232 using Fn = JSObject* (*)(JSContext*, HandleValue);
9233 callVM<Fn, BoxNonStrictThis>(lir);
9234
9235 StoreRegisterTo(output).generate(this);
9236 restoreLiveIgnore(lir, StoreRegisterTo(output).clobbered());
9237
9238 masm.jump(ool->rejoin());
9239}
9240
9241void CodeGenerator::visitImplicitThis(LImplicitThis* lir) {
9242 Register env = ToRegister(lir->env());
9243 ValueOperand output = ToOutValue(lir);
9244
9245 using Fn = void (*)(JSContext*, HandleObject, MutableHandleValue);
9246 auto* ool = oolCallVM<Fn, ImplicitThisOperation>(lir, ArgList(env),
9247 StoreValueTo(output));
9248
9249 masm.computeImplicitThis(env, output, ool->entry());
9250 masm.bind(ool->rejoin());
9251}
9252
9253void CodeGenerator::visitArrayLength(LArrayLength* lir) {
9254 Register elements = ToRegister(lir->elements());
9255 Register output = ToRegister(lir->output());
9256
9257 Address length(elements, ObjectElements::offsetOfLength());
9258 masm.load32(length, output);
9259
9260 // Bail out if the length doesn't fit in int32.
9261 bailoutTest32(Assembler::Signed, output, output, lir->snapshot());
9262}
9263
9264static void SetLengthFromIndex(MacroAssembler& masm, const LAllocation* index,
9265 const Address& length) {
9266 if (index->isConstant()) {
9267 masm.store32(Imm32(ToInt32(index) + 1), length);
9268 } else {
9269 Register newLength = ToRegister(index);
9270 masm.add32(Imm32(1), newLength);
9271 masm.store32(newLength, length);
9272 masm.sub32(Imm32(1), newLength);
9273 }
9274}
9275
9276void CodeGenerator::visitSetArrayLength(LSetArrayLength* lir) {
9277 Address length(ToRegister(lir->elements()), ObjectElements::offsetOfLength());
9278 SetLengthFromIndex(masm, lir->index(), length);
9279}
9280
9281void CodeGenerator::visitFunctionLength(LFunctionLength* lir) {
9282 Register function = ToRegister(lir->function());
9283 Register output = ToRegister(lir->output());
9284
9285 Label bail;
9286
9287 // Get the JSFunction flags.
9288 masm.load32(Address(function, JSFunction::offsetOfFlagsAndArgCount()),
9289 output);
9290
9291 // Functions with a SelfHostedLazyScript must be compiled with the slow-path
9292 // before the function length is known. If the length was previously resolved,
9293 // the length property may be shadowed.
9294 masm.branchTest32(
9295 Assembler::NonZero, output,
9296 Imm32(FunctionFlags::SELFHOSTLAZY | FunctionFlags::RESOLVED_LENGTH),
9297 &bail);
9298
9299 masm.loadFunctionLength(function, output, output, &bail);
9300
9301 bailoutFrom(&bail, lir->snapshot());
9302}
9303
9304void CodeGenerator::visitFunctionName(LFunctionName* lir) {
9305 Register function = ToRegister(lir->function());
9306 Register output = ToRegister(lir->output());
9307
9308 Label bail;
9309
9310 const JSAtomState& names = gen->runtime->names();
9311 masm.loadFunctionName(function, output, ImmGCPtr(names.empty_), &bail);
9312
9313 bailoutFrom(&bail, lir->snapshot());
9314}
9315
9316template <class OrderedHashTable>
9317static void RangeFront(MacroAssembler&, Register, Register, Register);
9318
9319template <>
9320void RangeFront<ValueMap>(MacroAssembler& masm, Register range, Register i,
9321 Register front) {
9322 masm.loadPtr(Address(range, ValueMap::Range::offsetOfHashTable()), front);
9323 masm.loadPtr(Address(front, ValueMap::offsetOfImplData()), front);
9324
9325 MOZ_ASSERT(ValueMap::offsetOfImplDataElement() == 0,do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ValueMap::offsetOfImplDataElement() == 0)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(ValueMap::offsetOfImplDataElement() == 0))), 0))) { do { }
while (false); MOZ_ReportAssertionFailure("ValueMap::offsetOfImplDataElement() == 0"
" (" "offsetof(Data, element) is 0" ")", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 9326); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ValueMap::offsetOfImplDataElement() == 0"
") (" "offsetof(Data, element) is 0" ")"); do { *((volatile int
*)__null) = 9326; __attribute__((nomerge)) ::abort(); } while
(false); } } while (false)
9326 "offsetof(Data, element) is 0")do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ValueMap::offsetOfImplDataElement() == 0)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(ValueMap::offsetOfImplDataElement() == 0))), 0))) { do { }
while (false); MOZ_ReportAssertionFailure("ValueMap::offsetOfImplDataElement() == 0"
" (" "offsetof(Data, element) is 0" ")", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 9326); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ValueMap::offsetOfImplDataElement() == 0"
") (" "offsetof(Data, element) is 0" ")"); do { *((volatile int
*)__null) = 9326; __attribute__((nomerge)) ::abort(); } while
(false); } } while (false)
;
9327 static_assert(ValueMap::sizeofImplData() == 24, "sizeof(Data) is 24");
9328 masm.mulBy3(i, i);
9329 masm.lshiftPtr(Imm32(3), i);
9330 masm.addPtr(i, front);
9331}
9332
9333template <>
9334void RangeFront<ValueSet>(MacroAssembler& masm, Register range, Register i,
9335 Register front) {
9336 masm.loadPtr(Address(range, ValueSet::Range::offsetOfHashTable()), front);
9337 masm.loadPtr(Address(front, ValueSet::offsetOfImplData()), front);
9338
9339 MOZ_ASSERT(ValueSet::offsetOfImplDataElement() == 0,do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ValueSet::offsetOfImplDataElement() == 0)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(ValueSet::offsetOfImplDataElement() == 0))), 0))) { do { }
while (false); MOZ_ReportAssertionFailure("ValueSet::offsetOfImplDataElement() == 0"
" (" "offsetof(Data, element) is 0" ")", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 9340); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ValueSet::offsetOfImplDataElement() == 0"
") (" "offsetof(Data, element) is 0" ")"); do { *((volatile int
*)__null) = 9340; __attribute__((nomerge)) ::abort(); } while
(false); } } while (false)
9340 "offsetof(Data, element) is 0")do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ValueSet::offsetOfImplDataElement() == 0)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(ValueSet::offsetOfImplDataElement() == 0))), 0))) { do { }
while (false); MOZ_ReportAssertionFailure("ValueSet::offsetOfImplDataElement() == 0"
" (" "offsetof(Data, element) is 0" ")", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 9340); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ValueSet::offsetOfImplDataElement() == 0"
") (" "offsetof(Data, element) is 0" ")"); do { *((volatile int
*)__null) = 9340; __attribute__((nomerge)) ::abort(); } while
(false); } } while (false)
;
9341 static_assert(ValueSet::sizeofImplData() == 16, "sizeof(Data) is 16");
9342 masm.lshiftPtr(Imm32(4), i);
9343 masm.addPtr(i, front);
9344}
9345
9346template <class OrderedHashTable>
9347static void RangePopFront(MacroAssembler& masm, Register range, Register front,
9348 Register dataLength, Register temp) {
9349 Register i = temp;
9350
9351 masm.add32(Imm32(1),
9352 Address(range, OrderedHashTable::Range::offsetOfCount()));
9353
9354 masm.load32(Address(range, OrderedHashTable::Range::offsetOfI()), i);
9355
9356 Label done, seek;
9357 masm.bind(&seek);
9358 masm.add32(Imm32(1), i);
9359 masm.branch32(Assembler::AboveOrEqual, i, dataLength, &done);
9360
9361 // We can add sizeof(Data) to |front| to select the next element, because
9362 // |front| and |range.ht.data[i]| point to the same location.
9363 MOZ_ASSERT(OrderedHashTable::offsetOfImplDataElement() == 0,do { static_assert( mozilla::detail::AssertionConditionType<
decltype(OrderedHashTable::offsetOfImplDataElement() == 0)>
::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(OrderedHashTable::offsetOfImplDataElement() == 0))),
0))) { do { } while (false); MOZ_ReportAssertionFailure("OrderedHashTable::offsetOfImplDataElement() == 0"
" (" "offsetof(Data, element) is 0" ")", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 9364); AnnotateMozCrashReason("MOZ_ASSERT" "(" "OrderedHashTable::offsetOfImplDataElement() == 0"
") (" "offsetof(Data, element) is 0" ")"); do { *((volatile int
*)__null) = 9364; __attribute__((nomerge)) ::abort(); } while
(false); } } while (false)
9364 "offsetof(Data, element) is 0")do { static_assert( mozilla::detail::AssertionConditionType<
decltype(OrderedHashTable::offsetOfImplDataElement() == 0)>
::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(OrderedHashTable::offsetOfImplDataElement() == 0))),
0))) { do { } while (false); MOZ_ReportAssertionFailure("OrderedHashTable::offsetOfImplDataElement() == 0"
" (" "offsetof(Data, element) is 0" ")", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 9364); AnnotateMozCrashReason("MOZ_ASSERT" "(" "OrderedHashTable::offsetOfImplDataElement() == 0"
") (" "offsetof(Data, element) is 0" ")"); do { *((volatile int
*)__null) = 9364; __attribute__((nomerge)) ::abort(); } while
(false); } } while (false)
;
9365 masm.addPtr(Imm32(OrderedHashTable::sizeofImplData()), front);
9366
9367 masm.branchTestMagic(Assembler::Equal,
9368 Address(front, OrderedHashTable::offsetOfEntryKey()),
9369 JS_HASH_KEY_EMPTY, &seek);
9370
9371 masm.bind(&done);
9372 masm.store32(i, Address(range, OrderedHashTable::Range::offsetOfI()));
9373}
9374
9375template <class OrderedHashTable>
9376static inline void RangeDestruct(MacroAssembler& masm, Register iter,
9377 Register range, Register temp0,
9378 Register temp1) {
9379 Register next = temp0;
9380 Register prevp = temp1;
9381
9382 masm.loadPtr(Address(range, OrderedHashTable::Range::offsetOfNext()), next);
9383 masm.loadPtr(Address(range, OrderedHashTable::Range::offsetOfPrevP()), prevp);
9384 masm.storePtr(next, Address(prevp, 0));
9385
9386 Label hasNoNext;
9387 masm.branchTestPtr(Assembler::Zero, next, next, &hasNoNext);
9388
9389 masm.storePtr(prevp, Address(next, OrderedHashTable::Range::offsetOfPrevP()));
9390
9391 masm.bind(&hasNoNext);
9392
9393 Label nurseryAllocated;
9394 masm.branchPtrInNurseryChunk(Assembler::Equal, iter, temp0,
9395 &nurseryAllocated);
9396
9397 masm.callFreeStub(range);
9398
9399 masm.bind(&nurseryAllocated);
9400}
9401
9402template <>
9403void CodeGenerator::emitLoadIteratorValues<ValueMap>(Register result,
9404 Register temp,
9405 Register front) {
9406 size_t elementsOffset = NativeObject::offsetOfFixedElements();
9407
9408 Address keyAddress(front, ValueMap::Entry::offsetOfKey());
9409 Address valueAddress(front, ValueMap::Entry::offsetOfValue());
9410 Address keyElemAddress(result, elementsOffset);
9411 Address valueElemAddress(result, elementsOffset + sizeof(Value));
9412 masm.guardedCallPreBarrier(keyElemAddress, MIRType::Value);
9413 masm.guardedCallPreBarrier(valueElemAddress, MIRType::Value);
9414 masm.storeValue(keyAddress, keyElemAddress, temp);
9415 masm.storeValue(valueAddress, valueElemAddress, temp);
9416
9417 Label emitBarrier, skipBarrier;
9418 masm.branchValueIsNurseryCell(Assembler::Equal, keyAddress, temp,
9419 &emitBarrier);
9420 masm.branchValueIsNurseryCell(Assembler::NotEqual, valueAddress, temp,
9421 &skipBarrier);
9422 {
9423 masm.bind(&emitBarrier);
9424 saveVolatile(temp);
9425 emitPostWriteBarrier(result);
9426 restoreVolatile(temp);
9427 }
9428 masm.bind(&skipBarrier);
9429}
9430
9431template <>
9432void CodeGenerator::emitLoadIteratorValues<ValueSet>(Register result,
9433 Register temp,
9434 Register front) {
9435 size_t elementsOffset = NativeObject::offsetOfFixedElements();
9436
9437 Address keyAddress(front, ValueSet::offsetOfEntryKey());
9438 Address keyElemAddress(result, elementsOffset);
9439 masm.guardedCallPreBarrier(keyElemAddress, MIRType::Value);
9440 masm.storeValue(keyAddress, keyElemAddress, temp);
9441
9442 Label skipBarrier;
9443 masm.branchValueIsNurseryCell(Assembler::NotEqual, keyAddress, temp,
9444 &skipBarrier);
9445 {
9446 saveVolatile(temp);
9447 emitPostWriteBarrier(result);
9448 restoreVolatile(temp);
9449 }
9450 masm.bind(&skipBarrier);
9451}
9452
9453template <class IteratorObject, class OrderedHashTable>
9454void CodeGenerator::emitGetNextEntryForIterator(LGetNextEntryForIterator* lir) {
9455 Register iter = ToRegister(lir->iter());
9456 Register result = ToRegister(lir->result());
9457 Register temp = ToRegister(lir->temp0());
9458 Register dataLength = ToRegister(lir->temp1());
9459 Register range = ToRegister(lir->temp2());
9460 Register output = ToRegister(lir->output());
9461
9462#ifdef DEBUG1
9463 // Self-hosted code is responsible for ensuring GetNextEntryForIterator is
9464 // only called with the correct iterator class. Assert here all self-
9465 // hosted callers of GetNextEntryForIterator perform this class check.
9466 // No Spectre mitigations are needed because this is DEBUG-only code.
9467 Label success;
9468 masm.branchTestObjClassNoSpectreMitigations(
9469 Assembler::Equal, iter, &IteratorObject::class_, temp, &success);
9470 masm.assumeUnreachable("Iterator object should have the correct class.");
9471 masm.bind(&success);
9472#endif
9473
9474 masm.loadPrivate(Address(iter, NativeObject::getFixedSlotOffset(
9475 IteratorObject::RangeSlot)),
9476 range);
9477
9478 Label iterAlreadyDone, iterDone, done;
9479 masm.branchTestPtr(Assembler::Zero, range, range, &iterAlreadyDone);
9480
9481 masm.load32(Address(range, OrderedHashTable::Range::offsetOfI()), temp);
9482 masm.loadPtr(Address(range, OrderedHashTable::Range::offsetOfHashTable()),
9483 dataLength);
9484 masm.load32(Address(dataLength, OrderedHashTable::offsetOfImplDataLength()),
9485 dataLength);
9486 masm.branch32(Assembler::AboveOrEqual, temp, dataLength, &iterDone);
9487 {
9488 masm.Push(iter);
9489
9490 Register front = iter;
9491 RangeFront<OrderedHashTable>(masm, range, temp, front);
9492
9493 emitLoadIteratorValues<OrderedHashTable>(result, temp, front);
9494
9495 RangePopFront<OrderedHashTable>(masm, range, front, dataLength, temp);
9496
9497 masm.Pop(iter);
9498 masm.move32(Imm32(0), output);
9499 }
9500 masm.jump(&done);
9501 {
9502 masm.bind(&iterDone);
9503
9504 RangeDestruct<OrderedHashTable>(masm, iter, range, temp, dataLength);
9505
9506 masm.storeValue(PrivateValue(nullptr),
9507 Address(iter, NativeObject::getFixedSlotOffset(
9508 IteratorObject::RangeSlot)));
9509
9510 masm.bind(&iterAlreadyDone);
9511
9512 masm.move32(Imm32(1), output);
9513 }
9514 masm.bind(&done);
9515}
9516
9517void CodeGenerator::visitGetNextEntryForIterator(
9518 LGetNextEntryForIterator* lir) {
9519 if (lir->mir()->mode() == MGetNextEntryForIterator::Map) {
9520 emitGetNextEntryForIterator<MapIteratorObject, ValueMap>(lir);
9521 } else {
9522 MOZ_ASSERT(lir->mir()->mode() == MGetNextEntryForIterator::Set)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(lir->mir()->mode() == MGetNextEntryForIterator
::Set)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(lir->mir()->mode() == MGetNextEntryForIterator
::Set))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("lir->mir()->mode() == MGetNextEntryForIterator::Set",
"/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 9522); AnnotateMozCrashReason("MOZ_ASSERT" "(" "lir->mir()->mode() == MGetNextEntryForIterator::Set"
")"); do { *((volatile int*)__null) = 9522; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
9523 emitGetNextEntryForIterator<SetIteratorObject, ValueSet>(lir);
9524 }
9525}
9526
9527// The point of these is to inform Ion of where these values already are; they
9528// don't normally generate (much) code.
9529void CodeGenerator::visitWasmRegisterPairResult(LWasmRegisterPairResult* lir) {}
9530void CodeGenerator::visitWasmStackResult(LWasmStackResult* lir) {}
9531void CodeGenerator::visitWasmStackResult64(LWasmStackResult64* lir) {}
9532
9533void CodeGenerator::visitWasmStackResultArea(LWasmStackResultArea* lir) {
9534 LAllocation* output = lir->getDef(0)->output();
9535 MOZ_ASSERT(output->isStackArea())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(output->isStackArea())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(output->isStackArea()))),
0))) { do { } while (false); MOZ_ReportAssertionFailure("output->isStackArea()"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 9535); AnnotateMozCrashReason("MOZ_ASSERT" "(" "output->isStackArea()"
")"); do { *((volatile int*)__null) = 9535; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
9536 bool tempInit = false;
9537 for (auto iter = output->toStackArea()->results(); iter; iter.next()) {
9538 // Zero out ref stack results.
9539 if (iter.isWasmAnyRef()) {
9540 Register temp = ToRegister(lir->temp0());
9541 if (!tempInit) {
9542 masm.xorPtr(temp, temp);
9543 tempInit = true;
9544 }
9545 masm.storePtr(temp, ToAddress(iter.alloc()));
9546 }
9547 }
9548}
9549
9550void CodeGenerator::visitWasmRegisterResult(LWasmRegisterResult* lir) {
9551#ifdef JS_64BIT1
9552 if (MWasmRegisterResult* mir = lir->mir()) {
9553 if (mir->type() == MIRType::Int32) {
9554 masm.widenInt32(ToRegister(lir->output()));
9555 }
9556 }
9557#endif
9558}
9559
9560void CodeGenerator::visitWasmCall(LWasmCall* lir) {
9561 const MWasmCallBase* callBase = lir->callBase();
9562 bool isReturnCall = lir->isReturnCall();
9563
9564 // If this call is in Wasm try code block, initialise a wasm::TryNote for this
9565 // call.
9566 bool inTry = callBase->inTry();
9567 if (inTry) {
9568 size_t tryNoteIndex = callBase->tryNoteIndex();
9569 wasm::TryNoteVector& tryNotes = masm.tryNotes();
9570 wasm::TryNote& tryNote = tryNotes[tryNoteIndex];
9571 tryNote.setTryBodyBegin(masm.currentOffset());
9572 }
9573
9574 MOZ_ASSERT((sizeof(wasm::Frame) + masm.framePushed()) % WasmStackAlignment ==do { static_assert( mozilla::detail::AssertionConditionType<
decltype((sizeof(wasm::Frame) + masm.framePushed()) % WasmStackAlignment
== 0)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!((sizeof(wasm::Frame) + masm.framePushed()) % WasmStackAlignment
== 0))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("(sizeof(wasm::Frame) + masm.framePushed()) % WasmStackAlignment == 0"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 9575); AnnotateMozCrashReason("MOZ_ASSERT" "(" "(sizeof(wasm::Frame) + masm.framePushed()) % WasmStackAlignment == 0"
")"); do { *((volatile int*)__null) = 9575; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
9575 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype((sizeof(wasm::Frame) + masm.framePushed()) % WasmStackAlignment
== 0)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!((sizeof(wasm::Frame) + masm.framePushed()) % WasmStackAlignment
== 0))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("(sizeof(wasm::Frame) + masm.framePushed()) % WasmStackAlignment == 0"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 9575); AnnotateMozCrashReason("MOZ_ASSERT" "(" "(sizeof(wasm::Frame) + masm.framePushed()) % WasmStackAlignment == 0"
")"); do { *((volatile int*)__null) = 9575; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
9576 static_assert(
9577 WasmStackAlignment >= ABIStackAlignment &&
9578 WasmStackAlignment % ABIStackAlignment == 0,
9579 "The wasm stack alignment should subsume the ABI-required alignment");
9580
9581#ifdef DEBUG1
9582 Label ok;
9583 masm.branchTestStackPtr(Assembler::Zero, Imm32(WasmStackAlignment - 1), &ok);
9584 masm.breakpoint();
9585 masm.bind(&ok);
9586#endif
9587
9588 // LWasmCallBase::isCallPreserved() assumes that all MWasmCalls preserve the
9589 // instance and pinned regs. The only case where where we don't have to
9590 // reload the instance and pinned regs is when the callee preserves them.
9591 bool reloadRegs = true;
9592 bool switchRealm = true;
9593
9594 const wasm::CallSiteDesc& desc = callBase->desc();
9595 const wasm::CalleeDesc& callee = callBase->callee();
9596 CodeOffset retOffset;
9597 CodeOffset secondRetOffset;
9598 switch (callee.which()) {
9599 case wasm::CalleeDesc::Func:
9600#ifdef ENABLE_WASM_TAIL_CALLS1
9601 if (isReturnCall) {
9602 ReturnCallAdjustmentInfo retCallInfo(
9603 callBase->stackArgAreaSizeUnaligned(), inboundStackArgBytes_);
9604 masm.wasmReturnCall(desc, callee.funcIndex(), retCallInfo);
9605 // The rest of the method is unnecessary for a return call.
9606 return;
9607 }
9608#endif
9609 MOZ_ASSERT(!isReturnCall)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!isReturnCall)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!isReturnCall))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("!isReturnCall",
"/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 9609); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!isReturnCall"
")"); do { *((volatile int*)__null) = 9609; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
9610 retOffset = masm.call(desc, callee.funcIndex());
9611 reloadRegs = false;
9612 switchRealm = false;
9613 break;
9614 case wasm::CalleeDesc::Import:
9615#ifdef ENABLE_WASM_TAIL_CALLS1
9616 if (isReturnCall) {
9617 ReturnCallAdjustmentInfo retCallInfo(
9618 callBase->stackArgAreaSizeUnaligned(), inboundStackArgBytes_);
9619 masm.wasmReturnCallImport(desc, callee, retCallInfo);
9620 // The rest of the method is unnecessary for a return call.
9621 return;
9622 }
9623#endif
9624 MOZ_ASSERT(!isReturnCall)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!isReturnCall)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!isReturnCall))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("!isReturnCall",
"/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 9624); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!isReturnCall"
")"); do { *((volatile int*)__null) = 9624; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
9625 retOffset = masm.wasmCallImport(desc, callee);
9626 break;
9627 case wasm::CalleeDesc::AsmJSTable:
9628 retOffset = masm.asmCallIndirect(desc, callee);
9629 break;
9630 case wasm::CalleeDesc::WasmTable: {
9631 Label* boundsCheckFailed = nullptr;
9632 if (lir->needsBoundsCheck()) {
9633 OutOfLineAbortingWasmTrap* ool =
9634 new (alloc()) OutOfLineAbortingWasmTrap(
9635 wasm::BytecodeOffset(desc.lineOrBytecode()),
9636 wasm::Trap::OutOfBounds);
9637 if (lir->isCatchable()) {
9638 addOutOfLineCode(ool, lir->mirCatchable());
9639 } else if (isReturnCall) {
9640#ifdef ENABLE_WASM_TAIL_CALLS1
9641 addOutOfLineCode(ool, lir->mirReturnCall());
9642#else
9643 MOZ_CRASH("Return calls are disabled.")do { do { } while (false); MOZ_ReportCrash("" "Return calls are disabled."
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 9643); AnnotateMozCrashReason("MOZ_CRASH(" "Return calls are disabled."
")"); do { *((volatile int*)__null) = 9643; __attribute__((nomerge
)) ::abort(); } while (false); } while (false)
;
9644#endif
9645 } else {
9646 addOutOfLineCode(ool, lir->mirUncatchable());
9647 }
9648 boundsCheckFailed = ool->entry();
9649 }
9650 Label* nullCheckFailed = nullptr;
9651#ifndef WASM_HAS_HEAPREG1
9652 {
9653 OutOfLineAbortingWasmTrap* ool =
9654 new (alloc()) OutOfLineAbortingWasmTrap(
9655 wasm::BytecodeOffset(desc.lineOrBytecode()),
9656 wasm::Trap::IndirectCallToNull);
9657 if (lir->isCatchable()) {
9658 addOutOfLineCode(ool, lir->mirCatchable());
9659 } else if (isReturnCall) {
9660# ifdef ENABLE_WASM_TAIL_CALLS1
9661 addOutOfLineCode(ool, lir->mirReturnCall());
9662# else
9663 MOZ_CRASH("Return calls are disabled.")do { do { } while (false); MOZ_ReportCrash("" "Return calls are disabled."
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 9663); AnnotateMozCrashReason("MOZ_CRASH(" "Return calls are disabled."
")"); do { *((volatile int*)__null) = 9663; __attribute__((nomerge
)) ::abort(); } while (false); } while (false)
;
9664# endif
9665 } else {
9666 addOutOfLineCode(ool, lir->mirUncatchable());
9667 }
9668 nullCheckFailed = ool->entry();
9669 }
9670#endif
9671#ifdef ENABLE_WASM_TAIL_CALLS1
9672 if (isReturnCall) {
9673 ReturnCallAdjustmentInfo retCallInfo(
9674 callBase->stackArgAreaSizeUnaligned(), inboundStackArgBytes_);
9675 masm.wasmReturnCallIndirect(desc, callee, boundsCheckFailed,
9676 nullCheckFailed, mozilla::Nothing(),
9677 retCallInfo);
9678 // The rest of the method is unnecessary for a return call.
9679 return;
9680 }
9681#endif
9682 MOZ_ASSERT(!isReturnCall)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!isReturnCall)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!isReturnCall))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("!isReturnCall",
"/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 9682); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!isReturnCall"
")"); do { *((volatile int*)__null) = 9682; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
9683 masm.wasmCallIndirect(desc, callee, boundsCheckFailed, nullCheckFailed,
9684 lir->tableSize(), &retOffset, &secondRetOffset);
9685 // Register reloading and realm switching are handled dynamically inside
9686 // wasmCallIndirect. There are two return offsets, one for each call
9687 // instruction (fast path and slow path).
9688 reloadRegs = false;
9689 switchRealm = false;
9690 break;
9691 }
9692 case wasm::CalleeDesc::Builtin:
9693 retOffset = masm.call(desc, callee.builtin());
9694 reloadRegs = false;
9695 switchRealm = false;
9696 break;
9697 case wasm::CalleeDesc::BuiltinInstanceMethod:
9698 retOffset = masm.wasmCallBuiltinInstanceMethod(
9699 desc, callBase->instanceArg(), callee.builtin(),
9700 callBase->builtinMethodFailureMode());
9701 switchRealm = false;
9702 break;
9703 case wasm::CalleeDesc::FuncRef:
9704#ifdef ENABLE_WASM_TAIL_CALLS1
9705 if (isReturnCall) {
9706 ReturnCallAdjustmentInfo retCallInfo(
9707 callBase->stackArgAreaSizeUnaligned(), inboundStackArgBytes_);
9708 masm.wasmReturnCallRef(desc, callee, retCallInfo);
9709 // The rest of the method is unnecessary for a return call.
9710 return;
9711 }
9712#endif
9713 MOZ_ASSERT(!isReturnCall)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!isReturnCall)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!isReturnCall))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("!isReturnCall",
"/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 9713); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!isReturnCall"
")"); do { *((volatile int*)__null) = 9713; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
9714 // Register reloading and realm switching are handled dynamically inside
9715 // wasmCallRef. There are two return offsets, one for each call
9716 // instruction (fast path and slow path).
9717 masm.wasmCallRef(desc, callee, &retOffset, &secondRetOffset);
9718 reloadRegs = false;
9719 switchRealm = false;
9720 break;
9721 }
9722
9723 // Note the assembler offset for the associated LSafePoint.
9724 MOZ_ASSERT(!isReturnCall)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!isReturnCall)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!isReturnCall))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("!isReturnCall",
"/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 9724); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!isReturnCall"
")"); do { *((volatile int*)__null) = 9724; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
9725 markSafepointAt(retOffset.offset(), lir);
9726
9727 // Now that all the outbound in-memory args are on the stack, note the
9728 // required lower boundary point of the associated StackMap.
9729 uint32_t framePushedAtStackMapBase =
9730 masm.framePushed() -
9731 wasm::AlignStackArgAreaSize(callBase->stackArgAreaSizeUnaligned());
9732 lir->safepoint()->setFramePushedAtStackMapBase(framePushedAtStackMapBase);
9733 MOZ_ASSERT(lir->safepoint()->wasmSafepointKind() ==do { static_assert( mozilla::detail::AssertionConditionType<
decltype(lir->safepoint()->wasmSafepointKind() == WasmSafepointKind
::LirCall)>::isValid, "invalid assertion condition"); if (
(__builtin_expect(!!(!(!!(lir->safepoint()->wasmSafepointKind
() == WasmSafepointKind::LirCall))), 0))) { do { } while (false
); MOZ_ReportAssertionFailure("lir->safepoint()->wasmSafepointKind() == WasmSafepointKind::LirCall"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 9734); AnnotateMozCrashReason("MOZ_ASSERT" "(" "lir->safepoint()->wasmSafepointKind() == WasmSafepointKind::LirCall"
")"); do { *((volatile int*)__null) = 9734; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
9734 WasmSafepointKind::LirCall)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(lir->safepoint()->wasmSafepointKind() == WasmSafepointKind
::LirCall)>::isValid, "invalid assertion condition"); if (
(__builtin_expect(!!(!(!!(lir->safepoint()->wasmSafepointKind
() == WasmSafepointKind::LirCall))), 0))) { do { } while (false
); MOZ_ReportAssertionFailure("lir->safepoint()->wasmSafepointKind() == WasmSafepointKind::LirCall"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 9734); AnnotateMozCrashReason("MOZ_ASSERT" "(" "lir->safepoint()->wasmSafepointKind() == WasmSafepointKind::LirCall"
")"); do { *((volatile int*)__null) = 9734; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
9735
9736 // Note the assembler offset and framePushed for use by the adjunct
9737 // LSafePoint, see visitor for LWasmCallIndirectAdjunctSafepoint below.
9738 if (callee.which() == wasm::CalleeDesc::WasmTable ||
9739 callee.which() == wasm::CalleeDesc::FuncRef) {
9740 lir->adjunctSafepoint()->recordSafepointInfo(secondRetOffset,
9741 framePushedAtStackMapBase);
9742 }
9743
9744 if (reloadRegs) {
9745 masm.loadPtr(
9746 Address(masm.getStackPointer(), WasmCallerInstanceOffsetBeforeCall),
9747 InstanceReg);
9748 masm.loadWasmPinnedRegsFromInstance();
9749 if (switchRealm) {
9750 masm.switchToWasmInstanceRealm(ABINonArgReturnReg0, ABINonArgReturnReg1);
9751 }
9752 } else {
9753 MOZ_ASSERT(!switchRealm)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!switchRealm)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!switchRealm))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("!switchRealm", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 9753); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!switchRealm"
")"); do { *((volatile int*)__null) = 9753; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
9754 }
9755
9756#ifdef ENABLE_WASM_TAIL_CALLS1
9757 switch (callee.which()) {
9758 case wasm::CalleeDesc::Func:
9759 case wasm::CalleeDesc::Import:
9760 case wasm::CalleeDesc::WasmTable:
9761 case wasm::CalleeDesc::FuncRef:
9762 // Stack allocation could change during Wasm (return) calls,
9763 // recover pre-call state.
9764 masm.freeStackTo(masm.framePushed());
9765 break;
9766 default:
9767 break;
9768 }
9769#endif // ENABLE_WASM_TAIL_CALLS
9770
9771 if (inTry) {
9772 // Set the end of the try note range
9773 size_t tryNoteIndex = callBase->tryNoteIndex();
9774 wasm::TryNoteVector& tryNotes = masm.tryNotes();
9775 wasm::TryNote& tryNote = tryNotes[tryNoteIndex];
9776
9777 // Don't set the end of the try note if we've OOM'ed, as the above
9778 // instructions may not have been emitted, which will trigger an assert
9779 // about zero-length try-notes. This is okay as this compilation will be
9780 // thrown away.
9781 if (!masm.oom()) {
9782 tryNote.setTryBodyEnd(masm.currentOffset());
9783 }
9784
9785 // This instruction or the adjunct safepoint must be the last instruction
9786 // in the block. No other instructions may be inserted.
9787 LBlock* block = lir->block();
9788 MOZ_RELEASE_ASSERT(*block->rbegin() == lir ||do { static_assert( mozilla::detail::AssertionConditionType<
decltype(*block->rbegin() == lir || (block->rbegin()->
isWasmCallIndirectAdjunctSafepoint() && *(++block->
rbegin()) == lir))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(*block->rbegin() == lir ||
(block->rbegin()->isWasmCallIndirectAdjunctSafepoint()
&& *(++block->rbegin()) == lir)))), 0))) { do { }
while (false); MOZ_ReportAssertionFailure("*block->rbegin() == lir || (block->rbegin()->isWasmCallIndirectAdjunctSafepoint() && *(++block->rbegin()) == lir)"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 9790); AnnotateMozCrashReason("MOZ_RELEASE_ASSERT" "(" "*block->rbegin() == lir || (block->rbegin()->isWasmCallIndirectAdjunctSafepoint() && *(++block->rbegin()) == lir)"
")"); do { *((volatile int*)__null) = 9790; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
9789 (block->rbegin()->isWasmCallIndirectAdjunctSafepoint() &&do { static_assert( mozilla::detail::AssertionConditionType<
decltype(*block->rbegin() == lir || (block->rbegin()->
isWasmCallIndirectAdjunctSafepoint() && *(++block->
rbegin()) == lir))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(*block->rbegin() == lir ||
(block->rbegin()->isWasmCallIndirectAdjunctSafepoint()
&& *(++block->rbegin()) == lir)))), 0))) { do { }
while (false); MOZ_ReportAssertionFailure("*block->rbegin() == lir || (block->rbegin()->isWasmCallIndirectAdjunctSafepoint() && *(++block->rbegin()) == lir)"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 9790); AnnotateMozCrashReason("MOZ_RELEASE_ASSERT" "(" "*block->rbegin() == lir || (block->rbegin()->isWasmCallIndirectAdjunctSafepoint() && *(++block->rbegin()) == lir)"
")"); do { *((volatile int*)__null) = 9790; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
9790 *(++block->rbegin()) == lir))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(*block->rbegin() == lir || (block->rbegin()->
isWasmCallIndirectAdjunctSafepoint() && *(++block->
rbegin()) == lir))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(*block->rbegin() == lir ||
(block->rbegin()->isWasmCallIndirectAdjunctSafepoint()
&& *(++block->rbegin()) == lir)))), 0))) { do { }
while (false); MOZ_ReportAssertionFailure("*block->rbegin() == lir || (block->rbegin()->isWasmCallIndirectAdjunctSafepoint() && *(++block->rbegin()) == lir)"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 9790); AnnotateMozCrashReason("MOZ_RELEASE_ASSERT" "(" "*block->rbegin() == lir || (block->rbegin()->isWasmCallIndirectAdjunctSafepoint() && *(++block->rbegin()) == lir)"
")"); do { *((volatile int*)__null) = 9790; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
9791
9792 // Jump to the fallthrough block
9793 jumpToBlock(lir->mirCatchable()->getSuccessor(
9794 MWasmCallCatchable::FallthroughBranchIndex));
9795 }
9796}
9797
9798#ifdef ENABLE_WASM_JSPI1
9799void CodeGenerator::callWasmUpdateSuspenderState(
9800 wasm::UpdateSuspenderStateAction kind, Register suspender, Register temp) {
9801 masm.Push(InstanceReg);
9802 int32_t framePushedAfterInstance = masm.framePushed();
9803
9804 masm.move32(Imm32(uint32_t(kind)), temp);
9805
9806 masm.setupWasmABICall();
9807 masm.passABIArg(InstanceReg);
9808 masm.passABIArg(suspender);
9809 masm.passABIArg(temp);
9810 int32_t instanceOffset = masm.framePushed() - framePushedAfterInstance;
9811 masm.callWithABI(wasm::BytecodeOffset(0),
9812 wasm::SymbolicAddress::UpdateSuspenderState,
9813 mozilla::Some(instanceOffset));
9814
9815 masm.Pop(InstanceReg);
9816}
9817
9818void CodeGenerator::prepareWasmStackSwitchTrampolineCall(Register suspender,
9819 Register data) {
9820 // Reserve stack space for the wasm call.
9821 unsigned argDecrement;
9822 {
9823 WasmABIArgGenerator abi;
9824 ABIArg arg;
9825 arg = abi.next(MIRType::Pointer);
9826 arg = abi.next(MIRType::Pointer);
9827 argDecrement = StackDecrementForCall(WasmStackAlignment, 0,
9828 abi.stackBytesConsumedSoFar());
9829 }
9830 masm.reserveStack(argDecrement);
9831
9832 // Pass the suspender and data params through the wasm function ABI registers.
9833 WasmABIArgGenerator abi;
9834 ABIArg arg;
9835 arg = abi.next(MIRType::Pointer);
9836 if (arg.kind() == ABIArg::GPR) {
9837 masm.movePtr(suspender, arg.gpr());
9838 } else {
9839 MOZ_ASSERT(arg.kind() == ABIArg::Stack)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(arg.kind() == ABIArg::Stack)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(arg.kind() == ABIArg::Stack)
)), 0))) { do { } while (false); MOZ_ReportAssertionFailure("arg.kind() == ABIArg::Stack"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 9839); AnnotateMozCrashReason("MOZ_ASSERT" "(" "arg.kind() == ABIArg::Stack"
")"); do { *((volatile int*)__null) = 9839; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
9840 masm.storePtr(suspender,
9841 Address(masm.getStackPointer(), arg.offsetFromArgBase()));
9842 }
9843 arg = abi.next(MIRType::Pointer);
9844 if (arg.kind() == ABIArg::GPR) {
9845 masm.movePtr(data, arg.gpr());
9846 } else {
9847 MOZ_ASSERT(arg.kind() == ABIArg::Stack)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(arg.kind() == ABIArg::Stack)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(arg.kind() == ABIArg::Stack)
)), 0))) { do { } while (false); MOZ_ReportAssertionFailure("arg.kind() == ABIArg::Stack"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 9847); AnnotateMozCrashReason("MOZ_ASSERT" "(" "arg.kind() == ABIArg::Stack"
")"); do { *((volatile int*)__null) = 9847; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
9848 masm.storePtr(data,
9849 Address(masm.getStackPointer(), arg.offsetFromArgBase()));
9850 }
9851
9852 masm.storePtr(InstanceReg, Address(masm.getStackPointer(),
9853 WasmCallerInstanceOffsetBeforeCall));
9854}
9855#endif // ENABLE_WASM_JSPI
9856
9857void CodeGenerator::visitWasmStackSwitchToSuspendable(
9858 LWasmStackSwitchToSuspendable* lir) {
9859#ifdef ENABLE_WASM_JSPI1
9860 const Register SuspenderReg = lir->suspender()->toRegister().gpr();
9861 const Register FnReg = lir->fn()->toRegister().gpr();
9862 const Register DataReg = lir->data()->toRegister().gpr();
9863 const Register SuspenderDataReg = ABINonArgReg3;
9864
9865# ifdef JS_CODEGEN_ARM64
9866 vixl::UseScratchRegisterScope temps(&masm);
9867 const Register ScratchReg1 = temps.AcquireX().asUnsized();
9868# elif defined(JS_CODEGEN_X86)
9869 const Register ScratchReg1 = ABINonArgReg3;
9870# elif defined(JS_CODEGEN_X641)
9871 const Register ScratchReg1 = ScratchReg;
9872# elif defined(JS_CODEGEN_ARM)
9873 const Register ScratchReg1 = ABINonArgReturnVolatileReg;
9874# else
9875# error "NYI: scratch register"
9876# endif
9877
9878 masm.Push(SuspenderReg);
9879 masm.Push(FnReg);
9880 masm.Push(DataReg);
9881
9882 callWasmUpdateSuspenderState(wasm::UpdateSuspenderStateAction::Enter,
9883 SuspenderReg, ScratchReg1);
9884 masm.Pop(DataReg);
9885 masm.Pop(FnReg);
9886 masm.Pop(SuspenderReg);
9887
9888 masm.Push(SuspenderReg);
9889 int32_t framePushedAtSuspender = masm.framePushed();
9890 masm.Push(InstanceReg);
9891
9892 wasm::CallSiteDesc desc(wasm::CallSiteDesc::Kind::StackSwitch);
9893 CodeLabel returnCallsite;
9894
9895 // Aligning stack before trampoline call.
9896 uint32_t reserve = ComputeByteAlignment(
9897 masm.framePushed() - sizeof(wasm::Frame), WasmStackAlignment);
9898 masm.reserveStack(reserve);
9899
9900 masm.loadPrivate(Address(SuspenderReg, NativeObject::getFixedSlotOffset(
9901 wasm::SuspenderObjectDataSlot)),
9902 SuspenderDataReg);
9903
9904 // Switch stacks to suspendable, keep original FP to maintain
9905 // frames chain between main and suspendable stack segments.
9906 masm.storeStackPtr(
9907 Address(SuspenderDataReg, wasm::SuspenderObjectData::offsetOfMainSP()));
9908 masm.storePtr(
9909 FramePointer,
9910 Address(SuspenderDataReg, wasm::SuspenderObjectData::offsetOfMainFP()));
9911
9912 masm.loadStackPtr(Address(
9913 SuspenderDataReg, wasm::SuspenderObjectData::offsetOfSuspendableSP()));
9914
9915 masm.assertStackAlignment(WasmStackAlignment);
9916
9917 // The FramePointer is not changed for SwitchToSuspendable.
9918 uint32_t framePushed = masm.framePushed();
9919
9920 // On different stack, reset framePushed. FramePointer is not valid here.
9921 masm.setFramePushed(0);
9922
9923 prepareWasmStackSwitchTrampolineCall(SuspenderReg, DataReg);
9924
9925 // Get wasm instance pointer for callee.
9926 size_t instanceSlotOffset = FunctionExtended::offsetOfExtendedSlot(
9927 FunctionExtended::WASM_INSTANCE_SLOT);
9928 masm.loadPtr(Address(FnReg, instanceSlotOffset), InstanceReg);
9929
9930 masm.storePtr(InstanceReg, Address(masm.getStackPointer(),
9931 WasmCalleeInstanceOffsetBeforeCall));
9932 masm.loadWasmPinnedRegsFromInstance();
9933
9934 masm.assertStackAlignment(WasmStackAlignment);
9935
9936 const Register ReturnAddressReg = ScratchReg1;
9937
9938 // DataReg is not needed anymore, using it as a scratch register.
9939 const Register ScratchReg2 = DataReg;
9940
9941 // Save future of suspendable stack exit frame pointer.
9942 masm.computeEffectiveAddress(
9943 Address(masm.getStackPointer(), -int32_t(sizeof(wasm::Frame))),
9944 ScratchReg2);
9945 masm.storePtr(
9946 ScratchReg2,
9947 Address(SuspenderDataReg,
9948 wasm::SuspenderObjectData::offsetOfSuspendableExitFP()));
9949
9950 masm.mov(&returnCallsite, ReturnAddressReg);
9951
9952 // Call wasm function fast.
9953# ifdef JS_USE_LINK_REGISTER
9954 masm.mov(ReturnAddressReg, lr);
9955# else
9956 masm.Push(ReturnAddressReg);
9957# endif
9958 // Get funcUncheckedCallEntry() from the function's
9959 // WASM_FUNC_UNCHECKED_ENTRY_SLOT extended slot.
9960 size_t uncheckedEntrySlotOffset = FunctionExtended::offsetOfExtendedSlot(
9961 FunctionExtended::WASM_FUNC_UNCHECKED_ENTRY_SLOT);
9962 masm.loadPtr(Address(FnReg, uncheckedEntrySlotOffset), ScratchReg2);
9963 masm.jump(ScratchReg2);
9964
9965 // About to use valid FramePointer -- restore framePushed.
9966 masm.setFramePushed(framePushed);
9967
9968 // For IsPlausibleStackMapKey check for the following callsite.
9969 masm.wasmTrapInstruction();
9970
9971 // Callsite for return from main stack.
9972 masm.bind(&returnCallsite);
9973 masm.append(desc, *returnCallsite.target());
9974 masm.addCodeLabel(returnCallsite);
9975
9976 masm.assertStackAlignment(WasmStackAlignment);
9977
9978 markSafepointAt(returnCallsite.target()->offset(), lir);
9979 lir->safepoint()->setFramePushedAtStackMapBase(framePushed);
9980 lir->safepoint()->setWasmSafepointKind(WasmSafepointKind::StackSwitch);
9981 // Rooting SuspenderReg.
9982 masm.propagateOOM(
9983 lir->safepoint()->addWasmAnyRefSlot(true, framePushedAtSuspender));
9984
9985 masm.freeStackTo(framePushed);
9986
9987 masm.freeStack(reserve);
9988 masm.Pop(InstanceReg);
9989 masm.Pop(SuspenderReg);
9990
9991 masm.switchToWasmInstanceRealm(ScratchReg1, ScratchReg2);
9992
9993 callWasmUpdateSuspenderState(wasm::UpdateSuspenderStateAction::Leave,
9994 SuspenderReg, ScratchReg1);
9995#else
9996 MOZ_CRASH("NYI")do { do { } while (false); MOZ_ReportCrash("" "NYI", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 9996); AnnotateMozCrashReason("MOZ_CRASH(" "NYI" ")"); do {
*((volatile int*)__null) = 9996; __attribute__((nomerge)) ::
abort(); } while (false); } while (false)
;
9997#endif // ENABLE_WASM_JSPI
9998}
9999
10000void CodeGenerator::visitWasmStackSwitchToMain(LWasmStackSwitchToMain* lir) {
10001#ifdef ENABLE_WASM_JSPI1
10002 const Register SuspenderReg = lir->suspender()->toRegister().gpr();
10003 const Register FnReg = lir->fn()->toRegister().gpr();
10004 const Register DataReg = lir->data()->toRegister().gpr();
10005 const Register SuspenderDataReg = ABINonArgReg3;
10006
10007# ifdef JS_CODEGEN_ARM64
10008 vixl::UseScratchRegisterScope temps(&masm);
10009 const Register ScratchReg1 = temps.AcquireX().asUnsized();
10010# elif defined(JS_CODEGEN_X86)
10011 const Register ScratchReg1 = ABINonArgReg3;
10012# elif defined(JS_CODEGEN_X641)
10013 const Register ScratchReg1 = ScratchReg;
10014# elif defined(JS_CODEGEN_ARM)
10015 const Register ScratchReg1 = ABINonArgReturnVolatileReg;
10016# else
10017# error "NYI: scratch register"
10018# endif
10019
10020 masm.Push(SuspenderReg);
10021 masm.Push(FnReg);
10022 masm.Push(DataReg);
10023
10024 callWasmUpdateSuspenderState(wasm::UpdateSuspenderStateAction::Suspend,
10025 SuspenderReg, ScratchReg1);
10026
10027 masm.Pop(DataReg);
10028 masm.Pop(FnReg);
10029 masm.Pop(SuspenderReg);
10030
10031 masm.Push(SuspenderReg);
10032 int32_t framePushedAtSuspender = masm.framePushed();
10033 masm.Push(InstanceReg);
10034
10035 wasm::CallSiteDesc desc(wasm::CallSiteDesc::Kind::StackSwitch);
10036 CodeLabel returnCallsite;
10037
10038 // Aligning stack before trampoline call.
10039 uint32_t reserve = ComputeByteAlignment(
10040 masm.framePushed() - sizeof(wasm::Frame), WasmStackAlignment);
10041 masm.reserveStack(reserve);
10042
10043 masm.loadPrivate(Address(SuspenderReg, NativeObject::getFixedSlotOffset(
10044 wasm::SuspenderObjectDataSlot)),
10045 SuspenderDataReg);
10046
10047 // Switch stacks to main.
10048 masm.storeStackPtr(Address(
10049 SuspenderDataReg, wasm::SuspenderObjectData::offsetOfSuspendableSP()));
10050 masm.storePtr(FramePointer,
10051 Address(SuspenderDataReg,
10052 wasm::SuspenderObjectData::offsetOfSuspendableFP()));
10053
10054 masm.loadStackPtr(
10055 Address(SuspenderDataReg, wasm::SuspenderObjectData::offsetOfMainSP()));
10056 masm.loadPtr(
10057 Address(SuspenderDataReg, wasm::SuspenderObjectData::offsetOfMainFP()),
10058 FramePointer);
10059
10060 // Set main_ra field to returnCallsite.
10061# ifdef JS_CODEGEN_X86
10062 // SuspenderDataReg is also ScratchReg1, use DataReg as a scratch register.
10063 MOZ_ASSERT(ScratchReg1 == SuspenderDataReg)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ScratchReg1 == SuspenderDataReg)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(ScratchReg1 == SuspenderDataReg
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"ScratchReg1 == SuspenderDataReg", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 10063); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ScratchReg1 == SuspenderDataReg"
")"); do { *((volatile int*)__null) = 10063; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
10064 masm.push(DataReg);
10065 masm.mov(&returnCallsite, DataReg);
10066 masm.storePtr(
10067 DataReg,
10068 Address(SuspenderDataReg,
10069 wasm::SuspenderObjectData::offsetOfSuspendedReturnAddress()));
10070 masm.pop(DataReg);
10071# else
10072 MOZ_ASSERT(ScratchReg1 != SuspenderDataReg)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ScratchReg1 != SuspenderDataReg)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(ScratchReg1 != SuspenderDataReg
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"ScratchReg1 != SuspenderDataReg", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 10072); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ScratchReg1 != SuspenderDataReg"
")"); do { *((volatile int*)__null) = 10072; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
10073 masm.mov(&returnCallsite, ScratchReg1);
10074 masm.storePtr(
10075 ScratchReg1,
10076 Address(SuspenderDataReg,
10077 wasm::SuspenderObjectData::offsetOfSuspendedReturnAddress()));
10078# endif
10079
10080 masm.assertStackAlignment(WasmStackAlignment);
10081
10082 // The FramePointer is pointing to the same
10083 // place as before switch happened.
10084 uint32_t framePushed = masm.framePushed();
10085
10086 // On different stack, reset framePushed. FramePointer is not valid here.
10087 masm.setFramePushed(0);
10088
10089 prepareWasmStackSwitchTrampolineCall(SuspenderReg, DataReg);
10090
10091 // Get wasm instance pointer for callee.
10092 size_t instanceSlotOffset = FunctionExtended::offsetOfExtendedSlot(
10093 FunctionExtended::WASM_INSTANCE_SLOT);
10094 masm.loadPtr(Address(FnReg, instanceSlotOffset), InstanceReg);
10095
10096 masm.storePtr(InstanceReg, Address(masm.getStackPointer(),
10097 WasmCalleeInstanceOffsetBeforeCall));
10098 masm.loadWasmPinnedRegsFromInstance();
10099
10100 masm.assertStackAlignment(WasmStackAlignment);
10101
10102 const Register ReturnAddressReg = ScratchReg1;
10103 // DataReg is not needed anymore, using it as a scratch register.
10104 const Register ScratchReg2 = DataReg;
10105
10106 // Load InstanceReg from suspendable stack exit frame.
10107 masm.loadPtr(Address(SuspenderDataReg,
10108 wasm::SuspenderObjectData::offsetOfSuspendableExitFP()),
10109 ScratchReg2);
10110 masm.loadPtr(
10111 Address(ScratchReg2, wasm::FrameWithInstances::callerInstanceOffset()),
10112 ScratchReg2);
10113 masm.storePtr(ScratchReg2, Address(masm.getStackPointer(),
10114 WasmCallerInstanceOffsetBeforeCall));
10115
10116 // Load RA from suspendable stack exit frame.
10117 masm.loadPtr(Address(SuspenderDataReg,
10118 wasm::SuspenderObjectData::offsetOfSuspendableExitFP()),
10119 ScratchReg1);
10120 masm.loadPtr(Address(ScratchReg1, wasm::Frame::returnAddressOffset()),
10121 ReturnAddressReg);
10122
10123 // Call wasm function fast.
10124# ifdef JS_USE_LINK_REGISTER
10125 masm.mov(ReturnAddressReg, lr);
10126# else
10127 masm.Push(ReturnAddressReg);
10128# endif
10129 // Get funcUncheckedCallEntry() from the function's
10130 // WASM_FUNC_UNCHECKED_ENTRY_SLOT extended slot.
10131 size_t uncheckedEntrySlotOffset = FunctionExtended::offsetOfExtendedSlot(
10132 FunctionExtended::WASM_FUNC_UNCHECKED_ENTRY_SLOT);
10133 masm.loadPtr(Address(FnReg, uncheckedEntrySlotOffset), ScratchReg2);
10134 masm.jump(ScratchReg2);
10135
10136 // About to use valid FramePointer -- restore framePushed.
10137 masm.setFramePushed(framePushed);
10138
10139 // For IsPlausibleStackMapKey check for the following callsite.
10140 masm.wasmTrapInstruction();
10141
10142 // Callsite for return from suspendable stack.
10143 masm.bind(&returnCallsite);
10144 masm.append(desc, *returnCallsite.target());
10145 masm.addCodeLabel(returnCallsite);
10146
10147 masm.assertStackAlignment(WasmStackAlignment);
10148
10149 markSafepointAt(returnCallsite.target()->offset(), lir);
10150 lir->safepoint()->setFramePushedAtStackMapBase(framePushed);
10151 lir->safepoint()->setWasmSafepointKind(WasmSafepointKind::StackSwitch);
10152 // Rooting SuspenderReg.
10153 masm.propagateOOM(
10154 lir->safepoint()->addWasmAnyRefSlot(true, framePushedAtSuspender));
10155
10156 masm.freeStackTo(framePushed);
10157
10158 masm.freeStack(reserve);
10159 masm.Pop(InstanceReg);
10160 masm.Pop(SuspenderReg);
10161
10162 masm.switchToWasmInstanceRealm(ScratchReg1, ScratchReg2);
10163
10164 callWasmUpdateSuspenderState(wasm::UpdateSuspenderStateAction::Resume,
10165 SuspenderReg, ScratchReg1);
10166#else
10167 MOZ_CRASH("NYI")do { do { } while (false); MOZ_ReportCrash("" "NYI", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 10167); AnnotateMozCrashReason("MOZ_CRASH(" "NYI" ")"); do {
*((volatile int*)__null) = 10167; __attribute__((nomerge)) ::
abort(); } while (false); } while (false)
;
10168#endif // ENABLE_WASM_JSPI
10169}
10170
10171void CodeGenerator::visitWasmStackContinueOnSuspendable(
10172 LWasmStackContinueOnSuspendable* lir) {
10173#ifdef ENABLE_WASM_JSPI1
10174 const Register SuspenderReg = lir->suspender()->toRegister().gpr();
10175 const Register SuspenderDataReg = ABINonArgReg3;
10176
10177# ifdef JS_CODEGEN_ARM64
10178 vixl::UseScratchRegisterScope temps(&masm);
10179 const Register ScratchReg1 = temps.AcquireX().asUnsized();
10180# elif defined(JS_CODEGEN_X86)
10181 const Register ScratchReg1 = ABINonArgReg2;
10182# elif defined(JS_CODEGEN_X641)
10183 const Register ScratchReg1 = ScratchReg;
10184# elif defined(JS_CODEGEN_ARM)
10185 const Register ScratchReg1 = ABINonArgReturnVolatileReg;
10186# else
10187# error "NYI: scratch register"
10188# endif
10189 const Register ScratchReg2 = ABINonArgReg1;
10190
10191 masm.Push(SuspenderReg);
10192 int32_t framePushedAtSuspender = masm.framePushed();
10193 masm.Push(InstanceReg);
10194
10195 wasm::CallSiteDesc desc(wasm::CallSiteDesc::Kind::StackSwitch);
10196 CodeLabel returnCallsite;
10197
10198 // Aligning stack before trampoline call.
10199 uint32_t reserve = ComputeByteAlignment(
10200 masm.framePushed() - sizeof(wasm::Frame), WasmStackAlignment);
10201 masm.reserveStack(reserve);
10202
10203 masm.loadPrivate(Address(SuspenderReg, NativeObject::getFixedSlotOffset(
10204 wasm::SuspenderObjectDataSlot)),
10205 SuspenderDataReg);
10206 masm.storeStackPtr(
10207 Address(SuspenderDataReg, wasm::SuspenderObjectData::offsetOfMainSP()));
10208 masm.storePtr(
10209 FramePointer,
10210 Address(SuspenderDataReg, wasm::SuspenderObjectData::offsetOfMainFP()));
10211
10212 // Adjust exit frame FP.
10213 masm.loadPtr(Address(SuspenderDataReg,
10214 wasm::SuspenderObjectData::offsetOfSuspendableExitFP()),
10215 ScratchReg1);
10216 masm.storePtr(FramePointer,
10217 Address(ScratchReg1, wasm::Frame::callerFPOffset()));
10218
10219 // Adjust exit frame RA.
10220 masm.mov(&returnCallsite, ScratchReg2);
10221
10222 masm.storePtr(ScratchReg2,
10223 Address(ScratchReg1, wasm::Frame::returnAddressOffset()));
10224 // Adjust exit frame caller instance slot.
10225 masm.storePtr(
10226 InstanceReg,
10227 Address(ScratchReg1, wasm::FrameWithInstances::callerInstanceOffset()));
10228
10229 // Switch stacks to suspendable.
10230 masm.loadStackPtr(Address(
10231 SuspenderDataReg, wasm::SuspenderObjectData::offsetOfSuspendableSP()));
10232 masm.loadPtr(Address(SuspenderDataReg,
10233 wasm::SuspenderObjectData::offsetOfSuspendableFP()),
10234 FramePointer);
10235
10236 masm.assertStackAlignment(WasmStackAlignment);
10237
10238 // The FramePointer is pointing to the same
10239 // place as before switch happened.
10240 uint32_t framePushed = masm.framePushed();
10241
10242 // On different stack, reset framePushed. FramePointer is not valid here.
10243 masm.setFramePushed(0);
10244
10245 // Restore shadow stack area and instance slots.
10246 WasmABIArgGenerator abi;
10247 unsigned reserveBeforeCall = abi.stackBytesConsumedSoFar();
10248 MOZ_ASSERT(masm.framePushed() == 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(masm.framePushed() == 0)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(masm.framePushed() == 0))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("masm.framePushed() == 0"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 10248); AnnotateMozCrashReason("MOZ_ASSERT" "(" "masm.framePushed() == 0"
")"); do { *((volatile int*)__null) = 10248; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
10249 unsigned argDecrement =
10250 StackDecrementForCall(WasmStackAlignment, 0, reserveBeforeCall);
10251 masm.reserveStack(argDecrement);
10252
10253 masm.storePtr(InstanceReg, Address(masm.getStackPointer(),
10254 WasmCallerInstanceOffsetBeforeCall));
10255 masm.storePtr(InstanceReg, Address(masm.getStackPointer(),
10256 WasmCalleeInstanceOffsetBeforeCall));
10257
10258 masm.assertStackAlignment(WasmStackAlignment);
10259
10260 const Register ReturnAddressReg = ScratchReg1;
10261
10262 // Pretend we just returned from the function.
10263 masm.loadPtr(
10264 Address(SuspenderDataReg,
10265 wasm::SuspenderObjectData::offsetOfSuspendedReturnAddress()),
10266 ReturnAddressReg);
10267 masm.jump(ReturnAddressReg);
10268
10269 // About to use valid FramePointer -- restore framePushed.
10270 masm.setFramePushed(framePushed);
10271
10272 // For IsPlausibleStackMapKey check for the following callsite.
10273 masm.wasmTrapInstruction();
10274
10275 // Callsite for return from suspendable stack.
10276 masm.bind(&returnCallsite);
10277 masm.append(desc, *returnCallsite.target());
10278 masm.addCodeLabel(returnCallsite);
10279
10280 masm.assertStackAlignment(WasmStackAlignment);
10281
10282 markSafepointAt(returnCallsite.target()->offset(), lir);
10283 lir->safepoint()->setFramePushedAtStackMapBase(framePushed);
10284 lir->safepoint()->setWasmSafepointKind(WasmSafepointKind::StackSwitch);
10285 // Rooting SuspenderReg.
10286 masm.propagateOOM(
10287 lir->safepoint()->addWasmAnyRefSlot(true, framePushedAtSuspender));
10288
10289 masm.freeStackTo(framePushed);
10290
10291 masm.freeStack(reserve);
10292 masm.Pop(InstanceReg);
10293 masm.Pop(SuspenderReg);
10294
10295 // Using SuspenderDataReg and ABINonArgReg2 as temps.
10296 masm.switchToWasmInstanceRealm(SuspenderDataReg, ABINonArgReg2);
10297
10298 callWasmUpdateSuspenderState(wasm::UpdateSuspenderStateAction::Leave,
10299 SuspenderReg, ScratchReg1);
10300#else
10301 MOZ_CRASH("NYI")do { do { } while (false); MOZ_ReportCrash("" "NYI", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 10301); AnnotateMozCrashReason("MOZ_CRASH(" "NYI" ")"); do {
*((volatile int*)__null) = 10301; __attribute__((nomerge)) ::
abort(); } while (false); } while (false)
;
10302#endif // ENABLE_WASM_JSPI
10303}
10304
10305void CodeGenerator::visitWasmCallLandingPrePad(LWasmCallLandingPrePad* lir) {
10306 LBlock* block = lir->block();
10307 MWasmCallLandingPrePad* mir = lir->mir();
10308 MBasicBlock* mirBlock = mir->block();
10309 MBasicBlock* callMirBlock = mir->callBlock();
10310
10311 // This block must be the pre-pad successor of the call block. No blocks may
10312 // be inserted between us, such as for critical edge splitting.
10313 MOZ_RELEASE_ASSERT(mirBlock == callMirBlock->getSuccessor(do { static_assert( mozilla::detail::AssertionConditionType<
decltype(mirBlock == callMirBlock->getSuccessor( MWasmCallCatchable
::PrePadBranchIndex))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(mirBlock == callMirBlock->
getSuccessor( MWasmCallCatchable::PrePadBranchIndex)))), 0)))
{ do { } while (false); MOZ_ReportAssertionFailure("mirBlock == callMirBlock->getSuccessor( MWasmCallCatchable::PrePadBranchIndex)"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 10314); AnnotateMozCrashReason("MOZ_RELEASE_ASSERT" "(" "mirBlock == callMirBlock->getSuccessor( MWasmCallCatchable::PrePadBranchIndex)"
")"); do { *((volatile int*)__null) = 10314; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
10314 MWasmCallCatchable::PrePadBranchIndex))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(mirBlock == callMirBlock->getSuccessor( MWasmCallCatchable
::PrePadBranchIndex))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(mirBlock == callMirBlock->
getSuccessor( MWasmCallCatchable::PrePadBranchIndex)))), 0)))
{ do { } while (false); MOZ_ReportAssertionFailure("mirBlock == callMirBlock->getSuccessor( MWasmCallCatchable::PrePadBranchIndex)"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 10314); AnnotateMozCrashReason("MOZ_RELEASE_ASSERT" "(" "mirBlock == callMirBlock->getSuccessor( MWasmCallCatchable::PrePadBranchIndex)"
")"); do { *((volatile int*)__null) = 10314; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
10315
10316 // This instruction or a move group must be the first instruction in the
10317 // block. No other instructions may be inserted.
10318 MOZ_RELEASE_ASSERT(*block->begin() == lir || (block->begin()->isMoveGroup() &&do { static_assert( mozilla::detail::AssertionConditionType<
decltype(*block->begin() == lir || (block->begin()->
isMoveGroup() && *(++block->begin()) == lir))>::
isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(*block->begin() == lir || (block->begin()->
isMoveGroup() && *(++block->begin()) == lir)))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("*block->begin() == lir || (block->begin()->isMoveGroup() && *(++block->begin()) == lir)"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 10319); AnnotateMozCrashReason("MOZ_RELEASE_ASSERT" "(" "*block->begin() == lir || (block->begin()->isMoveGroup() && *(++block->begin()) == lir)"
")"); do { *((volatile int*)__null) = 10319; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
10319 *(++block->begin()) == lir))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(*block->begin() == lir || (block->begin()->
isMoveGroup() && *(++block->begin()) == lir))>::
isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(*block->begin() == lir || (block->begin()->
isMoveGroup() && *(++block->begin()) == lir)))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("*block->begin() == lir || (block->begin()->isMoveGroup() && *(++block->begin()) == lir)"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 10319); AnnotateMozCrashReason("MOZ_RELEASE_ASSERT" "(" "*block->begin() == lir || (block->begin()->isMoveGroup() && *(++block->begin()) == lir)"
")"); do { *((volatile int*)__null) = 10319; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
10320
10321 wasm::TryNoteVector& tryNotes = masm.tryNotes();
10322 wasm::TryNote& tryNote = tryNotes[mir->tryNoteIndex()];
10323 // Set the entry point for the call try note to be the beginning of this
10324 // block. The above assertions (and assertions in visitWasmCall) guarantee
10325 // that we are not skipping over instructions that should be executed.
10326 tryNote.setLandingPad(block->label()->offset(), masm.framePushed());
10327}
10328
10329void CodeGenerator::visitWasmCallIndirectAdjunctSafepoint(
10330 LWasmCallIndirectAdjunctSafepoint* lir) {
10331 markSafepointAt(lir->safepointLocation().offset(), lir);
10332 lir->safepoint()->setFramePushedAtStackMapBase(
10333 lir->framePushedAtStackMapBase());
10334}
10335
10336template <typename InstructionWithMaybeTrapSite>
10337void EmitSignalNullCheckTrapSite(MacroAssembler& masm,
10338 InstructionWithMaybeTrapSite* ins,
10339 FaultingCodeOffset fco,
10340 wasm::TrapMachineInsn tmi) {
10341 if (!ins->maybeTrap()) {
10342 return;
10343 }
10344 wasm::BytecodeOffset trapOffset(ins->maybeTrap()->offset);
10345 masm.append(wasm::Trap::NullPointerDereference,
10346 wasm::TrapSite(tmi, fco, trapOffset));
10347}
10348
10349template <typename InstructionWithMaybeTrapSite, class AddressOrBaseIndex>
10350void CodeGenerator::emitWasmValueLoad(InstructionWithMaybeTrapSite* ins,
10351 MIRType type, MWideningOp wideningOp,
10352 AddressOrBaseIndex addr,
10353 AnyRegister dst) {
10354 FaultingCodeOffset fco;
10355 switch (type) {
10356 case MIRType::Int32:
10357 switch (wideningOp) {
10358 case MWideningOp::None:
10359 fco = masm.load32(addr, dst.gpr());
10360 EmitSignalNullCheckTrapSite(masm, ins, fco,
10361 wasm::TrapMachineInsn::Load32);
10362 break;
10363 case MWideningOp::FromU16:
10364 fco = masm.load16ZeroExtend(addr, dst.gpr());
10365 EmitSignalNullCheckTrapSite(masm, ins, fco,
10366 wasm::TrapMachineInsn::Load16);
10367 break;
10368 case MWideningOp::FromS16:
10369 fco = masm.load16SignExtend(addr, dst.gpr());
10370 EmitSignalNullCheckTrapSite(masm, ins, fco,
10371 wasm::TrapMachineInsn::Load16);
10372 break;
10373 case MWideningOp::FromU8:
10374 fco = masm.load8ZeroExtend(addr, dst.gpr());
10375 EmitSignalNullCheckTrapSite(masm, ins, fco,
10376 wasm::TrapMachineInsn::Load8);
10377 break;
10378 case MWideningOp::FromS8:
10379 fco = masm.load8SignExtend(addr, dst.gpr());
10380 EmitSignalNullCheckTrapSite(masm, ins, fco,
10381 wasm::TrapMachineInsn::Load8);
10382 break;
10383 default:
10384 MOZ_CRASH("unexpected widening op in ::visitWasmLoadElement")do { do { } while (false); MOZ_ReportCrash("" "unexpected widening op in ::visitWasmLoadElement"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 10384); AnnotateMozCrashReason("MOZ_CRASH(" "unexpected widening op in ::visitWasmLoadElement"
")"); do { *((volatile int*)__null) = 10384; __attribute__((
nomerge)) ::abort(); } while (false); } while (false)
;
10385 }
10386 break;
10387 case MIRType::Float32:
10388 MOZ_ASSERT(wideningOp == MWideningOp::None)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(wideningOp == MWideningOp::None)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(wideningOp == MWideningOp::None
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"wideningOp == MWideningOp::None", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 10388); AnnotateMozCrashReason("MOZ_ASSERT" "(" "wideningOp == MWideningOp::None"
")"); do { *((volatile int*)__null) = 10388; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
10389 fco = masm.loadFloat32(addr, dst.fpu());
10390 EmitSignalNullCheckTrapSite(masm, ins, fco,
10391 wasm::TrapMachineInsn::Load32);
10392 break;
10393 case MIRType::Double:
10394 MOZ_ASSERT(wideningOp == MWideningOp::None)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(wideningOp == MWideningOp::None)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(wideningOp == MWideningOp::None
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"wideningOp == MWideningOp::None", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 10394); AnnotateMozCrashReason("MOZ_ASSERT" "(" "wideningOp == MWideningOp::None"
")"); do { *((volatile int*)__null) = 10394; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
10395 fco = masm.loadDouble(addr, dst.fpu());
10396 EmitSignalNullCheckTrapSite(masm, ins, fco,
10397 wasm::TrapMachineInsn::Load64);
10398 break;
10399 case MIRType::Pointer:
10400 case MIRType::WasmAnyRef:
10401 case MIRType::WasmArrayData:
10402 MOZ_ASSERT(wideningOp == MWideningOp::None)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(wideningOp == MWideningOp::None)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(wideningOp == MWideningOp::None
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"wideningOp == MWideningOp::None", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 10402); AnnotateMozCrashReason("MOZ_ASSERT" "(" "wideningOp == MWideningOp::None"
")"); do { *((volatile int*)__null) = 10402; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
10403 fco = masm.loadPtr(addr, dst.gpr());
10404 EmitSignalNullCheckTrapSite(masm, ins, fco,
10405 wasm::TrapMachineInsnForLoadWord());
10406 break;
10407 default:
10408 MOZ_CRASH("unexpected type in ::emitWasmValueLoad")do { do { } while (false); MOZ_ReportCrash("" "unexpected type in ::emitWasmValueLoad"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 10408); AnnotateMozCrashReason("MOZ_CRASH(" "unexpected type in ::emitWasmValueLoad"
")"); do { *((volatile int*)__null) = 10408; __attribute__((
nomerge)) ::abort(); } while (false); } while (false)
;
10409 }
10410}
10411
10412template <typename InstructionWithMaybeTrapSite, class AddressOrBaseIndex>
10413void CodeGenerator::emitWasmValueStore(InstructionWithMaybeTrapSite* ins,
10414 MIRType type, MNarrowingOp narrowingOp,
10415 AnyRegister src,
10416 AddressOrBaseIndex addr) {
10417 FaultingCodeOffset fco;
10418 switch (type) {
10419 case MIRType::Int32:
10420 switch (narrowingOp) {
10421 case MNarrowingOp::None:
10422 fco = masm.store32(src.gpr(), addr);
10423 EmitSignalNullCheckTrapSite(masm, ins, fco,
10424 wasm::TrapMachineInsn::Store32);
10425 break;
10426 case MNarrowingOp::To16:
10427 fco = masm.store16(src.gpr(), addr);
10428 EmitSignalNullCheckTrapSite(masm, ins, fco,
10429 wasm::TrapMachineInsn::Store16);
10430 break;
10431 case MNarrowingOp::To8:
10432 fco = masm.store8(src.gpr(), addr);
10433 EmitSignalNullCheckTrapSite(masm, ins, fco,
10434 wasm::TrapMachineInsn::Store8);
10435 break;
10436 default:
10437 MOZ_CRASH()do { do { } while (false); MOZ_ReportCrash("" , "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 10437); AnnotateMozCrashReason("MOZ_CRASH(" ")"); do { *((volatile
int*)__null) = 10437; __attribute__((nomerge)) ::abort(); } while
(false); } while (false)
;
10438 }
10439 break;
10440 case MIRType::Float32:
10441 fco = masm.storeFloat32(src.fpu(), addr);
10442 EmitSignalNullCheckTrapSite(masm, ins, fco,
10443 wasm::TrapMachineInsn::Store32);
10444 break;
10445 case MIRType::Double:
10446 fco = masm.storeDouble(src.fpu(), addr);
10447 EmitSignalNullCheckTrapSite(masm, ins, fco,
10448 wasm::TrapMachineInsn::Store64);
10449 break;
10450 case MIRType::Pointer:
10451 // This could be correct, but it would be a new usage, so check carefully.
10452 MOZ_CRASH("Unexpected type in ::emitWasmValueStore.")do { do { } while (false); MOZ_ReportCrash("" "Unexpected type in ::emitWasmValueStore."
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 10452); AnnotateMozCrashReason("MOZ_CRASH(" "Unexpected type in ::emitWasmValueStore."
")"); do { *((volatile int*)__null) = 10452; __attribute__((
nomerge)) ::abort(); } while (false); } while (false)
;
10453 case MIRType::WasmAnyRef:
10454 MOZ_CRASH("Bad type in ::emitWasmValueStore. Use LWasmStoreElementRef.")do { do { } while (false); MOZ_ReportCrash("" "Bad type in ::emitWasmValueStore. Use LWasmStoreElementRef."
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 10454); AnnotateMozCrashReason("MOZ_CRASH(" "Bad type in ::emitWasmValueStore. Use LWasmStoreElementRef."
")"); do { *((volatile int*)__null) = 10454; __attribute__((
nomerge)) ::abort(); } while (false); } while (false)
;
10455 default:
10456 MOZ_CRASH("unexpected type in ::emitWasmValueStore")do { do { } while (false); MOZ_ReportCrash("" "unexpected type in ::emitWasmValueStore"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 10456); AnnotateMozCrashReason("MOZ_CRASH(" "unexpected type in ::emitWasmValueStore"
")"); do { *((volatile int*)__null) = 10456; __attribute__((
nomerge)) ::abort(); } while (false); } while (false)
;
10457 }
10458}
10459
10460void CodeGenerator::visitWasmLoadSlot(LWasmLoadSlot* ins) {
10461 MIRType type = ins->type();
10462 MWideningOp wideningOp = ins->wideningOp();
10463 Register container = ToRegister(ins->containerRef());
10464 Address addr(container, ins->offset());
10465 AnyRegister dst = ToAnyRegister(ins->output());
10466
10467#ifdef ENABLE_WASM_SIMD1
10468 if (type == MIRType::Simd128) {
10469 MOZ_ASSERT(wideningOp == MWideningOp::None)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(wideningOp == MWideningOp::None)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(wideningOp == MWideningOp::None
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"wideningOp == MWideningOp::None", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 10469); AnnotateMozCrashReason("MOZ_ASSERT" "(" "wideningOp == MWideningOp::None"
")"); do { *((volatile int*)__null) = 10469; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
10470 FaultingCodeOffset fco = masm.loadUnalignedSimd128(addr, dst.fpu());
10471 EmitSignalNullCheckTrapSite(masm, ins, fco, wasm::TrapMachineInsn::Load128);
10472 return;
10473 }
10474#endif
10475 emitWasmValueLoad(ins, type, wideningOp, addr, dst);
10476}
10477
10478void CodeGenerator::visitWasmLoadElement(LWasmLoadElement* ins) {
10479 MIRType type = ins->type();
10480 MWideningOp wideningOp = ins->wideningOp();
10481 Scale scale = ins->scale();
10482 Register base = ToRegister(ins->base());
10483 Register index = ToRegister(ins->index());
10484 AnyRegister dst = ToAnyRegister(ins->output());
10485
10486#ifdef ENABLE_WASM_SIMD1
10487 if (type == MIRType::Simd128) {
10488 MOZ_ASSERT(wideningOp == MWideningOp::None)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(wideningOp == MWideningOp::None)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(wideningOp == MWideningOp::None
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"wideningOp == MWideningOp::None", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 10488); AnnotateMozCrashReason("MOZ_ASSERT" "(" "wideningOp == MWideningOp::None"
")"); do { *((volatile int*)__null) = 10488; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
10489 FaultingCodeOffset fco;
10490 Register temp = ToRegister(ins->temp0());
10491 masm.movePtr(index, temp);
10492 masm.lshiftPtr(Imm32(4), temp);
10493 fco = masm.loadUnalignedSimd128(BaseIndex(base, temp, Scale::TimesOne),
10494 dst.fpu());
10495 EmitSignalNullCheckTrapSite(masm, ins, fco, wasm::TrapMachineInsn::Load128);
10496 return;
10497 }
10498#endif
10499 emitWasmValueLoad(ins, type, wideningOp, BaseIndex(base, index, scale), dst);
10500}
10501
10502void CodeGenerator::visitWasmStoreSlot(LWasmStoreSlot* ins) {
10503 MIRType type = ins->type();
10504 MNarrowingOp narrowingOp = ins->narrowingOp();
10505 Register container = ToRegister(ins->containerRef());
10506 Address addr(container, ins->offset());
10507 AnyRegister src = ToAnyRegister(ins->value());
10508 if (type != MIRType::Int32) {
10509 MOZ_RELEASE_ASSERT(narrowingOp == MNarrowingOp::None)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(narrowingOp == MNarrowingOp::None)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(narrowingOp == MNarrowingOp::
None))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("narrowingOp == MNarrowingOp::None", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 10509); AnnotateMozCrashReason("MOZ_RELEASE_ASSERT" "(" "narrowingOp == MNarrowingOp::None"
")"); do { *((volatile int*)__null) = 10509; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
10510 }
10511
10512#ifdef ENABLE_WASM_SIMD1
10513 if (type == MIRType::Simd128) {
10514 FaultingCodeOffset fco = masm.storeUnalignedSimd128(src.fpu(), addr);
10515 EmitSignalNullCheckTrapSite(masm, ins, fco,
10516 wasm::TrapMachineInsn::Store128);
10517 return;
10518 }
10519#endif
10520 emitWasmValueStore(ins, type, narrowingOp, src, addr);
10521}
10522
10523void CodeGenerator::visitWasmStoreElement(LWasmStoreElement* ins) {
10524 MIRType type = ins->type();
10525 MNarrowingOp narrowingOp = ins->narrowingOp();
10526 Scale scale = ins->scale();
10527 Register base = ToRegister(ins->base());
10528 Register index = ToRegister(ins->index());
10529 AnyRegister src = ToAnyRegister(ins->value());
10530 if (type != MIRType::Int32) {
10531 MOZ_RELEASE_ASSERT(narrowingOp == MNarrowingOp::None)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(narrowingOp == MNarrowingOp::None)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(narrowingOp == MNarrowingOp::
None))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("narrowingOp == MNarrowingOp::None", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 10531); AnnotateMozCrashReason("MOZ_RELEASE_ASSERT" "(" "narrowingOp == MNarrowingOp::None"
")"); do { *((volatile int*)__null) = 10531; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
10532 }
10533
10534#ifdef ENABLE_WASM_SIMD1
10535 if (type == MIRType::Simd128) {
10536 Register temp = ToRegister(ins->temp0());
10537 masm.movePtr(index, temp);
10538 masm.lshiftPtr(Imm32(4), temp);
10539 FaultingCodeOffset fco = masm.storeUnalignedSimd128(
10540 src.fpu(), BaseIndex(base, temp, Scale::TimesOne));
10541 EmitSignalNullCheckTrapSite(masm, ins, fco,
10542 wasm::TrapMachineInsn::Store128);
10543 return;
10544 }
10545#endif
10546 emitWasmValueStore(ins, type, narrowingOp, src,
10547 BaseIndex(base, index, scale));
10548}
10549
10550void CodeGenerator::visitWasmLoadTableElement(LWasmLoadTableElement* ins) {
10551 Register elements = ToRegister(ins->elements());
10552 Register index = ToRegister(ins->index());
10553 Register output = ToRegister(ins->output());
10554 masm.loadPtr(BaseIndex(elements, index, ScalePointer), output);
10555}
10556
10557void CodeGenerator::visitWasmDerivedPointer(LWasmDerivedPointer* ins) {
10558 masm.movePtr(ToRegister(ins->base()), ToRegister(ins->output()));
10559 masm.addPtr(Imm32(int32_t(ins->offset())), ToRegister(ins->output()));
10560}
10561
10562void CodeGenerator::visitWasmDerivedIndexPointer(
10563 LWasmDerivedIndexPointer* ins) {
10564 Register base = ToRegister(ins->base());
10565 Register index = ToRegister(ins->index());
10566 Register output = ToRegister(ins->output());
10567 masm.computeEffectiveAddress(BaseIndex(base, index, ins->scale()), output);
10568}
10569
10570void CodeGenerator::visitWasmStoreRef(LWasmStoreRef* ins) {
10571 Register instance = ToRegister(ins->instance());
10572 Register valueBase = ToRegister(ins->valueBase());
10573 size_t offset = ins->offset();
10574 Register value = ToRegister(ins->value());
10575 Register temp = ToRegister(ins->temp0());
10576
10577 if (ins->preBarrierKind() == WasmPreBarrierKind::Normal) {
10578 Label skipPreBarrier;
10579 wasm::EmitWasmPreBarrierGuard(
10580 masm, instance, temp, Address(valueBase, offset), &skipPreBarrier,
10581 ins->maybeTrap() ? &ins->maybeTrap()->offset : nullptr);
10582 wasm::EmitWasmPreBarrierCallImmediate(masm, instance, temp, valueBase,
10583 offset);
10584 masm.bind(&skipPreBarrier);
10585 }
10586
10587 FaultingCodeOffset fco = masm.storePtr(value, Address(valueBase, offset));
10588 EmitSignalNullCheckTrapSite(masm, ins, fco,
10589 wasm::TrapMachineInsnForStoreWord());
10590 // The postbarrier is handled separately.
10591}
10592
10593void CodeGenerator::visitWasmStoreElementRef(LWasmStoreElementRef* ins) {
10594 Register instance = ToRegister(ins->instance());
10595 Register base = ToRegister(ins->base());
10596 Register index = ToRegister(ins->index());
10597 Register value = ToRegister(ins->value());
10598 Register temp0 = ToTempRegisterOrInvalid(ins->temp0());
10599 Register temp1 = ToTempRegisterOrInvalid(ins->temp1());
10600
10601 BaseIndex addr(base, index, ScalePointer);
10602
10603 if (ins->preBarrierKind() == WasmPreBarrierKind::Normal) {
10604 Label skipPreBarrier;
10605 wasm::EmitWasmPreBarrierGuard(
10606 masm, instance, temp0, addr, &skipPreBarrier,
10607 ins->maybeTrap() ? &ins->maybeTrap()->offset : nullptr);
10608 wasm::EmitWasmPreBarrierCallIndex(masm, instance, temp0, temp1, addr);
10609 masm.bind(&skipPreBarrier);
10610 }
10611
10612 FaultingCodeOffset fco = masm.storePtr(value, addr);
10613 EmitSignalNullCheckTrapSite(masm, ins, fco,
10614 wasm::TrapMachineInsnForStoreWord());
10615 // The postbarrier is handled separately.
10616}
10617
10618// Out-of-line path to update the store buffer for wasm references.
10619class OutOfLineWasmCallPostWriteBarrierImmediate
10620 : public OutOfLineCodeBase<CodeGenerator> {
10621 LInstruction* lir_;
10622 Register valueBase_;
10623 Register temp_;
10624 uint32_t valueOffset_;
10625
10626 public:
10627 OutOfLineWasmCallPostWriteBarrierImmediate(LInstruction* lir,
10628 Register valueBase, Register temp,
10629 uint32_t valueOffset)
10630 : lir_(lir),
10631 valueBase_(valueBase),
10632 temp_(temp),
10633 valueOffset_(valueOffset) {}
10634
10635 void accept(CodeGenerator* codegen) override {
10636 codegen->visitOutOfLineWasmCallPostWriteBarrierImmediate(this);
10637 }
10638
10639 LInstruction* lir() const { return lir_; }
10640 Register valueBase() const { return valueBase_; }
10641 Register temp() const { return temp_; }
10642 uint32_t valueOffset() const { return valueOffset_; }
10643};
10644
10645void CodeGenerator::visitOutOfLineWasmCallPostWriteBarrierImmediate(
10646 OutOfLineWasmCallPostWriteBarrierImmediate* ool) {
10647 saveLiveVolatile(ool->lir());
10648 masm.Push(InstanceReg);
10649 int32_t framePushedAfterInstance = masm.framePushed();
10650
10651 // Fold the value offset into the value base
10652 Register valueAddr = ool->valueBase();
10653 Register temp = ool->temp();
10654 masm.computeEffectiveAddress(Address(valueAddr, ool->valueOffset()), temp);
10655
10656 // Call Instance::postBarrier
10657 masm.setupWasmABICall();
10658 masm.passABIArg(InstanceReg);
10659 masm.passABIArg(temp);
10660 int32_t instanceOffset = masm.framePushed() - framePushedAfterInstance;
10661 masm.callWithABI(wasm::BytecodeOffset(0), wasm::SymbolicAddress::PostBarrier,
10662 mozilla::Some(instanceOffset), ABIType::General);
10663
10664 masm.Pop(InstanceReg);
10665 restoreLiveVolatile(ool->lir());
10666
10667 masm.jump(ool->rejoin());
10668}
10669
10670void CodeGenerator::visitWasmPostWriteBarrierImmediate(
10671 LWasmPostWriteBarrierImmediate* lir) {
10672 Register object = ToRegister(lir->object());
10673 Register value = ToRegister(lir->value());
10674 Register valueBase = ToRegister(lir->valueBase());
10675 Register temp = ToRegister(lir->temp0());
10676 MOZ_ASSERT(ToRegister(lir->instance()) == InstanceReg)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ToRegister(lir->instance()) == InstanceReg)>::
isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(ToRegister(lir->instance()) == InstanceReg))), 0)
)) { do { } while (false); MOZ_ReportAssertionFailure("ToRegister(lir->instance()) == InstanceReg"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 10676); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ToRegister(lir->instance()) == InstanceReg"
")"); do { *((volatile int*)__null) = 10676; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
10677 auto* ool = new (alloc()) OutOfLineWasmCallPostWriteBarrierImmediate(
10678 lir, valueBase, temp, lir->valueOffset());
10679 addOutOfLineCode(ool, lir->mir());
10680
10681 wasm::EmitWasmPostBarrierGuard(masm, mozilla::Some(object), temp, value,
10682 ool->rejoin());
10683 masm.jump(ool->entry());
10684 masm.bind(ool->rejoin());
10685}
10686
10687// Out-of-line path to update the store buffer for wasm references.
10688class OutOfLineWasmCallPostWriteBarrierIndex
10689 : public OutOfLineCodeBase<CodeGenerator> {
10690 LInstruction* lir_;
10691 Register valueBase_;
10692 Register index_;
10693 Register temp_;
10694 uint32_t elemSize_;
10695
10696 public:
10697 OutOfLineWasmCallPostWriteBarrierIndex(LInstruction* lir, Register valueBase,
10698 Register index, Register temp,
10699 uint32_t elemSize)
10700 : lir_(lir),
10701 valueBase_(valueBase),
10702 index_(index),
10703 temp_(temp),
10704 elemSize_(elemSize) {
10705 MOZ_ASSERT(elemSize == 1 || elemSize == 2 || elemSize == 4 ||do { static_assert( mozilla::detail::AssertionConditionType<
decltype(elemSize == 1 || elemSize == 2 || elemSize == 4 || elemSize
== 8 || elemSize == 16)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(elemSize == 1 || elemSize ==
2 || elemSize == 4 || elemSize == 8 || elemSize == 16))), 0)
)) { do { } while (false); MOZ_ReportAssertionFailure("elemSize == 1 || elemSize == 2 || elemSize == 4 || elemSize == 8 || elemSize == 16"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 10706); AnnotateMozCrashReason("MOZ_ASSERT" "(" "elemSize == 1 || elemSize == 2 || elemSize == 4 || elemSize == 8 || elemSize == 16"
")"); do { *((volatile int*)__null) = 10706; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
10706 elemSize == 8 || elemSize == 16)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(elemSize == 1 || elemSize == 2 || elemSize == 4 || elemSize
== 8 || elemSize == 16)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(elemSize == 1 || elemSize ==
2 || elemSize == 4 || elemSize == 8 || elemSize == 16))), 0)
)) { do { } while (false); MOZ_ReportAssertionFailure("elemSize == 1 || elemSize == 2 || elemSize == 4 || elemSize == 8 || elemSize == 16"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 10706); AnnotateMozCrashReason("MOZ_ASSERT" "(" "elemSize == 1 || elemSize == 2 || elemSize == 4 || elemSize == 8 || elemSize == 16"
")"); do { *((volatile int*)__null) = 10706; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
10707 }
10708
10709 void accept(CodeGenerator* codegen) override {
10710 codegen->visitOutOfLineWasmCallPostWriteBarrierIndex(this);
10711 }
10712
10713 LInstruction* lir() const { return lir_; }
10714 Register valueBase() const { return valueBase_; }
10715 Register index() const { return index_; }
10716 Register temp() const { return temp_; }
10717 uint32_t elemSize() const { return elemSize_; }
10718};
10719
10720void CodeGenerator::visitOutOfLineWasmCallPostWriteBarrierIndex(
10721 OutOfLineWasmCallPostWriteBarrierIndex* ool) {
10722 saveLiveVolatile(ool->lir());
10723 masm.Push(InstanceReg);
10724 int32_t framePushedAfterInstance = masm.framePushed();
10725
10726 // Fold the value offset into the value base
10727 Register temp = ool->temp();
10728 if (ool->elemSize() == 16) {
10729 masm.movePtr(ool->index(), temp);
10730 masm.lshiftPtr(Imm32(4), temp);
10731 masm.addPtr(ool->valueBase(), temp);
10732 } else {
10733 masm.computeEffectiveAddress(BaseIndex(ool->valueBase(), ool->index(),
10734 ScaleFromElemWidth(ool->elemSize())),
10735 temp);
10736 }
10737
10738 // Call Instance::postBarrier
10739 masm.setupWasmABICall();
10740 masm.passABIArg(InstanceReg);
10741 masm.passABIArg(temp);
10742 int32_t instanceOffset = masm.framePushed() - framePushedAfterInstance;
10743 masm.callWithABI(wasm::BytecodeOffset(0), wasm::SymbolicAddress::PostBarrier,
10744 mozilla::Some(instanceOffset), ABIType::General);
10745
10746 masm.Pop(InstanceReg);
10747 restoreLiveVolatile(ool->lir());
10748
10749 masm.jump(ool->rejoin());
10750}
10751
10752void CodeGenerator::visitWasmPostWriteBarrierIndex(
10753 LWasmPostWriteBarrierIndex* lir) {
10754 Register object = ToRegister(lir->object());
10755 Register value = ToRegister(lir->value());
10756 Register valueBase = ToRegister(lir->valueBase());
10757 Register index = ToRegister(lir->index());
10758 Register temp = ToRegister(lir->temp0());
10759 MOZ_ASSERT(ToRegister(lir->instance()) == InstanceReg)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ToRegister(lir->instance()) == InstanceReg)>::
isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(ToRegister(lir->instance()) == InstanceReg))), 0)
)) { do { } while (false); MOZ_ReportAssertionFailure("ToRegister(lir->instance()) == InstanceReg"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 10759); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ToRegister(lir->instance()) == InstanceReg"
")"); do { *((volatile int*)__null) = 10759; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
10760 auto* ool = new (alloc()) OutOfLineWasmCallPostWriteBarrierIndex(
10761 lir, valueBase, index, temp, lir->elemSize());
10762 addOutOfLineCode(ool, lir->mir());
10763
10764 wasm::EmitWasmPostBarrierGuard(masm, mozilla::Some(object), temp, value,
10765 ool->rejoin());
10766 masm.jump(ool->entry());
10767 masm.bind(ool->rejoin());
10768}
10769
10770void CodeGenerator::visitWasmLoadSlotI64(LWasmLoadSlotI64* ins) {
10771 Register container = ToRegister(ins->containerRef());
10772 Address addr(container, ins->offset());
10773 Register64 output = ToOutRegister64(ins);
10774 // Either 1 or 2 words. On a 32-bit target, it is hard to argue that one
10775 // transaction will always trap before the other, so it seems safest to
10776 // register both of them as potentially trapping.
10777#ifdef JS_64BIT1
10778 FaultingCodeOffset fco = masm.load64(addr, output);
10779 EmitSignalNullCheckTrapSite(masm, ins, fco, wasm::TrapMachineInsn::Load64);
10780#else
10781 FaultingCodeOffsetPair fcop = masm.load64(addr, output);
10782 EmitSignalNullCheckTrapSite(masm, ins, fcop.first,
10783 wasm::TrapMachineInsn::Load32);
10784 EmitSignalNullCheckTrapSite(masm, ins, fcop.second,
10785 wasm::TrapMachineInsn::Load32);
10786#endif
10787}
10788
10789void CodeGenerator::visitWasmLoadElementI64(LWasmLoadElementI64* ins) {
10790 Register base = ToRegister(ins->base());
10791 Register index = ToRegister(ins->index());
10792 BaseIndex addr(base, index, Scale::TimesEight);
10793 Register64 output = ToOutRegister64(ins);
10794 // Either 1 or 2 words. On a 32-bit target, it is hard to argue that one
10795 // transaction will always trap before the other, so it seems safest to
10796 // register both of them as potentially trapping.
10797#ifdef JS_64BIT1
10798 FaultingCodeOffset fco = masm.load64(addr, output);
10799 EmitSignalNullCheckTrapSite(masm, ins, fco, wasm::TrapMachineInsn::Load64);
10800#else
10801 FaultingCodeOffsetPair fcop = masm.load64(addr, output);
10802 EmitSignalNullCheckTrapSite(masm, ins, fcop.first,
10803 wasm::TrapMachineInsn::Load32);
10804 EmitSignalNullCheckTrapSite(masm, ins, fcop.second,
10805 wasm::TrapMachineInsn::Load32);
10806#endif
10807}
10808
10809void CodeGenerator::visitWasmStoreSlotI64(LWasmStoreSlotI64* ins) {
10810 Register container = ToRegister(ins->containerRef());
10811 Address addr(container, ins->offset());
10812 Register64 value = ToRegister64(ins->value());
10813 // Either 1 or 2 words. As above we register both transactions in the
10814 // 2-word case.
10815#ifdef JS_64BIT1
10816 FaultingCodeOffset fco = masm.store64(value, addr);
10817 EmitSignalNullCheckTrapSite(masm, ins, fco, wasm::TrapMachineInsn::Store64);
10818#else
10819 FaultingCodeOffsetPair fcop = masm.store64(value, addr);
10820 EmitSignalNullCheckTrapSite(masm, ins, fcop.first,
10821 wasm::TrapMachineInsn::Store32);
10822 EmitSignalNullCheckTrapSite(masm, ins, fcop.second,
10823 wasm::TrapMachineInsn::Store32);
10824#endif
10825}
10826
10827void CodeGenerator::visitWasmStoreElementI64(LWasmStoreElementI64* ins) {
10828 Register base = ToRegister(ins->base());
10829 Register index = ToRegister(ins->index());
10830 BaseIndex addr(base, index, Scale::TimesEight);
10831 Register64 value = ToRegister64(ins->value());
10832 // Either 1 or 2 words. As above we register both transactions in the
10833 // 2-word case.
10834#ifdef JS_64BIT1
10835 FaultingCodeOffset fco = masm.store64(value, addr);
10836 EmitSignalNullCheckTrapSite(masm, ins, fco, wasm::TrapMachineInsn::Store64);
10837#else
10838 FaultingCodeOffsetPair fcop = masm.store64(value, addr);
10839 EmitSignalNullCheckTrapSite(masm, ins, fcop.first,
10840 wasm::TrapMachineInsn::Store32);
10841 EmitSignalNullCheckTrapSite(masm, ins, fcop.second,
10842 wasm::TrapMachineInsn::Store32);
10843#endif
10844}
10845
10846void CodeGenerator::visitWasmClampTable64Index(LWasmClampTable64Index* lir) {
10847#ifdef ENABLE_WASM_MEMORY641
10848 Register64 index = ToRegister64(lir->index());
10849 Register out = ToRegister(lir->output());
10850 masm.wasmClampTable64Index(index, out);
10851#else
10852 MOZ_CRASH("table64 indexes should not be valid without memory64")do { do { } while (false); MOZ_ReportCrash("" "table64 indexes should not be valid without memory64"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 10852); AnnotateMozCrashReason("MOZ_CRASH(" "table64 indexes should not be valid without memory64"
")"); do { *((volatile int*)__null) = 10852; __attribute__((
nomerge)) ::abort(); } while (false); } while (false)
;
10853#endif
10854}
10855
10856void CodeGenerator::visitArrayBufferByteLength(LArrayBufferByteLength* lir) {
10857 Register obj = ToRegister(lir->object());
10858 Register out = ToRegister(lir->output());
10859 masm.loadArrayBufferByteLengthIntPtr(obj, out);
10860}
10861
10862void CodeGenerator::visitArrayBufferViewLength(LArrayBufferViewLength* lir) {
10863 Register obj = ToRegister(lir->object());
10864 Register out = ToRegister(lir->output());
10865 masm.loadArrayBufferViewLengthIntPtr(obj, out);
10866}
10867
10868void CodeGenerator::visitArrayBufferViewByteOffset(
10869 LArrayBufferViewByteOffset* lir) {
10870 Register obj = ToRegister(lir->object());
10871 Register out = ToRegister(lir->output());
10872 masm.loadArrayBufferViewByteOffsetIntPtr(obj, out);
10873}
10874
10875void CodeGenerator::visitArrayBufferViewElements(
10876 LArrayBufferViewElements* lir) {
10877 Register obj = ToRegister(lir->object());
10878 Register out = ToRegister(lir->output());
10879 masm.loadPtr(Address(obj, ArrayBufferViewObject::dataOffset()), out);
10880}
10881
10882void CodeGenerator::visitTypedArrayElementSize(LTypedArrayElementSize* lir) {
10883 Register obj = ToRegister(lir->object());
10884 Register out = ToRegister(lir->output());
10885
10886 masm.typedArrayElementSize(obj, out);
10887}
10888
10889void CodeGenerator::visitResizableTypedArrayByteOffsetMaybeOutOfBounds(
10890 LResizableTypedArrayByteOffsetMaybeOutOfBounds* lir) {
10891 Register obj = ToRegister(lir->object());
10892 Register out = ToRegister(lir->output());
10893 Register temp = ToRegister(lir->temp0());
10894
10895 masm.loadResizableTypedArrayByteOffsetMaybeOutOfBoundsIntPtr(obj, out, temp);
10896}
10897
10898void CodeGenerator::visitResizableTypedArrayLength(
10899 LResizableTypedArrayLength* lir) {
10900 Register obj = ToRegister(lir->object());
10901 Register out = ToRegister(lir->output());
10902 Register temp = ToRegister(lir->temp0());
10903
10904 masm.loadResizableTypedArrayLengthIntPtr(lir->synchronization(), obj, out,
10905 temp);
10906}
10907
10908void CodeGenerator::visitResizableDataViewByteLength(
10909 LResizableDataViewByteLength* lir) {
10910 Register obj = ToRegister(lir->object());
10911 Register out = ToRegister(lir->output());
10912 Register temp = ToRegister(lir->temp0());
10913
10914 masm.loadResizableDataViewByteLengthIntPtr(lir->synchronization(), obj, out,
10915 temp);
10916}
10917
10918void CodeGenerator::visitGrowableSharedArrayBufferByteLength(
10919 LGrowableSharedArrayBufferByteLength* lir) {
10920 Register obj = ToRegister(lir->object());
10921 Register out = ToRegister(lir->output());
10922
10923 // Explicit |byteLength| accesses are seq-consistent atomic loads.
10924 auto sync = Synchronization::Load();
10925
10926 masm.loadGrowableSharedArrayBufferByteLengthIntPtr(sync, obj, out);
10927}
10928
10929void CodeGenerator::visitGuardResizableArrayBufferViewInBounds(
10930 LGuardResizableArrayBufferViewInBounds* lir) {
10931 Register obj = ToRegister(lir->object());
10932 Register temp = ToRegister(lir->temp0());
10933
10934 Label bail;
10935 masm.branchIfResizableArrayBufferViewOutOfBounds(obj, temp, &bail);
10936 bailoutFrom(&bail, lir->snapshot());
10937}
10938
10939void CodeGenerator::visitGuardResizableArrayBufferViewInBoundsOrDetached(
10940 LGuardResizableArrayBufferViewInBoundsOrDetached* lir) {
10941 Register obj = ToRegister(lir->object());
10942 Register temp = ToRegister(lir->temp0());
10943
10944 Label done, bail;
10945 masm.branchIfResizableArrayBufferViewInBounds(obj, temp, &done);
10946 masm.branchIfHasAttachedArrayBuffer(obj, temp, &bail);
10947 masm.bind(&done);
10948 bailoutFrom(&bail, lir->snapshot());
10949}
10950
10951void CodeGenerator::visitGuardHasAttachedArrayBuffer(
10952 LGuardHasAttachedArrayBuffer* lir) {
10953 Register obj = ToRegister(lir->object());
10954 Register temp = ToRegister(lir->temp0());
10955
10956 Label bail;
10957 masm.branchIfHasDetachedArrayBuffer(obj, temp, &bail);
10958 bailoutFrom(&bail, lir->snapshot());
10959}
10960
10961class OutOfLineGuardNumberToIntPtrIndex
10962 : public OutOfLineCodeBase<CodeGenerator> {
10963 LGuardNumberToIntPtrIndex* lir_;
10964
10965 public:
10966 explicit OutOfLineGuardNumberToIntPtrIndex(LGuardNumberToIntPtrIndex* lir)
10967 : lir_(lir) {}
10968
10969 void accept(CodeGenerator* codegen) override {
10970 codegen->visitOutOfLineGuardNumberToIntPtrIndex(this);
10971 }
10972 LGuardNumberToIntPtrIndex* lir() const { return lir_; }
10973};
10974
10975void CodeGenerator::visitGuardNumberToIntPtrIndex(
10976 LGuardNumberToIntPtrIndex* lir) {
10977 FloatRegister input = ToFloatRegister(lir->input());
10978 Register output = ToRegister(lir->output());
10979
10980 if (!lir->mir()->supportOOB()) {
10981 Label bail;
10982 masm.convertDoubleToPtr(input, output, &bail, false);
10983 bailoutFrom(&bail, lir->snapshot());
10984 return;
10985 }
10986
10987 auto* ool = new (alloc()) OutOfLineGuardNumberToIntPtrIndex(lir);
10988 addOutOfLineCode(ool, lir->mir());
10989
10990 masm.convertDoubleToPtr(input, output, ool->entry(), false);
10991 masm.bind(ool->rejoin());
10992}
10993
10994void CodeGenerator::visitOutOfLineGuardNumberToIntPtrIndex(
10995 OutOfLineGuardNumberToIntPtrIndex* ool) {
10996 // Substitute the invalid index with an arbitrary out-of-bounds index.
10997 masm.movePtr(ImmWord(-1), ToRegister(ool->lir()->output()));
10998 masm.jump(ool->rejoin());
10999}
11000
11001void CodeGenerator::visitStringLength(LStringLength* lir) {
11002 Register input = ToRegister(lir->string());
11003 Register output = ToRegister(lir->output());
11004
11005 masm.loadStringLength(input, output);
11006}
11007
11008void CodeGenerator::visitMinMaxI(LMinMaxI* ins) {
11009 Register first = ToRegister(ins->first());
11010 Register output = ToRegister(ins->output());
11011
11012 MOZ_ASSERT(first == output)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(first == output)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(first == output))), 0))) { do
{ } while (false); MOZ_ReportAssertionFailure("first == output"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 11012); AnnotateMozCrashReason("MOZ_ASSERT" "(" "first == output"
")"); do { *((volatile int*)__null) = 11012; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
11013
11014 Assembler::Condition cond =
11015 ins->mir()->isMax() ? Assembler::GreaterThan : Assembler::LessThan;
11016
11017 if (ins->second()->isConstant()) {
11018 Label done;
11019 masm.branch32(cond, first, Imm32(ToInt32(ins->second())), &done);
11020 masm.move32(Imm32(ToInt32(ins->second())), output);
11021 masm.bind(&done);
11022 } else {
11023 Register second = ToRegister(ins->second());
11024 masm.cmp32Move32(cond, second, first, second, output);
11025 }
11026}
11027
11028void CodeGenerator::visitMinMaxArrayI(LMinMaxArrayI* ins) {
11029 Register array = ToRegister(ins->array());
11030 Register output = ToRegister(ins->output());
11031 Register temp1 = ToRegister(ins->temp1());
11032 Register temp2 = ToRegister(ins->temp2());
11033 Register temp3 = ToRegister(ins->temp3());
11034 bool isMax = ins->isMax();
11035
11036 Label bail;
11037 masm.minMaxArrayInt32(array, output, temp1, temp2, temp3, isMax, &bail);
11038 bailoutFrom(&bail, ins->snapshot());
11039}
11040
11041void CodeGenerator::visitMinMaxArrayD(LMinMaxArrayD* ins) {
11042 Register array = ToRegister(ins->array());
11043 FloatRegister output = ToFloatRegister(ins->output());
11044 Register temp1 = ToRegister(ins->temp1());
11045 Register temp2 = ToRegister(ins->temp2());
11046 FloatRegister floatTemp = ToFloatRegister(ins->floatTemp());
11047 bool isMax = ins->isMax();
11048
11049 Label bail;
11050 masm.minMaxArrayNumber(array, output, floatTemp, temp1, temp2, isMax, &bail);
11051 bailoutFrom(&bail, ins->snapshot());
11052}
11053
11054// For Abs*, lowering will have tied input to output on platforms where that is
11055// sensible, and otherwise left them untied.
11056
11057void CodeGenerator::visitAbsI(LAbsI* ins) {
11058 Register input = ToRegister(ins->input());
11059 Register output = ToRegister(ins->output());
11060
11061 if (ins->mir()->fallible()) {
11062 Label positive;
11063 if (input != output) {
11064 masm.move32(input, output);
11065 }
11066 masm.branchTest32(Assembler::NotSigned, output, output, &positive);
11067 Label bail;
11068 masm.branchNeg32(Assembler::Overflow, output, &bail);
11069 bailoutFrom(&bail, ins->snapshot());
11070 masm.bind(&positive);
11071 } else {
11072 masm.abs32(input, output);
11073 }
11074}
11075
11076void CodeGenerator::visitAbsD(LAbsD* ins) {
11077 masm.absDouble(ToFloatRegister(ins->input()), ToFloatRegister(ins->output()));
11078}
11079
11080void CodeGenerator::visitAbsF(LAbsF* ins) {
11081 masm.absFloat32(ToFloatRegister(ins->input()),
11082 ToFloatRegister(ins->output()));
11083}
11084
11085void CodeGenerator::visitPowII(LPowII* ins) {
11086 Register value = ToRegister(ins->value());
11087 Register power = ToRegister(ins->power());
11088 Register output = ToRegister(ins->output());
11089 Register temp0 = ToRegister(ins->temp0());
11090 Register temp1 = ToRegister(ins->temp1());
11091
11092 Label bailout;
11093 masm.pow32(value, power, output, temp0, temp1, &bailout);
11094 bailoutFrom(&bailout, ins->snapshot());
11095}
11096
11097void CodeGenerator::visitPowI(LPowI* ins) {
11098 FloatRegister value = ToFloatRegister(ins->value());
11099 Register power = ToRegister(ins->power());
11100
11101 using Fn = double (*)(double x, int32_t y);
11102 masm.setupAlignedABICall();
11103 masm.passABIArg(value, ABIType::Float64);
11104 masm.passABIArg(power);
11105
11106 masm.callWithABI<Fn, js::powi>(ABIType::Float64);
11107 MOZ_ASSERT(ToFloatRegister(ins->output()) == ReturnDoubleReg)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ToFloatRegister(ins->output()) == ReturnDoubleReg
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(ToFloatRegister(ins->output()) == ReturnDoubleReg
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"ToFloatRegister(ins->output()) == ReturnDoubleReg", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 11107); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ToFloatRegister(ins->output()) == ReturnDoubleReg"
")"); do { *((volatile int*)__null) = 11107; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
11108}
11109
11110void CodeGenerator::visitPowD(LPowD* ins) {
11111 FloatRegister value = ToFloatRegister(ins->value());
11112 FloatRegister power = ToFloatRegister(ins->power());
11113
11114 using Fn = double (*)(double x, double y);
11115 masm.setupAlignedABICall();
11116 masm.passABIArg(value, ABIType::Float64);
11117 masm.passABIArg(power, ABIType::Float64);
11118 masm.callWithABI<Fn, ecmaPow>(ABIType::Float64);
11119
11120 MOZ_ASSERT(ToFloatRegister(ins->output()) == ReturnDoubleReg)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ToFloatRegister(ins->output()) == ReturnDoubleReg
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(ToFloatRegister(ins->output()) == ReturnDoubleReg
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"ToFloatRegister(ins->output()) == ReturnDoubleReg", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 11120); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ToFloatRegister(ins->output()) == ReturnDoubleReg"
")"); do { *((volatile int*)__null) = 11120; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
11121}
11122
11123void CodeGenerator::visitPowOfTwoI(LPowOfTwoI* ins) {
11124 Register power = ToRegister(ins->power());
11125 Register output = ToRegister(ins->output());
11126
11127 uint32_t base = ins->base();
11128 MOZ_ASSERT(mozilla::IsPowerOfTwo(base))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(mozilla::IsPowerOfTwo(base))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(mozilla::IsPowerOfTwo(base))
)), 0))) { do { } while (false); MOZ_ReportAssertionFailure("mozilla::IsPowerOfTwo(base)"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 11128); AnnotateMozCrashReason("MOZ_ASSERT" "(" "mozilla::IsPowerOfTwo(base)"
")"); do { *((volatile int*)__null) = 11128; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
11129
11130 uint32_t n = mozilla::FloorLog2(base);
11131 MOZ_ASSERT(n != 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(n != 0)>::isValid, "invalid assertion condition")
; if ((__builtin_expect(!!(!(!!(n != 0))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("n != 0", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 11131); AnnotateMozCrashReason("MOZ_ASSERT" "(" "n != 0" ")"
); do { *((volatile int*)__null) = 11131; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
11132
11133 // Hacker's Delight, 2nd edition, theorem D2.
11134 auto ceilingDiv = [](uint32_t x, uint32_t y) { return (x + y - 1) / y; };
11135
11136 // Take bailout if |power| is greater-or-equals |log_y(2^31)| or is negative.
11137 // |2^(n*y) < 2^31| must hold, hence |n*y < 31| resp. |y < 31/n|.
11138 //
11139 // Note: it's important for this condition to match the code in CacheIR.cpp
11140 // (CanAttachInt32Pow) to prevent failure loops.
11141 bailoutCmp32(Assembler::AboveOrEqual, power, Imm32(ceilingDiv(31, n)),
11142 ins->snapshot());
11143
11144 // Compute (2^n)^y as 2^(n*y) using repeated shifts. We could directly scale
11145 // |power| and perform a single shift, but due to the lack of necessary
11146 // MacroAssembler functionality, like multiplying a register with an
11147 // immediate, we restrict the number of generated shift instructions when
11148 // lowering this operation.
11149 masm.move32(Imm32(1), output);
11150 do {
11151 masm.lshift32(power, output);
11152 n--;
11153 } while (n > 0);
11154}
11155
11156void CodeGenerator::visitSqrtD(LSqrtD* ins) {
11157 FloatRegister input = ToFloatRegister(ins->input());
11158 FloatRegister output = ToFloatRegister(ins->output());
11159 masm.sqrtDouble(input, output);
11160}
11161
11162void CodeGenerator::visitSqrtF(LSqrtF* ins) {
11163 FloatRegister input = ToFloatRegister(ins->input());
11164 FloatRegister output = ToFloatRegister(ins->output());
11165 masm.sqrtFloat32(input, output);
11166}
11167
11168void CodeGenerator::visitSignI(LSignI* ins) {
11169 Register input = ToRegister(ins->input());
11170 Register output = ToRegister(ins->output());
11171 masm.signInt32(input, output);
11172}
11173
11174void CodeGenerator::visitSignD(LSignD* ins) {
11175 FloatRegister input = ToFloatRegister(ins->input());
11176 FloatRegister output = ToFloatRegister(ins->output());
11177 masm.signDouble(input, output);
11178}
11179
11180void CodeGenerator::visitSignDI(LSignDI* ins) {
11181 FloatRegister input = ToFloatRegister(ins->input());
11182 FloatRegister temp = ToFloatRegister(ins->temp0());
11183 Register output = ToRegister(ins->output());
11184
11185 Label bail;
11186 masm.signDoubleToInt32(input, output, temp, &bail);
11187 bailoutFrom(&bail, ins->snapshot());
11188}
11189
11190void CodeGenerator::visitMathFunctionD(LMathFunctionD* ins) {
11191 FloatRegister input = ToFloatRegister(ins->input());
11192 MOZ_ASSERT(ToFloatRegister(ins->output()) == ReturnDoubleReg)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ToFloatRegister(ins->output()) == ReturnDoubleReg
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(ToFloatRegister(ins->output()) == ReturnDoubleReg
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"ToFloatRegister(ins->output()) == ReturnDoubleReg", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 11192); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ToFloatRegister(ins->output()) == ReturnDoubleReg"
")"); do { *((volatile int*)__null) = 11192; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
11193
11194 UnaryMathFunction fun = ins->mir()->function();
11195 UnaryMathFunctionType funPtr = GetUnaryMathFunctionPtr(fun);
11196
11197 masm.setupAlignedABICall();
11198
11199 masm.passABIArg(input, ABIType::Float64);
11200 masm.callWithABI(DynamicFunction<UnaryMathFunctionType>(funPtr),
11201 ABIType::Float64);
11202}
11203
11204void CodeGenerator::visitMathFunctionF(LMathFunctionF* ins) {
11205 FloatRegister input = ToFloatRegister(ins->input());
11206 MOZ_ASSERT(ToFloatRegister(ins->output()) == ReturnFloat32Reg)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ToFloatRegister(ins->output()) == ReturnFloat32Reg
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(ToFloatRegister(ins->output()) == ReturnFloat32Reg
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"ToFloatRegister(ins->output()) == ReturnFloat32Reg", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 11206); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ToFloatRegister(ins->output()) == ReturnFloat32Reg"
")"); do { *((volatile int*)__null) = 11206; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
11207
11208 masm.setupAlignedABICall();
11209 masm.passABIArg(input, ABIType::Float32);
11210
11211 using Fn = float (*)(float x);
11212 Fn funptr = nullptr;
11213 CheckUnsafeCallWithABI check = CheckUnsafeCallWithABI::Check;
11214 switch (ins->mir()->function()) {
11215 case UnaryMathFunction::Floor:
11216 funptr = floorf;
11217 check = CheckUnsafeCallWithABI::DontCheckOther;
11218 break;
11219 case UnaryMathFunction::Round:
11220 funptr = math_roundf_impl;
11221 break;
11222 case UnaryMathFunction::Trunc:
11223 funptr = math_truncf_impl;
11224 break;
11225 case UnaryMathFunction::Ceil:
11226 funptr = ceilf;
11227 check = CheckUnsafeCallWithABI::DontCheckOther;
11228 break;
11229 default:
11230 MOZ_CRASH("Unknown or unsupported float32 math function")do { do { } while (false); MOZ_ReportCrash("" "Unknown or unsupported float32 math function"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 11230); AnnotateMozCrashReason("MOZ_CRASH(" "Unknown or unsupported float32 math function"
")"); do { *((volatile int*)__null) = 11230; __attribute__((
nomerge)) ::abort(); } while (false); } while (false)
;
11231 }
11232
11233 masm.callWithABI(DynamicFunction<Fn>(funptr), ABIType::Float32, check);
11234}
11235
11236void CodeGenerator::visitModD(LModD* ins) {
11237 MOZ_ASSERT(!gen->compilingWasm())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!gen->compilingWasm())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!gen->compilingWasm()))),
0))) { do { } while (false); MOZ_ReportAssertionFailure("!gen->compilingWasm()"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 11237); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!gen->compilingWasm()"
")"); do { *((volatile int*)__null) = 11237; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
11238
11239 FloatRegister lhs = ToFloatRegister(ins->lhs());
11240 FloatRegister rhs = ToFloatRegister(ins->rhs());
11241
11242 MOZ_ASSERT(ToFloatRegister(ins->output()) == ReturnDoubleReg)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ToFloatRegister(ins->output()) == ReturnDoubleReg
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(ToFloatRegister(ins->output()) == ReturnDoubleReg
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"ToFloatRegister(ins->output()) == ReturnDoubleReg", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 11242); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ToFloatRegister(ins->output()) == ReturnDoubleReg"
")"); do { *((volatile int*)__null) = 11242; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
11243
11244 using Fn = double (*)(double a, double b);
11245 masm.setupAlignedABICall();
11246 masm.passABIArg(lhs, ABIType::Float64);
11247 masm.passABIArg(rhs, ABIType::Float64);
11248 masm.callWithABI<Fn, NumberMod>(ABIType::Float64);
11249}
11250
11251void CodeGenerator::visitModPowTwoD(LModPowTwoD* ins) {
11252 FloatRegister lhs = ToFloatRegister(ins->lhs());
11253 uint32_t divisor = ins->divisor();
11254 MOZ_ASSERT(mozilla::IsPowerOfTwo(divisor))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(mozilla::IsPowerOfTwo(divisor))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(mozilla::IsPowerOfTwo(divisor
)))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("mozilla::IsPowerOfTwo(divisor)", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 11254); AnnotateMozCrashReason("MOZ_ASSERT" "(" "mozilla::IsPowerOfTwo(divisor)"
")"); do { *((volatile int*)__null) = 11254; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
11255
11256 FloatRegister output = ToFloatRegister(ins->output());
11257
11258 // Compute |n % d| using |copysign(n - (d * trunc(n / d)), n)|.
11259 //
11260 // This doesn't work if |d| isn't a power of two, because we may lose too much
11261 // precision. For example |Number.MAX_VALUE % 3 == 2|, but
11262 // |3 * trunc(Number.MAX_VALUE / 3) == Infinity|.
11263
11264 Label done;
11265 {
11266 ScratchDoubleScope scratch(masm);
11267
11268 // Subnormals can lead to performance degradation, which can make calling
11269 // |fmod| faster than this inline implementation. Work around this issue by
11270 // directly returning the input for any value in the interval ]-1, +1[.
11271 Label notSubnormal;
11272 masm.loadConstantDouble(1.0, scratch);
11273 masm.loadConstantDouble(-1.0, output);
11274 masm.branchDouble(Assembler::DoubleGreaterThanOrEqual, lhs, scratch,
11275 &notSubnormal);
11276 masm.branchDouble(Assembler::DoubleLessThanOrEqual, lhs, output,
11277 &notSubnormal);
11278
11279 masm.moveDouble(lhs, output);
11280 masm.jump(&done);
11281
11282 masm.bind(&notSubnormal);
11283
11284 if (divisor == 1) {
11285 // The pattern |n % 1 == 0| is used to detect integer numbers. We can skip
11286 // the multiplication by one in this case.
11287 masm.moveDouble(lhs, output);
11288 masm.nearbyIntDouble(RoundingMode::TowardsZero, output, scratch);
11289 masm.subDouble(scratch, output);
11290 } else {
11291 masm.loadConstantDouble(1.0 / double(divisor), scratch);
11292 masm.loadConstantDouble(double(divisor), output);
11293
11294 masm.mulDouble(lhs, scratch);
11295 masm.nearbyIntDouble(RoundingMode::TowardsZero, scratch, scratch);
11296 masm.mulDouble(output, scratch);
11297
11298 masm.moveDouble(lhs, output);
11299 masm.subDouble(scratch, output);
11300 }
11301 }
11302
11303 masm.copySignDouble(output, lhs, output);
11304 masm.bind(&done);
11305}
11306
11307void CodeGenerator::visitWasmBuiltinModD(LWasmBuiltinModD* ins) {
11308 masm.Push(InstanceReg);
11309 int32_t framePushedAfterInstance = masm.framePushed();
11310
11311 FloatRegister lhs = ToFloatRegister(ins->lhs());
11312 FloatRegister rhs = ToFloatRegister(ins->rhs());
11313
11314 MOZ_ASSERT(ToFloatRegister(ins->output()) == ReturnDoubleReg)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ToFloatRegister(ins->output()) == ReturnDoubleReg
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(ToFloatRegister(ins->output()) == ReturnDoubleReg
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"ToFloatRegister(ins->output()) == ReturnDoubleReg", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 11314); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ToFloatRegister(ins->output()) == ReturnDoubleReg"
")"); do { *((volatile int*)__null) = 11314; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
11315
11316 masm.setupWasmABICall();
11317 masm.passABIArg(lhs, ABIType::Float64);
11318 masm.passABIArg(rhs, ABIType::Float64);
11319
11320 int32_t instanceOffset = masm.framePushed() - framePushedAfterInstance;
11321 masm.callWithABI(ins->mir()->bytecodeOffset(), wasm::SymbolicAddress::ModD,
11322 mozilla::Some(instanceOffset), ABIType::Float64);
11323
11324 masm.Pop(InstanceReg);
11325}
11326
11327void CodeGenerator::visitClzI(LClzI* ins) {
11328 Register input = ToRegister(ins->input());
11329 Register output = ToRegister(ins->output());
11330 bool knownNotZero = ins->mir()->operandIsNeverZero();
11331
11332 masm.clz32(input, output, knownNotZero);
11333}
11334
11335void CodeGenerator::visitCtzI(LCtzI* ins) {
11336 Register input = ToRegister(ins->input());
11337 Register output = ToRegister(ins->output());
11338 bool knownNotZero = ins->mir()->operandIsNeverZero();
11339
11340 masm.ctz32(input, output, knownNotZero);
11341}
11342
11343void CodeGenerator::visitPopcntI(LPopcntI* ins) {
11344 Register input = ToRegister(ins->input());
11345 Register output = ToRegister(ins->output());
11346 Register temp = ToRegister(ins->temp0());
11347
11348 masm.popcnt32(input, output, temp);
11349}
11350
11351void CodeGenerator::visitClzI64(LClzI64* ins) {
11352 Register64 input = ToRegister64(ins->num());
11353 Register64 output = ToOutRegister64(ins);
11354
11355 masm.clz64(input, output);
11356}
11357
11358void CodeGenerator::visitCtzI64(LCtzI64* ins) {
11359 Register64 input = ToRegister64(ins->num());
11360 Register64 output = ToOutRegister64(ins);
11361
11362 masm.ctz64(input, output);
11363}
11364
11365void CodeGenerator::visitPopcntI64(LPopcntI64* ins) {
11366 Register64 input = ToRegister64(ins->num());
11367 Register64 output = ToOutRegister64(ins);
11368 Register temp = ToRegister(ins->temp0());
11369
11370 masm.popcnt64(input, output, temp);
11371}
11372
11373void CodeGenerator::visitBigIntAdd(LBigIntAdd* ins) {
11374 pushArg(ToRegister(ins->rhs()));
11375 pushArg(ToRegister(ins->lhs()));
11376
11377 using Fn = BigInt* (*)(JSContext*, HandleBigInt, HandleBigInt);
11378 callVM<Fn, BigInt::add>(ins);
11379}
11380
11381void CodeGenerator::visitBigIntSub(LBigIntSub* ins) {
11382 pushArg(ToRegister(ins->rhs()));
11383 pushArg(ToRegister(ins->lhs()));
11384
11385 using Fn = BigInt* (*)(JSContext*, HandleBigInt, HandleBigInt);
11386 callVM<Fn, BigInt::sub>(ins);
11387}
11388
11389void CodeGenerator::visitBigIntMul(LBigIntMul* ins) {
11390 pushArg(ToRegister(ins->rhs()));
11391 pushArg(ToRegister(ins->lhs()));
11392
11393 using Fn = BigInt* (*)(JSContext*, HandleBigInt, HandleBigInt);
11394 callVM<Fn, BigInt::mul>(ins);
11395}
11396
11397void CodeGenerator::visitBigIntDiv(LBigIntDiv* ins) {
11398 pushArg(ToRegister(ins->rhs()));
11399 pushArg(ToRegister(ins->lhs()));
11400
11401 using Fn = BigInt* (*)(JSContext*, HandleBigInt, HandleBigInt);
11402 callVM<Fn, BigInt::div>(ins);
11403}
11404
11405void CodeGenerator::visitBigIntMod(LBigIntMod* ins) {
11406 pushArg(ToRegister(ins->rhs()));
11407 pushArg(ToRegister(ins->lhs()));
11408
11409 using Fn = BigInt* (*)(JSContext*, HandleBigInt, HandleBigInt);
11410 callVM<Fn, BigInt::mod>(ins);
11411}
11412
11413void CodeGenerator::visitBigIntPow(LBigIntPow* ins) {
11414 pushArg(ToRegister(ins->rhs()));
11415 pushArg(ToRegister(ins->lhs()));
11416
11417 using Fn = BigInt* (*)(JSContext*, HandleBigInt, HandleBigInt);
11418 callVM<Fn, BigInt::pow>(ins);
11419}
11420
11421void CodeGenerator::visitBigIntBitAnd(LBigIntBitAnd* ins) {
11422 pushArg(ToRegister(ins->rhs()));
11423 pushArg(ToRegister(ins->lhs()));
11424
11425 using Fn = BigInt* (*)(JSContext*, HandleBigInt, HandleBigInt);
11426 callVM<Fn, BigInt::bitAnd>(ins);
11427}
11428
11429void CodeGenerator::visitBigIntBitOr(LBigIntBitOr* ins) {
11430 pushArg(ToRegister(ins->rhs()));
11431 pushArg(ToRegister(ins->lhs()));
11432
11433 using Fn = BigInt* (*)(JSContext*, HandleBigInt, HandleBigInt);
11434 callVM<Fn, BigInt::bitOr>(ins);
11435}
11436
11437void CodeGenerator::visitBigIntBitXor(LBigIntBitXor* ins) {
11438 pushArg(ToRegister(ins->rhs()));
11439 pushArg(ToRegister(ins->lhs()));
11440
11441 using Fn = BigInt* (*)(JSContext*, HandleBigInt, HandleBigInt);
11442 callVM<Fn, BigInt::bitXor>(ins);
11443}
11444
11445void CodeGenerator::visitBigIntLsh(LBigIntLsh* ins) {
11446 pushArg(ToRegister(ins->rhs()));
11447 pushArg(ToRegister(ins->lhs()));
11448
11449 using Fn = BigInt* (*)(JSContext*, HandleBigInt, HandleBigInt);
11450 callVM<Fn, BigInt::lsh>(ins);
11451}
11452
11453void CodeGenerator::visitBigIntRsh(LBigIntRsh* ins) {
11454 pushArg(ToRegister(ins->rhs()));
11455 pushArg(ToRegister(ins->lhs()));
11456
11457 using Fn = BigInt* (*)(JSContext*, HandleBigInt, HandleBigInt);
11458 callVM<Fn, BigInt::rsh>(ins);
11459}
11460
11461void CodeGenerator::visitBigIntIncrement(LBigIntIncrement* ins) {
11462 pushArg(ToRegister(ins->input()));
11463
11464 using Fn = BigInt* (*)(JSContext*, HandleBigInt);
11465 callVM<Fn, BigInt::inc>(ins);
11466}
11467
11468void CodeGenerator::visitBigIntDecrement(LBigIntDecrement* ins) {
11469 pushArg(ToRegister(ins->input()));
11470
11471 using Fn = BigInt* (*)(JSContext*, HandleBigInt);
11472 callVM<Fn, BigInt::dec>(ins);
11473}
11474
11475void CodeGenerator::visitBigIntNegate(LBigIntNegate* ins) {
11476 Register input = ToRegister(ins->input());
11477 Register temp = ToRegister(ins->temp0());
11478 Register output = ToRegister(ins->output());
11479
11480 using Fn = BigInt* (*)(JSContext*, HandleBigInt);
11481 auto* ool =
11482 oolCallVM<Fn, BigInt::neg>(ins, ArgList(input), StoreRegisterTo(output));
11483
11484 // -0n == 0n
11485 Label lhsNonZero;
11486 masm.branchIfBigIntIsNonZero(input, &lhsNonZero);
11487 masm.movePtr(input, output);
11488 masm.jump(ool->rejoin());
11489 masm.bind(&lhsNonZero);
11490
11491 // Call into the VM when the input uses heap digits.
11492 masm.copyBigIntWithInlineDigits(input, output, temp, initialBigIntHeap(),
11493 ool->entry());
11494
11495 // Flip the sign bit.
11496 masm.xor32(Imm32(BigInt::signBitMask()),
11497 Address(output, BigInt::offsetOfFlags()));
11498
11499 masm.bind(ool->rejoin());
11500}
11501
11502void CodeGenerator::visitBigIntBitNot(LBigIntBitNot* ins) {
11503 pushArg(ToRegister(ins->input()));
11504
11505 using Fn = BigInt* (*)(JSContext*, HandleBigInt);
11506 callVM<Fn, BigInt::bitNot>(ins);
11507}
11508
11509void CodeGenerator::visitBigIntToIntPtr(LBigIntToIntPtr* ins) {
11510 Register input = ToRegister(ins->input());
11511 Register output = ToRegister(ins->output());
11512
11513 Label bail;
11514 masm.loadBigIntPtr(input, output, &bail);
11515 bailoutFrom(&bail, ins->snapshot());
11516}
11517
11518void CodeGenerator::visitIntPtrToBigInt(LIntPtrToBigInt* ins) {
11519 Register input = ToRegister(ins->input());
11520 Register temp = ToRegister(ins->temp0());
11521 Register output = ToRegister(ins->output());
11522
11523 using Fn = BigInt* (*)(JSContext*, intptr_t);
11524 auto* ool = oolCallVM<Fn, JS::BigInt::createFromIntPtr>(
11525 ins, ArgList(input), StoreRegisterTo(output));
11526
11527 masm.newGCBigInt(output, temp, initialBigIntHeap(), ool->entry());
11528 masm.movePtr(input, temp);
11529 masm.initializeBigIntPtr(output, temp);
11530
11531 masm.bind(ool->rejoin());
11532}
11533
11534void CodeGenerator::visitBigIntPtrAdd(LBigIntPtrAdd* ins) {
11535 Register lhs = ToRegister(ins->lhs());
11536 const LAllocation* rhs = ins->rhs();
11537 Register output = ToRegister(ins->output());
11538
11539 if (rhs->isConstant()) {
11540 masm.movePtr(ImmWord(ToIntPtr(rhs)), output);
11541 } else {
11542 masm.movePtr(ToRegister(rhs), output);
11543 }
11544
11545 Label bail;
11546 masm.branchAddPtr(Assembler::Overflow, lhs, output, &bail);
11547 bailoutFrom(&bail, ins->snapshot());
11548}
11549
11550void CodeGenerator::visitBigIntPtrSub(LBigIntPtrSub* ins) {
11551 Register lhs = ToRegister(ins->lhs());
11552 Register rhs = ToRegister(ins->rhs());
11553 Register output = ToRegister(ins->output());
11554
11555 Label bail;
11556 masm.movePtr(lhs, output);
11557 masm.branchSubPtr(Assembler::Overflow, rhs, output, &bail);
11558 bailoutFrom(&bail, ins->snapshot());
11559}
11560
11561void CodeGenerator::visitBigIntPtrMul(LBigIntPtrMul* ins) {
11562 Register lhs = ToRegister(ins->lhs());
11563 const LAllocation* rhs = ins->rhs();
11564 Register output = ToRegister(ins->output());
11565
11566 if (rhs->isConstant()) {
11567 masm.movePtr(ImmWord(ToIntPtr(rhs)), output);
11568 } else {
11569 masm.movePtr(ToRegister(rhs), output);
11570 }
11571
11572 Label bail;
11573 masm.branchMulPtr(Assembler::Overflow, lhs, output, &bail);
11574 bailoutFrom(&bail, ins->snapshot());
11575}
11576
11577void CodeGenerator::visitBigIntPtrDiv(LBigIntPtrDiv* ins) {
11578 Register lhs = ToRegister(ins->lhs());
11579 Register rhs = ToRegister(ins->rhs());
11580 Register output = ToRegister(ins->output());
11581
11582 // x / 0 throws an error.
11583 Label bail;
11584 if (ins->mir()->canBeDivideByZero()) {
11585 masm.branchPtr(Assembler::Equal, rhs, Imm32(0), &bail);
11586 }
11587
11588 static constexpr auto DigitMin = std::numeric_limits<
11589 mozilla::SignedStdintTypeForSize<sizeof(BigInt::Digit)>::Type>::min();
11590
11591 // Handle an integer overflow from INT{32,64}_MIN / -1.
11592 Label notOverflow;
11593 masm.branchPtr(Assembler::NotEqual, lhs, ImmWord(DigitMin), &notOverflow);
11594 masm.branchPtr(Assembler::Equal, rhs, Imm32(-1), &bail);
11595 masm.bind(&notOverflow);
11596
11597 emitBigIntPtrDiv(ins, lhs, rhs, output);
11598
11599 bailoutFrom(&bail, ins->snapshot());
11600}
11601
11602void CodeGenerator::visitBigIntPtrDivPowTwo(LBigIntPtrDivPowTwo* ins) {
11603 Register lhs = ToRegister(ins->lhs());
11604 Register output = ToRegister(ins->output());
11605 int32_t shift = ins->shift();
11606 bool negativeDivisor = ins->negativeDivisor();
11607
11608 masm.movePtr(lhs, output);
11609
11610 if (shift) {
11611 // Adjust the value so that shifting produces a correctly rounded result
11612 // when the numerator is negative.
11613 // See 10-1 "Signed Division by a Known Power of 2" in Henry S. Warren,
11614 // Jr.'s Hacker's Delight.
11615
11616 constexpr size_t bits = BigInt::DigitBits;
11617
11618 if (shift > 1) {
11619 // Copy the sign bit of the numerator. (= (2^bits - 1) or 0)
11620 masm.rshiftPtrArithmetic(Imm32(bits - 1), output);
11621 }
11622
11623 // Divide by 2^(bits - shift)
11624 // i.e. (= (2^bits - 1) / 2^(bits - shift) or 0)
11625 // i.e. (= (2^shift - 1) or 0)
11626 masm.rshiftPtr(Imm32(bits - shift), output);
11627
11628 // If signed, make any 1 bit below the shifted bits to bubble up, such that
11629 // once shifted the value would be rounded towards 0.
11630 masm.addPtr(lhs, output);
11631
11632 masm.rshiftPtrArithmetic(Imm32(shift), output);
11633
11634 if (negativeDivisor) {
11635 masm.negPtr(output);
11636 }
11637 } else if (negativeDivisor) {
11638 Label bail;
11639 masm.branchNegPtr(Assembler::Overflow, output, &bail);
11640 bailoutFrom(&bail, ins->snapshot());
11641 }
11642}
11643
11644void CodeGenerator::visitBigIntPtrMod(LBigIntPtrMod* ins) {
11645 Register lhs = ToRegister(ins->lhs());
11646 Register rhs = ToRegister(ins->rhs());
11647 Register output = ToRegister(ins->output());
11648 Register temp = ToRegister(ins->temp0());
11649
11650 // x % 0 throws an error.
11651 if (ins->mir()->canBeDivideByZero()) {
11652 bailoutCmpPtr(Assembler::Equal, rhs, Imm32(0), ins->snapshot());
11653 }
11654
11655 static constexpr auto DigitMin = std::numeric_limits<
11656 mozilla::SignedStdintTypeForSize<sizeof(BigInt::Digit)>::Type>::min();
11657
11658 masm.movePtr(lhs, temp);
11659
11660 // Handle an integer overflow from INT{32,64}_MIN / -1.
11661 Label notOverflow;
11662 masm.branchPtr(Assembler::NotEqual, lhs, ImmWord(DigitMin), &notOverflow);
11663 masm.branchPtr(Assembler::NotEqual, rhs, Imm32(-1), &notOverflow);
11664 masm.movePtr(ImmWord(0), temp);
11665 masm.bind(&notOverflow);
11666
11667 emitBigIntPtrMod(ins, temp, rhs, output);
11668}
11669
11670void CodeGenerator::visitBigIntPtrModPowTwo(LBigIntPtrModPowTwo* ins) {
11671 Register lhs = ToRegister(ins->lhs());
11672 Register output = ToRegister(ins->output());
11673 Register temp = ToRegister(ins->temp0());
11674 int32_t shift = ins->shift();
11675
11676 masm.movePtr(lhs, output);
11677 masm.movePtr(ImmWord((uintptr_t(1) << shift) - uintptr_t(1)), temp);
11678
11679 // Switch based on sign of the lhs.
11680
11681 // Positive numbers are just a bitmask.
11682 Label negative;
11683 masm.branchTestPtr(Assembler::Signed, lhs, lhs, &negative);
11684
11685 masm.andPtr(temp, output);
11686
11687 Label done;
11688 masm.jump(&done);
11689
11690 // Negative numbers need a negate, bitmask, negate
11691 masm.bind(&negative);
11692
11693 masm.negPtr(output);
11694 masm.andPtr(temp, output);
11695 masm.negPtr(output);
11696
11697 masm.bind(&done);
11698}
11699
11700void CodeGenerator::visitBigIntPtrPow(LBigIntPtrPow* ins) {
11701 Register lhs = ToRegister(ins->lhs());
11702 Register rhs = ToRegister(ins->rhs());
11703 Register output = ToRegister(ins->output());
11704 Register temp0 = ToRegister(ins->temp0());
11705 Register temp1 = ToRegister(ins->temp1());
11706
11707 Label bail;
11708 masm.powPtr(lhs, rhs, output, temp0, temp1, &bail);
11709 bailoutFrom(&bail, ins->snapshot());
11710}
11711
11712void CodeGenerator::visitBigIntPtrBitAnd(LBigIntPtrBitAnd* ins) {
11713 Register lhs = ToRegister(ins->lhs());
11714 const LAllocation* rhs = ins->rhs();
11715 Register output = ToRegister(ins->output());
11716
11717 if (rhs->isConstant()) {
11718 masm.movePtr(ImmWord(ToIntPtr(rhs)), output);
11719 } else {
11720 masm.movePtr(ToRegister(rhs), output);
11721 }
11722 masm.andPtr(lhs, output);
11723}
11724
11725void CodeGenerator::visitBigIntPtrBitOr(LBigIntPtrBitOr* ins) {
11726 Register lhs = ToRegister(ins->lhs());
11727 const LAllocation* rhs = ins->rhs();
11728 Register output = ToRegister(ins->output());
11729
11730 if (rhs->isConstant()) {
11731 masm.movePtr(ImmWord(ToIntPtr(rhs)), output);
11732 } else {
11733 masm.movePtr(ToRegister(rhs), output);
11734 }
11735 masm.orPtr(lhs, output);
11736}
11737
11738void CodeGenerator::visitBigIntPtrBitXor(LBigIntPtrBitXor* ins) {
11739 Register lhs = ToRegister(ins->lhs());
11740 const LAllocation* rhs = ins->rhs();
11741 Register output = ToRegister(ins->output());
11742
11743 if (rhs->isConstant()) {
11744 masm.movePtr(ImmWord(ToIntPtr(rhs)), output);
11745 } else {
11746 masm.movePtr(ToRegister(rhs), output);
11747 }
11748 masm.xorPtr(lhs, output);
11749}
11750
11751void CodeGenerator::visitBigIntPtrLsh(LBigIntPtrLsh* ins) {
11752 Register lhs = ToRegister(ins->lhs());
11753 Register output = ToRegister(ins->output());
11754 Register temp = ToTempRegisterOrInvalid(ins->temp0());
11755 Register tempShift = ToTempRegisterOrInvalid(ins->temp1());
11756
11757 if (ins->rhs()->isConstant()) {
11758 intptr_t rhs = ToIntPtr(ins->rhs());
11759
11760 if (rhs >= intptr_t(BigInt::DigitBits)) {
11761 MOZ_ASSERT(ins->mir()->fallible())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ins->mir()->fallible())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(ins->mir()->fallible()
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"ins->mir()->fallible()", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 11761); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ins->mir()->fallible()"
")"); do { *((volatile int*)__null) = 11761; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
11762
11763 // x << DigitBits with x != 0n always exceeds pointer-sized storage.
11764 masm.movePtr(ImmWord(0), output);
11765 bailoutCmpPtr(Assembler::NotEqual, lhs, Imm32(0), ins->snapshot());
11766 } else if (rhs <= -intptr_t(BigInt::DigitBits)) {
11767 MOZ_ASSERT(!ins->mir()->fallible())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!ins->mir()->fallible())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!ins->mir()->fallible(
)))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("!ins->mir()->fallible()", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 11767); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!ins->mir()->fallible()"
")"); do { *((volatile int*)__null) = 11767; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
11768
11769 // x << -DigitBits == x >> DigitBits, which is either 0n or -1n.
11770 masm.movePtr(lhs, output);
11771 masm.rshiftPtrArithmetic(Imm32(BigInt::DigitBits - 1), output);
11772 } else if (rhs <= 0) {
11773 MOZ_ASSERT(!ins->mir()->fallible())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!ins->mir()->fallible())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!ins->mir()->fallible(
)))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("!ins->mir()->fallible()", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 11773); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!ins->mir()->fallible()"
")"); do { *((volatile int*)__null) = 11773; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
11774
11775 // |x << -y| is computed as |x >> y|.
11776 masm.movePtr(lhs, output);
11777 masm.rshiftPtrArithmetic(Imm32(-rhs), output);
11778 } else {
11779 MOZ_ASSERT(ins->mir()->fallible())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ins->mir()->fallible())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(ins->mir()->fallible()
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"ins->mir()->fallible()", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 11779); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ins->mir()->fallible()"
")"); do { *((volatile int*)__null) = 11779; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
11780
11781 masm.movePtr(lhs, output);
11782 masm.lshiftPtr(Imm32(rhs), output);
11783
11784 // Check for overflow: ((lhs << rhs) >> rhs) == lhs.
11785 masm.movePtr(output, temp);
11786 masm.rshiftPtrArithmetic(Imm32(rhs), temp);
11787 bailoutCmpPtr(Assembler::NotEqual, temp, lhs, ins->snapshot());
11788 }
11789 } else {
11790 Register rhs = ToRegister(ins->rhs());
11791
11792 Label done, bail;
11793 MOZ_ASSERT(ins->mir()->fallible())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ins->mir()->fallible())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(ins->mir()->fallible()
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"ins->mir()->fallible()", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 11793); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ins->mir()->fallible()"
")"); do { *((volatile int*)__null) = 11793; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
11794
11795 masm.movePtr(lhs, output);
11796
11797 // 0n << x == 0n
11798 masm.branchPtr(Assembler::Equal, lhs, Imm32(0), &done);
11799
11800 // x << DigitBits with x != 0n always exceeds pointer-sized storage.
11801 masm.branchPtr(Assembler::GreaterThanOrEqual, rhs, Imm32(BigInt::DigitBits),
11802 &bail);
11803
11804 // x << -DigitBits == x >> DigitBits, which is either 0n or -1n.
11805 Label shift;
11806 masm.branchPtr(Assembler::GreaterThan, rhs,
11807 Imm32(-int32_t(BigInt::DigitBits)), &shift);
11808 {
11809 masm.rshiftPtrArithmetic(Imm32(BigInt::DigitBits - 1), output);
11810 masm.jump(&done);
11811 }
11812 masm.bind(&shift);
11813
11814 // Move |rhs| into the designated shift register.
11815 masm.movePtr(rhs, tempShift);
11816
11817 // |x << -y| is computed as |x >> y|.
11818 Label leftShift;
11819 masm.branchPtr(Assembler::GreaterThanOrEqual, rhs, Imm32(0), &leftShift);
11820 {
11821 masm.negPtr(tempShift);
11822 masm.rshiftPtrArithmetic(tempShift, output);
11823 masm.jump(&done);
11824 }
11825 masm.bind(&leftShift);
11826
11827 masm.lshiftPtr(tempShift, output);
11828
11829 // Check for overflow: ((lhs << rhs) >> rhs) == lhs.
11830 masm.movePtr(output, temp);
11831 masm.rshiftPtrArithmetic(tempShift, temp);
11832 masm.branchPtr(Assembler::NotEqual, temp, lhs, &bail);
11833
11834 masm.bind(&done);
11835 bailoutFrom(&bail, ins->snapshot());
11836 }
11837}
11838
11839void CodeGenerator::visitBigIntPtrRsh(LBigIntPtrRsh* ins) {
11840 Register lhs = ToRegister(ins->lhs());
11841 Register output = ToRegister(ins->output());
11842 Register temp = ToTempRegisterOrInvalid(ins->temp0());
11843 Register tempShift = ToTempRegisterOrInvalid(ins->temp1());
11844
11845 if (ins->rhs()->isConstant()) {
11846 intptr_t rhs = ToIntPtr(ins->rhs());
11847
11848 if (rhs <= -intptr_t(BigInt::DigitBits)) {
11849 MOZ_ASSERT(ins->mir()->fallible())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ins->mir()->fallible())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(ins->mir()->fallible()
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"ins->mir()->fallible()", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 11849); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ins->mir()->fallible()"
")"); do { *((volatile int*)__null) = 11849; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
11850
11851 // x >> -DigitBits == x << DigitBits, which exceeds pointer-sized storage.
11852 masm.movePtr(ImmWord(0), output);
11853 bailoutCmpPtr(Assembler::NotEqual, lhs, Imm32(0), ins->snapshot());
11854 } else if (rhs >= intptr_t(BigInt::DigitBits)) {
11855 MOZ_ASSERT(!ins->mir()->fallible())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!ins->mir()->fallible())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!ins->mir()->fallible(
)))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("!ins->mir()->fallible()", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 11855); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!ins->mir()->fallible()"
")"); do { *((volatile int*)__null) = 11855; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
11856
11857 // x >> DigitBits is either 0n or -1n.
11858 masm.movePtr(lhs, output);
11859 masm.rshiftPtrArithmetic(Imm32(BigInt::DigitBits - 1), output);
11860 } else if (rhs < 0) {
11861 MOZ_ASSERT(ins->mir()->fallible())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ins->mir()->fallible())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(ins->mir()->fallible()
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"ins->mir()->fallible()", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 11861); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ins->mir()->fallible()"
")"); do { *((volatile int*)__null) = 11861; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
11862
11863 // |x >> -y| is computed as |x << y|.
11864 masm.movePtr(lhs, output);
11865 masm.lshiftPtr(Imm32(-rhs), output);
11866
11867 // Check for overflow: ((lhs << rhs) >> rhs) == lhs.
11868 masm.movePtr(output, temp);
11869 masm.rshiftPtrArithmetic(Imm32(-rhs), temp);
11870 bailoutCmpPtr(Assembler::NotEqual, temp, lhs, ins->snapshot());
11871 } else {
11872 MOZ_ASSERT(!ins->mir()->fallible())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!ins->mir()->fallible())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!ins->mir()->fallible(
)))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("!ins->mir()->fallible()", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 11872); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!ins->mir()->fallible()"
")"); do { *((volatile int*)__null) = 11872; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
11873
11874 masm.movePtr(lhs, output);
11875 masm.rshiftPtrArithmetic(Imm32(rhs), output);
11876 }
11877 } else {
11878 Register rhs = ToRegister(ins->rhs());
11879
11880 Label done, bail;
11881 MOZ_ASSERT(ins->mir()->fallible())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ins->mir()->fallible())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(ins->mir()->fallible()
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"ins->mir()->fallible()", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 11881); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ins->mir()->fallible()"
")"); do { *((volatile int*)__null) = 11881; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
11882
11883 masm.movePtr(lhs, output);
11884
11885 // 0n >> x == 0n
11886 masm.branchPtr(Assembler::Equal, lhs, Imm32(0), &done);
11887
11888 // x >> -DigitBits == x << DigitBits, which exceeds pointer-sized storage.
11889 masm.branchPtr(Assembler::LessThanOrEqual, rhs,
11890 Imm32(-int32_t(BigInt::DigitBits)), &bail);
11891
11892 // x >> DigitBits is either 0n or -1n.
11893 Label shift;
11894 masm.branchPtr(Assembler::LessThan, rhs, Imm32(BigInt::DigitBits), &shift);
11895 {
11896 masm.rshiftPtrArithmetic(Imm32(BigInt::DigitBits - 1), output);
11897 masm.jump(&done);
11898 }
11899 masm.bind(&shift);
11900
11901 // Move |rhs| into the designated shift register.
11902 masm.movePtr(rhs, tempShift);
11903
11904 // |x >> -y| is computed as |x << y|.
11905 Label rightShift;
11906 masm.branchPtr(Assembler::GreaterThanOrEqual, rhs, Imm32(0), &rightShift);
11907 {
11908 masm.negPtr(tempShift);
11909 masm.lshiftPtr(tempShift, output);
11910
11911 // Check for overflow: ((lhs << rhs) >> rhs) == lhs.
11912 masm.movePtr(output, temp);
11913 masm.rshiftPtrArithmetic(tempShift, temp);
11914 masm.branchPtr(Assembler::NotEqual, temp, lhs, &bail);
11915
11916 masm.jump(&done);
11917 }
11918 masm.bind(&rightShift);
11919
11920 masm.rshiftPtrArithmetic(tempShift, output);
11921
11922 masm.bind(&done);
11923 bailoutFrom(&bail, ins->snapshot());
11924 }
11925}
11926
11927void CodeGenerator::visitBigIntPtrBitNot(LBigIntPtrBitNot* ins) {
11928 Register input = ToRegister(ins->input());
11929 Register output = ToRegister(ins->output());
11930
11931 masm.movePtr(input, output);
11932 masm.notPtr(output);
11933}
11934
11935void CodeGenerator::visitInt32ToStringWithBase(LInt32ToStringWithBase* lir) {
11936 Register input = ToRegister(lir->input());
11937 RegisterOrInt32 base = ToRegisterOrInt32(lir->base());
11938 Register output = ToRegister(lir->output());
11939 Register temp0 = ToRegister(lir->temp0());
11940 Register temp1 = ToRegister(lir->temp1());
11941
11942 bool lowerCase = lir->mir()->lowerCase();
11943
11944 using Fn = JSString* (*)(JSContext*, int32_t, int32_t, bool);
11945 if (base.is<Register>()) {
11946 auto* ool = oolCallVM<Fn, js::Int32ToStringWithBase>(
11947 lir, ArgList(input, base.as<Register>(), Imm32(lowerCase)),
11948 StoreRegisterTo(output));
11949
11950 LiveRegisterSet liveRegs = liveVolatileRegs(lir);
11951 masm.loadInt32ToStringWithBase(input, base.as<Register>(), output, temp0,
11952 temp1, gen->runtime->staticStrings(),
11953 liveRegs, lowerCase, ool->entry());
11954 masm.bind(ool->rejoin());
11955 } else {
11956 auto* ool = oolCallVM<Fn, js::Int32ToStringWithBase>(
11957 lir, ArgList(input, Imm32(base.as<int32_t>()), Imm32(lowerCase)),
11958 StoreRegisterTo(output));
11959
11960 masm.loadInt32ToStringWithBase(input, base.as<int32_t>(), output, temp0,
11961 temp1, gen->runtime->staticStrings(),
11962 lowerCase, ool->entry());
11963 masm.bind(ool->rejoin());
11964 }
11965}
11966
11967void CodeGenerator::visitNumberParseInt(LNumberParseInt* lir) {
11968 Register string = ToRegister(lir->string());
11969 Register radix = ToRegister(lir->radix());
11970 ValueOperand output = ToOutValue(lir);
11971 Register temp = ToRegister(lir->temp0());
11972
11973#ifdef DEBUG1
11974 Label ok;
11975 masm.branch32(Assembler::Equal, radix, Imm32(0), &ok);
11976 masm.branch32(Assembler::Equal, radix, Imm32(10), &ok);
11977 masm.assumeUnreachable("radix must be 0 or 10 for indexed value fast path");
11978 masm.bind(&ok);
11979#endif
11980
11981 // Use indexed value as fast path if possible.
11982 Label vmCall, done;
11983 masm.loadStringIndexValue(string, temp, &vmCall);
11984 masm.tagValue(JSVAL_TYPE_INT32, temp, output);
11985 masm.jump(&done);
11986 {
11987 masm.bind(&vmCall);
11988
11989 pushArg(radix);
11990 pushArg(string);
11991
11992 using Fn = bool (*)(JSContext*, HandleString, int32_t, MutableHandleValue);
11993 callVM<Fn, js::NumberParseInt>(lir);
11994 }
11995 masm.bind(&done);
11996}
11997
11998void CodeGenerator::visitDoubleParseInt(LDoubleParseInt* lir) {
11999 FloatRegister number = ToFloatRegister(lir->number());
12000 Register output = ToRegister(lir->output());
12001 FloatRegister temp = ToFloatRegister(lir->temp0());
12002
12003 Label bail;
12004 masm.branchDouble(Assembler::DoubleUnordered, number, number, &bail);
12005 masm.branchTruncateDoubleToInt32(number, output, &bail);
12006
12007 Label ok;
12008 masm.branch32(Assembler::NotEqual, output, Imm32(0), &ok);
12009 {
12010 // Accept both +0 and -0 and return 0.
12011 masm.loadConstantDouble(0.0, temp);
12012 masm.branchDouble(Assembler::DoubleEqual, number, temp, &ok);
12013
12014 // Fail if a non-zero input is in the exclusive range (-1, 1.0e-6).
12015 masm.loadConstantDouble(DOUBLE_DECIMAL_IN_SHORTEST_LOW, temp);
12016 masm.branchDouble(Assembler::DoubleLessThan, number, temp, &bail);
12017 }
12018 masm.bind(&ok);
12019
12020 bailoutFrom(&bail, lir->snapshot());
12021}
12022
12023void CodeGenerator::visitFloor(LFloor* lir) {
12024 FloatRegister input = ToFloatRegister(lir->input());
12025 Register output = ToRegister(lir->output());
12026
12027 Label bail;
12028 masm.floorDoubleToInt32(input, output, &bail);
12029 bailoutFrom(&bail, lir->snapshot());
12030}
12031
12032void CodeGenerator::visitFloorF(LFloorF* lir) {
12033 FloatRegister input = ToFloatRegister(lir->input());
12034 Register output = ToRegister(lir->output());
12035
12036 Label bail;
12037 masm.floorFloat32ToInt32(input, output, &bail);
12038 bailoutFrom(&bail, lir->snapshot());
12039}
12040
12041void CodeGenerator::visitCeil(LCeil* lir) {
12042 FloatRegister input = ToFloatRegister(lir->input());
12043 Register output = ToRegister(lir->output());
12044
12045 Label bail;
12046 masm.ceilDoubleToInt32(input, output, &bail);
12047 bailoutFrom(&bail, lir->snapshot());
12048}
12049
12050void CodeGenerator::visitCeilF(LCeilF* lir) {
12051 FloatRegister input = ToFloatRegister(lir->input());
12052 Register output = ToRegister(lir->output());
12053
12054 Label bail;
12055 masm.ceilFloat32ToInt32(input, output, &bail);
12056 bailoutFrom(&bail, lir->snapshot());
12057}
12058
12059void CodeGenerator::visitRound(LRound* lir) {
12060 FloatRegister input = ToFloatRegister(lir->input());
12061 FloatRegister temp = ToFloatRegister(lir->temp0());
12062 Register output = ToRegister(lir->output());
12063
12064 Label bail;
12065 masm.roundDoubleToInt32(input, output, temp, &bail);
12066 bailoutFrom(&bail, lir->snapshot());
12067}
12068
12069void CodeGenerator::visitRoundF(LRoundF* lir) {
12070 FloatRegister input = ToFloatRegister(lir->input());
12071 FloatRegister temp = ToFloatRegister(lir->temp0());
12072 Register output = ToRegister(lir->output());
12073
12074 Label bail;
12075 masm.roundFloat32ToInt32(input, output, temp, &bail);
12076 bailoutFrom(&bail, lir->snapshot());
12077}
12078
12079void CodeGenerator::visitTrunc(LTrunc* lir) {
12080 FloatRegister input = ToFloatRegister(lir->input());
12081 Register output = ToRegister(lir->output());
12082
12083 Label bail;
12084 masm.truncDoubleToInt32(input, output, &bail);
12085 bailoutFrom(&bail, lir->snapshot());
12086}
12087
12088void CodeGenerator::visitTruncF(LTruncF* lir) {
12089 FloatRegister input = ToFloatRegister(lir->input());
12090 Register output = ToRegister(lir->output());
12091
12092 Label bail;
12093 masm.truncFloat32ToInt32(input, output, &bail);
12094 bailoutFrom(&bail, lir->snapshot());
12095}
12096
12097void CodeGenerator::visitCompareS(LCompareS* lir) {
12098 JSOp op = lir->mir()->jsop();
12099 Register left = ToRegister(lir->left());
12100 Register right = ToRegister(lir->right());
12101 Register output = ToRegister(lir->output());
12102
12103 OutOfLineCode* ool = nullptr;
12104
12105 using Fn = bool (*)(JSContext*, HandleString, HandleString, bool*);
12106 if (op == JSOp::Eq || op == JSOp::StrictEq) {
12107 ool = oolCallVM<Fn, jit::StringsEqual<EqualityKind::Equal>>(
12108 lir, ArgList(left, right), StoreRegisterTo(output));
12109 } else if (op == JSOp::Ne || op == JSOp::StrictNe) {
12110 ool = oolCallVM<Fn, jit::StringsEqual<EqualityKind::NotEqual>>(
12111 lir, ArgList(left, right), StoreRegisterTo(output));
12112 } else if (op == JSOp::Lt) {
12113 ool = oolCallVM<Fn, jit::StringsCompare<ComparisonKind::LessThan>>(
12114 lir, ArgList(left, right), StoreRegisterTo(output));
12115 } else if (op == JSOp::Le) {
12116 // Push the operands in reverse order for JSOp::Le:
12117 // - |left <= right| is implemented as |right >= left|.
12118 ool =
12119 oolCallVM<Fn, jit::StringsCompare<ComparisonKind::GreaterThanOrEqual>>(
12120 lir, ArgList(right, left), StoreRegisterTo(output));
12121 } else if (op == JSOp::Gt) {
12122 // Push the operands in reverse order for JSOp::Gt:
12123 // - |left > right| is implemented as |right < left|.
12124 ool = oolCallVM<Fn, jit::StringsCompare<ComparisonKind::LessThan>>(
12125 lir, ArgList(right, left), StoreRegisterTo(output));
12126 } else {
12127 MOZ_ASSERT(op == JSOp::Ge)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(op == JSOp::Ge)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(op == JSOp::Ge))), 0))) { do
{ } while (false); MOZ_ReportAssertionFailure("op == JSOp::Ge"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12127); AnnotateMozCrashReason("MOZ_ASSERT" "(" "op == JSOp::Ge"
")"); do { *((volatile int*)__null) = 12127; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
12128 ool =
12129 oolCallVM<Fn, jit::StringsCompare<ComparisonKind::GreaterThanOrEqual>>(
12130 lir, ArgList(left, right), StoreRegisterTo(output));
12131 }
12132
12133 masm.compareStrings(op, left, right, output, ool->entry());
12134
12135 masm.bind(ool->rejoin());
12136}
12137
12138void CodeGenerator::visitCompareSInline(LCompareSInline* lir) {
12139 JSOp op = lir->mir()->jsop();
12140 MOZ_ASSERT(IsEqualityOp(op))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(IsEqualityOp(op))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(IsEqualityOp(op)))), 0))) { do
{ } while (false); MOZ_ReportAssertionFailure("IsEqualityOp(op)"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12140); AnnotateMozCrashReason("MOZ_ASSERT" "(" "IsEqualityOp(op)"
")"); do { *((volatile int*)__null) = 12140; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
12141
12142 Register input = ToRegister(lir->input());
12143 Register output = ToRegister(lir->output());
12144
12145 const JSLinearString* str = lir->constant();
12146 MOZ_ASSERT(str->length() > 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(str->length() > 0)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(str->length() > 0))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("str->length() > 0"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12146); AnnotateMozCrashReason("MOZ_ASSERT" "(" "str->length() > 0"
")"); do { *((volatile int*)__null) = 12146; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
12147
12148 OutOfLineCode* ool = nullptr;
12149
12150 using Fn = bool (*)(JSContext*, HandleString, HandleString, bool*);
12151 if (op == JSOp::Eq || op == JSOp::StrictEq) {
12152 ool = oolCallVM<Fn, jit::StringsEqual<EqualityKind::Equal>>(
12153 lir, ArgList(ImmGCPtr(str), input), StoreRegisterTo(output));
12154 } else {
12155 MOZ_ASSERT(op == JSOp::Ne || op == JSOp::StrictNe)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(op == JSOp::Ne || op == JSOp::StrictNe)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(op == JSOp::Ne || op == JSOp::StrictNe))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("op == JSOp::Ne || op == JSOp::StrictNe"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12155); AnnotateMozCrashReason("MOZ_ASSERT" "(" "op == JSOp::Ne || op == JSOp::StrictNe"
")"); do { *((volatile int*)__null) = 12155; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
12156 ool = oolCallVM<Fn, jit::StringsEqual<EqualityKind::NotEqual>>(
12157 lir, ArgList(ImmGCPtr(str), input), StoreRegisterTo(output));
12158 }
12159
12160 Label compareChars;
12161 {
12162 Label notPointerEqual;
12163
12164 // If operands point to the same instance, the strings are trivially equal.
12165 masm.branchPtr(Assembler::NotEqual, input, ImmGCPtr(str), &notPointerEqual);
12166 masm.move32(Imm32(op == JSOp::Eq || op == JSOp::StrictEq), output);
12167 masm.jump(ool->rejoin());
12168
12169 masm.bind(&notPointerEqual);
12170
12171 Label setNotEqualResult;
12172
12173 if (str->isAtom()) {
12174 // Atoms cannot be equal to each other if they point to different strings.
12175 Imm32 atomBit(JSString::ATOM_BIT);
12176 masm.branchTest32(Assembler::NonZero,
12177 Address(input, JSString::offsetOfFlags()), atomBit,
12178 &setNotEqualResult);
12179 }
12180
12181 if (str->hasTwoByteChars()) {
12182 // Pure two-byte strings can't be equal to Latin-1 strings.
12183 JS::AutoCheckCannotGC nogc;
12184 if (!mozilla::IsUtf16Latin1(str->twoByteRange(nogc))) {
12185 masm.branchLatin1String(input, &setNotEqualResult);
12186 }
12187 }
12188
12189 // Strings of different length can never be equal.
12190 masm.branch32(Assembler::NotEqual,
12191 Address(input, JSString::offsetOfLength()),
12192 Imm32(str->length()), &setNotEqualResult);
12193
12194 if (str->isAtom()) {
12195 Label forwardedPtrEqual;
12196 masm.tryFastAtomize(input, output, output, &compareChars);
12197
12198 // We now have two atoms. Just check pointer equality.
12199 masm.branchPtr(Assembler::Equal, output, ImmGCPtr(str),
12200 &forwardedPtrEqual);
12201
12202 masm.move32(Imm32(op == JSOp::Ne || op == JSOp::StrictNe), output);
12203 masm.jump(ool->rejoin());
12204
12205 masm.bind(&forwardedPtrEqual);
12206 masm.move32(Imm32(op == JSOp::Eq || op == JSOp::StrictEq), output);
12207 masm.jump(ool->rejoin());
12208 } else {
12209 masm.jump(&compareChars);
12210 }
12211
12212 masm.bind(&setNotEqualResult);
12213 masm.move32(Imm32(op == JSOp::Ne || op == JSOp::StrictNe), output);
12214 masm.jump(ool->rejoin());
12215 }
12216
12217 masm.bind(&compareChars);
12218
12219 // Load the input string's characters.
12220 Register stringChars = output;
12221 masm.loadStringCharsForCompare(input, str, stringChars, ool->entry());
12222
12223 // Start comparing character by character.
12224 masm.compareStringChars(op, stringChars, str, output);
12225
12226 masm.bind(ool->rejoin());
12227}
12228
12229void CodeGenerator::visitCompareSSingle(LCompareSSingle* lir) {
12230 JSOp op = lir->jsop();
12231 MOZ_ASSERT(IsRelationalOp(op))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(IsRelationalOp(op))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(IsRelationalOp(op)))), 0))) {
do { } while (false); MOZ_ReportAssertionFailure("IsRelationalOp(op)"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12231); AnnotateMozCrashReason("MOZ_ASSERT" "(" "IsRelationalOp(op)"
")"); do { *((volatile int*)__null) = 12231; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
12232
12233 Register input = ToRegister(lir->input());
12234 Register output = ToRegister(lir->output());
12235 Register temp = ToRegister(lir->temp0());
12236
12237 const JSLinearString* str = lir->constant();
12238 MOZ_ASSERT(str->length() == 1)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(str->length() == 1)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(str->length() == 1))), 0)
)) { do { } while (false); MOZ_ReportAssertionFailure("str->length() == 1"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12238); AnnotateMozCrashReason("MOZ_ASSERT" "(" "str->length() == 1"
")"); do { *((volatile int*)__null) = 12238; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
12239
12240 char16_t ch = str->latin1OrTwoByteChar(0);
12241
12242 masm.movePtr(input, temp);
12243
12244 // Check if the string is empty.
12245 Label compareLength;
12246 masm.branch32(Assembler::Equal, Address(temp, JSString::offsetOfLength()),
12247 Imm32(0), &compareLength);
12248
12249 // The first character is in the left-most rope child.
12250 Label notRope;
12251 masm.branchIfNotRope(temp, &notRope);
12252 {
12253 // Unwind ropes at the start if possible.
12254 Label unwindRope;
12255 masm.bind(&unwindRope);
12256 masm.loadRopeLeftChild(temp, output);
12257 masm.movePtr(output, temp);
12258
12259#ifdef DEBUG1
12260 Label notEmpty;
12261 masm.branch32(Assembler::NotEqual,
12262 Address(temp, JSString::offsetOfLength()), Imm32(0),
12263 &notEmpty);
12264 masm.assumeUnreachable("rope children are non-empty");
12265 masm.bind(&notEmpty);
12266#endif
12267
12268 // Otherwise keep unwinding ropes.
12269 masm.branchIfRope(temp, &unwindRope);
12270 }
12271 masm.bind(&notRope);
12272
12273 // Load the first character into |output|.
12274 auto loadFirstChar = [&](auto encoding) {
12275 masm.loadStringChars(temp, output, encoding);
12276 masm.loadChar(Address(output, 0), output, encoding);
12277 };
12278
12279 Label done;
12280 if (ch <= JSString::MAX_LATIN1_CHAR) {
12281 // Handle both encodings when the search character is Latin-1.
12282 Label twoByte, compare;
12283 masm.branchTwoByteString(temp, &twoByte);
12284
12285 loadFirstChar(CharEncoding::Latin1);
12286 masm.jump(&compare);
12287
12288 masm.bind(&twoByte);
12289 loadFirstChar(CharEncoding::TwoByte);
12290
12291 masm.bind(&compare);
12292 } else {
12293 // The search character is a two-byte character, so it can't be equal to any
12294 // character of a Latin-1 string.
12295 masm.move32(Imm32(int32_t(op == JSOp::Lt || op == JSOp::Le)), output);
12296 masm.branchLatin1String(temp, &done);
12297
12298 loadFirstChar(CharEncoding::TwoByte);
12299 }
12300
12301 // Compare the string length when the search character is equal to the
12302 // input's first character.
12303 masm.branch32(Assembler::Equal, output, Imm32(ch), &compareLength);
12304
12305 // Otherwise compute the result and jump to the end.
12306 masm.cmp32Set(JSOpToCondition(op, /* isSigned = */ false), output, Imm32(ch),
12307 output);
12308 masm.jump(&done);
12309
12310 // Compare the string length to compute the overall result.
12311 masm.bind(&compareLength);
12312 masm.cmp32Set(JSOpToCondition(op, /* isSigned = */ false),
12313 Address(temp, JSString::offsetOfLength()), Imm32(1), output);
12314
12315 masm.bind(&done);
12316}
12317
12318void CodeGenerator::visitCompareBigInt(LCompareBigInt* lir) {
12319 JSOp op = lir->mir()->jsop();
12320 Register left = ToRegister(lir->left());
12321 Register right = ToRegister(lir->right());
12322 Register temp0 = ToRegister(lir->temp0());
12323 Register temp1 = ToRegister(lir->temp1());
12324 Register temp2 = ToRegister(lir->temp2());
12325 Register output = ToRegister(lir->output());
12326
12327 Label notSame;
12328 Label compareSign;
12329 Label compareLength;
12330 Label compareDigit;
12331
12332 Label* notSameSign;
12333 Label* notSameLength;
12334 Label* notSameDigit;
12335 if (IsEqualityOp(op)) {
12336 notSameSign = &notSame;
12337 notSameLength = &notSame;
12338 notSameDigit = &notSame;
12339 } else {
12340 notSameSign = &compareSign;
12341 notSameLength = &compareLength;
12342 notSameDigit = &compareDigit;
12343 }
12344
12345 masm.equalBigInts(left, right, temp0, temp1, temp2, output, notSameSign,
12346 notSameLength, notSameDigit);
12347
12348 Label done;
12349 masm.move32(Imm32(op == JSOp::Eq || op == JSOp::StrictEq || op == JSOp::Le ||
12350 op == JSOp::Ge),
12351 output);
12352 masm.jump(&done);
12353
12354 if (IsEqualityOp(op)) {
12355 masm.bind(&notSame);
12356 masm.move32(Imm32(op == JSOp::Ne || op == JSOp::StrictNe), output);
12357 } else {
12358 Label invertWhenNegative;
12359
12360 // There are two cases when sign(left) != sign(right):
12361 // 1. sign(left) = positive and sign(right) = negative,
12362 // 2. or the dual case with reversed signs.
12363 //
12364 // For case 1, |left| <cmp> |right| is true for cmp=Gt or cmp=Ge and false
12365 // for cmp=Lt or cmp=Le. Initialize the result for case 1 and handle case 2
12366 // with |invertWhenNegative|.
12367 masm.bind(&compareSign);
12368 masm.move32(Imm32(op == JSOp::Gt || op == JSOp::Ge), output);
12369 masm.jump(&invertWhenNegative);
12370
12371 // For sign(left) = sign(right) and len(digits(left)) != len(digits(right)),
12372 // we have to consider the two cases:
12373 // 1. len(digits(left)) < len(digits(right))
12374 // 2. len(digits(left)) > len(digits(right))
12375 //
12376 // For |left| <cmp> |right| with cmp=Lt:
12377 // Assume both BigInts are positive, then |left < right| is true for case 1
12378 // and false for case 2. When both are negative, the result is reversed.
12379 //
12380 // The other comparison operators can be handled similarly.
12381 //
12382 // |temp0| holds the digits length of the right-hand side operand.
12383 masm.bind(&compareLength);
12384 masm.cmp32Set(JSOpToCondition(op, /* isSigned = */ false),
12385 Address(left, BigInt::offsetOfLength()), temp0, output);
12386 masm.jump(&invertWhenNegative);
12387
12388 // Similar to the case above, compare the current digit to determine the
12389 // overall comparison result.
12390 //
12391 // |temp1| points to the current digit of the left-hand side operand.
12392 // |output| holds the current digit of the right-hand side operand.
12393 masm.bind(&compareDigit);
12394 masm.cmpPtrSet(JSOpToCondition(op, /* isSigned = */ false),
12395 Address(temp1, 0), output, output);
12396
12397 Label nonNegative;
12398 masm.bind(&invertWhenNegative);
12399 masm.branchIfBigIntIsNonNegative(left, &nonNegative);
12400 masm.xor32(Imm32(1), output);
12401 masm.bind(&nonNegative);
12402 }
12403
12404 masm.bind(&done);
12405}
12406
12407void CodeGenerator::visitCompareBigIntInt32(LCompareBigIntInt32* lir) {
12408 JSOp op = lir->mir()->jsop();
12409 Register left = ToRegister(lir->left());
12410 Register temp0 = ToRegister(lir->temp0());
12411 Register temp1 = ToTempRegisterOrInvalid(lir->temp1());
12412 Register output = ToRegister(lir->output());
12413
12414 Label ifTrue, ifFalse;
12415 if (lir->right()->isConstant()) {
12416 MOZ_ASSERT(temp1 == InvalidReg)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(temp1 == InvalidReg)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(temp1 == InvalidReg))), 0)))
{ do { } while (false); MOZ_ReportAssertionFailure("temp1 == InvalidReg"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12416); AnnotateMozCrashReason("MOZ_ASSERT" "(" "temp1 == InvalidReg"
")"); do { *((volatile int*)__null) = 12416; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
12417
12418 Imm32 right = Imm32(ToInt32(lir->right()));
12419 masm.compareBigIntAndInt32(op, left, right, temp0, &ifTrue, &ifFalse);
12420 } else {
12421 MOZ_ASSERT(temp1 != InvalidReg)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(temp1 != InvalidReg)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(temp1 != InvalidReg))), 0)))
{ do { } while (false); MOZ_ReportAssertionFailure("temp1 != InvalidReg"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12421); AnnotateMozCrashReason("MOZ_ASSERT" "(" "temp1 != InvalidReg"
")"); do { *((volatile int*)__null) = 12421; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
12422
12423 Register right = ToRegister(lir->right());
12424 masm.compareBigIntAndInt32(op, left, right, temp0, temp1, &ifTrue,
12425 &ifFalse);
12426 }
12427
12428 Label done;
12429 masm.bind(&ifFalse);
12430 masm.move32(Imm32(0), output);
12431 masm.jump(&done);
12432 masm.bind(&ifTrue);
12433 masm.move32(Imm32(1), output);
12434 masm.bind(&done);
12435}
12436
12437void CodeGenerator::visitCompareBigIntInt32AndBranch(
12438 LCompareBigIntInt32AndBranch* lir) {
12439 JSOp op = lir->cmpMir()->jsop();
12440 Register left = ToRegister(lir->left());
12441 Register temp1 = ToRegister(lir->temp1());
12442 Register temp2 = ToTempRegisterOrInvalid(lir->temp2());
12443
12444 Label* ifTrue = getJumpLabelForBranch(lir->ifTrue());
12445 Label* ifFalse = getJumpLabelForBranch(lir->ifFalse());
12446
12447 // compareBigIntAndInt32 falls through to the false case. If the next block
12448 // is the true case, negate the comparison so we can fall through.
12449 if (isNextBlock(lir->ifTrue()->lir())) {
12450 op = NegateCompareOp(op);
12451 std::swap(ifTrue, ifFalse);
12452 }
12453
12454 if (lir->right()->isConstant()) {
12455 MOZ_ASSERT(temp2 == InvalidReg)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(temp2 == InvalidReg)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(temp2 == InvalidReg))), 0)))
{ do { } while (false); MOZ_ReportAssertionFailure("temp2 == InvalidReg"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12455); AnnotateMozCrashReason("MOZ_ASSERT" "(" "temp2 == InvalidReg"
")"); do { *((volatile int*)__null) = 12455; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
12456
12457 Imm32 right = Imm32(ToInt32(lir->right()));
12458 masm.compareBigIntAndInt32(op, left, right, temp1, ifTrue, ifFalse);
12459 } else {
12460 MOZ_ASSERT(temp2 != InvalidReg)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(temp2 != InvalidReg)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(temp2 != InvalidReg))), 0)))
{ do { } while (false); MOZ_ReportAssertionFailure("temp2 != InvalidReg"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12460); AnnotateMozCrashReason("MOZ_ASSERT" "(" "temp2 != InvalidReg"
")"); do { *((volatile int*)__null) = 12460; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
12461
12462 Register right = ToRegister(lir->right());
12463 masm.compareBigIntAndInt32(op, left, right, temp1, temp2, ifTrue, ifFalse);
12464 }
12465
12466 if (!isNextBlock(lir->ifTrue()->lir())) {
12467 jumpToBlock(lir->ifFalse());
12468 }
12469}
12470
12471void CodeGenerator::visitCompareBigIntDouble(LCompareBigIntDouble* lir) {
12472 JSOp op = lir->mir()->jsop();
12473 Register left = ToRegister(lir->left());
12474 FloatRegister right = ToFloatRegister(lir->right());
12475 Register output = ToRegister(lir->output());
12476
12477 masm.setupAlignedABICall();
12478
12479 // Push the operands in reverse order for JSOp::Le and JSOp::Gt:
12480 // - |left <= right| is implemented as |right >= left|.
12481 // - |left > right| is implemented as |right < left|.
12482 if (op == JSOp::Le || op == JSOp::Gt) {
12483 masm.passABIArg(right, ABIType::Float64);
12484 masm.passABIArg(left);
12485 } else {
12486 masm.passABIArg(left);
12487 masm.passABIArg(right, ABIType::Float64);
12488 }
12489
12490 using FnBigIntNumber = bool (*)(BigInt*, double);
12491 using FnNumberBigInt = bool (*)(double, BigInt*);
12492 switch (op) {
12493 case JSOp::Eq: {
12494 masm.callWithABI<FnBigIntNumber,
12495 jit::BigIntNumberEqual<EqualityKind::Equal>>();
12496 break;
12497 }
12498 case JSOp::Ne: {
12499 masm.callWithABI<FnBigIntNumber,
12500 jit::BigIntNumberEqual<EqualityKind::NotEqual>>();
12501 break;
12502 }
12503 case JSOp::Lt: {
12504 masm.callWithABI<FnBigIntNumber,
12505 jit::BigIntNumberCompare<ComparisonKind::LessThan>>();
12506 break;
12507 }
12508 case JSOp::Gt: {
12509 masm.callWithABI<FnNumberBigInt,
12510 jit::NumberBigIntCompare<ComparisonKind::LessThan>>();
12511 break;
12512 }
12513 case JSOp::Le: {
12514 masm.callWithABI<
12515 FnNumberBigInt,
12516 jit::NumberBigIntCompare<ComparisonKind::GreaterThanOrEqual>>();
12517 break;
12518 }
12519 case JSOp::Ge: {
12520 masm.callWithABI<
12521 FnBigIntNumber,
12522 jit::BigIntNumberCompare<ComparisonKind::GreaterThanOrEqual>>();
12523 break;
12524 }
12525 default:
12526 MOZ_CRASH("unhandled op")do { do { } while (false); MOZ_ReportCrash("" "unhandled op",
"/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12526); AnnotateMozCrashReason("MOZ_CRASH(" "unhandled op" ")"
); do { *((volatile int*)__null) = 12526; __attribute__((nomerge
)) ::abort(); } while (false); } while (false)
;
12527 }
12528
12529 masm.storeCallBoolResult(output);
12530}
12531
12532void CodeGenerator::visitCompareBigIntString(LCompareBigIntString* lir) {
12533 JSOp op = lir->mir()->jsop();
12534 Register left = ToRegister(lir->left());
12535 Register right = ToRegister(lir->right());
12536
12537 // Push the operands in reverse order for JSOp::Le and JSOp::Gt:
12538 // - |left <= right| is implemented as |right >= left|.
12539 // - |left > right| is implemented as |right < left|.
12540 if (op == JSOp::Le || op == JSOp::Gt) {
12541 pushArg(left);
12542 pushArg(right);
12543 } else {
12544 pushArg(right);
12545 pushArg(left);
12546 }
12547
12548 using FnBigIntString =
12549 bool (*)(JSContext*, HandleBigInt, HandleString, bool*);
12550 using FnStringBigInt =
12551 bool (*)(JSContext*, HandleString, HandleBigInt, bool*);
12552
12553 switch (op) {
12554 case JSOp::Eq: {
12555 constexpr auto Equal = EqualityKind::Equal;
12556 callVM<FnBigIntString, BigIntStringEqual<Equal>>(lir);
12557 break;
12558 }
12559 case JSOp::Ne: {
12560 constexpr auto NotEqual = EqualityKind::NotEqual;
12561 callVM<FnBigIntString, BigIntStringEqual<NotEqual>>(lir);
12562 break;
12563 }
12564 case JSOp::Lt: {
12565 constexpr auto LessThan = ComparisonKind::LessThan;
12566 callVM<FnBigIntString, BigIntStringCompare<LessThan>>(lir);
12567 break;
12568 }
12569 case JSOp::Gt: {
12570 constexpr auto LessThan = ComparisonKind::LessThan;
12571 callVM<FnStringBigInt, StringBigIntCompare<LessThan>>(lir);
12572 break;
12573 }
12574 case JSOp::Le: {
12575 constexpr auto GreaterThanOrEqual = ComparisonKind::GreaterThanOrEqual;
12576 callVM<FnStringBigInt, StringBigIntCompare<GreaterThanOrEqual>>(lir);
12577 break;
12578 }
12579 case JSOp::Ge: {
12580 constexpr auto GreaterThanOrEqual = ComparisonKind::GreaterThanOrEqual;
12581 callVM<FnBigIntString, BigIntStringCompare<GreaterThanOrEqual>>(lir);
12582 break;
12583 }
12584 default:
12585 MOZ_CRASH("Unexpected compare op")do { do { } while (false); MOZ_ReportCrash("" "Unexpected compare op"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12585); AnnotateMozCrashReason("MOZ_CRASH(" "Unexpected compare op"
")"); do { *((volatile int*)__null) = 12585; __attribute__((
nomerge)) ::abort(); } while (false); } while (false)
;
12586 }
12587}
12588
12589void CodeGenerator::visitIsNullOrLikeUndefinedV(LIsNullOrLikeUndefinedV* lir) {
12590 MOZ_ASSERT(lir->mir()->compareType() == MCompare::Compare_Undefined ||do { static_assert( mozilla::detail::AssertionConditionType<
decltype(lir->mir()->compareType() == MCompare::Compare_Undefined
|| lir->mir()->compareType() == MCompare::Compare_Null
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(lir->mir()->compareType() == MCompare::Compare_Undefined
|| lir->mir()->compareType() == MCompare::Compare_Null
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"lir->mir()->compareType() == MCompare::Compare_Undefined || lir->mir()->compareType() == MCompare::Compare_Null"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12591); AnnotateMozCrashReason("MOZ_ASSERT" "(" "lir->mir()->compareType() == MCompare::Compare_Undefined || lir->mir()->compareType() == MCompare::Compare_Null"
")"); do { *((volatile int*)__null) = 12591; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
12591 lir->mir()->compareType() == MCompare::Compare_Null)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(lir->mir()->compareType() == MCompare::Compare_Undefined
|| lir->mir()->compareType() == MCompare::Compare_Null
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(lir->mir()->compareType() == MCompare::Compare_Undefined
|| lir->mir()->compareType() == MCompare::Compare_Null
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"lir->mir()->compareType() == MCompare::Compare_Undefined || lir->mir()->compareType() == MCompare::Compare_Null"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12591); AnnotateMozCrashReason("MOZ_ASSERT" "(" "lir->mir()->compareType() == MCompare::Compare_Undefined || lir->mir()->compareType() == MCompare::Compare_Null"
")"); do { *((volatile int*)__null) = 12591; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
12592
12593 JSOp op = lir->mir()->jsop();
12594 MOZ_ASSERT(IsLooseEqualityOp(op))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(IsLooseEqualityOp(op))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(IsLooseEqualityOp(op)))), 0)
)) { do { } while (false); MOZ_ReportAssertionFailure("IsLooseEqualityOp(op)"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12594); AnnotateMozCrashReason("MOZ_ASSERT" "(" "IsLooseEqualityOp(op)"
")"); do { *((volatile int*)__null) = 12594; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
12595
12596 const ValueOperand value = ToValue(lir, LIsNullOrLikeUndefinedV::ValueIndex);
12597 Register output = ToRegister(lir->output());
12598
12599 bool intact = hasSeenObjectEmulateUndefinedFuseIntactAndDependencyNoted();
12600 if (!intact) {
12601 auto* ool = new (alloc()) OutOfLineTestObjectWithLabels();
12602 addOutOfLineCode(ool, lir->mir());
12603
12604 Label* nullOrLikeUndefined = ool->label1();
12605 Label* notNullOrLikeUndefined = ool->label2();
12606
12607 {
12608 ScratchTagScope tag(masm, value);
12609 masm.splitTagForTest(value, tag);
12610
12611 masm.branchTestNull(Assembler::Equal, tag, nullOrLikeUndefined);
12612 masm.branchTestUndefined(Assembler::Equal, tag, nullOrLikeUndefined);
12613
12614 // Check whether it's a truthy object or a falsy object that emulates
12615 // undefined.
12616 masm.branchTestObject(Assembler::NotEqual, tag, notNullOrLikeUndefined);
12617 }
12618
12619 Register objreg =
12620 masm.extractObject(value, ToTempUnboxRegister(lir->temp0()));
12621 branchTestObjectEmulatesUndefined(objreg, nullOrLikeUndefined,
12622 notNullOrLikeUndefined, output, ool);
12623 // fall through
12624
12625 Label done;
12626
12627 // It's not null or undefined, and if it's an object it doesn't
12628 // emulate undefined, so it's not like undefined.
12629 masm.move32(Imm32(op == JSOp::Ne), output);
12630 masm.jump(&done);
12631
12632 masm.bind(nullOrLikeUndefined);
12633 masm.move32(Imm32(op == JSOp::Eq), output);
12634
12635 // Both branches meet here.
12636 masm.bind(&done);
12637 } else {
12638 Label nullOrUndefined, notNullOrLikeUndefined;
12639#if defined(DEBUG1) || defined(FUZZING)
12640 Register objreg = Register::Invalid();
12641#endif
12642 {
12643 ScratchTagScope tag(masm, value);
12644 masm.splitTagForTest(value, tag);
12645
12646 masm.branchTestNull(Assembler::Equal, tag, &nullOrUndefined);
12647 masm.branchTestUndefined(Assembler::Equal, tag, &nullOrUndefined);
12648
12649#if defined(DEBUG1) || defined(FUZZING)
12650 // Check whether it's a truthy object or a falsy object that emulates
12651 // undefined.
12652 masm.branchTestObject(Assembler::NotEqual, tag, &notNullOrLikeUndefined);
12653 objreg = masm.extractObject(value, ToTempUnboxRegister(lir->temp0()));
12654#endif
12655 }
12656
12657#if defined(DEBUG1) || defined(FUZZING)
12658 assertObjectDoesNotEmulateUndefined(objreg, output, lir->mir());
12659 masm.bind(&notNullOrLikeUndefined);
12660#endif
12661
12662 Label done;
12663
12664 // It's not null or undefined, and if it's an object it doesn't
12665 // emulate undefined.
12666 masm.move32(Imm32(op == JSOp::Ne), output);
12667 masm.jump(&done);
12668
12669 masm.bind(&nullOrUndefined);
12670 masm.move32(Imm32(op == JSOp::Eq), output);
12671
12672 // Both branches meet here.
12673 masm.bind(&done);
12674 }
12675}
12676
12677void CodeGenerator::visitIsNullOrLikeUndefinedAndBranchV(
12678 LIsNullOrLikeUndefinedAndBranchV* lir) {
12679 MOZ_ASSERT(lir->cmpMir()->compareType() == MCompare::Compare_Undefined ||do { static_assert( mozilla::detail::AssertionConditionType<
decltype(lir->cmpMir()->compareType() == MCompare::Compare_Undefined
|| lir->cmpMir()->compareType() == MCompare::Compare_Null
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(lir->cmpMir()->compareType() == MCompare::Compare_Undefined
|| lir->cmpMir()->compareType() == MCompare::Compare_Null
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"lir->cmpMir()->compareType() == MCompare::Compare_Undefined || lir->cmpMir()->compareType() == MCompare::Compare_Null"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12680); AnnotateMozCrashReason("MOZ_ASSERT" "(" "lir->cmpMir()->compareType() == MCompare::Compare_Undefined || lir->cmpMir()->compareType() == MCompare::Compare_Null"
")"); do { *((volatile int*)__null) = 12680; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
12680 lir->cmpMir()->compareType() == MCompare::Compare_Null)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(lir->cmpMir()->compareType() == MCompare::Compare_Undefined
|| lir->cmpMir()->compareType() == MCompare::Compare_Null
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(lir->cmpMir()->compareType() == MCompare::Compare_Undefined
|| lir->cmpMir()->compareType() == MCompare::Compare_Null
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"lir->cmpMir()->compareType() == MCompare::Compare_Undefined || lir->cmpMir()->compareType() == MCompare::Compare_Null"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12680); AnnotateMozCrashReason("MOZ_ASSERT" "(" "lir->cmpMir()->compareType() == MCompare::Compare_Undefined || lir->cmpMir()->compareType() == MCompare::Compare_Null"
")"); do { *((volatile int*)__null) = 12680; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
12681
12682 JSOp op = lir->cmpMir()->jsop();
12683 MOZ_ASSERT(IsLooseEqualityOp(op))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(IsLooseEqualityOp(op))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(IsLooseEqualityOp(op)))), 0)
)) { do { } while (false); MOZ_ReportAssertionFailure("IsLooseEqualityOp(op)"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12683); AnnotateMozCrashReason("MOZ_ASSERT" "(" "IsLooseEqualityOp(op)"
")"); do { *((volatile int*)__null) = 12683; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
12684
12685 const ValueOperand value =
12686 ToValue(lir, LIsNullOrLikeUndefinedAndBranchV::Value);
12687
12688 MBasicBlock* ifTrue = lir->ifTrue();
12689 MBasicBlock* ifFalse = lir->ifFalse();
12690
12691 if (op == JSOp::Ne) {
12692 // Swap branches.
12693 std::swap(ifTrue, ifFalse);
12694 }
12695
12696 bool intact = hasSeenObjectEmulateUndefinedFuseIntactAndDependencyNoted();
12697
12698 Label* ifTrueLabel = getJumpLabelForBranch(ifTrue);
12699 Label* ifFalseLabel = getJumpLabelForBranch(ifFalse);
12700
12701 {
12702 ScratchTagScope tag(masm, value);
12703 masm.splitTagForTest(value, tag);
12704
12705 masm.branchTestNull(Assembler::Equal, tag, ifTrueLabel);
12706 masm.branchTestUndefined(Assembler::Equal, tag, ifTrueLabel);
12707
12708 masm.branchTestObject(Assembler::NotEqual, tag, ifFalseLabel);
12709 }
12710
12711 bool extractObject = !intact;
Value stored to 'extractObject' during its initialization is never read
12712#if defined(DEBUG1) || defined(FUZZING)
12713 // always extract objreg if we're in debug and
12714 // assertObjectDoesNotEmulateUndefined;
12715 extractObject = true;
12716#endif
12717
12718 Register objreg = Register::Invalid();
12719 Register scratch = ToRegister(lir->temp());
12720 if (extractObject) {
12721 objreg = masm.extractObject(value, ToTempUnboxRegister(lir->tempToUnbox()));
12722 }
12723 if (!intact) {
12724 // Objects that emulate undefined are loosely equal to null/undefined.
12725 OutOfLineTestObject* ool = new (alloc()) OutOfLineTestObject();
12726 addOutOfLineCode(ool, lir->cmpMir());
12727 testObjectEmulatesUndefined(objreg, ifTrueLabel, ifFalseLabel, scratch,
12728 ool);
12729 } else {
12730 assertObjectDoesNotEmulateUndefined(objreg, scratch, lir->cmpMir());
12731 // Bug 1874905. This would be nice to optimize out at the MIR level.
12732 masm.jump(ifFalseLabel);
12733 }
12734}
12735
12736void CodeGenerator::visitIsNullOrLikeUndefinedT(LIsNullOrLikeUndefinedT* lir) {
12737 MOZ_ASSERT(lir->mir()->compareType() == MCompare::Compare_Undefined ||do { static_assert( mozilla::detail::AssertionConditionType<
decltype(lir->mir()->compareType() == MCompare::Compare_Undefined
|| lir->mir()->compareType() == MCompare::Compare_Null
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(lir->mir()->compareType() == MCompare::Compare_Undefined
|| lir->mir()->compareType() == MCompare::Compare_Null
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"lir->mir()->compareType() == MCompare::Compare_Undefined || lir->mir()->compareType() == MCompare::Compare_Null"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12738); AnnotateMozCrashReason("MOZ_ASSERT" "(" "lir->mir()->compareType() == MCompare::Compare_Undefined || lir->mir()->compareType() == MCompare::Compare_Null"
")"); do { *((volatile int*)__null) = 12738; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
12738 lir->mir()->compareType() == MCompare::Compare_Null)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(lir->mir()->compareType() == MCompare::Compare_Undefined
|| lir->mir()->compareType() == MCompare::Compare_Null
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(lir->mir()->compareType() == MCompare::Compare_Undefined
|| lir->mir()->compareType() == MCompare::Compare_Null
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"lir->mir()->compareType() == MCompare::Compare_Undefined || lir->mir()->compareType() == MCompare::Compare_Null"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12738); AnnotateMozCrashReason("MOZ_ASSERT" "(" "lir->mir()->compareType() == MCompare::Compare_Undefined || lir->mir()->compareType() == MCompare::Compare_Null"
")"); do { *((volatile int*)__null) = 12738; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
12739 MOZ_ASSERT(lir->mir()->lhs()->type() == MIRType::Object)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(lir->mir()->lhs()->type() == MIRType::Object
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(lir->mir()->lhs()->type() == MIRType::Object
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"lir->mir()->lhs()->type() == MIRType::Object", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12739); AnnotateMozCrashReason("MOZ_ASSERT" "(" "lir->mir()->lhs()->type() == MIRType::Object"
")"); do { *((volatile int*)__null) = 12739; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
12740
12741 bool intact = hasSeenObjectEmulateUndefinedFuseIntactAndDependencyNoted();
12742 JSOp op = lir->mir()->jsop();
12743 Register output = ToRegister(lir->output());
12744 Register objreg = ToRegister(lir->input());
12745 if (!intact) {
12746 MOZ_ASSERT(IsLooseEqualityOp(op),do { static_assert( mozilla::detail::AssertionConditionType<
decltype(IsLooseEqualityOp(op))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(IsLooseEqualityOp(op)))), 0)
)) { do { } while (false); MOZ_ReportAssertionFailure("IsLooseEqualityOp(op)"
" (" "Strict equality should have been folded" ")", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12747); AnnotateMozCrashReason("MOZ_ASSERT" "(" "IsLooseEqualityOp(op)"
") (" "Strict equality should have been folded" ")"); do { *
((volatile int*)__null) = 12747; __attribute__((nomerge)) ::abort
(); } while (false); } } while (false)
12747 "Strict equality should have been folded")do { static_assert( mozilla::detail::AssertionConditionType<
decltype(IsLooseEqualityOp(op))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(IsLooseEqualityOp(op)))), 0)
)) { do { } while (false); MOZ_ReportAssertionFailure("IsLooseEqualityOp(op)"
" (" "Strict equality should have been folded" ")", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12747); AnnotateMozCrashReason("MOZ_ASSERT" "(" "IsLooseEqualityOp(op)"
") (" "Strict equality should have been folded" ")"); do { *
((volatile int*)__null) = 12747; __attribute__((nomerge)) ::abort
(); } while (false); } } while (false)
;
12748
12749 auto* ool = new (alloc()) OutOfLineTestObjectWithLabels();
12750 addOutOfLineCode(ool, lir->mir());
12751
12752 Label* emulatesUndefined = ool->label1();
12753 Label* doesntEmulateUndefined = ool->label2();
12754
12755 branchTestObjectEmulatesUndefined(objreg, emulatesUndefined,
12756 doesntEmulateUndefined, output, ool);
12757
12758 Label done;
12759
12760 masm.move32(Imm32(op == JSOp::Ne), output);
12761 masm.jump(&done);
12762
12763 masm.bind(emulatesUndefined);
12764 masm.move32(Imm32(op == JSOp::Eq), output);
12765 masm.bind(&done);
12766 } else {
12767 assertObjectDoesNotEmulateUndefined(objreg, output, lir->mir());
12768 masm.move32(Imm32(op == JSOp::Ne), output);
12769 }
12770}
12771
12772void CodeGenerator::visitIsNullOrLikeUndefinedAndBranchT(
12773 LIsNullOrLikeUndefinedAndBranchT* lir) {
12774 MOZ_ASSERT(lir->cmpMir()->compareType() == MCompare::Compare_Undefined ||do { static_assert( mozilla::detail::AssertionConditionType<
decltype(lir->cmpMir()->compareType() == MCompare::Compare_Undefined
|| lir->cmpMir()->compareType() == MCompare::Compare_Null
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(lir->cmpMir()->compareType() == MCompare::Compare_Undefined
|| lir->cmpMir()->compareType() == MCompare::Compare_Null
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"lir->cmpMir()->compareType() == MCompare::Compare_Undefined || lir->cmpMir()->compareType() == MCompare::Compare_Null"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12775); AnnotateMozCrashReason("MOZ_ASSERT" "(" "lir->cmpMir()->compareType() == MCompare::Compare_Undefined || lir->cmpMir()->compareType() == MCompare::Compare_Null"
")"); do { *((volatile int*)__null) = 12775; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
12775 lir->cmpMir()->compareType() == MCompare::Compare_Null)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(lir->cmpMir()->compareType() == MCompare::Compare_Undefined
|| lir->cmpMir()->compareType() == MCompare::Compare_Null
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(lir->cmpMir()->compareType() == MCompare::Compare_Undefined
|| lir->cmpMir()->compareType() == MCompare::Compare_Null
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"lir->cmpMir()->compareType() == MCompare::Compare_Undefined || lir->cmpMir()->compareType() == MCompare::Compare_Null"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12775); AnnotateMozCrashReason("MOZ_ASSERT" "(" "lir->cmpMir()->compareType() == MCompare::Compare_Undefined || lir->cmpMir()->compareType() == MCompare::Compare_Null"
")"); do { *((volatile int*)__null) = 12775; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
12776 MOZ_ASSERT(lir->cmpMir()->lhs()->type() == MIRType::Object)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(lir->cmpMir()->lhs()->type() == MIRType::Object
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(lir->cmpMir()->lhs()->type() == MIRType::Object
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"lir->cmpMir()->lhs()->type() == MIRType::Object", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12776); AnnotateMozCrashReason("MOZ_ASSERT" "(" "lir->cmpMir()->lhs()->type() == MIRType::Object"
")"); do { *((volatile int*)__null) = 12776; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
12777
12778 bool intact = hasSeenObjectEmulateUndefinedFuseIntactAndDependencyNoted();
12779
12780 JSOp op = lir->cmpMir()->jsop();
12781 MOZ_ASSERT(IsLooseEqualityOp(op), "Strict equality should have been folded")do { static_assert( mozilla::detail::AssertionConditionType<
decltype(IsLooseEqualityOp(op))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(IsLooseEqualityOp(op)))), 0)
)) { do { } while (false); MOZ_ReportAssertionFailure("IsLooseEqualityOp(op)"
" (" "Strict equality should have been folded" ")", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12781); AnnotateMozCrashReason("MOZ_ASSERT" "(" "IsLooseEqualityOp(op)"
") (" "Strict equality should have been folded" ")"); do { *
((volatile int*)__null) = 12781; __attribute__((nomerge)) ::abort
(); } while (false); } } while (false)
;
12782
12783 MBasicBlock* ifTrue = lir->ifTrue();
12784 MBasicBlock* ifFalse = lir->ifFalse();
12785
12786 if (op == JSOp::Ne) {
12787 // Swap branches.
12788 std::swap(ifTrue, ifFalse);
12789 }
12790
12791 Register input = ToRegister(lir->getOperand(0));
12792 Register scratch = ToRegister(lir->temp());
12793 Label* ifTrueLabel = getJumpLabelForBranch(ifTrue);
12794 Label* ifFalseLabel = getJumpLabelForBranch(ifFalse);
12795
12796 if (intact) {
12797 // Bug 1874905. Ideally branches like this would be optimized out.
12798 assertObjectDoesNotEmulateUndefined(input, scratch, lir->mir());
12799 masm.jump(ifFalseLabel);
12800 } else {
12801 auto* ool = new (alloc()) OutOfLineTestObject();
12802 addOutOfLineCode(ool, lir->cmpMir());
12803
12804 // Objects that emulate undefined are loosely equal to null/undefined.
12805 testObjectEmulatesUndefined(input, ifTrueLabel, ifFalseLabel, scratch, ool);
12806 }
12807}
12808
12809void CodeGenerator::visitIsNull(LIsNull* lir) {
12810 MCompare::CompareType compareType = lir->mir()->compareType();
12811 MOZ_ASSERT(compareType == MCompare::Compare_Null)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(compareType == MCompare::Compare_Null)>::isValid,
"invalid assertion condition"); if ((__builtin_expect(!!(!(!
!(compareType == MCompare::Compare_Null))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("compareType == MCompare::Compare_Null"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12811); AnnotateMozCrashReason("MOZ_ASSERT" "(" "compareType == MCompare::Compare_Null"
")"); do { *((volatile int*)__null) = 12811; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
12812
12813 JSOp op = lir->mir()->jsop();
12814 MOZ_ASSERT(IsStrictEqualityOp(op))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(IsStrictEqualityOp(op))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(IsStrictEqualityOp(op)))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("IsStrictEqualityOp(op)"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12814); AnnotateMozCrashReason("MOZ_ASSERT" "(" "IsStrictEqualityOp(op)"
")"); do { *((volatile int*)__null) = 12814; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
12815
12816 const ValueOperand value = ToValue(lir, LIsNull::ValueIndex);
12817 Register output = ToRegister(lir->output());
12818
12819 Assembler::Condition cond = JSOpToCondition(compareType, op);
12820 masm.testNullSet(cond, value, output);
12821}
12822
12823void CodeGenerator::visitIsUndefined(LIsUndefined* lir) {
12824 MCompare::CompareType compareType = lir->mir()->compareType();
12825 MOZ_ASSERT(compareType == MCompare::Compare_Undefined)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(compareType == MCompare::Compare_Undefined)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(compareType == MCompare::Compare_Undefined))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("compareType == MCompare::Compare_Undefined"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12825); AnnotateMozCrashReason("MOZ_ASSERT" "(" "compareType == MCompare::Compare_Undefined"
")"); do { *((volatile int*)__null) = 12825; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
12826
12827 JSOp op = lir->mir()->jsop();
12828 MOZ_ASSERT(IsStrictEqualityOp(op))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(IsStrictEqualityOp(op))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(IsStrictEqualityOp(op)))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("IsStrictEqualityOp(op)"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12828); AnnotateMozCrashReason("MOZ_ASSERT" "(" "IsStrictEqualityOp(op)"
")"); do { *((volatile int*)__null) = 12828; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
12829
12830 const ValueOperand value = ToValue(lir, LIsUndefined::ValueIndex);
12831 Register output = ToRegister(lir->output());
12832
12833 Assembler::Condition cond = JSOpToCondition(compareType, op);
12834 masm.testUndefinedSet(cond, value, output);
12835}
12836
12837void CodeGenerator::visitIsNullAndBranch(LIsNullAndBranch* lir) {
12838 MCompare::CompareType compareType = lir->cmpMir()->compareType();
12839 MOZ_ASSERT(compareType == MCompare::Compare_Null)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(compareType == MCompare::Compare_Null)>::isValid,
"invalid assertion condition"); if ((__builtin_expect(!!(!(!
!(compareType == MCompare::Compare_Null))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("compareType == MCompare::Compare_Null"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12839); AnnotateMozCrashReason("MOZ_ASSERT" "(" "compareType == MCompare::Compare_Null"
")"); do { *((volatile int*)__null) = 12839; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
12840
12841 JSOp op = lir->cmpMir()->jsop();
12842 MOZ_ASSERT(IsStrictEqualityOp(op))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(IsStrictEqualityOp(op))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(IsStrictEqualityOp(op)))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("IsStrictEqualityOp(op)"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12842); AnnotateMozCrashReason("MOZ_ASSERT" "(" "IsStrictEqualityOp(op)"
")"); do { *((volatile int*)__null) = 12842; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
12843
12844 const ValueOperand value = ToValue(lir, LIsNullAndBranch::Value);
12845
12846 Assembler::Condition cond = JSOpToCondition(compareType, op);
12847
12848 MBasicBlock* ifTrue = lir->ifTrue();
12849 MBasicBlock* ifFalse = lir->ifFalse();
12850
12851 if (isNextBlock(ifFalse->lir())) {
12852 masm.branchTestNull(cond, value, getJumpLabelForBranch(ifTrue));
12853 } else {
12854 masm.branchTestNull(Assembler::InvertCondition(cond), value,
12855 getJumpLabelForBranch(ifFalse));
12856 jumpToBlock(ifTrue);
12857 }
12858}
12859
12860void CodeGenerator::visitIsUndefinedAndBranch(LIsUndefinedAndBranch* lir) {
12861 MCompare::CompareType compareType = lir->cmpMir()->compareType();
12862 MOZ_ASSERT(compareType == MCompare::Compare_Undefined)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(compareType == MCompare::Compare_Undefined)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(compareType == MCompare::Compare_Undefined))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("compareType == MCompare::Compare_Undefined"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12862); AnnotateMozCrashReason("MOZ_ASSERT" "(" "compareType == MCompare::Compare_Undefined"
")"); do { *((volatile int*)__null) = 12862; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
12863
12864 JSOp op = lir->cmpMir()->jsop();
12865 MOZ_ASSERT(IsStrictEqualityOp(op))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(IsStrictEqualityOp(op))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(IsStrictEqualityOp(op)))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("IsStrictEqualityOp(op)"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12865); AnnotateMozCrashReason("MOZ_ASSERT" "(" "IsStrictEqualityOp(op)"
")"); do { *((volatile int*)__null) = 12865; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
12866
12867 const ValueOperand value = ToValue(lir, LIsUndefinedAndBranch::Value);
12868
12869 Assembler::Condition cond = JSOpToCondition(compareType, op);
12870
12871 MBasicBlock* ifTrue = lir->ifTrue();
12872 MBasicBlock* ifFalse = lir->ifFalse();
12873
12874 if (isNextBlock(ifFalse->lir())) {
12875 masm.branchTestUndefined(cond, value, getJumpLabelForBranch(ifTrue));
12876 } else {
12877 masm.branchTestUndefined(Assembler::InvertCondition(cond), value,
12878 getJumpLabelForBranch(ifFalse));
12879 jumpToBlock(ifTrue);
12880 }
12881}
12882
12883void CodeGenerator::visitSameValueDouble(LSameValueDouble* lir) {
12884 FloatRegister left = ToFloatRegister(lir->left());
12885 FloatRegister right = ToFloatRegister(lir->right());
12886 FloatRegister temp = ToFloatRegister(lir->temp0());
12887 Register output = ToRegister(lir->output());
12888
12889 masm.sameValueDouble(left, right, temp, output);
12890}
12891
12892void CodeGenerator::visitSameValue(LSameValue* lir) {
12893 ValueOperand lhs = ToValue(lir, LSameValue::LhsIndex);
12894 ValueOperand rhs = ToValue(lir, LSameValue::RhsIndex);
12895 Register output = ToRegister(lir->output());
12896
12897 using Fn = bool (*)(JSContext*, HandleValue, HandleValue, bool*);
12898 OutOfLineCode* ool =
12899 oolCallVM<Fn, SameValue>(lir, ArgList(lhs, rhs), StoreRegisterTo(output));
12900
12901 // First check to see if the values have identical bits.
12902 // This is correct for SameValue because SameValue(NaN,NaN) is true,
12903 // and SameValue(0,-0) is false.
12904 masm.branch64(Assembler::NotEqual, lhs.toRegister64(), rhs.toRegister64(),
12905 ool->entry());
12906 masm.move32(Imm32(1), output);
12907
12908 // If this fails, call SameValue.
12909 masm.bind(ool->rejoin());
12910}
12911
12912void CodeGenerator::emitConcat(LInstruction* lir, Register lhs, Register rhs,
12913 Register output) {
12914 using Fn =
12915 JSString* (*)(JSContext*, HandleString, HandleString, js::gc::Heap);
12916 OutOfLineCode* ool = oolCallVM<Fn, ConcatStrings<CanGC>>(
12917 lir, ArgList(lhs, rhs, static_cast<Imm32>(int32_t(gc::Heap::Default))),
12918 StoreRegisterTo(output));
12919
12920 const JitZone* jitZone = gen->realm->zone()->jitZone();
12921 JitCode* stringConcatStub =
12922 jitZone->stringConcatStubNoBarrier(&zoneStubsToReadBarrier_);
12923 masm.call(stringConcatStub);
12924 masm.branchTestPtr(Assembler::Zero, output, output, ool->entry());
12925
12926 masm.bind(ool->rejoin());
12927}
12928
12929void CodeGenerator::visitConcat(LConcat* lir) {
12930 Register lhs = ToRegister(lir->lhs());
12931 Register rhs = ToRegister(lir->rhs());
12932
12933 Register output = ToRegister(lir->output());
12934
12935 MOZ_ASSERT(lhs == CallTempReg0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(lhs == CallTempReg0)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(lhs == CallTempReg0))), 0)))
{ do { } while (false); MOZ_ReportAssertionFailure("lhs == CallTempReg0"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12935); AnnotateMozCrashReason("MOZ_ASSERT" "(" "lhs == CallTempReg0"
")"); do { *((volatile int*)__null) = 12935; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
12936 MOZ_ASSERT(rhs == CallTempReg1)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(rhs == CallTempReg1)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(rhs == CallTempReg1))), 0)))
{ do { } while (false); MOZ_ReportAssertionFailure("rhs == CallTempReg1"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12936); AnnotateMozCrashReason("MOZ_ASSERT" "(" "rhs == CallTempReg1"
")"); do { *((volatile int*)__null) = 12936; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
12937 MOZ_ASSERT(ToRegister(lir->temp0()) == CallTempReg0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ToRegister(lir->temp0()) == CallTempReg0)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(ToRegister(lir->temp0()) == CallTempReg0))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("ToRegister(lir->temp0()) == CallTempReg0"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12937); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ToRegister(lir->temp0()) == CallTempReg0"
")"); do { *((volatile int*)__null) = 12937; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
12938 MOZ_ASSERT(ToRegister(lir->temp1()) == CallTempReg1)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ToRegister(lir->temp1()) == CallTempReg1)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(ToRegister(lir->temp1()) == CallTempReg1))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("ToRegister(lir->temp1()) == CallTempReg1"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12938); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ToRegister(lir->temp1()) == CallTempReg1"
")"); do { *((volatile int*)__null) = 12938; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
12939 MOZ_ASSERT(ToRegister(lir->temp2()) == CallTempReg2)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ToRegister(lir->temp2()) == CallTempReg2)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(ToRegister(lir->temp2()) == CallTempReg2))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("ToRegister(lir->temp2()) == CallTempReg2"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12939); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ToRegister(lir->temp2()) == CallTempReg2"
")"); do { *((volatile int*)__null) = 12939; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
12940 MOZ_ASSERT(ToRegister(lir->temp3()) == CallTempReg3)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ToRegister(lir->temp3()) == CallTempReg3)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(ToRegister(lir->temp3()) == CallTempReg3))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("ToRegister(lir->temp3()) == CallTempReg3"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12940); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ToRegister(lir->temp3()) == CallTempReg3"
")"); do { *((volatile int*)__null) = 12940; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
12941 MOZ_ASSERT(ToRegister(lir->temp4()) == CallTempReg4)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ToRegister(lir->temp4()) == CallTempReg4)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(ToRegister(lir->temp4()) == CallTempReg4))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("ToRegister(lir->temp4()) == CallTempReg4"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12941); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ToRegister(lir->temp4()) == CallTempReg4"
")"); do { *((volatile int*)__null) = 12941; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
12942 MOZ_ASSERT(output == CallTempReg5)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(output == CallTempReg5)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(output == CallTempReg5))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("output == CallTempReg5"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12942); AnnotateMozCrashReason("MOZ_ASSERT" "(" "output == CallTempReg5"
")"); do { *((volatile int*)__null) = 12942; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
12943
12944 emitConcat(lir, lhs, rhs, output);
12945}
12946
12947static void CopyStringChars(MacroAssembler& masm, Register to, Register from,
12948 Register len, Register byteOpScratch,
12949 CharEncoding fromEncoding, CharEncoding toEncoding,
12950 size_t maximumLength = SIZE_MAX(18446744073709551615UL)) {
12951 // Copy |len| char16_t code units from |from| to |to|. Assumes len > 0
12952 // (checked below in debug builds), and when done |to| must point to the
12953 // next available char.
12954
12955#ifdef DEBUG1
12956 Label ok;
12957 masm.branch32(Assembler::GreaterThan, len, Imm32(0), &ok);
12958 masm.assumeUnreachable("Length should be greater than 0.");
12959 masm.bind(&ok);
12960
12961 if (maximumLength != SIZE_MAX(18446744073709551615UL)) {
12962 MOZ_ASSERT(maximumLength <= INT32_MAX, "maximum length fits into int32")do { static_assert( mozilla::detail::AssertionConditionType<
decltype(maximumLength <= (2147483647))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(maximumLength <= (2147483647
)))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("maximumLength <= (2147483647)" " (" "maximum length fits into int32"
")", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12962); AnnotateMozCrashReason("MOZ_ASSERT" "(" "maximumLength <= (2147483647)"
") (" "maximum length fits into int32" ")"); do { *((volatile
int*)__null) = 12962; __attribute__((nomerge)) ::abort(); } while
(false); } } while (false)
;
12963
12964 Label ok;
12965 masm.branchPtr(Assembler::BelowOrEqual, len, Imm32(maximumLength), &ok);
12966 masm.assumeUnreachable("Length should not exceed maximum length.");
12967 masm.bind(&ok);
12968 }
12969#endif
12970
12971 MOZ_ASSERT_IF(toEncoding == CharEncoding::Latin1,do { if (toEncoding == CharEncoding::Latin1) { do { static_assert
( mozilla::detail::AssertionConditionType<decltype(fromEncoding
== CharEncoding::Latin1)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(fromEncoding == CharEncoding
::Latin1))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("fromEncoding == CharEncoding::Latin1", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12972); AnnotateMozCrashReason("MOZ_ASSERT" "(" "fromEncoding == CharEncoding::Latin1"
")"); do { *((volatile int*)__null) = 12972; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false); } } while
(false)
12972 fromEncoding == CharEncoding::Latin1)do { if (toEncoding == CharEncoding::Latin1) { do { static_assert
( mozilla::detail::AssertionConditionType<decltype(fromEncoding
== CharEncoding::Latin1)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(fromEncoding == CharEncoding
::Latin1))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("fromEncoding == CharEncoding::Latin1", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 12972); AnnotateMozCrashReason("MOZ_ASSERT" "(" "fromEncoding == CharEncoding::Latin1"
")"); do { *((volatile int*)__null) = 12972; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false); } } while
(false)
;
12973
12974 size_t fromWidth =
12975 fromEncoding == CharEncoding::Latin1 ? sizeof(char) : sizeof(char16_t);
12976 size_t toWidth =
12977 toEncoding == CharEncoding::Latin1 ? sizeof(char) : sizeof(char16_t);
12978
12979 // Try to copy multiple characters at once when both encoding are equal.
12980 if (fromEncoding == toEncoding) {
12981 constexpr size_t ptrWidth = sizeof(uintptr_t);
12982
12983 // Copy |width| bytes and then adjust |from| and |to|.
12984 auto copyCharacters = [&](size_t width) {
12985 static_assert(ptrWidth <= 8, "switch handles only up to eight bytes");
12986
12987 switch (width) {
12988 case 1:
12989 masm.load8ZeroExtend(Address(from, 0), byteOpScratch);
12990 masm.store8(byteOpScratch, Address(to, 0));
12991 break;
12992 case 2:
12993 masm.load16ZeroExtend(Address(from, 0), byteOpScratch);
12994 masm.store16(byteOpScratch, Address(to, 0));
12995 break;
12996 case 4:
12997 masm.load32(Address(from, 0), byteOpScratch);
12998 masm.store32(byteOpScratch, Address(to, 0));
12999 break;
13000 case 8:
13001 MOZ_ASSERT(width == ptrWidth)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(width == ptrWidth)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(width == ptrWidth))), 0))) {
do { } while (false); MOZ_ReportAssertionFailure("width == ptrWidth"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 13001); AnnotateMozCrashReason("MOZ_ASSERT" "(" "width == ptrWidth"
")"); do { *((volatile int*)__null) = 13001; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
13002 masm.loadPtr(Address(from, 0), byteOpScratch);
13003 masm.storePtr(byteOpScratch, Address(to, 0));
13004 break;
13005 }
13006
13007 masm.addPtr(Imm32(width), from);
13008 masm.addPtr(Imm32(width), to);
13009 };
13010
13011 // First align |len| to pointer width.
13012 Label done;
13013 for (size_t width = fromWidth; width < ptrWidth; width *= 2) {
13014 // Number of characters which fit into |width| bytes.
13015 size_t charsPerWidth = width / fromWidth;
13016
13017 if (charsPerWidth < maximumLength) {
13018 Label next;
13019 masm.branchTest32(Assembler::Zero, len, Imm32(charsPerWidth), &next);
13020
13021 copyCharacters(width);
13022
13023 masm.branchSub32(Assembler::Zero, Imm32(charsPerWidth), len, &done);
13024 masm.bind(&next);
13025 } else if (charsPerWidth == maximumLength) {
13026 copyCharacters(width);
13027 masm.sub32(Imm32(charsPerWidth), len);
13028 }
13029 }
13030
13031 size_t maxInlineLength;
13032 if (fromEncoding == CharEncoding::Latin1) {
13033 maxInlineLength = JSFatInlineString::MAX_LENGTH_LATIN1;
13034 } else {
13035 maxInlineLength = JSFatInlineString::MAX_LENGTH_TWO_BYTE;
13036 }
13037
13038 // Number of characters which fit into a single register.
13039 size_t charsPerPtr = ptrWidth / fromWidth;
13040
13041 // Unroll small loops.
13042 constexpr size_t unrollLoopLimit = 3;
13043 size_t loopCount = std::min(maxInlineLength, maximumLength) / charsPerPtr;
13044
13045#ifdef JS_64BIT1
13046 static constexpr size_t latin1MaxInlineByteLength =
13047 JSFatInlineString::MAX_LENGTH_LATIN1 * sizeof(char);
13048 static constexpr size_t twoByteMaxInlineByteLength =
13049 JSFatInlineString::MAX_LENGTH_TWO_BYTE * sizeof(char16_t);
13050
13051 // |unrollLoopLimit| should be large enough to allow loop unrolling on
13052 // 64-bit targets.
13053 static_assert(latin1MaxInlineByteLength / ptrWidth == unrollLoopLimit,
13054 "Latin-1 loops are unrolled on 64-bit");
13055 static_assert(twoByteMaxInlineByteLength / ptrWidth == unrollLoopLimit,
13056 "Two-byte loops are unrolled on 64-bit");
13057#endif
13058
13059 if (loopCount <= unrollLoopLimit) {
13060 Label labels[unrollLoopLimit];
13061
13062 // Check up front how many characters can be copied.
13063 for (size_t i = 1; i < loopCount; i++) {
13064 masm.branch32(Assembler::Below, len, Imm32((i + 1) * charsPerPtr),
13065 &labels[i]);
13066 }
13067
13068 // Generate the unrolled loop body.
13069 for (size_t i = loopCount; i > 0; i--) {
13070 copyCharacters(ptrWidth);
13071 masm.sub32(Imm32(charsPerPtr), len);
13072
13073 // Jump target for the previous length check.
13074 if (i != 1) {
13075 masm.bind(&labels[i - 1]);
13076 }
13077 }
13078 } else {
13079 Label start;
13080 masm.bind(&start);
13081 copyCharacters(ptrWidth);
13082 masm.branchSub32(Assembler::NonZero, Imm32(charsPerPtr), len, &start);
13083 }
13084
13085 masm.bind(&done);
13086 } else {
13087 Label start;
13088 masm.bind(&start);
13089 masm.loadChar(Address(from, 0), byteOpScratch, fromEncoding);
13090 masm.storeChar(byteOpScratch, Address(to, 0), toEncoding);
13091 masm.addPtr(Imm32(fromWidth), from);
13092 masm.addPtr(Imm32(toWidth), to);
13093 masm.branchSub32(Assembler::NonZero, Imm32(1), len, &start);
13094 }
13095}
13096
13097static void CopyStringChars(MacroAssembler& masm, Register to, Register from,
13098 Register len, Register byteOpScratch,
13099 CharEncoding encoding, size_t maximumLength) {
13100 CopyStringChars(masm, to, from, len, byteOpScratch, encoding, encoding,
13101 maximumLength);
13102}
13103
13104static void CopyStringCharsMaybeInflate(MacroAssembler& masm, Register input,
13105 Register destChars, Register temp1,
13106 Register temp2) {
13107 // destChars is TwoByte and input is a Latin1 or TwoByte string, so we may
13108 // have to inflate.
13109
13110 Label isLatin1, done;
13111 masm.loadStringLength(input, temp1);
13112 masm.branchLatin1String(input, &isLatin1);
13113 {
13114 masm.loadStringChars(input, temp2, CharEncoding::TwoByte);
13115 masm.movePtr(temp2, input);
13116 CopyStringChars(masm, destChars, input, temp1, temp2,
13117 CharEncoding::TwoByte);
13118 masm.jump(&done);
13119 }
13120 masm.bind(&isLatin1);
13121 {
13122 masm.loadStringChars(input, temp2, CharEncoding::Latin1);
13123 masm.movePtr(temp2, input);
13124 CopyStringChars(masm, destChars, input, temp1, temp2, CharEncoding::Latin1,
13125 CharEncoding::TwoByte);
13126 }
13127 masm.bind(&done);
13128}
13129
13130static void AllocateThinOrFatInlineString(MacroAssembler& masm, Register output,
13131 Register length, Register temp,
13132 gc::Heap initialStringHeap,
13133 Label* failure,
13134 CharEncoding encoding) {
13135#ifdef DEBUG1
13136 size_t maxInlineLength;
13137 if (encoding == CharEncoding::Latin1) {
13138 maxInlineLength = JSFatInlineString::MAX_LENGTH_LATIN1;
13139 } else {
13140 maxInlineLength = JSFatInlineString::MAX_LENGTH_TWO_BYTE;
13141 }
13142
13143 Label ok;
13144 masm.branch32(Assembler::BelowOrEqual, length, Imm32(maxInlineLength), &ok);
13145 masm.assumeUnreachable("string length too large to be allocated as inline");
13146 masm.bind(&ok);
13147#endif
13148
13149 size_t maxThinInlineLength;
13150 if (encoding == CharEncoding::Latin1) {
13151 maxThinInlineLength = JSThinInlineString::MAX_LENGTH_LATIN1;
13152 } else {
13153 maxThinInlineLength = JSThinInlineString::MAX_LENGTH_TWO_BYTE;
13154 }
13155
13156 Label isFat, allocDone;
13157 masm.branch32(Assembler::Above, length, Imm32(maxThinInlineLength), &isFat);
13158 {
13159 uint32_t flags = JSString::INIT_THIN_INLINE_FLAGS;
13160 if (encoding == CharEncoding::Latin1) {
13161 flags |= JSString::LATIN1_CHARS_BIT;
13162 }
13163 masm.newGCString(output, temp, initialStringHeap, failure);
13164 masm.store32(Imm32(flags), Address(output, JSString::offsetOfFlags()));
13165 masm.jump(&allocDone);
13166 }
13167 masm.bind(&isFat);
13168 {
13169 uint32_t flags = JSString::INIT_FAT_INLINE_FLAGS;
13170 if (encoding == CharEncoding::Latin1) {
13171 flags |= JSString::LATIN1_CHARS_BIT;
13172 }
13173 masm.newGCFatInlineString(output, temp, initialStringHeap, failure);
13174 masm.store32(Imm32(flags), Address(output, JSString::offsetOfFlags()));
13175 }
13176 masm.bind(&allocDone);
13177
13178 // Store length.
13179 masm.store32(length, Address(output, JSString::offsetOfLength()));
13180}
13181
13182static void ConcatInlineString(MacroAssembler& masm, Register lhs, Register rhs,
13183 Register output, Register temp1, Register temp2,
13184 Register temp3, gc::Heap initialStringHeap,
13185 Label* failure, CharEncoding encoding) {
13186 JitSpew(JitSpew_Codegen, "# Emitting ConcatInlineString (encoding=%s)",
13187 (encoding == CharEncoding::Latin1 ? "Latin-1" : "Two-Byte"));
13188
13189 // State: result length in temp2.
13190
13191 // Ensure both strings are linear.
13192 masm.branchIfRope(lhs, failure);
13193 masm.branchIfRope(rhs, failure);
13194
13195 // Allocate a JSThinInlineString or JSFatInlineString.
13196 AllocateThinOrFatInlineString(masm, output, temp2, temp1, initialStringHeap,
13197 failure, encoding);
13198
13199 // Load chars pointer in temp2.
13200 masm.loadInlineStringCharsForStore(output, temp2);
13201
13202 auto copyChars = [&](Register src) {
13203 if (encoding == CharEncoding::TwoByte) {
13204 CopyStringCharsMaybeInflate(masm, src, temp2, temp1, temp3);
13205 } else {
13206 masm.loadStringLength(src, temp3);
13207 masm.loadStringChars(src, temp1, CharEncoding::Latin1);
13208 masm.movePtr(temp1, src);
13209 CopyStringChars(masm, temp2, src, temp3, temp1, CharEncoding::Latin1);
13210 }
13211 };
13212
13213 // Copy lhs chars. Note that this advances temp2 to point to the next
13214 // char. This also clobbers the lhs register.
13215 copyChars(lhs);
13216
13217 // Copy rhs chars. Clobbers the rhs register.
13218 copyChars(rhs);
13219}
13220
13221void CodeGenerator::visitSubstr(LSubstr* lir) {
13222 Register string = ToRegister(lir->string());
13223 Register begin = ToRegister(lir->begin());
13224 Register length = ToRegister(lir->length());
13225 Register output = ToRegister(lir->output());
13226 Register temp0 = ToRegister(lir->temp0());
13227 Register temp2 = ToRegister(lir->temp2());
13228
13229 // On x86 there are not enough registers. In that case reuse the string
13230 // register as temporary.
13231 Register temp1 =
13232 lir->temp1()->isBogusTemp() ? string : ToRegister(lir->temp1());
13233
13234 size_t maximumLength = SIZE_MAX(18446744073709551615UL);
13235
13236 Range* range = lir->mir()->length()->range();
13237 if (range && range->hasInt32UpperBound()) {
13238 MOZ_ASSERT(range->upper() >= 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(range->upper() >= 0)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(range->upper() >= 0)))
, 0))) { do { } while (false); MOZ_ReportAssertionFailure("range->upper() >= 0"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 13238); AnnotateMozCrashReason("MOZ_ASSERT" "(" "range->upper() >= 0"
")"); do { *((volatile int*)__null) = 13238; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
13239 maximumLength = size_t(range->upper());
13240 }
13241
13242 static_assert(JSThinInlineString::MAX_LENGTH_TWO_BYTE <=
13243 JSThinInlineString::MAX_LENGTH_LATIN1);
13244
13245 static_assert(JSFatInlineString::MAX_LENGTH_TWO_BYTE <=
13246 JSFatInlineString::MAX_LENGTH_LATIN1);
13247
13248 bool tryFatInlineOrDependent =
13249 maximumLength > JSThinInlineString::MAX_LENGTH_TWO_BYTE;
13250 bool tryDependent = maximumLength > JSFatInlineString::MAX_LENGTH_TWO_BYTE;
13251
13252#ifdef DEBUG1
13253 if (maximumLength != SIZE_MAX(18446744073709551615UL)) {
13254 Label ok;
13255 masm.branch32(Assembler::BelowOrEqual, length, Imm32(maximumLength), &ok);
13256 masm.assumeUnreachable("length should not exceed maximum length");
13257 masm.bind(&ok);
13258 }
13259#endif
13260
13261 Label nonZero, nonInput;
13262
13263 // For every edge case use the C++ variant.
13264 // Note: we also use this upon allocation failure in newGCString and
13265 // newGCFatInlineString. To squeeze out even more performance those failures
13266 // can be handled by allocate in ool code and returning to jit code to fill
13267 // in all data.
13268 using Fn = JSString* (*)(JSContext* cx, HandleString str, int32_t begin,
13269 int32_t len);
13270 OutOfLineCode* ool = oolCallVM<Fn, SubstringKernel>(
13271 lir, ArgList(string, begin, length), StoreRegisterTo(output));
13272 Label* slowPath = ool->entry();
13273 Label* done = ool->rejoin();
13274
13275 // Zero length, return emptystring.
13276 masm.branchTest32(Assembler::NonZero, length, length, &nonZero);
13277 const JSAtomState& names = gen->runtime->names();
13278 masm.movePtr(ImmGCPtr(names.empty_), output);
13279 masm.jump(done);
13280
13281 // Substring from 0..|str.length|, return str.
13282 masm.bind(&nonZero);
13283 masm.branch32(Assembler::NotEqual,
13284 Address(string, JSString::offsetOfLength()), length, &nonInput);
13285#ifdef DEBUG1
13286 {
13287 Label ok;
13288 masm.branchTest32(Assembler::Zero, begin, begin, &ok);
13289 masm.assumeUnreachable("length == str.length implies begin == 0");
13290 masm.bind(&ok);
13291 }
13292#endif
13293 masm.movePtr(string, output);
13294 masm.jump(done);
13295
13296 // Use slow path for ropes.
13297 masm.bind(&nonInput);
13298 masm.branchIfRope(string, slowPath);
13299
13300 // Optimize one and two character strings.
13301 Label nonStatic;
13302 masm.branch32(Assembler::Above, length, Imm32(2), &nonStatic);
13303 {
13304 Label loadLengthOne, loadLengthTwo;
13305
13306 auto loadChars = [&](CharEncoding encoding, bool fallthru) {
13307 size_t size = encoding == CharEncoding::Latin1 ? sizeof(JS::Latin1Char)
13308 : sizeof(char16_t);
13309
13310 masm.loadStringChars(string, temp0, encoding);
13311 masm.loadChar(temp0, begin, temp2, encoding);
13312 masm.branch32(Assembler::Equal, length, Imm32(1), &loadLengthOne);
13313 masm.loadChar(temp0, begin, temp0, encoding, int32_t(size));
13314 if (!fallthru) {
13315 masm.jump(&loadLengthTwo);
13316 }
13317 };
13318
13319 Label isLatin1;
13320 masm.branchLatin1String(string, &isLatin1);
13321 loadChars(CharEncoding::TwoByte, /* fallthru = */ false);
13322
13323 masm.bind(&isLatin1);
13324 loadChars(CharEncoding::Latin1, /* fallthru = */ true);
13325
13326 // Try to load a length-two static string.
13327 masm.bind(&loadLengthTwo);
13328 masm.lookupStaticString(temp2, temp0, output, gen->runtime->staticStrings(),
13329 &nonStatic);
13330 masm.jump(done);
13331
13332 // Try to load a length-one static string.
13333 masm.bind(&loadLengthOne);
13334 masm.lookupStaticString(temp2, output, gen->runtime->staticStrings(),
13335 &nonStatic);
13336 masm.jump(done);
13337 }
13338 masm.bind(&nonStatic);
13339
13340 // Allocate either a JSThinInlineString or JSFatInlineString, or jump to
13341 // notInline if we need a dependent string.
13342 Label notInline;
13343 {
13344 static_assert(JSThinInlineString::MAX_LENGTH_LATIN1 <
13345 JSFatInlineString::MAX_LENGTH_LATIN1);
13346 static_assert(JSThinInlineString::MAX_LENGTH_TWO_BYTE <
13347 JSFatInlineString::MAX_LENGTH_TWO_BYTE);
13348
13349 // Use temp2 to store the JS(Thin|Fat)InlineString flags. This avoids having
13350 // duplicate newGCString/newGCFatInlineString codegen for Latin1 vs TwoByte
13351 // strings.
13352
13353 Label allocFat, allocDone;
13354 if (tryFatInlineOrDependent) {
13355 Label isLatin1, allocThin;
13356 masm.branchLatin1String(string, &isLatin1);
13357 {
13358 if (tryDependent) {
13359 masm.branch32(Assembler::Above, length,
13360 Imm32(JSFatInlineString::MAX_LENGTH_TWO_BYTE),
13361 &notInline);
13362 }
13363 masm.move32(Imm32(0), temp2);
13364 masm.branch32(Assembler::Above, length,
13365 Imm32(JSThinInlineString::MAX_LENGTH_TWO_BYTE),
13366 &allocFat);
13367 masm.jump(&allocThin);
13368 }
13369
13370 masm.bind(&isLatin1);
13371 {
13372 if (tryDependent) {
13373 masm.branch32(Assembler::Above, length,
13374 Imm32(JSFatInlineString::MAX_LENGTH_LATIN1),
13375 &notInline);
13376 }
13377 masm.move32(Imm32(JSString::LATIN1_CHARS_BIT), temp2);
13378 masm.branch32(Assembler::Above, length,
13379 Imm32(JSThinInlineString::MAX_LENGTH_LATIN1), &allocFat);
13380 }
13381
13382 masm.bind(&allocThin);
13383 } else {
13384 masm.load32(Address(string, JSString::offsetOfFlags()), temp2);
13385 masm.and32(Imm32(JSString::LATIN1_CHARS_BIT), temp2);
13386 }
13387
13388 {
13389 masm.newGCString(output, temp0, initialStringHeap(), slowPath);
13390 masm.or32(Imm32(JSString::INIT_THIN_INLINE_FLAGS), temp2);
13391 }
13392
13393 if (tryFatInlineOrDependent) {
13394 masm.jump(&allocDone);
13395
13396 masm.bind(&allocFat);
13397 {
13398 masm.newGCFatInlineString(output, temp0, initialStringHeap(), slowPath);
13399 masm.or32(Imm32(JSString::INIT_FAT_INLINE_FLAGS), temp2);
13400 }
13401
13402 masm.bind(&allocDone);
13403 }
13404
13405 masm.store32(temp2, Address(output, JSString::offsetOfFlags()));
13406 masm.store32(length, Address(output, JSString::offsetOfLength()));
13407
13408 auto initializeInlineString = [&](CharEncoding encoding) {
13409 masm.loadStringChars(string, temp0, encoding);
13410 masm.addToCharPtr(temp0, begin, encoding);
13411 if (temp1 == string) {
13412 masm.push(string);
13413 }
13414 masm.loadInlineStringCharsForStore(output, temp1);
13415 CopyStringChars(masm, temp1, temp0, length, temp2, encoding,
13416 maximumLength);
13417 masm.loadStringLength(output, length);
13418 if (temp1 == string) {
13419 masm.pop(string);
13420 }
13421 };
13422
13423 Label isInlineLatin1;
13424 masm.branchTest32(Assembler::NonZero, temp2,
13425 Imm32(JSString::LATIN1_CHARS_BIT), &isInlineLatin1);
13426 initializeInlineString(CharEncoding::TwoByte);
13427 masm.jump(done);
13428
13429 masm.bind(&isInlineLatin1);
13430 initializeInlineString(CharEncoding::Latin1);
13431 }
13432
13433 // Handle other cases with a DependentString.
13434 if (tryDependent) {
13435 masm.jump(done);
13436
13437 masm.bind(&notInline);
13438 masm.newGCString(output, temp0, gen->initialStringHeap(), slowPath);
13439 masm.store32(length, Address(output, JSString::offsetOfLength()));
13440
13441 // Note: no post barrier is needed because the dependent string is either
13442 // allocated in the nursery or both strings are tenured (if nursery strings
13443 // are disabled for this zone).
13444 EmitInitDependentStringBase(masm, output, string, temp0, temp2,
13445 /* needsPostBarrier = */ false);
13446
13447 auto initializeDependentString = [&](CharEncoding encoding) {
13448 uint32_t flags = JSString::INIT_DEPENDENT_FLAGS;
13449 if (encoding == CharEncoding::Latin1) {
13450 flags |= JSString::LATIN1_CHARS_BIT;
13451 }
13452 masm.store32(Imm32(flags), Address(output, JSString::offsetOfFlags()));
13453 masm.loadNonInlineStringChars(string, temp0, encoding);
13454 masm.addToCharPtr(temp0, begin, encoding);
13455 masm.storeNonInlineStringChars(temp0, output);
13456 };
13457
13458 Label isLatin1;
13459 masm.branchLatin1String(string, &isLatin1);
13460 initializeDependentString(CharEncoding::TwoByte);
13461 masm.jump(done);
13462
13463 masm.bind(&isLatin1);
13464 initializeDependentString(CharEncoding::Latin1);
13465 }
13466
13467 masm.bind(done);
13468}
13469
13470JitCode* JitZone::generateStringConcatStub(JSContext* cx) {
13471 JitSpew(JitSpew_Codegen, "# Emitting StringConcat stub");
13472
13473 TempAllocator temp(&cx->tempLifoAlloc());
13474 JitContext jcx(cx);
13475 StackMacroAssembler masm(cx, temp);
13476 AutoCreatedBy acb(masm, "JitZone::generateStringConcatStub");
13477
13478 Register lhs = CallTempReg0;
13479 Register rhs = CallTempReg1;
13480 Register temp1 = CallTempReg2;
13481 Register temp2 = CallTempReg3;
13482 Register temp3 = CallTempReg4;
13483 Register output = CallTempReg5;
13484
13485 Label failure;
13486#ifdef JS_USE_LINK_REGISTER
13487 masm.pushReturnAddress();
13488#endif
13489 masm.Push(FramePointer);
13490 masm.moveStackPtrTo(FramePointer);
13491
13492 // If lhs is empty, return rhs.
13493 Label leftEmpty;
13494 masm.loadStringLength(lhs, temp1);
13495 masm.branchTest32(Assembler::Zero, temp1, temp1, &leftEmpty);
13496
13497 // If rhs is empty, return lhs.
13498 Label rightEmpty;
13499 masm.loadStringLength(rhs, temp2);
13500 masm.branchTest32(Assembler::Zero, temp2, temp2, &rightEmpty);
13501
13502 masm.add32(temp1, temp2);
13503
13504 // Check if we can use a JSInlineString. The result is a Latin1 string if
13505 // lhs and rhs are both Latin1, so we AND the flags.
13506 Label isInlineTwoByte, isInlineLatin1;
13507 masm.load32(Address(lhs, JSString::offsetOfFlags()), temp1);
13508 masm.and32(Address(rhs, JSString::offsetOfFlags()), temp1);
13509
13510 Label isLatin1, notInline;
13511 masm.branchTest32(Assembler::NonZero, temp1,
13512 Imm32(JSString::LATIN1_CHARS_BIT), &isLatin1);
13513 {
13514 masm.branch32(Assembler::BelowOrEqual, temp2,
13515 Imm32(JSFatInlineString::MAX_LENGTH_TWO_BYTE),
13516 &isInlineTwoByte);
13517 masm.jump(&notInline);
13518 }
13519 masm.bind(&isLatin1);
13520 {
13521 masm.branch32(Assembler::BelowOrEqual, temp2,
13522 Imm32(JSFatInlineString::MAX_LENGTH_LATIN1), &isInlineLatin1);
13523 }
13524 masm.bind(&notInline);
13525
13526 // Keep AND'ed flags in temp1.
13527
13528 // Ensure result length <= JSString::MAX_LENGTH.
13529 masm.branch32(Assembler::Above, temp2, Imm32(JSString::MAX_LENGTH), &failure);
13530
13531 // Allocate a new rope, guaranteed to be in the nursery if initialStringHeap
13532 // == gc::Heap::Default. (As a result, no post barriers are needed below.)
13533 masm.newGCString(output, temp3, initialStringHeap, &failure);
13534
13535 // Store rope length and flags. temp1 still holds the result of AND'ing the
13536 // lhs and rhs flags, so we just have to clear the other flags to get our rope
13537 // flags (Latin1 if both lhs and rhs are Latin1).
13538 static_assert(JSString::INIT_ROPE_FLAGS == 0,
13539 "Rope type flags must have no bits set");
13540 masm.and32(Imm32(JSString::LATIN1_CHARS_BIT), temp1);
13541 masm.store32(temp1, Address(output, JSString::offsetOfFlags()));
13542 masm.store32(temp2, Address(output, JSString::offsetOfLength()));
13543
13544 // Store left and right nodes.
13545 masm.storeRopeChildren(lhs, rhs, output);
13546 masm.pop(FramePointer);
13547 masm.ret();
13548
13549 masm.bind(&leftEmpty);
13550 masm.mov(rhs, output);
13551 masm.pop(FramePointer);
13552 masm.ret();
13553
13554 masm.bind(&rightEmpty);
13555 masm.mov(lhs, output);
13556 masm.pop(FramePointer);
13557 masm.ret();
13558
13559 masm.bind(&isInlineTwoByte);
13560 ConcatInlineString(masm, lhs, rhs, output, temp1, temp2, temp3,
13561 initialStringHeap, &failure, CharEncoding::TwoByte);
13562 masm.pop(FramePointer);
13563 masm.ret();
13564
13565 masm.bind(&isInlineLatin1);
13566 ConcatInlineString(masm, lhs, rhs, output, temp1, temp2, temp3,
13567 initialStringHeap, &failure, CharEncoding::Latin1);
13568 masm.pop(FramePointer);
13569 masm.ret();
13570
13571 masm.bind(&failure);
13572 masm.movePtr(ImmPtr(nullptr), output);
13573 masm.pop(FramePointer);
13574 masm.ret();
13575
13576 Linker linker(masm);
13577 JitCode* code = linker.newCode(cx, CodeKind::Other);
13578
13579 CollectPerfSpewerJitCodeProfile(code, "StringConcatStub");
13580#ifdef MOZ_VTUNE1
13581 vtune::MarkStub(code, "StringConcatStub");
13582#endif
13583
13584 return code;
13585}
13586
13587void JitRuntime::generateFreeStub(MacroAssembler& masm) {
13588 AutoCreatedBy acb(masm, "JitRuntime::generateFreeStub");
13589
13590 const Register regSlots = CallTempReg0;
13591
13592 freeStubOffset_ = startTrampolineCode(masm);
13593
13594#ifdef JS_USE_LINK_REGISTER
13595 masm.pushReturnAddress();
13596#endif
13597 AllocatableRegisterSet regs(RegisterSet::Volatile());
13598 regs.takeUnchecked(regSlots);
13599 LiveRegisterSet save(regs.asLiveSet());
13600 masm.PushRegsInMask(save);
13601
13602 const Register regTemp = regs.takeAnyGeneral();
13603 MOZ_ASSERT(regTemp != regSlots)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(regTemp != regSlots)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(regTemp != regSlots))), 0)))
{ do { } while (false); MOZ_ReportAssertionFailure("regTemp != regSlots"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 13603); AnnotateMozCrashReason("MOZ_ASSERT" "(" "regTemp != regSlots"
")"); do { *((volatile int*)__null) = 13603; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
13604
13605 using Fn = void (*)(void* p);
13606 masm.setupUnalignedABICall(regTemp);
13607 masm.passABIArg(regSlots);
13608 masm.callWithABI<Fn, js_free>(ABIType::General,
13609 CheckUnsafeCallWithABI::DontCheckOther);
13610
13611 masm.PopRegsInMask(save);
13612
13613 masm.ret();
13614}
13615
13616void JitRuntime::generateLazyLinkStub(MacroAssembler& masm) {
13617 AutoCreatedBy acb(masm, "JitRuntime::generateLazyLinkStub");
13618
13619 lazyLinkStubOffset_ = startTrampolineCode(masm);
13620
13621#ifdef JS_USE_LINK_REGISTER
13622 masm.pushReturnAddress();
13623#endif
13624 masm.Push(FramePointer);
13625 masm.moveStackPtrTo(FramePointer);
13626
13627 AllocatableGeneralRegisterSet regs(GeneralRegisterSet::Volatile());
13628 Register temp0 = regs.takeAny();
13629 Register temp1 = regs.takeAny();
13630 Register temp2 = regs.takeAny();
13631
13632 masm.loadJSContext(temp0);
13633 masm.enterFakeExitFrame(temp0, temp2, ExitFrameType::LazyLink);
13634 masm.moveStackPtrTo(temp1);
13635
13636 using Fn = uint8_t* (*)(JSContext* cx, LazyLinkExitFrameLayout* frame);
13637 masm.setupUnalignedABICall(temp2);
13638 masm.passABIArg(temp0);
13639 masm.passABIArg(temp1);
13640 masm.callWithABI<Fn, LazyLinkTopActivation>(
13641 ABIType::General, CheckUnsafeCallWithABI::DontCheckHasExitFrame);
13642
13643 // Discard exit frame and restore frame pointer.
13644 masm.leaveExitFrame(0);
13645 masm.pop(FramePointer);
13646
13647#ifdef JS_USE_LINK_REGISTER
13648 // Restore the return address such that the emitPrologue function of the
13649 // CodeGenerator can push it back on the stack with pushReturnAddress.
13650 masm.popReturnAddress();
13651#endif
13652 masm.jump(ReturnReg);
13653}
13654
13655void JitRuntime::generateInterpreterStub(MacroAssembler& masm) {
13656 AutoCreatedBy acb(masm, "JitRuntime::generateInterpreterStub");
13657
13658 interpreterStubOffset_ = startTrampolineCode(masm);
13659
13660#ifdef JS_USE_LINK_REGISTER
13661 masm.pushReturnAddress();
13662#endif
13663 masm.Push(FramePointer);
13664 masm.moveStackPtrTo(FramePointer);
13665
13666 AllocatableGeneralRegisterSet regs(GeneralRegisterSet::Volatile());
13667 Register temp0 = regs.takeAny();
13668 Register temp1 = regs.takeAny();
13669 Register temp2 = regs.takeAny();
13670
13671 masm.loadJSContext(temp0);
13672 masm.enterFakeExitFrame(temp0, temp2, ExitFrameType::InterpreterStub);
13673 masm.moveStackPtrTo(temp1);
13674
13675 using Fn = bool (*)(JSContext* cx, InterpreterStubExitFrameLayout* frame);
13676 masm.setupUnalignedABICall(temp2);
13677 masm.passABIArg(temp0);
13678 masm.passABIArg(temp1);
13679 masm.callWithABI<Fn, InvokeFromInterpreterStub>(
13680 ABIType::General, CheckUnsafeCallWithABI::DontCheckHasExitFrame);
13681
13682 masm.branchIfFalseBool(ReturnReg, masm.failureLabel());
13683
13684 // Discard exit frame and restore frame pointer.
13685 masm.leaveExitFrame(0);
13686 masm.pop(FramePointer);
13687
13688 // InvokeFromInterpreterStub stores the return value in argv[0], where the
13689 // caller stored |this|. Subtract |sizeof(void*)| for the frame pointer we
13690 // just popped.
13691 masm.loadValue(Address(masm.getStackPointer(),
13692 JitFrameLayout::offsetOfThis() - sizeof(void*)),
13693 JSReturnOperand);
13694 masm.ret();
13695}
13696
13697void JitRuntime::generateDoubleToInt32ValueStub(MacroAssembler& masm) {
13698 AutoCreatedBy acb(masm, "JitRuntime::generateDoubleToInt32ValueStub");
13699 doubleToInt32ValueStubOffset_ = startTrampolineCode(masm);
13700
13701 Label done;
13702 masm.branchTestDouble(Assembler::NotEqual, R0, &done);
13703
13704 masm.unboxDouble(R0, FloatReg0);
13705 masm.convertDoubleToInt32(FloatReg0, R1.scratchReg(), &done,
13706 /* negativeZeroCheck = */ false);
13707 masm.tagValue(JSVAL_TYPE_INT32, R1.scratchReg(), R0);
13708
13709 masm.bind(&done);
13710 masm.abiret();
13711}
13712
13713void CodeGenerator::visitLinearizeString(LLinearizeString* lir) {
13714 Register str = ToRegister(lir->str());
13715 Register output = ToRegister(lir->output());
13716
13717 using Fn = JSLinearString* (*)(JSContext*, JSString*);
13718 auto* ool = oolCallVM<Fn, jit::LinearizeForCharAccess>(
13719 lir, ArgList(str), StoreRegisterTo(output));
13720
13721 masm.branchIfRope(str, ool->entry());
13722
13723 masm.movePtr(str, output);
13724 masm.bind(ool->rejoin());
13725}
13726
13727void CodeGenerator::visitLinearizeForCharAccess(LLinearizeForCharAccess* lir) {
13728 Register str = ToRegister(lir->str());
13729 Register index = ToRegister(lir->index());
13730 Register output = ToRegister(lir->output());
13731
13732 using Fn = JSLinearString* (*)(JSContext*, JSString*);
13733 auto* ool = oolCallVM<Fn, jit::LinearizeForCharAccess>(
13734 lir, ArgList(str), StoreRegisterTo(output));
13735
13736 masm.branchIfNotCanLoadStringChar(str, index, output, ool->entry());
13737
13738 masm.movePtr(str, output);
13739 masm.bind(ool->rejoin());
13740}
13741
13742void CodeGenerator::visitLinearizeForCodePointAccess(
13743 LLinearizeForCodePointAccess* lir) {
13744 Register str = ToRegister(lir->str());
13745 Register index = ToRegister(lir->index());
13746 Register output = ToRegister(lir->output());
13747 Register temp = ToRegister(lir->temp0());
13748
13749 using Fn = JSLinearString* (*)(JSContext*, JSString*);
13750 auto* ool = oolCallVM<Fn, jit::LinearizeForCharAccess>(
13751 lir, ArgList(str), StoreRegisterTo(output));
13752
13753 masm.branchIfNotCanLoadStringCodePoint(str, index, output, temp,
13754 ool->entry());
13755
13756 masm.movePtr(str, output);
13757 masm.bind(ool->rejoin());
13758}
13759
13760void CodeGenerator::visitToRelativeStringIndex(LToRelativeStringIndex* lir) {
13761 Register index = ToRegister(lir->index());
13762 Register length = ToRegister(lir->length());
13763 Register output = ToRegister(lir->output());
13764
13765 masm.move32(Imm32(0), output);
13766 masm.cmp32Move32(Assembler::LessThan, index, Imm32(0), length, output);
13767 masm.add32(index, output);
13768}
13769
13770void CodeGenerator::visitCharCodeAt(LCharCodeAt* lir) {
13771 Register str = ToRegister(lir->str());
13772 Register output = ToRegister(lir->output());
13773 Register temp0 = ToRegister(lir->temp0());
13774 Register temp1 = ToRegister(lir->temp1());
13775
13776 using Fn = bool (*)(JSContext*, HandleString, int32_t, uint32_t*);
13777
13778 if (lir->index()->isBogus()) {
13779 auto* ool = oolCallVM<Fn, jit::CharCodeAt>(lir, ArgList(str, Imm32(0)),
13780 StoreRegisterTo(output));
13781 masm.loadStringChar(str, 0, output, temp0, temp1, ool->entry());
13782 masm.bind(ool->rejoin());
13783 } else {
13784 Register index = ToRegister(lir->index());
13785
13786 auto* ool = oolCallVM<Fn, jit::CharCodeAt>(lir, ArgList(str, index),
13787 StoreRegisterTo(output));
13788 masm.loadStringChar(str, index, output, temp0, temp1, ool->entry());
13789 masm.bind(ool->rejoin());
13790 }
13791}
13792
13793void CodeGenerator::visitCharCodeAtOrNegative(LCharCodeAtOrNegative* lir) {
13794 Register str = ToRegister(lir->str());
13795 Register output = ToRegister(lir->output());
13796 Register temp0 = ToRegister(lir->temp0());
13797 Register temp1 = ToRegister(lir->temp1());
13798
13799 using Fn = bool (*)(JSContext*, HandleString, int32_t, uint32_t*);
13800
13801 // Return -1 for out-of-bounds access.
13802 masm.move32(Imm32(-1), output);
13803
13804 if (lir->index()->isBogus()) {
13805 auto* ool = oolCallVM<Fn, jit::CharCodeAt>(lir, ArgList(str, Imm32(0)),
13806 StoreRegisterTo(output));
13807
13808 masm.branch32(Assembler::Equal, Address(str, JSString::offsetOfLength()),
13809 Imm32(0), ool->rejoin());
13810 masm.loadStringChar(str, 0, output, temp0, temp1, ool->entry());
13811 masm.bind(ool->rejoin());
13812 } else {
13813 Register index = ToRegister(lir->index());
13814
13815 auto* ool = oolCallVM<Fn, jit::CharCodeAt>(lir, ArgList(str, index),
13816 StoreRegisterTo(output));
13817
13818 masm.spectreBoundsCheck32(index, Address(str, JSString::offsetOfLength()),
13819 temp0, ool->rejoin());
13820 masm.loadStringChar(str, index, output, temp0, temp1, ool->entry());
13821 masm.bind(ool->rejoin());
13822 }
13823}
13824
13825void CodeGenerator::visitCodePointAt(LCodePointAt* lir) {
13826 Register str = ToRegister(lir->str());
13827 Register index = ToRegister(lir->index());
13828 Register output = ToRegister(lir->output());
13829 Register temp0 = ToRegister(lir->temp0());
13830 Register temp1 = ToRegister(lir->temp1());
13831
13832 using Fn = bool (*)(JSContext*, HandleString, int32_t, uint32_t*);
13833 auto* ool = oolCallVM<Fn, jit::CodePointAt>(lir, ArgList(str, index),
13834 StoreRegisterTo(output));
13835
13836 masm.loadStringCodePoint(str, index, output, temp0, temp1, ool->entry());
13837 masm.bind(ool->rejoin());
13838}
13839
13840void CodeGenerator::visitCodePointAtOrNegative(LCodePointAtOrNegative* lir) {
13841 Register str = ToRegister(lir->str());
13842 Register index = ToRegister(lir->index());
13843 Register output = ToRegister(lir->output());
13844 Register temp0 = ToRegister(lir->temp0());
13845 Register temp1 = ToRegister(lir->temp1());
13846
13847 using Fn = bool (*)(JSContext*, HandleString, int32_t, uint32_t*);
13848 auto* ool = oolCallVM<Fn, jit::CodePointAt>(lir, ArgList(str, index),
13849 StoreRegisterTo(output));
13850
13851 // Return -1 for out-of-bounds access.
13852 masm.move32(Imm32(-1), output);
13853
13854 masm.spectreBoundsCheck32(index, Address(str, JSString::offsetOfLength()),
13855 temp0, ool->rejoin());
13856 masm.loadStringCodePoint(str, index, output, temp0, temp1, ool->entry());
13857 masm.bind(ool->rejoin());
13858}
13859
13860void CodeGenerator::visitNegativeToNaN(LNegativeToNaN* lir) {
13861 Register input = ToRegister(lir->input());
13862 ValueOperand output = ToOutValue(lir);
13863
13864 masm.tagValue(JSVAL_TYPE_INT32, input, output);
13865
13866 Label done;
13867 masm.branchTest32(Assembler::NotSigned, input, input, &done);
13868 masm.moveValue(JS::NaNValue(), output);
13869 masm.bind(&done);
13870}
13871
13872void CodeGenerator::visitNegativeToUndefined(LNegativeToUndefined* lir) {
13873 Register input = ToRegister(lir->input());
13874 ValueOperand output = ToOutValue(lir);
13875
13876 masm.tagValue(JSVAL_TYPE_INT32, input, output);
13877
13878 Label done;
13879 masm.branchTest32(Assembler::NotSigned, input, input, &done);
13880 masm.moveValue(JS::UndefinedValue(), output);
13881 masm.bind(&done);
13882}
13883
13884void CodeGenerator::visitFromCharCode(LFromCharCode* lir) {
13885 Register code = ToRegister(lir->code());
13886 Register output = ToRegister(lir->output());
13887
13888 using Fn = JSLinearString* (*)(JSContext*, int32_t);
13889 auto* ool = oolCallVM<Fn, js::StringFromCharCode>(lir, ArgList(code),
13890 StoreRegisterTo(output));
13891
13892 // OOL path if code >= UNIT_STATIC_LIMIT.
13893 masm.lookupStaticString(code, output, gen->runtime->staticStrings(),
13894 ool->entry());
13895
13896 masm.bind(ool->rejoin());
13897}
13898
13899void CodeGenerator::visitFromCharCodeEmptyIfNegative(
13900 LFromCharCodeEmptyIfNegative* lir) {
13901 Register code = ToRegister(lir->code());
13902 Register output = ToRegister(lir->output());
13903
13904 using Fn = JSLinearString* (*)(JSContext*, int32_t);
13905 auto* ool = oolCallVM<Fn, js::StringFromCharCode>(lir, ArgList(code),
13906 StoreRegisterTo(output));
13907
13908 // Return the empty string for negative inputs.
13909 const JSAtomState& names = gen->runtime->names();
13910 masm.movePtr(ImmGCPtr(names.empty_), output);
13911 masm.branchTest32(Assembler::Signed, code, code, ool->rejoin());
13912
13913 // OOL path if code >= UNIT_STATIC_LIMIT.
13914 masm.lookupStaticString(code, output, gen->runtime->staticStrings(),
13915 ool->entry());
13916
13917 masm.bind(ool->rejoin());
13918}
13919
13920void CodeGenerator::visitFromCharCodeUndefinedIfNegative(
13921 LFromCharCodeUndefinedIfNegative* lir) {
13922 Register code = ToRegister(lir->code());
13923 ValueOperand output = ToOutValue(lir);
13924 Register temp = output.scratchReg();
13925
13926 using Fn = JSLinearString* (*)(JSContext*, int32_t);
13927 auto* ool = oolCallVM<Fn, js::StringFromCharCode>(lir, ArgList(code),
13928 StoreRegisterTo(temp));
13929
13930 // Return |undefined| for negative inputs.
13931 Label done;
13932 masm.moveValue(UndefinedValue(), output);
13933 masm.branchTest32(Assembler::Signed, code, code, &done);
13934
13935 // OOL path if code >= UNIT_STATIC_LIMIT.
13936 masm.lookupStaticString(code, temp, gen->runtime->staticStrings(),
13937 ool->entry());
13938
13939 masm.bind(ool->rejoin());
13940 masm.tagValue(JSVAL_TYPE_STRING, temp, output);
13941
13942 masm.bind(&done);
13943}
13944
13945void CodeGenerator::visitFromCodePoint(LFromCodePoint* lir) {
13946 Register codePoint = ToRegister(lir->codePoint());
13947 Register output = ToRegister(lir->output());
13948 Register temp0 = ToRegister(lir->temp0());
13949 Register temp1 = ToRegister(lir->temp1());
13950 LSnapshot* snapshot = lir->snapshot();
13951
13952 // The OOL path is only taken when we can't allocate the inline string.
13953 using Fn = JSLinearString* (*)(JSContext*, char32_t);
13954 auto* ool = oolCallVM<Fn, js::StringFromCodePoint>(lir, ArgList(codePoint),
13955 StoreRegisterTo(output));
13956
13957 Label isTwoByte;
13958 Label* done = ool->rejoin();
13959
13960 static_assert(
13961 StaticStrings::UNIT_STATIC_LIMIT - 1 == JSString::MAX_LATIN1_CHAR,
13962 "Latin-1 strings can be loaded from static strings");
13963
13964 {
13965 masm.lookupStaticString(codePoint, output, gen->runtime->staticStrings(),
13966 &isTwoByte);
13967 masm.jump(done);
13968 }
13969 masm.bind(&isTwoByte);
13970 {
13971 // Use a bailout if the input is not a valid code point, because
13972 // MFromCodePoint is movable and it'd be observable when a moved
13973 // fromCodePoint throws an exception before its actual call site.
13974 bailoutCmp32(Assembler::Above, codePoint, Imm32(unicode::NonBMPMax),
13975 snapshot);
13976
13977 // Allocate a JSThinInlineString.
13978 {
13979 static_assert(JSThinInlineString::MAX_LENGTH_TWO_BYTE >= 2,
13980 "JSThinInlineString can hold a supplementary code point");
13981
13982 uint32_t flags = JSString::INIT_THIN_INLINE_FLAGS;
13983 masm.newGCString(output, temp0, gen->initialStringHeap(), ool->entry());
13984 masm.store32(Imm32(flags), Address(output, JSString::offsetOfFlags()));
13985 }
13986
13987 Label isSupplementary;
13988 masm.branch32(Assembler::AboveOrEqual, codePoint, Imm32(unicode::NonBMPMin),
13989 &isSupplementary);
13990 {
13991 // Store length.
13992 masm.store32(Imm32(1), Address(output, JSString::offsetOfLength()));
13993
13994 // Load chars pointer in temp0.
13995 masm.loadInlineStringCharsForStore(output, temp0);
13996
13997 masm.store16(codePoint, Address(temp0, 0));
13998
13999 masm.jump(done);
14000 }
14001 masm.bind(&isSupplementary);
14002 {
14003 // Store length.
14004 masm.store32(Imm32(2), Address(output, JSString::offsetOfLength()));
14005
14006 // Load chars pointer in temp0.
14007 masm.loadInlineStringCharsForStore(output, temp0);
14008
14009 // Inlined unicode::LeadSurrogate(uint32_t).
14010 masm.move32(codePoint, temp1);
14011 masm.rshift32(Imm32(10), temp1);
14012 masm.add32(Imm32(unicode::LeadSurrogateMin - (unicode::NonBMPMin >> 10)),
14013 temp1);
14014
14015 masm.store16(temp1, Address(temp0, 0));
14016
14017 // Inlined unicode::TrailSurrogate(uint32_t).
14018 masm.move32(codePoint, temp1);
14019 masm.and32(Imm32(0x3FF), temp1);
14020 masm.or32(Imm32(unicode::TrailSurrogateMin), temp1);
14021
14022 masm.store16(temp1, Address(temp0, sizeof(char16_t)));
14023 }
14024 }
14025
14026 masm.bind(done);
14027}
14028
14029void CodeGenerator::visitStringIncludes(LStringIncludes* lir) {
14030 pushArg(ToRegister(lir->searchString()));
14031 pushArg(ToRegister(lir->string()));
14032
14033 using Fn = bool (*)(JSContext*, HandleString, HandleString, bool*);
14034 callVM<Fn, js::StringIncludes>(lir);
14035}
14036
14037template <typename LIns>
14038static void CallStringMatch(MacroAssembler& masm, LIns* lir, OutOfLineCode* ool,
14039 LiveRegisterSet volatileRegs) {
14040 Register string = ToRegister(lir->string());
14041 Register output = ToRegister(lir->output());
14042 Register tempLength = ToRegister(lir->temp0());
14043 Register tempChars = ToRegister(lir->temp1());
14044 Register maybeTempPat = ToTempRegisterOrInvalid(lir->temp2());
14045
14046 const JSLinearString* searchString = lir->searchString();
14047 size_t length = searchString->length();
14048 MOZ_ASSERT(length == 1 || length == 2)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(length == 1 || length == 2)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(length == 1 || length == 2))
), 0))) { do { } while (false); MOZ_ReportAssertionFailure("length == 1 || length == 2"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 14048); AnnotateMozCrashReason("MOZ_ASSERT" "(" "length == 1 || length == 2"
")"); do { *((volatile int*)__null) = 14048; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
14049
14050 // The additional temp register is only needed when searching for two
14051 // pattern characters.
14052 MOZ_ASSERT_IF(length == 2, maybeTempPat != InvalidReg)do { if (length == 2) { do { static_assert( mozilla::detail::
AssertionConditionType<decltype(maybeTempPat != InvalidReg
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(maybeTempPat != InvalidReg))), 0))) { do { } while (
false); MOZ_ReportAssertionFailure("maybeTempPat != InvalidReg"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 14052); AnnotateMozCrashReason("MOZ_ASSERT" "(" "maybeTempPat != InvalidReg"
")"); do { *((volatile int*)__null) = 14052; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false); } } while
(false)
;
14053
14054 if constexpr (std::is_same_v<LIns, LStringIncludesSIMD>) {
14055 masm.move32(Imm32(0), output);
14056 } else {
14057 masm.move32(Imm32(-1), output);
14058 }
14059
14060 masm.loadStringLength(string, tempLength);
14061
14062 // Can't be a substring when the string is smaller than the search string.
14063 Label done;
14064 masm.branch32(Assembler::Below, tempLength, Imm32(length), ool->rejoin());
14065
14066 bool searchStringIsPureTwoByte = false;
14067 if (searchString->hasTwoByteChars()) {
14068 JS::AutoCheckCannotGC nogc;
14069 searchStringIsPureTwoByte =
14070 !mozilla::IsUtf16Latin1(searchString->twoByteRange(nogc));
14071 }
14072
14073 // Pure two-byte strings can't occur in a Latin-1 string.
14074 if (searchStringIsPureTwoByte) {
14075 masm.branchLatin1String(string, ool->rejoin());
14076 }
14077
14078 // Slow path when we need to linearize the string.
14079 masm.branchIfRope(string, ool->entry());
14080
14081 Label restoreVolatile;
14082
14083 auto callMatcher = [&](CharEncoding encoding) {
14084 masm.loadStringChars(string, tempChars, encoding);
14085
14086 LiveGeneralRegisterSet liveRegs;
14087 if constexpr (std::is_same_v<LIns, LStringIndexOfSIMD>) {
14088 // Save |tempChars| to compute the result index.
14089 liveRegs.add(tempChars);
14090
14091#ifdef DEBUG1
14092 // Save |tempLength| in debug-mode for assertions.
14093 liveRegs.add(tempLength);
14094#endif
14095
14096 // Exclude non-volatile registers.
14097 liveRegs.set() = GeneralRegisterSet::Intersect(
14098 liveRegs.set(), GeneralRegisterSet::Volatile());
14099
14100 masm.PushRegsInMask(liveRegs);
14101 }
14102
14103 if (length == 1) {
14104 char16_t pat = searchString->latin1OrTwoByteChar(0);
14105 MOZ_ASSERT_IF(encoding == CharEncoding::Latin1,do { if (encoding == CharEncoding::Latin1) { do { static_assert
( mozilla::detail::AssertionConditionType<decltype(pat <=
JSString::MAX_LATIN1_CHAR)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(pat <= JSString::MAX_LATIN1_CHAR
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"pat <= JSString::MAX_LATIN1_CHAR", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 14106); AnnotateMozCrashReason("MOZ_ASSERT" "(" "pat <= JSString::MAX_LATIN1_CHAR"
")"); do { *((volatile int*)__null) = 14106; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false); } } while
(false)
14106 pat <= JSString::MAX_LATIN1_CHAR)do { if (encoding == CharEncoding::Latin1) { do { static_assert
( mozilla::detail::AssertionConditionType<decltype(pat <=
JSString::MAX_LATIN1_CHAR)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(pat <= JSString::MAX_LATIN1_CHAR
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"pat <= JSString::MAX_LATIN1_CHAR", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 14106); AnnotateMozCrashReason("MOZ_ASSERT" "(" "pat <= JSString::MAX_LATIN1_CHAR"
")"); do { *((volatile int*)__null) = 14106; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false); } } while
(false)
;
14107
14108 masm.move32(Imm32(pat), output);
14109
14110 masm.setupAlignedABICall();
14111 masm.passABIArg(tempChars);
14112 masm.passABIArg(output);
14113 masm.passABIArg(tempLength);
14114 if (encoding == CharEncoding::Latin1) {
14115 using Fn = const char* (*)(const char*, char, size_t);
14116 masm.callWithABI<Fn, mozilla::SIMD::memchr8>(
14117 ABIType::General, CheckUnsafeCallWithABI::DontCheckOther);
14118 } else {
14119 using Fn = const char16_t* (*)(const char16_t*, char16_t, size_t);
14120 masm.callWithABI<Fn, mozilla::SIMD::memchr16>(
14121 ABIType::General, CheckUnsafeCallWithABI::DontCheckOther);
14122 }
14123 } else {
14124 char16_t pat0 = searchString->latin1OrTwoByteChar(0);
14125 MOZ_ASSERT_IF(encoding == CharEncoding::Latin1,do { if (encoding == CharEncoding::Latin1) { do { static_assert
( mozilla::detail::AssertionConditionType<decltype(pat0 <=
JSString::MAX_LATIN1_CHAR)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(pat0 <= JSString::MAX_LATIN1_CHAR
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"pat0 <= JSString::MAX_LATIN1_CHAR", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 14126); AnnotateMozCrashReason("MOZ_ASSERT" "(" "pat0 <= JSString::MAX_LATIN1_CHAR"
")"); do { *((volatile int*)__null) = 14126; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false); } } while
(false)
14126 pat0 <= JSString::MAX_LATIN1_CHAR)do { if (encoding == CharEncoding::Latin1) { do { static_assert
( mozilla::detail::AssertionConditionType<decltype(pat0 <=
JSString::MAX_LATIN1_CHAR)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(pat0 <= JSString::MAX_LATIN1_CHAR
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"pat0 <= JSString::MAX_LATIN1_CHAR", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 14126); AnnotateMozCrashReason("MOZ_ASSERT" "(" "pat0 <= JSString::MAX_LATIN1_CHAR"
")"); do { *((volatile int*)__null) = 14126; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false); } } while
(false)
;
14127
14128 char16_t pat1 = searchString->latin1OrTwoByteChar(1);
14129 MOZ_ASSERT_IF(encoding == CharEncoding::Latin1,do { if (encoding == CharEncoding::Latin1) { do { static_assert
( mozilla::detail::AssertionConditionType<decltype(pat1 <=
JSString::MAX_LATIN1_CHAR)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(pat1 <= JSString::MAX_LATIN1_CHAR
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"pat1 <= JSString::MAX_LATIN1_CHAR", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 14130); AnnotateMozCrashReason("MOZ_ASSERT" "(" "pat1 <= JSString::MAX_LATIN1_CHAR"
")"); do { *((volatile int*)__null) = 14130; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false); } } while
(false)
14130 pat1 <= JSString::MAX_LATIN1_CHAR)do { if (encoding == CharEncoding::Latin1) { do { static_assert
( mozilla::detail::AssertionConditionType<decltype(pat1 <=
JSString::MAX_LATIN1_CHAR)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(pat1 <= JSString::MAX_LATIN1_CHAR
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"pat1 <= JSString::MAX_LATIN1_CHAR", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 14130); AnnotateMozCrashReason("MOZ_ASSERT" "(" "pat1 <= JSString::MAX_LATIN1_CHAR"
")"); do { *((volatile int*)__null) = 14130; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false); } } while
(false)
;
14131
14132 masm.move32(Imm32(pat0), output);
14133 masm.move32(Imm32(pat1), maybeTempPat);
14134
14135 masm.setupAlignedABICall();
14136 masm.passABIArg(tempChars);
14137 masm.passABIArg(output);
14138 masm.passABIArg(maybeTempPat);
14139 masm.passABIArg(tempLength);
14140 if (encoding == CharEncoding::Latin1) {
14141 using Fn = const char* (*)(const char*, char, char, size_t);
14142 masm.callWithABI<Fn, mozilla::SIMD::memchr2x8>(
14143 ABIType::General, CheckUnsafeCallWithABI::DontCheckOther);
14144 } else {
14145 using Fn =
14146 const char16_t* (*)(const char16_t*, char16_t, char16_t, size_t);
14147 masm.callWithABI<Fn, mozilla::SIMD::memchr2x16>(
14148 ABIType::General, CheckUnsafeCallWithABI::DontCheckOther);
14149 }
14150 }
14151
14152 masm.storeCallPointerResult(output);
14153
14154 // Convert to string index for `indexOf`.
14155 if constexpr (std::is_same_v<LIns, LStringIndexOfSIMD>) {
14156 // Restore |tempChars|. (And in debug mode |tempLength|.)
14157 masm.PopRegsInMask(liveRegs);
14158
14159 Label found;
14160 masm.branchPtr(Assembler::NotEqual, output, ImmPtr(nullptr), &found);
14161 {
14162 masm.move32(Imm32(-1), output);
14163 masm.jump(&restoreVolatile);
14164 }
14165 masm.bind(&found);
14166
14167#ifdef DEBUG1
14168 // Check lower bound.
14169 Label lower;
14170 masm.branchPtr(Assembler::AboveOrEqual, output, tempChars, &lower);
14171 masm.assumeUnreachable("result pointer below string chars");
14172 masm.bind(&lower);
14173
14174 // Compute the end position of the characters.
14175 auto scale = encoding == CharEncoding::Latin1 ? TimesOne : TimesTwo;
14176 masm.computeEffectiveAddress(BaseIndex(tempChars, tempLength, scale),
14177 tempLength);
14178
14179 // Check upper bound.
14180 Label upper;
14181 masm.branchPtr(Assembler::Below, output, tempLength, &upper);
14182 masm.assumeUnreachable("result pointer above string chars");
14183 masm.bind(&upper);
14184#endif
14185
14186 masm.subPtr(tempChars, output);
14187
14188 if (encoding == CharEncoding::TwoByte) {
14189 masm.rshiftPtr(Imm32(1), output);
14190 }
14191 }
14192 };
14193
14194 volatileRegs.takeUnchecked(output);
14195 volatileRegs.takeUnchecked(tempLength);
14196 volatileRegs.takeUnchecked(tempChars);
14197 if (maybeTempPat != InvalidReg) {
14198 volatileRegs.takeUnchecked(maybeTempPat);
14199 }
14200 masm.PushRegsInMask(volatileRegs);
14201
14202 // Handle the case when the input is a Latin-1 string.
14203 if (!searchStringIsPureTwoByte) {
14204 Label twoByte;
14205 masm.branchTwoByteString(string, &twoByte);
14206 {
14207 callMatcher(CharEncoding::Latin1);
14208 masm.jump(&restoreVolatile);
14209 }
14210 masm.bind(&twoByte);
14211 }
14212
14213 // Handle the case when the input is a two-byte string.
14214 callMatcher(CharEncoding::TwoByte);
14215
14216 masm.bind(&restoreVolatile);
14217 masm.PopRegsInMask(volatileRegs);
14218
14219 // Convert to bool for `includes`.
14220 if constexpr (std::is_same_v<LIns, LStringIncludesSIMD>) {
14221 masm.cmpPtrSet(Assembler::NotEqual, output, ImmPtr(nullptr), output);
14222 }
14223
14224 masm.bind(ool->rejoin());
14225}
14226
14227void CodeGenerator::visitStringIncludesSIMD(LStringIncludesSIMD* lir) {
14228 Register string = ToRegister(lir->string());
14229 Register output = ToRegister(lir->output());
14230 const JSLinearString* searchString = lir->searchString();
14231
14232 using Fn = bool (*)(JSContext*, HandleString, HandleString, bool*);
14233 auto* ool = oolCallVM<Fn, js::StringIncludes>(
14234 lir, ArgList(string, ImmGCPtr(searchString)), StoreRegisterTo(output));
14235
14236 CallStringMatch(masm, lir, ool, liveVolatileRegs(lir));
14237}
14238
14239void CodeGenerator::visitStringIndexOf(LStringIndexOf* lir) {
14240 pushArg(ToRegister(lir->searchString()));
14241 pushArg(ToRegister(lir->string()));
14242
14243 using Fn = bool (*)(JSContext*, HandleString, HandleString, int32_t*);
14244 callVM<Fn, js::StringIndexOf>(lir);
14245}
14246
14247void CodeGenerator::visitStringIndexOfSIMD(LStringIndexOfSIMD* lir) {
14248 Register string = ToRegister(lir->string());
14249 Register output = ToRegister(lir->output());
14250 const JSLinearString* searchString = lir->searchString();
14251
14252 using Fn = bool (*)(JSContext*, HandleString, HandleString, int32_t*);
14253 auto* ool = oolCallVM<Fn, js::StringIndexOf>(
14254 lir, ArgList(string, ImmGCPtr(searchString)), StoreRegisterTo(output));
14255
14256 CallStringMatch(masm, lir, ool, liveVolatileRegs(lir));
14257}
14258
14259void CodeGenerator::visitStringLastIndexOf(LStringLastIndexOf* lir) {
14260 pushArg(ToRegister(lir->searchString()));
14261 pushArg(ToRegister(lir->string()));
14262
14263 using Fn = bool (*)(JSContext*, HandleString, HandleString, int32_t*);
14264 callVM<Fn, js::StringLastIndexOf>(lir);
14265}
14266
14267void CodeGenerator::visitStringStartsWith(LStringStartsWith* lir) {
14268 pushArg(ToRegister(lir->searchString()));
14269 pushArg(ToRegister(lir->string()));
14270
14271 using Fn = bool (*)(JSContext*, HandleString, HandleString, bool*);
14272 callVM<Fn, js::StringStartsWith>(lir);
14273}
14274
14275void CodeGenerator::visitStringStartsWithInline(LStringStartsWithInline* lir) {
14276 Register string = ToRegister(lir->string());
14277 Register output = ToRegister(lir->output());
14278 Register temp = ToRegister(lir->temp0());
14279
14280 const JSLinearString* searchString = lir->searchString();
14281
14282 size_t length = searchString->length();
14283 MOZ_ASSERT(length > 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(length > 0)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(length > 0))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("length > 0",
"/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 14283); AnnotateMozCrashReason("MOZ_ASSERT" "(" "length > 0"
")"); do { *((volatile int*)__null) = 14283; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
14284
14285 using Fn = bool (*)(JSContext*, HandleString, HandleString, bool*);
14286 auto* ool = oolCallVM<Fn, js::StringStartsWith>(
14287 lir, ArgList(string, ImmGCPtr(searchString)), StoreRegisterTo(output));
14288
14289 masm.move32(Imm32(0), output);
14290
14291 // Can't be a prefix when the string is smaller than the search string.
14292 masm.branch32(Assembler::Below, Address(string, JSString::offsetOfLength()),
14293 Imm32(length), ool->rejoin());
14294
14295 // Unwind ropes at the start if possible.
14296 Label compare;
14297 masm.movePtr(string, temp);
14298 masm.branchIfNotRope(temp, &compare);
14299
14300 Label unwindRope;
14301 masm.bind(&unwindRope);
14302 masm.loadRopeLeftChild(temp, output);
14303 masm.movePtr(output, temp);
14304
14305 // If the left child is smaller than the search string, jump into the VM to
14306 // linearize the string.
14307 masm.branch32(Assembler::Below, Address(temp, JSString::offsetOfLength()),
14308 Imm32(length), ool->entry());
14309
14310 // Otherwise keep unwinding ropes.
14311 masm.branchIfRope(temp, &unwindRope);
14312
14313 masm.bind(&compare);
14314
14315 // If operands point to the same instance, it's trivially a prefix.
14316 Label notPointerEqual;
14317 masm.branchPtr(Assembler::NotEqual, temp, ImmGCPtr(searchString),
14318 &notPointerEqual);
14319 masm.move32(Imm32(1), output);
14320 masm.jump(ool->rejoin());
14321 masm.bind(&notPointerEqual);
14322
14323 if (searchString->hasTwoByteChars()) {
14324 // Pure two-byte strings can't be a prefix of Latin-1 strings.
14325 JS::AutoCheckCannotGC nogc;
14326 if (!mozilla::IsUtf16Latin1(searchString->twoByteRange(nogc))) {
14327 Label compareChars;
14328 masm.branchTwoByteString(temp, &compareChars);
14329 masm.move32(Imm32(0), output);
14330 masm.jump(ool->rejoin());
14331 masm.bind(&compareChars);
14332 }
14333 }
14334
14335 // Load the input string's characters.
14336 Register stringChars = output;
14337 masm.loadStringCharsForCompare(temp, searchString, stringChars, ool->entry());
14338
14339 // Start comparing character by character.
14340 masm.compareStringChars(JSOp::Eq, stringChars, searchString, output);
14341
14342 masm.bind(ool->rejoin());
14343}
14344
14345void CodeGenerator::visitStringEndsWith(LStringEndsWith* lir) {
14346 pushArg(ToRegister(lir->searchString()));
14347 pushArg(ToRegister(lir->string()));
14348
14349 using Fn = bool (*)(JSContext*, HandleString, HandleString, bool*);
14350 callVM<Fn, js::StringEndsWith>(lir);
14351}
14352
14353void CodeGenerator::visitStringEndsWithInline(LStringEndsWithInline* lir) {
14354 Register string = ToRegister(lir->string());
14355 Register output = ToRegister(lir->output());
14356 Register temp = ToRegister(lir->temp0());
14357
14358 const JSLinearString* searchString = lir->searchString();
14359
14360 size_t length = searchString->length();
14361 MOZ_ASSERT(length > 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(length > 0)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(length > 0))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("length > 0",
"/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 14361); AnnotateMozCrashReason("MOZ_ASSERT" "(" "length > 0"
")"); do { *((volatile int*)__null) = 14361; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
14362
14363 using Fn = bool (*)(JSContext*, HandleString, HandleString, bool*);
14364 auto* ool = oolCallVM<Fn, js::StringEndsWith>(
14365 lir, ArgList(string, ImmGCPtr(searchString)), StoreRegisterTo(output));
14366
14367 masm.move32(Imm32(0), output);
14368
14369 // Can't be a suffix when the string is smaller than the search string.
14370 masm.branch32(Assembler::Below, Address(string, JSString::offsetOfLength()),
14371 Imm32(length), ool->rejoin());
14372
14373 // Unwind ropes at the end if possible.
14374 Label compare;
14375 masm.movePtr(string, temp);
14376 masm.branchIfNotRope(temp, &compare);
14377
14378 Label unwindRope;
14379 masm.bind(&unwindRope);
14380 masm.loadRopeRightChild(temp, output);
14381 masm.movePtr(output, temp);
14382
14383 // If the right child is smaller than the search string, jump into the VM to
14384 // linearize the string.
14385 masm.branch32(Assembler::Below, Address(temp, JSString::offsetOfLength()),
14386 Imm32(length), ool->entry());
14387
14388 // Otherwise keep unwinding ropes.
14389 masm.branchIfRope(temp, &unwindRope);
14390
14391 masm.bind(&compare);
14392
14393 // If operands point to the same instance, it's trivially a suffix.
14394 Label notPointerEqual;
14395 masm.branchPtr(Assembler::NotEqual, temp, ImmGCPtr(searchString),
14396 &notPointerEqual);
14397 masm.move32(Imm32(1), output);
14398 masm.jump(ool->rejoin());
14399 masm.bind(&notPointerEqual);
14400
14401 CharEncoding encoding = searchString->hasLatin1Chars()
14402 ? CharEncoding::Latin1
14403 : CharEncoding::TwoByte;
14404 if (encoding == CharEncoding::TwoByte) {
14405 // Pure two-byte strings can't be a suffix of Latin-1 strings.
14406 JS::AutoCheckCannotGC nogc;
14407 if (!mozilla::IsUtf16Latin1(searchString->twoByteRange(nogc))) {
14408 Label compareChars;
14409 masm.branchTwoByteString(temp, &compareChars);
14410 masm.move32(Imm32(0), output);
14411 masm.jump(ool->rejoin());
14412 masm.bind(&compareChars);
14413 }
14414 }
14415
14416 // Load the input string's characters.
14417 Register stringChars = output;
14418 masm.loadStringCharsForCompare(temp, searchString, stringChars, ool->entry());
14419
14420 // Move string-char pointer to the suffix string.
14421 masm.loadStringLength(temp, temp);
14422 masm.sub32(Imm32(length), temp);
14423 masm.addToCharPtr(stringChars, temp, encoding);
14424
14425 // Start comparing character by character.
14426 masm.compareStringChars(JSOp::Eq, stringChars, searchString, output);
14427
14428 masm.bind(ool->rejoin());
14429}
14430
14431void CodeGenerator::visitStringToLowerCase(LStringToLowerCase* lir) {
14432 Register string = ToRegister(lir->string());
14433 Register output = ToRegister(lir->output());
14434 Register temp0 = ToRegister(lir->temp0());
14435 Register temp1 = ToRegister(lir->temp1());
14436 Register temp2 = ToRegister(lir->temp2());
14437
14438 // On x86 there are not enough registers. In that case reuse the string
14439 // register as a temporary.
14440 Register temp3 =
14441 lir->temp3()->isBogusTemp() ? string : ToRegister(lir->temp3());
14442 Register temp4 = ToRegister(lir->temp4());
14443
14444 using Fn = JSString* (*)(JSContext*, HandleString);
14445 OutOfLineCode* ool = oolCallVM<Fn, js::StringToLowerCase>(
14446 lir, ArgList(string), StoreRegisterTo(output));
14447
14448 // Take the slow path if the string isn't a linear Latin-1 string.
14449 Imm32 linearLatin1Bits(JSString::LINEAR_BIT | JSString::LATIN1_CHARS_BIT);
14450 Register flags = temp0;
14451 masm.load32(Address(string, JSString::offsetOfFlags()), flags);
14452 masm.and32(linearLatin1Bits, flags);
14453 masm.branch32(Assembler::NotEqual, flags, linearLatin1Bits, ool->entry());
14454
14455 Register length = temp0;
14456 masm.loadStringLength(string, length);
14457
14458 // Return the input if it's the empty string.
14459 Label notEmptyString;
14460 masm.branch32(Assembler::NotEqual, length, Imm32(0), &notEmptyString);
14461 {
14462 masm.movePtr(string, output);
14463 masm.jump(ool->rejoin());
14464 }
14465 masm.bind(&notEmptyString);
14466
14467 Register inputChars = temp1;
14468 masm.loadStringChars(string, inputChars, CharEncoding::Latin1);
14469
14470 Register toLowerCaseTable = temp2;
14471 masm.movePtr(ImmPtr(unicode::latin1ToLowerCaseTable), toLowerCaseTable);
14472
14473 // Single element strings can be directly retrieved from static strings cache.
14474 Label notSingleElementString;
14475 masm.branch32(Assembler::NotEqual, length, Imm32(1), &notSingleElementString);
14476 {
14477 Register current = temp4;
14478
14479 masm.loadChar(Address(inputChars, 0), current, CharEncoding::Latin1);
14480 masm.load8ZeroExtend(BaseIndex(toLowerCaseTable, current, TimesOne),
14481 current);
14482 masm.lookupStaticString(current, output, gen->runtime->staticStrings());
14483
14484 masm.jump(ool->rejoin());
14485 }
14486 masm.bind(&notSingleElementString);
14487
14488 // Use the OOL-path when the string is too long. This prevents scanning long
14489 // strings which have upper case characters only near the end a second time in
14490 // the VM.
14491 constexpr int32_t MaxInlineLength = 64;
14492 masm.branch32(Assembler::Above, length, Imm32(MaxInlineLength), ool->entry());
14493
14494 {
14495 // Check if there are any characters which need to be converted.
14496 //
14497 // This extra loop gives a small performance improvement for strings which
14498 // are already lower cased and lets us avoid calling into the runtime for
14499 // non-inline, all lower case strings. But more importantly it avoids
14500 // repeated inline allocation failures:
14501 // |AllocateThinOrFatInlineString| below takes the OOL-path and calls the
14502 // |js::StringToLowerCase| runtime function when the result string can't be
14503 // allocated inline. And |js::StringToLowerCase| directly returns the input
14504 // string when no characters need to be converted. That means it won't
14505 // trigger GC to clear up the free nursery space, so the next toLowerCase()
14506 // call will again fail to inline allocate the result string.
14507 Label hasUpper;
14508 {
14509 Register checkInputChars = output;
14510 masm.movePtr(inputChars, checkInputChars);
14511
14512 Register current = temp4;
14513
14514 Label start;
14515 masm.bind(&start);
14516 masm.loadChar(Address(checkInputChars, 0), current, CharEncoding::Latin1);
14517 masm.branch8(Assembler::NotEqual,
14518 BaseIndex(toLowerCaseTable, current, TimesOne), current,
14519 &hasUpper);
14520 masm.addPtr(Imm32(sizeof(Latin1Char)), checkInputChars);
14521 masm.branchSub32(Assembler::NonZero, Imm32(1), length, &start);
14522
14523 // Input is already in lower case.
14524 masm.movePtr(string, output);
14525 masm.jump(ool->rejoin());
14526 }
14527 masm.bind(&hasUpper);
14528
14529 // |length| was clobbered above, reload.
14530 masm.loadStringLength(string, length);
14531
14532 // Call into the runtime when we can't create an inline string.
14533 masm.branch32(Assembler::Above, length,
14534 Imm32(JSFatInlineString::MAX_LENGTH_LATIN1), ool->entry());
14535
14536 AllocateThinOrFatInlineString(masm, output, length, temp4,
14537 initialStringHeap(), ool->entry(),
14538 CharEncoding::Latin1);
14539
14540 if (temp3 == string) {
14541 masm.push(string);
14542 }
14543
14544 Register outputChars = temp3;
14545 masm.loadInlineStringCharsForStore(output, outputChars);
14546
14547 {
14548 Register current = temp4;
14549
14550 Label start;
14551 masm.bind(&start);
14552 masm.loadChar(Address(inputChars, 0), current, CharEncoding::Latin1);
14553 masm.load8ZeroExtend(BaseIndex(toLowerCaseTable, current, TimesOne),
14554 current);
14555 masm.storeChar(current, Address(outputChars, 0), CharEncoding::Latin1);
14556 masm.addPtr(Imm32(sizeof(Latin1Char)), inputChars);
14557 masm.addPtr(Imm32(sizeof(Latin1Char)), outputChars);
14558 masm.branchSub32(Assembler::NonZero, Imm32(1), length, &start);
14559 }
14560
14561 if (temp3 == string) {
14562 masm.pop(string);
14563 }
14564 }
14565
14566 masm.bind(ool->rejoin());
14567}
14568
14569void CodeGenerator::visitStringToUpperCase(LStringToUpperCase* lir) {
14570 pushArg(ToRegister(lir->string()));
14571
14572 using Fn = JSString* (*)(JSContext*, HandleString);
14573 callVM<Fn, js::StringToUpperCase>(lir);
14574}
14575
14576void CodeGenerator::visitCharCodeToLowerCase(LCharCodeToLowerCase* lir) {
14577 Register code = ToRegister(lir->code());
14578 Register output = ToRegister(lir->output());
14579 Register temp = ToRegister(lir->temp0());
14580
14581 using Fn = JSString* (*)(JSContext*, int32_t);
14582 auto* ool = oolCallVM<Fn, jit::CharCodeToLowerCase>(lir, ArgList(code),
14583 StoreRegisterTo(output));
14584
14585 constexpr char16_t NonLatin1Min = char16_t(JSString::MAX_LATIN1_CHAR) + 1;
14586
14587 // OOL path if code >= NonLatin1Min.
14588 masm.boundsCheck32PowerOfTwo(code, NonLatin1Min, ool->entry());
14589
14590 // Convert to lower case.
14591 masm.movePtr(ImmPtr(unicode::latin1ToLowerCaseTable), temp);
14592 masm.load8ZeroExtend(BaseIndex(temp, code, TimesOne), temp);
14593
14594 // Load static string for lower case character.
14595 masm.lookupStaticString(temp, output, gen->runtime->staticStrings());
14596
14597 masm.bind(ool->rejoin());
14598}
14599
14600void CodeGenerator::visitCharCodeToUpperCase(LCharCodeToUpperCase* lir) {
14601 Register code = ToRegister(lir->code());
14602 Register output = ToRegister(lir->output());
14603 Register temp = ToRegister(lir->temp0());
14604
14605 using Fn = JSString* (*)(JSContext*, int32_t);
14606 auto* ool = oolCallVM<Fn, jit::CharCodeToUpperCase>(lir, ArgList(code),
14607 StoreRegisterTo(output));
14608
14609 constexpr char16_t NonLatin1Min = char16_t(JSString::MAX_LATIN1_CHAR) + 1;
14610
14611 // OOL path if code >= NonLatin1Min.
14612 masm.boundsCheck32PowerOfTwo(code, NonLatin1Min, ool->entry());
14613
14614 // Most one element Latin-1 strings can be directly retrieved from the
14615 // static strings cache, except the following three characters:
14616 //
14617 // 1. ToUpper(U+00B5) = 0+039C
14618 // 2. ToUpper(U+00FF) = 0+0178
14619 // 3. ToUpper(U+00DF) = 0+0053 0+0053
14620 masm.branch32(Assembler::Equal, code, Imm32(unicode::MICRO_SIGN),
14621 ool->entry());
14622 masm.branch32(Assembler::Equal, code,
14623 Imm32(unicode::LATIN_SMALL_LETTER_Y_WITH_DIAERESIS),
14624 ool->entry());
14625 masm.branch32(Assembler::Equal, code,
14626 Imm32(unicode::LATIN_SMALL_LETTER_SHARP_S), ool->entry());
14627
14628 // Inline unicode::ToUpperCase (without the special case for ASCII characters)
14629
14630 constexpr size_t shift = unicode::CharInfoShift;
14631
14632 // code >> shift
14633 masm.move32(code, temp);
14634 masm.rshift32(Imm32(shift), temp);
14635
14636 // index = index1[code >> shift];
14637 masm.movePtr(ImmPtr(unicode::index1), output);
14638 masm.load8ZeroExtend(BaseIndex(output, temp, TimesOne), temp);
14639
14640 // (code & ((1 << shift) - 1)
14641 masm.move32(code, output);
14642 masm.and32(Imm32((1 << shift) - 1), output);
14643
14644 // (index << shift) + (code & ((1 << shift) - 1))
14645 masm.lshift32(Imm32(shift), temp);
14646 masm.add32(output, temp);
14647
14648 // index = index2[(index << shift) + (code & ((1 << shift) - 1))]
14649 masm.movePtr(ImmPtr(unicode::index2), output);
14650 masm.load8ZeroExtend(BaseIndex(output, temp, TimesOne), temp);
14651
14652 // Compute |index * 6| through |(index * 3) * TimesTwo|.
14653 static_assert(sizeof(unicode::CharacterInfo) == 6);
14654 masm.mulBy3(temp, temp);
14655
14656 // upperCase = js_charinfo[index].upperCase
14657 masm.movePtr(ImmPtr(unicode::js_charinfo), output);
14658 masm.load16ZeroExtend(BaseIndex(output, temp, TimesTwo,
14659 offsetof(unicode::CharacterInfo, upperCase)__builtin_offsetof(unicode::CharacterInfo, upperCase)),
14660 temp);
14661
14662 // uint16_t(ch) + upperCase
14663 masm.add32(code, temp);
14664
14665 // Clear any high bits added when performing the unsigned 16-bit addition
14666 // through a signed 32-bit addition.
14667 masm.move8ZeroExtend(temp, temp);
14668
14669 // Load static string for upper case character.
14670 masm.lookupStaticString(temp, output, gen->runtime->staticStrings());
14671
14672 masm.bind(ool->rejoin());
14673}
14674
14675void CodeGenerator::visitStringTrimStartIndex(LStringTrimStartIndex* lir) {
14676 Register string = ToRegister(lir->string());
14677 Register output = ToRegister(lir->output());
14678
14679 auto volatileRegs = liveVolatileRegs(lir);
14680 volatileRegs.takeUnchecked(output);
14681
14682 masm.PushRegsInMask(volatileRegs);
14683
14684 using Fn = int32_t (*)(const JSString*);
14685 masm.setupAlignedABICall();
14686 masm.passABIArg(string);
14687 masm.callWithABI<Fn, jit::StringTrimStartIndex>();
14688 masm.storeCallInt32Result(output);
14689
14690 masm.PopRegsInMask(volatileRegs);
14691}
14692
14693void CodeGenerator::visitStringTrimEndIndex(LStringTrimEndIndex* lir) {
14694 Register string = ToRegister(lir->string());
14695 Register start = ToRegister(lir->start());
14696 Register output = ToRegister(lir->output());
14697
14698 auto volatileRegs = liveVolatileRegs(lir);
14699 volatileRegs.takeUnchecked(output);
14700
14701 masm.PushRegsInMask(volatileRegs);
14702
14703 using Fn = int32_t (*)(const JSString*, int32_t);
14704 masm.setupAlignedABICall();
14705 masm.passABIArg(string);
14706 masm.passABIArg(start);
14707 masm.callWithABI<Fn, jit::StringTrimEndIndex>();
14708 masm.storeCallInt32Result(output);
14709
14710 masm.PopRegsInMask(volatileRegs);
14711}
14712
14713void CodeGenerator::visitStringSplit(LStringSplit* lir) {
14714 pushArg(Imm32(INT32_MAX(2147483647)));
14715 pushArg(ToRegister(lir->separator()));
14716 pushArg(ToRegister(lir->string()));
14717
14718 using Fn = ArrayObject* (*)(JSContext*, HandleString, HandleString, uint32_t);
14719 callVM<Fn, js::StringSplitString>(lir);
14720}
14721
14722void CodeGenerator::visitInitializedLength(LInitializedLength* lir) {
14723 Address initLength(ToRegister(lir->elements()),
14724 ObjectElements::offsetOfInitializedLength());
14725 masm.load32(initLength, ToRegister(lir->output()));
14726}
14727
14728void CodeGenerator::visitSetInitializedLength(LSetInitializedLength* lir) {
14729 Address initLength(ToRegister(lir->elements()),
14730 ObjectElements::offsetOfInitializedLength());
14731 SetLengthFromIndex(masm, lir->index(), initLength);
14732}
14733
14734void CodeGenerator::visitNotI(LNotI* lir) {
14735 Register input = ToRegister(lir->input());
14736 Register output = ToRegister(lir->output());
14737
14738 masm.cmp32Set(Assembler::Equal, input, Imm32(0), output);
14739}
14740
14741void CodeGenerator::visitNotIPtr(LNotIPtr* lir) {
14742 Register input = ToRegister(lir->input());
14743 Register output = ToRegister(lir->output());
14744
14745 masm.cmpPtrSet(Assembler::Equal, input, ImmWord(0), output);
14746}
14747
14748void CodeGenerator::visitNotI64(LNotI64* lir) {
14749 Register64 input = ToRegister64(lir->inputI64());
14750 Register output = ToRegister(lir->output());
14751
14752 masm.cmp64Set(Assembler::Equal, input, Imm64(0), output);
14753}
14754
14755void CodeGenerator::visitNotBI(LNotBI* lir) {
14756 Register input = ToRegister(lir->input());
14757 Register output = ToRegister(lir->output());
14758
14759 masm.cmp32Set(Assembler::Equal, Address(input, BigInt::offsetOfLength()),
14760 Imm32(0), output);
14761}
14762
14763void CodeGenerator::visitNotO(LNotO* lir) {
14764 Register objreg = ToRegister(lir->input());
14765 Register output = ToRegister(lir->output());
14766
14767 bool intact = hasSeenObjectEmulateUndefinedFuseIntactAndDependencyNoted();
14768 if (intact) {
14769 // Bug 1874905: It would be fantastic if this could be optimized out.
14770 assertObjectDoesNotEmulateUndefined(objreg, output, lir->mir());
14771 masm.move32(Imm32(0), output);
14772 } else {
14773 auto* ool = new (alloc()) OutOfLineTestObjectWithLabels();
14774 addOutOfLineCode(ool, lir->mir());
14775
14776 Label* ifEmulatesUndefined = ool->label1();
14777 Label* ifDoesntEmulateUndefined = ool->label2();
14778
14779 branchTestObjectEmulatesUndefined(objreg, ifEmulatesUndefined,
14780 ifDoesntEmulateUndefined, output, ool);
14781 // fall through
14782
14783 Label join;
14784
14785 masm.move32(Imm32(0), output);
14786 masm.jump(&join);
14787
14788 masm.bind(ifEmulatesUndefined);
14789 masm.move32(Imm32(1), output);
14790
14791 masm.bind(&join);
14792 }
14793}
14794
14795void CodeGenerator::visitNotV(LNotV* lir) {
14796 auto* ool = new (alloc()) OutOfLineTestObjectWithLabels();
14797 addOutOfLineCode(ool, lir->mir());
14798
14799 Label* ifTruthy = ool->label1();
14800 Label* ifFalsy = ool->label2();
14801
14802 ValueOperand input = ToValue(lir, LNotV::InputIndex);
14803 Register tempToUnbox = ToTempUnboxRegister(lir->temp1());
14804 FloatRegister floatTemp = ToFloatRegister(lir->temp0());
14805 Register output = ToRegister(lir->output());
14806 const TypeDataList& observedTypes = lir->mir()->observedTypes();
14807
14808 testValueTruthy(input, tempToUnbox, output, floatTemp, observedTypes,
14809 ifTruthy, ifFalsy, ool);
14810
14811 Label join;
14812
14813 // Note that the testValueTruthy call above may choose to fall through
14814 // to ifTruthy instead of branching there.
14815 masm.bind(ifTruthy);
14816 masm.move32(Imm32(0), output);
14817 masm.jump(&join);
14818
14819 masm.bind(ifFalsy);
14820 masm.move32(Imm32(1), output);
14821
14822 // both branches meet here.
14823 masm.bind(&join);
14824}
14825
14826void CodeGenerator::visitBoundsCheck(LBoundsCheck* lir) {
14827 const LAllocation* index = lir->index();
14828 const LAllocation* length = lir->length();
14829 LSnapshot* snapshot = lir->snapshot();
14830
14831 MIRType type = lir->mir()->type();
14832
14833 auto bailoutCmp = [&](Assembler::Condition cond, auto lhs, auto rhs) {
14834 if (type == MIRType::Int32) {
14835 bailoutCmp32(cond, lhs, rhs, snapshot);
14836 } else {
14837 MOZ_ASSERT(type == MIRType::IntPtr)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(type == MIRType::IntPtr)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(type == MIRType::IntPtr))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("type == MIRType::IntPtr"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 14837); AnnotateMozCrashReason("MOZ_ASSERT" "(" "type == MIRType::IntPtr"
")"); do { *((volatile int*)__null) = 14837; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
14838 bailoutCmpPtr(cond, lhs, rhs, snapshot);
14839 }
14840 };
14841
14842 auto bailoutCmpConstant = [&](Assembler::Condition cond, auto lhs,
14843 int32_t rhs) {
14844 if (type == MIRType::Int32) {
14845 bailoutCmp32(cond, lhs, Imm32(rhs), snapshot);
14846 } else {
14847 MOZ_ASSERT(type == MIRType::IntPtr)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(type == MIRType::IntPtr)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(type == MIRType::IntPtr))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("type == MIRType::IntPtr"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 14847); AnnotateMozCrashReason("MOZ_ASSERT" "(" "type == MIRType::IntPtr"
")"); do { *((volatile int*)__null) = 14847; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
14848 bailoutCmpPtr(cond, lhs, ImmWord(rhs), snapshot);
14849 }
14850 };
14851
14852 if (index->isConstant()) {
14853 // Use uint32 so that the comparison is unsigned.
14854 uint32_t idx = ToInt32(index);
14855 if (length->isConstant()) {
14856 uint32_t len = ToInt32(lir->length());
14857 if (idx < len) {
14858 return;
14859 }
14860 bailout(snapshot);
14861 return;
14862 }
14863
14864 if (length->isRegister()) {
14865 bailoutCmpConstant(Assembler::BelowOrEqual, ToRegister(length), idx);
14866 } else {
14867 bailoutCmpConstant(Assembler::BelowOrEqual, ToAddress(length), idx);
14868 }
14869 return;
14870 }
14871
14872 Register indexReg = ToRegister(index);
14873 if (length->isConstant()) {
14874 bailoutCmpConstant(Assembler::AboveOrEqual, indexReg, ToInt32(length));
14875 } else if (length->isRegister()) {
14876 bailoutCmp(Assembler::BelowOrEqual, ToRegister(length), indexReg);
14877 } else {
14878 bailoutCmp(Assembler::BelowOrEqual, ToAddress(length), indexReg);
14879 }
14880}
14881
14882void CodeGenerator::visitBoundsCheckRange(LBoundsCheckRange* lir) {
14883 int32_t min = lir->mir()->minimum();
14884 int32_t max = lir->mir()->maximum();
14885 MOZ_ASSERT(max >= min)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(max >= min)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(max >= min))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("max >= min",
"/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 14885); AnnotateMozCrashReason("MOZ_ASSERT" "(" "max >= min"
")"); do { *((volatile int*)__null) = 14885; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
14886
14887 LSnapshot* snapshot = lir->snapshot();
14888 MIRType type = lir->mir()->type();
14889
14890 const LAllocation* length = lir->length();
14891 Register temp = ToRegister(lir->getTemp(0));
14892
14893 auto bailoutCmp = [&](Assembler::Condition cond, auto lhs, auto rhs) {
14894 if (type == MIRType::Int32) {
14895 bailoutCmp32(cond, lhs, rhs, snapshot);
14896 } else {
14897 MOZ_ASSERT(type == MIRType::IntPtr)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(type == MIRType::IntPtr)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(type == MIRType::IntPtr))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("type == MIRType::IntPtr"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 14897); AnnotateMozCrashReason("MOZ_ASSERT" "(" "type == MIRType::IntPtr"
")"); do { *((volatile int*)__null) = 14897; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
14898 bailoutCmpPtr(cond, lhs, rhs, snapshot);
14899 }
14900 };
14901
14902 auto bailoutCmpConstant = [&](Assembler::Condition cond, auto lhs,
14903 int32_t rhs) {
14904 if (type == MIRType::Int32) {
14905 bailoutCmp32(cond, lhs, Imm32(rhs), snapshot);
14906 } else {
14907 MOZ_ASSERT(type == MIRType::IntPtr)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(type == MIRType::IntPtr)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(type == MIRType::IntPtr))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("type == MIRType::IntPtr"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 14907); AnnotateMozCrashReason("MOZ_ASSERT" "(" "type == MIRType::IntPtr"
")"); do { *((volatile int*)__null) = 14907; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
14908 bailoutCmpPtr(cond, lhs, ImmWord(rhs), snapshot);
14909 }
14910 };
14911
14912 if (lir->index()->isConstant()) {
14913 int32_t nmin, nmax;
14914 int32_t index = ToInt32(lir->index());
14915 if (SafeAdd(index, min, &nmin) && SafeAdd(index, max, &nmax) && nmin >= 0) {
14916 if (length->isRegister()) {
14917 bailoutCmpConstant(Assembler::BelowOrEqual, ToRegister(length), nmax);
14918 } else {
14919 bailoutCmpConstant(Assembler::BelowOrEqual, ToAddress(length), nmax);
14920 }
14921 return;
14922 }
14923 masm.mov(ImmWord(index), temp);
14924 } else {
14925 masm.mov(ToRegister(lir->index()), temp);
14926 }
14927
14928 // If the minimum and maximum differ then do an underflow check first.
14929 // If the two are the same then doing an unsigned comparison on the
14930 // length will also catch a negative index.
14931 if (min != max) {
14932 if (min != 0) {
14933 Label bail;
14934 if (type == MIRType::Int32) {
14935 masm.branchAdd32(Assembler::Overflow, Imm32(min), temp, &bail);
14936 } else {
14937 masm.branchAddPtr(Assembler::Overflow, Imm32(min), temp, &bail);
14938 }
14939 bailoutFrom(&bail, snapshot);
14940 }
14941
14942 bailoutCmpConstant(Assembler::LessThan, temp, 0);
14943
14944 if (min != 0) {
14945 int32_t diff;
14946 if (SafeSub(max, min, &diff)) {
14947 max = diff;
14948 } else {
14949 if (type == MIRType::Int32) {
14950 masm.sub32(Imm32(min), temp);
14951 } else {
14952 masm.subPtr(Imm32(min), temp);
14953 }
14954 }
14955 }
14956 }
14957
14958 // Compute the maximum possible index. No overflow check is needed when
14959 // max > 0. We can only wraparound to a negative number, which will test as
14960 // larger than all nonnegative numbers in the unsigned comparison, and the
14961 // length is required to be nonnegative (else testing a negative length
14962 // would succeed on any nonnegative index).
14963 if (max != 0) {
14964 if (max < 0) {
14965 Label bail;
14966 if (type == MIRType::Int32) {
14967 masm.branchAdd32(Assembler::Overflow, Imm32(max), temp, &bail);
14968 } else {
14969 masm.branchAddPtr(Assembler::Overflow, Imm32(max), temp, &bail);
14970 }
14971 bailoutFrom(&bail, snapshot);
14972 } else {
14973 if (type == MIRType::Int32) {
14974 masm.add32(Imm32(max), temp);
14975 } else {
14976 masm.addPtr(Imm32(max), temp);
14977 }
14978 }
14979 }
14980
14981 if (length->isRegister()) {
14982 bailoutCmp(Assembler::BelowOrEqual, ToRegister(length), temp);
14983 } else {
14984 bailoutCmp(Assembler::BelowOrEqual, ToAddress(length), temp);
14985 }
14986}
14987
14988void CodeGenerator::visitBoundsCheckLower(LBoundsCheckLower* lir) {
14989 int32_t min = lir->mir()->minimum();
14990 bailoutCmp32(Assembler::LessThan, ToRegister(lir->index()), Imm32(min),
14991 lir->snapshot());
14992}
14993
14994void CodeGenerator::visitSpectreMaskIndex(LSpectreMaskIndex* lir) {
14995 MOZ_ASSERT(JitOptions.spectreIndexMasking)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(JitOptions.spectreIndexMasking)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(JitOptions.spectreIndexMasking
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"JitOptions.spectreIndexMasking", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 14995); AnnotateMozCrashReason("MOZ_ASSERT" "(" "JitOptions.spectreIndexMasking"
")"); do { *((volatile int*)__null) = 14995; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
14996
14997 const LAllocation* length = lir->length();
14998 Register index = ToRegister(lir->index());
14999 Register output = ToRegister(lir->output());
15000
15001 if (lir->mir()->type() == MIRType::Int32) {
15002 if (length->isRegister()) {
15003 masm.spectreMaskIndex32(index, ToRegister(length), output);
15004 } else {
15005 masm.spectreMaskIndex32(index, ToAddress(length), output);
15006 }
15007 } else {
15008 MOZ_ASSERT(lir->mir()->type() == MIRType::IntPtr)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(lir->mir()->type() == MIRType::IntPtr)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(lir->mir()->type() == MIRType::IntPtr))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("lir->mir()->type() == MIRType::IntPtr"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 15008); AnnotateMozCrashReason("MOZ_ASSERT" "(" "lir->mir()->type() == MIRType::IntPtr"
")"); do { *((volatile int*)__null) = 15008; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
15009 if (length->isRegister()) {
15010 masm.spectreMaskIndexPtr(index, ToRegister(length), output);
15011 } else {
15012 masm.spectreMaskIndexPtr(index, ToAddress(length), output);
15013 }
15014 }
15015}
15016
15017class OutOfLineStoreElementHole : public OutOfLineCodeBase<CodeGenerator> {
15018 LInstruction* ins_;
15019
15020 public:
15021 explicit OutOfLineStoreElementHole(LInstruction* ins) : ins_(ins) {
15022 MOZ_ASSERT(ins->isStoreElementHoleV() || ins->isStoreElementHoleT())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ins->isStoreElementHoleV() || ins->isStoreElementHoleT
())>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(ins->isStoreElementHoleV() || ins->isStoreElementHoleT
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("ins->isStoreElementHoleV() || ins->isStoreElementHoleT()"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 15022); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ins->isStoreElementHoleV() || ins->isStoreElementHoleT()"
")"); do { *((volatile int*)__null) = 15022; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
15023 }
15024
15025 void accept(CodeGenerator* codegen) override {
15026 codegen->visitOutOfLineStoreElementHole(this);
15027 }
15028
15029 MStoreElementHole* mir() const {
15030 return ins_->isStoreElementHoleV() ? ins_->toStoreElementHoleV()->mir()
15031 : ins_->toStoreElementHoleT()->mir();
15032 }
15033 LInstruction* ins() const { return ins_; }
15034};
15035
15036void CodeGenerator::emitStoreHoleCheck(Register elements,
15037 const LAllocation* index,
15038 LSnapshot* snapshot) {
15039 Label bail;
15040 if (index->isConstant()) {
15041 Address dest(elements, ToInt32(index) * sizeof(js::Value));
15042 masm.branchTestMagic(Assembler::Equal, dest, &bail);
15043 } else {
15044 BaseObjectElementIndex dest(elements, ToRegister(index));
15045 masm.branchTestMagic(Assembler::Equal, dest, &bail);
15046 }
15047 bailoutFrom(&bail, snapshot);
15048}
15049
15050void CodeGenerator::emitStoreElementTyped(const LAllocation* value,
15051 MIRType valueType, Register elements,
15052 const LAllocation* index) {
15053 MOZ_ASSERT(valueType != MIRType::MagicHole)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(valueType != MIRType::MagicHole)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(valueType != MIRType::MagicHole
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"valueType != MIRType::MagicHole", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 15053); AnnotateMozCrashReason("MOZ_ASSERT" "(" "valueType != MIRType::MagicHole"
")"); do { *((volatile int*)__null) = 15053; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
15054 ConstantOrRegister v = ToConstantOrRegister(value, valueType);
15055 if (index->isConstant()) {
15056 Address dest(elements, ToInt32(index) * sizeof(js::Value));
15057 masm.storeUnboxedValue(v, valueType, dest);
15058 } else {
15059 BaseObjectElementIndex dest(elements, ToRegister(index));
15060 masm.storeUnboxedValue(v, valueType, dest);
15061 }
15062}
15063
15064void CodeGenerator::visitStoreElementT(LStoreElementT* store) {
15065 Register elements = ToRegister(store->elements());
15066 const LAllocation* index = store->index();
15067
15068 if (store->mir()->needsBarrier()) {
15069 emitPreBarrier(elements, index);
15070 }
15071
15072 if (store->mir()->needsHoleCheck()) {
15073 emitStoreHoleCheck(elements, index, store->snapshot());
15074 }
15075
15076 emitStoreElementTyped(store->value(), store->mir()->value()->type(), elements,
15077 index);
15078}
15079
15080void CodeGenerator::visitStoreElementV(LStoreElementV* lir) {
15081 const ValueOperand value = ToValue(lir, LStoreElementV::Value);
15082 Register elements = ToRegister(lir->elements());
15083 const LAllocation* index = lir->index();
15084
15085 if (lir->mir()->needsBarrier()) {
15086 emitPreBarrier(elements, index);
15087 }
15088
15089 if (lir->mir()->needsHoleCheck()) {
15090 emitStoreHoleCheck(elements, index, lir->snapshot());
15091 }
15092
15093 if (lir->index()->isConstant()) {
15094 Address dest(elements, ToInt32(lir->index()) * sizeof(js::Value));
15095 masm.storeValue(value, dest);
15096 } else {
15097 BaseObjectElementIndex dest(elements, ToRegister(lir->index()));
15098 masm.storeValue(value, dest);
15099 }
15100}
15101
15102void CodeGenerator::visitStoreHoleValueElement(LStoreHoleValueElement* lir) {
15103 Register elements = ToRegister(lir->elements());
15104 Register index = ToRegister(lir->index());
15105
15106 Address elementsFlags(elements, ObjectElements::offsetOfFlags());
15107 masm.or32(Imm32(ObjectElements::NON_PACKED), elementsFlags);
15108
15109 BaseObjectElementIndex element(elements, index);
15110 masm.storeValue(MagicValue(JS_ELEMENTS_HOLE), element);
15111}
15112
15113void CodeGenerator::visitStoreElementHoleT(LStoreElementHoleT* lir) {
15114 auto* ool = new (alloc()) OutOfLineStoreElementHole(lir);
15115 addOutOfLineCode(ool, lir->mir());
15116
15117 Register obj = ToRegister(lir->object());
15118 Register elements = ToRegister(lir->elements());
15119 Register index = ToRegister(lir->index());
15120 Register temp = ToRegister(lir->temp0());
15121
15122 Address initLength(elements, ObjectElements::offsetOfInitializedLength());
15123 masm.spectreBoundsCheck32(index, initLength, temp, ool->entry());
15124
15125 emitPreBarrier(elements, lir->index());
15126
15127 masm.bind(ool->rejoin());
15128 emitStoreElementTyped(lir->value(), lir->mir()->value()->type(), elements,
15129 lir->index());
15130
15131 if (ValueNeedsPostBarrier(lir->mir()->value())) {
15132 LiveRegisterSet regs = liveVolatileRegs(lir);
15133 ConstantOrRegister val =
15134 ToConstantOrRegister(lir->value(), lir->mir()->value()->type());
15135 emitElementPostWriteBarrier(lir->mir(), regs, obj, lir->index(), temp, val);
15136 }
15137}
15138
15139void CodeGenerator::visitStoreElementHoleV(LStoreElementHoleV* lir) {
15140 auto* ool = new (alloc()) OutOfLineStoreElementHole(lir);
15141 addOutOfLineCode(ool, lir->mir());
15142
15143 Register obj = ToRegister(lir->object());
15144 Register elements = ToRegister(lir->elements());
15145 Register index = ToRegister(lir->index());
15146 const ValueOperand value = ToValue(lir, LStoreElementHoleV::ValueIndex);
15147 Register temp = ToRegister(lir->temp0());
15148
15149 Address initLength(elements, ObjectElements::offsetOfInitializedLength());
15150 masm.spectreBoundsCheck32(index, initLength, temp, ool->entry());
15151
15152 emitPreBarrier(elements, lir->index());
15153
15154 masm.bind(ool->rejoin());
15155 masm.storeValue(value, BaseObjectElementIndex(elements, index));
15156
15157 if (ValueNeedsPostBarrier(lir->mir()->value())) {
15158 LiveRegisterSet regs = liveVolatileRegs(lir);
15159 emitElementPostWriteBarrier(lir->mir(), regs, obj, lir->index(), temp,
15160 ConstantOrRegister(value));
15161 }
15162}
15163
15164void CodeGenerator::visitOutOfLineStoreElementHole(
15165 OutOfLineStoreElementHole* ool) {
15166 Register object, elements, index;
15167 LInstruction* ins = ool->ins();
15168 mozilla::Maybe<ConstantOrRegister> value;
15169 Register temp;
15170
15171 if (ins->isStoreElementHoleV()) {
15172 LStoreElementHoleV* store = ins->toStoreElementHoleV();
15173 object = ToRegister(store->object());
15174 elements = ToRegister(store->elements());
15175 index = ToRegister(store->index());
15176 value.emplace(
15177 TypedOrValueRegister(ToValue(store, LStoreElementHoleV::ValueIndex)));
15178 temp = ToRegister(store->temp0());
15179 } else {
15180 LStoreElementHoleT* store = ins->toStoreElementHoleT();
15181 object = ToRegister(store->object());
15182 elements = ToRegister(store->elements());
15183 index = ToRegister(store->index());
15184 if (store->value()->isConstant()) {
15185 value.emplace(
15186 ConstantOrRegister(store->value()->toConstant()->toJSValue()));
15187 } else {
15188 MIRType valueType = store->mir()->value()->type();
15189 value.emplace(
15190 TypedOrValueRegister(valueType, ToAnyRegister(store->value())));
15191 }
15192 temp = ToRegister(store->temp0());
15193 }
15194
15195 Address initLength(elements, ObjectElements::offsetOfInitializedLength());
15196
15197 // We're out-of-bounds. We only handle the index == initlength case.
15198 // If index > initializedLength, bail out. Note that this relies on the
15199 // condition flags sticking from the incoming branch.
15200 // Also note: this branch does not need Spectre mitigations, doing that for
15201 // the capacity check below is sufficient.
15202 Label allocElement, addNewElement;
15203#if defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || \
15204 defined(JS_CODEGEN_LOONG64) || defined(JS_CODEGEN_RISCV64)
15205 // Had to reimplement for MIPS because there are no flags.
15206 bailoutCmp32(Assembler::NotEqual, initLength, index, ins->snapshot());
15207#else
15208 bailoutIf(Assembler::NotEqual, ins->snapshot());
15209#endif
15210
15211 // If index < capacity, we can add a dense element inline. If not, we need
15212 // to allocate more elements first.
15213 masm.spectreBoundsCheck32(
15214 index, Address(elements, ObjectElements::offsetOfCapacity()), temp,
15215 &allocElement);
15216 masm.jump(&addNewElement);
15217
15218 masm.bind(&allocElement);
15219
15220 // Save all live volatile registers, except |temp|.
15221 LiveRegisterSet liveRegs = liveVolatileRegs(ins);
15222 liveRegs.takeUnchecked(temp);
15223 masm.PushRegsInMask(liveRegs);
15224
15225 masm.setupAlignedABICall();
15226 masm.loadJSContext(temp);
15227 masm.passABIArg(temp);
15228 masm.passABIArg(object);
15229
15230 using Fn = bool (*)(JSContext*, NativeObject*);
15231 masm.callWithABI<Fn, NativeObject::addDenseElementPure>();
15232 masm.storeCallPointerResult(temp);
15233
15234 masm.PopRegsInMask(liveRegs);
15235 bailoutIfFalseBool(temp, ins->snapshot());
15236
15237 // Load the reallocated elements pointer.
15238 masm.loadPtr(Address(object, NativeObject::offsetOfElements()), elements);
15239
15240 masm.bind(&addNewElement);
15241
15242 // Increment initLength
15243 masm.add32(Imm32(1), initLength);
15244
15245 // If length is now <= index, increment length too.
15246 Label skipIncrementLength;
15247 Address length(elements, ObjectElements::offsetOfLength());
15248 masm.branch32(Assembler::Above, length, index, &skipIncrementLength);
15249 masm.add32(Imm32(1), length);
15250 masm.bind(&skipIncrementLength);
15251
15252 // Jump to the inline path where we will store the value.
15253 // We rejoin after the prebarrier, because the memory is uninitialized.
15254 masm.jump(ool->rejoin());
15255}
15256
15257void CodeGenerator::visitArrayPopShift(LArrayPopShift* lir) {
15258 Register obj = ToRegister(lir->object());
15259 Register temp1 = ToRegister(lir->temp0());
15260 Register temp2 = ToRegister(lir->temp1());
15261 ValueOperand out = ToOutValue(lir);
15262
15263 Label bail;
15264 if (lir->mir()->mode() == MArrayPopShift::Pop) {
15265 masm.packedArrayPop(obj, out, temp1, temp2, &bail);
15266 } else {
15267 MOZ_ASSERT(lir->mir()->mode() == MArrayPopShift::Shift)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(lir->mir()->mode() == MArrayPopShift::Shift)>
::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(lir->mir()->mode() == MArrayPopShift::Shift)))
, 0))) { do { } while (false); MOZ_ReportAssertionFailure("lir->mir()->mode() == MArrayPopShift::Shift"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 15267); AnnotateMozCrashReason("MOZ_ASSERT" "(" "lir->mir()->mode() == MArrayPopShift::Shift"
")"); do { *((volatile int*)__null) = 15267; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
15268 LiveRegisterSet volatileRegs = liveVolatileRegs(lir);
15269 masm.packedArrayShift(obj, out, temp1, temp2, volatileRegs, &bail);
15270 }
15271 bailoutFrom(&bail, lir->snapshot());
15272}
15273
15274class OutOfLineArrayPush : public OutOfLineCodeBase<CodeGenerator> {
15275 LArrayPush* ins_;
15276
15277 public:
15278 explicit OutOfLineArrayPush(LArrayPush* ins) : ins_(ins) {}
15279
15280 void accept(CodeGenerator* codegen) override {
15281 codegen->visitOutOfLineArrayPush(this);
15282 }
15283
15284 LArrayPush* ins() const { return ins_; }
15285};
15286
15287void CodeGenerator::visitArrayPush(LArrayPush* lir) {
15288 Register obj = ToRegister(lir->object());
15289 Register elementsTemp = ToRegister(lir->temp0());
15290 Register length = ToRegister(lir->output());
15291 ValueOperand value = ToValue(lir, LArrayPush::ValueIndex);
15292 Register spectreTemp = ToTempRegisterOrInvalid(lir->temp1());
15293
15294 auto* ool = new (alloc()) OutOfLineArrayPush(lir);
15295 addOutOfLineCode(ool, lir->mir());
15296
15297 // Load obj->elements in elementsTemp.
15298 masm.loadPtr(Address(obj, NativeObject::offsetOfElements()), elementsTemp);
15299
15300 Address initLengthAddr(elementsTemp,
15301 ObjectElements::offsetOfInitializedLength());
15302 Address lengthAddr(elementsTemp, ObjectElements::offsetOfLength());
15303 Address capacityAddr(elementsTemp, ObjectElements::offsetOfCapacity());
15304
15305 // Bail out if length != initLength.
15306 masm.load32(lengthAddr, length);
15307 bailoutCmp32(Assembler::NotEqual, initLengthAddr, length, lir->snapshot());
15308
15309 // If length < capacity, we can add a dense element inline. If not, we
15310 // need to allocate more elements.
15311 masm.spectreBoundsCheck32(length, capacityAddr, spectreTemp, ool->entry());
15312 masm.bind(ool->rejoin());
15313
15314 // Store the value.
15315 masm.storeValue(value, BaseObjectElementIndex(elementsTemp, length));
15316
15317 // Update length and initialized length.
15318 masm.add32(Imm32(1), length);
15319 masm.store32(length, Address(elementsTemp, ObjectElements::offsetOfLength()));
15320 masm.store32(length, Address(elementsTemp,
15321 ObjectElements::offsetOfInitializedLength()));
15322
15323 if (ValueNeedsPostBarrier(lir->mir()->value())) {
15324 LiveRegisterSet regs = liveVolatileRegs(lir);
15325 regs.addUnchecked(length);
15326 emitElementPostWriteBarrier(lir->mir(), regs, obj, lir->output()->output(),
15327 elementsTemp, ConstantOrRegister(value),
15328 /* indexDiff = */ -1);
15329 }
15330}
15331
15332void CodeGenerator::visitOutOfLineArrayPush(OutOfLineArrayPush* ool) {
15333 LArrayPush* ins = ool->ins();
15334
15335 Register object = ToRegister(ins->object());
15336 Register temp = ToRegister(ins->temp0());
15337
15338 LiveRegisterSet liveRegs = liveVolatileRegs(ins);
15339 liveRegs.takeUnchecked(temp);
15340 liveRegs.addUnchecked(ToRegister(ins->output()));
15341 liveRegs.addUnchecked(ToValue(ins, LArrayPush::ValueIndex));
15342
15343 masm.PushRegsInMask(liveRegs);
15344
15345 masm.setupAlignedABICall();
15346 masm.loadJSContext(temp);
15347 masm.passABIArg(temp);
15348 masm.passABIArg(object);
15349
15350 using Fn = bool (*)(JSContext*, NativeObject* obj);
15351 masm.callWithABI<Fn, NativeObject::addDenseElementPure>();
15352 masm.storeCallPointerResult(temp);
15353
15354 masm.PopRegsInMask(liveRegs);
15355 bailoutIfFalseBool(temp, ins->snapshot());
15356
15357 // Load the reallocated elements pointer.
15358 masm.loadPtr(Address(object, NativeObject::offsetOfElements()), temp);
15359
15360 masm.jump(ool->rejoin());
15361}
15362
15363void CodeGenerator::visitArraySlice(LArraySlice* lir) {
15364 Register object = ToRegister(lir->object());
15365 Register begin = ToRegister(lir->begin());
15366 Register end = ToRegister(lir->end());
15367 Register temp0 = ToRegister(lir->temp0());
15368 Register temp1 = ToRegister(lir->temp1());
15369
15370 Label call, fail;
15371
15372 Label bail;
15373 masm.branchArrayIsNotPacked(object, temp0, temp1, &bail);
15374 bailoutFrom(&bail, lir->snapshot());
15375
15376 // Try to allocate an object.
15377 TemplateObject templateObject(lir->mir()->templateObj());
15378 masm.createGCObject(temp0, temp1, templateObject, lir->mir()->initialHeap(),
15379 &fail);
15380
15381 masm.jump(&call);
15382 {
15383 masm.bind(&fail);
15384 masm.movePtr(ImmPtr(nullptr), temp0);
15385 }
15386 masm.bind(&call);
15387
15388 pushArg(temp0);
15389 pushArg(end);
15390 pushArg(begin);
15391 pushArg(object);
15392
15393 using Fn =
15394 JSObject* (*)(JSContext*, HandleObject, int32_t, int32_t, HandleObject);
15395 callVM<Fn, ArraySliceDense>(lir);
15396}
15397
15398void CodeGenerator::visitArgumentsSlice(LArgumentsSlice* lir) {
15399 Register object = ToRegister(lir->object());
15400 Register begin = ToRegister(lir->begin());
15401 Register end = ToRegister(lir->end());
15402 Register temp0 = ToRegister(lir->temp0());
15403 Register temp1 = ToRegister(lir->temp1());
15404
15405 Label call, fail;
15406
15407 // Try to allocate an object.
15408 TemplateObject templateObject(lir->mir()->templateObj());
15409 masm.createGCObject(temp0, temp1, templateObject, lir->mir()->initialHeap(),
15410 &fail);
15411
15412 masm.jump(&call);
15413 {
15414 masm.bind(&fail);
15415 masm.movePtr(ImmPtr(nullptr), temp0);
15416 }
15417 masm.bind(&call);
15418
15419 pushArg(temp0);
15420 pushArg(end);
15421 pushArg(begin);
15422 pushArg(object);
15423
15424 using Fn =
15425 JSObject* (*)(JSContext*, HandleObject, int32_t, int32_t, HandleObject);
15426 callVM<Fn, ArgumentsSliceDense>(lir);
15427}
15428
15429#ifdef DEBUG1
15430void CodeGenerator::emitAssertArgumentsSliceBounds(const RegisterOrInt32& begin,
15431 const RegisterOrInt32& count,
15432 Register numActualArgs) {
15433 // |begin| must be positive or zero.
15434 if (begin.is<Register>()) {
15435 Label beginOk;
15436 masm.branch32(Assembler::GreaterThanOrEqual, begin.as<Register>(), Imm32(0),
15437 &beginOk);
15438 masm.assumeUnreachable("begin < 0");
15439 masm.bind(&beginOk);
15440 } else {
15441 MOZ_ASSERT(begin.as<int32_t>() >= 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(begin.as<int32_t>() >= 0)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(begin.as<int32_t>() >=
0))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("begin.as<int32_t>() >= 0", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 15441); AnnotateMozCrashReason("MOZ_ASSERT" "(" "begin.as<int32_t>() >= 0"
")"); do { *((volatile int*)__null) = 15441; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
15442 }
15443
15444 // |count| must be positive or zero.
15445 if (count.is<Register>()) {
15446 Label countOk;
15447 masm.branch32(Assembler::GreaterThanOrEqual, count.as<Register>(), Imm32(0),
15448 &countOk);
15449 masm.assumeUnreachable("count < 0");
15450 masm.bind(&countOk);
15451 } else {
15452 MOZ_ASSERT(count.as<int32_t>() >= 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(count.as<int32_t>() >= 0)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(count.as<int32_t>() >=
0))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("count.as<int32_t>() >= 0", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 15452); AnnotateMozCrashReason("MOZ_ASSERT" "(" "count.as<int32_t>() >= 0"
")"); do { *((volatile int*)__null) = 15452; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
15453 }
15454
15455 // |begin| must be less-or-equal to |numActualArgs|.
15456 Label argsBeginOk;
15457 if (begin.is<Register>()) {
15458 masm.branchPtr(Assembler::AboveOrEqual, numActualArgs, begin.as<Register>(),
15459 &argsBeginOk);
15460 } else {
15461 masm.branchPtr(Assembler::AboveOrEqual, numActualArgs,
15462 Imm32(begin.as<int32_t>()), &argsBeginOk);
15463 }
15464 masm.assumeUnreachable("begin <= numActualArgs");
15465 masm.bind(&argsBeginOk);
15466
15467 // |count| must be less-or-equal to |numActualArgs|.
15468 Label argsCountOk;
15469 if (count.is<Register>()) {
15470 masm.branchPtr(Assembler::AboveOrEqual, numActualArgs, count.as<Register>(),
15471 &argsCountOk);
15472 } else {
15473 masm.branchPtr(Assembler::AboveOrEqual, numActualArgs,
15474 Imm32(count.as<int32_t>()), &argsCountOk);
15475 }
15476 masm.assumeUnreachable("count <= numActualArgs");
15477 masm.bind(&argsCountOk);
15478
15479 // |begin| and |count| must be preserved, but |numActualArgs| can be changed.
15480 //
15481 // Pre-condition: |count| <= |numActualArgs|
15482 // Condition to test: |begin + count| <= |numActualArgs|
15483 // Transform to: |begin| <= |numActualArgs - count|
15484 if (count.is<Register>()) {
15485 masm.subPtr(count.as<Register>(), numActualArgs);
15486 } else {
15487 masm.subPtr(Imm32(count.as<int32_t>()), numActualArgs);
15488 }
15489
15490 // |begin + count| must be less-or-equal to |numActualArgs|.
15491 Label argsBeginCountOk;
15492 if (begin.is<Register>()) {
15493 masm.branchPtr(Assembler::AboveOrEqual, numActualArgs, begin.as<Register>(),
15494 &argsBeginCountOk);
15495 } else {
15496 masm.branchPtr(Assembler::AboveOrEqual, numActualArgs,
15497 Imm32(begin.as<int32_t>()), &argsBeginCountOk);
15498 }
15499 masm.assumeUnreachable("begin + count <= numActualArgs");
15500 masm.bind(&argsBeginCountOk);
15501}
15502#endif
15503
15504template <class ArgumentsSlice>
15505void CodeGenerator::emitNewArray(ArgumentsSlice* lir,
15506 const RegisterOrInt32& count, Register output,
15507 Register temp) {
15508 using Fn = ArrayObject* (*)(JSContext*, int32_t);
15509 auto* ool = count.match(
15510 [&](Register count) {
15511 return oolCallVM<Fn, NewArrayObjectEnsureDenseInitLength>(
15512 lir, ArgList(count), StoreRegisterTo(output));
15513 },
15514 [&](int32_t count) {
15515 return oolCallVM<Fn, NewArrayObjectEnsureDenseInitLength>(
15516 lir, ArgList(Imm32(count)), StoreRegisterTo(output));
15517 });
15518
15519 TemplateObject templateObject(lir->mir()->templateObj());
15520 MOZ_ASSERT(templateObject.isArrayObject())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(templateObject.isArrayObject())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(templateObject.isArrayObject
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("templateObject.isArrayObject()", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 15520); AnnotateMozCrashReason("MOZ_ASSERT" "(" "templateObject.isArrayObject()"
")"); do { *((volatile int*)__null) = 15520; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
15521
15522 auto templateNativeObj = templateObject.asTemplateNativeObject();
15523 MOZ_ASSERT(templateNativeObj.getArrayLength() == 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(templateNativeObj.getArrayLength() == 0)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(templateNativeObj.getArrayLength() == 0))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("templateNativeObj.getArrayLength() == 0"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 15523); AnnotateMozCrashReason("MOZ_ASSERT" "(" "templateNativeObj.getArrayLength() == 0"
")"); do { *((volatile int*)__null) = 15523; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
15524 MOZ_ASSERT(templateNativeObj.getDenseInitializedLength() == 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(templateNativeObj.getDenseInitializedLength() == 0)>
::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(templateNativeObj.getDenseInitializedLength() == 0))
), 0))) { do { } while (false); MOZ_ReportAssertionFailure("templateNativeObj.getDenseInitializedLength() == 0"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 15524); AnnotateMozCrashReason("MOZ_ASSERT" "(" "templateNativeObj.getDenseInitializedLength() == 0"
")"); do { *((volatile int*)__null) = 15524; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
15525 MOZ_ASSERT(!templateNativeObj.hasDynamicElements())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!templateNativeObj.hasDynamicElements())>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(!templateNativeObj.hasDynamicElements()))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("!templateNativeObj.hasDynamicElements()"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 15525); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!templateNativeObj.hasDynamicElements()"
")"); do { *((volatile int*)__null) = 15525; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
15526
15527 // Check array capacity. Call into the VM if the template object's capacity
15528 // is too small.
15529 bool tryAllocate = count.match(
15530 [&](Register count) {
15531 masm.branch32(Assembler::Above, count,
15532 Imm32(templateNativeObj.getDenseCapacity()),
15533 ool->entry());
15534 return true;
15535 },
15536 [&](int32_t count) {
15537 MOZ_ASSERT(count >= 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(count >= 0)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(count >= 0))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("count >= 0",
"/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 15537); AnnotateMozCrashReason("MOZ_ASSERT" "(" "count >= 0"
")"); do { *((volatile int*)__null) = 15537; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
15538 if (uint32_t(count) > templateNativeObj.getDenseCapacity()) {
15539 masm.jump(ool->entry());
15540 return false;
15541 }
15542 return true;
15543 });
15544
15545 if (tryAllocate) {
15546 // Try to allocate an object.
15547 masm.createGCObject(output, temp, templateObject, lir->mir()->initialHeap(),
15548 ool->entry());
15549
15550 auto setInitializedLengthAndLength = [&](auto count) {
15551 const int elementsOffset = NativeObject::offsetOfFixedElements();
15552
15553 // Update initialized length.
15554 Address initLength(
15555 output, elementsOffset + ObjectElements::offsetOfInitializedLength());
15556 masm.store32(count, initLength);
15557
15558 // Update length.
15559 Address length(output, elementsOffset + ObjectElements::offsetOfLength());
15560 masm.store32(count, length);
15561 };
15562
15563 // The array object was successfully created. Set the length and initialized
15564 // length and then proceed to fill the elements.
15565 count.match([&](Register count) { setInitializedLengthAndLength(count); },
15566 [&](int32_t count) {
15567 if (count > 0) {
15568 setInitializedLengthAndLength(Imm32(count));
15569 }
15570 });
15571 }
15572
15573 masm.bind(ool->rejoin());
15574}
15575
15576void CodeGenerator::visitFrameArgumentsSlice(LFrameArgumentsSlice* lir) {
15577 Register begin = ToRegister(lir->begin());
15578 Register count = ToRegister(lir->count());
15579 Register temp = ToRegister(lir->temp0());
15580 Register output = ToRegister(lir->output());
15581
15582#ifdef DEBUG1
15583 masm.loadNumActualArgs(FramePointer, temp);
15584 emitAssertArgumentsSliceBounds(RegisterOrInt32(begin), RegisterOrInt32(count),
15585 temp);
15586#endif
15587
15588 emitNewArray(lir, RegisterOrInt32(count), output, temp);
15589
15590 Label done;
15591 masm.branch32(Assembler::Equal, count, Imm32(0), &done);
15592 {
15593 AllocatableGeneralRegisterSet allRegs(GeneralRegisterSet::All());
15594 allRegs.take(begin);
15595 allRegs.take(count);
15596 allRegs.take(temp);
15597 allRegs.take(output);
15598
15599 ValueOperand value = allRegs.takeAnyValue();
15600
15601 LiveRegisterSet liveRegs;
15602 liveRegs.add(output);
15603 liveRegs.add(begin);
15604 liveRegs.add(value);
15605
15606 masm.PushRegsInMask(liveRegs);
15607
15608 // Initialize all elements.
15609
15610 Register elements = output;
15611 masm.loadPtr(Address(output, NativeObject::offsetOfElements()), elements);
15612
15613 Register argIndex = begin;
15614
15615 Register index = temp;
15616 masm.move32(Imm32(0), index);
15617
15618 size_t argvOffset = JitFrameLayout::offsetOfActualArgs();
15619 BaseValueIndex argPtr(FramePointer, argIndex, argvOffset);
15620
15621 Label loop;
15622 masm.bind(&loop);
15623
15624 masm.loadValue(argPtr, value);
15625
15626 // We don't need a pre-barrier, because the element at |index| is guaranteed
15627 // to be a non-GC thing (either uninitialized memory or the magic hole
15628 // value).
15629 masm.storeValue(value, BaseObjectElementIndex(elements, index));
15630
15631 masm.add32(Imm32(1), index);
15632 masm.add32(Imm32(1), argIndex);
15633
15634 masm.branch32(Assembler::LessThan, index, count, &loop);
15635
15636 masm.PopRegsInMask(liveRegs);
15637
15638 // Emit a post-write barrier if |output| is tenured.
15639 //
15640 // We expect that |output| is nursery allocated, so it isn't worth the
15641 // trouble to check if no frame argument is a nursery thing, which would
15642 // allow to omit the post-write barrier.
15643 masm.branchPtrInNurseryChunk(Assembler::Equal, output, temp, &done);
15644
15645 LiveRegisterSet volatileRegs = liveVolatileRegs(lir);
15646 volatileRegs.takeUnchecked(temp);
15647 if (output.volatile_()) {
15648 volatileRegs.addUnchecked(output);
15649 }
15650
15651 masm.PushRegsInMask(volatileRegs);
15652 emitPostWriteBarrier(output);
15653 masm.PopRegsInMask(volatileRegs);
15654 }
15655 masm.bind(&done);
15656}
15657
15658CodeGenerator::RegisterOrInt32 CodeGenerator::ToRegisterOrInt32(
15659 const LAllocation* allocation) {
15660 if (allocation->isConstant()) {
15661 return RegisterOrInt32(allocation->toConstant()->toInt32());
15662 }
15663 return RegisterOrInt32(ToRegister(allocation));
15664}
15665
15666void CodeGenerator::visitInlineArgumentsSlice(LInlineArgumentsSlice* lir) {
15667 RegisterOrInt32 begin = ToRegisterOrInt32(lir->begin());
15668 RegisterOrInt32 count = ToRegisterOrInt32(lir->count());
15669 Register temp = ToRegister(lir->temp());
15670 Register output = ToRegister(lir->output());
15671
15672 uint32_t numActuals = lir->mir()->numActuals();
15673
15674#ifdef DEBUG1
15675 masm.move32(Imm32(numActuals), temp);
15676
15677 emitAssertArgumentsSliceBounds(begin, count, temp);
15678#endif
15679
15680 emitNewArray(lir, count, output, temp);
15681
15682 // We're done if there are no actual arguments.
15683 if (numActuals == 0) {
15684 return;
15685 }
15686
15687 // Check if any arguments have to be copied.
15688 Label done;
15689 if (count.is<Register>()) {
15690 masm.branch32(Assembler::Equal, count.as<Register>(), Imm32(0), &done);
15691 } else if (count.as<int32_t>() == 0) {
15692 return;
15693 }
15694
15695 auto getArg = [&](uint32_t i) {
15696 return toConstantOrRegister(lir, LInlineArgumentsSlice::ArgIndex(i),
15697 lir->mir()->getArg(i)->type());
15698 };
15699
15700 auto storeArg = [&](uint32_t i, auto dest) {
15701 // We don't need a pre-barrier because the element at |index| is guaranteed
15702 // to be a non-GC thing (either uninitialized memory or the magic hole
15703 // value).
15704 masm.storeConstantOrRegister(getArg(i), dest);
15705 };
15706
15707 // Initialize all elements.
15708 if (numActuals == 1) {
15709 // There's exactly one argument. We've checked that |count| is non-zero,
15710 // which implies that |begin| must be zero.
15711 MOZ_ASSERT_IF(begin.is<int32_t>(), begin.as<int32_t>() == 0)do { if (begin.is<int32_t>()) { do { static_assert( mozilla
::detail::AssertionConditionType<decltype(begin.as<int32_t
>() == 0)>::isValid, "invalid assertion condition"); if
((__builtin_expect(!!(!(!!(begin.as<int32_t>() == 0)))
, 0))) { do { } while (false); MOZ_ReportAssertionFailure("begin.as<int32_t>() == 0"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 15711); AnnotateMozCrashReason("MOZ_ASSERT" "(" "begin.as<int32_t>() == 0"
")"); do { *((volatile int*)__null) = 15711; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false); } } while
(false)
;
15712
15713 Register elements = temp;
15714 masm.loadPtr(Address(output, NativeObject::offsetOfElements()), elements);
15715
15716 storeArg(0, Address(elements, 0));
15717 } else if (begin.is<Register>()) {
15718 // There is more than one argument and |begin| isn't a compile-time
15719 // constant. Iterate through 0..numActuals to search for |begin| and then
15720 // start copying |count| arguments from that index.
15721
15722 LiveGeneralRegisterSet liveRegs;
15723 liveRegs.add(output);
15724 liveRegs.add(begin.as<Register>());
15725
15726 masm.PushRegsInMask(liveRegs);
15727
15728 Register elements = output;
15729 masm.loadPtr(Address(output, NativeObject::offsetOfElements()), elements);
15730
15731 Register argIndex = begin.as<Register>();
15732
15733 Register index = temp;
15734 masm.move32(Imm32(0), index);
15735
15736 Label doneLoop;
15737 for (uint32_t i = 0; i < numActuals; ++i) {
15738 Label next;
15739 masm.branch32(Assembler::NotEqual, argIndex, Imm32(i), &next);
15740
15741 storeArg(i, BaseObjectElementIndex(elements, index));
15742
15743 masm.add32(Imm32(1), index);
15744 masm.add32(Imm32(1), argIndex);
15745
15746 if (count.is<Register>()) {
15747 masm.branch32(Assembler::GreaterThanOrEqual, index,
15748 count.as<Register>(), &doneLoop);
15749 } else {
15750 masm.branch32(Assembler::GreaterThanOrEqual, index,
15751 Imm32(count.as<int32_t>()), &doneLoop);
15752 }
15753
15754 masm.bind(&next);
15755 }
15756 masm.bind(&doneLoop);
15757
15758 masm.PopRegsInMask(liveRegs);
15759 } else {
15760 // There is more than one argument and |begin| is a compile-time constant.
15761
15762 Register elements = temp;
15763 masm.loadPtr(Address(output, NativeObject::offsetOfElements()), elements);
15764
15765 int32_t argIndex = begin.as<int32_t>();
15766
15767 int32_t index = 0;
15768
15769 Label doneLoop;
15770 for (uint32_t i = argIndex; i < numActuals; ++i) {
15771 storeArg(i, Address(elements, index * sizeof(Value)));
15772
15773 index += 1;
15774
15775 if (count.is<Register>()) {
15776 masm.branch32(Assembler::LessThanOrEqual, count.as<Register>(),
15777 Imm32(index), &doneLoop);
15778 } else {
15779 if (index >= count.as<int32_t>()) {
15780 break;
15781 }
15782 }
15783 }
15784 masm.bind(&doneLoop);
15785 }
15786
15787 // Determine if we have to emit post-write barrier.
15788 //
15789 // If either |begin| or |count| is a constant, use their value directly.
15790 // Otherwise assume we copy all inline arguments from 0..numActuals.
15791 bool postWriteBarrier = false;
15792 uint32_t actualBegin = begin.match([](Register) { return 0; },
15793 [](int32_t value) { return value; });
15794 uint32_t actualCount =
15795 count.match([=](Register) { return numActuals; },
15796 [](int32_t value) -> uint32_t { return value; });
15797 for (uint32_t i = 0; i < actualCount; ++i) {
15798 ConstantOrRegister arg = getArg(actualBegin + i);
15799 if (arg.constant()) {
15800 Value v = arg.value();
15801 if (v.isGCThing() && IsInsideNursery(v.toGCThing())) {
15802 postWriteBarrier = true;
15803 }
15804 } else {
15805 MIRType type = arg.reg().type();
15806 if (type == MIRType::Value || NeedsPostBarrier(type)) {
15807 postWriteBarrier = true;
15808 }
15809 }
15810 }
15811
15812 // Emit a post-write barrier if |output| is tenured and we couldn't
15813 // determine at compile-time that no barrier is needed.
15814 if (postWriteBarrier) {
15815 masm.branchPtrInNurseryChunk(Assembler::Equal, output, temp, &done);
15816
15817 LiveRegisterSet volatileRegs = liveVolatileRegs(lir);
15818 volatileRegs.takeUnchecked(temp);
15819 if (output.volatile_()) {
15820 volatileRegs.addUnchecked(output);
15821 }
15822
15823 masm.PushRegsInMask(volatileRegs);
15824 emitPostWriteBarrier(output);
15825 masm.PopRegsInMask(volatileRegs);
15826 }
15827
15828 masm.bind(&done);
15829}
15830
15831void CodeGenerator::visitNormalizeSliceTerm(LNormalizeSliceTerm* lir) {
15832 Register value = ToRegister(lir->value());
15833 Register length = ToRegister(lir->length());
15834 Register output = ToRegister(lir->output());
15835
15836 masm.move32(value, output);
15837
15838 Label positive;
15839 masm.branch32(Assembler::GreaterThanOrEqual, value, Imm32(0), &positive);
15840
15841 Label done;
15842 masm.add32(length, output);
15843 masm.branch32(Assembler::GreaterThanOrEqual, output, Imm32(0), &done);
15844 masm.move32(Imm32(0), output);
15845 masm.jump(&done);
15846
15847 masm.bind(&positive);
15848 masm.cmp32Move32(Assembler::LessThan, length, value, length, output);
15849
15850 masm.bind(&done);
15851}
15852
15853void CodeGenerator::visitArrayJoin(LArrayJoin* lir) {
15854 Label skipCall;
15855
15856 Register output = ToRegister(lir->output());
15857 Register sep = ToRegister(lir->separator());
15858 Register array = ToRegister(lir->array());
15859 Register temp = ToRegister(lir->temp0());
15860
15861 // Fast path for simple length <= 1 cases.
15862 {
15863 masm.loadPtr(Address(array, NativeObject::offsetOfElements()), temp);
15864 Address length(temp, ObjectElements::offsetOfLength());
15865 Address initLength(temp, ObjectElements::offsetOfInitializedLength());
15866
15867 // Check for length == 0
15868 Label notEmpty;
15869 masm.branch32(Assembler::NotEqual, length, Imm32(0), &notEmpty);
15870 const JSAtomState& names = gen->runtime->names();
15871 masm.movePtr(ImmGCPtr(names.empty_), output);
15872 masm.jump(&skipCall);
15873
15874 masm.bind(&notEmpty);
15875 Label notSingleString;
15876 // Check for length == 1, initializedLength >= 1, arr[0].isString()
15877 masm.branch32(Assembler::NotEqual, length, Imm32(1), &notSingleString);
15878 masm.branch32(Assembler::LessThan, initLength, Imm32(1), &notSingleString);
15879
15880 Address elem0(temp, 0);
15881 masm.branchTestString(Assembler::NotEqual, elem0, &notSingleString);
15882
15883 // At this point, 'output' can be used as a scratch register, since we're
15884 // guaranteed to succeed.
15885 masm.unboxString(elem0, output);
15886 masm.jump(&skipCall);
15887 masm.bind(&notSingleString);
15888 }
15889
15890 pushArg(sep);
15891 pushArg(array);
15892
15893 using Fn = JSString* (*)(JSContext*, HandleObject, HandleString);
15894 callVM<Fn, jit::ArrayJoin>(lir);
15895 masm.bind(&skipCall);
15896}
15897
15898void CodeGenerator::visitObjectKeys(LObjectKeys* lir) {
15899 Register object = ToRegister(lir->object());
15900
15901 pushArg(object);
15902
15903 using Fn = JSObject* (*)(JSContext*, HandleObject);
15904 callVM<Fn, jit::ObjectKeys>(lir);
15905}
15906
15907void CodeGenerator::visitObjectKeysLength(LObjectKeysLength* lir) {
15908 Register object = ToRegister(lir->object());
15909
15910 pushArg(object);
15911
15912 using Fn = bool (*)(JSContext*, HandleObject, int32_t*);
15913 callVM<Fn, jit::ObjectKeysLength>(lir);
15914}
15915
15916void CodeGenerator::visitGetIteratorCache(LGetIteratorCache* lir) {
15917 LiveRegisterSet liveRegs = lir->safepoint()->liveRegs();
15918 TypedOrValueRegister val =
15919 toConstantOrRegister(lir, LGetIteratorCache::ValueIndex,
15920 lir->mir()->value()->type())
15921 .reg();
15922 Register output = ToRegister(lir->output());
15923 Register temp0 = ToRegister(lir->temp0());
15924 Register temp1 = ToRegister(lir->temp1());
15925
15926 IonGetIteratorIC ic(liveRegs, val, output, temp0, temp1);
15927 addIC(lir, allocateIC(ic));
15928}
15929
15930void CodeGenerator::visitOptimizeSpreadCallCache(
15931 LOptimizeSpreadCallCache* lir) {
15932 LiveRegisterSet liveRegs = lir->safepoint()->liveRegs();
15933 ValueOperand val = ToValue(lir, LOptimizeSpreadCallCache::ValueIndex);
15934 ValueOperand output = ToOutValue(lir);
15935 Register temp = ToRegister(lir->temp0());
15936
15937 IonOptimizeSpreadCallIC ic(liveRegs, val, output, temp);
15938 addIC(lir, allocateIC(ic));
15939}
15940
15941void CodeGenerator::visitCloseIterCache(LCloseIterCache* lir) {
15942 LiveRegisterSet liveRegs = lir->safepoint()->liveRegs();
15943 Register iter = ToRegister(lir->iter());
15944 Register temp = ToRegister(lir->temp0());
15945 CompletionKind kind = CompletionKind(lir->mir()->completionKind());
15946
15947 IonCloseIterIC ic(liveRegs, iter, temp, kind);
15948 addIC(lir, allocateIC(ic));
15949}
15950
15951void CodeGenerator::visitOptimizeGetIteratorCache(
15952 LOptimizeGetIteratorCache* lir) {
15953 LiveRegisterSet liveRegs = lir->safepoint()->liveRegs();
15954 ValueOperand val = ToValue(lir, LOptimizeGetIteratorCache::ValueIndex);
15955 Register output = ToRegister(lir->output());
15956 Register temp = ToRegister(lir->temp0());
15957
15958 IonOptimizeGetIteratorIC ic(liveRegs, val, output, temp);
15959 addIC(lir, allocateIC(ic));
15960}
15961
15962void CodeGenerator::visitIteratorMore(LIteratorMore* lir) {
15963 const Register obj = ToRegister(lir->iterator());
15964 const ValueOperand output = ToOutValue(lir);
15965 const Register temp = ToRegister(lir->temp0());
15966
15967 masm.iteratorMore(obj, output, temp);
15968}
15969
15970void CodeGenerator::visitIsNoIterAndBranch(LIsNoIterAndBranch* lir) {
15971 ValueOperand input = ToValue(lir, LIsNoIterAndBranch::Input);
15972 Label* ifTrue = getJumpLabelForBranch(lir->ifTrue());
15973 Label* ifFalse = getJumpLabelForBranch(lir->ifFalse());
15974
15975 masm.branchTestMagic(Assembler::Equal, input, ifTrue);
15976
15977 if (!isNextBlock(lir->ifFalse()->lir())) {
15978 masm.jump(ifFalse);
15979 }
15980}
15981
15982void CodeGenerator::visitIteratorEnd(LIteratorEnd* lir) {
15983 const Register obj = ToRegister(lir->object());
15984 const Register temp0 = ToRegister(lir->temp0());
15985 const Register temp1 = ToRegister(lir->temp1());
15986 const Register temp2 = ToRegister(lir->temp2());
15987
15988 masm.iteratorClose(obj, temp0, temp1, temp2);
15989}
15990
15991void CodeGenerator::visitArgumentsLength(LArgumentsLength* lir) {
15992 // read number of actual arguments from the JS frame.
15993 Register argc = ToRegister(lir->output());
15994 masm.loadNumActualArgs(FramePointer, argc);
15995}
15996
15997void CodeGenerator::visitGetFrameArgument(LGetFrameArgument* lir) {
15998 ValueOperand result = ToOutValue(lir);
15999 const LAllocation* index = lir->index();
16000 size_t argvOffset = JitFrameLayout::offsetOfActualArgs();
16001
16002 // This instruction is used to access actual arguments and formal arguments.
16003 // The number of Values on the stack is |max(numFormals, numActuals)|, so we
16004 // assert |index < numFormals || index < numActuals| in debug builds.
16005 DebugOnly<size_t> numFormals = gen->outerInfo().script()->function()->nargs();
16006
16007 if (index->isConstant()) {
16008 int32_t i = index->toConstant()->toInt32();
16009#ifdef DEBUG1
16010 if (uint32_t(i) >= numFormals) {
16011 Label ok;
16012 Register argc = result.scratchReg();
16013 masm.loadNumActualArgs(FramePointer, argc);
16014 masm.branch32(Assembler::Above, argc, Imm32(i), &ok);
16015 masm.assumeUnreachable("Invalid argument index");
16016 masm.bind(&ok);
16017 }
16018#endif
16019 Address argPtr(FramePointer, sizeof(Value) * i + argvOffset);
16020 masm.loadValue(argPtr, result);
16021 } else {
16022 Register i = ToRegister(index);
16023#ifdef DEBUG1
16024 Label ok;
16025 Register argc = result.scratchReg();
16026 masm.branch32(Assembler::Below, i, Imm32(numFormals), &ok);
16027 masm.loadNumActualArgs(FramePointer, argc);
16028 masm.branch32(Assembler::Above, argc, i, &ok);
16029 masm.assumeUnreachable("Invalid argument index");
16030 masm.bind(&ok);
16031#endif
16032 BaseValueIndex argPtr(FramePointer, i, argvOffset);
16033 masm.loadValue(argPtr, result);
16034 }
16035}
16036
16037void CodeGenerator::visitGetFrameArgumentHole(LGetFrameArgumentHole* lir) {
16038 ValueOperand result = ToOutValue(lir);
16039 Register index = ToRegister(lir->index());
16040 Register length = ToRegister(lir->length());
16041 Register spectreTemp = ToTempRegisterOrInvalid(lir->temp0());
16042 size_t argvOffset = JitFrameLayout::offsetOfActualArgs();
16043
16044 Label outOfBounds, done;
16045 masm.spectreBoundsCheck32(index, length, spectreTemp, &outOfBounds);
16046
16047 BaseValueIndex argPtr(FramePointer, index, argvOffset);
16048 masm.loadValue(argPtr, result);
16049 masm.jump(&done);
16050
16051 masm.bind(&outOfBounds);
16052 bailoutCmp32(Assembler::LessThan, index, Imm32(0), lir->snapshot());
16053 masm.moveValue(UndefinedValue(), result);
16054
16055 masm.bind(&done);
16056}
16057
16058void CodeGenerator::visitRest(LRest* lir) {
16059 Register numActuals = ToRegister(lir->numActuals());
16060 Register temp0 = ToRegister(lir->temp0());
16061 Register temp1 = ToRegister(lir->temp1());
16062 Register temp2 = ToRegister(lir->temp2());
16063 Register temp3 = ToRegister(lir->temp3());
16064 unsigned numFormals = lir->mir()->numFormals();
16065
16066 constexpr uint32_t arrayCapacity = 2;
16067
16068 if (Shape* shape = lir->mir()->shape()) {
16069 uint32_t arrayLength = 0;
16070 gc::AllocKind allocKind = GuessArrayGCKind(arrayCapacity);
16071 MOZ_ASSERT(CanChangeToBackgroundAllocKind(allocKind, &ArrayObject::class_))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(CanChangeToBackgroundAllocKind(allocKind, &ArrayObject
::class_))>::isValid, "invalid assertion condition"); if (
(__builtin_expect(!!(!(!!(CanChangeToBackgroundAllocKind(allocKind
, &ArrayObject::class_)))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("CanChangeToBackgroundAllocKind(allocKind, &ArrayObject::class_)"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 16071); AnnotateMozCrashReason("MOZ_ASSERT" "(" "CanChangeToBackgroundAllocKind(allocKind, &ArrayObject::class_)"
")"); do { *((volatile int*)__null) = 16071; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
16072 allocKind = ForegroundToBackgroundAllocKind(allocKind);
16073 MOZ_ASSERT(GetGCKindSlots(allocKind) ==do { static_assert( mozilla::detail::AssertionConditionType<
decltype(GetGCKindSlots(allocKind) == arrayCapacity + ObjectElements
::VALUES_PER_HEADER)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(GetGCKindSlots(allocKind) ==
arrayCapacity + ObjectElements::VALUES_PER_HEADER))), 0))) {
do { } while (false); MOZ_ReportAssertionFailure("GetGCKindSlots(allocKind) == arrayCapacity + ObjectElements::VALUES_PER_HEADER"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 16074); AnnotateMozCrashReason("MOZ_ASSERT" "(" "GetGCKindSlots(allocKind) == arrayCapacity + ObjectElements::VALUES_PER_HEADER"
")"); do { *((volatile int*)__null) = 16074; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
16074 arrayCapacity + ObjectElements::VALUES_PER_HEADER)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(GetGCKindSlots(allocKind) == arrayCapacity + ObjectElements
::VALUES_PER_HEADER)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(GetGCKindSlots(allocKind) ==
arrayCapacity + ObjectElements::VALUES_PER_HEADER))), 0))) {
do { } while (false); MOZ_ReportAssertionFailure("GetGCKindSlots(allocKind) == arrayCapacity + ObjectElements::VALUES_PER_HEADER"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 16074); AnnotateMozCrashReason("MOZ_ASSERT" "(" "GetGCKindSlots(allocKind) == arrayCapacity + ObjectElements::VALUES_PER_HEADER"
")"); do { *((volatile int*)__null) = 16074; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
16075
16076 Label joinAlloc, failAlloc;
16077 masm.movePtr(ImmGCPtr(shape), temp0);
16078 masm.createArrayWithFixedElements(temp2, temp0, temp1, InvalidReg,
16079 arrayLength, arrayCapacity, 0, 0,
16080 allocKind, gc::Heap::Default, &failAlloc);
16081 masm.jump(&joinAlloc);
16082 {
16083 masm.bind(&failAlloc);
16084 masm.movePtr(ImmPtr(nullptr), temp2);
16085 }
16086 masm.bind(&joinAlloc);
16087 } else {
16088 masm.movePtr(ImmPtr(nullptr), temp2);
16089 }
16090
16091 // Set temp1 to the address of the first actual argument.
16092 size_t actualsOffset = JitFrameLayout::offsetOfActualArgs();
16093 masm.computeEffectiveAddress(Address(FramePointer, actualsOffset), temp1);
16094
16095 // Compute array length: max(numActuals - numFormals, 0).
16096 Register lengthReg;
16097 if (numFormals) {
16098 lengthReg = temp0;
16099 Label emptyLength, joinLength;
16100 masm.branch32(Assembler::LessThanOrEqual, numActuals, Imm32(numFormals),
16101 &emptyLength);
16102 {
16103 masm.move32(numActuals, lengthReg);
16104 masm.sub32(Imm32(numFormals), lengthReg);
16105
16106 // Skip formal arguments.
16107 masm.addPtr(Imm32(sizeof(Value) * numFormals), temp1);
16108
16109 masm.jump(&joinLength);
16110 }
16111 masm.bind(&emptyLength);
16112 {
16113 masm.move32(Imm32(0), lengthReg);
16114
16115 // Leave temp1 pointed to the start of actuals() when the rest-array
16116 // length is zero. We don't use |actuals() + numFormals| because
16117 // |numFormals| can be any non-negative int32 value when this MRest was
16118 // created from scalar replacement optimizations. And it seems
16119 // questionable to compute a Value* pointer which points to who knows
16120 // where.
16121 }
16122 masm.bind(&joinLength);
16123 } else {
16124 // Use numActuals directly when there are no formals.
16125 lengthReg = numActuals;
16126 }
16127
16128 // Try to initialize the array elements.
16129 Label vmCall, done;
16130 if (lir->mir()->shape()) {
16131 // Call into C++ if we failed to allocate an array or there are more than
16132 // |arrayCapacity| elements.
16133 masm.branchTestPtr(Assembler::Zero, temp2, temp2, &vmCall);
16134 masm.branch32(Assembler::Above, lengthReg, Imm32(arrayCapacity), &vmCall);
16135
16136 // The array must be nursery allocated so no post barrier is needed.
16137#ifdef DEBUG1
16138 Label ok;
16139 masm.branchPtrInNurseryChunk(Assembler::Equal, temp2, temp3, &ok);
16140 masm.assumeUnreachable("Unexpected tenured object for LRest");
16141 masm.bind(&ok);
16142#endif
16143
16144 Label initialized;
16145 masm.branch32(Assembler::Equal, lengthReg, Imm32(0), &initialized);
16146
16147 // Store length and initializedLength.
16148 Register elements = temp3;
16149 masm.loadPtr(Address(temp2, NativeObject::offsetOfElements()), elements);
16150 Address lengthAddr(elements, ObjectElements::offsetOfLength());
16151 Address initLengthAddr(elements,
16152 ObjectElements::offsetOfInitializedLength());
16153 masm.store32(lengthReg, lengthAddr);
16154 masm.store32(lengthReg, initLengthAddr);
16155
16156 // Store either one or two elements. This may clobber lengthReg (temp0).
16157 static_assert(arrayCapacity == 2, "code handles 1 or 2 elements");
16158 Label storeFirst;
16159 masm.branch32(Assembler::Equal, lengthReg, Imm32(1), &storeFirst);
16160 masm.storeValue(Address(temp1, sizeof(Value)),
16161 Address(elements, sizeof(Value)), temp0);
16162 masm.bind(&storeFirst);
16163 masm.storeValue(Address(temp1, 0), Address(elements, 0), temp0);
16164
16165 // Done.
16166 masm.bind(&initialized);
16167 masm.movePtr(temp2, ReturnReg);
16168 masm.jump(&done);
16169 }
16170
16171 masm.bind(&vmCall);
16172
16173 pushArg(temp2);
16174 pushArg(temp1);
16175 pushArg(lengthReg);
16176
16177 using Fn =
16178 ArrayObject* (*)(JSContext*, uint32_t, Value*, Handle<ArrayObject*>);
16179 callVM<Fn, InitRestParameter>(lir);
16180
16181 masm.bind(&done);
16182}
16183
16184// Create a stackmap from the given safepoint, with the structure:
16185//
16186// <reg dump, if any>
16187// | ++ <body (general spill)>
16188// | | ++ <space for Frame>
16189// | | ++ <inbound args>
16190// | | |
16191// Lowest Addr Highest Addr
16192// |
16193// framePushedAtStackMapBase
16194//
16195// The caller owns the resulting stackmap. This assumes a grow-down stack.
16196//
16197// For non-debug builds, if the stackmap would contain no pointers, no
16198// stackmap is created, and nullptr is returned. For a debug build, a
16199// stackmap is always created and returned.
16200//
16201// Depending on the type of safepoint, the stackmap may need to account for
16202// spilled registers. WasmSafepointKind::LirCall corresponds to LIR nodes where
16203// isCall() == true, for which the register allocator will spill/restore all
16204// live registers at the LIR level - in this case, the LSafepoint sees only live
16205// values on the stack, never in registers. WasmSafepointKind::CodegenCall, on
16206// the other hand, is for LIR nodes which may manually spill/restore live
16207// registers in codegen, in which case the stackmap must account for this. Traps
16208// also require tracking of live registers, but spilling is handled by the trap
16209// mechanism.
16210static bool CreateStackMapFromLSafepoint(LSafepoint& safepoint,
16211 const RegisterOffsets& trapExitLayout,
16212 size_t trapExitLayoutNumWords,
16213 size_t nInboundStackArgBytes,
16214 wasm::StackMap** result) {
16215 // Ensure this is defined on all return paths.
16216 *result = nullptr;
16217
16218 // The size of the wasm::Frame itself.
16219 const size_t nFrameBytes = sizeof(wasm::Frame);
16220
16221 // This is the number of bytes spilled for live registers, outside of a trap.
16222 // For traps, trapExitLayout and trapExitLayoutNumWords will be used.
16223 const size_t nRegisterDumpBytes =
16224 MacroAssembler::PushRegsInMaskSizeInBytes(safepoint.liveRegs());
16225
16226 // As mentioned above, for WasmSafepointKind::LirCall, register spills and
16227 // restores are handled at the LIR level and there should therefore be no live
16228 // registers to handle here.
16229 MOZ_ASSERT_IF(safepoint.wasmSafepointKind() == WasmSafepointKind::LirCall,do { if (safepoint.wasmSafepointKind() == WasmSafepointKind::
LirCall) { do { static_assert( mozilla::detail::AssertionConditionType
<decltype(nRegisterDumpBytes == 0)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(nRegisterDumpBytes == 0))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("nRegisterDumpBytes == 0"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 16230); AnnotateMozCrashReason("MOZ_ASSERT" "(" "nRegisterDumpBytes == 0"
")"); do { *((volatile int*)__null) = 16230; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false); } } while
(false)
16230 nRegisterDumpBytes == 0)do { if (safepoint.wasmSafepointKind() == WasmSafepointKind::
LirCall) { do { static_assert( mozilla::detail::AssertionConditionType
<decltype(nRegisterDumpBytes == 0)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(nRegisterDumpBytes == 0))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("nRegisterDumpBytes == 0"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 16230); AnnotateMozCrashReason("MOZ_ASSERT" "(" "nRegisterDumpBytes == 0"
")"); do { *((volatile int*)__null) = 16230; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false); } } while
(false)
;
16231 MOZ_ASSERT(nRegisterDumpBytes % sizeof(void*) == 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(nRegisterDumpBytes % sizeof(void*) == 0)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(nRegisterDumpBytes % sizeof(void*) == 0))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("nRegisterDumpBytes % sizeof(void*) == 0"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 16231); AnnotateMozCrashReason("MOZ_ASSERT" "(" "nRegisterDumpBytes % sizeof(void*) == 0"
")"); do { *((volatile int*)__null) = 16231; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
16232
16233 // This is the number of bytes in the general spill area, below the Frame.
16234 const size_t nBodyBytes = safepoint.framePushedAtStackMapBase();
16235
16236 // The stack map owns any alignment padding around inbound stack args.
16237 const size_t nInboundStackArgBytesAligned =
16238 wasm::AlignStackArgAreaSize(nInboundStackArgBytes);
16239
16240 // This is the number of bytes in the general spill area, the Frame, and the
16241 // incoming args, but not including any register dump area.
16242 const size_t nNonRegisterBytes =
16243 nBodyBytes + nFrameBytes + nInboundStackArgBytesAligned;
16244 MOZ_ASSERT(nNonRegisterBytes % sizeof(void*) == 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(nNonRegisterBytes % sizeof(void*) == 0)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(nNonRegisterBytes % sizeof(void*) == 0))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("nNonRegisterBytes % sizeof(void*) == 0"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 16244); AnnotateMozCrashReason("MOZ_ASSERT" "(" "nNonRegisterBytes % sizeof(void*) == 0"
")"); do { *((volatile int*)__null) = 16244; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
16245
16246 // This is the number of bytes in the register dump area, if any, below the
16247 // general spill area.
16248 const size_t nRegisterBytes =
16249 (safepoint.wasmSafepointKind() == WasmSafepointKind::Trap)
16250 ? (trapExitLayoutNumWords * sizeof(void*))
16251 : nRegisterDumpBytes;
16252
16253 // This is the total number of bytes covered by the map.
16254 const size_t nTotalBytes = nNonRegisterBytes + nRegisterBytes;
16255
16256#ifndef DEBUG1
16257 bool needStackMap = !(safepoint.wasmAnyRefRegs().empty() &&
16258 safepoint.wasmAnyRefSlots().empty() &&
16259 safepoint.slotsOrElementsSlots().empty());
16260
16261 // There are no references, and this is a non-debug build, so don't bother
16262 // building the stackmap.
16263 if (!needStackMap) {
16264 return true;
16265 }
16266#endif
16267
16268 wasm::StackMap* stackMap =
16269 wasm::StackMap::create(nTotalBytes / sizeof(void*));
16270 if (!stackMap) {
16271 return false;
16272 }
16273 if (safepoint.wasmSafepointKind() == WasmSafepointKind::Trap) {
16274 stackMap->setExitStubWords(trapExitLayoutNumWords);
16275 }
16276
16277 // REG DUMP AREA, if any.
16278 size_t regDumpWords = 0;
16279 const LiveGeneralRegisterSet wasmAnyRefRegs = safepoint.wasmAnyRefRegs();
16280 const LiveGeneralRegisterSet slotsOrElementsRegs =
16281 safepoint.slotsOrElementsRegs();
16282 const LiveGeneralRegisterSet refRegs(GeneralRegisterSet::Union(
16283 wasmAnyRefRegs.set(), slotsOrElementsRegs.set()));
16284 GeneralRegisterForwardIterator refRegsIter(refRegs);
16285 switch (safepoint.wasmSafepointKind()) {
16286 case WasmSafepointKind::LirCall:
16287 case WasmSafepointKind::StackSwitch:
16288 case WasmSafepointKind::CodegenCall: {
16289 size_t spilledNumWords = nRegisterDumpBytes / sizeof(void*);
16290 regDumpWords += spilledNumWords;
16291
16292 for (; refRegsIter.more(); ++refRegsIter) {
16293 Register reg = *refRegsIter;
16294 size_t offsetFromSpillBase =
16295 safepoint.liveRegs().gprs().offsetOfPushedRegister(reg) /
16296 sizeof(void*);
16297 MOZ_ASSERT(0 < offsetFromSpillBase &&do { static_assert( mozilla::detail::AssertionConditionType<
decltype(0 < offsetFromSpillBase && offsetFromSpillBase
<= spilledNumWords)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(0 < offsetFromSpillBase &&
offsetFromSpillBase <= spilledNumWords))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("0 < offsetFromSpillBase && offsetFromSpillBase <= spilledNumWords"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 16298); AnnotateMozCrashReason("MOZ_ASSERT" "(" "0 < offsetFromSpillBase && offsetFromSpillBase <= spilledNumWords"
")"); do { *((volatile int*)__null) = 16298; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
16298 offsetFromSpillBase <= spilledNumWords)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(0 < offsetFromSpillBase && offsetFromSpillBase
<= spilledNumWords)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(0 < offsetFromSpillBase &&
offsetFromSpillBase <= spilledNumWords))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("0 < offsetFromSpillBase && offsetFromSpillBase <= spilledNumWords"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 16298); AnnotateMozCrashReason("MOZ_ASSERT" "(" "0 < offsetFromSpillBase && offsetFromSpillBase <= spilledNumWords"
")"); do { *((volatile int*)__null) = 16298; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
16299 size_t index = spilledNumWords - offsetFromSpillBase;
16300
16301 if (wasmAnyRefRegs.has(reg)) {
16302 stackMap->set(index, wasm::StackMap::AnyRef);
16303 } else {
16304 MOZ_ASSERT(slotsOrElementsRegs.has(reg))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(slotsOrElementsRegs.has(reg))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(slotsOrElementsRegs.has(reg)
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"slotsOrElementsRegs.has(reg)", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 16304); AnnotateMozCrashReason("MOZ_ASSERT" "(" "slotsOrElementsRegs.has(reg)"
")"); do { *((volatile int*)__null) = 16304; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
16305 stackMap->set(index, wasm::StackMap::ArrayDataPointer);
16306 }
16307 }
16308 // Float and vector registers do not have to be handled; they cannot
16309 // contain wasm anyrefs, and they are spilled after general-purpose
16310 // registers. Gprs are therefore closest to the spill base and thus their
16311 // offset calculation does not need to account for other spills.
16312 } break;
16313 case WasmSafepointKind::Trap: {
16314 regDumpWords += trapExitLayoutNumWords;
16315
16316 for (; refRegsIter.more(); ++refRegsIter) {
16317 Register reg = *refRegsIter;
16318 size_t offsetFromTop = trapExitLayout.getOffset(reg);
16319
16320 // If this doesn't hold, the associated register wasn't saved by
16321 // the trap exit stub. Better to crash now than much later, in
16322 // some obscure place, and possibly with security consequences.
16323 MOZ_RELEASE_ASSERT(offsetFromTop < trapExitLayoutNumWords)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(offsetFromTop < trapExitLayoutNumWords)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(offsetFromTop < trapExitLayoutNumWords))), 0))) { do { }
while (false); MOZ_ReportAssertionFailure("offsetFromTop < trapExitLayoutNumWords"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 16323); AnnotateMozCrashReason("MOZ_RELEASE_ASSERT" "(" "offsetFromTop < trapExitLayoutNumWords"
")"); do { *((volatile int*)__null) = 16323; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
16324
16325 // offsetFromTop is an offset in words down from the highest
16326 // address in the exit stub save area. Switch it around to be an
16327 // offset up from the bottom of the (integer register) save area.
16328 size_t offsetFromBottom = trapExitLayoutNumWords - 1 - offsetFromTop;
16329
16330 if (wasmAnyRefRegs.has(reg)) {
16331 stackMap->set(offsetFromBottom, wasm::StackMap::AnyRef);
16332 } else {
16333 MOZ_ASSERT(slotsOrElementsRegs.has(reg))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(slotsOrElementsRegs.has(reg))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(slotsOrElementsRegs.has(reg)
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"slotsOrElementsRegs.has(reg)", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 16333); AnnotateMozCrashReason("MOZ_ASSERT" "(" "slotsOrElementsRegs.has(reg)"
")"); do { *((volatile int*)__null) = 16333; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
16334 stackMap->set(offsetFromBottom, wasm::StackMap::ArrayDataPointer);
16335 }
16336 }
16337 } break;
16338 default:
16339 MOZ_CRASH("unreachable")do { do { } while (false); MOZ_ReportCrash("" "unreachable", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 16339); AnnotateMozCrashReason("MOZ_CRASH(" "unreachable" ")"
); do { *((volatile int*)__null) = 16339; __attribute__((nomerge
)) ::abort(); } while (false); } while (false)
;
16340 }
16341
16342 // Ensure other reg/slot collections on LSafepoint are empty.
16343 MOZ_ASSERT(safepoint.gcRegs().empty() && safepoint.gcSlots().empty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(safepoint.gcRegs().empty() && safepoint.gcSlots
().empty())>::isValid, "invalid assertion condition"); if (
(__builtin_expect(!!(!(!!(safepoint.gcRegs().empty() &&
safepoint.gcSlots().empty()))), 0))) { do { } while (false);
MOZ_ReportAssertionFailure("safepoint.gcRegs().empty() && safepoint.gcSlots().empty()"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 16343); AnnotateMozCrashReason("MOZ_ASSERT" "(" "safepoint.gcRegs().empty() && safepoint.gcSlots().empty()"
")"); do { *((volatile int*)__null) = 16343; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
16344#ifdef JS_NUNBOX32
16345 MOZ_ASSERT(safepoint.nunboxParts().empty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(safepoint.nunboxParts().empty())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(safepoint.nunboxParts().empty
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("safepoint.nunboxParts().empty()", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 16345); AnnotateMozCrashReason("MOZ_ASSERT" "(" "safepoint.nunboxParts().empty()"
")"); do { *((volatile int*)__null) = 16345; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
16346#elif JS_PUNBOX641
16347 MOZ_ASSERT(safepoint.valueRegs().empty() && safepoint.valueSlots().empty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(safepoint.valueRegs().empty() && safepoint.valueSlots
().empty())>::isValid, "invalid assertion condition"); if (
(__builtin_expect(!!(!(!!(safepoint.valueRegs().empty() &&
safepoint.valueSlots().empty()))), 0))) { do { } while (false
); MOZ_ReportAssertionFailure("safepoint.valueRegs().empty() && safepoint.valueSlots().empty()"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 16347); AnnotateMozCrashReason("MOZ_ASSERT" "(" "safepoint.valueRegs().empty() && safepoint.valueSlots().empty()"
")"); do { *((volatile int*)__null) = 16347; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
16348#endif
16349
16350 // BODY (GENERAL SPILL) AREA and FRAME and INCOMING ARGS
16351 // Deal with roots on the stack.
16352 const LSafepoint::SlotList& wasmAnyRefSlots = safepoint.wasmAnyRefSlots();
16353 for (SafepointSlotEntry wasmAnyRefSlot : wasmAnyRefSlots) {
16354 // The following needs to correspond with JitFrameLayout::slotRef
16355 // wasmAnyRefSlot.stack == 0 means the slot is in the args area
16356 if (wasmAnyRefSlot.stack) {
16357 // It's a slot in the body allocation, so .slot is interpreted
16358 // as an index downwards from the Frame*
16359 MOZ_ASSERT(wasmAnyRefSlot.slot <= nBodyBytes)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(wasmAnyRefSlot.slot <= nBodyBytes)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(wasmAnyRefSlot.slot <= nBodyBytes
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"wasmAnyRefSlot.slot <= nBodyBytes", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 16359); AnnotateMozCrashReason("MOZ_ASSERT" "(" "wasmAnyRefSlot.slot <= nBodyBytes"
")"); do { *((volatile int*)__null) = 16359; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
16360 uint32_t offsetInBytes = nBodyBytes - wasmAnyRefSlot.slot;
16361 MOZ_ASSERT(offsetInBytes % sizeof(void*) == 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(offsetInBytes % sizeof(void*) == 0)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(offsetInBytes % sizeof(void*
) == 0))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("offsetInBytes % sizeof(void*) == 0", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 16361); AnnotateMozCrashReason("MOZ_ASSERT" "(" "offsetInBytes % sizeof(void*) == 0"
")"); do { *((volatile int*)__null) = 16361; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
16362 stackMap->set(regDumpWords + offsetInBytes / sizeof(void*),
16363 wasm::StackMap::AnyRef);
16364 } else {
16365 // It's an argument slot
16366 MOZ_ASSERT(wasmAnyRefSlot.slot < nInboundStackArgBytes)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(wasmAnyRefSlot.slot < nInboundStackArgBytes)>::
isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(wasmAnyRefSlot.slot < nInboundStackArgBytes))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("wasmAnyRefSlot.slot < nInboundStackArgBytes"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 16366); AnnotateMozCrashReason("MOZ_ASSERT" "(" "wasmAnyRefSlot.slot < nInboundStackArgBytes"
")"); do { *((volatile int*)__null) = 16366; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
16367 uint32_t offsetInBytes = nBodyBytes + nFrameBytes + wasmAnyRefSlot.slot;
16368 MOZ_ASSERT(offsetInBytes % sizeof(void*) == 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(offsetInBytes % sizeof(void*) == 0)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(offsetInBytes % sizeof(void*
) == 0))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("offsetInBytes % sizeof(void*) == 0", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 16368); AnnotateMozCrashReason("MOZ_ASSERT" "(" "offsetInBytes % sizeof(void*) == 0"
")"); do { *((volatile int*)__null) = 16368; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
16369 stackMap->set(regDumpWords + offsetInBytes / sizeof(void*),
16370 wasm::StackMap::AnyRef);
16371 }
16372 }
16373
16374 // Track array data pointers on the stack
16375 const LSafepoint::SlotList& slots = safepoint.slotsOrElementsSlots();
16376 for (SafepointSlotEntry slot : slots) {
16377 MOZ_ASSERT(slot.stack)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(slot.stack)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(slot.stack))), 0))) { do { }
while (false); MOZ_ReportAssertionFailure("slot.stack", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 16377); AnnotateMozCrashReason("MOZ_ASSERT" "(" "slot.stack"
")"); do { *((volatile int*)__null) = 16377; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
16378
16379 // It's a slot in the body allocation, so .slot is interpreted
16380 // as an index downwards from the Frame*
16381 MOZ_ASSERT(slot.slot <= nBodyBytes)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(slot.slot <= nBodyBytes)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(slot.slot <= nBodyBytes))
), 0))) { do { } while (false); MOZ_ReportAssertionFailure("slot.slot <= nBodyBytes"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 16381); AnnotateMozCrashReason("MOZ_ASSERT" "(" "slot.slot <= nBodyBytes"
")"); do { *((volatile int*)__null) = 16381; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
16382 uint32_t offsetInBytes = nBodyBytes - slot.slot;
16383 MOZ_ASSERT(offsetInBytes % sizeof(void*) == 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(offsetInBytes % sizeof(void*) == 0)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(offsetInBytes % sizeof(void*
) == 0))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("offsetInBytes % sizeof(void*) == 0", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 16383); AnnotateMozCrashReason("MOZ_ASSERT" "(" "offsetInBytes % sizeof(void*) == 0"
")"); do { *((volatile int*)__null) = 16383; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
16384 stackMap->set(regDumpWords + offsetInBytes / sizeof(void*),
16385 wasm::StackMap::Kind::ArrayDataPointer);
16386 }
16387
16388 // Record in the map, how far down from the highest address the Frame* is.
16389 // Take the opportunity to check that we haven't marked any part of the
16390 // Frame itself as a pointer.
16391 stackMap->setFrameOffsetFromTop((nInboundStackArgBytesAligned + nFrameBytes) /
16392 sizeof(void*));
16393#ifdef DEBUG1
16394 for (uint32_t i = 0; i < nFrameBytes / sizeof(void*); i++) {
16395 MOZ_ASSERT(stackMap->get(stackMap->header.numMappedWords -do { static_assert( mozilla::detail::AssertionConditionType<
decltype(stackMap->get(stackMap->header.numMappedWords -
stackMap->header.frameOffsetFromTop + i) == wasm::StackMap
::Kind::POD)>::isValid, "invalid assertion condition"); if
((__builtin_expect(!!(!(!!(stackMap->get(stackMap->header
.numMappedWords - stackMap->header.frameOffsetFromTop + i)
== wasm::StackMap::Kind::POD))), 0))) { do { } while (false)
; MOZ_ReportAssertionFailure("stackMap->get(stackMap->header.numMappedWords - stackMap->header.frameOffsetFromTop + i) == wasm::StackMap::Kind::POD"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 16397); AnnotateMozCrashReason("MOZ_ASSERT" "(" "stackMap->get(stackMap->header.numMappedWords - stackMap->header.frameOffsetFromTop + i) == wasm::StackMap::Kind::POD"
")"); do { *((volatile int*)__null) = 16397; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
16396 stackMap->header.frameOffsetFromTop + i) ==do { static_assert( mozilla::detail::AssertionConditionType<
decltype(stackMap->get(stackMap->header.numMappedWords -
stackMap->header.frameOffsetFromTop + i) == wasm::StackMap
::Kind::POD)>::isValid, "invalid assertion condition"); if
((__builtin_expect(!!(!(!!(stackMap->get(stackMap->header
.numMappedWords - stackMap->header.frameOffsetFromTop + i)
== wasm::StackMap::Kind::POD))), 0))) { do { } while (false)
; MOZ_ReportAssertionFailure("stackMap->get(stackMap->header.numMappedWords - stackMap->header.frameOffsetFromTop + i) == wasm::StackMap::Kind::POD"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 16397); AnnotateMozCrashReason("MOZ_ASSERT" "(" "stackMap->get(stackMap->header.numMappedWords - stackMap->header.frameOffsetFromTop + i) == wasm::StackMap::Kind::POD"
")"); do { *((volatile int*)__null) = 16397; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
16397 wasm::StackMap::Kind::POD)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(stackMap->get(stackMap->header.numMappedWords -
stackMap->header.frameOffsetFromTop + i) == wasm::StackMap
::Kind::POD)>::isValid, "invalid assertion condition"); if
((__builtin_expect(!!(!(!!(stackMap->get(stackMap->header
.numMappedWords - stackMap->header.frameOffsetFromTop + i)
== wasm::StackMap::Kind::POD))), 0))) { do { } while (false)
; MOZ_ReportAssertionFailure("stackMap->get(stackMap->header.numMappedWords - stackMap->header.frameOffsetFromTop + i) == wasm::StackMap::Kind::POD"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 16397); AnnotateMozCrashReason("MOZ_ASSERT" "(" "stackMap->get(stackMap->header.numMappedWords - stackMap->header.frameOffsetFromTop + i) == wasm::StackMap::Kind::POD"
")"); do { *((volatile int*)__null) = 16397; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
16398 }
16399#endif
16400
16401 *result = stackMap;
16402 return true;
16403}
16404
16405bool CodeGenerator::generateWasm(
16406 wasm::CallIndirectId callIndirectId, wasm::BytecodeOffset trapOffset,
16407 const wasm::ArgTypeVector& argTypes, const RegisterOffsets& trapExitLayout,
16408 size_t trapExitLayoutNumWords, wasm::FuncOffsets* offsets,
16409 wasm::StackMaps* stackMaps, wasm::Decoder* decoder) {
16410 AutoCreatedBy acb(masm, "CodeGenerator::generateWasm");
16411
16412 JitSpew(JitSpew_Codegen, "# Emitting wasm code");
16413
16414 size_t nInboundStackArgBytes = StackArgAreaSizeUnaligned(argTypes);
16415 inboundStackArgBytes_ = nInboundStackArgBytes;
16416
16417 wasm::GenerateFunctionPrologue(masm, callIndirectId, mozilla::Nothing(),
16418 offsets);
16419
16420 MOZ_ASSERT(masm.framePushed() == 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(masm.framePushed() == 0)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(masm.framePushed() == 0))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("masm.framePushed() == 0"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 16420); AnnotateMozCrashReason("MOZ_ASSERT" "(" "masm.framePushed() == 0"
")"); do { *((volatile int*)__null) = 16420; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
16421
16422 // Very large frames are implausible, probably an attack.
16423 if (frameSize() > wasm::MaxFrameSize) {
16424 return decoder->fail(decoder->beginOffset(), "stack frame is too large");
16425 }
16426
16427 if (omitOverRecursedCheck()) {
16428 masm.reserveStack(frameSize());
16429 } else {
16430 std::pair<CodeOffset, uint32_t> pair =
16431 masm.wasmReserveStackChecked(frameSize(), trapOffset);
16432 CodeOffset trapInsnOffset = pair.first;
16433 size_t nBytesReservedBeforeTrap = pair.second;
16434
16435 wasm::StackMap* functionEntryStackMap = nullptr;
16436 if (!CreateStackMapForFunctionEntryTrap(
16437 argTypes, trapExitLayout, trapExitLayoutNumWords,
16438 nBytesReservedBeforeTrap, nInboundStackArgBytes,
16439 &functionEntryStackMap)) {
16440 return false;
16441 }
16442
16443 // In debug builds, we'll always have a stack map, even if there are no
16444 // refs to track.
16445 MOZ_ASSERT(functionEntryStackMap)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(functionEntryStackMap)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(functionEntryStackMap))), 0)
)) { do { } while (false); MOZ_ReportAssertionFailure("functionEntryStackMap"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 16445); AnnotateMozCrashReason("MOZ_ASSERT" "(" "functionEntryStackMap"
")"); do { *((volatile int*)__null) = 16445; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
16446
16447 if (functionEntryStackMap &&
16448 !stackMaps->add((uint8_t*)(uintptr_t)trapInsnOffset.offset(),
16449 functionEntryStackMap)) {
16450 functionEntryStackMap->destroy();
16451 return false;
16452 }
16453 }
16454
16455 MOZ_ASSERT(masm.framePushed() == frameSize())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(masm.framePushed() == frameSize())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(masm.framePushed() == frameSize
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("masm.framePushed() == frameSize()", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 16455); AnnotateMozCrashReason("MOZ_ASSERT" "(" "masm.framePushed() == frameSize()"
")"); do { *((volatile int*)__null) = 16455; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
16456
16457 if (!generateBody()) {
16458 return false;
16459 }
16460
16461 masm.bind(&returnLabel_);
16462 wasm::GenerateFunctionEpilogue(masm, frameSize(), offsets);
16463
16464 if (!generateOutOfLineCode()) {
16465 return false;
16466 }
16467
16468 masm.flush();
16469 if (masm.oom()) {
16470 return false;
16471 }
16472
16473 offsets->end = masm.currentOffset();
16474
16475 MOZ_ASSERT(!masm.failureLabel()->used())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!masm.failureLabel()->used())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!masm.failureLabel()->used
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("!masm.failureLabel()->used()", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 16475); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!masm.failureLabel()->used()"
")"); do { *((volatile int*)__null) = 16475; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
16476 MOZ_ASSERT(snapshots_.listSize() == 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(snapshots_.listSize() == 0)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(snapshots_.listSize() == 0))
), 0))) { do { } while (false); MOZ_ReportAssertionFailure("snapshots_.listSize() == 0"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 16476); AnnotateMozCrashReason("MOZ_ASSERT" "(" "snapshots_.listSize() == 0"
")"); do { *((volatile int*)__null) = 16476; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
16477 MOZ_ASSERT(snapshots_.RVATableSize() == 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(snapshots_.RVATableSize() == 0)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(snapshots_.RVATableSize() ==
0))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("snapshots_.RVATableSize() == 0", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 16477); AnnotateMozCrashReason("MOZ_ASSERT" "(" "snapshots_.RVATableSize() == 0"
")"); do { *((volatile int*)__null) = 16477; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
16478 MOZ_ASSERT(recovers_.size() == 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(recovers_.size() == 0)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(recovers_.size() == 0))), 0)
)) { do { } while (false); MOZ_ReportAssertionFailure("recovers_.size() == 0"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 16478); AnnotateMozCrashReason("MOZ_ASSERT" "(" "recovers_.size() == 0"
")"); do { *((volatile int*)__null) = 16478; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
16479 MOZ_ASSERT(graph.numConstants() == 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(graph.numConstants() == 0)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(graph.numConstants() == 0)))
, 0))) { do { } while (false); MOZ_ReportAssertionFailure("graph.numConstants() == 0"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 16479); AnnotateMozCrashReason("MOZ_ASSERT" "(" "graph.numConstants() == 0"
")"); do { *((volatile int*)__null) = 16479; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
16480 MOZ_ASSERT(osiIndices_.empty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(osiIndices_.empty())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(osiIndices_.empty()))), 0)))
{ do { } while (false); MOZ_ReportAssertionFailure("osiIndices_.empty()"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 16480); AnnotateMozCrashReason("MOZ_ASSERT" "(" "osiIndices_.empty()"
")"); do { *((volatile int*)__null) = 16480; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
16481 MOZ_ASSERT(icList_.empty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(icList_.empty())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(icList_.empty()))), 0))) { do
{ } while (false); MOZ_ReportAssertionFailure("icList_.empty()"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 16481); AnnotateMozCrashReason("MOZ_ASSERT" "(" "icList_.empty()"
")"); do { *((volatile int*)__null) = 16481; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
16482 MOZ_ASSERT(safepoints_.size() == 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(safepoints_.size() == 0)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(safepoints_.size() == 0))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("safepoints_.size() == 0"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 16482); AnnotateMozCrashReason("MOZ_ASSERT" "(" "safepoints_.size() == 0"
")"); do { *((volatile int*)__null) = 16482; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
16483 MOZ_ASSERT(!scriptCounts_)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!scriptCounts_)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!scriptCounts_))), 0))) { do
{ } while (false); MOZ_ReportAssertionFailure("!scriptCounts_"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 16483); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!scriptCounts_"
")"); do { *((volatile int*)__null) = 16483; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
16484
16485 // Convert the safepoints to stackmaps and add them to our running
16486 // collection thereof.
16487 for (CodegenSafepointIndex& index : safepointIndices_) {
16488 wasm::StackMap* stackMap = nullptr;
16489 if (!CreateStackMapFromLSafepoint(*index.safepoint(), trapExitLayout,
16490 trapExitLayoutNumWords,
16491 nInboundStackArgBytes, &stackMap)) {
16492 return false;
16493 }
16494
16495 // In debug builds, we'll always have a stack map.
16496 MOZ_ASSERT(stackMap)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(stackMap)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(stackMap))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("stackMap", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 16496); AnnotateMozCrashReason("MOZ_ASSERT" "(" "stackMap" ")"
); do { *((volatile int*)__null) = 16496; __attribute__((nomerge
)) ::abort(); } while (false); } } while (false)
;
16497 if (!stackMap) {
16498 continue;
16499 }
16500
16501 if (!stackMaps->add((uint8_t*)(uintptr_t)index.displacement(), stackMap)) {
16502 stackMap->destroy();
16503 return false;
16504 }
16505 }
16506
16507 return true;
16508}
16509
16510bool CodeGenerator::generate() {
16511 AutoCreatedBy acb(masm, "CodeGenerator::generate");
16512
16513 JitSpew(JitSpew_Codegen, "# Emitting code for script %s:%u:%u",
16514 gen->outerInfo().script()->filename(),
16515 gen->outerInfo().script()->lineno(),
16516 gen->outerInfo().script()->column().oneOriginValue());
16517
16518 // Initialize native code table with an entry to the start of
16519 // top-level script.
16520 InlineScriptTree* tree = gen->outerInfo().inlineScriptTree();
16521 jsbytecode* startPC = tree->script()->code();
16522 BytecodeSite* startSite = new (gen->alloc()) BytecodeSite(tree, startPC);
16523 if (!addNativeToBytecodeEntry(startSite)) {
16524 return false;
16525 }
16526
16527 if (!safepoints_.init(gen->alloc())) {
16528 return false;
16529 }
16530
16531 perfSpewer_.recordOffset(masm, "Prologue");
16532 if (!generatePrologue()) {
16533 return false;
16534 }
16535
16536 // Reset native => bytecode map table with top-level script and startPc.
16537 if (!addNativeToBytecodeEntry(startSite)) {
16538 return false;
16539 }
16540
16541 if (!generateBody()) {
16542 return false;
16543 }
16544
16545 // Reset native => bytecode map table with top-level script and startPc.
16546 if (!addNativeToBytecodeEntry(startSite)) {
16547 return false;
16548 }
16549
16550 perfSpewer_.recordOffset(masm, "Epilogue");
16551 if (!generateEpilogue()) {
16552 return false;
16553 }
16554
16555 // Reset native => bytecode map table with top-level script and startPc.
16556 if (!addNativeToBytecodeEntry(startSite)) {
16557 return false;
16558 }
16559
16560 perfSpewer_.recordOffset(masm, "InvalidateEpilogue");
16561 generateInvalidateEpilogue();
16562
16563 // native => bytecode entries for OOL code will be added
16564 // by CodeGeneratorShared::generateOutOfLineCode
16565 perfSpewer_.recordOffset(masm, "OOLCode");
16566 if (!generateOutOfLineCode()) {
16567 return false;
16568 }
16569
16570 // Add terminal entry.
16571 if (!addNativeToBytecodeEntry(startSite)) {
16572 return false;
16573 }
16574
16575 // Dump Native to bytecode entries to spew.
16576 dumpNativeToBytecodeEntries();
16577
16578 // We encode safepoints after the OSI-point offsets have been determined.
16579 if (!encodeSafepoints()) {
16580 return false;
16581 }
16582
16583 return !masm.oom();
16584}
16585
16586static bool AddInlinedCompilations(JSContext* cx, HandleScript script,
16587 IonCompilationId compilationId,
16588 const WarpSnapshot* snapshot,
16589 bool* isValid) {
16590 MOZ_ASSERT(!*isValid)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!*isValid)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!*isValid))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("!*isValid", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 16590); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!*isValid"
")"); do { *((volatile int*)__null) = 16590; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
16591 RecompileInfo recompileInfo(script, compilationId);
16592
16593 JitZone* jitZone = cx->zone()->jitZone();
16594
16595 for (const auto* scriptSnapshot : snapshot->scripts()) {
16596 JSScript* inlinedScript = scriptSnapshot->script();
16597 if (inlinedScript == script) {
16598 continue;
16599 }
16600
16601 // TODO(post-Warp): This matches FinishCompilation and is necessary to
16602 // ensure in-progress compilations are canceled when an inlined functon
16603 // becomes a debuggee. See the breakpoint-14.js jit-test.
16604 // When TI is gone, try to clean this up by moving AddInlinedCompilations to
16605 // WarpOracle so that we can handle this as part of addPendingRecompile
16606 // instead of requiring this separate check.
16607 if (inlinedScript->isDebuggee()) {
16608 *isValid = false;
16609 return true;
16610 }
16611
16612 if (!jitZone->addInlinedCompilation(recompileInfo, inlinedScript)) {
16613 return false;
16614 }
16615 }
16616
16617 *isValid = true;
16618 return true;
16619}
16620
16621struct EmulatesUndefinedDependency final : public CompilationDependency {
16622 CompileRuntime* runtime;
16623 explicit EmulatesUndefinedDependency(CompileRuntime* runtime)
16624 : CompilationDependency(CompilationDependency::Type::EmulatesUndefined),
16625 runtime(runtime) {};
16626
16627 virtual bool operator==(CompilationDependency& dep) {
16628 // Since the emulates undefined fuse is runtime wide, they are all equal
16629 return dep.type == type;
16630 }
16631
16632 virtual bool checkDependency() {
16633 return runtime->hasSeenObjectEmulateUndefinedFuseIntact();
16634 }
16635
16636 virtual bool registerDependency(JSContext* cx, HandleScript script) {
16637 return cx->runtime()
16638 ->hasSeenObjectEmulateUndefinedFuse.ref()
16639 .addFuseDependency(cx, script);
16640 }
16641
16642 virtual UniquePtr<CompilationDependency> clone() {
16643 return MakeUnique<EmulatesUndefinedDependency>(runtime);
16644 }
16645};
16646
16647bool CodeGenerator::addHasSeenObjectEmulateUndefinedFuseDependency() {
16648 EmulatesUndefinedDependency dep(gen->runtime);
16649 return mirGen().tracker.addDependency(dep);
16650}
16651
16652bool CodeGenerator::link(JSContext* cx, const WarpSnapshot* snapshot) {
16653 AutoCreatedBy acb(masm, "CodeGenerator::link");
16654
16655 // We cancel off-thread Ion compilations in a few places during GC, but if
16656 // this compilation was performed off-thread it will already have been
16657 // removed from the relevant lists by this point. Don't allow GC here.
16658 JS::AutoAssertNoGC nogc(cx);
16659
16660 RootedScript script(cx, gen->outerInfo().script());
16661 MOZ_ASSERT(!script->hasIonScript())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!script->hasIonScript())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!script->hasIonScript()))
), 0))) { do { } while (false); MOZ_ReportAssertionFailure("!script->hasIonScript()"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 16661); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!script->hasIonScript()"
")"); do { *((volatile int*)__null) = 16661; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
16662
16663 // Perform any read barriers which were skipped while compiling the
16664 // script, which may have happened off-thread.
16665 JitZone* jitZone = cx->zone()->jitZone();
16666 jitZone->performStubReadBarriers(zoneStubsToReadBarrier_);
16667
16668 if (scriptCounts_ && !script->hasScriptCounts() &&
16669 !script->initScriptCounts(cx)) {
16670 return false;
16671 }
16672
16673 IonCompilationId compilationId =
16674 cx->runtime()->jitRuntime()->nextCompilationId();
16675 jitZone->currentCompilationIdRef().emplace(compilationId);
16676 auto resetCurrentId = mozilla::MakeScopeExit(
16677 [jitZone] { jitZone->currentCompilationIdRef().reset(); });
16678
16679 // Record constraints. If an error occured, returns false and potentially
16680 // prevent future compilations. Otherwise, if an invalidation occured, then
16681 // skip the current compilation.
16682 bool isValid = false;
16683
16684 // If an inlined script is invalidated (for example, by attaching
16685 // a debugger), we must also invalidate the parent IonScript.
16686 if (!AddInlinedCompilations(cx, script, compilationId, snapshot, &isValid)) {
16687 return false;
16688 }
16689
16690 // This compilation is no longer valid; don't proceed, but return true as this
16691 // isn't an error case either.
16692 if (!isValid) {
16693 return true;
16694 }
16695
16696 CompilationDependencyTracker& tracker = mirGen().tracker;
16697 if (!tracker.checkDependencies()) {
16698 return true;
16699 }
16700
16701 for (auto& dep : tracker.dependencies) {
16702 if (!dep->registerDependency(cx, script)) {
16703 return false; // Should we make sure we only return false on OOM and then
16704 // eat the OOM here?
16705 }
16706 }
16707
16708 uint32_t argumentSlots = (gen->outerInfo().nargs() + 1) * sizeof(Value);
16709
16710 size_t numNurseryObjects = snapshot->nurseryObjects().length();
16711
16712 IonScript* ionScript = IonScript::New(
16713 cx, compilationId, graph.localSlotsSize(), argumentSlots, frameDepth_,
16714 snapshots_.listSize(), snapshots_.RVATableSize(), recovers_.size(),
16715 graph.numConstants(), numNurseryObjects, safepointIndices_.length(),
16716 osiIndices_.length(), icList_.length(), runtimeData_.length(),
16717 safepoints_.size());
16718 if (!ionScript) {
16719 return false;
16720 }
16721#ifdef DEBUG1
16722 ionScript->setICHash(snapshot->icHash());
16723#endif
16724
16725 auto freeIonScript = mozilla::MakeScopeExit([&ionScript] {
16726 // Use js_free instead of IonScript::Destroy: the cache list is still
16727 // uninitialized.
16728 js_free(ionScript);
16729 });
16730
16731 Linker linker(masm);
16732 JitCode* code = linker.newCode(cx, CodeKind::Ion);
16733 if (!code) {
16734 return false;
16735 }
16736
16737 // Encode native to bytecode map if profiling is enabled.
16738 if (isProfilerInstrumentationEnabled()) {
16739 // Generate native-to-bytecode main table.
16740 IonEntry::ScriptList scriptList;
16741 if (!generateCompactNativeToBytecodeMap(cx, code, scriptList)) {
16742 return false;
16743 }
16744
16745 uint8_t* ionTableAddr =
16746 ((uint8_t*)nativeToBytecodeMap_.get()) + nativeToBytecodeTableOffset_;
16747 JitcodeIonTable* ionTable = (JitcodeIonTable*)ionTableAddr;
16748
16749 // Construct the IonEntry that will go into the global table.
16750 auto entry = MakeJitcodeGlobalEntry<IonEntry>(
16751 cx, code, code->raw(), code->rawEnd(), std::move(scriptList), ionTable);
16752 if (!entry) {
16753 return false;
16754 }
16755 (void)nativeToBytecodeMap_.release(); // Table is now owned by |entry|.
16756
16757 // Add entry to the global table.
16758 JitcodeGlobalTable* globalTable =
16759 cx->runtime()->jitRuntime()->getJitcodeGlobalTable();
16760 if (!globalTable->addEntry(std::move(entry))) {
16761 return false;
16762 }
16763
16764 // Mark the jitcode as having a bytecode map.
16765 code->setHasBytecodeMap();
16766 } else {
16767 // Add a dumy jitcodeGlobalTable entry.
16768 auto entry = MakeJitcodeGlobalEntry<DummyEntry>(cx, code, code->raw(),
16769 code->rawEnd());
16770 if (!entry) {
16771 return false;
16772 }
16773
16774 // Add entry to the global table.
16775 JitcodeGlobalTable* globalTable =
16776 cx->runtime()->jitRuntime()->getJitcodeGlobalTable();
16777 if (!globalTable->addEntry(std::move(entry))) {
16778 return false;
16779 }
16780
16781 // Mark the jitcode as having a bytecode map.
16782 code->setHasBytecodeMap();
16783 }
16784
16785 ionScript->setMethod(code);
16786
16787 // If the Gecko Profiler is enabled, mark IonScript as having been
16788 // instrumented accordingly.
16789 if (isProfilerInstrumentationEnabled()) {
16790 ionScript->setHasProfilingInstrumentation();
16791 }
16792
16793 Assembler::PatchDataWithValueCheck(
16794 CodeLocationLabel(code, invalidateEpilogueData_), ImmPtr(ionScript),
16795 ImmPtr((void*)-1));
16796
16797 for (CodeOffset offset : ionScriptLabels_) {
16798 Assembler::PatchDataWithValueCheck(CodeLocationLabel(code, offset),
16799 ImmPtr(ionScript), ImmPtr((void*)-1));
16800 }
16801
16802 for (NurseryObjectLabel label : ionNurseryObjectLabels_) {
16803 void* entry = ionScript->addressOfNurseryObject(label.nurseryIndex);
16804 Assembler::PatchDataWithValueCheck(CodeLocationLabel(code, label.offset),
16805 ImmPtr(entry), ImmPtr((void*)-1));
16806 }
16807
16808 // for generating inline caches during the execution.
16809 if (runtimeData_.length()) {
16810 ionScript->copyRuntimeData(&runtimeData_[0]);
16811 }
16812 if (icList_.length()) {
16813 ionScript->copyICEntries(&icList_[0]);
16814 }
16815
16816 for (size_t i = 0; i < icInfo_.length(); i++) {
16817 IonIC& ic = ionScript->getICFromIndex(i);
16818 Assembler::PatchDataWithValueCheck(
16819 CodeLocationLabel(code, icInfo_[i].icOffsetForJump),
16820 ImmPtr(ic.codeRawPtr()), ImmPtr((void*)-1));
16821 Assembler::PatchDataWithValueCheck(
16822 CodeLocationLabel(code, icInfo_[i].icOffsetForPush), ImmPtr(&ic),
16823 ImmPtr((void*)-1));
16824 }
16825
16826 JitSpew(JitSpew_Codegen, "Created IonScript %p (raw %p)", (void*)ionScript,
16827 (void*)code->raw());
16828
16829 ionScript->setInvalidationEpilogueDataOffset(
16830 invalidateEpilogueData_.offset());
16831 if (jsbytecode* osrPc = gen->outerInfo().osrPc()) {
16832 ionScript->setOsrPc(osrPc);
16833 ionScript->setOsrEntryOffset(getOsrEntryOffset());
16834 }
16835 ionScript->setInvalidationEpilogueOffset(invalidate_.offset());
16836
16837 perfSpewer_.saveProfile(cx, script, code);
16838
16839#ifdef MOZ_VTUNE1
16840 vtune::MarkScript(code, script, "ion");
16841#endif
16842
16843 // Set a Ion counter hint for this script.
16844 if (cx->runtime()->jitRuntime()->hasJitHintsMap()) {
16845 JitHintsMap* jitHints = cx->runtime()->jitRuntime()->getJitHintsMap();
16846 jitHints->recordIonCompilation(script);
16847 }
16848
16849 // for marking during GC.
16850 if (safepointIndices_.length()) {
16851 ionScript->copySafepointIndices(&safepointIndices_[0]);
16852 }
16853 if (safepoints_.size()) {
16854 ionScript->copySafepoints(&safepoints_);
16855 }
16856
16857 // for recovering from an Ion Frame.
16858 if (osiIndices_.length()) {
16859 ionScript->copyOsiIndices(&osiIndices_[0]);
16860 }
16861 if (snapshots_.listSize()) {
16862 ionScript->copySnapshots(&snapshots_);
16863 }
16864 MOZ_ASSERT_IF(snapshots_.listSize(), recovers_.size())do { if (snapshots_.listSize()) { do { static_assert( mozilla
::detail::AssertionConditionType<decltype(recovers_.size()
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(recovers_.size()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("recovers_.size()", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 16864); AnnotateMozCrashReason("MOZ_ASSERT" "(" "recovers_.size()"
")"); do { *((volatile int*)__null) = 16864; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false); } } while
(false)
;
16865 if (recovers_.size()) {
16866 ionScript->copyRecovers(&recovers_);
16867 }
16868 if (graph.numConstants()) {
16869 const Value* vp = graph.constantPool();
16870 ionScript->copyConstants(vp);
16871 for (size_t i = 0; i < graph.numConstants(); i++) {
16872 const Value& v = vp[i];
16873 if (v.isGCThing()) {
16874 if (gc::StoreBuffer* sb = v.toGCThing()->storeBuffer()) {
16875 sb->putWholeCell(script);
16876 break;
16877 }
16878 }
16879 }
16880 }
16881
16882 // Attach any generated script counts to the script.
16883 if (IonScriptCounts* counts = extractScriptCounts()) {
16884 script->addIonCounts(counts);
16885 }
16886 // WARNING: Code after this point must be infallible!
16887
16888 // Copy the list of nursery objects. Note that the store buffer can add
16889 // HeapPtr edges that must be cleared in IonScript::Destroy. See the
16890 // infallibility warning above.
16891 const auto& nurseryObjects = snapshot->nurseryObjects();
16892 for (size_t i = 0; i < nurseryObjects.length(); i++) {
16893 ionScript->nurseryObjects()[i].init(nurseryObjects[i]);
16894 }
16895
16896 // Transfer ownership of the IonScript to the JitScript. At this point enough
16897 // of the IonScript must be initialized for IonScript::Destroy to work.
16898 freeIonScript.release();
16899 script->jitScript()->setIonScript(script, ionScript);
16900
16901 return true;
16902}
16903
16904// An out-of-line path to convert a boxed int32 to either a float or double.
16905class OutOfLineUnboxFloatingPoint : public OutOfLineCodeBase<CodeGenerator> {
16906 LUnboxFloatingPoint* unboxFloatingPoint_;
16907
16908 public:
16909 explicit OutOfLineUnboxFloatingPoint(LUnboxFloatingPoint* unboxFloatingPoint)
16910 : unboxFloatingPoint_(unboxFloatingPoint) {}
16911
16912 void accept(CodeGenerator* codegen) override {
16913 codegen->visitOutOfLineUnboxFloatingPoint(this);
16914 }
16915
16916 LUnboxFloatingPoint* unboxFloatingPoint() const {
16917 return unboxFloatingPoint_;
16918 }
16919};
16920
16921void CodeGenerator::visitUnboxFloatingPoint(LUnboxFloatingPoint* lir) {
16922 const ValueOperand box = ToValue(lir, LUnboxFloatingPoint::Input);
16923 const LDefinition* result = lir->output();
16924
16925 // Out-of-line path to convert int32 to double or bailout
16926 // if this instruction is fallible.
16927 OutOfLineUnboxFloatingPoint* ool =
16928 new (alloc()) OutOfLineUnboxFloatingPoint(lir);
16929 addOutOfLineCode(ool, lir->mir());
16930
16931 FloatRegister resultReg = ToFloatRegister(result);
16932 masm.branchTestDouble(Assembler::NotEqual, box, ool->entry());
16933 masm.unboxDouble(box, resultReg);
16934 if (lir->type() == MIRType::Float32) {
16935 masm.convertDoubleToFloat32(resultReg, resultReg);
16936 }
16937 masm.bind(ool->rejoin());
16938}
16939
16940void CodeGenerator::visitOutOfLineUnboxFloatingPoint(
16941 OutOfLineUnboxFloatingPoint* ool) {
16942 LUnboxFloatingPoint* ins = ool->unboxFloatingPoint();
16943 const ValueOperand value = ToValue(ins, LUnboxFloatingPoint::Input);
16944
16945 if (ins->mir()->fallible()) {
16946 Label bail;
16947 masm.branchTestInt32(Assembler::NotEqual, value, &bail);
16948 bailoutFrom(&bail, ins->snapshot());
16949 }
16950 if (ins->type() == MIRType::Float32) {
16951 masm.convertInt32ToFloat32(value.payloadOrValueReg(),
16952 ToFloatRegister(ins->output()));
16953 } else {
16954 masm.convertInt32ToDouble(value.payloadOrValueReg(),
16955 ToFloatRegister(ins->output()));
16956 }
16957 masm.jump(ool->rejoin());
16958}
16959
16960void CodeGenerator::visitCallBindVar(LCallBindVar* lir) {
16961 pushArg(ToRegister(lir->environmentChain()));
16962
16963 using Fn = JSObject* (*)(JSContext*, JSObject*);
16964 callVM<Fn, BindVarOperation>(lir);
16965}
16966
16967void CodeGenerator::visitMegamorphicSetElement(LMegamorphicSetElement* lir) {
16968 Register obj = ToRegister(lir->getOperand(0));
16969 ValueOperand idVal = ToValue(lir, LMegamorphicSetElement::IndexIndex);
16970 ValueOperand value = ToValue(lir, LMegamorphicSetElement::ValueIndex);
16971
16972 Register temp0 = ToRegister(lir->temp0());
16973 // See comment in LIROps.yaml (x86 is short on registers)
16974#ifndef JS_CODEGEN_X86
16975 Register temp1 = ToRegister(lir->temp1());
16976 Register temp2 = ToRegister(lir->temp2());
16977#endif
16978
16979 // The instruction is marked as call-instruction so only these registers are
16980 // live.
16981 LiveRegisterSet liveRegs;
16982 liveRegs.addUnchecked(obj);
16983 liveRegs.addUnchecked(idVal);
16984 liveRegs.addUnchecked(value);
16985 liveRegs.addUnchecked(temp0);
16986#ifndef JS_CODEGEN_X86
16987 liveRegs.addUnchecked(temp1);
16988 liveRegs.addUnchecked(temp2);
16989#endif
16990
16991 Label cacheHit, done;
16992#ifdef JS_CODEGEN_X86
16993 masm.emitMegamorphicCachedSetSlot(
16994 idVal, obj, temp0, value, liveRegs, &cacheHit,
16995 [](MacroAssembler& masm, const Address& addr, MIRType mirType) {
16996 EmitPreBarrier(masm, addr, mirType);
16997 });
16998#else
16999 masm.emitMegamorphicCachedSetSlot(
17000 idVal, obj, temp0, temp1, temp2, value, liveRegs, &cacheHit,
17001 [](MacroAssembler& masm, const Address& addr, MIRType mirType) {
17002 EmitPreBarrier(masm, addr, mirType);
17003 });
17004#endif
17005
17006 pushArg(Imm32(lir->mir()->strict()));
17007 pushArg(ToValue(lir, LMegamorphicSetElement::ValueIndex));
17008 pushArg(ToValue(lir, LMegamorphicSetElement::IndexIndex));
17009 pushArg(obj);
17010
17011 using Fn = bool (*)(JSContext*, HandleObject, HandleValue, HandleValue, bool);
17012 callVM<Fn, js::jit::SetElementMegamorphic<true>>(lir);
17013
17014 masm.jump(&done);
17015 masm.bind(&cacheHit);
17016
17017 masm.branchPtrInNurseryChunk(Assembler::Equal, obj, temp0, &done);
17018 masm.branchValueIsNurseryCell(Assembler::NotEqual, value, temp0, &done);
17019
17020 // Note: because this is a call-instruction, no registers need to be saved.
17021 MOZ_ASSERT(lir->isCall())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(lir->isCall())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(lir->isCall()))), 0))) { do
{ } while (false); MOZ_ReportAssertionFailure("lir->isCall()"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 17021); AnnotateMozCrashReason("MOZ_ASSERT" "(" "lir->isCall()"
")"); do { *((volatile int*)__null) = 17021; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
17022 emitPostWriteBarrier(obj);
17023
17024 masm.bind(&done);
17025}
17026
17027void CodeGenerator::visitLoadScriptedProxyHandler(
17028 LLoadScriptedProxyHandler* ins) {
17029 Register obj = ToRegister(ins->getOperand(0));
17030 Register output = ToRegister(ins->output());
17031
17032 masm.loadPtr(Address(obj, ProxyObject::offsetOfReservedSlots()), output);
17033
17034 Label bail;
17035 Address handlerAddr(output, js::detail::ProxyReservedSlots::offsetOfSlot(
17036 ScriptedProxyHandler::HANDLER_EXTRA));
17037 masm.fallibleUnboxObject(handlerAddr, output, &bail);
17038 bailoutFrom(&bail, ins->snapshot());
17039}
17040
17041#ifdef JS_PUNBOX641
17042void CodeGenerator::visitCheckScriptedProxyGetResult(
17043 LCheckScriptedProxyGetResult* ins) {
17044 ValueOperand target = ToValue(ins, LCheckScriptedProxyGetResult::TargetIndex);
17045 ValueOperand value = ToValue(ins, LCheckScriptedProxyGetResult::ValueIndex);
17046 ValueOperand id = ToValue(ins, LCheckScriptedProxyGetResult::IdIndex);
17047 Register scratch = ToRegister(ins->temp0());
17048 Register scratch2 = ToRegister(ins->temp1());
17049
17050 using Fn = bool (*)(JSContext*, HandleObject, HandleValue, HandleValue,
17051 MutableHandleValue);
17052 OutOfLineCode* ool = oolCallVM<Fn, CheckProxyGetByValueResult>(
17053 ins, ArgList(scratch, id, value), StoreValueTo(value));
17054
17055 masm.unboxObject(target, scratch);
17056 masm.branchTestObjectNeedsProxyResultValidation(Assembler::NonZero, scratch,
17057 scratch2, ool->entry());
17058 masm.bind(ool->rejoin());
17059}
17060#endif
17061
17062void CodeGenerator::visitIdToStringOrSymbol(LIdToStringOrSymbol* ins) {
17063 ValueOperand id = ToValue(ins, LIdToStringOrSymbol::IdIndex);
17064 ValueOperand output = ToOutValue(ins);
17065 Register scratch = ToRegister(ins->temp0());
17066
17067 masm.moveValue(id, output);
17068
17069 Label done, callVM;
17070 Label bail;
17071 {
17072 ScratchTagScope tag(masm, output);
17073 masm.splitTagForTest(output, tag);
17074 masm.branchTestString(Assembler::Equal, tag, &done);
17075 masm.branchTestSymbol(Assembler::Equal, tag, &done);
17076 masm.branchTestInt32(Assembler::NotEqual, tag, &bail);
17077 }
17078
17079 masm.unboxInt32(output, scratch);
17080
17081 using Fn = JSLinearString* (*)(JSContext*, int);
17082 OutOfLineCode* ool = oolCallVM<Fn, Int32ToString<CanGC>>(
17083 ins, ArgList(scratch), StoreRegisterTo(output.scratchReg()));
17084
17085 masm.lookupStaticIntString(scratch, output.scratchReg(),
17086 gen->runtime->staticStrings(), ool->entry());
17087
17088 masm.bind(ool->rejoin());
17089 masm.tagValue(JSVAL_TYPE_STRING, output.scratchReg(), output);
17090 masm.bind(&done);
17091
17092 bailoutFrom(&bail, ins->snapshot());
17093}
17094
17095void CodeGenerator::visitLoadFixedSlotV(LLoadFixedSlotV* ins) {
17096 const Register obj = ToRegister(ins->getOperand(0));
17097 size_t slot = ins->mir()->slot();
17098 ValueOperand result = ToOutValue(ins);
17099
17100 masm.loadValue(Address(obj, NativeObject::getFixedSlotOffset(slot)), result);
17101}
17102
17103void CodeGenerator::visitLoadFixedSlotT(LLoadFixedSlotT* ins) {
17104 const Register obj = ToRegister(ins->getOperand(0));
17105 size_t slot = ins->mir()->slot();
17106 AnyRegister result = ToAnyRegister(ins->getDef(0));
17107 MIRType type = ins->mir()->type();
17108
17109 masm.loadUnboxedValue(Address(obj, NativeObject::getFixedSlotOffset(slot)),
17110 type, result);
17111}
17112
17113template <typename T>
17114static void EmitLoadAndUnbox(MacroAssembler& masm, const T& src, MIRType type,
17115 bool fallible, AnyRegister dest, Label* fail) {
17116 if (type == MIRType::Double) {
17117 MOZ_ASSERT(dest.isFloat())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(dest.isFloat())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(dest.isFloat()))), 0))) { do
{ } while (false); MOZ_ReportAssertionFailure("dest.isFloat()"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 17117); AnnotateMozCrashReason("MOZ_ASSERT" "(" "dest.isFloat()"
")"); do { *((volatile int*)__null) = 17117; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
17118 masm.ensureDouble(src, dest.fpu(), fail);
17119 return;
17120 }
17121 if (fallible) {
17122 switch (type) {
17123 case MIRType::Int32:
17124 masm.fallibleUnboxInt32(src, dest.gpr(), fail);
17125 break;
17126 case MIRType::Boolean:
17127 masm.fallibleUnboxBoolean(src, dest.gpr(), fail);
17128 break;
17129 case MIRType::Object:
17130 masm.fallibleUnboxObject(src, dest.gpr(), fail);
17131 break;
17132 case MIRType::String:
17133 masm.fallibleUnboxString(src, dest.gpr(), fail);
17134 break;
17135 case MIRType::Symbol:
17136 masm.fallibleUnboxSymbol(src, dest.gpr(), fail);
17137 break;
17138 case MIRType::BigInt:
17139 masm.fallibleUnboxBigInt(src, dest.gpr(), fail);
17140 break;
17141 default:
17142 MOZ_CRASH("Unexpected MIRType")do { do { } while (false); MOZ_ReportCrash("" "Unexpected MIRType"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 17142); AnnotateMozCrashReason("MOZ_CRASH(" "Unexpected MIRType"
")"); do { *((volatile int*)__null) = 17142; __attribute__((
nomerge)) ::abort(); } while (false); } while (false)
;
17143 }
17144 return;
17145 }
17146 masm.loadUnboxedValue(src, type, dest);
17147}
17148
17149void CodeGenerator::visitLoadFixedSlotAndUnbox(LLoadFixedSlotAndUnbox* ins) {
17150 const MLoadFixedSlotAndUnbox* mir = ins->mir();
17151 MIRType type = mir->type();
17152 Register input = ToRegister(ins->object());
17153 AnyRegister result = ToAnyRegister(ins->output());
17154 size_t slot = mir->slot();
17155
17156 Address address(input, NativeObject::getFixedSlotOffset(slot));
17157
17158 Label bail;
17159 EmitLoadAndUnbox(masm, address, type, mir->fallible(), result, &bail);
17160 if (mir->fallible()) {
17161 bailoutFrom(&bail, ins->snapshot());
17162 }
17163}
17164
17165void CodeGenerator::visitLoadDynamicSlotAndUnbox(
17166 LLoadDynamicSlotAndUnbox* ins) {
17167 const MLoadDynamicSlotAndUnbox* mir = ins->mir();
17168 MIRType type = mir->type();
17169 Register input = ToRegister(ins->slots());
17170 AnyRegister result = ToAnyRegister(ins->output());
17171 size_t slot = mir->slot();
17172
17173 Address address(input, slot * sizeof(JS::Value));
17174
17175 Label bail;
17176 EmitLoadAndUnbox(masm, address, type, mir->fallible(), result, &bail);
17177 if (mir->fallible()) {
17178 bailoutFrom(&bail, ins->snapshot());
17179 }
17180}
17181
17182void CodeGenerator::visitLoadElementAndUnbox(LLoadElementAndUnbox* ins) {
17183 const MLoadElementAndUnbox* mir = ins->mir();
17184 MIRType type = mir->type();
17185 Register elements = ToRegister(ins->elements());
17186 AnyRegister result = ToAnyRegister(ins->output());
17187
17188 Label bail;
17189 if (ins->index()->isConstant()) {
17190 NativeObject::elementsSizeMustNotOverflow();
17191 int32_t offset = ToInt32(ins->index()) * sizeof(Value);
17192 Address address(elements, offset);
17193 EmitLoadAndUnbox(masm, address, type, mir->fallible(), result, &bail);
17194 } else {
17195 BaseObjectElementIndex address(elements, ToRegister(ins->index()));
17196 EmitLoadAndUnbox(masm, address, type, mir->fallible(), result, &bail);
17197 }
17198
17199 if (mir->fallible()) {
17200 bailoutFrom(&bail, ins->snapshot());
17201 }
17202}
17203
17204class OutOfLineAtomizeSlot : public OutOfLineCodeBase<CodeGenerator> {
17205 LInstruction* lir_;
17206 Register stringReg_;
17207 Address slotAddr_;
17208 TypedOrValueRegister dest_;
17209
17210 public:
17211 OutOfLineAtomizeSlot(LInstruction* lir, Register stringReg, Address slotAddr,
17212 TypedOrValueRegister dest)
17213 : lir_(lir), stringReg_(stringReg), slotAddr_(slotAddr), dest_(dest) {}
17214
17215 void accept(CodeGenerator* codegen) final {
17216 codegen->visitOutOfLineAtomizeSlot(this);
17217 }
17218 LInstruction* lir() const { return lir_; }
17219 Register stringReg() const { return stringReg_; }
17220 Address slotAddr() const { return slotAddr_; }
17221 TypedOrValueRegister dest() const { return dest_; }
17222};
17223
17224void CodeGenerator::visitOutOfLineAtomizeSlot(OutOfLineAtomizeSlot* ool) {
17225 LInstruction* lir = ool->lir();
17226 Register stringReg = ool->stringReg();
17227 Address slotAddr = ool->slotAddr();
17228 TypedOrValueRegister dest = ool->dest();
17229
17230 // This code is called with a non-atomic string in |stringReg|.
17231 // When it returns, |stringReg| contains an unboxed pointer to an
17232 // atomized version of that string, and |slotAddr| contains a
17233 // StringValue pointing to that atom. If |dest| is a ValueOperand,
17234 // it contains the same StringValue; otherwise we assert that |dest|
17235 // is |stringReg|.
17236
17237 saveLive(lir);
17238 pushArg(stringReg);
17239
17240 using Fn = JSAtom* (*)(JSContext*, JSString*);
17241 callVM<Fn, js::AtomizeString>(lir);
17242 StoreRegisterTo(stringReg).generate(this);
17243 restoreLiveIgnore(lir, StoreRegisterTo(stringReg).clobbered());
17244
17245 if (dest.hasValue()) {
17246 masm.moveValue(
17247 TypedOrValueRegister(MIRType::String, AnyRegister(stringReg)),
17248 dest.valueReg());
17249 } else {
17250 MOZ_ASSERT(dest.typedReg().gpr() == stringReg)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(dest.typedReg().gpr() == stringReg)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(dest.typedReg().gpr() == stringReg
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"dest.typedReg().gpr() == stringReg", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 17250); AnnotateMozCrashReason("MOZ_ASSERT" "(" "dest.typedReg().gpr() == stringReg"
")"); do { *((volatile int*)__null) = 17250; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
17251 }
17252
17253 emitPreBarrier(slotAddr);
17254 masm.storeTypedOrValue(dest, slotAddr);
17255
17256 // We don't need a post-barrier because atoms aren't nursery-allocated.
17257#ifdef DEBUG1
17258 // We need a temp register for the nursery check. Spill something.
17259 AllocatableGeneralRegisterSet allRegs(GeneralRegisterSet::All());
17260 allRegs.take(stringReg);
17261 Register temp = allRegs.takeAny();
17262 masm.push(temp);
17263
17264 Label tenured;
17265 masm.branchPtrInNurseryChunk(Assembler::NotEqual, stringReg, temp, &tenured);
17266 masm.assumeUnreachable("AtomizeString returned a nursery pointer");
17267 masm.bind(&tenured);
17268
17269 masm.pop(temp);
17270#endif
17271
17272 masm.jump(ool->rejoin());
17273}
17274
17275void CodeGenerator::emitMaybeAtomizeSlot(LInstruction* ins, Register stringReg,
17276 Address slotAddr,
17277 TypedOrValueRegister dest) {
17278 OutOfLineAtomizeSlot* ool =
17279 new (alloc()) OutOfLineAtomizeSlot(ins, stringReg, slotAddr, dest);
17280 addOutOfLineCode(ool, ins->mirRaw()->toInstruction());
17281 masm.branchTest32(Assembler::NonZero,
17282 Address(stringReg, JSString::offsetOfFlags()),
17283 Imm32(JSString::ATOM_BIT), ool->rejoin());
17284
17285 masm.branchTest32(Assembler::Zero,
17286 Address(stringReg, JSString::offsetOfFlags()),
17287 Imm32(JSString::ATOM_REF_BIT), ool->entry());
17288 masm.loadPtr(Address(stringReg, JSAtomRefString::offsetOfAtom()), stringReg);
17289
17290 if (dest.hasValue()) {
17291 masm.moveValue(
17292 TypedOrValueRegister(MIRType::String, AnyRegister(stringReg)),
17293 dest.valueReg());
17294 } else {
17295 MOZ_ASSERT(dest.typedReg().gpr() == stringReg)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(dest.typedReg().gpr() == stringReg)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(dest.typedReg().gpr() == stringReg
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"dest.typedReg().gpr() == stringReg", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 17295); AnnotateMozCrashReason("MOZ_ASSERT" "(" "dest.typedReg().gpr() == stringReg"
")"); do { *((volatile int*)__null) = 17295; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
17296 }
17297
17298 emitPreBarrier(slotAddr);
17299 masm.storeTypedOrValue(dest, slotAddr);
17300
17301 masm.bind(ool->rejoin());
17302}
17303
17304void CodeGenerator::visitLoadFixedSlotAndAtomize(
17305 LLoadFixedSlotAndAtomize* ins) {
17306 Register obj = ToRegister(ins->getOperand(0));
17307 Register temp = ToRegister(ins->temp0());
17308 size_t slot = ins->mir()->slot();
17309 ValueOperand result = ToOutValue(ins);
17310
17311 Address slotAddr(obj, NativeObject::getFixedSlotOffset(slot));
17312 masm.loadValue(slotAddr, result);
17313
17314 Label notString;
17315 masm.branchTestString(Assembler::NotEqual, result, &notString);
17316 masm.unboxString(result, temp);
17317 emitMaybeAtomizeSlot(ins, temp, slotAddr, result);
17318 masm.bind(&notString);
17319}
17320
17321void CodeGenerator::visitLoadDynamicSlotAndAtomize(
17322 LLoadDynamicSlotAndAtomize* ins) {
17323 ValueOperand result = ToOutValue(ins);
17324 Register temp = ToRegister(ins->temp0());
17325 Register base = ToRegister(ins->input());
17326 int32_t offset = ins->mir()->slot() * sizeof(js::Value);
17327
17328 Address slotAddr(base, offset);
17329 masm.loadValue(slotAddr, result);
17330
17331 Label notString;
17332 masm.branchTestString(Assembler::NotEqual, result, &notString);
17333 masm.unboxString(result, temp);
17334 emitMaybeAtomizeSlot(ins, temp, slotAddr, result);
17335 masm.bind(&notString);
17336}
17337
17338void CodeGenerator::visitLoadFixedSlotUnboxAndAtomize(
17339 LLoadFixedSlotUnboxAndAtomize* ins) {
17340 const MLoadFixedSlotAndUnbox* mir = ins->mir();
17341 MOZ_ASSERT(mir->type() == MIRType::String)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(mir->type() == MIRType::String)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(mir->type() == MIRType::String
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"mir->type() == MIRType::String", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 17341); AnnotateMozCrashReason("MOZ_ASSERT" "(" "mir->type() == MIRType::String"
")"); do { *((volatile int*)__null) = 17341; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
17342 Register input = ToRegister(ins->object());
17343 AnyRegister result = ToAnyRegister(ins->output());
17344 size_t slot = mir->slot();
17345
17346 Address slotAddr(input, NativeObject::getFixedSlotOffset(slot));
17347
17348 Label bail;
17349 EmitLoadAndUnbox(masm, slotAddr, MIRType::String, mir->fallible(), result,
17350 &bail);
17351 emitMaybeAtomizeSlot(ins, result.gpr(), slotAddr,
17352 TypedOrValueRegister(MIRType::String, result));
17353
17354 if (mir->fallible()) {
17355 bailoutFrom(&bail, ins->snapshot());
17356 }
17357}
17358
17359void CodeGenerator::visitLoadDynamicSlotUnboxAndAtomize(
17360 LLoadDynamicSlotUnboxAndAtomize* ins) {
17361 const MLoadDynamicSlotAndUnbox* mir = ins->mir();
17362 MOZ_ASSERT(mir->type() == MIRType::String)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(mir->type() == MIRType::String)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(mir->type() == MIRType::String
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"mir->type() == MIRType::String", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 17362); AnnotateMozCrashReason("MOZ_ASSERT" "(" "mir->type() == MIRType::String"
")"); do { *((volatile int*)__null) = 17362; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
17363 Register input = ToRegister(ins->slots());
17364 AnyRegister result = ToAnyRegister(ins->output());
17365 size_t slot = mir->slot();
17366
17367 Address slotAddr(input, slot * sizeof(JS::Value));
17368
17369 Label bail;
17370 EmitLoadAndUnbox(masm, slotAddr, MIRType::String, mir->fallible(), result,
17371 &bail);
17372 emitMaybeAtomizeSlot(ins, result.gpr(), slotAddr,
17373 TypedOrValueRegister(MIRType::String, result));
17374
17375 if (mir->fallible()) {
17376 bailoutFrom(&bail, ins->snapshot());
17377 }
17378}
17379
17380void CodeGenerator::visitAddAndStoreSlot(LAddAndStoreSlot* ins) {
17381 const Register obj = ToRegister(ins->getOperand(0));
17382 const ValueOperand value = ToValue(ins, LAddAndStoreSlot::ValueIndex);
17383 const Register maybeTemp = ToTempRegisterOrInvalid(ins->temp0());
17384
17385 Shape* shape = ins->mir()->shape();
17386 masm.storeObjShape(shape, obj, [](MacroAssembler& masm, const Address& addr) {
17387 EmitPreBarrier(masm, addr, MIRType::Shape);
17388 });
17389
17390 // Perform the store. No pre-barrier required since this is a new
17391 // initialization.
17392
17393 uint32_t offset = ins->mir()->slotOffset();
17394 if (ins->mir()->kind() == MAddAndStoreSlot::Kind::FixedSlot) {
17395 Address slot(obj, offset);
17396 masm.storeValue(value, slot);
17397 } else {
17398 masm.loadPtr(Address(obj, NativeObject::offsetOfSlots()), maybeTemp);
17399 Address slot(maybeTemp, offset);
17400 masm.storeValue(value, slot);
17401 }
17402}
17403
17404void CodeGenerator::visitAllocateAndStoreSlot(LAllocateAndStoreSlot* ins) {
17405 const Register obj = ToRegister(ins->getOperand(0));
17406 const ValueOperand value = ToValue(ins, LAllocateAndStoreSlot::ValueIndex);
17407 const Register temp0 = ToRegister(ins->temp0());
17408 const Register temp1 = ToRegister(ins->temp1());
17409
17410 masm.Push(obj);
17411 masm.Push(value);
17412
17413 using Fn = bool (*)(JSContext* cx, NativeObject* obj, uint32_t newCount);
17414 masm.setupAlignedABICall();
17415 masm.loadJSContext(temp0);
17416 masm.passABIArg(temp0);
17417 masm.passABIArg(obj);
17418 masm.move32(Imm32(ins->mir()->numNewSlots()), temp1);
17419 masm.passABIArg(temp1);
17420 masm.callWithABI<Fn, NativeObject::growSlotsPure>();
17421 masm.storeCallPointerResult(temp0);
17422
17423 masm.Pop(value);
17424 masm.Pop(obj);
17425
17426 bailoutIfFalseBool(temp0, ins->snapshot());
17427
17428 masm.storeObjShape(ins->mir()->shape(), obj,
17429 [](MacroAssembler& masm, const Address& addr) {
17430 EmitPreBarrier(masm, addr, MIRType::Shape);
17431 });
17432
17433 // Perform the store. No pre-barrier required since this is a new
17434 // initialization.
17435 masm.loadPtr(Address(obj, NativeObject::offsetOfSlots()), temp0);
17436 Address slot(temp0, ins->mir()->slotOffset());
17437 masm.storeValue(value, slot);
17438}
17439
17440void CodeGenerator::visitAddSlotAndCallAddPropHook(
17441 LAddSlotAndCallAddPropHook* ins) {
17442 const Register obj = ToRegister(ins->object());
17443 const ValueOperand value =
17444 ToValue(ins, LAddSlotAndCallAddPropHook::ValueIndex);
17445
17446 pushArg(ImmGCPtr(ins->mir()->shape()));
17447 pushArg(value);
17448 pushArg(obj);
17449
17450 using Fn =
17451 bool (*)(JSContext*, Handle<NativeObject*>, HandleValue, Handle<Shape*>);
17452 callVM<Fn, AddSlotAndCallAddPropHook>(ins);
17453}
17454
17455void CodeGenerator::visitStoreFixedSlotV(LStoreFixedSlotV* ins) {
17456 const Register obj = ToRegister(ins->getOperand(0));
17457 size_t slot = ins->mir()->slot();
17458
17459 const ValueOperand value = ToValue(ins, LStoreFixedSlotV::ValueIndex);
17460
17461 Address address(obj, NativeObject::getFixedSlotOffset(slot));
17462 if (ins->mir()->needsBarrier()) {
17463 emitPreBarrier(address);
17464 }
17465
17466 masm.storeValue(value, address);
17467}
17468
17469void CodeGenerator::visitStoreFixedSlotT(LStoreFixedSlotT* ins) {
17470 const Register obj = ToRegister(ins->getOperand(0));
17471 size_t slot = ins->mir()->slot();
17472
17473 const LAllocation* value = ins->value();
17474 MIRType valueType = ins->mir()->value()->type();
17475
17476 Address address(obj, NativeObject::getFixedSlotOffset(slot));
17477 if (ins->mir()->needsBarrier()) {
17478 emitPreBarrier(address);
17479 }
17480
17481 ConstantOrRegister nvalue =
17482 value->isConstant()
17483 ? ConstantOrRegister(value->toConstant()->toJSValue())
17484 : TypedOrValueRegister(valueType, ToAnyRegister(value));
17485 masm.storeConstantOrRegister(nvalue, address);
17486}
17487
17488void CodeGenerator::visitGetNameCache(LGetNameCache* ins) {
17489 LiveRegisterSet liveRegs = ins->safepoint()->liveRegs();
17490 Register envChain = ToRegister(ins->envObj());
17491 ValueOperand output = ToOutValue(ins);
17492 Register temp = ToRegister(ins->temp0());
17493
17494 IonGetNameIC ic(liveRegs, envChain, output, temp);
17495 addIC(ins, allocateIC(ic));
17496}
17497
17498void CodeGenerator::addGetPropertyCache(LInstruction* ins,
17499 LiveRegisterSet liveRegs,
17500 TypedOrValueRegister value,
17501 const ConstantOrRegister& id,
17502 ValueOperand output) {
17503 CacheKind kind = CacheKind::GetElem;
17504 if (id.constant() && id.value().isString()) {
17505 JSString* idString = id.value().toString();
17506 if (idString->isAtom() && !idString->asAtom().isIndex()) {
17507 kind = CacheKind::GetProp;
17508 }
17509 }
17510 IonGetPropertyIC cache(kind, liveRegs, value, id, output);
17511 addIC(ins, allocateIC(cache));
17512}
17513
17514void CodeGenerator::addSetPropertyCache(LInstruction* ins,
17515 LiveRegisterSet liveRegs,
17516 Register objReg, Register temp,
17517 const ConstantOrRegister& id,
17518 const ConstantOrRegister& value,
17519 bool strict) {
17520 CacheKind kind = CacheKind::SetElem;
17521 if (id.constant() && id.value().isString()) {
17522 JSString* idString = id.value().toString();
17523 if (idString->isAtom() && !idString->asAtom().isIndex()) {
17524 kind = CacheKind::SetProp;
17525 }
17526 }
17527 IonSetPropertyIC cache(kind, liveRegs, objReg, temp, id, value, strict);
17528 addIC(ins, allocateIC(cache));
17529}
17530
17531ConstantOrRegister CodeGenerator::toConstantOrRegister(LInstruction* lir,
17532 size_t n, MIRType type) {
17533 if (type == MIRType::Value) {
17534 return TypedOrValueRegister(ToValue(lir, n));
17535 }
17536
17537 const LAllocation* value = lir->getOperand(n);
17538 if (value->isConstant()) {
17539 return ConstantOrRegister(value->toConstant()->toJSValue());
17540 }
17541
17542 return TypedOrValueRegister(type, ToAnyRegister(value));
17543}
17544
17545void CodeGenerator::visitGetPropertyCache(LGetPropertyCache* ins) {
17546 LiveRegisterSet liveRegs = ins->safepoint()->liveRegs();
17547 TypedOrValueRegister value =
17548 toConstantOrRegister(ins, LGetPropertyCache::ValueIndex,
17549 ins->mir()->value()->type())
17550 .reg();
17551 ConstantOrRegister id = toConstantOrRegister(ins, LGetPropertyCache::IdIndex,
17552 ins->mir()->idval()->type());
17553 ValueOperand output = ToOutValue(ins);
17554 addGetPropertyCache(ins, liveRegs, value, id, output);
17555}
17556
17557void CodeGenerator::visitGetPropSuperCache(LGetPropSuperCache* ins) {
17558 LiveRegisterSet liveRegs = ins->safepoint()->liveRegs();
17559 Register obj = ToRegister(ins->obj());
17560 TypedOrValueRegister receiver =
17561 toConstantOrRegister(ins, LGetPropSuperCache::ReceiverIndex,
17562 ins->mir()->receiver()->type())
17563 .reg();
17564 ConstantOrRegister id = toConstantOrRegister(ins, LGetPropSuperCache::IdIndex,
17565 ins->mir()->idval()->type());
17566 ValueOperand output = ToOutValue(ins);
17567
17568 CacheKind kind = CacheKind::GetElemSuper;
17569 if (id.constant() && id.value().isString()) {
17570 JSString* idString = id.value().toString();
17571 if (idString->isAtom() && !idString->asAtom().isIndex()) {
17572 kind = CacheKind::GetPropSuper;
17573 }
17574 }
17575
17576 IonGetPropSuperIC cache(kind, liveRegs, obj, receiver, id, output);
17577 addIC(ins, allocateIC(cache));
17578}
17579
17580void CodeGenerator::visitBindNameCache(LBindNameCache* ins) {
17581 LiveRegisterSet liveRegs = ins->safepoint()->liveRegs();
17582 Register envChain = ToRegister(ins->environmentChain());
17583 Register output = ToRegister(ins->output());
17584 Register temp = ToRegister(ins->temp0());
17585
17586 IonBindNameIC ic(liveRegs, envChain, output, temp);
17587 addIC(ins, allocateIC(ic));
17588}
17589
17590void CodeGenerator::visitHasOwnCache(LHasOwnCache* ins) {
17591 LiveRegisterSet liveRegs = ins->safepoint()->liveRegs();
17592 TypedOrValueRegister value =
17593 toConstantOrRegister(ins, LHasOwnCache::ValueIndex,
17594 ins->mir()->value()->type())
17595 .reg();
17596 TypedOrValueRegister id = toConstantOrRegister(ins, LHasOwnCache::IdIndex,
17597 ins->mir()->idval()->type())
17598 .reg();
17599 Register output = ToRegister(ins->output());
17600
17601 IonHasOwnIC cache(liveRegs, value, id, output);
17602 addIC(ins, allocateIC(cache));
17603}
17604
17605void CodeGenerator::visitCheckPrivateFieldCache(LCheckPrivateFieldCache* ins) {
17606 LiveRegisterSet liveRegs = ins->safepoint()->liveRegs();
17607 TypedOrValueRegister value =
17608 toConstantOrRegister(ins, LCheckPrivateFieldCache::ValueIndex,
17609 ins->mir()->value()->type())
17610 .reg();
17611 TypedOrValueRegister id =
17612 toConstantOrRegister(ins, LCheckPrivateFieldCache::IdIndex,
17613 ins->mir()->idval()->type())
17614 .reg();
17615 Register output = ToRegister(ins->output());
17616
17617 IonCheckPrivateFieldIC cache(liveRegs, value, id, output);
17618 addIC(ins, allocateIC(cache));
17619}
17620
17621void CodeGenerator::visitNewPrivateName(LNewPrivateName* ins) {
17622 pushArg(ImmGCPtr(ins->mir()->name()));
17623
17624 using Fn = JS::Symbol* (*)(JSContext*, Handle<JSAtom*>);
17625 callVM<Fn, NewPrivateName>(ins);
17626}
17627
17628void CodeGenerator::visitCallDeleteProperty(LCallDeleteProperty* lir) {
17629 pushArg(ImmGCPtr(lir->mir()->name()));
17630 pushArg(ToValue(lir, LCallDeleteProperty::ValueIndex));
17631
17632 using Fn = bool (*)(JSContext*, HandleValue, Handle<PropertyName*>, bool*);
17633 if (lir->mir()->strict()) {
17634 callVM<Fn, DelPropOperation<true>>(lir);
17635 } else {
17636 callVM<Fn, DelPropOperation<false>>(lir);
17637 }
17638}
17639
17640void CodeGenerator::visitCallDeleteElement(LCallDeleteElement* lir) {
17641 pushArg(ToValue(lir, LCallDeleteElement::IndexIndex));
17642 pushArg(ToValue(lir, LCallDeleteElement::ValueIndex));
17643
17644 using Fn = bool (*)(JSContext*, HandleValue, HandleValue, bool*);
17645 if (lir->mir()->strict()) {
17646 callVM<Fn, DelElemOperation<true>>(lir);
17647 } else {
17648 callVM<Fn, DelElemOperation<false>>(lir);
17649 }
17650}
17651
17652void CodeGenerator::visitObjectToIterator(LObjectToIterator* lir) {
17653 Register obj = ToRegister(lir->object());
17654 Register iterObj = ToRegister(lir->output());
17655 Register temp = ToRegister(lir->temp0());
17656 Register temp2 = ToRegister(lir->temp1());
17657 Register temp3 = ToRegister(lir->temp2());
17658
17659 using Fn = PropertyIteratorObject* (*)(JSContext*, HandleObject);
17660 OutOfLineCode* ool = (lir->mir()->wantsIndices())
17661 ? oolCallVM<Fn, GetIteratorWithIndices>(
17662 lir, ArgList(obj), StoreRegisterTo(iterObj))
17663 : oolCallVM<Fn, GetIterator>(
17664 lir, ArgList(obj), StoreRegisterTo(iterObj));
17665
17666 masm.maybeLoadIteratorFromShape(obj, iterObj, temp, temp2, temp3,
17667 ool->entry());
17668
17669 Register nativeIter = temp;
17670 masm.loadPrivate(
17671 Address(iterObj, PropertyIteratorObject::offsetOfIteratorSlot()),
17672 nativeIter);
17673
17674 if (lir->mir()->wantsIndices()) {
17675 // At least one consumer of the output of this iterator has been optimized
17676 // to use iterator indices. If the cached iterator doesn't include indices,
17677 // but it was marked to indicate that we can create them if needed, then we
17678 // do a VM call to replace the cached iterator with a fresh iterator
17679 // including indices.
17680 masm.branchNativeIteratorIndices(Assembler::Equal, nativeIter, temp2,
17681 NativeIteratorIndices::AvailableOnRequest,
17682 ool->entry());
17683 }
17684
17685 Address iterFlagsAddr(nativeIter, NativeIterator::offsetOfFlagsAndCount());
17686 masm.storePtr(
17687 obj, Address(nativeIter, NativeIterator::offsetOfObjectBeingIterated()));
17688 masm.or32(Imm32(NativeIterator::Flags::Active), iterFlagsAddr);
17689
17690 Register enumeratorsAddr = temp2;
17691 masm.movePtr(ImmPtr(lir->mir()->enumeratorsAddr()), enumeratorsAddr);
17692 masm.registerIterator(enumeratorsAddr, nativeIter, temp3);
17693
17694 // Generate post-write barrier for storing to |iterObj->objectBeingIterated_|.
17695 // We already know that |iterObj| is tenured, so we only have to check |obj|.
17696 Label skipBarrier;
17697 masm.branchPtrInNurseryChunk(Assembler::NotEqual, obj, temp2, &skipBarrier);
17698 {
17699 LiveRegisterSet save = liveVolatileRegs(lir);
17700 save.takeUnchecked(temp);
17701 save.takeUnchecked(temp2);
17702 save.takeUnchecked(temp3);
17703 if (iterObj.volatile_()) {
17704 save.addUnchecked(iterObj);
17705 }
17706
17707 masm.PushRegsInMask(save);
17708 emitPostWriteBarrier(iterObj);
17709 masm.PopRegsInMask(save);
17710 }
17711 masm.bind(&skipBarrier);
17712
17713 masm.bind(ool->rejoin());
17714}
17715
17716void CodeGenerator::visitValueToIterator(LValueToIterator* lir) {
17717 pushArg(ToValue(lir, LValueToIterator::ValueIndex));
17718
17719 using Fn = PropertyIteratorObject* (*)(JSContext*, HandleValue);
17720 callVM<Fn, ValueToIterator>(lir);
17721}
17722
17723void CodeGenerator::visitIteratorHasIndicesAndBranch(
17724 LIteratorHasIndicesAndBranch* lir) {
17725 Register iterator = ToRegister(lir->iterator());
17726 Register object = ToRegister(lir->object());
17727 Register temp = ToRegister(lir->temp());
17728 Register temp2 = ToRegister(lir->temp2());
17729 Label* ifTrue = getJumpLabelForBranch(lir->ifTrue());
17730 Label* ifFalse = getJumpLabelForBranch(lir->ifFalse());
17731
17732 // Check that the iterator has indices available.
17733 Address nativeIterAddr(iterator,
17734 PropertyIteratorObject::offsetOfIteratorSlot());
17735 masm.loadPrivate(nativeIterAddr, temp);
17736 masm.branchNativeIteratorIndices(Assembler::NotEqual, temp, temp2,
17737 NativeIteratorIndices::Valid, ifFalse);
17738
17739 // Guard that the first shape stored in the iterator matches the current
17740 // shape of the iterated object.
17741 Address firstShapeAddr(temp, NativeIterator::offsetOfFirstShape());
17742 masm.loadPtr(firstShapeAddr, temp);
17743 masm.branchTestObjShape(Assembler::NotEqual, object, temp, temp2, object,
17744 ifFalse);
17745
17746 if (!isNextBlock(lir->ifTrue()->lir())) {
17747 masm.jump(ifTrue);
17748 }
17749}
17750
17751void CodeGenerator::visitLoadSlotByIteratorIndex(
17752 LLoadSlotByIteratorIndex* lir) {
17753 Register object = ToRegister(lir->object());
17754 Register iterator = ToRegister(lir->iterator());
17755 Register temp = ToRegister(lir->temp0());
17756 Register temp2 = ToRegister(lir->temp1());
17757 ValueOperand result = ToOutValue(lir);
17758
17759 masm.extractCurrentIndexAndKindFromIterator(iterator, temp, temp2);
17760
17761 Label notDynamicSlot, notFixedSlot, done;
17762 masm.branch32(Assembler::NotEqual, temp2,
17763 Imm32(uint32_t(PropertyIndex::Kind::DynamicSlot)),
17764 &notDynamicSlot);
17765 masm.loadPtr(Address(object, NativeObject::offsetOfSlots()), temp2);
17766 masm.loadValue(BaseValueIndex(temp2, temp), result);
17767 masm.jump(&done);
17768
17769 masm.bind(&notDynamicSlot);
17770 masm.branch32(Assembler::NotEqual, temp2,
17771 Imm32(uint32_t(PropertyIndex::Kind::FixedSlot)), &notFixedSlot);
17772 // Fixed slot
17773 masm.loadValue(BaseValueIndex(object, temp, sizeof(NativeObject)), result);
17774 masm.jump(&done);
17775 masm.bind(&notFixedSlot);
17776
17777#ifdef DEBUG1
17778 Label kindOkay;
17779 masm.branch32(Assembler::Equal, temp2,
17780 Imm32(uint32_t(PropertyIndex::Kind::Element)), &kindOkay);
17781 masm.assumeUnreachable("Invalid PropertyIndex::Kind");
17782 masm.bind(&kindOkay);
17783#endif
17784
17785 // Dense element
17786 masm.loadPtr(Address(object, NativeObject::offsetOfElements()), temp2);
17787 Label indexOkay;
17788 Address initLength(temp2, ObjectElements::offsetOfInitializedLength());
17789 masm.branch32(Assembler::Above, initLength, temp, &indexOkay);
17790 masm.assumeUnreachable("Dense element out of bounds");
17791 masm.bind(&indexOkay);
17792
17793 masm.loadValue(BaseObjectElementIndex(temp2, temp), result);
17794 masm.bind(&done);
17795}
17796
17797void CodeGenerator::visitStoreSlotByIteratorIndex(
17798 LStoreSlotByIteratorIndex* lir) {
17799 Register object = ToRegister(lir->object());
17800 Register iterator = ToRegister(lir->iterator());
17801 ValueOperand value = ToValue(lir, LStoreSlotByIteratorIndex::ValueIndex);
17802 Register temp = ToRegister(lir->temp0());
17803 Register temp2 = ToRegister(lir->temp1());
17804
17805 masm.extractCurrentIndexAndKindFromIterator(iterator, temp, temp2);
17806
17807 Label notDynamicSlot, notFixedSlot, done, doStore;
17808 masm.branch32(Assembler::NotEqual, temp2,
17809 Imm32(uint32_t(PropertyIndex::Kind::DynamicSlot)),
17810 &notDynamicSlot);
17811 masm.loadPtr(Address(object, NativeObject::offsetOfSlots()), temp2);
17812 masm.computeEffectiveAddress(BaseValueIndex(temp2, temp), temp);
17813 masm.jump(&doStore);
17814
17815 masm.bind(&notDynamicSlot);
17816 masm.branch32(Assembler::NotEqual, temp2,
17817 Imm32(uint32_t(PropertyIndex::Kind::FixedSlot)), &notFixedSlot);
17818 // Fixed slot
17819 masm.computeEffectiveAddress(
17820 BaseValueIndex(object, temp, sizeof(NativeObject)), temp);
17821 masm.jump(&doStore);
17822 masm.bind(&notFixedSlot);
17823
17824#ifdef DEBUG1
17825 Label kindOkay;
17826 masm.branch32(Assembler::Equal, temp2,
17827 Imm32(uint32_t(PropertyIndex::Kind::Element)), &kindOkay);
17828 masm.assumeUnreachable("Invalid PropertyIndex::Kind");
17829 masm.bind(&kindOkay);
17830#endif
17831
17832 // Dense element
17833 masm.loadPtr(Address(object, NativeObject::offsetOfElements()), temp2);
17834 Label indexOkay;
17835 Address initLength(temp2, ObjectElements::offsetOfInitializedLength());
17836 masm.branch32(Assembler::Above, initLength, temp, &indexOkay);
17837 masm.assumeUnreachable("Dense element out of bounds");
17838 masm.bind(&indexOkay);
17839
17840 BaseObjectElementIndex elementAddress(temp2, temp);
17841 masm.computeEffectiveAddress(elementAddress, temp);
17842
17843 masm.bind(&doStore);
17844 Address storeAddress(temp, 0);
17845 emitPreBarrier(storeAddress);
17846 masm.storeValue(value, storeAddress);
17847
17848 masm.branchPtrInNurseryChunk(Assembler::Equal, object, temp2, &done);
17849 masm.branchValueIsNurseryCell(Assembler::NotEqual, value, temp2, &done);
17850
17851 saveVolatile(temp2);
17852 emitPostWriteBarrier(object);
17853 restoreVolatile(temp2);
17854
17855 masm.bind(&done);
17856}
17857
17858void CodeGenerator::visitSetPropertyCache(LSetPropertyCache* ins) {
17859 LiveRegisterSet liveRegs = ins->safepoint()->liveRegs();
17860 Register objReg = ToRegister(ins->object());
17861 Register temp = ToRegister(ins->temp0());
17862
17863 ConstantOrRegister id = toConstantOrRegister(ins, LSetPropertyCache::IdIndex,
17864 ins->mir()->idval()->type());
17865 ConstantOrRegister value = toConstantOrRegister(
17866 ins, LSetPropertyCache::ValueIndex, ins->mir()->value()->type());
17867
17868 addSetPropertyCache(ins, liveRegs, objReg, temp, id, value,
17869 ins->mir()->strict());
17870}
17871
17872void CodeGenerator::visitThrow(LThrow* lir) {
17873 pushArg(ToValue(lir, LThrow::ValueIndex));
17874
17875 using Fn = bool (*)(JSContext*, HandleValue);
17876 callVM<Fn, js::ThrowOperation>(lir);
17877}
17878
17879void CodeGenerator::visitThrowWithStack(LThrowWithStack* lir) {
17880 pushArg(ToValue(lir, LThrowWithStack::StackIndex));
17881 pushArg(ToValue(lir, LThrowWithStack::ValueIndex));
17882
17883 using Fn = bool (*)(JSContext*, HandleValue, HandleValue);
17884 callVM<Fn, js::ThrowWithStackOperation>(lir);
17885}
17886
17887class OutOfLineTypeOfV : public OutOfLineCodeBase<CodeGenerator> {
17888 LTypeOfV* ins_;
17889
17890 public:
17891 explicit OutOfLineTypeOfV(LTypeOfV* ins) : ins_(ins) {}
17892
17893 void accept(CodeGenerator* codegen) override {
17894 codegen->visitOutOfLineTypeOfV(this);
17895 }
17896 LTypeOfV* ins() const { return ins_; }
17897};
17898
17899void CodeGenerator::emitTypeOfJSType(JSValueType type, Register output) {
17900 switch (type) {
17901 case JSVAL_TYPE_OBJECT:
17902 masm.move32(Imm32(JSTYPE_OBJECT), output);
17903 break;
17904 case JSVAL_TYPE_DOUBLE:
17905 case JSVAL_TYPE_INT32:
17906 masm.move32(Imm32(JSTYPE_NUMBER), output);
17907 break;
17908 case JSVAL_TYPE_BOOLEAN:
17909 masm.move32(Imm32(JSTYPE_BOOLEAN), output);
17910 break;
17911 case JSVAL_TYPE_UNDEFINED:
17912 masm.move32(Imm32(JSTYPE_UNDEFINED), output);
17913 break;
17914 case JSVAL_TYPE_NULL:
17915 masm.move32(Imm32(JSTYPE_OBJECT), output);
17916 break;
17917 case JSVAL_TYPE_STRING:
17918 masm.move32(Imm32(JSTYPE_STRING), output);
17919 break;
17920 case JSVAL_TYPE_SYMBOL:
17921 masm.move32(Imm32(JSTYPE_SYMBOL), output);
17922 break;
17923 case JSVAL_TYPE_BIGINT:
17924 masm.move32(Imm32(JSTYPE_BIGINT), output);
17925 break;
17926 default:
17927 MOZ_CRASH("Unsupported JSValueType")do { do { } while (false); MOZ_ReportCrash("" "Unsupported JSValueType"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 17927); AnnotateMozCrashReason("MOZ_CRASH(" "Unsupported JSValueType"
")"); do { *((volatile int*)__null) = 17927; __attribute__((
nomerge)) ::abort(); } while (false); } while (false)
;
17928 }
17929}
17930
17931void CodeGenerator::emitTypeOfCheck(JSValueType type, Register tag,
17932 Register output, Label* done,
17933 Label* oolObject) {
17934 Label notMatch;
17935 switch (type) {
17936 case JSVAL_TYPE_OBJECT:
17937 // The input may be a callable object (result is "function") or
17938 // may emulate undefined (result is "undefined"). Use an OOL path.
17939 masm.branchTestObject(Assembler::Equal, tag, oolObject);
17940 return;
17941 case JSVAL_TYPE_DOUBLE:
17942 case JSVAL_TYPE_INT32:
17943 masm.branchTestNumber(Assembler::NotEqual, tag, &notMatch);
17944 break;
17945 default:
17946 masm.branchTestType(Assembler::NotEqual, tag, type, &notMatch);
17947 break;
17948 }
17949
17950 emitTypeOfJSType(type, output);
17951 masm.jump(done);
17952 masm.bind(&notMatch);
17953}
17954
17955void CodeGenerator::visitTypeOfV(LTypeOfV* lir) {
17956 const ValueOperand value = ToValue(lir, LTypeOfV::InputIndex);
17957 Register output = ToRegister(lir->output());
17958 Register tag = masm.extractTag(value, output);
17959
17960 Label done;
17961
17962 auto* ool = new (alloc()) OutOfLineTypeOfV(lir);
17963 addOutOfLineCode(ool, lir->mir());
17964
17965 const std::initializer_list<JSValueType> defaultOrder = {
17966 JSVAL_TYPE_OBJECT, JSVAL_TYPE_DOUBLE, JSVAL_TYPE_UNDEFINED,
17967 JSVAL_TYPE_NULL, JSVAL_TYPE_BOOLEAN, JSVAL_TYPE_STRING,
17968 JSVAL_TYPE_SYMBOL, JSVAL_TYPE_BIGINT};
17969
17970 mozilla::EnumSet<JSValueType, uint32_t> remaining(defaultOrder);
17971
17972 // Generate checks for previously observed types first.
17973 // The TypeDataList is sorted by descending frequency.
17974 for (auto& observed : lir->mir()->observedTypes()) {
17975 JSValueType type = observed.type();
17976
17977 // Unify number types.
17978 if (type == JSVAL_TYPE_INT32) {
17979 type = JSVAL_TYPE_DOUBLE;
17980 }
17981
17982 remaining -= type;
17983
17984 emitTypeOfCheck(type, tag, output, &done, ool->entry());
17985 }
17986
17987 // Generate checks for remaining types.
17988 for (auto type : defaultOrder) {
17989 if (!remaining.contains(type)) {
17990 continue;
17991 }
17992 remaining -= type;
17993
17994 if (remaining.isEmpty() && type != JSVAL_TYPE_OBJECT) {
17995 // We can skip the check for the last remaining type, unless the type is
17996 // JSVAL_TYPE_OBJECT, which may have to go through the OOL path.
17997#ifdef DEBUG1
17998 emitTypeOfCheck(type, tag, output, &done, ool->entry());
17999 masm.assumeUnreachable("Unexpected Value type in visitTypeOfV");
18000#else
18001 emitTypeOfJSType(type, output);
18002#endif
18003 } else {
18004 emitTypeOfCheck(type, tag, output, &done, ool->entry());
18005 }
18006 }
18007 MOZ_ASSERT(remaining.isEmpty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(remaining.isEmpty())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(remaining.isEmpty()))), 0)))
{ do { } while (false); MOZ_ReportAssertionFailure("remaining.isEmpty()"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 18007); AnnotateMozCrashReason("MOZ_ASSERT" "(" "remaining.isEmpty()"
")"); do { *((volatile int*)__null) = 18007; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
18008
18009 masm.bind(&done);
18010 masm.bind(ool->rejoin());
18011}
18012
18013void CodeGenerator::emitTypeOfObject(Register obj, Register output,
18014 Label* done) {
18015 Label slowCheck, isObject, isCallable, isUndefined;
18016 masm.typeOfObject(obj, output, &slowCheck, &isObject, &isCallable,
18017 &isUndefined);
18018
18019 masm.bind(&isCallable);
18020 masm.move32(Imm32(JSTYPE_FUNCTION), output);
18021 masm.jump(done);
18022
18023 masm.bind(&isUndefined);
18024 masm.move32(Imm32(JSTYPE_UNDEFINED), output);
18025 masm.jump(done);
18026
18027 masm.bind(&isObject);
18028 masm.move32(Imm32(JSTYPE_OBJECT), output);
18029 masm.jump(done);
18030
18031 masm.bind(&slowCheck);
18032
18033 saveVolatile(output);
18034 using Fn = JSType (*)(JSObject*);
18035 masm.setupAlignedABICall();
18036 masm.passABIArg(obj);
18037 masm.callWithABI<Fn, js::TypeOfObject>();
18038 masm.storeCallInt32Result(output);
18039 restoreVolatile(output);
18040}
18041
18042void CodeGenerator::visitOutOfLineTypeOfV(OutOfLineTypeOfV* ool) {
18043 LTypeOfV* ins = ool->ins();
18044
18045 ValueOperand input = ToValue(ins, LTypeOfV::InputIndex);
18046 Register temp = ToTempUnboxRegister(ins->temp0());
18047 Register output = ToRegister(ins->output());
18048
18049 Register obj = masm.extractObject(input, temp);
18050 emitTypeOfObject(obj, output, ool->rejoin());
18051 masm.jump(ool->rejoin());
18052}
18053
18054void CodeGenerator::visitTypeOfO(LTypeOfO* lir) {
18055 Register obj = ToRegister(lir->object());
18056 Register output = ToRegister(lir->output());
18057
18058 Label done;
18059 emitTypeOfObject(obj, output, &done);
18060 masm.bind(&done);
18061}
18062
18063void CodeGenerator::visitTypeOfName(LTypeOfName* lir) {
18064 Register input = ToRegister(lir->input());
18065 Register output = ToRegister(lir->output());
18066
18067#ifdef DEBUG1
18068 Label ok;
18069 masm.branch32(Assembler::Below, input, Imm32(JSTYPE_LIMIT), &ok);
18070 masm.assumeUnreachable("bad JSType");
18071 masm.bind(&ok);
18072#endif
18073
18074 static_assert(JSTYPE_UNDEFINED == 0);
18075
18076 masm.movePtr(ImmPtr(&gen->runtime->names().undefined), output);
18077 masm.loadPtr(BaseIndex(output, input, ScalePointer), output);
18078}
18079
18080class OutOfLineTypeOfIsNonPrimitiveV : public OutOfLineCodeBase<CodeGenerator> {
18081 LTypeOfIsNonPrimitiveV* ins_;
18082
18083 public:
18084 explicit OutOfLineTypeOfIsNonPrimitiveV(LTypeOfIsNonPrimitiveV* ins)
18085 : ins_(ins) {}
18086
18087 void accept(CodeGenerator* codegen) override {
18088 codegen->visitOutOfLineTypeOfIsNonPrimitiveV(this);
18089 }
18090 auto* ins() const { return ins_; }
18091};
18092
18093class OutOfLineTypeOfIsNonPrimitiveO : public OutOfLineCodeBase<CodeGenerator> {
18094 LTypeOfIsNonPrimitiveO* ins_;
18095
18096 public:
18097 explicit OutOfLineTypeOfIsNonPrimitiveO(LTypeOfIsNonPrimitiveO* ins)
18098 : ins_(ins) {}
18099
18100 void accept(CodeGenerator* codegen) override {
18101 codegen->visitOutOfLineTypeOfIsNonPrimitiveO(this);
18102 }
18103 auto* ins() const { return ins_; }
18104};
18105
18106void CodeGenerator::emitTypeOfIsObjectOOL(MTypeOfIs* mir, Register obj,
18107 Register output) {
18108 saveVolatile(output);
18109 using Fn = JSType (*)(JSObject*);
18110 masm.setupAlignedABICall();
18111 masm.passABIArg(obj);
18112 masm.callWithABI<Fn, js::TypeOfObject>();
18113 masm.storeCallInt32Result(output);
18114 restoreVolatile(output);
18115
18116 auto cond = JSOpToCondition(mir->jsop(), /* isSigned = */ false);
18117 masm.cmp32Set(cond, output, Imm32(mir->jstype()), output);
18118}
18119
18120void CodeGenerator::visitOutOfLineTypeOfIsNonPrimitiveV(
18121 OutOfLineTypeOfIsNonPrimitiveV* ool) {
18122 auto* ins = ool->ins();
18123 ValueOperand input = ToValue(ins, LTypeOfIsNonPrimitiveV::InputIndex);
18124 Register output = ToRegister(ins->output());
18125 Register temp = ToTempUnboxRegister(ins->temp0());
18126
18127 Register obj = masm.extractObject(input, temp);
18128
18129 emitTypeOfIsObjectOOL(ins->mir(), obj, output);
18130
18131 masm.jump(ool->rejoin());
18132}
18133
18134void CodeGenerator::visitOutOfLineTypeOfIsNonPrimitiveO(
18135 OutOfLineTypeOfIsNonPrimitiveO* ool) {
18136 auto* ins = ool->ins();
18137 Register input = ToRegister(ins->input());
18138 Register output = ToRegister(ins->output());
18139
18140 emitTypeOfIsObjectOOL(ins->mir(), input, output);
18141
18142 masm.jump(ool->rejoin());
18143}
18144
18145void CodeGenerator::emitTypeOfIsObject(MTypeOfIs* mir, Register obj,
18146 Register output, Label* success,
18147 Label* fail, Label* slowCheck) {
18148 Label* isObject = fail;
18149 Label* isFunction = fail;
18150 Label* isUndefined = fail;
18151
18152 switch (mir->jstype()) {
18153 case JSTYPE_UNDEFINED:
18154 isUndefined = success;
18155 break;
18156
18157 case JSTYPE_OBJECT:
18158 isObject = success;
18159 break;
18160
18161 case JSTYPE_FUNCTION:
18162 isFunction = success;
18163 break;
18164
18165 case JSTYPE_STRING:
18166 case JSTYPE_NUMBER:
18167 case JSTYPE_BOOLEAN:
18168 case JSTYPE_SYMBOL:
18169 case JSTYPE_BIGINT:
18170#ifdef ENABLE_RECORD_TUPLE
18171 case JSTYPE_RECORD:
18172 case JSTYPE_TUPLE:
18173#endif
18174 case JSTYPE_LIMIT:
18175 MOZ_CRASH("Primitive type")do { do { } while (false); MOZ_ReportCrash("" "Primitive type"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 18175); AnnotateMozCrashReason("MOZ_CRASH(" "Primitive type"
")"); do { *((volatile int*)__null) = 18175; __attribute__((
nomerge)) ::abort(); } while (false); } while (false)
;
18176 }
18177
18178 masm.typeOfObject(obj, output, slowCheck, isObject, isFunction, isUndefined);
18179
18180 auto op = mir->jsop();
18181
18182 Label done;
18183 masm.bind(fail);
18184 masm.move32(Imm32(op == JSOp::Ne || op == JSOp::StrictNe), output);
18185 masm.jump(&done);
18186 masm.bind(success);
18187 masm.move32(Imm32(op == JSOp::Eq || op == JSOp::StrictEq), output);
18188 masm.bind(&done);
18189}
18190
18191void CodeGenerator::visitTypeOfIsNonPrimitiveV(LTypeOfIsNonPrimitiveV* lir) {
18192 ValueOperand input = ToValue(lir, LTypeOfIsNonPrimitiveV::InputIndex);
18193 Register output = ToRegister(lir->output());
18194 Register temp = ToTempUnboxRegister(lir->temp0());
18195
18196 auto* mir = lir->mir();
18197
18198 auto* ool = new (alloc()) OutOfLineTypeOfIsNonPrimitiveV(lir);
18199 addOutOfLineCode(ool, mir);
18200
18201 Label success, fail;
18202
18203 switch (mir->jstype()) {
18204 case JSTYPE_UNDEFINED: {
18205 ScratchTagScope tag(masm, input);
18206 masm.splitTagForTest(input, tag);
18207
18208 masm.branchTestUndefined(Assembler::Equal, tag, &success);
18209 masm.branchTestObject(Assembler::NotEqual, tag, &fail);
18210 break;
18211 }
18212
18213 case JSTYPE_OBJECT: {
18214 ScratchTagScope tag(masm, input);
18215 masm.splitTagForTest(input, tag);
18216
18217 masm.branchTestNull(Assembler::Equal, tag, &success);
18218 masm.branchTestObject(Assembler::NotEqual, tag, &fail);
18219 break;
18220 }
18221
18222 case JSTYPE_FUNCTION: {
18223 masm.branchTestObject(Assembler::NotEqual, input, &fail);
18224 break;
18225 }
18226
18227 case JSTYPE_STRING:
18228 case JSTYPE_NUMBER:
18229 case JSTYPE_BOOLEAN:
18230 case JSTYPE_SYMBOL:
18231 case JSTYPE_BIGINT:
18232#ifdef ENABLE_RECORD_TUPLE
18233 case JSTYPE_RECORD:
18234 case JSTYPE_TUPLE:
18235#endif
18236 case JSTYPE_LIMIT:
18237 MOZ_CRASH("Primitive type")do { do { } while (false); MOZ_ReportCrash("" "Primitive type"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 18237); AnnotateMozCrashReason("MOZ_CRASH(" "Primitive type"
")"); do { *((volatile int*)__null) = 18237; __attribute__((
nomerge)) ::abort(); } while (false); } while (false)
;
18238 }
18239
18240 Register obj = masm.extractObject(input, temp);
18241
18242 emitTypeOfIsObject(mir, obj, output, &success, &fail, ool->entry());
18243
18244 masm.bind(ool->rejoin());
18245}
18246
18247void CodeGenerator::visitTypeOfIsNonPrimitiveO(LTypeOfIsNonPrimitiveO* lir) {
18248 Register input = ToRegister(lir->input());
18249 Register output = ToRegister(lir->output());
18250
18251 auto* mir = lir->mir();
18252
18253 auto* ool = new (alloc()) OutOfLineTypeOfIsNonPrimitiveO(lir);
18254 addOutOfLineCode(ool, mir);
18255
18256 Label success, fail;
18257 emitTypeOfIsObject(mir, input, output, &success, &fail, ool->entry());
18258
18259 masm.bind(ool->rejoin());
18260}
18261
18262void CodeGenerator::visitTypeOfIsPrimitive(LTypeOfIsPrimitive* lir) {
18263 ValueOperand input = ToValue(lir, LTypeOfIsPrimitive::InputIndex);
18264 Register output = ToRegister(lir->output());
18265
18266 auto* mir = lir->mir();
18267 auto cond = JSOpToCondition(mir->jsop(), /* isSigned = */ false);
18268
18269 switch (mir->jstype()) {
18270 case JSTYPE_STRING:
18271 masm.testStringSet(cond, input, output);
18272 break;
18273 case JSTYPE_NUMBER:
18274 masm.testNumberSet(cond, input, output);
18275 break;
18276 case JSTYPE_BOOLEAN:
18277 masm.testBooleanSet(cond, input, output);
18278 break;
18279 case JSTYPE_SYMBOL:
18280 masm.testSymbolSet(cond, input, output);
18281 break;
18282 case JSTYPE_BIGINT:
18283 masm.testBigIntSet(cond, input, output);
18284 break;
18285
18286 case JSTYPE_UNDEFINED:
18287 case JSTYPE_OBJECT:
18288 case JSTYPE_FUNCTION:
18289#ifdef ENABLE_RECORD_TUPLE
18290 case JSTYPE_RECORD:
18291 case JSTYPE_TUPLE:
18292#endif
18293 case JSTYPE_LIMIT:
18294 MOZ_CRASH("Non-primitive type")do { do { } while (false); MOZ_ReportCrash("" "Non-primitive type"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 18294); AnnotateMozCrashReason("MOZ_CRASH(" "Non-primitive type"
")"); do { *((volatile int*)__null) = 18294; __attribute__((
nomerge)) ::abort(); } while (false); } while (false)
;
18295 }
18296}
18297
18298void CodeGenerator::visitToAsyncIter(LToAsyncIter* lir) {
18299 pushArg(ToValue(lir, LToAsyncIter::NextMethodIndex));
18300 pushArg(ToRegister(lir->iterator()));
18301
18302 using Fn = JSObject* (*)(JSContext*, HandleObject, HandleValue);
18303 callVM<Fn, js::CreateAsyncFromSyncIterator>(lir);
18304}
18305
18306void CodeGenerator::visitToPropertyKeyCache(LToPropertyKeyCache* lir) {
18307 LiveRegisterSet liveRegs = lir->safepoint()->liveRegs();
18308 ValueOperand input = ToValue(lir, LToPropertyKeyCache::InputIndex);
18309 ValueOperand output = ToOutValue(lir);
18310
18311 IonToPropertyKeyIC ic(liveRegs, input, output);
18312 addIC(lir, allocateIC(ic));
18313}
18314
18315void CodeGenerator::visitLoadElementV(LLoadElementV* load) {
18316 Register elements = ToRegister(load->elements());
18317 const ValueOperand out = ToOutValue(load);
18318
18319 if (load->index()->isConstant()) {
18320 NativeObject::elementsSizeMustNotOverflow();
18321 int32_t offset = ToInt32(load->index()) * sizeof(Value);
18322 masm.loadValue(Address(elements, offset), out);
18323 } else {
18324 masm.loadValue(BaseObjectElementIndex(elements, ToRegister(load->index())),
18325 out);
18326 }
18327
18328 Label testMagic;
18329 masm.branchTestMagic(Assembler::Equal, out, &testMagic);
18330 bailoutFrom(&testMagic, load->snapshot());
18331}
18332
18333void CodeGenerator::visitLoadElementHole(LLoadElementHole* lir) {
18334 Register elements = ToRegister(lir->elements());
18335 Register index = ToRegister(lir->index());
18336 Register initLength = ToRegister(lir->initLength());
18337 const ValueOperand out = ToOutValue(lir);
18338
18339 const MLoadElementHole* mir = lir->mir();
18340
18341 // If the index is out of bounds, load |undefined|. Otherwise, load the
18342 // value.
18343 Label outOfBounds, done;
18344 masm.spectreBoundsCheck32(index, initLength, out.scratchReg(), &outOfBounds);
18345
18346 masm.loadValue(BaseObjectElementIndex(elements, index), out);
18347
18348 // If the value wasn't a hole, we're done. Otherwise, we'll load undefined.
18349 masm.branchTestMagic(Assembler::NotEqual, out, &done);
18350
18351 if (mir->needsNegativeIntCheck()) {
18352 Label loadUndefined;
18353 masm.jump(&loadUndefined);
18354
18355 masm.bind(&outOfBounds);
18356
18357 bailoutCmp32(Assembler::LessThan, index, Imm32(0), lir->snapshot());
18358
18359 masm.bind(&loadUndefined);
18360 } else {
18361 masm.bind(&outOfBounds);
18362 }
18363 masm.moveValue(UndefinedValue(), out);
18364
18365 masm.bind(&done);
18366}
18367
18368void CodeGenerator::visitLoadUnboxedScalar(LLoadUnboxedScalar* lir) {
18369 Register elements = ToRegister(lir->elements());
18370 Register temp0 = ToTempRegisterOrInvalid(lir->temp0());
18371 Register temp1 = ToTempRegisterOrInvalid(lir->temp1());
18372 AnyRegister out = ToAnyRegister(lir->output());
18373
18374 const MLoadUnboxedScalar* mir = lir->mir();
18375
18376 Scalar::Type storageType = mir->storageType();
18377
18378 LiveRegisterSet volatileRegs;
18379 if (MacroAssembler::LoadRequiresCall(storageType)) {
18380 volatileRegs = liveVolatileRegs(lir);
18381 }
18382
18383 Label fail;
18384 if (lir->index()->isConstant()) {
18385 Address source =
18386 ToAddress(elements, lir->index(), storageType, mir->offsetAdjustment());
18387 masm.loadFromTypedArray(storageType, source, out, temp0, temp1, &fail,
18388 volatileRegs);
18389 } else {
18390 BaseIndex source(elements, ToRegister(lir->index()),
18391 ScaleFromScalarType(storageType), mir->offsetAdjustment());
18392 masm.loadFromTypedArray(storageType, source, out, temp0, temp1, &fail,
18393 volatileRegs);
18394 }
18395
18396 if (fail.used()) {
18397 bailoutFrom(&fail, lir->snapshot());
18398 }
18399}
18400
18401void CodeGenerator::visitLoadUnboxedInt64(LLoadUnboxedInt64* lir) {
18402 Register elements = ToRegister(lir->elements());
18403 Register64 out = ToOutRegister64(lir);
18404
18405 const MLoadUnboxedScalar* mir = lir->mir();
18406
18407 Scalar::Type storageType = mir->storageType();
18408
18409 if (lir->index()->isConstant()) {
18410 Address source =
18411 ToAddress(elements, lir->index(), storageType, mir->offsetAdjustment());
18412 masm.load64(source, out);
18413 } else {
18414 BaseIndex source(elements, ToRegister(lir->index()),
18415 ScaleFromScalarType(storageType), mir->offsetAdjustment());
18416 masm.load64(source, out);
18417 }
18418}
18419
18420void CodeGenerator::visitLoadDataViewElement(LLoadDataViewElement* lir) {
18421 Register elements = ToRegister(lir->elements());
18422 const LAllocation* littleEndian = lir->littleEndian();
18423 Register temp1 = ToTempRegisterOrInvalid(lir->temp1());
18424 Register temp2 = ToTempRegisterOrInvalid(lir->temp2());
18425 Register64 temp64 = ToTempRegister64OrInvalid(lir->temp64());
18426 AnyRegister out = ToAnyRegister(lir->output());
18427
18428 const MLoadDataViewElement* mir = lir->mir();
18429 Scalar::Type storageType = mir->storageType();
18430
18431 LiveRegisterSet volatileRegs;
18432 if (MacroAssembler::LoadRequiresCall(storageType)) {
18433 volatileRegs = liveVolatileRegs(lir);
18434 }
18435
18436 BaseIndex source(elements, ToRegister(lir->index()), TimesOne);
18437
18438 bool noSwap = littleEndian->isConstant() &&
18439 ToBoolean(littleEndian) == MOZ_LITTLE_ENDIAN()1;
18440
18441 // Directly load if no byte swap is needed and the platform supports unaligned
18442 // accesses for the access. (Such support is assumed for integer types.)
18443 if (noSwap && (!Scalar::isFloatingType(storageType) ||
18444 MacroAssembler::SupportsFastUnalignedFPAccesses())) {
18445 Label fail;
18446 masm.loadFromTypedArray(storageType, source, out, temp1, temp2, &fail,
18447 volatileRegs);
18448
18449 if (fail.used()) {
18450 bailoutFrom(&fail, lir->snapshot());
18451 }
18452 return;
18453 }
18454
18455 // Load the value into a gpr register.
18456 switch (storageType) {
18457 case Scalar::Int16:
18458 masm.load16UnalignedSignExtend(source, out.gpr());
18459 break;
18460 case Scalar::Uint16:
18461 masm.load16UnalignedZeroExtend(source, out.gpr());
18462 break;
18463 case Scalar::Int32:
18464 masm.load32Unaligned(source, out.gpr());
18465 break;
18466 case Scalar::Uint32:
18467 masm.load32Unaligned(source, out.isFloat() ? temp1 : out.gpr());
18468 break;
18469 case Scalar::Float16:
18470 masm.load16UnalignedZeroExtend(source, temp1);
18471 break;
18472 case Scalar::Float32:
18473 masm.load32Unaligned(source, temp1);
18474 break;
18475 case Scalar::Float64:
18476 masm.load64Unaligned(source, temp64);
18477 break;
18478 case Scalar::Int8:
18479 case Scalar::Uint8:
18480 case Scalar::Uint8Clamped:
18481 case Scalar::BigInt64:
18482 case Scalar::BigUint64:
18483 default:
18484 MOZ_CRASH("Invalid typed array type")do { do { } while (false); MOZ_ReportCrash("" "Invalid typed array type"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 18484); AnnotateMozCrashReason("MOZ_CRASH(" "Invalid typed array type"
")"); do { *((volatile int*)__null) = 18484; __attribute__((
nomerge)) ::abort(); } while (false); } while (false)
;
18485 }
18486
18487 if (!noSwap) {
18488 // Swap the bytes in the loaded value.
18489 Label skip;
18490 if (!littleEndian->isConstant()) {
18491 masm.branch32(
18492 MOZ_LITTLE_ENDIAN()1 ? Assembler::NotEqual : Assembler::Equal,
18493 ToRegister(littleEndian), Imm32(0), &skip);
18494 }
18495
18496 switch (storageType) {
18497 case Scalar::Int16:
18498 masm.byteSwap16SignExtend(out.gpr());
18499 break;
18500 case Scalar::Uint16:
18501 masm.byteSwap16ZeroExtend(out.gpr());
18502 break;
18503 case Scalar::Int32:
18504 masm.byteSwap32(out.gpr());
18505 break;
18506 case Scalar::Uint32:
18507 masm.byteSwap32(out.isFloat() ? temp1 : out.gpr());
18508 break;
18509 case Scalar::Float16:
18510 masm.byteSwap16ZeroExtend(temp1);
18511 break;
18512 case Scalar::Float32:
18513 masm.byteSwap32(temp1);
18514 break;
18515 case Scalar::Float64:
18516 masm.byteSwap64(temp64);
18517 break;
18518 case Scalar::Int8:
18519 case Scalar::Uint8:
18520 case Scalar::Uint8Clamped:
18521 case Scalar::BigInt64:
18522 case Scalar::BigUint64:
18523 default:
18524 MOZ_CRASH("Invalid typed array type")do { do { } while (false); MOZ_ReportCrash("" "Invalid typed array type"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 18524); AnnotateMozCrashReason("MOZ_CRASH(" "Invalid typed array type"
")"); do { *((volatile int*)__null) = 18524; __attribute__((
nomerge)) ::abort(); } while (false); } while (false)
;
18525 }
18526
18527 if (skip.used()) {
18528 masm.bind(&skip);
18529 }
18530 }
18531
18532 // Move the value into the output register.
18533 switch (storageType) {
18534 case Scalar::Int16:
18535 case Scalar::Uint16:
18536 case Scalar::Int32:
18537 break;
18538 case Scalar::Uint32:
18539 if (out.isFloat()) {
18540 masm.convertUInt32ToDouble(temp1, out.fpu());
18541 } else {
18542 // Bail out if the value doesn't fit into a signed int32 value. This
18543 // is what allows MLoadDataViewElement to have a type() of
18544 // MIRType::Int32 for UInt32 array loads.
18545 bailoutTest32(Assembler::Signed, out.gpr(), out.gpr(), lir->snapshot());
18546 }
18547 break;
18548 case Scalar::Float16:
18549 masm.moveGPRToFloat16(temp1, out.fpu(), temp2, volatileRegs);
18550 masm.canonicalizeFloat(out.fpu());
18551 break;
18552 case Scalar::Float32:
18553 masm.moveGPRToFloat32(temp1, out.fpu());
18554 masm.canonicalizeFloat(out.fpu());
18555 break;
18556 case Scalar::Float64:
18557 masm.moveGPR64ToDouble(temp64, out.fpu());
18558 masm.canonicalizeDouble(out.fpu());
18559 break;
18560 case Scalar::Int8:
18561 case Scalar::Uint8:
18562 case Scalar::Uint8Clamped:
18563 case Scalar::BigInt64:
18564 case Scalar::BigUint64:
18565 default:
18566 MOZ_CRASH("Invalid typed array type")do { do { } while (false); MOZ_ReportCrash("" "Invalid typed array type"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 18566); AnnotateMozCrashReason("MOZ_CRASH(" "Invalid typed array type"
")"); do { *((volatile int*)__null) = 18566; __attribute__((
nomerge)) ::abort(); } while (false); } while (false)
;
18567 }
18568}
18569
18570void CodeGenerator::visitLoadDataViewElement64(LLoadDataViewElement64* lir) {
18571 Register elements = ToRegister(lir->elements());
18572 const LAllocation* littleEndian = lir->littleEndian();
18573 Register64 out = ToOutRegister64(lir);
18574
18575 MOZ_ASSERT(Scalar::isBigIntType(lir->mir()->storageType()))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(Scalar::isBigIntType(lir->mir()->storageType()
))>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(Scalar::isBigIntType(lir->mir()->storageType()
)))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("Scalar::isBigIntType(lir->mir()->storageType())", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 18575); AnnotateMozCrashReason("MOZ_ASSERT" "(" "Scalar::isBigIntType(lir->mir()->storageType())"
")"); do { *((volatile int*)__null) = 18575; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
18576
18577 BaseIndex source(elements, ToRegister(lir->index()), TimesOne);
18578
18579 bool noSwap = littleEndian->isConstant() &&
18580 ToBoolean(littleEndian) == MOZ_LITTLE_ENDIAN()1;
18581
18582 // Load the value into a register.
18583 masm.load64Unaligned(source, out);
18584
18585 if (!noSwap) {
18586 // Swap the bytes in the loaded value.
18587 Label skip;
18588 if (!littleEndian->isConstant()) {
18589 masm.branch32(
18590 MOZ_LITTLE_ENDIAN()1 ? Assembler::NotEqual : Assembler::Equal,
18591 ToRegister(littleEndian), Imm32(0), &skip);
18592 }
18593
18594 masm.byteSwap64(out);
18595
18596 if (skip.used()) {
18597 masm.bind(&skip);
18598 }
18599 }
18600}
18601
18602void CodeGenerator::visitLoadTypedArrayElementHole(
18603 LLoadTypedArrayElementHole* lir) {
18604 Register elements = ToRegister(lir->elements());
18605 Register index = ToRegister(lir->index());
18606 Register length = ToRegister(lir->length());
18607 Register temp = ToTempRegisterOrInvalid(lir->temp0());
18608 const ValueOperand out = ToOutValue(lir);
18609
18610 Register scratch = out.scratchReg();
18611
18612 // Load undefined if index >= length.
18613 Label outOfBounds, done;
18614 masm.spectreBoundsCheckPtr(index, length, scratch, &outOfBounds);
18615
18616 Scalar::Type arrayType = lir->mir()->arrayType();
18617
18618 LiveRegisterSet volatileRegs;
18619 if (MacroAssembler::LoadRequiresCall(arrayType)) {
18620 volatileRegs = liveVolatileRegs(lir);
18621 }
18622
18623 Label fail;
18624 BaseIndex source(elements, index, ScaleFromScalarType(arrayType));
18625 MacroAssembler::Uint32Mode uint32Mode =
18626 lir->mir()->forceDouble() ? MacroAssembler::Uint32Mode::ForceDouble
18627 : MacroAssembler::Uint32Mode::FailOnDouble;
18628 masm.loadFromTypedArray(arrayType, source, out, uint32Mode, temp, &fail,
18629 volatileRegs);
18630 masm.jump(&done);
18631
18632 masm.bind(&outOfBounds);
18633 masm.moveValue(UndefinedValue(), out);
18634
18635 if (fail.used()) {
18636 bailoutFrom(&fail, lir->snapshot());
18637 }
18638
18639 masm.bind(&done);
18640}
18641
18642void CodeGenerator::visitLoadTypedArrayElementHoleBigInt(
18643 LLoadTypedArrayElementHoleBigInt* lir) {
18644 Register elements = ToRegister(lir->elements());
18645 Register index = ToRegister(lir->index());
18646 Register length = ToRegister(lir->length());
18647 const ValueOperand out = ToOutValue(lir);
18648
18649 Register temp = ToRegister(lir->temp());
18650
18651 // On x86 there are not enough registers. In that case reuse the output
18652 // registers as temporaries.
18653#ifdef JS_CODEGEN_X86
18654 MOZ_ASSERT(lir->temp64().isBogusTemp())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(lir->temp64().isBogusTemp())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(lir->temp64().isBogusTemp
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("lir->temp64().isBogusTemp()", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 18654); AnnotateMozCrashReason("MOZ_ASSERT" "(" "lir->temp64().isBogusTemp()"
")"); do { *((volatile int*)__null) = 18654; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
18655 Register64 temp64 = out.toRegister64();
18656#else
18657 Register64 temp64 = ToRegister64(lir->temp64());
18658#endif
18659
18660 // Load undefined if index >= length.
18661 Label outOfBounds, done;
18662 masm.spectreBoundsCheckPtr(index, length, temp, &outOfBounds);
18663
18664 Scalar::Type arrayType = lir->mir()->arrayType();
18665 BaseIndex source(elements, index, ScaleFromScalarType(arrayType));
18666 masm.load64(source, temp64);
18667
18668#ifdef JS_CODEGEN_X86
18669 Register bigInt = temp;
18670 Register maybeTemp = InvalidReg;
18671#else
18672 Register bigInt = out.scratchReg();
18673 Register maybeTemp = temp;
18674#endif
18675 emitCreateBigInt(lir, arrayType, temp64, bigInt, maybeTemp);
18676
18677 masm.tagValue(JSVAL_TYPE_BIGINT, bigInt, out);
18678 masm.jump(&done);
18679
18680 masm.bind(&outOfBounds);
18681 masm.moveValue(UndefinedValue(), out);
18682
18683 masm.bind(&done);
18684}
18685
18686template <SwitchTableType tableType>
18687class OutOfLineSwitch : public OutOfLineCodeBase<CodeGenerator> {
18688 using LabelsVector = Vector<Label, 0, JitAllocPolicy>;
18689 using CodeLabelsVector = Vector<CodeLabel, 0, JitAllocPolicy>;
18690 LabelsVector labels_;
18691 CodeLabelsVector codeLabels_;
18692 CodeLabel start_;
18693 bool isOutOfLine_;
18694
18695 void accept(CodeGenerator* codegen) override {
18696 codegen->visitOutOfLineSwitch(this);
18697 }
18698
18699 public:
18700 explicit OutOfLineSwitch(TempAllocator& alloc)
18701 : labels_(alloc), codeLabels_(alloc), isOutOfLine_(false) {}
18702
18703 CodeLabel* start() { return &start_; }
18704
18705 CodeLabelsVector& codeLabels() { return codeLabels_; }
18706 LabelsVector& labels() { return labels_; }
18707
18708 void jumpToCodeEntries(MacroAssembler& masm, Register index, Register temp) {
18709 Register base;
18710 if (tableType == SwitchTableType::Inline) {
18711#if defined(JS_CODEGEN_ARM)
18712 base = ::js::jit::pc;
18713#else
18714 MOZ_CRASH("NYI: SwitchTableType::Inline")do { do { } while (false); MOZ_ReportCrash("" "NYI: SwitchTableType::Inline"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 18714); AnnotateMozCrashReason("MOZ_CRASH(" "NYI: SwitchTableType::Inline"
")"); do { *((volatile int*)__null) = 18714; __attribute__((
nomerge)) ::abort(); } while (false); } while (false)
;
18715#endif
18716 } else {
18717#if defined(JS_CODEGEN_ARM)
18718 MOZ_CRASH("NYI: SwitchTableType::OutOfLine")do { do { } while (false); MOZ_ReportCrash("" "NYI: SwitchTableType::OutOfLine"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 18718); AnnotateMozCrashReason("MOZ_CRASH(" "NYI: SwitchTableType::OutOfLine"
")"); do { *((volatile int*)__null) = 18718; __attribute__((
nomerge)) ::abort(); } while (false); } while (false)
;
18719#else
18720 masm.mov(start(), temp);
18721 base = temp;
18722#endif
18723 }
18724 BaseIndex jumpTarget(base, index, ScalePointer);
18725 masm.branchToComputedAddress(jumpTarget);
18726 }
18727
18728 // Register an entry in the switch table.
18729 void addTableEntry(MacroAssembler& masm) {
18730 if ((!isOutOfLine_ && tableType == SwitchTableType::Inline) ||
18731 (isOutOfLine_ && tableType == SwitchTableType::OutOfLine)) {
18732 CodeLabel cl;
18733 masm.writeCodePointer(&cl);
18734 masm.propagateOOM(codeLabels_.append(std::move(cl)));
18735 }
18736 }
18737 // Register the code, to which the table will jump to.
18738 void addCodeEntry(MacroAssembler& masm) {
18739 Label entry;
18740 masm.bind(&entry);
18741 masm.propagateOOM(labels_.append(std::move(entry)));
18742 }
18743
18744 void setOutOfLine() { isOutOfLine_ = true; }
18745};
18746
18747template <SwitchTableType tableType>
18748void CodeGenerator::visitOutOfLineSwitch(
18749 OutOfLineSwitch<tableType>* jumpTable) {
18750 jumpTable->setOutOfLine();
18751 auto& labels = jumpTable->labels();
18752
18753 if (tableType == SwitchTableType::OutOfLine) {
18754#if defined(JS_CODEGEN_ARM)
18755 MOZ_CRASH("NYI: SwitchTableType::OutOfLine")do { do { } while (false); MOZ_ReportCrash("" "NYI: SwitchTableType::OutOfLine"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 18755); AnnotateMozCrashReason("MOZ_CRASH(" "NYI: SwitchTableType::OutOfLine"
")"); do { *((volatile int*)__null) = 18755; __attribute__((
nomerge)) ::abort(); } while (false); } while (false)
;
18756#elif defined(JS_CODEGEN_NONE)
18757 MOZ_CRASH()do { do { } while (false); MOZ_ReportCrash("" , "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 18757); AnnotateMozCrashReason("MOZ_CRASH(" ")"); do { *((volatile
int*)__null) = 18757; __attribute__((nomerge)) ::abort(); } while
(false); } while (false)
;
18758#else
18759
18760# if defined(JS_CODEGEN_ARM64)
18761 AutoForbidPoolsAndNops afp(
18762 &masm,
18763 (labels.length() + 1) * (sizeof(void*) / vixl::kInstructionSize));
18764# endif
18765
18766 masm.haltingAlign(sizeof(void*));
18767
18768 // Bind the address of the jump table and reserve the space for code
18769 // pointers to jump in the newly generated code.
18770 masm.bind(jumpTable->start());
18771 masm.addCodeLabel(*jumpTable->start());
18772 for (size_t i = 0, e = labels.length(); i < e; i++) {
18773 jumpTable->addTableEntry(masm);
18774 }
18775#endif
18776 }
18777
18778 // Register all reserved pointers of the jump table to target labels. The
18779 // entries of the jump table need to be absolute addresses and thus must be
18780 // patched after codegen is finished.
18781 auto& codeLabels = jumpTable->codeLabels();
18782 for (size_t i = 0, e = codeLabels.length(); i < e; i++) {
18783 auto& cl = codeLabels[i];
18784 cl.target()->bind(labels[i].offset());
18785 masm.addCodeLabel(cl);
18786 }
18787}
18788
18789template void CodeGenerator::visitOutOfLineSwitch(
18790 OutOfLineSwitch<SwitchTableType::Inline>* jumpTable);
18791template void CodeGenerator::visitOutOfLineSwitch(
18792 OutOfLineSwitch<SwitchTableType::OutOfLine>* jumpTable);
18793
18794template <typename T>
18795static inline void StoreToTypedArray(MacroAssembler& masm,
18796 Scalar::Type writeType,
18797 const LAllocation* value, const T& dest,
18798 Register temp,
18799 LiveRegisterSet volatileRegs) {
18800 if (Scalar::isFloatingType(writeType)) {
18801 masm.storeToTypedFloatArray(writeType, ToFloatRegister(value), dest, temp,
18802 volatileRegs);
18803 } else {
18804 if (value->isConstant()) {
18805 masm.storeToTypedIntArray(writeType, Imm32(ToInt32(value)), dest);
18806 } else {
18807 masm.storeToTypedIntArray(writeType, ToRegister(value), dest);
18808 }
18809 }
18810}
18811
18812void CodeGenerator::visitStoreUnboxedScalar(LStoreUnboxedScalar* lir) {
18813 Register elements = ToRegister(lir->elements());
18814 Register temp = ToTempRegisterOrInvalid(lir->temp0());
18815 const LAllocation* value = lir->value();
18816
18817 const MStoreUnboxedScalar* mir = lir->mir();
18818
18819 Scalar::Type writeType = mir->writeType();
18820
18821 LiveRegisterSet volatileRegs;
18822 if (MacroAssembler::StoreRequiresCall(writeType)) {
18823 volatileRegs = liveVolatileRegs(lir);
18824 }
18825
18826 if (lir->index()->isConstant()) {
18827 Address dest = ToAddress(elements, lir->index(), writeType);
18828 StoreToTypedArray(masm, writeType, value, dest, temp, volatileRegs);
18829 } else {
18830 BaseIndex dest(elements, ToRegister(lir->index()),
18831 ScaleFromScalarType(writeType));
18832 StoreToTypedArray(masm, writeType, value, dest, temp, volatileRegs);
18833 }
18834}
18835
18836void CodeGenerator::visitStoreUnboxedInt64(LStoreUnboxedInt64* lir) {
18837 Register elements = ToRegister(lir->elements());
18838 Register64 value = ToRegister64(lir->value());
18839
18840 Scalar::Type writeType = lir->mir()->writeType();
18841
18842 if (lir->index()->isConstant()) {
18843 Address dest = ToAddress(elements, lir->index(), writeType);
18844 masm.storeToTypedBigIntArray(writeType, value, dest);
18845 } else {
18846 BaseIndex dest(elements, ToRegister(lir->index()),
18847 ScaleFromScalarType(writeType));
18848 masm.storeToTypedBigIntArray(writeType, value, dest);
18849 }
18850}
18851
18852void CodeGenerator::visitStoreDataViewElement(LStoreDataViewElement* lir) {
18853 Register elements = ToRegister(lir->elements());
18854 const LAllocation* value = lir->value();
18855 const LAllocation* littleEndian = lir->littleEndian();
18856 Register temp = ToTempRegisterOrInvalid(lir->temp());
18857 Register64 temp64 = ToTempRegister64OrInvalid(lir->temp64());
18858
18859 const MStoreDataViewElement* mir = lir->mir();
18860 Scalar::Type writeType = mir->writeType();
18861
18862 LiveRegisterSet volatileRegs;
18863 if (MacroAssembler::StoreRequiresCall(writeType)) {
18864 volatileRegs = liveVolatileRegs(lir);
18865 }
18866
18867 BaseIndex dest(elements, ToRegister(lir->index()), TimesOne);
18868
18869 bool noSwap = littleEndian->isConstant() &&
18870 ToBoolean(littleEndian) == MOZ_LITTLE_ENDIAN()1;
18871
18872 // Directly store if no byte swap is needed and the platform supports
18873 // unaligned accesses for the access. (Such support is assumed for integer
18874 // types.)
18875 if (noSwap && (!Scalar::isFloatingType(writeType) ||
18876 MacroAssembler::SupportsFastUnalignedFPAccesses())) {
18877 StoreToTypedArray(masm, writeType, value, dest, temp, volatileRegs);
18878 return;
18879 }
18880
18881 // Load the value into a gpr register.
18882 switch (writeType) {
18883 case Scalar::Int16:
18884 case Scalar::Uint16:
18885 case Scalar::Int32:
18886 case Scalar::Uint32:
18887 if (value->isConstant()) {
18888 masm.move32(Imm32(ToInt32(value)), temp);
18889 } else {
18890 masm.move32(ToRegister(value), temp);
18891 }
18892 break;
18893 case Scalar::Float16: {
18894 FloatRegister fvalue = ToFloatRegister(value);
18895 masm.canonicalizeFloatIfDeterministic(fvalue);
18896 masm.moveFloat16ToGPR(fvalue, temp, volatileRegs);
18897 break;
18898 }
18899 case Scalar::Float32: {
18900 FloatRegister fvalue = ToFloatRegister(value);
18901 masm.canonicalizeFloatIfDeterministic(fvalue);
18902 masm.moveFloat32ToGPR(fvalue, temp);
18903 break;
18904 }
18905 case Scalar::Float64: {
18906 FloatRegister fvalue = ToFloatRegister(value);
18907 masm.canonicalizeDoubleIfDeterministic(fvalue);
18908 masm.moveDoubleToGPR64(fvalue, temp64);
18909 break;
18910 }
18911 case Scalar::Int8:
18912 case Scalar::Uint8:
18913 case Scalar::Uint8Clamped:
18914 case Scalar::BigInt64:
18915 case Scalar::BigUint64:
18916 default:
18917 MOZ_CRASH("Invalid typed array type")do { do { } while (false); MOZ_ReportCrash("" "Invalid typed array type"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 18917); AnnotateMozCrashReason("MOZ_CRASH(" "Invalid typed array type"
")"); do { *((volatile int*)__null) = 18917; __attribute__((
nomerge)) ::abort(); } while (false); } while (false)
;
18918 }
18919
18920 if (!noSwap) {
18921 // Swap the bytes in the loaded value.
18922 Label skip;
18923 if (!littleEndian->isConstant()) {
18924 masm.branch32(
18925 MOZ_LITTLE_ENDIAN()1 ? Assembler::NotEqual : Assembler::Equal,
18926 ToRegister(littleEndian), Imm32(0), &skip);
18927 }
18928
18929 switch (writeType) {
18930 case Scalar::Int16:
18931 masm.byteSwap16SignExtend(temp);
18932 break;
18933 case Scalar::Uint16:
18934 case Scalar::Float16:
18935 masm.byteSwap16ZeroExtend(temp);
18936 break;
18937 case Scalar::Int32:
18938 case Scalar::Uint32:
18939 case Scalar::Float32:
18940 masm.byteSwap32(temp);
18941 break;
18942 case Scalar::Float64:
18943 masm.byteSwap64(temp64);
18944 break;
18945 case Scalar::Int8:
18946 case Scalar::Uint8:
18947 case Scalar::Uint8Clamped:
18948 case Scalar::BigInt64:
18949 case Scalar::BigUint64:
18950 default:
18951 MOZ_CRASH("Invalid typed array type")do { do { } while (false); MOZ_ReportCrash("" "Invalid typed array type"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 18951); AnnotateMozCrashReason("MOZ_CRASH(" "Invalid typed array type"
")"); do { *((volatile int*)__null) = 18951; __attribute__((
nomerge)) ::abort(); } while (false); } while (false)
;
18952 }
18953
18954 if (skip.used()) {
18955 masm.bind(&skip);
18956 }
18957 }
18958
18959 // Store the value into the destination.
18960 switch (writeType) {
18961 case Scalar::Int16:
18962 case Scalar::Uint16:
18963 case Scalar::Float16:
18964 masm.store16Unaligned(temp, dest);
18965 break;
18966 case Scalar::Int32:
18967 case Scalar::Uint32:
18968 case Scalar::Float32:
18969 masm.store32Unaligned(temp, dest);
18970 break;
18971 case Scalar::Float64:
18972 masm.store64Unaligned(temp64, dest);
18973 break;
18974 case Scalar::Int8:
18975 case Scalar::Uint8:
18976 case Scalar::Uint8Clamped:
18977 case Scalar::BigInt64:
18978 case Scalar::BigUint64:
18979 default:
18980 MOZ_CRASH("Invalid typed array type")do { do { } while (false); MOZ_ReportCrash("" "Invalid typed array type"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 18980); AnnotateMozCrashReason("MOZ_CRASH(" "Invalid typed array type"
")"); do { *((volatile int*)__null) = 18980; __attribute__((
nomerge)) ::abort(); } while (false); } while (false)
;
18981 }
18982}
18983
18984void CodeGenerator::visitStoreDataViewElement64(LStoreDataViewElement64* lir) {
18985 Register elements = ToRegister(lir->elements());
18986 Register64 value = ToRegister64(lir->value());
18987 const LAllocation* littleEndian = lir->littleEndian();
18988 Register64 temp = ToTempRegister64OrInvalid(lir->temp());
18989
18990 MOZ_ASSERT(Scalar::isBigIntType(lir->mir()->writeType()))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(Scalar::isBigIntType(lir->mir()->writeType()))
>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(Scalar::isBigIntType(lir->mir()->writeType()))
)), 0))) { do { } while (false); MOZ_ReportAssertionFailure("Scalar::isBigIntType(lir->mir()->writeType())"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 18990); AnnotateMozCrashReason("MOZ_ASSERT" "(" "Scalar::isBigIntType(lir->mir()->writeType())"
")"); do { *((volatile int*)__null) = 18990; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
18991
18992 BaseIndex dest(elements, ToRegister(lir->index()), TimesOne);
18993
18994 bool noSwap = littleEndian->isConstant() &&
18995 ToBoolean(littleEndian) == MOZ_LITTLE_ENDIAN()1;
18996
18997 if (!noSwap) {
18998 // Preserve the input value.
18999 if (temp != Register64::Invalid()) {
19000 masm.move64(value, temp);
19001 value = temp;
19002 } else {
19003 masm.Push(value);
19004 }
19005
19006 // Swap the bytes in the loaded value.
19007 Label skip;
19008 if (!littleEndian->isConstant()) {
19009 masm.branch32(
19010 MOZ_LITTLE_ENDIAN()1 ? Assembler::NotEqual : Assembler::Equal,
19011 ToRegister(littleEndian), Imm32(0), &skip);
19012 }
19013
19014 masm.byteSwap64(value);
19015
19016 if (skip.used()) {
19017 masm.bind(&skip);
19018 }
19019 }
19020
19021 // Store the value into the destination.
19022 masm.store64Unaligned(value, dest);
19023
19024 // Restore |value| if it was modified.
19025 if (!noSwap && temp == Register64::Invalid()) {
19026 masm.Pop(value);
19027 }
19028}
19029
19030void CodeGenerator::visitStoreTypedArrayElementHole(
19031 LStoreTypedArrayElementHole* lir) {
19032 Register elements = ToRegister(lir->elements());
19033 const LAllocation* value = lir->value();
19034
19035 Scalar::Type arrayType = lir->mir()->arrayType();
19036
19037 Register index = ToRegister(lir->index());
19038 const LAllocation* length = lir->length();
19039 Register temp = ToTempRegisterOrInvalid(lir->temp0());
19040
19041 LiveRegisterSet volatileRegs;
19042 if (MacroAssembler::StoreRequiresCall(arrayType)) {
19043 volatileRegs = liveVolatileRegs(lir);
19044 }
19045
19046 Label skip;
19047 if (length->isRegister()) {
19048 masm.spectreBoundsCheckPtr(index, ToRegister(length), temp, &skip);
19049 } else {
19050 masm.spectreBoundsCheckPtr(index, ToAddress(length), temp, &skip);
19051 }
19052
19053 BaseIndex dest(elements, index, ScaleFromScalarType(arrayType));
19054 StoreToTypedArray(masm, arrayType, value, dest, temp, volatileRegs);
19055
19056 masm.bind(&skip);
19057}
19058
19059void CodeGenerator::visitStoreTypedArrayElementHoleInt64(
19060 LStoreTypedArrayElementHoleInt64* lir) {
19061 Register elements = ToRegister(lir->elements());
19062 Register64 value = ToRegister64(lir->value());
19063
19064 Scalar::Type arrayType = lir->mir()->arrayType();
19065
19066 Register index = ToRegister(lir->index());
19067 const LAllocation* length = lir->length();
19068 Register spectreTemp = ToTempRegisterOrInvalid(lir->temp());
19069
19070 Label skip;
19071 if (length->isRegister()) {
19072 masm.spectreBoundsCheckPtr(index, ToRegister(length), spectreTemp, &skip);
19073 } else {
19074 masm.spectreBoundsCheckPtr(index, ToAddress(length), spectreTemp, &skip);
19075 }
19076
19077 BaseIndex dest(elements, index, ScaleFromScalarType(arrayType));
19078 masm.storeToTypedBigIntArray(arrayType, value, dest);
19079
19080 masm.bind(&skip);
19081}
19082
19083void CodeGenerator::visitMemoryBarrier(LMemoryBarrier* ins) {
19084 masm.memoryBarrier(ins->type());
19085}
19086
19087void CodeGenerator::visitAtomicIsLockFree(LAtomicIsLockFree* lir) {
19088 Register value = ToRegister(lir->value());
19089 Register output = ToRegister(lir->output());
19090
19091 masm.atomicIsLockFreeJS(value, output);
19092}
19093
19094void CodeGenerator::visitClampIToUint8(LClampIToUint8* lir) {
19095 Register output = ToRegister(lir->output());
19096 MOZ_ASSERT(output == ToRegister(lir->input()))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(output == ToRegister(lir->input()))>::isValid,
"invalid assertion condition"); if ((__builtin_expect(!!(!(!
!(output == ToRegister(lir->input())))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("output == ToRegister(lir->input())"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 19096); AnnotateMozCrashReason("MOZ_ASSERT" "(" "output == ToRegister(lir->input())"
")"); do { *((volatile int*)__null) = 19096; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
19097 masm.clampIntToUint8(output);
19098}
19099
19100void CodeGenerator::visitClampDToUint8(LClampDToUint8* lir) {
19101 FloatRegister input = ToFloatRegister(lir->input());
19102 Register output = ToRegister(lir->output());
19103 masm.clampDoubleToUint8(input, output);
19104}
19105
19106void CodeGenerator::visitClampVToUint8(LClampVToUint8* lir) {
19107 ValueOperand operand = ToValue(lir, LClampVToUint8::InputIndex);
19108 FloatRegister tempFloat = ToFloatRegister(lir->temp0());
19109 Register output = ToRegister(lir->output());
19110
19111 using Fn = bool (*)(JSContext*, JSString*, double*);
19112 OutOfLineCode* oolString = oolCallVM<Fn, StringToNumber>(
19113 lir, ArgList(output), StoreFloatRegisterTo(tempFloat));
19114 Label* stringEntry = oolString->entry();
19115 Label* stringRejoin = oolString->rejoin();
19116
19117 Label fails;
19118 masm.clampValueToUint8(operand, stringEntry, stringRejoin, output, tempFloat,
19119 output, &fails);
19120
19121 bailoutFrom(&fails, lir->snapshot());
19122}
19123
19124void CodeGenerator::visitInCache(LInCache* ins) {
19125 LiveRegisterSet liveRegs = ins->safepoint()->liveRegs();
19126
19127 ConstantOrRegister key =
19128 toConstantOrRegister(ins, LInCache::LhsIndex, ins->mir()->key()->type());
19129 Register object = ToRegister(ins->rhs());
19130 Register output = ToRegister(ins->output());
19131 Register temp = ToRegister(ins->temp0());
19132
19133 IonInIC cache(liveRegs, key, object, output, temp);
19134 addIC(ins, allocateIC(cache));
19135}
19136
19137void CodeGenerator::visitInArray(LInArray* lir) {
19138 const MInArray* mir = lir->mir();
19139 Register elements = ToRegister(lir->elements());
19140 Register initLength = ToRegister(lir->initLength());
19141 Register output = ToRegister(lir->output());
19142
19143 Label falseBranch, done, trueBranch;
19144
19145 if (lir->index()->isConstant()) {
19146 int32_t index = ToInt32(lir->index());
19147
19148 if (index < 0) {
19149 MOZ_ASSERT(mir->needsNegativeIntCheck())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(mir->needsNegativeIntCheck())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(mir->needsNegativeIntCheck
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("mir->needsNegativeIntCheck()", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 19149); AnnotateMozCrashReason("MOZ_ASSERT" "(" "mir->needsNegativeIntCheck()"
")"); do { *((volatile int*)__null) = 19149; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
19150 bailout(lir->snapshot());
19151 return;
19152 }
19153
19154 masm.branch32(Assembler::BelowOrEqual, initLength, Imm32(index),
19155 &falseBranch);
19156
19157 NativeObject::elementsSizeMustNotOverflow();
19158 Address address = Address(elements, index * sizeof(Value));
19159 masm.branchTestMagic(Assembler::Equal, address, &falseBranch);
19160 } else {
19161 Register index = ToRegister(lir->index());
19162
19163 Label negativeIntCheck;
19164 Label* failedInitLength = &falseBranch;
19165 if (mir->needsNegativeIntCheck()) {
19166 failedInitLength = &negativeIntCheck;
19167 }
19168
19169 masm.branch32(Assembler::BelowOrEqual, initLength, index, failedInitLength);
19170
19171 BaseObjectElementIndex address(elements, index);
19172 masm.branchTestMagic(Assembler::Equal, address, &falseBranch);
19173
19174 if (mir->needsNegativeIntCheck()) {
19175 masm.jump(&trueBranch);
19176 masm.bind(&negativeIntCheck);
19177
19178 bailoutCmp32(Assembler::LessThan, index, Imm32(0), lir->snapshot());
19179
19180 masm.jump(&falseBranch);
19181 }
19182 }
19183
19184 masm.bind(&trueBranch);
19185 masm.move32(Imm32(1), output);
19186 masm.jump(&done);
19187
19188 masm.bind(&falseBranch);
19189 masm.move32(Imm32(0), output);
19190 masm.bind(&done);
19191}
19192
19193void CodeGenerator::visitGuardElementNotHole(LGuardElementNotHole* lir) {
19194 Register elements = ToRegister(lir->elements());
19195 const LAllocation* index = lir->index();
19196
19197 Label testMagic;
19198 if (index->isConstant()) {
19199 Address address(elements, ToInt32(index) * sizeof(js::Value));
19200 masm.branchTestMagic(Assembler::Equal, address, &testMagic);
19201 } else {
19202 BaseObjectElementIndex address(elements, ToRegister(index));
19203 masm.branchTestMagic(Assembler::Equal, address, &testMagic);
19204 }
19205 bailoutFrom(&testMagic, lir->snapshot());
19206}
19207
19208void CodeGenerator::visitInstanceOfO(LInstanceOfO* ins) {
19209 Register protoReg = ToRegister(ins->rhs());
19210 emitInstanceOf(ins, protoReg);
19211}
19212
19213void CodeGenerator::visitInstanceOfV(LInstanceOfV* ins) {
19214 Register protoReg = ToRegister(ins->rhs());
19215 emitInstanceOf(ins, protoReg);
19216}
19217
19218void CodeGenerator::emitInstanceOf(LInstruction* ins, Register protoReg) {
19219 // This path implements fun_hasInstance when the function's prototype is
19220 // known to be the object in protoReg
19221
19222 Label done;
19223 Register output = ToRegister(ins->getDef(0));
19224
19225 // If the lhs is a primitive, the result is false.
19226 Register objReg;
19227 if (ins->isInstanceOfV()) {
19228 Label isObject;
19229 ValueOperand lhsValue = ToValue(ins, LInstanceOfV::LhsIndex);
19230 masm.branchTestObject(Assembler::Equal, lhsValue, &isObject);
19231 masm.mov(ImmWord(0), output);
19232 masm.jump(&done);
19233 masm.bind(&isObject);
19234 objReg = masm.extractObject(lhsValue, output);
19235 } else {
19236 objReg = ToRegister(ins->toInstanceOfO()->lhs());
19237 }
19238
19239 // Crawl the lhs's prototype chain in a loop to search for prototypeObject.
19240 // This follows the main loop of js::IsPrototypeOf, though additionally breaks
19241 // out of the loop on Proxy::LazyProto.
19242
19243 // Load the lhs's prototype.
19244 masm.loadObjProto(objReg, output);
19245
19246 Label testLazy;
19247 {
19248 Label loopPrototypeChain;
19249 masm.bind(&loopPrototypeChain);
19250
19251 // Test for the target prototype object.
19252 Label notPrototypeObject;
19253 masm.branchPtr(Assembler::NotEqual, output, protoReg, &notPrototypeObject);
19254 masm.mov(ImmWord(1), output);
19255 masm.jump(&done);
19256 masm.bind(&notPrototypeObject);
19257
19258 MOZ_ASSERT(uintptr_t(TaggedProto::LazyProto) == 1)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(uintptr_t(TaggedProto::LazyProto) == 1)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(uintptr_t(TaggedProto::LazyProto) == 1))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("uintptr_t(TaggedProto::LazyProto) == 1"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 19258); AnnotateMozCrashReason("MOZ_ASSERT" "(" "uintptr_t(TaggedProto::LazyProto) == 1"
")"); do { *((volatile int*)__null) = 19258; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
19259
19260 // Test for nullptr or Proxy::LazyProto
19261 masm.branchPtr(Assembler::BelowOrEqual, output, ImmWord(1), &testLazy);
19262
19263 // Load the current object's prototype.
19264 masm.loadObjProto(output, output);
19265
19266 masm.jump(&loopPrototypeChain);
19267 }
19268
19269 // Make a VM call if an object with a lazy proto was found on the prototype
19270 // chain. This currently occurs only for cross compartment wrappers, which
19271 // we do not expect to be compared with non-wrapper functions from this
19272 // compartment. Otherwise, we stopped on a nullptr prototype and the output
19273 // register is already correct.
19274
19275 using Fn = bool (*)(JSContext*, HandleObject, JSObject*, bool*);
19276 auto* ool = oolCallVM<Fn, IsPrototypeOf>(ins, ArgList(protoReg, objReg),
19277 StoreRegisterTo(output));
19278
19279 // Regenerate the original lhs object for the VM call.
19280 Label regenerate, *lazyEntry;
19281 if (objReg != output) {
19282 lazyEntry = ool->entry();
19283 } else {
19284 masm.bind(&regenerate);
19285 lazyEntry = &regenerate;
19286 if (ins->isInstanceOfV()) {
19287 ValueOperand lhsValue = ToValue(ins, LInstanceOfV::LhsIndex);
19288 objReg = masm.extractObject(lhsValue, output);
19289 } else {
19290 objReg = ToRegister(ins->toInstanceOfO()->lhs());
19291 }
19292 MOZ_ASSERT(objReg == output)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(objReg == output)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(objReg == output))), 0))) { do
{ } while (false); MOZ_ReportAssertionFailure("objReg == output"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 19292); AnnotateMozCrashReason("MOZ_ASSERT" "(" "objReg == output"
")"); do { *((volatile int*)__null) = 19292; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
19293 masm.jump(ool->entry());
19294 }
19295
19296 masm.bind(&testLazy);
19297 masm.branchPtr(Assembler::Equal, output, ImmWord(1), lazyEntry);
19298
19299 masm.bind(&done);
19300 masm.bind(ool->rejoin());
19301}
19302
19303void CodeGenerator::visitInstanceOfCache(LInstanceOfCache* ins) {
19304 // The Lowering ensures that RHS is an object, and that LHS is a value.
19305 LiveRegisterSet liveRegs = ins->safepoint()->liveRegs();
19306 TypedOrValueRegister lhs =
19307 TypedOrValueRegister(ToValue(ins, LInstanceOfCache::LHS));
19308 Register rhs = ToRegister(ins->rhs());
19309 Register output = ToRegister(ins->output());
19310
19311 IonInstanceOfIC ic(liveRegs, lhs, rhs, output);
19312 addIC(ins, allocateIC(ic));
19313}
19314
19315void CodeGenerator::visitGetDOMProperty(LGetDOMProperty* ins) {
19316 const Register JSContextReg = ToRegister(ins->getJSContextReg());
19317 const Register ObjectReg = ToRegister(ins->getObjectReg());
19318 const Register PrivateReg = ToRegister(ins->getPrivReg());
19319 const Register ValueReg = ToRegister(ins->getValueReg());
19320
19321 Label haveValue;
19322 if (ins->mir()->valueMayBeInSlot()) {
19323 size_t slot = ins->mir()->domMemberSlotIndex();
19324 // It's a bit annoying to redo these slot calculations, which duplcate
19325 // LSlots and a few other things like that, but I'm not sure there's a
19326 // way to reuse those here.
19327 //
19328 // If this ever gets fixed to work with proxies (by not assuming that
19329 // reserved slot indices, which is what domMemberSlotIndex() returns,
19330 // match fixed slot indices), we can reenable MGetDOMProperty for
19331 // proxies in IonBuilder.
19332 if (slot < NativeObject::MAX_FIXED_SLOTS) {
19333 masm.loadValue(Address(ObjectReg, NativeObject::getFixedSlotOffset(slot)),
19334 JSReturnOperand);
19335 } else {
19336 // It's a dynamic slot.
19337 slot -= NativeObject::MAX_FIXED_SLOTS;
19338 // Use PrivateReg as a scratch register for the slots pointer.
19339 masm.loadPtr(Address(ObjectReg, NativeObject::offsetOfSlots()),
19340 PrivateReg);
19341 masm.loadValue(Address(PrivateReg, slot * sizeof(js::Value)),
19342 JSReturnOperand);
19343 }
19344 masm.branchTestUndefined(Assembler::NotEqual, JSReturnOperand, &haveValue);
19345 }
19346
19347 DebugOnly<uint32_t> initialStack = masm.framePushed();
19348
19349 masm.checkStackAlignment();
19350
19351 // Make space for the outparam. Pre-initialize it to UndefinedValue so we
19352 // can trace it at GC time.
19353 masm.Push(UndefinedValue());
19354 // We pass the pointer to our out param as an instance of
19355 // JSJitGetterCallArgs, since on the binary level it's the same thing.
19356 static_assert(sizeof(JSJitGetterCallArgs) == sizeof(Value*));
19357 masm.moveStackPtrTo(ValueReg);
19358
19359 masm.Push(ObjectReg);
19360
19361 LoadDOMPrivate(masm, ObjectReg, PrivateReg, ins->mir()->objectKind());
19362
19363 // Rooting will happen at GC time.
19364 masm.moveStackPtrTo(ObjectReg);
19365
19366 Realm* getterRealm = ins->mir()->getterRealm();
19367 if (gen->realm->realmPtr() != getterRealm) {
19368 // We use JSContextReg as scratch register here.
19369 masm.switchToRealm(getterRealm, JSContextReg);
19370 }
19371
19372 uint32_t safepointOffset = masm.buildFakeExitFrame(JSContextReg);
19373 masm.loadJSContext(JSContextReg);
19374 masm.enterFakeExitFrame(JSContextReg, JSContextReg,
19375 ExitFrameType::IonDOMGetter);
19376
19377 markSafepointAt(safepointOffset, ins);
19378
19379 masm.setupAlignedABICall();
19380 masm.loadJSContext(JSContextReg);
19381 masm.passABIArg(JSContextReg);
19382 masm.passABIArg(ObjectReg);
19383 masm.passABIArg(PrivateReg);
19384 masm.passABIArg(ValueReg);
19385 ensureOsiSpace();
19386 masm.callWithABI(DynamicFunction<JSJitGetterOp>(ins->mir()->fun()),
19387 ABIType::General,
19388 CheckUnsafeCallWithABI::DontCheckHasExitFrame);
19389
19390 if (ins->mir()->isInfallible()) {
19391 masm.loadValue(Address(masm.getStackPointer(),
19392 IonDOMExitFrameLayout::offsetOfResult()),
19393 JSReturnOperand);
19394 } else {
19395 masm.branchIfFalseBool(ReturnReg, masm.exceptionLabel());
19396
19397 masm.loadValue(Address(masm.getStackPointer(),
19398 IonDOMExitFrameLayout::offsetOfResult()),
19399 JSReturnOperand);
19400 }
19401
19402 // Switch back to the current realm if needed. Note: if the getter threw an
19403 // exception, the exception handler will do this.
19404 if (gen->realm->realmPtr() != getterRealm) {
19405 static_assert(!JSReturnOperand.aliases(ReturnReg),
19406 "Clobbering ReturnReg should not affect the return value");
19407 masm.switchToRealm(gen->realm->realmPtr(), ReturnReg);
19408 }
19409
19410 // Until C++ code is instrumented against Spectre, prevent speculative
19411 // execution from returning any private data.
19412 if (JitOptions.spectreJitToCxxCalls && ins->mir()->hasLiveDefUses()) {
19413 masm.speculationBarrier();
19414 }
19415
19416 masm.adjustStack(IonDOMExitFrameLayout::Size());
19417
19418 masm.bind(&haveValue);
19419
19420 MOZ_ASSERT(masm.framePushed() == initialStack)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(masm.framePushed() == initialStack)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(masm.framePushed() == initialStack
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"masm.framePushed() == initialStack", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 19420); AnnotateMozCrashReason("MOZ_ASSERT" "(" "masm.framePushed() == initialStack"
")"); do { *((volatile int*)__null) = 19420; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
19421}
19422
19423void CodeGenerator::visitGetDOMMemberV(LGetDOMMemberV* ins) {
19424 // It's simpler to duplicate visitLoadFixedSlotV here than it is to try to
19425 // use an LLoadFixedSlotV or some subclass of it for this case: that would
19426 // require us to have MGetDOMMember inherit from MLoadFixedSlot, and then
19427 // we'd have to duplicate a bunch of stuff we now get for free from
19428 // MGetDOMProperty.
19429 //
19430 // If this ever gets fixed to work with proxies (by not assuming that
19431 // reserved slot indices, which is what domMemberSlotIndex() returns,
19432 // match fixed slot indices), we can reenable MGetDOMMember for
19433 // proxies in IonBuilder.
19434 Register object = ToRegister(ins->object());
19435 size_t slot = ins->mir()->domMemberSlotIndex();
19436 ValueOperand result = ToOutValue(ins);
19437
19438 masm.loadValue(Address(object, NativeObject::getFixedSlotOffset(slot)),
19439 result);
19440}
19441
19442void CodeGenerator::visitGetDOMMemberT(LGetDOMMemberT* ins) {
19443 // It's simpler to duplicate visitLoadFixedSlotT here than it is to try to
19444 // use an LLoadFixedSlotT or some subclass of it for this case: that would
19445 // require us to have MGetDOMMember inherit from MLoadFixedSlot, and then
19446 // we'd have to duplicate a bunch of stuff we now get for free from
19447 // MGetDOMProperty.
19448 //
19449 // If this ever gets fixed to work with proxies (by not assuming that
19450 // reserved slot indices, which is what domMemberSlotIndex() returns,
19451 // match fixed slot indices), we can reenable MGetDOMMember for
19452 // proxies in IonBuilder.
19453 Register object = ToRegister(ins->object());
19454 size_t slot = ins->mir()->domMemberSlotIndex();
19455 AnyRegister result = ToAnyRegister(ins->getDef(0));
19456 MIRType type = ins->mir()->type();
19457
19458 masm.loadUnboxedValue(Address(object, NativeObject::getFixedSlotOffset(slot)),
19459 type, result);
19460}
19461
19462void CodeGenerator::visitSetDOMProperty(LSetDOMProperty* ins) {
19463 const Register JSContextReg = ToRegister(ins->getJSContextReg());
19464 const Register ObjectReg = ToRegister(ins->getObjectReg());
19465 const Register PrivateReg = ToRegister(ins->getPrivReg());
19466 const Register ValueReg = ToRegister(ins->getValueReg());
19467
19468 DebugOnly<uint32_t> initialStack = masm.framePushed();
19469
19470 masm.checkStackAlignment();
19471
19472 // Push the argument. Rooting will happen at GC time.
19473 ValueOperand argVal = ToValue(ins, LSetDOMProperty::Value);
19474 masm.Push(argVal);
19475 // We pass the pointer to our out param as an instance of
19476 // JSJitGetterCallArgs, since on the binary level it's the same thing.
19477 static_assert(sizeof(JSJitSetterCallArgs) == sizeof(Value*));
19478 masm.moveStackPtrTo(ValueReg);
19479
19480 masm.Push(ObjectReg);
19481
19482 LoadDOMPrivate(masm, ObjectReg, PrivateReg, ins->mir()->objectKind());
19483
19484 // Rooting will happen at GC time.
19485 masm.moveStackPtrTo(ObjectReg);
19486
19487 Realm* setterRealm = ins->mir()->setterRealm();
19488 if (gen->realm->realmPtr() != setterRealm) {
19489 // We use JSContextReg as scratch register here.
19490 masm.switchToRealm(setterRealm, JSContextReg);
19491 }
19492
19493 uint32_t safepointOffset = masm.buildFakeExitFrame(JSContextReg);
19494 masm.loadJSContext(JSContextReg);
19495 masm.enterFakeExitFrame(JSContextReg, JSContextReg,
19496 ExitFrameType::IonDOMSetter);
19497
19498 markSafepointAt(safepointOffset, ins);
19499
19500 masm.setupAlignedABICall();
19501 masm.loadJSContext(JSContextReg);
19502 masm.passABIArg(JSContextReg);
19503 masm.passABIArg(ObjectReg);
19504 masm.passABIArg(PrivateReg);
19505 masm.passABIArg(ValueReg);
19506 ensureOsiSpace();
19507 masm.callWithABI(DynamicFunction<JSJitSetterOp>(ins->mir()->fun()),
19508 ABIType::General,
19509 CheckUnsafeCallWithABI::DontCheckHasExitFrame);
19510
19511 masm.branchIfFalseBool(ReturnReg, masm.exceptionLabel());
19512
19513 // Switch back to the current realm if needed. Note: if the setter threw an
19514 // exception, the exception handler will do this.
19515 if (gen->realm->realmPtr() != setterRealm) {
19516 masm.switchToRealm(gen->realm->realmPtr(), ReturnReg);
19517 }
19518
19519 masm.adjustStack(IonDOMExitFrameLayout::Size());
19520
19521 MOZ_ASSERT(masm.framePushed() == initialStack)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(masm.framePushed() == initialStack)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(masm.framePushed() == initialStack
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"masm.framePushed() == initialStack", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 19521); AnnotateMozCrashReason("MOZ_ASSERT" "(" "masm.framePushed() == initialStack"
")"); do { *((volatile int*)__null) = 19521; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
19522}
19523
19524void CodeGenerator::visitLoadDOMExpandoValue(LLoadDOMExpandoValue* ins) {
19525 Register proxy = ToRegister(ins->proxy());
19526 ValueOperand out = ToOutValue(ins);
19527
19528 masm.loadPtr(Address(proxy, ProxyObject::offsetOfReservedSlots()),
19529 out.scratchReg());
19530 masm.loadValue(Address(out.scratchReg(),
19531 js::detail::ProxyReservedSlots::offsetOfPrivateSlot()),
19532 out);
19533}
19534
19535void CodeGenerator::visitLoadDOMExpandoValueGuardGeneration(
19536 LLoadDOMExpandoValueGuardGeneration* ins) {
19537 Register proxy = ToRegister(ins->proxy());
19538 ValueOperand out = ToOutValue(ins);
19539
19540 Label bail;
19541 masm.loadDOMExpandoValueGuardGeneration(proxy, out,
19542 ins->mir()->expandoAndGeneration(),
19543 ins->mir()->generation(), &bail);
19544 bailoutFrom(&bail, ins->snapshot());
19545}
19546
19547void CodeGenerator::visitLoadDOMExpandoValueIgnoreGeneration(
19548 LLoadDOMExpandoValueIgnoreGeneration* ins) {
19549 Register proxy = ToRegister(ins->proxy());
19550 ValueOperand out = ToOutValue(ins);
19551
19552 masm.loadPtr(Address(proxy, ProxyObject::offsetOfReservedSlots()),
19553 out.scratchReg());
19554
19555 // Load the ExpandoAndGeneration* from the PrivateValue.
19556 masm.loadPrivate(
19557 Address(out.scratchReg(),
19558 js::detail::ProxyReservedSlots::offsetOfPrivateSlot()),
19559 out.scratchReg());
19560
19561 // Load expandoAndGeneration->expando into the output Value register.
19562 masm.loadValue(
19563 Address(out.scratchReg(), ExpandoAndGeneration::offsetOfExpando()), out);
19564}
19565
19566void CodeGenerator::visitGuardDOMExpandoMissingOrGuardShape(
19567 LGuardDOMExpandoMissingOrGuardShape* ins) {
19568 Register temp = ToRegister(ins->temp0());
19569 ValueOperand input =
19570 ToValue(ins, LGuardDOMExpandoMissingOrGuardShape::InputIndex);
19571
19572 Label done;
19573 masm.branchTestUndefined(Assembler::Equal, input, &done);
19574
19575 masm.debugAssertIsObject(input);
19576 masm.unboxObject(input, temp);
19577 // The expando object is not used in this case, so we don't need Spectre
19578 // mitigations.
19579 Label bail;
19580 masm.branchTestObjShapeNoSpectreMitigations(Assembler::NotEqual, temp,
19581 ins->mir()->shape(), &bail);
19582 bailoutFrom(&bail, ins->snapshot());
19583
19584 masm.bind(&done);
19585}
19586
19587class OutOfLineIsCallable : public OutOfLineCodeBase<CodeGenerator> {
19588 Register object_;
19589 Register output_;
19590
19591 public:
19592 OutOfLineIsCallable(Register object, Register output)
19593 : object_(object), output_(output) {}
19594
19595 void accept(CodeGenerator* codegen) override {
19596 codegen->visitOutOfLineIsCallable(this);
19597 }
19598 Register object() const { return object_; }
19599 Register output() const { return output_; }
19600};
19601
19602void CodeGenerator::visitIsCallableO(LIsCallableO* ins) {
19603 Register object = ToRegister(ins->object());
19604 Register output = ToRegister(ins->output());
19605
19606 OutOfLineIsCallable* ool = new (alloc()) OutOfLineIsCallable(object, output);
19607 addOutOfLineCode(ool, ins->mir());
19608
19609 masm.isCallable(object, output, ool->entry());
19610
19611 masm.bind(ool->rejoin());
19612}
19613
19614void CodeGenerator::visitIsCallableV(LIsCallableV* ins) {
19615 ValueOperand val = ToValue(ins, LIsCallableV::ObjectIndex);
19616 Register output = ToRegister(ins->output());
19617 Register temp = ToRegister(ins->temp0());
19618
19619 Label notObject;
19620 masm.fallibleUnboxObject(val, temp, &notObject);
19621
19622 OutOfLineIsCallable* ool = new (alloc()) OutOfLineIsCallable(temp, output);
19623 addOutOfLineCode(ool, ins->mir());
19624
19625 masm.isCallable(temp, output, ool->entry());
19626 masm.jump(ool->rejoin());
19627
19628 masm.bind(&notObject);
19629 masm.move32(Imm32(0), output);
19630
19631 masm.bind(ool->rejoin());
19632}
19633
19634void CodeGenerator::visitOutOfLineIsCallable(OutOfLineIsCallable* ool) {
19635 Register object = ool->object();
19636 Register output = ool->output();
19637
19638 saveVolatile(output);
19639 using Fn = bool (*)(JSObject* obj);
19640 masm.setupAlignedABICall();
19641 masm.passABIArg(object);
19642 masm.callWithABI<Fn, ObjectIsCallable>();
19643 masm.storeCallBoolResult(output);
19644 restoreVolatile(output);
19645 masm.jump(ool->rejoin());
19646}
19647
19648class OutOfLineIsConstructor : public OutOfLineCodeBase<CodeGenerator> {
19649 LIsConstructor* ins_;
19650
19651 public:
19652 explicit OutOfLineIsConstructor(LIsConstructor* ins) : ins_(ins) {}
19653
19654 void accept(CodeGenerator* codegen) override {
19655 codegen->visitOutOfLineIsConstructor(this);
19656 }
19657 LIsConstructor* ins() const { return ins_; }
19658};
19659
19660void CodeGenerator::visitIsConstructor(LIsConstructor* ins) {
19661 Register object = ToRegister(ins->object());
19662 Register output = ToRegister(ins->output());
19663
19664 OutOfLineIsConstructor* ool = new (alloc()) OutOfLineIsConstructor(ins);
19665 addOutOfLineCode(ool, ins->mir());
19666
19667 masm.isConstructor(object, output, ool->entry());
19668
19669 masm.bind(ool->rejoin());
19670}
19671
19672void CodeGenerator::visitOutOfLineIsConstructor(OutOfLineIsConstructor* ool) {
19673 LIsConstructor* ins = ool->ins();
19674 Register object = ToRegister(ins->object());
19675 Register output = ToRegister(ins->output());
19676
19677 saveVolatile(output);
19678 using Fn = bool (*)(JSObject* obj);
19679 masm.setupAlignedABICall();
19680 masm.passABIArg(object);
19681 masm.callWithABI<Fn, ObjectIsConstructor>();
19682 masm.storeCallBoolResult(output);
19683 restoreVolatile(output);
19684 masm.jump(ool->rejoin());
19685}
19686
19687void CodeGenerator::visitIsCrossRealmArrayConstructor(
19688 LIsCrossRealmArrayConstructor* ins) {
19689 Register object = ToRegister(ins->object());
19690 Register output = ToRegister(ins->output());
19691
19692 masm.setIsCrossRealmArrayConstructor(object, output);
19693}
19694
19695static void EmitObjectIsArray(MacroAssembler& masm, OutOfLineCode* ool,
19696 Register obj, Register output,
19697 Label* notArray = nullptr) {
19698 masm.loadObjClassUnsafe(obj, output);
19699
19700 Label isArray;
19701 masm.branchPtr(Assembler::Equal, output, ImmPtr(&ArrayObject::class_),
19702 &isArray);
19703
19704 // Branch to OOL path if it's a proxy.
19705 masm.branchTestClassIsProxy(true, output, ool->entry());
19706
19707 if (notArray) {
19708 masm.bind(notArray);
19709 }
19710 masm.move32(Imm32(0), output);
19711 masm.jump(ool->rejoin());
19712
19713 masm.bind(&isArray);
19714 masm.move32(Imm32(1), output);
19715
19716 masm.bind(ool->rejoin());
19717}
19718
19719void CodeGenerator::visitIsArrayO(LIsArrayO* lir) {
19720 Register object = ToRegister(lir->object());
19721 Register output = ToRegister(lir->output());
19722
19723 using Fn = bool (*)(JSContext*, HandleObject, bool*);
19724 OutOfLineCode* ool = oolCallVM<Fn, js::IsArrayFromJit>(
19725 lir, ArgList(object), StoreRegisterTo(output));
19726 EmitObjectIsArray(masm, ool, object, output);
19727}
19728
19729void CodeGenerator::visitIsArrayV(LIsArrayV* lir) {
19730 ValueOperand val = ToValue(lir, LIsArrayV::ValueIndex);
19731 Register output = ToRegister(lir->output());
19732 Register temp = ToRegister(lir->temp0());
19733
19734 Label notArray;
19735 masm.fallibleUnboxObject(val, temp, &notArray);
19736
19737 using Fn = bool (*)(JSContext*, HandleObject, bool*);
19738 OutOfLineCode* ool = oolCallVM<Fn, js::IsArrayFromJit>(
19739 lir, ArgList(temp), StoreRegisterTo(output));
19740 EmitObjectIsArray(masm, ool, temp, output, &notArray);
19741}
19742
19743void CodeGenerator::visitIsTypedArray(LIsTypedArray* lir) {
19744 Register object = ToRegister(lir->object());
19745 Register output = ToRegister(lir->output());
19746
19747 OutOfLineCode* ool = nullptr;
19748 if (lir->mir()->isPossiblyWrapped()) {
19749 using Fn = bool (*)(JSContext*, JSObject*, bool*);
19750 ool = oolCallVM<Fn, jit::IsPossiblyWrappedTypedArray>(
19751 lir, ArgList(object), StoreRegisterTo(output));
19752 }
19753
19754 Label notTypedArray;
19755 Label done;
19756
19757 masm.loadObjClassUnsafe(object, output);
19758 masm.branchIfClassIsNotTypedArray(output, &notTypedArray);
19759
19760 masm.move32(Imm32(1), output);
19761 masm.jump(&done);
19762 masm.bind(&notTypedArray);
19763 if (ool) {
19764 masm.branchTestClassIsProxy(true, output, ool->entry());
19765 }
19766 masm.move32(Imm32(0), output);
19767 masm.bind(&done);
19768 if (ool) {
19769 masm.bind(ool->rejoin());
19770 }
19771}
19772
19773void CodeGenerator::visitIsObject(LIsObject* ins) {
19774 Register output = ToRegister(ins->output());
19775 ValueOperand value = ToValue(ins, LIsObject::ObjectIndex);
19776 masm.testObjectSet(Assembler::Equal, value, output);
19777}
19778
19779void CodeGenerator::visitIsObjectAndBranch(LIsObjectAndBranch* ins) {
19780 ValueOperand value = ToValue(ins, LIsObjectAndBranch::Input);
19781
19782 MBasicBlock* ifTrue = ins->ifTrue();
19783 MBasicBlock* ifFalse = ins->ifFalse();
19784
19785 if (isNextBlock(ifFalse->lir())) {
19786 masm.branchTestObject(Assembler::Equal, value,
19787 getJumpLabelForBranch(ifTrue));
19788 } else {
19789 masm.branchTestObject(Assembler::NotEqual, value,
19790 getJumpLabelForBranch(ifFalse));
19791 jumpToBlock(ifTrue);
19792 }
19793}
19794
19795void CodeGenerator::visitIsNullOrUndefined(LIsNullOrUndefined* ins) {
19796 Register output = ToRegister(ins->output());
19797 ValueOperand value = ToValue(ins, LIsNullOrUndefined::InputIndex);
19798
19799 Label isNotNull, done;
19800 masm.branchTestNull(Assembler::NotEqual, value, &isNotNull);
19801
19802 masm.move32(Imm32(1), output);
19803 masm.jump(&done);
19804
19805 masm.bind(&isNotNull);
19806 masm.testUndefinedSet(Assembler::Equal, value, output);
19807
19808 masm.bind(&done);
19809}
19810
19811void CodeGenerator::visitIsNullOrUndefinedAndBranch(
19812 LIsNullOrUndefinedAndBranch* ins) {
19813 Label* ifTrue = getJumpLabelForBranch(ins->ifTrue());
19814 Label* ifFalse = getJumpLabelForBranch(ins->ifFalse());
19815 ValueOperand value = ToValue(ins, LIsNullOrUndefinedAndBranch::Input);
19816
19817 ScratchTagScope tag(masm, value);
19818 masm.splitTagForTest(value, tag);
19819
19820 masm.branchTestNull(Assembler::Equal, tag, ifTrue);
19821 masm.branchTestUndefined(Assembler::Equal, tag, ifTrue);
19822
19823 if (!isNextBlock(ins->ifFalse()->lir())) {
19824 masm.jump(ifFalse);
19825 }
19826}
19827
19828void CodeGenerator::loadOutermostJSScript(Register reg) {
19829 // The "outermost" JSScript means the script that we are compiling
19830 // basically; this is not always the script associated with the
19831 // current basic block, which might be an inlined script.
19832
19833 MIRGraph& graph = current->mir()->graph();
19834 MBasicBlock* entryBlock = graph.entryBlock();
19835 masm.movePtr(ImmGCPtr(entryBlock->info().script()), reg);
19836}
19837
19838void CodeGenerator::loadJSScriptForBlock(MBasicBlock* block, Register reg) {
19839 // The current JSScript means the script for the current
19840 // basic block. This may be an inlined script.
19841
19842 JSScript* script = block->info().script();
19843 masm.movePtr(ImmGCPtr(script), reg);
19844}
19845
19846void CodeGenerator::visitHasClass(LHasClass* ins) {
19847 Register lhs = ToRegister(ins->lhs());
19848 Register output = ToRegister(ins->output());
19849
19850 masm.loadObjClassUnsafe(lhs, output);
19851 masm.cmpPtrSet(Assembler::Equal, output, ImmPtr(ins->mir()->getClass()),
19852 output);
19853}
19854
19855void CodeGenerator::visitGuardToClass(LGuardToClass* ins) {
19856 Register lhs = ToRegister(ins->lhs());
19857 Register temp = ToRegister(ins->temp0());
19858
19859 // branchTestObjClass may zero the object register on speculative paths
19860 // (we should have a defineReuseInput allocation in this case).
19861 Register spectreRegToZero = lhs;
19862
19863 Label notEqual;
19864
19865 masm.branchTestObjClass(Assembler::NotEqual, lhs, ins->mir()->getClass(),
19866 temp, spectreRegToZero, &notEqual);
19867
19868 // Can't return null-return here, so bail.
19869 bailoutFrom(&notEqual, ins->snapshot());
19870}
19871
19872void CodeGenerator::visitGuardToEitherClass(LGuardToEitherClass* ins) {
19873 Register lhs = ToRegister(ins->lhs());
19874 Register temp = ToRegister(ins->temp0());
19875
19876 // branchTestObjClass may zero the object register on speculative paths
19877 // (we should have a defineReuseInput allocation in this case).
19878 Register spectreRegToZero = lhs;
19879
19880 Label notEqual;
19881
19882 masm.branchTestObjClass(Assembler::NotEqual, lhs,
19883 {ins->mir()->getClass1(), ins->mir()->getClass2()},
19884 temp, spectreRegToZero, &notEqual);
19885
19886 // Can't return null-return here, so bail.
19887 bailoutFrom(&notEqual, ins->snapshot());
19888}
19889
19890void CodeGenerator::visitGuardToFunction(LGuardToFunction* ins) {
19891 Register lhs = ToRegister(ins->lhs());
19892 Register temp = ToRegister(ins->temp0());
19893
19894 // branchTestObjClass may zero the object register on speculative paths
19895 // (we should have a defineReuseInput allocation in this case).
19896 Register spectreRegToZero = lhs;
19897
19898 Label notEqual;
19899
19900 masm.branchTestObjIsFunction(Assembler::NotEqual, lhs, temp, spectreRegToZero,
19901 &notEqual);
19902
19903 // Can't return null-return here, so bail.
19904 bailoutFrom(&notEqual, ins->snapshot());
19905}
19906
19907void CodeGenerator::visitObjectClassToString(LObjectClassToString* lir) {
19908 Register obj = ToRegister(lir->lhs());
19909 Register temp = ToRegister(lir->temp0());
19910
19911 using Fn = JSString* (*)(JSContext*, JSObject*);
19912 masm.setupAlignedABICall();
19913 masm.loadJSContext(temp);
19914 masm.passABIArg(temp);
19915 masm.passABIArg(obj);
19916 masm.callWithABI<Fn, js::ObjectClassToString>();
19917
19918 bailoutCmpPtr(Assembler::Equal, ReturnReg, ImmWord(0), lir->snapshot());
19919}
19920
19921void CodeGenerator::visitWasmParameter(LWasmParameter* lir) {}
19922
19923void CodeGenerator::visitWasmParameterI64(LWasmParameterI64* lir) {}
19924
19925void CodeGenerator::visitWasmReturn(LWasmReturn* lir) {
19926 // Don't emit a jump to the return label if this is the last block.
19927 if (current->mir() != *gen->graph().poBegin()) {
19928 masm.jump(&returnLabel_);
19929 }
19930}
19931
19932void CodeGenerator::visitWasmReturnI64(LWasmReturnI64* lir) {
19933 // Don't emit a jump to the return label if this is the last block.
19934 if (current->mir() != *gen->graph().poBegin()) {
19935 masm.jump(&returnLabel_);
19936 }
19937}
19938
19939void CodeGenerator::visitWasmReturnVoid(LWasmReturnVoid* lir) {
19940 // Don't emit a jump to the return label if this is the last block.
19941 if (current->mir() != *gen->graph().poBegin()) {
19942 masm.jump(&returnLabel_);
19943 }
19944}
19945
19946void CodeGenerator::emitAssertRangeI(MIRType type, const Range* r,
19947 Register input) {
19948 // Check the lower bound.
19949 if (r->hasInt32LowerBound() && r->lower() > INT32_MIN(-2147483647-1)) {
19950 Label success;
19951 if (type == MIRType::Int32 || type == MIRType::Boolean) {
19952 masm.branch32(Assembler::GreaterThanOrEqual, input, Imm32(r->lower()),
19953 &success);
19954 } else {
19955 MOZ_ASSERT(type == MIRType::IntPtr)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(type == MIRType::IntPtr)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(type == MIRType::IntPtr))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("type == MIRType::IntPtr"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 19955); AnnotateMozCrashReason("MOZ_ASSERT" "(" "type == MIRType::IntPtr"
")"); do { *((volatile int*)__null) = 19955; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
19956 masm.branchPtr(Assembler::GreaterThanOrEqual, input, Imm32(r->lower()),
19957 &success);
19958 }
19959 masm.assumeUnreachable(
19960 "Integer input should be equal or higher than Lowerbound.");
19961 masm.bind(&success);
19962 }
19963
19964 // Check the upper bound.
19965 if (r->hasInt32UpperBound() && r->upper() < INT32_MAX(2147483647)) {
19966 Label success;
19967 if (type == MIRType::Int32 || type == MIRType::Boolean) {
19968 masm.branch32(Assembler::LessThanOrEqual, input, Imm32(r->upper()),
19969 &success);
19970 } else {
19971 MOZ_ASSERT(type == MIRType::IntPtr)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(type == MIRType::IntPtr)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(type == MIRType::IntPtr))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("type == MIRType::IntPtr"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 19971); AnnotateMozCrashReason("MOZ_ASSERT" "(" "type == MIRType::IntPtr"
")"); do { *((volatile int*)__null) = 19971; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
19972 masm.branchPtr(Assembler::LessThanOrEqual, input, Imm32(r->upper()),
19973 &success);
19974 }
19975 masm.assumeUnreachable(
19976 "Integer input should be lower or equal than Upperbound.");
19977 masm.bind(&success);
19978 }
19979
19980 // For r->canHaveFractionalPart(), r->canBeNegativeZero(), and
19981 // r->exponent(), there's nothing to check, because if we ended up in the
19982 // integer range checking code, the value is already in an integer register
19983 // in the integer range.
19984}
19985
19986void CodeGenerator::emitAssertRangeD(const Range* r, FloatRegister input,
19987 FloatRegister temp) {
19988 // Check the lower bound.
19989 if (r->hasInt32LowerBound()) {
19990 Label success;
19991 masm.loadConstantDouble(r->lower(), temp);
19992 if (r->canBeNaN()) {
19993 masm.branchDouble(Assembler::DoubleUnordered, input, input, &success);
19994 }
19995 masm.branchDouble(Assembler::DoubleGreaterThanOrEqual, input, temp,
19996 &success);
19997 masm.assumeUnreachable(
19998 "Double input should be equal or higher than Lowerbound.");
19999 masm.bind(&success);
20000 }
20001 // Check the upper bound.
20002 if (r->hasInt32UpperBound()) {
20003 Label success;
20004 masm.loadConstantDouble(r->upper(), temp);
20005 if (r->canBeNaN()) {
20006 masm.branchDouble(Assembler::DoubleUnordered, input, input, &success);
20007 }
20008 masm.branchDouble(Assembler::DoubleLessThanOrEqual, input, temp, &success);
20009 masm.assumeUnreachable(
20010 "Double input should be lower or equal than Upperbound.");
20011 masm.bind(&success);
20012 }
20013
20014 // This code does not yet check r->canHaveFractionalPart(). This would require
20015 // new assembler interfaces to make rounding instructions available.
20016
20017 if (!r->canBeNegativeZero()) {
20018 Label success;
20019
20020 // First, test for being equal to 0.0, which also includes -0.0.
20021 masm.loadConstantDouble(0.0, temp);
20022 masm.branchDouble(Assembler::DoubleNotEqualOrUnordered, input, temp,
20023 &success);
20024
20025 // The easiest way to distinguish -0.0 from 0.0 is that 1.0/-0.0 is
20026 // -Infinity instead of Infinity.
20027 masm.loadConstantDouble(1.0, temp);
20028 masm.divDouble(input, temp);
20029 masm.branchDouble(Assembler::DoubleGreaterThan, temp, input, &success);
20030
20031 masm.assumeUnreachable("Input shouldn't be negative zero.");
20032
20033 masm.bind(&success);
20034 }
20035
20036 if (!r->hasInt32Bounds() && !r->canBeInfiniteOrNaN() &&
20037 r->exponent() < FloatingPoint<double>::kExponentBias) {
20038 // Check the bounds implied by the maximum exponent.
20039 Label exponentLoOk;
20040 masm.loadConstantDouble(pow(2.0, r->exponent() + 1), temp);
20041 masm.branchDouble(Assembler::DoubleUnordered, input, input, &exponentLoOk);
20042 masm.branchDouble(Assembler::DoubleLessThanOrEqual, input, temp,
20043 &exponentLoOk);
20044 masm.assumeUnreachable("Check for exponent failed.");
20045 masm.bind(&exponentLoOk);
20046
20047 Label exponentHiOk;
20048 masm.loadConstantDouble(-pow(2.0, r->exponent() + 1), temp);
20049 masm.branchDouble(Assembler::DoubleUnordered, input, input, &exponentHiOk);
20050 masm.branchDouble(Assembler::DoubleGreaterThanOrEqual, input, temp,
20051 &exponentHiOk);
20052 masm.assumeUnreachable("Check for exponent failed.");
20053 masm.bind(&exponentHiOk);
20054 } else if (!r->hasInt32Bounds() && !r->canBeNaN()) {
20055 // If we think the value can't be NaN, check that it isn't.
20056 Label notnan;
20057 masm.branchDouble(Assembler::DoubleOrdered, input, input, &notnan);
20058 masm.assumeUnreachable("Input shouldn't be NaN.");
20059 masm.bind(&notnan);
20060
20061 // If we think the value also can't be an infinity, check that it isn't.
20062 if (!r->canBeInfiniteOrNaN()) {
20063 Label notposinf;
20064 masm.loadConstantDouble(PositiveInfinity<double>(), temp);
20065 masm.branchDouble(Assembler::DoubleLessThan, input, temp, &notposinf);
20066 masm.assumeUnreachable("Input shouldn't be +Inf.");
20067 masm.bind(&notposinf);
20068
20069 Label notneginf;
20070 masm.loadConstantDouble(NegativeInfinity<double>(), temp);
20071 masm.branchDouble(Assembler::DoubleGreaterThan, input, temp, &notneginf);
20072 masm.assumeUnreachable("Input shouldn't be -Inf.");
20073 masm.bind(&notneginf);
20074 }
20075 }
20076}
20077
20078void CodeGenerator::visitAssertClass(LAssertClass* ins) {
20079 Register obj = ToRegister(ins->input());
20080 Register temp = ToRegister(ins->getTemp(0));
20081
20082 Label success;
20083 if (ins->mir()->getClass() == &FunctionClass) {
20084 // Allow both possible function classes here.
20085 masm.branchTestObjIsFunctionNoSpectreMitigations(Assembler::Equal, obj,
20086 temp, &success);
20087 } else {
20088 masm.branchTestObjClassNoSpectreMitigations(
20089 Assembler::Equal, obj, ins->mir()->getClass(), temp, &success);
20090 }
20091 masm.assumeUnreachable("Wrong KnownClass during run-time");
20092 masm.bind(&success);
20093}
20094
20095void CodeGenerator::visitAssertShape(LAssertShape* ins) {
20096 Register obj = ToRegister(ins->input());
20097
20098 Label success;
20099 masm.branchTestObjShapeNoSpectreMitigations(Assembler::Equal, obj,
20100 ins->mir()->shape(), &success);
20101 masm.assumeUnreachable("Wrong Shape during run-time");
20102 masm.bind(&success);
20103}
20104
20105void CodeGenerator::visitAssertRangeI(LAssertRangeI* ins) {
20106 Register input = ToRegister(ins->input());
20107 const Range* r = ins->range();
20108
20109 emitAssertRangeI(ins->mir()->input()->type(), r, input);
20110}
20111
20112void CodeGenerator::visitAssertRangeD(LAssertRangeD* ins) {
20113 FloatRegister input = ToFloatRegister(ins->input());
20114 FloatRegister temp = ToFloatRegister(ins->temp());
20115 const Range* r = ins->range();
20116
20117 emitAssertRangeD(r, input, temp);
20118}
20119
20120void CodeGenerator::visitAssertRangeF(LAssertRangeF* ins) {
20121 FloatRegister input = ToFloatRegister(ins->input());
20122 FloatRegister temp = ToFloatRegister(ins->temp());
20123 FloatRegister temp2 = ToFloatRegister(ins->temp2());
20124
20125 const Range* r = ins->range();
20126
20127 masm.convertFloat32ToDouble(input, temp);
20128 emitAssertRangeD(r, temp, temp2);
20129}
20130
20131void CodeGenerator::visitAssertRangeV(LAssertRangeV* ins) {
20132 const Range* r = ins->range();
20133 const ValueOperand value = ToValue(ins, LAssertRangeV::Input);
20134 Label done;
20135
20136 {
20137 ScratchTagScope tag(masm, value);
20138 masm.splitTagForTest(value, tag);
20139
20140 {
20141 Label isNotInt32;
20142 masm.branchTestInt32(Assembler::NotEqual, tag, &isNotInt32);
20143 {
20144 ScratchTagScopeRelease _(&tag);
20145 Register unboxInt32 = ToTempUnboxRegister(ins->temp());
20146 Register input = masm.extractInt32(value, unboxInt32);
20147 emitAssertRangeI(MIRType::Int32, r, input);
20148 masm.jump(&done);
20149 }
20150 masm.bind(&isNotInt32);
20151 }
20152
20153 {
20154 Label isNotDouble;
20155 masm.branchTestDouble(Assembler::NotEqual, tag, &isNotDouble);
20156 {
20157 ScratchTagScopeRelease _(&tag);
20158 FloatRegister input = ToFloatRegister(ins->floatTemp1());
20159 FloatRegister temp = ToFloatRegister(ins->floatTemp2());
20160 masm.unboxDouble(value, input);
20161 emitAssertRangeD(r, input, temp);
20162 masm.jump(&done);
20163 }
20164 masm.bind(&isNotDouble);
20165 }
20166 }
20167
20168 masm.assumeUnreachable("Incorrect range for Value.");
20169 masm.bind(&done);
20170}
20171
20172void CodeGenerator::visitInterruptCheck(LInterruptCheck* lir) {
20173 using Fn = bool (*)(JSContext*);
20174 OutOfLineCode* ool =
20175 oolCallVM<Fn, InterruptCheck>(lir, ArgList(), StoreNothing());
20176
20177 const void* interruptAddr = gen->runtime->addressOfInterruptBits();
20178 masm.branch32(Assembler::NotEqual, AbsoluteAddress(interruptAddr), Imm32(0),
20179 ool->entry());
20180 masm.bind(ool->rejoin());
20181}
20182
20183void CodeGenerator::visitOutOfLineResumableWasmTrap(
20184 OutOfLineResumableWasmTrap* ool) {
20185 LInstruction* lir = ool->lir();
20186 masm.wasmTrap(ool->trap(), ool->bytecodeOffset());
20187
20188 markSafepointAt(masm.currentOffset(), lir);
20189
20190 // Note that masm.framePushed() doesn't include the register dump area.
20191 // That will be taken into account when the StackMap is created from the
20192 // LSafepoint.
20193 lir->safepoint()->setFramePushedAtStackMapBase(ool->framePushed());
20194 lir->safepoint()->setWasmSafepointKind(WasmSafepointKind::Trap);
20195
20196 masm.jump(ool->rejoin());
20197}
20198
20199void CodeGenerator::visitOutOfLineAbortingWasmTrap(
20200 OutOfLineAbortingWasmTrap* ool) {
20201 masm.wasmTrap(ool->trap(), ool->bytecodeOffset());
20202}
20203
20204void CodeGenerator::visitWasmInterruptCheck(LWasmInterruptCheck* lir) {
20205 MOZ_ASSERT(gen->compilingWasm())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(gen->compilingWasm())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(gen->compilingWasm()))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("gen->compilingWasm()"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 20205); AnnotateMozCrashReason("MOZ_ASSERT" "(" "gen->compilingWasm()"
")"); do { *((volatile int*)__null) = 20205; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
20206
20207 OutOfLineResumableWasmTrap* ool = new (alloc()) OutOfLineResumableWasmTrap(
20208 lir, masm.framePushed(), lir->mir()->bytecodeOffset(),
20209 wasm::Trap::CheckInterrupt);
20210 addOutOfLineCode(ool, lir->mir());
20211 masm.branch32(
20212 Assembler::NotEqual,
20213 Address(ToRegister(lir->instance()), wasm::Instance::offsetOfInterrupt()),
20214 Imm32(0), ool->entry());
20215 masm.bind(ool->rejoin());
20216}
20217
20218void CodeGenerator::visitWasmTrap(LWasmTrap* lir) {
20219 MOZ_ASSERT(gen->compilingWasm())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(gen->compilingWasm())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(gen->compilingWasm()))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("gen->compilingWasm()"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 20219); AnnotateMozCrashReason("MOZ_ASSERT" "(" "gen->compilingWasm()"
")"); do { *((volatile int*)__null) = 20219; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
20220 const MWasmTrap* mir = lir->mir();
20221
20222 masm.wasmTrap(mir->trap(), mir->bytecodeOffset());
20223}
20224
20225void CodeGenerator::visitWasmTrapIfNull(LWasmTrapIfNull* lir) {
20226 MOZ_ASSERT(gen->compilingWasm())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(gen->compilingWasm())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(gen->compilingWasm()))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("gen->compilingWasm()"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 20226); AnnotateMozCrashReason("MOZ_ASSERT" "(" "gen->compilingWasm()"
")"); do { *((volatile int*)__null) = 20226; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
20227 const MWasmTrapIfNull* mir = lir->mir();
20228 Label nonNull;
20229 Register ref = ToRegister(lir->ref());
20230
20231 masm.branchWasmAnyRefIsNull(false, ref, &nonNull);
20232 masm.wasmTrap(mir->trap(), mir->bytecodeOffset());
20233 masm.bind(&nonNull);
20234}
20235
20236void CodeGenerator::visitWasmRefIsSubtypeOfAbstract(
20237 LWasmRefIsSubtypeOfAbstract* ins) {
20238 MOZ_ASSERT(gen->compilingWasm())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(gen->compilingWasm())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(gen->compilingWasm()))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("gen->compilingWasm()"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 20238); AnnotateMozCrashReason("MOZ_ASSERT" "(" "gen->compilingWasm()"
")"); do { *((volatile int*)__null) = 20238; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
20239
20240 const MWasmRefIsSubtypeOfAbstract* mir = ins->mir();
20241 MOZ_ASSERT(!mir->destType().isTypeRef())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!mir->destType().isTypeRef())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!mir->destType().isTypeRef
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("!mir->destType().isTypeRef()", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 20241); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!mir->destType().isTypeRef()"
")"); do { *((volatile int*)__null) = 20241; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
20242
20243 Register ref = ToRegister(ins->ref());
20244 Register superSTV = Register::Invalid();
20245 Register scratch1 = ToTempRegisterOrInvalid(ins->temp0());
20246 Register scratch2 = Register::Invalid();
20247 Register result = ToRegister(ins->output());
20248 Label onSuccess;
20249 Label onFail;
20250 Label join;
20251 masm.branchWasmRefIsSubtype(ref, mir->sourceType(), mir->destType(),
20252 &onSuccess, /*onSuccess=*/true, superSTV,
20253 scratch1, scratch2);
20254 masm.bind(&onFail);
20255 masm.xor32(result, result);
20256 masm.jump(&join);
20257 masm.bind(&onSuccess);
20258 masm.move32(Imm32(1), result);
20259 masm.bind(&join);
20260}
20261
20262void CodeGenerator::visitWasmRefIsSubtypeOfConcrete(
20263 LWasmRefIsSubtypeOfConcrete* ins) {
20264 MOZ_ASSERT(gen->compilingWasm())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(gen->compilingWasm())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(gen->compilingWasm()))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("gen->compilingWasm()"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 20264); AnnotateMozCrashReason("MOZ_ASSERT" "(" "gen->compilingWasm()"
")"); do { *((volatile int*)__null) = 20264; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
20265
20266 const MWasmRefIsSubtypeOfConcrete* mir = ins->mir();
20267 MOZ_ASSERT(mir->destType().isTypeRef())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(mir->destType().isTypeRef())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(mir->destType().isTypeRef
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("mir->destType().isTypeRef()", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 20267); AnnotateMozCrashReason("MOZ_ASSERT" "(" "mir->destType().isTypeRef()"
")"); do { *((volatile int*)__null) = 20267; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
20268
20269 Register ref = ToRegister(ins->ref());
20270 Register superSTV = ToRegister(ins->superSTV());
20271 Register scratch1 = ToRegister(ins->temp0());
20272 Register scratch2 = ToTempRegisterOrInvalid(ins->temp1());
20273 Register result = ToRegister(ins->output());
20274 Label onSuccess;
20275 Label join;
20276 masm.branchWasmRefIsSubtype(ref, mir->sourceType(), mir->destType(),
20277 &onSuccess, /*onSuccess=*/true, superSTV,
20278 scratch1, scratch2);
20279 masm.move32(Imm32(0), result);
20280 masm.jump(&join);
20281 masm.bind(&onSuccess);
20282 masm.move32(Imm32(1), result);
20283 masm.bind(&join);
20284}
20285
20286void CodeGenerator::visitWasmRefIsSubtypeOfAbstractAndBranch(
20287 LWasmRefIsSubtypeOfAbstractAndBranch* ins) {
20288 MOZ_ASSERT(gen->compilingWasm())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(gen->compilingWasm())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(gen->compilingWasm()))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("gen->compilingWasm()"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 20288); AnnotateMozCrashReason("MOZ_ASSERT" "(" "gen->compilingWasm()"
")"); do { *((volatile int*)__null) = 20288; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
20289 Register ref = ToRegister(ins->ref());
20290 Register scratch1 = ToTempRegisterOrInvalid(ins->temp0());
20291 Label* onSuccess = getJumpLabelForBranch(ins->ifTrue());
20292 Label* onFail = getJumpLabelForBranch(ins->ifFalse());
20293 masm.branchWasmRefIsSubtype(
20294 ref, ins->sourceType(), ins->destType(), onSuccess, /*onSuccess=*/true,
20295 Register::Invalid(), scratch1, Register::Invalid());
20296 masm.jump(onFail);
20297}
20298
20299void CodeGenerator::visitWasmRefIsSubtypeOfConcreteAndBranch(
20300 LWasmRefIsSubtypeOfConcreteAndBranch* ins) {
20301 MOZ_ASSERT(gen->compilingWasm())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(gen->compilingWasm())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(gen->compilingWasm()))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("gen->compilingWasm()"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 20301); AnnotateMozCrashReason("MOZ_ASSERT" "(" "gen->compilingWasm()"
")"); do { *((volatile int*)__null) = 20301; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
20302 Register ref = ToRegister(ins->ref());
20303 Register superSTV = ToRegister(ins->superSTV());
20304 Register scratch1 = ToRegister(ins->temp0());
20305 Register scratch2 = ToTempRegisterOrInvalid(ins->temp1());
20306 Label* onSuccess = getJumpLabelForBranch(ins->ifTrue());
20307 Label* onFail = getJumpLabelForBranch(ins->ifFalse());
20308 masm.branchWasmRefIsSubtype(ref, ins->sourceType(), ins->destType(),
20309 onSuccess, /*onSuccess=*/true, superSTV, scratch1,
20310 scratch2);
20311 masm.jump(onFail);
20312}
20313
20314void CodeGenerator::callWasmStructAllocFun(LInstruction* lir,
20315 wasm::SymbolicAddress fun,
20316 Register typeDefData,
20317 Register output) {
20318 masm.Push(InstanceReg);
20319 int32_t framePushedAfterInstance = masm.framePushed();
20320 saveLive(lir);
20321
20322 masm.setupWasmABICall();
20323 masm.passABIArg(InstanceReg);
20324 masm.passABIArg(typeDefData);
20325 int32_t instanceOffset = masm.framePushed() - framePushedAfterInstance;
20326 CodeOffset offset =
20327 masm.callWithABI(wasm::BytecodeOffset(0), fun,
20328 mozilla::Some(instanceOffset), ABIType::General);
20329 masm.storeCallPointerResult(output);
20330
20331 markSafepointAt(offset.offset(), lir);
20332 lir->safepoint()->setFramePushedAtStackMapBase(framePushedAfterInstance);
20333 lir->safepoint()->setWasmSafepointKind(WasmSafepointKind::CodegenCall);
20334
20335 restoreLive(lir);
20336 masm.Pop(InstanceReg);
20337#if JS_CODEGEN_ARM64
20338 masm.syncStackPtr();
20339#endif
20340}
20341
20342// Out-of-line path to allocate wasm GC structs
20343class OutOfLineWasmNewStruct : public OutOfLineCodeBase<CodeGenerator> {
20344 LInstruction* lir_;
20345 wasm::SymbolicAddress fun_;
20346 Register typeDefData_;
20347 Register output_;
20348
20349 public:
20350 OutOfLineWasmNewStruct(LInstruction* lir, wasm::SymbolicAddress fun,
20351 Register typeDefData, Register output)
20352 : lir_(lir), fun_(fun), typeDefData_(typeDefData), output_(output) {}
20353
20354 void accept(CodeGenerator* codegen) override {
20355 codegen->visitOutOfLineWasmNewStruct(this);
20356 }
20357
20358 LInstruction* lir() const { return lir_; }
20359 wasm::SymbolicAddress fun() const { return fun_; }
20360 Register typeDefData() const { return typeDefData_; }
20361 Register output() const { return output_; }
20362};
20363
20364void CodeGenerator::visitOutOfLineWasmNewStruct(OutOfLineWasmNewStruct* ool) {
20365 callWasmStructAllocFun(ool->lir(), ool->fun(), ool->typeDefData(),
20366 ool->output());
20367 masm.jump(ool->rejoin());
20368}
20369
20370void CodeGenerator::visitWasmNewStructObject(LWasmNewStructObject* lir) {
20371 MOZ_ASSERT(gen->compilingWasm())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(gen->compilingWasm())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(gen->compilingWasm()))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("gen->compilingWasm()"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 20371); AnnotateMozCrashReason("MOZ_ASSERT" "(" "gen->compilingWasm()"
")"); do { *((volatile int*)__null) = 20371; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
20372
20373 MWasmNewStructObject* mir = lir->mir();
20374
20375 Register typeDefData = ToRegister(lir->typeDefData());
20376 Register output = ToRegister(lir->output());
20377
20378 if (mir->isOutline()) {
20379 wasm::SymbolicAddress fun = mir->zeroFields()
20380 ? wasm::SymbolicAddress::StructNewOOL_true
20381 : wasm::SymbolicAddress::StructNewOOL_false;
20382 callWasmStructAllocFun(lir, fun, typeDefData, output);
20383 } else {
20384 wasm::SymbolicAddress fun = mir->zeroFields()
20385 ? wasm::SymbolicAddress::StructNewIL_true
20386 : wasm::SymbolicAddress::StructNewIL_false;
20387
20388 Register instance = ToRegister(lir->instance());
20389 MOZ_ASSERT(instance == InstanceReg)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(instance == InstanceReg)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(instance == InstanceReg))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("instance == InstanceReg"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 20389); AnnotateMozCrashReason("MOZ_ASSERT" "(" "instance == InstanceReg"
")"); do { *((volatile int*)__null) = 20389; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
20390
20391 auto ool =
20392 new (alloc()) OutOfLineWasmNewStruct(lir, fun, typeDefData, output);
20393 addOutOfLineCode(ool, lir->mir());
20394
20395 Register temp1 = ToRegister(lir->temp0());
20396 Register temp2 = ToRegister(lir->temp1());
20397 masm.wasmNewStructObject(instance, output, typeDefData, temp1, temp2,
20398 ool->entry(), mir->allocKind(), mir->zeroFields());
20399
20400 masm.bind(ool->rejoin());
20401 }
20402}
20403
20404void CodeGenerator::callWasmArrayAllocFun(LInstruction* lir,
20405 wasm::SymbolicAddress fun,
20406 Register numElements,
20407 Register typeDefData, Register output,
20408 wasm::BytecodeOffset bytecodeOffset) {
20409 masm.Push(InstanceReg);
20410 int32_t framePushedAfterInstance = masm.framePushed();
20411 saveLive(lir);
20412
20413 masm.setupWasmABICall();
20414 masm.passABIArg(InstanceReg);
20415 masm.passABIArg(numElements);
20416 masm.passABIArg(typeDefData);
20417 int32_t instanceOffset = masm.framePushed() - framePushedAfterInstance;
20418 CodeOffset offset = masm.callWithABI(
20419 bytecodeOffset, fun, mozilla::Some(instanceOffset), ABIType::General);
20420 masm.storeCallPointerResult(output);
20421
20422 markSafepointAt(offset.offset(), lir);
20423 lir->safepoint()->setFramePushedAtStackMapBase(framePushedAfterInstance);
20424 lir->safepoint()->setWasmSafepointKind(WasmSafepointKind::CodegenCall);
20425
20426 restoreLive(lir);
20427 masm.Pop(InstanceReg);
20428#if JS_CODEGEN_ARM64
20429 masm.syncStackPtr();
20430#endif
20431
20432 Label ok;
20433 masm.branchPtr(Assembler::NonZero, output, ImmWord(0), &ok);
20434 masm.wasmTrap(wasm::Trap::ThrowReported, bytecodeOffset);
20435 masm.bind(&ok);
20436}
20437
20438// Out-of-line path to allocate wasm GC arrays
20439class OutOfLineWasmNewArray : public OutOfLineCodeBase<CodeGenerator> {
20440 LInstruction* lir_;
20441 wasm::SymbolicAddress fun_;
20442 Register numElementsReg_;
20443 mozilla::Maybe<uint32_t> numElements_;
20444 Register typeDefData_;
20445 Register output_;
20446 wasm::BytecodeOffset bytecodeOffset_;
20447
20448 public:
20449 OutOfLineWasmNewArray(LInstruction* lir, wasm::SymbolicAddress fun,
20450 Register numElementsReg,
20451 mozilla::Maybe<uint32_t> numElements,
20452 Register typeDefData, Register output,
20453 wasm::BytecodeOffset bytecodeOffset)
20454 : lir_(lir),
20455 fun_(fun),
20456 numElementsReg_(numElementsReg),
20457 numElements_(numElements),
20458 typeDefData_(typeDefData),
20459 output_(output),
20460 bytecodeOffset_(bytecodeOffset) {}
20461
20462 void accept(CodeGenerator* codegen) override {
20463 codegen->visitOutOfLineWasmNewArray(this);
20464 }
20465
20466 LInstruction* lir() const { return lir_; }
20467 wasm::SymbolicAddress fun() const { return fun_; }
20468 Register numElementsReg() const { return numElementsReg_; }
20469 mozilla::Maybe<uint32_t> numElements() const { return numElements_; }
20470 Register typeDefData() const { return typeDefData_; }
20471 Register output() const { return output_; }
20472 wasm::BytecodeOffset bytecodeOffset() const { return bytecodeOffset_; }
20473};
20474
20475void CodeGenerator::visitOutOfLineWasmNewArray(OutOfLineWasmNewArray* ool) {
20476 if (ool->numElements().isSome()) {
20477 masm.move32(Imm32(ool->numElements().value()), ool->numElementsReg());
20478 }
20479 callWasmArrayAllocFun(ool->lir(), ool->fun(), ool->numElementsReg(),
20480 ool->typeDefData(), ool->output(),
20481 ool->bytecodeOffset());
20482 masm.jump(ool->rejoin());
20483}
20484
20485void CodeGenerator::visitWasmNewArrayObject(LWasmNewArrayObject* lir) {
20486 MOZ_ASSERT(gen->compilingWasm())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(gen->compilingWasm())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(gen->compilingWasm()))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("gen->compilingWasm()"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 20486); AnnotateMozCrashReason("MOZ_ASSERT" "(" "gen->compilingWasm()"
")"); do { *((volatile int*)__null) = 20486; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
20487
20488 MWasmNewArrayObject* mir = lir->mir();
20489
20490 Register typeDefData = ToRegister(lir->typeDefData());
20491 Register output = ToRegister(lir->output());
20492 Register temp1 = ToRegister(lir->temp0());
20493 Register temp2 = ToRegister(lir->temp1());
20494
20495 wasm::SymbolicAddress fun = mir->zeroFields()
20496 ? wasm::SymbolicAddress::ArrayNew_true
20497 : wasm::SymbolicAddress::ArrayNew_false;
20498
20499 if (lir->numElements()->isConstant()) {
20500 // numElements is constant, so we can do optimized code generation.
20501 uint32_t numElements = lir->numElements()->toConstant()->toInt32();
20502 CheckedUint32 storageBytes =
20503 WasmArrayObject::calcStorageBytesChecked(mir->elemSize(), numElements);
20504 if (!storageBytes.isValid() ||
20505 storageBytes.value() > WasmArrayObject_MaxInlineBytes) {
20506 // Too much array data to store inline. Immediately perform an instance
20507 // call to handle the out-of-line storage (or the trap).
20508 masm.move32(Imm32(numElements), temp1);
20509 callWasmArrayAllocFun(lir, fun, temp1, typeDefData, output,
20510 mir->bytecodeOffset());
20511 } else {
20512 // storageBytes is small enough to be stored inline in WasmArrayObject.
20513 // Attempt a nursery allocation and fall back to an instance call if it
20514 // fails.
20515 Register instance = ToRegister(lir->instance());
20516 MOZ_ASSERT(instance == InstanceReg)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(instance == InstanceReg)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(instance == InstanceReg))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("instance == InstanceReg"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 20516); AnnotateMozCrashReason("MOZ_ASSERT" "(" "instance == InstanceReg"
")"); do { *((volatile int*)__null) = 20516; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
20517
20518 auto ool = new (alloc())
20519 OutOfLineWasmNewArray(lir, fun, temp1, mozilla::Some(numElements),
20520 typeDefData, output, mir->bytecodeOffset());
20521 addOutOfLineCode(ool, lir->mir());
20522
20523 masm.wasmNewArrayObjectFixed(instance, output, typeDefData, temp1, temp2,
20524 ool->entry(), numElements,
20525 storageBytes.value(), mir->zeroFields());
20526
20527 masm.bind(ool->rejoin());
20528 }
20529 } else {
20530 // numElements is dynamic. Attempt a dynamic inline-storage nursery
20531 // allocation and fall back to an instance call if it fails.
20532 Register instance = ToRegister(lir->instance());
20533 MOZ_ASSERT(instance == InstanceReg)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(instance == InstanceReg)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(instance == InstanceReg))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("instance == InstanceReg"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 20533); AnnotateMozCrashReason("MOZ_ASSERT" "(" "instance == InstanceReg"
")"); do { *((volatile int*)__null) = 20533; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
20534 Register numElements = ToRegister(lir->numElements());
20535
20536 auto ool = new (alloc())
20537 OutOfLineWasmNewArray(lir, fun, numElements, mozilla::Nothing(),
20538 typeDefData, output, mir->bytecodeOffset());
20539 addOutOfLineCode(ool, lir->mir());
20540
20541 masm.wasmNewArrayObject(instance, output, numElements, typeDefData, temp1,
20542 ool->entry(), mir->elemSize(), mir->zeroFields());
20543
20544 masm.bind(ool->rejoin());
20545 }
20546}
20547
20548void CodeGenerator::visitWasmHeapReg(LWasmHeapReg* ins) {
20549#ifdef WASM_HAS_HEAPREG1
20550 masm.movePtr(HeapReg, ToRegister(ins->output()));
20551#else
20552 MOZ_CRASH()do { do { } while (false); MOZ_ReportCrash("" , "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 20552); AnnotateMozCrashReason("MOZ_CRASH(" ")"); do { *((volatile
int*)__null) = 20552; __attribute__((nomerge)) ::abort(); } while
(false); } while (false)
;
20553#endif
20554}
20555
20556void CodeGenerator::visitWasmBoundsCheck(LWasmBoundsCheck* ins) {
20557 const MWasmBoundsCheck* mir = ins->mir();
20558 Register ptr = ToRegister(ins->ptr());
20559 Register boundsCheckLimit = ToRegister(ins->boundsCheckLimit());
20560 // When there are no spectre mitigations in place, branching out-of-line to
20561 // the trap is a big performance win, but with mitigations it's trickier. See
20562 // bug 1680243.
20563 if (JitOptions.spectreIndexMasking) {
20564 Label ok;
20565 masm.wasmBoundsCheck32(Assembler::Below, ptr, boundsCheckLimit, &ok);
20566 masm.wasmTrap(wasm::Trap::OutOfBounds, mir->bytecodeOffset());
20567 masm.bind(&ok);
20568 } else {
20569 OutOfLineAbortingWasmTrap* ool = new (alloc()) OutOfLineAbortingWasmTrap(
20570 mir->bytecodeOffset(), wasm::Trap::OutOfBounds);
20571 addOutOfLineCode(ool, mir);
20572 masm.wasmBoundsCheck32(Assembler::AboveOrEqual, ptr, boundsCheckLimit,
20573 ool->entry());
20574 }
20575}
20576
20577void CodeGenerator::visitWasmBoundsCheck64(LWasmBoundsCheck64* ins) {
20578 const MWasmBoundsCheck* mir = ins->mir();
20579 Register64 ptr = ToRegister64(ins->ptr());
20580 Register64 boundsCheckLimit = ToRegister64(ins->boundsCheckLimit());
20581 // See above.
20582 if (JitOptions.spectreIndexMasking) {
20583 Label ok;
20584 masm.wasmBoundsCheck64(Assembler::Below, ptr, boundsCheckLimit, &ok);
20585 masm.wasmTrap(wasm::Trap::OutOfBounds, mir->bytecodeOffset());
20586 masm.bind(&ok);
20587 } else {
20588 OutOfLineAbortingWasmTrap* ool = new (alloc()) OutOfLineAbortingWasmTrap(
20589 mir->bytecodeOffset(), wasm::Trap::OutOfBounds);
20590 addOutOfLineCode(ool, mir);
20591 masm.wasmBoundsCheck64(Assembler::AboveOrEqual, ptr, boundsCheckLimit,
20592 ool->entry());
20593 }
20594}
20595
20596void CodeGenerator::visitWasmBoundsCheckRange32(LWasmBoundsCheckRange32* ins) {
20597 const MWasmBoundsCheckRange32* mir = ins->mir();
20598 Register index = ToRegister(ins->index());
20599 Register length = ToRegister(ins->length());
20600 Register limit = ToRegister(ins->limit());
20601 Register tmp = ToRegister(ins->temp0());
20602
20603 masm.wasmBoundsCheckRange32(index, length, limit, tmp, mir->bytecodeOffset());
20604}
20605
20606void CodeGenerator::visitWasmAlignmentCheck(LWasmAlignmentCheck* ins) {
20607 const MWasmAlignmentCheck* mir = ins->mir();
20608 Register ptr = ToRegister(ins->ptr());
20609 OutOfLineAbortingWasmTrap* ool = new (alloc()) OutOfLineAbortingWasmTrap(
20610 mir->bytecodeOffset(), wasm::Trap::UnalignedAccess);
20611 addOutOfLineCode(ool, mir);
20612 masm.branchTest32(Assembler::NonZero, ptr, Imm32(mir->byteSize() - 1),
20613 ool->entry());
20614}
20615
20616void CodeGenerator::visitWasmAlignmentCheck64(LWasmAlignmentCheck64* ins) {
20617 const MWasmAlignmentCheck* mir = ins->mir();
20618 Register64 ptr = ToRegister64(ins->ptr());
20619#ifdef JS_64BIT1
20620 Register r = ptr.reg;
20621#else
20622 Register r = ptr.low;
20623#endif
20624 OutOfLineAbortingWasmTrap* ool = new (alloc()) OutOfLineAbortingWasmTrap(
20625 mir->bytecodeOffset(), wasm::Trap::UnalignedAccess);
20626 addOutOfLineCode(ool, mir);
20627 masm.branchTestPtr(Assembler::NonZero, r, Imm32(mir->byteSize() - 1),
20628 ool->entry());
20629}
20630
20631void CodeGenerator::visitWasmLoadInstance(LWasmLoadInstance* ins) {
20632 switch (ins->mir()->type()) {
20633 case MIRType::WasmAnyRef:
20634 case MIRType::Pointer:
20635 masm.loadPtr(Address(ToRegister(ins->instance()), ins->mir()->offset()),
20636 ToRegister(ins->output()));
20637 break;
20638 case MIRType::Int32:
20639 masm.load32(Address(ToRegister(ins->instance()), ins->mir()->offset()),
20640 ToRegister(ins->output()));
20641 break;
20642 default:
20643 MOZ_CRASH("MIRType not supported in WasmLoadInstance")do { do { } while (false); MOZ_ReportCrash("" "MIRType not supported in WasmLoadInstance"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 20643); AnnotateMozCrashReason("MOZ_CRASH(" "MIRType not supported in WasmLoadInstance"
")"); do { *((volatile int*)__null) = 20643; __attribute__((
nomerge)) ::abort(); } while (false); } while (false)
;
20644 }
20645}
20646
20647void CodeGenerator::visitWasmLoadInstance64(LWasmLoadInstance64* ins) {
20648 MOZ_ASSERT(ins->mir()->type() == MIRType::Int64)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ins->mir()->type() == MIRType::Int64)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(ins->mir()->type() == MIRType::Int64))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("ins->mir()->type() == MIRType::Int64"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 20648); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ins->mir()->type() == MIRType::Int64"
")"); do { *((volatile int*)__null) = 20648; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
20649 masm.load64(Address(ToRegister(ins->instance()), ins->mir()->offset()),
20650 ToOutRegister64(ins));
20651}
20652
20653void CodeGenerator::incrementWarmUpCounter(AbsoluteAddress warmUpCount,
20654 JSScript* script, Register tmp) {
20655 // The code depends on the JitScript* not being discarded without also
20656 // invalidating Ion code. Assert this.
20657#ifdef DEBUG1
20658 Label ok;
20659 masm.movePtr(ImmGCPtr(script), tmp);
20660 masm.loadJitScript(tmp, tmp);
20661 masm.branchPtr(Assembler::Equal, tmp, ImmPtr(script->jitScript()), &ok);
20662 masm.assumeUnreachable("Didn't find JitScript?");
20663 masm.bind(&ok);
20664#endif
20665
20666 masm.load32(warmUpCount, tmp);
20667 masm.add32(Imm32(1), tmp);
20668 masm.store32(tmp, warmUpCount);
20669}
20670
20671void CodeGenerator::visitIncrementWarmUpCounter(LIncrementWarmUpCounter* ins) {
20672 Register tmp = ToRegister(ins->temp0());
20673
20674 AbsoluteAddress warmUpCount =
20675 AbsoluteAddress(ins->mir()->script()->jitScript())
20676 .offset(JitScript::offsetOfWarmUpCount());
20677 incrementWarmUpCounter(warmUpCount, ins->mir()->script(), tmp);
20678}
20679
20680void CodeGenerator::visitLexicalCheck(LLexicalCheck* ins) {
20681 ValueOperand inputValue = ToValue(ins, LLexicalCheck::InputIndex);
20682 Label bail;
20683 masm.branchTestMagicValue(Assembler::Equal, inputValue,
20684 JS_UNINITIALIZED_LEXICAL, &bail);
20685 bailoutFrom(&bail, ins->snapshot());
20686}
20687
20688void CodeGenerator::visitThrowRuntimeLexicalError(
20689 LThrowRuntimeLexicalError* ins) {
20690 pushArg(Imm32(ins->mir()->errorNumber()));
20691
20692 using Fn = bool (*)(JSContext*, unsigned);
20693 callVM<Fn, jit::ThrowRuntimeLexicalError>(ins);
20694}
20695
20696void CodeGenerator::visitThrowMsg(LThrowMsg* ins) {
20697 pushArg(Imm32(static_cast<int32_t>(ins->mir()->throwMsgKind())));
20698
20699 using Fn = bool (*)(JSContext*, unsigned);
20700 callVM<Fn, js::ThrowMsgOperation>(ins);
20701}
20702
20703void CodeGenerator::visitGlobalDeclInstantiation(
20704 LGlobalDeclInstantiation* ins) {
20705 pushArg(ImmPtr(ins->mir()->resumePoint()->pc()));
20706 pushArg(ImmGCPtr(ins->mir()->block()->info().script()));
20707
20708 using Fn = bool (*)(JSContext*, HandleScript, const jsbytecode*);
20709 callVM<Fn, GlobalDeclInstantiationFromIon>(ins);
20710}
20711
20712void CodeGenerator::visitDebugger(LDebugger* ins) {
20713 Register cx = ToRegister(ins->temp0());
20714
20715 masm.loadJSContext(cx);
20716 using Fn = bool (*)(JSContext* cx);
20717 masm.setupAlignedABICall();
20718 masm.passABIArg(cx);
20719 masm.callWithABI<Fn, GlobalHasLiveOnDebuggerStatement>();
20720
20721 Label bail;
20722 masm.branchIfTrueBool(ReturnReg, &bail);
20723 bailoutFrom(&bail, ins->snapshot());
20724}
20725
20726void CodeGenerator::visitNewTarget(LNewTarget* ins) {
20727 ValueOperand output = ToOutValue(ins);
20728
20729 // if (isConstructing) output = argv[Max(numActualArgs, numFormalArgs)]
20730 Label notConstructing, done;
20731 Address calleeToken(FramePointer, JitFrameLayout::offsetOfCalleeToken());
20732 masm.branchTestPtr(Assembler::Zero, calleeToken,
20733 Imm32(CalleeToken_FunctionConstructing), &notConstructing);
20734
20735 Register argvLen = output.scratchReg();
20736 masm.loadNumActualArgs(FramePointer, argvLen);
20737
20738 Label useNFormals;
20739
20740 size_t numFormalArgs = ins->mirRaw()->block()->info().nargs();
20741 masm.branchPtr(Assembler::Below, argvLen, Imm32(numFormalArgs), &useNFormals);
20742
20743 size_t argsOffset = JitFrameLayout::offsetOfActualArgs();
20744 {
20745 BaseValueIndex newTarget(FramePointer, argvLen, argsOffset);
20746 masm.loadValue(newTarget, output);
20747 masm.jump(&done);
20748 }
20749
20750 masm.bind(&useNFormals);
20751
20752 {
20753 Address newTarget(FramePointer,
20754 argsOffset + (numFormalArgs * sizeof(Value)));
20755 masm.loadValue(newTarget, output);
20756 masm.jump(&done);
20757 }
20758
20759 // else output = undefined
20760 masm.bind(&notConstructing);
20761 masm.moveValue(UndefinedValue(), output);
20762 masm.bind(&done);
20763}
20764
20765void CodeGenerator::visitCheckReturn(LCheckReturn* ins) {
20766 ValueOperand returnValue = ToValue(ins, LCheckReturn::ReturnValueIndex);
20767 ValueOperand thisValue = ToValue(ins, LCheckReturn::ThisValueIndex);
20768 ValueOperand output = ToOutValue(ins);
20769
20770 using Fn = bool (*)(JSContext*, HandleValue);
20771 OutOfLineCode* ool = oolCallVM<Fn, ThrowBadDerivedReturnOrUninitializedThis>(
20772 ins, ArgList(returnValue), StoreNothing());
20773
20774 Label noChecks;
20775 masm.branchTestObject(Assembler::Equal, returnValue, &noChecks);
20776 masm.branchTestUndefined(Assembler::NotEqual, returnValue, ool->entry());
20777 masm.branchTestMagic(Assembler::Equal, thisValue, ool->entry());
20778 masm.moveValue(thisValue, output);
20779 masm.jump(ool->rejoin());
20780 masm.bind(&noChecks);
20781 masm.moveValue(returnValue, output);
20782 masm.bind(ool->rejoin());
20783}
20784
20785void CodeGenerator::visitCheckIsObj(LCheckIsObj* ins) {
20786 ValueOperand value = ToValue(ins, LCheckIsObj::ValueIndex);
20787 Register output = ToRegister(ins->output());
20788
20789 using Fn = bool (*)(JSContext*, CheckIsObjectKind);
20790 OutOfLineCode* ool = oolCallVM<Fn, ThrowCheckIsObject>(
20791 ins, ArgList(Imm32(ins->mir()->checkKind())), StoreNothing());
20792
20793 masm.fallibleUnboxObject(value, output, ool->entry());
20794 masm.bind(ool->rejoin());
20795}
20796
20797void CodeGenerator::visitCheckObjCoercible(LCheckObjCoercible* ins) {
20798 ValueOperand checkValue = ToValue(ins, LCheckObjCoercible::ValueIndex);
20799
20800 using Fn = bool (*)(JSContext*, HandleValue);
20801 OutOfLineCode* ool = oolCallVM<Fn, ThrowObjectCoercible>(
20802 ins, ArgList(checkValue), StoreNothing());
20803 masm.branchTestNull(Assembler::Equal, checkValue, ool->entry());
20804 masm.branchTestUndefined(Assembler::Equal, checkValue, ool->entry());
20805 masm.bind(ool->rejoin());
20806}
20807
20808void CodeGenerator::visitCheckClassHeritage(LCheckClassHeritage* ins) {
20809 ValueOperand heritage = ToValue(ins, LCheckClassHeritage::HeritageIndex);
20810 Register temp0 = ToRegister(ins->temp0());
20811 Register temp1 = ToRegister(ins->temp1());
20812
20813 using Fn = bool (*)(JSContext*, HandleValue);
20814 OutOfLineCode* ool = oolCallVM<Fn, CheckClassHeritageOperation>(
20815 ins, ArgList(heritage), StoreNothing());
20816
20817 masm.branchTestNull(Assembler::Equal, heritage, ool->rejoin());
20818 masm.fallibleUnboxObject(heritage, temp0, ool->entry());
20819
20820 masm.isConstructor(temp0, temp1, ool->entry());
20821 masm.branchTest32(Assembler::Zero, temp1, temp1, ool->entry());
20822
20823 masm.bind(ool->rejoin());
20824}
20825
20826void CodeGenerator::visitCheckThis(LCheckThis* ins) {
20827 ValueOperand thisValue = ToValue(ins, LCheckThis::ValueIndex);
20828
20829 using Fn = bool (*)(JSContext*);
20830 OutOfLineCode* ool =
20831 oolCallVM<Fn, ThrowUninitializedThis>(ins, ArgList(), StoreNothing());
20832 masm.branchTestMagic(Assembler::Equal, thisValue, ool->entry());
20833 masm.bind(ool->rejoin());
20834}
20835
20836void CodeGenerator::visitCheckThisReinit(LCheckThisReinit* ins) {
20837 ValueOperand thisValue = ToValue(ins, LCheckThisReinit::ThisValueIndex);
20838
20839 using Fn = bool (*)(JSContext*);
20840 OutOfLineCode* ool =
20841 oolCallVM<Fn, ThrowInitializedThis>(ins, ArgList(), StoreNothing());
20842 masm.branchTestMagic(Assembler::NotEqual, thisValue, ool->entry());
20843 masm.bind(ool->rejoin());
20844}
20845
20846void CodeGenerator::visitGenerator(LGenerator* lir) {
20847 Register callee = ToRegister(lir->callee());
20848 Register environmentChain = ToRegister(lir->environmentChain());
20849 Register argsObject = ToRegister(lir->argsObject());
20850
20851 pushArg(argsObject);
20852 pushArg(environmentChain);
20853 pushArg(ImmGCPtr(current->mir()->info().script()));
20854 pushArg(callee);
20855
20856 using Fn = JSObject* (*)(JSContext* cx, HandleFunction, HandleScript,
20857 HandleObject, HandleObject);
20858 callVM<Fn, CreateGenerator>(lir);
20859}
20860
20861void CodeGenerator::visitAsyncResolve(LAsyncResolve* lir) {
20862 Register generator = ToRegister(lir->generator());
20863 ValueOperand value = ToValue(lir, LAsyncResolve::ValueIndex);
20864
20865 pushArg(value);
20866 pushArg(generator);
20867
20868 using Fn = JSObject* (*)(JSContext*, Handle<AsyncFunctionGeneratorObject*>,
20869 HandleValue);
20870 callVM<Fn, js::AsyncFunctionResolve>(lir);
20871}
20872
20873void CodeGenerator::visitAsyncReject(LAsyncReject* lir) {
20874 Register generator = ToRegister(lir->generator());
20875 ValueOperand reason = ToValue(lir, LAsyncReject::ReasonIndex);
20876 ValueOperand stack = ToValue(lir, LAsyncReject::StackIndex);
20877
20878 pushArg(stack);
20879 pushArg(reason);
20880 pushArg(generator);
20881
20882 using Fn = JSObject* (*)(JSContext*, Handle<AsyncFunctionGeneratorObject*>,
20883 HandleValue, HandleValue);
20884 callVM<Fn, js::AsyncFunctionReject>(lir);
20885}
20886
20887void CodeGenerator::visitAsyncAwait(LAsyncAwait* lir) {
20888 ValueOperand value = ToValue(lir, LAsyncAwait::ValueIndex);
20889 Register generator = ToRegister(lir->generator());
20890
20891 pushArg(value);
20892 pushArg(generator);
20893
20894 using Fn =
20895 JSObject* (*)(JSContext* cx, Handle<AsyncFunctionGeneratorObject*> genObj,
20896 HandleValue value);
20897 callVM<Fn, js::AsyncFunctionAwait>(lir);
20898}
20899
20900void CodeGenerator::visitCanSkipAwait(LCanSkipAwait* lir) {
20901 ValueOperand value = ToValue(lir, LCanSkipAwait::ValueIndex);
20902
20903 pushArg(value);
20904
20905 using Fn = bool (*)(JSContext*, HandleValue, bool* canSkip);
20906 callVM<Fn, js::CanSkipAwait>(lir);
20907}
20908
20909void CodeGenerator::visitMaybeExtractAwaitValue(LMaybeExtractAwaitValue* lir) {
20910 ValueOperand value = ToValue(lir, LMaybeExtractAwaitValue::ValueIndex);
20911 ValueOperand output = ToOutValue(lir);
20912 Register canSkip = ToRegister(lir->canSkip());
20913
20914 Label cantExtract, finished;
20915 masm.branchIfFalseBool(canSkip, &cantExtract);
20916
20917 pushArg(value);
20918
20919 using Fn = bool (*)(JSContext*, HandleValue, MutableHandleValue);
20920 callVM<Fn, js::ExtractAwaitValue>(lir);
20921 masm.jump(&finished);
20922 masm.bind(&cantExtract);
20923
20924 masm.moveValue(value, output);
20925
20926 masm.bind(&finished);
20927}
20928
20929void CodeGenerator::visitDebugCheckSelfHosted(LDebugCheckSelfHosted* ins) {
20930 ValueOperand checkValue = ToValue(ins, LDebugCheckSelfHosted::ValueIndex);
20931 pushArg(checkValue);
20932 using Fn = bool (*)(JSContext*, HandleValue);
20933 callVM<Fn, js::Debug_CheckSelfHosted>(ins);
20934}
20935
20936void CodeGenerator::visitRandom(LRandom* ins) {
20937 using mozilla::non_crypto::XorShift128PlusRNG;
20938
20939 FloatRegister output = ToFloatRegister(ins->output());
20940 Register rngReg = ToRegister(ins->temp0());
20941
20942 Register64 temp1 = ToRegister64(ins->temp1());
20943 Register64 temp2 = ToRegister64(ins->temp2());
20944
20945 const XorShift128PlusRNG* rng = gen->realm->addressOfRandomNumberGenerator();
20946 masm.movePtr(ImmPtr(rng), rngReg);
20947
20948 masm.randomDouble(rngReg, output, temp1, temp2);
20949 if (js::SupportDifferentialTesting()) {
20950 masm.loadConstantDouble(0.0, output);
20951 }
20952}
20953
20954void CodeGenerator::visitSignExtendInt32(LSignExtendInt32* ins) {
20955 Register input = ToRegister(ins->input());
20956 Register output = ToRegister(ins->output());
20957
20958 switch (ins->mode()) {
20959 case MSignExtendInt32::Byte:
20960 masm.move8SignExtend(input, output);
20961 break;
20962 case MSignExtendInt32::Half:
20963 masm.move16SignExtend(input, output);
20964 break;
20965 }
20966}
20967
20968void CodeGenerator::visitRotate(LRotate* ins) {
20969 MRotate* mir = ins->mir();
20970 Register input = ToRegister(ins->input());
20971 Register dest = ToRegister(ins->output());
20972
20973 const LAllocation* count = ins->count();
20974 if (count->isConstant()) {
20975 int32_t c = ToInt32(count) & 0x1F;
20976 if (mir->isLeftRotate()) {
20977 masm.rotateLeft(Imm32(c), input, dest);
20978 } else {
20979 masm.rotateRight(Imm32(c), input, dest);
20980 }
20981 } else {
20982 Register creg = ToRegister(count);
20983 if (mir->isLeftRotate()) {
20984 masm.rotateLeft(creg, input, dest);
20985 } else {
20986 masm.rotateRight(creg, input, dest);
20987 }
20988 }
20989}
20990
20991class OutOfLineNaNToZero : public OutOfLineCodeBase<CodeGenerator> {
20992 LNaNToZero* lir_;
20993
20994 public:
20995 explicit OutOfLineNaNToZero(LNaNToZero* lir) : lir_(lir) {}
20996
20997 void accept(CodeGenerator* codegen) override {
20998 codegen->visitOutOfLineNaNToZero(this);
20999 }
21000 LNaNToZero* lir() const { return lir_; }
21001};
21002
21003void CodeGenerator::visitOutOfLineNaNToZero(OutOfLineNaNToZero* ool) {
21004 FloatRegister output = ToFloatRegister(ool->lir()->output());
21005 masm.loadConstantDouble(0.0, output);
21006 masm.jump(ool->rejoin());
21007}
21008
21009void CodeGenerator::visitNaNToZero(LNaNToZero* lir) {
21010 FloatRegister input = ToFloatRegister(lir->input());
21011
21012 OutOfLineNaNToZero* ool = new (alloc()) OutOfLineNaNToZero(lir);
21013 addOutOfLineCode(ool, lir->mir());
21014
21015 if (lir->mir()->operandIsNeverNegativeZero()) {
21016 masm.branchDouble(Assembler::DoubleUnordered, input, input, ool->entry());
21017 } else {
21018 FloatRegister scratch = ToFloatRegister(lir->temp0());
21019 masm.loadConstantDouble(0.0, scratch);
21020 masm.branchDouble(Assembler::DoubleEqualOrUnordered, input, scratch,
21021 ool->entry());
21022 }
21023 masm.bind(ool->rejoin());
21024}
21025
21026void CodeGenerator::visitIsPackedArray(LIsPackedArray* lir) {
21027 Register obj = ToRegister(lir->object());
21028 Register output = ToRegister(lir->output());
21029 Register temp = ToRegister(lir->temp0());
21030
21031 masm.setIsPackedArray(obj, output, temp);
21032}
21033
21034void CodeGenerator::visitGuardArrayIsPacked(LGuardArrayIsPacked* lir) {
21035 Register array = ToRegister(lir->array());
21036 Register temp0 = ToRegister(lir->temp0());
21037 Register temp1 = ToRegister(lir->temp1());
21038
21039 Label bail;
21040 masm.branchArrayIsNotPacked(array, temp0, temp1, &bail);
21041 bailoutFrom(&bail, lir->snapshot());
21042}
21043
21044void CodeGenerator::visitGetPrototypeOf(LGetPrototypeOf* lir) {
21045 Register target = ToRegister(lir->target());
21046 ValueOperand out = ToOutValue(lir);
21047 Register scratch = out.scratchReg();
21048
21049 using Fn = bool (*)(JSContext*, HandleObject, MutableHandleValue);
21050 OutOfLineCode* ool = oolCallVM<Fn, jit::GetPrototypeOf>(lir, ArgList(target),
21051 StoreValueTo(out));
21052
21053 MOZ_ASSERT(uintptr_t(TaggedProto::LazyProto) == 1)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(uintptr_t(TaggedProto::LazyProto) == 1)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(uintptr_t(TaggedProto::LazyProto) == 1))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("uintptr_t(TaggedProto::LazyProto) == 1"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 21053); AnnotateMozCrashReason("MOZ_ASSERT" "(" "uintptr_t(TaggedProto::LazyProto) == 1"
")"); do { *((volatile int*)__null) = 21053; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
21054
21055 masm.loadObjProto(target, scratch);
21056
21057 Label hasProto;
21058 masm.branchPtr(Assembler::Above, scratch, ImmWord(1), &hasProto);
21059
21060 // Call into the VM for lazy prototypes.
21061 masm.branchPtr(Assembler::Equal, scratch, ImmWord(1), ool->entry());
21062
21063 masm.moveValue(NullValue(), out);
21064 masm.jump(ool->rejoin());
21065
21066 masm.bind(&hasProto);
21067 masm.tagValue(JSVAL_TYPE_OBJECT, scratch, out);
21068
21069 masm.bind(ool->rejoin());
21070}
21071
21072void CodeGenerator::visitObjectWithProto(LObjectWithProto* lir) {
21073 pushArg(ToValue(lir, LObjectWithProto::PrototypeIndex));
21074
21075 using Fn = PlainObject* (*)(JSContext*, HandleValue);
21076 callVM<Fn, js::ObjectWithProtoOperation>(lir);
21077}
21078
21079void CodeGenerator::visitObjectStaticProto(LObjectStaticProto* lir) {
21080 Register obj = ToRegister(lir->input());
21081 Register output = ToRegister(lir->output());
21082
21083 masm.loadObjProto(obj, output);
21084
21085#ifdef DEBUG1
21086 // We shouldn't encounter a null or lazy proto.
21087 MOZ_ASSERT(uintptr_t(TaggedProto::LazyProto) == 1)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(uintptr_t(TaggedProto::LazyProto) == 1)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(uintptr_t(TaggedProto::LazyProto) == 1))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("uintptr_t(TaggedProto::LazyProto) == 1"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 21087); AnnotateMozCrashReason("MOZ_ASSERT" "(" "uintptr_t(TaggedProto::LazyProto) == 1"
")"); do { *((volatile int*)__null) = 21087; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
21088
21089 Label done;
21090 masm.branchPtr(Assembler::Above, output, ImmWord(1), &done);
21091 masm.assumeUnreachable("Unexpected null or lazy proto in MObjectStaticProto");
21092 masm.bind(&done);
21093#endif
21094}
21095
21096void CodeGenerator::visitBuiltinObject(LBuiltinObject* lir) {
21097 pushArg(Imm32(static_cast<int32_t>(lir->mir()->builtinObjectKind())));
21098
21099 using Fn = JSObject* (*)(JSContext*, BuiltinObjectKind);
21100 callVM<Fn, js::BuiltinObjectOperation>(lir);
21101}
21102
21103void CodeGenerator::visitSuperFunction(LSuperFunction* lir) {
21104 Register callee = ToRegister(lir->callee());
21105 ValueOperand out = ToOutValue(lir);
21106 Register temp = ToRegister(lir->temp0());
21107
21108#ifdef DEBUG1
21109 Label classCheckDone;
21110 masm.branchTestObjIsFunction(Assembler::Equal, callee, temp, callee,
21111 &classCheckDone);
21112 masm.assumeUnreachable("Unexpected non-JSFunction callee in JSOp::SuperFun");
21113 masm.bind(&classCheckDone);
21114#endif
21115
21116 // Load prototype of callee
21117 masm.loadObjProto(callee, temp);
21118
21119#ifdef DEBUG1
21120 // We won't encounter a lazy proto, because |callee| is guaranteed to be a
21121 // JSFunction and only proxy objects can have a lazy proto.
21122 MOZ_ASSERT(uintptr_t(TaggedProto::LazyProto) == 1)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(uintptr_t(TaggedProto::LazyProto) == 1)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(uintptr_t(TaggedProto::LazyProto) == 1))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("uintptr_t(TaggedProto::LazyProto) == 1"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 21122); AnnotateMozCrashReason("MOZ_ASSERT" "(" "uintptr_t(TaggedProto::LazyProto) == 1"
")"); do { *((volatile int*)__null) = 21122; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
21123
21124 Label proxyCheckDone;
21125 masm.branchPtr(Assembler::NotEqual, temp, ImmWord(1), &proxyCheckDone);
21126 masm.assumeUnreachable("Unexpected lazy proto in JSOp::SuperFun");
21127 masm.bind(&proxyCheckDone);
21128#endif
21129
21130 Label nullProto, done;
21131 masm.branchPtr(Assembler::Equal, temp, ImmWord(0), &nullProto);
21132
21133 // Box prototype and return
21134 masm.tagValue(JSVAL_TYPE_OBJECT, temp, out);
21135 masm.jump(&done);
21136
21137 masm.bind(&nullProto);
21138 masm.moveValue(NullValue(), out);
21139
21140 masm.bind(&done);
21141}
21142
21143void CodeGenerator::visitInitHomeObject(LInitHomeObject* lir) {
21144 Register func = ToRegister(lir->function());
21145 ValueOperand homeObject = ToValue(lir, LInitHomeObject::HomeObjectIndex);
21146
21147 masm.assertFunctionIsExtended(func);
21148
21149 Address addr(func, FunctionExtended::offsetOfMethodHomeObjectSlot());
21150
21151 emitPreBarrier(addr);
21152 masm.storeValue(homeObject, addr);
21153}
21154
21155void CodeGenerator::visitIsTypedArrayConstructor(
21156 LIsTypedArrayConstructor* lir) {
21157 Register object = ToRegister(lir->object());
21158 Register output = ToRegister(lir->output());
21159
21160 masm.setIsDefinitelyTypedArrayConstructor(object, output);
21161}
21162
21163void CodeGenerator::visitLoadValueTag(LLoadValueTag* lir) {
21164 ValueOperand value = ToValue(lir, LLoadValueTag::ValueIndex);
21165 Register output = ToRegister(lir->output());
21166
21167 Register tag = masm.extractTag(value, output);
21168 if (tag != output) {
21169 masm.mov(tag, output);
21170 }
21171}
21172
21173void CodeGenerator::visitGuardTagNotEqual(LGuardTagNotEqual* lir) {
21174 Register lhs = ToRegister(lir->lhs());
21175 Register rhs = ToRegister(lir->rhs());
21176
21177 bailoutCmp32(Assembler::Equal, lhs, rhs, lir->snapshot());
21178
21179 // If both lhs and rhs are numbers, can't use tag comparison to do inequality
21180 // comparison
21181 Label done;
21182 masm.branchTestNumber(Assembler::NotEqual, lhs, &done);
21183 masm.branchTestNumber(Assembler::NotEqual, rhs, &done);
21184 bailout(lir->snapshot());
21185
21186 masm.bind(&done);
21187}
21188
21189void CodeGenerator::visitLoadWrapperTarget(LLoadWrapperTarget* lir) {
21190 Register object = ToRegister(lir->object());
21191 Register output = ToRegister(lir->output());
21192
21193 masm.loadPtr(Address(object, ProxyObject::offsetOfReservedSlots()), output);
21194
21195 // Bail for revoked proxies.
21196 Label bail;
21197 Address targetAddr(output,
21198 js::detail::ProxyReservedSlots::offsetOfPrivateSlot());
21199 if (lir->mir()->fallible()) {
21200 masm.fallibleUnboxObject(targetAddr, output, &bail);
21201 bailoutFrom(&bail, lir->snapshot());
21202 } else {
21203 masm.unboxObject(targetAddr, output);
21204 }
21205}
21206
21207void CodeGenerator::visitGuardHasGetterSetter(LGuardHasGetterSetter* lir) {
21208 Register object = ToRegister(lir->object());
21209 Register temp0 = ToRegister(lir->temp0());
21210 Register temp1 = ToRegister(lir->temp1());
21211 Register temp2 = ToRegister(lir->temp2());
21212
21213 masm.movePropertyKey(lir->mir()->propId(), temp1);
21214 masm.movePtr(ImmGCPtr(lir->mir()->getterSetter()), temp2);
21215
21216 using Fn = bool (*)(JSContext* cx, JSObject* obj, jsid id,
21217 GetterSetter* getterSetter);
21218 masm.setupAlignedABICall();
21219 masm.loadJSContext(temp0);
21220 masm.passABIArg(temp0);
21221 masm.passABIArg(object);
21222 masm.passABIArg(temp1);
21223 masm.passABIArg(temp2);
21224 masm.callWithABI<Fn, ObjectHasGetterSetterPure>();
21225
21226 bailoutIfFalseBool(ReturnReg, lir->snapshot());
21227}
21228
21229void CodeGenerator::visitGuardIsExtensible(LGuardIsExtensible* lir) {
21230 Register object = ToRegister(lir->object());
21231 Register temp = ToRegister(lir->temp0());
21232
21233 Label bail;
21234 masm.branchIfObjectNotExtensible(object, temp, &bail);
21235 bailoutFrom(&bail, lir->snapshot());
21236}
21237
21238void CodeGenerator::visitGuardInt32IsNonNegative(
21239 LGuardInt32IsNonNegative* lir) {
21240 Register index = ToRegister(lir->index());
21241
21242 bailoutCmp32(Assembler::LessThan, index, Imm32(0), lir->snapshot());
21243}
21244
21245void CodeGenerator::visitGuardInt32Range(LGuardInt32Range* lir) {
21246 Register input = ToRegister(lir->input());
21247
21248 bailoutCmp32(Assembler::LessThan, input, Imm32(lir->mir()->minimum()),
21249 lir->snapshot());
21250 bailoutCmp32(Assembler::GreaterThan, input, Imm32(lir->mir()->maximum()),
21251 lir->snapshot());
21252}
21253
21254void CodeGenerator::visitGuardIndexIsNotDenseElement(
21255 LGuardIndexIsNotDenseElement* lir) {
21256 Register object = ToRegister(lir->object());
21257 Register index = ToRegister(lir->index());
21258 Register temp = ToRegister(lir->temp0());
21259 Register spectreTemp = ToTempRegisterOrInvalid(lir->temp1());
21260
21261 // Load obj->elements.
21262 masm.loadPtr(Address(object, NativeObject::offsetOfElements()), temp);
21263
21264 // Ensure index >= initLength or the element is a hole.
21265 Label notDense;
21266 Address capacity(temp, ObjectElements::offsetOfInitializedLength());
21267 masm.spectreBoundsCheck32(index, capacity, spectreTemp, &notDense);
21268
21269 BaseValueIndex element(temp, index);
21270 masm.branchTestMagic(Assembler::Equal, element, &notDense);
21271
21272 bailout(lir->snapshot());
21273
21274 masm.bind(&notDense);
21275}
21276
21277void CodeGenerator::visitGuardIndexIsValidUpdateOrAdd(
21278 LGuardIndexIsValidUpdateOrAdd* lir) {
21279 Register object = ToRegister(lir->object());
21280 Register index = ToRegister(lir->index());
21281 Register temp = ToRegister(lir->temp0());
21282 Register spectreTemp = ToTempRegisterOrInvalid(lir->temp1());
21283
21284 // Load obj->elements.
21285 masm.loadPtr(Address(object, NativeObject::offsetOfElements()), temp);
21286
21287 Label success;
21288
21289 // If length is writable, branch to &success. All indices are writable.
21290 Address flags(temp, ObjectElements::offsetOfFlags());
21291 masm.branchTest32(Assembler::Zero, flags,
21292 Imm32(ObjectElements::Flags::NONWRITABLE_ARRAY_LENGTH),
21293 &success);
21294
21295 // Otherwise, ensure index is in bounds.
21296 Label bail;
21297 Address length(temp, ObjectElements::offsetOfLength());
21298 masm.spectreBoundsCheck32(index, length, spectreTemp, &bail);
21299 masm.bind(&success);
21300
21301 bailoutFrom(&bail, lir->snapshot());
21302}
21303
21304void CodeGenerator::visitCallAddOrUpdateSparseElement(
21305 LCallAddOrUpdateSparseElement* lir) {
21306 Register object = ToRegister(lir->object());
21307 Register index = ToRegister(lir->index());
21308 ValueOperand value = ToValue(lir, LCallAddOrUpdateSparseElement::ValueIndex);
21309
21310 pushArg(Imm32(lir->mir()->strict()));
21311 pushArg(value);
21312 pushArg(index);
21313 pushArg(object);
21314
21315 using Fn =
21316 bool (*)(JSContext*, Handle<NativeObject*>, int32_t, HandleValue, bool);
21317 callVM<Fn, js::AddOrUpdateSparseElementHelper>(lir);
21318}
21319
21320void CodeGenerator::visitCallGetSparseElement(LCallGetSparseElement* lir) {
21321 Register object = ToRegister(lir->object());
21322 Register index = ToRegister(lir->index());
21323
21324 pushArg(index);
21325 pushArg(object);
21326
21327 using Fn =
21328 bool (*)(JSContext*, Handle<NativeObject*>, int32_t, MutableHandleValue);
21329 callVM<Fn, js::GetSparseElementHelper>(lir);
21330}
21331
21332void CodeGenerator::visitCallNativeGetElement(LCallNativeGetElement* lir) {
21333 Register object = ToRegister(lir->object());
21334 Register index = ToRegister(lir->index());
21335
21336 pushArg(index);
21337 pushArg(TypedOrValueRegister(MIRType::Object, AnyRegister(object)));
21338 pushArg(object);
21339
21340 using Fn = bool (*)(JSContext*, Handle<NativeObject*>, HandleValue, int32_t,
21341 MutableHandleValue);
21342 callVM<Fn, js::NativeGetElement>(lir);
21343}
21344
21345void CodeGenerator::visitCallNativeGetElementSuper(
21346 LCallNativeGetElementSuper* lir) {
21347 Register object = ToRegister(lir->object());
21348 Register index = ToRegister(lir->index());
21349 ValueOperand receiver =
21350 ToValue(lir, LCallNativeGetElementSuper::ReceiverIndex);
21351
21352 pushArg(index);
21353 pushArg(receiver);
21354 pushArg(object);
21355
21356 using Fn = bool (*)(JSContext*, Handle<NativeObject*>, HandleValue, int32_t,
21357 MutableHandleValue);
21358 callVM<Fn, js::NativeGetElement>(lir);
21359}
21360
21361void CodeGenerator::visitCallObjectHasSparseElement(
21362 LCallObjectHasSparseElement* lir) {
21363 Register object = ToRegister(lir->object());
21364 Register index = ToRegister(lir->index());
21365 Register temp0 = ToRegister(lir->temp0());
21366 Register temp1 = ToRegister(lir->temp1());
21367 Register output = ToRegister(lir->output());
21368
21369 masm.reserveStack(sizeof(Value));
21370 masm.moveStackPtrTo(temp1);
21371
21372 using Fn = bool (*)(JSContext*, NativeObject*, int32_t, Value*);
21373 masm.setupAlignedABICall();
21374 masm.loadJSContext(temp0);
21375 masm.passABIArg(temp0);
21376 masm.passABIArg(object);
21377 masm.passABIArg(index);
21378 masm.passABIArg(temp1);
21379 masm.callWithABI<Fn, HasNativeElementPure>();
21380 masm.storeCallPointerResult(temp0);
21381
21382 Label bail, ok;
21383 uint32_t framePushed = masm.framePushed();
21384 masm.branchIfTrueBool(temp0, &ok);
21385 masm.adjustStack(sizeof(Value));
21386 masm.jump(&bail);
21387
21388 masm.bind(&ok);
21389 masm.setFramePushed(framePushed);
21390 masm.unboxBoolean(Address(masm.getStackPointer(), 0), output);
21391 masm.adjustStack(sizeof(Value));
21392
21393 bailoutFrom(&bail, lir->snapshot());
21394}
21395
21396void CodeGenerator::visitBigIntAsIntN(LBigIntAsIntN* ins) {
21397 Register bits = ToRegister(ins->bits());
21398 Register input = ToRegister(ins->input());
21399
21400 pushArg(bits);
21401 pushArg(input);
21402
21403 using Fn = BigInt* (*)(JSContext*, HandleBigInt, int32_t);
21404 callVM<Fn, jit::BigIntAsIntN>(ins);
21405}
21406
21407void CodeGenerator::visitBigIntAsIntN64(LBigIntAsIntN64* ins) {
21408 Register input = ToRegister(ins->input());
21409 Register temp = ToRegister(ins->temp());
21410 Register64 temp64 = ToRegister64(ins->temp64());
21411 Register output = ToRegister(ins->output());
21412
21413 Label done, create;
21414
21415 masm.movePtr(input, output);
21416
21417 // Load the BigInt value as an int64.
21418 masm.loadBigInt64(input, temp64);
21419
21420 // Create a new BigInt when the input exceeds the int64 range.
21421 masm.branch32(Assembler::Above, Address(input, BigInt::offsetOfLength()),
21422 Imm32(64 / BigInt::DigitBits), &create);
21423
21424 // And create a new BigInt when the value and the BigInt have different signs.
21425 Label nonNegative;
21426 masm.branchIfBigIntIsNonNegative(input, &nonNegative);
21427 masm.branchTest64(Assembler::NotSigned, temp64, temp64, temp, &create);
21428 masm.jump(&done);
21429
21430 masm.bind(&nonNegative);
21431 masm.branchTest64(Assembler::NotSigned, temp64, temp64, temp, &done);
21432
21433 masm.bind(&create);
21434 emitCreateBigInt(ins, Scalar::BigInt64, temp64, output, temp);
21435
21436 masm.bind(&done);
21437}
21438
21439void CodeGenerator::visitBigIntAsIntN32(LBigIntAsIntN32* ins) {
21440 Register input = ToRegister(ins->input());
21441 Register temp = ToRegister(ins->temp());
21442 Register64 temp64 = ToRegister64(ins->temp64());
21443 Register output = ToRegister(ins->output());
21444
21445 Label done, create;
21446
21447 masm.movePtr(input, output);
21448
21449 // Load the absolute value of the first digit.
21450 masm.loadBigIntDigit(input, temp);
21451
21452 // If the absolute value exceeds the int32 range, create a new BigInt.
21453 masm.branchPtr(Assembler::Above, temp, Imm32(INT32_MAX(2147483647)), &create);
21454
21455 // Also create a new BigInt if we have more than one digit.
21456 masm.branch32(Assembler::BelowOrEqual,
21457 Address(input, BigInt::offsetOfLength()), Imm32(1), &done);
21458
21459 masm.bind(&create);
21460
21461 // |temp| stores the absolute value, negate it when the sign flag is set.
21462 Label nonNegative;
21463 masm.branchIfBigIntIsNonNegative(input, &nonNegative);
21464 masm.negPtr(temp);
21465 masm.bind(&nonNegative);
21466
21467 masm.move32To64SignExtend(temp, temp64);
21468 emitCreateBigInt(ins, Scalar::BigInt64, temp64, output, temp);
21469
21470 masm.bind(&done);
21471}
21472
21473void CodeGenerator::visitBigIntAsUintN(LBigIntAsUintN* ins) {
21474 Register bits = ToRegister(ins->bits());
21475 Register input = ToRegister(ins->input());
21476
21477 pushArg(bits);
21478 pushArg(input);
21479
21480 using Fn = BigInt* (*)(JSContext*, HandleBigInt, int32_t);
21481 callVM<Fn, jit::BigIntAsUintN>(ins);
21482}
21483
21484void CodeGenerator::visitBigIntAsUintN64(LBigIntAsUintN64* ins) {
21485 Register input = ToRegister(ins->input());
21486 Register temp = ToRegister(ins->temp());
21487 Register64 temp64 = ToRegister64(ins->temp64());
21488 Register output = ToRegister(ins->output());
21489
21490 Label done, create;
21491
21492 masm.movePtr(input, output);
21493
21494 // Load the BigInt value as an uint64.
21495 masm.loadBigInt64(input, temp64);
21496
21497 // Create a new BigInt when the input exceeds the uint64 range.
21498 masm.branch32(Assembler::Above, Address(input, BigInt::offsetOfLength()),
21499 Imm32(64 / BigInt::DigitBits), &create);
21500
21501 // And create a new BigInt when the input has the sign flag set.
21502 masm.branchIfBigIntIsNonNegative(input, &done);
21503
21504 masm.bind(&create);
21505 emitCreateBigInt(ins, Scalar::BigUint64, temp64, output, temp);
21506
21507 masm.bind(&done);
21508}
21509
21510void CodeGenerator::visitBigIntAsUintN32(LBigIntAsUintN32* ins) {
21511 Register input = ToRegister(ins->input());
21512 Register temp = ToRegister(ins->temp());
21513 Register64 temp64 = ToRegister64(ins->temp64());
21514 Register output = ToRegister(ins->output());
21515
21516 Label done, create;
21517
21518 masm.movePtr(input, output);
21519
21520 // Load the absolute value of the first digit.
21521 masm.loadBigIntDigit(input, temp);
21522
21523 // If the absolute value exceeds the uint32 range, create a new BigInt.
21524#if JS_PUNBOX641
21525 masm.branchPtr(Assembler::Above, temp, ImmWord(UINT32_MAX(4294967295U)), &create);
21526#endif
21527
21528 // Also create a new BigInt if we have more than one digit.
21529 masm.branch32(Assembler::Above, Address(input, BigInt::offsetOfLength()),
21530 Imm32(1), &create);
21531
21532 // And create a new BigInt when the input has the sign flag set.
21533 masm.branchIfBigIntIsNonNegative(input, &done);
21534
21535 masm.bind(&create);
21536
21537 // |temp| stores the absolute value, negate it when the sign flag is set.
21538 Label nonNegative;
21539 masm.branchIfBigIntIsNonNegative(input, &nonNegative);
21540 masm.negPtr(temp);
21541 masm.bind(&nonNegative);
21542
21543 masm.move32To64ZeroExtend(temp, temp64);
21544 emitCreateBigInt(ins, Scalar::BigUint64, temp64, output, temp);
21545
21546 masm.bind(&done);
21547}
21548
21549void CodeGenerator::visitGuardNonGCThing(LGuardNonGCThing* ins) {
21550 ValueOperand input = ToValue(ins, LGuardNonGCThing::InputIndex);
21551
21552 Label bail;
21553 masm.branchTestGCThing(Assembler::Equal, input, &bail);
21554 bailoutFrom(&bail, ins->snapshot());
21555}
21556
21557void CodeGenerator::visitToHashableNonGCThing(LToHashableNonGCThing* ins) {
21558 ValueOperand input = ToValue(ins, LToHashableNonGCThing::InputIndex);
21559 FloatRegister tempFloat = ToFloatRegister(ins->temp0());
21560 ValueOperand output = ToOutValue(ins);
21561
21562 masm.toHashableNonGCThing(input, output, tempFloat);
21563}
21564
21565void CodeGenerator::visitToHashableString(LToHashableString* ins) {
21566 Register input = ToRegister(ins->input());
21567 Register output = ToRegister(ins->output());
21568
21569 using Fn = JSAtom* (*)(JSContext*, JSString*);
21570 auto* ool = oolCallVM<Fn, js::AtomizeString>(ins, ArgList(input),
21571 StoreRegisterTo(output));
21572
21573 Label isAtom;
21574 masm.branchTest32(Assembler::NonZero,
21575 Address(input, JSString::offsetOfFlags()),
21576 Imm32(JSString::ATOM_BIT), &isAtom);
21577
21578 masm.tryFastAtomize(input, output, output, ool->entry());
21579 masm.jump(ool->rejoin());
21580 masm.bind(&isAtom);
21581 masm.movePtr(input, output);
21582 masm.bind(ool->rejoin());
21583}
21584
21585void CodeGenerator::visitToHashableValue(LToHashableValue* ins) {
21586 ValueOperand input = ToValue(ins, LToHashableValue::InputIndex);
21587 FloatRegister tempFloat = ToFloatRegister(ins->temp0());
21588 ValueOperand output = ToOutValue(ins);
21589
21590 Register str = output.scratchReg();
21591
21592 using Fn = JSAtom* (*)(JSContext*, JSString*);
21593 auto* ool =
21594 oolCallVM<Fn, js::AtomizeString>(ins, ArgList(str), StoreRegisterTo(str));
21595
21596 masm.toHashableValue(input, output, tempFloat, ool->entry(), ool->rejoin());
21597}
21598
21599void CodeGenerator::visitHashNonGCThing(LHashNonGCThing* ins) {
21600 ValueOperand input = ToValue(ins, LHashNonGCThing::InputIndex);
21601 Register temp = ToRegister(ins->temp0());
21602 Register output = ToRegister(ins->output());
21603
21604 masm.prepareHashNonGCThing(input, output, temp);
21605}
21606
21607void CodeGenerator::visitHashString(LHashString* ins) {
21608 Register input = ToRegister(ins->input());
21609 Register temp = ToRegister(ins->temp0());
21610 Register output = ToRegister(ins->output());
21611
21612 masm.prepareHashString(input, output, temp);
21613}
21614
21615void CodeGenerator::visitHashSymbol(LHashSymbol* ins) {
21616 Register input = ToRegister(ins->input());
21617 Register output = ToRegister(ins->output());
21618
21619 masm.prepareHashSymbol(input, output);
21620}
21621
21622void CodeGenerator::visitHashBigInt(LHashBigInt* ins) {
21623 Register input = ToRegister(ins->input());
21624 Register temp0 = ToRegister(ins->temp0());
21625 Register temp1 = ToRegister(ins->temp1());
21626 Register temp2 = ToRegister(ins->temp2());
21627 Register output = ToRegister(ins->output());
21628
21629 masm.prepareHashBigInt(input, output, temp0, temp1, temp2);
21630}
21631
21632void CodeGenerator::visitHashObject(LHashObject* ins) {
21633 Register setObj = ToRegister(ins->setObject());
21634 ValueOperand input = ToValue(ins, LHashObject::InputIndex);
21635 Register temp0 = ToRegister(ins->temp0());
21636 Register temp1 = ToRegister(ins->temp1());
21637 Register temp2 = ToRegister(ins->temp2());
21638 Register temp3 = ToRegister(ins->temp3());
21639 Register output = ToRegister(ins->output());
21640
21641 masm.prepareHashObject(setObj, input, output, temp0, temp1, temp2, temp3);
21642}
21643
21644void CodeGenerator::visitHashValue(LHashValue* ins) {
21645 Register setObj = ToRegister(ins->setObject());
21646 ValueOperand input = ToValue(ins, LHashValue::InputIndex);
21647 Register temp0 = ToRegister(ins->temp0());
21648 Register temp1 = ToRegister(ins->temp1());
21649 Register temp2 = ToRegister(ins->temp2());
21650 Register temp3 = ToRegister(ins->temp3());
21651 Register output = ToRegister(ins->output());
21652
21653 masm.prepareHashValue(setObj, input, output, temp0, temp1, temp2, temp3);
21654}
21655
21656void CodeGenerator::visitSetObjectHasNonBigInt(LSetObjectHasNonBigInt* ins) {
21657 Register setObj = ToRegister(ins->setObject());
21658 ValueOperand input = ToValue(ins, LSetObjectHasNonBigInt::InputIndex);
21659 Register hash = ToRegister(ins->hash());
21660 Register temp0 = ToRegister(ins->temp0());
21661 Register temp1 = ToRegister(ins->temp1());
21662 Register output = ToRegister(ins->output());
21663
21664 masm.setObjectHasNonBigInt(setObj, input, hash, output, temp0, temp1);
21665}
21666
21667void CodeGenerator::visitSetObjectHasBigInt(LSetObjectHasBigInt* ins) {
21668 Register setObj = ToRegister(ins->setObject());
21669 ValueOperand input = ToValue(ins, LSetObjectHasBigInt::InputIndex);
21670 Register hash = ToRegister(ins->hash());
21671 Register temp0 = ToRegister(ins->temp0());
21672 Register temp1 = ToRegister(ins->temp1());
21673 Register temp2 = ToRegister(ins->temp2());
21674 Register temp3 = ToRegister(ins->temp3());
21675 Register output = ToRegister(ins->output());
21676
21677 masm.setObjectHasBigInt(setObj, input, hash, output, temp0, temp1, temp2,
21678 temp3);
21679}
21680
21681void CodeGenerator::visitSetObjectHasValue(LSetObjectHasValue* ins) {
21682 Register setObj = ToRegister(ins->setObject());
21683 ValueOperand input = ToValue(ins, LSetObjectHasValue::InputIndex);
21684 Register hash = ToRegister(ins->hash());
21685 Register temp0 = ToRegister(ins->temp0());
21686 Register temp1 = ToRegister(ins->temp1());
21687 Register temp2 = ToRegister(ins->temp2());
21688 Register temp3 = ToRegister(ins->temp3());
21689 Register output = ToRegister(ins->output());
21690
21691 masm.setObjectHasValue(setObj, input, hash, output, temp0, temp1, temp2,
21692 temp3);
21693}
21694
21695void CodeGenerator::visitSetObjectHasValueVMCall(
21696 LSetObjectHasValueVMCall* ins) {
21697 pushArg(ToValue(ins, LSetObjectHasValueVMCall::InputIndex));
21698 pushArg(ToRegister(ins->setObject()));
21699
21700 using Fn = bool (*)(JSContext*, HandleObject, HandleValue, bool*);
21701 callVM<Fn, jit::SetObjectHas>(ins);
21702}
21703
21704void CodeGenerator::visitSetObjectSize(LSetObjectSize* ins) {
21705 Register setObj = ToRegister(ins->setObject());
21706 Register output = ToRegister(ins->output());
21707
21708 masm.loadSetObjectSize(setObj, output);
21709}
21710
21711void CodeGenerator::visitMapObjectHasNonBigInt(LMapObjectHasNonBigInt* ins) {
21712 Register mapObj = ToRegister(ins->mapObject());
21713 ValueOperand input = ToValue(ins, LMapObjectHasNonBigInt::InputIndex);
21714 Register hash = ToRegister(ins->hash());
21715 Register temp0 = ToRegister(ins->temp0());
21716 Register temp1 = ToRegister(ins->temp1());
21717 Register output = ToRegister(ins->output());
21718
21719 masm.mapObjectHasNonBigInt(mapObj, input, hash, output, temp0, temp1);
21720}
21721
21722void CodeGenerator::visitMapObjectHasBigInt(LMapObjectHasBigInt* ins) {
21723 Register mapObj = ToRegister(ins->mapObject());
21724 ValueOperand input = ToValue(ins, LMapObjectHasBigInt::InputIndex);
21725 Register hash = ToRegister(ins->hash());
21726 Register temp0 = ToRegister(ins->temp0());
21727 Register temp1 = ToRegister(ins->temp1());
21728 Register temp2 = ToRegister(ins->temp2());
21729 Register temp3 = ToRegister(ins->temp3());
21730 Register output = ToRegister(ins->output());
21731
21732 masm.mapObjectHasBigInt(mapObj, input, hash, output, temp0, temp1, temp2,
21733 temp3);
21734}
21735
21736void CodeGenerator::visitMapObjectHasValue(LMapObjectHasValue* ins) {
21737 Register mapObj = ToRegister(ins->mapObject());
21738 ValueOperand input = ToValue(ins, LMapObjectHasValue::InputIndex);
21739 Register hash = ToRegister(ins->hash());
21740 Register temp0 = ToRegister(ins->temp0());
21741 Register temp1 = ToRegister(ins->temp1());
21742 Register temp2 = ToRegister(ins->temp2());
21743 Register temp3 = ToRegister(ins->temp3());
21744 Register output = ToRegister(ins->output());
21745
21746 masm.mapObjectHasValue(mapObj, input, hash, output, temp0, temp1, temp2,
21747 temp3);
21748}
21749
21750void CodeGenerator::visitMapObjectHasValueVMCall(
21751 LMapObjectHasValueVMCall* ins) {
21752 pushArg(ToValue(ins, LMapObjectHasValueVMCall::InputIndex));
21753 pushArg(ToRegister(ins->mapObject()));
21754
21755 using Fn = bool (*)(JSContext*, HandleObject, HandleValue, bool*);
21756 callVM<Fn, jit::MapObjectHas>(ins);
21757}
21758
21759void CodeGenerator::visitMapObjectGetNonBigInt(LMapObjectGetNonBigInt* ins) {
21760 Register mapObj = ToRegister(ins->mapObject());
21761 ValueOperand input = ToValue(ins, LMapObjectGetNonBigInt::InputIndex);
21762 Register hash = ToRegister(ins->hash());
21763 Register temp0 = ToRegister(ins->temp0());
21764 Register temp1 = ToRegister(ins->temp1());
21765 ValueOperand output = ToOutValue(ins);
21766
21767 masm.mapObjectGetNonBigInt(mapObj, input, hash, output, temp0, temp1,
21768 output.scratchReg());
21769}
21770
21771void CodeGenerator::visitMapObjectGetBigInt(LMapObjectGetBigInt* ins) {
21772 Register mapObj = ToRegister(ins->mapObject());
21773 ValueOperand input = ToValue(ins, LMapObjectGetBigInt::InputIndex);
21774 Register hash = ToRegister(ins->hash());
21775 Register temp0 = ToRegister(ins->temp0());
21776 Register temp1 = ToRegister(ins->temp1());
21777 Register temp2 = ToRegister(ins->temp2());
21778 Register temp3 = ToRegister(ins->temp3());
21779 ValueOperand output = ToOutValue(ins);
21780
21781 masm.mapObjectGetBigInt(mapObj, input, hash, output, temp0, temp1, temp2,
21782 temp3, output.scratchReg());
21783}
21784
21785void CodeGenerator::visitMapObjectGetValue(LMapObjectGetValue* ins) {
21786 Register mapObj = ToRegister(ins->map());
21787 ValueOperand input = ToValue(ins, LMapObjectGetValue::ValueIndex);
21788 Register hash = ToRegister(ins->hash());
21789 Register temp0 = ToRegister(ins->temp0());
21790 Register temp1 = ToRegister(ins->temp1());
21791 Register temp2 = ToRegister(ins->temp2());
21792 Register temp3 = ToRegister(ins->temp3());
21793 ValueOperand output = ToOutValue(ins);
21794
21795 masm.mapObjectGetValue(mapObj, input, hash, output, temp0, temp1, temp2,
21796 temp3, output.scratchReg());
21797}
21798
21799void CodeGenerator::visitMapObjectGetValueVMCall(
21800 LMapObjectGetValueVMCall* ins) {
21801 pushArg(ToValue(ins, LMapObjectGetValueVMCall::InputIndex));
21802 pushArg(ToRegister(ins->mapObject()));
21803
21804 using Fn =
21805 bool (*)(JSContext*, HandleObject, HandleValue, MutableHandleValue);
21806 callVM<Fn, jit::MapObjectGet>(ins);
21807}
21808
21809void CodeGenerator::visitMapObjectSize(LMapObjectSize* ins) {
21810 Register mapObj = ToRegister(ins->mapObject());
21811 Register output = ToRegister(ins->output());
21812
21813 masm.loadMapObjectSize(mapObj, output);
21814}
21815
21816template <size_t NumDefs>
21817void CodeGenerator::emitIonToWasmCallBase(LIonToWasmCallBase<NumDefs>* lir) {
21818 wasm::JitCallStackArgVector stackArgs;
21819 masm.propagateOOM(stackArgs.reserve(lir->numOperands()));
21820 if (masm.oom()) {
21821 return;
21822 }
21823
21824 MIonToWasmCall* mir = lir->mir();
21825 const wasm::FuncExport& funcExport = mir->funcExport();
21826 const wasm::FuncType& sig =
21827 mir->instance()->code().codeMeta().getFuncType(funcExport.funcIndex());
21828
21829 WasmABIArgGenerator abi;
21830 for (size_t i = 0; i < lir->numOperands(); i++) {
21831 MIRType argMir;
21832 switch (sig.args()[i].kind()) {
21833 case wasm::ValType::I32:
21834 case wasm::ValType::I64:
21835 case wasm::ValType::F32:
21836 case wasm::ValType::F64:
21837 argMir = sig.args()[i].toMIRType();
21838 break;
21839 case wasm::ValType::V128:
21840 MOZ_CRASH("unexpected argument type when calling from ion to wasm")do { do { } while (false); MOZ_ReportCrash("" "unexpected argument type when calling from ion to wasm"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 21840); AnnotateMozCrashReason("MOZ_CRASH(" "unexpected argument type when calling from ion to wasm"
")"); do { *((volatile int*)__null) = 21840; __attribute__((
nomerge)) ::abort(); } while (false); } while (false)
;
21841 case wasm::ValType::Ref:
21842 // temporarilyUnsupportedReftypeForEntry() restricts args to externref
21843 MOZ_RELEASE_ASSERT(sig.args()[i].refType().isExtern())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(sig.args()[i].refType().isExtern())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(sig.args()[i].refType().isExtern
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("sig.args()[i].refType().isExtern()", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 21843); AnnotateMozCrashReason("MOZ_RELEASE_ASSERT" "(" "sig.args()[i].refType().isExtern()"
")"); do { *((volatile int*)__null) = 21843; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
21844 // Argument is boxed on the JS side to an anyref, so passed as a
21845 // pointer here.
21846 argMir = sig.args()[i].toMIRType();
21847 break;
21848 }
21849
21850 ABIArg arg = abi.next(argMir);
21851 switch (arg.kind()) {
21852 case ABIArg::GPR:
21853 case ABIArg::FPU: {
21854 MOZ_ASSERT(ToAnyRegister(lir->getOperand(i)) == arg.reg())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ToAnyRegister(lir->getOperand(i)) == arg.reg())>
::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(ToAnyRegister(lir->getOperand(i)) == arg.reg())))
, 0))) { do { } while (false); MOZ_ReportAssertionFailure("ToAnyRegister(lir->getOperand(i)) == arg.reg()"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 21854); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ToAnyRegister(lir->getOperand(i)) == arg.reg()"
")"); do { *((volatile int*)__null) = 21854; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
21855 stackArgs.infallibleEmplaceBack(wasm::JitCallStackArg());
21856 break;
21857 }
21858 case ABIArg::Stack: {
21859 const LAllocation* larg = lir->getOperand(i);
21860 if (larg->isConstant()) {
21861 stackArgs.infallibleEmplaceBack(ToInt32(larg));
21862 } else if (larg->isGeneralReg()) {
21863 stackArgs.infallibleEmplaceBack(ToRegister(larg));
21864 } else if (larg->isFloatReg()) {
21865 stackArgs.infallibleEmplaceBack(ToFloatRegister(larg));
21866 } else {
21867 // Always use the stack pointer here because GenerateDirectCallFromJit
21868 // depends on this.
21869 Address addr = ToAddress<BaseRegForAddress::SP>(larg);
21870 stackArgs.infallibleEmplaceBack(addr);
21871 }
21872 break;
21873 }
21874#ifdef JS_CODEGEN_REGISTER_PAIR
21875 case ABIArg::GPR_PAIR: {
21876 MOZ_CRASH(do { do { } while (false); MOZ_ReportCrash("" "no way to pass i64, and wasm uses hardfp for function calls"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 21877); AnnotateMozCrashReason("MOZ_CRASH(" "no way to pass i64, and wasm uses hardfp for function calls"
")"); do { *((volatile int*)__null) = 21877; __attribute__((
nomerge)) ::abort(); } while (false); } while (false)
21877 "no way to pass i64, and wasm uses hardfp for function calls")do { do { } while (false); MOZ_ReportCrash("" "no way to pass i64, and wasm uses hardfp for function calls"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 21877); AnnotateMozCrashReason("MOZ_CRASH(" "no way to pass i64, and wasm uses hardfp for function calls"
")"); do { *((volatile int*)__null) = 21877; __attribute__((
nomerge)) ::abort(); } while (false); } while (false)
;
21878 }
21879#endif
21880 case ABIArg::Uninitialized: {
21881 MOZ_CRASH("Uninitialized ABIArg kind")do { do { } while (false); MOZ_ReportCrash("" "Uninitialized ABIArg kind"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 21881); AnnotateMozCrashReason("MOZ_CRASH(" "Uninitialized ABIArg kind"
")"); do { *((volatile int*)__null) = 21881; __attribute__((
nomerge)) ::abort(); } while (false); } while (false)
;
21882 }
21883 }
21884 }
21885
21886 const wasm::ValTypeVector& results = sig.results();
21887 if (results.length() == 0) {
21888 MOZ_ASSERT(lir->mir()->type() == MIRType::Value)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(lir->mir()->type() == MIRType::Value)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(lir->mir()->type() == MIRType::Value))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("lir->mir()->type() == MIRType::Value"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 21888); AnnotateMozCrashReason("MOZ_ASSERT" "(" "lir->mir()->type() == MIRType::Value"
")"); do { *((volatile int*)__null) = 21888; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
21889 } else {
21890 MOZ_ASSERT(results.length() == 1, "multi-value return unimplemented")do { static_assert( mozilla::detail::AssertionConditionType<
decltype(results.length() == 1)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(results.length() == 1))), 0)
)) { do { } while (false); MOZ_ReportAssertionFailure("results.length() == 1"
" (" "multi-value return unimplemented" ")", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 21890); AnnotateMozCrashReason("MOZ_ASSERT" "(" "results.length() == 1"
") (" "multi-value return unimplemented" ")"); do { *((volatile
int*)__null) = 21890; __attribute__((nomerge)) ::abort(); } while
(false); } } while (false)
;
21891 switch (results[0].kind()) {
21892 case wasm::ValType::I32:
21893 MOZ_ASSERT(lir->mir()->type() == MIRType::Int32)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(lir->mir()->type() == MIRType::Int32)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(lir->mir()->type() == MIRType::Int32))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("lir->mir()->type() == MIRType::Int32"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 21893); AnnotateMozCrashReason("MOZ_ASSERT" "(" "lir->mir()->type() == MIRType::Int32"
")"); do { *((volatile int*)__null) = 21893; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
21894 MOZ_ASSERT(ToRegister(lir->output()) == ReturnReg)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ToRegister(lir->output()) == ReturnReg)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(ToRegister(lir->output()) == ReturnReg))), 0))) { do { }
while (false); MOZ_ReportAssertionFailure("ToRegister(lir->output()) == ReturnReg"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 21894); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ToRegister(lir->output()) == ReturnReg"
")"); do { *((volatile int*)__null) = 21894; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
21895 break;
21896 case wasm::ValType::I64:
21897 MOZ_ASSERT(lir->mir()->type() == MIRType::Int64)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(lir->mir()->type() == MIRType::Int64)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(lir->mir()->type() == MIRType::Int64))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("lir->mir()->type() == MIRType::Int64"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 21897); AnnotateMozCrashReason("MOZ_ASSERT" "(" "lir->mir()->type() == MIRType::Int64"
")"); do { *((volatile int*)__null) = 21897; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
21898 MOZ_ASSERT(ToOutRegister64(lir) == ReturnReg64)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ToOutRegister64(lir) == ReturnReg64)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(ToOutRegister64(lir) == ReturnReg64
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"ToOutRegister64(lir) == ReturnReg64", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 21898); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ToOutRegister64(lir) == ReturnReg64"
")"); do { *((volatile int*)__null) = 21898; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
21899 break;
21900 case wasm::ValType::F32:
21901 MOZ_ASSERT(lir->mir()->type() == MIRType::Float32)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(lir->mir()->type() == MIRType::Float32)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(lir->mir()->type() == MIRType::Float32))), 0))) { do
{ } while (false); MOZ_ReportAssertionFailure("lir->mir()->type() == MIRType::Float32"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 21901); AnnotateMozCrashReason("MOZ_ASSERT" "(" "lir->mir()->type() == MIRType::Float32"
")"); do { *((volatile int*)__null) = 21901; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
21902 MOZ_ASSERT(ToFloatRegister(lir->output()) == ReturnFloat32Reg)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ToFloatRegister(lir->output()) == ReturnFloat32Reg
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(ToFloatRegister(lir->output()) == ReturnFloat32Reg
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"ToFloatRegister(lir->output()) == ReturnFloat32Reg", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 21902); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ToFloatRegister(lir->output()) == ReturnFloat32Reg"
")"); do { *((volatile int*)__null) = 21902; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
21903 break;
21904 case wasm::ValType::F64:
21905 MOZ_ASSERT(lir->mir()->type() == MIRType::Double)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(lir->mir()->type() == MIRType::Double)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(lir->mir()->type() == MIRType::Double))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("lir->mir()->type() == MIRType::Double"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 21905); AnnotateMozCrashReason("MOZ_ASSERT" "(" "lir->mir()->type() == MIRType::Double"
")"); do { *((volatile int*)__null) = 21905; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
21906 MOZ_ASSERT(ToFloatRegister(lir->output()) == ReturnDoubleReg)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ToFloatRegister(lir->output()) == ReturnDoubleReg
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(ToFloatRegister(lir->output()) == ReturnDoubleReg
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"ToFloatRegister(lir->output()) == ReturnDoubleReg", "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 21906); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ToFloatRegister(lir->output()) == ReturnDoubleReg"
")"); do { *((volatile int*)__null) = 21906; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
21907 break;
21908 case wasm::ValType::V128:
21909 MOZ_CRASH("unexpected return type when calling from ion to wasm")do { do { } while (false); MOZ_ReportCrash("" "unexpected return type when calling from ion to wasm"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 21909); AnnotateMozCrashReason("MOZ_CRASH(" "unexpected return type when calling from ion to wasm"
")"); do { *((volatile int*)__null) = 21909; __attribute__((
nomerge)) ::abort(); } while (false); } while (false)
;
21910 case wasm::ValType::Ref:
21911 // The wasm stubs layer unboxes anything that needs to be unboxed
21912 // and leaves it in a Value. A FuncRef/EqRef we could in principle
21913 // leave it as a raw object pointer but for now it complicates the
21914 // API to do so.
21915 MOZ_ASSERT(lir->mir()->type() == MIRType::Value)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(lir->mir()->type() == MIRType::Value)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(lir->mir()->type() == MIRType::Value))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("lir->mir()->type() == MIRType::Value"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 21915); AnnotateMozCrashReason("MOZ_ASSERT" "(" "lir->mir()->type() == MIRType::Value"
")"); do { *((volatile int*)__null) = 21915; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
21916 break;
21917 }
21918 }
21919
21920 WasmInstanceObject* instObj = lir->mir()->instanceObject();
21921
21922 Register scratch = ToRegister(lir->temp());
21923
21924 uint32_t callOffset;
21925 ensureOsiSpace();
21926 GenerateDirectCallFromJit(masm, funcExport, instObj->instance(), stackArgs,
21927 scratch, &callOffset);
21928
21929 // Add the instance object to the constant pool, so it is transferred to
21930 // the owning IonScript and so that it gets traced as long as the IonScript
21931 // lives.
21932
21933 uint32_t unused;
21934 masm.propagateOOM(graph.addConstantToPool(ObjectValue(*instObj), &unused));
21935
21936 markSafepointAt(callOffset, lir);
21937}
21938
21939void CodeGenerator::visitIonToWasmCall(LIonToWasmCall* lir) {
21940 emitIonToWasmCallBase(lir);
21941}
21942void CodeGenerator::visitIonToWasmCallV(LIonToWasmCallV* lir) {
21943 emitIonToWasmCallBase(lir);
21944}
21945void CodeGenerator::visitIonToWasmCallI64(LIonToWasmCallI64* lir) {
21946 emitIonToWasmCallBase(lir);
21947}
21948
21949void CodeGenerator::visitWasmNullConstant(LWasmNullConstant* lir) {
21950 masm.xorPtr(ToRegister(lir->output()), ToRegister(lir->output()));
21951}
21952
21953void CodeGenerator::visitWasmFence(LWasmFence* lir) {
21954 MOZ_ASSERT(gen->compilingWasm())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(gen->compilingWasm())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(gen->compilingWasm()))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("gen->compilingWasm()"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 21954); AnnotateMozCrashReason("MOZ_ASSERT" "(" "gen->compilingWasm()"
")"); do { *((volatile int*)__null) = 21954; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
21955 masm.memoryBarrier(MembarFull);
21956}
21957
21958void CodeGenerator::visitWasmAnyRefFromJSValue(LWasmAnyRefFromJSValue* lir) {
21959 ValueOperand input = ToValue(lir, LWasmAnyRefFromJSValue::InputIndex);
21960 Register output = ToRegister(lir->output());
21961 FloatRegister tempFloat = ToFloatRegister(lir->temp0());
21962
21963 using Fn = JSObject* (*)(JSContext* cx, HandleValue value);
21964 OutOfLineCode* oolBoxValue = oolCallVM<Fn, wasm::AnyRef::boxValue>(
21965 lir, ArgList(input), StoreRegisterTo(output));
21966 masm.convertValueToWasmAnyRef(input, output, tempFloat, oolBoxValue->entry());
21967 masm.bind(oolBoxValue->rejoin());
21968}
21969
21970void CodeGenerator::visitWasmAnyRefFromJSObject(LWasmAnyRefFromJSObject* lir) {
21971 Register input = ToRegister(lir->input());
21972 Register output = ToRegister(lir->output());
21973 masm.convertObjectToWasmAnyRef(input, output);
21974}
21975
21976void CodeGenerator::visitWasmAnyRefFromJSString(LWasmAnyRefFromJSString* lir) {
21977 Register input = ToRegister(lir->input());
21978 Register output = ToRegister(lir->output());
21979 masm.convertStringToWasmAnyRef(input, output);
21980}
21981
21982void CodeGenerator::visitWasmAnyRefIsJSString(LWasmAnyRefIsJSString* lir) {
21983 Register input = ToRegister(lir->input());
21984 Register output = ToRegister(lir->output());
21985 Register temp = ToRegister(lir->temp0());
21986 Label fallthrough;
21987 Label isJSString;
21988 masm.branchWasmAnyRefIsJSString(true, input, temp, &isJSString);
21989 masm.move32(Imm32(0), output);
21990 masm.jump(&fallthrough);
21991 masm.bind(&isJSString);
21992 masm.move32(Imm32(1), output);
21993 masm.bind(&fallthrough);
21994}
21995
21996void CodeGenerator::visitWasmTrapIfAnyRefIsNotJSString(
21997 LWasmTrapIfAnyRefIsNotJSString* lir) {
21998 Register input = ToRegister(lir->input());
21999 Register temp = ToRegister(lir->temp0());
22000 Label isJSString;
22001 masm.branchWasmAnyRefIsJSString(true, input, temp, &isJSString);
22002 masm.wasmTrap(lir->mir()->trap(), lir->mir()->bytecodeOffset());
22003 masm.bind(&isJSString);
22004}
22005
22006void CodeGenerator::visitWasmNewI31Ref(LWasmNewI31Ref* lir) {
22007 if (lir->value()->isConstant()) {
22008 // i31ref are often created with constants. If that's the case we will
22009 // do the operation statically here. This is similar to what is done
22010 // in masm.truncate32ToWasmI31Ref.
22011 Register output = ToRegister(lir->output());
22012 uint32_t value =
22013 static_cast<uint32_t>(lir->value()->toConstant()->toInt32());
22014 uintptr_t ptr = wasm::AnyRef::fromUint32Truncate(value).rawValue();
22015 masm.movePtr(ImmWord(ptr), output);
22016 } else {
22017 Register value = ToRegister(lir->value());
22018 Register output = ToRegister(lir->output());
22019 masm.truncate32ToWasmI31Ref(value, output);
22020 }
22021}
22022
22023void CodeGenerator::visitWasmI31RefGet(LWasmI31RefGet* lir) {
22024 Register value = ToRegister(lir->value());
22025 Register output = ToRegister(lir->output());
22026 if (lir->mir()->wideningOp() == wasm::FieldWideningOp::Signed) {
22027 masm.convertWasmI31RefTo32Signed(value, output);
22028 } else {
22029 masm.convertWasmI31RefTo32Unsigned(value, output);
22030 }
22031}
22032
22033#ifdef ENABLE_EXPLICIT_RESOURCE_MANAGEMENT
22034void CodeGenerator::visitAddDisposableResource(LAddDisposableResource* lir) {
22035 Register environment = ToRegister(lir->environment());
22036 ValueOperand resource = ToValue(lir, LAddDisposableResource::ResourceIndex);
22037 ValueOperand method = ToValue(lir, LAddDisposableResource::MethodIndex);
22038 Register needsClosure = ToRegister(lir->needsClosure());
22039 uint8_t hint = lir->hint();
22040
22041 pushArg(Imm32(hint));
22042 pushArg(needsClosure);
22043 pushArg(method);
22044 pushArg(resource);
22045 pushArg(environment);
22046
22047 using Fn = bool (*)(JSContext*, JS::Handle<JSObject*>, JS::Handle<JS::Value>,
22048 JS::Handle<JS::Value>, bool, UsingHint);
22049 callVM<Fn, js::AddDisposableResourceToCapability>(lir);
22050}
22051
22052void CodeGenerator::visitTakeDisposeCapability(LTakeDisposeCapability* lir) {
22053 Register environment = ToRegister(lir->environment());
22054 ValueOperand output = ToOutValue(lir);
22055
22056 Address capabilityAddr(
22057 environment, DisposableEnvironmentObject::offsetOfDisposeCapability());
22058 masm.loadValue(capabilityAddr, output);
22059 masm.storeValue(JS::UndefinedValue(), capabilityAddr);
22060}
22061
22062void CodeGenerator::visitCreateSuppressedError(LCreateSuppressedError* lir) {
22063 ValueOperand error = ToValue(lir, LCreateSuppressedError::ErrorIndex);
22064 ValueOperand suppressed =
22065 ToValue(lir, LCreateSuppressedError::SuppressedIndex);
22066
22067 pushArg(suppressed);
22068 pushArg(error);
22069
22070 using Fn = ErrorObject* (*)(JSContext*, JS::Handle<JS::Value>,
22071 JS::Handle<JS::Value>);
22072 callVM<Fn, js::CreateSuppressedError>(lir);
22073}
22074#endif
22075
22076#ifdef FUZZING_JS_FUZZILLI
22077void CodeGenerator::emitFuzzilliHashObject(LInstruction* lir, Register obj,
22078 Register output) {
22079 using Fn = void (*)(JSContext* cx, JSObject* obj, uint32_t* out);
22080 OutOfLineCode* ool = oolCallVM<Fn, FuzzilliHashObjectInl>(
22081 lir, ArgList(obj), StoreRegisterTo(output));
22082
22083 masm.jump(ool->entry());
22084 masm.bind(ool->rejoin());
22085}
22086
22087void CodeGenerator::emitFuzzilliHashBigInt(LInstruction* lir, Register bigInt,
22088 Register output) {
22089 LiveRegisterSet volatileRegs = liveVolatileRegs(lir);
22090 volatileRegs.takeUnchecked(output);
22091
22092 masm.PushRegsInMask(volatileRegs);
22093
22094 using Fn = uint32_t (*)(BigInt* bigInt);
22095 masm.setupUnalignedABICall(output);
22096 masm.passABIArg(bigInt);
22097 masm.callWithABI<Fn, js::FuzzilliHashBigInt>();
22098 masm.storeCallInt32Result(output);
22099
22100 masm.PopRegsInMask(volatileRegs);
22101}
22102
22103void CodeGenerator::visitFuzzilliHashV(LFuzzilliHashV* ins) {
22104 ValueOperand value = ToValue(ins, 0);
22105
22106 FloatRegister scratchFloat = ToFloatRegister(ins->getTemp(1));
22107 Register scratch = ToRegister(ins->getTemp(0));
22108 Register output = ToRegister(ins->output());
22109 MOZ_ASSERT(scratch != output)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(scratch != output)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(scratch != output))), 0))) {
do { } while (false); MOZ_ReportAssertionFailure("scratch != output"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 22109); AnnotateMozCrashReason("MOZ_ASSERT" "(" "scratch != output"
")"); do { *((volatile int*)__null) = 22109; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
22110
22111 Label hashDouble, done;
22112
22113 Label isInt32, isDouble, isNull, isUndefined, isBoolean, isBigInt, isObject;
22114 {
22115 ScratchTagScope tag(masm, value);
22116 masm.splitTagForTest(value, tag);
22117
22118 masm.branchTestInt32(Assembler::Equal, tag, &isInt32);
22119 masm.branchTestDouble(Assembler::Equal, tag, &isDouble);
22120 masm.branchTestNull(Assembler::Equal, tag, &isNull);
22121 masm.branchTestUndefined(Assembler::Equal, tag, &isUndefined);
22122 masm.branchTestBoolean(Assembler::Equal, tag, &isBoolean);
22123 masm.branchTestBigInt(Assembler::Equal, tag, &isBigInt);
22124 masm.branchTestObject(Assembler::Equal, tag, &isObject);
22125
22126 // Symbol or String.
22127 masm.move32(Imm32(0), output);
22128 masm.jump(&done);
22129 }
22130
22131 masm.bind(&isInt32);
22132 {
22133 masm.unboxInt32(value, scratch);
22134 masm.convertInt32ToDouble(scratch, scratchFloat);
22135 masm.jump(&hashDouble);
22136 }
22137
22138 masm.bind(&isDouble);
22139 {
22140 masm.unboxDouble(value, scratchFloat);
22141 masm.jump(&hashDouble);
22142 }
22143
22144 masm.bind(&isNull);
22145 {
22146 masm.loadConstantDouble(1.0, scratchFloat);
22147 masm.jump(&hashDouble);
22148 }
22149
22150 masm.bind(&isUndefined);
22151 {
22152 masm.loadConstantDouble(2.0, scratchFloat);
22153 masm.jump(&hashDouble);
22154 }
22155
22156 masm.bind(&isBoolean);
22157 {
22158 masm.unboxBoolean(value, scratch);
22159 masm.add32(Imm32(3), scratch);
22160 masm.convertInt32ToDouble(scratch, scratchFloat);
22161 masm.jump(&hashDouble);
22162 }
22163
22164 masm.bind(&isBigInt);
22165 {
22166 masm.unboxBigInt(value, scratch);
22167 emitFuzzilliHashBigInt(ins, scratch, output);
22168 masm.jump(&done);
22169 }
22170
22171 masm.bind(&isObject);
22172 {
22173 masm.unboxObject(value, scratch);
22174 emitFuzzilliHashObject(ins, scratch, output);
22175 masm.jump(&done);
22176 }
22177
22178 masm.bind(&hashDouble);
22179 masm.fuzzilliHashDouble(scratchFloat, output, scratch);
22180
22181 masm.bind(&done);
22182}
22183
22184void CodeGenerator::visitFuzzilliHashT(LFuzzilliHashT* ins) {
22185 const LAllocation* value = ins->value();
22186 MIRType mirType = ins->mir()->getOperand(0)->type();
22187
22188 Register scratch = ToTempRegisterOrInvalid(ins->getTemp(0));
22189 FloatRegister scratchFloat = ToTempFloatRegisterOrInvalid(ins->getTemp(1));
22190
22191 Register output = ToRegister(ins->output());
22192 MOZ_ASSERT(scratch != output)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(scratch != output)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(scratch != output))), 0))) {
do { } while (false); MOZ_ReportAssertionFailure("scratch != output"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 22192); AnnotateMozCrashReason("MOZ_ASSERT" "(" "scratch != output"
")"); do { *((volatile int*)__null) = 22192; __attribute__((
nomerge)) ::abort(); } while (false); } } while (false)
;
22193
22194 switch (mirType) {
22195 case MIRType::Undefined: {
22196 masm.loadConstantDouble(2.0, scratchFloat);
22197 masm.fuzzilliHashDouble(scratchFloat, output, scratch);
22198 break;
22199 }
22200
22201 case MIRType::Null: {
22202 masm.loadConstantDouble(1.0, scratchFloat);
22203 masm.fuzzilliHashDouble(scratchFloat, output, scratch);
22204 break;
22205 }
22206
22207 case MIRType::Int32: {
22208 masm.move32(ToRegister(value), scratch);
22209 masm.convertInt32ToDouble(scratch, scratchFloat);
22210 masm.fuzzilliHashDouble(scratchFloat, output, scratch);
22211 break;
22212 }
22213
22214 case MIRType::Double: {
22215 masm.moveDouble(ToFloatRegister(value), scratchFloat);
22216 masm.fuzzilliHashDouble(scratchFloat, output, scratch);
22217 break;
22218 }
22219
22220 case MIRType::Float32: {
22221 masm.convertFloat32ToDouble(ToFloatRegister(value), scratchFloat);
22222 masm.fuzzilliHashDouble(scratchFloat, output, scratch);
22223 break;
22224 }
22225
22226 case MIRType::Boolean: {
22227 masm.move32(ToRegister(value), scratch);
22228 masm.add32(Imm32(3), scratch);
22229 masm.convertInt32ToDouble(scratch, scratchFloat);
22230 masm.fuzzilliHashDouble(scratchFloat, output, scratch);
22231 break;
22232 }
22233
22234 case MIRType::BigInt: {
22235 emitFuzzilliHashBigInt(ins, ToRegister(value), output);
22236 break;
22237 }
22238
22239 case MIRType::Object: {
22240 emitFuzzilliHashObject(ins, ToRegister(value), output);
22241 break;
22242 }
22243
22244 default:
22245 MOZ_CRASH("unexpected type")do { do { } while (false); MOZ_ReportCrash("" "unexpected type"
, "/var/lib/jenkins/workspace/firefox-scan-build/js/src/jit/CodeGenerator.cpp"
, 22245); AnnotateMozCrashReason("MOZ_CRASH(" "unexpected type"
")"); do { *((volatile int*)__null) = 22245; __attribute__((
nomerge)) ::abort(); } while (false); } while (false)
;
22246 }
22247}
22248
22249void CodeGenerator::visitFuzzilliHashStore(LFuzzilliHashStore* ins) {
22250 Register value = ToRegister(ins->value());
22251 Register temp0 = ToRegister(ins->getTemp(0));
22252 Register temp1 = ToRegister(ins->getTemp(1));
22253
22254 masm.fuzzilliStoreHash(value, temp0, temp1);
22255}
22256#endif
22257
22258static_assert(!std::is_polymorphic_v<CodeGenerator>,
22259 "CodeGenerator should not have any virtual methods");
22260
22261} // namespace jit
22262} // namespace js