Bug Summary

File:root/firefox-clang/obj-x86_64-pc-linux-gnu/js/src/gc/./../../../../js/src/gc/BufferAllocator.cpp
Warning:line 1074, column 9
Value stored to 'buffer' during its initialization is never read

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -O3 -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name Unified_cpp_js_src_gc0.cpp -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -fhalf-no-semantic-interposition -mframe-pointer=all -relaxed-aliasing -ffp-contract=off -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -fdebug-compilation-dir=/root/firefox-clang/obj-x86_64-pc-linux-gnu/js/src/gc -fcoverage-compilation-dir=/root/firefox-clang/obj-x86_64-pc-linux-gnu/js/src/gc -resource-dir /usr/lib/llvm-23/lib/clang/23 -include /root/firefox-clang/config/gcc_hidden.h -include /root/firefox-clang/obj-x86_64-pc-linux-gnu/mozilla-config.h -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/stl_wrappers -D _GLIBCXX_ASSERTIONS=1 -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/system_wrappers -U _FORTIFY_SOURCE -D _FORTIFY_SOURCE=2 -D DEBUG=1 -D WASM_SUPPORTS_HUGE_MEMORY -D JS_CACHEIR_SPEW -D JS_STRUCTURED_SPEW -D JS_HAS_CTYPES -D FFI_BUILDING -D EXPORT_JS_API -D MOZ_HAS_MOZGLUE -D MOZ_SUPPORT_LEAKCHECKING -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/js/src -I /root/firefox-clang/js/src -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/include -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/include/nspr -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/include/nss -D MOZILLA_CLIENT -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/16/../../../../include/c++/16 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/16/../../../../include/x86_64-linux-gnu/c++/16 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/16/../../../../include/c++/16/backward -internal-isystem /usr/lib/llvm-23/lib/clang/23/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/16/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -Wno-error=pessimizing-move -Wno-error=large-by-value-copy=128 -Wno-error=implicit-int-float-conversion -Wno-error=thread-safety-analysis -Wno-error=tautological-type-limit-compare -Wno-invalid-offsetof -Wno-range-loop-analysis -Wno-deprecated-anon-enum-enum-conversion -Wno-deprecated-enum-enum-conversion -Wno-inline-new-delete -Wno-error=deprecated-declarations -Wno-error=array-bounds -Wno-error=free-nonheap-object -Wno-error=atomic-alignment -Wno-error=deprecated-builtins -Wno-psabi -Wno-error=builtin-macro-redefined -Wno-vla-cxx-extension -Wno-unknown-warning-option -Wno-character-conversion -std=gnu++20 -fdeprecated-macro -ferror-limit 19 -fstrict-flex-arrays=1 -stack-protector 2 -fstack-clash-protection -ftrivial-auto-var-init=pattern -fno-rtti -fgnuc-version=4.2.1 -fno-implicit-modules -fskip-odr-check-in-gmf -fno-sized-deallocation -fno-aligned-allocation -fdiagnostics-absolute-paths -vectorize-loops -vectorize-slp -analyzer-checker optin.performance.Padding -analyzer-output=html -analyzer-config stable-report-filename=true -mllvm -dwarf-linkage-names=Abstract -faddrsig -fdwarf2-cfi-asm -o /tmp/scan-build-2026-09-01-224014-2642839-1 -x c++ Unified_cpp_js_src_gc0.cpp
1/* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4
5#include "mozilla/Likely.h"
6#include "mozilla/ScopeExit.h"
7
8#include <bit>
9
10#include "gc/BufferAllocator-inl.h"
11
12#ifdef XP_DARWIN
13# include <mach/mach_init.h>
14# include <mach/vm_map.h>
15#endif
16
17#include "gc/BufferAllocatorInternals.h"
18#include "gc/GCInternals.h"
19#include "gc/GCLock.h"
20#include "gc/PublicIterators.h"
21#include "gc/Tenuring.h"
22#include "gc/Zone.h"
23#include "js/HeapAPI.h"
24#include "util/Poison.h"
25
26#include "gc/Heap-inl.h"
27#include "gc/Marking-inl.h"
28
29using namespace js;
30using namespace js::gc;
31
32namespace js::gc {
33
34AutoLockBufferAllocator::AutoLockBufferAllocator(
35 BufferAllocatorRuntime* runtime)
36 : LockGuard(runtime->lock) {}
37
38static void CheckHighBitsOfPointer(void* ptr) {
39#ifdef JS_64BIT1
40 // We require bit 48 and higher be clear.
41 MOZ_DIAGNOSTIC_ASSERT((uintptr_t(ptr) >> 47) == 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype((uintptr_t(ptr) >> 47) == 0)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!((uintptr_t(ptr) >> 47)
== 0))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("(uintptr_t(ptr) >> 47) == 0", "./../../../../js/src/gc/BufferAllocator.cpp"
, 41); AnnotateMozCrashReason("MOZ_DIAGNOSTIC_ASSERT" "(" "(uintptr_t(ptr) >> 47) == 0"
")"); do { MOZ_CrashSequence(__null, 41); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
42#endif
43}
44
45BufferAllocator::FreeLists::FreeLists(FreeLists&& other) {
46 MOZ_ASSERT(this != &other)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(this != &other)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(this != &other))), 0))) {
do { } while (false); MOZ_ReportAssertionFailure("this != &other"
, "./../../../../js/src/gc/BufferAllocator.cpp", 46); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "this != &other" ")"); do { MOZ_CrashSequence
(__null, 46); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
47 assertEmpty();
48 std::swap(lists, other.lists);
49 std::swap(available, other.available);
50 other.assertEmpty();
51}
52
53BufferAllocator::FreeLists& BufferAllocator::FreeLists::operator=(
54 FreeLists&& other) {
55 MOZ_ASSERT(this != &other)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(this != &other)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(this != &other))), 0))) {
do { } while (false); MOZ_ReportAssertionFailure("this != &other"
, "./../../../../js/src/gc/BufferAllocator.cpp", 55); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "this != &other" ")"); do { MOZ_CrashSequence
(__null, 55); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
56 assertEmpty();
57 std::swap(lists, other.lists);
58 std::swap(available, other.available);
59 other.assertEmpty();
60 return *this;
61}
62
63BufferAllocator::FreeLists::FreeListIter
64BufferAllocator::FreeLists::freeListIter() {
65 return FreeListIter(*this);
66}
67
68BufferAllocator::FreeLists::FreeRegionIter
69BufferAllocator::FreeLists::freeRegionIter() {
70 return FreeRegionIter(*this);
71}
72
73bool BufferAllocator::FreeLists::hasSizeClass(size_t sizeClass) const {
74 MOZ_ASSERT(sizeClass <= MaxMediumAllocClass)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(sizeClass <= MaxMediumAllocClass)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(sizeClass <= MaxMediumAllocClass
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"sizeClass <= MaxMediumAllocClass", "./../../../../js/src/gc/BufferAllocator.cpp"
, 74); AnnotateMozCrashReason("MOZ_ASSERT" "(" "sizeClass <= MaxMediumAllocClass"
")"); do { MOZ_CrashSequence(__null, 74); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
75 return available[sizeClass];
76}
77bool BufferAllocator::FreeLists::hasAnySizeClass(size_t minSizeClass,
78 size_t maxSizeClass) const {
79 return getFirstAvailableSizeClass(minSizeClass, maxSizeClass) != SIZE_MAX(18446744073709551615UL);
80}
81
82size_t BufferAllocator::FreeLists::getFirstAvailableSizeClass(
83 size_t minSizeClass, size_t maxSizeClass) const {
84 MOZ_ASSERT(maxSizeClass <= MaxMediumAllocClass)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(maxSizeClass <= MaxMediumAllocClass)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(maxSizeClass <= MaxMediumAllocClass))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("maxSizeClass <= MaxMediumAllocClass"
, "./../../../../js/src/gc/BufferAllocator.cpp", 84); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "maxSizeClass <= MaxMediumAllocClass" ")"
); do { MOZ_CrashSequence(__null, 84); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
85
86 size_t result = available.FindNext(minSizeClass);
87 MOZ_ASSERT(result >= minSizeClass)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(result >= minSizeClass)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(result >= minSizeClass)))
, 0))) { do { } while (false); MOZ_ReportAssertionFailure("result >= minSizeClass"
, "./../../../../js/src/gc/BufferAllocator.cpp", 87); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "result >= minSizeClass" ")"); do { MOZ_CrashSequence
(__null, 87); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
88 MOZ_ASSERT_IF(result != SIZE_MAX, !lists[result].isEmpty())do { if (result != (18446744073709551615UL)) { do { static_assert
( mozilla::detail::AssertionConditionType<decltype(!lists[
result].isEmpty())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!lists[result].isEmpty()))),
0))) { do { } while (false); MOZ_ReportAssertionFailure("!lists[result].isEmpty()"
, "./../../../../js/src/gc/BufferAllocator.cpp", 88); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!lists[result].isEmpty()" ")"); do { MOZ_CrashSequence
(__null, 88); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false); } } while (false);
89
90 if (result > maxSizeClass) {
91 return SIZE_MAX(18446744073709551615UL);
92 }
93
94 return result;
95}
96
97size_t BufferAllocator::FreeLists::getLastAvailableSizeClass(
98 size_t minSizeClass, size_t maxSizeClass) const {
99 MOZ_ASSERT(maxSizeClass <= MaxMediumAllocClass)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(maxSizeClass <= MaxMediumAllocClass)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(maxSizeClass <= MaxMediumAllocClass))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("maxSizeClass <= MaxMediumAllocClass"
, "./../../../../js/src/gc/BufferAllocator.cpp", 99); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "maxSizeClass <= MaxMediumAllocClass" ")"
); do { MOZ_CrashSequence(__null, 99); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
100
101 size_t result = available.FindPrev(maxSizeClass);
102 MOZ_ASSERT(result <= maxSizeClass || result == SIZE_MAX)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(result <= maxSizeClass || result == (18446744073709551615UL
))>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(result <= maxSizeClass || result == (18446744073709551615UL
)))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("result <= maxSizeClass || result == (18446744073709551615UL)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 102); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "result <= maxSizeClass || result == (18446744073709551615UL)"
")"); do { MOZ_CrashSequence(__null, 102); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
103 MOZ_ASSERT_IF(result != SIZE_MAX, !lists[result].isEmpty())do { if (result != (18446744073709551615UL)) { do { static_assert
( mozilla::detail::AssertionConditionType<decltype(!lists[
result].isEmpty())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!lists[result].isEmpty()))),
0))) { do { } while (false); MOZ_ReportAssertionFailure("!lists[result].isEmpty()"
, "./../../../../js/src/gc/BufferAllocator.cpp", 103); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!lists[result].isEmpty()" ")"); do { MOZ_CrashSequence
(__null, 103); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false); } } while (false);
104
105 if (result < minSizeClass) {
106 return SIZE_MAX(18446744073709551615UL);
107 }
108
109 return result;
110}
111
112BufferAllocator::FreeRegion* BufferAllocator::FreeLists::getFirstRegion(
113 size_t sizeClass) {
114 MOZ_ASSERT(!lists[sizeClass].isEmpty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!lists[sizeClass].isEmpty())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!lists[sizeClass].isEmpty())
)), 0))) { do { } while (false); MOZ_ReportAssertionFailure("!lists[sizeClass].isEmpty()"
, "./../../../../js/src/gc/BufferAllocator.cpp", 114); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!lists[sizeClass].isEmpty()" ")"); do { MOZ_CrashSequence
(__null, 114); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
115 return lists[sizeClass].getFirst();
116}
117
118void BufferAllocator::FreeLists::pushFront(FreeRegion* region, SizeKind kind) {
119 MOZ_ASSERT(region)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(region)>::isValid, "invalid assertion condition")
; if ((__builtin_expect(!!(!(!!(region))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("region", "./../../../../js/src/gc/BufferAllocator.cpp"
, 119); AnnotateMozCrashReason("MOZ_ASSERT" "(" "region" ")")
; do { MOZ_CrashSequence(__null, 119); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
120 size_t sizeClass = SizeClassForFreeRegion(region->size(), kind);
121 CheckFreeRegionClass(region, sizeClass);
122 pushFront(sizeClass, region);
123}
124
125void BufferAllocator::FreeLists::pushBack(FreeRegion* region, SizeKind kind) {
126 MOZ_ASSERT(region)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(region)>::isValid, "invalid assertion condition")
; if ((__builtin_expect(!!(!(!!(region))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("region", "./../../../../js/src/gc/BufferAllocator.cpp"
, 126); AnnotateMozCrashReason("MOZ_ASSERT" "(" "region" ")")
; do { MOZ_CrashSequence(__null, 126); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
127 size_t sizeClass = SizeClassForFreeRegion(region->size(), kind);
128 CheckFreeRegionClass(region, sizeClass);
129 pushBack(sizeClass, region);
130}
131
132void BufferAllocator::FreeLists::pushFront(size_t sizeClass,
133 FreeRegion* region) {
134 MOZ_ASSERT(sizeClass < AllocSizeClasses)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(sizeClass < AllocSizeClasses)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(sizeClass < AllocSizeClasses
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"sizeClass < AllocSizeClasses", "./../../../../js/src/gc/BufferAllocator.cpp"
, 134); AnnotateMozCrashReason("MOZ_ASSERT" "(" "sizeClass < AllocSizeClasses"
")"); do { MOZ_CrashSequence(__null, 134); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
135 lists[sizeClass].pushFront(region);
136 available[sizeClass] = true;
137}
138
139void BufferAllocator::FreeLists::pushBack(size_t sizeClass,
140 FreeRegion* region) {
141 MOZ_ASSERT(sizeClass < AllocSizeClasses)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(sizeClass < AllocSizeClasses)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(sizeClass < AllocSizeClasses
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"sizeClass < AllocSizeClasses", "./../../../../js/src/gc/BufferAllocator.cpp"
, 141); AnnotateMozCrashReason("MOZ_ASSERT" "(" "sizeClass < AllocSizeClasses"
")"); do { MOZ_CrashSequence(__null, 141); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
142 lists[sizeClass].pushBack(region);
143 available[sizeClass] = true;
144}
145
146void BufferAllocator::FreeLists::append(FreeLists&& other) {
147 for (size_t i = 0; i < AllocSizeClasses; i++) {
148 if (!other.lists[i].isEmpty()) {
149 lists[i].append(std::move(other.lists[i]));
150 available[i] = true;
151 }
152 }
153 other.available.ResetAll();
154 other.assertEmpty();
155}
156
157void BufferAllocator::FreeLists::remove(size_t sizeClass, FreeRegion* region) {
158 MOZ_ASSERT(sizeClass < AllocSizeClasses)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(sizeClass < AllocSizeClasses)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(sizeClass < AllocSizeClasses
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"sizeClass < AllocSizeClasses", "./../../../../js/src/gc/BufferAllocator.cpp"
, 158); AnnotateMozCrashReason("MOZ_ASSERT" "(" "sizeClass < AllocSizeClasses"
")"); do { MOZ_CrashSequence(__null, 158); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
159 lists[sizeClass].remove(region);
160 available[sizeClass] = !lists[sizeClass].isEmpty();
161}
162
163void BufferAllocator::FreeLists::clear() {
164 for (auto freeList = freeListIter(); !freeList.done(); freeList.next()) {
165 new (&freeList.get()) FreeList; // clear() is less efficient.
166 }
167 available.ResetAll();
168}
169
170template <typename Func>
171void BufferAllocator::FreeLists::forEachRegion(Func&& func) {
172 for (size_t i = 0; i <= MaxMediumAllocClass; i++) {
173 FreeList& freeList = lists[i];
174 FreeRegion* region = freeList.getFirst();
175 while (region) {
176 FreeRegion* next = region->getNext();
177 func(freeList, i, region);
178 region = next;
179 }
180 available[i] = !freeList.isEmpty();
181 }
182}
183
184inline void BufferAllocator::FreeLists::assertEmpty() const {
185#ifdef DEBUG1
186 for (size_t i = 0; i < AllocSizeClasses; i++) {
187 MOZ_ASSERT(lists[i].isEmpty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(lists[i].isEmpty())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(lists[i].isEmpty()))), 0))) {
do { } while (false); MOZ_ReportAssertionFailure("lists[i].isEmpty()"
, "./../../../../js/src/gc/BufferAllocator.cpp", 187); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "lists[i].isEmpty()" ")"); do { MOZ_CrashSequence
(__null, 187); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
188 }
189 MOZ_ASSERT(available.IsEmpty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(available.IsEmpty())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(available.IsEmpty()))), 0)))
{ do { } while (false); MOZ_ReportAssertionFailure("available.IsEmpty()"
, "./../../../../js/src/gc/BufferAllocator.cpp", 189); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "available.IsEmpty()" ")"); do { MOZ_CrashSequence
(__null, 189); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
190#endif
191}
192
193inline void BufferAllocator::FreeLists::assertContains(
194 size_t sizeClass, FreeRegion* region) const {
195#ifdef DEBUG1
196 MOZ_ASSERT(available[sizeClass])do { static_assert( mozilla::detail::AssertionConditionType<
decltype(available[sizeClass])>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(available[sizeClass]))), 0))
) { do { } while (false); MOZ_ReportAssertionFailure("available[sizeClass]"
, "./../../../../js/src/gc/BufferAllocator.cpp", 196); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "available[sizeClass]" ")"); do { MOZ_CrashSequence
(__null, 196); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
197 MOZ_ASSERT(lists[sizeClass].contains(region))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(lists[sizeClass].contains(region))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(lists[sizeClass].contains(region
)))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("lists[sizeClass].contains(region)", "./../../../../js/src/gc/BufferAllocator.cpp"
, 197); AnnotateMozCrashReason("MOZ_ASSERT" "(" "lists[sizeClass].contains(region)"
")"); do { MOZ_CrashSequence(__null, 197); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
198#endif
199}
200
201inline void BufferAllocator::FreeLists::checkAvailable() const {
202#ifdef DEBUG1
203 for (size_t i = 0; i < AllocSizeClasses; i++) {
204 MOZ_ASSERT(available[i] == !lists[i].isEmpty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(available[i] == !lists[i].isEmpty())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(available[i] == !lists[i].isEmpty
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("available[i] == !lists[i].isEmpty()", "./../../../../js/src/gc/BufferAllocator.cpp"
, 204); AnnotateMozCrashReason("MOZ_ASSERT" "(" "available[i] == !lists[i].isEmpty()"
")"); do { MOZ_CrashSequence(__null, 204); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
205 }
206#endif
207}
208
209BufferAllocator::ChunkLists::ChunkIter
210BufferAllocator::ChunkLists::chunkIter() {
211 return ChunkIter(*this);
212}
213
214BufferChunk* BufferAllocator::ChunkLists::popFirstChunk(ContentKind kind,
215 size_t sizeClass) {
216 MOZ_ASSERT(sizeClass < AllocSizeClasses)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(sizeClass < AllocSizeClasses)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(sizeClass < AllocSizeClasses
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"sizeClass < AllocSizeClasses", "./../../../../js/src/gc/BufferAllocator.cpp"
, 216); AnnotateMozCrashReason("MOZ_ASSERT" "(" "sizeClass < AllocSizeClasses"
")"); do { MOZ_CrashSequence(__null, 216); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
217 MOZ_ASSERT(availableSizeClasses(kind)[(sizeClass)])do { static_assert( mozilla::detail::AssertionConditionType<
decltype(availableSizeClasses(kind)[(sizeClass)])>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(availableSizeClasses(kind)[(sizeClass)]))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("availableSizeClasses(kind)[(sizeClass)]"
, "./../../../../js/src/gc/BufferAllocator.cpp", 217); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "availableSizeClasses(kind)[(sizeClass)]" ")"
); do { MOZ_CrashSequence(__null, 217); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
218
219 auto& list = lists[sizeClass];
220 MOZ_ASSERT(!list.isEmpty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!list.isEmpty())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!list.isEmpty()))), 0))) { do
{ } while (false); MOZ_ReportAssertionFailure("!list.isEmpty()"
, "./../../../../js/src/gc/BufferAllocator.cpp", 220); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!list.isEmpty()" ")"); do { MOZ_CrashSequence
(__null, 220); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
221
222 BufferChunk* chunk =
223 kind == ContentKind::Mixed ? list.getFirst() : list.getLast();
224 MOZ_ASSERT(chunk->kind() == kind)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(chunk->kind() == kind)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(chunk->kind() == kind))),
0))) { do { } while (false); MOZ_ReportAssertionFailure("chunk->kind() == kind"
, "./../../../../js/src/gc/BufferAllocator.cpp", 224); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "chunk->kind() == kind" ")"); do { MOZ_CrashSequence
(__null, 224); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
225
226 remove(kind, sizeClass, chunk);
227
228 return chunk;
229}
230
231void BufferAllocator::ChunkLists::remove(size_t sizeClass, BufferChunk* chunk) {
232 remove(chunk->kind(), sizeClass, chunk);
233}
234
235void BufferAllocator::ChunkLists::remove(ContentKind kind, size_t sizeClass,
236 BufferChunk* chunk) {
237 MOZ_ASSERT(chunk->kind() == kind)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(chunk->kind() == kind)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(chunk->kind() == kind))),
0))) { do { } while (false); MOZ_ReportAssertionFailure("chunk->kind() == kind"
, "./../../../../js/src/gc/BufferAllocator.cpp", 237); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "chunk->kind() == kind" ")"); do { MOZ_CrashSequence
(__null, 237); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
238 MOZ_ASSERT(sizeClass <= AllocSizeClasses)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(sizeClass <= AllocSizeClasses)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(sizeClass <= AllocSizeClasses
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"sizeClass <= AllocSizeClasses", "./../../../../js/src/gc/BufferAllocator.cpp"
, 238); AnnotateMozCrashReason("MOZ_ASSERT" "(" "sizeClass <= AllocSizeClasses"
")"); do { MOZ_CrashSequence(__null, 238); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
239
240 auto& list = lists[sizeClass];
241 MOZ_ASSERT(!list.isEmpty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!list.isEmpty())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!list.isEmpty()))), 0))) { do
{ } while (false); MOZ_ReportAssertionFailure("!list.isEmpty()"
, "./../../../../js/src/gc/BufferAllocator.cpp", 241); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!list.isEmpty()" ")"); do { MOZ_CrashSequence
(__null, 241); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
242 list.remove(chunk);
243
244 availableMixed[sizeClass] =
245 !list.isEmpty() && list.getFirst()->kind() == ContentKind::Mixed;
246 availableTenured[sizeClass] =
247 !list.isEmpty() && list.getLast()->kind() == ContentKind::Tenured;
248
249 checkAvailable();
250}
251
252void BufferAllocator::ChunkLists::addChunk(BufferChunk* chunk) {
253 addChunk(chunk->sizeClassForAvailableLists(), chunk);
254}
255
256void BufferAllocator::ChunkLists::addChunk(size_t sizeClass,
257 BufferChunk* chunk) {
258 if (chunk->kind() == ContentKind::Mixed) {
259 addMixedChunk(sizeClass, chunk);
260 return;
261 }
262
263 addTenuredChunk(sizeClass, chunk);
264}
265
266void BufferAllocator::ChunkLists::addMixedChunk(size_t sizeClass,
267 BufferChunk* chunk) {
268 MOZ_ASSERT(chunk->kind() == ContentKind::Mixed)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(chunk->kind() == ContentKind::Mixed)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(chunk->kind() == ContentKind::Mixed))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("chunk->kind() == ContentKind::Mixed"
, "./../../../../js/src/gc/BufferAllocator.cpp", 268); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "chunk->kind() == ContentKind::Mixed" ")"
); do { MOZ_CrashSequence(__null, 268); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
269 MOZ_ASSERT(chunk->ownsFreeLists)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(chunk->ownsFreeLists)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(chunk->ownsFreeLists))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("chunk->ownsFreeLists"
, "./../../../../js/src/gc/BufferAllocator.cpp", 269); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "chunk->ownsFreeLists" ")"); do { MOZ_CrashSequence
(__null, 269); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
270 MOZ_ASSERT(sizeClass <= AllocSizeClasses)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(sizeClass <= AllocSizeClasses)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(sizeClass <= AllocSizeClasses
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"sizeClass <= AllocSizeClasses", "./../../../../js/src/gc/BufferAllocator.cpp"
, 270); AnnotateMozCrashReason("MOZ_ASSERT" "(" "sizeClass <= AllocSizeClasses"
")"); do { MOZ_CrashSequence(__null, 270); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
271 lists[sizeClass].pushFront(chunk);
272 availableMixed[sizeClass] = true;
273}
274
275void BufferAllocator::ChunkLists::addTenuredChunk(BufferChunk* chunk) {
276 addTenuredChunk(chunk->sizeClassForAvailableLists(), chunk);
277}
278
279void BufferAllocator::ChunkLists::addTenuredChunk(size_t sizeClass,
280 BufferChunk* chunk) {
281 MOZ_ASSERT(chunk->kind() == ContentKind::Tenured)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(chunk->kind() == ContentKind::Tenured)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(chunk->kind() == ContentKind::Tenured))), 0))) { do { }
while (false); MOZ_ReportAssertionFailure("chunk->kind() == ContentKind::Tenured"
, "./../../../../js/src/gc/BufferAllocator.cpp", 281); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "chunk->kind() == ContentKind::Tenured" ")"
); do { MOZ_CrashSequence(__null, 281); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
282 MOZ_ASSERT(chunk->ownsFreeLists)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(chunk->ownsFreeLists)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(chunk->ownsFreeLists))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("chunk->ownsFreeLists"
, "./../../../../js/src/gc/BufferAllocator.cpp", 282); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "chunk->ownsFreeLists" ")"); do { MOZ_CrashSequence
(__null, 282); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
283 MOZ_ASSERT(sizeClass <= AllocSizeClasses)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(sizeClass <= AllocSizeClasses)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(sizeClass <= AllocSizeClasses
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"sizeClass <= AllocSizeClasses", "./../../../../js/src/gc/BufferAllocator.cpp"
, 283); AnnotateMozCrashReason("MOZ_ASSERT" "(" "sizeClass <= AllocSizeClasses"
")"); do { MOZ_CrashSequence(__null, 283); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
284 lists[sizeClass].pushBack(chunk);
285 availableTenured[sizeClass] = true;
286}
287
288// Extract chunks to new lists ordered so that chunks with the largest
289// free regions are last.
290
291BufferAllocator::BufferChunkList
292BufferAllocator::ChunkLists::extractAllChunks() {
293 BufferChunkList result = std::move(lists[FullChunkSizeClass]);
294 for (auto iter = ChunkListIter(*this); !iter.done(); iter.next()) {
295 result.append(std::move(iter.get()));
296 }
297 availableMixed.ResetAll();
298 availableTenured.ResetAll();
299 return result;
300}
301
302BufferAllocator::BufferChunkList
303BufferAllocator::ChunkLists::extractMixedChunks() {
304 BufferChunkList result;
305
306 auto extractMixedPrefix = [&](BufferChunkList& list) {
307 BufferChunk* lastMixedChunk = nullptr;
308 BufferChunk* chunk = list.getFirst();
309 while (chunk && chunk->kind() == ContentKind::Mixed) {
310 lastMixedChunk = chunk;
311 chunk = chunk->getNext();
312 }
313 if (lastMixedChunk) {
314 result.append(list.removeRange(list.getFirst(), lastMixedChunk));
315 }
316 };
317
318 if (availableMixed[FullChunkSizeClass]) {
319 extractMixedPrefix(lists[FullChunkSizeClass]);
320 availableMixed[FullChunkSizeClass] = false;
321 }
322
323 for (auto iter = ChunkListIter(*this, availableMixed); !iter.done();
324 iter.next()) {
325 extractMixedPrefix(iter.get());
326 }
327 availableMixed.ResetAll();
328
329 checkAvailable();
330 return result;
331}
332
333inline bool BufferAllocator::ChunkLists::isEmpty() const {
334 checkAvailable();
335 return availableMixed.IsEmpty() && availableTenured.IsEmpty();
336}
337
338inline void BufferAllocator::ChunkLists::checkAvailable() const {
339#ifdef DEBUG1
340 for (size_t i = 0; i < AllocSizeClasses; i++) {
341 bool hasMixed = false;
342 bool hasTenured = false;
343 for (const BufferChunk* chunk : lists[i]) {
344 MOZ_ASSERT_IF(hasTenured, chunk->kind() == ContentKind::Tenured)do { if (hasTenured) { do { static_assert( mozilla::detail::AssertionConditionType
<decltype(chunk->kind() == ContentKind::Tenured)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(chunk->kind() == ContentKind::Tenured))), 0))) { do { }
while (false); MOZ_ReportAssertionFailure("chunk->kind() == ContentKind::Tenured"
, "./../../../../js/src/gc/BufferAllocator.cpp", 344); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "chunk->kind() == ContentKind::Tenured" ")"
); do { MOZ_CrashSequence(__null, 344); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false);
345 if (chunk->kind() == ContentKind::Mixed) {
346 hasMixed = true;
347 } else {
348 hasTenured = true;
349 }
350 }
351 MOZ_ASSERT(availableMixed[i] == hasMixed)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(availableMixed[i] == hasMixed)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(availableMixed[i] == hasMixed
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"availableMixed[i] == hasMixed", "./../../../../js/src/gc/BufferAllocator.cpp"
, 351); AnnotateMozCrashReason("MOZ_ASSERT" "(" "availableMixed[i] == hasMixed"
")"); do { MOZ_CrashSequence(__null, 351); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
352 MOZ_ASSERT(availableTenured[i] == hasTenured)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(availableTenured[i] == hasTenured)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(availableTenured[i] == hasTenured
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"availableTenured[i] == hasTenured", "./../../../../js/src/gc/BufferAllocator.cpp"
, 352); AnnotateMozCrashReason("MOZ_ASSERT" "(" "availableTenured[i] == hasTenured"
")"); do { MOZ_CrashSequence(__null, 352); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
353 }
354#endif
355}
356
357} // namespace js::gc
358
359MOZ_ALWAYS_INLINEinline void PoisonAlloc(void* alloc, uint8_t value, size_t bytes,
360 MemCheckKind kind) {
361#if !(defined(NIGHTLY_BUILD1) || defined(MOZ_DEV_EDITION) || defined(DEBUG1))
362 // Limit poisoning in release builds.
363 bytes = std::min(bytes, size_t(256));
364#endif
365 AlwaysPoison(alloc, value, bytes, kind);
366}
367
368template <typename D, size_t S, size_t G>
369void AllocSpace<D, S, G>::setAllocated(void* alloc, size_t bytes,
370 bool allocated) {
371 MOZ_ASSERT(bytes != 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(bytes != 0)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(bytes != 0))), 0))) { do { }
while (false); MOZ_ReportAssertionFailure("bytes != 0", "./../../../../js/src/gc/BufferAllocator.cpp"
, 371); AnnotateMozCrashReason("MOZ_ASSERT" "(" "bytes != 0" ")"
); do { MOZ_CrashSequence(__null, 371); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
372 MOZ_ASSERT(bytes % GranularityBytes == 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(bytes % GranularityBytes == 0)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(bytes % GranularityBytes == 0
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"bytes % GranularityBytes == 0", "./../../../../js/src/gc/BufferAllocator.cpp"
, 372); AnnotateMozCrashReason("MOZ_ASSERT" "(" "bytes % GranularityBytes == 0"
")"); do { MOZ_CrashSequence(__null, 372); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
373
374 size_t startBit = ptrToIndex(alloc);
375 size_t endBit = endBitIndex(startBit, bytes);
376 MOZ_ASSERT(allocStartBitmap.ref()[startBit] != allocated)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(allocStartBitmap.ref()[startBit] != allocated)>::
isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(allocStartBitmap.ref()[startBit] != allocated))), 0)
)) { do { } while (false); MOZ_ReportAssertionFailure("allocStartBitmap.ref()[startBit] != allocated"
, "./../../../../js/src/gc/BufferAllocator.cpp", 376); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "allocStartBitmap.ref()[startBit] != allocated"
")"); do { MOZ_CrashSequence(__null, 376); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
377 MOZ_ASSERT(allocStartBitmap.ref()[startBit] == allocEndBitmap.ref()[endBit])do { static_assert( mozilla::detail::AssertionConditionType<
decltype(allocStartBitmap.ref()[startBit] == allocEndBitmap.ref
()[endBit])>::isValid, "invalid assertion condition"); if (
(__builtin_expect(!!(!(!!(allocStartBitmap.ref()[startBit] ==
allocEndBitmap.ref()[endBit]))), 0))) { do { } while (false)
; MOZ_ReportAssertionFailure("allocStartBitmap.ref()[startBit] == allocEndBitmap.ref()[endBit]"
, "./../../../../js/src/gc/BufferAllocator.cpp", 377); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "allocStartBitmap.ref()[startBit] == allocEndBitmap.ref()[endBit]"
")"); do { MOZ_CrashSequence(__null, 377); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
378 MOZ_ASSERT(findEndBit(startBit) >= endBit)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(findEndBit(startBit) >= endBit)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(findEndBit(startBit) >= endBit
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"findEndBit(startBit) >= endBit", "./../../../../js/src/gc/BufferAllocator.cpp"
, 378); AnnotateMozCrashReason("MOZ_ASSERT" "(" "findEndBit(startBit) >= endBit"
")"); do { MOZ_CrashSequence(__null, 378); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
379
380 allocStartBitmap.ref()[startBit] = allocated;
381 allocEndBitmap.ref()[endBit] = allocated;
382}
383
384template <typename D, size_t S, size_t G>
385void AllocSpace<D, S, G>::setDeallocated(void* alloc, size_t bytes) {
386 MOZ_ASSERT(allocBytes(alloc) == bytes)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(allocBytes(alloc) == bytes)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(allocBytes(alloc) == bytes))
), 0))) { do { } while (false); MOZ_ReportAssertionFailure("allocBytes(alloc) == bytes"
, "./../../../../js/src/gc/BufferAllocator.cpp", 386); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "allocBytes(alloc) == bytes" ")"); do { MOZ_CrashSequence
(__null, 386); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
387 setNurseryOwned(alloc, false);
388 setAllocated(alloc, bytes, false);
389}
390
391template <typename D, size_t S, size_t G>
392void AllocSpace<D, S, G>::updateEndOffset(void* alloc, size_t oldBytes,
393 size_t newBytes) {
394 MOZ_ASSERT(isAllocated(alloc))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(isAllocated(alloc))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(isAllocated(alloc)))), 0))) {
do { } while (false); MOZ_ReportAssertionFailure("isAllocated(alloc)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 394); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "isAllocated(alloc)" ")"); do { MOZ_CrashSequence
(__null, 394); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
395 MOZ_ASSERT(newBytes != oldBytes)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(newBytes != oldBytes)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(newBytes != oldBytes))), 0))
) { do { } while (false); MOZ_ReportAssertionFailure("newBytes != oldBytes"
, "./../../../../js/src/gc/BufferAllocator.cpp", 395); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "newBytes != oldBytes" ")"); do { MOZ_CrashSequence
(__null, 395); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
396
397 size_t startBit = ptrToIndex(alloc);
398 size_t oldEndBit = endBitIndex(startBit, oldBytes);
399 MOZ_ASSERT(allocEndBitmap.ref()[oldEndBit])do { static_assert( mozilla::detail::AssertionConditionType<
decltype(allocEndBitmap.ref()[oldEndBit])>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(allocEndBitmap.ref()[oldEndBit
]))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("allocEndBitmap.ref()[oldEndBit]", "./../../../../js/src/gc/BufferAllocator.cpp"
, 399); AnnotateMozCrashReason("MOZ_ASSERT" "(" "allocEndBitmap.ref()[oldEndBit]"
")"); do { MOZ_CrashSequence(__null, 399); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
400 allocEndBitmap.ref()[oldEndBit] = false;
401
402 size_t newEndBit = endBitIndex(startBit, newBytes);
403 MOZ_ASSERT(allocStartBitmap.ref().FindNext(startBit + 1) > newEndBit)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(allocStartBitmap.ref().FindNext(startBit + 1) > newEndBit
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(allocStartBitmap.ref().FindNext(startBit + 1) > newEndBit
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"allocStartBitmap.ref().FindNext(startBit + 1) > newEndBit"
, "./../../../../js/src/gc/BufferAllocator.cpp", 403); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "allocStartBitmap.ref().FindNext(startBit + 1) > newEndBit"
")"); do { MOZ_CrashSequence(__null, 403); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
404 MOZ_ASSERT(findEndBit(startBit) > newEndBit)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(findEndBit(startBit) > newEndBit)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(findEndBit(startBit) > newEndBit
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"findEndBit(startBit) > newEndBit", "./../../../../js/src/gc/BufferAllocator.cpp"
, 404); AnnotateMozCrashReason("MOZ_ASSERT" "(" "findEndBit(startBit) > newEndBit"
")"); do { MOZ_CrashSequence(__null, 404); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
405 allocEndBitmap.ref()[newEndBit] = true;
406}
407
408template <typename D, size_t S, size_t G>
409size_t AllocSpace<D, S, G>::allocBytes(const void* alloc) const {
410 MOZ_ASSERT(isAllocated(alloc))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(isAllocated(alloc))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(isAllocated(alloc)))), 0))) {
do { } while (false); MOZ_ReportAssertionFailure("isAllocated(alloc)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 410); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "isAllocated(alloc)" ")"); do { MOZ_CrashSequence
(__null, 410); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
411
412 size_t startBit = ptrToIndex(alloc);
413 size_t endBit = findEndBit(startBit);
414 MOZ_ASSERT(endBit >= startBit)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(endBit >= startBit)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(endBit >= startBit))), 0)
)) { do { } while (false); MOZ_ReportAssertionFailure("endBit >= startBit"
, "./../../../../js/src/gc/BufferAllocator.cpp", 414); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "endBit >= startBit" ")"); do { MOZ_CrashSequence
(__null, 414); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
415 MOZ_ASSERT(endBit < MaxAllocCount)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(endBit < MaxAllocCount)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(endBit < MaxAllocCount)))
, 0))) { do { } while (false); MOZ_ReportAssertionFailure("endBit < MaxAllocCount"
, "./../../../../js/src/gc/BufferAllocator.cpp", 415); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "endBit < MaxAllocCount" ")"); do { MOZ_CrashSequence
(__null, 415); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
416
417 return (endBit - startBit + 1) * GranularityBytes;
418}
419
420template <typename D, size_t S, size_t G>
421bool AllocSpace<D, S, G>::setMarked(void* alloc) {
422 MOZ_ASSERT(isAllocated(alloc))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(isAllocated(alloc))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(isAllocated(alloc)))), 0))) {
do { } while (false); MOZ_ReportAssertionFailure("isAllocated(alloc)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 422); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "isAllocated(alloc)" ")"); do { MOZ_CrashSequence
(__null, 422); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
423 size_t bit = ptrToIndex(alloc);
424
425 // This is thread safe but can return false positives if another thread also
426 // marked the same allocation at the same time;
427 if (markBits.ref().getBit(bit)) {
428 return false;
429 }
430
431 markBits.ref().setBit(bit, true);
432 return true;
433}
434
435template <typename D, size_t S, size_t G>
436size_t AllocSpace<D, S, G>::findNextAllocated(uintptr_t offset) const {
437 size_t bit = offsetToIndex(offset);
438 size_t next = allocStartBitmap.ref().FindNext(bit);
439 if (next == SIZE_MAX(18446744073709551615UL)) {
440 return SizeBytes;
441 }
442
443 return next * GranularityBytes;
444}
445
446template <typename D, size_t S, size_t G>
447size_t AllocSpace<D, S, G>::findPrevAllocated(uintptr_t offset) const {
448 size_t bit = offsetToIndex(offset);
449 size_t prev = allocStartBitmap.ref().FindPrev(bit);
450 if (prev == SIZE_MAX(18446744073709551615UL)) {
451 return SizeBytes;
452 }
453
454 return prev * GranularityBytes;
455}
456
457template <typename D, size_t S, size_t G>
458BufferAllocator::FreeRegion* AllocSpace<D, S, G>::findFollowingFreeRegion(
459 uintptr_t startAddr) {
460 // Find the free region that starts at |startAddr|, which is not allocated and
461 // not at the end of the chunk. Always returns a region.
462
463 uintptr_t offset = uintptr_t(startAddr) & AddressMask;
464 MOZ_ASSERT(isValidOffset(offset))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(isValidOffset(offset))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(isValidOffset(offset)))), 0)
)) { do { } while (false); MOZ_ReportAssertionFailure("isValidOffset(offset)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 464); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "isValidOffset(offset)" ")"); do { MOZ_CrashSequence
(__null, 464); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
465 MOZ_ASSERT((offset % GranularityBytes) == 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype((offset % GranularityBytes) == 0)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!((offset % GranularityBytes) ==
0))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("(offset % GranularityBytes) == 0", "./../../../../js/src/gc/BufferAllocator.cpp"
, 465); AnnotateMozCrashReason("MOZ_ASSERT" "(" "(offset % GranularityBytes) == 0"
")"); do { MOZ_CrashSequence(__null, 465); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
466
467 MOZ_ASSERT(!isAllocated(offset))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!isAllocated(offset))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!isAllocated(offset)))), 0))
) { do { } while (false); MOZ_ReportAssertionFailure("!isAllocated(offset)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 467); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!isAllocated(offset)" ")"); do { MOZ_CrashSequence
(__null, 467); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false); // Already marked as not allocated.
468 offset = findNextAllocated(offset);
469 MOZ_ASSERT(offset <= SizeBytes)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(offset <= SizeBytes)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(offset <= SizeBytes))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("offset <= SizeBytes"
, "./../../../../js/src/gc/BufferAllocator.cpp", 469); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "offset <= SizeBytes" ")"); do { MOZ_CrashSequence
(__null, 469); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
470
471 auto* region = FreeRegion::fromEndAddr(startAddress() + offset);
472 MOZ_ASSERT(region->startAddr == startAddr)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(region->startAddr == startAddr)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(region->startAddr == startAddr
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"region->startAddr == startAddr", "./../../../../js/src/gc/BufferAllocator.cpp"
, 472); AnnotateMozCrashReason("MOZ_ASSERT" "(" "region->startAddr == startAddr"
")"); do { MOZ_CrashSequence(__null, 472); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
473
474 return region;
475}
476
477template <typename D, size_t S, size_t G>
478BufferAllocator::FreeRegion* AllocSpace<D, S, G>::findPrecedingFreeRegion(
479 uintptr_t endAddr) {
480 // Find the free region, if any, that ends at |endAddr|, which may be
481 // allocated or at the start of the chunk.
482
483 uintptr_t offset = uintptr_t(endAddr) & AddressMask;
484 MOZ_ASSERT(isValidOffset(offset))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(isValidOffset(offset))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(isValidOffset(offset)))), 0)
)) { do { } while (false); MOZ_ReportAssertionFailure("isValidOffset(offset)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 484); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "isValidOffset(offset)" ")"); do { MOZ_CrashSequence
(__null, 484); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
485 MOZ_ASSERT((offset % GranularityBytes) == 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype((offset % GranularityBytes) == 0)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!((offset % GranularityBytes) ==
0))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("(offset % GranularityBytes) == 0", "./../../../../js/src/gc/BufferAllocator.cpp"
, 485); AnnotateMozCrashReason("MOZ_ASSERT" "(" "(offset % GranularityBytes) == 0"
")"); do { MOZ_CrashSequence(__null, 485); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
486
487 if (offset == firstAllocOffset()) {
488 return nullptr; // Already at start of chunk.
489 }
490
491 MOZ_ASSERT(!isAllocated(offset))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!isAllocated(offset))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!isAllocated(offset)))), 0))
) { do { } while (false); MOZ_ReportAssertionFailure("!isAllocated(offset)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 491); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!isAllocated(offset)" ")"); do { MOZ_CrashSequence
(__null, 491); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
492 offset = findPrevAllocated(offset);
493
494 if (offset != SizeBytes) {
495 // Found a preceding allocation.
496 const void* alloc = ptrFromOffset(offset);
497 size_t bytes = allocBytes(alloc);
498 MOZ_ASSERT(uintptr_t(alloc) + bytes <= endAddr)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(uintptr_t(alloc) + bytes <= endAddr)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(uintptr_t(alloc) + bytes <= endAddr))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("uintptr_t(alloc) + bytes <= endAddr"
, "./../../../../js/src/gc/BufferAllocator.cpp", 498); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "uintptr_t(alloc) + bytes <= endAddr" ")"
); do { MOZ_CrashSequence(__null, 498); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
499 if (uintptr_t(alloc) + bytes == endAddr) {
500 // No free space between preceding allocation and |endAddr|.
501 return nullptr;
502 }
503 }
504
505 auto* region = FreeRegion::fromEndAddr(endAddr);
506
507#ifdef DEBUG1
508 region->check();
509 if (offset != SizeBytes) {
510 const void* alloc = ptrFromOffset(offset);
511 size_t bytes = allocBytes(alloc);
512 MOZ_ASSERT(region->startAddr == uintptr_t(alloc) + bytes)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(region->startAddr == uintptr_t(alloc) + bytes)>
::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(region->startAddr == uintptr_t(alloc) + bytes))),
0))) { do { } while (false); MOZ_ReportAssertionFailure("region->startAddr == uintptr_t(alloc) + bytes"
, "./../../../../js/src/gc/BufferAllocator.cpp", 512); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "region->startAddr == uintptr_t(alloc) + bytes"
")"); do { MOZ_CrashSequence(__null, 512); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
513 } else {
514 MOZ_ASSERT(region->startAddr == startAddress() + firstAllocOffset())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(region->startAddr == startAddress() + firstAllocOffset
())>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(region->startAddr == startAddress() + firstAllocOffset
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("region->startAddr == startAddress() + firstAllocOffset()"
, "./../../../../js/src/gc/BufferAllocator.cpp", 514); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "region->startAddr == startAddress() + firstAllocOffset()"
")"); do { MOZ_CrashSequence(__null, 514); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
515 }
516#endif
517
518 return region;
519}
520
521BufferChunk::BufferChunk(Zone* zone)
522 : ChunkBase(zone->runtimeFromAnyThread(), ChunkKind::Buffers), zone(zone) {
523 MOZ_ASSERT(decommittedPages.ref().IsEmpty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(decommittedPages.ref().IsEmpty())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(decommittedPages.ref().IsEmpty
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("decommittedPages.ref().IsEmpty()", "./../../../../js/src/gc/BufferAllocator.cpp"
, 523); AnnotateMozCrashReason("MOZ_ASSERT" "(" "decommittedPages.ref().IsEmpty()"
")"); do { MOZ_CrashSequence(__null, 523); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
524 MOZ_ASSERT(!allocatedDuringCollection)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!allocatedDuringCollection)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!allocatedDuringCollection))
), 0))) { do { } while (false); MOZ_ReportAssertionFailure("!allocatedDuringCollection"
, "./../../../../js/src/gc/BufferAllocator.cpp", 524); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!allocatedDuringCollection" ")"); do { MOZ_CrashSequence
(__null, 524); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
525 MOZ_ASSERT(!stolenFromSweepList)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!stolenFromSweepList)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!stolenFromSweepList))), 0))
) { do { } while (false); MOZ_ReportAssertionFailure("!stolenFromSweepList"
, "./../../../../js/src/gc/BufferAllocator.cpp", 525); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!stolenFromSweepList" ")"); do { MOZ_CrashSequence
(__null, 525); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
526}
527
528BufferChunk::~BufferChunk() {
529#ifdef DEBUG1
530 MOZ_ASSERT(allocStartBitmap.ref().IsEmpty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(allocStartBitmap.ref().IsEmpty())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(allocStartBitmap.ref().IsEmpty
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("allocStartBitmap.ref().IsEmpty()", "./../../../../js/src/gc/BufferAllocator.cpp"
, 530); AnnotateMozCrashReason("MOZ_ASSERT" "(" "allocStartBitmap.ref().IsEmpty()"
")"); do { MOZ_CrashSequence(__null, 530); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
531 MOZ_ASSERT(allocEndBitmap.ref().IsEmpty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(allocEndBitmap.ref().IsEmpty())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(allocEndBitmap.ref().IsEmpty
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("allocEndBitmap.ref().IsEmpty()", "./../../../../js/src/gc/BufferAllocator.cpp"
, 531); AnnotateMozCrashReason("MOZ_ASSERT" "(" "allocEndBitmap.ref().IsEmpty()"
")"); do { MOZ_CrashSequence(__null, 531); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
532 MOZ_ASSERT(nurseryOwnedBitmap.ref().IsEmpty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(nurseryOwnedBitmap.ref().IsEmpty())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(nurseryOwnedBitmap.ref().IsEmpty
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("nurseryOwnedBitmap.ref().IsEmpty()", "./../../../../js/src/gc/BufferAllocator.cpp"
, 532); AnnotateMozCrashReason("MOZ_ASSERT" "(" "nurseryOwnedBitmap.ref().IsEmpty()"
")"); do { MOZ_CrashSequence(__null, 532); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
533#endif
534}
535
536void BufferChunk::setSmallBufferRegion(void* alloc, bool smallAlloc) {
537 MOZ_ASSERT(isAllocated(alloc))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(isAllocated(alloc))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(isAllocated(alloc)))), 0))) {
do { } while (false); MOZ_ReportAssertionFailure("isAllocated(alloc)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 537); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "isAllocated(alloc)" ")"); do { MOZ_CrashSequence
(__null, 537); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
538 size_t bit = ptrToIndex<SmallRegionSize, SmallRegionSize>(alloc);
539 smallRegionBitmap.ref().setBit(bit, smallAlloc);
540}
541
542bool BufferChunk::isSmallBufferRegion(const void* alloc) const {
543 // Allow any valid small alloc pointer within the region.
544 size_t bit = ptrToIndex<SmallRegionSize, SmallAllocGranularity>(alloc);
545 return smallRegionBitmap.ref().getBit(bit);
546}
547
548size_t BufferChunk::sizeClassForAvailableLists() const {
549 MOZ_ASSERT(ownsFreeLists)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ownsFreeLists)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(ownsFreeLists))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("ownsFreeLists",
"./../../../../js/src/gc/BufferAllocator.cpp", 549); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "ownsFreeLists" ")"); do { MOZ_CrashSequence
(__null, 549); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
550
551 // To quickly find an available chunk we bin them by the size of their largest
552 // free region. This allows us to select a chunk we know will be able to
553 // satisfy a request.
554 //
555 // This prioritises allocating into chunks with large free regions first. It
556 // might be better for memory use to allocate into chunks with less free space
557 // first instead.
558 size_t sizeClass =
559 freeLists.ref().getLastAvailableSizeClass(0, MaxMediumAllocClass);
560
561 // Use a special size class for completely full chunks.
562 if (sizeClass == SIZE_MAX(18446744073709551615UL)) {
563 return BufferAllocator::FullChunkSizeClass;
564 }
565
566 return sizeClass;
567}
568
569void SmallBufferRegion::setHasNurseryOwnedAllocs(bool value) {
570 hasNurseryOwnedAllocs_ = value;
571}
572bool SmallBufferRegion::hasNurseryOwnedAllocs() const {
573 return hasNurseryOwnedAllocs_.ref();
574}
575
576BufferAllocatorRuntime::BufferAllocatorRuntime()
577 : lock(mutexid::BufferAllocator) {}
578
579void BufferAllocatorRuntime::incOffThreadCount() { offThreadAccessCount++; }
580
581void BufferAllocatorRuntime::decOffThreadCount() {
582 MOZ_ALWAYS_TRUE(offThreadAccessCount-- != 0)do { if ((__builtin_expect(!!(offThreadAccessCount-- != 0), 1
))) { } else { do { do { } while (false); MOZ_ReportCrash("" "offThreadAccessCount-- != 0"
, "./../../../../js/src/gc/BufferAllocator.cpp", 582); AnnotateMozCrashReason
("MOZ_CRASH(" "offThreadAccessCount-- != 0" ")"); do { MOZ_CrashSequence
(__null, 582); __attribute__((nomerge)) ::abort(); } while (false
); } while (false); } } while (false);
583}
584
585void BufferAllocatorRuntime::resetRetainedStats() {
586 usedBytesInRetainedChunks = 0;
587 freeBytesInRetainedChunks = 0;
588 adminBytesInRetainedChunks = 0;
589}
590
591void BufferAllocatorRuntime::addRetainedStats(size_t usedBytes,
592 size_t freeBytes,
593 size_t adminBytes) {
594 usedBytesInRetainedChunks += usedBytes;
595 freeBytesInRetainedChunks += freeBytes;
596 adminBytesInRetainedChunks += adminBytes;
597}
598
599void BufferAllocatorRuntime::getRetainedStats(size_t* usedBytesOut,
600 size_t* freeBytesOut,
601 size_t* adminBytesOut) {
602 *usedBytesOut = usedBytesInRetainedChunks;
603 *freeBytesOut = freeBytesInRetainedChunks;
604 *adminBytesOut = adminBytesInRetainedChunks;
605}
606
607bool BufferAllocatorRuntime::needLockToAccessBufferMap() const {
608 return offThreadAccessCount != 0;
609}
610
611LargeBuffer* BufferAllocatorRuntime::lookupLargeBuffer(void* alloc) {
612 MaybeLock lock;
613 return lookupLargeBuffer(alloc, lock);
614}
615
616LargeBuffer* BufferAllocatorRuntime::lookupLargeBuffer(void* alloc,
617 MaybeLock& lock) {
618 MOZ_ASSERT(lock.isNothing())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(lock.isNothing())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(lock.isNothing()))), 0))) { do
{ } while (false); MOZ_ReportAssertionFailure("lock.isNothing()"
, "./../../../../js/src/gc/BufferAllocator.cpp", 618); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "lock.isNothing()" ")"); do { MOZ_CrashSequence
(__null, 618); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
619 if (needLockToAccessBufferMap()) {
620 lock.emplace(this);
621 }
622
623 auto ptr = largeAllocMap.ref().readonlyThreadsafeLookup(alloc);
624 MOZ_ASSERT(ptr)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ptr)>::isValid, "invalid assertion condition"); if
((__builtin_expect(!!(!(!!(ptr))), 0))) { do { } while (false
); MOZ_ReportAssertionFailure("ptr", "./../../../../js/src/gc/BufferAllocator.cpp"
, 624); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ptr" ")"); do
{ MOZ_CrashSequence(__null, 624); __attribute__((nomerge)) ::
abort(); } while (false); } } while (false);
625
626 LargeBuffer* buffer = ptr->value();
627 MOZ_ASSERT(buffer->data() == alloc)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(buffer->data() == alloc)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(buffer->data() == alloc))
), 0))) { do { } while (false); MOZ_ReportAssertionFailure("buffer->data() == alloc"
, "./../../../../js/src/gc/BufferAllocator.cpp", 627); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "buffer->data() == alloc" ")"); do { MOZ_CrashSequence
(__null, 627); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
628 return buffer;
629}
630
631void BufferAllocatorRuntime::checkGCStateNotInUse() {
632 MOZ_ASSERT(offThreadAccessCount == 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(offThreadAccessCount == 0)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(offThreadAccessCount == 0)))
, 0))) { do { } while (false); MOZ_ReportAssertionFailure("offThreadAccessCount == 0"
, "./../../../../js/src/gc/BufferAllocator.cpp", 632); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "offThreadAccessCount == 0" ")"); do { MOZ_CrashSequence
(__null, 632); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
633}
634
635BufferAllocator::BufferAllocator(GCRuntime* gc, Zone* zone)
636 : gc(gc),
637 zone(zone),
638 sweptMixedChunks(gc->bufferRuntime().lock),
639 sweptTenuredChunks(gc->bufferRuntime().lock),
640 sweptLargeTenuredAllocs(gc->bufferRuntime().lock),
641 minorState(State::NotCollecting),
642 majorState(State::NotCollecting),
643 minorSweepingFinished(gc->bufferRuntime().lock),
644 majorSweepingFinished(gc->bufferRuntime().lock),
645 allocTenuredInMixedChunks(true) {}
646
647BufferAllocator::~BufferAllocator() {
648#ifdef DEBUG1
649 checkGCStateNotInUse();
650 MOZ_ASSERT(currentMixedChunks.ref().isEmpty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(currentMixedChunks.ref().isEmpty())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(currentMixedChunks.ref().isEmpty
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("currentMixedChunks.ref().isEmpty()", "./../../../../js/src/gc/BufferAllocator.cpp"
, 650); AnnotateMozCrashReason("MOZ_ASSERT" "(" "currentMixedChunks.ref().isEmpty()"
")"); do { MOZ_CrashSequence(__null, 650); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
651 MOZ_ASSERT(currentTenuredChunks.ref().isEmpty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(currentTenuredChunks.ref().isEmpty())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(currentTenuredChunks.ref().isEmpty
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("currentTenuredChunks.ref().isEmpty()", "./../../../../js/src/gc/BufferAllocator.cpp"
, 651); AnnotateMozCrashReason("MOZ_ASSERT" "(" "currentTenuredChunks.ref().isEmpty()"
")"); do { MOZ_CrashSequence(__null, 651); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
652 mixedFreeLists.ref().assertEmpty();
653 tenuredFreeLists.ref().assertEmpty();
654 MOZ_ASSERT(availableChunks.ref().isEmpty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(availableChunks.ref().isEmpty())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(availableChunks.ref().isEmpty
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("availableChunks.ref().isEmpty()", "./../../../../js/src/gc/BufferAllocator.cpp"
, 654); AnnotateMozCrashReason("MOZ_ASSERT" "(" "availableChunks.ref().isEmpty()"
")"); do { MOZ_CrashSequence(__null, 654); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
655 MOZ_ASSERT(totalChunkCount == 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(totalChunkCount == 0)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(totalChunkCount == 0))), 0))
) { do { } while (false); MOZ_ReportAssertionFailure("totalChunkCount == 0"
, "./../../../../js/src/gc/BufferAllocator.cpp", 655); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "totalChunkCount == 0" ")"); do { MOZ_CrashSequence
(__null, 655); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
656 MOZ_ASSERT(largeNurseryAllocs.ref().isEmpty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(largeNurseryAllocs.ref().isEmpty())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(largeNurseryAllocs.ref().isEmpty
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("largeNurseryAllocs.ref().isEmpty()", "./../../../../js/src/gc/BufferAllocator.cpp"
, 656); AnnotateMozCrashReason("MOZ_ASSERT" "(" "largeNurseryAllocs.ref().isEmpty()"
")"); do { MOZ_CrashSequence(__null, 656); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
657 MOZ_ASSERT(largeTenuredAllocs.ref().isEmpty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(largeTenuredAllocs.ref().isEmpty())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(largeTenuredAllocs.ref().isEmpty
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("largeTenuredAllocs.ref().isEmpty()", "./../../../../js/src/gc/BufferAllocator.cpp"
, 657); AnnotateMozCrashReason("MOZ_ASSERT" "(" "largeTenuredAllocs.ref().isEmpty()"
")"); do { MOZ_CrashSequence(__null, 657); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
658#endif
659}
660
661bool BufferAllocator::isEmpty() const {
662 checkMainThread();
663 MOZ_ASSERT(!zone->wasGCStarted() || zone->isGCFinished())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!zone->wasGCStarted() || zone->isGCFinished())
>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(!zone->wasGCStarted() || zone->isGCFinished())
)), 0))) { do { } while (false); MOZ_ReportAssertionFailure("!zone->wasGCStarted() || zone->isGCFinished()"
, "./../../../../js/src/gc/BufferAllocator.cpp", 663); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!zone->wasGCStarted() || zone->isGCFinished()"
")"); do { MOZ_CrashSequence(__null, 663); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
664 MOZ_ASSERT(minorState == State::NotCollecting)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(minorState == State::NotCollecting)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(minorState == State::NotCollecting
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"minorState == State::NotCollecting", "./../../../../js/src/gc/BufferAllocator.cpp"
, 664); AnnotateMozCrashReason("MOZ_ASSERT" "(" "minorState == State::NotCollecting"
")"); do { MOZ_CrashSequence(__null, 664); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
665 MOZ_ASSERT(majorState == State::NotCollecting)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(majorState == State::NotCollecting)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(majorState == State::NotCollecting
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"majorState == State::NotCollecting", "./../../../../js/src/gc/BufferAllocator.cpp"
, 665); AnnotateMozCrashReason("MOZ_ASSERT" "(" "majorState == State::NotCollecting"
")"); do { MOZ_CrashSequence(__null, 665); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
666 return currentMixedChunks.ref().isEmpty() &&
667 currentTenuredChunks.ref().isEmpty() &&
668 availableChunks.ref().isEmpty() &&
669 largeNurseryAllocs.ref().isEmpty() &&
670 largeTenuredAllocs.ref().isEmpty();
671}
672
673void BufferAllocator::setMultiThreadedUse(Mutex* mutex) {
674 MOZ_ASSERT(CurrentThreadCanAccessZone(zone))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(CurrentThreadCanAccessZone(zone))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(CurrentThreadCanAccessZone(zone
)))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("CurrentThreadCanAccessZone(zone)", "./../../../../js/src/gc/BufferAllocator.cpp"
, 674); AnnotateMozCrashReason("MOZ_ASSERT" "(" "CurrentThreadCanAccessZone(zone)"
")"); do { MOZ_CrashSequence(__null, 674); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
675 MOZ_ASSERT(!multiThreadedMutex)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!multiThreadedMutex)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!multiThreadedMutex))), 0)))
{ do { } while (false); MOZ_ReportAssertionFailure("!multiThreadedMutex"
, "./../../../../js/src/gc/BufferAllocator.cpp", 675); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!multiThreadedMutex" ")"); do { MOZ_CrashSequence
(__null, 675); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
676 MOZ_ASSERT(majorState != State::Sweeping)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(majorState != State::Sweeping)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(majorState != State::Sweeping
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"majorState != State::Sweeping", "./../../../../js/src/gc/BufferAllocator.cpp"
, 676); AnnotateMozCrashReason("MOZ_ASSERT" "(" "majorState != State::Sweeping"
")"); do { MOZ_CrashSequence(__null, 676); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
677
678 multiThreadedMutex = mutex;
679}
680
681void BufferAllocator::clearMultiThreadedUse() {
682 MOZ_ASSERT(CurrentThreadCanAccessZone(zone))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(CurrentThreadCanAccessZone(zone))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(CurrentThreadCanAccessZone(zone
)))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("CurrentThreadCanAccessZone(zone)", "./../../../../js/src/gc/BufferAllocator.cpp"
, 682); AnnotateMozCrashReason("MOZ_ASSERT" "(" "CurrentThreadCanAccessZone(zone)"
")"); do { MOZ_CrashSequence(__null, 682); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
683 MOZ_ASSERT(multiThreadedMutex)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(multiThreadedMutex)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(multiThreadedMutex))), 0))) {
do { } while (false); MOZ_ReportAssertionFailure("multiThreadedMutex"
, "./../../../../js/src/gc/BufferAllocator.cpp", 683); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "multiThreadedMutex" ")"); do { MOZ_CrashSequence
(__null, 683); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
684 MOZ_ASSERT(majorState != State::Sweeping)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(majorState != State::Sweeping)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(majorState != State::Sweeping
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"majorState != State::Sweeping", "./../../../../js/src/gc/BufferAllocator.cpp"
, 684); AnnotateMozCrashReason("MOZ_ASSERT" "(" "majorState != State::Sweeping"
")"); do { MOZ_CrashSequence(__null, 684); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
685
686 multiThreadedMutex = nullptr;
687}
688
689void BufferAllocator::checkAccess() const {
690#ifdef DEBUG1
691 if (multiThreadedMutex) {
692 MOZ_ASSERT(multiThreadedMutex->isOwnedByCurrentThread())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(multiThreadedMutex->isOwnedByCurrentThread())>
::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(multiThreadedMutex->isOwnedByCurrentThread()))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("multiThreadedMutex->isOwnedByCurrentThread()"
, "./../../../../js/src/gc/BufferAllocator.cpp", 692); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "multiThreadedMutex->isOwnedByCurrentThread()"
")"); do { MOZ_CrashSequence(__null, 692); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
693 } else {
694 MOZ_ASSERT(CurrentThreadCanAccessZone(zone))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(CurrentThreadCanAccessZone(zone))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(CurrentThreadCanAccessZone(zone
)))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("CurrentThreadCanAccessZone(zone)", "./../../../../js/src/gc/BufferAllocator.cpp"
, 694); AnnotateMozCrashReason("MOZ_ASSERT" "(" "CurrentThreadCanAccessZone(zone)"
")"); do { MOZ_CrashSequence(__null, 694); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
695 }
696#endif
697}
698
699void BufferAllocator::checkMainThread() const {
700 MOZ_ASSERT(!multiThreadedMutex)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!multiThreadedMutex)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!multiThreadedMutex))), 0)))
{ do { } while (false); MOZ_ReportAssertionFailure("!multiThreadedMutex"
, "./../../../../js/src/gc/BufferAllocator.cpp", 700); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!multiThreadedMutex" ")"); do { MOZ_CrashSequence
(__null, 700); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
701 MOZ_ASSERT(CurrentThreadCanAccessZone(zone))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(CurrentThreadCanAccessZone(zone))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(CurrentThreadCanAccessZone(zone
)))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("CurrentThreadCanAccessZone(zone)", "./../../../../js/src/gc/BufferAllocator.cpp"
, 701); AnnotateMozCrashReason("MOZ_ASSERT" "(" "CurrentThreadCanAccessZone(zone)"
")"); do { MOZ_CrashSequence(__null, 701); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
702}
703
704bool BufferAllocator::isUsedByMainThread() const { return !multiThreadedMutex; }
705
706BufferAllocatorRuntime* BufferAllocator::runtime() const {
707 return &gc->bufferRuntime();
708}
709
710void* BufferAllocator::alloc(size_t bytes, bool nurseryOwned) {
711 MOZ_ASSERT_IF(zone->isGCMarkingOrSweeping(), majorState == State::Marking)do { if (zone->isGCMarkingOrSweeping()) { do { static_assert
( mozilla::detail::AssertionConditionType<decltype(majorState
== State::Marking)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(majorState == State::Marking
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"majorState == State::Marking", "./../../../../js/src/gc/BufferAllocator.cpp"
, 711); AnnotateMozCrashReason("MOZ_ASSERT" "(" "majorState == State::Marking"
")"); do { MOZ_CrashSequence(__null, 711); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false);
712 checkAccess();
713
714 if (IsLargeAllocSize(bytes)) {
715 return allocLarge(bytes, nurseryOwned, false);
716 }
717
718 if (IsSmallAllocSize(bytes)) {
719 return allocSmall(bytes, nurseryOwned, false);
720 }
721
722 return allocMedium(bytes, nurseryOwned, false);
723}
724
725void* BufferAllocator::allocInGC(size_t bytes, bool nurseryOwned) {
726 // Currently this is used during tenuring only.
727 MOZ_ASSERT(minorState == State::Marking)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(minorState == State::Marking)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(minorState == State::Marking
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"minorState == State::Marking", "./../../../../js/src/gc/BufferAllocator.cpp"
, 727); AnnotateMozCrashReason("MOZ_ASSERT" "(" "minorState == State::Marking"
")"); do { MOZ_CrashSequence(__null, 727); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
728
729 MOZ_ASSERT_IF(zone->isGCMarkingOrSweeping(), majorState == State::Marking)do { if (zone->isGCMarkingOrSweeping()) { do { static_assert
( mozilla::detail::AssertionConditionType<decltype(majorState
== State::Marking)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(majorState == State::Marking
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"majorState == State::Marking", "./../../../../js/src/gc/BufferAllocator.cpp"
, 729); AnnotateMozCrashReason("MOZ_ASSERT" "(" "majorState == State::Marking"
")"); do { MOZ_CrashSequence(__null, 729); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false);
730 checkAccess();
731
732 void* result;
733 if (IsLargeAllocSize(bytes)) {
734 result = allocLarge(bytes, nurseryOwned, true);
735 } else if (IsSmallAllocSize(bytes)) {
736 result = allocSmall(bytes, nurseryOwned, true);
737 } else {
738 result = allocMedium(bytes, nurseryOwned, true);
739 }
740
741 if (!result) {
742 return nullptr;
743 }
744
745 // Barrier to mark nursery-owned allocations that happen during collection. We
746 // don't need to do this for tenured-owned allocations because we don't sweep
747 // tenured-owned allocations that happened after the start of a major
748 // collection.
749 if (nurseryOwned) {
750 markNurseryOwnedAlloc(result, true);
751 }
752
753 return result;
754}
755
756inline Zone* LargeBuffer::zone() {
757 Zone* zone = zoneFromAnyThread();
758 MOZ_ASSERT(CurrentThreadCanAccessZone(zone))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(CurrentThreadCanAccessZone(zone))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(CurrentThreadCanAccessZone(zone
)))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("CurrentThreadCanAccessZone(zone)", "./../../../../js/src/gc/BufferAllocator.cpp"
, 758); AnnotateMozCrashReason("MOZ_ASSERT" "(" "CurrentThreadCanAccessZone(zone)"
")"); do { MOZ_CrashSequence(__null, 758); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
759 return zone;
760}
761
762inline Zone* LargeBuffer::zoneFromAnyThread() {
763 return BufferChunk::from(this)->zone;
764}
765
766#ifdef XP_DARWIN
767static inline void VirtualCopyPages(void* dst, const void* src, size_t bytes) {
768 MOZ_ASSERT((uintptr_t(dst) & PageMask) == 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype((uintptr_t(dst) & PageMask) == 0)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!((uintptr_t(dst) & PageMask
) == 0))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("(uintptr_t(dst) & PageMask) == 0", "./../../../../js/src/gc/BufferAllocator.cpp"
, 768); AnnotateMozCrashReason("MOZ_ASSERT" "(" "(uintptr_t(dst) & PageMask) == 0"
")"); do { MOZ_CrashSequence(__null, 768); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
769 MOZ_ASSERT((uintptr_t(src) & PageMask) == 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype((uintptr_t(src) & PageMask) == 0)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!((uintptr_t(src) & PageMask
) == 0))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("(uintptr_t(src) & PageMask) == 0", "./../../../../js/src/gc/BufferAllocator.cpp"
, 769); AnnotateMozCrashReason("MOZ_ASSERT" "(" "(uintptr_t(src) & PageMask) == 0"
")"); do { MOZ_CrashSequence(__null, 769); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
770 MOZ_ASSERT(bytes >= ChunkSize)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(bytes >= ChunkSize)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(bytes >= ChunkSize))), 0)
)) { do { } while (false); MOZ_ReportAssertionFailure("bytes >= ChunkSize"
, "./../../../../js/src/gc/BufferAllocator.cpp", 770); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "bytes >= ChunkSize" ")"); do { MOZ_CrashSequence
(__null, 770); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
771
772 kern_return_t r = vm_copy(mach_task_self(), vm_address_t(src),
773 vm_size_t(bytes), vm_address_t(dst));
774 if (r != KERN_SUCCESS) {
775 MOZ_CRASH("vm_copy() failed")do { do { } while (false); MOZ_ReportCrash("" "vm_copy() failed"
, "./../../../../js/src/gc/BufferAllocator.cpp", 775); AnnotateMozCrashReason
("MOZ_CRASH(" "vm_copy() failed" ")"); do { MOZ_CrashSequence
(__null, 775); __attribute__((nomerge)) ::abort(); } while (false
); } while (false);
776 }
777}
778#endif
779
780void* BufferAllocator::realloc(void* alloc, size_t bytes, bool nurseryOwned) {
781 // Reallocate a buffer. This has the same semantics as standard libarary
782 // realloc: if |ptr| is null it creates a new allocation, and if it fails it
783 // returns |nullptr| and the original |ptr| is still valid.
784
785 checkAccess();
786
787 if (!alloc) {
788 return this->alloc(bytes, nurseryOwned);
789 }
790
791 MOZ_ASSERT(isNurseryOwned(alloc) == nurseryOwned)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(isNurseryOwned(alloc) == nurseryOwned)>::isValid,
"invalid assertion condition"); if ((__builtin_expect(!!(!(!
!(isNurseryOwned(alloc) == nurseryOwned))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("isNurseryOwned(alloc) == nurseryOwned"
, "./../../../../js/src/gc/BufferAllocator.cpp", 791); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "isNurseryOwned(alloc) == nurseryOwned" ")"
); do { MOZ_CrashSequence(__null, 791); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
792 MOZ_ASSERT_IF(zone->isGCMarkingOrSweeping(), majorState == State::Marking)do { if (zone->isGCMarkingOrSweeping()) { do { static_assert
( mozilla::detail::AssertionConditionType<decltype(majorState
== State::Marking)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(majorState == State::Marking
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"majorState == State::Marking", "./../../../../js/src/gc/BufferAllocator.cpp"
, 792); AnnotateMozCrashReason("MOZ_ASSERT" "(" "majorState == State::Marking"
")"); do { MOZ_CrashSequence(__null, 792); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false);
793
794 bytes = GetGoodAllocSize(bytes);
795
796 size_t currentBytes;
797 if (IsLargeAlloc(alloc)) {
798 LargeBuffer* buffer = lookupLargeBuffer(alloc);
799 currentBytes = buffer->allocBytes();
800
801 // We can shrink large allocations (on some platforms).
802 if (bytes < buffer->allocBytes() && IsLargeAllocSize(bytes)) {
803 if (shrinkLarge(buffer, bytes)) {
804 return alloc;
805 }
806 }
807 } else if (IsMediumAlloc(alloc)) {
808 BufferChunk* chunk = BufferChunk::from(alloc);
809 MOZ_ASSERT(!chunk->isSmallBufferRegion(alloc))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!chunk->isSmallBufferRegion(alloc))>::isValid,
"invalid assertion condition"); if ((__builtin_expect(!!(!(!
!(!chunk->isSmallBufferRegion(alloc)))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("!chunk->isSmallBufferRegion(alloc)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 809); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!chunk->isSmallBufferRegion(alloc)" ")"
); do { MOZ_CrashSequence(__null, 809); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
810
811 currentBytes = chunk->allocBytes(alloc);
812
813 // We can grow or shrink medium allocations.
814 if (bytes < currentBytes && !IsSmallAllocSize(bytes)) {
815 if (shrinkMedium(alloc, bytes)) {
816 return alloc;
817 }
818 }
819
820 if (bytes > currentBytes && !IsLargeAllocSize(bytes)) {
821 if (growMedium(alloc, bytes)) {
822 return alloc;
823 }
824 }
825 } else {
826 // TODO: Grow and shrink small allocations.
827 auto* region = SmallBufferRegion::from(alloc);
828 currentBytes = region->allocBytes(alloc);
829 }
830
831 if (bytes == currentBytes) {
832 return alloc;
833 }
834
835 void* newAlloc = this->alloc(bytes, nurseryOwned);
836 if (!newAlloc) {
837 return nullptr;
838 }
839
840 auto freeGuard = mozilla::MakeScopeExit([&]() { free(alloc); });
841
842 size_t bytesToCopy = std::min(bytes, currentBytes);
843
844#ifdef XP_DARWIN
845 if (bytesToCopy >= ChunkSize) {
846 MOZ_ASSERT(IsLargeAlloc(alloc))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(IsLargeAlloc(alloc))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(IsLargeAlloc(alloc)))), 0)))
{ do { } while (false); MOZ_ReportAssertionFailure("IsLargeAlloc(alloc)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 846); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "IsLargeAlloc(alloc)" ")"); do { MOZ_CrashSequence
(__null, 846); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
847 MOZ_ASSERT(IsLargeAlloc(newAlloc))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(IsLargeAlloc(newAlloc))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(IsLargeAlloc(newAlloc)))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("IsLargeAlloc(newAlloc)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 847); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "IsLargeAlloc(newAlloc)" ")"); do { MOZ_CrashSequence
(__null, 847); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
848 VirtualCopyPages(newAlloc, alloc, bytesToCopy);
849 return newAlloc;
850 }
851#endif
852
853 memcpy(newAlloc, alloc, bytesToCopy);
854 return newAlloc;
855}
856
857void BufferAllocator::free(void* alloc) {
858 MOZ_ASSERT(alloc)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(alloc)>::isValid, "invalid assertion condition");
if ((__builtin_expect(!!(!(!!(alloc))), 0))) { do { } while (
false); MOZ_ReportAssertionFailure("alloc", "./../../../../js/src/gc/BufferAllocator.cpp"
, 858); AnnotateMozCrashReason("MOZ_ASSERT" "(" "alloc" ")");
do { MOZ_CrashSequence(__null, 858); __attribute__((nomerge)
) ::abort(); } while (false); } } while (false);
859 checkAccess();
860
861 if (IsLargeAlloc(alloc)) {
862 freeLarge(alloc);
863 return;
864 }
865
866 if (IsMediumAlloc(alloc)) {
867 freeMedium(alloc);
868 return;
869 }
870
871 // Can't free small allocations.
872}
873
874/* static */
875bool BufferAllocator::IsBufferAlloc(void* alloc) {
876 // Precondition: |alloc| is a pointer to a buffer allocation, a GC thing or a
877 // direct nursery allocation returned by Nursery::allocateBuffer.
878
879 if (IsLargeAlloc(alloc)) {
880 return true;
881 }
882
883 ChunkBase* chunk = detail::GetGCAddressChunkBase(alloc);
884 return chunk->getKind() == ChunkKind::Buffers;
885}
886
887#ifdef DEBUG1
888bool BufferAllocator::hasAlloc(void* alloc) {
889 MOZ_ASSERT(IsBufferAlloc(alloc))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(IsBufferAlloc(alloc))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(IsBufferAlloc(alloc)))), 0))
) { do { } while (false); MOZ_ReportAssertionFailure("IsBufferAlloc(alloc)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 889); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "IsBufferAlloc(alloc)" ")"); do { MOZ_CrashSequence
(__null, 889); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
890
891 if (IsLargeAlloc(alloc)) {
892 MaybeLock lock;
893 if (needLockToAccessBufferMap()) {
894 lock.emplace(runtime());
895 }
896 auto ptr = runtime()->largeAllocMap.ref().readonlyThreadsafeLookup(alloc);
897 return ptr.found();
898 }
899
900 BufferChunk* chunk = BufferChunk::from(alloc);
901 return chunk->zone == zone;
902}
903#endif
904
905size_t BufferAllocator::getAllocSize(void* alloc) {
906 checkAccess();
907
908 if (!alloc) {
909 return 0;
910 }
911
912 if (IsLargeAlloc(alloc)) {
913 LargeBuffer* buffer = lookupLargeBuffer(alloc);
914 return buffer->allocBytes();
915 }
916
917 if (IsSmallAlloc(alloc)) {
918 auto* region = SmallBufferRegion::from(alloc);
919 return region->allocBytes(alloc);
920 }
921
922 MOZ_ASSERT(IsMediumAlloc(alloc))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(IsMediumAlloc(alloc))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(IsMediumAlloc(alloc)))), 0))
) { do { } while (false); MOZ_ReportAssertionFailure("IsMediumAlloc(alloc)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 922); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "IsMediumAlloc(alloc)" ")"); do { MOZ_CrashSequence
(__null, 922); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
923 BufferChunk* chunk = BufferChunk::from(alloc);
924 return chunk->allocBytes(alloc);
925}
926
927bool BufferAllocator::isNurseryOwned(void* alloc) {
928 if (IsLargeAlloc(alloc)) {
929 LargeBuffer* buffer = lookupLargeBuffer(alloc);
930 return buffer->isNurseryOwned;
931 }
932
933 if (IsSmallAlloc(alloc)) {
934 auto* region = SmallBufferRegion::from(alloc);
935 return region->isNurseryOwned(alloc);
936 }
937
938 BufferChunk* chunk = BufferChunk::from(alloc);
939 return chunk->isNurseryOwned(alloc);
940}
941
942void BufferAllocator::markNurseryOwnedAlloc(void* alloc, bool nurseryOwned) {
943 MOZ_ASSERT(alloc)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(alloc)>::isValid, "invalid assertion condition");
if ((__builtin_expect(!!(!(!!(alloc))), 0))) { do { } while (
false); MOZ_ReportAssertionFailure("alloc", "./../../../../js/src/gc/BufferAllocator.cpp"
, 943); AnnotateMozCrashReason("MOZ_ASSERT" "(" "alloc" ")");
do { MOZ_CrashSequence(__null, 943); __attribute__((nomerge)
) ::abort(); } while (false); } } while (false);
944 MOZ_ASSERT(isNurseryOwned(alloc))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(isNurseryOwned(alloc))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(isNurseryOwned(alloc)))), 0)
)) { do { } while (false); MOZ_ReportAssertionFailure("isNurseryOwned(alloc)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 944); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "isNurseryOwned(alloc)" ")"); do { MOZ_CrashSequence
(__null, 944); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
945 MOZ_ASSERT(minorState == State::Marking)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(minorState == State::Marking)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(minorState == State::Marking
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"minorState == State::Marking", "./../../../../js/src/gc/BufferAllocator.cpp"
, 945); AnnotateMozCrashReason("MOZ_ASSERT" "(" "minorState == State::Marking"
")"); do { MOZ_CrashSequence(__null, 945); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
946
947 if (IsLargeAlloc(alloc)) {
948 LargeBuffer* buffer = lookupLargeBuffer(alloc);
949 MOZ_ASSERT(buffer->zone() == zone)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(buffer->zone() == zone)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(buffer->zone() == zone)))
, 0))) { do { } while (false); MOZ_ReportAssertionFailure("buffer->zone() == zone"
, "./../../../../js/src/gc/BufferAllocator.cpp", 949); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "buffer->zone() == zone" ")"); do { MOZ_CrashSequence
(__null, 949); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
950 markLargeNurseryOwnedBuffer(buffer, nurseryOwned);
951 return;
952 }
953
954 if (IsSmallAlloc(alloc)) {
955 markSmallNurseryOwnedBuffer(alloc, nurseryOwned);
956 return;
957 }
958
959 MOZ_ASSERT(IsMediumAlloc(alloc))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(IsMediumAlloc(alloc))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(IsMediumAlloc(alloc)))), 0))
) { do { } while (false); MOZ_ReportAssertionFailure("IsMediumAlloc(alloc)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 959); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "IsMediumAlloc(alloc)" ")"); do { MOZ_CrashSequence
(__null, 959); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
960 markMediumNurseryOwnedBuffer(alloc, nurseryOwned);
961}
962
963void BufferAllocator::markSmallNurseryOwnedBuffer(void* alloc,
964 bool nurseryOwned) {
965#ifdef DEBUG1
966 BufferChunk* chunk = BufferChunk::from(alloc);
967 MOZ_ASSERT(chunk->zone == zone)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(chunk->zone == zone)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(chunk->zone == zone))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("chunk->zone == zone"
, "./../../../../js/src/gc/BufferAllocator.cpp", 967); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "chunk->zone == zone" ")"); do { MOZ_CrashSequence
(__null, 967); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
968 MOZ_ASSERT(chunk->hasNurseryOwnedAllocs)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(chunk->hasNurseryOwnedAllocs)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(chunk->hasNurseryOwnedAllocs
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"chunk->hasNurseryOwnedAllocs", "./../../../../js/src/gc/BufferAllocator.cpp"
, 968); AnnotateMozCrashReason("MOZ_ASSERT" "(" "chunk->hasNurseryOwnedAllocs"
")"); do { MOZ_CrashSequence(__null, 968); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
969#endif
970
971 auto* region = SmallBufferRegion::from(alloc);
972 MOZ_ASSERT(region->hasNurseryOwnedAllocs())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(region->hasNurseryOwnedAllocs())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(region->hasNurseryOwnedAllocs
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("region->hasNurseryOwnedAllocs()", "./../../../../js/src/gc/BufferAllocator.cpp"
, 972); AnnotateMozCrashReason("MOZ_ASSERT" "(" "region->hasNurseryOwnedAllocs()"
")"); do { MOZ_CrashSequence(__null, 972); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
973 MOZ_ASSERT(region->isNurseryOwned(alloc))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(region->isNurseryOwned(alloc))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(region->isNurseryOwned(alloc
)))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("region->isNurseryOwned(alloc)", "./../../../../js/src/gc/BufferAllocator.cpp"
, 973); AnnotateMozCrashReason("MOZ_ASSERT" "(" "region->isNurseryOwned(alloc)"
")"); do { MOZ_CrashSequence(__null, 973); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
974
975 if (region->isMarked(alloc)) {
976 MOZ_ASSERT(nurseryOwned)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(nurseryOwned)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(nurseryOwned))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("nurseryOwned", "./../../../../js/src/gc/BufferAllocator.cpp"
, 976); AnnotateMozCrashReason("MOZ_ASSERT" "(" "nurseryOwned"
")"); do { MOZ_CrashSequence(__null, 976); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
977 return;
978 }
979
980 if (!nurseryOwned) {
981 region->setNurseryOwned(alloc, false);
982 // If all nursery owned allocations in the region were tenured then
983 // chunk->isNurseryOwned(region) will now be stale. It will be updated when
984 // the region is swept.
985 return;
986 }
987
988 region->setMarked(alloc);
989}
990
991void BufferAllocator::markMediumNurseryOwnedBuffer(void* alloc,
992 bool nurseryOwned) {
993 BufferChunk* chunk = BufferChunk::from(alloc);
994 MOZ_ASSERT(chunk->zone == zone)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(chunk->zone == zone)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(chunk->zone == zone))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("chunk->zone == zone"
, "./../../../../js/src/gc/BufferAllocator.cpp", 994); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "chunk->zone == zone" ")"); do { MOZ_CrashSequence
(__null, 994); __attribute__((nomerge)) ::abort(); } while (false
); } } while (false);
995 MOZ_ASSERT(chunk->hasNurseryOwnedAllocs)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(chunk->hasNurseryOwnedAllocs)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(chunk->hasNurseryOwnedAllocs
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"chunk->hasNurseryOwnedAllocs", "./../../../../js/src/gc/BufferAllocator.cpp"
, 995); AnnotateMozCrashReason("MOZ_ASSERT" "(" "chunk->hasNurseryOwnedAllocs"
")"); do { MOZ_CrashSequence(__null, 995); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
996 MOZ_ASSERT(chunk->isAllocated(alloc))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(chunk->isAllocated(alloc))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(chunk->isAllocated(alloc)
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"chunk->isAllocated(alloc)", "./../../../../js/src/gc/BufferAllocator.cpp"
, 996); AnnotateMozCrashReason("MOZ_ASSERT" "(" "chunk->isAllocated(alloc)"
")"); do { MOZ_CrashSequence(__null, 996); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
997 MOZ_ASSERT(chunk->isNurseryOwned(alloc))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(chunk->isNurseryOwned(alloc))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(chunk->isNurseryOwned(alloc
)))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("chunk->isNurseryOwned(alloc)", "./../../../../js/src/gc/BufferAllocator.cpp"
, 997); AnnotateMozCrashReason("MOZ_ASSERT" "(" "chunk->isNurseryOwned(alloc)"
")"); do { MOZ_CrashSequence(__null, 997); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
998
999 if (chunk->isMarked(alloc)) {
1000 MOZ_ASSERT(nurseryOwned)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(nurseryOwned)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(nurseryOwned))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("nurseryOwned", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1000); AnnotateMozCrashReason("MOZ_ASSERT" "(" "nurseryOwned"
")"); do { MOZ_CrashSequence(__null, 1000); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1001 return;
1002 }
1003
1004 size_t size = chunk->allocBytes(alloc);
1005 increaseHeapSize(size, nurseryOwned, false, false);
1006
1007 if (!nurseryOwned) {
1008 // Change the allocation to a tenured owned one. This prevents sweeping in a
1009 // minor collection.
1010 chunk->setNurseryOwned(alloc, false);
1011 return;
1012 }
1013
1014 chunk->setMarked(alloc);
1015}
1016
1017void BufferAllocator::markLargeNurseryOwnedBuffer(LargeBuffer* buffer,
1018 bool nurseryOwned) {
1019 MOZ_ASSERT(buffer->isNurseryOwned)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(buffer->isNurseryOwned)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(buffer->isNurseryOwned)))
, 0))) { do { } while (false); MOZ_ReportAssertionFailure("buffer->isNurseryOwned"
, "./../../../../js/src/gc/BufferAllocator.cpp", 1019); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "buffer->isNurseryOwned" ")"); do { MOZ_CrashSequence
(__null, 1019); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
1020
1021 // The buffer metadata is held in a small buffer. Check whether it has already
1022 // been marked.
1023 auto* region = SmallBufferRegion::from(buffer);
1024 MOZ_ASSERT(region->isNurseryOwned(buffer))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(region->isNurseryOwned(buffer))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(region->isNurseryOwned(buffer
)))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("region->isNurseryOwned(buffer)", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1024); AnnotateMozCrashReason("MOZ_ASSERT" "(" "region->isNurseryOwned(buffer)"
")"); do { MOZ_CrashSequence(__null, 1024); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1025
1026 if (region->isMarked(buffer)) {
1027 MOZ_ASSERT(nurseryOwned)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(nurseryOwned)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(nurseryOwned))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("nurseryOwned", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1027); AnnotateMozCrashReason("MOZ_ASSERT" "(" "nurseryOwned"
")"); do { MOZ_CrashSequence(__null, 1027); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1028 return;
1029 }
1030
1031 markSmallNurseryOwnedBuffer(buffer, nurseryOwned);
1032
1033 largeNurseryAllocsToSweep.ref().remove(buffer);
1034
1035 size_t usableSize = buffer->allocBytes();
1036 increaseHeapSize(usableSize, nurseryOwned, false, false);
1037
1038 if (!nurseryOwned) {
1039 buffer->isNurseryOwned = false;
1040 buffer->allocatedDuringCollection = majorState != State::NotCollecting;
1041 largeTenuredAllocs.ref().pushBack(buffer);
1042 return;
1043 }
1044
1045 largeNurseryAllocs.ref().pushBack(buffer);
1046}
1047
1048bool BufferAllocator::isMarkedBlack(void* alloc) {
1049 checkMainThread();
1050
1051 if (IsLargeAlloc(alloc)) {
1052 // The buffer metadata is held in a small buffer.
1053 alloc = lookupLargeBuffer(alloc);
1054 } else if (!IsSmallAlloc(alloc)) {
1055 MOZ_ASSERT(IsMediumAlloc(alloc))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(IsMediumAlloc(alloc))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(IsMediumAlloc(alloc)))), 0))
) { do { } while (false); MOZ_ReportAssertionFailure("IsMediumAlloc(alloc)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 1055); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "IsMediumAlloc(alloc)" ")"); do { MOZ_CrashSequence
(__null, 1055); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
1056 BufferChunk* chunk = BufferChunk::from(alloc);
1057 return chunk->isMarked(alloc);
1058 }
1059
1060 auto* region = SmallBufferRegion::from(alloc);
1061 return region->isMarked(alloc);
1062}
1063
1064/* static */
1065void* BufferAllocator::TraceEdge(JSTracer* trc, void** bufferp,
1066 const char* name) {
1067 // Buffers are conceptually part of the owning cell and are not reported to
1068 // the tracer.
1069
1070 // TODO: This should be unified with the rest of the tracing system.
1071
1072 MOZ_ASSERT(bufferp)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(bufferp)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(bufferp))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("bufferp", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1072); AnnotateMozCrashReason("MOZ_ASSERT" "(" "bufferp" ")"
); do { MOZ_CrashSequence(__null, 1072); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1073
1074 void* buffer = *bufferp;
Value stored to 'buffer' during its initialization is never read
1075#ifdef JS_GC_CONCURRENT_MARKING
1076 // Conservatively perform an atomic load even when marking is not concurrent.
1077 buffer = __atomic_load_n(bufferp, __ATOMIC_RELAXED0);
1078#else
1079 buffer = *bufferp;
1080#endif
1081
1082 if (!buffer) {
1083 return nullptr;
1084 }
1085
1086 if (!IsLargeAlloc(buffer) &&
1087 js::gc::detail::GetGCAddressChunkBase(buffer)->isNurseryChunk()) {
1088 // JSObject slots and elements can be allocated in the nursery and this is
1089 // handled separately.
1090 return buffer;
1091 }
1092
1093 MOZ_ASSERT(IsBufferAlloc(buffer))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(IsBufferAlloc(buffer))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(IsBufferAlloc(buffer)))), 0)
)) { do { } while (false); MOZ_ReportAssertionFailure("IsBufferAlloc(buffer)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 1093); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "IsBufferAlloc(buffer)" ")"); do { MOZ_CrashSequence
(__null, 1093); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
1094
1095 if (MOZ_UNLIKELY(IsLargeAlloc(buffer))(__builtin_expect(!!(IsLargeAlloc(buffer)), 0))) {
1096 TraceLargeAlloc(trc, bufferp, name);
1097 return buffer;
1098 }
1099
1100 BufferChunk* chunk = BufferChunk::from(buffer);
1101 BufferAllocator& allocator = chunk->zone->bufferAllocator;
1102
1103 if (IsSmallAlloc(buffer)) {
1104 allocator.traceSmallAlloc(trc, buffer, name);
1105 return buffer;
1106 }
1107
1108 allocator.traceMediumAlloc(trc, buffer, name);
1109 return buffer;
1110}
1111
1112void BufferAllocator::traceSmallAlloc(JSTracer* trc, void* alloc,
1113 const char* name) {
1114 auto* region = SmallBufferRegion::from(alloc);
1115
1116 if (trc->isTenuringTracer()) {
1117 if (region->isNurseryOwned(alloc)) {
1118 bool nurseryOwned = TenuringTracer::From(trc)->sourceIsInNursery.value();
1119 markSmallNurseryOwnedBuffer(alloc, nurseryOwned);
1120 }
1121 return;
1122 }
1123
1124 if (trc->isMarkingTracer()) {
1125 if (zone->isGCMarking() && !region->isNurseryOwned(alloc)) {
1126 markSmallTenuredAlloc(alloc);
1127 }
1128 return;
1129 }
1130}
1131
1132void BufferAllocator::traceMediumAlloc(JSTracer* trc, void* alloc,
1133 const char* name) {
1134 BufferChunk* chunk = BufferChunk::from(alloc);
1135
1136 if (trc->isTenuringTracer()) {
1137 if (chunk->isNurseryOwned(alloc)) {
1138 bool nurseryOwned = TenuringTracer::From(trc)->sourceIsInNursery.value();
1139 markMediumNurseryOwnedBuffer(alloc, nurseryOwned);
1140 }
1141 return;
1142 }
1143
1144 if (trc->isMarkingTracer()) {
1145 if (zone->isGCMarking() && !chunk->isNurseryOwned(alloc)) {
1146 markMediumTenuredAlloc(alloc);
1147 }
1148 return;
1149 }
1150}
1151
1152/* static */
1153void BufferAllocator::TraceLargeAlloc(JSTracer* trc, void** allocp,
1154 const char* name) {
1155 void* alloc = *allocp;
1156 BufferAllocatorRuntime* runtime = &trc->runtime()->gc.bufferRuntime();
1157 LargeBuffer* buffer = runtime->lookupLargeBuffer(alloc);
1158 Zone* zone = buffer->zoneFromAnyThread(); // May be parallel marking here.
1159 zone->bufferAllocator.traceLargeBuffer(trc, buffer, name);
1160}
1161
1162void BufferAllocator::traceLargeBuffer(JSTracer* trc, LargeBuffer* buffer,
1163 const char* name) {
1164 if (trc->isTenuringTracer()) {
1165 if (buffer->isNurseryOwned) {
1166 bool nurseryOwned = TenuringTracer::From(trc)->sourceIsInNursery.value();
1167 markLargeNurseryOwnedBuffer(buffer, nurseryOwned);
1168 }
1169 return;
1170 }
1171
1172 if (trc->isMarkingTracer()) {
1173 if (zone->isGCMarking() && !buffer->isNurseryOwned) {
1174 markLargeTenuredBuffer(buffer);
1175 }
1176 return;
1177 }
1178}
1179
1180bool BufferAllocator::markTenuredAlloc(void* alloc) {
1181 MOZ_ASSERT(alloc)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(alloc)>::isValid, "invalid assertion condition");
if ((__builtin_expect(!!(!(!!(alloc))), 0))) { do { } while (
false); MOZ_ReportAssertionFailure("alloc", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1181); AnnotateMozCrashReason("MOZ_ASSERT" "(" "alloc" ")")
; do { MOZ_CrashSequence(__null, 1181); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1182 MOZ_ASSERT(!isNurseryOwned(alloc))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!isNurseryOwned(alloc))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!isNurseryOwned(alloc)))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("!isNurseryOwned(alloc)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 1182); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!isNurseryOwned(alloc)" ")"); do { MOZ_CrashSequence
(__null, 1182); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
1183
1184 if (IsLargeAlloc(alloc)) {
1185 LargeBuffer* buffer = lookupLargeBuffer(alloc);
1186 return markLargeTenuredBuffer(buffer);
1187 }
1188
1189 if (IsSmallAlloc(alloc)) {
1190 return markSmallTenuredAlloc(alloc);
1191 }
1192
1193 return markMediumTenuredAlloc(alloc);
1194}
1195
1196bool BufferAllocator::markSmallTenuredAlloc(void* alloc) {
1197 auto* chunk = BufferChunk::from(alloc);
1198 if (chunk->allocatedDuringCollection) {
1199 // Will not be swept, already counted as marked.
1200 return false;
1201 }
1202
1203 auto* region = SmallBufferRegion::from(alloc);
1204 MOZ_ASSERT(region->isAllocated(alloc))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(region->isAllocated(alloc))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(region->isAllocated(alloc
)))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("region->isAllocated(alloc)", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1204); AnnotateMozCrashReason("MOZ_ASSERT" "(" "region->isAllocated(alloc)"
")"); do { MOZ_CrashSequence(__null, 1204); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1205 return region->setMarked(alloc);
1206}
1207
1208bool BufferAllocator::markMediumTenuredAlloc(void* alloc) {
1209 BufferChunk* chunk = BufferChunk::from(alloc);
1210 MOZ_ASSERT(chunk->isAllocated(alloc))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(chunk->isAllocated(alloc))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(chunk->isAllocated(alloc)
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"chunk->isAllocated(alloc)", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1210); AnnotateMozCrashReason("MOZ_ASSERT" "(" "chunk->isAllocated(alloc)"
")"); do { MOZ_CrashSequence(__null, 1210); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1211 if (chunk->allocatedDuringCollection) {
1212 // Will not be swept, already counted as marked.
1213 return false;
1214 }
1215
1216 return chunk->setMarked(alloc);
1217}
1218
1219void BufferAllocator::startMinorCollection(MaybeLock& lock) {
1220 checkMainThread();
1221 maybeMergeSweptData(lock);
1222
1223#ifdef DEBUG1
1224 MOZ_ASSERT(minorState == State::NotCollecting)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(minorState == State::NotCollecting)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(minorState == State::NotCollecting
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"minorState == State::NotCollecting", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1224); AnnotateMozCrashReason("MOZ_ASSERT" "(" "minorState == State::NotCollecting"
")"); do { MOZ_CrashSequence(__null, 1224); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1225 if (majorState == State::NotCollecting) {
1226 if (gc->hasZealMode(ZealMode::CheckHeapBeforeMinorGC)) {
1227 // This is too expensive to run on every minor GC.
1228 checkGCStateNotInUse(lock);
1229 }
1230 }
1231#endif
1232
1233 // Large allocations that are marked when tracing the nursery will be moved
1234 // back to the main list.
1235 MOZ_ASSERT(largeNurseryAllocsToSweep.ref().isEmpty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(largeNurseryAllocsToSweep.ref().isEmpty())>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(largeNurseryAllocsToSweep.ref().isEmpty()))), 0))) { do { }
while (false); MOZ_ReportAssertionFailure("largeNurseryAllocsToSweep.ref().isEmpty()"
, "./../../../../js/src/gc/BufferAllocator.cpp", 1235); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "largeNurseryAllocsToSweep.ref().isEmpty()"
")"); do { MOZ_CrashSequence(__null, 1235); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1236 std::swap(largeNurseryAllocs.ref(), largeNurseryAllocsToSweep.ref());
1237
1238 minorState = State::Marking;
1239}
1240
1241bool BufferAllocator::startMinorSweeping() {
1242 // Called during minor GC. Operates on the active allocs/chunks lists. The 'to
1243 // sweep' lists do not contain nursery owned allocations.
1244
1245#ifdef DEBUG1
1246 checkMainThread();
1247 MOZ_ASSERT(minorState == State::Marking)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(minorState == State::Marking)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(minorState == State::Marking
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"minorState == State::Marking", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1247); AnnotateMozCrashReason("MOZ_ASSERT" "(" "minorState == State::Marking"
")"); do { MOZ_CrashSequence(__null, 1247); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1248 {
1249 AutoLock lock(runtime());
1250 MOZ_ASSERT(!minorSweepingFinished)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!minorSweepingFinished)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!minorSweepingFinished))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("!minorSweepingFinished"
, "./../../../../js/src/gc/BufferAllocator.cpp", 1250); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!minorSweepingFinished" ")"); do { MOZ_CrashSequence
(__null, 1250); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
1251 MOZ_ASSERT(sweptMixedChunks.ref().isEmpty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(sweptMixedChunks.ref().isEmpty())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(sweptMixedChunks.ref().isEmpty
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("sweptMixedChunks.ref().isEmpty()", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1251); AnnotateMozCrashReason("MOZ_ASSERT" "(" "sweptMixedChunks.ref().isEmpty()"
")"); do { MOZ_CrashSequence(__null, 1251); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1252 }
1253 for (LargeBuffer* buffer : largeNurseryAllocs.ref()) {
1254 MOZ_ASSERT(buffer->isNurseryOwned)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(buffer->isNurseryOwned)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(buffer->isNurseryOwned)))
, 0))) { do { } while (false); MOZ_ReportAssertionFailure("buffer->isNurseryOwned"
, "./../../../../js/src/gc/BufferAllocator.cpp", 1254); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "buffer->isNurseryOwned" ")"); do { MOZ_CrashSequence
(__null, 1254); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
1255 }
1256 for (LargeBuffer* buffer : largeNurseryAllocsToSweep.ref()) {
1257 MOZ_ASSERT(buffer->isNurseryOwned)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(buffer->isNurseryOwned)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(buffer->isNurseryOwned)))
, 0))) { do { } while (false); MOZ_ReportAssertionFailure("buffer->isNurseryOwned"
, "./../../../../js/src/gc/BufferAllocator.cpp", 1257); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "buffer->isNurseryOwned" ")"); do { MOZ_CrashSequence
(__null, 1257); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
1258 }
1259#endif
1260
1261 // Check whether there are any medium chunks containing nursery owned
1262 // allocations that need to be swept.
1263 if (currentMixedChunks.ref().isEmpty() &&
1264 !availableChunks.ref().hasMixedChunks() &&
1265 largeNurseryAllocsToSweep.ref().isEmpty()) {
1266 // Nothing more to do. Don't transition to sweeping state.
1267 minorState = State::NotCollecting;
1268 return false;
1269 }
1270
1271#ifdef DEBUG1
1272 for (BufferChunk* chunk : currentMixedChunks.ref()) {
1273 MOZ_ASSERT(!chunk->ownsFreeLists)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!chunk->ownsFreeLists)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!chunk->ownsFreeLists))),
0))) { do { } while (false); MOZ_ReportAssertionFailure("!chunk->ownsFreeLists"
, "./../../../../js/src/gc/BufferAllocator.cpp", 1273); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!chunk->ownsFreeLists" ")"); do { MOZ_CrashSequence
(__null, 1273); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
1274 chunk->freeLists.ref().assertEmpty();
1275 }
1276#endif
1277
1278 // Discard free regions in |mixedFreeLists| (these live in
1279 // |currentMixedChunks|). They will be rebuilt by sweeping.
1280 mixedFreeLists.ref().clear();
1281
1282 // Move all mixed chunks to the list of chunks to sweep.
1283 mixedChunksToSweep.ref() = std::move(currentMixedChunks.ref());
1284 mixedChunksToSweep.ref().append(availableChunks.ref().extractMixedChunks());
1285
1286 minorState = State::Sweeping;
1287 runtime()->incOffThreadCount();
1288
1289 return true;
1290}
1291
1292struct LargeAllocToFree {
1293 size_t bytes;
1294 LargeAllocToFree* next = nullptr;
1295
1296 explicit LargeAllocToFree(size_t bytes) : bytes(bytes) {}
1297};
1298
1299static void PushLargeAllocToFree(LargeAllocToFree** listHead,
1300 LargeBuffer* buffer) {
1301 auto* alloc = new (buffer->data()) LargeAllocToFree(buffer->bytes);
1302 alloc->next = *listHead;
1303 *listHead = alloc;
1304}
1305
1306static void FreeLargeAllocs(LargeAllocToFree* listHead) {
1307 while (listHead) {
1308 LargeAllocToFree* alloc = listHead;
1309 LargeAllocToFree* next = alloc->next;
1310 UnmapPages(alloc, alloc->bytes);
1311 listHead = next;
1312 }
1313}
1314
1315void BufferAllocator::sweepForMinorCollection() {
1316 // Called on a background thread.
1317
1318 MOZ_ASSERT(minorState.refNoCheck() == State::Sweeping)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(minorState.refNoCheck() == State::Sweeping)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(minorState.refNoCheck() == State::Sweeping))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("minorState.refNoCheck() == State::Sweeping"
, "./../../../../js/src/gc/BufferAllocator.cpp", 1318); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "minorState.refNoCheck() == State::Sweeping"
")"); do { MOZ_CrashSequence(__null, 1318); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1319 {
1320 AutoLock lock(runtime());
1321 MOZ_ASSERT(sweptMixedChunks.ref().isEmpty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(sweptMixedChunks.ref().isEmpty())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(sweptMixedChunks.ref().isEmpty
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("sweptMixedChunks.ref().isEmpty()", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1321); AnnotateMozCrashReason("MOZ_ASSERT" "(" "sweptMixedChunks.ref().isEmpty()"
")"); do { MOZ_CrashSequence(__null, 1321); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1322 }
1323
1324 // Bug 1961749: Freeing large buffers can be slow so it might be worth
1325 // splitting sweeping into two phases so that all zones get their medium
1326 // buffers swept and made available for allocation before any large buffers
1327 // are freed.
1328
1329 // Freeing large buffers may be slow, so leave that till the end. However
1330 // large buffer metadata is stored in small buffers so form a list of large
1331 // buffers to free before sweeping small buffers.
1332 LargeAllocToFree* largeAllocsToFree = nullptr;
1333 while (!largeNurseryAllocsToSweep.ref().isEmpty()) {
1334 LargeBuffer* buffer = largeNurseryAllocsToSweep.ref().popFirst();
1335 PushLargeAllocToFree(&largeAllocsToFree, buffer);
1336 MaybeLock lock(std::in_place, runtime());
1337 unregisterLarge(buffer, true, lock);
1338 }
1339
1340 while (!mixedChunksToSweep.ref().isEmpty()) {
1341 BufferChunk* chunk = mixedChunksToSweep.ref().popFirst();
1342 if (sweepChunk(chunk, SweepKind::Nursery, false)) {
1343 {
1344 AutoLock lock(runtime());
1345 sweptMixedChunks.ref().pushBack(chunk);
1346 }
1347
1348 // Signal to the main thread that swept data is available by setting this
1349 // relaxed atomic flag.
1350 hasSweepDataToMerge = true;
1351 }
1352 }
1353
1354 // Unmap large buffers.
1355 FreeLargeAllocs(largeAllocsToFree);
1356
1357 // Signal to main thread to update minorState.
1358 {
1359 AutoLock lock(runtime());
1360 MOZ_ASSERT(!minorSweepingFinished)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!minorSweepingFinished)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!minorSweepingFinished))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("!minorSweepingFinished"
, "./../../../../js/src/gc/BufferAllocator.cpp", 1360); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!minorSweepingFinished" ")"); do { MOZ_CrashSequence
(__null, 1360); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
1361 minorSweepingFinished = true;
1362 hasSweepDataToMerge = true;
1363 }
1364}
1365
1366void BufferAllocator::startMajorCollection(MaybeLock& lock) {
1367 checkMainThread();
1368
1369 maybeMergeSweptData(lock);
1370
1371#ifdef DEBUG1
1372 MOZ_ASSERT(majorState == State::NotCollecting)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(majorState == State::NotCollecting)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(majorState == State::NotCollecting
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"majorState == State::NotCollecting", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1372); AnnotateMozCrashReason("MOZ_ASSERT" "(" "majorState == State::NotCollecting"
")"); do { MOZ_CrashSequence(__null, 1372); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1373 checkGCStateNotInUse(lock);
1374
1375 // Everything is tenured since we just evicted the nursery, or will be by the
1376 // time minor sweeping finishes.
1377 MOZ_ASSERT(mixedFreeLists.ref().isEmpty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(mixedFreeLists.ref().isEmpty())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(mixedFreeLists.ref().isEmpty
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("mixedFreeLists.ref().isEmpty()", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1377); AnnotateMozCrashReason("MOZ_ASSERT" "(" "mixedFreeLists.ref().isEmpty()"
")"); do { MOZ_CrashSequence(__null, 1377); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1378 MOZ_ASSERT(currentMixedChunks.ref().isEmpty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(currentMixedChunks.ref().isEmpty())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(currentMixedChunks.ref().isEmpty
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("currentMixedChunks.ref().isEmpty()", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1378); AnnotateMozCrashReason("MOZ_ASSERT" "(" "currentMixedChunks.ref().isEmpty()"
")"); do { MOZ_CrashSequence(__null, 1378); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1379 MOZ_ASSERT(!availableChunks.ref().hasMixedChunks())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!availableChunks.ref().hasMixedChunks())>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(!availableChunks.ref().hasMixedChunks()))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("!availableChunks.ref().hasMixedChunks()"
, "./../../../../js/src/gc/BufferAllocator.cpp", 1379); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!availableChunks.ref().hasMixedChunks()" ")"
); do { MOZ_CrashSequence(__null, 1379); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1380 MOZ_ASSERT(largeNurseryAllocs.ref().isEmpty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(largeNurseryAllocs.ref().isEmpty())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(largeNurseryAllocs.ref().isEmpty
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("largeNurseryAllocs.ref().isEmpty()", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1380); AnnotateMozCrashReason("MOZ_ASSERT" "(" "largeNurseryAllocs.ref().isEmpty()"
")"); do { MOZ_CrashSequence(__null, 1380); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1381
1382 for (BufferChunk* chunk : currentTenuredChunks.ref()) {
1383 MOZ_ASSERT(!chunk->ownsFreeLists)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!chunk->ownsFreeLists)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!chunk->ownsFreeLists))),
0))) { do { } while (false); MOZ_ReportAssertionFailure("!chunk->ownsFreeLists"
, "./../../../../js/src/gc/BufferAllocator.cpp", 1383); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!chunk->ownsFreeLists" ")"); do { MOZ_CrashSequence
(__null, 1383); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
1384 chunk->freeLists.ref().assertEmpty();
1385 }
1386#endif
1387
1388 gc->bufferRuntime().resetRetainedStats();
1389
1390 largeTenuredAllocsToSweep.ref() = std::move(largeTenuredAllocs.ref());
1391
1392 // Discard free regions in |tenuredFreeLists| (these live in
1393 // |currentTenuredChunks|). They will be rebuilt by sweeping or on abort.
1394 tenuredFreeLists.ref().clear();
1395
1396 // Move all tenured chunks to the list of chunks to sweep.
1397 tenuredChunksToSweep.ref() = std::move(currentTenuredChunks.ref());
1398 MOZ_ASSERT(!availableChunks.ref().hasMixedChunks())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!availableChunks.ref().hasMixedChunks())>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(!availableChunks.ref().hasMixedChunks()))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("!availableChunks.ref().hasMixedChunks()"
, "./../../../../js/src/gc/BufferAllocator.cpp", 1398); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!availableChunks.ref().hasMixedChunks()" ")"
); do { MOZ_CrashSequence(__null, 1398); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1399 tenuredChunksToSweep.ref().append(availableChunks.ref().extractAllChunks());
1400
1401 if (minorState == State::Sweeping) {
1402 // Ensure swept nursery chunks are moved to the currentTenuredChunks lists
1403 // in mergeSweptData.
1404 majorStartedWhileMinorSweeping = true;
1405 }
1406
1407#ifdef DEBUG1
1408 MOZ_ASSERT(currentTenuredChunks.ref().isEmpty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(currentTenuredChunks.ref().isEmpty())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(currentTenuredChunks.ref().isEmpty
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("currentTenuredChunks.ref().isEmpty()", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1408); AnnotateMozCrashReason("MOZ_ASSERT" "(" "currentTenuredChunks.ref().isEmpty()"
")"); do { MOZ_CrashSequence(__null, 1408); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1409 MOZ_ASSERT(availableChunks.ref().isEmpty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(availableChunks.ref().isEmpty())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(availableChunks.ref().isEmpty
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("availableChunks.ref().isEmpty()", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1409); AnnotateMozCrashReason("MOZ_ASSERT" "(" "availableChunks.ref().isEmpty()"
")"); do { MOZ_CrashSequence(__null, 1409); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1410 tenuredFreeLists.ref().assertEmpty();
1411 MOZ_ASSERT(largeTenuredAllocs.ref().isEmpty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(largeTenuredAllocs.ref().isEmpty())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(largeTenuredAllocs.ref().isEmpty
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("largeTenuredAllocs.ref().isEmpty()", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1411); AnnotateMozCrashReason("MOZ_ASSERT" "(" "largeTenuredAllocs.ref().isEmpty()"
")"); do { MOZ_CrashSequence(__null, 1411); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1412#endif
1413
1414 majorState = State::Marking;
1415}
1416
1417void BufferAllocator::startMajorSweeping(MaybeLock& lock) {
1418 // Called when a zone transitions from marking to sweeping.
1419
1420#ifdef DEBUG1
1421 checkMainThread();
1422 MOZ_ASSERT(majorState == State::Marking)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(majorState == State::Marking)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(majorState == State::Marking
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"majorState == State::Marking", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1422); AnnotateMozCrashReason("MOZ_ASSERT" "(" "majorState == State::Marking"
")"); do { MOZ_CrashSequence(__null, 1422); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1423 MOZ_ASSERT(zone->isGCFinished())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(zone->isGCFinished())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(zone->isGCFinished()))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("zone->isGCFinished()"
, "./../../../../js/src/gc/BufferAllocator.cpp", 1423); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "zone->isGCFinished()" ")"); do { MOZ_CrashSequence
(__null, 1423); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
1424 MOZ_ASSERT(!majorSweepingFinished.refNoCheck())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!majorSweepingFinished.refNoCheck())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!majorSweepingFinished.refNoCheck
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("!majorSweepingFinished.refNoCheck()", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1424); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!majorSweepingFinished.refNoCheck()"
")"); do { MOZ_CrashSequence(__null, 1424); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1425#endif
1426
1427 maybeMergeSweptData(lock);
1428 MOZ_ASSERT(!majorStartedWhileMinorSweeping)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!majorStartedWhileMinorSweeping)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!majorStartedWhileMinorSweeping
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"!majorStartedWhileMinorSweeping", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1428); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!majorStartedWhileMinorSweeping"
")"); do { MOZ_CrashSequence(__null, 1428); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1429
1430 clearMarkBitsInStolenChunks();
1431
1432 if (minorState == State::Sweeping) {
1433 majorSweepingStartedWhileMinorSweeping = true;
1434 }
1435
1436 majorState = State::Sweeping;
1437 runtime()->incOffThreadCount();
1438}
1439
1440void BufferAllocator::sweepForMajorCollection(bool shouldDecommit) {
1441 // Called on a background thread.
1442
1443 MOZ_ASSERT(majorState.refNoCheck() == State::Sweeping)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(majorState.refNoCheck() == State::Sweeping)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(majorState.refNoCheck() == State::Sweeping))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("majorState.refNoCheck() == State::Sweeping"
, "./../../../../js/src/gc/BufferAllocator.cpp", 1443); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "majorState.refNoCheck() == State::Sweeping"
")"); do { MOZ_CrashSequence(__null, 1443); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1444
1445 // Sweep large allocs first since they rely on the mark bits of their
1446 // corresponding LargeBuffer structures which are stored small buffers.
1447 //
1448 // It's tempting to try and optimize this by moving the allocations between
1449 // lists when they are marked, in the same way as for nursery sweeping. This
1450 // would require synchronizing the list modification when marking in parallel,
1451 // so is probably not worth it.
1452 LargeAllocList sweptLargeAllocs;
1453 LargeAllocToFree* largeAllocsToFree = nullptr;
1454 while (!largeTenuredAllocsToSweep.ref().isEmpty()) {
1455 LargeBuffer* buffer = largeTenuredAllocsToSweep.ref().popFirst();
1456 if (isLargeTenuredMarked(buffer)) {
1457 buffer->isMarked = false;
1458 sweptLargeAllocs.pushBack(buffer);
1459 } else {
1460 PushLargeAllocToFree(&largeAllocsToFree, buffer);
1461 MaybeLock lock(std::in_place, runtime());
1462 unregisterLarge(buffer, true, lock);
1463 }
1464 }
1465
1466 while (!tenuredChunksToSweep.ref().isEmpty()) {
1467 BufferChunk* chunk = tenuredChunksToSweep.ref().popFirst();
1468 if (sweepChunk(chunk, SweepKind::Tenured, shouldDecommit)) {
1469 {
1470 AutoLock lock(runtime());
1471 sweptTenuredChunks.ref().pushBack(chunk);
1472 }
1473
1474 // Signal to the main thread that swept data is available by setting this
1475 // relaxed atomic flag.
1476 hasSweepDataToMerge = true;
1477 }
1478 }
1479
1480 // Unmap large buffers.
1481 //
1482 // Bug 1961749: This could possibly run after signalling sweeping is finished
1483 // or concurrently with other sweeping.
1484 FreeLargeAllocs(largeAllocsToFree);
1485
1486 AutoLock lock(runtime());
1487 sweptLargeTenuredAllocs.ref() = std::move(sweptLargeAllocs);
1488
1489 // Signal to main thread to update majorState.
1490 MOZ_ASSERT(!majorSweepingFinished)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!majorSweepingFinished)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!majorSweepingFinished))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("!majorSweepingFinished"
, "./../../../../js/src/gc/BufferAllocator.cpp", 1490); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!majorSweepingFinished" ")"); do { MOZ_CrashSequence
(__null, 1490); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
1491 majorSweepingFinished = true;
1492}
1493
1494void BufferAllocator::finishMajorCollection(const AutoLock& lock) {
1495 // This can be called in any state:
1496 //
1497 // - NotCollecting: after major sweeping has finished and the state has been
1498 // reset to NotCollecting in mergeSweptData.
1499 //
1500 // - Marking: if collection was aborted and startMajorSweeping was not
1501 // called.
1502 //
1503 // - Sweeping: if sweeping has finished and mergeSweptData has not been
1504 // called yet.
1505
1506 checkMainThread();
1507 MOZ_ASSERT_IF(majorState == State::Sweeping, majorSweepingFinished)do { if (majorState == State::Sweeping) { do { static_assert(
mozilla::detail::AssertionConditionType<decltype(majorSweepingFinished
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(majorSweepingFinished))), 0))) { do { } while (false
); MOZ_ReportAssertionFailure("majorSweepingFinished", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1507); AnnotateMozCrashReason("MOZ_ASSERT" "(" "majorSweepingFinished"
")"); do { MOZ_CrashSequence(__null, 1507); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false);
1508
1509 if (minorState == State::Sweeping || majorState == State::Sweeping) {
1510 mergeSweptData(lock);
1511 }
1512
1513 if (majorState == State::Marking) {
1514 abortMajorSweeping(lock);
1515 }
1516
1517 // Set whether to use mixed chunks for tenured allocations too. Use the
1518 // same chunks for both kinds of allocation if the heap is small.
1519 allocTenuredInMixedChunks = totalChunkCount <= 4;
1520
1521#ifdef DEBUG1
1522 checkGCStateNotInUse(lock);
1523#endif
1524}
1525
1526void BufferAllocator::abortMajorSweeping(const AutoLock& lock) {
1527 // We have aborted collection without sweeping this zone. Restore or rebuild
1528 // the original state.
1529
1530#ifdef DEBUG1
1531 checkMainThread();
1532 MOZ_ASSERT(majorState == State::Marking)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(majorState == State::Marking)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(majorState == State::Marking
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"majorState == State::Marking", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1532); AnnotateMozCrashReason("MOZ_ASSERT" "(" "majorState == State::Marking"
")"); do { MOZ_CrashSequence(__null, 1532); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1533 MOZ_ASSERT(sweptTenuredChunks.ref().isEmpty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(sweptTenuredChunks.ref().isEmpty())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(sweptTenuredChunks.ref().isEmpty
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("sweptTenuredChunks.ref().isEmpty()", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1533); AnnotateMozCrashReason("MOZ_ASSERT" "(" "sweptTenuredChunks.ref().isEmpty()"
")"); do { MOZ_CrashSequence(__null, 1533); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1534 for (auto chunk = availableChunks.ref().chunkIter(); !chunk.done();
1535 chunk.next()) {
1536 MOZ_ASSERT(chunk->allocatedDuringCollection)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(chunk->allocatedDuringCollection)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(chunk->allocatedDuringCollection
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"chunk->allocatedDuringCollection", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1536); AnnotateMozCrashReason("MOZ_ASSERT" "(" "chunk->allocatedDuringCollection"
")"); do { MOZ_CrashSequence(__null, 1536); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1537 }
1538#endif
1539
1540 clearMarkBitsInStolenChunks();
1541
1542 clearAllocatedDuringCollectionState(lock);
1543
1544 if (minorState == State::Sweeping) {
1545 // If we are minor sweeping then stolen chunks may be in
1546 // |mixedChunksToSweep|; ensure mergeSweptData clears their
1547 // stolenFromSweepList flag and stale mark bits.
1548 majorSweepingStartedWhileMinorSweeping = true;
1549 // Also, chunks with allocatedDuringCollection set may be present in
1550 // |mixedChunksToSweep|. Set a flag so these are cleared when they are
1551 // merged later.
1552 majorFinishedWhileMinorSweeping = true;
1553 }
1554
1555 // Clear mark bits for chunks we didn't end up sweeping.
1556 while (BufferChunk* chunk = tenuredChunksToSweep.ref().popFirst()) {
1557 if (!chunk->ownsFreeLists) {
1558 rebuildFreeLists(chunk);
1559 }
1560 MOZ_ASSERT(chunk->ownsFreeLists)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(chunk->ownsFreeLists)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(chunk->ownsFreeLists))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("chunk->ownsFreeLists"
, "./../../../../js/src/gc/BufferAllocator.cpp", 1560); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "chunk->ownsFreeLists" ")"); do { MOZ_CrashSequence
(__null, 1560); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
1561 clearChunkMarkBits(chunk);
1562 availableChunks.ref().addTenuredChunk(chunk);
1563 }
1564
1565 // Clear mark bits for large allocations we didn't end up sweeping.
1566 while (LargeBuffer* buffer = largeTenuredAllocsToSweep.ref().popLast()) {
1567 buffer->isMarked = false;
1568 largeTenuredAllocs.ref().pushFront(buffer);
1569 }
1570
1571 majorState = State::NotCollecting;
1572}
1573
1574void BufferAllocator::clearMarkBitsInStolenChunks() {
1575#ifdef JS_GC_CONCURRENT_MARKING
1576 // Clear mark bits for any chunks stolen from the sweep list. These are in the
1577 // current or available lists and have stolenFromSweepList == true. There
1578 // shouldn't be too many of these as everything got moved to the sweep list at
1579 // the start of marking.
1580 //
1581 // We also need to do the same for chunks coming back from minor sweeping when
1582 // majorSweepingStartedWhileMinorSweeping is set.
1583
1584 // TODO: Optimize this. We could keep a count of such chunks and stop when we
1585 // find that number, or thread a list pointer through the stolen chunks so we
1586 // can find them easily.
1587
1588 for (BufferChunkList* list :
1589 {&currentMixedChunks.ref(), &currentTenuredChunks.ref()}) {
1590 for (BufferChunk* chunk : *list) {
1591 chunk->clearMarkBitsIfStolenChunk();
1592 }
1593 }
1594
1595 for (auto chunk = availableChunks.ref().chunkIter(); !chunk.done();
1596 chunk.next()) {
1597 chunk->clearMarkBitsIfStolenChunk();
1598 }
1599#endif
1600}
1601
1602void BufferChunk::clearMarkBitsIfStolenChunk() {
1603#ifdef JS_GC_CONCURRENT_MARKING
1604 if (stolenFromSweepList) {
1605 clearMarkBits();
1606 stolenFromSweepList = false;
1607 }
1608#else
1609 MOZ_ASSERT(!stolenFromSweepList)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!stolenFromSweepList)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!stolenFromSweepList))), 0))
) { do { } while (false); MOZ_ReportAssertionFailure("!stolenFromSweepList"
, "./../../../../js/src/gc/BufferAllocator.cpp", 1609); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!stolenFromSweepList" ")"); do { MOZ_CrashSequence
(__null, 1609); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
1610#endif
1611}
1612
1613void BufferAllocator::clearAllocatedDuringCollectionState(
1614 const AutoLock& lock) {
1615#ifdef DEBUG1
1616 // This flag is not set for large nursery-owned allocations.
1617 for (LargeBuffer* buffer : largeNurseryAllocs.ref()) {
1618 MOZ_ASSERT(!buffer->allocatedDuringCollection)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!buffer->allocatedDuringCollection)>::isValid,
"invalid assertion condition"); if ((__builtin_expect(!!(!(!
!(!buffer->allocatedDuringCollection))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("!buffer->allocatedDuringCollection"
, "./../../../../js/src/gc/BufferAllocator.cpp", 1618); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!buffer->allocatedDuringCollection" ")"
); do { MOZ_CrashSequence(__null, 1618); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1619 }
1620#endif
1621
1622 ClearAllocatedDuringCollection(currentMixedChunks.ref());
1623 ClearAllocatedDuringCollection(currentTenuredChunks.ref());
1624 ClearAllocatedDuringCollection(availableChunks.ref());
1625 ClearAllocatedDuringCollection(largeTenuredAllocs.ref());
1626}
1627
1628/* static */
1629void BufferAllocator::ClearAllocatedDuringCollection(ChunkLists& chunks) {
1630 for (auto chunk = chunks.chunkIter(); !chunk.done(); chunk.next()) {
1631 chunk->allocatedDuringCollection = false;
1632 }
1633}
1634/* static */
1635void BufferAllocator::ClearAllocatedDuringCollection(BufferChunkList& list) {
1636 for (auto* chunk : list) {
1637 chunk->allocatedDuringCollection = false;
1638 }
1639}
1640/* static */
1641void BufferAllocator::ClearAllocatedDuringCollection(LargeAllocList& list) {
1642 for (auto* element : list) {
1643 element->allocatedDuringCollection = false;
1644 }
1645}
1646
1647void BufferAllocator::maybeMergeSweptData() {
1648 if (minorState == State::Sweeping || majorState == State::Sweeping) {
1649 mergeSweptData();
1650 }
1651}
1652
1653void BufferAllocator::mergeSweptData() {
1654 AutoLock lock(runtime());
1655 mergeSweptData(lock);
1656}
1657
1658void BufferAllocator::maybeMergeSweptData(MaybeLock& lock) {
1659 if (minorState == State::Sweeping || majorState == State::Sweeping) {
1660 if (lock.isNothing()) {
1661 lock.emplace(runtime());
1662 }
1663 mergeSweptData(lock.ref());
1664 }
1665}
1666
1667void BufferAllocator::mergeSweptData(const AutoLock& lock) {
1668 checkAccess();
1669 MOZ_ASSERT(minorState == State::Sweeping || majorState == State::Sweeping)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(minorState == State::Sweeping || majorState == State
::Sweeping)>::isValid, "invalid assertion condition"); if (
(__builtin_expect(!!(!(!!(minorState == State::Sweeping || majorState
== State::Sweeping))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("minorState == State::Sweeping || majorState == State::Sweeping"
, "./../../../../js/src/gc/BufferAllocator.cpp", 1669); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "minorState == State::Sweeping || majorState == State::Sweeping"
")"); do { MOZ_CrashSequence(__null, 1669); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1670
1671 if (majorSweepingFinished) {
1672 clearAllocatedDuringCollectionState(lock);
1673
1674 if (minorState == State::Sweeping) {
1675 majorFinishedWhileMinorSweeping = true;
1676 }
1677 }
1678
1679 // Merge swept chunks that previously contained nursery owned allocations. If
1680 // semispace nursery collection is in use then these chunks may contain both
1681 // nursery and tenured-owned allocations, otherwise all allocations will be
1682 // tenured-owned.
1683 while (!sweptMixedChunks.ref().isEmpty()) {
1684 BufferChunk* chunk = sweptMixedChunks.ref().popLast();
1685 MOZ_ASSERT(chunk->ownsFreeLists)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(chunk->ownsFreeLists)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(chunk->ownsFreeLists))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("chunk->ownsFreeLists"
, "./../../../../js/src/gc/BufferAllocator.cpp", 1685); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "chunk->ownsFreeLists" ")"); do { MOZ_CrashSequence
(__null, 1685); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
1686 MOZ_ASSERT(chunk->hasNurseryOwnedAllocs)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(chunk->hasNurseryOwnedAllocs)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(chunk->hasNurseryOwnedAllocs
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"chunk->hasNurseryOwnedAllocs", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1686); AnnotateMozCrashReason("MOZ_ASSERT" "(" "chunk->hasNurseryOwnedAllocs"
")"); do { MOZ_CrashSequence(__null, 1686); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1687 chunk->hasNurseryOwnedAllocs = chunk->hasNurseryOwnedAllocsAfterSweep;
1688
1689 MOZ_ASSERT_IF(do { if (majorState == State::NotCollecting && !majorFinishedWhileMinorSweeping
) { do { static_assert( mozilla::detail::AssertionConditionType
<decltype(!chunk->allocatedDuringCollection)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(!chunk->allocatedDuringCollection))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("!chunk->allocatedDuringCollection"
, "./../../../../js/src/gc/BufferAllocator.cpp", 1691); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!chunk->allocatedDuringCollection" ")")
; do { MOZ_CrashSequence(__null, 1691); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false)
1690 majorState == State::NotCollecting && !majorFinishedWhileMinorSweeping,do { if (majorState == State::NotCollecting && !majorFinishedWhileMinorSweeping
) { do { static_assert( mozilla::detail::AssertionConditionType
<decltype(!chunk->allocatedDuringCollection)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(!chunk->allocatedDuringCollection))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("!chunk->allocatedDuringCollection"
, "./../../../../js/src/gc/BufferAllocator.cpp", 1691); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!chunk->allocatedDuringCollection" ")")
; do { MOZ_CrashSequence(__null, 1691); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false)
1691 !chunk->allocatedDuringCollection)do { if (majorState == State::NotCollecting && !majorFinishedWhileMinorSweeping
) { do { static_assert( mozilla::detail::AssertionConditionType
<decltype(!chunk->allocatedDuringCollection)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(!chunk->allocatedDuringCollection))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("!chunk->allocatedDuringCollection"
, "./../../../../js/src/gc/BufferAllocator.cpp", 1691); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!chunk->allocatedDuringCollection" ")")
; do { MOZ_CrashSequence(__null, 1691); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false);
1692
1693 if (majorSweepingStartedWhileMinorSweeping) {
1694 chunk->clearMarkBitsIfStolenChunk();
1695 }
1696
1697 if (majorFinishedWhileMinorSweeping) {
1698 chunk->allocatedDuringCollection = false;
1699 }
1700
1701 if (chunk->kind() == ContentKind::Tenured &&
1702 majorStartedWhileMinorSweeping) {
1703 // We need to sweep this again for the major collection.
1704 tenuredChunksToSweep.ref().pushFront(chunk);
1705 } else {
1706 availableChunks.ref().addChunk(chunk);
1707 }
1708 }
1709
1710 // Merge swept chunks that did not contain nursery owned allocations.
1711#ifdef DEBUG1
1712 for (BufferChunk* chunk : sweptTenuredChunks.ref()) {
1713 MOZ_ASSERT(!chunk->hasNurseryOwnedAllocs)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!chunk->hasNurseryOwnedAllocs)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!chunk->hasNurseryOwnedAllocs
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"!chunk->hasNurseryOwnedAllocs", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1713); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!chunk->hasNurseryOwnedAllocs"
")"); do { MOZ_CrashSequence(__null, 1713); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1714 MOZ_ASSERT(!chunk->hasNurseryOwnedAllocsAfterSweep)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!chunk->hasNurseryOwnedAllocsAfterSweep)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(!chunk->hasNurseryOwnedAllocsAfterSweep))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("!chunk->hasNurseryOwnedAllocsAfterSweep"
, "./../../../../js/src/gc/BufferAllocator.cpp", 1714); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!chunk->hasNurseryOwnedAllocsAfterSweep"
")"); do { MOZ_CrashSequence(__null, 1714); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1715 MOZ_ASSERT(!chunk->allocatedDuringCollection)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!chunk->allocatedDuringCollection)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!chunk->allocatedDuringCollection
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"!chunk->allocatedDuringCollection", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1715); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!chunk->allocatedDuringCollection"
")"); do { MOZ_CrashSequence(__null, 1715); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1716 }
1717#endif
1718 while (BufferChunk* chunk = sweptTenuredChunks.ref().popFirst()) {
1719 availableChunks.ref().addTenuredChunk(chunk);
1720 mergeChunkStatsToRuntime(chunk);
1721 }
1722
1723 largeTenuredAllocs.ref().prepend(std::move(sweptLargeTenuredAllocs.ref()));
1724
1725 hasSweepDataToMerge = false;
1726
1727 if (minorSweepingFinished) {
1728 MOZ_ASSERT(minorState == State::Sweeping)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(minorState == State::Sweeping)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(minorState == State::Sweeping
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"minorState == State::Sweeping", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1728); AnnotateMozCrashReason("MOZ_ASSERT" "(" "minorState == State::Sweeping"
")"); do { MOZ_CrashSequence(__null, 1728); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1729 runtime()->decOffThreadCount();
1730 minorState = State::NotCollecting;
1731 minorSweepingFinished = false;
1732 majorStartedWhileMinorSweeping = false;
1733 majorSweepingStartedWhileMinorSweeping = false;
1734 majorFinishedWhileMinorSweeping = false;
1735
1736#ifdef DEBUG1
1737 for (BufferChunk* chunk : currentMixedChunks.ref()) {
1738 verifyChunk(chunk, true);
1739 }
1740 for (BufferChunk* chunk : currentTenuredChunks.ref()) {
1741 verifyChunk(chunk, false);
1742 }
1743#endif
1744 }
1745
1746 if (majorSweepingFinished) {
1747 MOZ_ASSERT(majorState == State::Sweeping)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(majorState == State::Sweeping)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(majorState == State::Sweeping
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"majorState == State::Sweeping", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1747); AnnotateMozCrashReason("MOZ_ASSERT" "(" "majorState == State::Sweeping"
")"); do { MOZ_CrashSequence(__null, 1747); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1748 runtime()->decOffThreadCount();
1749 majorState = State::NotCollecting;
1750 majorSweepingFinished = false;
1751
1752 MOZ_ASSERT(tenuredChunksToSweep.ref().isEmpty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(tenuredChunksToSweep.ref().isEmpty())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(tenuredChunksToSweep.ref().isEmpty
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("tenuredChunksToSweep.ref().isEmpty()", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1752); AnnotateMozCrashReason("MOZ_ASSERT" "(" "tenuredChunksToSweep.ref().isEmpty()"
")"); do { MOZ_CrashSequence(__null, 1752); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1753 }
1754}
1755
1756void BufferAllocator::mergeChunkStatsToRuntime(BufferChunk* chunk) {
1757 size_t freeBytesAfterSweep =
1758 ChunkSize - chunk->usedBytesAfterSweep - chunk->adminBytesAfterSweep;
1759 MOZ_ASSERT(freeBytesAfterSweep < ChunkSize)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(freeBytesAfterSweep < ChunkSize)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(freeBytesAfterSweep < ChunkSize
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"freeBytesAfterSweep < ChunkSize", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1759); AnnotateMozCrashReason("MOZ_ASSERT" "(" "freeBytesAfterSweep < ChunkSize"
")"); do { MOZ_CrashSequence(__null, 1759); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1760 runtime()->addRetainedStats(chunk->usedBytesAfterSweep, freeBytesAfterSweep,
1761 chunk->adminBytesAfterSweep);
1762}
1763
1764void BufferAllocator::clearMarkStateAfterBarrierVerification() {
1765 checkMainThread();
1766 MOZ_ASSERT(!zone->wasGCStarted())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!zone->wasGCStarted())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!zone->wasGCStarted()))),
0))) { do { } while (false); MOZ_ReportAssertionFailure("!zone->wasGCStarted()"
, "./../../../../js/src/gc/BufferAllocator.cpp", 1766); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!zone->wasGCStarted()" ")"); do { MOZ_CrashSequence
(__null, 1766); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
1767
1768 maybeMergeSweptData();
1769 MOZ_ASSERT(minorState == State::NotCollecting)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(minorState == State::NotCollecting)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(minorState == State::NotCollecting
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"minorState == State::NotCollecting", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1769); AnnotateMozCrashReason("MOZ_ASSERT" "(" "minorState == State::NotCollecting"
")"); do { MOZ_CrashSequence(__null, 1769); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1770 MOZ_ASSERT(majorState == State::NotCollecting)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(majorState == State::NotCollecting)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(majorState == State::NotCollecting
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"majorState == State::NotCollecting", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1770); AnnotateMozCrashReason("MOZ_ASSERT" "(" "majorState == State::NotCollecting"
")"); do { MOZ_CrashSequence(__null, 1770); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1771
1772 for (auto* chunks :
1773 {&currentMixedChunks.ref(), &currentTenuredChunks.ref()}) {
1774 for (auto* chunk : *chunks) {
1775 clearChunkMarkBits(chunk);
1776 }
1777 }
1778
1779 for (auto chunk = availableChunks.ref().chunkIter(); !chunk.done();
1780 chunk.next()) {
1781 clearChunkMarkBits(chunk);
1782 }
1783
1784#ifdef DEBUG1
1785 checkGCStateNotInUse();
1786#endif
1787}
1788
1789void BufferAllocator::clearChunkMarkBits(BufferChunk* chunk) {
1790 checkAccess();
1791 chunk->clearMarkBits();
1792}
1793
1794void BufferChunk::clearMarkBits() {
1795 markBits.ref().clear();
1796 for (auto iter = smallRegionIter(); !iter.done(); iter.next()) {
1797 SmallBufferRegion* region = iter.get();
1798 region->markBits.ref().clear();
1799 }
1800}
1801
1802bool BufferAllocator::isPointerWithinBuffer(void* ptr) {
1803 checkMainThread();
1804
1805 maybeMergeSweptData();
1806
1807 MOZ_ASSERT(mixedChunksToSweep.ref().isEmpty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(mixedChunksToSweep.ref().isEmpty())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(mixedChunksToSweep.ref().isEmpty
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("mixedChunksToSweep.ref().isEmpty()", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1807); AnnotateMozCrashReason("MOZ_ASSERT" "(" "mixedChunksToSweep.ref().isEmpty()"
")"); do { MOZ_CrashSequence(__null, 1807); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1808 MOZ_ASSERT_IF(majorState != State::Marking,do { if (majorState != State::Marking) { do { static_assert( mozilla
::detail::AssertionConditionType<decltype(tenuredChunksToSweep
.ref().isEmpty())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(tenuredChunksToSweep.ref().isEmpty
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("tenuredChunksToSweep.ref().isEmpty()", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1809); AnnotateMozCrashReason("MOZ_ASSERT" "(" "tenuredChunksToSweep.ref().isEmpty()"
")"); do { MOZ_CrashSequence(__null, 1809); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false)
1809 tenuredChunksToSweep.ref().isEmpty())do { if (majorState != State::Marking) { do { static_assert( mozilla
::detail::AssertionConditionType<decltype(tenuredChunksToSweep
.ref().isEmpty())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(tenuredChunksToSweep.ref().isEmpty
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("tenuredChunksToSweep.ref().isEmpty()", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1809); AnnotateMozCrashReason("MOZ_ASSERT" "(" "tenuredChunksToSweep.ref().isEmpty()"
")"); do { MOZ_CrashSequence(__null, 1809); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false);
1810
1811 for (const auto* chunks :
1812 {&currentMixedChunks.ref(), &currentTenuredChunks.ref(),
1813 &tenuredChunksToSweep.ref()}) {
1814 for (auto* chunk : *chunks) {
1815 if (chunk->isPointerWithinAllocation(ptr)) {
1816 return true;
1817 }
1818 }
1819 }
1820
1821 for (auto chunk = availableChunks.ref().chunkIter(); !chunk.done();
1822 chunk.next()) {
1823 if (chunk->isPointerWithinAllocation(ptr)) {
1824 return true;
1825 }
1826 }
1827
1828 // Note we cannot safely access data that is being swept on another thread.
1829
1830 for (const auto* allocs :
1831 {&largeNurseryAllocs.ref(), &largeTenuredAllocs.ref()}) {
1832 for (auto* alloc : *allocs) {
1833 if (alloc->isPointerWithinAllocation(ptr)) {
1834 return true;
1835 }
1836 }
1837 }
1838
1839 return false;
1840}
1841
1842bool BufferChunk::isPointerWithinAllocation(void* ptr) const {
1843 uintptr_t offset = uintptr_t(ptr) - uintptr_t(this);
1844 if (offset >= ChunkSize || offset < FirstMediumAllocOffset) {
1845 return false;
1846 }
1847
1848 if (smallRegionBitmap.ref().getBit(offset / SmallRegionSize)) {
1849 auto* region = SmallBufferRegion::from(ptr);
1850 return region->isPointerWithinAllocation(ptr);
1851 }
1852
1853 uintptr_t allocOffset =
1854 findPrevAllocated(RoundDown(offset, MinMediumAllocSize));
1855 MOZ_ASSERT(allocOffset <= ChunkSize)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(allocOffset <= ChunkSize)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(allocOffset <= ChunkSize)
)), 0))) { do { } while (false); MOZ_ReportAssertionFailure("allocOffset <= ChunkSize"
, "./../../../../js/src/gc/BufferAllocator.cpp", 1855); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "allocOffset <= ChunkSize" ")"); do { MOZ_CrashSequence
(__null, 1855); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
1856 if (allocOffset == ChunkSize) {
1857 return false;
1858 }
1859
1860 const void* alloc = ptrFromOffset(allocOffset);
1861 size_t size = allocBytes(alloc);
1862 return offset < allocOffset + size;
1863}
1864
1865bool SmallBufferRegion::isPointerWithinAllocation(void* ptr) const {
1866 uintptr_t offset = uintptr_t(ptr) - uintptr_t(this);
1867 MOZ_ASSERT(offset < SmallRegionSize)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(offset < SmallRegionSize)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(offset < SmallRegionSize)
)), 0))) { do { } while (false); MOZ_ReportAssertionFailure("offset < SmallRegionSize"
, "./../../../../js/src/gc/BufferAllocator.cpp", 1867); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "offset < SmallRegionSize" ")"); do { MOZ_CrashSequence
(__null, 1867); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
1868
1869 uintptr_t allocOffset =
1870 findPrevAllocated(RoundDown(offset, SmallAllocGranularity));
1871 MOZ_ASSERT(allocOffset <= SmallRegionSize)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(allocOffset <= SmallRegionSize)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(allocOffset <= SmallRegionSize
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"allocOffset <= SmallRegionSize", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1871); AnnotateMozCrashReason("MOZ_ASSERT" "(" "allocOffset <= SmallRegionSize"
")"); do { MOZ_CrashSequence(__null, 1871); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1872 if (allocOffset == SmallRegionSize) {
1873 return false;
1874 }
1875
1876 const void* alloc = ptrFromOffset(allocOffset);
1877 size_t size = allocBytes(alloc);
1878 return offset < allocOffset + size;
1879}
1880
1881bool LargeBuffer::isPointerWithinAllocation(void* ptr) const {
1882 return uintptr_t(ptr) - uintptr_t(alloc) < bytes;
1883}
1884
1885#ifdef DEBUG1
1886
1887void BufferAllocator::checkGCStateNotInUse() {
1888 MaybeLock lock;
1889 maybeMergeSweptData(lock);
1890 checkGCStateNotInUse(lock);
1891}
1892
1893void BufferAllocator::checkGCStateNotInUse(MaybeLock& maybeLock) {
1894 if (maybeLock.isNothing()) {
1895 // Some fields are protected by this lock.
1896 maybeLock.emplace(runtime());
1897 }
1898
1899 checkGCStateNotInUse(maybeLock.ref());
1900}
1901
1902void BufferAllocator::checkGCStateNotInUse(const AutoLock& lock) {
1903 MOZ_ASSERT(majorState == State::NotCollecting)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(majorState == State::NotCollecting)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(majorState == State::NotCollecting
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"majorState == State::NotCollecting", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1903); AnnotateMozCrashReason("MOZ_ASSERT" "(" "majorState == State::NotCollecting"
")"); do { MOZ_CrashSequence(__null, 1903); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1904 bool isNurserySweeping = minorState == State::Sweeping;
1905
1906 checkChunkListGCStateNotInUse(currentMixedChunks.ref(), true, false, false);
1907 checkChunkListGCStateNotInUse(currentTenuredChunks.ref(), false, false,
1908 false);
1909 checkChunkListsGCStateNotInUse(availableChunks.ref(), false);
1910
1911 if (isNurserySweeping) {
1912 checkChunkListGCStateNotInUse(sweptMixedChunks.ref(), true,
1913 majorFinishedWhileMinorSweeping, true);
1914 checkChunkListGCStateNotInUse(sweptTenuredChunks.ref(), false, false, true);
1915 } else {
1916 MOZ_ASSERT(mixedChunksToSweep.ref().isEmpty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(mixedChunksToSweep.ref().isEmpty())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(mixedChunksToSweep.ref().isEmpty
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("mixedChunksToSweep.ref().isEmpty()", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1916); AnnotateMozCrashReason("MOZ_ASSERT" "(" "mixedChunksToSweep.ref().isEmpty()"
")"); do { MOZ_CrashSequence(__null, 1916); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1917 MOZ_ASSERT(largeNurseryAllocsToSweep.ref().isEmpty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(largeNurseryAllocsToSweep.ref().isEmpty())>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(largeNurseryAllocsToSweep.ref().isEmpty()))), 0))) { do { }
while (false); MOZ_ReportAssertionFailure("largeNurseryAllocsToSweep.ref().isEmpty()"
, "./../../../../js/src/gc/BufferAllocator.cpp", 1917); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "largeNurseryAllocsToSweep.ref().isEmpty()"
")"); do { MOZ_CrashSequence(__null, 1917); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1918
1919 MOZ_ASSERT(sweptMixedChunks.ref().isEmpty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(sweptMixedChunks.ref().isEmpty())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(sweptMixedChunks.ref().isEmpty
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("sweptMixedChunks.ref().isEmpty()", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1919); AnnotateMozCrashReason("MOZ_ASSERT" "(" "sweptMixedChunks.ref().isEmpty()"
")"); do { MOZ_CrashSequence(__null, 1919); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1920 MOZ_ASSERT(sweptTenuredChunks.ref().isEmpty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(sweptTenuredChunks.ref().isEmpty())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(sweptTenuredChunks.ref().isEmpty
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("sweptTenuredChunks.ref().isEmpty()", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1920); AnnotateMozCrashReason("MOZ_ASSERT" "(" "sweptTenuredChunks.ref().isEmpty()"
")"); do { MOZ_CrashSequence(__null, 1920); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1921
1922 MOZ_ASSERT(!majorStartedWhileMinorSweeping)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!majorStartedWhileMinorSweeping)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!majorStartedWhileMinorSweeping
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"!majorStartedWhileMinorSweeping", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1922); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!majorStartedWhileMinorSweeping"
")"); do { MOZ_CrashSequence(__null, 1922); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1923 MOZ_ASSERT(!majorSweepingStartedWhileMinorSweeping)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!majorSweepingStartedWhileMinorSweeping)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(!majorSweepingStartedWhileMinorSweeping))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("!majorSweepingStartedWhileMinorSweeping"
, "./../../../../js/src/gc/BufferAllocator.cpp", 1923); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!majorSweepingStartedWhileMinorSweeping" ")"
); do { MOZ_CrashSequence(__null, 1923); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1924 MOZ_ASSERT(!majorFinishedWhileMinorSweeping)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!majorFinishedWhileMinorSweeping)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!majorFinishedWhileMinorSweeping
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"!majorFinishedWhileMinorSweeping", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1924); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!majorFinishedWhileMinorSweeping"
")"); do { MOZ_CrashSequence(__null, 1924); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1925 MOZ_ASSERT(!hasSweepDataToMerge)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!hasSweepDataToMerge)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!hasSweepDataToMerge))), 0))
) { do { } while (false); MOZ_ReportAssertionFailure("!hasSweepDataToMerge"
, "./../../../../js/src/gc/BufferAllocator.cpp", 1925); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!hasSweepDataToMerge" ")"); do { MOZ_CrashSequence
(__null, 1925); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
1926 MOZ_ASSERT(!minorSweepingFinished)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!minorSweepingFinished)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!minorSweepingFinished))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("!minorSweepingFinished"
, "./../../../../js/src/gc/BufferAllocator.cpp", 1926); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!minorSweepingFinished" ")"); do { MOZ_CrashSequence
(__null, 1926); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
1927 MOZ_ASSERT(!majorSweepingFinished)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!majorSweepingFinished)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!majorSweepingFinished))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("!majorSweepingFinished"
, "./../../../../js/src/gc/BufferAllocator.cpp", 1927); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!majorSweepingFinished" ")"); do { MOZ_CrashSequence
(__null, 1927); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
1928 }
1929
1930 MOZ_ASSERT(tenuredChunksToSweep.ref().isEmpty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(tenuredChunksToSweep.ref().isEmpty())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(tenuredChunksToSweep.ref().isEmpty
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("tenuredChunksToSweep.ref().isEmpty()", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1930); AnnotateMozCrashReason("MOZ_ASSERT" "(" "tenuredChunksToSweep.ref().isEmpty()"
")"); do { MOZ_CrashSequence(__null, 1930); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1931
1932 checkAllocListGCStateNotInUse(largeNurseryAllocs.ref(), true);
1933 checkAllocListGCStateNotInUse(largeTenuredAllocs.ref(), false);
1934
1935 MOZ_ASSERT(largeTenuredAllocsToSweep.ref().isEmpty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(largeTenuredAllocsToSweep.ref().isEmpty())>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(largeTenuredAllocsToSweep.ref().isEmpty()))), 0))) { do { }
while (false); MOZ_ReportAssertionFailure("largeTenuredAllocsToSweep.ref().isEmpty()"
, "./../../../../js/src/gc/BufferAllocator.cpp", 1935); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "largeTenuredAllocsToSweep.ref().isEmpty()"
")"); do { MOZ_CrashSequence(__null, 1935); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1936 MOZ_ASSERT(sweptLargeTenuredAllocs.ref().isEmpty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(sweptLargeTenuredAllocs.ref().isEmpty())>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(sweptLargeTenuredAllocs.ref().isEmpty()))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("sweptLargeTenuredAllocs.ref().isEmpty()"
, "./../../../../js/src/gc/BufferAllocator.cpp", 1936); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "sweptLargeTenuredAllocs.ref().isEmpty()" ")"
); do { MOZ_CrashSequence(__null, 1936); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1937}
1938
1939void BufferAllocator::checkChunkListsGCStateNotInUse(
1940 ChunkLists& chunkLists, bool allowAllocatedDuringCollection) {
1941 for (auto chunk = chunkLists.chunkIter(); !chunk.done(); chunk.next()) {
1942 checkChunkGCStateNotInUse(chunk, allowAllocatedDuringCollection, true);
1943 verifyChunk(chunk, chunk->hasNurseryOwnedAllocs);
1944
1945 MOZ_ASSERT(chunk->ownsFreeLists)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(chunk->ownsFreeLists)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(chunk->ownsFreeLists))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("chunk->ownsFreeLists"
, "./../../../../js/src/gc/BufferAllocator.cpp", 1945); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "chunk->ownsFreeLists" ")"); do { MOZ_CrashSequence
(__null, 1945); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
1946 size_t sizeClass = chunk.getSizeClass();
1947
1948 MOZ_ASSERT(chunk->sizeClassForAvailableLists() == sizeClass)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(chunk->sizeClassForAvailableLists() == sizeClass)
>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(chunk->sizeClassForAvailableLists() == sizeClass)
)), 0))) { do { } while (false); MOZ_ReportAssertionFailure("chunk->sizeClassForAvailableLists() == sizeClass"
, "./../../../../js/src/gc/BufferAllocator.cpp", 1948); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "chunk->sizeClassForAvailableLists() == sizeClass"
")"); do { MOZ_CrashSequence(__null, 1948); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1949 MOZ_ASSERT_IF(sizeClass != FullChunkSizeClass,do { if (sizeClass != FullChunkSizeClass) { do { static_assert
( mozilla::detail::AssertionConditionType<decltype(chunk->
freeLists.ref().hasSizeClass(sizeClass))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(chunk->freeLists.ref().hasSizeClass
(sizeClass)))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("chunk->freeLists.ref().hasSizeClass(sizeClass)", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1950); AnnotateMozCrashReason("MOZ_ASSERT" "(" "chunk->freeLists.ref().hasSizeClass(sizeClass)"
")"); do { MOZ_CrashSequence(__null, 1950); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false)
1950 chunk->freeLists.ref().hasSizeClass(sizeClass))do { if (sizeClass != FullChunkSizeClass) { do { static_assert
( mozilla::detail::AssertionConditionType<decltype(chunk->
freeLists.ref().hasSizeClass(sizeClass))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(chunk->freeLists.ref().hasSizeClass
(sizeClass)))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("chunk->freeLists.ref().hasSizeClass(sizeClass)", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1950); AnnotateMozCrashReason("MOZ_ASSERT" "(" "chunk->freeLists.ref().hasSizeClass(sizeClass)"
")"); do { MOZ_CrashSequence(__null, 1950); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false);
1951 }
1952}
1953
1954void BufferAllocator::checkChunkListGCStateNotInUse(
1955 BufferChunkList& chunks, bool hasNurseryOwnedAllocs,
1956 bool allowAllocatedDuringCollection, bool allowFreeLists) {
1957 for (BufferChunk* chunk : chunks) {
1958 checkChunkGCStateNotInUse(chunk, allowAllocatedDuringCollection,
1959 allowFreeLists);
1960 verifyChunk(chunk, hasNurseryOwnedAllocs);
1961 }
1962}
1963
1964void BufferAllocator::checkChunkGCStateNotInUse(
1965 BufferChunk* chunk, bool allowAllocatedDuringCollection,
1966 bool allowFreeLists) {
1967 MOZ_ASSERT_IF(!allowAllocatedDuringCollection,do { if (!allowAllocatedDuringCollection) { do { static_assert
( mozilla::detail::AssertionConditionType<decltype(!chunk->
allocatedDuringCollection)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!chunk->allocatedDuringCollection
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"!chunk->allocatedDuringCollection", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1968); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!chunk->allocatedDuringCollection"
")"); do { MOZ_CrashSequence(__null, 1968); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false)
1968 !chunk->allocatedDuringCollection)do { if (!allowAllocatedDuringCollection) { do { static_assert
( mozilla::detail::AssertionConditionType<decltype(!chunk->
allocatedDuringCollection)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!chunk->allocatedDuringCollection
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"!chunk->allocatedDuringCollection", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1968); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!chunk->allocatedDuringCollection"
")"); do { MOZ_CrashSequence(__null, 1968); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false);
1969 MOZ_ASSERT(!chunk->stolenFromSweepList)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!chunk->stolenFromSweepList)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!chunk->stolenFromSweepList
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"!chunk->stolenFromSweepList", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1969); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!chunk->stolenFromSweepList"
")"); do { MOZ_CrashSequence(__null, 1969); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1970 MOZ_ASSERT(chunk->markBits.ref().isEmpty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(chunk->markBits.ref().isEmpty())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(chunk->markBits.ref().isEmpty
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("chunk->markBits.ref().isEmpty()", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1970); AnnotateMozCrashReason("MOZ_ASSERT" "(" "chunk->markBits.ref().isEmpty()"
")"); do { MOZ_CrashSequence(__null, 1970); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1971 for (auto iter = chunk->smallRegionIter(); !iter.done(); iter.next()) {
1972 SmallBufferRegion* region = iter.get();
1973 MOZ_ASSERT(region->markBits.ref().isEmpty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(region->markBits.ref().isEmpty())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(region->markBits.ref().isEmpty
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("region->markBits.ref().isEmpty()", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1973); AnnotateMozCrashReason("MOZ_ASSERT" "(" "region->markBits.ref().isEmpty()"
")"); do { MOZ_CrashSequence(__null, 1973); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1974 }
1975 MOZ_ASSERT(allowFreeLists == chunk->ownsFreeLists)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(allowFreeLists == chunk->ownsFreeLists)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(allowFreeLists == chunk->ownsFreeLists))), 0))) { do { }
while (false); MOZ_ReportAssertionFailure("allowFreeLists == chunk->ownsFreeLists"
, "./../../../../js/src/gc/BufferAllocator.cpp", 1975); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "allowFreeLists == chunk->ownsFreeLists"
")"); do { MOZ_CrashSequence(__null, 1975); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1976 if (!chunk->ownsFreeLists) {
1977 chunk->freeLists.ref().assertEmpty();
1978 }
1979}
1980
1981void BufferAllocator::verifyChunk(BufferChunk* chunk,
1982 bool hasNurseryOwnedAllocs) {
1983 MOZ_ASSERT(chunk->hasNurseryOwnedAllocs == hasNurseryOwnedAllocs)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(chunk->hasNurseryOwnedAllocs == hasNurseryOwnedAllocs
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(chunk->hasNurseryOwnedAllocs == hasNurseryOwnedAllocs
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"chunk->hasNurseryOwnedAllocs == hasNurseryOwnedAllocs", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1983); AnnotateMozCrashReason("MOZ_ASSERT" "(" "chunk->hasNurseryOwnedAllocs == hasNurseryOwnedAllocs"
")"); do { MOZ_CrashSequence(__null, 1983); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1984
1985 static constexpr size_t StepBytes = MediumAllocGranularity;
1986
1987 size_t freeOffset = FirstMediumAllocOffset;
1988
1989 size_t freeListsFreeRegionCount = 0;
1990 if (chunk->ownsFreeLists) {
1991 chunk->freeLists.ref().checkAvailable();
1992 for (auto region = chunk->freeLists.ref().freeRegionIter(); !region.done();
1993 region.next()) {
1994 MOZ_ASSERT(BufferChunk::from(region) == chunk)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(BufferChunk::from(region) == chunk)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(BufferChunk::from(region) ==
chunk))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("BufferChunk::from(region) == chunk", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1994); AnnotateMozCrashReason("MOZ_ASSERT" "(" "BufferChunk::from(region) == chunk"
")"); do { MOZ_CrashSequence(__null, 1994); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1995 freeListsFreeRegionCount++;
1996 }
1997 } else {
1998 MOZ_ASSERT(chunk->freeLists.ref().isEmpty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(chunk->freeLists.ref().isEmpty())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(chunk->freeLists.ref().isEmpty
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("chunk->freeLists.ref().isEmpty()", "./../../../../js/src/gc/BufferAllocator.cpp"
, 1998); AnnotateMozCrashReason("MOZ_ASSERT" "(" "chunk->freeLists.ref().isEmpty()"
")"); do { MOZ_CrashSequence(__null, 1998); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
1999 }
2000
2001 size_t chunkFreeRegionCount = 0;
2002 for (auto iter = chunk->allocIter(); !iter.done(); iter.next()) {
2003 // Check any free region preceding this allocation.
2004 size_t offset = iter.getOffset();
2005 MOZ_ASSERT(offset >= FirstMediumAllocOffset)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(offset >= FirstMediumAllocOffset)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(offset >= FirstMediumAllocOffset
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"offset >= FirstMediumAllocOffset", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2005); AnnotateMozCrashReason("MOZ_ASSERT" "(" "offset >= FirstMediumAllocOffset"
")"); do { MOZ_CrashSequence(__null, 2005); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2006 if (offset > freeOffset) {
2007 verifyFreeRegion(chunk, offset, offset - freeOffset,
2008 chunkFreeRegionCount);
2009 }
2010
2011 // Check this allocation.
2012 void* alloc = iter.get();
2013 MOZ_ASSERT_IF(chunk->isNurseryOwned(alloc), hasNurseryOwnedAllocs)do { if (chunk->isNurseryOwned(alloc)) { do { static_assert
( mozilla::detail::AssertionConditionType<decltype(hasNurseryOwnedAllocs
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(hasNurseryOwnedAllocs))), 0))) { do { } while (false
); MOZ_ReportAssertionFailure("hasNurseryOwnedAllocs", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2013); AnnotateMozCrashReason("MOZ_ASSERT" "(" "hasNurseryOwnedAllocs"
")"); do { MOZ_CrashSequence(__null, 2013); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false);
2014 size_t bytes = chunk->allocBytes(alloc);
2015 uintptr_t endOffset = offset + bytes;
2016 MOZ_ASSERT(endOffset <= ChunkSize)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(endOffset <= ChunkSize)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(endOffset <= ChunkSize)))
, 0))) { do { } while (false); MOZ_ReportAssertionFailure("endOffset <= ChunkSize"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2016); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "endOffset <= ChunkSize" ")"); do { MOZ_CrashSequence
(__null, 2016); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
2017 for (size_t i = offset + StepBytes; i < endOffset; i += StepBytes) {
2018 MOZ_ASSERT(!chunk->isAllocated(i))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!chunk->isAllocated(i))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!chunk->isAllocated(i))))
, 0))) { do { } while (false); MOZ_ReportAssertionFailure("!chunk->isAllocated(i)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2018); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!chunk->isAllocated(i)" ")"); do { MOZ_CrashSequence
(__null, 2018); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
2019 }
2020
2021 if (chunk->isSmallBufferRegion(alloc)) {
2022 auto* region = SmallBufferRegion::from(alloc);
2023 MOZ_ASSERT_IF(region->hasNurseryOwnedAllocs(), hasNurseryOwnedAllocs)do { if (region->hasNurseryOwnedAllocs()) { do { static_assert
( mozilla::detail::AssertionConditionType<decltype(hasNurseryOwnedAllocs
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(hasNurseryOwnedAllocs))), 0))) { do { } while (false
); MOZ_ReportAssertionFailure("hasNurseryOwnedAllocs", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2023); AnnotateMozCrashReason("MOZ_ASSERT" "(" "hasNurseryOwnedAllocs"
")"); do { MOZ_CrashSequence(__null, 2023); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false);
2024 verifySmallBufferRegion(region, chunkFreeRegionCount);
2025 }
2026
2027 freeOffset = endOffset;
2028 }
2029
2030 // Check any free region following the last allocation.
2031 if (freeOffset < ChunkSize) {
2032 verifyFreeRegion(chunk, ChunkSize, ChunkSize - freeOffset,
2033 chunkFreeRegionCount);
2034 }
2035
2036 MOZ_ASSERT_IF(chunk->ownsFreeLists,do { if (chunk->ownsFreeLists) { do { static_assert( mozilla
::detail::AssertionConditionType<decltype(freeListsFreeRegionCount
== chunkFreeRegionCount)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(freeListsFreeRegionCount == chunkFreeRegionCount
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"freeListsFreeRegionCount == chunkFreeRegionCount", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2037); AnnotateMozCrashReason("MOZ_ASSERT" "(" "freeListsFreeRegionCount == chunkFreeRegionCount"
")"); do { MOZ_CrashSequence(__null, 2037); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false)
2037 freeListsFreeRegionCount == chunkFreeRegionCount)do { if (chunk->ownsFreeLists) { do { static_assert( mozilla
::detail::AssertionConditionType<decltype(freeListsFreeRegionCount
== chunkFreeRegionCount)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(freeListsFreeRegionCount == chunkFreeRegionCount
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"freeListsFreeRegionCount == chunkFreeRegionCount", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2037); AnnotateMozCrashReason("MOZ_ASSERT" "(" "freeListsFreeRegionCount == chunkFreeRegionCount"
")"); do { MOZ_CrashSequence(__null, 2037); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false);
2038}
2039
2040void BufferAllocator::verifyFreeRegion(BufferChunk* chunk, uintptr_t endOffset,
2041 size_t expectedSize,
2042 size_t& freeRegionCount) {
2043 MOZ_ASSERT(expectedSize >= MinFreeRegionSize)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(expectedSize >= MinFreeRegionSize)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(expectedSize >= MinFreeRegionSize
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"expectedSize >= MinFreeRegionSize", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2043); AnnotateMozCrashReason("MOZ_ASSERT" "(" "expectedSize >= MinFreeRegionSize"
")"); do { MOZ_CrashSequence(__null, 2043); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2044 auto* freeRegion = FreeRegion::fromEndOffset(chunk, endOffset);
2045 MOZ_ASSERT(freeRegion->isInList())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(freeRegion->isInList())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(freeRegion->isInList())))
, 0))) { do { } while (false); MOZ_ReportAssertionFailure("freeRegion->isInList()"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2045); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "freeRegion->isInList()" ")"); do { MOZ_CrashSequence
(__null, 2045); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
2046 MOZ_ASSERT(freeRegion->size() == expectedSize)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(freeRegion->size() == expectedSize)>::isValid,
"invalid assertion condition"); if ((__builtin_expect(!!(!(!
!(freeRegion->size() == expectedSize))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("freeRegion->size() == expectedSize"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2046); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "freeRegion->size() == expectedSize" ")"
); do { MOZ_CrashSequence(__null, 2046); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2047 freeRegionCount++;
2048}
2049
2050void BufferAllocator::verifySmallBufferRegion(SmallBufferRegion* region,
2051 size_t& freeRegionCount) {
2052 bool foundNurseryOwnedAllocs = false;
2053
2054 static constexpr size_t StepBytes = SmallAllocGranularity;
2055
2056 size_t freeOffset = FirstSmallAllocOffset;
2057
2058 for (auto iter = region->allocIter(); !iter.done(); iter.next()) {
2059 // Check any free region preceding this allocation.
2060 size_t offset = iter.getOffset();
2061 MOZ_ASSERT(offset >= FirstSmallAllocOffset)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(offset >= FirstSmallAllocOffset)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(offset >= FirstSmallAllocOffset
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"offset >= FirstSmallAllocOffset", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2061); AnnotateMozCrashReason("MOZ_ASSERT" "(" "offset >= FirstSmallAllocOffset"
")"); do { MOZ_CrashSequence(__null, 2061); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2062 if (offset > freeOffset) {
2063 verifyFreeRegion(region, offset, offset - freeOffset, freeRegionCount);
2064 }
2065
2066 // Check this allocation.
2067 void* alloc = iter.get();
2068 MOZ_ASSERT_IF(region->isNurseryOwned(alloc),do { if (region->isNurseryOwned(alloc)) { do { static_assert
( mozilla::detail::AssertionConditionType<decltype(region->
hasNurseryOwnedAllocs())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(region->hasNurseryOwnedAllocs
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("region->hasNurseryOwnedAllocs()", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2069); AnnotateMozCrashReason("MOZ_ASSERT" "(" "region->hasNurseryOwnedAllocs()"
")"); do { MOZ_CrashSequence(__null, 2069); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false)
2069 region->hasNurseryOwnedAllocs())do { if (region->isNurseryOwned(alloc)) { do { static_assert
( mozilla::detail::AssertionConditionType<decltype(region->
hasNurseryOwnedAllocs())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(region->hasNurseryOwnedAllocs
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("region->hasNurseryOwnedAllocs()", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2069); AnnotateMozCrashReason("MOZ_ASSERT" "(" "region->hasNurseryOwnedAllocs()"
")"); do { MOZ_CrashSequence(__null, 2069); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false);
2070 size_t bytes = region->allocBytes(alloc);
2071 uintptr_t endOffset = offset + bytes;
2072 MOZ_ASSERT(endOffset <= SmallRegionSize)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(endOffset <= SmallRegionSize)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(endOffset <= SmallRegionSize
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"endOffset <= SmallRegionSize", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2072); AnnotateMozCrashReason("MOZ_ASSERT" "(" "endOffset <= SmallRegionSize"
")"); do { MOZ_CrashSequence(__null, 2072); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2073 for (size_t i = offset + StepBytes; i < endOffset; i += StepBytes) {
2074 MOZ_ASSERT(!region->isAllocated(i))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!region->isAllocated(i))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!region->isAllocated(i)))
), 0))) { do { } while (false); MOZ_ReportAssertionFailure("!region->isAllocated(i)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2074); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!region->isAllocated(i)" ")"); do { MOZ_CrashSequence
(__null, 2074); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
2075 }
2076
2077 if (region->isNurseryOwned(alloc)) {
2078 foundNurseryOwnedAllocs = true;
2079 }
2080
2081 freeOffset = endOffset;
2082 }
2083
2084 MOZ_ASSERT(foundNurseryOwnedAllocs == region->hasNurseryOwnedAllocs())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(foundNurseryOwnedAllocs == region->hasNurseryOwnedAllocs
())>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(foundNurseryOwnedAllocs == region->hasNurseryOwnedAllocs
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("foundNurseryOwnedAllocs == region->hasNurseryOwnedAllocs()"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2084); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "foundNurseryOwnedAllocs == region->hasNurseryOwnedAllocs()"
")"); do { MOZ_CrashSequence(__null, 2084); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2085
2086 // Check any free region following the last allocation.
2087 if (freeOffset < SmallRegionSize) {
2088 verifyFreeRegion(region, SmallRegionSize, SmallRegionSize - freeOffset,
2089 freeRegionCount);
2090 }
2091}
2092
2093void BufferAllocator::verifyFreeRegion(SmallBufferRegion* region,
2094 uintptr_t endOffset, size_t expectedSize,
2095 size_t& freeRegionCount) {
2096 if (expectedSize < MinFreeRegionSize) {
2097 return;
2098 }
2099
2100 auto* freeRegion = FreeRegion::fromEndOffset(region, endOffset);
2101 MOZ_ASSERT(freeRegion->isInList())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(freeRegion->isInList())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(freeRegion->isInList())))
, 0))) { do { } while (false); MOZ_ReportAssertionFailure("freeRegion->isInList()"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2101); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "freeRegion->isInList()" ")"); do { MOZ_CrashSequence
(__null, 2101); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
2102 MOZ_ASSERT(freeRegion->size() == expectedSize)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(freeRegion->size() == expectedSize)>::isValid,
"invalid assertion condition"); if ((__builtin_expect(!!(!(!
!(freeRegion->size() == expectedSize))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("freeRegion->size() == expectedSize"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2102); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "freeRegion->size() == expectedSize" ")"
); do { MOZ_CrashSequence(__null, 2102); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2103 freeRegionCount++;
2104}
2105
2106void BufferAllocator::checkAllocListGCStateNotInUse(LargeAllocList& list,
2107 bool isNurseryOwned) {
2108 for (LargeBuffer* buffer : list) {
2109 MOZ_ASSERT(buffer->isNurseryOwned == isNurseryOwned)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(buffer->isNurseryOwned == isNurseryOwned)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(buffer->isNurseryOwned == isNurseryOwned))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("buffer->isNurseryOwned == isNurseryOwned"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2109); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "buffer->isNurseryOwned == isNurseryOwned"
")"); do { MOZ_CrashSequence(__null, 2109); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2110 MOZ_ASSERT_IF(!isNurseryOwned, !buffer->allocatedDuringCollection)do { if (!isNurseryOwned) { do { static_assert( mozilla::detail
::AssertionConditionType<decltype(!buffer->allocatedDuringCollection
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(!buffer->allocatedDuringCollection))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("!buffer->allocatedDuringCollection"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2110); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!buffer->allocatedDuringCollection" ")"
); do { MOZ_CrashSequence(__null, 2110); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false);
2111 }
2112}
2113
2114#endif
2115
2116void* BufferAllocator::allocSmall(size_t bytes, bool nurseryOwned, bool inGC) {
2117 MOZ_ASSERT(IsSmallAllocSize(bytes))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(IsSmallAllocSize(bytes))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(IsSmallAllocSize(bytes)))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("IsSmallAllocSize(bytes)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2117); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "IsSmallAllocSize(bytes)" ")"); do { MOZ_CrashSequence
(__null, 2117); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
2118
2119 // Round up to next available size.
2120 bytes = RoundUp(std::max(bytes, MinSmallAllocSize), SmallAllocGranularity);
2121 MOZ_ASSERT(bytes <= MaxSmallAllocSize)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(bytes <= MaxSmallAllocSize)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(bytes <= MaxSmallAllocSize
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"bytes <= MaxSmallAllocSize", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2121); AnnotateMozCrashReason("MOZ_ASSERT" "(" "bytes <= MaxSmallAllocSize"
")"); do { MOZ_CrashSequence(__null, 2121); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2122
2123 // Get size class from |bytes|.
2124 size_t sizeClass = SizeClassForSmallAlloc(bytes);
2125
2126 void* alloc = bumpAlloc(bytes, sizeClass, MaxSmallAllocClass, nurseryOwned);
2127 if (MOZ_UNLIKELY(!alloc)(__builtin_expect(!!(!alloc), 0))) {
2128 alloc = retrySmallAlloc(bytes, sizeClass, nurseryOwned, inGC);
2129 if (!alloc) {
2130 return nullptr;
2131 }
2132 }
2133
2134 SmallBufferRegion* region = SmallBufferRegion::from(alloc);
2135 region->setAllocated(alloc, bytes, true);
2136 MOZ_ASSERT(region->allocBytes(alloc) == bytes)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(region->allocBytes(alloc) == bytes)>::isValid,
"invalid assertion condition"); if ((__builtin_expect(!!(!(!
!(region->allocBytes(alloc) == bytes))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("region->allocBytes(alloc) == bytes"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2136); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "region->allocBytes(alloc) == bytes" ")"
); do { MOZ_CrashSequence(__null, 2136); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2137
2138 MOZ_ASSERT(!region->isNurseryOwned(alloc))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!region->isNurseryOwned(alloc))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!region->isNurseryOwned(alloc
)))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("!region->isNurseryOwned(alloc)", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2138); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!region->isNurseryOwned(alloc)"
")"); do { MOZ_CrashSequence(__null, 2138); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2139 if (nurseryOwned) {
2140 MOZ_ASSERT(BufferChunk::from(alloc)->hasNurseryOwnedAllocs)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(BufferChunk::from(alloc)->hasNurseryOwnedAllocs)>
::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(BufferChunk::from(alloc)->hasNurseryOwnedAllocs))
), 0))) { do { } while (false); MOZ_ReportAssertionFailure("BufferChunk::from(alloc)->hasNurseryOwnedAllocs"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2140); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "BufferChunk::from(alloc)->hasNurseryOwnedAllocs"
")"); do { MOZ_CrashSequence(__null, 2140); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2141 region->setNurseryOwned(alloc, nurseryOwned);
2142 if (!region->hasNurseryOwnedAllocs()) {
2143 region->setHasNurseryOwnedAllocs(true);
2144 }
2145 }
2146
2147 // Heap size updates are done for the small buffer region as a whole, not
2148 // individual allocations within it.
2149
2150 MOZ_ASSERT(!region->isMarked(alloc))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!region->isMarked(alloc))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!region->isMarked(alloc))
)), 0))) { do { } while (false); MOZ_ReportAssertionFailure("!region->isMarked(alloc)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2150); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!region->isMarked(alloc)" ")"); do { MOZ_CrashSequence
(__null, 2150); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
2151 MOZ_ASSERT(IsSmallAlloc(alloc))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(IsSmallAlloc(alloc))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(IsSmallAlloc(alloc)))), 0)))
{ do { } while (false); MOZ_ReportAssertionFailure("IsSmallAlloc(alloc)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2151); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "IsSmallAlloc(alloc)" ")"); do { MOZ_CrashSequence
(__null, 2151); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
2152
2153 return alloc;
2154}
2155
2156MOZ_NEVER_INLINE__attribute__((noinline)) void* BufferAllocator::retrySmallAlloc(size_t bytes,
2157 size_t sizeClass,
2158 bool nurseryOwned,
2159 bool inGC) {
2160 auto alloc = [&]() {
2161 return bumpAlloc(bytes, sizeClass, MaxSmallAllocClass, nurseryOwned);
2162 };
2163 auto growHeap = [&]() { return allocNewSmallRegion(nurseryOwned, inGC); };
2164
2165 return refillFreeListsAndRetryAlloc(sizeClass, MaxSmallAllocClass,
2166 nurseryOwned, alloc, growHeap);
2167}
2168
2169bool BufferAllocator::allocNewSmallRegion(bool nurseryOwned, bool inGC) {
2170 void* ptr = allocMediumAligned(SmallRegionSize, nurseryOwned, inGC);
2171 if (!ptr) {
2172 return false;
2173 }
2174
2175 auto* region = new (ptr) SmallBufferRegion;
2176
2177 BufferChunk* chunk = BufferChunk::from(region);
2178 chunk->setSmallBufferRegion(region, true);
2179
2180 uintptr_t freeStart = uintptr_t(region) + FirstSmallAllocOffset;
2181 uintptr_t freeEnd = uintptr_t(region) + SmallRegionSize;
2182
2183 size_t sizeClass =
2184 SizeClassForFreeRegion(freeEnd - freeStart, SizeKind::Small);
2185
2186 ptr = reinterpret_cast<void*>(freeEnd - sizeof(FreeRegion));
2187 FreeRegion* freeRegion = new (ptr) FreeRegion(freeStart);
2188 MOZ_ASSERT(freeRegion->getEnd() == freeEnd)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(freeRegion->getEnd() == freeEnd)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(freeRegion->getEnd() == freeEnd
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"freeRegion->getEnd() == freeEnd", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2188); AnnotateMozCrashReason("MOZ_ASSERT" "(" "freeRegion->getEnd() == freeEnd"
")"); do { MOZ_CrashSequence(__null, 2188); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2189
2190 MOZ_ASSERT_IF(nurseryOwned, chunk->kind() == ContentKind::Mixed)do { if (nurseryOwned) { do { static_assert( mozilla::detail::
AssertionConditionType<decltype(chunk->kind() == ContentKind
::Mixed)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(chunk->kind() == ContentKind::Mixed))), 0))) { do
{ } while (false); MOZ_ReportAssertionFailure("chunk->kind() == ContentKind::Mixed"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2190); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "chunk->kind() == ContentKind::Mixed" ")"
); do { MOZ_CrashSequence(__null, 2190); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false);
2191 if (nurseryOwned) {
2192 mixedFreeLists.ref().pushFront(sizeClass, freeRegion);
2193 } else {
2194 tenuredFreeLists.ref().pushFront(sizeClass, freeRegion);
2195 }
2196
2197 return true;
2198}
2199
2200/* static */
2201bool BufferAllocator::IsSmallAlloc(void* alloc) {
2202 MOZ_ASSERT(IsBufferAlloc(alloc))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(IsBufferAlloc(alloc))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(IsBufferAlloc(alloc)))), 0))
) { do { } while (false); MOZ_ReportAssertionFailure("IsBufferAlloc(alloc)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2202); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "IsBufferAlloc(alloc)" ")"); do { MOZ_CrashSequence
(__null, 2202); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
2203
2204 // Test for large buffers before calling this so we can assume |alloc| is
2205 // inside a chunk.
2206 MOZ_ASSERT(!IsLargeAlloc(alloc))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!IsLargeAlloc(alloc))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!IsLargeAlloc(alloc)))), 0))
) { do { } while (false); MOZ_ReportAssertionFailure("!IsLargeAlloc(alloc)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2206); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!IsLargeAlloc(alloc)" ")"); do { MOZ_CrashSequence
(__null, 2206); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
2207
2208 BufferChunk* chunk = BufferChunk::from(alloc);
2209 return chunk->isSmallBufferRegion(alloc);
2210}
2211
2212void* BufferAllocator::allocMedium(size_t bytes, bool nurseryOwned, bool inGC) {
2213 MOZ_ASSERT(!IsSmallAllocSize(bytes))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!IsSmallAllocSize(bytes))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!IsSmallAllocSize(bytes)))),
0))) { do { } while (false); MOZ_ReportAssertionFailure("!IsSmallAllocSize(bytes)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2213); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!IsSmallAllocSize(bytes)" ")"); do { MOZ_CrashSequence
(__null, 2213); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
2214 MOZ_ASSERT(!IsLargeAllocSize(bytes))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!IsLargeAllocSize(bytes))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!IsLargeAllocSize(bytes)))),
0))) { do { } while (false); MOZ_ReportAssertionFailure("!IsLargeAllocSize(bytes)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2214); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!IsLargeAllocSize(bytes)" ")"); do { MOZ_CrashSequence
(__null, 2214); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
2215
2216 // Round up to next allowed size.
2217 bytes = RoundUp(bytes, MediumAllocGranularity);
2218 MOZ_ASSERT(bytes <= MaxMediumAllocSize)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(bytes <= MaxMediumAllocSize)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(bytes <= MaxMediumAllocSize
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"bytes <= MaxMediumAllocSize", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2218); AnnotateMozCrashReason("MOZ_ASSERT" "(" "bytes <= MaxMediumAllocSize"
")"); do { MOZ_CrashSequence(__null, 2218); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2219
2220 // Get size class from |bytes|.
2221 size_t sizeClass = SizeClassForMediumAlloc(bytes);
2222
2223 void* alloc = bumpAlloc(bytes, sizeClass, MaxMediumAllocClass, nurseryOwned);
2224 if (MOZ_UNLIKELY(!alloc)(__builtin_expect(!!(!alloc), 0))) {
2225 alloc = retryMediumAlloc(bytes, sizeClass, nurseryOwned, inGC);
2226 if (!alloc) {
2227 return nullptr;
2228 }
2229 }
2230
2231 setAllocated(alloc, bytes, nurseryOwned, inGC);
2232 return alloc;
2233}
2234
2235MOZ_NEVER_INLINE__attribute__((noinline)) void* BufferAllocator::retryMediumAlloc(size_t bytes,
2236 size_t sizeClass,
2237 bool nurseryOwned,
2238 bool inGC) {
2239 auto alloc = [&]() {
2240 return bumpAlloc(bytes, sizeClass, MaxMediumAllocClass, nurseryOwned);
2241 };
2242 auto growHeap = [&]() {
2243 return stealOrAllocNewChunk(sizeClass, nurseryOwned, inGC);
2244 };
2245 return refillFreeListsAndRetryAlloc(sizeClass, MaxMediumAllocClass,
2246 nurseryOwned, alloc, growHeap);
2247}
2248
2249template <typename Alloc, typename GrowHeap>
2250void* BufferAllocator::refillFreeListsAndRetryAlloc(size_t sizeClass,
2251 size_t maxSizeClass,
2252 bool nurseryOwned,
2253 Alloc&& alloc,
2254 GrowHeap&& growHeap) {
2255 RefillResult r;
2256 do {
2257 r = refillFreeLists(sizeClass, maxSizeClass, nurseryOwned, growHeap);
2258 if (r == RefillResult::Fail) {
2259 return nullptr;
2260 }
2261 } while (r == RefillResult::Retry);
2262
2263 void* ptr = alloc();
2264 MOZ_ASSERT(ptr)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ptr)>::isValid, "invalid assertion condition"); if
((__builtin_expect(!!(!(!!(ptr))), 0))) { do { } while (false
); MOZ_ReportAssertionFailure("ptr", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2264); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ptr" ")"); do
{ MOZ_CrashSequence(__null, 2264); __attribute__((nomerge)) ::
abort(); } while (false); } } while (false);
2265 return ptr;
2266}
2267
2268template <typename GrowHeap>
2269BufferAllocator::RefillResult BufferAllocator::refillFreeLists(
2270 size_t sizeClass, size_t maxSizeClass, bool nurseryOwned,
2271 GrowHeap&& growHeap) {
2272 MOZ_ASSERT(sizeClass <= maxSizeClass)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(sizeClass <= maxSizeClass)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(sizeClass <= maxSizeClass
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"sizeClass <= maxSizeClass", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2272); AnnotateMozCrashReason("MOZ_ASSERT" "(" "sizeClass <= maxSizeClass"
")"); do { MOZ_CrashSequence(__null, 2272); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2273
2274 // Take chunks from the available lists and add their free regions to the
2275 // free lists.
2276 if (useAvailableChunk(sizeClass, maxSizeClass, nurseryOwned)) {
2277 return RefillResult::Success;
2278 }
2279
2280 // If that fails try to merge swept data and retry, avoiding taking the lock
2281 // unless we know there is data to merge. This reduces context switches.
2282 if (hasSweepDataToMerge) {
2283 mergeSweptData();
2284 return RefillResult::Retry;
2285 }
2286
2287 // If we were trying to use separate chunks for nursery and tenured owned
2288 // allocations stop doing that to avoid growing the heap more than necessary.
2289 // TODO: For large heaps we could try keeping this enabled.
2290 if (!nurseryOwned && !allocTenuredInMixedChunks) {
2291 allocTenuredInMixedChunks = true;
2292 return RefillResult::Retry;
2293 }
2294
2295 // Try to grow the heap.
2296 if (MOZ_LIKELY(growHeap())(__builtin_expect(!!(growHeap()), 1))) {
2297 return RefillResult::Success;
2298 }
2299
2300 // If all else fails (and we're on the main thread), try waiting for any
2301 // background GC activity to finish.
2302 if (isUsedByMainThread()) {
2303 if (gc->waitForBackgroundTasksOnAllocFailure()) {
2304 return RefillResult::Retry;
2305 }
2306 }
2307
2308 return RefillResult::Fail;
2309}
2310
2311bool BufferAllocator::useAvailableChunk(size_t sizeClass, size_t maxSizeClass,
2312 bool nurseryOwned) {
2313 if (nurseryOwned) {
2314 return useAvailableChunk(sizeClass, maxSizeClass, true, ContentKind::Mixed,
2315 currentMixedChunks.ref(), mixedFreeLists.ref()) ||
2316 useAvailableChunk(sizeClass, maxSizeClass, true,
2317 ContentKind::Tenured, currentMixedChunks.ref(),
2318 mixedFreeLists.ref());
2319 }
2320
2321 return useAvailableChunk(sizeClass, maxSizeClass, false, ContentKind::Tenured,
2322 currentTenuredChunks.ref(), tenuredFreeLists.ref());
2323}
2324
2325bool BufferAllocator::useAvailableChunk(size_t minSizeClass,
2326 size_t maxSizeClass, bool nurseryOwned,
2327 ContentKind srcKind,
2328 BufferChunkList& dstChunks,
2329 FreeLists& dstFreeLists) {
2330 // Move an available chunk of kind |srcKind| that can satisfy the allocation
2331 // request to |dstChunks| and move put its free lists to |dstFreeLists|. If no
2332 // such chunk exists return false.
2333
2334 MOZ_ASSERT(!dstFreeLists.hasAnySizeClass(minSizeClass, maxSizeClass))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!dstFreeLists.hasAnySizeClass(minSizeClass, maxSizeClass
))>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(!dstFreeLists.hasAnySizeClass(minSizeClass, maxSizeClass
)))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("!dstFreeLists.hasAnySizeClass(minSizeClass, maxSizeClass)",
"./../../../../js/src/gc/BufferAllocator.cpp", 2334); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!dstFreeLists.hasAnySizeClass(minSizeClass, maxSizeClass)"
")"); do { MOZ_CrashSequence(__null, 2334); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2335
2336 auto& available = availableChunks.ref().availableSizeClasses(srcKind);
2337 size_t sizeClass = available.FindNext(minSizeClass);
2338 if (sizeClass > maxSizeClass) {
2339 return false;
2340 }
2341
2342 BufferChunk* chunk = availableChunks.ref().popFirstChunk(srcKind, sizeClass);
2343 MOZ_ASSERT(chunk->ownsFreeLists)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(chunk->ownsFreeLists)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(chunk->ownsFreeLists))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("chunk->ownsFreeLists"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2343); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "chunk->ownsFreeLists" ")"); do { MOZ_CrashSequence
(__null, 2343); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
2344 MOZ_ASSERT(chunk->freeLists.ref().hasSizeClass(sizeClass))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(chunk->freeLists.ref().hasSizeClass(sizeClass))>
::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(chunk->freeLists.ref().hasSizeClass(sizeClass))))
, 0))) { do { } while (false); MOZ_ReportAssertionFailure("chunk->freeLists.ref().hasSizeClass(sizeClass)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2344); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "chunk->freeLists.ref().hasSizeClass(sizeClass)"
")"); do { MOZ_CrashSequence(__null, 2344); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2345 MOZ_ASSERT(chunk->kind() == srcKind)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(chunk->kind() == srcKind)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(chunk->kind() == srcKind)
)), 0))) { do { } while (false); MOZ_ReportAssertionFailure("chunk->kind() == srcKind"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2345); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "chunk->kind() == srcKind" ")"); do { MOZ_CrashSequence
(__null, 2345); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
2346
2347 if (nurseryOwned && srcKind == ContentKind::Tenured) {
2348 MOZ_ASSERT(!chunk->hasNurseryOwnedAllocs)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!chunk->hasNurseryOwnedAllocs)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!chunk->hasNurseryOwnedAllocs
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"!chunk->hasNurseryOwnedAllocs", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2348); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!chunk->hasNurseryOwnedAllocs"
")"); do { MOZ_CrashSequence(__null, 2348); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2349 chunk->hasNurseryOwnedAllocs = true;
2350 }
2351
2352 dstChunks.pushBack(chunk);
2353 dstFreeLists.append(std::move(chunk->freeLists.ref()));
2354 chunk->ownsFreeLists = false;
2355 chunk->freeLists.ref().assertEmpty();
2356
2357 MOZ_ASSERT(dstFreeLists.hasAnySizeClass(minSizeClass, maxSizeClass))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(dstFreeLists.hasAnySizeClass(minSizeClass, maxSizeClass
))>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(dstFreeLists.hasAnySizeClass(minSizeClass, maxSizeClass
)))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("dstFreeLists.hasAnySizeClass(minSizeClass, maxSizeClass)", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2357); AnnotateMozCrashReason("MOZ_ASSERT" "(" "dstFreeLists.hasAnySizeClass(minSizeClass, maxSizeClass)"
")"); do { MOZ_CrashSequence(__null, 2357); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2358 return true;
2359}
2360
2361// Differentiate between small and medium size classes. Large allocations do not
2362// use size classes.
2363static bool IsMediumSizeClass(size_t sizeClass) {
2364 MOZ_ASSERT(sizeClass < BufferAllocator::AllocSizeClasses)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(sizeClass < BufferAllocator::AllocSizeClasses)>
::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(sizeClass < BufferAllocator::AllocSizeClasses))),
0))) { do { } while (false); MOZ_ReportAssertionFailure("sizeClass < BufferAllocator::AllocSizeClasses"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2364); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "sizeClass < BufferAllocator::AllocSizeClasses"
")"); do { MOZ_CrashSequence(__null, 2364); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2365 return sizeClass >= MinMediumAllocClass;
2366}
2367
2368/* static */
2369BufferAllocator::SizeKind BufferAllocator::SizeClassKind(size_t sizeClass) {
2370 return IsMediumSizeClass(sizeClass) ? SizeKind::Medium : SizeKind::Small;
2371}
2372
2373void* BufferAllocator::bumpAlloc(size_t bytes, size_t minSizeClass,
2374 size_t maxSizeClass, bool nurseryOwned) {
2375 MOZ_ASSERT(SizeClassKind(minSizeClass) == SizeClassKind(maxSizeClass))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(SizeClassKind(minSizeClass) == SizeClassKind(maxSizeClass
))>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(SizeClassKind(minSizeClass) == SizeClassKind(maxSizeClass
)))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("SizeClassKind(minSizeClass) == SizeClassKind(maxSizeClass)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2375); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "SizeClassKind(minSizeClass) == SizeClassKind(maxSizeClass)"
")"); do { MOZ_CrashSequence(__null, 2375); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2376
2377 // Find smallest suitable size class that has free regions.
2378 FreeLists* freeLists = &getFreeListsForAlloc(nurseryOwned);
2379 freeLists->checkAvailable();
2380 size_t sizeClass =
2381 freeLists->getFirstAvailableSizeClass(minSizeClass, maxSizeClass);
2382
2383 // Retry using mixed chunks for tenured allocations if the
2384 // |allocTenuredInMixedChunks| flag has been set.
2385 //
2386 // This does not do the converse for nursery owned allocations because it's
2387 // inconvenient to move free regions from |tenuredFreeLists| to
2388 // |mixedFreeLists|. In that case we fail here and grow the heap. This does
2389 // not appear to affect maximum heap size much.
2390 if (sizeClass == SIZE_MAX(18446744073709551615UL) && !nurseryOwned && allocTenuredInMixedChunks) {
2391 freeLists = &getFreeListsForAlloc(!nurseryOwned);
2392 freeLists->checkAvailable();
2393 sizeClass =
2394 freeLists->getFirstAvailableSizeClass(minSizeClass, maxSizeClass);
2395 }
2396
2397 if (sizeClass == SIZE_MAX(18446744073709551615UL)) {
2398 return nullptr;
2399 }
2400
2401 FreeRegion* region = freeLists->getFirstRegion(sizeClass);
2402 MOZ_ASSERT(region->size() >= bytes)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(region->size() >= bytes)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(region->size() >= bytes
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"region->size() >= bytes", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2402); AnnotateMozCrashReason("MOZ_ASSERT" "(" "region->size() >= bytes"
")"); do { MOZ_CrashSequence(__null, 2402); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2403
2404 void* ptr = allocFromRegion(region, bytes, sizeClass);
2405 updateFreeListsAfterAlloc(freeLists, region, sizeClass);
2406
2407 DebugOnlyPoison(ptr, JS_ALLOCATED_BUFFER_PATTERN, bytes,
2408 MemCheckKind::MakeUndefined);
2409
2410 return ptr;
2411}
2412
2413inline BufferAllocator::FreeLists& BufferAllocator::getFreeListsForAlloc(
2414 bool nurseryOwned) {
2415 return nurseryOwned ? mixedFreeLists.ref() : tenuredFreeLists.ref();
2416}
2417
2418#ifdef DEBUG1
2419static size_t GranularityForSizeClass(size_t sizeClass) {
2420 return IsMediumSizeClass(sizeClass) ? MediumAllocGranularity
2421 : SmallAllocGranularity;
2422}
2423#endif // DEBUG
2424
2425void* BufferAllocator::allocFromRegion(FreeRegion* region, size_t bytes,
2426 size_t sizeClass) {
2427 uintptr_t start = region->startAddr;
2428 MOZ_ASSERT(region->getEnd() > start)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(region->getEnd() > start)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(region->getEnd() > start
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"region->getEnd() > start", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2428); AnnotateMozCrashReason("MOZ_ASSERT" "(" "region->getEnd() > start"
")"); do { MOZ_CrashSequence(__null, 2428); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2429 MOZ_ASSERT_IF(sizeClass != MaxMediumAllocClass,do { if (sizeClass != MaxMediumAllocClass) { do { static_assert
( mozilla::detail::AssertionConditionType<decltype(region->
size() >= SizeClassBytes(sizeClass))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(region->size() >= SizeClassBytes
(sizeClass)))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("region->size() >= SizeClassBytes(sizeClass)", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2430); AnnotateMozCrashReason("MOZ_ASSERT" "(" "region->size() >= SizeClassBytes(sizeClass)"
")"); do { MOZ_CrashSequence(__null, 2430); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false)
2430 region->size() >= SizeClassBytes(sizeClass))do { if (sizeClass != MaxMediumAllocClass) { do { static_assert
( mozilla::detail::AssertionConditionType<decltype(region->
size() >= SizeClassBytes(sizeClass))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(region->size() >= SizeClassBytes
(sizeClass)))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("region->size() >= SizeClassBytes(sizeClass)", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2430); AnnotateMozCrashReason("MOZ_ASSERT" "(" "region->size() >= SizeClassBytes(sizeClass)"
")"); do { MOZ_CrashSequence(__null, 2430); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false);
2431 MOZ_ASSERT_IF(sizeClass == MaxMediumAllocClass,do { if (sizeClass == MaxMediumAllocClass) { do { static_assert
( mozilla::detail::AssertionConditionType<decltype(region->
size() >= MaxMediumAllocSize)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(region->size() >= MaxMediumAllocSize
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"region->size() >= MaxMediumAllocSize", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2432); AnnotateMozCrashReason("MOZ_ASSERT" "(" "region->size() >= MaxMediumAllocSize"
")"); do { MOZ_CrashSequence(__null, 2432); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false)
2432 region->size() >= MaxMediumAllocSize)do { if (sizeClass == MaxMediumAllocClass) { do { static_assert
( mozilla::detail::AssertionConditionType<decltype(region->
size() >= MaxMediumAllocSize)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(region->size() >= MaxMediumAllocSize
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"region->size() >= MaxMediumAllocSize", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2432); AnnotateMozCrashReason("MOZ_ASSERT" "(" "region->size() >= MaxMediumAllocSize"
")"); do { MOZ_CrashSequence(__null, 2432); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false);
2433 MOZ_ASSERT(start % GranularityForSizeClass(sizeClass) == 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(start % GranularityForSizeClass(sizeClass) == 0)>
::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(start % GranularityForSizeClass(sizeClass) == 0))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("start % GranularityForSizeClass(sizeClass) == 0"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2433); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "start % GranularityForSizeClass(sizeClass) == 0"
")"); do { MOZ_CrashSequence(__null, 2433); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2434 MOZ_ASSERT(region->size() % GranularityForSizeClass(sizeClass) == 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(region->size() % GranularityForSizeClass(sizeClass
) == 0)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(region->size() % GranularityForSizeClass(sizeClass
) == 0))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("region->size() % GranularityForSizeClass(sizeClass) == 0"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2434); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "region->size() % GranularityForSizeClass(sizeClass) == 0"
")"); do { MOZ_CrashSequence(__null, 2434); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2435
2436 // Ensure whole region is commited.
2437 if (region->hasDecommittedPages) {
2438 recommitRegion(region);
2439 }
2440
2441 // Allocate from start of region.
2442 void* ptr = reinterpret_cast<void*>(start);
2443 start += bytes;
2444 MOZ_ASSERT(region->getEnd() >= start)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(region->getEnd() >= start)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(region->getEnd() >= start
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"region->getEnd() >= start", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2444); AnnotateMozCrashReason("MOZ_ASSERT" "(" "region->getEnd() >= start"
")"); do { MOZ_CrashSequence(__null, 2444); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2445
2446 // Update region start.
2447 region->startAddr = start;
2448
2449 return ptr;
2450}
2451
2452// Allocate a region of size |bytes| aligned to |bytes|. The maximum size is
2453// limited to 256KB. In practice this is only ever used to allocate
2454// SmallBufferRegions.
2455void* BufferAllocator::allocMediumAligned(size_t bytes, bool nurseryOwned,
2456 bool inGC) {
2457 MOZ_ASSERT(bytes >= MinMediumAllocSize)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(bytes >= MinMediumAllocSize)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(bytes >= MinMediumAllocSize
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"bytes >= MinMediumAllocSize", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2457); AnnotateMozCrashReason("MOZ_ASSERT" "(" "bytes >= MinMediumAllocSize"
")"); do { MOZ_CrashSequence(__null, 2457); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2458 MOZ_ASSERT(bytes <= MaxAlignedAllocSize)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(bytes <= MaxAlignedAllocSize)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(bytes <= MaxAlignedAllocSize
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"bytes <= MaxAlignedAllocSize", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2458); AnnotateMozCrashReason("MOZ_ASSERT" "(" "bytes <= MaxAlignedAllocSize"
")"); do { MOZ_CrashSequence(__null, 2458); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2459 MOZ_ASSERT(std::has_single_bit(bytes))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(std::has_single_bit(bytes))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(std::has_single_bit(bytes)))
), 0))) { do { } while (false); MOZ_ReportAssertionFailure("std::has_single_bit(bytes)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2459); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "std::has_single_bit(bytes)" ")"); do { MOZ_CrashSequence
(__null, 2459); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
2460
2461 // Get size class from |bytes|.
2462 size_t sizeClass = SizeClassForMediumAlloc(bytes);
2463
2464 void* alloc = alignedAlloc(sizeClass, nurseryOwned);
2465 if (MOZ_UNLIKELY(!alloc)(__builtin_expect(!!(!alloc), 0))) {
2466 alloc = retryAlignedAlloc(sizeClass, nurseryOwned, inGC);
2467 if (!alloc) {
2468 return nullptr;
2469 }
2470 }
2471
2472 setAllocated(alloc, bytes, false, inGC);
2473
2474 return alloc;
2475}
2476
2477MOZ_NEVER_INLINE__attribute__((noinline)) void* BufferAllocator::retryAlignedAlloc(size_t sizeClass,
2478 bool nurseryOwned,
2479 bool inGC) {
2480 // Ensure aligned allocation is possible.
2481 MOZ_ASSERT(sizeClass < MaxMediumAllocClass)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(sizeClass < MaxMediumAllocClass)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(sizeClass < MaxMediumAllocClass
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"sizeClass < MaxMediumAllocClass", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2481); AnnotateMozCrashReason("MOZ_ASSERT" "(" "sizeClass < MaxMediumAllocClass"
")"); do { MOZ_CrashSequence(__null, 2481); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2482 size_t expandedSizeClass = sizeClass + 1;
2483
2484 auto alloc = [&]() { return alignedAlloc(sizeClass, nurseryOwned); };
2485 auto growHeap = [&]() {
2486 return stealOrAllocNewChunk(expandedSizeClass, nurseryOwned, inGC);
2487 };
2488 return refillFreeListsAndRetryAlloc(expandedSizeClass, MaxMediumAllocClass,
2489 nurseryOwned, alloc, growHeap);
2490}
2491
2492void* BufferAllocator::alignedAlloc(size_t sizeClass, bool nurseryOwned) {
2493 FreeLists& freeLists = getFreeListsForAlloc(nurseryOwned);
2494 freeLists.checkAvailable();
2495
2496 // Try the first free region for the smallest possible size class. This will
2497 // fail if that region is for the exact size class requested but the region is
2498 // not aligned.
2499 size_t allocClass =
2500 freeLists.getFirstAvailableSizeClass(sizeClass, MaxMediumAllocClass);
2501 MOZ_ASSERT(allocClass >= sizeClass)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(allocClass >= sizeClass)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(allocClass >= sizeClass))
), 0))) { do { } while (false); MOZ_ReportAssertionFailure("allocClass >= sizeClass"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2501); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "allocClass >= sizeClass" ")"); do { MOZ_CrashSequence
(__null, 2501); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
2502 if (allocClass == SIZE_MAX(18446744073709551615UL)) {
2503 return nullptr;
2504 }
2505
2506 FreeRegion* region = freeLists.getFirstRegion(allocClass);
2507 void* ptr = alignedAllocFromRegion(region, sizeClass, freeLists);
2508 if (ptr) {
2509 updateFreeListsAfterAlloc(&freeLists, region, allocClass);
2510 return ptr;
2511 }
2512
2513 // If we couldn't allocate an aligned region, try a larger size class. This
2514 // only happens if we selected the size class equal to the requested size.
2515 MOZ_ASSERT(allocClass == sizeClass)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(allocClass == sizeClass)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(allocClass == sizeClass))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("allocClass == sizeClass"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2515); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "allocClass == sizeClass" ")"); do { MOZ_CrashSequence
(__null, 2515); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
2516 allocClass =
2517 freeLists.getFirstAvailableSizeClass(sizeClass + 1, MaxMediumAllocClass);
2518 if (allocClass == SIZE_MAX(18446744073709551615UL)) {
2519 return nullptr;
2520 }
2521
2522 region = freeLists.getFirstRegion(allocClass);
2523 ptr = alignedAllocFromRegion(region, sizeClass, freeLists);
2524 MOZ_ASSERT(ptr)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ptr)>::isValid, "invalid assertion condition"); if
((__builtin_expect(!!(!(!!(ptr))), 0))) { do { } while (false
); MOZ_ReportAssertionFailure("ptr", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2524); AnnotateMozCrashReason("MOZ_ASSERT" "(" "ptr" ")"); do
{ MOZ_CrashSequence(__null, 2524); __attribute__((nomerge)) ::
abort(); } while (false); } } while (false);
2525 updateFreeListsAfterAlloc(&freeLists, region, allocClass);
2526 return ptr;
2527}
2528
2529void* BufferAllocator::alignedAllocFromRegion(FreeRegion* region,
2530 size_t sizeClass,
2531 FreeLists& freeLists) {
2532 // Attempt to allocate an aligned region from |region|.
2533
2534 uintptr_t start = region->startAddr;
2535 MOZ_ASSERT(region->getEnd() > start)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(region->getEnd() > start)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(region->getEnd() > start
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"region->getEnd() > start", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2535); AnnotateMozCrashReason("MOZ_ASSERT" "(" "region->getEnd() > start"
")"); do { MOZ_CrashSequence(__null, 2535); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2536 MOZ_ASSERT(region->size() >= SizeClassBytes(sizeClass))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(region->size() >= SizeClassBytes(sizeClass))>
::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(region->size() >= SizeClassBytes(sizeClass))))
, 0))) { do { } while (false); MOZ_ReportAssertionFailure("region->size() >= SizeClassBytes(sizeClass)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2536); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "region->size() >= SizeClassBytes(sizeClass)"
")"); do { MOZ_CrashSequence(__null, 2536); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2537 MOZ_ASSERT((region->size() % MinMediumAllocSize) == 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype((region->size() % MinMediumAllocSize) == 0)>::
isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!((region->size() % MinMediumAllocSize) == 0))), 0)
)) { do { } while (false); MOZ_ReportAssertionFailure("(region->size() % MinMediumAllocSize) == 0"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2537); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "(region->size() % MinMediumAllocSize) == 0"
")"); do { MOZ_CrashSequence(__null, 2537); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2538
2539 size_t bytes = SizeClassBytes(sizeClass);
2540 size_t alignedStart = RoundUp(start, bytes);
2541 size_t end = alignedStart + bytes;
2542 if (end > region->getEnd()) {
2543 return nullptr;
2544 }
2545
2546 // Align the start of the region, creating a new free region out of the space
2547 // at the start if necessary.
2548 if (alignedStart != start) {
2549 size_t alignBytes = alignedStart - start;
2550 void* prefix = allocFromRegion(region, alignBytes, sizeClass);
2551 MOZ_ASSERT(uintptr_t(prefix) == start)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(uintptr_t(prefix) == start)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(uintptr_t(prefix) == start))
), 0))) { do { } while (false); MOZ_ReportAssertionFailure("uintptr_t(prefix) == start"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2551); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "uintptr_t(prefix) == start" ")"); do { MOZ_CrashSequence
(__null, 2551); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
2552 (void)prefix;
2553 MOZ_ASSERT(!region->hasDecommittedPages)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!region->hasDecommittedPages)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!region->hasDecommittedPages
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"!region->hasDecommittedPages", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2553); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!region->hasDecommittedPages"
")"); do { MOZ_CrashSequence(__null, 2553); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2554 FreeRegion* region = FreeRegion::create(start, alignBytes, false);
2555 freeLists.pushBack(region, SizeKind::Medium);
2556 }
2557
2558 // Now the start is aligned we can use the normal allocation method.
2559 MOZ_ASSERT(region->startAddr % bytes == 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(region->startAddr % bytes == 0)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(region->startAddr % bytes
== 0))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("region->startAddr % bytes == 0", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2559); AnnotateMozCrashReason("MOZ_ASSERT" "(" "region->startAddr % bytes == 0"
")"); do { MOZ_CrashSequence(__null, 2559); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2560 return allocFromRegion(region, bytes, sizeClass);
2561}
2562
2563void BufferAllocator::setAllocated(void* alloc, size_t bytes, bool nurseryOwned,
2564 bool inGC) {
2565 BufferChunk* chunk = BufferChunk::from(alloc);
2566 chunk->setAllocated(alloc, bytes, true);
2567 MOZ_ASSERT(chunk->allocBytes(alloc) == bytes)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(chunk->allocBytes(alloc) == bytes)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(chunk->allocBytes(alloc) ==
bytes))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("chunk->allocBytes(alloc) == bytes", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2567); AnnotateMozCrashReason("MOZ_ASSERT" "(" "chunk->allocBytes(alloc) == bytes"
")"); do { MOZ_CrashSequence(__null, 2567); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2568
2569 MOZ_ASSERT(!chunk->isNurseryOwned(alloc))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!chunk->isNurseryOwned(alloc))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!chunk->isNurseryOwned(alloc
)))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("!chunk->isNurseryOwned(alloc)", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2569); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!chunk->isNurseryOwned(alloc)"
")"); do { MOZ_CrashSequence(__null, 2569); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2570 if (nurseryOwned) {
2571 MOZ_ASSERT(chunk->hasNurseryOwnedAllocs)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(chunk->hasNurseryOwnedAllocs)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(chunk->hasNurseryOwnedAllocs
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"chunk->hasNurseryOwnedAllocs", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2571); AnnotateMozCrashReason("MOZ_ASSERT" "(" "chunk->hasNurseryOwnedAllocs"
")"); do { MOZ_CrashSequence(__null, 2571); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2572 chunk->setNurseryOwned(alloc, nurseryOwned);
2573 }
2574
2575 MOZ_ASSERT(!chunk->isMarked(alloc))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!chunk->isMarked(alloc))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!chunk->isMarked(alloc)))
), 0))) { do { } while (false); MOZ_ReportAssertionFailure("!chunk->isMarked(alloc)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2575); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!chunk->isMarked(alloc)" ")"); do { MOZ_CrashSequence
(__null, 2575); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
2576
2577 bool checkThresholds = !inGC;
2578 increaseHeapSize(bytes, nurseryOwned, checkThresholds, false);
2579
2580 MOZ_ASSERT(!chunk->isSmallBufferRegion(alloc))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!chunk->isSmallBufferRegion(alloc))>::isValid,
"invalid assertion condition"); if ((__builtin_expect(!!(!(!
!(!chunk->isSmallBufferRegion(alloc)))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("!chunk->isSmallBufferRegion(alloc)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2580); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!chunk->isSmallBufferRegion(alloc)" ")"
); do { MOZ_CrashSequence(__null, 2580); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2581}
2582
2583void BufferAllocator::updateFreeListsAfterAlloc(FreeLists* freeLists,
2584 FreeRegion* region,
2585 size_t sizeClass) {
2586 // Updates |freeLists| after an allocation from |region| which is currently in
2587 // the |sizeClass| free list. This may move the region to a different free
2588 // list.
2589
2590 freeLists->assertContains(sizeClass, region);
2591
2592 // If the region is still valid for further allocations of this size class
2593 // then there's nothing to do.
2594 size_t classBytes = SizeClassBytes(sizeClass);
2595 size_t newSize = region->size();
2596 MOZ_ASSERT(newSize % GranularityForSizeClass(sizeClass) == 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(newSize % GranularityForSizeClass(sizeClass) == 0)>
::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(newSize % GranularityForSizeClass(sizeClass) == 0)))
, 0))) { do { } while (false); MOZ_ReportAssertionFailure("newSize % GranularityForSizeClass(sizeClass) == 0"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2596); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "newSize % GranularityForSizeClass(sizeClass) == 0"
")"); do { MOZ_CrashSequence(__null, 2596); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2597 if (newSize >= classBytes) {
2598 return;
2599 }
2600
2601 // Remove region from this free list.
2602 freeLists->remove(sizeClass, region);
2603
2604 // If the region is now empty then we're done.
2605 if (newSize == 0) {
2606 return;
2607 }
2608
2609 // Otherwise region is now too small. Move it to the appropriate bucket for
2610 // its reduced size if possible.
2611
2612 if (newSize < MinFreeRegionSize) {
2613 // We can't record a region this small. The free space will not be reused
2614 // until enough adjacent space become free
2615 return;
2616 }
2617
2618 size_t newSizeClass =
2619 SizeClassForFreeRegion(newSize, SizeClassKind(sizeClass));
2620 MOZ_ASSERT_IF(newSizeClass != MaxMediumAllocClass,do { if (newSizeClass != MaxMediumAllocClass) { do { static_assert
( mozilla::detail::AssertionConditionType<decltype(newSize
>= SizeClassBytes(newSizeClass))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(newSize >= SizeClassBytes
(newSizeClass)))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("newSize >= SizeClassBytes(newSizeClass)", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2621); AnnotateMozCrashReason("MOZ_ASSERT" "(" "newSize >= SizeClassBytes(newSizeClass)"
")"); do { MOZ_CrashSequence(__null, 2621); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false)
2621 newSize >= SizeClassBytes(newSizeClass))do { if (newSizeClass != MaxMediumAllocClass) { do { static_assert
( mozilla::detail::AssertionConditionType<decltype(newSize
>= SizeClassBytes(newSizeClass))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(newSize >= SizeClassBytes
(newSizeClass)))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("newSize >= SizeClassBytes(newSizeClass)", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2621); AnnotateMozCrashReason("MOZ_ASSERT" "(" "newSize >= SizeClassBytes(newSizeClass)"
")"); do { MOZ_CrashSequence(__null, 2621); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false);
2622 MOZ_ASSERT(newSizeClass <= sizeClass)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(newSizeClass <= sizeClass)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(newSizeClass <= sizeClass
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"newSizeClass <= sizeClass", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2622); AnnotateMozCrashReason("MOZ_ASSERT" "(" "newSizeClass <= sizeClass"
")"); do { MOZ_CrashSequence(__null, 2622); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2623 MOZ_ASSERT_IF(newSizeClass != MaxMediumAllocClass, newSizeClass < sizeClass)do { if (newSizeClass != MaxMediumAllocClass) { do { static_assert
( mozilla::detail::AssertionConditionType<decltype(newSizeClass
< sizeClass)>::isValid, "invalid assertion condition")
; if ((__builtin_expect(!!(!(!!(newSizeClass < sizeClass))
), 0))) { do { } while (false); MOZ_ReportAssertionFailure("newSizeClass < sizeClass"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2623); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "newSizeClass < sizeClass" ")"); do { MOZ_CrashSequence
(__null, 2623); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false); } } while (false);
2624 MOZ_ASSERT(SizeClassKind(newSizeClass) == SizeClassKind(sizeClass))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(SizeClassKind(newSizeClass) == SizeClassKind(sizeClass
))>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(SizeClassKind(newSizeClass) == SizeClassKind(sizeClass
)))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("SizeClassKind(newSizeClass) == SizeClassKind(sizeClass)", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2624); AnnotateMozCrashReason("MOZ_ASSERT" "(" "SizeClassKind(newSizeClass) == SizeClassKind(sizeClass)"
")"); do { MOZ_CrashSequence(__null, 2624); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2625 freeLists->pushFront(newSizeClass, region);
2626}
2627
2628void BufferAllocator::recommitRegion(FreeRegion* region) {
2629 MOZ_ASSERT(region->hasDecommittedPages)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(region->hasDecommittedPages)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(region->hasDecommittedPages
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"region->hasDecommittedPages", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2629); AnnotateMozCrashReason("MOZ_ASSERT" "(" "region->hasDecommittedPages"
")"); do { MOZ_CrashSequence(__null, 2629); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2630 MOZ_ASSERT(DecommitEnabled())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(DecommitEnabled())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(DecommitEnabled()))), 0))) {
do { } while (false); MOZ_ReportAssertionFailure("DecommitEnabled()"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2630); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "DecommitEnabled()" ")"); do { MOZ_CrashSequence
(__null, 2630); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
2631
2632 BufferChunk* chunk = BufferChunk::from(region);
2633 uintptr_t startAddr = RoundUp(region->startAddr, PageSize);
2634 uintptr_t endAddr = RoundDown(uintptr_t(region), PageSize);
2635
2636 size_t startPage = (startAddr - uintptr_t(chunk)) / PageSize;
2637 size_t endPage = (endAddr - uintptr_t(chunk)) / PageSize;
2638
2639 // If the start of the region does not lie on a page boundary the page it is
2640 // in should be committed as it must either contain the start of the chunk, a
2641 // FreeRegion or an allocation.
2642 MOZ_ASSERT_IF((region->startAddr % PageSize) != 0,do { if ((region->startAddr % PageSize) != 0) { do { static_assert
( mozilla::detail::AssertionConditionType<decltype(!chunk->
decommittedPages.ref()[startPage - 1])>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!chunk->decommittedPages.
ref()[startPage - 1]))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("!chunk->decommittedPages.ref()[startPage - 1]", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2643); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!chunk->decommittedPages.ref()[startPage - 1]"
")"); do { MOZ_CrashSequence(__null, 2643); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false)
2643 !chunk->decommittedPages.ref()[startPage - 1])do { if ((region->startAddr % PageSize) != 0) { do { static_assert
( mozilla::detail::AssertionConditionType<decltype(!chunk->
decommittedPages.ref()[startPage - 1])>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!chunk->decommittedPages.
ref()[startPage - 1]))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("!chunk->decommittedPages.ref()[startPage - 1]", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2643); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!chunk->decommittedPages.ref()[startPage - 1]"
")"); do { MOZ_CrashSequence(__null, 2643); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false);
2644
2645 // The end of the region should be committed as it holds FreeRegion |region|.
2646 MOZ_ASSERT(!chunk->decommittedPages.ref()[endPage])do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!chunk->decommittedPages.ref()[endPage])>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(!chunk->decommittedPages.ref()[endPage]))), 0))) { do {
} while (false); MOZ_ReportAssertionFailure("!chunk->decommittedPages.ref()[endPage]"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2646); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!chunk->decommittedPages.ref()[endPage]"
")"); do { MOZ_CrashSequence(__null, 2646); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2647
2648 MarkPagesInUseSoft(reinterpret_cast<void*>(startAddr), endAddr - startAddr);
2649 for (size_t i = startPage; i != endPage; i++) {
2650 chunk->decommittedPages.ref()[i] = false;
2651 }
2652
2653 region->hasDecommittedPages = false;
2654}
2655
2656static inline StallAndRetry ShouldStallAndRetry(bool inGC) {
2657 return inGC ? StallAndRetry::Yes : StallAndRetry::No;
2658}
2659
2660bool BufferAllocator::stealOrAllocNewChunk(size_t sizeClass, bool nurseryOwned,
2661 bool inGC) {
2662 if (majorState == State::Marking && !tenuredChunksToSweep.ref().isEmpty() &&
2663 gc->isNormalGC()) {
2664 if (tryToStealQueuedChunk(nurseryOwned, sizeClass)) {
2665 return true;
2666 }
2667 }
2668
2669 return allocNewChunk(nurseryOwned, inGC);
2670}
2671
2672bool BufferAllocator::tryToStealQueuedChunk(bool nurseryOwned,
2673 size_t sizeClass) {
2674 // Attempt to steal a nearly empty chunk that would otherwise be swept so we
2675 // can continue allocating from it.
2676
2677 MOZ_ASSERT(majorState == State::Marking)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(majorState == State::Marking)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(majorState == State::Marking
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"majorState == State::Marking", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2677); AnnotateMozCrashReason("MOZ_ASSERT" "(" "majorState == State::Marking"
")"); do { MOZ_CrashSequence(__null, 2677); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2678 MOZ_ASSERT(!tenuredChunksToSweep.ref().isEmpty())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!tenuredChunksToSweep.ref().isEmpty())>::isValid,
"invalid assertion condition"); if ((__builtin_expect(!!(!(!
!(!tenuredChunksToSweep.ref().isEmpty()))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("!tenuredChunksToSweep.ref().isEmpty()"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2678); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!tenuredChunksToSweep.ref().isEmpty()" ")"
); do { MOZ_CrashSequence(__null, 2678); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2679
2680 BufferChunk* chunk = tenuredChunksToSweep.ref().getLast();
2681 if (!chunk->ownsFreeLists) {
2682 return false;
2683 }
2684
2685 // Look for chunks that have a free region at least 1/2 the size of a chunk,
2686 // and that can fit the allocation.
2687 size_t minSizeClass = std::max(sizeClass, MaxMediumAllocClass - 1);
2688 if (!chunk->freeLists.ref().hasAnySizeClass(minSizeClass,
2689 MaxMediumAllocClass)) {
2690 return false;
2691 }
2692
2693 // Remove the chunk from the sweep list.
2694 tenuredChunksToSweep.ref().remove(chunk);
2695
2696 // Make the chunk look like a newly allocated one and disable barriers.
2697 chunk->allocatedDuringCollection = true;
2698
2699#ifdef JS_GC_CONCURRENT_MARKING
2700 // Set a flag so we know to clear the mark bits later. We can't do this
2701 // now because concurrent marking may be accessing them.
2702 chunk->stolenFromSweepList = true;
2703#else
2704 chunk->clearMarkBits();
2705#endif
2706
2707 // Move the chunk to the current allocation lists.
2708 MOZ_ASSERT(!chunk->hasNurseryOwnedAllocs)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!chunk->hasNurseryOwnedAllocs)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!chunk->hasNurseryOwnedAllocs
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"!chunk->hasNurseryOwnedAllocs", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2708); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!chunk->hasNurseryOwnedAllocs"
")"); do { MOZ_CrashSequence(__null, 2708); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2709
2710 if (nurseryOwned) {
2711 chunk->hasNurseryOwnedAllocs = true;
2712 currentMixedChunks.ref().pushBack(chunk);
2713 mixedFreeLists.ref().append(std::move(chunk->freeLists.ref()));
2714 } else {
2715 currentTenuredChunks.ref().pushBack(chunk);
2716 tenuredFreeLists.ref().append(std::move(chunk->freeLists.ref()));
2717 }
2718
2719 chunk->ownsFreeLists = false;
2720 chunk->freeLists.ref().assertEmpty();
2721
2722 return true;
2723}
2724
2725bool BufferAllocator::allocNewChunk(bool nurseryOwned, bool inGC) {
2726 if (!inGC && js::oom::ShouldFailWithOOM()) {
2727 return false;
2728 }
2729
2730 AutoLockGCBgAlloc lock(gc);
2731 ArenaChunk* baseChunk = gc->getOrAllocChunk(ShouldStallAndRetry(inGC), lock);
2732 if (!baseChunk) {
2733 return false;
2734 }
2735
2736 CheckHighBitsOfPointer(baseChunk);
2737
2738 // Ensure all memory is initially committed.
2739 if (!baseChunk->decommittedPages.IsEmpty()) {
2740 MOZ_ASSERT(DecommitEnabled())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(DecommitEnabled())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(DecommitEnabled()))), 0))) {
do { } while (false); MOZ_ReportAssertionFailure("DecommitEnabled()"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2740); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "DecommitEnabled()" ")"); do { MOZ_CrashSequence
(__null, 2740); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
2741 MarkPagesInUseSoft(baseChunk, ChunkSize);
2742 }
2743
2744 // Unpoison past the ChunkBase header.
2745 void* ptr = reinterpret_cast<void*>(uintptr_t(baseChunk) + sizeof(ChunkBase));
2746 size_t size = ChunkSize - sizeof(ChunkBase);
2747 SetMemCheckKind(ptr, size, MemCheckKind::MakeUndefined);
2748
2749 BufferChunk* chunk = new (baseChunk) BufferChunk(zone);
2750 chunk->allocatedDuringCollection = majorState != State::NotCollecting;
2751
2752 if (nurseryOwned) {
2753 chunk->hasNurseryOwnedAllocs = true;
2754 currentMixedChunks.ref().pushBack(chunk);
2755 } else {
2756 currentTenuredChunks.ref().pushBack(chunk);
2757 }
2758
2759 uintptr_t freeStart = uintptr_t(chunk) + FirstMediumAllocOffset;
2760 uintptr_t freeEnd = uintptr_t(chunk) + ChunkSize;
2761
2762 size_t sizeClass =
2763 SizeClassForFreeRegion(freeEnd - freeStart, SizeKind::Medium);
2764 MOZ_ASSERT(sizeClass > MaxSmallAllocClass)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(sizeClass > MaxSmallAllocClass)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(sizeClass > MaxSmallAllocClass
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"sizeClass > MaxSmallAllocClass", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2764); AnnotateMozCrashReason("MOZ_ASSERT" "(" "sizeClass > MaxSmallAllocClass"
")"); do { MOZ_CrashSequence(__null, 2764); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2765 MOZ_ASSERT(sizeClass <= MaxMediumAllocClass)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(sizeClass <= MaxMediumAllocClass)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(sizeClass <= MaxMediumAllocClass
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"sizeClass <= MaxMediumAllocClass", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2765); AnnotateMozCrashReason("MOZ_ASSERT" "(" "sizeClass <= MaxMediumAllocClass"
")"); do { MOZ_CrashSequence(__null, 2765); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2766
2767 ptr = reinterpret_cast<void*>(freeEnd - sizeof(FreeRegion));
2768 FreeRegion* region = new (ptr) FreeRegion(freeStart);
2769 MOZ_ASSERT(region->getEnd() == freeEnd)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(region->getEnd() == freeEnd)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(region->getEnd() == freeEnd
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"region->getEnd() == freeEnd", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2769); AnnotateMozCrashReason("MOZ_ASSERT" "(" "region->getEnd() == freeEnd"
")"); do { MOZ_CrashSequence(__null, 2769); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2770
2771 if (nurseryOwned) {
2772 mixedFreeLists.ref().pushFront(sizeClass, region);
2773 } else {
2774 tenuredFreeLists.ref().pushFront(sizeClass, region);
2775 }
2776
2777 totalChunkCount++;
2778
2779 return true;
2780}
2781
2782bool BufferAllocator::sweepChunk(BufferChunk* chunk, SweepKind sweepKind,
2783 bool shouldDecommit) {
2784 // Find all regions of free space in |chunk| and add them to the swept free
2785 // lists.
2786
2787 // TODO: It could be beneficialy to allocate from most-full chunks first. This
2788 // could happen by sweeping all chunks and then sorting them by how much free
2789 // space they had and then adding their free regions to the free lists in that
2790 // order.
2791
2792 MOZ_ASSERT_IF(sweepKind != SweepKind::Nursery, !chunk->stolenFromSweepList)do { if (sweepKind != SweepKind::Nursery) { do { static_assert
( mozilla::detail::AssertionConditionType<decltype(!chunk->
stolenFromSweepList)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!chunk->stolenFromSweepList
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"!chunk->stolenFromSweepList", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2792); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!chunk->stolenFromSweepList"
")"); do { MOZ_CrashSequence(__null, 2792); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false);
2793 MOZ_ASSERT_IF(sweepKind != SweepKind::Nursery,do { if (sweepKind != SweepKind::Nursery) { do { static_assert
( mozilla::detail::AssertionConditionType<decltype(!chunk->
allocatedDuringCollection)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!chunk->allocatedDuringCollection
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"!chunk->allocatedDuringCollection", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2794); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!chunk->allocatedDuringCollection"
")"); do { MOZ_CrashSequence(__null, 2794); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false)
2794 !chunk->allocatedDuringCollection)do { if (sweepKind != SweepKind::Nursery) { do { static_assert
( mozilla::detail::AssertionConditionType<decltype(!chunk->
allocatedDuringCollection)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!chunk->allocatedDuringCollection
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"!chunk->allocatedDuringCollection", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2794); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!chunk->allocatedDuringCollection"
")"); do { MOZ_CrashSequence(__null, 2794); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false);
2795 FreeLists& freeLists = chunk->freeLists.ref();
2796
2797 // Try and check that existing free regions are unchanged if we don't free
2798 // anything. Chunks that don't own their free lists may have been modified by
2799 // realloc while queued for sweeping without their free regions being updated.
2800 bool mayBeUnchanged = chunk->ownsFreeLists;
2801
2802 freeLists.clear();
2803 chunk->ownsFreeLists = true;
2804
2805 if (sweepKind == SweepKind::Tenured) {
2806 chunk->usedBytesAfterSweep = 0;
2807 chunk->adminBytesAfterSweep = 0;
2808 }
2809
2810 // First sweep any small buffer regions.
2811 bool hasNurseryOwnedSmallRegions = false;
2812 size_t smallRegionBytesFreed = 0;
2813 size_t smallRegionBytesRetained = 0;
2814
2815 for (auto iter = chunk->smallRegionIter(); !iter.done(); iter.next()) {
2816 SmallBufferRegion* region = iter.get();
2817 MOZ_ASSERT(!chunk->isMarked(region))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!chunk->isMarked(region))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!chunk->isMarked(region))
)), 0))) { do { } while (false); MOZ_ReportAssertionFailure("!chunk->isMarked(region)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2817); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!chunk->isMarked(region)" ")"); do { MOZ_CrashSequence
(__null, 2817); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
2818 MOZ_ASSERT(!chunk->isNurseryOwned(region))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!chunk->isNurseryOwned(region))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!chunk->isNurseryOwned(region
)))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("!chunk->isNurseryOwned(region)", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2818); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!chunk->isNurseryOwned(region)"
")"); do { MOZ_CrashSequence(__null, 2818); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2819 MOZ_ASSERT(chunk->allocBytes(region) == SmallRegionSize)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(chunk->allocBytes(region) == SmallRegionSize)>
::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(chunk->allocBytes(region) == SmallRegionSize))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("chunk->allocBytes(region) == SmallRegionSize"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2819); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "chunk->allocBytes(region) == SmallRegionSize"
")"); do { MOZ_CrashSequence(__null, 2819); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2820
2821 size_t regionUsedBytes = 0;
2822 if (!sweepSmallBufferRegion(chunk, region, sweepKind, &regionUsedBytes)) {
2823 chunk->setSmallBufferRegion(region, false);
2824 chunk->setDeallocated(region, SmallRegionSize);
2825 PoisonAlloc(region, JS_SWEPT_TENURED_PATTERN, sizeof(SmallBufferRegion),
2826 MemCheckKind::MakeUndefined);
2827 smallRegionBytesFreed += SmallRegionSize;
2828 mayBeUnchanged = false;
2829 } else {
2830 if (sweepKind == SweepKind::Tenured) {
2831 chunk->setMarked(region);
2832 chunk->usedBytesAfterSweep += regionUsedBytes;
2833 chunk->adminBytesAfterSweep += FirstSmallAllocOffset;
2834 smallRegionBytesRetained += SmallRegionSize;
2835 }
2836 if (region->hasNurseryOwnedAllocs()) {
2837 hasNurseryOwnedSmallRegions = true;
2838 }
2839 }
2840 }
2841
2842 if (smallRegionBytesFreed) {
2843 bool inMajorGC = sweepKind == SweepKind::Tenured;
2844 decreaseHeapSize(smallRegionBytesFreed, false, inMajorGC);
2845 }
2846
2847 auto result =
2848 chunk->sweep(freeLists, sweepKind, mayBeUnchanged, shouldDecommit);
2849
2850 if (sweepKind == SweepKind::Tenured && result.bytesFreed) {
2851 decreaseHeapSize(result.bytesFreed, false, true);
2852 }
2853
2854 if (sweepKind != SweepKind::RebuildFreeLists && result.isEmpty) {
2855 // Chunk is empty. Give it back to the system. It will never be merged, so
2856 // it simply won't contribute to BufferAllocatorRuntime's used/free/admin
2857 // byte totals for this GC (see mergeSweptData/resetRetainedStats).
2858 MOZ_ASSERT(totalChunkCount != 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(totalChunkCount != 0)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(totalChunkCount != 0))), 0))
) { do { } while (false); MOZ_ReportAssertionFailure("totalChunkCount != 0"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2858); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "totalChunkCount != 0" ")"); do { MOZ_CrashSequence
(__null, 2858); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
2859 totalChunkCount--;
2860 bool allMemoryCommitted = chunk->decommittedPages.ref().IsEmpty();
2861 chunk->~BufferChunk();
2862 ArenaChunk* tenuredChunk = ArenaChunk::init(chunk, gc, allMemoryCommitted);
2863 AutoLockGC gcLock(gc);
2864 gc->recycleChunk(tenuredChunk, gcLock);
2865 return false;
2866 }
2867
2868 if (sweepKind == SweepKind::Tenured) {
2869 // Don't double count small regions.
2870 chunk->usedBytesAfterSweep += result.usedBytes - smallRegionBytesRetained;
2871 chunk->adminBytesAfterSweep += FirstMediumAllocOffset;
2872 }
2873
2874 if (sweepKind != SweepKind::RebuildFreeLists) {
2875 chunk->hasNurseryOwnedAllocsAfterSweep =
2876 hasNurseryOwnedSmallRegions || result.hasNurseryOwnedAllocs;
2877 }
2878
2879 return true;
2880}
2881
2882/* static */
2883bool BufferAllocator::CanSweepAlloc(bool nurseryOwned,
2884 BufferAllocator::SweepKind sweepKind) {
2885 static_assert(SweepKind::Nursery == SweepKind(uint8_t(true)));
2886 static_assert(SweepKind::Tenured == SweepKind(uint8_t(false)));
2887 SweepKind requiredKind = SweepKind(uint8_t(nurseryOwned));
2888 return sweepKind == requiredKind;
2889}
2890
2891void BufferChunk::addSweptRegion(uintptr_t freeStart, uintptr_t freeEnd,
2892 bool shouldDecommit, bool expectUnchanged,
2893 FreeLists& freeLists) {
2894 // Add the region from |freeStart| to |freeEnd| to the appropriate swept free
2895 // list based on its size.
2896
2897 MOZ_ASSERT(freeStart >= FirstMediumAllocOffset)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(freeStart >= FirstMediumAllocOffset)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(freeStart >= FirstMediumAllocOffset))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("freeStart >= FirstMediumAllocOffset"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2897); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "freeStart >= FirstMediumAllocOffset" ")"
); do { MOZ_CrashSequence(__null, 2897); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2898 MOZ_ASSERT(freeStart < freeEnd)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(freeStart < freeEnd)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(freeStart < freeEnd))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("freeStart < freeEnd"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2898); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "freeStart < freeEnd" ")"); do { MOZ_CrashSequence
(__null, 2898); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
2899 MOZ_ASSERT(freeEnd <= ChunkSize)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(freeEnd <= ChunkSize)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(freeEnd <= ChunkSize))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("freeEnd <= ChunkSize"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2899); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "freeEnd <= ChunkSize" ")"); do { MOZ_CrashSequence
(__null, 2899); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
2900 MOZ_ASSERT((freeStart % MediumAllocGranularity) == 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype((freeStart % MediumAllocGranularity) == 0)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!((freeStart % MediumAllocGranularity) == 0))), 0))) { do { }
while (false); MOZ_ReportAssertionFailure("(freeStart % MediumAllocGranularity) == 0"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2900); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "(freeStart % MediumAllocGranularity) == 0"
")"); do { MOZ_CrashSequence(__null, 2900); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2901 MOZ_ASSERT((freeEnd % MediumAllocGranularity) == 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype((freeEnd % MediumAllocGranularity) == 0)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!((freeEnd % MediumAllocGranularity) == 0))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("(freeEnd % MediumAllocGranularity) == 0"
, "./../../../../js/src/gc/BufferAllocator.cpp", 2901); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "(freeEnd % MediumAllocGranularity) == 0" ")"
); do { MOZ_CrashSequence(__null, 2901); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2902 MOZ_ASSERT_IF(shouldDecommit, DecommitEnabled())do { if (shouldDecommit) { do { static_assert( mozilla::detail
::AssertionConditionType<decltype(DecommitEnabled())>::
isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(DecommitEnabled()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("DecommitEnabled()", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2902); AnnotateMozCrashReason("MOZ_ASSERT" "(" "DecommitEnabled()"
")"); do { MOZ_CrashSequence(__null, 2902); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false);
2903
2904 // Decommit pages if |shouldDecommit| was specified, but leave space for
2905 // the FreeRegion structure at the end.
2906 bool anyDecommitted = false;
2907 uintptr_t decommitStart = RoundUp(freeStart, PageSize);
2908 uintptr_t decommitEnd = RoundDown(freeEnd - sizeof(FreeRegion), PageSize);
2909 size_t endPage = decommitEnd / PageSize;
2910 if (shouldDecommit && decommitEnd > decommitStart) {
2911 void* ptr = reinterpret_cast<void*>(decommitStart + uintptr_t(this));
2912 MarkPagesUnusedSoft(ptr, decommitEnd - decommitStart);
2913 size_t startPage = decommitStart / PageSize;
2914 for (size_t i = startPage; i != endPage; i++) {
2915 decommittedPages.ref()[i] = true;
2916 }
2917 anyDecommitted = true;
2918 } else {
2919 // Check for any previously decommitted pages.
2920 uintptr_t startPage = RoundDown(freeStart, PageSize) / PageSize;
2921 for (size_t i = startPage; i != endPage; i++) {
2922 if (decommittedPages.ref()[i]) {
2923 anyDecommitted = true;
2924 }
2925 }
2926 }
2927
2928 // The last page must have previously been either a live allocation or a
2929 // FreeRegion, so it must already be committed.
2930 MOZ_ASSERT(!decommittedPages.ref()[endPage])do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!decommittedPages.ref()[endPage])>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!decommittedPages.ref()[endPage
]))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("!decommittedPages.ref()[endPage]", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2930); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!decommittedPages.ref()[endPage]"
")"); do { MOZ_CrashSequence(__null, 2930); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2931
2932 freeStart += uintptr_t(this);
2933 freeEnd += uintptr_t(this);
2934
2935 size_t bytes = freeEnd - freeStart;
2936 FreeRegion* region =
2937 FreeRegion::create(freeStart, bytes, anyDecommitted, expectUnchanged);
2938 freeLists.pushBack(region, SizeKind::Medium);
2939}
2940
2941bool BufferAllocator::sweepSmallBufferRegion(BufferChunk* chunk,
2942 SmallBufferRegion* region,
2943 SweepKind sweepKind,
2944 size_t* usedBytesOut) {
2945 FreeLists& freeLists = chunk->freeLists.ref();
2946 auto result = region->sweep(freeLists, sweepKind, false, false);
2947
2948 if (result.isEmpty) {
2949 return false;
2950 }
2951
2952 region->setHasNurseryOwnedAllocs(result.hasNurseryOwnedAllocs);
2953 *usedBytesOut = result.usedBytes;
2954 return true;
2955}
2956
2957template <typename D, size_t S, size_t G>
2958AllocSpace<D, S, G>::SweepResult AllocSpace<D, S, G>::sweep(
2959 FreeLists& freeLists, SweepKind sweepKind, bool mayBeUnchanged,
2960 bool shouldDecommit) {
2961 SweepResult result;
2962
2963 size_t freeStart = firstAllocOffset();
2964
2965 bool regionUnchanged = mayBeUnchanged;
2966 for (auto iter = allocIter(); !iter.done(); iter.next()) {
2967 void* alloc = iter.get();
2968
2969 size_t bytes = allocBytes(alloc);
2970 uintptr_t allocEnd = iter.getOffset() + bytes;
2971
2972 bool nurseryOwned = isNurseryOwned(alloc);
2973 bool canSweep = BufferAllocator::CanSweepAlloc(nurseryOwned, sweepKind);
2974 bool shouldSweep = canSweep && !isMarked(alloc);
2975 if (shouldSweep) {
2976 // Dead. Update allocated bitmap, metadata and heap size accounting.
2977 setDeallocated(alloc, bytes);
2978 PoisonAlloc(alloc, JS_SWEPT_TENURED_PATTERN, bytes,
2979 MemCheckKind::MakeUndefined);
2980 result.bytesFreed += bytes;
2981 regionUnchanged = false;
2982 } else {
2983 // Alive. Add any free space before this allocation.
2984 uintptr_t allocStart = iter.getOffset();
2985 if (freeStart != allocStart) {
2986 asDerived()->addSweptRegion(freeStart, allocStart, shouldDecommit,
2987 regionUnchanged, freeLists);
2988 }
2989 freeStart = allocEnd;
2990 result.usedBytes += bytes;
2991 if (canSweep) {
2992 setUnmarked(alloc);
2993 }
2994 if (nurseryOwned) {
2995 MOZ_ASSERT(sweepKind == SweepKind::Nursery)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(sweepKind == SweepKind::Nursery)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(sweepKind == SweepKind::Nursery
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"sweepKind == SweepKind::Nursery", "./../../../../js/src/gc/BufferAllocator.cpp"
, 2995); AnnotateMozCrashReason("MOZ_ASSERT" "(" "sweepKind == SweepKind::Nursery"
")"); do { MOZ_CrashSequence(__null, 2995); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
2996 result.hasNurseryOwnedAllocs = true;
2997 }
2998 regionUnchanged = mayBeUnchanged;
2999 }
3000 }
3001
3002 // Add any free space from the end of the last allocation to the end of the
3003 // space. We don't always need to do this if the space is entirely empty.
3004 result.isEmpty = freeStart == firstAllocOffset();
3005 bool hasFreeRegionAtEnd = freeStart != SizeBytes;
3006 bool addFreeRegionIfEmpty = sweepKind == SweepKind::RebuildFreeLists &&
3007 std::is_same_v<D, BufferChunk>;
3008 if (hasFreeRegionAtEnd && (!result.isEmpty || addFreeRegionIfEmpty)) {
3009 asDerived()->addSweptRegion(freeStart, SizeBytes, shouldDecommit,
3010 regionUnchanged, freeLists);
3011 }
3012
3013 return result;
3014}
3015
3016void SmallBufferRegion::addSweptRegion(uintptr_t freeStart, uintptr_t freeEnd,
3017 bool shouldDecommit,
3018 bool expectUnchanged,
3019 FreeLists& freeLists) {
3020 // Add the region from |freeStart| to |freeEnd| to the appropriate swept free
3021 // list based on its size. Unused pages in small buffer regions are not
3022 // decommitted.
3023
3024 MOZ_ASSERT(freeStart >= FirstSmallAllocOffset)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(freeStart >= FirstSmallAllocOffset)>::isValid,
"invalid assertion condition"); if ((__builtin_expect(!!(!(!
!(freeStart >= FirstSmallAllocOffset))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("freeStart >= FirstSmallAllocOffset"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3024); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "freeStart >= FirstSmallAllocOffset" ")"
); do { MOZ_CrashSequence(__null, 3024); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3025 MOZ_ASSERT(freeStart < freeEnd)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(freeStart < freeEnd)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(freeStart < freeEnd))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("freeStart < freeEnd"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3025); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "freeStart < freeEnd" ")"); do { MOZ_CrashSequence
(__null, 3025); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
3026 MOZ_ASSERT(freeEnd <= SmallRegionSize)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(freeEnd <= SmallRegionSize)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(freeEnd <= SmallRegionSize
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"freeEnd <= SmallRegionSize", "./../../../../js/src/gc/BufferAllocator.cpp"
, 3026); AnnotateMozCrashReason("MOZ_ASSERT" "(" "freeEnd <= SmallRegionSize"
")"); do { MOZ_CrashSequence(__null, 3026); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3027 MOZ_ASSERT(freeStart % SmallAllocGranularity == 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(freeStart % SmallAllocGranularity == 0)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(freeStart % SmallAllocGranularity == 0))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("freeStart % SmallAllocGranularity == 0"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3027); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "freeStart % SmallAllocGranularity == 0" ")"
); do { MOZ_CrashSequence(__null, 3027); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3028 MOZ_ASSERT(freeEnd % SmallAllocGranularity == 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(freeEnd % SmallAllocGranularity == 0)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(freeEnd % SmallAllocGranularity
== 0))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("freeEnd % SmallAllocGranularity == 0", "./../../../../js/src/gc/BufferAllocator.cpp"
, 3028); AnnotateMozCrashReason("MOZ_ASSERT" "(" "freeEnd % SmallAllocGranularity == 0"
")"); do { MOZ_CrashSequence(__null, 3028); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3029 MOZ_ASSERT(!shouldDecommit)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!shouldDecommit)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!shouldDecommit))), 0))) { do
{ } while (false); MOZ_ReportAssertionFailure("!shouldDecommit"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3029); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!shouldDecommit" ")"); do { MOZ_CrashSequence
(__null, 3029); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
3030
3031 freeStart += uintptr_t(this);
3032 freeEnd += uintptr_t(this);
3033
3034 size_t bytes = freeEnd - freeStart;
3035 FreeRegion* freeRegion =
3036 FreeRegion::create(freeStart, bytes, false, expectUnchanged);
3037 if (freeRegion) {
3038 freeLists.pushBack(freeRegion, SizeKind::Small);
3039 }
3040}
3041
3042void BufferAllocator::rebuildFreeLists(BufferChunk* chunk) {
3043 // Called when sweeping is aborted for chunks that don't have their own free
3044 // lists because they were being used for allocation.
3045 MOZ_ASSERT(!chunk->ownsFreeLists)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!chunk->ownsFreeLists)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!chunk->ownsFreeLists))),
0))) { do { } while (false); MOZ_ReportAssertionFailure("!chunk->ownsFreeLists"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3045); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!chunk->ownsFreeLists" ")"); do { MOZ_CrashSequence
(__null, 3045); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
3046 sweepChunk(chunk, SweepKind::RebuildFreeLists, false);
3047}
3048
3049void BufferAllocator::freeMedium(void* alloc) {
3050 // Free a medium sized allocation. This coalesces the free space with any
3051 // neighboring free regions. Coalescing is necessary for resize to work
3052 // properly.
3053
3054 BufferChunk* chunk = BufferChunk::from(alloc);
3055 MOZ_ASSERT(chunk->zone == zone)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(chunk->zone == zone)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(chunk->zone == zone))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("chunk->zone == zone"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3055); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "chunk->zone == zone" ")"); do { MOZ_CrashSequence
(__null, 3055); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
3056
3057 if (!canModifyAllocations(chunk)) {
3058 return;
3059 }
3060
3061 size_t bytes = chunk->allocBytes(alloc);
3062 PoisonAlloc(alloc, JS_FREED_BUFFER_PATTERN, bytes,
3063 MemCheckKind::MakeUndefined);
3064
3065 // Update heap size.
3066 bool updateRetained =
3067 majorState == State::Marking && !chunk->allocatedDuringCollection;
3068 decreaseHeapSize(bytes, chunk->isNurseryOwned(alloc), updateRetained);
3069
3070 // TODO: Since the mark bits are atomic, it's probably OK to unmark even if
3071 // the chunk is currently being swept. If we get lucky the memory will be
3072 // freed sooner.
3073 chunk->setUnmarked(alloc);
3074
3075 // Set region as not allocated and clear metadata.
3076 chunk->setDeallocated(alloc, bytes);
3077
3078 uintptr_t startAddr = uintptr_t(alloc);
3079 uintptr_t endAddr = startAddr + bytes;
3080
3081 // If the chunk is in one of the available lists we may need to move it.
3082 ChunkLists* availableChunks = getChunkAvailableLists(chunk);
3083 size_t oldChunkSizeClass = SIZE_MAX(18446744073709551615UL);
3084 if (availableChunks) {
3085 oldChunkSizeClass = chunk->sizeClassForAvailableLists();
3086 }
3087
3088 FreeLists* freeLists = getChunkFreeLists(chunk);
3089
3090 // First check whether there is a free region following the allocation.
3091 FreeRegion* region;
3092 uintptr_t endOffset = endAddr & ChunkMask;
3093 if (endOffset == 0 || chunk->isAllocated(endOffset)) {
3094 // The allocation abuts the end of the chunk or another allocation. Add
3095 // the allocation as a new free region.
3096 //
3097 // The new region is added to the front of relevant list so as to reuse
3098 // recently freed memory preferentially. This may reduce fragmentation.
3099 // See "The Memory Fragmentation Problem: Solved?" by Johnstone et al.
3100 region = FreeRegion::create(startAddr, bytes, false);
3101 if (freeLists) {
3102 freeLists->pushFront(region, SizeKind::Medium);
3103 }
3104 } else {
3105 // There is a free region following this allocation. Expand the existing
3106 // region down to cover the newly freed space.
3107 region = chunk->findFollowingFreeRegion(endAddr);
3108 MOZ_ASSERT(region->startAddr == endAddr)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(region->startAddr == endAddr)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(region->startAddr == endAddr
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"region->startAddr == endAddr", "./../../../../js/src/gc/BufferAllocator.cpp"
, 3108); AnnotateMozCrashReason("MOZ_ASSERT" "(" "region->startAddr == endAddr"
")"); do { MOZ_CrashSequence(__null, 3108); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3109 updateFreeRegionStart(freeLists, region, startAddr, SizeKind::Medium);
3110 }
3111
3112 // Next check for any preceding free region and coalesce.
3113 FreeRegion* precRegion = chunk->findPrecedingFreeRegion(startAddr);
3114 if (precRegion) {
3115 if (freeLists) {
3116 size_t sizeClass =
3117 SizeClassForFreeRegion(precRegion->size(), SizeKind::Medium);
3118 freeLists->remove(sizeClass, precRegion);
3119 }
3120
3121 updateFreeRegionStart(freeLists, region, precRegion->startAddr,
3122 SizeKind::Medium);
3123 if (precRegion->hasDecommittedPages) {
3124 region->hasDecommittedPages = true;
3125 }
3126 }
3127
3128 if (availableChunks) {
3129 maybeUpdateAvailableLists(availableChunks, chunk, oldChunkSizeClass);
3130 }
3131}
3132
3133void BufferAllocator::maybeUpdateAvailableLists(ChunkLists* availableChunks,
3134 BufferChunk* chunk,
3135 size_t oldChunkSizeClass) {
3136 // A realloc or free operation can change the amount of free space in an
3137 // available chunk, so we may need to move it to a different list.
3138 size_t newChunkSizeClass = chunk->sizeClassForAvailableLists();
3139 if (newChunkSizeClass != oldChunkSizeClass) {
3140 availableChunks->remove(oldChunkSizeClass, chunk);
3141 availableChunks->addChunk(newChunkSizeClass, chunk);
3142 }
3143}
3144
3145bool BufferAllocator::canModifyAllocations(BufferChunk* chunk) {
3146 // Don't free or resize allocations that may be accessed by concurrent
3147 // marking. Dead allocations will be freed during sweeping.
3148 if (isConcurrentMarking()) {
3149 return false;
3150 }
3151
3152 // We can't change anything if the chunk is currently being swept.
3153 return !isSweepingChunk(chunk);
3154}
3155
3156bool BufferAllocator::isConcurrentMarking() const {
3157#ifdef JS_GC_CONCURRENT_MARKING
3158 return majorState == State::Marking && gc->isConcurrentMarkingEnabled();
3159#else
3160 return false;
3161#endif
3162}
3163
3164bool BufferAllocator::isSweepingChunk(BufferChunk* chunk) {
3165 if (minorState == State::Sweeping && chunk->hasNurseryOwnedAllocs) {
3166 // We are currently sweeping nursery owned allocations.
3167
3168 // TODO: We could set a flag for nursery chunks allocated during minor
3169 // collection to allow operations on chunks that are not being swept here.
3170
3171 if (!hasSweepDataToMerge) {
3172#ifdef DEBUG1
3173 {
3174 AutoLock lock(runtime());
3175 MOZ_ASSERT_IF(!hasSweepDataToMerge, !minorSweepingFinished)do { if (!hasSweepDataToMerge) { do { static_assert( mozilla::
detail::AssertionConditionType<decltype(!minorSweepingFinished
)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(!minorSweepingFinished))), 0))) { do { } while (false
); MOZ_ReportAssertionFailure("!minorSweepingFinished", "./../../../../js/src/gc/BufferAllocator.cpp"
, 3175); AnnotateMozCrashReason("MOZ_ASSERT" "(" "!minorSweepingFinished"
")"); do { MOZ_CrashSequence(__null, 3175); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false);
3176 }
3177#endif
3178
3179 // Likely no data to merge so don't bother taking the lock.
3180 return true;
3181 }
3182
3183 // Merge swept data and recheck.
3184 //
3185 // TODO: It would be good to know how often this helps and if it is
3186 // worthwhile.
3187 mergeSweptData();
3188 if (minorState == State::Sweeping && chunk->hasNurseryOwnedAllocs) {
3189 return true;
3190 }
3191 }
3192
3193 if (majorState == State::Sweeping && !chunk->allocatedDuringCollection) {
3194 // We are currently sweeping tenured owned allocations.
3195 return true;
3196 }
3197
3198 return false;
3199}
3200
3201/* static */
3202BufferAllocator::FreeRegion* BufferAllocator::FreeRegion::create(
3203 uintptr_t start, size_t bytes, bool anyDecommitted,
3204 bool expectUnchanged /* = false */) {
3205 static_assert(sizeof(FreeRegion) <= MinFreeRegionSize);
3206 if (bytes < MinFreeRegionSize) {
3207 // We can't record a region this small. The free space will not be reused
3208 // until enough adjacent space become free.
3209 return nullptr;
3210 }
3211
3212 uintptr_t end = start + bytes;
3213#ifdef DEBUG1
3214 if (expectUnchanged) {
3215 // We didn't free any allocations so there should already be a FreeRegion
3216 // from |start| to |end|.
3217 auto* region = FreeRegion::fromEndAddr(end);
3218 MOZ_ASSERT(region->startAddr == start)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(region->startAddr == start)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(region->startAddr == start
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"region->startAddr == start", "./../../../../js/src/gc/BufferAllocator.cpp"
, 3218); AnnotateMozCrashReason("MOZ_ASSERT" "(" "region->startAddr == start"
")"); do { MOZ_CrashSequence(__null, 3218); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3219 }
3220#endif
3221
3222 void* ptr = reinterpret_cast<void*>(end - sizeof(FreeRegion));
3223 FreeRegion* region = new (ptr) FreeRegion(start, anyDecommitted);
3224 MOZ_ASSERT(region->getEnd() == end)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(region->getEnd() == end)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(region->getEnd() == end))
), 0))) { do { } while (false); MOZ_ReportAssertionFailure("region->getEnd() == end"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3224); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "region->getEnd() == end" ")"); do { MOZ_CrashSequence
(__null, 3224); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
3225
3226 return region;
3227}
3228
3229/* static */
3230inline void BufferAllocator::CheckFreeRegionClass(FreeRegion* region,
3231 size_t sizeClass) {
3232#ifdef DEBUG1
3233 size_t bytes = region->size();
3234 MOZ_ASSERT_IF(sizeClass != MaxMediumAllocClass,do { if (sizeClass != MaxMediumAllocClass) { do { static_assert
( mozilla::detail::AssertionConditionType<decltype(bytes >=
SizeClassBytes(sizeClass))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(bytes >= SizeClassBytes(sizeClass
)))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("bytes >= SizeClassBytes(sizeClass)", "./../../../../js/src/gc/BufferAllocator.cpp"
, 3235); AnnotateMozCrashReason("MOZ_ASSERT" "(" "bytes >= SizeClassBytes(sizeClass)"
")"); do { MOZ_CrashSequence(__null, 3235); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false)
3235 bytes >= SizeClassBytes(sizeClass))do { if (sizeClass != MaxMediumAllocClass) { do { static_assert
( mozilla::detail::AssertionConditionType<decltype(bytes >=
SizeClassBytes(sizeClass))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(bytes >= SizeClassBytes(sizeClass
)))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("bytes >= SizeClassBytes(sizeClass)", "./../../../../js/src/gc/BufferAllocator.cpp"
, 3235); AnnotateMozCrashReason("MOZ_ASSERT" "(" "bytes >= SizeClassBytes(sizeClass)"
")"); do { MOZ_CrashSequence(__null, 3235); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false);
3236 MOZ_ASSERT(region->startAddr % GranularityForSizeClass(sizeClass) == 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(region->startAddr % GranularityForSizeClass(sizeClass
) == 0)>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(region->startAddr % GranularityForSizeClass(sizeClass
) == 0))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("region->startAddr % GranularityForSizeClass(sizeClass) == 0"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3236); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "region->startAddr % GranularityForSizeClass(sizeClass) == 0"
")"); do { MOZ_CrashSequence(__null, 3236); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3237 MOZ_ASSERT(bytes % GranularityForSizeClass(sizeClass) == 0)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(bytes % GranularityForSizeClass(sizeClass) == 0)>
::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(bytes % GranularityForSizeClass(sizeClass) == 0))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("bytes % GranularityForSizeClass(sizeClass) == 0"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3237); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "bytes % GranularityForSizeClass(sizeClass) == 0"
")"); do { MOZ_CrashSequence(__null, 3237); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3238#endif
3239}
3240
3241void BufferAllocator::updateFreeRegionStart(FreeLists* freeLists,
3242 FreeRegion* region,
3243 uintptr_t newStart, SizeKind kind) {
3244 MOZ_ASSERT((newStart & ~ChunkMask) == (uintptr_t(region) & ~ChunkMask))do { static_assert( mozilla::detail::AssertionConditionType<
decltype((newStart & ~ChunkMask) == (uintptr_t(region) &
~ChunkMask))>::isValid, "invalid assertion condition"); if
((__builtin_expect(!!(!(!!((newStart & ~ChunkMask) == (uintptr_t
(region) & ~ChunkMask)))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("(newStart & ~ChunkMask) == (uintptr_t(region) & ~ChunkMask)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3244); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "(newStart & ~ChunkMask) == (uintptr_t(region) & ~ChunkMask)"
")"); do { MOZ_CrashSequence(__null, 3244); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3245 MOZ_ASSERT(region->startAddr != newStart)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(region->startAddr != newStart)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(region->startAddr != newStart
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"region->startAddr != newStart", "./../../../../js/src/gc/BufferAllocator.cpp"
, 3245); AnnotateMozCrashReason("MOZ_ASSERT" "(" "region->startAddr != newStart"
")"); do { MOZ_CrashSequence(__null, 3245); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3246
3247 // TODO: Support realloc for small regions.
3248 MOZ_ASSERT(kind == SizeKind::Medium)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(kind == SizeKind::Medium)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(kind == SizeKind::Medium))),
0))) { do { } while (false); MOZ_ReportAssertionFailure("kind == SizeKind::Medium"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3248); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "kind == SizeKind::Medium" ")"); do { MOZ_CrashSequence
(__null, 3248); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
3249
3250 size_t oldSize = region->size();
3251 region->startAddr = newStart;
3252
3253 if (!freeLists) {
3254 return;
3255 }
3256
3257 size_t currentSizeClass = SizeClassForFreeRegion(oldSize, kind);
3258 size_t newSizeClass = SizeClassForFreeRegion(region->size(), kind);
3259 MOZ_ASSERT(SizeClassKind(newSizeClass) == SizeClassKind(currentSizeClass))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(SizeClassKind(newSizeClass) == SizeClassKind(currentSizeClass
))>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(SizeClassKind(newSizeClass) == SizeClassKind(currentSizeClass
)))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("SizeClassKind(newSizeClass) == SizeClassKind(currentSizeClass)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3259); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "SizeClassKind(newSizeClass) == SizeClassKind(currentSizeClass)"
")"); do { MOZ_CrashSequence(__null, 3259); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3260 if (currentSizeClass != newSizeClass) {
3261 freeLists->remove(currentSizeClass, region);
3262 freeLists->pushFront(newSizeClass, region);
3263 }
3264}
3265
3266bool BufferAllocator::growMedium(void* alloc, size_t newBytes) {
3267 MOZ_ASSERT(!IsSmallAllocSize(newBytes))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!IsSmallAllocSize(newBytes))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!IsSmallAllocSize(newBytes))
)), 0))) { do { } while (false); MOZ_ReportAssertionFailure("!IsSmallAllocSize(newBytes)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3267); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!IsSmallAllocSize(newBytes)" ")"); do { MOZ_CrashSequence
(__null, 3267); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
3268 MOZ_ASSERT(!IsLargeAllocSize(newBytes))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!IsLargeAllocSize(newBytes))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!IsLargeAllocSize(newBytes))
)), 0))) { do { } while (false); MOZ_ReportAssertionFailure("!IsLargeAllocSize(newBytes)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3268); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!IsLargeAllocSize(newBytes)" ")"); do { MOZ_CrashSequence
(__null, 3268); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
3269 newBytes = std::max(newBytes, MinMediumAllocSize);
3270 MOZ_ASSERT(newBytes == GetGoodAllocSize(newBytes))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(newBytes == GetGoodAllocSize(newBytes))>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(newBytes == GetGoodAllocSize(newBytes)))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("newBytes == GetGoodAllocSize(newBytes)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3270); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "newBytes == GetGoodAllocSize(newBytes)" ")"
); do { MOZ_CrashSequence(__null, 3270); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3271
3272 BufferChunk* chunk = BufferChunk::from(alloc);
3273 MOZ_ASSERT(chunk->zone == zone)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(chunk->zone == zone)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(chunk->zone == zone))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("chunk->zone == zone"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3273); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "chunk->zone == zone" ")"); do { MOZ_CrashSequence
(__null, 3273); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
3274
3275 if (!canModifyAllocations(chunk)) {
3276 return false;
3277 }
3278
3279 size_t currentBytes = chunk->allocBytes(alloc);
3280 MOZ_ASSERT(newBytes > currentBytes)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(newBytes > currentBytes)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(newBytes > currentBytes))
), 0))) { do { } while (false); MOZ_ReportAssertionFailure("newBytes > currentBytes"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3280); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "newBytes > currentBytes" ")"); do { MOZ_CrashSequence
(__null, 3280); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
3281
3282 uintptr_t startOffset = uintptr_t(alloc) & ChunkMask;
3283 uintptr_t endOffset = startOffset + currentBytes;
3284 MOZ_ASSERT(endOffset <= ChunkSize)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(endOffset <= ChunkSize)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(endOffset <= ChunkSize)))
, 0))) { do { } while (false); MOZ_ReportAssertionFailure("endOffset <= ChunkSize"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3284); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "endOffset <= ChunkSize" ")"); do { MOZ_CrashSequence
(__null, 3284); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
3285
3286 uintptr_t newEndOffset = startOffset + newBytes;
3287 if (newEndOffset > ChunkSize) {
3288 return false; // Can't extend because there's not enough room in a chunk.
3289 }
3290
3291 uintptr_t nextAllocOffset = chunk->findNextAllocated(endOffset);
3292 if (nextAllocOffset < newEndOffset) {
3293 return false; // Can't extend because another allocation is in the way.
3294 }
3295
3296 FreeRegion* region =
3297 FreeRegion::fromEndAddr(chunk->startAddress() + nextAllocOffset);
3298 MOZ_ASSERT((region->startAddr & ChunkMask) == endOffset)do { static_assert( mozilla::detail::AssertionConditionType<
decltype((region->startAddr & ChunkMask) == endOffset)
>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!((region->startAddr & ChunkMask) == endOffset)
)), 0))) { do { } while (false); MOZ_ReportAssertionFailure("(region->startAddr & ChunkMask) == endOffset"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3298); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "(region->startAddr & ChunkMask) == endOffset"
")"); do { MOZ_CrashSequence(__null, 3298); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3299 size_t extraBytes = newBytes - currentBytes;
3300 size_t sizeClass = SizeClassForFreeRegion(region->size(), SizeKind::Medium);
3301 allocFromRegion(region, extraBytes, sizeClass);
3302
3303 // If the chunk is in one of the available lists we may need to move it if the
3304 // largest free region has shrunk too much.
3305 ChunkLists* availableChunks = getChunkAvailableLists(chunk);
3306 size_t oldChunkSizeClass = SIZE_MAX(18446744073709551615UL);
3307 if (availableChunks) {
3308 oldChunkSizeClass = chunk->sizeClassForAvailableLists();
3309 }
3310
3311 if (FreeLists* freeLists = getChunkFreeLists(chunk)) {
3312 updateFreeListsAfterAlloc(freeLists, region, sizeClass);
3313 }
3314
3315 if (availableChunks) {
3316 maybeUpdateAvailableLists(availableChunks, chunk, oldChunkSizeClass);
3317 }
3318
3319 chunk->updateEndOffset(alloc, currentBytes, newBytes);
3320 MOZ_ASSERT(chunk->allocBytes(alloc) == newBytes)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(chunk->allocBytes(alloc) == newBytes)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(chunk->allocBytes(alloc) == newBytes))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("chunk->allocBytes(alloc) == newBytes"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3320); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "chunk->allocBytes(alloc) == newBytes" ")"
); do { MOZ_CrashSequence(__null, 3320); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3321
3322 bool updateRetained =
3323 majorState == State::Marking && !chunk->allocatedDuringCollection;
3324 increaseHeapSize(extraBytes, chunk->isNurseryOwned(alloc), true,
3325 updateRetained);
3326
3327 return true;
3328}
3329
3330bool BufferAllocator::shrinkMedium(void* alloc, size_t newBytes) {
3331 MOZ_ASSERT(!IsSmallAllocSize(newBytes))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!IsSmallAllocSize(newBytes))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!IsSmallAllocSize(newBytes))
)), 0))) { do { } while (false); MOZ_ReportAssertionFailure("!IsSmallAllocSize(newBytes)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3331); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!IsSmallAllocSize(newBytes)" ")"); do { MOZ_CrashSequence
(__null, 3331); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
3332 MOZ_ASSERT(!IsLargeAllocSize(newBytes))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!IsLargeAllocSize(newBytes))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!IsLargeAllocSize(newBytes))
)), 0))) { do { } while (false); MOZ_ReportAssertionFailure("!IsLargeAllocSize(newBytes)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3332); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!IsLargeAllocSize(newBytes)" ")"); do { MOZ_CrashSequence
(__null, 3332); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
3333 newBytes = std::max(newBytes, MinMediumAllocSize);
3334 MOZ_ASSERT(newBytes == GetGoodAllocSize(newBytes))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(newBytes == GetGoodAllocSize(newBytes))>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(newBytes == GetGoodAllocSize(newBytes)))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("newBytes == GetGoodAllocSize(newBytes)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3334); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "newBytes == GetGoodAllocSize(newBytes)" ")"
); do { MOZ_CrashSequence(__null, 3334); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3335
3336 BufferChunk* chunk = BufferChunk::from(alloc);
3337 MOZ_ASSERT(chunk->zone == zone)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(chunk->zone == zone)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(chunk->zone == zone))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("chunk->zone == zone"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3337); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "chunk->zone == zone" ")"); do { MOZ_CrashSequence
(__null, 3337); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
3338
3339 if (!canModifyAllocations(chunk)) {
3340 return false;
3341 }
3342
3343 size_t currentBytes = chunk->allocBytes(alloc);
3344 if (newBytes == currentBytes) {
3345 // Requested size is the same after adjusting to a valid medium alloc size.
3346 return false;
3347 }
3348
3349 MOZ_ASSERT(newBytes < currentBytes)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(newBytes < currentBytes)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(newBytes < currentBytes))
), 0))) { do { } while (false); MOZ_ReportAssertionFailure("newBytes < currentBytes"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3349); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "newBytes < currentBytes" ")"); do { MOZ_CrashSequence
(__null, 3349); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
3350 size_t sizeChange = currentBytes - newBytes;
3351
3352 // Update allocation size.
3353 chunk->updateEndOffset(alloc, currentBytes, newBytes);
3354 MOZ_ASSERT(chunk->allocBytes(alloc) == newBytes)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(chunk->allocBytes(alloc) == newBytes)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(chunk->allocBytes(alloc) == newBytes))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("chunk->allocBytes(alloc) == newBytes"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3354); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "chunk->allocBytes(alloc) == newBytes" ")"
); do { MOZ_CrashSequence(__null, 3354); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3355 bool updateRetained =
3356 majorState == State::Marking && !chunk->allocatedDuringCollection;
3357 decreaseHeapSize(sizeChange, chunk->isNurseryOwned(alloc), updateRetained);
3358
3359 uintptr_t startOffset = uintptr_t(alloc) & ChunkMask;
3360 uintptr_t oldEndOffset = startOffset + currentBytes;
3361 uintptr_t newEndOffset = startOffset + newBytes;
3362 MOZ_ASSERT(oldEndOffset <= ChunkSize)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(oldEndOffset <= ChunkSize)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(oldEndOffset <= ChunkSize
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"oldEndOffset <= ChunkSize", "./../../../../js/src/gc/BufferAllocator.cpp"
, 3362); AnnotateMozCrashReason("MOZ_ASSERT" "(" "oldEndOffset <= ChunkSize"
")"); do { MOZ_CrashSequence(__null, 3362); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3363
3364 // Poison freed memory.
3365 uintptr_t chunkAddr = uintptr_t(chunk);
3366 PoisonAlloc(reinterpret_cast<void*>(chunkAddr + newEndOffset),
3367 JS_SWEPT_TENURED_PATTERN, sizeChange,
3368 MemCheckKind::MakeUndefined);
3369
3370 // If the chunk is in one of the available lists we may need to move it.
3371 ChunkLists* availableChunks = getChunkAvailableLists(chunk);
3372 size_t oldChunkSizeClass = SIZE_MAX(18446744073709551615UL);
3373 if (availableChunks) {
3374 oldChunkSizeClass = chunk->sizeClassForAvailableLists();
3375 }
3376
3377 FreeLists* freeLists = getChunkFreeLists(chunk);
3378 if (oldEndOffset == ChunkSize || chunk->isAllocated(oldEndOffset)) {
3379 // If we abut another allocation then add a new free region.
3380 uintptr_t freeStart = chunkAddr + newEndOffset;
3381 FreeRegion* region = FreeRegion::create(freeStart, sizeChange, false);
3382 if (freeLists) {
3383 freeLists->pushFront(region, SizeKind::Medium);
3384 }
3385 } else {
3386 // Otherwise find the following free region and extend it down.
3387 FreeRegion* region =
3388 chunk->findFollowingFreeRegion(chunkAddr + oldEndOffset);
3389 MOZ_ASSERT(region->startAddr == chunkAddr + oldEndOffset)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(region->startAddr == chunkAddr + oldEndOffset)>
::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(region->startAddr == chunkAddr + oldEndOffset))),
0))) { do { } while (false); MOZ_ReportAssertionFailure("region->startAddr == chunkAddr + oldEndOffset"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3389); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "region->startAddr == chunkAddr + oldEndOffset"
")"); do { MOZ_CrashSequence(__null, 3389); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3390 updateFreeRegionStart(freeLists, region, chunkAddr + newEndOffset,
3391 SizeKind::Medium);
3392 }
3393
3394 if (availableChunks) {
3395 maybeUpdateAvailableLists(availableChunks, chunk, oldChunkSizeClass);
3396 }
3397
3398 return true;
3399}
3400
3401BufferAllocator::FreeLists* BufferAllocator::getChunkFreeLists(
3402 BufferChunk* chunk) {
3403 MOZ_ASSERT_IF(majorState == State::Sweeping,do { if (majorState == State::Sweeping) { do { static_assert(
mozilla::detail::AssertionConditionType<decltype(chunk->
allocatedDuringCollection)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(chunk->allocatedDuringCollection
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"chunk->allocatedDuringCollection", "./../../../../js/src/gc/BufferAllocator.cpp"
, 3404); AnnotateMozCrashReason("MOZ_ASSERT" "(" "chunk->allocatedDuringCollection"
")"); do { MOZ_CrashSequence(__null, 3404); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false)
3404 chunk->allocatedDuringCollection)do { if (majorState == State::Sweeping) { do { static_assert(
mozilla::detail::AssertionConditionType<decltype(chunk->
allocatedDuringCollection)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(chunk->allocatedDuringCollection
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"chunk->allocatedDuringCollection", "./../../../../js/src/gc/BufferAllocator.cpp"
, 3404); AnnotateMozCrashReason("MOZ_ASSERT" "(" "chunk->allocatedDuringCollection"
")"); do { MOZ_CrashSequence(__null, 3404); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false);
3405
3406 if (chunk->ownsFreeLists) {
3407 // The chunk is in one of the available lists.
3408 return &chunk->freeLists.ref();
3409 }
3410
3411 if (majorState == State::Marking && !chunk->allocatedDuringCollection) {
3412 // The chunk is queued for sweeping and the free lists with be rebuilt.
3413 return nullptr;
3414 }
3415
3416 if (chunk->kind() == ContentKind::Mixed) {
3417 return &mixedFreeLists.ref();
3418 }
3419
3420 return &tenuredFreeLists.ref();
3421}
3422
3423BufferAllocator::ChunkLists* BufferAllocator::getChunkAvailableLists(
3424 BufferChunk* chunk) {
3425 MOZ_ASSERT_IF(majorState == State::Sweeping,do { if (majorState == State::Sweeping) { do { static_assert(
mozilla::detail::AssertionConditionType<decltype(chunk->
allocatedDuringCollection)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(chunk->allocatedDuringCollection
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"chunk->allocatedDuringCollection", "./../../../../js/src/gc/BufferAllocator.cpp"
, 3426); AnnotateMozCrashReason("MOZ_ASSERT" "(" "chunk->allocatedDuringCollection"
")"); do { MOZ_CrashSequence(__null, 3426); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false)
3426 chunk->allocatedDuringCollection)do { if (majorState == State::Sweeping) { do { static_assert(
mozilla::detail::AssertionConditionType<decltype(chunk->
allocatedDuringCollection)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(chunk->allocatedDuringCollection
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"chunk->allocatedDuringCollection", "./../../../../js/src/gc/BufferAllocator.cpp"
, 3426); AnnotateMozCrashReason("MOZ_ASSERT" "(" "chunk->allocatedDuringCollection"
")"); do { MOZ_CrashSequence(__null, 3426); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false);
3427
3428 if (!chunk->ownsFreeLists) {
3429 return nullptr; // Chunk is not in the available lists.
3430 }
3431
3432 if (majorState == State::Marking && !chunk->allocatedDuringCollection) {
3433 return nullptr; // Chunk is waiting to be swept.
3434 }
3435
3436 return &availableChunks.ref();
3437}
3438
3439/* static */
3440size_t BufferAllocator::SizeClassForSmallAlloc(size_t bytes) {
3441 MOZ_ASSERT(bytes >= MinSmallAllocSize)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(bytes >= MinSmallAllocSize)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(bytes >= MinSmallAllocSize
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"bytes >= MinSmallAllocSize", "./../../../../js/src/gc/BufferAllocator.cpp"
, 3441); AnnotateMozCrashReason("MOZ_ASSERT" "(" "bytes >= MinSmallAllocSize"
")"); do { MOZ_CrashSequence(__null, 3441); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3442 MOZ_ASSERT(bytes <= MaxSmallAllocSize)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(bytes <= MaxSmallAllocSize)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(bytes <= MaxSmallAllocSize
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"bytes <= MaxSmallAllocSize", "./../../../../js/src/gc/BufferAllocator.cpp"
, 3442); AnnotateMozCrashReason("MOZ_ASSERT" "(" "bytes <= MaxSmallAllocSize"
")"); do { MOZ_CrashSequence(__null, 3442); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3443
3444 size_t log2Size = mozilla::CeilingLog2(bytes);
3445 MOZ_ASSERT((size_t(1) << log2Size) >= bytes)do { static_assert( mozilla::detail::AssertionConditionType<
decltype((size_t(1) << log2Size) >= bytes)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!((size_t(1) << log2Size) >= bytes))), 0))) { do { }
while (false); MOZ_ReportAssertionFailure("(size_t(1) << log2Size) >= bytes"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3445); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "(size_t(1) << log2Size) >= bytes"
")"); do { MOZ_CrashSequence(__null, 3445); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3446 MOZ_ASSERT(MinSizeClassShift == mozilla::CeilingLog2(MinFreeRegionSize))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(MinSizeClassShift == mozilla::CeilingLog2(MinFreeRegionSize
))>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(MinSizeClassShift == mozilla::CeilingLog2(MinFreeRegionSize
)))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("MinSizeClassShift == mozilla::CeilingLog2(MinFreeRegionSize)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3446); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "MinSizeClassShift == mozilla::CeilingLog2(MinFreeRegionSize)"
")"); do { MOZ_CrashSequence(__null, 3446); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3447 if (log2Size < MinSizeClassShift) {
3448 return 0;
3449 }
3450
3451 size_t sizeClass = log2Size - MinSizeClassShift;
3452 MOZ_ASSERT(sizeClass <= MaxSmallAllocClass)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(sizeClass <= MaxSmallAllocClass)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(sizeClass <= MaxSmallAllocClass
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"sizeClass <= MaxSmallAllocClass", "./../../../../js/src/gc/BufferAllocator.cpp"
, 3452); AnnotateMozCrashReason("MOZ_ASSERT" "(" "sizeClass <= MaxSmallAllocClass"
")"); do { MOZ_CrashSequence(__null, 3452); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3453 return sizeClass;
3454}
3455
3456/* static */
3457size_t BufferAllocator::SizeClassForMediumAlloc(size_t bytes) {
3458 MOZ_ASSERT(bytes >= MinMediumAllocSize)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(bytes >= MinMediumAllocSize)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(bytes >= MinMediumAllocSize
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"bytes >= MinMediumAllocSize", "./../../../../js/src/gc/BufferAllocator.cpp"
, 3458); AnnotateMozCrashReason("MOZ_ASSERT" "(" "bytes >= MinMediumAllocSize"
")"); do { MOZ_CrashSequence(__null, 3458); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3459 MOZ_ASSERT(bytes <= MaxMediumAllocSize)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(bytes <= MaxMediumAllocSize)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(bytes <= MaxMediumAllocSize
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"bytes <= MaxMediumAllocSize", "./../../../../js/src/gc/BufferAllocator.cpp"
, 3459); AnnotateMozCrashReason("MOZ_ASSERT" "(" "bytes <= MaxMediumAllocSize"
")"); do { MOZ_CrashSequence(__null, 3459); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3460
3461 size_t log2Size = mozilla::CeilingLog2(bytes);
3462 MOZ_ASSERT((size_t(1) << log2Size) >= bytes)do { static_assert( mozilla::detail::AssertionConditionType<
decltype((size_t(1) << log2Size) >= bytes)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!((size_t(1) << log2Size) >= bytes))), 0))) { do { }
while (false); MOZ_ReportAssertionFailure("(size_t(1) << log2Size) >= bytes"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3462); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "(size_t(1) << log2Size) >= bytes"
")"); do { MOZ_CrashSequence(__null, 3462); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3463
3464 MOZ_ASSERT(log2Size >= MinMediumAllocShift)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(log2Size >= MinMediumAllocShift)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(log2Size >= MinMediumAllocShift
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"log2Size >= MinMediumAllocShift", "./../../../../js/src/gc/BufferAllocator.cpp"
, 3464); AnnotateMozCrashReason("MOZ_ASSERT" "(" "log2Size >= MinMediumAllocShift"
")"); do { MOZ_CrashSequence(__null, 3464); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3465 size_t sizeClass = log2Size - MinMediumAllocShift + MinMediumAllocClass;
3466
3467 MOZ_ASSERT(sizeClass >= MinMediumAllocClass)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(sizeClass >= MinMediumAllocClass)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(sizeClass >= MinMediumAllocClass
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"sizeClass >= MinMediumAllocClass", "./../../../../js/src/gc/BufferAllocator.cpp"
, 3467); AnnotateMozCrashReason("MOZ_ASSERT" "(" "sizeClass >= MinMediumAllocClass"
")"); do { MOZ_CrashSequence(__null, 3467); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3468 MOZ_ASSERT(sizeClass < AllocSizeClasses)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(sizeClass < AllocSizeClasses)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(sizeClass < AllocSizeClasses
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"sizeClass < AllocSizeClasses", "./../../../../js/src/gc/BufferAllocator.cpp"
, 3468); AnnotateMozCrashReason("MOZ_ASSERT" "(" "sizeClass < AllocSizeClasses"
")"); do { MOZ_CrashSequence(__null, 3468); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3469 return sizeClass;
3470}
3471
3472/* static */
3473size_t BufferAllocator::SizeClassForFreeRegion(size_t bytes, SizeKind kind) {
3474 MOZ_ASSERT(bytes >= MinFreeRegionSize)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(bytes >= MinFreeRegionSize)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(bytes >= MinFreeRegionSize
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"bytes >= MinFreeRegionSize", "./../../../../js/src/gc/BufferAllocator.cpp"
, 3474); AnnotateMozCrashReason("MOZ_ASSERT" "(" "bytes >= MinFreeRegionSize"
")"); do { MOZ_CrashSequence(__null, 3474); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3475 MOZ_ASSERT(bytes < ChunkSize)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(bytes < ChunkSize)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(bytes < ChunkSize))), 0))
) { do { } while (false); MOZ_ReportAssertionFailure("bytes < ChunkSize"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3475); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "bytes < ChunkSize" ")"); do { MOZ_CrashSequence
(__null, 3475); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
3476
3477 if (kind == SizeKind::Medium && bytes >= MaxMediumAllocSize) {
3478 // Free regions large enough for MaxMediumAllocSize don't have to have
3479 // enough space for that size rounded up to the next power of two, as is the
3480 // case for smaller regions.
3481 return MaxMediumAllocClass;
3482 }
3483
3484 size_t log2Size = mozilla::FloorLog2(bytes);
3485 MOZ_ASSERT((size_t(1) << log2Size) <= bytes)do { static_assert( mozilla::detail::AssertionConditionType<
decltype((size_t(1) << log2Size) <= bytes)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!((size_t(1) << log2Size) <= bytes))), 0))) { do { }
while (false); MOZ_ReportAssertionFailure("(size_t(1) << log2Size) <= bytes"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3485); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "(size_t(1) << log2Size) <= bytes"
")"); do { MOZ_CrashSequence(__null, 3485); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3486 MOZ_ASSERT(log2Size >= MinSizeClassShift)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(log2Size >= MinSizeClassShift)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(log2Size >= MinSizeClassShift
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"log2Size >= MinSizeClassShift", "./../../../../js/src/gc/BufferAllocator.cpp"
, 3486); AnnotateMozCrashReason("MOZ_ASSERT" "(" "log2Size >= MinSizeClassShift"
")"); do { MOZ_CrashSequence(__null, 3486); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3487 size_t sizeClass =
3488 std::min(log2Size - MinSizeClassShift, AllocSizeClasses - 1);
3489
3490 if (kind == SizeKind::Small) {
3491 return std::min(sizeClass, MaxSmallAllocClass);
3492 }
3493
3494 sizeClass++; // Medium size classes start after small ones.
3495
3496 MOZ_ASSERT(sizeClass >= MinMediumAllocClass)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(sizeClass >= MinMediumAllocClass)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(sizeClass >= MinMediumAllocClass
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"sizeClass >= MinMediumAllocClass", "./../../../../js/src/gc/BufferAllocator.cpp"
, 3496); AnnotateMozCrashReason("MOZ_ASSERT" "(" "sizeClass >= MinMediumAllocClass"
")"); do { MOZ_CrashSequence(__null, 3496); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3497 MOZ_ASSERT(sizeClass < AllocSizeClasses)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(sizeClass < AllocSizeClasses)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(sizeClass < AllocSizeClasses
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"sizeClass < AllocSizeClasses", "./../../../../js/src/gc/BufferAllocator.cpp"
, 3497); AnnotateMozCrashReason("MOZ_ASSERT" "(" "sizeClass < AllocSizeClasses"
")"); do { MOZ_CrashSequence(__null, 3497); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3498 return sizeClass;
3499}
3500
3501/* static */
3502inline size_t BufferAllocator::SizeClassBytes(size_t sizeClass) {
3503 MOZ_ASSERT(sizeClass < AllocSizeClasses)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(sizeClass < AllocSizeClasses)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(sizeClass < AllocSizeClasses
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"sizeClass < AllocSizeClasses", "./../../../../js/src/gc/BufferAllocator.cpp"
, 3503); AnnotateMozCrashReason("MOZ_ASSERT" "(" "sizeClass < AllocSizeClasses"
")"); do { MOZ_CrashSequence(__null, 3503); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3504
3505 // The first medium size class is the same size as the last small size class.
3506 if (sizeClass >= MinMediumAllocClass) {
3507 sizeClass--;
3508 }
3509
3510 return 1 << (sizeClass + MinSizeClassShift);
3511}
3512
3513/* static */
3514bool BufferAllocator::IsMediumAlloc(void* alloc) {
3515 MOZ_ASSERT(IsBufferAlloc(alloc))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(IsBufferAlloc(alloc))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(IsBufferAlloc(alloc)))), 0))
) { do { } while (false); MOZ_ReportAssertionFailure("IsBufferAlloc(alloc)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3515); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "IsBufferAlloc(alloc)" ")"); do { MOZ_CrashSequence
(__null, 3515); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
3516
3517 // Test for large buffers before calling this so we can assume |alloc| is
3518 // inside a chunk.
3519 MOZ_ASSERT(!IsLargeAlloc(alloc))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!IsLargeAlloc(alloc))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!IsLargeAlloc(alloc)))), 0))
) { do { } while (false); MOZ_ReportAssertionFailure("!IsLargeAlloc(alloc)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3519); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!IsLargeAlloc(alloc)" ")"); do { MOZ_CrashSequence
(__null, 3519); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
3520
3521 BufferChunk* chunk = BufferChunk::from(alloc);
3522 return !chunk->isSmallBufferRegion(alloc);
3523}
3524
3525bool BufferAllocator::needLockToAccessBufferMap() const {
3526 MOZ_ASSERT(CurrentThreadCanAccessZone(zone) || CurrentThreadIsPerformingGC())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(CurrentThreadCanAccessZone(zone) || CurrentThreadIsPerformingGC
())>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(CurrentThreadCanAccessZone(zone) || CurrentThreadIsPerformingGC
()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("CurrentThreadCanAccessZone(zone) || CurrentThreadIsPerformingGC()"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3526); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "CurrentThreadCanAccessZone(zone) || CurrentThreadIsPerformingGC()"
")"); do { MOZ_CrashSequence(__null, 3526); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3527 return runtime()->needLockToAccessBufferMap();
3528}
3529
3530LargeBuffer* BufferAllocator::lookupLargeBuffer(void* alloc) {
3531 MaybeLock lock;
3532 return lookupLargeBuffer(alloc, lock);
3533}
3534
3535LargeBuffer* BufferAllocator::lookupLargeBuffer(void* alloc, MaybeLock& lock) {
3536 LargeBuffer* buffer = runtime()->lookupLargeBuffer(alloc, lock);
3537 MOZ_ASSERT(buffer->zoneFromAnyThread() == zone)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(buffer->zoneFromAnyThread() == zone)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(buffer->zoneFromAnyThread() == zone))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("buffer->zoneFromAnyThread() == zone"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3537); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "buffer->zoneFromAnyThread() == zone" ")"
); do { MOZ_CrashSequence(__null, 3537); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3538 return buffer;
3539}
3540
3541void* BufferAllocator::allocLarge(size_t requestedBytes, bool nurseryOwned,
3542 bool inGC) {
3543 size_t bytes = RoundUp(requestedBytes, ChunkSize);
3544 if (MOZ_UNLIKELY(bytes < requestedBytes)(__builtin_expect(!!(bytes < requestedBytes), 0))) {
3545 return nullptr;
3546 }
3547 MOZ_ASSERT(bytes > MaxMediumAllocSize)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(bytes > MaxMediumAllocSize)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(bytes > MaxMediumAllocSize
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"bytes > MaxMediumAllocSize", "./../../../../js/src/gc/BufferAllocator.cpp"
, 3547); AnnotateMozCrashReason("MOZ_ASSERT" "(" "bytes > MaxMediumAllocSize"
")"); do { MOZ_CrashSequence(__null, 3547); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3548
3549 // Allocate a small buffer the size of a LargeBuffer to hold the metadata.
3550 static_assert(sizeof(LargeBuffer) <= MaxSmallAllocSize);
3551 void* bufferPtr = allocSmall(sizeof(LargeBuffer), nurseryOwned, inGC);
3552 if (!bufferPtr) {
3553 return nullptr;
3554 }
3555
3556 // Large allocations are aligned to the chunk size, even if they are smaller
3557 // than a chunk. This allows us to tell large buffer allocations apart by
3558 // looking at the pointer alignment.
3559 void* alloc = MapAlignedPages(bytes, ChunkSize, ShouldStallAndRetry(inGC));
3560 if (!alloc) {
3561 return nullptr;
3562 }
3563 auto freeGuard = mozilla::MakeScopeExit([&]() { UnmapPages(alloc, bytes); });
3564
3565 CheckHighBitsOfPointer(alloc);
3566
3567 auto* buffer = new (bufferPtr) LargeBuffer(alloc, bytes, nurseryOwned);
3568
3569 {
3570 MaybeLock lock;
3571 if (needLockToAccessBufferMap()) {
3572 lock.emplace(runtime());
3573 }
3574 if (!runtime()->largeAllocMap.ref().putNew(alloc, buffer)) {
3575 return nullptr;
3576 }
3577 }
3578
3579 freeGuard.release();
3580
3581 if (nurseryOwned) {
3582 largeNurseryAllocs.ref().pushBack(buffer);
3583 } else {
3584 buffer->allocatedDuringCollection = majorState != State::NotCollecting;
3585 largeTenuredAllocs.ref().pushBack(buffer);
3586 }
3587
3588 // Update memory accounting and trigger an incremental slice if needed.
3589 bool checkThresholds = !inGC;
3590 increaseHeapSize(bytes, nurseryOwned, checkThresholds, false);
3591
3592 MOZ_ASSERT(IsLargeAlloc(alloc))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(IsLargeAlloc(alloc))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(IsLargeAlloc(alloc)))), 0)))
{ do { } while (false); MOZ_ReportAssertionFailure("IsLargeAlloc(alloc)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3592); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "IsLargeAlloc(alloc)" ")"); do { MOZ_CrashSequence
(__null, 3592); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
3593 return alloc;
3594}
3595
3596void BufferAllocator::increaseHeapSize(size_t bytes, bool nurseryOwned,
3597 bool checkThresholds,
3598 bool updateRetainedSize) {
3599 // Update memory accounting and trigger an incremental slice if needed.
3600 // TODO: This will eventually be attributed to gcHeapSize.
3601 if (nurseryOwned) {
3602 gc->nursery().addMallocedBufferBytes(bytes);
3603 } else {
3604 zone->mallocHeapSize.addBytes(bytes, updateRetainedSize);
3605 if (checkThresholds) {
3606 gc->maybeTriggerGCAfterMalloc(zone);
3607 }
3608 }
3609}
3610
3611void BufferAllocator::decreaseHeapSize(size_t bytes, bool nurseryOwned,
3612 bool updateRetainedSize) {
3613 if (nurseryOwned) {
3614 gc->nursery().removeMallocedBufferBytes(bytes);
3615 } else {
3616 zone->mallocHeapSize.removeBytes(bytes, updateRetainedSize);
3617 }
3618}
3619
3620/* static */
3621bool BufferAllocator::IsLargeAlloc(void* alloc) {
3622 return (uintptr_t(alloc) & ChunkMask) == 0;
3623}
3624
3625bool BufferAllocator::markLargeTenuredBuffer(LargeBuffer* buffer) {
3626 MOZ_ASSERT(!buffer->isNurseryOwned)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!buffer->isNurseryOwned)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!buffer->isNurseryOwned))
), 0))) { do { } while (false); MOZ_ReportAssertionFailure("!buffer->isNurseryOwned"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3626); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!buffer->isNurseryOwned" ")"); do { MOZ_CrashSequence
(__null, 3626); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
3627
3628 if (buffer->allocatedDuringCollection) {
3629 return false;
3630 }
3631
3632 // Also mark the allocation containing the LargeBuffer.
3633 markSmallTenuredAlloc(buffer);
3634
3635 return buffer->isMarked.compareExchange(false, true);
3636}
3637
3638bool BufferAllocator::isLargeTenuredMarked(LargeBuffer* buffer) {
3639 MOZ_ASSERT(!buffer->isNurseryOwned)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!buffer->isNurseryOwned)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!buffer->isNurseryOwned))
), 0))) { do { } while (false); MOZ_ReportAssertionFailure("!buffer->isNurseryOwned"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3639); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!buffer->isNurseryOwned" ")"); do { MOZ_CrashSequence
(__null, 3639); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
3640 MOZ_ASSERT(buffer->zoneFromAnyThread() == zone)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(buffer->zoneFromAnyThread() == zone)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(buffer->zoneFromAnyThread() == zone))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("buffer->zoneFromAnyThread() == zone"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3640); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "buffer->zoneFromAnyThread() == zone" ")"
); do { MOZ_CrashSequence(__null, 3640); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3641 MOZ_ASSERT(!buffer->isInList())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!buffer->isInList())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!buffer->isInList()))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("!buffer->isInList()"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3641); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!buffer->isInList()" ")"); do { MOZ_CrashSequence
(__null, 3641); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
3642 return buffer->isMarked;
3643}
3644
3645void BufferAllocator::freeLarge(void* alloc) {
3646 MaybeLock lock;
3647 LargeBuffer* buffer = lookupLargeBuffer(alloc, lock);
3648 MOZ_ASSERT(buffer->zoneFromAnyThread() == zone)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(buffer->zoneFromAnyThread() == zone)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(buffer->zoneFromAnyThread() == zone))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("buffer->zoneFromAnyThread() == zone"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3648); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "buffer->zoneFromAnyThread() == zone" ")"
); do { MOZ_CrashSequence(__null, 3648); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3649
3650 // Don't free data that may be accessed by concurrent marking.
3651 if (isConcurrentMarking()) {
3652 return;
3653 }
3654
3655 DebugOnlyPoison(alloc, JS_FREED_BUFFER_PATTERN, buffer->allocBytes(),
3656 MemCheckKind::MakeUndefined);
3657
3658 if (!buffer->isNurseryOwned && majorState == State::Sweeping &&
3659 !buffer->allocatedDuringCollection) {
3660 return; // Large allocations are currently being swept.
3661 }
3662
3663 MOZ_ASSERT(buffer->isInList())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(buffer->isInList())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(buffer->isInList()))), 0)
)) { do { } while (false); MOZ_ReportAssertionFailure("buffer->isInList()"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3663); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "buffer->isInList()" ")"); do { MOZ_CrashSequence
(__null, 3663); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
3664
3665 if (buffer->isNurseryOwned) {
3666 largeNurseryAllocs.ref().remove(buffer);
3667 } else if (majorState == State::Marking &&
3668 !buffer->allocatedDuringCollection) {
3669 largeTenuredAllocsToSweep.ref().remove(buffer);
3670 } else {
3671 largeTenuredAllocs.ref().remove(buffer);
3672 }
3673
3674 unmapLarge(buffer, false, lock);
3675}
3676
3677bool BufferAllocator::shrinkLarge(LargeBuffer* buffer, size_t newBytes) {
3678 MOZ_ASSERT(IsLargeAllocSize(newBytes))do { static_assert( mozilla::detail::AssertionConditionType<
decltype(IsLargeAllocSize(newBytes))>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(IsLargeAllocSize(newBytes)))
), 0))) { do { } while (false); MOZ_ReportAssertionFailure("IsLargeAllocSize(newBytes)"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3678); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "IsLargeAllocSize(newBytes)" ")"); do { MOZ_CrashSequence
(__null, 3678); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
3679#ifdef XP_WIN
3680 // Can't unmap part of a region mapped with VirtualAlloc on Windows.
3681 //
3682 // It is possible to decommit the physical pages so we could do that and
3683 // track virtual size as well as committed size. This would also allow us to
3684 // grow the allocation again if necessary.
3685 return false;
3686#else
3687 MOZ_ASSERT(buffer->zone() == zone)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(buffer->zone() == zone)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(buffer->zone() == zone)))
, 0))) { do { } while (false); MOZ_ReportAssertionFailure("buffer->zone() == zone"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3687); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "buffer->zone() == zone" ")"); do { MOZ_CrashSequence
(__null, 3687); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
3688
3689 // Don't free data that may be accessed by concurrent marking.
3690 if (isConcurrentMarking()) {
3691 return false;
3692 }
3693
3694 if (!buffer->isNurseryOwned && majorState == State::Sweeping &&
3695 !buffer->allocatedDuringCollection) {
3696 return false; // Large allocations are currently being swept.
3697 }
3698
3699 MOZ_ASSERT(buffer->isInList())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(buffer->isInList())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(buffer->isInList()))), 0)
)) { do { } while (false); MOZ_ReportAssertionFailure("buffer->isInList()"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3699); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "buffer->isInList()" ")"); do { MOZ_CrashSequence
(__null, 3699); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
3700
3701 newBytes = RoundUp(newBytes, ChunkSize);
3702 size_t oldBytes = buffer->bytes;
3703 MOZ_ASSERT(oldBytes > newBytes)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(oldBytes > newBytes)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(oldBytes > newBytes))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("oldBytes > newBytes"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3703); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "oldBytes > newBytes" ")"); do { MOZ_CrashSequence
(__null, 3703); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
3704 size_t shrinkBytes = oldBytes - newBytes;
3705
3706 decreaseHeapSize(shrinkBytes, buffer->isNurseryOwned, false);
3707
3708 buffer->bytes = newBytes;
3709
3710 void* endPtr = reinterpret_cast<void*>(uintptr_t(buffer->data()) + newBytes);
3711 UnmapPages(endPtr, shrinkBytes);
3712
3713 return true;
3714#endif
3715}
3716
3717void BufferAllocator::unmapLarge(LargeBuffer* buffer, bool isSweeping,
3718 MaybeLock& lock) {
3719 unregisterLarge(buffer, isSweeping, lock);
3720 UnmapPages(buffer->data(), buffer->bytes);
3721}
3722
3723void BufferAllocator::unregisterLarge(LargeBuffer* buffer, bool isSweeping,
3724 MaybeLock& lock) {
3725 MOZ_ASSERT(buffer->zoneFromAnyThread() == zone)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(buffer->zoneFromAnyThread() == zone)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(buffer->zoneFromAnyThread() == zone))), 0))) { do { } while
(false); MOZ_ReportAssertionFailure("buffer->zoneFromAnyThread() == zone"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3725); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "buffer->zoneFromAnyThread() == zone" ")"
); do { MOZ_CrashSequence(__null, 3725); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3726 MOZ_ASSERT(!buffer->isInList())do { static_assert( mozilla::detail::AssertionConditionType<
decltype(!buffer->isInList())>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(!buffer->isInList()))), 0
))) { do { } while (false); MOZ_ReportAssertionFailure("!buffer->isInList()"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3726); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "!buffer->isInList()" ")"); do { MOZ_CrashSequence
(__null, 3726); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
3727 MOZ_ASSERT_IF(isSweeping || needLockToAccessBufferMap(), lock.isSome())do { if (isSweeping || needLockToAccessBufferMap()) { do { static_assert
( mozilla::detail::AssertionConditionType<decltype(lock.isSome
())>::isValid, "invalid assertion condition"); if ((__builtin_expect
(!!(!(!!(lock.isSome()))), 0))) { do { } while (false); MOZ_ReportAssertionFailure
("lock.isSome()", "./../../../../js/src/gc/BufferAllocator.cpp"
, 3727); AnnotateMozCrashReason("MOZ_ASSERT" "(" "lock.isSome()"
")"); do { MOZ_CrashSequence(__null, 3727); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false); } } while (
false);
3728
3729#ifdef DEBUG1
3730 auto ptr = runtime()->largeAllocMap.ref().lookup(buffer->data());
3731 MOZ_ASSERT(ptr && ptr->value() == buffer)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(ptr && ptr->value() == buffer)>::isValid
, "invalid assertion condition"); if ((__builtin_expect(!!(!(
!!(ptr && ptr->value() == buffer))), 0))) { do { }
while (false); MOZ_ReportAssertionFailure("ptr && ptr->value() == buffer"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3731); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "ptr && ptr->value() == buffer" ")"
); do { MOZ_CrashSequence(__null, 3731); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3732#endif
3733
3734 runtime()->largeAllocMap.ref().remove(buffer->data());
3735
3736 // Drop the lock now we've updated the map.
3737 lock.reset();
3738
3739 if (!buffer->isNurseryOwned || !isSweeping) {
3740 decreaseHeapSize(buffer->bytes, buffer->isNurseryOwned, isSweeping);
3741 }
3742}
3743
3744#include "js/Printer.h"
3745#include "util/GetPidProvider.h"
3746
3747static const char* const BufferAllocatorStatsPrefix = "BufAllc:";
3748
3749#define FOR_EACH_BUFFER_STATS_FIELD(_)_("PID", 7, "%7zu", pid) _("Runtime", 14, "0x%12p", runtime) _
("Timestamp", 10, "%10.6f", timestamp.ToSeconds()) _("Reason"
, 20, "%-20.20s", reason) _("", 2, "%2s", "") _("TotalKB", 8,
"%8zu", totalBytes / 1024) _("UsedKB", 8, "%8zu", stats.usedBytes
/ 1024) _("FreeKB", 8, "%8zu", stats.freeBytes / 1024) _("Zs"
, 3, "%3zu", zoneCount) _("", 7, "%7s", "") _("MixSRs", 6, "%6zu"
, stats.mixedSmallRegions) _("TnrSRs", 6, "%6zu", stats.tenuredSmallRegions
) _("MixCs", 6, "%6zu", stats.mixedChunks) _("TnrCs", 6, "%6zu"
, stats.tenuredChunks) _("AMixCs", 6, "%6zu", stats.availableMixedChunks
) _("ATnrCs", 6, "%6zu", stats.availableTenuredChunks) _("FreeRs"
, 6, "%6zu", stats.freeRegions) _("LNurAs", 6, "%6zu", stats.
largeNurseryAllocs) _("LTnrAs", 6, "%6zu", stats.largeTenuredAllocs
) \
3750 _("PID", 7, "%7zu", pid) \
3751 _("Runtime", 14, "0x%12p", runtime) \
3752 _("Timestamp", 10, "%10.6f", timestamp.ToSeconds()) \
3753 _("Reason", 20, "%-20.20s", reason) \
3754 _("", 2, "%2s", "") \
3755 _("TotalKB", 8, "%8zu", totalBytes / 1024) \
3756 _("UsedKB", 8, "%8zu", stats.usedBytes / 1024) \
3757 _("FreeKB", 8, "%8zu", stats.freeBytes / 1024) \
3758 _("Zs", 3, "%3zu", zoneCount) \
3759 _("", 7, "%7s", "") \
3760 _("MixSRs", 6, "%6zu", stats.mixedSmallRegions) \
3761 _("TnrSRs", 6, "%6zu", stats.tenuredSmallRegions) \
3762 _("MixCs", 6, "%6zu", stats.mixedChunks) \
3763 _("TnrCs", 6, "%6zu", stats.tenuredChunks) \
3764 _("AMixCs", 6, "%6zu", stats.availableMixedChunks) \
3765 _("ATnrCs", 6, "%6zu", stats.availableTenuredChunks) \
3766 _("FreeRs", 6, "%6zu", stats.freeRegions) \
3767 _("LNurAs", 6, "%6zu", stats.largeNurseryAllocs) \
3768 _("LTnrAs", 6, "%6zu", stats.largeTenuredAllocs)
3769
3770/* static */
3771void BufferAllocator::printStatsHeader(FILE* file) {
3772 Sprinter sprinter;
3773 if (!sprinter.init()) {
3774 return;
3775 }
3776 sprinter.put(BufferAllocatorStatsPrefix);
3777
3778#define PRINT_METADATA_NAME(name, width, _1, _2) \
3779 sprinter.printf(" %-*s", width, name);
3780
3781 FOR_EACH_BUFFER_STATS_FIELD(PRINT_METADATA_NAME)PRINT_METADATA_NAME("PID", 7, "%7zu", pid) PRINT_METADATA_NAME
("Runtime", 14, "0x%12p", runtime) PRINT_METADATA_NAME("Timestamp"
, 10, "%10.6f", timestamp.ToSeconds()) PRINT_METADATA_NAME("Reason"
, 20, "%-20.20s", reason) PRINT_METADATA_NAME("", 2, "%2s", ""
) PRINT_METADATA_NAME("TotalKB", 8, "%8zu", totalBytes / 1024
) PRINT_METADATA_NAME("UsedKB", 8, "%8zu", stats.usedBytes / 1024
) PRINT_METADATA_NAME("FreeKB", 8, "%8zu", stats.freeBytes / 1024
) PRINT_METADATA_NAME("Zs", 3, "%3zu", zoneCount) PRINT_METADATA_NAME
("", 7, "%7s", "") PRINT_METADATA_NAME("MixSRs", 6, "%6zu", stats
.mixedSmallRegions) PRINT_METADATA_NAME("TnrSRs", 6, "%6zu", stats
.tenuredSmallRegions) PRINT_METADATA_NAME("MixCs", 6, "%6zu",
stats.mixedChunks) PRINT_METADATA_NAME("TnrCs", 6, "%6zu", stats
.tenuredChunks) PRINT_METADATA_NAME("AMixCs", 6, "%6zu", stats
.availableMixedChunks) PRINT_METADATA_NAME("ATnrCs", 6, "%6zu"
, stats.availableTenuredChunks) PRINT_METADATA_NAME("FreeRs",
6, "%6zu", stats.freeRegions) PRINT_METADATA_NAME("LNurAs", 6
, "%6zu", stats.largeNurseryAllocs) PRINT_METADATA_NAME("LTnrAs"
, 6, "%6zu", stats.largeTenuredAllocs)
3782#undef PRINT_METADATA_NAME
3783
3784 sprinter.put("\n");
3785
3786 JS::UniqueChars str = sprinter.release();
3787 if (!str) {
3788 return;
3789 }
3790 fputs(str.get(), file);
3791}
3792
3793/* static */
3794void BufferAllocator::printStats(GCRuntime* gc, mozilla::TimeStamp creationTime,
3795 bool isMajorGC, FILE* file) {
3796 Sprinter sprinter;
3797 if (!sprinter.init()) {
3798 return;
3799 }
3800 sprinter.put(BufferAllocatorStatsPrefix);
3801
3802 size_t pid = getpid();
3803 JSRuntime* runtime = gc->rt;
3804 mozilla::TimeDuration timestamp = mozilla::TimeStamp::Now() - creationTime;
3805 const char* reason = isMajorGC ? "post major GC" : "pre minor GC";
3806
3807 size_t zoneCount = 0;
3808 Stats stats;
3809 for (AllZonesIter zone(gc); !zone.done(); zone.next()) {
3810 zoneCount++;
3811 zone->bufferAllocator.getStats(stats);
3812 }
3813
3814 size_t totalBytes = stats.usedBytes + stats.freeBytes + stats.adminBytes;
3815
3816#define PRINT_FIELD_VALUE(_1, _2, format, value) \
3817 sprinter.printf(" " format, value);
3818
3819 FOR_EACH_BUFFER_STATS_FIELD(PRINT_FIELD_VALUE)PRINT_FIELD_VALUE("PID", 7, "%7zu", pid) PRINT_FIELD_VALUE("Runtime"
, 14, "0x%12p", runtime) PRINT_FIELD_VALUE("Timestamp", 10, "%10.6f"
, timestamp.ToSeconds()) PRINT_FIELD_VALUE("Reason", 20, "%-20.20s"
, reason) PRINT_FIELD_VALUE("", 2, "%2s", "") PRINT_FIELD_VALUE
("TotalKB", 8, "%8zu", totalBytes / 1024) PRINT_FIELD_VALUE("UsedKB"
, 8, "%8zu", stats.usedBytes / 1024) PRINT_FIELD_VALUE("FreeKB"
, 8, "%8zu", stats.freeBytes / 1024) PRINT_FIELD_VALUE("Zs", 3
, "%3zu", zoneCount) PRINT_FIELD_VALUE("", 7, "%7s", "") PRINT_FIELD_VALUE
("MixSRs", 6, "%6zu", stats.mixedSmallRegions) PRINT_FIELD_VALUE
("TnrSRs", 6, "%6zu", stats.tenuredSmallRegions) PRINT_FIELD_VALUE
("MixCs", 6, "%6zu", stats.mixedChunks) PRINT_FIELD_VALUE("TnrCs"
, 6, "%6zu", stats.tenuredChunks) PRINT_FIELD_VALUE("AMixCs",
6, "%6zu", stats.availableMixedChunks) PRINT_FIELD_VALUE("ATnrCs"
, 6, "%6zu", stats.availableTenuredChunks) PRINT_FIELD_VALUE(
"FreeRs", 6, "%6zu", stats.freeRegions) PRINT_FIELD_VALUE("LNurAs"
, 6, "%6zu", stats.largeNurseryAllocs) PRINT_FIELD_VALUE("LTnrAs"
, 6, "%6zu", stats.largeTenuredAllocs)
3820#undef PRINT_FIELD_VALUE
3821
3822 sprinter.put("\n");
3823
3824 JS::UniqueChars str = sprinter.release();
3825 if (!str) {
3826 return;
3827 }
3828
3829 fputs(str.get(), file);
3830}
3831
3832size_t BufferAllocator::getSizeOfNurseryBuffers() {
3833 checkMainThread();
3834
3835 maybeMergeSweptData();
3836
3837 MOZ_ASSERT(minorState == State::NotCollecting)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(minorState == State::NotCollecting)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(minorState == State::NotCollecting
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"minorState == State::NotCollecting", "./../../../../js/src/gc/BufferAllocator.cpp"
, 3837); AnnotateMozCrashReason("MOZ_ASSERT" "(" "minorState == State::NotCollecting"
")"); do { MOZ_CrashSequence(__null, 3837); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3838 MOZ_ASSERT(majorState == State::NotCollecting)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(majorState == State::NotCollecting)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(majorState == State::NotCollecting
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"majorState == State::NotCollecting", "./../../../../js/src/gc/BufferAllocator.cpp"
, 3838); AnnotateMozCrashReason("MOZ_ASSERT" "(" "majorState == State::NotCollecting"
")"); do { MOZ_CrashSequence(__null, 3838); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3839
3840 size_t bytes = 0;
3841
3842 for (BufferChunk* chunk : currentMixedChunks.ref()) {
3843 for (auto alloc = chunk->allocIter(); !alloc.done(); alloc.next()) {
3844 if (chunk->isNurseryOwned(alloc)) {
3845 bytes += chunk->allocBytes(alloc);
3846 }
3847 }
3848 }
3849
3850 for (const LargeBuffer* buffer : largeNurseryAllocs.ref()) {
3851 bytes += buffer->allocBytes();
3852 }
3853
3854 return bytes;
3855}
3856
3857void BufferAllocator::addBufferSizesAndCounts(
3858 size_t* usedBytesOut, size_t* freeBytesOut, size_t* adminBytesOut,
3859 size_t* totalChunksOut, size_t* freeRegionsOut, size_t* largeAllocsOut) {
3860 checkMainThread();
3861
3862 maybeMergeSweptData();
3863
3864 MOZ_ASSERT(minorState == State::NotCollecting)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(minorState == State::NotCollecting)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(minorState == State::NotCollecting
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"minorState == State::NotCollecting", "./../../../../js/src/gc/BufferAllocator.cpp"
, 3864); AnnotateMozCrashReason("MOZ_ASSERT" "(" "minorState == State::NotCollecting"
")"); do { MOZ_CrashSequence(__null, 3864); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3865 MOZ_ASSERT(majorState == State::NotCollecting)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(majorState == State::NotCollecting)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(majorState == State::NotCollecting
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"majorState == State::NotCollecting", "./../../../../js/src/gc/BufferAllocator.cpp"
, 3865); AnnotateMozCrashReason("MOZ_ASSERT" "(" "majorState == State::NotCollecting"
")"); do { MOZ_CrashSequence(__null, 3865); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3866
3867 Stats stats;
3868 getStats(stats);
3869
3870 *usedBytesOut += stats.usedBytes;
3871 *freeBytesOut += stats.freeBytes;
3872 *adminBytesOut += stats.adminBytes;
3873 *totalChunksOut += stats.mixedChunks + stats.tenuredChunks +
3874 stats.availableMixedChunks + stats.availableTenuredChunks;
3875 *freeRegionsOut += stats.freeRegions;
3876 *largeAllocsOut += stats.largeNurseryAllocs + stats.largeTenuredAllocs;
3877}
3878
3879void BufferAllocator::getStats(Stats& stats) {
3880 checkMainThread();
3881
3882 maybeMergeSweptData();
3883
3884 MOZ_ASSERT(minorState == State::NotCollecting)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(minorState == State::NotCollecting)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(minorState == State::NotCollecting
))), 0))) { do { } while (false); MOZ_ReportAssertionFailure(
"minorState == State::NotCollecting", "./../../../../js/src/gc/BufferAllocator.cpp"
, 3884); AnnotateMozCrashReason("MOZ_ASSERT" "(" "minorState == State::NotCollecting"
")"); do { MOZ_CrashSequence(__null, 3884); __attribute__((nomerge
)) ::abort(); } while (false); } } while (false);
3885
3886 for (BufferChunk* chunk : currentMixedChunks.ref()) {
3887 stats.mixedChunks++;
3888 chunk->getStats(stats);
3889 }
3890 for (BufferChunk* chunk : currentTenuredChunks.ref()) {
3891 stats.tenuredChunks++;
3892 chunk->getStats(stats);
3893 }
3894 for (auto chunk = availableChunks.ref().chunkIter(); !chunk.done();
3895 chunk.next()) {
3896 if (chunk->hasNurseryOwnedAllocs) {
3897 stats.availableMixedChunks++;
3898 } else {
3899 stats.availableTenuredChunks++;
3900 }
3901 chunk->getStats(stats);
3902 }
3903 for (const LargeBuffer* buffer : largeNurseryAllocs.ref()) {
3904 stats.largeNurseryAllocs++;
3905 stats.usedBytes += buffer->allocBytes();
3906 stats.adminBytes += sizeof(LargeBuffer);
3907 }
3908 for (const LargeBuffer* buffer : largeTenuredAllocs.ref()) {
3909 stats.largeTenuredAllocs++;
3910 stats.usedBytes += buffer->allocBytes();
3911 stats.adminBytes += sizeof(LargeBuffer);
3912 }
3913 mixedFreeLists.ref().getStats(stats);
3914 tenuredFreeLists.ref().getStats(stats);
3915}
3916
3917void BufferChunk::getStats(BufferAllocator::Stats& stats) {
3918 stats.usedBytes += ChunkSize - FirstMediumAllocOffset;
3919 stats.adminBytes += FirstMediumAllocOffset;
3920
3921 for (auto iter = smallRegionIter(); !iter.done(); iter.next()) {
3922 SmallBufferRegion* region = iter.get();
3923 if (region->hasNurseryOwnedAllocs()) {
3924 stats.mixedSmallRegions++;
3925 } else {
3926 stats.tenuredSmallRegions++;
3927 }
3928 stats.usedBytes -= FirstSmallAllocOffset;
3929 stats.adminBytes += FirstSmallAllocOffset;
3930 }
3931
3932 if (ownsFreeLists) {
3933 freeLists.ref().getStats(stats);
3934 }
3935}
3936
3937void BufferAllocator::FreeLists::getStats(Stats& stats) {
3938 for (auto region = freeRegionIter(); !region.done(); region.next()) {
3939 stats.freeRegions++;
3940 size_t size = region->size();
3941 MOZ_ASSERT(stats.usedBytes >= size)do { static_assert( mozilla::detail::AssertionConditionType<
decltype(stats.usedBytes >= size)>::isValid, "invalid assertion condition"
); if ((__builtin_expect(!!(!(!!(stats.usedBytes >= size))
), 0))) { do { } while (false); MOZ_ReportAssertionFailure("stats.usedBytes >= size"
, "./../../../../js/src/gc/BufferAllocator.cpp", 3941); AnnotateMozCrashReason
("MOZ_ASSERT" "(" "stats.usedBytes >= size" ")"); do { MOZ_CrashSequence
(__null, 3941); __attribute__((nomerge)) ::abort(); } while (
false); } } while (false);
3942 stats.usedBytes -= size;
3943 stats.freeBytes += size;
3944 }
3945}