Attachment 316200 Details for Bug 174727 – starting to compile

[patch] starting to compile

blah.patch (text/plain), 91.30 KB, created by Filip Pizlo on 2017-07-22 10:42:10 PDT

(hide)

Description:

Filename:

MIME Type:

Creator: Filip Pizlo

Created: 2017-07-22 10:42:10 PDT

Size: 91.30 KB

patch

obsolete

>Index: Source/JavaScriptCore/API/JSTypedArray.cpp
>===================================================================
>--- Source/JavaScriptCore/API/JSTypedArray.cpp	(revision 219600)
>+++ Source/JavaScriptCore/API/JSTypedArray.cpp	(working copy)
>@@ -304,7 +304,7 @@ JSObjectRef JSObjectMakeArrayBufferWithB
> {
>     ExecState* exec = toJS(ctx);
>     JSLockHolder locker(exec);
>-
>+    
>     auto buffer = ArrayBuffer::createFromBytes(bytes, byteLength, [=](void* p) {
>         if (bytesDeallocator)
>             bytesDeallocator(p, deallocatorContext);
>Index: Source/JavaScriptCore/JavaScriptCore.xcodeproj/project.pbxproj
>===================================================================
>--- Source/JavaScriptCore/JavaScriptCore.xcodeproj/project.pbxproj	(revision 219600)
>+++ Source/JavaScriptCore/JavaScriptCore.xcodeproj/project.pbxproj	(working copy)
>@@ -463,6 +463,8 @@
> 		0F5A6284188C98D40072C9DF /* FTLValueRange.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F5A6282188C98D40072C9DF /* FTLValueRange.h */; settings = {ATTRIBUTES = (Private, ); }; };
> 		0F5AE2C41DF4F2800066EFE1 /* VMInlines.h in Headers */ = {isa = PBXBuildFile; fileRef = FE90BB3A1B7CF64E006B3F03 /* VMInlines.h */; settings = {ATTRIBUTES = (Private, ); }; };
> 		0F5B4A331C84F0D600F1B17E /* SlowPathReturnType.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F5B4A321C84F0D600F1B17E /* SlowPathReturnType.h */; settings = {ATTRIBUTES = (Private, ); }; };
>+		0F5BF1561F22EB170029D91D /* GigacageSubspace.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 0F5BF1541F22EB170029D91D /* GigacageSubspace.cpp */; };
>+		0F5BF1571F22EB170029D91D /* GigacageSubspace.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F5BF1551F22EB170029D91D /* GigacageSubspace.h */; settings = {ATTRIBUTES = (Private, ); }; };
> 		0F5CF9811E96F17F00C18692 /* AirTmpMap.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F5CF9801E96F17D00C18692 /* AirTmpMap.h */; };
> 		0F5CF9841E9D537700C18692 /* AirLowerStackArgs.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F5CF9831E9D537500C18692 /* AirLowerStackArgs.h */; };
> 		0F5CF9851E9D537A00C18692 /* AirLowerStackArgs.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 0F5CF9821E9D537500C18692 /* AirLowerStackArgs.cpp */; };
>@@ -3028,6 +3030,8 @@
> 		0F5A6281188C98D40072C9DF /* FTLValueRange.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = FTLValueRange.cpp; path = ftl/FTLValueRange.cpp; sourceTree = "<group>"; };
> 		0F5A6282188C98D40072C9DF /* FTLValueRange.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = FTLValueRange.h; path = ftl/FTLValueRange.h; sourceTree = "<group>"; };
> 		0F5B4A321C84F0D600F1B17E /* SlowPathReturnType.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SlowPathReturnType.h; sourceTree = "<group>"; };
>+		0F5BF1541F22EB170029D91D /* GigacageSubspace.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = GigacageSubspace.cpp; sourceTree = "<group>"; };
>+		0F5BF1551F22EB170029D91D /* GigacageSubspace.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = GigacageSubspace.h; sourceTree = "<group>"; };
> 		0F5CF9801E96F17D00C18692 /* AirTmpMap.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = AirTmpMap.h; path = b3/air/AirTmpMap.h; sourceTree = "<group>"; };
> 		0F5CF9821E9D537500C18692 /* AirLowerStackArgs.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = AirLowerStackArgs.cpp; path = b3/air/AirLowerStackArgs.cpp; sourceTree = "<group>"; };
> 		0F5CF9831E9D537500C18692 /* AirLowerStackArgs.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = AirLowerStackArgs.h; path = b3/air/AirLowerStackArgs.h; sourceTree = "<group>"; };
>@@ -6021,6 +6025,8 @@
> 				2A343F7418A1748B0039B085 /* GCSegmentedArray.h */,
> 				2A343F7718A1749D0039B085 /* GCSegmentedArrayInlines.h */,
> 				0F86A26E1D6F7B3100CB0C92 /* GCTypeMap.h */,
>+				0F5BF1541F22EB170029D91D /* GigacageSubspace.cpp */,
>+				0F5BF1551F22EB170029D91D /* GigacageSubspace.h */,
> 				142E312B134FF0A600AFADB5 /* Handle.h */,
> 				C28318FF16FE4B7D00157BFD /* HandleBlock.h */,
> 				C283190116FE533E00157BFD /* HandleBlockInlines.h */,
>@@ -9477,6 +9483,7 @@
> 				0F5AE2C41DF4F2800066EFE1 /* VMInlines.h in Headers */,
> 				FE3022D71E42857300BAC493 /* VMInspector.h in Headers */,
> 				FE6F56DE1E64EAD600D17801 /* VMTraps.h in Headers */,
>+				0F5BF1571F22EB170029D91D /* GigacageSubspace.h in Headers */,
> 				53F40E931D5A4AB30099A1B6 /* WasmB3IRGenerator.h in Headers */,
> 				53CA730A1EA533D80076049D /* WasmBBQPlan.h in Headers */,
> 				53F8D2001E8387D400D21116 /* WasmBBQPlanInlines.h in Headers */,
>@@ -10893,6 +10900,7 @@
> 				70EC0EC61AA0D7DA00B6AAFA /* StringIteratorPrototype.cpp in Sources */,
> 				14469DEC107EC7E700650446 /* StringObject.cpp in Sources */,
> 				14469DED107EC7E700650446 /* StringPrototype.cpp in Sources */,
>+				0F5BF1561F22EB170029D91D /* GigacageSubspace.cpp in Sources */,
> 				9335F24D12E6765B002B5553 /* StringRecursionChecker.cpp in Sources */,
> 				BCDE3B430E6C832D001453A7 /* Structure.cpp in Sources */,
> 				7E4EE70F0EBB7A5B005934AA /* StructureChain.cpp in Sources */,
>Index: Source/JavaScriptCore/ftl/FTLLowerDFGToB3.cpp
>===================================================================
>--- Source/JavaScriptCore/ftl/FTLLowerDFGToB3.cpp	(revision 219600)
>+++ Source/JavaScriptCore/ftl/FTLLowerDFGToB3.cpp	(working copy)
>@@ -85,6 +85,7 @@
> #include "VirtualRegister.h"
> #include "Watchdog.h"
> #include <atomic>
>+#include <bmalloc/Gigacage.h>
> #include <unordered_set>
> #include <wtf/Box.h>
> 
>@@ -3231,7 +3232,7 @@ private:
>     
>     void compileGetButterfly()
>     {
>-        setStorage(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.JSObject_butterfly));
>+        setStorage(caged(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.JSObject_butterfly)));
>     }
> 
>     void compileConstantStoragePointer()
>@@ -3267,7 +3268,7 @@ private:
>         }
> 
>         DFG_ASSERT(m_graph, m_node, isTypedView(m_node->arrayMode().typedArrayType()));
>-        setStorage(m_out.loadPtr(cell, m_heaps.JSArrayBufferView_vector));
>+        setStorage(caged(m_out.loadPtr(cell, m_heaps.JSArrayBufferView_vector)));
>     }
>     
>     void compileCheckArray()
>@@ -3509,6 +3510,8 @@ private:
>                     index,
>                     m_out.load32NonNegative(base, m_heaps.DirectArguments_length)));
> 
>+            // FIXME: I guess we need to cage DirectArguments?
>+            // OOPS!!! (need bug URL)
>             TypedPointer address = m_out.baseIndex(
>                 m_heaps.DirectArguments_storage, base, m_out.zeroExtPtr(index));
>             setJSValue(m_out.load64(address));
>@@ -3540,6 +3543,8 @@ private:
>             LValue scope = m_out.loadPtr(base, m_heaps.ScopedArguments_scope);
>             LValue arguments = m_out.loadPtr(table, m_heaps.ScopedArgumentsTable_arguments);
>             
>+            // FIXME: I guess we need to cage ScopedArguments?
>+            // OOPS!!! (need bug URL)
>             TypedPointer address = m_out.baseIndex(
>                 m_heaps.scopedArgumentsTableArguments, arguments, m_out.zeroExtPtr(index));
>             LValue scopeOffset = m_out.load32(address);
>@@ -3548,6 +3553,8 @@ private:
>                 ExoticObjectMode, noValue(), nullptr,
>                 m_out.equal(scopeOffset, m_out.constInt32(ScopeOffset::invalidOffset)));
>             
>+            // FIXME: I guess we need to cage JSEnvironmentRecord?
>+            // OOPS!!! (need bug URL)
>             address = m_out.baseIndex(
>                 m_heaps.JSEnvironmentRecord_variables, scope, m_out.zeroExtPtr(scopeOffset));
>             ValueFromBlock namedResult = m_out.anchor(m_out.load64(address));
>@@ -3555,6 +3562,8 @@ private:
>             
>             m_out.appendTo(overflowCase, continuation);
>             
>+            // FIXME: I guess we need to cage overflow storage?
>+            // OOPS!!! (need bug URL)
>             address = m_out.baseIndex(
>                 m_heaps.ScopedArguments_overflowStorage, base,
>                 m_out.zeroExtPtr(m_out.sub(index, namedLength)));
>@@ -5378,6 +5387,8 @@ private:
>             
>         m_out.appendTo(is8Bit, is16Bit);
>             
>+        // FIXME: Need to cage strings!
>+        // OOPS!! (need bug URL)
>         ValueFromBlock char8Bit = m_out.anchor(
>             m_out.load8ZeroExt32(m_out.baseIndex(
>                 m_heaps.characters8, storage, m_out.zeroExtPtr(index),
>@@ -5479,6 +5490,8 @@ private:
>             
>         LBasicBlock lastNext = m_out.appendTo(is8Bit, is16Bit);
>             
>+        // FIXME: need to cage strings!
>+        // OOPS!!! (need bug URL)
>         ValueFromBlock char8Bit = m_out.anchor(
>             m_out.load8ZeroExt32(m_out.baseIndex(
>                 m_heaps.characters8, storage, m_out.zeroExtPtr(index),
>@@ -8075,6 +8088,8 @@ private:
>         m_out.appendTo(loopStart, notEmptyValue);
>         LValue unmaskedIndex = m_out.phi(Int32, indexStart);
>         LValue index = m_out.bitAnd(mask, unmaskedIndex);
>+        // FIXME: I think these buffers are caged?
>+        // OOPS!! (need bug URL)
>         LValue hashMapBucket = m_out.load64(m_out.baseIndex(m_heaps.properties.atAnyNumber(), buffer, m_out.zeroExt(index, Int64), ScaleEight));
>         ValueFromBlock bucketResult = m_out.anchor(hashMapBucket);
>         m_out.branch(m_out.equal(hashMapBucket, m_out.constIntPtr(bitwise_cast<intptr_t>(HashMapImpl<HashMapBucket<HashMapBucketDataKey>>::emptyValue()))),
>@@ -8850,7 +8865,7 @@ private:
>             m_out.neg(m_out.sub(index, m_out.load32(enumerator, m_heaps.JSPropertyNameEnumerator_cachedInlineCapacity))));
>         int32_t offsetOfFirstProperty = static_cast<int32_t>(offsetInButterfly(firstOutOfLineOffset)) * sizeof(EncodedJSValue);
>         ValueFromBlock outOfLineResult = m_out.anchor(
>-            m_out.load64(m_out.baseIndex(m_heaps.properties.atAnyNumber(), storage, realIndex, ScaleEight, offsetOfFirstProperty)));
>+            m_out.load64(m_out.baseIndex(m_heaps.properties.atAnyNumber(), caged(storage), realIndex, ScaleEight, offsetOfFirstProperty)));
>         m_out.jump(continuation);
> 
>         m_out.appendTo(slowCase, continuation);
>@@ -10268,6 +10283,8 @@ private:
> 
>         m_out.appendTo(loopBody, slowPath);
> 
>+        // FIXME: Strings needs to be caged.
>+        // OOPS!!! (need bug URL)
>         LValue byte = m_out.load8ZeroExt32(m_out.baseIndex(m_heaps.characters8, buffer, m_out.zeroExtPtr(index)));
>         LValue isInvalidAsciiRange = m_out.bitAnd(byte, m_out.constInt32(~0x7F));
>         LValue isUpperCase = m_out.belowOrEqual(m_out.sub(byte, m_out.constInt32('A')), m_out.constInt32('Z' - 'A'));
>@@ -11584,6 +11601,19 @@ private:
>         }
>     }
>     
>+    LValue caged(LValue ptr)
>+    {
>+        if (vm().gigacageEnabled().isStillValid()) {
>+            m_graph.watchpoints().addLazily(vm().gigacageEnabled());
>+            
>+            return m_out.add(
>+                m_out.constIntPtr(g_gigacageBasePtr),
>+                m_out.bitAnd(ptr, m_constIntPtr(GIGACAGE_MASK)));
>+        }
>+        
>+        return ptr;
>+    }
>+    
>     void buildSwitch(SwitchData* data, LType type, LValue switchValue)
>     {
>         ASSERT(type == pointerType() || type == Int32);
>Index: Source/JavaScriptCore/heap/GigacageSubspace.cpp
>===================================================================
>--- Source/JavaScriptCore/heap/GigacageSubspace.cpp	(nonexistent)
>+++ Source/JavaScriptCore/heap/GigacageSubspace.cpp	(working copy)
>@@ -0,0 +1,83 @@
>+/*
>+ * Copyright (C) 2017 Apple Inc. All rights reserved.
>+ *
>+ * Redistribution and use in source and binary forms, with or without
>+ * modification, are permitted provided that the following conditions
>+ * are met:
>+ * 1. Redistributions of source code must retain the above copyright
>+ *    notice, this list of conditions and the following disclaimer.
>+ * 2. Redistributions in binary form must reproduce the above copyright
>+ *    notice, this list of conditions and the following disclaimer in the
>+ *    documentation and/or other materials provided with the distribution.
>+ *
>+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
>+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
>+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
>+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
>+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
>+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
>+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
>+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
>+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
>+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
>+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
>+ */
>+
>+#include "config.h"
>+#include "GigacageSubspace.h"
>+
>+#if !(defined(USE_SYSTEM_MALLOC) && USE_SYSTEM_MALLOC)
>+#include <bmalloc/bmalloc.h>
>+#endif
>+
>+namespace JSC {
>+
>+#if defined(USE_SYSTEM_MALLOC) && USE_SYSTEM_MALLOC
>+
>+// Even if the Gigacage isn't "enabled", this still implies that we avoid reusing memory with other things.
>+// That's a slight security advantage.
>+
>+GigacageSubspace::GigacageSubspace()
>+{
>+}
>+
>+void* GigacageSubspace::tryAllocateAlignedMemory(size_t alignment, size_t size)
>+{
>+    return Subspace::tryAllocateAlignedMemory(alignment, size);
>+}
>+
>+void GigacageSubspace::freeAlignedMemory(void* basePtr)
>+{
>+    Subspace::freeAlignedMemory(basePtr);
>+}
>+
>+#else // defined(USE_SYSTEM_MALLOC) && USE_SYSTEM_MALLOC
>+
>+GigacageSubspace::GigacageSubspace()
>+{
>+    bmalloc::api::ensureGigacage();
>+}
>+
>+void* GigacageSubspace::tryAllocateAlignedMemory(size_t alignment, size_t size)
>+{
>+    return bmalloc::api::tryMemalign(bmalloc::HeapKind::Gigacage, alignment, size);
>+}
>+
>+void GigacageSubspace::freeAlignedMemory(void* basePtr)
>+{
>+    bmalloc::api::free(bmalloc::HeapKind::Gigacage, basePtr);
>+}
>+
>+#endif // defined(USE_SYSTEM_MALLOC) && USE_SYSTEM_MALLOC
>+
>+GigacageSubspace::~GigacageSubspace()
>+{
>+}
>+
>+bool GigacageSubspace::canTradeBlocksWith(Subspace* other)
>+{
>+    return this == other;
>+}
>+
>+} // namespace JSC
>+
>Index: Source/JavaScriptCore/heap/GigacageSubspace.h
>===================================================================
>--- Source/JavaScriptCore/heap/GigacageSubspace.h	(nonexistent)
>+++ Source/JavaScriptCore/heap/GigacageSubspace.h	(working copy)
>@@ -0,0 +1,45 @@
>+/*
>+ * Copyright (C) 2017 Apple Inc. All rights reserved.
>+ *
>+ * Redistribution and use in source and binary forms, with or without
>+ * modification, are permitted provided that the following conditions
>+ * are met:
>+ * 1. Redistributions of source code must retain the above copyright
>+ *    notice, this list of conditions and the following disclaimer.
>+ * 2. Redistributions in binary form must reproduce the above copyright
>+ *    notice, this list of conditions and the following disclaimer in the
>+ *    documentation and/or other materials provided with the distribution.
>+ *
>+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
>+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
>+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
>+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
>+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
>+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
>+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
>+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
>+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
>+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
>+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
>+ */
>+
>+#pragma once
>+
>+#include "Subspace.h"
>+#include <bmalloc/Gigacage.h>
>+
>+namespace JSC {
>+
>+// We use a GigacageSubspace for the auxiliary space.
>+class GigacageSubspace : public Subspace {
>+public:
>+    GigacageSubspace(CString name, Heap&, AllocatorAttributes);
>+    ~GigacageSubsace();
>+    
>+    bool canTradeBlocksWith(Subspace* other) override;
>+    void* tryAllocateAlignedMemory(size_t alignment, size_t size) override;
>+    void freeAlignedMemory(void*) override;
>+};
>+
>+} // namespace JSC
>+
>Index: Source/JavaScriptCore/heap/LargeAllocation.cpp
>===================================================================
>--- Source/JavaScriptCore/heap/LargeAllocation.cpp	(revision 219600)
>+++ Source/JavaScriptCore/heap/LargeAllocation.cpp	(working copy)
>@@ -34,7 +34,7 @@ namespace JSC {
> 
> LargeAllocation* LargeAllocation::tryCreate(Heap& heap, size_t size, Subspace* subspace)
> {
>-    void* space = tryFastAlignedMalloc(alignment, headerSize() + size);
>+    void* space = subspace->tryAllocateAlignedMemory(alignment, headerSize() + size);
>     if (!space)
>         return nullptr;
>     if (scribbleFreeCells())
>@@ -106,8 +106,9 @@ void LargeAllocation::sweep()
> 
> void LargeAllocation::destroy()
> {
>+    Subspace* subspace = m_subspace;
>     this->~LargeAllocation();
>-    fastAlignedFree(this);
>+    subspace->freeAlignedMemory(this);
> }
> 
> void LargeAllocation::dump(PrintStream& out) const
>Index: Source/JavaScriptCore/heap/MarkedAllocator.cpp
>===================================================================
>--- Source/JavaScriptCore/heap/MarkedAllocator.cpp	(revision 219600)
>+++ Source/JavaScriptCore/heap/MarkedAllocator.cpp	(working copy)
>@@ -113,7 +113,10 @@ void* MarkedAllocator::tryAllocateWithou
>     
>     if (Options::stealEmptyBlocksFromOtherAllocators()
>         && shouldStealEmptyBlocksFromOtherAllocators()) {
>-        if (MarkedBlock::Handle* block = markedSpace().findEmptyBlockToSteal()) {
>+        if (MarkedBlock::Handle* block = m_subspace->findEmptyBlockToSteal()) {
>+            RELEASE_ASSERT(block->subspace()->canTradeBlocksWith(m_subspace));
>+            RELEASE_ASSERT(m_subspace->canTradeBlocksWith(block->subspace()));
>+            
>             block->sweep(nullptr);
>             
>             // It's good that this clears canAllocateButNotEmpty as well as all other bits,
>Index: Source/JavaScriptCore/heap/MarkedBlock.cpp
>===================================================================
>--- Source/JavaScriptCore/heap/MarkedBlock.cpp	(revision 219600)
>+++ Source/JavaScriptCore/heap/MarkedBlock.cpp	(working copy)
>@@ -43,14 +43,14 @@ const size_t MarkedBlock::blockSize;
> static const bool computeBalance = false;
> static size_t balance;
> 
>-MarkedBlock::Handle* MarkedBlock::tryCreate(Heap& heap)
>+MarkedBlock::Handle* MarkedBlock::tryCreate(Heap& heap, Subspace* subspace)
> {
>     if (computeBalance) {
>         balance++;
>         if (!(balance % 10))
>             dataLog("MarkedBlock Balance: ", balance, "\n");
>     }
>-    void* blockSpace = tryFastAlignedMalloc(blockSize, blockSize);
>+    void* blockSpace = subspace->tryAllocateAlignedMemory(blockSize, blockSize);
>     if (!blockSpace)
>         return nullptr;
>     if (scribbleFreeCells())
>@@ -72,6 +72,7 @@ MarkedBlock::Handle::Handle(Heap& heap, 
> MarkedBlock::Handle::~Handle()
> {
>     Heap& heap = *this->heap();
>+    Subspace* subspace = this->subspace();
>     if (computeBalance) {
>         balance--;
>         if (!(balance % 10))
>@@ -79,7 +80,7 @@ MarkedBlock::Handle::~Handle()
>     }
>     removeFromAllocator();
>     m_block->~MarkedBlock();
>-    fastAlignedFree(m_block);
>+    subspace->freeAlignedMemory(m_block);
>     heap.didFreeBlock(blockSize);
> }
> 
>Index: Source/JavaScriptCore/heap/MarkedBlock.h
>===================================================================
>--- Source/JavaScriptCore/heap/MarkedBlock.h	(revision 219600)
>+++ Source/JavaScriptCore/heap/MarkedBlock.h	(working copy)
>@@ -238,7 +238,7 @@ public:
>         MarkedBlock* m_block { nullptr };
>     };
>         
>-    static MarkedBlock::Handle* tryCreate(Heap&);
>+    static MarkedBlock::Handle* tryCreate(Heap&, Subspace*);
>         
>     Handle& handle();
>         
>Index: Source/JavaScriptCore/heap/MarkedSpace.cpp
>===================================================================
>--- Source/JavaScriptCore/heap/MarkedSpace.cpp	(revision 219600)
>+++ Source/JavaScriptCore/heap/MarkedSpace.cpp	(working copy)
>@@ -254,11 +254,8 @@ void MarkedSpace::sweepLargeAllocations(
> 
> void MarkedSpace::prepareForAllocation()
> {
>-    forEachAllocator(
>-        [&] (MarkedAllocator& allocator) -> IterationStatus {
>-            allocator.prepareForAllocation();
>-            return IterationStatus::Continue;
>-        });
>+    for (Subspace* subspace : m_subspaces)
>+        subspace->prepareForAllocation();
> 
>     m_activeWeakSets.takeFrom(m_newActiveWeakSets);
>     
>@@ -267,8 +264,6 @@ void MarkedSpace::prepareForAllocation()
>     else
>         m_largeAllocationsNurseryOffsetForSweep = 0;
>     m_largeAllocationsNurseryOffset = m_largeAllocations.size();
>-    
>-    m_allocatorForEmptyAllocation = m_firstAllocator;
> }
> 
> void MarkedSpace::visitWeakSets(SlotVisitor& visitor)
>@@ -514,15 +509,6 @@ void MarkedSpace::didAllocateInBlock(Mar
>     }
> }
> 
>-MarkedBlock::Handle* MarkedSpace::findEmptyBlockToSteal()
>-{
>-    for (; m_allocatorForEmptyAllocation; m_allocatorForEmptyAllocation = m_allocatorForEmptyAllocation->nextAllocator()) {
>-        if (MarkedBlock::Handle* block = m_allocatorForEmptyAllocation->findEmptyBlockToSteal())
>-            return block;
>-    }
>-    return nullptr;
>-}
>-
> void MarkedSpace::snapshotUnswept()
> {
>     if (m_heap->collectionScope() == CollectionScope::Eden) {
>@@ -572,7 +558,8 @@ MarkedAllocator* MarkedSpace::addMarkedA
>     if (!m_firstAllocator) {
>         m_firstAllocator = allocator;
>         m_lastAllocator = allocator;
>-        m_allocatorForEmptyAllocation = allocator;
>+        for (Subspace* subspace : m_subspaces)
>+            subspace->didCreateFirstAllocator(allocator);
>     } else {
>         m_lastAllocator->setNextAllocator(allocator);
>         m_lastAllocator = allocator;
>Index: Source/JavaScriptCore/heap/MarkedSpace.h
>===================================================================
>--- Source/JavaScriptCore/heap/MarkedSpace.h	(revision 219600)
>+++ Source/JavaScriptCore/heap/MarkedSpace.h	(working copy)
>@@ -155,9 +155,6 @@ public:
>     unsigned largeAllocationsForThisCollectionSize() const { return m_largeAllocationsForThisCollectionSize; }
>     
>     MarkedAllocator* firstAllocator() const { return m_firstAllocator; }
>-    MarkedAllocator* allocatorForEmptyAllocation() const { return m_allocatorForEmptyAllocation; }
>-    
>-    MarkedBlock::Handle* findEmptyBlockToSteal();
>     
>     Lock& allocatorLock() { return m_allocatorLock; }
>     MarkedAllocator* addMarkedAllocator(const AbstractLocker&, Subspace*, size_t cellSize);
>@@ -215,7 +212,6 @@ private:
>     Bag<MarkedAllocator> m_bagOfAllocators;
>     MarkedAllocator* m_firstAllocator { nullptr };
>     MarkedAllocator* m_lastAllocator { nullptr };
>-    MarkedAllocator* m_allocatorForEmptyAllocation { nullptr };
> 
>     friend class HeapVerifier;
> };
>Index: Source/JavaScriptCore/heap/Subspace.cpp
>===================================================================
>--- Source/JavaScriptCore/heap/Subspace.cpp	(revision 219600)
>+++ Source/JavaScriptCore/heap/Subspace.cpp	(working copy)
>@@ -58,6 +58,7 @@ Subspace::Subspace(CString name, Heap& h
>     : m_space(heap.objectSpace())
>     , m_name(name)
>     , m_attributes(attributes)
>+    , m_allocatorForEmptyAllocation(m_space.firstAllocator())
> {
>     // It's remotely possible that we're GCing right now even if the client is careful to only
>     // create subspaces right after VM creation, since collectContinuously (and probably other
>@@ -87,6 +88,21 @@ void Subspace::destroy(VM& vm, JSCell* c
>     DestroyFunc()(vm, cell);
> }
> 
>+bool Subspace::canTradeBlocksWith(Subspace*)
>+{
>+    return true;
>+}
>+
>+void* Subspace::tryAllocateAlignedMemory(size_t alignment, size_t size)
>+{
>+    return tryFastAlignedMalloc(alignment, headerSize() + size);
>+}
>+
>+void Subspace::freeAlignedMemory(void* basePtr)
>+{
>+    fastAlignedFree(basePtr);
>+}
>+
> // The reason why we distinguish between allocate and tryAllocate is to minimize the number of
> // checks on the allocation path in both cases. Likewise, the reason why we have overloads with and
> // without deferralContext is to minimize the amount of code for calling allocate when you don't
>@@ -135,6 +151,31 @@ void* Subspace::tryAllocate(GCDeferralCo
>     return result;
> }
> 
>+void Subspace::prepareForAllocation()
>+{
>+    forEachAllocator(
>+        [&] (MarkedAllocator& allocator) {
>+            allocator.prepareForAllocation();
>+        });
>+
>+    m_allocatorForEmptyAllocation = m_space.firstAllocator();
>+}
>+
>+MarkedBlock::Handle* Subspace::findEmptyBlockToSteal()
>+{
>+    for (; m_allocatorForEmptyAllocation; m_allocatorForEmptyAllocation = m_allocatorForEmptyAllocation->nextAllocator()) {
>+        Subspace* otherSubspace = m_allocatorForEmptyAllocation->subspace();
>+        if (!canTradeBlocksWith(otherSubspace))
>+            continue;
>+        if (!otherSubspae->canTradeBlocksWith(this))
>+            continue;
>+        
>+        if (MarkedBlock::Handle* block = m_allocatorForEmptyAllocation->findEmptyBlockToSteal())
>+            return block;
>+    }
>+    return nullptr;
>+}
>+
> MarkedAllocator* Subspace::allocatorForSlow(size_t size)
> {
>     size_t index = MarkedSpace::sizeClassToIndex(size);
>Index: Source/JavaScriptCore/heap/Subspace.h
>===================================================================
>--- Source/JavaScriptCore/heap/Subspace.h	(revision 219600)
>+++ Source/JavaScriptCore/heap/Subspace.h	(working copy)
>@@ -59,6 +59,10 @@ public:
>     // These get called for large objects.
>     virtual void destroy(VM&, JSCell*);
>     
>+    virtual bool canTradeBlocksWith(Subspace* other);
>+    virtual void* tryAllocateAlignedMemory(size_t alignment, size_t size);
>+    virtual void freeAlignedMemory(void*);
>+    
>     MarkedAllocator* tryAllocatorFor(size_t);
>     MarkedAllocator* allocatorFor(size_t);
>     
>@@ -68,6 +72,16 @@ public:
>     JS_EXPORT_PRIVATE void* tryAllocate(size_t);
>     JS_EXPORT_PRIVATE void* tryAllocate(GCDeferralContext*, size_t);
>     
>+    void prepareForAllocation();
>+    
>+    void didCreateFirstAllocator(MarkedAllocator* allocator) { m_allocatorForEmptyAllocation = allocator; }
>+    
>+    // Finds an empty block from any Subspace that agrees to trade blocks with us.
>+    MarkedBlock::Handle* findEmptyBlockToSteal();
>+    
>+    template<typename Func>
>+    void forEachAllocator(const Func&);
>+    
>     template<typename Func>
>     void forEachMarkedBlock(const Func&);
>     
>@@ -103,6 +117,7 @@ private:
>     
>     std::array<MarkedAllocator*, MarkedSpace::numSizeClasses> m_allocatorForSizeStep;
>     MarkedAllocator* m_firstAllocator { nullptr };
>+    MarkedAllocator* m_allocatorForEmptyAllocation { nullptr }; // Uses the MarkedSpace linked list of blocks.
>     SentinelLinkedList<LargeAllocation, BasicRawSentinelNode<LargeAllocation>> m_largeAllocations;
> };
> 
>Index: Source/JavaScriptCore/heap/SubspaceInlines.h
>===================================================================
>--- Source/JavaScriptCore/heap/SubspaceInlines.h	(revision 219600)
>+++ Source/JavaScriptCore/heap/SubspaceInlines.h	(working copy)
>@@ -34,17 +34,28 @@
> namespace JSC {
> 
> template<typename Func>
>-void Subspace::forEachMarkedBlock(const Func& func)
>+void Subspace::forEachAllocator(const Func& func)
> {
>     for (MarkedAllocator* allocator = m_firstAllocator; allocator; allocator = allocator->nextAllocatorInSubspace())
>-        allocator->forEachBlock(func);
>+        func(*allocator);
>+}
>+
>+template<typename Func>
>+void Subspace::forEachMarkedBlock(const Func& func)
>+{
>+    forEachAllocator(
>+        [&] (MarkedAllocator& allocator) {
>+            allocator.forEachBlock(func);
>+        });
> }
> 
> template<typename Func>
> void Subspace::forEachNotEmptyMarkedBlock(const Func& func)
> {
>-    for (MarkedAllocator* allocator = m_firstAllocator; allocator; allocator = allocator->nextAllocatorInSubspace())
>-        allocator->forEachNotEmptyBlock(func);
>+    forEachAllocator(
>+        [&] (MarkedAllocator& allocator) {
>+            allocator.forEachBlock(func);
>+        });
> }
> 
> template<typename Func>
>Index: Source/JavaScriptCore/runtime/ArrayBuffer.cpp
>===================================================================
>--- Source/JavaScriptCore/runtime/ArrayBuffer.cpp	(revision 219600)
>+++ Source/JavaScriptCore/runtime/ArrayBuffer.cpp	(working copy)
>@@ -180,11 +180,21 @@ Ref<ArrayBuffer> ArrayBuffer::create(Arr
>     return adoptRef(*new ArrayBuffer(WTFMove(contents)));
> }
> 
>+// FIXME: We cannot use this except if the memory comes from the cage.
>+// Current this is only used from:
>+// - JSGenericTypedArrayView<>::slowDownAndWasteMemory. But in that case, the memory should have already come
>+//   from the cage.
> Ref<ArrayBuffer> ArrayBuffer::createAdopted(const void* data, unsigned byteLength)
> {
>     return createFromBytes(data, byteLength, [] (void* p) { fastFree(p); });
> }
> 
>+// FIXME: We cannot use this except if the memory comes from the cage.
>+// Currently this is only used from:
>+// - The C API. We could support that by either having the system switch to a mode where typed arrays are no
>+//   longer caged, or we could introduce a new set of typed array types that are uncaged and get accessed
>+//   differently.
>+// - WebAssembly. Wasm should allocate from the cage.
> Ref<ArrayBuffer> ArrayBuffer::createFromBytes(const void* data, unsigned byteLength, ArrayBufferDestructorFunction&& destructor)
> {
>     ArrayBufferContents contents(const_cast<void*>(data), byteLength, WTFMove(destructor));
>Index: Source/JavaScriptCore/runtime/JSArrayBuffer.cpp
>===================================================================
>--- Source/JavaScriptCore/runtime/JSArrayBuffer.cpp	(revision 219600)
>+++ Source/JavaScriptCore/runtime/JSArrayBuffer.cpp	(working copy)
>@@ -52,6 +52,9 @@ void JSArrayBuffer::finishCreation(VM& v
> JSArrayBuffer* JSArrayBuffer::create(
>     VM& vm, Structure* structure, RefPtr<ArrayBuffer>&& buffer)
> {
>+    if (!Gigacage::isCaged(buffer->data()))
>+        vm.gigacageEnabled().fireAll("JSArrayBuffer::create() saw uncaged data");
>+    
>     JSArrayBuffer* result =
>         new (NotNull, allocateCell<JSArrayBuffer>(vm.heap))
>         JSArrayBuffer(vm, structure, WTFMove(buffer));
>Index: Source/JavaScriptCore/runtime/VM.cpp
>===================================================================
>--- Source/JavaScriptCore/runtime/VM.cpp	(revision 219600)
>+++ Source/JavaScriptCore/runtime/VM.cpp	(working copy)
>@@ -109,6 +109,7 @@
> #include "Watchdog.h"
> #include "WeakGCMapInlines.h"
> #include "WeakMapData.h"
>+#include <bmalloc/bmalloc.h>
> #include <wtf/CurrentTime.h>
> #include <wtf/ProcessID.h>
> #include <wtf/SimpleStats.h>
>@@ -207,6 +208,7 @@ VM::VM(VMType vmType, HeapType heapType)
>     , m_codeCache(std::make_unique<CodeCache>())
>     , m_builtinExecutables(std::make_unique<BuiltinExecutables>(*this))
>     , m_typeProfilerEnabledCount(0)
>+    , m_gigacageEnabled(IsWatched)
>     , m_controlFlowProfilerEnabledCount(0)
>     , m_shadowChicken(std::make_unique<ShadowChicken>())
> {
>@@ -283,6 +285,11 @@ VM::VM(VMType vmType, HeapType heapType)
> #if ENABLE(JIT)
>     initializeHostCallReturnValue(); // This is needed to convince the linker not to drop host call return support.
> #endif
>+    
>+    bmalloc::api::ensureGigapage();
>+    
>+    if (!g_gigacageBasePtr)
>+        m_gigacageEnabled.fireAll(*this, "Gigacage was never enabled");
> 
>     heap.notifyIsSafeToCollect();
>     
>Index: Source/JavaScriptCore/runtime/VM.h
>===================================================================
>--- Source/JavaScriptCore/runtime/VM.h	(revision 219600)
>+++ Source/JavaScriptCore/runtime/VM.h	(working copy)
>@@ -36,6 +36,7 @@
> #include "ExceptionEventLocation.h"
> #include "ExecutableAllocator.h"
> #include "FunctionHasExecutedCache.h"
>+#include "GigacageSubspace.h"
> #include "Heap.h"
> #include "Intrinsic.h"
> #include "JITThunks.h"
>@@ -285,7 +286,7 @@ private:
> public:
>     Heap heap;
>     
>-    Subspace auxiliarySpace;
>+    GigacageSubspace auxiliarySpace;
>     
>     // Whenever possible, use subspaceFor<CellType>(vm) to get one of these subspaces.
>     Subspace cellSpace;
>@@ -622,6 +623,8 @@ public:
>     
>     // FIXME: Use AtomicString once it got merged with Identifier.
>     JS_EXPORT_PRIVATE void addImpureProperty(const String&);
>+    
>+    InlineWatchpointSet& gigacageEnabled() { return m_gigacageEnabled; }
> 
>     BuiltinExecutables* builtinExecutables() { return m_builtinExecutables.get(); }
> 
>@@ -772,6 +775,7 @@ private:
>     std::unique_ptr<TypeProfiler> m_typeProfiler;
>     std::unique_ptr<TypeProfilerLog> m_typeProfilerLog;
>     unsigned m_typeProfilerEnabledCount;
>+    InlineWatchpointSet m_gigacageEnabled;
>     FunctionHasExecutedCache m_functionHasExecutedCache;
>     std::unique_ptr<ControlFlowProfiler> m_controlFlowProfiler;
>     unsigned m_controlFlowProfilerEnabledCount;
>Index: Source/WTF/wtf/FastMalloc.cpp
>===================================================================
>--- Source/WTF/wtf/FastMalloc.cpp	(revision 219600)
>+++ Source/WTF/wtf/FastMalloc.cpp	(working copy)
>@@ -1,6 +1,6 @@
> /*
>  * Copyright (c) 2005, 2007, Google Inc. All rights reserved.
>- * Copyright (C) 2005-2009, 2011, 2015-2016 Apple Inc. All rights reserved.
>+ * Copyright (C) 2005-2017 Apple Inc. All rights reserved.
>  * Redistribution and use in source and binary forms, with or without
>  * modification, are permitted provided that the following conditions
>  * are met:
>@@ -249,13 +249,13 @@ namespace WTF {
> 
> bool isFastMallocEnabled()
> {
>-    return bmalloc::api::isEnabled();
>+    return bmalloc::api::isEnabled(bmalloc::HeapKind::Primary);
> }
> 
> void* fastMalloc(size_t size)
> {
>     ASSERT_IS_WITHIN_LIMIT(size);
>-    return bmalloc::api::malloc(size);
>+    return bmalloc::api::malloc(bmalloc::HeapKind::Primary, size);
> }
> 
> void* fastCalloc(size_t numElements, size_t elementSize)
>@@ -272,12 +272,12 @@ void* fastCalloc(size_t numElements, siz
> void* fastRealloc(void* object, size_t size)
> {
>     ASSERT_IS_WITHIN_LIMIT(size);
>-    return bmalloc::api::realloc(object, size);
>+    return bmalloc::api::realloc(bmalloc::HeapKind::Primary, object, size);
> }
> 
> void fastFree(void* object)
> {
>-    bmalloc::api::free(object);
>+    bmalloc::api::free(bmalloc::HeapKind::Primary, object);
> }
> 
> size_t fastMallocSize(const void*)
>@@ -296,24 +296,24 @@ size_t fastMallocGoodSize(size_t size)
> void* fastAlignedMalloc(size_t alignment, size_t size) 
> {
>     ASSERT_IS_WITHIN_LIMIT(size);
>-    return bmalloc::api::memalign(alignment, size);
>+    return bmalloc::api::memalign(bmalloc::HeapKind::Primary, alignment, size);
> }
> 
> void* tryFastAlignedMalloc(size_t alignment, size_t size) 
> {
>     FAIL_IF_EXCEEDS_LIMIT(size);
>-    return bmalloc::api::tryMemalign(alignment, size);
>+    return bmalloc::api::tryMemalign(bmalloc::HeapKind::Primary, alignment, size);
> }
> 
> void fastAlignedFree(void* p) 
> {
>-    bmalloc::api::free(p);
>+    bmalloc::api::free(bmalloc::HeapKind::Primary, p);
> }
> 
> TryMallocReturnValue tryFastMalloc(size_t size)
> {
>     FAIL_IF_EXCEEDS_LIMIT(size);
>-    return bmalloc::api::tryMalloc(size);
>+    return bmalloc::api::tryMalloc(bmalloc::HeapKind::Primary, size);
> }
>     
> TryMallocReturnValue tryFastCalloc(size_t numElements, size_t elementSize)
>Index: Source/bmalloc/bmalloc.xcodeproj/project.pbxproj
>===================================================================
>--- Source/bmalloc/bmalloc.xcodeproj/project.pbxproj	(revision 219600)
>+++ Source/bmalloc/bmalloc.xcodeproj/project.pbxproj	(working copy)
>@@ -7,6 +7,13 @@
> 	objects = {
> 
> /* Begin PBXBuildFile section */
>+		0F5BF1471F22A8B10029D91D /* HeapKind.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F5BF1461F22A8B10029D91D /* HeapKind.h */; settings = {ATTRIBUTES = (Private, ); }; };
>+		0F5BF1491F22A8D80029D91D /* PerHeapKind.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F5BF1481F22A8D80029D91D /* PerHeapKind.h */; settings = {ATTRIBUTES = (Private, ); }; };
>+		0F5BF14D1F22B0C30029D91D /* Gigacage.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F5BF14C1F22B0C30029D91D /* Gigacage.h */; settings = {ATTRIBUTES = (Private, ); }; };
>+		0F5BF14F1F22DEAF0029D91D /* Gigacage.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 0F5BF14E1F22DEAF0029D91D /* Gigacage.cpp */; };
>+		0F5BF1521F22E1570029D91D /* Scavenger.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 0F5BF1501F22E1570029D91D /* Scavenger.cpp */; };
>+		0F5BF1531F22E1570029D91D /* Scavenger.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F5BF1511F22E1570029D91D /* Scavenger.h */; settings = {ATTRIBUTES = (Private, ); }; };
>+		0F5BF1731F23C5710029D91D /* BExport.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F5BF1721F23C5710029D91D /* BExport.h */; settings = {ATTRIBUTES = (Private, ); }; };
> 		1400274918F89C1300115C97 /* Heap.h in Headers */ = {isa = PBXBuildFile; fileRef = 14DA320C18875B09007269E0 /* Heap.h */; settings = {ATTRIBUTES = (Private, ); }; };
> 		1400274A18F89C2300115C97 /* VMHeap.h in Headers */ = {isa = PBXBuildFile; fileRef = 144F7BFC18BFC517003537F3 /* VMHeap.h */; settings = {ATTRIBUTES = (Private, ); }; };
> 		140FA00319CE429C00FFD3C8 /* BumpRange.h in Headers */ = {isa = PBXBuildFile; fileRef = 140FA00219CE429C00FFD3C8 /* BumpRange.h */; settings = {ATTRIBUTES = (Private, ); }; };
>@@ -75,6 +82,13 @@
> /* End PBXContainerItemProxy section */
> 
> /* Begin PBXFileReference section */
>+		0F5BF1461F22A8B10029D91D /* HeapKind.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = HeapKind.h; path = bmalloc/HeapKind.h; sourceTree = "<group>"; };
>+		0F5BF1481F22A8D80029D91D /* PerHeapKind.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = PerHeapKind.h; path = bmalloc/PerHeapKind.h; sourceTree = "<group>"; };
>+		0F5BF14C1F22B0C30029D91D /* Gigacage.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = Gigacage.h; path = bmalloc/Gigacage.h; sourceTree = "<group>"; };
>+		0F5BF14E1F22DEAF0029D91D /* Gigacage.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = Gigacage.cpp; path = bmalloc/Gigacage.cpp; sourceTree = "<group>"; };
>+		0F5BF1501F22E1570029D91D /* Scavenger.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = Scavenger.cpp; path = bmalloc/Scavenger.cpp; sourceTree = "<group>"; };
>+		0F5BF1511F22E1570029D91D /* Scavenger.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = Scavenger.h; path = bmalloc/Scavenger.h; sourceTree = "<group>"; };
>+		0F5BF1721F23C5710029D91D /* BExport.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = BExport.h; path = bmalloc/BExport.h; sourceTree = "<group>"; };
> 		140FA00219CE429C00FFD3C8 /* BumpRange.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = BumpRange.h; path = bmalloc/BumpRange.h; sourceTree = "<group>"; };
> 		140FA00419CE4B6800FFD3C8 /* LineMetadata.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = LineMetadata.h; path = bmalloc/LineMetadata.h; sourceTree = "<group>"; };
> 		14105E8318E14374003A106E /* ObjectType.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = ObjectType.cpp; path = bmalloc/ObjectType.cpp; sourceTree = "<group>"; };
>@@ -241,12 +255,16 @@
> 				142B44351E2839E7001DA6E9 /* DebugHeap.h */,
> 				14895D8F1A3A319C0006235D /* Environment.cpp */,
> 				14895D901A3A319C0006235D /* Environment.h */,
>+				0F5BF14E1F22DEAF0029D91D /* Gigacage.cpp */,
>+				0F5BF14C1F22B0C30029D91D /* Gigacage.h */,
> 				14DA320E18875D9F007269E0 /* Heap.cpp */,
> 				14DA320C18875B09007269E0 /* Heap.h */,
> 				140FA00419CE4B6800FFD3C8 /* LineMetadata.h */,
> 				144BE11E1CA346520099C8C0 /* Object.h */,
> 				14105E8318E14374003A106E /* ObjectType.cpp */,
> 				1485656018A43DBA00ED6942 /* ObjectType.h */,
>+				0F5BF1501F22E1570029D91D /* Scavenger.cpp */,
>+				0F5BF1511F22E1570029D91D /* Scavenger.h */,
> 				145F6874179DF84100D65598 /* Sizes.h */,
> 				144F7BFB18BFC517003537F3 /* VMHeap.cpp */,
> 				144F7BFC18BFC517003537F3 /* VMHeap.h */,
>@@ -265,14 +283,17 @@
> 				6599C5CA1EC3F15900A2F7BB /* AvailableMemory.cpp */,
> 				6599C5CB1EC3F15900A2F7BB /* AvailableMemory.h */,
> 				1413E468189EEDE400546D68 /* BAssert.h */,
>+				0F5BF1721F23C5710029D91D /* BExport.h */,
> 				14C919C818FCC59F0028DB43 /* BPlatform.h */,
> 				14D9DB4517F2447100EAAB79 /* FixedVector.h */,
>+				0F5BF1461F22A8B10029D91D /* HeapKind.h */,
> 				1413E460189DCE1E00546D68 /* Inline.h */,
> 				141D9AFF1C8E51C0000ABBA0 /* List.h */,
> 				4426E27E1C838EE0008EB042 /* Logging.cpp */,
> 				4426E27F1C838EE0008EB042 /* Logging.h */,
> 				14C8992A1CC485E70027A057 /* Map.h */,
> 				144DCED617A649D90093B2F2 /* Mutex.h */,
>+				0F5BF1481F22A8D80029D91D /* PerHeapKind.h */,
> 				14446A0717A61FA400F9EA1D /* PerProcess.h */,
> 				144469FD17A61F1F00F9EA1D /* PerThread.h */,
> 				145F6878179E3A4400D65598 /* Range.h */,
>@@ -310,6 +331,8 @@
> 			files = (
> 				14DD78C518F48D7500950702 /* Algorithm.h in Headers */,
> 				14DD789818F48D4A00950702 /* Allocator.h in Headers */,
>+				0F5BF1531F22E1570029D91D /* Scavenger.h in Headers */,
>+				0F5BF1471F22A8B10029D91D /* HeapKind.h in Headers */,
> 				14DD78C618F48D7500950702 /* AsyncTask.h in Headers */,
> 				6599C5CD1EC3F15900A2F7BB /* AvailableMemory.h in Headers */,
> 				14DD78C718F48D7500950702 /* BAssert.h in Headers */,
>@@ -320,11 +343,13 @@
> 				140FA00319CE429C00FFD3C8 /* BumpRange.h in Headers */,
> 				14DD789918F48D4A00950702 /* Cache.h in Headers */,
> 				147DC6E31CA5B70B00724E8D /* Chunk.h in Headers */,
>+				0F5BF1731F23C5710029D91D /* BExport.h in Headers */,
> 				14DD789A18F48D4A00950702 /* Deallocator.h in Headers */,
> 				142B44371E2839E7001DA6E9 /* DebugHeap.h in Headers */,
> 				14895D921A3A319C0006235D /* Environment.h in Headers */,
> 				14DD78C818F48D7500950702 /* FixedVector.h in Headers */,
> 				1400274918F89C1300115C97 /* Heap.h in Headers */,
>+				0F5BF1491F22A8D80029D91D /* PerHeapKind.h in Headers */,
> 				14DD78C918F48D7500950702 /* Inline.h in Headers */,
> 				144C07F51C7B70260051BB6A /* LargeMap.h in Headers */,
> 				14C8992D1CC578330027A057 /* LargeRange.h in Headers */,
>@@ -344,6 +369,7 @@
> 				14DD78BD18F48D6B00950702 /* SmallPage.h in Headers */,
> 				143CB81D19022BC900B16A45 /* StaticMutex.h in Headers */,
> 				14DD78CE18F48D7500950702 /* Syscall.h in Headers */,
>+				0F5BF14D1F22B0C30029D91D /* Gigacage.h in Headers */,
> 				14DD78CF18F48D7500950702 /* Vector.h in Headers */,
> 				14DD78D018F48D7500950702 /* VMAllocate.h in Headers */,
> 				1400274A18F89C2300115C97 /* VMHeap.h in Headers */,
>@@ -429,6 +455,7 @@
> 			isa = PBXSourcesBuildPhase;
> 			buildActionMask = 2147483647;
> 			files = (
>+				0F5BF1521F22E1570029D91D /* Scavenger.cpp in Sources */,
> 				14F271C318EA3978008C152F /* Allocator.cpp in Sources */,
> 				6599C5CC1EC3F15900A2F7BB /* AvailableMemory.cpp in Sources */,
> 				14F271C418EA397B008C152F /* Cache.cpp in Sources */,
>@@ -436,6 +463,7 @@
> 				142B44361E2839E7001DA6E9 /* DebugHeap.cpp in Sources */,
> 				14895D911A3A319C0006235D /* Environment.cpp in Sources */,
> 				14F271C718EA3990008C152F /* Heap.cpp in Sources */,
>+				0F5BF14F1F22DEAF0029D91D /* Gigacage.cpp in Sources */,
> 				144C07F41C7B70260051BB6A /* LargeMap.cpp in Sources */,
> 				4426E2801C838EE0008EB042 /* Logging.cpp in Sources */,
> 				14F271C818EA3990008C152F /* ObjectType.cpp in Sources */,
>Index: Source/bmalloc/bmalloc/Allocator.cpp
>===================================================================
>--- Source/bmalloc/bmalloc/Allocator.cpp	(revision 219600)
>+++ Source/bmalloc/bmalloc/Allocator.cpp	(working copy)
>@@ -38,8 +38,9 @@ using namespace std;
> 
> namespace bmalloc {
> 
>-Allocator::Allocator(Heap* heap, Deallocator& deallocator)
>-    : m_debugHeap(heap->debugHeap())
>+Allocator::Allocator(Heap& heap, Deallocator& deallocator)
>+    : m_heap(heap)
>+    , m_debugHeap(heap.debugHeap())
>     , m_deallocator(deallocator)
> {
>     for (size_t sizeClass = 0; sizeClass < sizeClassCount; ++sizeClass)
>@@ -59,8 +60,8 @@ void* Allocator::tryAllocate(size_t size
>     if (size <= smallMax)
>         return allocate(size);
> 
>-    std::lock_guard<StaticMutex> lock(PerProcess<Heap>::mutex());
>-    return PerProcess<Heap>::getFastCase()->tryAllocateLarge(lock, alignment, size);
>+    std::lock_guard<StaticMutex> lock(Heap::mutex());
>+    return m_heap.tryAllocateLarge(lock, alignment, size);
> }
> 
> void* Allocator::allocate(size_t alignment, size_t size)
>@@ -88,11 +89,10 @@ void* Allocator::allocateImpl(size_t ali
>     if (size <= smallMax && alignment <= smallMax)
>         return allocate(roundUpToMultipleOf(alignment, size));
> 
>-    std::lock_guard<StaticMutex> lock(PerProcess<Heap>::mutex());
>-    Heap* heap = PerProcess<Heap>::getFastCase();
>+    std::lock_guard<StaticMutex> lock(Heap::mutex());
>     if (crashOnFailure)
>-        return heap->allocateLarge(lock, alignment, size);
>-    return heap->tryAllocateLarge(lock, alignment, size);
>+        return m_heap.allocateLarge(lock, alignment, size);
>+    return m_heap.tryAllocateLarge(lock, alignment, size);
> }
> 
> void* Allocator::reallocate(void* object, size_t newSize)
>@@ -101,9 +101,9 @@ void* Allocator::reallocate(void* object
>         return m_debugHeap->realloc(object, newSize);
> 
>     size_t oldSize = 0;
>-    switch (objectType(object)) {
>+    switch (objectType(m_heap.kind(), object)) {
>     case ObjectType::Small: {
>-        BASSERT(objectType(nullptr) == ObjectType::Small);
>+        BASSERT(objectType(m_heap.kind(), nullptr) == ObjectType::Small);
>         if (!object)
>             break;
> 
>@@ -112,11 +112,11 @@ void* Allocator::reallocate(void* object
>         break;
>     }
>     case ObjectType::Large: {
>-        std::lock_guard<StaticMutex> lock(PerProcess<Heap>::mutex());
>-        oldSize = PerProcess<Heap>::getFastCase()->largeSize(lock, object);
>+        std::lock_guard<StaticMutex> lock(Heap::mutex());
>+        oldSize = m_heap.largeSize(lock, object);
> 
>         if (newSize < oldSize && newSize > smallMax) {
>-            PerProcess<Heap>::getFastCase()->shrinkLarge(lock, Range(object, oldSize), newSize);
>+            m_heap.shrinkLarge(lock, Range(object, oldSize), newSize);
>             return object;
>         }
>         break;
>@@ -153,10 +153,9 @@ NO_INLINE void Allocator::refillAllocato
> {
>     BumpRangeCache& bumpRangeCache = m_bumpRangeCaches[sizeClass];
> 
>-    std::lock_guard<StaticMutex> lock(PerProcess<Heap>::mutex());
>+    std::lock_guard<StaticMutex> lock(Heap::mutex());
>     m_deallocator.processObjectLog(lock);
>-    PerProcess<Heap>::getFastCase()->allocateSmallBumpRanges(
>-        lock, sizeClass, allocator, bumpRangeCache, m_deallocator.lineCache(lock));
>+    m_heap.allocateSmallBumpRanges(lock, sizeClass, allocator, bumpRangeCache, m_deallocator.lineCache(lock));
> }
> 
> INLINE void Allocator::refillAllocator(BumpAllocator& allocator, size_t sizeClass)
>@@ -169,8 +168,8 @@ INLINE void Allocator::refillAllocator(B
> 
> NO_INLINE void* Allocator::allocateLarge(size_t size)
> {
>-    std::lock_guard<StaticMutex> lock(PerProcess<Heap>::mutex());
>-    return PerProcess<Heap>::getFastCase()->allocateLarge(lock, alignment, size);
>+    std::lock_guard<StaticMutex> lock(Heap::mutex());
>+    return m_heap.allocateLarge(lock, alignment, size);
> }
> 
> NO_INLINE void* Allocator::allocateLogSizeClass(size_t size)
>Index: Source/bmalloc/bmalloc/Allocator.h
>===================================================================
>--- Source/bmalloc/bmalloc/Allocator.h	(revision 219600)
>+++ Source/bmalloc/bmalloc/Allocator.h	(working copy)
>@@ -26,6 +26,7 @@
> #ifndef Allocator_h
> #define Allocator_h
> 
>+#include "BExport.h"
> #include "BumpAllocator.h"
> #include <array>
> 
>@@ -39,7 +40,7 @@ class Heap;
> 
> class Allocator {
> public:
>-    Allocator(Heap*, Deallocator&);
>+    Allocator(Heap&, Deallocator&);
>     ~Allocator();
> 
>     void* tryAllocate(size_t);
>@@ -54,7 +55,7 @@ private:
>     void* allocateImpl(size_t alignment, size_t, bool crashOnFailure);
>     
>     bool allocateFastCase(size_t, void*&);
>-    void* allocateSlowCase(size_t);
>+    BEXPORT void* allocateSlowCase(size_t);
>     
>     void* allocateLogSizeClass(size_t);
>     void* allocateLarge(size_t);
>@@ -65,6 +66,7 @@ private:
>     std::array<BumpAllocator, sizeClassCount> m_bumpAllocators;
>     std::array<BumpRangeCache, sizeClassCount> m_bumpRangeCaches;
> 
>+    Heap& m_heap;
>     DebugHeap* m_debugHeap;
>     Deallocator& m_deallocator;
> };
>Index: Source/bmalloc/bmalloc/BExport.h
>===================================================================
>--- Source/bmalloc/bmalloc/BExport.h	(nonexistent)
>+++ Source/bmalloc/bmalloc/BExport.h	(working copy)
>@@ -0,0 +1,29 @@
>+/*
>+ * Copyright (C) 2017 Apple Inc. All rights reserved.
>+ *
>+ * Redistribution and use in source and binary forms, with or without
>+ * modification, are permitted provided that the following conditions
>+ * are met:
>+ * 1. Redistributions of source code must retain the above copyright
>+ *    notice, this list of conditions and the following disclaimer.
>+ * 2. Redistributions in binary form must reproduce the above copyright
>+ *    notice, this list of conditions and the following disclaimer in the
>+ *    documentation and/or other materials provided with the distribution.
>+ *
>+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
>+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
>+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
>+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
>+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
>+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
>+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
>+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
>+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
>+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
>+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
>+ */
>+
>+#pragma once
>+
>+#define BEXPORT __attribute__((visibility("default")))
>+
>Index: Source/bmalloc/bmalloc/Cache.cpp
>===================================================================
>--- Source/bmalloc/bmalloc/Cache.cpp	(revision 219600)
>+++ Source/bmalloc/bmalloc/Cache.cpp	(working copy)
>@@ -1,5 +1,5 @@
> /*
>- * Copyright (C) 2014, 2015 Apple Inc. All rights reserved.
>+ * Copyright (C) 2014-2017 Apple Inc. All rights reserved.
>  *
>  * Redistribution and use in source and binary forms, with or without
>  * modification, are permitted provided that the following conditions
>@@ -30,55 +30,45 @@
> 
> namespace bmalloc {
> 
>-void* Cache::operator new(size_t size)
>+void Cache::scavenge(HeapKind heapKind)
> {
>-    return vmAllocate(vmSize(size));
>-}
>-
>-void Cache::operator delete(void* p, size_t size)
>-{
>-    vmDeallocate(p, vmSize(size));
>-}
>-
>-void Cache::scavenge()
>-{
>-    Cache* cache = PerThread<Cache>::getFastCase();
>-    if (!cache)
>+    PerHeapKind<Cache>* caches = PerThread<PerHeapKind<Cache>>::getFastCase();
>+    if (!caches)
>         return;
> 
>-    cache->allocator().scavenge();
>-    cache->deallocator().scavenge();
>+    caches->at(heapKind).allocator().scavenge();
>+    caches->at(heapKind).deallocator().scavenge();
> }
> 
>-Cache::Cache()
>-    : m_deallocator(PerProcess<Heap>::get())
>-    , m_allocator(PerProcess<Heap>::get(), m_deallocator)
>+Cache::Cache(HeapKind heapKind)
>+    : m_deallocator(PerProcess<PerHeapKind<Heap>>::get()->at(heapKind))
>+    , m_allocator(PerProcess<PerHeapKind<Heap>>::get()->at(heapKind), m_deallocator)
> {
> }
> 
>-NO_INLINE void* Cache::tryAllocateSlowCaseNullCache(size_t size)
>+NO_INLINE void* Cache::tryAllocateSlowCaseNullCache(HeapKind heapKind, size_t size)
> {
>-    return PerThread<Cache>::getSlowCase()->allocator().tryAllocate(size);
>+    return PerThread<PerHeapKind<Cache>>::getSlowCase()->at(heapKind).allocator().tryAllocate(size);
> }
> 
>-NO_INLINE void* Cache::allocateSlowCaseNullCache(size_t size)
>+NO_INLINE void* Cache::allocateSlowCaseNullCache(HeapKind heapKind, size_t size)
> {
>-    return PerThread<Cache>::getSlowCase()->allocator().allocate(size);
>+    return PerThread<PerHeapKind<Cache>>::getSlowCase()->at(heapKind).allocator().allocate(size);
> }
> 
>-NO_INLINE void* Cache::allocateSlowCaseNullCache(size_t alignment, size_t size)
>+NO_INLINE void* Cache::allocateSlowCaseNullCache(HeapKind heapKind, size_t alignment, size_t size)
> {
>-    return PerThread<Cache>::getSlowCase()->allocator().allocate(alignment, size);
>+    return PerThread<PerHeapKind<Cache>>::getSlowCase()->at(heapKind).allocator().allocate(alignment, size);
> }
> 
>-NO_INLINE void Cache::deallocateSlowCaseNullCache(void* object)
>+NO_INLINE void Cache::deallocateSlowCaseNullCache(HeapKind heapKind, void* object)
> {
>-    PerThread<Cache>::getSlowCase()->deallocator().deallocate(object);
>+    PerThread<PerHeapKind<Cache>>::getSlowCase()->at(heapKind).deallocator().deallocate(object);
> }
> 
>-NO_INLINE void* Cache::reallocateSlowCaseNullCache(void* object, size_t newSize)
>+NO_INLINE void* Cache::reallocateSlowCaseNullCache(HeapKind heapKind, void* object, size_t newSize)
> {
>-    return PerThread<Cache>::getSlowCase()->allocator().reallocate(object, newSize);
>+    return PerThread<PerHeapKind<Cache>>::getSlowCase()->at(heapKind).allocator().reallocate(object, newSize);
> }
> 
> } // namespace bmalloc
>Index: Source/bmalloc/bmalloc/Cache.h
>===================================================================
>--- Source/bmalloc/bmalloc/Cache.h	(revision 219600)
>+++ Source/bmalloc/bmalloc/Cache.h	(working copy)
>@@ -1,5 +1,5 @@
> /*
>- * Copyright (C) 2014, 2016 Apple Inc. All rights reserved.
>+ * Copyright (C) 2014-2017 Apple Inc. All rights reserved.
>  *
>  * Redistribution and use in source and binary forms, with or without
>  * modification, are permitted provided that the following conditions
>@@ -27,7 +27,9 @@
> #define Cache_h
> 
> #include "Allocator.h"
>+#include "BExport.h"
> #include "Deallocator.h"
>+#include "HeapKind.h"
> #include "PerThread.h"
> 
> namespace bmalloc {
>@@ -36,80 +38,77 @@ namespace bmalloc {
> 
> class Cache {
> public:
>-    void* operator new(size_t);
>-    void operator delete(void*, size_t);
>+    static void* tryAllocate(HeapKind, size_t);
>+    static void* allocate(HeapKind, size_t);
>+    static void* tryAllocate(HeapKind, size_t alignment, size_t);
>+    static void* allocate(HeapKind, size_t alignment, size_t);
>+    static void deallocate(HeapKind, void*);
>+    static void* reallocate(HeapKind, void*, size_t);
> 
>-    static void* tryAllocate(size_t);
>-    static void* allocate(size_t);
>-    static void* tryAllocate(size_t alignment, size_t);
>-    static void* allocate(size_t alignment, size_t);
>-    static void deallocate(void*);
>-    static void* reallocate(void*, size_t);
>+    static void scavenge(HeapKind);
> 
>-    static void scavenge();
>-
>-    Cache();
>+    Cache(HeapKind);
> 
>     Allocator& allocator() { return m_allocator; }
>     Deallocator& deallocator() { return m_deallocator; }
> 
> private:
>-    static void* tryAllocateSlowCaseNullCache(size_t);
>-    static void* allocateSlowCaseNullCache(size_t);
>-    static void* allocateSlowCaseNullCache(size_t alignment, size_t);
>-    static void deallocateSlowCaseNullCache(void*);
>-    static void* reallocateSlowCaseNullCache(void*, size_t);
>+    BEXPORT static void* tryAllocateSlowCaseNullCache(HeapKind, size_t);
>+    BEXPORT static void* allocateSlowCaseNullCache(HeapKind, size_t);
>+    BEXPORT static void* allocateSlowCaseNullCache(HeapKind, size_t alignment, size_t);
>+    BEXPORT static void deallocateSlowCaseNullCache(HeapKind, void*);
>+    BEXPORT static void* reallocateSlowCaseNullCache(HeapKind, void*, size_t);
> 
>     Deallocator m_deallocator;
>     Allocator m_allocator;
> };
> 
>-inline void* Cache::tryAllocate(size_t size)
>+inline void* Cache::tryAllocate(HeapKind heapKind, size_t size)
> {
>-    Cache* cache = PerThread<Cache>::getFastCase();
>-    if (!cache)
>-        return tryAllocateSlowCaseNullCache(size);
>-    return cache->allocator().tryAllocate(size);
>+    PerHeapKind<Cache>* caches = PerThread<PerHeapKind<Cache>>::getFastCase();
>+    if (!caches)
>+        return tryAllocateSlowCaseNullCache(heapKind, size);
>+    return caches->at(heapKind).allocator().tryAllocate(size);
> }
> 
>-inline void* Cache::allocate(size_t size)
>+inline void* Cache::allocate(HeapKind heapKind, size_t size)
> {
>-    Cache* cache = PerThread<Cache>::getFastCase();
>-    if (!cache)
>-        return allocateSlowCaseNullCache(size);
>-    return cache->allocator().allocate(size);
>+    PerHeapKind<Cache>* caches = PerThread<PerHeapKind<Cache>>::getFastCase();
>+    if (!caches)
>+        return allocateSlowCaseNullCache(heapKind, size);
>+    return caches->at(heapKind).allocator().allocate(size);
> }
> 
>-inline void* Cache::tryAllocate(size_t alignment, size_t size)
>+inline void* Cache::tryAllocate(HeapKind heapKind, size_t alignment, size_t size)
> {
>-    Cache* cache = PerThread<Cache>::getFastCase();
>-    if (!cache)
>-        return allocateSlowCaseNullCache(alignment, size);
>-    return cache->allocator().tryAllocate(alignment, size);
>+    PerHeapKind<Cache>* caches = PerThread<PerHeapKind<Cache>>::getFastCase();
>+    if (!caches)
>+        return allocateSlowCaseNullCache(heapKind, alignment, size);
>+    return caches->at(heapKind).allocator().tryAllocate(alignment, size);
> }
> 
>-inline void* Cache::allocate(size_t alignment, size_t size)
>+inline void* Cache::allocate(HeapKind heapKind, size_t alignment, size_t size)
> {
>-    Cache* cache = PerThread<Cache>::getFastCase();
>-    if (!cache)
>-        return allocateSlowCaseNullCache(alignment, size);
>-    return cache->allocator().allocate(alignment, size);
>+    PerHeapKind<Cache>* caches = PerThread<PerHeapKind<Cache>>::getFastCase();
>+    if (!caches)
>+        return allocateSlowCaseNullCache(heapKind, alignment, size);
>+    return caches->at(heapKind).allocator().allocate(alignment, size);
> }
> 
>-inline void Cache::deallocate(void* object)
>+inline void Cache::deallocate(HeapKind heapKind, void* object)
> {
>-    Cache* cache = PerThread<Cache>::getFastCase();
>-    if (!cache)
>-        return deallocateSlowCaseNullCache(object);
>-    return cache->deallocator().deallocate(object);
>+    PerHeapKind<Cache>* caches = PerThread<PerHeapKind<Cache>>::getFastCase();
>+    if (!caches)
>+        return deallocateSlowCaseNullCache(heapKind, object);
>+    return caches->at(heapKind).deallocator().deallocate(object);
> }
> 
>-inline void* Cache::reallocate(void* object, size_t newSize)
>+inline void* Cache::reallocate(HeapKind heapKind, void* object, size_t newSize)
> {
>-    Cache* cache = PerThread<Cache>::getFastCase();
>-    if (!cache)
>-        return reallocateSlowCaseNullCache(object, newSize);
>-    return cache->allocator().reallocate(object, newSize);
>+    PerHeapKind<Cache>* caches = PerThread<PerHeapKind<Cache>>::getFastCase();
>+    if (!caches)
>+        return reallocateSlowCaseNullCache(heapKind, object, newSize);
>+    return caches->at(heapKind).allocator().reallocate(object, newSize);
> }
> 
> } // namespace bmalloc
>Index: Source/bmalloc/bmalloc/Deallocator.cpp
>===================================================================
>--- Source/bmalloc/bmalloc/Deallocator.cpp	(revision 219600)
>+++ Source/bmalloc/bmalloc/Deallocator.cpp	(working copy)
>@@ -39,8 +39,9 @@ using namespace std;
> 
> namespace bmalloc {
> 
>-Deallocator::Deallocator(Heap* heap)
>-    : m_debugHeap(heap->debugHeap())
>+Deallocator::Deallocator(Heap& heap)
>+    : m_heap(heap)
>+    , m_debugHeap(heap.debugHeap())
> {
>     if (m_debugHeap) {
>         // Fill the object log in order to disable the fast path.
>@@ -59,18 +60,16 @@ void Deallocator::scavenge()
>     if (m_debugHeap)
>         return;
> 
>-    std::lock_guard<StaticMutex> lock(PerProcess<Heap>::mutex());
>+    std::lock_guard<StaticMutex> lock(Heap::mutex());
> 
>     processObjectLog(lock);
>-    PerProcess<Heap>::getFastCase()->deallocateLineCache(lock, lineCache(lock));
>+    m_heap.deallocateLineCache(lock, lineCache(lock));
> }
> 
> void Deallocator::processObjectLog(std::lock_guard<StaticMutex>& lock)
> {
>-    Heap* heap = PerProcess<Heap>::getFastCase();
>-    
>     for (Object object : m_objectLog)
>-        heap->derefSmallLine(lock, object, lineCache(lock));
>+        m_heap.derefSmallLine(lock, object, lineCache(lock));
>     m_objectLog.clear();
> }
> 
>@@ -82,9 +81,9 @@ void Deallocator::deallocateSlowCase(voi
>     if (!object)
>         return;
> 
>-    std::lock_guard<StaticMutex> lock(PerProcess<Heap>::mutex());
>-    if (PerProcess<Heap>::getFastCase()->isLarge(lock, object)) {
>-        PerProcess<Heap>::getFastCase()->deallocateLarge(lock, object);
>+    std::lock_guard<StaticMutex> lock(Heap::mutex());
>+    if (m_heap.isLarge(lock, object)) {
>+        m_heap.deallocateLarge(lock, object);
>         return;
>     }
> 
>Index: Source/bmalloc/bmalloc/Deallocator.h
>===================================================================
>--- Source/bmalloc/bmalloc/Deallocator.h	(revision 219600)
>+++ Source/bmalloc/bmalloc/Deallocator.h	(working copy)
>@@ -26,6 +26,7 @@
> #ifndef Deallocator_h
> #define Deallocator_h
> 
>+#include "BExport.h"
> #include "FixedVector.h"
> #include "SmallPage.h"
> #include <mutex>
>@@ -40,7 +41,7 @@ class StaticMutex;
> 
> class Deallocator {
> public:
>-    Deallocator(Heap*);
>+    Deallocator(Heap&);
>     ~Deallocator();
> 
>     void deallocate(void*);
>@@ -52,8 +53,9 @@ public:
> 
> private:
>     bool deallocateFastCase(void*);
>-    void deallocateSlowCase(void*);
>+    BEXPORT void deallocateSlowCase(void*);
> 
>+    Heap& m_heap;
>     FixedVector<void*, deallocatorLogCapacity> m_objectLog;
>     LineCache m_lineCache; // The Heap removes items from this cache.
>     DebugHeap* m_debugHeap;
>Index: Source/bmalloc/bmalloc/Gigacage.cpp
>===================================================================
>--- Source/bmalloc/bmalloc/Gigacage.cpp	(nonexistent)
>+++ Source/bmalloc/bmalloc/Gigacage.cpp	(working copy)
>@@ -0,0 +1,57 @@
>+/*
>+ * Copyright (C) 2014-2017 Apple Inc. All rights reserved.
>+ *
>+ * Redistribution and use in source and binary forms, with or without
>+ * modification, are permitted provided that the following conditions
>+ * are met:
>+ * 1. Redistributions of source code must retain the above copyright
>+ *    notice, this list of conditions and the following disclaimer.
>+ * 2. Redistributions in binary form must reproduce the above copyright
>+ *    notice, this list of conditions and the following disclaimer in the
>+ *    documentation and/or other materials provided with the distribution.
>+ *
>+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
>+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
>+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
>+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
>+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
>+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
>+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
>+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
>+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
>+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
>+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
>+ */
>+
>+#include "Gigacage.h"
>+
>+#include "VMAllocate.h"
>+#include "bmalloc.h"
>+#include <mutex>
>+
>+// FIXME: Ask dyld to put this in its own page, and mprotect the page after we ensure the gigacage.
>+// OOPS! (need bug title)
>+void* g_gigacageBasePtr;
>+
>+namespace bmalloc { namespace api {
>+
>+void ensureGigacage()
>+{
>+#if BOS(DARWIN) && BCPU(X86_64)
>+    std::once_flag onceFlag;
>+    std::call_once(
>+        onceFlag,
>+        [] {
>+            void* basePtr = tryVMAllocate(GIGACAGE_SIZE, GIGACAGE_SIZE);
>+            if (!basePtr)
>+                return;
>+            
>+            vmDeallocatePhysicalPages(basePtr, GIGACAGE_SIZE);
>+            
>+            g_gigacageBasePtr = basePtr;
>+        });
>+#endif // BOS(DARWIN) && BCPU(X86_64)
>+}
>+
>+} } // namespace bmalloc::api
>+
>Index: Source/bmalloc/bmalloc/Gigacage.h
>===================================================================
>--- Source/bmalloc/bmalloc/Gigacage.h	(nonexistent)
>+++ Source/bmalloc/bmalloc/Gigacage.h	(working copy)
>@@ -0,0 +1,56 @@
>+/*
>+ * Copyright (C) 2014-2017 Apple Inc. All rights reserved.
>+ *
>+ * Redistribution and use in source and binary forms, with or without
>+ * modification, are permitted provided that the following conditions
>+ * are met:
>+ * 1. Redistributions of source code must retain the above copyright
>+ *    notice, this list of conditions and the following disclaimer.
>+ * 2. Redistributions in binary form must reproduce the above copyright
>+ *    notice, this list of conditions and the following disclaimer in the
>+ *    documentation and/or other materials provided with the distribution.
>+ *
>+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
>+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
>+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
>+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
>+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
>+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
>+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
>+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
>+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
>+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
>+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
>+ */
>+
>+#pragma once
>+
>+#include "BExport.h"
>+#include <inttypes.h>
>+
>+// The Gigacage is 64GB.
>+#define GIGACAGE_MASK 0xffffffffffllu
>+#define GIGACAGE_SIZE (GIGACAGE_MASK + 1)
>+
>+BEXPORT extern "C" void* g_gigacageBasePtr;
>+
>+namespace Gigacage {
>+
>+template<typename T>
>+T* caged(T* ptr)
>+{
>+    void* gigacageBasePtr = g_gigacageBasePtr;
>+    if (!gigacageBasePtr)
>+        return ptr;
>+    return reinterpret_cast<T*>(
>+        reinterpret_cast<uintptr_t>(gigacageBasePtr) + (
>+            reinterpret_cast<uintptr_t>(ptr) & static_cast<uintptr_t>(GIGACAGE_MASK)));
>+}
>+
>+inline bool isCaged(void* ptr)
>+{
>+    return caged(ptr) == ptr;
>+}
>+
>+}
>+
>Index: Source/bmalloc/bmalloc/Heap.cpp
>===================================================================
>--- Source/bmalloc/bmalloc/Heap.cpp	(revision 219600)
>+++ Source/bmalloc/bmalloc/Heap.cpp	(working copy)
>@@ -1,5 +1,5 @@
> /*
>- * Copyright (C) 2014-2016 Apple Inc. All rights reserved.
>+ * Copyright (C) 2014-2017 Apple Inc. All rights reserved.
>  *
>  * Redistribution and use in source and binary forms, with or without
>  * modification, are permitted provided that the following conditions
>@@ -28,16 +28,21 @@
> #include "AvailableMemory.h"
> #include "BumpAllocator.h"
> #include "Chunk.h"
>+#include "Gigacage.h"
> #include "DebugHeap.h"
> #include "PerProcess.h"
>+#include "Scavenger.h"
> #include "SmallLine.h"
> #include "SmallPage.h"
>+#include "VMHeap.h"
>+#include "bmalloc.h"
> #include <thread>
> 
> namespace bmalloc {
> 
>-Heap::Heap(std::lock_guard<StaticMutex>&)
>-    : m_vmPageSizePhysical(vmPageSizePhysical())
>+Heap::Heap(HeapKind kind, std::lock_guard<StaticMutex>&)
>+    : m_kind(kind)
>+    , m_vmPageSizePhysical(vmPageSizePhysical())
>     , m_scavenger(*this, &Heap::concurrentScavenge)
>     , m_debugHeap(nullptr)
> {
>@@ -49,17 +54,22 @@ Heap::Heap(std::lock_guard<StaticMutex>&
>     
>     if (m_environment.isDebugHeapEnabled())
>         m_debugHeap = PerProcess<DebugHeap>::get();
>+    else {
>+        if (m_kind == HeapKind::Gigacage)
>+            api::ensureGigacage();
>+        
>+        if (usingGigacage()) {
>+            RELEASE_BASSERT(g_gigacageBasePtr);
>+            m_largeFree.add(LargeRange(g_gigacageBasePtr, GIGACAGE_SIZE, 0));
>+        }
>+    }
>+    
>+    PerProcess<Scavenger>::get();
>+}
> 
>-#if BOS(DARWIN)
>-    auto queue = dispatch_queue_create("WebKit Malloc Memory Pressure Handler", DISPATCH_QUEUE_SERIAL);
>-    m_pressureHandlerDispatchSource = dispatch_source_create(DISPATCH_SOURCE_TYPE_MEMORYPRESSURE, 0, DISPATCH_MEMORYPRESSURE_CRITICAL, queue);
>-    dispatch_source_set_event_handler(m_pressureHandlerDispatchSource, ^{
>-        std::lock_guard<StaticMutex> lock(PerProcess<Heap>::mutex());
>-        scavenge(lock);
>-    });
>-    dispatch_resume(m_pressureHandlerDispatchSource);
>-    dispatch_release(queue);
>-#endif
>+bool Heap::usingGigacage()
>+{
>+    return m_kind == HeapKind::Gigacage && g_gigacageBasePtr;
> }
> 
> void Heap::initializeLineMetadata()
>@@ -120,10 +130,10 @@ void Heap::initializePageMetadata()
> 
> void Heap::concurrentScavenge()
> {
>-    std::lock_guard<StaticMutex> lock(PerProcess<Heap>::mutex());
>+    std::lock_guard<StaticMutex> lock(mutex());
> 
> #if BOS(DARWIN)
>-    pthread_set_qos_class_self_np(m_requestedScavengerThreadQOSClass, 0);
>+    pthread_set_qos_class_self_np(PerProcess<Scavenger>::get()->requestedScavengerThreadQOSClass(), 0);
> #endif
> 
>     if (m_isGrowing && !isUnderMemoryPressure()) {
>@@ -494,12 +504,15 @@ void* Heap::tryAllocateLarge(std::lock_g
> 
>     LargeRange range = m_largeFree.remove(alignment, size);
>     if (!range) {
>-        range = m_vmHeap.tryAllocateLargeChunk(alignment, size);
>-        if (!range)
>+        if (usingGigacage())
>             return nullptr;
> 
>+        range = PerProcess<VMHeap>::get()->tryAllocateLargeChunk(alignment, size);
>+        if (!range)
>+            return nullptr;
>+        
>         m_largeFree.add(range);
>-
>+        
>         range = m_largeFree.remove(alignment, size);
>     }
> 
>Index: Source/bmalloc/bmalloc/Heap.h
>===================================================================
>--- Source/bmalloc/bmalloc/Heap.h	(revision 219600)
>+++ Source/bmalloc/bmalloc/Heap.h	(working copy)
>@@ -1,5 +1,5 @@
> /*
>- * Copyright (C) 2014-2016 Apple Inc. All rights reserved.
>+ * Copyright (C) 2014-2017 Apple Inc. All rights reserved.
>  *
>  * Redistribution and use in source and binary forms, with or without
>  * modification, are permitted provided that the following conditions
>@@ -28,24 +28,23 @@
> 
> #include "AsyncTask.h"
> #include "BumpRange.h"
>+#include "Chunk.h"
> #include "Environment.h"
>+#include "HeapKind.h"
> #include "LargeMap.h"
> #include "LineMetadata.h"
> #include "List.h"
> #include "Map.h"
> #include "Mutex.h"
> #include "Object.h"
>+#include "PerHeapKind.h"
>+#include "PerProcess.h"
> #include "SmallLine.h"
> #include "SmallPage.h"
>-#include "VMHeap.h"
> #include "Vector.h"
> #include <array>
> #include <mutex>
> 
>-#if BOS(DARWIN)
>-#include <dispatch/dispatch.h>
>-#endif
>-
> namespace bmalloc {
> 
> class BeginTag;
>@@ -55,7 +54,11 @@ class EndTag;
> 
> class Heap {
> public:
>-    Heap(std::lock_guard<StaticMutex>&);
>+    Heap(HeapKind, std::lock_guard<StaticMutex>&);
>+    
>+    static StaticMutex& mutex() { return PerProcess<PerHeapKind<Heap>>::mutex(); }
>+    
>+    HeapKind kind() const { return m_kind; }
>     
>     DebugHeap* debugHeap() { return m_debugHeap; }
> 
>@@ -74,10 +77,6 @@ public:
> 
>     void scavenge(std::lock_guard<StaticMutex>&);
> 
>-#if BOS(DARWIN)
>-    void setScavengerThreadQOSClass(qos_class_t overrideClass) { m_requestedScavengerThreadQOSClass = overrideClass; }
>-#endif
>-
> private:
>     struct LargeObjectHash {
>         static unsigned hash(void* key)
>@@ -89,6 +88,8 @@ private:
> 
>     ~Heap() = delete;
>     
>+    bool usingGigacage();
>+    
>     void initializeLineMetadata();
>     void initializePageMetadata();
> 
>@@ -114,6 +115,8 @@ private:
>     
>     void concurrentScavenge();
>     
>+    HeapKind m_kind;
>+    
>     size_t m_vmPageSizePhysical;
>     Vector<LineMetadata> m_smallLineMetadata;
>     std::array<size_t, sizeClassCount> m_pageClasses;
>@@ -134,13 +137,6 @@ private:
> 
>     Environment m_environment;
>     DebugHeap* m_debugHeap;
>-
>-    VMHeap m_vmHeap;
>-
>-#if BOS(DARWIN)
>-    dispatch_source_t m_pressureHandlerDispatchSource;
>-    qos_class_t m_requestedScavengerThreadQOSClass { QOS_CLASS_USER_INITIATED };
>-#endif
> };
> 
> inline void Heap::allocateSmallBumpRanges(
>Index: Source/bmalloc/bmalloc/HeapKind.h
>===================================================================
>--- Source/bmalloc/bmalloc/HeapKind.h	(nonexistent)
>+++ Source/bmalloc/bmalloc/HeapKind.h	(working copy)
>@@ -0,0 +1,38 @@
>+/*
>+ * Copyright (C) 2017 Apple Inc. All rights reserved.
>+ *
>+ * Redistribution and use in source and binary forms, with or without
>+ * modification, are permitted provided that the following conditions
>+ * are met:
>+ * 1. Redistributions of source code must retain the above copyright
>+ *    notice, this list of conditions and the following disclaimer.
>+ * 2. Redistributions in binary form must reproduce the above copyright
>+ *    notice, this list of conditions and the following disclaimer in the
>+ *    documentation and/or other materials provided with the distribution.
>+ *
>+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
>+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
>+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
>+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
>+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
>+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
>+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
>+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
>+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
>+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
>+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
>+ */
>+
>+#pragma once
>+
>+namespace bmalloc {
>+
>+enum class HeapKind {
>+    Primary,
>+    Gigacage
>+};
>+
>+static const unsigned numHeaps = 2;
>+
>+} // namespace bmalloc
>+
>Index: Source/bmalloc/bmalloc/ObjectType.cpp
>===================================================================
>--- Source/bmalloc/bmalloc/ObjectType.cpp	(revision 219600)
>+++ Source/bmalloc/bmalloc/ObjectType.cpp	(working copy)
>@@ -1,5 +1,5 @@
> /*
>- * Copyright (C) 2014 Apple Inc. All rights reserved.
>+ * Copyright (C) 2014-2017 Apple Inc. All rights reserved.
>  *
>  * Redistribution and use in source and binary forms, with or without
>  * modification, are permitted provided that the following conditions
>@@ -32,14 +32,14 @@
> 
> namespace bmalloc {
> 
>-ObjectType objectType(void* object)
>+ObjectType objectType(HeapKind kind, void* object)
> {
>     if (mightBeLarge(object)) {
>         if (!object)
>             return ObjectType::Small;
> 
>-        std::lock_guard<StaticMutex> lock(PerProcess<Heap>::mutex());
>-        if (PerProcess<Heap>::getFastCase()->isLarge(lock, object))
>+        std::lock_guard<StaticMutex> lock(Heap::mutex());
>+        if (PerProcess<PerHeapKind<Heap>>::getFastCase()->at(kind).isLarge(lock, object))
>             return ObjectType::Large;
>     }
>     
>Index: Source/bmalloc/bmalloc/ObjectType.h
>===================================================================
>--- Source/bmalloc/bmalloc/ObjectType.h	(revision 219600)
>+++ Source/bmalloc/bmalloc/ObjectType.h	(working copy)
>@@ -1,5 +1,5 @@
> /*
>- * Copyright (C) 2014 Apple Inc. All rights reserved.
>+ * Copyright (C) 2014-2017 Apple Inc. All rights reserved.
>  *
>  * Redistribution and use in source and binary forms, with or without
>  * modification, are permitted provided that the following conditions
>@@ -27,13 +27,14 @@
> #define ObjectType_h
> 
> #include "BAssert.h"
>+#include "HeapKind.h"
> #include "Sizes.h"
> 
> namespace bmalloc {
> 
> enum class ObjectType : unsigned char { Small, Large };
> 
>-ObjectType objectType(void*);
>+ObjectType objectType(HeapKind, void*);
> 
> inline bool mightBeLarge(void* object)
> {
>Index: Source/bmalloc/bmalloc/PerHeapKind.h
>===================================================================
>--- Source/bmalloc/bmalloc/PerHeapKind.h	(nonexistent)
>+++ Source/bmalloc/bmalloc/PerHeapKind.h	(working copy)
>@@ -0,0 +1,106 @@
>+/*
>+ * Copyright (C) 2017 Apple Inc. All rights reserved.
>+ *
>+ * Redistribution and use in source and binary forms, with or without
>+ * modification, are permitted provided that the following conditions
>+ * are met:
>+ * 1. Redistributions of source code must retain the above copyright
>+ *    notice, this list of conditions and the following disclaimer.
>+ * 2. Redistributions in binary form must reproduce the above copyright
>+ *    notice, this list of conditions and the following disclaimer in the
>+ *    documentation and/or other materials provided with the distribution.
>+ *
>+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
>+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
>+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
>+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
>+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
>+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
>+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
>+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
>+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
>+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
>+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
>+ */
>+
>+#pragma once
>+
>+#include "HeapKind.h"
>+
>+namespace bmalloc {
>+
>+template<typename T>
>+class PerHeapKindBase {
>+public:
>+    PerHeapKindBase(const PerHeapKindBase&) = delete;
>+    PerHeapKindBase& operator=(const PerHeapKindBase&) = delete;
>+    
>+    template<typename... Arguments>
>+    PerHeapKindBase(Arguments&&... arguments)
>+    {
>+        for (unsigned i = numHeaps; i--;)
>+            new (&at(i)) T(static_cast<HeapKind>(i), std::forward<Arguments>(arguments)...);
>+    }
>+    
>+    static size_t size() { return numHeaps; }
>+    
>+    T& at(size_t i)
>+    {
>+        return reinterpret_cast<T*>(&m_memory)[i];
>+    }
>+    
>+    const T& at(size_t i) const
>+    {
>+        return reinterpret_cast<T*>(&m_memory)[i];
>+    }
>+    
>+    T& at(HeapKind heapKind)
>+    {
>+        return at(static_cast<size_t>(heapKind));
>+    }
>+    
>+    const T& at(HeapKind heapKind) const
>+    {
>+        return at(static_cast<size_t>(heapKind));
>+    }
>+    
>+    T& operator[](size_t i) { return at(i); }
>+    const T& operator[](size_t i) const { return at(i); }
>+    T& operator[](HeapKind heapKind) { return at(heapKind); }
>+    const T& operator[](HeapKind heapKind) const { return at(heapKind); }
>+
>+private:
>+    typedef typename std::aligned_storage<sizeof(T), std::alignment_of<T>::value> Memory[numHeaps];
>+    Memory m_memory;
>+};
>+
>+template<typename T>
>+class StaticPerHeapKind : public PerHeapKindBase<T> {
>+public:
>+    template<typename... Arguments>
>+    StaticPerHeapKind(Arguments&&... arguments)
>+        : PerHeapKindBase<T>(std::forward<Arguments>(arguments)...)
>+    {
>+    }
>+    
>+    ~StaticPerHeapKind() = delete;
>+};
>+
>+template<typename T>
>+class PerHeapKind : public PerHeapKindBase<T> {
>+public:
>+    template<typename... Arguments>
>+    PerHeapKind(Arguments&&... arguments)
>+        : PerHeapKindBase<T>(std::forward<Arguments>(arguments)...)
>+    {
>+    }
>+    
>+    ~PerHeapKind()
>+    {
>+        for (unsigned i = numHeaps; i--;)
>+            this->at(i).~T();
>+    }
>+};
>+
>+} // namespace bmalloc
>+
>Index: Source/bmalloc/bmalloc/PerThread.h
>===================================================================
>--- Source/bmalloc/bmalloc/PerThread.h	(revision 219600)
>+++ Source/bmalloc/bmalloc/PerThread.h	(working copy)
>@@ -1,5 +1,5 @@
> /*
>- * Copyright (C) 2014 Apple Inc. All rights reserved.
>+ * Copyright (C) 2014-2017 Apple Inc. All rights reserved.
>  *
>  * Redistribution and use in source and binary forms, with or without
>  * modification, are permitted provided that the following conditions
>@@ -28,6 +28,8 @@
> 
> #include "BPlatform.h"
> #include "Inline.h"
>+#include "PerHeapKind.h"
>+#include "VMAllocate.h"
> #include <mutex>
> #include <pthread.h>
> 
>@@ -63,9 +65,9 @@ private:
> class Cache;
> template<typename T> struct PerThreadStorage;
> 
>-// For now, we only support PerThread<Cache>. We can expand to other types by
>+// For now, we only support PerThread<PerHeapKind<Cache>>. We can expand to other types by
> // using more keys.
>-template<> struct PerThreadStorage<Cache> {
>+template<> struct PerThreadStorage<PerHeapKind<Cache>> {
>     static const pthread_key_t key = __PTK_FRAMEWORK_JAVASCRIPTCORE_KEY0;
> 
>     static void* get()
>@@ -131,14 +133,16 @@ template<typename T>
> void PerThread<T>::destructor(void* p)
> {
>     T* t = static_cast<T*>(p);
>-    delete t;
>+    t->~T();
>+    vmDeallocate(t, vmSize(sizeof(T)));
> }
> 
> template<typename T>
> T* PerThread<T>::getSlowCase()
> {
>     BASSERT(!getFastCase());
>-    T* t = new T;
>+    T* t = static_cast<T*>(vmAllocate(vmSize(sizeof(T))));
>+    new (t) T();
>     PerThreadStorage<T>::init(t, destructor);
>     return t;
> }
>Index: Source/bmalloc/bmalloc/Scavenger.cpp
>===================================================================
>--- Source/bmalloc/bmalloc/Scavenger.cpp	(nonexistent)
>+++ Source/bmalloc/bmalloc/Scavenger.cpp	(working copy)
>@@ -0,0 +1,54 @@
>+/*
>+ * Copyright (C) 2017 Apple Inc. All rights reserved.
>+ *
>+ * Redistribution and use in source and binary forms, with or without
>+ * modification, are permitted provided that the following conditions
>+ * are met:
>+ * 1. Redistributions of source code must retain the above copyright
>+ *    notice, this list of conditions and the following disclaimer.
>+ * 2. Redistributions in binary form must reproduce the above copyright
>+ *    notice, this list of conditions and the following disclaimer in the
>+ *    documentation and/or other materials provided with the distribution.
>+ *
>+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
>+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
>+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
>+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
>+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
>+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
>+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
>+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
>+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
>+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
>+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
>+ */
>+
>+#include "Scavenger.h"
>+
>+#include "Heap.h"
>+#include <thread>
>+
>+namespace bmalloc {
>+
>+Scavenger::Scavenger(std::lock_guard<StaticMutex>&)
>+{
>+#if BOS(DARWIN)
>+    auto queue = dispatch_queue_create("WebKit Malloc Memory Pressure Handler", DISPATCH_QUEUE_SERIAL);
>+    m_pressureHandlerDispatchSource = dispatch_source_create(DISPATCH_SOURCE_TYPE_MEMORYPRESSURE, 0, DISPATCH_MEMORYPRESSURE_CRITICAL, queue);
>+    dispatch_source_set_event_handler(m_pressureHandlerDispatchSource, ^{
>+        scavenge();
>+    });
>+    dispatch_resume(m_pressureHandlerDispatchSource);
>+    dispatch_release(queue);
>+#endif
>+}
>+
>+void Scavenger::scavenge()
>+{
>+    std::lock_guard<StaticMutex> lock(Heap::mutex());
>+    for (unsigned i = numHeaps; i--;)
>+        PerProcess<PerHeapKind<Heap>>::get()->at(i).scavenge(lock);
>+}
>+
>+} // namespace bmalloc
>+
>Index: Source/bmalloc/bmalloc/Scavenger.h
>===================================================================
>--- Source/bmalloc/bmalloc/Scavenger.h	(nonexistent)
>+++ Source/bmalloc/bmalloc/Scavenger.h	(working copy)
>@@ -0,0 +1,64 @@
>+/*
>+ * Copyright (C) 2017 Apple Inc. All rights reserved.
>+ *
>+ * Redistribution and use in source and binary forms, with or without
>+ * modification, are permitted provided that the following conditions
>+ * are met:
>+ * 1. Redistributions of source code must retain the above copyright
>+ *    notice, this list of conditions and the following disclaimer.
>+ * 2. Redistributions in binary form must reproduce the above copyright
>+ *    notice, this list of conditions and the following disclaimer in the
>+ *    documentation and/or other materials provided with the distribution.
>+ *
>+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
>+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
>+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
>+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
>+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
>+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
>+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
>+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
>+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
>+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
>+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
>+ */
>+
>+#pragma once
>+
>+#include "BPlatform.h"
>+#include "StaticMutex.h"
>+#include <mutex>
>+
>+#if BOS(DARWIN)
>+#include <dispatch/dispatch.h>
>+#endif
>+
>+namespace bmalloc {
>+
>+// FIXME: This class should become a common scavenger mechanism for all heaps.
>+// OOPS! (need bug title)
>+
>+class Scavenger {
>+public:
>+    Scavenger(std::lock_guard<StaticMutex>&);
>+    
>+    ~Scavenger() = delete;
>+    
>+    void scavenge();
>+    
>+#if BOS(DARWIN)
>+    void setScavengerThreadQOSClass(qos_class_t overrideClass) { m_requestedScavengerThreadQOSClass = overrideClass; }
>+#endif
>+    
>+    qos_class_t requestedScavengerThreadQOSClass() const { return m_requestedScavengerThreadQOSClass; }
>+
>+private:
>+#if BOS(DARWIN)
>+    dispatch_source_t m_pressureHandlerDispatchSource;
>+    qos_class_t m_requestedScavengerThreadQOSClass { QOS_CLASS_USER_INITIATED };
>+#endif
>+};
>+
>+} // namespace bmalloc
>+
>+
>Index: Source/bmalloc/bmalloc/VMHeap.cpp
>===================================================================
>--- Source/bmalloc/bmalloc/VMHeap.cpp	(revision 219600)
>+++ Source/bmalloc/bmalloc/VMHeap.cpp	(working copy)
>@@ -1,5 +1,5 @@
> /*
>- * Copyright (C) 2014, 2015 Apple Inc. All rights reserved.
>+ * Copyright (C) 2014-2017 Apple Inc. All rights reserved.
>  *
>  * Redistribution and use in source and binary forms, with or without
>  * modification, are permitted provided that the following conditions
>@@ -29,6 +29,10 @@
> 
> namespace bmalloc {
> 
>+VMHeap::VMHeap(std::lock_guard<StaticMutex>&)
>+{
>+}
>+
> LargeRange VMHeap::tryAllocateLargeChunk(size_t alignment, size_t size)
> {
>     // We allocate VM in aligned multiples to increase the chances that
>@@ -50,7 +54,7 @@ LargeRange VMHeap::tryAllocateLargeChunk
>     Chunk* chunk = static_cast<Chunk*>(memory);
>     
> #if BOS(DARWIN)
>-    m_zone.addRange(Range(chunk->bytes(), size));
>+    PerProcess<Zone>::get()->addRange(Range(chunk->bytes(), size));
> #endif
> 
>     return LargeRange(chunk->bytes(), size, 0);
>Index: Source/bmalloc/bmalloc/VMHeap.h
>===================================================================
>--- Source/bmalloc/bmalloc/VMHeap.h	(revision 219600)
>+++ Source/bmalloc/bmalloc/VMHeap.h	(working copy)
>@@ -1,5 +1,5 @@
> /*
>- * Copyright (C) 2014-2016 Apple Inc. All rights reserved.
>+ * Copyright (C) 2014-2017 Apple Inc. All rights reserved.
>  *
>  * Redistribution and use in source and binary forms, with or without
>  * modification, are permitted provided that the following conditions
>@@ -28,6 +28,7 @@
> 
> #include "Chunk.h"
> #include "FixedVector.h"
>+#include "HeapKind.h"
> #include "LargeRange.h"
> #include "Map.h"
> #include "Vector.h"
>@@ -45,12 +46,9 @@ typedef enum { Sync, Async } ScavengeMod
> 
> class VMHeap {
> public:
>-    LargeRange tryAllocateLargeChunk(size_t alignment, size_t);
>+    VMHeap(std::lock_guard<StaticMutex>&);
>     
>-private:
>-#if BOS(DARWIN)
>-    Zone m_zone;
>-#endif
>+    LargeRange tryAllocateLargeChunk(size_t alignment, size_t);
> };
> 
> } // namespace bmalloc
>Index: Source/bmalloc/bmalloc/Zone.cpp
>===================================================================
>--- Source/bmalloc/bmalloc/Zone.cpp	(revision 219600)
>+++ Source/bmalloc/bmalloc/Zone.cpp	(working copy)
>@@ -115,7 +115,7 @@ static const malloc_introspection_t zone
>     .statistics = bmalloc::statistics
> };
> 
>-Zone::Zone()
>+Zone::Zone(std::lock_guard<StaticMutex>&)
> {
>     malloc_zone_t::size = &bmalloc::zoneSize;
>     malloc_zone_t::zone_name = "WebKit Malloc";
>Index: Source/bmalloc/bmalloc/Zone.h
>===================================================================
>--- Source/bmalloc/bmalloc/Zone.h	(revision 219600)
>+++ Source/bmalloc/bmalloc/Zone.h	(working copy)
>@@ -28,7 +28,9 @@
> 
> #include "FixedVector.h"
> #include "Range.h"
>+#include "StaticMutex.h"
> #include <malloc/malloc.h>
>+#include <mutex>
> 
> namespace bmalloc {
> 
>@@ -39,7 +41,7 @@ public:
>     // Enough capacity to track a 64GB heap, so probably enough for anything.
>     static const size_t capacity = 2048;
> 
>-    Zone();
>+    Zone(std::lock_guard<StaticMutex>&);
>     Zone(task_t, memory_reader_t, vm_address_t);
> 
>     void addRange(Range);
>Index: Source/bmalloc/bmalloc/bmalloc.h
>===================================================================
>--- Source/bmalloc/bmalloc/bmalloc.h	(revision 219600)
>+++ Source/bmalloc/bmalloc/bmalloc.h	(working copy)
>@@ -1,5 +1,5 @@
> /*
>- * Copyright (C) 2014-2016 Apple Inc. All rights reserved.
>+ * Copyright (C) 2014-2017 Apple Inc. All rights reserved.
>  *
>  * Redistribution and use in source and binary forms, with or without
>  * modification, are permitted provided that the following conditions
>@@ -25,65 +25,68 @@
> 
> #include "AvailableMemory.h"
> #include "Cache.h"
>+#include "Gigacage.h"
> #include "Heap.h"
>+#include "PerHeapKind.h"
> #include "PerProcess.h"
>+#include "Scavenger.h"
> #include "StaticMutex.h"
> 
> namespace bmalloc {
> namespace api {
> 
> // Returns null on failure.
>-inline void* tryMalloc(size_t size)
>+inline void* tryMalloc(HeapKind kind, size_t size)
> {
>-    return Cache::tryAllocate(size);
>+    return Cache::tryAllocate(kind, size);
> }
> 
> // Crashes on failure.
>-inline void* malloc(size_t size)
>+inline void* malloc(HeapKind kind, size_t size)
> {
>-    return Cache::allocate(size);
>+    return Cache::allocate(kind, size);
> }
> 
> // Returns null on failure.
>-inline void* tryMemalign(size_t alignment, size_t size)
>+inline void* tryMemalign(HeapKind kind, size_t alignment, size_t size)
> {
>-    return Cache::tryAllocate(alignment, size);
>+    return Cache::tryAllocate(kind, alignment, size);
> }
> 
> // Crashes on failure.
>-inline void* memalign(size_t alignment, size_t size)
>+inline void* memalign(HeapKind kind, size_t alignment, size_t size)
> {
>-    return Cache::allocate(alignment, size);
>+    return Cache::allocate(kind, alignment, size);
> }
> 
> // Crashes on failure.
>-inline void* realloc(void* object, size_t newSize)
>+inline void* realloc(HeapKind kind, void* object, size_t newSize)
> {
>-    return Cache::reallocate(object, newSize);
>+    return Cache::reallocate(kind, object, newSize);
> }
> 
>-inline void free(void* object)
>+inline void free(HeapKind kind, void* object)
> {
>-    Cache::deallocate(object);
>+    Cache::deallocate(kind, object);
> }
> 
> inline void scavengeThisThread()
> {
>-    Cache::scavenge();
>+    for (unsigned i = numHeaps; i--;)
>+        Cache::scavenge(static_cast<HeapKind>(i));
> }
> 
> inline void scavenge()
> {
>     scavengeThisThread();
> 
>-    std::lock_guard<StaticMutex> lock(PerProcess<Heap>::mutex());
>-    PerProcess<Heap>::get()->scavenge(lock);
>+    PerProcess<Scavenger>::get()->scavenge();
> }
> 
>-inline bool isEnabled()
>+inline bool isEnabled(HeapKind kind)
> {
>-    std::unique_lock<StaticMutex> lock(PerProcess<Heap>::mutex());
>-    return !PerProcess<Heap>::getFastCase()->debugHeap();
>+    std::unique_lock<StaticMutex> lock(Heap::mutex());
>+    return !PerProcess<PerHeapKind<Heap>>::getFastCase()->at(kind).debugHeap();
> }
>     
> inline size_t availableMemory()
>@@ -106,10 +109,12 @@ inline double percentAvailableMemoryInUs
> #if BOS(DARWIN)
> inline void setScavengerThreadQOSClass(qos_class_t overrideClass)
> {
>-    std::unique_lock<StaticMutex> lock(PerProcess<Heap>::mutex());
>-    PerProcess<Heap>::getFastCase()->setScavengerThreadQOSClass(overrideClass);
>+    std::unique_lock<StaticMutex> lock(Heap::mutex());
>+    PerProcess<Scavenger>::get()->setScavengerThreadQOSClass(overrideClass);
> }
> #endif
> 
>+BEXPORT void ensureGigacage();
>+
> } // namespace api
> } // namespace bmalloc
>Index: Source/bmalloc/bmalloc/mbmalloc.cpp
>===================================================================
>--- Source/bmalloc/bmalloc/mbmalloc.cpp	(revision 219600)
>+++ Source/bmalloc/bmalloc/mbmalloc.cpp	(working copy)
>@@ -25,15 +25,15 @@
> 
> #include "bmalloc.h"
> 
>-#define EXPORT __attribute__((visibility("default")))
>+#include "Export.h"
> 
> extern "C" {
> 
>-EXPORT void* mbmalloc(size_t);
>-EXPORT void* mbmemalign(size_t, size_t);
>-EXPORT void mbfree(void*, size_t);
>-EXPORT void* mbrealloc(void*, size_t, size_t);
>-EXPORT void mbscavenge();
>+BEXPORT void* mbmalloc(size_t);
>+BEXPORT void* mbmemalign(size_t, size_t);
>+BEXPORT void mbfree(void*, size_t);
>+BEXPORT void* mbrealloc(void*, size_t, size_t);
>+BEXPORT void mbscavenge();
>     
> void* mbmalloc(size_t size)
> {

Actions: View | Formatted Diff | Diff

Attachments on bug 174727: 316124 | 316156 | 316161 | 316167 | 316199 | 316200 | 316210 | 316212 | 316221 | 316245 | 316250 | 316255 | 316257 | 316258 | 316260 | 316349 | 316418 | 316424 | 316427 | 316431 | 316560 | 316572 | 316574 | 316583 | 316587 | 316593 | 316602 | 316603 | 316604 | 316606 | 316607 | 316608 | 316612 | 316617 | 316680 | 316689 | 316721 | 316723 | 316726 | 316727 | 316728 | 316729 | 316730 | 316879 | 316887 | 316900 | 316901