Attachment 288866 Details for Bug 161581 – it just ran a program

[patch] it just ran a program

blah.patch (text/plain), 120.15 KB, created by Filip Pizlo on 2016-09-14 14:50:20 PDT

(hide)

Description:

Filename:

MIME Type:

Creator: Filip Pizlo

Created: 2016-09-14 14:50:20 PDT

Size: 120.15 KB

patch

obsolete

>Index: Source/JavaScriptCore/JavaScriptCore.xcodeproj/project.pbxproj
>===================================================================
>--- Source/JavaScriptCore/JavaScriptCore.xcodeproj/project.pbxproj	(revision 205859)
>+++ Source/JavaScriptCore/JavaScriptCore.xcodeproj/project.pbxproj	(working copy)
>@@ -498,6 +498,8 @@
> 		0F7C39FB1C8F629300480151 /* RegExpInlines.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F7C39FA1C8F629300480151 /* RegExpInlines.h */; };
> 		0F7C39FD1C8F659500480151 /* RegExpObjectInlines.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F7C39FC1C8F659500480151 /* RegExpObjectInlines.h */; };
> 		0F7C39FF1C90C55B00480151 /* DFGOpInfo.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F7C39FE1C90C55B00480151 /* DFGOpInfo.h */; };
>+		0F7C5FB81D888A0C0044F5E2 /* MarkedBlockInlines.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F7C5FB71D888A010044F5E2 /* MarkedBlockInlines.h */; };
>+		0F7C5FBA1D8895070044F5E2 /* MarkedSpaceInlines.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F7C5FB91D8895050044F5E2 /* MarkedSpaceInlines.h */; };
> 		0F8023EA1613832B00A0BA45 /* ByValInfo.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F8023E91613832300A0BA45 /* ByValInfo.h */; settings = {ATTRIBUTES = (Private, ); }; };
> 		0F8335B71639C1E6001443B5 /* ArrayAllocationProfile.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 0F8335B41639C1E3001443B5 /* ArrayAllocationProfile.cpp */; };
> 		0F8335B81639C1EA001443B5 /* ArrayAllocationProfile.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F8335B51639C1E3001443B5 /* ArrayAllocationProfile.h */; settings = {ATTRIBUTES = (Private, ); }; };
>@@ -2731,6 +2733,8 @@
> 		0F7C39FA1C8F629300480151 /* RegExpInlines.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = RegExpInlines.h; sourceTree = "<group>"; };
> 		0F7C39FC1C8F659500480151 /* RegExpObjectInlines.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = RegExpObjectInlines.h; sourceTree = "<group>"; };
> 		0F7C39FE1C90C55B00480151 /* DFGOpInfo.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = DFGOpInfo.h; path = dfg/DFGOpInfo.h; sourceTree = "<group>"; };
>+		0F7C5FB71D888A010044F5E2 /* MarkedBlockInlines.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = MarkedBlockInlines.h; sourceTree = "<group>"; };
>+		0F7C5FB91D8895050044F5E2 /* MarkedSpaceInlines.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = MarkedSpaceInlines.h; sourceTree = "<group>"; };
> 		0F8023E91613832300A0BA45 /* ByValInfo.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ByValInfo.h; sourceTree = "<group>"; };
> 		0F8335B41639C1E3001443B5 /* ArrayAllocationProfile.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = ArrayAllocationProfile.cpp; sourceTree = "<group>"; };
> 		0F8335B51639C1E3001443B5 /* ArrayAllocationProfile.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ArrayAllocationProfile.h; sourceTree = "<group>"; };
>@@ -5338,9 +5342,11 @@
> 				C2B916C114DA014E00CBAC86 /* MarkedAllocator.h */,
> 				142D6F0613539A2800B02E86 /* MarkedBlock.cpp */,
> 				142D6F0713539A2800B02E86 /* MarkedBlock.h */,
>+				0F7C5FB71D888A010044F5E2 /* MarkedBlockInlines.h */,
> 				141448CA13A176EC00F5BA1A /* MarkedBlockSet.h */,
> 				14D2F3D8139F4BE200491031 /* MarkedSpace.cpp */,
> 				14D2F3D9139F4BE200491031 /* MarkedSpace.h */,
>+				0F7C5FB91D8895050044F5E2 /* MarkedSpaceInlines.h */,
> 				142D6F0E13539A4100B02E86 /* MarkStack.cpp */,
> 				142D6F0F13539A4100B02E86 /* MarkStack.h */,
> 				ADDB1F6218D77DB7009B58A8 /* OpaqueRootSet.h */,
>@@ -7394,6 +7400,7 @@
> 				0F66E16B14DF3F1600B7B2E4 /* DFGAdjacencyList.h in Headers */,
> 				0FFB921816D02EB20055A5DB /* DFGAllocator.h in Headers */,
> 				0F1E3A461534CBAF000F9456 /* DFGArgumentPosition.h in Headers */,
>+				0F7C5FB81D888A0C0044F5E2 /* MarkedBlockInlines.h in Headers */,
> 				0F2DD8121AB3D8BE00BBB8E8 /* DFGArgumentsEliminationPhase.h in Headers */,
> 				0F2DD8141AB3D8BE00BBB8E8 /* DFGArgumentsUtilities.h in Headers */,
> 				0F485322187750560083B687 /* DFGArithMode.h in Headers */,
>@@ -7911,6 +7918,7 @@
> 				A503FA2A188F105900110F14 /* JSGlobalObjectScriptDebugServer.h in Headers */,
> 				79A228361D35D71F00D8E067 /* ArithProfile.h in Headers */,
> 				A513E5C0185BFACC007E95AD /* JSInjectedScriptHost.h in Headers */,
>+				0F7C5FBA1D8895070044F5E2 /* MarkedSpaceInlines.h in Headers */,
> 				A513E5C2185BFACC007E95AD /* JSInjectedScriptHostPrototype.h in Headers */,
> 				C442CB251A6CDB8C005D3D7C /* JSInputs.json in Headers */,
> 				0F2B66F817B6B5AB00A7AE3F /* JSInt16Array.h in Headers */,
>Index: Source/JavaScriptCore/debugger/Debugger.cpp
>===================================================================
>--- Source/JavaScriptCore/debugger/Debugger.cpp	(revision 205859)
>+++ Source/JavaScriptCore/debugger/Debugger.cpp	(working copy)
>@@ -31,6 +31,7 @@
> #include "JSFunction.h"
> #include "JSGlobalObject.h"
> #include "JSCInlines.h"
>+#include "MarkedSpaceInlines.h"
> #include "Parser.h"
> #include "Protect.h"
> #include "VMEntryScope.h"
>Index: Source/JavaScriptCore/heap/CellContainer.h
>===================================================================
>--- Source/JavaScriptCore/heap/CellContainer.h	(revision 205859)
>+++ Source/JavaScriptCore/heap/CellContainer.h	(working copy)
>@@ -29,10 +29,12 @@
> 
> namespace JSC {
> 
>+class Heap;
> class HeapCell;
> class LargeAllocation;
> class MarkedBlock;
> class WeakSet;
>+class VM;
> 
> typedef uint32_t HeapVersion;
> 
>@@ -56,6 +58,9 @@ public:
>     {
>     }
>     
>+    VM* vm() const;
>+    Heap* heap() const;
>+    
>     explicit operator bool() const { return !!m_encodedPointer; }
>     
>     bool isMarkedBlock() const { return m_encodedPointer && !(m_encodedPointer & isLargeAllocationBit); }
>@@ -73,13 +78,12 @@ public:
>         return *bitwise_cast<LargeAllocation*>(m_encodedPointer - isLargeAllocationBit);
>     }
>     
>-    void flipIfNecessary(HeapVersion);
>-    void flipIfNecessary();
>+    void aboutToMark(HeapVersion);
>     bool needsFlip() const;
>     
>-    bool isMarked() const;
>     bool isMarked(HeapCell*) const;
>     bool isMarkedOrNewlyAllocated(HeapCell*) const;
>+    bool isMarkedOrNewlyAllocated(HeapVersion, HeapCell*) const;
>     
>     void noteMarked();
>     
>Index: Source/JavaScriptCore/heap/CellContainerInlines.h
>===================================================================
>--- Source/JavaScriptCore/heap/CellContainerInlines.h	(revision 205859)
>+++ Source/JavaScriptCore/heap/CellContainerInlines.h	(working copy)
>@@ -29,14 +29,20 @@
> #include "JSCell.h"
> #include "LargeAllocation.h"
> #include "MarkedBlock.h"
>+#include "VM.h"
> 
> namespace JSC {
> 
>-inline bool CellContainer::isMarked() const
>+inline VM* CellContainer::vm() const
> {
>     if (isLargeAllocation())
>-        return true;
>-    return markedBlock().handle().isMarked();
>+        return largeAllocation().vm();
>+    return markedBlock().vm();
>+}
>+
>+inline Heap* CellContainer::heap() const
>+{
>+    return &vm()->heap;
> }
> 
> inline bool CellContainer::isMarked(HeapCell* cell) const
>@@ -53,6 +59,13 @@ inline bool CellContainer::isMarkedOrNew
>     return markedBlock().isMarkedOrNewlyAllocated(cell);
> }
> 
>+inline bool CellContainer::isMarkedOrNewlyAllocated(HeapVersion version, HeapCell* cell) const
>+{
>+    if (isLargeAllocation())
>+        return largeAllocation().isMarkedOrNewlyAllocated();
>+    return markedBlock().isMarkedOrNewlyAllocated(version, cell);
>+}
>+
> inline void CellContainer::noteMarked()
> {
>     if (!isLargeAllocation())
>@@ -73,16 +86,10 @@ inline WeakSet& CellContainer::weakSet()
>     return markedBlock().weakSet();
> }
> 
>-inline void CellContainer::flipIfNecessary(HeapVersion heapVersion)
>-{
>-    if (!isLargeAllocation())
>-        markedBlock().flipIfNecessary(heapVersion);
>-}
>-
>-inline void CellContainer::flipIfNecessary()
>+inline void CellContainer::aboutToMark(HeapVersion heapVersion)
> {
>     if (!isLargeAllocation())
>-        markedBlock().flipIfNecessary();
>+        markedBlock().aboutToMark(heapVersion);
> }
> 
> inline bool CellContainer::needsFlip() const
>Index: Source/JavaScriptCore/heap/ConservativeRoots.cpp
>===================================================================
>--- Source/JavaScriptCore/heap/ConservativeRoots.cpp	(revision 205859)
>+++ Source/JavaScriptCore/heap/ConservativeRoots.cpp	(working copy)
>@@ -34,6 +34,7 @@
> #include "JSCell.h"
> #include "JSObject.h"
> #include "JSCInlines.h"
>+#include "MarkedBlockInlines.h"
> #include "Structure.h"
> #include <wtf/OSAllocator.h>
> 
>@@ -65,12 +66,12 @@ void ConservativeRoots::grow()
> }
> 
> template<typename MarkHook>
>-inline void ConservativeRoots::genericAddPointer(void* p, HeapVersion version, TinyBloomFilter filter, MarkHook& markHook)
>+inline void ConservativeRoots::genericAddPointer(void* p, TinyBloomFilter filter, MarkHook& markHook)
> {
>     markHook.mark(p);
> 
>     HeapUtil::findGCObjectPointersForMarking(
>-        m_heap, version, filter, p,
>+        m_heap, filter, p,
>         [&] (void* p) {
>             if (m_size == m_capacity)
>                 grow();
>@@ -93,9 +94,8 @@ void ConservativeRoots::genericAddSpan(v
>     RELEASE_ASSERT(isPointerAligned(end));
> 
>     TinyBloomFilter filter = m_heap.objectSpace().blocks().filter(); // Make a local copy of filter to show the compiler it won't alias, and can be register-allocated.
>-    HeapVersion version = m_heap.objectSpace().version();
>     for (char** it = static_cast<char**>(begin); it != static_cast<char**>(end); ++it)
>-        genericAddPointer(*it, version, filter, markHook);
>+        genericAddPointer(*it, filter, markHook);
> }
> 
> class DummyMarkHook {
>Index: Source/JavaScriptCore/heap/ConservativeRoots.h
>===================================================================
>--- Source/JavaScriptCore/heap/ConservativeRoots.h	(revision 205859)
>+++ Source/JavaScriptCore/heap/ConservativeRoots.h	(working copy)
>@@ -51,7 +51,7 @@ private:
>     static const size_t nonInlineCapacity = 8192 / sizeof(HeapCell*);
>     
>     template<typename MarkHook>
>-    void genericAddPointer(void*, HeapVersion, TinyBloomFilter, MarkHook&);
>+    void genericAddPointer(void*, TinyBloomFilter, MarkHook&);
> 
>     template<typename MarkHook>
>     void genericAddSpan(void*, void* end, MarkHook&);
>Index: Source/JavaScriptCore/heap/Heap.cpp
>===================================================================
>--- Source/JavaScriptCore/heap/Heap.cpp	(revision 205859)
>+++ Source/JavaScriptCore/heap/Heap.cpp	(working copy)
>@@ -44,6 +44,7 @@
> #include "JSGlobalObject.h"
> #include "JSLock.h"
> #include "JSVirtualMachineInternal.h"
>+#include "MarkedSpaceInlines.h"
> #include "SamplingProfiler.h"
> #include "ShadowChicken.h"
> #include "SuperSampler.h"
>@@ -526,11 +527,9 @@ void Heap::beginMarking()
>     }
>     
>     {
>-        TimingScope clearMarksTimingScope(*this, "m_objectSpace.clearMarks");
>-        m_objectSpace.flip();
>+        TimingScope clearMarksTimingScope(*this, "m_objectSpace.beginMarking");
>+        m_objectSpace.beginMarking();
>     }
>-    
>-    m_objectSpace.setIsMarking(true);
> }
> 
> void Heap::visitExternalRememberedSet()
>@@ -779,7 +778,7 @@ void Heap::endMarking()
>     ASSERT(m_sharedMarkStack.isEmpty());
>     m_weakReferenceHarvesters.removeAll();
>     
>-    m_objectSpace.setIsMarking(false);
>+    m_objectSpace.endMarking();
> }
> 
> size_t Heap::objectCount()
>@@ -939,7 +938,7 @@ void Heap::collectAllGarbage()
>         m_objectSpace.sweep();
>         m_objectSpace.shrink();
>     }
>-    ASSERT(m_blockSnapshot.isEmpty());
>+    m_objectSpace.assertSnapshotEmpty();
> 
>     sweepAllLogicallyEmptyWeakBlocks();
> }
>@@ -1048,7 +1047,7 @@ NEVER_INLINE void Heap::collectImpl(Heap
>     notifyIncrementalSweeper();
>     writeBarrierCurrentlyExecutingCodeBlocks();
> 
>-    resetAllocators();
>+    prepareForAllocation();
>     updateAllocationLimits();
>     didFinishCollection(gcStartTime);
>     resumeCompilerThreads();
>@@ -1162,44 +1161,10 @@ void Heap::sweepArrayBuffers()
>     m_arrayBuffers.sweep();
> }
> 
>-struct MarkedBlockSnapshotFunctor : public MarkedBlock::VoidFunctor {
>-    MarkedBlockSnapshotFunctor(Vector<MarkedBlock::Handle*>& blocks) 
>-        : m_index(0) 
>-        , m_blocks(blocks)
>-    {
>-    }
>-
>-    void operator()(MarkedBlock::Handle* block) const
>-    {
>-        block->setIsOnBlocksToSweep(true);
>-        m_blocks[m_index++] = block;
>-    }
>-
>-    // FIXME: This is a mutable field becaue this isn't a C++ lambda.
>-    // https://bugs.webkit.org/show_bug.cgi?id=159644
>-    mutable size_t m_index;
>-    Vector<MarkedBlock::Handle*>& m_blocks;
>-};
>-
> void Heap::snapshotMarkedSpace()
> {
>     TimingScope timingScope(*this, "Heap::snapshotMarkedSpace");
>-    // FIXME: This should probably be renamed. It's not actually snapshotting all of MarkedSpace.
>-    // This is used by IncrementalSweeper, so it only needs to snapshot blocks. However, if we ever
>-    // wanted to add other snapshotting login, we'd probably put it here.
>-    
>-    if (m_operationInProgress == EdenCollection) {
>-        for (MarkedBlock::Handle* handle : m_objectSpace.blocksWithNewObjects()) {
>-            if (handle->isOnBlocksToSweep())
>-                continue;
>-            m_blockSnapshot.append(handle);
>-            handle->setIsOnBlocksToSweep(true);
>-        }
>-    } else {
>-        m_blockSnapshot.resizeToFit(m_objectSpace.blocks().set().size());
>-        MarkedBlockSnapshotFunctor functor(m_blockSnapshot);
>-        m_objectSpace.forEachBlock(functor);
>-    }
>+    m_objectSpace.snapshot();
> }
> 
> void Heap::deleteSourceProviderCaches()
>@@ -1222,9 +1187,9 @@ void Heap::writeBarrierCurrentlyExecutin
>     m_codeBlocks->writeBarrierCurrentlyExecutingCodeBlocks(this);
> }
> 
>-void Heap::resetAllocators()
>+void Heap::prepareForAllocation()
> {
>-    m_objectSpace.resetAllocators();
>+    m_objectSpace.prepareForAllocation();
> }
> 
> void Heap::updateAllocationLimits()
>@@ -1473,7 +1438,7 @@ public:
> void Heap::zombifyDeadObjects()
> {
>     // Sweep now because destructors will crash once we're zombified.
>-    m_objectSpace.zombifySweep();
>+    m_objectSpace.sweep();
>     HeapIterationScope iterationScope(*this);
>     m_objectSpace.forEachDeadCell(iterationScope, Zombify());
> }
>Index: Source/JavaScriptCore/heap/Heap.h
>===================================================================
>--- Source/JavaScriptCore/heap/Heap.h	(revision 205859)
>+++ Source/JavaScriptCore/heap/Heap.h	(working copy)
>@@ -97,11 +97,9 @@ public:
>     // our scan to run faster. 
>     static const unsigned s_timeCheckResolution = 16;
> 
>-    static bool isLive(const void*);
>     static bool isMarked(const void*);
>     static bool isMarkedConcurrently(const void*);
>     static bool testAndSetMarked(HeapVersion, const void*);
>-    static void setMarked(const void*);
>     
>     static size_t cellSize(const void*);
> 
>@@ -333,7 +331,7 @@ private:
>     void deleteSourceProviderCaches();
>     void notifyIncrementalSweeper();
>     void writeBarrierCurrentlyExecutingCodeBlocks();
>-    void resetAllocators();
>+    void prepareForAllocation();
>     void harvestWeakReferences();
>     void finalizeUnconditionalFinalizers();
>     void clearUnmarkedExecutables();
>Index: Source/JavaScriptCore/heap/HeapInlines.h
>===================================================================
>--- Source/JavaScriptCore/heap/HeapInlines.h	(revision 205859)
>+++ Source/JavaScriptCore/heap/HeapInlines.h	(working copy)
>@@ -74,17 +74,6 @@ inline Heap* Heap::heap(const JSValue v)
>     return heap(v.asCell());
> }
> 
>-inline bool Heap::isLive(const void* rawCell)
>-{
>-    ASSERT(!mayBeGCThread());
>-    HeapCell* cell = bitwise_cast<HeapCell*>(rawCell);
>-    if (cell->isLargeAllocation())
>-        return cell->largeAllocation().isLive();
>-    MarkedBlock& block = cell->markedBlock();
>-    block.flipIfNecessary(block.vm()->heap.objectSpace().version());
>-    return block.handle().isLiveCell(cell);
>-}
>-
> ALWAYS_INLINE bool Heap::isMarked(const void* rawCell)
> {
>     ASSERT(!mayBeGCThread());
>@@ -92,9 +81,8 @@ ALWAYS_INLINE bool Heap::isMarked(const 
>     if (cell->isLargeAllocation())
>         return cell->largeAllocation().isMarked();
>     MarkedBlock& block = cell->markedBlock();
>-    if (block.needsFlip(block.vm()->heap.objectSpace().version()))
>-        return false;
>-    return block.isMarked(cell);
>+    return block.isMarked(
>+        block.vm()->heap.objectSpace().version(), cell);
> }
> 
> ALWAYS_INLINE bool Heap::isMarkedConcurrently(const void* rawCell)
>@@ -103,10 +91,8 @@ ALWAYS_INLINE bool Heap::isMarkedConcurr
>     if (cell->isLargeAllocation())
>         return cell->largeAllocation().isMarked();
>     MarkedBlock& block = cell->markedBlock();
>-    if (block.needsFlip(block.vm()->heap.objectSpace().version()))
>-        return false;
>-    WTF::loadLoadFence();
>-    return block.isMarked(cell);
>+    return block.isMarkedConcurrently(
>+        block.vm()->heap.objectSpace().version(), cell);
> }
> 
> ALWAYS_INLINE bool Heap::testAndSetMarked(HeapVersion version, const void* rawCell)
>@@ -115,22 +101,10 @@ ALWAYS_INLINE bool Heap::testAndSetMarke
>     if (cell->isLargeAllocation())
>         return cell->largeAllocation().testAndSetMarked();
>     MarkedBlock& block = cell->markedBlock();
>-    block.flipIfNecessaryDuringMarking(version);
>+    block.aboutToMark(version);
>     return block.testAndSetMarked(cell);
> }
> 
>-inline void Heap::setMarked(const void* rawCell)
>-{
>-    HeapCell* cell = bitwise_cast<HeapCell*>(rawCell);
>-    if (cell->isLargeAllocation()) {
>-        cell->largeAllocation().setMarked();
>-        return;
>-    }
>-    MarkedBlock& block = cell->markedBlock();
>-    block.flipIfNecessary(block.vm()->heap.objectSpace().version());
>-    block.setMarked(cell);
>-}
>-
> ALWAYS_INLINE size_t Heap::cellSize(const void* rawCell)
> {
>     return bitwise_cast<HeapCell*>(rawCell)->cellSize();
>Index: Source/JavaScriptCore/heap/HeapStatistics.cpp
>===================================================================
>--- Source/JavaScriptCore/heap/HeapStatistics.cpp	(revision 205859)
>+++ Source/JavaScriptCore/heap/HeapStatistics.cpp	(working copy)
>@@ -30,6 +30,7 @@
> #include "HeapIterationScope.h"
> #include "JSCInlines.h"
> #include "JSObject.h"
>+#include "MarkedSpaceInlines.h"
> #include "Options.h"
> #include <stdlib.h>
> #include <wtf/CurrentTime.h>
>Index: Source/JavaScriptCore/heap/HeapUtil.h
>===================================================================
>--- Source/JavaScriptCore/heap/HeapUtil.h	(revision 205859)
>+++ Source/JavaScriptCore/heap/HeapUtil.h	(working copy)
>@@ -46,8 +46,7 @@ public:
>     // before liveness data is cleared to be accurate.
>     template<typename Func>
>     static void findGCObjectPointersForMarking(
>-        Heap& heap, HeapVersion heapVersion, TinyBloomFilter filter, void* passedPointer,
>-        const Func& func)
>+        Heap& heap, TinyBloomFilter filter, void* passedPointer, const Func& func)
>     {
>         const HashSet<MarkedBlock*>& set = heap.objectSpace().blocks().set();
>         
>@@ -84,7 +83,6 @@ public:
>             if (!filter.ruleOut(bitwise_cast<Bits>(previousCandidate))
>                 && set.contains(previousCandidate)
>                 && previousCandidate->handle().cellKind() == HeapCell::Auxiliary) {
>-                previousCandidate->flipIfNecessary(heapVersion);
>                 previousPointer = static_cast<char*>(previousCandidate->handle().cellAlign(previousPointer));
>                 if (previousCandidate->handle().isLiveCell(previousPointer))
>                     func(previousPointer);
>@@ -99,8 +97,6 @@ public:
>         if (!set.contains(candidate))
>             return;
>         
>-        candidate->flipIfNecessary(heapVersion);
>-
>         auto tryPointer = [&] (void* pointer) {
>             if (candidate->handle().isLiveCell(pointer))
>                 func(pointer);
>@@ -170,7 +166,6 @@ public:
>         if (candidate->handle().cellKind() != HeapCell::JSCell)
>             return false;
>         
>-        candidate->flipIfNecessary();
>         if (!candidate->handle().isLiveCell(pointer))
>             return false;
>         
>Index: Source/JavaScriptCore/heap/HeapVerifier.cpp
>===================================================================
>--- Source/JavaScriptCore/heap/HeapVerifier.cpp	(revision 205859)
>+++ Source/JavaScriptCore/heap/HeapVerifier.cpp	(working copy)
>@@ -1,5 +1,5 @@
> /*
>- * Copyright (C) 2014 Apple Inc. All rights reserved.
>+ * Copyright (C) 2014, 2016 Apple Inc. All rights reserved.
>  *
>  * Redistribution and use in source and binary forms, with or without
>  * modification, are permitted provided that the following conditions
>@@ -30,6 +30,7 @@
> #include "HeapIterationScope.h"
> #include "JSCInlines.h"
> #include "JSObject.h"
>+#include "MarkedSpaceInlines.h"
> 
> namespace JSC {
> 
>Index: Source/JavaScriptCore/heap/IncrementalSweeper.cpp
>===================================================================
>--- Source/JavaScriptCore/heap/IncrementalSweeper.cpp	(revision 205859)
>+++ Source/JavaScriptCore/heap/IncrementalSweeper.cpp	(working copy)
>@@ -50,7 +50,6 @@ static const double sweepTimeMultiplier 
> #if USE(CF)
> IncrementalSweeper::IncrementalSweeper(Heap* heap, CFRunLoopRef runLoop)
>     : HeapTimer(heap->vm(), runLoop)
>-    , m_blocksToSweep(heap->m_blockSnapshot)
> {
> }
> 
>@@ -66,7 +65,6 @@ void IncrementalSweeper::cancelTimer()
> #elif PLATFORM(EFL)
> IncrementalSweeper::IncrementalSweeper(Heap* heap)
>     : HeapTimer(heap->vm())
>-    , m_blocksToSweep(heap->m_blockSnapshot)
> {
> }
> 
>@@ -86,7 +84,6 @@ void IncrementalSweeper::cancelTimer()
> #elif USE(GLIB)
> IncrementalSweeper::IncrementalSweeper(Heap* heap)
>     : HeapTimer(heap->vm())
>-    , m_blocksToSweep(heap->m_blockSnapshot)
> {
> }
> 
>@@ -121,19 +118,20 @@ void IncrementalSweeper::doSweep(double 
>         return;
>     }
> 
>-    m_blocksToSweep.clear();
>     cancelTimer();
> }
> 
> bool IncrementalSweeper::sweepNextBlock()
> {
>-    while (!m_blocksToSweep.isEmpty()) {
>-        MarkedBlock::Handle* block = m_blocksToSweep.takeLast();
>-        block->setIsOnBlocksToSweep(false);
>-
>-        if (!block->needsSweeping())
>-            continue;
>-
>+    MarkedBlock::Handle* block = nullptr;
>+    
>+    for (; m_currentAllocator; m_currentAllocator = m_currentAllocator->nextAllocator()) {
>+        block = m_currentAllocator->findBlockToSweep();
>+        if (block)
>+            break;
>+    }
>+    
>+    if (block) {
>         DeferGCForAWhile deferGC(m_vm->heap);
>         block->sweep();
>         m_vm->heap.objectSpace().freeOrShrinkBlock(block);
>@@ -146,11 +144,12 @@ bool IncrementalSweeper::sweepNextBlock(
> void IncrementalSweeper::startSweeping()
> {
>     scheduleTimer();
>+    m_currentAllocator = m_vm->heap.objectSpace().firstAllocator();
> }
> 
> void IncrementalSweeper::willFinishSweeping()
> {
>-    m_blocksToSweep.clear();
>+    m_currentAllocator = nullptr;
>     if (m_vm)
>         cancelTimer();
> }
>Index: Source/JavaScriptCore/heap/IncrementalSweeper.h
>===================================================================
>--- Source/JavaScriptCore/heap/IncrementalSweeper.h	(revision 205859)
>+++ Source/JavaScriptCore/heap/IncrementalSweeper.h	(working copy)
>@@ -27,13 +27,12 @@
> #define IncrementalSweeper_h
> 
> #include "HeapTimer.h"
>-#include "MarkedBlock.h"
> #include <wtf/Vector.h>
> 
> namespace JSC {
> 
> class Heap;
>-class MarkedBlock;
>+class MarkedAllocator;
> 
> class IncrementalSweeper : public HeapTimer {
>     WTF_MAKE_FAST_ALLOCATED;
>@@ -56,7 +55,7 @@ private:
>     void scheduleTimer();
>     void cancelTimer();
>     
>-    Vector<MarkedBlock::Handle*>& m_blocksToSweep;
>+    MarkedAllocator* m_currentAllocator;
> #endif
> };
> 
>Index: Source/JavaScriptCore/heap/LargeAllocation.h
>===================================================================
>--- Source/JavaScriptCore/heap/LargeAllocation.h	(revision 205859)
>+++ Source/JavaScriptCore/heap/LargeAllocation.h	(working copy)
>@@ -72,7 +72,8 @@ public:
>     ALWAYS_INLINE bool isMarked() { return m_isMarked.load(std::memory_order_relaxed); }
>     bool isMarkedOrNewlyAllocated() { return isMarked() || isNewlyAllocated(); }
>     bool isMarkedOrNewlyAllocated(HeapCell*) { return isMarkedOrNewlyAllocated(); }
>-    bool isMarkedOrNewlyAllocatedDuringWeakVisiting(HeapVersion, HeapCell*) { return isMarkedOrNewlyAllocated(); }
>+    bool isMarkedOrNewlyAllocated(HeapVersion, HeapCell*) { return isMarkedOrNewlyAllocated(); }
>+    bool isMarkedOrNewlyAllocatedConcurrently(HeapVersion, HeapCell*) { return isMarkedOrNewlyAllocated(); }
>     bool isLive() { return isMarkedOrNewlyAllocated(); }
>     
>     bool hasValidCell() const { return m_hasValidCell; }
>@@ -106,8 +107,7 @@ public:
>     
>     const AllocatorAttributes& attributes() const { return m_attributes; }
>     
>-    void flipIfNecessary(uint64_t) { }
>-    void flipIfNecessaryDuringMarking(uint64_t) { }
>+    void aboutToMark(HeapVersion) { }
>     
>     ALWAYS_INLINE bool testAndSetMarked()
>     {
>Index: Source/JavaScriptCore/heap/MarkedAllocator.cpp
>===================================================================
>--- Source/JavaScriptCore/heap/MarkedAllocator.cpp	(revision 205859)
>+++ Source/JavaScriptCore/heap/MarkedAllocator.cpp	(working copy)
>@@ -30,6 +30,7 @@
> #include "Heap.h"
> #include "IncrementalSweeper.h"
> #include "JSCInlines.h"
>+#include "MarkedBlockInlines.h"
> #include "SuperSampler.h"
> #include "VM.h"
> #include <wtf/CurrentTime.h>
>@@ -39,7 +40,6 @@ namespace JSC {
> MarkedAllocator::MarkedAllocator(Heap* heap, MarkedSpace* markedSpace, size_t cellSize, const AllocatorAttributes& attributes)
>     : m_currentBlock(0)
>     , m_lastActiveBlock(0)
>-    , m_nextBlockToSweep(nullptr)
>     , m_cellSize(static_cast<unsigned>(cellSize))
>     , m_attributes(attributes)
>     , m_heap(heap)
>@@ -50,11 +50,13 @@ MarkedAllocator::MarkedAllocator(Heap* h
> bool MarkedAllocator::isPagedOut(double deadline)
> {
>     unsigned itersSinceLastTimeCheck = 0;
>-    MarkedBlock::Handle* block = m_blockList.begin();
>-    while (block) {
>-        block = filterNextBlock(block->next());
>-        if (block)
>-            block->flipIfNecessary(); // Forces us to touch the memory of the block, but has no semantic effect.
>+    for (size_t index = 0; index < m_blocks.size(); ++index) {
>+        MarkedBlock::Handle* block = m_blocks[index];
>+        if (block) {
>+            // Forces us to touch the memory of the block, but has no semantic effect.
>+            if (block->needsFlip())
>+                block->block().resetVersion();
>+        }
>         ++itersSinceLastTimeCheck;
>         if (itersSinceLastTimeCheck >= Heap::s_timeCheckResolution) {
>             double currentTime = WTF::monotonicallyIncreasingTime();
>@@ -66,84 +68,77 @@ bool MarkedAllocator::isPagedOut(double 
>     return false;
> }
> 
>-void MarkedAllocator::retire(MarkedBlock::Handle* block)
>-{
>-    LockHolder locker(m_lock); // This will be called in parallel during GC.
>-    if (block == m_currentBlock) {
>-        // This happens when the mutator is running. We finished a full GC and marked too few things
>-        // to retire. Then we started allocating in this block. Then a barrier ran, which marked an
>-        // object in this block, which put it over the retirement threshold. It's OK to simply do
>-        // nothing in that case.
>-        return;
>-    }
>-    if (block == m_lastActiveBlock) {
>-        // This can easily happen during marking. It would be easy to handle this case, but it's
>-        // just as easy to ignore it.
>-        return;
>-    }
>-    RELEASE_ASSERT(block->isOnList());
>-    if (block == m_nextBlockToSweep)
>-        m_nextBlockToSweep = filterNextBlock(block->next());
>-    block->remove();
>-    m_retiredBlocks.push(block);
>-}
>-
>-MarkedBlock::Handle* MarkedAllocator::filterNextBlock(MarkedBlock::Handle* block)
>+MarkedBlock::Handle* MarkedAllocator::findEmptyBlock()
> {
>-    if (block == m_blockList.end())
>+    m_emptyCursor = m_empty.findBit(m_emptyCursor, true);
>+    if (m_emptyCursor >= m_blocks.size())
>         return nullptr;
>-    return block;
>-}
>-
>-void MarkedAllocator::setNextBlockToSweep(MarkedBlock::Handle* block)
>-{
>-    m_nextBlockToSweep = filterNextBlock(block);
>+    return m_blocks[m_emptyCursor];
> }
> 
> void* MarkedAllocator::tryAllocateWithoutCollectingImpl()
> {
>     SuperSamplerScope superSamplerScope(false);
>     
>-    if (m_currentBlock) {
>-        ASSERT(m_currentBlock == m_nextBlockToSweep);
>+    if (m_currentBlock)
>         m_currentBlock->didConsumeFreeList();
>-        setNextBlockToSweep(m_currentBlock->next());
>-    }
>     
>     setFreeList(FreeList());
>-
>-    RELEASE_ASSERT(m_nextBlockToSweep != m_blockList.end());
>-
>-    MarkedBlock::Handle* next;
>-    for (MarkedBlock::Handle*& block = m_nextBlockToSweep; block; block = next) {
>-        next = filterNextBlock(block->next());
>-
>-        // It would be super weird if the blocks we are sweeping have anything allocated during this
>-        // cycle.
>-        ASSERT(!block->hasAnyNewlyAllocated());
>-        
>-        FreeList freeList = block->sweep(MarkedBlock::Handle::SweepToFreeList);
>+    m_currentBlock = nullptr;
>+    
>+    for (;;) {
>+        m_allocationCursor = m_canAllocate.findBit(m_allocationCursor, true);
>+        if (m_allocationCursor >= m_blocks.size())
>+            break;
>         
>-        // It's possible to stumble on a complete-full block. Marking tries to retire these, but
>-        // that algorithm is racy and may forget to do it sometimes.
>-        if (freeList.allocationWillFail()) {
>-            ASSERT(block->isFreeListed());
>-            block->unsweepWithNoNewlyAllocated();
>-            ASSERT(block->isMarked());
>-            retire(block);
>-            continue;
>-        }
>+        setIsCanAllocate(m_allocationCursor, false);
> 
>-        m_currentBlock = block;
>-        setFreeList(freeList);
>-        break;
>+        if (void* result = tryAllocateIn(m_blocks[m_allocationCursor]))
>+            return result;
>     }
>     
>-    if (!m_freeList) {
>-        m_currentBlock = 0;
>-        return 0;
>+    if (MarkedBlock::Handle* block = findEmptyBlock())
>+        return allocateIn(block);
>+    
>+    if (MarkedBlock::Handle* block = m_markedSpace->findEmptyBlock()) {
>+        block->sweep();
>+        block->removeFromAllocator();
>+        addBlock(block);
>+        return allocateIn(block);
>     }
>+    
>+    return nullptr;
>+}
>+
>+void* MarkedAllocator::allocateIn(MarkedBlock::Handle* block)
>+{
>+    void* result = tryAllocateIn(block);
>+    RELEASE_ASSERT(result);
>+    return result;
>+}
> 
>+void* MarkedAllocator::tryAllocateIn(MarkedBlock::Handle* block)
>+{
>+    ASSERT(block);
>+    ASSERT(!block->hasAnyNewlyAllocated());
>+    ASSERT(!block->isFreeListed());
>+    
>+    FreeList freeList = block->sweep(MarkedBlock::Handle::SweepToFreeList);
>+    
>+    // It's possible to stumble on a completely full block. Marking tries to retire these, but
>+    // that algorithm is racy and may forget to do it sometimes.
>+    if (freeList.allocationWillFail()) {
>+        ASSERT(block->isFreeListed());
>+        block->unsweepWithNoNewlyAllocated();
>+        ASSERT(!block->isFreeListed());
>+        ASSERT(!isEmpty(block));
>+        ASSERT(!isCanAllocate(block));
>+        return nullptr;
>+    }
>+    
>+    m_currentBlock = block;
>+    setFreeList(freeList);
>+    
>     void* result;
>     if (m_freeList.remaining) {
>         unsigned cellSize = m_cellSize;
>@@ -155,6 +150,7 @@ void* MarkedAllocator::tryAllocateWithou
>         result = head;
>     }
>     RELEASE_ASSERT(result);
>+    setIsEden(m_currentBlock, true);
>     m_markedSpace->didAllocateInBlock(m_currentBlock);
>     return result;
> }
>@@ -227,8 +223,7 @@ void* MarkedAllocator::allocateSlowCaseI
>             return nullptr;
>     }
>     addBlock(block);
>-        
>-    result = tryAllocateWithoutCollecting();
>+    result = allocateIn(block);
>     ASSERT(result);
>     return result;
> }
>@@ -249,37 +244,72 @@ size_t MarkedAllocator::blockSizeForByte
> MarkedBlock::Handle* MarkedAllocator::tryAllocateBlock()
> {
>     SuperSamplerScope superSamplerScope(false);
>-    return MarkedBlock::tryCreate(*m_heap, this, m_cellSize, m_attributes);
>+    
>+    MarkedBlock::Handle* handle = MarkedBlock::tryCreate(*m_heap, m_cellSize, m_attributes);
>+    if (!handle)
>+        return nullptr;
>+    
>+    m_markedSpace->didAddBlock(handle);
>+    
>+    return handle;
> }
> 
> void MarkedAllocator::addBlock(MarkedBlock::Handle* block)
> {
>-    ASSERT(!m_currentBlock);
>-    ASSERT(!m_freeList);
>+    size_t index;
>+    if (m_freeBlockIndices.isEmpty()) {
>+        index = m_blocks.size();
>+
>+        size_t oldCapacity = m_blocks.capacity();
>+        m_blocks.append(block);
>+        if (m_blocks.capacity() != oldCapacity) {
>+            forEachBitVector(
>+                [&] (FastBitVector& vector) {
>+                    ASSERT(vector.numBits() == oldCapacity);
>+                });
>+            
>+            ASSERT(m_blocks.capacity() > oldCapacity);
>+            
>+            forEachBitVector(
>+                [&] (FastBitVector& vector) {
>+                    vector.resize(m_blocks.capacity());
>+                });
>+        }
>+    } else {
>+        index = m_freeBlockIndices.takeLast();
>+        ASSERT(!m_blocks[index]);
>+        m_blocks[index] = block;
>+    }
>+    
>+    forEachBitVector(
>+        [&] (FastBitVector& vector) {
>+            ASSERT(!vector[index]);
>+        });
>+
>+    block->didAddToAllocator(this, index);
>     
>-    m_blockList.append(block);
>-    setNextBlockToSweep(block);
>-    m_markedSpace->didAddBlock(block);
>+    setIsLive(index, true);
>+    setIsEmpty(index, true);
> }
> 
> void MarkedAllocator::removeBlock(MarkedBlock::Handle* block)
> {
>-    if (m_currentBlock == block) {
>-        m_currentBlock = filterNextBlock(m_currentBlock->next());
>-        setFreeList(FreeList());
>-    }
>-    if (m_nextBlockToSweep == block)
>-        setNextBlockToSweep(m_nextBlockToSweep->next());
>+    ASSERT(block->allocator() == this);
>+    ASSERT(m_blocks[block->index()] == block);
> 
>-    block->willRemoveBlock();
>-    m_blockList.remove(block);
>+    m_blocks[block->index()] = nullptr;
>+    m_freeBlockIndices.append(block->index());
>+    
>+    forEachBitVector(
>+        [&] (FastBitVector& vector) {
>+            vector[block->index()] = false;
>+        });
>+    
>+    block->didRemoveFromAllocator();
> }
> 
> void MarkedAllocator::stopAllocating()
> {
>-    if (m_heap->operationInProgress() == FullCollection)
>-        m_blockList.takeFrom(m_retiredBlocks);
>-
>     ASSERT(!m_lastActiveBlock);
>     if (!m_currentBlock) {
>         ASSERT(!m_freeList);
>@@ -292,29 +322,25 @@ void MarkedAllocator::stopAllocating()
>     m_freeList = FreeList();
> }
> 
>-void MarkedAllocator::reset()
>+void MarkedAllocator::prepareForAllocation()
> {
>-    m_lastActiveBlock = 0;
>-    m_currentBlock = 0;
>+    m_lastActiveBlock = nullptr;
>+    m_currentBlock = nullptr;
>     setFreeList(FreeList());
> 
>-    setNextBlockToSweep(m_blockList.begin());
>+    m_allocationCursor = 0;
>+    m_emptyCursor = 0;
>+    m_snapshotCursor = 0;
>+    
>+    m_eden.clearAll();
> 
>     if (UNLIKELY(Options::useImmortalObjects())) {
>-        MarkedBlock::Handle* next;
>-        for (MarkedBlock::Handle*& block = m_nextBlockToSweep; block; block = next) {
>-            next = filterNextBlock(block->next());
>-
>-            FreeList freeList = block->sweep(MarkedBlock::Handle::SweepToFreeList);
>-            block->zap(freeList);
>-            retire(block);
>-        }
>+        // FIXME: Things?
>     }
> }
> 
> void MarkedAllocator::lastChanceToFinalize()
> {
>-    m_blockList.takeFrom(m_retiredBlocks);
>     forEachBlock(
>         [&] (MarkedBlock::Handle* block) {
>             block->lastChanceToFinalize();
>@@ -326,4 +352,103 @@ void MarkedAllocator::setFreeList(const 
>     m_freeList = freeList;
> }
> 
>+void MarkedAllocator::resumeAllocating()
>+{
>+    if (!m_lastActiveBlock)
>+        return;
>+
>+    m_freeList = m_lastActiveBlock->resumeAllocating();
>+    m_currentBlock = m_lastActiveBlock;
>+    m_lastActiveBlock = nullptr;
>+}
>+
>+void MarkedAllocator::beginMarkingForFullCollection()
>+{
>+    // Mark bits are sticky and so is our summary of mark bits. We only clear these during full
>+    // collections, so if you survived the last collection you will survive the next one so long
>+    // as the next one is eden.
>+    m_markingNotEmpty.clearAll();
>+    m_markingRetired.clearAll();
>+}
>+
>+void MarkedAllocator::endMarking()
>+{
>+    m_allocated.clearAll();
>+    
>+    // It's surprising and frustrating to comprehend, but the end-of-marking flip does not need to
>+    // know what kind of collection it is. That knowledge is already encoded in the m_markingXYZ
>+    // vectors.
>+    
>+    if (needsDestruction()) {
>+        // If blocks need destruction then nothing is empty! This is a correct assertion but may
>+        // become wrong once we go full concurrent: when we create a new block, it will flicker
>+        // into the empty set for a tiny moment. On the other hand, this code is likely to be run
>+        // in stopTheWorld.
>+        ASSERT(m_empty.isEmpty());
>+        m_canAllocate = m_live & ~m_markingRetired;
>+        return;
>+    }
>+    
>+    m_empty = m_live & ~m_markingNotEmpty;
>+    m_canAllocate = m_live & m_markingNotEmpty & ~m_markingRetired;
>+}
>+
>+void MarkedAllocator::snapshotForEdenCollection()
>+{
>+    m_snapshot |= m_eden;
>+}
>+
>+void MarkedAllocator::snapshotForFullCollection()
>+{
>+    m_snapshot = m_live;
>+}
>+
>+MarkedBlock::Handle* MarkedAllocator::findBlockToSweep()
>+{
>+    m_snapshotCursor = m_snapshot.findBit(m_snapshotCursor, true);
>+    if (m_snapshotCursor >= m_blocks.size())
>+        return nullptr;
>+    return m_blocks[m_snapshotCursor];
>+}
>+
>+void MarkedAllocator::sweep()
>+{
>+    m_snapshot.forEachSetBit(
>+        [&] (size_t index) {
>+            m_blocks[index]->sweep();
>+        });
>+}
>+
>+void MarkedAllocator::shrink()
>+{
>+    m_empty.forEachSetBit(
>+        [&] (size_t index) {
>+            m_markedSpace->freeBlock(m_blocks[index]);
>+        });
>+}
>+
>+void MarkedAllocator::assertSnapshotEmpty()
>+{
>+    ASSERT(m_snapshot.isEmpty());
>+}
>+
>+void MarkedAllocator::dumpBits(PrintStream& out)
>+{
>+    unsigned maxNameLength = 0;
>+    forEachBitVectorWithName(
>+        [&] (FastBitVector&, const char* name) {
>+            unsigned length = strlen(name);
>+            maxNameLength = std::max(maxNameLength, length);
>+        });
>+    
>+    forEachBitVectorWithName(
>+        [&] (FastBitVector& vector, const char* name) {
>+            out.print("    ", name, ": ");
>+            for (unsigned i = maxNameLength - strlen(name); i--;)
>+                out.print(" ");
>+            out.print(vector, "\n");
>+        });
>+}
>+
> } // namespace JSC
>+
>Index: Source/JavaScriptCore/heap/MarkedAllocator.h
>===================================================================
>--- Source/JavaScriptCore/heap/MarkedAllocator.h	(revision 205859)
>+++ Source/JavaScriptCore/heap/MarkedAllocator.h	(working copy)
>@@ -29,7 +29,9 @@
> #include "AllocatorAttributes.h"
> #include "FreeList.h"
> #include "MarkedBlock.h"
>+#include <wtf/FastBitVector.h>
> #include <wtf/SentinelLinkedList.h>
>+#include <wtf/Vector.h>
> 
> namespace JSC {
> 
>@@ -37,6 +39,134 @@ class Heap;
> class MarkedSpace;
> class LLIntOffsetsExtractor;
> 
>+#define FOR_EACH_MARKED_ALLOCATOR_BIT(macro) \
>+    macro(live, Live) /* The set of block indices that have actual blocks. */\
>+    macro(empty, Empty) /* The set of all blocks that have no live objects and nothing to destroy. */ \
>+    macro(allocated, Allocated) /* The set of allblocks that are full of live objects. */\
>+    macro(canAllocate, CanAllocate) /* The set of all blocks that can be allocated in right now. */ \
>+    macro(eden, Eden) /* The set of all blocks that have new objects since the last GC. */\
>+    macro(snapshot, Snapshot) /* The set of all blocks that could be swept by the incremental sweeper. */\
>+    \
>+    /* These are computed during marking. */\
>+    macro(markingNotEmpty, MarkingNotEmpty) /* The set of all blocks that are not empty. */ \
>+    macro(markingRetired, MarkingRetired) /* The set of all blocks that are retired. */
>+
>+// Note that this collector supports overlapping allocator state with marking state, since in a
>+// concurrent collector you allow allocation while marking is running. So it's best to visualize a
>+// full mutable->eden collect->mutate->full collect cycle and see how the bits above get affected.
>+// The example below tries to be exhaustive about what happens to the bits, but omits a lot of
>+// things that happen to other state.
>+//
>+// Create allocator
>+// - all bits are empty
>+// Start allocating in some block
>+// - allocate the block and set the live bit.
>+// - the empty bit for the block flickers on and then gets immediately cleared by sweeping.
>+// - set the eden bit.
>+// Finish allocating in that block
>+// - set the allocated bit.
>+// Do that to a lot of blocks and then start an eden collection.
>+// - beginMarking() has nothing to do.
>+// - by default we have clear markingNotEmpty/markingRetired bits.
>+// - marking builds up markingNotEmpty/markingRetired bits.
>+// We do endMarking()
>+// - clear all allocated bits.
>+// - for destructor blocks: canAllocate = live & ~markingRetired
>+// - for non-destructor blocks:
>+//       empty = live & ~markingNotEmpty
>+//       canAllocate = live & markingNotEmpty & ~markingRetired
>+// Snapshotting.
>+// - snapshot |= eden
>+// Prepare for allocation.
>+// - clear eden
>+// Finish collection.
>+// Allocate in some block that had some free and some live objects.
>+// - clear the canAllocate bit
>+// - clear the snapshot bit
>+// - set the eden bit
>+// Finish allocating (set the allocated bit).
>+// Allocate in some block that was completely empty.
>+// - clear the empty bit
>+// - clear the snapshot bit
>+// - set the eden bit.
>+// Finish allocating (set the allocated bit).
>+// Allocate in some block that was completely empty in another allocator.
>+// - clear the empty bit
>+// - clear all bits in that allocator
>+// - set the live bit in another allocator and the empty bit.
>+// - clear the empty, snapshot bits.
>+// - set the eden bit.
>+// Finish allocating (set the allocated bit).
>+// Start a full collection.
>+// - beginMarking() clears markingNotEmpty, markingRetired
>+// - the heap version is incremented
>+// - marking rebuilds markingNotEmpty/markingretired bits.
>+// We do endMarking()
>+// - clear all allocated bits.
>+// - set canAllocate/empty the same way as in eden collection.
>+// Snapshotting.
>+// - snapshot |= live
>+// prepare for allocation.
>+// - clear eden.
>+// Finish collection.
>+//
>+// Notice how in this scheme, the empty/canAllocate state stays separate from the markingNotEmpty/
>+// markingRetired state. This is one step towards having separated allocation and marking state.
>+//
>+// After this, we just need to answer the question: what does an object allocated during eden GC
>+// look like to the GC?
>+//
>+// I've come up with two compelling answers.
>+//
>+// 1. ALLOCATE BLACK
>+//
>+// I think that the easiest answer is to allocate the object white and then
>+// immediately mark it. This makes the object behave as if it was allocated just before the start
>+// of GC and it was on the stack at the beginning of GC. We wouldn't have to do anything different
>+// with the various block bits when allocating during GC: we could still set eden and allocated
>+// since they will be cleared anyway.
>+//
>+// But this means that the object won't be eligible for freeing until the next full GC. Ideally
>+// we'd make the object is eligible for freeing at the next eden GC. There's a bunch of things we
>+// could do to make this happen. Here are some ideas:
>+//
>+// - Introduce a new CellState, called Float, which means that the object was allocated during the
>+//   GC. We would make sure that marking does nothing for Floats. This would make marking a tiny
>+//   bit more expensive. We would make sure that the barrier recognizes stores to a Float the same
>+//   way that it sees a store to OldBlack: it puts the object on the mark stack. Floats would be
>+//   put on the mark stack as FloatGrey to indicate that they will be scanned by the GC but that
>+//   they aren't actually marked.
>+//
>+//   Note that this implies having to change the state of those cells from Float to something else
>+//   at the end of GC.  I'm not sure that this is so great.
>+//
>+// - Mark the objects after allocation, but also set some other bits to indicate that the those
>+//   objects should be unmarked at the end of the GC.
>+//
>+// Either of these approaches would make collections more expensive, but it feels like the latter
>+// approach would be a profitable optimization. I'm also really getting worried about how this GC
>+// experiences backwards time travel, with all the remarking of things like new objects and
>+// whatnot.
>+//
>+// 2. ALLOCATE WHITE
>+//
>+// I don't know if this is sound, but if it was then it would be the optimal solution. Allocation
>+// would work as now and consequently, everything we know about what was allocated during a GC is
>+// forgotten at the end of GC. But that's fine. End of GC must rescan any heap locations that got
>+// stored into and it must rescan the stack. So, white objects allocated during GC would get
>+// marked if they somehow survived.
>+//
>+// This means that for example if there is a field that often gets stored to with new objects, the
>+// GC will not see all of the values in that field but instead it would poll that field, hopefully
>+// somewhat rarely. We can make it very rare by only infrequently donating from the main thread's
>+// slot visitor to any of the others. So, we would only mark some of the new objects that got
>+// stored into that field. It also means that any object allocated during the GC that didn't get
>+// stored into any object that was live since before the start of GC would simply disappear. This
>+// means much less rescanning than any of the other proposals.
>+//
>+// I really want to go with the allocate-white approach, but I need to think more deeply about it
>+// before I commit to it.
>+
> class MarkedAllocator {
>     friend class LLIntOffsetsExtractor;
> 
>@@ -46,9 +176,16 @@ public:
> 
>     MarkedAllocator(Heap*, MarkedSpace*, size_t cellSize, const AllocatorAttributes&);
>     void lastChanceToFinalize();
>-    void reset();
>+    void prepareForAllocation();
>     void stopAllocating();
>     void resumeAllocating();
>+    void beginMarkingForFullCollection();
>+    void endMarking();
>+    void snapshotForEdenCollection();
>+    void snapshotForFullCollection();
>+    void sweep();
>+    void shrink();
>+    void assertSnapshotEmpty();
>     size_t cellSize() const { return m_cellSize; }
>     const AllocatorAttributes& attributes() const { return m_attributes; }
>     bool needsDestruction() const { return m_attributes.destruction == NeedsDestruction; }
>@@ -72,6 +209,45 @@ public:
>     bool isPagedOut(double deadline);
>     
>     static size_t blockSizeForBytes(size_t);
>+    
>+#define MARKED_ALLOCATOR_BIT_ACCESSORS(lowerBitName, capitalBitName)     \
>+    bool is ## capitalBitName(size_t index) const { return m_ ## lowerBitName[index]; } \
>+    bool is ## capitalBitName(MarkedBlock::Handle* block) const { return is ## capitalBitName(block->index()); } \
>+    void setIs ## capitalBitName(size_t index, bool value) { m_ ## lowerBitName[index] = value; } \
>+    void setIs ## capitalBitName(MarkedBlock::Handle* block, bool value) { setIs ## capitalBitName(block->index(), value); } \
>+    bool atomicSetAndCheckIs ## capitalBitName(size_t index, bool value) { return m_ ## lowerBitName.atomicSetAndCheck(index, value); } \
>+    bool atomicSetAndCheckIs ## capitalBitName(MarkedBlock::Handle* block, bool value) { return m_ ## lowerBitName.atomicSetAndCheck(block->index(), value); }
>+    FOR_EACH_MARKED_ALLOCATOR_BIT(MARKED_ALLOCATOR_BIT_ACCESSORS)
>+#undef MARKED_ALLOCATOR_BIT_ACCESSORS
>+
>+    template<typename Func>
>+    void forEachBitVector(const Func& func)
>+    {
>+#define MARKED_ALLOCATOR_BIT_CALLBACK(lowerBitName, capitalBitName) \
>+        func(m_ ## lowerBitName);
>+        FOR_EACH_MARKED_ALLOCATOR_BIT(MARKED_ALLOCATOR_BIT_CALLBACK);
>+#undef MARKED_ALLOCATOR_BIT_CALLBACK
>+    }
>+    
>+    template<typename Func>
>+    void forEachBitVectorWithName(const Func& func)
>+    {
>+#define MARKED_ALLOCATOR_BIT_CALLBACK(lowerBitName, capitalBitName) \
>+        func(m_ ## lowerBitName, #capitalBitName);
>+        FOR_EACH_MARKED_ALLOCATOR_BIT(MARKED_ALLOCATOR_BIT_CALLBACK);
>+#undef MARKED_ALLOCATOR_BIT_CALLBACK
>+    }
>+    
>+    MarkedAllocator* nextAllocator() const { return m_nextAllocator; }
>+    
>+    // MarkedSpace calls this during init.
>+    void setNextAllocator(MarkedAllocator* allocator) { m_nextAllocator = allocator; }
>+    
>+    MarkedBlock::Handle* findEmptyBlock();
>+    
>+    MarkedBlock::Handle* findBlockToSweep();
>+    
>+    void dumpBits(PrintStream&);
>    
> private:
>     friend class MarkedBlock;
>@@ -82,25 +258,37 @@ private:
>     void* tryAllocateWithoutCollecting();
>     void* tryAllocateWithoutCollectingImpl();
>     MarkedBlock::Handle* tryAllocateBlock();
>+    void* tryAllocateIn(MarkedBlock::Handle*);
>+    void* allocateIn(MarkedBlock::Handle*);
>     ALWAYS_INLINE void doTestCollectionsIfNeeded();
>-    void retire(MarkedBlock::Handle*);
>     
>     void setFreeList(const FreeList&);
>     
>-    MarkedBlock::Handle* filterNextBlock(MarkedBlock::Handle*);
>-    void setNextBlockToSweep(MarkedBlock::Handle*);
>-    
>     FreeList m_freeList;
>+    
>+    Vector<MarkedBlock::Handle*> m_blocks;
>+    Vector<unsigned> m_freeBlockIndices;
>+    
>+#define MARKED_ALLOCATOR_BIT_DECLARATION(lowerBitName, capitalBitName) \
>+    FastBitVector m_ ## lowerBitName;
>+    FOR_EACH_MARKED_ALLOCATOR_BIT(MARKED_ALLOCATOR_BIT_DECLARATION)
>+#undef MARKED_ALLOCATOR_BIT_DECLARATION
>+    
>+    // After you do something to a block based on one of these cursors, you clear the bit in the
>+    // corresponding bitvector and leave the cursor where it was.
>+    size_t m_allocationCursor { 0 }; // Points to the next block that is a candidate for allocation.
>+    size_t m_emptyCursor { 0 }; // Points to the next block that is a candidate for empty allocation (allocating in empty blocks).
>+    size_t m_snapshotCursor { 0 }; // Points to the next block that is a candidate for incremental sweeping.
>+    
>     MarkedBlock::Handle* m_currentBlock;
>     MarkedBlock::Handle* m_lastActiveBlock;
>-    MarkedBlock::Handle* m_nextBlockToSweep;
>-    SentinelLinkedList<MarkedBlock::Handle, BasicRawSentinelNode<MarkedBlock::Handle>> m_blockList;
>-    SentinelLinkedList<MarkedBlock::Handle, BasicRawSentinelNode<MarkedBlock::Handle>> m_retiredBlocks;
>+
>     Lock m_lock;
>     unsigned m_cellSize;
>     AllocatorAttributes m_attributes;
>     Heap* m_heap;
>     MarkedSpace* m_markedSpace;
>+    MarkedAllocator* m_nextAllocator;
> };
> 
> inline ptrdiff_t MarkedAllocator::offsetOfFreeList()
>@@ -149,20 +337,12 @@ ALWAYS_INLINE void* MarkedAllocator::all
>     return head;
> }
> 
>-inline void MarkedAllocator::resumeAllocating()
>-{
>-    if (!m_lastActiveBlock)
>-        return;
>-
>-    m_freeList = m_lastActiveBlock->resumeAllocating();
>-    m_currentBlock = m_lastActiveBlock;
>-    m_lastActiveBlock = 0;
>-}
>-
> template <typename Functor> inline void MarkedAllocator::forEachBlock(const Functor& functor)
> {
>-    m_blockList.forEach(functor);
>-    m_retiredBlocks.forEach(functor);
>+    m_live.forEachSetBit(
>+        [&] (size_t index) {
>+            functor(m_blocks[index]);
>+        });
> }
> 
> } // namespace JSC
>Index: Source/JavaScriptCore/heap/MarkedBlock.cpp
>===================================================================
>--- Source/JavaScriptCore/heap/MarkedBlock.cpp	(revision 205859)
>+++ Source/JavaScriptCore/heap/MarkedBlock.cpp	(working copy)
>@@ -36,7 +36,7 @@ namespace JSC {
> static const bool computeBalance = false;
> static size_t balance;
> 
>-MarkedBlock::Handle* MarkedBlock::tryCreate(Heap& heap, MarkedAllocator* allocator, size_t cellSize, const AllocatorAttributes& attributes)
>+MarkedBlock::Handle* MarkedBlock::tryCreate(Heap& heap, size_t cellSize, const AllocatorAttributes& attributes)
> {
>     if (computeBalance) {
>         balance++;
>@@ -48,24 +48,20 @@ MarkedBlock::Handle* MarkedBlock::tryCre
>         return nullptr;
>     if (scribbleFreeCells())
>         scribble(blockSpace, blockSize);
>-    return new Handle(heap, allocator, cellSize, attributes, blockSpace);
>+    return new Handle(heap, cellSize, attributes, blockSpace);
> }
> 
>-MarkedBlock::Handle::Handle(Heap& heap, MarkedAllocator* allocator, size_t cellSize, const AllocatorAttributes& attributes, void* blockSpace)
>+MarkedBlock::Handle::Handle(Heap& heap, size_t cellSize, const AllocatorAttributes& attributes, void* blockSpace)
>     : m_atomsPerCell((cellSize + atomSize - 1) / atomSize)
>     , m_endAtom(atomsPerBlock - m_atomsPerCell + 1)
>     , m_attributes(attributes)
>-    , m_state(New) // All cells start out unmarked.
>-    , m_allocator(allocator)
>-    , m_weakSet(allocator->heap()->vm(), CellContainer())
>+    , m_weakSet(heap.vm(), CellContainer())
> {
>     m_block = new (NotNull, blockSpace) MarkedBlock(*heap.vm(), *this);
>     
>     m_weakSet.setContainer(*m_block);
>     
>     heap.didAllocateBlock(blockSize);
>-    HEAP_LOG_BLOCK_STATE_TRANSITION(this);
>-    ASSERT(allocator);
>     if (m_attributes.cellKind != HeapCell::JSCell)
>         RELEASE_ASSERT(m_attributes.destruction == DoesNotNeedDestruction);
> }
>@@ -78,6 +74,7 @@ MarkedBlock::Handle::~Handle()
>         if (!(balance % 10))
>             dataLog("MarkedBlock Balance: ", balance, "\n");
>     }
>+    removeFromAllocator();
>     m_block->~MarkedBlock();
>     fastAlignedFree(m_block);
>     heap.didFreeBlock(blockSize);
>@@ -85,7 +82,7 @@ MarkedBlock::Handle::~Handle()
> 
> MarkedBlock::MarkedBlock(VM& vm, Handle& handle)
>     : m_needsDestruction(handle.needsDestruction())
>-    , m_version(vm.heap.objectSpace().version())
>+    , m_version(MarkedSpace::nullVersion)
>     , m_handle(handle)
>     , m_vm(&vm)
> {
>@@ -101,22 +98,21 @@ MarkedBlock::MarkedBlock(VM& vm, Handle&
>     m_biasedMarkCount = m_markCountBias; // This means we haven't marked anything yet.
> }
> 
>-template<MarkedBlock::BlockState blockState, MarkedBlock::Handle::SweepMode sweepMode, DestructionMode destructionMode, MarkedBlock::Handle::ScribbleMode scribbleMode, MarkedBlock::Handle::NewlyAllocatedMode newlyAllocatedMode>
>+template<MarkedBlock::Handle::EmptyMode emptyMode, MarkedBlock::Handle::SweepMode sweepMode, DestructionMode destructionMode, MarkedBlock::Handle::ScribbleMode scribbleMode, MarkedBlock::Handle::NewlyAllocatedMode newlyAllocatedMode, MarkedBlock::Handle::FlipMode flipMode>
> FreeList MarkedBlock::Handle::specializedSweep()
> {
>-    SuperSamplerScope superSamplerScope(false);
>-    ASSERT(blockState == New || blockState == Marked);
>-    ASSERT(!(destructionMode == DoesNotNeedDestruction && sweepMode == SweepOnly));
>+    RELEASE_ASSERT(!(destructionMode == DoesNotNeedDestruction && sweepMode == SweepOnly));
>     
>-    assertFlipped();
>+    SuperSamplerScope superSamplerScope(false);
>+
>     MarkedBlock& block = this->block();
>     
>-    bool isNewBlock = blockState == New;
>-    bool isEmptyBlock = !block.hasAnyMarked()
>-        && newlyAllocatedMode == DoesNotHaveNewlyAllocated
>-        && destructionMode == DoesNotNeedDestruction;
>-    if (Options::useBumpAllocator() && (isNewBlock || isEmptyBlock)) {
>-        ASSERT(block.m_marks.isEmpty());
>+    m_allocator->setIsSnapshot(this, false);
>+    
>+    if (Options::useBumpAllocator()
>+        && emptyMode == IsEmpty
>+        && newlyAllocatedMode == DoesNotHaveNewlyAllocated) {
>+        ASSERT(flipMode == NeedsFlip);
>         
>         char* startOfLastCell = static_cast<char*>(cellAlign(block.atoms() + m_endAtom - 1));
>         char* payloadEnd = startOfLastCell + cellSize();
>@@ -124,7 +120,10 @@ FreeList MarkedBlock::Handle::specialize
>         char* payloadBegin = bitwise_cast<char*>(block.atoms() + firstAtom());
>         if (scribbleMode == Scribble)
>             scribble(payloadBegin, payloadEnd - payloadBegin);
>-        m_state = ((sweepMode == SweepToFreeList) ? FreeListed : Marked);
>+        if (sweepMode == SweepToFreeList)
>+            setIsFreeListed();
>+        else
>+            m_allocator->setIsEmpty(this, true);
>         FreeList result = FreeList::bump(payloadEnd, payloadEnd - payloadBegin);
>         if (false)
>             dataLog("Quickly swept block ", RawPointer(this), " with cell size ", cellSize(), " and attributes ", m_attributes, ": ", result, "\n");
>@@ -137,14 +136,14 @@ FreeList MarkedBlock::Handle::specialize
>     FreeCell* head = 0;
>     size_t count = 0;
>     for (size_t i = firstAtom(); i < m_endAtom; i += m_atomsPerCell) {
>-        if (blockState == Marked
>-            && (block.m_marks.get(i)
>+        if (emptyMode == NotEmpty
>+            && ((flipMode == DoesNotNeedFlip && block.m_marks.get(i))
>                 || (newlyAllocatedMode == HasNewlyAllocated && m_newlyAllocated->get(i))))
>             continue;
>-
>+        
>         HeapCell* cell = reinterpret_cast_ptr<HeapCell*>(&block.atoms()[i]);
> 
>-        if (destructionMode == NeedsDestruction && blockState != New)
>+        if (destructionMode == NeedsDestruction && emptyMode == NotEmpty)
>             static_cast<JSCell*>(cell)->callDestructor(*vm());
> 
>         if (sweepMode == SweepToFreeList) {
>@@ -163,7 +162,13 @@ FreeList MarkedBlock::Handle::specialize
>         m_newlyAllocated = nullptr;
> 
>     FreeList result = FreeList::list(head, count * cellSize());
>-    m_state = (sweepMode == SweepToFreeList ? FreeListed : Marked);
>+    if (sweepMode == SweepToFreeList)
>+        setIsFreeListed();
>+    else {
>+        unsigned cellsPerBlock = MarkedSpace::blockPayload / cellSize();
>+        if (count == cellsPerBlock)
>+            m_allocator->setIsEmpty(this, true);
>+    }
>     if (false)
>         dataLog("Slowly swept block ", RawPointer(&block), " with cell size ", cellSize(), " and attributes ", m_attributes, ": ", result, "\n");
>     return result;
>@@ -171,15 +176,18 @@ FreeList MarkedBlock::Handle::specialize
> 
> FreeList MarkedBlock::Handle::sweep(SweepMode sweepMode)
> {
>-    flipIfNecessary();
>-    
>-    HEAP_LOG_BLOCK_STATE_TRANSITION(this);
>-
>     m_weakSet.sweep();
> 
>     if (sweepMode == SweepOnly && m_attributes.destruction == DoesNotNeedDestruction)
>         return FreeList();
> 
>+    if (UNLIKELY(m_isFreeListed)) {
>+        RELEASE_ASSERT(sweepMode == SweepToFreeList);
>+        return FreeList();
>+    }
>+    
>+    ASSERT(!m_allocator->isAllocated(this));
>+    
>     if (m_attributes.destruction == NeedsDestruction)
>         return sweepHelperSelectScribbleMode<NeedsDestruction>(sweepMode);
>     return sweepHelperSelectScribbleMode<DoesNotNeedDestruction>(sweepMode);
>@@ -189,47 +197,52 @@ template<DestructionMode destructionMode
> FreeList MarkedBlock::Handle::sweepHelperSelectScribbleMode(SweepMode sweepMode)
> {
>     if (scribbleFreeCells())
>-        return sweepHelperSelectStateAndSweepMode<destructionMode, Scribble>(sweepMode);
>-    return sweepHelperSelectStateAndSweepMode<destructionMode, DontScribble>(sweepMode);
>+        return sweepHelperSelectEmptyMode<destructionMode, Scribble>(sweepMode);
>+    return sweepHelperSelectEmptyMode<destructionMode, DontScribble>(sweepMode);
> }
> 
> template<DestructionMode destructionMode, MarkedBlock::Handle::ScribbleMode scribbleMode>
>-FreeList MarkedBlock::Handle::sweepHelperSelectStateAndSweepMode(SweepMode sweepMode)
>+FreeList MarkedBlock::Handle::sweepHelperSelectEmptyMode(SweepMode sweepMode)
> {
>-    switch (m_state) {
>-    case New:
>-        ASSERT(sweepMode == SweepToFreeList);
>-        return specializedSweep<New, SweepToFreeList, destructionMode, scribbleMode, DoesNotHaveNewlyAllocated>();
>-    case FreeListed:
>-        // Happens when a block transitions to fully allocated.
>-        ASSERT(sweepMode == SweepToFreeList);
>-        return FreeList();
>-    case Allocated:
>-        RELEASE_ASSERT_NOT_REACHED();
>-        return FreeList();
>-    case Marked:
>-        if (m_newlyAllocated) {
>-            return sweepMode == SweepToFreeList
>-                ? specializedSweep<Marked, SweepToFreeList, destructionMode, scribbleMode, HasNewlyAllocated>()
>-                : specializedSweep<Marked, SweepOnly, destructionMode, scribbleMode, HasNewlyAllocated>();
>-        } else {
>-            return sweepMode == SweepToFreeList
>-                ? specializedSweep<Marked, SweepToFreeList, destructionMode, scribbleMode, DoesNotHaveNewlyAllocated>()
>-                : specializedSweep<Marked, SweepOnly, destructionMode, scribbleMode, DoesNotHaveNewlyAllocated>();
>-        }
>-    }
>-    RELEASE_ASSERT_NOT_REACHED();
>-    return FreeList();
>+    if (m_allocator->isEmpty(this))
>+        return sweepHelperSelectHasNewlyAllocated<IsEmpty, destructionMode, scribbleMode>(sweepMode);
>+    return sweepHelperSelectHasNewlyAllocated<NotEmpty, destructionMode, scribbleMode>(sweepMode);
>+}
>+
>+template<MarkedBlock::Handle::EmptyMode emptyMode, DestructionMode destructionMode, MarkedBlock::Handle::ScribbleMode scribbleMode>
>+FreeList MarkedBlock::Handle::sweepHelperSelectHasNewlyAllocated(SweepMode sweepMode)
>+{
>+    if (m_newlyAllocated)
>+        return sweepHelperSelectSweepMode<emptyMode, destructionMode, scribbleMode, HasNewlyAllocated>(sweepMode);
>+    return sweepHelperSelectSweepMode<emptyMode, destructionMode, scribbleMode, DoesNotHaveNewlyAllocated>(sweepMode);
>+}
>+
>+template<MarkedBlock::Handle::EmptyMode emptyMode, DestructionMode destructionMode, MarkedBlock::Handle::ScribbleMode scribbleMode, MarkedBlock::Handle::NewlyAllocatedMode newlyAllocatedMode>
>+FreeList MarkedBlock::Handle::sweepHelperSelectSweepMode(SweepMode sweepMode)
>+{
>+    if (sweepMode == SweepToFreeList)
>+        return sweepHelperSelectFlipMode<emptyMode, SweepToFreeList, destructionMode, scribbleMode, newlyAllocatedMode>();
>+    return sweepHelperSelectFlipMode<emptyMode, SweepOnly, destructionMode, scribbleMode, newlyAllocatedMode>();
>+}
>+
>+template<MarkedBlock::Handle::EmptyMode emptyMode, MarkedBlock::Handle::SweepMode sweepMode, DestructionMode destructionMode, MarkedBlock::Handle::ScribbleMode scribbleMode, MarkedBlock::Handle::NewlyAllocatedMode newlyAllocatedMode>
>+FreeList MarkedBlock::Handle::sweepHelperSelectFlipMode()
>+{
>+    if (needsFlip())
>+        return specializedSweep<emptyMode, SweepToFreeList, destructionMode, scribbleMode, newlyAllocatedMode, NeedsFlip>();
>+    return specializedSweep<emptyMode, SweepToFreeList, destructionMode, scribbleMode, newlyAllocatedMode, DoesNotNeedFlip>();
> }
> 
> void MarkedBlock::Handle::unsweepWithNoNewlyAllocated()
> {
>-    flipIfNecessary();
>-    
>-    HEAP_LOG_BLOCK_STATE_TRANSITION(this);
>-    
>-    RELEASE_ASSERT(m_state == FreeListed);
>-    m_state = Marked;
>+    RELEASE_ASSERT(m_isFreeListed);
>+    m_isFreeListed = false;
>+}
>+
>+void MarkedBlock::Handle::setIsFreeListed()
>+{
>+    m_allocator->setIsEmpty(this, false);
>+    m_isFreeListed = true;
> }
> 
> class SetNewlyAllocatedFunctor : public MarkedBlock::VoidFunctor {
>@@ -252,26 +265,15 @@ private:
> 
> void MarkedBlock::Handle::stopAllocating(const FreeList& freeList)
> {
>-    flipIfNecessary();
>-    HEAP_LOG_BLOCK_STATE_TRANSITION(this);
>+    ASSERT(!allocator()->isAllocated(this));
> 
>-    if (m_state == Marked) {
>-        // If the block is in the Marked state then we know that one of these
>-        // conditions holds:
>-        //
>-        // - It was not used for allocation during the previous allocation cycle.
>-        //   It may have dead objects, and we only know them to be dead by the
>-        //   fact that their mark bits are unset.
>-        //
>-        // - Someone had already done stopAllocating(), for example because of
>-        //   heap iteration, and they had already 
>-        // Hence if the block is Marked we need to leave it Marked.
>+    if (!isFreeListed()) {
>+        // This means that we either didn't use this block at all for allocation since last GC,
>+        // or someone had already done stopAllocating() before.
>         ASSERT(freeList.allocationWillFail());
>         return;
>     }
>     
>-    ASSERT(m_state == FreeListed);
>-    
>     // Roll back to a coherent state for Heap introspection. Cells newly
>     // allocated from our free list are not currently marked, so we need another
>     // way to tell what's live vs dead. 
>@@ -290,7 +292,7 @@ void MarkedBlock::Handle::stopAllocating
>             clearNewlyAllocated(cell);
>         });
>     
>-    m_state = Marked;
>+    m_isFreeListed = false;
> }
> 
> void MarkedBlock::Handle::lastChanceToFinalize()
>@@ -304,14 +306,11 @@ void MarkedBlock::Handle::lastChanceToFi
> 
> FreeList MarkedBlock::Handle::resumeAllocating()
> {
>-    flipIfNecessary();
>-    HEAP_LOG_BLOCK_STATE_TRANSITION(this);
>-
>-    ASSERT(m_state == Marked);
>+    ASSERT(!allocator()->isAllocated(this));
>+    ASSERT(!isFreeListed());
> 
>     if (!m_newlyAllocated) {
>-        // We didn't have to create a "newly allocated" bitmap. That means we were already Marked
>-        // when we last stopped allocation, so return an empty free list and stay in the Marked state.
>+        // This means we had already exhausted the block when we stopped allocation.
>         return FreeList();
>     }
> 
>@@ -345,40 +344,28 @@ void MarkedBlock::Handle::forEachFreeCel
>     }
> }
> 
>-void MarkedBlock::flipIfNecessary()
>-{
>-    flipIfNecessary(vm()->heap.objectSpace().version());
>-}
>-
>-void MarkedBlock::Handle::flipIfNecessary()
>-{
>-    block().flipIfNecessary();
>-}
>-
>-void MarkedBlock::flipIfNecessarySlow()
>-{
>-    ASSERT(needsFlip());
>-    ASSERT(!vm()->heap.objectSpace().isMarking());
>-    clearMarks();
>-}
>-
>-void MarkedBlock::flipIfNecessaryDuringMarkingSlow()
>+void MarkedBlock::aboutToMarkSlow(HeapVersion heapVersion)
> {
>     ASSERT(vm()->heap.objectSpace().isMarking());
>     LockHolder locker(m_lock);
>-    if (needsFlip())
>-        clearMarks();
>+    if (needsFlip(heapVersion)) {
>+        clearMarks(heapVersion);
>+        // This means we're the first ones to mark any object in this block.
>+        handle().allocator()->atomicSetAndCheckIsMarkingNotEmpty(&handle(), true);
>+    }
> }
> 
> void MarkedBlock::clearMarks()
> {
>+    clearMarks(vm()->heap.objectSpace().version());
>+}
>+
>+void MarkedBlock::clearMarks(HeapVersion heapVersion)
>+{
>     m_marks.clearAll();
>     clearHasAnyMarked();
>-    // This will become true at the end of the mark phase. We set it now to
>-    // avoid an extra pass to do so later.
>-    handle().m_state = Marked;
>     WTF::storeStoreFence();
>-    m_version = vm()->heap.objectSpace().version();
>+    m_version = heapVersion;
> }
> 
> #if !ASSERT_DISABLED
>@@ -398,31 +385,36 @@ bool MarkedBlock::Handle::needsFlip()
>     return m_block->needsFlip();
> }
> 
>-void MarkedBlock::Handle::willRemoveBlock()
>+bool MarkedBlock::isMarked(const void* p)
> {
>-    flipIfNecessary();
>+    return isMarked(vm()->heap.objectSpace().version(), p);
> }
> 
>-void MarkedBlock::Handle::didConsumeFreeList()
>+bool MarkedBlock::Handle::isMarkedOrNewlyAllocated(const HeapCell* cell)
> {
>-    flipIfNecessary();
>-    HEAP_LOG_BLOCK_STATE_TRANSITION(this);
>-    
>-    ASSERT(m_state == FreeListed);
>+    return isMarkedOrNewlyAllocated(vm()->heap.objectSpace().version(), cell);
>+}
>+
>+bool MarkedBlock::isMarkedOrNewlyAllocated(const HeapCell* cell)
>+{
>+    return isMarkedOrNewlyAllocated(vm()->heap.objectSpace().version(), cell);
>+}
> 
>-    m_state = Allocated;
>+void MarkedBlock::Handle::didConsumeFreeList()
>+{
>+    ASSERT(isFreeListed());
>+    m_isFreeListed = false;
>+    allocator()->setIsAllocated(this, true);
> }
> 
> size_t MarkedBlock::markCount()
> {
>-    flipIfNecessary();
>-    return m_marks.count();
>+    return needsFlip() ? 0 : m_marks.count();
> }
> 
> bool MarkedBlock::Handle::isEmpty()
> {
>-    flipIfNecessary();
>-    return m_state == Marked && !block().hasAnyMarked() && m_weakSet.isEmpty() && (!m_newlyAllocated || m_newlyAllocated->isEmpty());
>+    return m_allocator->isEmpty(this);
> }
> 
> void MarkedBlock::clearHasAnyMarked()
>@@ -432,33 +424,34 @@ void MarkedBlock::clearHasAnyMarked()
> 
> void MarkedBlock::noteMarkedSlow()
> {
>-    handle().m_allocator->retire(&handle());
>+    handle().allocator()->atomicSetAndCheckIsMarkingRetired(&handle(), true);
> }
> 
>-} // namespace JSC
>-
>-namespace WTF {
>+void MarkedBlock::Handle::removeFromAllocator()
>+{
>+    if (!m_allocator)
>+        return;
>+    
>+    m_allocator->removeBlock(this);
>+}
> 
>-using namespace JSC;
>+void MarkedBlock::Handle::didAddToAllocator(MarkedAllocator* allocator, size_t index)
>+{
>+    ASSERT(m_index == std::numeric_limits<size_t>::max());
>+    ASSERT(!m_allocator);
>+    
>+    m_index = index;
>+    m_allocator = allocator;
>+}
> 
>-void printInternal(PrintStream& out, MarkedBlock::BlockState blockState)
>+void MarkedBlock::Handle::didRemoveFromAllocator()
> {
>-    switch (blockState) {
>-    case MarkedBlock::New:
>-        out.print("New");
>-        return;
>-    case MarkedBlock::FreeListed:
>-        out.print("FreeListed");
>-        return;
>-    case MarkedBlock::Allocated:
>-        out.print("Allocated");
>-        return;
>-    case MarkedBlock::Marked:
>-        out.print("Marked");
>-        return;
>-    }
>-    RELEASE_ASSERT_NOT_REACHED();
>+    ASSERT(m_index != std::numeric_limits<size_t>::max());
>+    ASSERT(m_allocator);
>+    
>+    m_index = std::numeric_limits<size_t>::max();
>+    m_allocator = nullptr;
> }
> 
>-} // namespace WTF
>+} // namespace JSC
> 
>Index: Source/JavaScriptCore/heap/MarkedBlock.h
>===================================================================
>--- Source/JavaScriptCore/heap/MarkedBlock.h	(revision 205859)
>+++ Source/JavaScriptCore/heap/MarkedBlock.h	(working copy)
>@@ -44,20 +44,6 @@ class MarkedAllocator;
> typedef uintptr_t Bits;
> typedef uint32_t HeapVersion;
> 
>-// Set to log state transitions of blocks.
>-#define HEAP_LOG_BLOCK_STATE_TRANSITIONS 0
>-
>-#if HEAP_LOG_BLOCK_STATE_TRANSITIONS
>-#define HEAP_LOG_BLOCK_STATE_TRANSITION(handle) do {            \
>-        dataLogF(                                               \
>-            "%s:%d %s: block %s = %p, %d\n",                    \
>-            __FILE__, __LINE__, __FUNCTION__,                   \
>-            #handle, &(handle)->block(), (handle)->m_state);    \
>-    } while (false)
>-#else
>-#define HEAP_LOG_BLOCK_STATE_TRANSITION(handle) ((void)0)
>-#endif
>-
> // A marked block is a page-aligned container for heap-allocated objects.
> // Objects are allocated within cells of the marked block. For a given
> // marked block, all cells have the same size. Objects smaller than the
>@@ -76,8 +62,6 @@ public:
> private:
>     friend class Handle;
> public:
>-    enum BlockState : uint8_t { New, FreeListed, Allocated, Marked };
>-        
>     static const size_t atomSize = 16; // bytes
>     static const size_t blockSize = 16 * KB;
>     static const size_t blockMask = ~(blockSize - 1); // blockSize must be a power of two.
>@@ -86,12 +70,12 @@ public:
> 
>     static_assert(!(MarkedBlock::atomSize & (MarkedBlock::atomSize - 1)), "MarkedBlock::atomSize must be a power of two.");
>     static_assert(!(MarkedBlock::blockSize & (MarkedBlock::blockSize - 1)), "MarkedBlock::blockSize must be a power of two.");
>-
>+    
>     struct VoidFunctor {
>         typedef void ReturnType;
>         void returnValue() { }
>     };
>-
>+    
>     class CountFunctor {
>     public:
>         typedef size_t ReturnType;
>@@ -130,6 +114,14 @@ public:
>         VM* vm() const;
>         WeakSet& weakSet();
>             
>+        // Sweeping ensures that destructors get called and removes the block from the snapshot.
>+        // Sweeping to free list also removes the block from the empty set, if it was in that set.
>+        // Sweeping only may add this block to the empty set, if the block is found to be empty.
>+        //
>+        // Note that you need to make sure that the empty bit reflects reality. If it's not set
>+        // and the block is freshly created, then we'll make the mistake of running destructors in
>+        // the block. If it's not set and the block has nothing marked, then we'll make the
>+        // mistake of making a pop freelist rather than a bump freelist.
>         enum SweepMode { SweepOnly, SweepToFreeList };
>         FreeList sweep(SweepMode = SweepOnly);
>         
>@@ -153,8 +145,6 @@ public:
>         // and was successfully cleared and false otherwise.
>         bool clearNewlyAllocated();
>             
>-        void flipForEdenCollection();
>-            
>         size_t cellSize();
>         const AllocatorAttributes& attributes() const;
>         DestructionMode destruction() const;
>@@ -164,9 +154,11 @@ public:
>         size_t markCount();
>         size_t size();
>             
>-        bool isLive(const HeapCell*);
>-        bool isLiveCell(const void*);
>+        inline bool isLive(const HeapCell*);
>+        inline bool isLiveCell(const void*);
>         bool isMarkedOrNewlyAllocated(const HeapCell*);
>+        bool isMarkedOrNewlyAllocated(HeapVersion, const HeapCell*);
>+        bool isMarkedOrNewlyAllocatedConcurrently(HeapVersion, const HeapCell*);
>             
>         bool isNewlyAllocated(const void*);
>         void setNewlyAllocated(const void*);
>@@ -174,31 +166,25 @@ public:
>         
>         bool hasAnyNewlyAllocated() const { return !!m_newlyAllocated; }
>             
>-        bool isAllocated() const;
>-        bool isMarked() const;
>-        bool isFreeListed() const;
>-        bool needsSweeping() const;
>-        void willRemoveBlock();
>-
>         template <typename Functor> IterationStatus forEachCell(const Functor&);
>-        template <typename Functor> IterationStatus forEachLiveCell(const Functor&);
>-        template <typename Functor> IterationStatus forEachDeadCell(const Functor&);
>+        template <typename Functor> inline IterationStatus forEachLiveCell(const Functor&);
>+        template <typename Functor> inline IterationStatus forEachDeadCell(const Functor&);
>             
>         bool needsFlip();
>-            
>-        void flipIfNecessaryDuringMarking(HeapVersion);
>-        void flipIfNecessary(HeapVersion);
>-        void flipIfNecessary();
>-            
>+        
>         void assertFlipped();
>             
>-        bool isOnBlocksToSweep() const { return m_isOnBlocksToSweep; }
>-        void setIsOnBlocksToSweep(bool value) { m_isOnBlocksToSweep = value; }
>+        bool isFreeListed() const { return m_isFreeListed; }
>+        
>+        size_t index() const { return m_index; }
>+        
>+        void removeFromAllocator();
>+        
>+        void didAddToAllocator(MarkedAllocator*, size_t index);
>+        void didRemoveFromAllocator();
>         
>-        BlockState state() const { return m_state; }
>-            
>     private:
>-        Handle(Heap&, MarkedAllocator*, size_t cellSize, const AllocatorAttributes&, void*);
>+        Handle(Heap&, size_t cellSize, const AllocatorAttributes&, void*);
>             
>         template<DestructionMode>
>         FreeList sweepHelperSelectScribbleMode(SweepMode = SweepOnly);
>@@ -206,16 +192,31 @@ public:
>         enum ScribbleMode { DontScribble, Scribble };
>             
>         template<DestructionMode, ScribbleMode>
>-        FreeList sweepHelperSelectStateAndSweepMode(SweepMode = SweepOnly);
>+        FreeList sweepHelperSelectEmptyMode(SweepMode = SweepOnly);
>             
>+        enum EmptyMode { IsEmpty, NotEmpty };
>+        
>+        template<EmptyMode, DestructionMode, ScribbleMode>
>+        FreeList sweepHelperSelectHasNewlyAllocated(SweepMode = SweepOnly);
>+        
>         enum NewlyAllocatedMode { HasNewlyAllocated, DoesNotHaveNewlyAllocated };
>-            
>-        template<BlockState, SweepMode, DestructionMode, ScribbleMode, NewlyAllocatedMode>
>+        
>+        template<EmptyMode, DestructionMode, ScribbleMode, NewlyAllocatedMode>
>+        FreeList sweepHelperSelectSweepMode(SweepMode = SweepOnly);
>+        
>+        template<EmptyMode, SweepMode, DestructionMode, ScribbleMode, NewlyAllocatedMode>
>+        FreeList sweepHelperSelectFlipMode();
>+        
>+        enum FlipMode { NeedsFlip, DoesNotNeedFlip };
>+        
>+        template<EmptyMode, SweepMode, DestructionMode, ScribbleMode, NewlyAllocatedMode, FlipMode>
>         FreeList specializedSweep();
>             
>         template<typename Func>
>         void forEachFreeCell(const FreeList&, const Func&);
>-            
>+        
>+        void setIsFreeListed();
>+        
>         MarkedBlock::Handle* m_prev;
>         MarkedBlock::Handle* m_next;
>             
>@@ -225,16 +226,16 @@ public:
>         std::unique_ptr<WTF::Bitmap<atomsPerBlock>> m_newlyAllocated;
>             
>         AllocatorAttributes m_attributes;
>-        BlockState m_state;
>-        bool m_isOnBlocksToSweep { false };
>+        bool m_isFreeListed { false };
>             
>-        MarkedAllocator* m_allocator;
>+        MarkedAllocator* m_allocator { nullptr };
>+        size_t m_index { std::numeric_limits<size_t>::max() };
>         WeakSet m_weakSet;
>             
>         MarkedBlock* m_block;
>     };
>         
>-    static MarkedBlock::Handle* tryCreate(Heap&, MarkedAllocator*, size_t cellSize, const AllocatorAttributes&);
>+    static MarkedBlock::Handle* tryCreate(Heap&, size_t cellSize, const AllocatorAttributes&);
>         
>     Handle& handle();
>         
>@@ -248,34 +249,37 @@ public:
>     size_t markCount();
> 
>     bool isMarked(const void*);
>+    bool isMarked(HeapVersion, const void*);
>+    bool isMarkedConcurrently(HeapVersion, const void*);
>     bool testAndSetMarked(const void*);
>         
>     bool isMarkedOrNewlyAllocated(const HeapCell*);
>-    bool isMarkedOrNewlyAllocatedDuringWeakVisiting(HeapVersion, const HeapCell*);
>-
>+    bool isMarkedOrNewlyAllocated(HeapVersion, const HeapCell*);
>+    bool isMarkedOrNewlyAllocatedConcurrently(HeapVersion, const HeapCell*);
>+    
>     bool isAtom(const void*);
>     void setMarked(const void*);
>     void clearMarked(const void*);
>-        
>+    
>     size_t cellSize();
>     const AllocatorAttributes& attributes() const;
>-
>+    
>     bool hasAnyMarked() const;
>     void noteMarked();
>         
>     WeakSet& weakSet();
>-
>+    
>     bool needsFlip(HeapVersion);
>     bool needsFlip();
>-        
>-    void flipIfNecessaryDuringMarking(HeapVersion);
>-    void flipIfNecessary(HeapVersion);
>-    void flipIfNecessary();
>+    
>+    void aboutToMark(HeapVersion);
>         
>     void assertFlipped();
>         
>     bool needsDestruction() const { return m_needsDestruction; }
>-        
>+    
>+    inline void resetVersion();
>+    
> private:
>     static const size_t atomAlignmentMask = atomSize - 1;
> 
>@@ -284,9 +288,9 @@ private:
>     MarkedBlock(VM&, Handle&);
>     Atom* atoms();
>         
>-    void flipIfNecessaryDuringMarkingSlow();
>-    void flipIfNecessarySlow();
>+    void aboutToMarkSlow(HeapVersion);
>     void clearMarks();
>+    void clearMarks(HeapVersion);
>     void clearHasAnyMarked();
>     
>     void noteMarkedSlow();
>@@ -469,40 +473,13 @@ inline bool MarkedBlock::needsFlip(HeapV
>     return heapVersion != m_version;
> }
> 
>-inline void MarkedBlock::flipIfNecessary(HeapVersion heapVersion)
>-{
>-    if (UNLIKELY(needsFlip(heapVersion)))
>-        flipIfNecessarySlow();
>-}
>-
>-inline void MarkedBlock::flipIfNecessaryDuringMarking(HeapVersion heapVersion)
>+inline void MarkedBlock::aboutToMark(HeapVersion heapVersion)
> {
>     if (UNLIKELY(needsFlip(heapVersion)))
>-        flipIfNecessaryDuringMarkingSlow();
>+        aboutToMarkSlow(heapVersion);
>     WTF::loadLoadFence();
> }
> 
>-inline void MarkedBlock::Handle::flipIfNecessary(HeapVersion heapVersion)
>-{
>-    block().flipIfNecessary(heapVersion);
>-}
>-
>-inline void MarkedBlock::Handle::flipIfNecessaryDuringMarking(HeapVersion heapVersion)
>-{
>-    block().flipIfNecessaryDuringMarking(heapVersion);
>-}
>-
>-inline void MarkedBlock::Handle::flipForEdenCollection()
>-{
>-    assertFlipped();
>-        
>-    HEAP_LOG_BLOCK_STATE_TRANSITION(this);
>-    
>-    ASSERT(m_state != New && m_state != FreeListed);
>-    
>-    m_state = Marked;
>-}
>-
> #if ASSERT_DISABLED
> inline void MarkedBlock::assertFlipped()
> {
>@@ -514,9 +491,16 @@ inline void MarkedBlock::Handle::assertF
>     block().assertFlipped();
> }
> 
>-inline bool MarkedBlock::isMarked(const void* p)
>+inline bool MarkedBlock::isMarked(HeapVersion heapVersion, const void* p)
> {
>-    assertFlipped();
>+    return needsFlip(heapVersion) ? false : m_marks.get(atomNumber(p));
>+}
>+
>+inline bool MarkedBlock::isMarkedConcurrently(HeapVersion heapVersion, const void* p)
>+{
>+    if (needsFlip(heapVersion))
>+        return false;
>+    WTF::loadLoadFence();
>     return m_marks.get(atomNumber(p));
> }
> 
>@@ -550,43 +534,24 @@ inline bool MarkedBlock::Handle::clearNe
>     return false;
> }
> 
>-inline bool MarkedBlock::Handle::isMarkedOrNewlyAllocated(const HeapCell* cell)
>+inline bool MarkedBlock::Handle::isMarkedOrNewlyAllocated(HeapVersion version, const HeapCell* cell)
> {
>-    ASSERT(m_state == Marked);
>-    return m_block->isMarked(cell) || (m_newlyAllocated && isNewlyAllocated(cell));
>+    return m_block->isMarked(version, cell) || (m_newlyAllocated && isNewlyAllocated(cell));
> }
> 
>-inline bool MarkedBlock::isMarkedOrNewlyAllocated(const HeapCell* cell)
>+inline bool MarkedBlock::Handle::isMarkedOrNewlyAllocatedConcurrently(HeapVersion version, const HeapCell* cell)
> {
>-    ASSERT(m_handle.m_state == Marked);
>-    return isMarked(cell) || (m_handle.m_newlyAllocated && m_handle.isNewlyAllocated(cell));
>+    return m_block->isMarkedConcurrently(version, cell) || (m_newlyAllocated && isNewlyAllocated(cell));
> }
> 
>-inline bool MarkedBlock::isMarkedOrNewlyAllocatedDuringWeakVisiting(HeapVersion heapVersion, const HeapCell* cell)
>+inline bool MarkedBlock::isMarkedOrNewlyAllocated(HeapVersion version, const HeapCell* cell)
> {
>-    if (needsFlip(heapVersion))
>-        return false;
>-    return isMarkedOrNewlyAllocated(cell);
>+    return isMarked(version, cell) || (m_handle.m_newlyAllocated && m_handle.isNewlyAllocated(cell));
> }
> 
>-inline bool MarkedBlock::Handle::isLive(const HeapCell* cell)
>+inline bool MarkedBlock::isMarkedOrNewlyAllocatedConcurrently(HeapVersion version, const HeapCell* cell)
> {
>-    assertFlipped();
>-    switch (m_state) {
>-    case Allocated:
>-        return true;
>-
>-    case Marked:
>-        return isMarkedOrNewlyAllocated(cell);
>-
>-    case New:
>-    case FreeListed:
>-        RELEASE_ASSERT_NOT_REACHED();
>-        return false;
>-    }
>-
>-    RELEASE_ASSERT_NOT_REACHED();
>-    return false;
>+    return isMarkedConcurrently(version, cell) || (m_handle.m_newlyAllocated && m_handle.isNewlyAllocated(cell));
> }
> 
> inline bool MarkedBlock::isAtom(const void* p)
>@@ -603,13 +568,6 @@ inline bool MarkedBlock::isAtom(const vo
>     return true;
> }
> 
>-inline bool MarkedBlock::Handle::isLiveCell(const void* p)
>-{
>-    if (!m_block->isAtom(p))
>-        return false;
>-    return isLive(static_cast<const HeapCell*>(p));
>-}
>-
> template <typename Functor>
> inline IterationStatus MarkedBlock::Handle::forEachCell(const Functor& functor)
> {
>@@ -622,62 +580,6 @@ inline IterationStatus MarkedBlock::Hand
>     return IterationStatus::Continue;
> }
> 
>-template <typename Functor>
>-inline IterationStatus MarkedBlock::Handle::forEachLiveCell(const Functor& functor)
>-{
>-    flipIfNecessary();
>-    HeapCell::Kind kind = m_attributes.cellKind;
>-    for (size_t i = firstAtom(); i < m_endAtom; i += m_atomsPerCell) {
>-        HeapCell* cell = reinterpret_cast_ptr<HeapCell*>(&m_block->atoms()[i]);
>-        if (!isLive(cell))
>-            continue;
>-
>-        if (functor(cell, kind) == IterationStatus::Done)
>-            return IterationStatus::Done;
>-    }
>-    return IterationStatus::Continue;
>-}
>-
>-template <typename Functor>
>-inline IterationStatus MarkedBlock::Handle::forEachDeadCell(const Functor& functor)
>-{
>-    flipIfNecessary();
>-    HeapCell::Kind kind = m_attributes.cellKind;
>-    for (size_t i = firstAtom(); i < m_endAtom; i += m_atomsPerCell) {
>-        HeapCell* cell = reinterpret_cast_ptr<HeapCell*>(&m_block->atoms()[i]);
>-        if (isLive(cell))
>-            continue;
>-
>-        if (functor(cell, kind) == IterationStatus::Done)
>-            return IterationStatus::Done;
>-    }
>-    return IterationStatus::Continue;
>-}
>-
>-inline bool MarkedBlock::Handle::needsSweeping() const
>-{
>-    const_cast<MarkedBlock::Handle*>(this)->flipIfNecessary();
>-    return m_state == Marked;
>-}
>-
>-inline bool MarkedBlock::Handle::isAllocated() const
>-{
>-    const_cast<MarkedBlock::Handle*>(this)->flipIfNecessary();
>-    return m_state == Allocated;
>-}
>-
>-inline bool MarkedBlock::Handle::isMarked() const
>-{
>-    const_cast<MarkedBlock::Handle*>(this)->flipIfNecessary();
>-    return m_state == Marked;
>-}
>-
>-inline bool MarkedBlock::Handle::isFreeListed() const
>-{
>-    const_cast<MarkedBlock::Handle*>(this)->flipIfNecessary();
>-    return m_state == FreeListed;
>-}
>-
> inline bool MarkedBlock::hasAnyMarked() const
> {
>     return m_biasedMarkCount != m_markCountBias;
>@@ -711,8 +613,6 @@ template<> struct DefaultHash<JSC::Marke
>     typedef MarkedBlockHash Hash;
> };
> 
>-void printInternal(PrintStream& out, JSC::MarkedBlock::BlockState);
>-
> } // namespace WTF
> 
> #endif // MarkedBlock_h
>Index: Source/JavaScriptCore/heap/MarkedBlockInlines.h
>===================================================================
>--- Source/JavaScriptCore/heap/MarkedBlockInlines.h	(nonexistent)
>+++ Source/JavaScriptCore/heap/MarkedBlockInlines.h	(working copy)
>@@ -0,0 +1,89 @@
>+/*
>+ * Copyright (C) 2016 Apple Inc. All rights reserved.
>+ *
>+ * Redistribution and use in source and binary forms, with or without
>+ * modification, are permitted provided that the following conditions
>+ * are met:
>+ * 1. Redistributions of source code must retain the above copyright
>+ *    notice, this list of conditions and the following disclaimer.
>+ * 2. Redistributions in binary form must reproduce the above copyright
>+ *    notice, this list of conditions and the following disclaimer in the
>+ *    documentation and/or other materials provided with the distribution.
>+ *
>+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
>+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
>+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
>+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
>+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
>+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
>+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
>+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
>+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
>+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
>+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
>+ */
>+
>+#pragma once
>+
>+#include "MarkedAllocator.h"
>+#include "MarkedBlock.h"
>+
>+namespace JSC {
>+
>+inline bool MarkedBlock::Handle::isLive(const HeapCell* cell)
>+{
>+    ASSERT(!isFreeListed());
>+    
>+    if (needsFlip())
>+        return false;
>+
>+    if (allocator()->isAllocated(this))
>+        return true;
>+    
>+    return isMarkedOrNewlyAllocated(cell);
>+}
>+
>+inline bool MarkedBlock::Handle::isLiveCell(const void* p)
>+{
>+    if (!m_block->isAtom(p))
>+        return false;
>+    return isLive(static_cast<const HeapCell*>(p));
>+}
>+
>+template <typename Functor>
>+inline IterationStatus MarkedBlock::Handle::forEachLiveCell(const Functor& functor)
>+{
>+    HeapCell::Kind kind = m_attributes.cellKind;
>+    for (size_t i = firstAtom(); i < m_endAtom; i += m_atomsPerCell) {
>+        HeapCell* cell = reinterpret_cast_ptr<HeapCell*>(&m_block->atoms()[i]);
>+        if (!isLive(cell))
>+            continue;
>+
>+        if (functor(cell, kind) == IterationStatus::Done)
>+            return IterationStatus::Done;
>+    }
>+    return IterationStatus::Continue;
>+}
>+
>+template <typename Functor>
>+inline IterationStatus MarkedBlock::Handle::forEachDeadCell(const Functor& functor)
>+{
>+    HeapCell::Kind kind = m_attributes.cellKind;
>+    for (size_t i = firstAtom(); i < m_endAtom; i += m_atomsPerCell) {
>+        HeapCell* cell = reinterpret_cast_ptr<HeapCell*>(&m_block->atoms()[i]);
>+        if (isLive(cell))
>+            continue;
>+
>+        if (functor(cell, kind) == IterationStatus::Done)
>+            return IterationStatus::Done;
>+    }
>+    return IterationStatus::Continue;
>+}
>+
>+inline void MarkedBlock::resetVersion()
>+{
>+    m_version = MarkedSpace::nullVersion;
>+}
>+
>+} // namespace JSC
>+
>Index: Source/JavaScriptCore/heap/MarkedSpace.cpp
>===================================================================
>--- Source/JavaScriptCore/heap/MarkedSpace.cpp	(revision 205859)
>+++ Source/JavaScriptCore/heap/MarkedSpace.cpp	(working copy)
>@@ -24,6 +24,7 @@
> #include "IncrementalSweeper.h"
> #include "JSObject.h"
> #include "JSCInlines.h"
>+#include "MarkedBlockInlines.h"
> #include "SuperSampler.h"
> #include <wtf/ListDump.h>
> 
>@@ -195,6 +196,19 @@ MarkedSpace::MarkedSpace(Heap* heap)
>             
>             return IterationStatus::Continue;
>         });
>+    
>+    MarkedAllocator* previous = nullptr;
>+    forEachAllocator(
>+        [&] (MarkedAllocator& allocator) -> IterationStatus {
>+            // We can set this in any order. The point is just that we want to be able to
>+            // cursor over allocators.
>+            allocator.setNextAllocator(previous);
>+            
>+            previous = &allocator;
>+            return IterationStatus::Continue;
>+        });
>+    m_firstAllocator = previous;
>+    m_allocatorForEmptyAllocation = previous;
> }
> 
> MarkedSpace::~MarkedSpace()
>@@ -257,9 +271,10 @@ void* MarkedSpace::tryAllocateLarge(Subs
> void MarkedSpace::sweep()
> {
>     m_heap->sweeper()->willFinishSweeping();
>-    forEachBlock(
>-        [&] (MarkedBlock::Handle* block) {
>-            block->sweep();
>+    forEachAllocator(
>+        [&] (MarkedAllocator& allocator) -> IterationStatus {
>+            allocator.sweep();
>+            return IterationStatus::Continue;
>         });
> }
> 
>@@ -282,28 +297,16 @@ void MarkedSpace::sweepLargeAllocations(
>     m_largeAllocationsNurseryOffset = m_largeAllocations.size();
> }
> 
>-void MarkedSpace::zombifySweep()
>-{
>-    if (Options::logGC())
>-        dataLog("Zombifying sweep...");
>-    m_heap->sweeper()->willFinishSweeping();
>-    forEachBlock(
>-        [&] (MarkedBlock::Handle* block) {
>-            if (block->needsSweeping())
>-                block->sweep();
>-        });
>-}
>-
>-void MarkedSpace::resetAllocators()
>+void MarkedSpace::prepareForAllocation()
> {
>     forEachAllocator(
>         [&] (MarkedAllocator& allocator) -> IterationStatus {
>-            allocator.reset();
>+            allocator.prepareForAllocation();
>             return IterationStatus::Continue;
>         });
> 
>-    m_blocksWithNewObjects.clear();
>     m_activeWeakSets.takeFrom(m_newActiveWeakSets);
>+    
>     if (m_heap->operationInProgress() == EdenCollection)
>         m_largeAllocationsNurseryOffsetForSweep = m_largeAllocationsNurseryOffset;
>     else
>@@ -408,11 +411,11 @@ void MarkedSpace::freeOrShrinkBlock(Mark
> 
> void MarkedSpace::shrink()
> {
>-    forEachBlock(
>-        [&] (MarkedBlock::Handle* block) {
>-            freeOrShrinkBlock(block);
>+    forEachAllocator(
>+        [&] (MarkedAllocator& allocator) -> IterationStatus {
>+            allocator.shrink();
>+            return IterationStatus::Continue;
>         });
>-    // For LargeAllocations, we do the moral equivalent in sweepLargeAllocations().
> }
> 
> void MarkedSpace::clearNewlyAllocated()
>@@ -427,57 +430,61 @@ void MarkedSpace::clearNewlyAllocated()
>     for (unsigned i = m_largeAllocationsOffsetForThisCollection; i < m_largeAllocations.size(); ++i)
>         m_largeAllocations[i]->clearNewlyAllocated();
> 
>-#if !ASSERT_DISABLED
>-    forEachBlock(
>-        [&] (MarkedBlock::Handle* block) {
>-            ASSERT(!block->clearNewlyAllocated());
>-        });
>-
>-    for (LargeAllocation* allocation : m_largeAllocations)
>-        ASSERT(!allocation->isNewlyAllocated());
>-#endif // !ASSERT_DISABLED
>-}
>-
>-#ifndef NDEBUG 
>-struct VerifyMarked : MarkedBlock::VoidFunctor { 
>-    void operator()(MarkedBlock::Handle* block) const
>-    {
>-        if (block->needsFlip())
>-            return;
>-        switch (block->m_state) {
>-        case MarkedBlock::Marked:
>-            return;
>-        default:
>-            RELEASE_ASSERT_NOT_REACHED();
>-        }
>+    if (!ASSERT_DISABLED) {
>+        forEachBlock(
>+            [&] (MarkedBlock::Handle* block) {
>+                ASSERT(!block->clearNewlyAllocated());
>+            });
>+        
>+        for (LargeAllocation* allocation : m_largeAllocations)
>+            ASSERT(!allocation->isNewlyAllocated());
>     }
>-}; 
>-#endif 
>+}
> 
>-void MarkedSpace::flip()
>+void MarkedSpace::beginMarking()
> {
>-    if (m_heap->operationInProgress() == EdenCollection) {
>-        for (unsigned i = 0; i < m_blocksWithNewObjects.size(); ++i)
>-            m_blocksWithNewObjects[i]->flipForEdenCollection();
>-    } else {
>-        HeapVersion nextVersion = m_version + 1;
>-        if (UNLIKELY(nextVersion == initialVersion)) {
>-            // Oh no! Version wrap-around! We handle this by flipping all blocks. This happens
>-            // super rarely, probably never for most users.
>+    if (m_heap->operationInProgress() == FullCollection) {
>+        forEachAllocator(
>+            [&] (MarkedAllocator& allocator) -> IterationStatus {
>+                allocator.beginMarkingForFullCollection();
>+                return IterationStatus::Continue;
>+            });
>+
>+        m_version = nextVersion(m_version);
>+        
>+        if (UNLIKELY(m_version == initialVersion)) {
>+            // Oh no! Version wrap-around! We handle this by setting all block versions to null.
>             forEachBlock(
>                 [&] (MarkedBlock::Handle* handle) {
>-                    handle->flipIfNecessary();
>+                    handle->block().resetVersion();
>                 });
>         }
>-        m_version = nextVersion; // Henceforth, flipIfNecessary() will trigger on all blocks.
>+        
>         for (LargeAllocation* allocation : m_largeAllocations)
>             allocation->flip();
>     }
> 
>-#ifndef NDEBUG
>-    VerifyMarked verifyFunctor;
>-    forEachBlock(verifyFunctor);
>-#endif
>+    if (!ASSERT_DISABLED) {
>+        forEachBlock(
>+            [&] (MarkedBlock::Handle* block) {
>+                if (block->needsFlip())
>+                    return;
>+                ASSERT(!block->isFreeListed());
>+            });
>+    }
>+    
>+    m_isMarking = true;
>+}
>+
>+void MarkedSpace::endMarking()
>+{
>+    forEachAllocator(
>+        [&] (MarkedAllocator& allocator) -> IterationStatus {
>+            allocator.endMarking();
>+            return IterationStatus::Continue;
>+        });
>+    
>+    m_isMarking = false;
> }
> 
> void MarkedSpace::willStartIterating()
>@@ -544,13 +551,47 @@ void MarkedSpace::didAddBlock(MarkedBloc
> 
> void MarkedSpace::didAllocateInBlock(MarkedBlock::Handle* block)
> {
>-    block->assertFlipped();
>-    m_blocksWithNewObjects.append(block);
>-    
>     if (block->weakSet().isOnList()) {
>         block->weakSet().remove();
>         m_newActiveWeakSets.append(&block->weakSet());
>     }
> }
> 
>+MarkedBlock::Handle* MarkedSpace::findEmptyBlock()
>+{
>+    for (; m_allocatorForEmptyAllocation; m_allocatorForEmptyAllocation = m_allocatorForEmptyAllocation->nextAllocator()) {
>+        if (MarkedBlock::Handle* block = m_allocatorForEmptyAllocation->findEmptyBlock())
>+            return block;
>+    }
>+    return nullptr;
>+}
>+
>+void MarkedSpace::snapshot()
>+{
>+    if (m_heap->operationInProgress() == EdenCollection) {
>+        forEachAllocator(
>+            [&] (MarkedAllocator& allocator) -> IterationStatus {
>+                allocator.snapshotForEdenCollection();
>+                return IterationStatus::Continue;
>+            });
>+    } else {
>+        forEachAllocator(
>+            [&] (MarkedAllocator& allocator) -> IterationStatus {
>+                allocator.snapshotForFullCollection();
>+                return IterationStatus::Continue;
>+            });
>+    }
>+}
>+
>+void MarkedSpace::assertSnapshotEmpty()
>+{
>+    if (ASSERT_DISABLED)
>+        return;
>+    forEachAllocator(
>+        [&] (MarkedAllocator& allocator) -> IterationStatus {
>+            allocator.assertSnapshotEmpty();
>+            return IterationStatus::Continue;
>+        });
>+}
>+
> } // namespace JSC
>Index: Source/JavaScriptCore/heap/MarkedSpace.h
>===================================================================
>--- Source/JavaScriptCore/heap/MarkedSpace.h	(revision 205859)
>+++ Source/JavaScriptCore/heap/MarkedSpace.h	(working copy)
>@@ -65,7 +65,16 @@ public:
> 
>     static const size_t numSizeClasses = largeCutoff / sizeStep;
>     
>-    static const HeapVersion initialVersion = 42;  // This can be any value, including random garbage, so long as it's consistent for the lifetime of the process.
>+    static const HeapVersion nullVersion = 0; // The version of freshly allocated blocks.
>+    static const HeapVersion initialVersion = 1; // The version that the heap starts out with.
>+    
>+    HeapVersion nextVersion(HeapVersion version)
>+    {
>+        version++;
>+        if (version == nullVersion)
>+            version++;
>+        return version;
>+    }
>     
>     static size_t sizeClassToIndex(size_t size)
>     {
>@@ -114,7 +123,7 @@ public:
>     Subspace& subspaceForObjectsWithoutDestructor() { return m_normalSpace; }
>     Subspace& subspaceForAuxiliaryData() { return m_auxiliarySpace; }
>     
>-    void resetAllocators();
>+    void prepareForAllocation();
> 
>     void visitWeakSets(HeapRootVisitor&);
>     void reapWeakSets();
>@@ -144,11 +153,13 @@ public:
>     void didConsumeFreeList(MarkedBlock::Handle*);
>     void didAllocateInBlock(MarkedBlock::Handle*);
> 
>-    void flip();
>+    void beginMarking();
>+    void endMarking();
>+    void snapshot();
>     void clearNewlyAllocated();
>     void sweep();
>     void sweepLargeAllocations();
>-    void zombifySweep();
>+    void assertSnapshotEmpty();
>     size_t objectCount();
>     size_t size();
>     size_t capacity();
>@@ -157,8 +168,6 @@ public:
>     
>     HeapVersion version() const { return m_version; }
> 
>-    const Vector<MarkedBlock::Handle*>& blocksWithNewObjects() const { return m_blocksWithNewObjects; }
>-    
>     const Vector<LargeAllocation*>& largeAllocations() const { return m_largeAllocations; }
>     unsigned largeAllocationsNurseryOffset() const { return m_largeAllocationsNurseryOffset; }
>     unsigned largeAllocationsOffsetForThisCollection() const { return m_largeAllocationsOffsetForThisCollection; }
>@@ -169,13 +178,14 @@ public:
>     LargeAllocation** largeAllocationsForThisCollectionEnd() const { return m_largeAllocationsForThisCollectionEnd; }
>     unsigned largeAllocationsForThisCollectionSize() const { return m_largeAllocationsForThisCollectionSize; }
>     
>+    MarkedAllocator* firstAllocator() const { return m_firstAllocator; }
>+    MarkedAllocator* allocatorForEmptyAllocation() const { return m_allocatorForEmptyAllocation; }
>+    
>+    MarkedBlock::Handle* findEmptyBlock();
>+    
>     // When this is true it means that we have flipped but the mark bits haven't converged yet.
>     bool isMarking() const { return m_isMarking; }
>     
>-    // FIXME: After https://bugs.webkit.org/show_bug.cgi?id=161581, MarkedSpace will control this
>-    // flag directly.
>-    void setIsMarking(bool value) { m_isMarking = value; }
>-
> private:
>     friend class LLIntOffsetsExtractor;
>     friend class JIT;
>@@ -190,8 +200,8 @@ private:
>     
>     void initializeSubspace(Subspace&);
> 
>-    template<typename Functor> void forEachAllocator(const Functor&);
>-    template<typename Functor> void forEachSubspace(const Functor&);
>+    template<typename Functor> inline void forEachAllocator(const Functor&);
>+    template<typename Functor> inline void forEachSubspace(const Functor&);
>     
>     void addActiveWeakSet(WeakSet*);
> 
>@@ -205,7 +215,6 @@ private:
>     bool m_isIterating;
>     bool m_isMarking { false };
>     MarkedBlockSet m_blocks;
>-    Vector<MarkedBlock::Handle*> m_blocksWithNewObjects;
>     Vector<LargeAllocation*> m_largeAllocations;
>     unsigned m_largeAllocationsNurseryOffset { 0 };
>     unsigned m_largeAllocationsOffsetForThisCollection { 0 };
>@@ -215,40 +224,11 @@ private:
>     unsigned m_largeAllocationsForThisCollectionSize { 0 };
>     SentinelLinkedList<WeakSet, BasicRawSentinelNode<WeakSet>> m_activeWeakSets;
>     SentinelLinkedList<WeakSet, BasicRawSentinelNode<WeakSet>> m_newActiveWeakSets;
>+    
>+    MarkedAllocator* m_firstAllocator { nullptr };
>+    MarkedAllocator* m_allocatorForEmptyAllocation { nullptr };
> };
> 
>-template<typename Functor> inline void MarkedSpace::forEachLiveCell(HeapIterationScope&, const Functor& functor)
>-{
>-    ASSERT(isIterating());
>-    BlockIterator end = m_blocks.set().end();
>-    for (BlockIterator it = m_blocks.set().begin(); it != end; ++it) {
>-        if ((*it)->handle().forEachLiveCell(functor) == IterationStatus::Done)
>-            return;
>-    }
>-    for (LargeAllocation* allocation : m_largeAllocations) {
>-        if (allocation->isLive()) {
>-            if (functor(allocation->cell(), allocation->attributes().cellKind) == IterationStatus::Done)
>-                return;
>-        }
>-    }
>-}
>-
>-template<typename Functor> inline void MarkedSpace::forEachDeadCell(HeapIterationScope&, const Functor& functor)
>-{
>-    ASSERT(isIterating());
>-    BlockIterator end = m_blocks.set().end();
>-    for (BlockIterator it = m_blocks.set().begin(); it != end; ++it) {
>-        if ((*it)->handle().forEachDeadCell(functor) == IterationStatus::Done)
>-            return;
>-    }
>-    for (LargeAllocation* allocation : m_largeAllocations) {
>-        if (!allocation->isLive()) {
>-            if (functor(allocation->cell(), allocation->attributes().cellKind) == IterationStatus::Done)
>-                return;
>-        }
>-    }
>-}
>-
> inline MarkedAllocator* MarkedSpace::allocatorFor(Subspace& space, size_t bytes)
> {
>     ASSERT(bytes);
>Index: Source/JavaScriptCore/heap/MarkedSpaceInlines.h
>===================================================================
>--- Source/JavaScriptCore/heap/MarkedSpaceInlines.h	(nonexistent)
>+++ Source/JavaScriptCore/heap/MarkedSpaceInlines.h	(working copy)
>@@ -0,0 +1,66 @@
>+/*
>+ * Copyright (C) 2016 Apple Inc. All rights reserved.
>+ *
>+ * Redistribution and use in source and binary forms, with or without
>+ * modification, are permitted provided that the following conditions
>+ * are met:
>+ * 1. Redistributions of source code must retain the above copyright
>+ *    notice, this list of conditions and the following disclaimer.
>+ * 2. Redistributions in binary form must reproduce the above copyright
>+ *    notice, this list of conditions and the following disclaimer in the
>+ *    documentation and/or other materials provided with the distribution.
>+ *
>+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
>+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
>+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
>+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
>+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
>+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
>+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
>+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
>+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
>+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
>+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
>+ */
>+
>+#pragma once
>+
>+#include "MarkedBlockInlines.h"
>+#include "MarkedSpace.h"
>+
>+namespace JSC {
>+
>+template<typename Functor> inline void MarkedSpace::forEachLiveCell(HeapIterationScope&, const Functor& functor)
>+{
>+    ASSERT(isIterating());
>+    BlockIterator end = m_blocks.set().end();
>+    for (BlockIterator it = m_blocks.set().begin(); it != end; ++it) {
>+        if ((*it)->handle().forEachLiveCell(functor) == IterationStatus::Done)
>+            return;
>+    }
>+    for (LargeAllocation* allocation : m_largeAllocations) {
>+        if (allocation->isLive()) {
>+            if (functor(allocation->cell(), allocation->attributes().cellKind) == IterationStatus::Done)
>+                return;
>+        }
>+    }
>+}
>+
>+template<typename Functor> inline void MarkedSpace::forEachDeadCell(HeapIterationScope&, const Functor& functor)
>+{
>+    ASSERT(isIterating());
>+    BlockIterator end = m_blocks.set().end();
>+    for (BlockIterator it = m_blocks.set().begin(); it != end; ++it) {
>+        if ((*it)->handle().forEachDeadCell(functor) == IterationStatus::Done)
>+            return;
>+    }
>+    for (LargeAllocation* allocation : m_largeAllocations) {
>+        if (!allocation->isLive()) {
>+            if (functor(allocation->cell(), allocation->attributes().cellKind) == IterationStatus::Done)
>+                return;
>+        }
>+    }
>+}
>+
>+} // namespace JSC
>+
>Index: Source/JavaScriptCore/heap/SlotVisitor.cpp
>===================================================================
>--- Source/JavaScriptCore/heap/SlotVisitor.cpp	(revision 205859)
>+++ Source/JavaScriptCore/heap/SlotVisitor.cpp	(working copy)
>@@ -198,7 +198,7 @@ void SlotVisitor::setMarkedAndAppendToMa
> template<typename ContainerType>
> ALWAYS_INLINE void SlotVisitor::setMarkedAndAppendToMarkStack(ContainerType& container, JSCell* cell)
> {
>-    container.flipIfNecessaryDuringMarking(m_version);
>+    container.aboutToMark(m_version);
>     
>     if (container.testAndSetMarked(cell))
>         return;
>@@ -246,10 +246,8 @@ void SlotVisitor::markAuxiliary(const vo
>     
>     ASSERT(cell->heap() == heap());
>     
>-    if (Heap::testAndSetMarked(m_version, cell)) {
>-        RELEASE_ASSERT(Heap::isMarkedConcurrently(cell));
>+    if (Heap::testAndSetMarked(m_version, cell))
>         return;
>-    }
>     
>     noteLiveAuxiliaryCell(cell);
> }
>Index: Source/JavaScriptCore/heap/Weak.h
>===================================================================
>--- Source/JavaScriptCore/heap/Weak.h	(revision 205859)
>+++ Source/JavaScriptCore/heap/Weak.h	(working copy)
>@@ -1,5 +1,5 @@
> /*
>- * Copyright (C) 2009, 2012, 2013 Apple Inc. All rights reserved.
>+ * Copyright (C) 2009, 2012, 2013, 2016 Apple Inc. All rights reserved.
>  *
>  * Redistribution and use in source and binary forms, with or without
>  * modification, are permitted provided that the following conditions
>@@ -28,7 +28,9 @@
> 
> #include "JSExportMacros.h"
> #include <cstddef>
>+#include <wtf/HashTraits.h>
> #include <wtf/Noncopyable.h>
>+#include <wtf/VectorTraits.h>
> 
> namespace JSC {
> 
>@@ -93,4 +95,23 @@ private:
> 
> } // namespace JSC
> 
>+namespace WTF {
>+
>+template<typename T> struct VectorTraits<JSC::Weak<T>> : SimpleClassVectorTraits {
>+    static const bool canCompareWithMemcmp = false;
>+};
>+
>+template<typename T> struct HashTraits<JSC::Weak<T>> : SimpleClassHashTraits<JSC::Weak<T>> {
>+    typedef JSC::Weak<T> StorageType;
>+
>+    typedef std::nullptr_t EmptyValueType;
>+    static EmptyValueType emptyValue() { return nullptr; }
>+
>+    typedef T* PeekType;
>+    static PeekType peek(const StorageType& value) { return value.get(); }
>+    static PeekType peek(EmptyValueType) { return PeekType(); }
>+};
>+
>+} // namespace WTF
>+
> #endif // Weak_h
>Index: Source/JavaScriptCore/heap/WeakBlock.cpp
>===================================================================
>--- Source/JavaScriptCore/heap/WeakBlock.cpp	(revision 205859)
>+++ Source/JavaScriptCore/heap/WeakBlock.cpp	(working copy)
>@@ -113,7 +113,7 @@ void WeakBlock::specializedVisit(Contain
>             continue;
> 
>         const JSValue& jsValue = weakImpl->jsValue();
>-        if (container.isMarkedOrNewlyAllocatedDuringWeakVisiting(version, jsValue.asCell()))
>+        if (container.isMarkedOrNewlyAllocatedConcurrently(version, jsValue.asCell()))
>             continue;
>         
>         if (!weakHandleOwner->isReachableFromOpaqueRoots(Handle<Unknown>::wrapSlot(&const_cast<JSValue&>(jsValue)), weakImpl->context(), visitor))
>@@ -147,17 +147,14 @@ void WeakBlock::reap()
>     // If this WeakBlock doesn't belong to a CellContainer, we won't even be here.
>     ASSERT(m_container);
>     
>-    m_container.flipIfNecessary();
>-
>-    // We only reap after marking.
>-    ASSERT(m_container.isMarked());
>+    HeapVersion version = m_container.heap()->objectSpace().version();
> 
>     for (size_t i = 0; i < weakImplCount(); ++i) {
>         WeakImpl* weakImpl = &weakImpls()[i];
>         if (weakImpl->state() > WeakImpl::Dead)
>             continue;
> 
>-        if (m_container.isMarkedOrNewlyAllocated(weakImpl->jsValue().asCell())) {
>+        if (m_container.isMarkedOrNewlyAllocated(version, weakImpl->jsValue().asCell())) {
>             ASSERT(weakImpl->state() == WeakImpl::Live);
>             continue;
>         }
>Index: Source/JavaScriptCore/heap/WeakInlines.h
>===================================================================
>--- Source/JavaScriptCore/heap/WeakInlines.h	(revision 205859)
>+++ Source/JavaScriptCore/heap/WeakInlines.h	(working copy)
>@@ -1,5 +1,5 @@
> /*
>- * Copyright (C) 2009, 2012, 2013 Apple Inc. All rights reserved.
>+ * Copyright (C) 2009, 2012, 2013, 2016 Apple Inc. All rights reserved.
>  *
>  * Redistribution and use in source and binary forms, with or without
>  * modification, are permitted provided that the following conditions
>@@ -29,7 +29,6 @@
> #include "JSCell.h"
> #include "WeakSetInlines.h"
> #include <wtf/Assertions.h>
>-#include <wtf/HashTraits.h>
> 
> namespace JSC {
> 
>@@ -149,23 +148,4 @@ template<typename T> inline void weakCle
> 
> } // namespace JSC
> 
>-namespace WTF {
>-
>-template<typename T> struct VectorTraits<JSC::Weak<T>> : SimpleClassVectorTraits {
>-    static const bool canCompareWithMemcmp = false;
>-};
>-
>-template<typename T> struct HashTraits<JSC::Weak<T>> : SimpleClassHashTraits<JSC::Weak<T>> {
>-    typedef JSC::Weak<T> StorageType;
>-
>-    typedef std::nullptr_t EmptyValueType;
>-    static EmptyValueType emptyValue() { return nullptr; }
>-
>-    typedef T* PeekType;
>-    static PeekType peek(const StorageType& value) { return value.get(); }
>-    static PeekType peek(EmptyValueType) { return PeekType(); }
>-};
>-
>-} // namespace WTF
>-
> #endif // WeakInlines_h
>Index: Source/JavaScriptCore/jit/JITThunks.h
>===================================================================
>--- Source/JavaScriptCore/jit/JITThunks.h	(revision 205859)
>+++ Source/JavaScriptCore/jit/JITThunks.h	(working copy)
>@@ -34,10 +34,10 @@
> #include "ThunkGenerator.h"
> #include "Weak.h"
> #include "WeakHandleOwner.h"
>-#include "WeakInlines.h"
> #include <tuple>
> #include <wtf/HashMap.h>
> #include <wtf/ThreadingPrimitives.h>
>+#include <wtf/text/StringHash.h>
> 
> namespace JSC {
> 
>Index: Source/JavaScriptCore/runtime/JSGlobalObject.cpp
>===================================================================
>--- Source/JavaScriptCore/runtime/JSGlobalObject.cpp	(revision 205859)
>+++ Source/JavaScriptCore/runtime/JSGlobalObject.cpp	(working copy)
>@@ -116,6 +116,7 @@
> #include "MapConstructor.h"
> #include "MapIteratorPrototype.h"
> #include "MapPrototype.h"
>+#include "MarkedSpaceInlines.h"
> #include "MathObject.h"
> #include "Microtask.h"
> #include "ModuleLoaderPrototype.h"
>Index: Source/JavaScriptCore/runtime/PrototypeMap.h
>===================================================================
>--- Source/JavaScriptCore/runtime/PrototypeMap.h	(revision 205859)
>+++ Source/JavaScriptCore/runtime/PrototypeMap.h	(working copy)
>@@ -26,6 +26,8 @@
> #ifndef PrototypeMap_h
> #define PrototypeMap_h
> 
>+#include "IndexingType.h"
>+#include "JSTypeInfo.h"
> #include "WeakGCMap.h"
> #include <wtf/TriState.h>
> 
>Index: Source/JavaScriptCore/runtime/SamplingProfiler.cpp
>===================================================================
>--- Source/JavaScriptCore/runtime/SamplingProfiler.cpp	(revision 205859)
>+++ Source/JavaScriptCore/runtime/SamplingProfiler.cpp	(working copy)
>@@ -42,6 +42,7 @@
> #include "LLIntPCRanges.h"
> #include "MarkedBlock.h"
> #include "MarkedBlockSet.h"
>+#include "MarkedSpaceInlines.h"
> #include "PCToCodeOriginMap.h"
> #include "SlotVisitor.h"
> #include "SlotVisitorInlines.h"
>Index: Source/JavaScriptCore/runtime/WeakGCMap.h
>===================================================================
>--- Source/JavaScriptCore/runtime/WeakGCMap.h	(revision 205859)
>+++ Source/JavaScriptCore/runtime/WeakGCMap.h	(working copy)
>@@ -27,7 +27,6 @@
> #define WeakGCMap_h
> 
> #include "Weak.h"
>-#include "WeakInlines.h"
> #include <wtf/HashMap.h>
> 
> namespace JSC {
>Index: Source/JavaScriptCore/tools/JSDollarVMPrototype.cpp
>===================================================================
>--- Source/JavaScriptCore/tools/JSDollarVMPrototype.cpp	(revision 205859)
>+++ Source/JavaScriptCore/tools/JSDollarVMPrototype.cpp	(working copy)
>@@ -31,6 +31,7 @@
> #include "HeapIterationScope.h"
> #include "JSCInlines.h"
> #include "JSFunction.h"
>+#include "MarkedSpaceInlines.h"
> #include "StackVisitor.h"
> #include <wtf/DataLog.h>
> #include <wtf/StringPrintStream.h>
>Index: Source/WTF/WTF.xcodeproj/project.pbxproj
>===================================================================
>--- Source/WTF/WTF.xcodeproj/project.pbxproj	(revision 205859)
>+++ Source/WTF/WTF.xcodeproj/project.pbxproj	(working copy)
>@@ -28,6 +28,7 @@
> 		0F4570431BE5B58F0062A629 /* Dominators.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F4570421BE5B58F0062A629 /* Dominators.h */; };
> 		0F4570451BE834410062A629 /* BubbleSort.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F4570441BE834410062A629 /* BubbleSort.h */; };
> 		0F725CAC1C50461600AD943A /* RangeSet.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F725CAB1C50461600AD943A /* RangeSet.h */; };
>+		0F7C5FB61D885CF20044F5E2 /* FastBitVector.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 0F7C5FB51D885CF20044F5E2 /* FastBitVector.cpp */; };
> 		0F824A681B7443A0002E345D /* ParkingLot.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 0F824A641B7443A0002E345D /* ParkingLot.cpp */; };
> 		0F824A691B7443A0002E345D /* ParkingLot.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F824A651B7443A0002E345D /* ParkingLot.h */; };
> 		0F87105A16643F190090B0AD /* RawPointer.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F87105916643F190090B0AD /* RawPointer.h */; };
>@@ -366,6 +367,7 @@
> 		0F4570421BE5B58F0062A629 /* Dominators.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = Dominators.h; sourceTree = "<group>"; };
> 		0F4570441BE834410062A629 /* BubbleSort.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = BubbleSort.h; sourceTree = "<group>"; };
> 		0F725CAB1C50461600AD943A /* RangeSet.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = RangeSet.h; sourceTree = "<group>"; };
>+		0F7C5FB51D885CF20044F5E2 /* FastBitVector.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = FastBitVector.cpp; sourceTree = "<group>"; };
> 		0F824A641B7443A0002E345D /* ParkingLot.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = ParkingLot.cpp; sourceTree = "<group>"; };
> 		0F824A651B7443A0002E345D /* ParkingLot.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ParkingLot.h; sourceTree = "<group>"; };
> 		0F87105916643F190090B0AD /* RawPointer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = RawPointer.h; sourceTree = "<group>"; };
>@@ -880,6 +882,7 @@
> 				A8A47298151A825A004123FF /* dtoa.h */,
> 				1AEA88E11D6BBCF400E5AD64 /* EnumTraits.h */,
> 				A8A4729F151A825A004123FF /* ExportMacros.h */,
>+				0F7C5FB51D885CF20044F5E2 /* FastBitVector.cpp */,
> 				0FD81AC4154FB22E00983E72 /* FastBitVector.h */,
> 				A8A472A1151A825A004123FF /* FastMalloc.cpp */,
> 				A8A472A2151A825A004123FF /* FastMalloc.h */,
>@@ -1594,6 +1597,7 @@
> 				CD5497AC15857D0300B5BC30 /* MediaTime.cpp in Sources */,
> 				A8A473EC151A825B004123FF /* MetaAllocator.cpp in Sources */,
> 				A8A473F4151A825B004123FF /* NumberOfCores.cpp in Sources */,
>+				0F7C5FB61D885CF20044F5E2 /* FastBitVector.cpp in Sources */,
> 				A8A473F7151A825B004123FF /* OSAllocatorPosix.cpp in Sources */,
> 				A8A473F9151A825B004123FF /* OSRandomSource.cpp in Sources */,
> 				A8A47402151A825B004123FF /* PageBlock.cpp in Sources */,
>Index: Source/WTF/wtf/CMakeLists.txt
>===================================================================
>--- Source/WTF/wtf/CMakeLists.txt	(revision 205859)
>+++ Source/WTF/wtf/CMakeLists.txt	(working copy)
>@@ -181,6 +181,7 @@ set(WTF_SOURCES
>     DataLog.cpp
>     DateMath.cpp
>     DecimalNumber.cpp
>+    FastBitVector.cpp
>     FastMalloc.cpp
>     FilePrintStream.cpp
>     FunctionDispatcher.cpp
>Index: Source/WTF/wtf/FastBitVector.cpp
>===================================================================
>--- Source/WTF/wtf/FastBitVector.cpp	(nonexistent)
>+++ Source/WTF/wtf/FastBitVector.cpp	(working copy)
>@@ -0,0 +1,57 @@
>+/*
>+ * Copyright (C) 2016 Apple Inc. All rights reserved.
>+ *
>+ * Redistribution and use in source and binary forms, with or without
>+ * modification, are permitted provided that the following conditions
>+ * are met:
>+ * 1. Redistributions of source code must retain the above copyright
>+ *    notice, this list of conditions and the following disclaimer.
>+ * 2. Redistributions in binary form must reproduce the above copyright
>+ *    notice, this list of conditions and the following disclaimer in the
>+ *    documentation and/or other materials provided with the distribution.
>+ *
>+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
>+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
>+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
>+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
>+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
>+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
>+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
>+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
>+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
>+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
>+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
>+ */
>+
>+#include "config.h"
>+#include "FastBitVector.h"
>+
>+namespace WTF {
>+
>+void FastBitVectorWordOwner::setEqualsSlow(const FastBitVectorWordOwner& other)
>+{
>+    uint32_t* newArray = static_cast<uint32_t*>(
>+        fastCalloc(other.arrayLength(), sizeof(uint32_t)));
>+    memcpy(newArray, other.m_words, other.arrayLength() * sizeof(uint32_t));
>+    if (m_words)
>+        fastFree(m_words);
>+    m_words = newArray;
>+    m_numBits = other.m_numBits;
>+}
>+
>+void FastBitVectorWordOwner::resizeSlow(size_t numBits)
>+{
>+    size_t newLength = fastBitVectorArrayLength(numBits);
>+    
>+    // Use fastCalloc instead of fastRealloc because we expect the common
>+    // use case for this method to be initializing the size of the bitvector.
>+    
>+    uint32_t* newArray = static_cast<uint32_t*>(fastCalloc(newLength, sizeof(uint32_t)));
>+    memcpy(newArray, m_words, arrayLength() * sizeof(uint32_t));
>+    if (m_words)
>+        fastFree(m_words);
>+    m_words = newArray;
>+}
>+
>+} // namespace WTF
>+
>Index: Source/WTF/wtf/FastBitVector.h
>===================================================================
>--- Source/WTF/wtf/FastBitVector.h	(revision 205859)
>+++ Source/WTF/wtf/FastBitVector.h	(working copy)
>@@ -26,6 +26,7 @@
> #pragma once
> 
> #include <string.h>
>+#include <wtf/Atomics.h>
> #include <wtf/FastMalloc.h>
> #include <wtf/PrintStream.h>
> #include <wtf/StdLibExtras.h>
>@@ -91,17 +92,12 @@ public:
>     
>     FastBitVectorWordOwner& operator=(const FastBitVectorWordOwner& other)
>     {
>-        size_t length = other.arrayLength();
>-        if (length == arrayLength()) {
>-            memcpy(m_words, other.m_words, length * sizeof(uint32_t));
>-            return *this;
>+        if (arrayLength() != other.arrayLength())
>+            setEqualsSlow(other);
>+        else {
>+            memcpy(m_words, other.m_words, arrayLength() * sizeof(uint32_t));
>+            m_numBits = other.m_numBits;
>         }
>-        uint32_t* newArray = static_cast<uint32_t*>(fastCalloc(length, sizeof(uint32_t)));
>-        memcpy(newArray, other.m_words, length * sizeof(uint32_t));
>-        if (m_words)
>-            fastFree(m_words);
>-        m_words = newArray;
>-        m_numBits = other.m_numBits;
>         return *this;
>     }
>     
>@@ -140,18 +136,8 @@ public:
>     
>     void resize(size_t numBits)
>     {
>-        if (numBits == m_numBits)
>-            return;
>-        
>-        // Use fastCalloc instead of fastRealloc because we expect the common
>-        // use case for this method to be initializing the size of the bitvector.
>-        
>-        size_t newLength = fastBitVectorArrayLength(numBits);
>-        uint32_t* newArray = static_cast<uint32_t*>(fastCalloc(newLength, sizeof(uint32_t)));
>-        memcpy(newArray, m_words, arrayLength() * sizeof(uint32_t));
>-        if (m_words)
>-            fastFree(m_words);
>-        m_words = newArray;
>+        if (arrayLength() != fastBitVectorArrayLength(numBits))
>+            resizeSlow(numBits);
>         m_numBits = numBits;
>     }
>     
>@@ -171,6 +157,9 @@ public:
>     uint32_t* words() { return m_words; }
> 
> private:
>+    WTF_EXPORT_PRIVATE void setEqualsSlow(const FastBitVectorWordOwner& other);
>+    WTF_EXPORT_PRIVATE void resizeSlow(size_t numBits);
>+    
>     uint32_t* m_words { nullptr };
>     size_t m_numBits { 0 };
> };
>@@ -320,6 +309,15 @@ public:
>         return result;
>     }
>     
>+    bool isEmpty() const
>+    {
>+        for (size_t index = arrayLength(); index--;) {
>+            if (m_words.word(index))
>+                return false;
>+        }
>+        return true;
>+    }
>+    
>     template<typename OtherWords>
>     FastBitVectorImpl<FastBitVectorAndWords<typename Words::ViewType, typename OtherWords::ViewType>> operator&(const FastBitVectorImpl<OtherWords>& other) const
>     {
>@@ -471,6 +469,7 @@ public:
>         m_words.clearAll();
>     }
> 
>+    // Returns true if the contents of this bitvector changed.
>     template<typename OtherWords>
>     bool setAndCheck(const FastBitVectorImpl<OtherWords>& other)
>     {
>@@ -550,6 +549,28 @@ public:
>     {
>         return at(index);
>     }
>+    
>+    // Returns true if the contents changed.
>+    ALWAYS_INLINE bool atomicSetAndCheck(size_t index, bool value)
>+    {
>+        uint32_t* pointer = &m_words.word(index >> 5);
>+        uint32_t mask = 1 << (index & 31);
>+        for (;;) {
>+            uint32_t oldValue = *pointer;
>+            uint32_t newValue;
>+            if (value) {
>+                if (oldValue & mask)
>+                    return false;
>+                newValue = oldValue | mask;
>+            } else {
>+                if (!(oldValue & mask))
>+                    return false;
>+                newValue = oldValue & ~mask;
>+            }
>+            if (weakCompareAndSwap(pointer, oldValue, newValue))
>+                return true;
>+        }
>+    }
> };
> 
> } // namespace WTF

Actions: View | Formatted Diff | Diff

Attachments on bug 161581: 288519 | 288574 | 288578 | 288595 | 288602 | 288690 | 288699 | 288702 | 288714 | 288716 | 288750 | 288757 | 288855 | 288866 | 288976 | 289068 | 289080 | 289175 | 289177 | 289178 | 289179 | 289180 | 289181 | 289182 | 289211 | 289212 | 289232 | 289238 | 289239 | 289291 | 289306 | 289307 | 289364 | 289367