WebKit Bugzilla
New
Browse
Search+
Log In
×
Sign in with GitHub
or
Remember my login
Create Account
·
Forgot Password
Forgotten password account recovery
[patch]
fixed some more races
blah.patch (text/plain), 60.82 KB, created by
Filip Pizlo
on 2016-11-06 19:17:17 PST
(
hide
)
Description:
fixed some more races
Filename:
MIME Type:
Creator:
Filip Pizlo
Created:
2016-11-06 19:17:17 PST
Size:
60.82 KB
patch
obsolete
>Index: Source/JavaScriptCore/API/JSVirtualMachine.mm >=================================================================== >--- Source/JavaScriptCore/API/JSVirtualMachine.mm (revision 208415) >+++ Source/JavaScriptCore/API/JSVirtualMachine.mm (working copy) >@@ -1,5 +1,5 @@ > /* >- * Copyright (C) 2013, 2015 Apple Inc. All rights reserved. >+ * Copyright (C) 2013-2016 Apple Inc. All rights reserved. > * > * Redistribution and use in source and binary forms, with or without > * modification, are permitted provided that the following conditions >@@ -151,7 +151,7 @@ static id getInternalObjcObject(id objec > - (bool)isOldExternalObject:(id)object > { > JSC::VM* vm = toJS(m_group); >- return vm->heap.slotVisitor().containsOpaqueRoot(object); >+ return vm->heap.collectorSlotVisitor().containsOpaqueRoot(object); > } > > - (void)addExternalRememberedObject:(id)object >Index: Source/JavaScriptCore/JavaScriptCore.xcodeproj/project.pbxproj >=================================================================== >--- Source/JavaScriptCore/JavaScriptCore.xcodeproj/project.pbxproj (revision 208415) >+++ Source/JavaScriptCore/JavaScriptCore.xcodeproj/project.pbxproj (working copy) >@@ -613,6 +613,7 @@ > 0FA7A8EB18B413C80052371D /* Reg.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 0FA7A8E918B413C80052371D /* Reg.cpp */; }; > 0FA7A8EC18B413C80052371D /* Reg.h in Headers */ = {isa = PBXBuildFile; fileRef = 0FA7A8EA18B413C80052371D /* Reg.h */; settings = {ATTRIBUTES = (Private, ); }; }; > 0FA7A8EE18CE4FD80052371D /* ScratchRegisterAllocator.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 0FA7A8ED18CE4FD80052371D /* ScratchRegisterAllocator.cpp */; }; >+ 0FADDD1D1DD01EDE008275E8 /* ResumeTheWorldScope.h in Headers */ = {isa = PBXBuildFile; fileRef = 0FADDD1C1DD01EDC008275E8 /* ResumeTheWorldScope.h */; }; > 0FADE6731D4D23BE00768457 /* HeapUtil.h in Headers */ = {isa = PBXBuildFile; fileRef = 0FADE6721D4D23BC00768457 /* HeapUtil.h */; }; > 0FAF7EFD165BA91B000C8455 /* JITDisassembler.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 0FAF7EFA165BA919000C8455 /* JITDisassembler.cpp */; }; > 0FAF7EFE165BA91F000C8455 /* JITDisassembler.h in Headers */ = {isa = PBXBuildFile; fileRef = 0FAF7EFB165BA919000C8455 /* JITDisassembler.h */; settings = {ATTRIBUTES = (Private, ); }; }; >@@ -3003,6 +3004,7 @@ > 0FA7A8E918B413C80052371D /* Reg.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = Reg.cpp; sourceTree = "<group>"; }; > 0FA7A8EA18B413C80052371D /* Reg.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = Reg.h; sourceTree = "<group>"; }; > 0FA7A8ED18CE4FD80052371D /* ScratchRegisterAllocator.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = ScratchRegisterAllocator.cpp; sourceTree = "<group>"; }; >+ 0FADDD1C1DD01EDC008275E8 /* ResumeTheWorldScope.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ResumeTheWorldScope.h; sourceTree = "<group>"; }; > 0FADE6721D4D23BC00768457 /* HeapUtil.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = HeapUtil.h; sourceTree = "<group>"; }; > 0FAF7EFA165BA919000C8455 /* JITDisassembler.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = JITDisassembler.cpp; sourceTree = "<group>"; }; > 0FAF7EFB165BA919000C8455 /* JITDisassembler.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = JITDisassembler.h; sourceTree = "<group>"; }; >@@ -5684,6 +5686,7 @@ > 0FA762031DB9242300B7A2FD /* MutatorState.h */, > ADDB1F6218D77DB7009B58A8 /* OpaqueRootSet.h */, > 0F7CF94E1DBEEE860098CC12 /* ReleaseHeapAccessScope.h */, >+ 0FADDD1C1DD01EDC008275E8 /* ResumeTheWorldScope.h */, > C225494215F7DBAA0065E898 /* SlotVisitor.cpp */, > 14BA78F013AAB88F005B7C2C /* SlotVisitor.h */, > 0FCB408515C0A3C30048932B /* SlotVisitorInlines.h */, >@@ -7915,6 +7918,7 @@ > 0FFB921A16D02EC50055A5DB /* DFGBasicBlockInlines.h in Headers */, > A7D89CF417A0B8CC00773AD8 /* DFGBlockInsertionSet.h in Headers */, > 0F2BBD961C5FF3F50023EF23 /* B3SparseCollection.h in Headers */, >+ 0FADDD1D1DD01EDE008275E8 /* ResumeTheWorldScope.h in Headers */, > DC3D2B0A1D34316200BA918C /* HeapCell.h in Headers */, > 0FC3CCFC19ADA410006AC72A /* DFGBlockMap.h in Headers */, > 0FC3CCFD19ADA410006AC72A /* DFGBlockMapInlines.h in Headers */, >@@ -10076,7 +10080,6 @@ > 0FF4275715914A20004CB9FF /* LinkBuffer.cpp in Sources */, > A7E2EA6C0FB460CF00601F06 /* LiteralParser.cpp in Sources */, > A5A1A0951D8CB341004C2EB8 /* DebuggerParseData.cpp in Sources */, >- 53FF7F9D1DC00DB100A26CCC /* WasmFormat.cpp in Sources */, > FE3913541B794F6E00EDAF71 /* LiveObjectList.cpp in Sources */, > FE20CE9D15F04A9500DF3430 /* LLIntCLoop.cpp in Sources */, > 0F4680D214BBD16500BFE272 /* LLIntData.cpp in Sources */, >Index: Source/JavaScriptCore/dfg/DFGPlan.cpp >=================================================================== >--- Source/JavaScriptCore/dfg/DFGPlan.cpp (revision 208415) >+++ Source/JavaScriptCore/dfg/DFGPlan.cpp (working copy) >@@ -610,16 +610,34 @@ CompilationKey Plan::key() > return CompilationKey(codeBlock->alternative(), mode); > } > >-void Plan::rememberCodeBlocks() >+void Plan::markCodeBlocks(SlotVisitor& slotVisitor) > { >+ if (!isKnownToBeLiveDuringGC()) >+ return; >+ >+ // Compilation writes lots of values to a CodeBlock without performing >+ // an explicit barrier. So, we need to be pessimistic and assume that >+ // all our CodeBlocks must be visited during GC. >+ >+ slotVisitor.appendUnbarrieredReadOnlyPointer(codeBlock); >+ slotVisitor.appendUnbarrieredReadOnlyPointer(codeBlock->alternative()); >+ if (profiledDFGCodeBlock) >+ slotVisitor.appendUnbarrieredReadOnlyPointer(profiledDFGCodeBlock); >+} >+ >+void Plan::rememberCodeBlocks(VM& vm) >+{ >+ if (!isKnownToBeLiveDuringGC()) >+ return; >+ > // Compilation writes lots of values to a CodeBlock without performing > // an explicit barrier. So, we need to be pessimistic and assume that > // all our CodeBlocks must be visited during GC. > >- Heap::heap(codeBlock)->writeBarrier(codeBlock); >- Heap::heap(codeBlock)->writeBarrier(codeBlock->alternative()); >+ vm.heap.writeBarrier(codeBlock); >+ vm.heap.writeBarrier(codeBlock->alternative()); > if (profiledDFGCodeBlock) >- Heap::heap(profiledDFGCodeBlock)->writeBarrier(profiledDFGCodeBlock); >+ vm.heap.writeBarrier(profiledDFGCodeBlock); > } > > void Plan::checkLivenessAndVisitChildren(SlotVisitor& visitor) >Index: Source/JavaScriptCore/dfg/DFGPlan.h >=================================================================== >--- Source/JavaScriptCore/dfg/DFGPlan.h (revision 208415) >+++ Source/JavaScriptCore/dfg/DFGPlan.h (working copy) >@@ -69,7 +69,8 @@ struct Plan : public ThreadSafeRefCounte > > CompilationKey key(); > >- void rememberCodeBlocks(); >+ void markCodeBlocks(SlotVisitor&); >+ void rememberCodeBlocks(VM&); > void checkLivenessAndVisitChildren(SlotVisitor&); > bool isKnownToBeLiveDuringGC(); > void cancel(); >Index: Source/JavaScriptCore/dfg/DFGWorklist.cpp >=================================================================== >--- Source/JavaScriptCore/dfg/DFGWorklist.cpp (revision 208415) >+++ Source/JavaScriptCore/dfg/DFGWorklist.cpp (working copy) >@@ -26,8 +26,6 @@ > #include "config.h" > #include "DFGWorklist.h" > >-#if ENABLE(DFG_JIT) >- > #include "CodeBlock.h" > #include "DFGLongLivedState.h" > #include "DFGSafepoint.h" >@@ -38,6 +36,8 @@ > > namespace JSC { namespace DFG { > >+#if ENABLE(DFG_JIT) >+ > class Worklist::ThreadBody : public AutomaticThread { > public: > ThreadBody(const LockHolder& locker, Worklist& worklist, ThreadData& data, Box<Lock> lock, RefPtr<AutomaticThreadCondition> condition, int relativePriority) >@@ -349,6 +349,17 @@ void Worklist::completeAllPlansForVM(VM& > completeAllReadyPlansForVM(vm); > } > >+void Worklist::markCodeBlocks(VM& vm, SlotVisitor& slotVisitor) >+{ >+ LockHolder locker(*m_lock); >+ for (PlanMap::iterator iter = m_plans.begin(); iter != m_plans.end(); ++iter) { >+ Plan* plan = iter->value.get(); >+ if (plan->vm != &vm) >+ continue; >+ plan->markCodeBlocks(slotVisitor); >+ } >+} >+ > void Worklist::rememberCodeBlocks(VM& vm) > { > LockHolder locker(*m_lock); >@@ -356,7 +367,7 @@ void Worklist::rememberCodeBlocks(VM& vm > Plan* plan = iter->value.get(); > if (plan->vm != &vm) > continue; >- plan->rememberCodeBlocks(); >+ plan->rememberCodeBlocks(vm); > } > } > >@@ -553,6 +564,14 @@ void completeAllPlansForVM(VM& vm) > } > } > >+void markCodeBlocks(VM& vm, SlotVisitor& slotVisitor) >+{ >+ for (unsigned i = DFG::numberOfWorklists(); i--;) { >+ if (DFG::Worklist* worklist = DFG::existingWorklistForIndexOrNull(i)) >+ worklist->markCodeBlocks(vm, slotVisitor); >+ } >+} >+ > void rememberCodeBlocks(VM& vm) > { > for (unsigned i = DFG::numberOfWorklists(); i--;) { >@@ -561,7 +580,22 @@ void rememberCodeBlocks(VM& vm) > } > } > >-} } // namespace JSC::DFG >+#else // ENABLE(DFG_JIT) >+ >+void completeAllPlansForVM(VM&) >+{ >+} >+ >+void markCodeBlocks(VM&, SlotVisitor&) >+{ >+} >+ >+void rememberCodeBlocks(VM&) >+{ >+} > > #endif // ENABLE(DFG_JIT) > >+} } // namespace JSC::DFG >+ >+ >Index: Source/JavaScriptCore/dfg/DFGWorklist.h >=================================================================== >--- Source/JavaScriptCore/dfg/DFGWorklist.h (revision 208415) >+++ Source/JavaScriptCore/dfg/DFGWorklist.h (working copy) >@@ -25,8 +25,6 @@ > > #pragma once > >-#if ENABLE(DFG_JIT) >- > #include "DFGPlan.h" > #include "DFGThreadData.h" > #include <wtf/AutomaticThread.h> >@@ -42,6 +40,8 @@ class SlotVisitor; > > namespace DFG { > >+#if ENABLE(DFG_JIT) >+ > class Worklist : public RefCounted<Worklist> { > public: > enum State { NotKnown, Compiling, Compiled }; >@@ -57,6 +57,7 @@ public: > // worklist->completeAllReadyPlansForVM(vm); > void completeAllPlansForVM(VM&); > >+ void markCodeBlocks(VM&, SlotVisitor&); > void rememberCodeBlocks(VM&); > > void waitUntilAllPlansForVMAreReady(VM&); >@@ -163,9 +164,11 @@ inline Worklist& existingWorklistForInde > return *result; > } > >+#endif // ENABLE(DFG_JIT) >+ > void completeAllPlansForVM(VM&); >+void markCodeBlocks(VM&, SlotVisitor&); > void rememberCodeBlocks(VM&); > > } } // namespace JSC::DFG > >-#endif // ENABLE(DFG_JIT) >Index: Source/JavaScriptCore/heap/CellState.h >=================================================================== >--- Source/JavaScriptCore/heap/CellState.h (revision 208415) >+++ Source/JavaScriptCore/heap/CellState.h (working copy) >@@ -37,17 +37,11 @@ enum class CellState : uint8_t { > // SlotVisitors are done. > AnthraciteOrBlack = 0, > >- // The object is in eden. During GC, this means that the object has not been marked yet. >+ // The object is going to be is in eden. During GC, this means that the object has not been marked yet. > NewWhite = 1, > >- // The object is grey - i.e. it will be scanned - and this is the first time in this GC that we are >- // going to scan it. If this is an eden GC, this also means that the object is in eden. >- NewGrey = 2, >- >- // The object is grey - i.e. it will be scanned - but it either belongs to old gen (if this is eden >- // GC) or it is grey a second time in this current GC (because a concurrent store barrier requested >- // re-greying). >- OldGrey = 3 >+ // The object is grey - i.e. it will be scanned. >+ Grey = 2, > }; > > static const unsigned blackThreshold = 0; // x <= blackThreshold means x is AnthraciteOrBlack. >Index: Source/JavaScriptCore/heap/GCSegmentedArray.h >=================================================================== >--- Source/JavaScriptCore/heap/GCSegmentedArray.h (revision 208415) >+++ Source/JavaScriptCore/heap/GCSegmentedArray.h (working copy) >@@ -1,5 +1,5 @@ > /* >- * Copyright (C) 2014 Apple Inc. All rights reserved. >+ * Copyright (C) 2014-2016 Apple Inc. All rights reserved. > * > * Redistribution and use in source and binary forms, with or without > * modification, are permitted provided that the following conditions >@@ -26,6 +26,8 @@ > #pragma once > > #include <wtf/DoublyLinkedList.h> >+#include <wtf/ListDump.h> >+#include <wtf/PrintStream.h> > #include <wtf/Vector.h> > > namespace JSC { >@@ -162,3 +164,4 @@ private: > }; > > } // namespace JSC >+ >Index: Source/JavaScriptCore/heap/Heap.cpp >=================================================================== >--- Source/JavaScriptCore/heap/Heap.cpp (revision 208415) >+++ Source/JavaScriptCore/heap/Heap.cpp (working copy) >@@ -78,6 +78,35 @@ using namespace std; > > namespace JSC { > >+class Heap::ResumeTheWorldScope { >+public: >+ ResumeTheWorldScope(Heap& heap) >+ : m_heap(heap) >+ { >+ if (!Options::useConcurrentGC()) >+ return; >+ >+ if (Options::logGC()) >+ dataLog((MonotonicTime::now() - m_heap.m_stopTime).milliseconds(), " ms...]\n"); >+ >+ m_heap.resumeTheWorld(); >+ } >+ >+ ~ResumeTheWorldScope() >+ { >+ if (!Options::useConcurrentGC()) >+ return; >+ >+ if (Options::logGC()) >+ m_heap.stopTheWorld(); >+ >+ dataLog("[Increment: "); >+ } >+ >+private: >+ Heap& m_heap; >+}; >+ > namespace { > > static const size_t largeHeapSize = 32 * MB; // About 1.5X the average webpage. >@@ -244,7 +273,8 @@ Heap::Heap(VM* vm, HeapType heapType) > , m_extraMemorySize(0) > , m_deprecatedExtraMemorySize(0) > , m_machineThreads(this) >- , m_slotVisitor(*this) >+ , m_collectorSlotVisitor(std::make_unique<SlotVisitor>(*this)) >+ , m_mutatorMarkStack(std::make_unique<MarkStackArray>()) > , m_handleSet(vm) > , m_codeBlocks(std::make_unique<CodeBlockSet>()) > , m_jitStubRoutines(std::make_unique<JITStubRoutineSet>()) >@@ -266,6 +296,8 @@ Heap::Heap(VM* vm, HeapType heapType) > #if USE(FOUNDATION) > , m_delayedReleaseRecursionCount(0) > #endif >+ , m_sharedCollectorMarkStack(std::make_unique<MarkStackArray>()) >+ , m_sharedMutatorMarkStack(std::make_unique<MarkStackArray>()) > , m_helperClient(&heapHelperPool()) > , m_threadLock(Box<Lock>::create()) > , m_threadCondition(AutomaticThreadCondition::create()) >@@ -416,12 +448,12 @@ void Heap::addReference(JSCell* cell, Ar > > void Heap::harvestWeakReferences() > { >- m_slotVisitor.harvestWeakReferences(); >+ m_collectorSlotVisitor->harvestWeakReferences(); > } > > void Heap::finalizeUnconditionalFinalizers() > { >- m_slotVisitor.finalizeUnconditionalFinalizers(); >+ m_collectorSlotVisitor->finalizeUnconditionalFinalizers(); > } > > void Heap::willStartIterating() >@@ -439,24 +471,18 @@ void Heap::completeAllJITPlans() > #if ENABLE(JIT) > JITWorklist::instance()->completeAllForVM(*m_vm); > #endif // ENABLE(JIT) >-#if ENABLE(DFG_JIT) > DFG::completeAllPlansForVM(*m_vm); >-#endif > } > > void Heap::markRoots(double gcStartTime) > { > TimingScope markRootsTimingScope(*this, "Heap::markRoots"); > >- HeapRootVisitor heapRootVisitor(m_slotVisitor); >+ HeapRootVisitor heapRootVisitor(*m_collectorSlotVisitor); > > { > TimingScope preConvergenceTimingScope(*this, "Heap::markRoots before convergence"); > >-#if ENABLE(DFG_JIT) >- DFG::rememberCodeBlocks(*m_vm); >-#endif >- > #if ENABLE(SAMPLING_PROFILER) > if (SamplingProfiler* samplingProfiler = m_vm->samplingProfiler()) { > // Note that we need to own the lock from now until we're done >@@ -471,7 +497,8 @@ void Heap::markRoots(double gcStartTime) > > if (m_collectionScope == CollectionScope::Full) { > m_opaqueRoots.clear(); >- m_slotVisitor.clearMarkStack(); >+ m_collectorSlotVisitor->clearMarkStacks(); >+ m_mutatorMarkStack->clear(); > } > > beginMarking(); >@@ -506,30 +533,16 @@ void Heap::markRoots(double gcStartTime) > } > }); > >- m_slotVisitor.didStartMarking(); >+ m_collectorSlotVisitor->didStartMarking(); > } > > { > SuperSamplerScope superSamplerScope(false); >- TimingScope convergenceTimingScope(*this, "Heap::markRoots convergence"); >- ParallelModeEnabler enabler(m_slotVisitor); >+ TimingScope convergenceTimingScope(*this, "Heap::markRoots early convergence"); >+ ParallelModeEnabler enabler(*m_collectorSlotVisitor); >+ ResumeTheWorldScope resumeTheWorldScope(*this); > >- m_slotVisitor.donateAndDrain(); >- >- { >- TimingScope preConvergenceTimingScope(*this, "Heap::markRoots conservative scan"); >- ConservativeRoots conservativeRoots(*this); >- SuperSamplerScope superSamplerScope(false); >- gatherStackRoots(conservativeRoots); >- gatherJSStackRoots(conservativeRoots); >- gatherScratchBufferRoots(conservativeRoots); >- visitConservativeRoots(conservativeRoots); >- >- // We want to do this to conservatively ensure that we rescan any code blocks that are >- // running right now. However, we need to be sure to do it *after* we mark the code block >- // so that we know for sure if it really needs a barrier. >- m_codeBlocks->writeBarrierCurrentlyExecuting(this); >- } >+ m_collectorSlotVisitor->donateAndDrain(); > > visitExternalRememberedSet(); > visitSmallStrings(); >@@ -541,14 +554,12 @@ void Heap::markRoots(double gcStartTime) > visitSamplingProfiler(); > visitShadowChicken(); > traceCodeBlocksAndJITStubRoutines(); >- m_slotVisitor.drainFromShared(SlotVisitor::MasterDrain); >+ m_collectorSlotVisitor->drainFromShared(SlotVisitor::MasterDrain); > } > >- TimingScope postConvergenceTimingScope(*this, "Heap::markRoots after convergence"); >+ TimingScope postConvergenceTimingScope(*this, "Heap::markRoots late convergence"); > >- // Weak references must be marked last because their liveness depends on >- // the liveness of the rest of the object graph. >- visitWeakHandles(heapRootVisitor); >+ markToFixpoint(heapRootVisitor); > > { > std::lock_guard<Lock> lock(m_markingMutex); >@@ -562,7 +573,6 @@ void Heap::markRoots(double gcStartTime) > > void Heap::gatherStackRoots(ConservativeRoots& roots) > { >- m_jitStubRoutines->clearMarks(); > m_machineThreads.gatherConservativeRoots(roots, *m_jitStubRoutines, *m_codeBlocks); > } > >@@ -589,6 +599,7 @@ void Heap::beginMarking() > TimingScope timingScope(*this, "Heap::beginMarking"); > if (m_collectionScope == CollectionScope::Full) > m_codeBlocks->clearMarksForFullCollection(); >+ m_jitStubRoutines->clearMarks(); > > { > TimingScope clearMarksTimingScope(*this, "m_objectSpace.beginMarking"); >@@ -599,7 +610,7 @@ void Heap::beginMarking() > void Heap::visitExternalRememberedSet() > { > #if JSC_OBJC_API_ENABLED >- scanExternalRememberedSet(*m_vm, m_slotVisitor); >+ scanExternalRememberedSet(*m_vm, *m_collectorSlotVisitor); > #endif > } > >@@ -608,30 +619,28 @@ void Heap::visitSmallStrings() > if (!m_vm->smallStrings.needsToBeVisited(*m_collectionScope)) > return; > >- m_vm->smallStrings.visitStrongReferences(m_slotVisitor); >+ m_vm->smallStrings.visitStrongReferences(*m_collectorSlotVisitor); > if (Options::logGC() == GCLogging::Verbose) >- dataLog("Small strings:\n", m_slotVisitor); >- m_slotVisitor.donateAndDrain(); >+ dataLog("Small strings:\n", *m_collectorSlotVisitor); >+ m_collectorSlotVisitor->donateAndDrain(); > } > > void Heap::visitConservativeRoots(ConservativeRoots& roots) > { >- m_slotVisitor.append(roots); >+ m_collectorSlotVisitor->append(roots); > > if (Options::logGC() == GCLogging::Verbose) >- dataLog("Conservative Roots:\n", m_slotVisitor); >- >- m_slotVisitor.donateAndDrain(); >+ dataLog("Conservative Roots:\n", *m_collectorSlotVisitor); > } > > void Heap::visitCompilerWorklistWeakReferences() > { > #if ENABLE(DFG_JIT) > for (unsigned i = DFG::numberOfWorklists(); i--;) >- DFG::existingWorklistForIndex(i).visitWeakReferences(m_slotVisitor); >+ DFG::existingWorklistForIndex(i).visitWeakReferences(*m_collectorSlotVisitor); > > if (Options::logGC() == GCLogging::Verbose) >- dataLog("DFG Worklists:\n", m_slotVisitor); >+ dataLog("DFG Worklists:\n", *m_collectorSlotVisitor); > #endif > } > >@@ -710,9 +719,9 @@ void Heap::visitProtectedObjects(HeapRoo > heapRootVisitor.visit(&pair.key); > > if (Options::logGC() == GCLogging::Verbose) >- dataLog("Protected Objects:\n", m_slotVisitor); >+ dataLog("Protected Objects:\n", *m_collectorSlotVisitor); > >- m_slotVisitor.donateAndDrain(); >+ m_collectorSlotVisitor->donateAndDrain(); > } > > void Heap::visitArgumentBuffers(HeapRootVisitor& visitor) >@@ -723,9 +732,9 @@ void Heap::visitArgumentBuffers(HeapRoot > MarkedArgumentBuffer::markLists(visitor, *m_markListSet); > > if (Options::logGC() == GCLogging::Verbose) >- dataLog("Argument Buffers:\n", m_slotVisitor); >+ dataLog("Argument Buffers:\n", *m_collectorSlotVisitor); > >- m_slotVisitor.donateAndDrain(); >+ m_collectorSlotVisitor->donateAndDrain(); > } > > void Heap::visitException(HeapRootVisitor& visitor) >@@ -737,9 +746,9 @@ void Heap::visitException(HeapRootVisito > visitor.visit(m_vm->addressOfLastException()); > > if (Options::logGC() == GCLogging::Verbose) >- dataLog("Exceptions:\n", m_slotVisitor); >+ dataLog("Exceptions:\n", *m_collectorSlotVisitor); > >- m_slotVisitor.donateAndDrain(); >+ m_collectorSlotVisitor->donateAndDrain(); > } > > void Heap::visitStrongHandles(HeapRootVisitor& visitor) >@@ -747,9 +756,9 @@ void Heap::visitStrongHandles(HeapRootVi > m_handleSet.visitStrongHandles(visitor); > > if (Options::logGC() == GCLogging::Verbose) >- dataLog("Strong Handles:\n", m_slotVisitor); >+ dataLog("Strong Handles:\n", m_collectorSlotVisitor); > >- m_slotVisitor.donateAndDrain(); >+ m_collectorSlotVisitor->donateAndDrain(); > } > > void Heap::visitHandleStack(HeapRootVisitor& visitor) >@@ -757,9 +766,9 @@ void Heap::visitHandleStack(HeapRootVisi > m_handleStack.visit(visitor); > > if (Options::logGC() == GCLogging::Verbose) >- dataLog("Handle Stack:\n", m_slotVisitor); >+ dataLog("Handle Stack:\n", *m_collectorSlotVisitor); > >- m_slotVisitor.donateAndDrain(); >+ m_collectorSlotVisitor->donateAndDrain(); > } > > void Heap::visitSamplingProfiler() >@@ -767,11 +776,11 @@ void Heap::visitSamplingProfiler() > #if ENABLE(SAMPLING_PROFILER) > if (SamplingProfiler* samplingProfiler = m_vm->samplingProfiler()) { > ASSERT(samplingProfiler->getLock().isLocked()); >- samplingProfiler->visit(m_slotVisitor); >+ samplingProfiler->visit(*m_collectorSlotVisitor); > if (Options::logGC() == GCLogging::Verbose) >- dataLog("Sampling Profiler data:\n", m_slotVisitor); >+ dataLog("Sampling Profiler data:\n", *m_collectorSlotVisitor); > >- m_slotVisitor.donateAndDrain(); >+ m_collectorSlotVisitor->donateAndDrain(); > samplingProfiler->getLock().unlock(); > } > #endif // ENABLE(SAMPLING_PROFILER) >@@ -779,47 +788,90 @@ void Heap::visitSamplingProfiler() > > void Heap::visitShadowChicken() > { >- m_vm->shadowChicken().visitChildren(m_slotVisitor); >+ m_vm->shadowChicken().visitChildren(*m_collectorSlotVisitor); > } > > void Heap::traceCodeBlocksAndJITStubRoutines() > { >- m_jitStubRoutines->traceMarkedStubRoutines(m_slotVisitor); >+ m_jitStubRoutines->traceMarkedStubRoutines(*m_collectorSlotVisitor); > > if (Options::logGC() == GCLogging::Verbose) >- dataLog("Code Blocks and JIT Stub Routines:\n", m_slotVisitor); >+ dataLog("Code Blocks and JIT Stub Routines:\n", *m_collectorSlotVisitor); > >- m_slotVisitor.donateAndDrain(); >+ m_collectorSlotVisitor->donateAndDrain(); > } > >-void Heap::visitWeakHandles(HeapRootVisitor& visitor) >+void Heap::markToFixpoint(HeapRootVisitor& visitor) > { >- TimingScope timingScope(*this, "Heap::visitWeakHandles"); >+ TimingScope timingScope(*this, "Heap::fixpoint"); >+ Seconds concurrentTime; >+ double stoppedOverConcurrentRatio = 0.1; >+ const double ratioStep = 1.3; >+ MonotonicTime initialTime = MonotonicTime::now(); >+ MonotonicTime topOfLoopTime = initialTime; > while (true) { >+ MonotonicTime timeToResume = topOfLoopTime + concurrentTime * stoppedOverConcurrentRatio; >+ stoppedOverConcurrentRatio *= ratioStep; >+ { >+ TimingScope preConvergenceTimingScope(*this, "Heap::visit conservative scan"); >+ ConservativeRoots conservativeRoots(*this); >+ SuperSamplerScope superSamplerScope(false); >+ gatherStackRoots(conservativeRoots); >+ gatherJSStackRoots(conservativeRoots); >+ gatherScratchBufferRoots(conservativeRoots); >+ visitConservativeRoots(conservativeRoots); >+ } >+ > { > TimingScope timingScope(*this, "m_objectSpace.visitWeakSets"); > m_objectSpace.visitWeakSets(visitor); > } > harvestWeakReferences(); > visitCompilerWorklistWeakReferences(); >- if (m_slotVisitor.isEmpty()) >+ DFG::markCodeBlocks(*m_vm, *m_collectorSlotVisitor); >+ bool shouldTerminate = m_collectorSlotVisitor->isEmpty() && m_mutatorMarkStack->isEmpty(); >+ >+ // We want to do this to conservatively ensure that we rescan any code blocks that are >+ // running right now. However, we need to be sure to do it *after* we mark the code block >+ // so that we know for sure if it really needs a barrier. Also, this has to happen after the >+ // fixpoint check - otherwise we might loop forever. Incidentally, we also want to do this >+ // at the end of GC so that anything at the end of the last GC gets barriered in the next >+ // GC. >+ m_codeBlocks->writeBarrierCurrentlyExecuting(this); >+ DFG::rememberCodeBlocks(*m_vm); >+ >+ if (shouldTerminate) > break; >- >+ >+ // The SlotVisitor's mark stacks are accessed by the collector thread (i.e. this thread) >+ // without locks. That's why we double-buffer. >+ m_mutatorMarkStack->transferTo(m_collectorSlotVisitor->mutatorMarkStack()); >+ > if (Options::logGC() == GCLogging::Verbose) >- dataLog("Live Weak Handles:\n", m_slotVisitor); >+ dataLog("Live Weak Handles:\n", *m_collectorSlotVisitor); > > { >- ParallelModeEnabler enabler(m_slotVisitor); >- m_slotVisitor.donateAndDrain(); >- m_slotVisitor.drainFromShared(SlotVisitor::MasterDrain); >+ ParallelModeEnabler enabler(*m_collectorSlotVisitor); >+ m_collectorSlotVisitor->donateAndDrain(timeToResume); >+ SlotVisitor::SharedDrainResult drainResult = >+ m_collectorSlotVisitor->drainFromShared(SlotVisitor::MasterDrain, timeToResume); >+ if (drainResult != SlotVisitor::SharedDrainResult::Done) { >+ ResumeTheWorldScope resumeTheWorldScope(*this); >+ m_collectorSlotVisitor->donateAndDrain(); >+ m_collectorSlotVisitor->drainFromShared(SlotVisitor::MasterDrain); >+ } > } >+ >+ MonotonicTime timeOfStop = MonotonicTime::now(); >+ concurrentTime = timeOfStop - timeToResume; >+ topOfLoopTime = timeOfStop; > } > } > > void Heap::updateObjectCounts(double gcStartTime) > { > if (Options::logGC() == GCLogging::Verbose) { >- size_t visitCount = m_slotVisitor.visitCount(); >+ size_t visitCount = m_collectorSlotVisitor->visitCount(); > visitCount += threadVisitCount(); > dataLogF("\nNumber of live Objects after GC %lu, took %.6f secs\n", static_cast<unsigned long>(visitCount), WTF::monotonicallyIncreasingTime() - gcStartTime); > } >@@ -827,19 +879,22 @@ void Heap::updateObjectCounts(double gcS > if (m_collectionScope == CollectionScope::Full) > m_totalBytesVisited = 0; > >- m_totalBytesVisitedThisCycle = m_slotVisitor.bytesVisited() + threadBytesVisited(); >+ m_totalBytesVisitedThisCycle = >+ m_collectorSlotVisitor->bytesVisited() + >+ threadBytesVisited(); > > m_totalBytesVisited += m_totalBytesVisitedThisCycle; > } > > void Heap::endMarking() > { >- m_slotVisitor.reset(); >+ m_collectorSlotVisitor->reset(); > > for (auto& parallelVisitor : m_parallelSlotVisitors) > parallelVisitor->reset(); > >- ASSERT(m_sharedMarkStack.isEmpty()); >+ RELEASE_ASSERT(m_sharedCollectorMarkStack->isEmpty()); >+ RELEASE_ASSERT(m_sharedMutatorMarkStack->isEmpty()); > m_weakReferenceHarvesters.removeAll(); > > m_objectSpace.endMarking(); >@@ -977,14 +1032,20 @@ void Heap::addToRememberedSet(const JSCe > { > ASSERT(cell); > ASSERT(!Options::useConcurrentJIT() || !isCompilationThread()); >- ASSERT(cell->cellState() == CellState::AnthraciteOrBlack); >- // Indicate that this object is grey and that it's one of the following: >- // - A re-greyed object during a concurrent collection. >- // - An old remembered object. >- // "OldGrey" doesn't tell us which of these things is true, but we usually treat the two cases the >- // same. >- cell->setCellState(CellState::OldGrey); >- m_slotVisitor.appendToMarkStack(const_cast<JSCell*>(cell)); >+ if (!Heap::isMarkedConcurrently(cell)) { >+ // During a full collection a store into an unmarked object that had surivived past >+ // collections will manifest as a store to an unmarked black object. If the object gets >+ // marked at some time after this then it will go down the normal marking path. We can >+ // safely ignore these stores. >+ return; >+ } >+ // It could be that the object was *just* marked. This means that the collector may set the >+ // state to Grey and then to AnthraciteOrBlack at any time. It's OK for us to race with the >+ // collector here. If we win then this is accurate because the object _will_ get scanned again. >+ // If we lose then someone else will barrier the object again. That would be unfortunate but not >+ // the end of the world. >+ cell->setCellState(CellState::Grey); >+ m_mutatorMarkStack->append(cell); > } > > void Heap::collectAllGarbage() >@@ -1083,29 +1144,14 @@ void Heap::collectInThread() > > stopTheWorld(); > >- double before = 0; >+ MonotonicTime before; > if (Options::logGC()) { > dataLog("[GC: ", capacity() / 1024, " kb "); >- before = currentTimeMS(); >+ before = MonotonicTime::now(); > } > > double gcStartTime; > >- if (vm()->typeProfiler()) { >- DeferGCForAWhile awhile(*this); >- vm()->typeProfilerLog()->processLogEntries(ASCIILiteral("GC")); >- } >- >-#if ENABLE(JIT) >- { >- DeferGCForAWhile awhile(*this); >- if (JITWorklist::instance()->completeAllForVM(*m_vm)) >- setGCDidJIT(); >- } >-#endif // ENABLE(JIT) >- >- vm()->shadowChicken().update(*vm(), vm()->topCallFrame); >- > ASSERT(m_isSafeToCollect); > if (m_collectionScope) { > dataLog("Collection scope already set during GC: ", m_collectionScope, "\n"); >@@ -1125,10 +1171,7 @@ void Heap::collectInThread() > m_verifier->gatherLiveObjects(HeapVerifier::Phase::BeforeMarking); > } > >- flushOldStructureIDTables(); >- stopAllocation(); > prepareForMarking(); >- flushWriteBarrierBuffer(); > > if (HasOwnPropertyCache* cache = vm()->hasOwnPropertyCache()) > cache->clear(); >@@ -1170,8 +1213,8 @@ void Heap::collectInThread() > } > > if (Options::logGC()) { >- double after = currentTimeMS(); >- dataLog(after - before, " ms]\n"); >+ MonotonicTime after = MonotonicTime::now(); >+ dataLog((after - m_stopTime).milliseconds(), " ms, cycle ", (after - before).milliseconds(), " ms]\n"); > } > > { >@@ -1193,10 +1236,38 @@ void Heap::stopTheWorld() > stopTheMutator(); > suspendCompilerThreads(); > m_collectorBelievesThatTheWorldIsStopped = true; >+ >+ if (vm()->typeProfiler()) { >+ DeferGCForAWhile awhile(*this); >+ vm()->typeProfilerLog()->processLogEntries(ASCIILiteral("GC")); >+ } >+ >+#if ENABLE(JIT) >+ { >+ DeferGCForAWhile awhile(*this); >+ if (JITWorklist::instance()->completeAllForVM(*m_vm)) >+ setGCDidJIT(); >+ } >+#endif // ENABLE(JIT) >+ >+ vm()->shadowChicken().update(*vm(), vm()->topCallFrame); >+ >+ flushWriteBarrierBuffer(); >+ m_structureIDTable.flushOldTables(); >+ m_objectSpace.stopAllocating(); >+ >+ m_stopTime = MonotonicTime::now(); > } > > void Heap::resumeTheWorld() > { >+ // Calling resumeAllocating does the Right Thing depending on whether this is the end of a >+ // collection cycle or this is just a concurrent phase within a collection cycle: >+ // - At end of collection cycle: it's a no-op because prepareForAllocation already cleared the >+ // last active block. >+ // - During collection cycle: it reinstates the last active block. >+ m_objectSpace.resumeAllocating(); >+ > RELEASE_ASSERT(m_collectorBelievesThatTheWorldIsStopped); > m_collectorBelievesThatTheWorldIsStopped = false; > resumeCompilerThreads(); >@@ -1549,23 +1620,9 @@ void Heap::willStartCollection(Optional< > observer->willGarbageCollect(); > } > >-void Heap::flushOldStructureIDTables() >-{ >- m_structureIDTable.flushOldTables(); >-} >- > void Heap::flushWriteBarrierBuffer() > { >- if (m_collectionScope == CollectionScope::Eden) { >- m_writeBarrierBuffer.flush(*this); >- return; >- } >- m_writeBarrierBuffer.reset(); >-} >- >-void Heap::stopAllocation() >-{ >- m_objectSpace.stopAllocating(); >+ m_writeBarrierBuffer.flush(*this); > } > > void Heap::prepareForMarking() >@@ -1949,7 +2006,7 @@ bool Heap::canCollect() > return false; > if (!m_isSafeToCollect) > return false; >- if (collectionScope() || mutatorState() == MutatorState::HelpingGC) >+ if (mutatorState() == MutatorState::HelpingGC) > return false; > return true; > } >@@ -1971,12 +2028,8 @@ bool Heap::isCurrentThreadBusy() > return mayBeGCThread() || mutatorState() != MutatorState::Running; > } > >-void Heap::reportExtraMemoryVisited(CellState oldState, size_t size) >+void Heap::reportExtraMemoryVisited(size_t size) > { >- // We don't want to double-count the extra memory that was reported in previous collections. >- if (collectionScope() == CollectionScope::Eden && oldState == CellState::OldGrey) >- return; >- > size_t* counter = &m_extraMemorySize; > > for (;;) { >@@ -1987,12 +2040,8 @@ void Heap::reportExtraMemoryVisited(Cell > } > > #if ENABLE(RESOURCE_USAGE) >-void Heap::reportExternalMemoryVisited(CellState oldState, size_t size) >+void Heap::reportExternalMemoryVisited(size_t size) > { >- // We don't want to double-count the external memory that was reported in previous collections. >- if (collectionScope() == CollectionScope::Eden && oldState == CellState::OldGrey) >- return; >- > size_t* counter = &m_externalMemorySize; > > for (;;) { >Index: Source/JavaScriptCore/heap/Heap.h >=================================================================== >--- Source/JavaScriptCore/heap/Heap.h (revision 208415) >+++ Source/JavaScriptCore/heap/Heap.h (working copy) >@@ -22,6 +22,7 @@ > #pragma once > > #include "ArrayBuffer.h" >+#include "CellState.h" > #include "CollectionScope.h" > #include "GCIncomingRefCountedSet.h" > #include "HandleSet.h" >@@ -35,7 +36,6 @@ > #include "MarkedSpace.h" > #include "MutatorState.h" > #include "Options.h" >-#include "SlotVisitor.h" > #include "StructureIDTable.h" > #include "TinyBloomFilter.h" > #include "UnconditionalFinalizer.h" >@@ -70,7 +70,9 @@ class JITStubRoutineSet; > class JSCell; > class JSValue; > class LLIntOffsetsExtractor; >+class MarkStackArray; > class MarkedArgumentBuffer; >+class SlotVisitor; > class StopIfNecessaryTimer; > class VM; > >@@ -129,7 +131,8 @@ public: > MarkedSpace& objectSpace() { return m_objectSpace; } > MachineThreads& machineThreads() { return m_machineThreads; } > >- const SlotVisitor& slotVisitor() const { return m_slotVisitor; } >+ SlotVisitor& collectorSlotVisitor() { return *m_collectorSlotVisitor; } >+ MarkStackArray& mutatorMarkStack() { return *m_mutatorMarkStack; } > > JS_EXPORT_PRIVATE GCActivityCallback* fullActivityCallback(); > JS_EXPORT_PRIVATE GCActivityCallback* edenActivityCallback(); >@@ -203,11 +206,11 @@ public: > // call both of these functions: Calling only one may trigger catastropic > // memory growth. > void reportExtraMemoryAllocated(size_t); >- JS_EXPORT_PRIVATE void reportExtraMemoryVisited(CellState oldState, size_t); >+ JS_EXPORT_PRIVATE void reportExtraMemoryVisited(size_t); > > #if ENABLE(RESOURCE_USAGE) > // Use this API to report the subset of extra memory that lives outside this process. >- JS_EXPORT_PRIVATE void reportExternalMemoryVisited(CellState oldState, size_t); >+ JS_EXPORT_PRIVATE void reportExternalMemoryVisited(size_t); > size_t externalMemorySize() { return m_externalMemorySize; } > #endif > >@@ -391,6 +394,9 @@ private: > void stopTheWorld(); > void resumeTheWorld(); > >+ class ResumeTheWorldScope; >+ friend class ResumeTheWorldScope; >+ > void stopTheMutator(); > void resumeTheMutator(); > >@@ -422,9 +428,7 @@ private: > > void suspendCompilerThreads(); > void willStartCollection(Optional<CollectionScope>); >- void flushOldStructureIDTables(); > void flushWriteBarrierBuffer(); >- void stopAllocation(); > void prepareForMarking(); > > void markRoots(double gcStartTime); >@@ -445,7 +449,7 @@ private: > void visitSamplingProfiler(); > void visitShadowChicken(); > void traceCodeBlocksAndJITStubRoutines(); >- void visitWeakHandles(HeapRootVisitor&); >+ void markToFixpoint(HeapRootVisitor&); > void updateObjectCounts(double gcStartTime); > void endMarking(); > >@@ -517,7 +521,8 @@ private: > > MachineThreads m_machineThreads; > >- SlotVisitor m_slotVisitor; >+ std::unique_ptr<SlotVisitor> m_collectorSlotVisitor; >+ std::unique_ptr<MarkStackArray> m_mutatorMarkStack; > > // We pool the slot visitors used by parallel marking threads. It's useful to be able to > // enumerate over them, and it's useful to have them cache some small amount of memory from >@@ -571,7 +576,8 @@ private: > > Lock m_markingMutex; > Condition m_markingConditionVariable; >- MarkStackArray m_sharedMarkStack; >+ std::unique_ptr<MarkStackArray> m_sharedCollectorMarkStack; >+ std::unique_ptr<MarkStackArray> m_sharedMutatorMarkStack; > unsigned m_numberOfActiveParallelMarkers { 0 }; > unsigned m_numberOfWaitingParallelMarkers { 0 }; > bool m_parallelMarkersShouldExit { false }; >@@ -599,6 +605,7 @@ private: > static const unsigned mutatorWaitingBit = 1u << 5u; // Allows the mutator to use this as a condition variable. > Atomic<unsigned> m_worldState; > bool m_collectorBelievesThatTheWorldIsStopped { false }; >+ MonotonicTime m_stopTime; > > Deque<Optional<CollectionScope>> m_requests; > Ticket m_lastServedTicket { 0 }; >Index: Source/JavaScriptCore/heap/MarkStack.cpp >=================================================================== >--- Source/JavaScriptCore/heap/MarkStack.cpp (revision 208415) >+++ Source/JavaScriptCore/heap/MarkStack.cpp (working copy) >@@ -1,5 +1,5 @@ > /* >- * Copyright (C) 2009, 2011, 2015 Apple Inc. All rights reserved. >+ * Copyright (C) 2009-2016 Apple Inc. All rights reserved. > * > * Redistribution and use in source and binary forms, with or without > * modification, are permitted provided that the following conditions >@@ -36,6 +36,34 @@ MarkStackArray::MarkStackArray() > { > } > >+void MarkStackArray::transferTo(MarkStackArray& other) >+{ >+ RELEASE_ASSERT(this != &other); >+ >+ // Remove our head and the head of the other list. >+ GCArraySegment<const JSCell*>* myHead = m_segments.removeHead(); >+ GCArraySegment<const JSCell*>* otherHead = other.m_segments.removeHead(); >+ m_numberOfSegments--; >+ other.m_numberOfSegments--; >+ >+ other.m_segments.append(m_segments); >+ >+ other.m_numberOfSegments += m_numberOfSegments; >+ m_numberOfSegments = 0; >+ >+ // Put the original heads back in their places. >+ m_segments.push(myHead); >+ other.m_segments.push(otherHead); >+ m_numberOfSegments++; >+ other.m_numberOfSegments++; >+ >+ while (!isEmpty()) { >+ refill(); >+ while (canRemoveLast()) >+ other.append(removeLast()); >+ } >+} >+ > void MarkStackArray::donateSomeCellsTo(MarkStackArray& other) > { > // Try to donate about 1 / 2 of our cells. To reduce copying costs, >Index: Source/JavaScriptCore/heap/MarkStack.h >=================================================================== >--- Source/JavaScriptCore/heap/MarkStack.h (revision 208415) >+++ Source/JavaScriptCore/heap/MarkStack.h (working copy) >@@ -1,5 +1,5 @@ > /* >- * Copyright (C) 2009, 2011 Apple Inc. All rights reserved. >+ * Copyright (C) 2009-2016 Apple Inc. All rights reserved. > * > * Redistribution and use in source and binary forms, with or without > * modification, are permitted provided that the following conditions >@@ -35,8 +35,9 @@ class MarkStackArray : public GCSegmente > public: > MarkStackArray(); > >- void donateSomeCellsTo(MarkStackArray& other); >- void stealSomeCellsFrom(MarkStackArray& other, size_t idleThreadCount); >+ void transferTo(MarkStackArray&); >+ void donateSomeCellsTo(MarkStackArray&); >+ void stealSomeCellsFrom(MarkStackArray&, size_t idleThreadCount); > }; > > } // namespace JSC >Index: Source/JavaScriptCore/heap/MarkedSpace.cpp >=================================================================== >--- Source/JavaScriptCore/heap/MarkedSpace.cpp (revision 208415) >+++ Source/JavaScriptCore/heap/MarkedSpace.cpp (working copy) >@@ -412,7 +412,6 @@ void MarkedSpace::prepareForMarking() > > void MarkedSpace::resumeAllocating() > { >- ASSERT(isIterating()); > forEachAllocator( > [&] (MarkedAllocator& allocator) -> IterationStatus { > allocator.resumeAllocating(); >Index: Source/JavaScriptCore/heap/ResumeTheWorldScope.h >=================================================================== >--- Source/JavaScriptCore/heap/ResumeTheWorldScope.h (nonexistent) >+++ Source/JavaScriptCore/heap/ResumeTheWorldScope.h (working copy) >@@ -0,0 +1,24 @@ >+/* >+ * Copyright (C) 2016 Apple Inc. All rights reserved. >+ * >+ * Redistribution and use in source and binary forms, with or without >+ * modification, are permitted provided that the following conditions >+ * are met: >+ * 1. Redistributions of source code must retain the above copyright >+ * notice, this list of conditions and the following disclaimer. >+ * 2. Redistributions in binary form must reproduce the above copyright >+ * notice, this list of conditions and the following disclaimer in the >+ * documentation and/or other materials provided with the distribution. >+ * >+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' >+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, >+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR >+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS >+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR >+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF >+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS >+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN >+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) >+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF >+ * THE POSSIBILITY OF SUCH DAMAGE. >+ */ >Index: Source/JavaScriptCore/heap/SlotVisitor.cpp >=================================================================== >--- Source/JavaScriptCore/heap/SlotVisitor.cpp (revision 208415) >+++ Source/JavaScriptCore/heap/SlotVisitor.cpp (working copy) >@@ -75,9 +75,7 @@ static void validate(JSCell* cell) > #endif > > SlotVisitor::SlotVisitor(Heap& heap) >- : m_stack() >- , m_bytesVisited(0) >- , m_bytesCopied(0) >+ : m_bytesVisited(0) > , m_visitCount(0) > , m_isInParallelMode(false) > , m_markingVersion(MarkedSpace::initialVersion) >@@ -91,7 +89,7 @@ SlotVisitor::SlotVisitor(Heap& heap) > > SlotVisitor::~SlotVisitor() > { >- clearMarkStack(); >+ clearMarkStacks(); > } > > void SlotVisitor::didStartMarking() >@@ -110,15 +108,15 @@ void SlotVisitor::didStartMarking() > void SlotVisitor::reset() > { > m_bytesVisited = 0; >- m_bytesCopied = 0; > m_visitCount = 0; > m_heapSnapshotBuilder = nullptr; > ASSERT(!m_currentCell); > } > >-void SlotVisitor::clearMarkStack() >+void SlotVisitor::clearMarkStacks() > { >- m_stack.clear(); >+ m_collectorStack.clear(); >+ m_mutatorStack.clear(); > } > > void SlotVisitor::append(ConservativeRoots& conservativeRoots) >@@ -148,7 +146,7 @@ void SlotVisitor::appendJSCellOrAuxiliar > return; > } > >- jsCell->setCellState(CellState::NewGrey); >+ jsCell->setCellState(CellState::Grey); > > appendToMarkStack(jsCell); > return; >@@ -210,7 +208,7 @@ ALWAYS_INLINE void SlotVisitor::setMarke > // Indicate that the object is grey and that: > // In case of concurrent GC: it's the first time it is grey in this GC cycle. > // In case of eden collection: it's a new object that became grey rather than an old remembered object. >- cell->setCellState(CellState::NewGrey); >+ cell->setCellState(CellState::Grey); > > appendToMarkStack(container, cell); > } >@@ -228,16 +226,19 @@ ALWAYS_INLINE void SlotVisitor::appendTo > { > ASSERT(Heap::isMarkedConcurrently(cell)); > ASSERT(!cell->isZapped()); >- ASSERT(cell->cellState() == CellState::NewGrey || cell->cellState() == CellState::OldGrey); >+ ASSERT(cell->cellState() == CellState::Grey); > > container.noteMarked(); > >- // FIXME: These "just work" because the GC resets these fields before doing anything else. But >- // that won't be the case when we do concurrent GC. > m_visitCount++; > m_bytesVisited += container.cellSize(); > >- m_stack.append(cell); >+ m_collectorStack.append(cell); >+} >+ >+void SlotVisitor::appendToMutatorMarkStack(const JSCell* cell) >+{ >+ m_mutatorStack.append(cell); > } > > void SlotVisitor::markAuxiliary(const void* base) >@@ -294,12 +295,10 @@ ALWAYS_INLINE void SlotVisitor::visitChi > > SetCurrentCellScope currentCellScope(*this, cell); > >- m_oldCellState = cell->cellState(); >- >- // There is no race here - the cell state cannot change right now. Grey objects can only be >- // visited by one marking thread. Neither the barrier nor marking will change the state of an >- // object that is already grey. >- ASSERT(m_oldCellState == CellState::OldGrey || m_oldCellState == CellState::NewGrey); >+ // Funny story: it's possible for the object to be black already, if we barrier the object at >+ // about the same time that it's marked. That's fine. It's a gnarly and super-rare race. It's >+ // not clear to me that it would be correct or profitable to bail here if the object is already >+ // black. > > cell->setCellState(CellState::AnthraciteOrBlack); > >@@ -326,23 +325,23 @@ ALWAYS_INLINE void SlotVisitor::visitChi > } > > if (UNLIKELY(m_heapSnapshotBuilder)) { >- if (m_oldCellState == CellState::NewGrey) >+ if (!m_isVisitingMutatorStack) > m_heapSnapshotBuilder->appendNode(const_cast<JSCell*>(cell)); > } > } > >-void SlotVisitor::donateKnownParallel() >+void SlotVisitor::donateKnownParallel(MarkStackArray& from, MarkStackArray& to) > { > // NOTE: Because we re-try often, we can afford to be conservative, and > // assume that donating is not profitable. > > // Avoid locking when a thread reaches a dead end in the object graph. >- if (m_stack.size() < 2) >+ if (from.size() < 2) > return; > > // If there's already some shared work queued up, be conservative and assume > // that donating more is not profitable. >- if (m_heap.m_sharedMarkStack.size()) >+ if (to.size()) > return; > > // If we're contending on the lock, be conservative and assume that another >@@ -352,26 +351,43 @@ void SlotVisitor::donateKnownParallel() > return; > > // Otherwise, assume that a thread will go idle soon, and donate. >- m_stack.donateSomeCellsTo(m_heap.m_sharedMarkStack); >+ from.donateSomeCellsTo(to); > > m_heap.m_markingConditionVariable.notifyAll(); > } > >-void SlotVisitor::drain() >+void SlotVisitor::donateKnownParallel() >+{ >+ donateKnownParallel(m_collectorStack, *m_heap.m_sharedCollectorMarkStack); >+ donateKnownParallel(m_mutatorStack, *m_heap.m_sharedMutatorMarkStack); >+} >+ >+void SlotVisitor::drain(MonotonicTime timeout) > { > ASSERT(m_isInParallelMode); > >- while (!m_stack.isEmpty()) { >- m_stack.refill(); >- for (unsigned countdown = Options::minimumNumberOfScansBetweenRebalance(); m_stack.canRemoveLast() && countdown--;) >- visitChildren(m_stack.removeLast()); >+ while ((!m_collectorStack.isEmpty() || !m_mutatorStack.isEmpty()) && !hasElapsed(timeout)) { >+ if (!m_collectorStack.isEmpty()) { >+ m_collectorStack.refill(); >+ m_isVisitingMutatorStack = false; >+ for (unsigned countdown = Options::minimumNumberOfScansBetweenRebalance(); m_collectorStack.canRemoveLast() && countdown--;) >+ visitChildren(m_collectorStack.removeLast()); >+ } else if (!m_mutatorStack.isEmpty()) { >+ m_mutatorStack.refill(); >+ // We know for sure that we are visiting objects because of the barrier, not because of >+ // marking. Marking will visit an object exactly once. The barrier will visit it >+ // possibly many times, and always after it was already marked. >+ m_isVisitingMutatorStack = true; >+ for (unsigned countdown = Options::minimumNumberOfScansBetweenRebalance(); m_mutatorStack.canRemoveLast() && countdown--;) >+ visitChildren(m_mutatorStack.removeLast()); >+ } > donateKnownParallel(); > } > > mergeOpaqueRootsIfNecessary(); > } > >-void SlotVisitor::drainFromShared(SharedDrainMode sharedDrainMode) >+SlotVisitor::SharedDrainResult SlotVisitor::drainFromShared(SharedDrainMode sharedDrainMode, MonotonicTime timeout) > { > ASSERT(m_isInParallelMode); > >@@ -394,46 +410,58 @@ void SlotVisitor::drainFromShared(Shared > while (true) { > // Did we reach termination? > if (!m_heap.m_numberOfActiveParallelMarkers >- && m_heap.m_sharedMarkStack.isEmpty()) { >+ && m_heap.m_sharedCollectorMarkStack->isEmpty() >+ && m_heap.m_sharedMutatorMarkStack->isEmpty()) { > // Let any sleeping slaves know it's time for them to return; > m_heap.m_markingConditionVariable.notifyAll(); >- return; >+ return SharedDrainResult::Done; > } > >+ if (hasElapsed(timeout)) >+ return SharedDrainResult::TimedOut; >+ > // Is there work to be done? >- if (!m_heap.m_sharedMarkStack.isEmpty()) >+ if (!m_heap.m_sharedCollectorMarkStack->isEmpty() >+ || !m_heap.m_sharedMutatorMarkStack->isEmpty()) > break; > > // Otherwise wait. >- m_heap.m_markingConditionVariable.wait(lock); >+ m_heap.m_markingConditionVariable.waitUntil(lock, timeout); > } > } else { > ASSERT(sharedDrainMode == SlaveDrain); > > // Did we detect termination? If so, let the master know. > if (!m_heap.m_numberOfActiveParallelMarkers >- && m_heap.m_sharedMarkStack.isEmpty()) >+ && m_heap.m_sharedCollectorMarkStack->isEmpty() >+ && m_heap.m_sharedMutatorMarkStack->isEmpty()) > m_heap.m_markingConditionVariable.notifyAll(); > >- m_heap.m_markingConditionVariable.wait( >- lock, >+ m_heap.m_markingConditionVariable.waitUntil( >+ lock, timeout, > [this] { >- return !m_heap.m_sharedMarkStack.isEmpty() >+ return !m_heap.m_sharedCollectorMarkStack->isEmpty() >+ || !m_heap.m_sharedMutatorMarkStack->isEmpty() > || m_heap.m_parallelMarkersShouldExit; > }); > > // Is the current phase done? If so, return from this function. > if (m_heap.m_parallelMarkersShouldExit) >- return; >+ return SharedDrainResult::Done; >+ >+ if (hasElapsed(timeout)) >+ return SharedDrainResult::TimedOut; > } > >- m_stack.stealSomeCellsFrom( >- m_heap.m_sharedMarkStack, m_heap.m_numberOfWaitingParallelMarkers); >+ m_collectorStack.stealSomeCellsFrom( >+ *m_heap.m_sharedCollectorMarkStack, m_heap.m_numberOfWaitingParallelMarkers); >+ m_mutatorStack.stealSomeCellsFrom( >+ *m_heap.m_sharedMutatorMarkStack, m_heap.m_numberOfWaitingParallelMarkers); > m_heap.m_numberOfActiveParallelMarkers++; > m_heap.m_numberOfWaitingParallelMarkers--; > } > >- drain(); >+ drain(timeout); > } > } > >@@ -497,10 +525,10 @@ void SlotVisitor::donate() > donateKnownParallel(); > } > >-void SlotVisitor::donateAndDrain() >+void SlotVisitor::donateAndDrain(MonotonicTime timeout) > { > donate(); >- drain(); >+ drain(timeout); > } > > void SlotVisitor::mergeOpaqueRoots() >@@ -526,10 +554,9 @@ void SlotVisitor::finalizeUnconditionalF > m_heap.m_unconditionalFinalizers.removeNext()->finalizeUnconditionally(); > } > >-void SlotVisitor::dump(PrintStream&) const >+void SlotVisitor::dump(PrintStream& out) const > { >- for (const JSCell* cell : markStack()) >- dataLog(*cell, "\n"); >+ out.print("Collector: [", pointerListDump(collectorMarkStack()), "], Mutator: [", pointerListDump(mutatorMarkStack()), "]"); > } > > } // namespace JSC >Index: Source/JavaScriptCore/heap/SlotVisitor.h >=================================================================== >--- Source/JavaScriptCore/heap/SlotVisitor.h (revision 208415) >+++ Source/JavaScriptCore/heap/SlotVisitor.h (working copy) >@@ -29,6 +29,7 @@ > #include "HandleTypes.h" > #include "MarkStack.h" > #include "OpaqueRootSet.h" >+#include <wtf/MonotonicTime.h> > > namespace JSC { > >@@ -57,9 +58,11 @@ public: > SlotVisitor(Heap&); > ~SlotVisitor(); > >- MarkStackArray& markStack() { return m_stack; } >- const MarkStackArray& markStack() const { return m_stack; } >- >+ MarkStackArray& collectorMarkStack() { return m_collectorStack; } >+ MarkStackArray& mutatorMarkStack() { return m_mutatorStack; } >+ const MarkStackArray& collectorMarkStack() const { return m_collectorStack; } >+ const MarkStackArray& mutatorMarkStack() const { return m_mutatorStack; } >+ > VM& vm(); > const VM& vm() const; > Heap* heap() const; >@@ -86,22 +89,22 @@ public: > TriState containsOpaqueRootTriState(void*) const; > int opaqueRootCount(); > >- bool isEmpty() { return m_stack.isEmpty(); } >+ bool isEmpty() { return m_collectorStack.isEmpty() && m_mutatorStack.isEmpty(); } > > void didStartMarking(); > void reset(); >- void clearMarkStack(); >+ void clearMarkStacks(); > > size_t bytesVisited() const { return m_bytesVisited; } >- size_t bytesCopied() const { return m_bytesCopied; } > size_t visitCount() const { return m_visitCount; } > > void donate(); >- void drain(); >- void donateAndDrain(); >+ void drain(MonotonicTime timeout = MonotonicTime::infinity()); >+ void donateAndDrain(MonotonicTime timeout = MonotonicTime::infinity()); > > enum SharedDrainMode { SlaveDrain, MasterDrain }; >- void drainFromShared(SharedDrainMode); >+ enum class SharedDrainResult { Done, TimedOut }; >+ SharedDrainResult drainFromShared(SharedDrainMode, MonotonicTime timeout = MonotonicTime::infinity()); > > void harvestWeakReferences(); > void finalizeUnconditionalFinalizers(); >@@ -141,6 +144,8 @@ private: > template<typename ContainerType> > void appendToMarkStack(ContainerType&, JSCell*); > >+ void appendToMutatorMarkStack(const JSCell*); >+ > void noteLiveAuxiliaryCell(HeapCell*); > > JS_EXPORT_PRIVATE void mergeOpaqueRoots(); >@@ -150,12 +155,13 @@ private: > void visitChildren(const JSCell*); > > void donateKnownParallel(); >+ void donateKnownParallel(MarkStackArray& from, MarkStackArray& to); > >- MarkStackArray m_stack; >+ MarkStackArray m_collectorStack; >+ MarkStackArray m_mutatorStack; > OpaqueRootSet m_opaqueRoots; // Handle-owning data structures not visible to the garbage collector. > > size_t m_bytesVisited; >- size_t m_bytesCopied; > size_t m_visitCount; > bool m_isInParallelMode; > >@@ -165,7 +171,7 @@ private: > > HeapSnapshotBuilder* m_heapSnapshotBuilder { nullptr }; > JSCell* m_currentCell { nullptr }; >- CellState m_oldCellState; >+ bool m_isVisitingMutatorStack { false }; > > public: > #if !ASSERT_DISABLED >Index: Source/JavaScriptCore/heap/SlotVisitorInlines.h >=================================================================== >--- Source/JavaScriptCore/heap/SlotVisitorInlines.h (revision 208415) >+++ Source/JavaScriptCore/heap/SlotVisitorInlines.h (working copy) >@@ -105,13 +105,15 @@ inline void SlotVisitor::addUnconditiona > > inline void SlotVisitor::reportExtraMemoryVisited(size_t size) > { >- heap()->reportExtraMemoryVisited(m_oldCellState, size); >+ if (!m_isVisitingMutatorStack) >+ heap()->reportExtraMemoryVisited(size); > } > > #if ENABLE(RESOURCE_USAGE) > inline void SlotVisitor::reportExternalMemoryVisited(size_t size) > { >- heap()->reportExternalMemoryVisited(m_oldCellState, size); >+ if (!m_isVisitingMutatorStack) >+ heap()->reportExternalMemoryVisited(size); > } > #endif > >Index: Source/JavaScriptCore/runtime/JSCell.h >=================================================================== >--- Source/JavaScriptCore/runtime/JSCell.h (revision 208415) >+++ Source/JavaScriptCore/runtime/JSCell.h (working copy) >@@ -167,6 +167,16 @@ public: > CellState cellState() const { return m_cellState; } > > void setCellState(CellState data) const { const_cast<JSCell*>(this)->m_cellState = data; } >+ >+ bool atomicCompareExchangeCellStateWeakRelaxed(CellState oldState, CellState newState) >+ { >+ return WTF::atomicCompareExchangeWeakRelaxed(&m_cellState, oldState, newState); >+ } >+ >+ CellState atomicCompareExchangeCellStateStrong(CellState oldState, CellState newState) >+ { >+ return WTF::atomicCompareExchangeStrong(&m_cellState, oldState, newState); >+ } > > static ptrdiff_t structureIDOffset() > { >Index: Source/JavaScriptCore/runtime/Options.h >=================================================================== >--- Source/JavaScriptCore/runtime/Options.h (revision 208415) >+++ Source/JavaScriptCore/runtime/Options.h (working copy) >@@ -184,6 +184,7 @@ typedef const char* optionString; > v(bool, verboseSanitizeStack, false, Normal, nullptr) \ > v(bool, useGenerationalGC, true, Normal, nullptr) \ > v(bool, useConcurrentBarriers, true, Normal, nullptr) \ >+ v(bool, useConcurrentGC, false, Normal, nullptr) \ > v(bool, forceFencedBarrier, false, Normal, nullptr) \ > v(bool, scribbleFreeCells, false, Normal, nullptr) \ > v(double, sizeClassProgression, 1.4, Normal, nullptr) \ >Index: Source/WTF/wtf/ListDump.h >=================================================================== >--- Source/WTF/wtf/ListDump.h (revision 208415) >+++ Source/WTF/wtf/ListDump.h (working copy) >@@ -177,10 +177,11 @@ ListDumpInContext<T, U> listDumpInContex > } // namespace WTF > > using WTF::listDump; >-using WTF::sortedListDump; >+using WTF::listDumpInContext; > using WTF::mapDump; >+using WTF::pointerListDump; >+using WTF::sortedListDump; > using WTF::sortedMapDump; >-using WTF::listDumpInContext; > > #endif // ListDump_h > >Index: Source/WTF/wtf/TimeWithDynamicClockType.cpp >=================================================================== >--- Source/WTF/wtf/TimeWithDynamicClockType.cpp (revision 208415) >+++ Source/WTF/wtf/TimeWithDynamicClockType.cpp (working copy) >@@ -29,6 +29,9 @@ > #include "Condition.h" > #include "Lock.h" > #include "PrintStream.h" >+#include <cfloat> >+#include <cmath> >+#include <wtf/DataLog.h> > > namespace WTF { > >@@ -128,6 +131,17 @@ void sleep(const TimeWithDynamicClockTyp > fakeCondition.waitUntil(fakeLock, time); > } > >+bool hasElapsed(const TimeWithDynamicClockType& time) >+{ >+ // Avoid doing now(). >+ if (!(time > time.withSameClockAndRawSeconds(0))) >+ return true; >+ if (std::isinf(time.secondsSinceEpoch().value())) >+ return false; >+ >+ return time <= time.nowWithSameClock(); >+} >+ > } // namespace WTF > > >Index: Source/WTF/wtf/TimeWithDynamicClockType.h >=================================================================== >--- Source/WTF/wtf/TimeWithDynamicClockType.h (revision 208415) >+++ Source/WTF/wtf/TimeWithDynamicClockType.h (working copy) >@@ -134,8 +134,12 @@ private: > > WTF_EXPORT_PRIVATE void sleep(const TimeWithDynamicClockType&); > >+WTF_EXPORT_PRIVATE bool hasElapsed(const TimeWithDynamicClockType&); >+ > } // namespace WTF > > using WTF::TimeWithDynamicClockType; >+using WTF::hasElapsed; >+using WTF::sleep; > > #endif // WTF_TimeWithDynamicClockType_h
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Diff
View Attachment As Raw
Actions:
View
|
Formatted Diff
|
Diff
Attachments on
bug 164454
:
294001
|
294004
|
294007
|
294042
|
294043
|
294073
|
294074
|
294098
|
294110
|
294172
|
294191
|
294193
|
294205
|
294324
|
294327
|
294449
|
294512
|
294526
|
294613
|
294616
|
294623
|
294688
|
294724
|
294733
|
294738
|
294739
|
294740