Imported WebKit commit 3a8c29f35d00659d2ce7a0ccdfa8304f14e82327 (http://svn.webkit.org/repository/webkit/trunk@120813)

New snapshot with Windows build fixes

26 files changed, 454 insertions, 305 deletions

diff --git a/Source/JavaScriptCore/heap/BlockAllocator.cpp b/Source/JavaScriptCore/heap/BlockAllocator.cpp
index 485ec8dd1..fc4f8a4c6 100644
--- a/Source/JavaScriptCore/heap/BlockAllocator.cpp
+++ b/Source/JavaScriptCore/heap/BlockAllocator.cpp
@@ -37,13 +37,15 @@ BlockAllocator::BlockAllocator()
     , m_blockFreeingThread(createThread(blockFreeingThreadStartFunc, this, "JavaScriptCore::BlockFree"))
 {
     ASSERT(m_blockFreeingThread);
+    m_freeBlockLock.Init();
 }
 
 BlockAllocator::~BlockAllocator()
 {
     releaseFreeBlocks();
     {
-        MutexLocker locker(m_freeBlockLock);
+        MutexLocker locker(m_freeBlockConditionLock);
+
         m_blockFreeingThreadShouldQuit = true;
         m_freeBlockCondition.broadcast();
     }
@@ -55,7 +57,7 @@ void BlockAllocator::releaseFreeBlocks()
     while (true) {
         HeapBlock* block;
         {
-            MutexLocker locker(m_freeBlockLock);
+            SpinLockHolder locker(&m_freeBlockLock);
             if (!m_numberOfFreeBlocks)
                 block = 0;
             else {
@@ -76,7 +78,8 @@ void BlockAllocator::waitForRelativeTimeWhileHoldingLock(double relative)
 {
     if (m_blockFreeingThreadShouldQuit)
         return;
-    m_freeBlockCondition.timedWait(m_freeBlockLock, currentTime() + relative);
+
+    m_freeBlockCondition.timedWait(m_freeBlockConditionLock, currentTime() + relative);
 }
 
 void BlockAllocator::waitForRelativeTime(double relative)
@@ -85,7 +88,7 @@ void BlockAllocator::waitForRelativeTime(double relative)
     // frequently. It would only be a bug if this function failed to return
     // when it was asked to do so.
     
-    MutexLocker locker(m_freeBlockLock);
+    MutexLocker locker(m_freeBlockConditionLock);
     waitForRelativeTimeWhileHoldingLock(relative);
 }
 
@@ -120,7 +123,7 @@ void BlockAllocator::blockFreeingThreadMain()
         while (!m_blockFreeingThreadShouldQuit) {
             HeapBlock* block;
             {
-                MutexLocker locker(m_freeBlockLock);
+                SpinLockHolder locker(&m_freeBlockLock);
                 if (m_numberOfFreeBlocks <= desiredNumberOfFreeBlocks)
                     block = 0;
                 else {
diff --git a/Source/JavaScriptCore/heap/BlockAllocator.h b/Source/JavaScriptCore/heap/BlockAllocator.h
index 846bdfa2a..7a99d2edd 100644
--- a/Source/JavaScriptCore/heap/BlockAllocator.h
+++ b/Source/JavaScriptCore/heap/BlockAllocator.h
@@ -30,6 +30,7 @@
 #include <wtf/DoublyLinkedList.h>
 #include <wtf/Forward.h>
 #include <wtf/PageAllocationAligned.h>
+#include <wtf/TCSpinLock.h>
 #include <wtf/Threading.h>
 
 namespace JSC {
@@ -58,19 +59,22 @@ private:
     size_t m_numberOfFreeBlocks;
     bool m_isCurrentlyAllocating;
     bool m_blockFreeingThreadShouldQuit;
-    Mutex m_freeBlockLock;
+    SpinLock m_freeBlockLock;
+    Mutex m_freeBlockConditionLock;
     ThreadCondition m_freeBlockCondition;
     ThreadIdentifier m_blockFreeingThread;
 };
 
 inline PageAllocationAligned BlockAllocator::allocate()
 {
-    MutexLocker locker(m_freeBlockLock);
-    m_isCurrentlyAllocating = true;
-    if (m_numberOfFreeBlocks) {
-        ASSERT(!m_freeBlocks.isEmpty());
-        m_numberOfFreeBlocks--;
-        return m_freeBlocks.removeHead()->m_allocation;
+    {
+        SpinLockHolder locker(&m_freeBlockLock);
+        m_isCurrentlyAllocating = true;
+        if (m_numberOfFreeBlocks) {
+            ASSERT(!m_freeBlocks.isEmpty());
+            m_numberOfFreeBlocks--;
+            return m_freeBlocks.removeHead()->m_allocation;
+        }
     }
 
     ASSERT(m_freeBlocks.isEmpty());
@@ -82,7 +86,7 @@ inline PageAllocationAligned BlockAllocator::allocate()
 
 inline void BlockAllocator::deallocate(PageAllocationAligned allocation)
 {
-    MutexLocker locker(m_freeBlockLock);
+    SpinLockHolder locker(&m_freeBlockLock);
     HeapBlock* heapBlock = new(NotNull, allocation.base()) HeapBlock(allocation);
     m_freeBlocks.push(heapBlock);
     m_numberOfFreeBlocks++;
diff --git a/Source/JavaScriptCore/heap/CopiedBlock.h b/Source/JavaScriptCore/heap/CopiedBlock.h
index b408aa40b..5ed58008e 100644
--- a/Source/JavaScriptCore/heap/CopiedBlock.h
+++ b/Source/JavaScriptCore/heap/CopiedBlock.h
@@ -39,6 +39,7 @@ class CopiedBlock : public HeapBlock {
     friend class CopiedAllocator;
 public:
     static CopiedBlock* create(const PageAllocationAligned&);
+    static CopiedBlock* createNoZeroFill(const PageAllocationAligned&);
     static PageAllocationAligned destroy(CopiedBlock*);
 
     char* payload();
@@ -47,16 +48,37 @@ public:
 
 private:
     CopiedBlock(const PageAllocationAligned&);
+    void zeroFillToEnd(); // Can be called at any time to zero-fill to the end of the block.
 
     void* m_offset;
     uintptr_t m_isPinned;
 };
 
-inline CopiedBlock* CopiedBlock::create(const PageAllocationAligned& allocation)
+inline CopiedBlock* CopiedBlock::createNoZeroFill(const PageAllocationAligned& allocation)
 {
     return new(NotNull, allocation.base()) CopiedBlock(allocation);
 }
 
+inline CopiedBlock* CopiedBlock::create(const PageAllocationAligned& allocation)
+{
+    CopiedBlock* block = createNoZeroFill(allocation);
+    block->zeroFillToEnd();
+    return block;
+}
+
+inline void CopiedBlock::zeroFillToEnd()
+{
+#if USE(JSVALUE64)
+    char* offset = static_cast<char*>(m_offset);
+    memset(static_cast<void*>(offset), 0, static_cast<size_t>((reinterpret_cast<char*>(this) + m_allocation.size()) - offset));
+#else
+    JSValue emptyValue;
+    JSValue* limit = reinterpret_cast_ptr<JSValue*>(reinterpret_cast<char*>(this) + m_allocation.size());
+    for (JSValue* currentValue = reinterpret_cast<JSValue*>(m_offset); currentValue < limit; currentValue++)
+        *currentValue = emptyValue;
+#endif
+}
+
 inline PageAllocationAligned CopiedBlock::destroy(CopiedBlock* block)
 {
     PageAllocationAligned allocation;
@@ -72,15 +94,6 @@ inline CopiedBlock::CopiedBlock(const PageAllocationAligned& allocation)
     , m_isPinned(false)
 {
     ASSERT(is8ByteAligned(static_cast<void*>(m_offset)));
-#if USE(JSVALUE64)
-    char* offset = static_cast<char*>(m_offset);
-    memset(static_cast<void*>(offset), 0, static_cast<size_t>((reinterpret_cast<char*>(this) + allocation.size()) - offset));
-#else
-    JSValue emptyValue;
-    JSValue* limit = reinterpret_cast_ptr<JSValue*>(reinterpret_cast<char*>(this) + allocation.size());
-    for (JSValue* currentValue = reinterpret_cast<JSValue*>(m_offset); currentValue < limit; currentValue++)
-        *currentValue = emptyValue;
-#endif
 }
 
 inline char* CopiedBlock::payload()
diff --git a/Source/JavaScriptCore/heap/CopiedSpace.cpp b/Source/JavaScriptCore/heap/CopiedSpace.cpp
index 7f5a665df..631e829ec 100644
--- a/Source/JavaScriptCore/heap/CopiedSpace.cpp
+++ b/Source/JavaScriptCore/heap/CopiedSpace.cpp
@@ -38,6 +38,7 @@ CopiedSpace::CopiedSpace(Heap* heap)
     , m_inCopyingPhase(false)
     , m_numberOfLoanedBlocks(0)
 {
+    m_toSpaceLock.Init();
 }
 
 CopiedSpace::~CopiedSpace()
@@ -57,8 +58,7 @@ void CopiedSpace::init()
     m_toSpace = &m_blocks1;
     m_fromSpace = &m_blocks2;
     
-    if (!addNewBlock())
-        CRASH();
+    allocateBlock();
 }   
 
 CheckedBoolean CopiedSpace::tryAllocateSlowCase(size_t bytes, void** outPtr)
@@ -68,10 +68,8 @@ CheckedBoolean CopiedSpace::tryAllocateSlowCase(size_t bytes, void** outPtr)
     
     m_heap->didAllocate(m_allocator.currentCapacity());
 
-    if (!addNewBlock()) {
-        *outPtr = 0;
-        return false;
-    }
+    allocateBlock();
+
     *outPtr = m_allocator.allocate(bytes);
     ASSERT(*outPtr);
     return true;
@@ -167,8 +165,10 @@ void CopiedSpace::doneFillingBlock(CopiedBlock* block)
         return;
     }
 
+    block->zeroFillToEnd();
+
     {
-        MutexLocker locker(m_toSpaceLock);
+        SpinLockHolder locker(&m_toSpaceLock);
         m_toSpace->push(block);
         m_blockSet.add(block);
         m_blockFilter.add(reinterpret_cast<Bits>(block));
@@ -222,35 +222,12 @@ void CopiedSpace::doneCopying()
         curr = next;
     }
 
-    if (!m_toSpace->head()) {
-        if (!addNewBlock())
-            CRASH();
-    } else
+    if (!m_toSpace->head())
+        allocateBlock();
+    else
         m_allocator.resetCurrentBlock(static_cast<CopiedBlock*>(m_toSpace->head()));
 }
 
-CheckedBoolean CopiedSpace::getFreshBlock(AllocationEffort allocationEffort, CopiedBlock** outBlock)
-{
-    CopiedBlock* block = 0;
-    if (allocationEffort == AllocationMustSucceed)
-        block = CopiedBlock::create(m_heap->blockAllocator().allocate());
-    else {
-        ASSERT(allocationEffort == AllocationCanFail);
-        if (m_heap->shouldCollect())
-            m_heap->collect(Heap::DoNotSweep);
-        
-        if (!getFreshBlock(AllocationMustSucceed, &block)) {
-            *outBlock = 0;
-            ASSERT_NOT_REACHED();
-            return false;
-        }
-    }
-    ASSERT(block);
-    ASSERT(is8ByteAligned(block->m_offset));
-    *outBlock = block;
-    return true;
-}
-
 size_t CopiedSpace::size()
 {
     size_t calculatedSize = 0;
diff --git a/Source/JavaScriptCore/heap/CopiedSpace.h b/Source/JavaScriptCore/heap/CopiedSpace.h
index 27011781d..530e989da 100644
--- a/Source/JavaScriptCore/heap/CopiedSpace.h
+++ b/Source/JavaScriptCore/heap/CopiedSpace.h
@@ -37,6 +37,7 @@
 #include <wtf/PageAllocationAligned.h>
 #include <wtf/PageBlock.h>
 #include <wtf/StdLibExtras.h>
+#include <wtf/TCSpinLock.h>
 #include <wtf/ThreadingPrimitives.h>
 
 namespace JSC {
@@ -76,22 +77,20 @@ public:
     static CopiedBlock* blockFor(void*);
 
 private:
-    CheckedBoolean tryAllocateSlowCase(size_t, void**);
-    CheckedBoolean addNewBlock();
-    CheckedBoolean allocateNewBlock(CopiedBlock**);
-    
     static void* allocateFromBlock(CopiedBlock*, size_t);
+    static bool isOversize(size_t);
+    static bool fitsInBlock(CopiedBlock*, size_t);
+    static CopiedBlock* oversizeBlockFor(void* ptr);
+
+    CheckedBoolean tryAllocateSlowCase(size_t, void**);
     CheckedBoolean tryAllocateOversize(size_t, void**);
     CheckedBoolean tryReallocateOversize(void**, size_t, size_t);
     
-    static bool isOversize(size_t);
-    
-    CheckedBoolean borrowBlock(CopiedBlock**);
-    CheckedBoolean getFreshBlock(AllocationEffort, CopiedBlock**);
+    void allocateBlock();
+    CopiedBlock* allocateBlockForCopyingPhase();
+
     void doneFillingBlock(CopiedBlock*);
     void recycleBlock(CopiedBlock*);
-    static bool fitsInBlock(CopiedBlock*, size_t);
-    static CopiedBlock* oversizeBlockFor(void* ptr);
 
     Heap* m_heap;
 
@@ -100,7 +99,7 @@ private:
     TinyBloomFilter m_blockFilter;
     HashSet<CopiedBlock*> m_blockSet;
 
-    Mutex m_toSpaceLock;
+    SpinLock m_toSpaceLock;
 
     DoublyLinkedList<HeapBlock>* m_toSpace;
     DoublyLinkedList<HeapBlock>* m_fromSpace;
diff --git a/Source/JavaScriptCore/heap/CopiedSpaceInlineMethods.h b/Source/JavaScriptCore/heap/CopiedSpaceInlineMethods.h
index c97762598..1366cd8a7 100644
--- a/Source/JavaScriptCore/heap/CopiedSpaceInlineMethods.h
+++ b/Source/JavaScriptCore/heap/CopiedSpaceInlineMethods.h
@@ -84,46 +84,31 @@ inline void CopiedSpace::recycleBlock(CopiedBlock* block)
     }
 }
 
-inline CheckedBoolean CopiedSpace::borrowBlock(CopiedBlock** outBlock)
+inline CopiedBlock* CopiedSpace::allocateBlockForCopyingPhase()
 {
-    CopiedBlock* block = 0;
-    if (!getFreshBlock(AllocationMustSucceed, &block)) {
-        *outBlock = 0;
-        return false;
-    }
-
     ASSERT(m_inCopyingPhase);
-    MutexLocker locker(m_loanedBlocksLock);
-    m_numberOfLoanedBlocks++;
+    CopiedBlock* block = CopiedBlock::createNoZeroFill(m_heap->blockAllocator().allocate());
+
+    {
+        MutexLocker locker(m_loanedBlocksLock);
+        m_numberOfLoanedBlocks++;
+    }
 
     ASSERT(block->m_offset == block->payload());
-    *outBlock = block;
-    return true;
+    return block;
 }
 
-inline CheckedBoolean CopiedSpace::addNewBlock()
+inline void CopiedSpace::allocateBlock()
 {
-    CopiedBlock* block = 0;
-    if (!getFreshBlock(AllocationCanFail, &block))
-        return false;
+    if (m_heap->shouldCollect())
+        m_heap->collect(Heap::DoNotSweep);
+
+    CopiedBlock* block = CopiedBlock::create(m_heap->blockAllocator().allocate());
         
     m_toSpace->push(block);
     m_blockFilter.add(reinterpret_cast<Bits>(block));
     m_blockSet.add(block);
     m_allocator.resetCurrentBlock(block);
-    return true;
-}
-
-inline CheckedBoolean CopiedSpace::allocateNewBlock(CopiedBlock** outBlock)
-{
-    PageAllocationAligned allocation = PageAllocationAligned::allocate(HeapBlock::s_blockSize, HeapBlock::s_blockSize, OSAllocator::JSGCHeapPages);
-    if (!static_cast<bool>(allocation)) {
-        *outBlock = 0;
-        return false;
-    }
-
-    *outBlock = new (NotNull, allocation.base()) CopiedBlock(allocation);
-    return true;
 }
 
 inline bool CopiedSpace::fitsInBlock(CopiedBlock* block, size_t bytes)
diff --git a/Source/JavaScriptCore/heap/Handle.h b/Source/JavaScriptCore/heap/Handle.h
index 8bf2bd896..62f267e12 100644
--- a/Source/JavaScriptCore/heap/Handle.h
+++ b/Source/JavaScriptCore/heap/Handle.h
@@ -114,7 +114,7 @@ private:
 
 template <typename T> class Handle : public HandleBase, public HandleConverter<Handle<T>, T> {
 public:
-    template <typename A, typename B> friend class HandleConverter;
+    template <typename A, typename B> friend struct HandleConverter;
     typedef typename HandleTypes<T>::ExternalType ExternalType;
     template <typename U> Handle(Handle<U> o)
     {
diff --git a/Source/JavaScriptCore/heap/Heap.cpp b/Source/JavaScriptCore/heap/Heap.cpp
index 90c4bb72c..ef062c9ce 100644
--- a/Source/JavaScriptCore/heap/Heap.cpp
+++ b/Source/JavaScriptCore/heap/Heap.cpp
@@ -245,8 +245,6 @@ Heap::Heap(JSGlobalData* globalData, HeapType heapType)
     , m_operationInProgress(NoOperation)
     , m_objectSpace(this)
     , m_storageSpace(this)
-    , m_activityCallback(DefaultGCActivityCallback::create(this))
-    , m_sweeper(IncrementalSweeper::create(this))
     , m_machineThreads(this)
     , m_sharedData(globalData)
     , m_slotVisitor(m_sharedData)
@@ -255,6 +253,8 @@ Heap::Heap(JSGlobalData* globalData, HeapType heapType)
     , m_globalData(globalData)
     , m_lastGCLength(0)
     , m_lastCodeDiscardTime(WTF::currentTime())
+    , m_activityCallback(DefaultGCActivityCallback::create(this))
+    , m_sweeper(IncrementalSweeper::create(this))
 {
     m_storageSpace.init();
 }
@@ -403,9 +403,6 @@ inline RegisterFile& Heap::registerFile()
 void Heap::getConservativeRegisterRoots(HashSet<JSCell*>& roots)
 {
     ASSERT(isValidThreadState(m_globalData));
-    if (m_operationInProgress != NoOperation)
-        CRASH();
-    m_operationInProgress = Collection;
     ConservativeRoots registerFileRoots(&m_objectSpace.blocks(), &m_storageSpace);
     registerFile().gatherConservativeRoots(registerFileRoots);
     size_t registerFileRootCount = registerFileRoots.size();
@@ -414,7 +411,6 @@ void Heap::getConservativeRegisterRoots(HashSet<JSCell*>& roots)
         setMarked(registerRoots[i]);
         roots.add(registerRoots[i]);
     }
-    m_operationInProgress = NoOperation;
 }
 
 void Heap::markRoots(bool fullGC)
@@ -424,9 +420,10 @@ void Heap::markRoots(bool fullGC)
     COND_GCPHASE(fullGC, MarkFullRoots, MarkYoungRoots);
     UNUSED_PARAM(fullGC);
     ASSERT(isValidThreadState(m_globalData));
-    if (m_operationInProgress != NoOperation)
-        CRASH();
-    m_operationInProgress = Collection;
+
+#if ENABLE(OBJECT_MARK_LOGGING)
+    double gcStartTime = WTF::currentTime();
+#endif
 
     void* dummy;
     
@@ -491,28 +488,33 @@ void Heap::markRoots(bool fullGC)
     
         {
             GCPHASE(VisitMachineRoots);
+            MARK_LOG_ROOT(visitor, "C++ Stack");
             visitor.append(machineThreadRoots);
             visitor.donateAndDrain();
         }
         {
             GCPHASE(VisitRegisterFileRoots);
+            MARK_LOG_ROOT(visitor, "Register File");
             visitor.append(registerFileRoots);
             visitor.donateAndDrain();
         }
 #if ENABLE(DFG_JIT)
         {
             GCPHASE(VisitScratchBufferRoots);
+            MARK_LOG_ROOT(visitor, "Scratch Buffers");
             visitor.append(scratchBufferRoots);
             visitor.donateAndDrain();
         }
 #endif
         {
             GCPHASE(VisitProtectedObjects);
+            MARK_LOG_ROOT(visitor, "Protected Objects");
             markProtectedObjects(heapRootVisitor);
             visitor.donateAndDrain();
         }
         {
             GCPHASE(VisitTempSortVectors);
+            MARK_LOG_ROOT(visitor, "Temp Sort Vectors");
             markTempSortVectors(heapRootVisitor);
             visitor.donateAndDrain();
         }
@@ -520,30 +522,35 @@ void Heap::markRoots(bool fullGC)
         {
             GCPHASE(MarkingArgumentBuffers);
             if (m_markListSet && m_markListSet->size()) {
+                MARK_LOG_ROOT(visitor, "Argument Buffers");
                 MarkedArgumentBuffer::markLists(heapRootVisitor, *m_markListSet);
                 visitor.donateAndDrain();
             }
         }
         if (m_globalData->exception) {
             GCPHASE(MarkingException);
+            MARK_LOG_ROOT(visitor, "Exceptions");
             heapRootVisitor.visit(&m_globalData->exception);
             visitor.donateAndDrain();
         }
     
         {
             GCPHASE(VisitStrongHandles);
+            MARK_LOG_ROOT(visitor, "Strong Handles");
             m_handleSet.visitStrongHandles(heapRootVisitor);
             visitor.donateAndDrain();
         }
     
         {
             GCPHASE(HandleStack);
+            MARK_LOG_ROOT(visitor, "Handle Stack");
             m_handleStack.visit(heapRootVisitor);
             visitor.donateAndDrain();
         }
     
         {
             GCPHASE(TraceCodeBlocks);
+            MARK_LOG_ROOT(visitor, "Trace Code Blocks");
             m_dfgCodeBlocks.traceMarkedCodeBlocks(visitor);
             visitor.donateAndDrain();
         }
@@ -560,6 +567,7 @@ void Heap::markRoots(bool fullGC)
     // the liveness of the rest of the object graph.
     {
         GCPHASE(VisitingLiveWeakHandles);
+        MARK_LOG_ROOT(visitor, "Live Weak Handles");
         while (true) {
             m_objectSpace.visitWeakSets(heapRootVisitor);
             harvestWeakReferences();
@@ -578,11 +586,21 @@ void Heap::markRoots(bool fullGC)
     GCCOUNTER(VisitedValueCount, visitor.visitCount());
 
     visitor.doneCopying();
+#if ENABLE(OBJECT_MARK_LOGGING)
+    size_t visitCount = visitor.visitCount();
+#if ENABLE(PARALLEL_GC)
+    visitCount += m_sharedData.childVisitCount();
+#endif
+    MARK_LOG_MESSAGE2("\nNumber of live Objects after full GC %lu, took %.6f secs\n", visitCount, WTF::currentTime() - gcStartTime);
+#endif
+
     visitor.reset();
     m_sharedData.reset();
+#if ENABLE(PARALLEL_GC)
+    m_sharedData.resetChildren();
+#endif
     m_storageSpace.doneCopying();
 
-    m_operationInProgress = NoOperation;
 }
 
 size_t Heap::objectCount()
@@ -625,15 +643,38 @@ PassOwnPtr<TypeCountSet> Heap::objectTypeCounts()
     return m_objectSpace.forEachCell<RecordType>();
 }
 
-void Heap::discardAllCompiledCode()
+void Heap::deleteAllCompiledCode()
 {
-    // If JavaScript is running, it's not safe to recompile, since we'll end
-    // up throwing away code that is live on the stack.
+    // If JavaScript is running, it's not safe to delete code, since we'll end
+    // up deleting code that is live on the stack.
     if (m_globalData->dynamicGlobalObject)
         return;
 
-    for (FunctionExecutable* current = m_functions.head(); current; current = current->next())
-        current->discardCode();
+    for (ExecutableBase* current = m_compiledCode.head(); current; current = current->next()) {
+        if (!current->isFunctionExecutable())
+            continue;
+        static_cast<FunctionExecutable*>(current)->clearCodeIfNotCompiling();
+    }
+
+    m_dfgCodeBlocks.clearMarks();
+    m_dfgCodeBlocks.deleteUnmarkedJettisonedCodeBlocks();
+}
+
+void Heap::deleteUnmarkedCompiledCode()
+{
+    ExecutableBase* next;
+    for (ExecutableBase* current = m_compiledCode.head(); current; current = next) {
+        next = current->next();
+        if (isMarked(current))
+            continue;
+
+        // We do this because executable memory is limited on some platforms and because
+        // CodeBlock requires eager finalization.
+        ExecutableBase::clearCodeVirtual(current);
+        m_compiledCode.remove(current);
+    }
+
+    m_dfgCodeBlocks.deleteUnmarkedJettisonedCodeBlocks();
 }
 
 void Heap::collectAllGarbage()
@@ -654,12 +695,15 @@ void Heap::collect(SweepToggle sweepToggle)
     ASSERT(globalData()->identifierTable == wtfThreadData().currentIdentifierTable());
     ASSERT(m_isSafeToCollect);
     JAVASCRIPTCORE_GC_BEGIN();
+    if (m_operationInProgress != NoOperation)
+        CRASH();
+    m_operationInProgress = Collection;
 
     m_activityCallback->willCollect();
 
     double lastGCStartTime = WTF::currentTime();
     if (lastGCStartTime - m_lastCodeDiscardTime > minute) {
-        discardAllCompiledCode();
+        deleteAllCompiledCode();
         m_lastCodeDiscardTime = WTF::currentTime();
     }
 
@@ -682,22 +726,21 @@ void Heap::collect(SweepToggle sweepToggle)
         m_objectSpace.reapWeakSets();
     }
 
+    JAVASCRIPTCORE_GC_MARKED();
+
     {
         GCPHASE(FinalizeUnconditionalFinalizers);
         finalizeUnconditionalFinalizers();
     }
 
     {
-        GCPHASE(FinalizeWeakHandles);
-        m_objectSpace.sweepWeakSets();
+        GCPHASE(finalizeSmallStrings);
         m_globalData->smallStrings.finalizeSmallStrings();
     }
-    
-    JAVASCRIPTCORE_GC_MARKED();
 
     {
         GCPHASE(DeleteCodeBlocks);
-        m_dfgCodeBlocks.deleteUnmarkedJettisonedCodeBlocks();
+        deleteUnmarkedCompiledCode();
     }
 
     if (sweepToggle == DoSweep) {
@@ -728,6 +771,9 @@ void Heap::collect(SweepToggle sweepToggle)
     m_bytesAllocated = 0;
     double lastGCEndTime = WTF::currentTime();
     m_lastGCLength = lastGCEndTime - lastGCStartTime;
+    if (m_operationInProgress != Collection)
+        CRASH();
+    m_operationInProgress = NoOperation;
     JAVASCRIPTCORE_GC_END();
 }
 
@@ -784,14 +830,9 @@ void Heap::FinalizerOwner::finalize(Handle<Unknown> handle, void* context)
     WeakSet::deallocate(WeakImpl::asWeakImpl(slot));
 }
 
-void Heap::addFunctionExecutable(FunctionExecutable* executable)
-{
-    m_functions.append(executable);
-}
-
-void Heap::removeFunctionExecutable(FunctionExecutable* executable)
+void Heap::addCompiledCode(ExecutableBase* executable)
 {
-    m_functions.remove(executable);
+    m_compiledCode.append(executable);
 }
 
 } // namespace JSC
diff --git a/Source/JavaScriptCore/heap/Heap.h b/Source/JavaScriptCore/heap/Heap.h
index 296447d7b..91c3aa58f 100644
--- a/Source/JavaScriptCore/heap/Heap.h
+++ b/Source/JavaScriptCore/heap/Heap.h
@@ -42,7 +42,7 @@ namespace JSC {
 
     class CopiedSpace;
     class CodeBlock;
-    class FunctionExecutable;
+    class ExecutableBase;
     class GCActivityCallback;
     class GlobalCodeBlock;
     class Heap;
@@ -85,6 +85,7 @@ namespace JSC {
         static bool testAndSetMarked(const void*);
         static void setMarked(const void*);
 
+        static bool isWriteBarrierEnabled();
         static void writeBarrier(const JSCell*, JSValue);
         static void writeBarrier(const JSCell*, JSCell*);
         static uint8_t* addressOfCardFor(JSCell*);
@@ -115,10 +116,10 @@ namespace JSC {
 
         typedef void (*Finalizer)(JSCell*);
         JS_EXPORT_PRIVATE void addFinalizer(JSCell*, Finalizer);
-        void addFunctionExecutable(FunctionExecutable*);
-        void removeFunctionExecutable(FunctionExecutable*);
+        void addCompiledCode(ExecutableBase*);
 
         void notifyIsSafeToCollect() { m_isSafeToCollect = true; }
+        bool isSafeToCollect() const { return m_isSafeToCollect; }
 
         JS_EXPORT_PRIVATE void collectAllGarbage();
         enum SweepToggle { DoNotSweep, DoSweep };
@@ -158,7 +159,7 @@ namespace JSC {
         double lastGCLength() { return m_lastGCLength; }
         void increaseLastGCLength(double amount) { m_lastGCLength += amount; }
 
-        JS_EXPORT_PRIVATE void discardAllCompiledCode();
+        JS_EXPORT_PRIVATE void deleteAllCompiledCode();
 
         void didAllocate(size_t);
         void didAbandon(size_t);
@@ -193,6 +194,7 @@ namespace JSC {
         void markTempSortVectors(HeapRootVisitor&);
         void harvestWeakReferences();
         void finalizeUnconditionalFinalizers();
+        void deleteUnmarkedCompiledCode();
         
         RegisterFile& registerFile();
         BlockAllocator& blockAllocator();
@@ -219,9 +221,6 @@ namespace JSC {
         Vector<Vector<ValueStringPair>* > m_tempSortingVectors;
         OwnPtr<HashSet<MarkedArgumentBuffer*> > m_markListSet;
 
-        OwnPtr<GCActivityCallback> m_activityCallback;
-        OwnPtr<IncrementalSweeper> m_sweeper;
-        
         MachineThreads m_machineThreads;
         
         MarkStackThreadSharedData m_sharedData;
@@ -238,7 +237,10 @@ namespace JSC {
         double m_lastGCLength;
         double m_lastCodeDiscardTime;
 
-        DoublyLinkedList<FunctionExecutable> m_functions;
+        OwnPtr<GCActivityCallback> m_activityCallback;
+        OwnPtr<IncrementalSweeper> m_sweeper;
+        
+        DoublyLinkedList<ExecutableBase> m_compiledCode;
     };
 
     inline bool Heap::shouldCollect()
@@ -282,6 +284,15 @@ namespace JSC {
         MarkedBlock::blockFor(cell)->setMarked(cell);
     }
 
+    inline bool Heap::isWriteBarrierEnabled()
+    {
+#if ENABLE(GGC) || ENABLE(WRITE_BARRIER_PROFILING)
+        return true;
+#else
+        return false;
+#endif
+    }
+
 #if ENABLE(GGC)
     inline uint8_t* Heap::addressOfCardFor(JSCell* cell)
     {
diff --git a/Source/JavaScriptCore/heap/HeapTimer.cpp b/Source/JavaScriptCore/heap/HeapTimer.cpp
new file mode 100644
index 000000000..bc42032f5
--- /dev/null
+++ b/Source/JavaScriptCore/heap/HeapTimer.cpp
@@ -0,0 +1,71 @@
+#include "config.h"
+#include "HeapTimer.h"
+
+#include <wtf/Threading.h>
+
+namespace JSC {
+
+#if USE(CF)
+    
+const CFTimeInterval HeapTimer::s_decade = 60 * 60 * 24 * 365 * 10;
+
+HeapTimer::HeapTimer(JSGlobalData* globalData, CFRunLoopRef runLoop)
+    : m_globalData(globalData)
+    , m_runLoop(runLoop)
+{
+    memset(&m_context, 0, sizeof(CFRunLoopTimerContext));
+    m_context.info = this;
+    m_timer.adoptCF(CFRunLoopTimerCreate(0, s_decade, s_decade, 0, 0, HeapTimer::timerDidFire, &m_context));
+    CFRunLoopAddTimer(m_runLoop.get(), m_timer.get(), kCFRunLoopCommonModes);
+}
+
+HeapTimer::~HeapTimer()
+{
+    invalidate();
+}
+
+void HeapTimer::synchronize()
+{
+    if (CFRunLoopGetCurrent() == m_runLoop.get())
+        return;
+    CFRunLoopRemoveTimer(m_runLoop.get(), m_timer.get(), kCFRunLoopCommonModes);
+    m_runLoop = CFRunLoopGetCurrent();
+    CFRunLoopAddTimer(m_runLoop.get(), m_timer.get(), kCFRunLoopCommonModes);
+}
+
+void HeapTimer::invalidate()
+{
+    CFRunLoopRemoveTimer(m_runLoop.get(), m_timer.get(), kCFRunLoopCommonModes);
+    CFRunLoopTimerInvalidate(m_timer.get());
+}
+
+void HeapTimer::timerDidFire(CFRunLoopTimerRef, void* info)
+{
+    HeapTimer* agent = static_cast<HeapTimer*>(info);
+    agent->doWork();
+}
+
+#else
+    
+HeapTimer::HeapTimer(JSGlobalData* globalData)
+    : m_globalData(globalData)
+{
+}
+
+HeapTimer::~HeapTimer()
+{
+}
+
+void HeapTimer::synchronize()
+{
+}
+
+void HeapTimer::invalidate()
+{
+}
+
+
+#endif
+    
+
+} // namespace JSC
diff --git a/Source/JavaScriptCore/heap/HeapTimer.h b/Source/JavaScriptCore/heap/HeapTimer.h
new file mode 100644
index 000000000..ccd6ba8c5
--- /dev/null
+++ b/Source/JavaScriptCore/heap/HeapTimer.h
@@ -0,0 +1,45 @@
+#ifndef HeapTimer_h
+#define HeapTimer_h
+
+#include <wtf/RetainPtr.h>
+
+#if USE(CF)
+#include <CoreFoundation/CoreFoundation.h>
+#endif
+
+namespace JSC {
+
+class JSGlobalData;
+    
+class HeapTimer {
+public:
+#if USE(CF)
+    HeapTimer(JSGlobalData*, CFRunLoopRef);
+    static void timerDidFire(CFRunLoopTimerRef, void*);
+#else
+    HeapTimer(JSGlobalData*);
+#endif
+    
+    virtual ~HeapTimer();
+    
+    virtual void synchronize();
+    virtual void doWork() = 0;
+    
+protected:
+    JSGlobalData* m_globalData;
+
+#if USE(CF)
+    static const CFTimeInterval s_decade;
+
+    RetainPtr<CFRunLoopTimerRef> m_timer;
+    RetainPtr<CFRunLoopRef> m_runLoop;
+    CFRunLoopTimerContext m_context;
+#endif
+    
+private:
+    void invalidate();
+};
+
+} // namespace JSC
+
+#endif
diff --git a/Source/JavaScriptCore/heap/IncrementalSweeper.cpp b/Source/JavaScriptCore/heap/IncrementalSweeper.cpp
index 08a9f6c73..0d0116f42 100644
--- a/Source/JavaScriptCore/heap/IncrementalSweeper.cpp
+++ b/Source/JavaScriptCore/heap/IncrementalSweeper.cpp
@@ -14,33 +14,20 @@ namespace JSC {
 
 #if USE(CF)
 
-static const CFTimeInterval decade = 60 * 60 * 24 * 365 * 10;
 static const CFTimeInterval sweepTimeSlicePerBlock = 0.01;
 static const CFTimeInterval sweepTimeMultiplier = 1.0 / sweepTimeSlicePerBlock;
  
-void IncrementalSweeper::timerDidFire(CFRunLoopTimerRef, void* info)
+void IncrementalSweeper::doWork()
 {
-    Heap* heap = static_cast<Heap*>(info);
-    APIEntryShim shim(heap->globalData());
-    heap->sweeper()->doSweep(WTF::monotonicallyIncreasingTime());
+    APIEntryShim shim(m_globalData);
+    doSweep(WTF::monotonicallyIncreasingTime());
 }
     
 IncrementalSweeper::IncrementalSweeper(Heap* heap, CFRunLoopRef runLoop)
-    : m_heap(heap)
+    : HeapTimer(heap->globalData(), runLoop)
     , m_currentBlockToSweepIndex(0)
     , m_lengthOfLastSweepIncrement(0.0)
 {
-    memset(&m_context, 0, sizeof(CFRunLoopTimerContext));
-    m_context.info = m_heap;
-    m_runLoop = runLoop;
-    m_timer.adoptCF(CFRunLoopTimerCreate(0, CFAbsoluteTimeGetCurrent(), decade, 0, 0, &timerDidFire, &m_context));
-    CFRunLoopAddTimer(m_runLoop.get(), m_timer.get(), kCFRunLoopCommonModes);
-}
-
-IncrementalSweeper::~IncrementalSweeper()
-{
-    CFRunLoopRemoveTimer(m_runLoop.get(), m_timer.get(), kCFRunLoopCommonModes);
-    CFRunLoopTimerInvalidate(m_timer.get());
 }
 
 PassOwnPtr<IncrementalSweeper> IncrementalSweeper::create(Heap* heap)
@@ -55,7 +42,7 @@ void IncrementalSweeper::scheduleTimer()
 
 void IncrementalSweeper::cancelTimer()
 {
-    CFRunLoopTimerSetNextFireDate(m_timer.get(), CFAbsoluteTimeGetCurrent() + decade);
+    CFRunLoopTimerSetNextFireDate(m_timer.get(), CFAbsoluteTimeGetCurrent() + s_decade);
 }
 
 void IncrementalSweeper::doSweep(double sweepBeginTime)
@@ -85,17 +72,18 @@ void IncrementalSweeper::startSweeping(const HashSet<MarkedBlock*>& blockSnapsho
 
 #else
 
-IncrementalSweeper::IncrementalSweeper()
+IncrementalSweeper::IncrementalSweeper(JSGlobalData* globalData)
+    : HeapTimer(globalData)
 {
 }
 
-IncrementalSweeper::~IncrementalSweeper()
+void IncrementalSweeper::doWork()
 {
 }
 
-PassOwnPtr<IncrementalSweeper> IncrementalSweeper::create(Heap*)
+PassOwnPtr<IncrementalSweeper> IncrementalSweeper::create(Heap* heap)
 {
-    return adoptPtr(new IncrementalSweeper());
+    return adoptPtr(new IncrementalSweeper(heap->globalData()));
 }
 
 void IncrementalSweeper::startSweeping(const HashSet<MarkedBlock*>&)
diff --git a/Source/JavaScriptCore/heap/IncrementalSweeper.h b/Source/JavaScriptCore/heap/IncrementalSweeper.h
index 48f040409..80d674ca9 100644
--- a/Source/JavaScriptCore/heap/IncrementalSweeper.h
+++ b/Source/JavaScriptCore/heap/IncrementalSweeper.h
@@ -1,47 +1,37 @@
 #ifndef IncrementalSweeper_h
 #define IncrementalSweeper_h
 
+#include "HeapTimer.h"
 #include "MarkedBlock.h"
 #include <wtf/HashSet.h>
 #include <wtf/PassOwnPtr.h>
 #include <wtf/RetainPtr.h>
 #include <wtf/Vector.h>
 
-#if USE(CF)
-#include <CoreFoundation/CoreFoundation.h>
-#endif
-
 namespace JSC {
 
 class Heap;
     
-class IncrementalSweeper {
+class IncrementalSweeper : public HeapTimer {
 public:
-    ~IncrementalSweeper();
-
     static PassOwnPtr<IncrementalSweeper> create(Heap*);
     void startSweeping(const HashSet<MarkedBlock*>& blockSnapshot);
+    virtual void doWork();
 
 private:
 #if USE(CF)
     IncrementalSweeper(Heap*, CFRunLoopRef);
     
-    static void timerDidFire(CFRunLoopTimerRef, void*);
     void doSweep(double startTime);
     void scheduleTimer();
     void cancelTimer();
     
-    Heap* m_heap;
     unsigned m_currentBlockToSweepIndex;
-    RetainPtr<CFRunLoopTimerRef> m_timer;
-    RetainPtr<CFRunLoopRef> m_runLoop;
-    CFRunLoopTimerContext m_context;
-   
     double m_lengthOfLastSweepIncrement; 
     Vector<MarkedBlock*> m_blocksToSweep;
 #else
     
-    IncrementalSweeper();
+    IncrementalSweeper(JSGlobalData*);
     
 #endif
 };
diff --git a/Source/JavaScriptCore/heap/ListableHandler.h b/Source/JavaScriptCore/heap/ListableHandler.h
index 41f18fbce..2cb03251f 100644
--- a/Source/JavaScriptCore/heap/ListableHandler.h
+++ b/Source/JavaScriptCore/heap/ListableHandler.h
@@ -24,6 +24,7 @@
 #include <wtf/Locker.h>
 #include <wtf/Noncopyable.h>
 #include <wtf/ThreadingPrimitives.h>
+#include <wtf/TCSpinLock.h>
 
 namespace JSC {
 
@@ -60,15 +61,12 @@ private:
         List()
             : m_first(0)
         {
+            m_lock.Init();
         }
         
         void addThreadSafe(T* handler)
         {
-            // NOTE: If we ever want this to be faster, we could turn it into
-            // a CAS loop, since this is a singly-linked-list that, in parallel
-            // tracing mode, can only grow. I.e. we don't have to worry about
-            // any ABA problems.
-            MutexLocker locker(m_lock);
+            SpinLockHolder locker(&m_lock);
             addNotThreadSafe(handler);
         }
         
@@ -106,7 +104,7 @@ private:
             m_first = handler;
         }
         
-        Mutex m_lock;
+        SpinLock m_lock;
         T* m_first;
     };
     
diff --git a/Source/JavaScriptCore/heap/MarkStack.cpp b/Source/JavaScriptCore/heap/MarkStack.cpp
index 678f1cb45..3eb02c4e8 100644
--- a/Source/JavaScriptCore/heap/MarkStack.cpp
+++ b/Source/JavaScriptCore/heap/MarkStack.cpp
@@ -36,6 +36,7 @@
 #include "JSObject.h"
 #include "ScopeChain.h"
 #include "Structure.h"
+#include "UString.h"
 #include "WriteBarrier.h"
 #include <wtf/DataLog.h>
 #include <wtf/MainThread.h>
@@ -45,6 +46,7 @@ namespace JSC {
 MarkStackSegmentAllocator::MarkStackSegmentAllocator()
     : m_nextFreeSegment(0)
 {
+    m_lock.Init();
 }
 
 MarkStackSegmentAllocator::~MarkStackSegmentAllocator()
@@ -55,7 +57,7 @@ MarkStackSegmentAllocator::~MarkStackSegmentAllocator()
 MarkStackSegment* MarkStackSegmentAllocator::allocate()
 {
     {
-        MutexLocker locker(m_lock);
+        SpinLockHolder locker(&m_lock);
         if (m_nextFreeSegment) {
             MarkStackSegment* result = m_nextFreeSegment;
             m_nextFreeSegment = result->m_previous;
@@ -68,7 +70,7 @@ MarkStackSegment* MarkStackSegmentAllocator::allocate()
 
 void MarkStackSegmentAllocator::release(MarkStackSegment* segment)
 {
-    MutexLocker locker(m_lock);
+    SpinLockHolder locker(&m_lock);
     segment->m_previous = m_nextFreeSegment;
     m_nextFreeSegment = segment;
 }
@@ -77,7 +79,7 @@ void MarkStackSegmentAllocator::shrinkReserve()
 {
     MarkStackSegment* segments;
     {
-        MutexLocker locker(m_lock);
+        SpinLockHolder locker(&m_lock);
         segments = m_nextFreeSegment;
         m_nextFreeSegment = 0;
     }
@@ -141,23 +143,31 @@ bool MarkStackArray::refill()
     return true;
 }
 
-bool MarkStackArray::donateSomeCellsTo(MarkStackArray& other)
+void MarkStackArray::donateSomeCellsTo(MarkStackArray& other)
 {
+    // Try to donate about 1 / 2 of our cells. To reduce copying costs,
+    // we prefer donating whole segments over donating individual cells,
+    // even if this skews away from our 1 / 2 target.
+
     ASSERT(m_segmentCapacity == other.m_segmentCapacity);
+
+    size_t segmentsToDonate = (m_numberOfPreviousSegments + 2 - 1) / 2; // Round up to donate 1 / 1 previous segments.
+
+    if (!segmentsToDonate) {
+        size_t cellsToDonate = m_top / 2; // Round down to donate 0 / 1 cells.
+        while (cellsToDonate--) {
+            ASSERT(m_top);
+            other.append(removeLast());
+        }
+        return;
+    }
+
     validatePrevious();
     other.validatePrevious();
-        
-    // Fast check: see if the other mark stack already has enough segments.
-    if (other.m_numberOfPreviousSegments + 1 >= Options::maximumNumberOfSharedSegments)
-        return false;
-        
-    size_t numberOfCellsToKeep = Options::minimumNumberOfCellsToKeep;
-    ASSERT(m_top > numberOfCellsToKeep || m_topSegment->m_previous);
-        
-    // Looks like we should donate! Give the other mark stack all of our
-    // previous segments, and then top it off.
+
     MarkStackSegment* previous = m_topSegment->m_previous;
-    while (previous) {
+    while (segmentsToDonate--) {
+        ASSERT(previous);
         ASSERT(m_numberOfPreviousSegments);
 
         MarkStackSegment* current = previous;
@@ -169,23 +179,18 @@ bool MarkStackArray::donateSomeCellsTo(MarkStackArray& other)
         m_numberOfPreviousSegments--;
         other.m_numberOfPreviousSegments++;
     }
-    ASSERT(!m_numberOfPreviousSegments);
-    m_topSegment->m_previous = 0;
+    m_topSegment->m_previous = previous;
+
     validatePrevious();
     other.validatePrevious();
-        
-    // Now top off. We want to keep at a minimum numberOfCellsToKeep, but if
-    // we really have a lot of work, we give up half.
-    if (m_top > numberOfCellsToKeep * 2)
-        numberOfCellsToKeep = m_top / 2;
-    while (m_top > numberOfCellsToKeep)
-        other.append(removeLast());
-        
-    return true;
 }
 
-void MarkStackArray::stealSomeCellsFrom(MarkStackArray& other)
+void MarkStackArray::stealSomeCellsFrom(MarkStackArray& other, size_t idleThreadCount)
 {
+    // Try to steal 1 / Nth of the shared array, where N is the number of idle threads.
+    // To reduce copying costs, we prefer stealing a whole segment over stealing
+    // individual cells, even if this skews away from our 1 / N target.
+
     ASSERT(m_segmentCapacity == other.m_segmentCapacity);
     validatePrevious();
     other.validatePrevious();
@@ -210,28 +215,42 @@ void MarkStackArray::stealSomeCellsFrom(MarkStackArray& other)
         other.validatePrevious();
         return;
     }
-        
-    // Otherwise drain 1/Nth of the shared array where N is the number of
-    // workers, or Options::minimumNumberOfCellsToKeep, whichever is bigger.
-    size_t numberOfCellsToSteal = std::max((size_t)Options::minimumNumberOfCellsToKeep, other.size() / Options::numberOfGCMarkers);
+
+    size_t numberOfCellsToSteal = (other.size() + idleThreadCount - 1) / idleThreadCount; // Round up to steal 1 / 1.
     while (numberOfCellsToSteal-- > 0 && other.canRemoveLast())
         append(other.removeLast());
 }
 
 #if ENABLE(PARALLEL_GC)
-void MarkStackThreadSharedData::markingThreadMain()
+void MarkStackThreadSharedData::resetChildren()
+{
+    for (unsigned i = 0; i < m_markingThreadsMarkStack.size(); ++i)
+       m_markingThreadsMarkStack[i]->reset();
+}   
+
+size_t MarkStackThreadSharedData::childVisitCount()
+{       
+    unsigned long result = 0;
+    for (unsigned i = 0; i < m_markingThreadsMarkStack.size(); ++i)
+        result += m_markingThreadsMarkStack[i]->visitCount();
+    return result;
+}
+
+void MarkStackThreadSharedData::markingThreadMain(SlotVisitor* slotVisitor)
 {
     WTF::registerGCThread();
     {
-        SlotVisitor slotVisitor(*this);
-        ParallelModeEnabler enabler(slotVisitor);
-        slotVisitor.drainFromShared(SlotVisitor::SlaveDrain);
+        ParallelModeEnabler enabler(*slotVisitor);
+        slotVisitor->drainFromShared(SlotVisitor::SlaveDrain);
     }
+    delete slotVisitor;
 }
 
-void MarkStackThreadSharedData::markingThreadStartFunc(void* shared)
-{
-    static_cast<MarkStackThreadSharedData*>(shared)->markingThreadMain();
+void MarkStackThreadSharedData::markingThreadStartFunc(void* myVisitor)
+{               
+    SlotVisitor* slotVisitor = static_cast<SlotVisitor*>(myVisitor);
+
+    slotVisitor->sharedData().markingThreadMain(slotVisitor);
 }
 #endif
 
@@ -244,7 +263,9 @@ MarkStackThreadSharedData::MarkStackThreadSharedData(JSGlobalData* globalData)
 {
 #if ENABLE(PARALLEL_GC)
     for (unsigned i = 1; i < Options::numberOfGCMarkers; ++i) {
-        m_markingThreads.append(createThread(markingThreadStartFunc, this, "JavaScriptCore::Marking"));
+        SlotVisitor* slotVisitor = new SlotVisitor(*this);
+        m_markingThreadsMarkStack.append(slotVisitor);
+        m_markingThreads.append(createThread(markingThreadStartFunc, slotVisitor, "JavaScriptCore::Marking"));
         ASSERT(m_markingThreads.last());
     }
 #endif
@@ -276,7 +297,6 @@ void MarkStackThreadSharedData::reset()
 #else
     ASSERT(m_opaqueRoots.isEmpty());
 #endif
-    
     m_weakReferenceHarvesters.removeAll();
 }
 
@@ -325,18 +345,31 @@ ALWAYS_INLINE static void visitChildren(SlotVisitor& visitor, const JSCell* cell
     cell->methodTable()->visitChildren(const_cast<JSCell*>(cell), visitor);
 }
 
-void SlotVisitor::donateSlow()
+void SlotVisitor::donateKnownParallel()
 {
-    // Refuse to donate if shared has more entries than I do.
-    if (m_shared.m_sharedMarkStack.size() > m_stack.size())
+    // NOTE: Because we re-try often, we can afford to be conservative, and
+    // assume that donating is not profitable.
+
+    // Avoid locking when a thread reaches a dead end in the object graph.
+    if (m_stack.size() < 2)
         return;
-    MutexLocker locker(m_shared.m_markingLock);
-    if (m_stack.donateSomeCellsTo(m_shared.m_sharedMarkStack)) {
-        // Only wake up threads if the shared stack is big enough; otherwise assume that
-        // it's more profitable for us to just scan this ourselves later.
-        if (m_shared.m_sharedMarkStack.size() >= Options::sharedStackWakeupThreshold)
-            m_shared.m_markingCondition.broadcast();
-    }
+
+    // If there's already some shared work queued up, be conservative and assume
+    // that donating more is not profitable.
+    if (m_shared.m_sharedMarkStack.size())
+        return;
+
+    // If we're contending on the lock, be conservative and assume that another
+    // thread is already donating.
+    MutexTryLocker locker(m_shared.m_markingLock);
+    if (!locker.locked())
+        return;
+
+    // Otherwise, assume that a thread will go idle soon, and donate.
+    m_stack.donateSomeCellsTo(m_shared.m_sharedMarkStack);
+
+    if (m_shared.m_numberOfActiveParallelMarkers < Options::numberOfGCMarkers)
+        m_shared.m_markingCondition.broadcast();
 }
 
 void SlotVisitor::drain()
@@ -436,7 +469,8 @@ void SlotVisitor::drainFromShared(SharedDrainMode sharedDrainMode)
                 }
             }
            
-            m_stack.stealSomeCellsFrom(m_shared.m_sharedMarkStack);
+            size_t idleThreadCount = Options::numberOfGCMarkers - m_shared.m_numberOfActiveParallelMarkers;
+            m_stack.stealSomeCellsFrom(m_shared.m_sharedMarkStack, idleThreadCount);
             m_shared.m_numberOfActiveParallelMarkers++;
         }
         
@@ -461,8 +495,7 @@ void MarkStack::mergeOpaqueRoots()
 void SlotVisitor::startCopying()
 {
     ASSERT(!m_copyBlock);
-    if (!m_shared.m_copiedSpace->borrowBlock(&m_copyBlock))
-        CRASH();
+    m_copyBlock = m_shared.m_copiedSpace->allocateBlockForCopyingPhase();
 }    
 
 void* SlotVisitor::allocateNewSpace(void* ptr, size_t bytes)
@@ -483,8 +516,7 @@ void* SlotVisitor::allocateNewSpace(void* ptr, size_t bytes)
         // We don't need to lock across these two calls because the master thread won't 
         // call doneCopying() because this thread is considered active.
         m_shared.m_copiedSpace->doneFillingBlock(m_copyBlock);
-        if (!m_shared.m_copiedSpace->borrowBlock(&m_copyBlock))
-            CRASH();
+        m_copyBlock = m_shared.m_copiedSpace->allocateBlockForCopyingPhase();
     }
     return CopiedSpace::allocateFromBlock(m_copyBlock, bytes);
 }
diff --git a/Source/JavaScriptCore/heap/MarkStack.h b/Source/JavaScriptCore/heap/MarkStack.h
index 0695b1b32..c3065e7d6 100644
--- a/Source/JavaScriptCore/heap/MarkStack.h
+++ b/Source/JavaScriptCore/heap/MarkStack.h
@@ -28,18 +28,46 @@
 
 #include "CopiedSpace.h"
 #include "HandleTypes.h"
-#include "Options.h"
 #include "JSValue.h"
+#include "Options.h"
 #include "Register.h"
 #include "UnconditionalFinalizer.h"
 #include "VTableSpectrum.h"
 #include "WeakReferenceHarvester.h"
+#include <wtf/DataLog.h>
+#include <wtf/Forward.h>
 #include <wtf/HashMap.h>
 #include <wtf/HashSet.h>
-#include <wtf/Vector.h>
 #include <wtf/Noncopyable.h>
 #include <wtf/OSAllocator.h>
 #include <wtf/PageBlock.h>
+#include <wtf/TCSpinLock.h>
+#include <wtf/text/StringHash.h>
+#include <wtf/Vector.h>
+
+#if ENABLE(OBJECT_MARK_LOGGING)
+#define MARK_LOG_MESSAGE0(message) dataLog(message)
+#define MARK_LOG_MESSAGE1(message, arg1) dataLog(message, arg1)
+#define MARK_LOG_MESSAGE2(message, arg1, arg2) dataLog(message, arg1, arg2)
+#define MARK_LOG_ROOT(visitor, rootName) \
+    dataLog("\n%s: ", rootName); \
+    (visitor).resetChildCount()
+#define MARK_LOG_PARENT(visitor, parent) \
+    dataLog("\n%p (%s): ", parent, parent->className() ? parent->className() : "unknown"); \
+    (visitor).resetChildCount()
+#define MARK_LOG_CHILD(visitor, child) \
+    if ((visitor).childCount()) \
+    dataLogString(", "); \
+    dataLog("%p", child); \
+    (visitor).incrementChildCount()
+#else
+#define MARK_LOG_MESSAGE0(message) do { } while (false)
+#define MARK_LOG_MESSAGE1(message, arg1) do { } while (false)
+#define MARK_LOG_MESSAGE2(message, arg1, arg2) do { } while (false)
+#define MARK_LOG_ROOT(visitor, rootName) do { } while (false)
+#define MARK_LOG_PARENT(visitor, parent) do { } while (false)
+#define MARK_LOG_CHILD(visitor, child) do { } while (false)
+#endif
 
 namespace JSC {
 
@@ -85,7 +113,7 @@ namespace JSC {
         void shrinkReserve();
         
     private:
-        Mutex m_lock;
+        SpinLock m_lock;
         MarkStackSegment* m_nextFreeSegment;
     };
 
@@ -102,10 +130,9 @@ namespace JSC {
         
         bool isEmpty();
         
-        bool canDonateSomeCells(); // Returns false if you should definitely not call doanteSomeCellsTo().
-        bool donateSomeCellsTo(MarkStackArray& other); // Returns true if some cells were donated.
+        void donateSomeCellsTo(MarkStackArray& other);
         
-        void stealSomeCellsFrom(MarkStackArray& other);
+        void stealSomeCellsFrom(MarkStackArray& other, size_t idleThreadCount);
 
         size_t size();
 
@@ -171,13 +198,19 @@ namespace JSC {
         ~MarkStackThreadSharedData();
         
         void reset();
+
+#if ENABLE(PARALLEL_GC)
+        void resetChildren();
+        size_t childVisitCount();
+        size_t childDupStrings();
+#endif
     
     private:
         friend class MarkStack;
         friend class SlotVisitor;
 
 #if ENABLE(PARALLEL_GC)
-        void markingThreadMain();
+        void markingThreadMain(SlotVisitor*);
         static void markingThreadStartFunc(void* heap);
 #endif
 
@@ -187,6 +220,7 @@ namespace JSC {
         MarkStackSegmentAllocator m_segmentAllocator;
         
         Vector<ThreadIdentifier> m_markingThreads;
+        Vector<MarkStack*> m_markingThreadsMarkStack;
         
         Mutex m_markingLock;
         ThreadCondition m_markingCondition;
@@ -221,7 +255,8 @@ namespace JSC {
         void addOpaqueRoot(void*);
         bool containsOpaqueRoot(void*);
         int opaqueRootCount();
-        
+
+        MarkStackThreadSharedData& sharedData() { return m_shared; }
         bool isEmpty() { return m_stack.isEmpty(); }
 
         void reset();
@@ -242,6 +277,12 @@ namespace JSC {
             m_shared.m_unconditionalFinalizers.addThreadSafe(unconditionalFinalizer);
         }
 
+#if ENABLE(OBJECT_MARK_LOGGING)
+        inline void resetChildCount() { m_logChildCount = 0; }
+        inline unsigned childCount() { return m_logChildCount; }
+        inline void incrementChildCount() { m_logChildCount++; }
+#endif
+
     protected:
         JS_EXPORT_PRIVATE static void validate(JSCell*);
 
@@ -283,6 +324,10 @@ namespace JSC {
         bool m_isInParallelMode;
         
         MarkStackThreadSharedData& m_shared;
+
+#if ENABLE(OBJECT_MARK_LOGGING)
+        unsigned m_logChildCount;
+#endif
     };
 
     inline MarkStack::MarkStack(MarkStackThreadSharedData& shared)
@@ -369,22 +414,6 @@ namespace JSC {
         return true;
     }
 
-    inline bool MarkStackArray::canDonateSomeCells()
-    {
-        size_t numberOfCellsToKeep = Options::minimumNumberOfCellsToKeep;
-        // Another check: see if we have enough cells to warrant donation.
-        if (m_top <= numberOfCellsToKeep) {
-            // This indicates that we might not want to donate anything; check if we have
-            // another full segment. If not, then don't donate.
-            if (!m_topSegment->m_previous)
-                return false;
-            
-            ASSERT(m_topSegment->m_previous->m_top == m_segmentCapacity);
-        }
-        
-        return true;
-    }
-
     inline size_t MarkStackArray::size()
     {
         return m_top + m_segmentCapacity * m_numberOfPreviousSegments;
diff --git a/Source/JavaScriptCore/heap/MarkedAllocator.cpp b/Source/JavaScriptCore/heap/MarkedAllocator.cpp
index ac0cf570a..9552a54ea 100644
--- a/Source/JavaScriptCore/heap/MarkedAllocator.cpp
+++ b/Source/JavaScriptCore/heap/MarkedAllocator.cpp
@@ -47,6 +47,7 @@ inline void* MarkedAllocator::tryAllocateHelper()
     
 inline void* MarkedAllocator::tryAllocate()
 {
+    ASSERT(!m_heap->isBusy());
     m_heap->m_operationInProgress = Allocation;
     void* result = tryAllocateHelper();
     m_heap->m_operationInProgress = NoOperation;
diff --git a/Source/JavaScriptCore/heap/MarkedBlock.cpp b/Source/JavaScriptCore/heap/MarkedBlock.cpp
index 0075f78d7..01e4237cb 100644
--- a/Source/JavaScriptCore/heap/MarkedBlock.cpp
+++ b/Source/JavaScriptCore/heap/MarkedBlock.cpp
@@ -114,6 +114,8 @@ MarkedBlock::FreeList MarkedBlock::sweep(SweepMode sweepMode)
 {
     HEAP_LOG_BLOCK_STATE_TRANSITION(this);
 
+    m_weakSet.sweep();
+
     if (sweepMode == SweepOnly && !m_cellsNeedDestruction)
         return FreeList();
 
diff --git a/Source/JavaScriptCore/heap/MarkedBlock.h b/Source/JavaScriptCore/heap/MarkedBlock.h
index b94c1e2b0..eb1cb681b 100644
--- a/Source/JavaScriptCore/heap/MarkedBlock.h
+++ b/Source/JavaScriptCore/heap/MarkedBlock.h
@@ -129,11 +129,9 @@ namespace JSC {
         FreeList sweep(SweepMode = SweepOnly);
 
         void shrink();
-        void resetAllocator();
 
         void visitWeakSet(HeapRootVisitor&);
         void reapWeakSet();
-        void sweepWeakSet();
 
         // While allocating from a free list, MarkedBlock temporarily has bogus
         // cell liveness data. To restore accurate cell liveness data, call one
@@ -274,11 +272,6 @@ namespace JSC {
         m_weakSet.shrink();
     }
 
-    inline void MarkedBlock::resetAllocator()
-    {
-        m_weakSet.resetAllocator();
-    }
-
     inline void MarkedBlock::visitWeakSet(HeapRootVisitor& heapRootVisitor)
     {
         m_weakSet.visit(heapRootVisitor);
@@ -289,11 +282,6 @@ namespace JSC {
         m_weakSet.reap();
     }
 
-    inline void MarkedBlock::sweepWeakSet()
-    {
-        m_weakSet.sweep();
-    }
-
     inline void MarkedBlock::didConsumeFreeList()
     {
         HEAP_LOG_BLOCK_STATE_TRANSITION(this);
diff --git a/Source/JavaScriptCore/heap/MarkedSpace.cpp b/Source/JavaScriptCore/heap/MarkedSpace.cpp
index 1604d2d63..42247a385 100644
--- a/Source/JavaScriptCore/heap/MarkedSpace.cpp
+++ b/Source/JavaScriptCore/heap/MarkedSpace.cpp
@@ -77,10 +77,6 @@ struct ReapWeakSet : MarkedBlock::VoidFunctor {
     void operator()(MarkedBlock* block) { block->reapWeakSet(); }
 };
 
-struct SweepWeakSet : MarkedBlock::VoidFunctor {
-    void operator()(MarkedBlock* block) { block->sweepWeakSet(); }
-};
-
 MarkedSpace::MarkedSpace(Heap* heap)
     : m_heap(heap)
 {
@@ -112,10 +108,6 @@ void MarkedSpace::lastChanceToFinalize()
     forEachBlock<LastChanceToFinalize>();
 }
 
-struct ResetAllocator : MarkedBlock::VoidFunctor {
-    void operator()(MarkedBlock* block) { block->resetAllocator(); }
-};
-
 void MarkedSpace::resetAllocators()
 {
     for (size_t cellSize = preciseStep; cellSize <= preciseCutoff; cellSize += preciseStep) {
@@ -127,8 +119,6 @@ void MarkedSpace::resetAllocators()
         allocatorFor(cellSize).reset();
         destructorAllocatorFor(cellSize).reset();
     }
-
-    forEachBlock<ResetAllocator>();
 }
 
 void MarkedSpace::visitWeakSets(HeapRootVisitor& heapRootVisitor)
@@ -142,11 +132,6 @@ void MarkedSpace::reapWeakSets()
     forEachBlock<ReapWeakSet>();
 }
 
-void MarkedSpace::sweepWeakSets()
-{
-    forEachBlock<SweepWeakSet>();
-}
-
 void MarkedSpace::canonicalizeCellLivenessData()
 {
     for (size_t cellSize = preciseStep; cellSize <= preciseCutoff; cellSize += preciseStep) {
diff --git a/Source/JavaScriptCore/heap/MarkedSpace.h b/Source/JavaScriptCore/heap/MarkedSpace.h
index 18b57c6d0..3f82bac96 100644
--- a/Source/JavaScriptCore/heap/MarkedSpace.h
+++ b/Source/JavaScriptCore/heap/MarkedSpace.h
@@ -85,7 +85,6 @@ public:
 
     void visitWeakSets(HeapRootVisitor&);
     void reapWeakSets();
-    void sweepWeakSets();
 
     MarkedBlockSet& blocks() { return m_blocks; }
     
diff --git a/Source/JavaScriptCore/heap/SlotVisitor.h b/Source/JavaScriptCore/heap/SlotVisitor.h
index 01eb219fc..715e2008c 100644
--- a/Source/JavaScriptCore/heap/SlotVisitor.h
+++ b/Source/JavaScriptCore/heap/SlotVisitor.h
@@ -68,15 +68,8 @@ public:
 private:
     void* allocateNewSpace(void*, size_t);
 
-    void donateSlow();
-    
-    void donateKnownParallel()
-    {
-        if (!m_stack.canDonateSomeCells())
-            return;
-        donateSlow();
-    }
-    
+    void donateKnownParallel();
+
     CopiedBlock* m_copyBlock;
 };
 
diff --git a/Source/JavaScriptCore/heap/Weak.h b/Source/JavaScriptCore/heap/Weak.h
index e5e0a97ec..07698fd06 100644
--- a/Source/JavaScriptCore/heap/Weak.h
+++ b/Source/JavaScriptCore/heap/Weak.h
@@ -153,15 +153,15 @@ template<typename T> inline WeakImpl* Weak<T>::hashTableDeletedValue()
 
 // This function helps avoid modifying a weak table while holding an iterator into it. (Object allocation
 // can run a finalizer that modifies the table. We avoid that by requiring a pre-constructed object as our value.)
-template<typename T, typename U> inline void weakAdd(HashMap<T, Weak<U> >& map, const T& key, PassWeak<U> value)
+template<typename Map, typename Key, typename Value> inline void weakAdd(Map& map, const Key& key, Value value)
 {
     ASSERT(!map.get(key));
     map.set(key, value); // The table may still have a zombie for value.
 }
 
-template<typename T, typename U> inline void weakRemove(HashMap<T, Weak<U> >& map, const T& key, typename Weak<U>::GetType value)
+template<typename Map, typename Key, typename Value> inline void weakRemove(Map& map, const Key& key, Value value)
 {
-    typename HashMap<T, Weak<U> >::iterator it = map.find(key);
+    typename Map::iterator it = map.find(key);
     ASSERT_UNUSED(value, value);
     ASSERT(it != map.end());
     ASSERT(it->second.was(value));
diff --git a/Source/JavaScriptCore/heap/WeakBlock.cpp b/Source/JavaScriptCore/heap/WeakBlock.cpp
index 685779d3a..8900e73df 100644
--- a/Source/JavaScriptCore/heap/WeakBlock.cpp
+++ b/Source/JavaScriptCore/heap/WeakBlock.cpp
@@ -69,7 +69,8 @@ void WeakBlock::lastChanceToFinalize()
 
 void WeakBlock::sweep()
 {
-    if (!m_sweepResult.isNull())
+    // If a block is completely empty, a sweep won't have any effect.
+    if (isEmpty())
         return;
 
     SweepResult sweepResult;
diff --git a/Source/JavaScriptCore/heap/WeakSet.cpp b/Source/JavaScriptCore/heap/WeakSet.cpp
index 9374fd8ff..4a510b899 100644
--- a/Source/JavaScriptCore/heap/WeakSet.cpp
+++ b/Source/JavaScriptCore/heap/WeakSet.cpp
@@ -42,17 +42,10 @@ WeakSet::~WeakSet()
 
 void WeakSet::sweep()
 {
-    WeakBlock* next;
-    for (WeakBlock* block = m_blocks.head(); block; block = next) {
-        next = block->next();
-
-        // If a block is completely empty, a new sweep won't have any effect.
-        if (block->isEmpty())
-            continue;
-
-        block->takeSweepResult(); // Force a new sweep by discarding the last sweep.
+    for (WeakBlock* block = m_blocks.head(); block; block = block->next())
         block->sweep();
-    }
+
+    resetAllocator();
 }
 
 WeakBlock::FreeCell* WeakSet::findAllocator()
@@ -69,7 +62,6 @@ WeakBlock::FreeCell* WeakSet::tryFindAllocator()
         WeakBlock* block = m_nextAllocator;
         m_nextAllocator = m_nextAllocator->next();
 
-        block->sweep();
         WeakBlock::SweepResult sweepResult = block->takeSweepResult();
         if (sweepResult.freeList)
             return sweepResult.freeList;
diff --git a/Source/JavaScriptCore/heap/WeakSet.h b/Source/JavaScriptCore/heap/WeakSet.h
index be9844a64..291d0aebd 100644
--- a/Source/JavaScriptCore/heap/WeakSet.h
+++ b/Source/JavaScriptCore/heap/WeakSet.h
@@ -118,6 +118,8 @@ inline void WeakSet::shrink()
         if (block->isEmpty())
             removeAllocator(block);
     }
+
+    resetAllocator();
 }
 
 inline void WeakSet::resetAllocator()