From 03e12282df9aa1e1fb05a8b90f1cfc2e08764cec Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 9 Feb 2012 14:16:12 +0100 Subject: Imported WebKit commit e09a82039aa4273ab318b71122e92d8e5f233525 (http://svn.webkit.org/repository/webkit/trunk@107223) --- Source/JavaScriptCore/heap/HandleTypes.h | 2 +- Source/JavaScriptCore/heap/Heap.cpp | 6 +- Source/JavaScriptCore/heap/Heap.h | 6 +- Source/JavaScriptCore/heap/MarkedAllocator.cpp | 128 +++++++++++++++++++++ Source/JavaScriptCore/heap/MarkedAllocator.h | 97 ++++++++++++++++ Source/JavaScriptCore/heap/MarkedBlock.cpp | 2 +- Source/JavaScriptCore/heap/MarkedBlock.h | 2 +- Source/JavaScriptCore/heap/MarkedSpace.cpp | 147 +++---------------------- Source/JavaScriptCore/heap/MarkedSpace.h | 82 +++----------- 9 files changed, 267 insertions(+), 205 deletions(-) create mode 100644 Source/JavaScriptCore/heap/MarkedAllocator.cpp create mode 100644 Source/JavaScriptCore/heap/MarkedAllocator.h (limited to 'Source/JavaScriptCore/heap') diff --git a/Source/JavaScriptCore/heap/HandleTypes.h b/Source/JavaScriptCore/heap/HandleTypes.h index 780ab85cd..bdfbcfda3 100644 --- a/Source/JavaScriptCore/heap/HandleTypes.h +++ b/Source/JavaScriptCore/heap/HandleTypes.h @@ -35,7 +35,7 @@ typedef JSValue* HandleSlot; template struct HandleTypes { typedef T* ExternalType; - static ExternalType getFromSlot(HandleSlot slot) { return (slot && *slot) ? reinterpret_cast(slot->asCell()) : 0; } + static ExternalType getFromSlot(HandleSlot slot) { return (slot && *slot) ? reinterpret_cast(static_cast(slot->asCell())) : 0; } static JSValue toJSValue(T* cell) { return reinterpret_cast(cell); } template static void validateUpcast() { T* temp; temp = (U*)0; } }; diff --git a/Source/JavaScriptCore/heap/Heap.cpp b/Source/JavaScriptCore/heap/Heap.cpp index a2136083a..9f5094a58 100644 --- a/Source/JavaScriptCore/heap/Heap.cpp +++ b/Source/JavaScriptCore/heap/Heap.cpp @@ -811,7 +811,7 @@ void Heap::collect(SweepToggle sweepToggle) { GCPHASE(ResetAllocator); - resetAllocator(); + resetAllocators(); } { @@ -846,10 +846,10 @@ void Heap::canonicalizeCellLivenessData() m_objectSpace.canonicalizeCellLivenessData(); } -void Heap::resetAllocator() +void Heap::resetAllocators() { m_extraCost = 0; - m_objectSpace.resetAllocator(); + m_objectSpace.resetAllocators(); } void Heap::setActivityCallback(PassOwnPtr activityCallback) diff --git a/Source/JavaScriptCore/heap/Heap.h b/Source/JavaScriptCore/heap/Heap.h index 40a8376f0..1d0ac5407 100644 --- a/Source/JavaScriptCore/heap/Heap.h +++ b/Source/JavaScriptCore/heap/Heap.h @@ -25,6 +25,7 @@ #include "DFGCodeBlocks.h" #include "HandleHeap.h" #include "HandleStack.h" +#include "MarkedAllocator.h" #include "MarkedBlock.h" #include "MarkedBlockSet.h" #include "MarkedSpace.h" @@ -94,7 +95,7 @@ namespace JSC { // true if an allocation or collection is in progress inline bool isBusy(); - MarkedSpace::SizeClass& sizeClassForObject(size_t bytes) { return m_objectSpace.sizeClassFor(bytes); } + MarkedAllocator& allocatorForObject(size_t bytes) { return m_objectSpace.allocatorFor(bytes); } void* allocate(size_t); CheckedBoolean tryAllocateStorage(size_t, void**); CheckedBoolean tryReallocateStorage(void**, size_t, size_t); @@ -136,6 +137,7 @@ namespace JSC { private: friend class MarkedSpace; + friend class MarkedAllocator; friend class MarkedBlock; friend class BumpSpace; friend class SlotVisitor; @@ -160,7 +162,7 @@ namespace JSC { // conservative marking, eager sweeping, or iterating the cells in a MarkedBlock.) void canonicalizeCellLivenessData(); - void resetAllocator(); + void resetAllocators(); void freeBlocks(MarkedBlock*); void clearMarks(); diff --git a/Source/JavaScriptCore/heap/MarkedAllocator.cpp b/Source/JavaScriptCore/heap/MarkedAllocator.cpp new file mode 100644 index 000000000..8239fbaed --- /dev/null +++ b/Source/JavaScriptCore/heap/MarkedAllocator.cpp @@ -0,0 +1,128 @@ +#include "config.h" +#include "MarkedAllocator.h" + +#include "Heap.h" + +namespace JSC { + +inline void* MarkedAllocator::tryAllocateHelper() +{ + MarkedBlock::FreeCell* firstFreeCell = m_firstFreeCell; + if (!firstFreeCell) { + for (MarkedBlock*& block = m_currentBlock; block; block = static_cast(block->next())) { + firstFreeCell = block->sweep(MarkedBlock::SweepToFreeList); + if (firstFreeCell) + break; + m_markedSpace->didConsumeFreeList(block); + block->didConsumeFreeList(); + } + + if (!firstFreeCell) + return 0; + } + + ASSERT(firstFreeCell); + m_firstFreeCell = firstFreeCell->next; + return firstFreeCell; +} + +inline void* MarkedAllocator::tryAllocate() +{ + m_heap->m_operationInProgress = Allocation; + void* result = tryAllocateHelper(); + m_heap->m_operationInProgress = NoOperation; + return result; +} + +void* MarkedAllocator::allocateSlowCase() +{ +#if COLLECT_ON_EVERY_ALLOCATION + m_heap->collectAllGarbage(); + ASSERT(m_heap->m_operationInProgress == NoOperation); +#endif + + void* result = tryAllocate(); + + if (LIKELY(result != 0)) + return result; + + AllocationEffort allocationEffort; + + if (( +#if ENABLE(GGC) + nurseryWaterMark() < m_heap->m_minBytesPerCycle +#else + m_heap->waterMark() < m_heap->highWaterMark() +#endif + ) || !m_heap->m_isSafeToCollect) + allocationEffort = AllocationMustSucceed; + else + allocationEffort = AllocationCanFail; + + MarkedBlock* block = allocateBlock(allocationEffort); + if (block) { + addBlock(block); + void* result = tryAllocate(); + ASSERT(result); + return result; + } + + m_heap->collect(Heap::DoNotSweep); + + result = tryAllocate(); + + if (result) + return result; + + ASSERT(m_heap->waterMark() < m_heap->highWaterMark()); + + addBlock(allocateBlock(AllocationMustSucceed)); + + result = tryAllocate(); + ASSERT(result); + return result; +} + +MarkedBlock* MarkedAllocator::allocateBlock(AllocationEffort allocationEffort) +{ + MarkedBlock* block; + + { + MutexLocker locker(m_heap->m_freeBlockLock); + if (m_heap->m_numberOfFreeBlocks) { + block = static_cast(m_heap->m_freeBlocks.removeHead()); + ASSERT(block); + m_heap->m_numberOfFreeBlocks--; + } else + block = 0; + } + if (block) + block = MarkedBlock::recycle(block, m_heap, m_cellSize); + else if (allocationEffort == AllocationCanFail) + return 0; + else + block = MarkedBlock::create(m_heap, m_cellSize); + + m_markedSpace->didAddBlock(block); + + return block; +} + +void MarkedAllocator::addBlock(MarkedBlock* block) +{ + ASSERT(!m_currentBlock); + ASSERT(!m_firstFreeCell); + + m_blockList.append(block); + m_currentBlock = block; + m_firstFreeCell = block->sweep(MarkedBlock::SweepToFreeList); +} + +void MarkedAllocator::removeBlock(MarkedBlock* block) +{ + if (m_currentBlock == block) + m_currentBlock = 0; + m_blockList.remove(block); +} + +} // namespace JSC diff --git a/Source/JavaScriptCore/heap/MarkedAllocator.h b/Source/JavaScriptCore/heap/MarkedAllocator.h new file mode 100644 index 000000000..5644c691b --- /dev/null +++ b/Source/JavaScriptCore/heap/MarkedAllocator.h @@ -0,0 +1,97 @@ +#ifndef MarkedAllocator_h +#define MarkedAllocator_h + +#include "MarkedBlock.h" +#include + +namespace JSC { + +class Heap; +class MarkedSpace; + +namespace DFG { +class SpeculativeJIT; +} + +class MarkedAllocator { + friend class JIT; + friend class DFG::SpeculativeJIT; + +public: + MarkedAllocator(); + void reset(); + void zapFreeList(); + size_t cellSize() { return m_cellSize; } + void* allocate(); + Heap* heap() { return m_heap; } + + template void forEachBlock(Functor&); + + void addBlock(MarkedBlock*); + void removeBlock(MarkedBlock*); + void setHeap(Heap* heap) { m_heap = heap; } + void setCellSize(size_t cellSize) { m_cellSize = cellSize; } + void setMarkedSpace(MarkedSpace* space) { m_markedSpace = space; } + +private: + JS_EXPORT_PRIVATE void* allocateSlowCase(); + void* tryAllocate(); + void* tryAllocateHelper(); + MarkedBlock* allocateBlock(AllocationEffort); + + MarkedBlock::FreeCell* m_firstFreeCell; + MarkedBlock* m_currentBlock; + DoublyLinkedList m_blockList; + size_t m_cellSize; + Heap* m_heap; + MarkedSpace* m_markedSpace; +}; + +inline MarkedAllocator::MarkedAllocator() + : m_firstFreeCell(0) + , m_currentBlock(0) + , m_cellSize(0) + , m_heap(0) + , m_markedSpace(0) +{ +} + +inline void* MarkedAllocator::allocate() +{ + MarkedBlock::FreeCell* firstFreeCell = m_firstFreeCell; + // This is a light-weight fast path to cover the most common case. + if (UNLIKELY(!firstFreeCell)) + return allocateSlowCase(); + + m_firstFreeCell = firstFreeCell->next; + return firstFreeCell; +} + +inline void MarkedAllocator::reset() +{ + m_currentBlock = static_cast(m_blockList.head()); +} + +inline void MarkedAllocator::zapFreeList() +{ + if (!m_currentBlock) { + ASSERT(!m_firstFreeCell); + return; + } + + m_currentBlock->zapFreeList(m_firstFreeCell); + m_firstFreeCell = 0; +} + +template inline void MarkedAllocator::forEachBlock(Functor& functor) +{ + HeapBlock* next; + for (HeapBlock* block = m_blockList.head(); block; block = next) { + next = block->next(); + functor(static_cast(block)); + } +} + +} // namespace JSC + +#endif diff --git a/Source/JavaScriptCore/heap/MarkedBlock.cpp b/Source/JavaScriptCore/heap/MarkedBlock.cpp index 715f25d92..dd9233300 100644 --- a/Source/JavaScriptCore/heap/MarkedBlock.cpp +++ b/Source/JavaScriptCore/heap/MarkedBlock.cpp @@ -89,7 +89,7 @@ MarkedBlock::FreeCell* MarkedBlock::specializedSweep() if (blockState == Marked && m_marks.get(i)) continue; - JSCell* cell = reinterpret_cast(&atoms()[i]); + JSCell* cell = reinterpret_cast_ptr(&atoms()[i]); if (blockState == Zapped && !cell->isZapped()) continue; diff --git a/Source/JavaScriptCore/heap/MarkedBlock.h b/Source/JavaScriptCore/heap/MarkedBlock.h index 00eb54b1f..0a4ebe47e 100644 --- a/Source/JavaScriptCore/heap/MarkedBlock.h +++ b/Source/JavaScriptCore/heap/MarkedBlock.h @@ -317,7 +317,7 @@ namespace JSC { template inline void MarkedBlock::forEachCell(Functor& functor) { for (size_t i = firstAtom(); i < m_endAtom; i += m_atomsPerCell) { - JSCell* cell = reinterpret_cast(&atoms()[i]); + JSCell* cell = reinterpret_cast_ptr(&atoms()[i]); if (!isLive(cell)) continue; diff --git a/Source/JavaScriptCore/heap/MarkedSpace.cpp b/Source/JavaScriptCore/heap/MarkedSpace.cpp index fcca188e4..87dc0493d 100644 --- a/Source/JavaScriptCore/heap/MarkedSpace.cpp +++ b/Source/JavaScriptCore/heap/MarkedSpace.cpp @@ -35,155 +35,40 @@ MarkedSpace::MarkedSpace(Heap* heap) , m_nurseryWaterMark(0) , m_heap(heap) { - for (size_t cellSize = preciseStep; cellSize <= preciseCutoff; cellSize += preciseStep) - sizeClassFor(cellSize).cellSize = cellSize; - - for (size_t cellSize = impreciseStep; cellSize <= impreciseCutoff; cellSize += impreciseStep) - sizeClassFor(cellSize).cellSize = cellSize; -} - -void MarkedSpace::addBlock(SizeClass& sizeClass, MarkedBlock* block) -{ - ASSERT(!sizeClass.currentBlock); - ASSERT(!sizeClass.firstFreeCell); - - sizeClass.blockList.append(block); - sizeClass.currentBlock = block; - sizeClass.firstFreeCell = block->sweep(MarkedBlock::SweepToFreeList); -} + for (size_t cellSize = preciseStep; cellSize <= preciseCutoff; cellSize += preciseStep) { + allocatorFor(cellSize).setCellSize(cellSize); + allocatorFor(cellSize).setHeap(heap); + allocatorFor(cellSize).setMarkedSpace(this); + } -void MarkedSpace::removeBlock(MarkedBlock* block) -{ - SizeClass& sizeClass = sizeClassFor(block->cellSize()); - if (sizeClass.currentBlock == block) - sizeClass.currentBlock = 0; - sizeClass.blockList.remove(block); + for (size_t cellSize = impreciseStep; cellSize <= impreciseCutoff; cellSize += impreciseStep) { + allocatorFor(cellSize).setCellSize(cellSize); + allocatorFor(cellSize).setHeap(heap); + allocatorFor(cellSize).setMarkedSpace(this); + } } -void MarkedSpace::resetAllocator() +void MarkedSpace::resetAllocators() { m_waterMark = 0; m_nurseryWaterMark = 0; for (size_t cellSize = preciseStep; cellSize <= preciseCutoff; cellSize += preciseStep) - sizeClassFor(cellSize).resetAllocator(); + allocatorFor(cellSize).reset(); for (size_t cellSize = impreciseStep; cellSize <= impreciseCutoff; cellSize += impreciseStep) - sizeClassFor(cellSize).resetAllocator(); + allocatorFor(cellSize).reset(); } void MarkedSpace::canonicalizeCellLivenessData() { for (size_t cellSize = preciseStep; cellSize <= preciseCutoff; cellSize += preciseStep) - sizeClassFor(cellSize).zapFreeList(); + allocatorFor(cellSize).zapFreeList(); for (size_t cellSize = impreciseStep; cellSize <= impreciseCutoff; cellSize += impreciseStep) - sizeClassFor(cellSize).zapFreeList(); + allocatorFor(cellSize).zapFreeList(); } -inline void* MarkedSpace::tryAllocateHelper(MarkedSpace::SizeClass& sizeClass) -{ - MarkedBlock::FreeCell* firstFreeCell = sizeClass.firstFreeCell; - if (!firstFreeCell) { - for (MarkedBlock*& block = sizeClass.currentBlock; block; block = static_cast(block->next())) { - firstFreeCell = block->sweep(MarkedBlock::SweepToFreeList); - if (firstFreeCell) - break; - m_nurseryWaterMark += block->capacity() - block->size(); - m_waterMark += block->capacity(); - block->didConsumeFreeList(); - } - - if (!firstFreeCell) - return 0; - } - - ASSERT(firstFreeCell); - sizeClass.firstFreeCell = firstFreeCell->next; - return firstFreeCell; -} - -inline void* MarkedSpace::tryAllocate(MarkedSpace::SizeClass& sizeClass) -{ - m_heap->m_operationInProgress = Allocation; - void* result = tryAllocateHelper(sizeClass); - m_heap->m_operationInProgress = NoOperation; - return result; -} - -void* MarkedSpace::allocateSlowCase(MarkedSpace::SizeClass& sizeClass) -{ -#if COLLECT_ON_EVERY_ALLOCATION - m_heap->collectAllGarbage(); - ASSERT(m_heap->m_operationInProgress == NoOperation); -#endif - - void* result = tryAllocate(sizeClass); - - if (LIKELY(result != 0)) - return result; - - AllocationEffort allocationEffort; - - if (( -#if ENABLE(GGC) - nurseryWaterMark() < m_heap->m_minBytesPerCycle -#else - m_heap->waterMark() < m_heap->highWaterMark() -#endif - ) || !m_heap->m_isSafeToCollect) - allocationEffort = AllocationMustSucceed; - else - allocationEffort = AllocationCanFail; - - MarkedBlock* block = allocateBlock(sizeClass.cellSize, allocationEffort); - if (block) { - addBlock(sizeClass, block); - void* result = tryAllocate(sizeClass); - ASSERT(result); - return result; - } - - m_heap->collect(Heap::DoNotSweep); - - result = tryAllocate(sizeClass); - - if (result) - return result; - - ASSERT(m_heap->waterMark() < m_heap->highWaterMark()); - - addBlock(sizeClass, allocateBlock(sizeClass.cellSize, AllocationMustSucceed)); - - result = tryAllocate(sizeClass); - ASSERT(result); - return result; -} - -MarkedBlock* MarkedSpace::allocateBlock(size_t cellSize, AllocationEffort allocationEffort) -{ - MarkedBlock* block; - - { - MutexLocker locker(m_heap->m_freeBlockLock); - if (m_heap->m_numberOfFreeBlocks) { - block = static_cast(m_heap->m_freeBlocks.removeHead()); - ASSERT(block); - m_heap->m_numberOfFreeBlocks--; - } else - block = 0; - } - if (block) - block = MarkedBlock::recycle(block, m_heap, cellSize); - else if (allocationEffort == AllocationCanFail) - return 0; - else - block = MarkedBlock::create(m_heap, cellSize); - - m_blocks.add(block); - - return block; -} void MarkedSpace::freeBlocks(MarkedBlock* head) { @@ -222,7 +107,7 @@ inline void TakeIfUnmarked::operator()(MarkedBlock* block) if (!block->markCountIsZero()) return; - m_markedSpace->removeBlock(block); + m_markedSpace->allocatorFor(block->cellSize()).removeBlock(block); m_empties.append(block); } diff --git a/Source/JavaScriptCore/heap/MarkedSpace.h b/Source/JavaScriptCore/heap/MarkedSpace.h index f7d96c774..21a0b48de 100644 --- a/Source/JavaScriptCore/heap/MarkedSpace.h +++ b/Source/JavaScriptCore/heap/MarkedSpace.h @@ -23,6 +23,7 @@ #define MarkedSpace_h #include "MachineStackMarker.h" +#include "MarkedAllocator.h" #include "MarkedBlock.h" #include "MarkedBlockSet.h" #include "PageAllocationAligned.h" @@ -48,27 +49,13 @@ class MarkedSpace { public: static const size_t maxCellSize = 2048; - struct SizeClass { - SizeClass(); - void resetAllocator(); - void zapFreeList(); - - MarkedBlock::FreeCell* firstFreeCell; - MarkedBlock* currentBlock; - DoublyLinkedList blockList; - size_t cellSize; - }; - MarkedSpace(Heap*); - SizeClass& sizeClassFor(size_t); + MarkedAllocator& allocatorFor(size_t); void* allocate(size_t); - void* allocate(SizeClass&); - void resetAllocator(); + void resetAllocators(); - void addBlock(SizeClass&, MarkedBlock*); - void removeBlock(MarkedBlock*); MarkedBlockSet& blocks() { return m_blocks; } void canonicalizeCellLivenessData(); @@ -85,13 +72,10 @@ public: void shrink(); void freeBlocks(MarkedBlock* head); + void didAddBlock(MarkedBlock*); + void didConsumeFreeList(MarkedBlock*); private: - JS_EXPORT_PRIVATE void* allocateSlowCase(SizeClass&); - void* tryAllocateHelper(MarkedSpace::SizeClass&); - void* tryAllocate(MarkedSpace::SizeClass&); - MarkedBlock* allocateBlock(size_t, AllocationEffort); - // [ 32... 256 ] static const size_t preciseStep = MarkedBlock::atomSize; static const size_t preciseCutoff = 256; @@ -102,8 +86,8 @@ private: static const size_t impreciseCutoff = maxCellSize; static const size_t impreciseCount = impreciseCutoff / impreciseStep; - FixedArray m_preciseSizeClasses; - FixedArray m_impreciseSizeClasses; + FixedArray m_preciseSizeClasses; + FixedArray m_impreciseSizeClasses; size_t m_waterMark; size_t m_nurseryWaterMark; Heap* m_heap; @@ -136,7 +120,7 @@ template inline typename Functor::ReturnType MarkedSpace::forE return forEachCell(functor); } -inline MarkedSpace::SizeClass& MarkedSpace::sizeClassFor(size_t bytes) +inline MarkedAllocator& MarkedSpace::allocatorFor(size_t bytes) { ASSERT(bytes && bytes <= maxCellSize); if (bytes <= preciseCutoff) @@ -146,39 +130,17 @@ inline MarkedSpace::SizeClass& MarkedSpace::sizeClassFor(size_t bytes) inline void* MarkedSpace::allocate(size_t bytes) { - SizeClass& sizeClass = sizeClassFor(bytes); - return allocate(sizeClass); -} - -inline void* MarkedSpace::allocate(SizeClass& sizeClass) -{ - // This is a light-weight fast path to cover the most common case. - MarkedBlock::FreeCell* firstFreeCell = sizeClass.firstFreeCell; - if (UNLIKELY(!firstFreeCell)) - return allocateSlowCase(sizeClass); - - sizeClass.firstFreeCell = firstFreeCell->next; - return firstFreeCell; + return allocatorFor(bytes).allocate(); } template inline typename Functor::ReturnType MarkedSpace::forEachBlock(Functor& functor) { for (size_t i = 0; i < preciseCount; ++i) { - SizeClass& sizeClass = m_preciseSizeClasses[i]; - HeapBlock* next; - for (HeapBlock* block = sizeClass.blockList.head(); block; block = next) { - next = block->next(); - functor(static_cast(block)); - } + m_preciseSizeClasses[i].forEachBlock(functor); } for (size_t i = 0; i < impreciseCount; ++i) { - SizeClass& sizeClass = m_impreciseSizeClasses[i]; - HeapBlock* next; - for (HeapBlock* block = sizeClass.blockList.head(); block; block = next) { - next = block->next(); - functor(static_cast(block)); - } + m_impreciseSizeClasses[i].forEachBlock(functor); } return functor.returnValue(); @@ -190,27 +152,15 @@ template inline typename Functor::ReturnType MarkedSpace::for return forEachBlock(functor); } -inline MarkedSpace::SizeClass::SizeClass() - : firstFreeCell(0) - , currentBlock(0) - , cellSize(0) -{ -} - -inline void MarkedSpace::SizeClass::resetAllocator() +inline void MarkedSpace::didAddBlock(MarkedBlock* block) { - currentBlock = static_cast(blockList.head()); + m_blocks.add(block); } -inline void MarkedSpace::SizeClass::zapFreeList() +inline void MarkedSpace::didConsumeFreeList(MarkedBlock* block) { - if (!currentBlock) { - ASSERT(!firstFreeCell); - return; - } - - currentBlock->zapFreeList(firstFreeCell); - firstFreeCell = 0; + m_nurseryWaterMark += block->capacity() - block->size(); + m_waterMark += block->capacity(); } } // namespace JSC -- cgit v1.2.1