diff options
Diffstat (limited to 'Source/JavaScriptCore/heap')
86 files changed, 3109 insertions, 4673 deletions
diff --git a/Source/JavaScriptCore/heap/BlockAllocator.cpp b/Source/JavaScriptCore/heap/BlockAllocator.cpp new file mode 100644 index 000000000..7a7474913 --- /dev/null +++ b/Source/JavaScriptCore/heap/BlockAllocator.cpp @@ -0,0 +1,172 @@ +/* + * Copyright (C) 2012 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "BlockAllocator.h" + +#include "CopiedBlock.h" +#include "CopyWorkList.h" +#include "MarkedBlock.h" +#include "WeakBlock.h" +#include <wtf/CurrentTime.h> + +namespace JSC { + +inline ThreadIdentifier createBlockFreeingThread(BlockAllocator* allocator) +{ + if (!GCActivityCallback::s_shouldCreateGCTimer) + return 0; // No block freeing thread. + ThreadIdentifier identifier = createThread(allocator->blockFreeingThreadStartFunc, allocator, "JavaScriptCore::BlockFree"); + RELEASE_ASSERT(identifier); + return identifier; +} + +BlockAllocator::BlockAllocator() + : m_superRegion() + , m_copiedRegionSet(CopiedBlock::blockSize) + , m_markedRegionSet(MarkedBlock::blockSize) + , m_fourKBBlockRegionSet(WeakBlock::blockSize) + , m_workListRegionSet(CopyWorkListSegment::blockSize) + , m_numberOfEmptyRegions(0) + , m_isCurrentlyAllocating(false) + , m_blockFreeingThreadShouldQuit(false) + , m_blockFreeingThread(createBlockFreeingThread(this)) +{ + m_regionLock.Init(); +} + +BlockAllocator::~BlockAllocator() +{ + releaseFreeRegions(); + { + std::lock_guard<std::mutex> lock(m_emptyRegionConditionMutex); + m_blockFreeingThreadShouldQuit = true; + m_emptyRegionCondition.notify_all(); + } + if (m_blockFreeingThread) + waitForThreadCompletion(m_blockFreeingThread); + ASSERT(allRegionSetsAreEmpty()); + ASSERT(m_emptyRegions.isEmpty()); +} + +bool BlockAllocator::allRegionSetsAreEmpty() const +{ + return m_copiedRegionSet.isEmpty() + && m_markedRegionSet.isEmpty() + && m_fourKBBlockRegionSet.isEmpty() + && m_workListRegionSet.isEmpty(); +} + +void BlockAllocator::releaseFreeRegions() +{ + while (true) { + Region* region; + { + SpinLockHolder locker(&m_regionLock); + if (!m_numberOfEmptyRegions) + region = 0; + else { + region = m_emptyRegions.removeHead(); + RELEASE_ASSERT(region); + m_numberOfEmptyRegions--; + } + } + + if (!region) + break; + + region->destroy(); + } +} + +void BlockAllocator::waitForDuration(std::chrono::milliseconds duration) +{ + std::unique_lock<std::mutex> lock(m_emptyRegionConditionMutex); + + // If this returns early, that's fine, so long as it doesn't do it too + // frequently. It would only be a bug if this function failed to return + // when it was asked to do so. + if (m_blockFreeingThreadShouldQuit) + return; + + m_emptyRegionCondition.wait_for(lock, duration); +} + +void BlockAllocator::blockFreeingThreadStartFunc(void* blockAllocator) +{ + static_cast<BlockAllocator*>(blockAllocator)->blockFreeingThreadMain(); +} + +void BlockAllocator::blockFreeingThreadMain() +{ + size_t currentNumberOfEmptyRegions; + while (!m_blockFreeingThreadShouldQuit) { + // Generally wait for one second before scavenging free blocks. This + // may return early, particularly when we're being asked to quit. + waitForDuration(std::chrono::seconds(1)); + if (m_blockFreeingThreadShouldQuit) + break; + + if (m_isCurrentlyAllocating) { + m_isCurrentlyAllocating = false; + continue; + } + + // Sleep until there is actually work to do rather than waking up every second to check. + { + std::unique_lock<std::mutex> lock(m_emptyRegionConditionMutex); + SpinLockHolder regionLocker(&m_regionLock); + while (!m_numberOfEmptyRegions && !m_blockFreeingThreadShouldQuit) { + m_regionLock.Unlock(); + m_emptyRegionCondition.wait(lock); + m_regionLock.Lock(); + } + currentNumberOfEmptyRegions = m_numberOfEmptyRegions; + } + + size_t desiredNumberOfEmptyRegions = currentNumberOfEmptyRegions / 2; + + while (!m_blockFreeingThreadShouldQuit) { + Region* region; + { + SpinLockHolder locker(&m_regionLock); + if (m_numberOfEmptyRegions <= desiredNumberOfEmptyRegions) + region = 0; + else { + region = m_emptyRegions.removeHead(); + RELEASE_ASSERT(region); + m_numberOfEmptyRegions--; + } + } + + if (!region) + break; + + region->destroy(); + } + } +} + +} // namespace JSC diff --git a/Source/JavaScriptCore/heap/BlockAllocator.h b/Source/JavaScriptCore/heap/BlockAllocator.h new file mode 100644 index 000000000..a52b90f19 --- /dev/null +++ b/Source/JavaScriptCore/heap/BlockAllocator.h @@ -0,0 +1,300 @@ +/* + * Copyright (C) 2012 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef BlockAllocator_h +#define BlockAllocator_h + +#include "GCActivityCallback.h" +#include "HeapBlock.h" +#include "Region.h" +#include <condition_variable> +#include <wtf/DoublyLinkedList.h> +#include <wtf/Forward.h> +#include <wtf/PageAllocationAligned.h> +#include <wtf/TCSpinLock.h> +#include <wtf/Threading.h> + +namespace JSC { + +class BlockAllocator; +class CopiedBlock; +class CopyWorkListSegment; +class HandleBlock; +class VM; +class MarkStackSegment; +class MarkedBlock; +class WeakBlock; + +// Simple allocator to reduce VM cost by holding onto blocks of memory for +// short periods of time and then freeing them on a secondary thread. + +class BlockAllocator { +public: + BlockAllocator(); + ~BlockAllocator(); + + template <typename T> DeadBlock* allocate(); + DeadBlock* allocateCustomSize(size_t blockSize, size_t blockAlignment); + template <typename T> void deallocate(T*); + template <typename T> void deallocateCustomSize(T*); + +private: + void waitForDuration(std::chrono::milliseconds); + + friend ThreadIdentifier createBlockFreeingThread(BlockAllocator*); + void blockFreeingThreadMain(); + static void blockFreeingThreadStartFunc(void* heap); + + struct RegionSet { + RegionSet(size_t blockSize) + : m_numberOfPartialRegions(0) + , m_blockSize(blockSize) + { + } + + bool isEmpty() const + { + return m_fullRegions.isEmpty() && m_partialRegions.isEmpty(); + } + + DoublyLinkedList<Region> m_fullRegions; + DoublyLinkedList<Region> m_partialRegions; + size_t m_numberOfPartialRegions; + size_t m_blockSize; + }; + + DeadBlock* tryAllocateFromRegion(RegionSet&, DoublyLinkedList<Region>&, size_t&); + + bool allRegionSetsAreEmpty() const; + void releaseFreeRegions(); + + template <typename T> RegionSet& regionSetFor(); + + SuperRegion m_superRegion; + RegionSet m_copiedRegionSet; + RegionSet m_markedRegionSet; + // WeakBlocks and MarkStackSegments use the same RegionSet since they're the same size. + RegionSet m_fourKBBlockRegionSet; + RegionSet m_workListRegionSet; + + DoublyLinkedList<Region> m_emptyRegions; + size_t m_numberOfEmptyRegions; + + bool m_isCurrentlyAllocating; + bool m_blockFreeingThreadShouldQuit; + SpinLock m_regionLock; + std::mutex m_emptyRegionConditionMutex; + std::condition_variable m_emptyRegionCondition; + ThreadIdentifier m_blockFreeingThread; +}; + +inline DeadBlock* BlockAllocator::tryAllocateFromRegion(RegionSet& set, DoublyLinkedList<Region>& regions, size_t& numberOfRegions) +{ + if (numberOfRegions) { + ASSERT(!regions.isEmpty()); + Region* region = regions.head(); + ASSERT(!region->isFull()); + + if (region->isEmpty()) { + ASSERT(region == m_emptyRegions.head()); + m_numberOfEmptyRegions--; + set.m_numberOfPartialRegions++; + region = m_emptyRegions.removeHead()->reset(set.m_blockSize); + set.m_partialRegions.push(region); + } + + DeadBlock* block = region->allocate(); + + if (region->isFull()) { + set.m_numberOfPartialRegions--; + set.m_fullRegions.push(set.m_partialRegions.removeHead()); + } + + return block; + } + return 0; +} + +template<typename T> +inline DeadBlock* BlockAllocator::allocate() +{ + RegionSet& set = regionSetFor<T>(); + DeadBlock* block; + m_isCurrentlyAllocating = true; + { + SpinLockHolder locker(&m_regionLock); + if ((block = tryAllocateFromRegion(set, set.m_partialRegions, set.m_numberOfPartialRegions))) + return block; + if ((block = tryAllocateFromRegion(set, m_emptyRegions, m_numberOfEmptyRegions))) + return block; + } + + Region* newRegion = Region::create(&m_superRegion, T::blockSize); + + SpinLockHolder locker(&m_regionLock); + m_emptyRegions.push(newRegion); + m_numberOfEmptyRegions++; + block = tryAllocateFromRegion(set, m_emptyRegions, m_numberOfEmptyRegions); + ASSERT(block); + return block; +} + +inline DeadBlock* BlockAllocator::allocateCustomSize(size_t blockSize, size_t blockAlignment) +{ + size_t realSize = WTF::roundUpToMultipleOf(blockAlignment, blockSize); + Region* newRegion = Region::createCustomSize(&m_superRegion, realSize, blockAlignment); + DeadBlock* block = newRegion->allocate(); + ASSERT(block); + return block; +} + +template<typename T> +inline void BlockAllocator::deallocate(T* block) +{ + RegionSet& set = regionSetFor<T>(); + bool shouldWakeBlockFreeingThread = false; + { + SpinLockHolder locker(&m_regionLock); + Region* region = block->region(); + ASSERT(!region->isEmpty()); + if (region->isFull()) + set.m_fullRegions.remove(region); + else { + set.m_partialRegions.remove(region); + set.m_numberOfPartialRegions--; + } + + region->deallocate(block); + + if (region->isEmpty()) { + m_emptyRegions.push(region); + shouldWakeBlockFreeingThread = !m_numberOfEmptyRegions; + m_numberOfEmptyRegions++; + } else { + set.m_partialRegions.push(region); + set.m_numberOfPartialRegions++; + } + } + + if (shouldWakeBlockFreeingThread) { + std::lock_guard<std::mutex> lock(m_emptyRegionConditionMutex); + m_emptyRegionCondition.notify_one(); + } + + if (!m_blockFreeingThread) + releaseFreeRegions(); +} + +template<typename T> +inline void BlockAllocator::deallocateCustomSize(T* block) +{ + Region* region = block->region(); + ASSERT(region->isCustomSize()); + region->deallocate(block); + region->destroy(); +} + +template <> +inline BlockAllocator::RegionSet& BlockAllocator::regionSetFor<CopiedBlock>() +{ + return m_copiedRegionSet; +} + +template <> +inline BlockAllocator::RegionSet& BlockAllocator::regionSetFor<MarkedBlock>() +{ + return m_markedRegionSet; +} + +template <> +inline BlockAllocator::RegionSet& BlockAllocator::regionSetFor<WeakBlock>() +{ + return m_fourKBBlockRegionSet; +} + +template <> +inline BlockAllocator::RegionSet& BlockAllocator::regionSetFor<MarkStackSegment>() +{ + return m_fourKBBlockRegionSet; +} + +template <> +inline BlockAllocator::RegionSet& BlockAllocator::regionSetFor<CopyWorkListSegment>() +{ + return m_workListRegionSet; +} + +template <> +inline BlockAllocator::RegionSet& BlockAllocator::regionSetFor<HandleBlock>() +{ + return m_fourKBBlockRegionSet; +} + +template <> +inline BlockAllocator::RegionSet& BlockAllocator::regionSetFor<HeapBlock<CopiedBlock>>() +{ + return m_copiedRegionSet; +} + +template <> +inline BlockAllocator::RegionSet& BlockAllocator::regionSetFor<HeapBlock<MarkedBlock>>() +{ + return m_markedRegionSet; +} + +template <> +inline BlockAllocator::RegionSet& BlockAllocator::regionSetFor<HeapBlock<WeakBlock>>() +{ + return m_fourKBBlockRegionSet; +} + +template <> +inline BlockAllocator::RegionSet& BlockAllocator::regionSetFor<HeapBlock<MarkStackSegment>>() +{ + return m_fourKBBlockRegionSet; +} + +template <> +inline BlockAllocator::RegionSet& BlockAllocator::regionSetFor<HeapBlock<CopyWorkListSegment>>() +{ + return m_workListRegionSet; +} + +template <> +inline BlockAllocator::RegionSet& BlockAllocator::regionSetFor<HeapBlock<HandleBlock>>() +{ + return m_fourKBBlockRegionSet; +} + +template <typename T> +inline BlockAllocator::RegionSet& BlockAllocator::regionSetFor() +{ + RELEASE_ASSERT_NOT_REACHED(); + return *(RegionSet*)0; +} + +} // namespace JSC + +#endif // BlockAllocator_h diff --git a/Source/JavaScriptCore/heap/CodeBlockSet.cpp b/Source/JavaScriptCore/heap/CodeBlockSet.cpp index 9cfce4f97..c04cbacd6 100644 --- a/Source/JavaScriptCore/heap/CodeBlockSet.cpp +++ b/Source/JavaScriptCore/heap/CodeBlockSet.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013, 2014 Apple Inc. All rights reserved. + * Copyright (C) 2013 Apple Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -27,79 +27,54 @@ #include "CodeBlockSet.h" #include "CodeBlock.h" -#include "JSCInlines.h" #include "SlotVisitor.h" -#include <wtf/CommaPrinter.h> namespace JSC { static const bool verbose = false; -CodeBlockSet::CodeBlockSet() -{ -} +CodeBlockSet::CodeBlockSet() { } CodeBlockSet::~CodeBlockSet() { - for (CodeBlock* codeBlock : m_oldCodeBlocks) - codeBlock->deref(); - - for (CodeBlock* codeBlock : m_newCodeBlocks) - codeBlock->deref(); + HashSet<CodeBlock*>::iterator iter = m_set.begin(); + HashSet<CodeBlock*>::iterator end = m_set.end(); + for (; iter != end; ++iter) + (*iter)->deref(); } void CodeBlockSet::add(PassRefPtr<CodeBlock> codeBlock) { CodeBlock* block = codeBlock.leakRef(); - bool isNewEntry = m_newCodeBlocks.add(block).isNewEntry; + bool isNewEntry = m_set.add(block).isNewEntry; ASSERT_UNUSED(isNewEntry, isNewEntry); } -void CodeBlockSet::promoteYoungCodeBlocks() -{ - m_oldCodeBlocks.add(m_newCodeBlocks.begin(), m_newCodeBlocks.end()); - m_newCodeBlocks.clear(); -} - -void CodeBlockSet::clearMarksForFullCollection() +void CodeBlockSet::clearMarks() { - for (CodeBlock* codeBlock : m_oldCodeBlocks) { + HashSet<CodeBlock*>::iterator iter = m_set.begin(); + HashSet<CodeBlock*>::iterator end = m_set.end(); + for (; iter != end; ++iter) { + CodeBlock* codeBlock = *iter; codeBlock->m_mayBeExecuting = false; - codeBlock->m_visitAggregateHasBeenCalled.store(false, std::memory_order_relaxed); - } - - // We promote after we clear marks on the old generation CodeBlocks because - // none of the young generations CodeBlocks need to be cleared. - promoteYoungCodeBlocks(); -} - -void CodeBlockSet::clearMarksForEdenCollection(const Vector<const JSCell*>& rememberedSet) -{ - // This ensures that we will revisit CodeBlocks in remembered Executables even if they were previously marked. - for (const JSCell* cell : rememberedSet) { - ScriptExecutable* executable = const_cast<ScriptExecutable*>(jsDynamicCast<const ScriptExecutable*>(cell)); - if (!executable) - continue; - executable->forEachCodeBlock([](CodeBlock* codeBlock) { - codeBlock->m_mayBeExecuting = false; - codeBlock->m_visitAggregateHasBeenCalled.store(false, std::memory_order_relaxed); - }); + codeBlock->m_visitAggregateHasBeenCalled = false; } } -void CodeBlockSet::deleteUnmarkedAndUnreferenced(HeapOperation collectionType) +void CodeBlockSet::deleteUnmarkedAndUnreferenced() { - HashSet<CodeBlock*>& set = collectionType == EdenCollection ? m_newCodeBlocks : m_oldCodeBlocks; - // This needs to be a fixpoint because code blocks that are unmarked may // refer to each other. For example, a DFG code block that is owned by // the GC may refer to an FTL for-entry code block that is also owned by // the GC. Vector<CodeBlock*, 16> toRemove; if (verbose) - dataLog("Fixpointing over unmarked, set size = ", set.size(), "...\n"); + dataLog("Fixpointing over unmarked, set size = ", m_set.size(), "...\n"); for (;;) { - for (CodeBlock* codeBlock : set) { + HashSet<CodeBlock*>::iterator iter = m_set.begin(); + HashSet<CodeBlock*>::iterator end = m_set.end(); + for (; iter != end; ++iter) { + CodeBlock* codeBlock = *iter; if (!codeBlock->hasOneRef()) continue; if (codeBlock->m_mayBeExecuting) @@ -111,33 +86,22 @@ void CodeBlockSet::deleteUnmarkedAndUnreferenced(HeapOperation collectionType) dataLog(" Removing ", toRemove.size(), " blocks.\n"); if (toRemove.isEmpty()) break; - for (CodeBlock* codeBlock : toRemove) - set.remove(codeBlock); + for (unsigned i = toRemove.size(); i--;) + m_set.remove(toRemove[i]); toRemove.resize(0); } - - // Any remaining young CodeBlocks are live and need to be promoted to the set of old CodeBlocks. - if (collectionType == EdenCollection) - promoteYoungCodeBlocks(); -} - -void CodeBlockSet::remove(CodeBlock* codeBlock) -{ - codeBlock->deref(); - if (m_oldCodeBlocks.contains(codeBlock)) { - m_oldCodeBlocks.remove(codeBlock); - return; - } - ASSERT(m_newCodeBlocks.contains(codeBlock)); - m_newCodeBlocks.remove(codeBlock); } void CodeBlockSet::traceMarked(SlotVisitor& visitor) { if (verbose) - dataLog("Tracing ", m_currentlyExecuting.size(), " code blocks.\n"); - for (CodeBlock* codeBlock : m_currentlyExecuting) { - ASSERT(codeBlock->m_mayBeExecuting); + dataLog("Tracing ", m_set.size(), " code blocks.\n"); + HashSet<CodeBlock*>::iterator iter = m_set.begin(); + HashSet<CodeBlock*>::iterator end = m_set.end(); + for (; iter != end; ++iter) { + CodeBlock* codeBlock = *iter; + if (!codeBlock->m_mayBeExecuting) + continue; codeBlock->visitAggregate(visitor); } } @@ -145,34 +109,13 @@ void CodeBlockSet::traceMarked(SlotVisitor& visitor) void CodeBlockSet::rememberCurrentlyExecutingCodeBlocks(Heap* heap) { #if ENABLE(GGC) - if (verbose) - dataLog("Remembering ", m_currentlyExecuting.size(), " code blocks.\n"); - for (CodeBlock* codeBlock : m_currentlyExecuting) { - heap->addToRememberedSet(codeBlock->ownerExecutable()); - ASSERT(codeBlock->m_mayBeExecuting); - } + for (size_t i = 0; i < m_currentlyExecuting.size(); ++i) + heap->addToRememberedSet(m_currentlyExecuting[i]->ownerExecutable()); m_currentlyExecuting.clear(); #else UNUSED_PARAM(heap); #endif // ENABLE(GGC) } -void CodeBlockSet::dump(PrintStream& out) const -{ - CommaPrinter comma; - out.print("{old = ["); - for (CodeBlock* codeBlock : m_oldCodeBlocks) - out.print(comma, pointerDump(codeBlock)); - out.print("], new = ["); - comma = CommaPrinter(); - for (CodeBlock* codeBlock : m_newCodeBlocks) - out.print(comma, pointerDump(codeBlock)); - out.print("], currentlyExecuting = ["); - comma = CommaPrinter(); - for (CodeBlock* codeBlock : m_currentlyExecuting) - out.print(comma, pointerDump(codeBlock)); - out.print("]}"); -} - } // namespace JSC diff --git a/Source/JavaScriptCore/heap/CodeBlockSet.h b/Source/JavaScriptCore/heap/CodeBlockSet.h index 6cab875c9..791d18699 100644 --- a/Source/JavaScriptCore/heap/CodeBlockSet.h +++ b/Source/JavaScriptCore/heap/CodeBlockSet.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013, 2014 Apple Inc. All rights reserved. + * Copyright (C) 2013 Apple Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -26,19 +26,16 @@ #ifndef CodeBlockSet_h #define CodeBlockSet_h -#include "GCSegmentedArray.h" -#include "HeapOperation.h" #include <wtf/HashSet.h> #include <wtf/Noncopyable.h> #include <wtf/PassRefPtr.h> -#include <wtf/PrintStream.h> #include <wtf/RefPtr.h> +#include <wtf/Vector.h> namespace JSC { class CodeBlock; class Heap; -class JSCell; class SlotVisitor; // CodeBlockSet tracks all CodeBlocks. Every CodeBlock starts out with one @@ -55,22 +52,16 @@ public: // Add a CodeBlock. This is only called by CodeBlock constructors. void add(PassRefPtr<CodeBlock>); - // Clear mark bits for certain CodeBlocks depending on the type of collection. - void clearMarksForEdenCollection(const Vector<const JSCell*>&); - - // Clear all mark bits for all CodeBlocks. - void clearMarksForFullCollection(); - + // Clear all mark bits associated with DFG code blocks. + void clearMarks(); + // Mark a pointer that may be a CodeBlock that belongs to the set of DFG // blocks. This is defined in CodeBlock.h. - void mark(CodeBlock* candidateCodeBlock); void mark(void* candidateCodeBlock); // Delete all code blocks that are only referenced by this set (i.e. owned // by this set), and that have not been marked. - void deleteUnmarkedAndUnreferenced(HeapOperation); - - void remove(CodeBlock*); + void deleteUnmarkedAndUnreferenced(); // Trace all marked code blocks. The CodeBlock is free to make use of // mayBeExecuting. @@ -85,30 +76,18 @@ public: // visited. template<typename Functor> void iterate(Functor& functor) { - for (auto& codeBlock : m_oldCodeBlocks) { - bool done = functor(codeBlock); - if (done) - return; - } - - for (auto& codeBlock : m_newCodeBlocks) { + for (auto &codeBlock : m_set) { bool done = functor(codeBlock); if (done) - return; + break; } } - - void dump(PrintStream&) const; private: - void clearMarksForCodeBlocksInRememberedExecutables(const Vector<const JSCell*>&); - void promoteYoungCodeBlocks(); - // This is not a set of RefPtr<CodeBlock> because we need to be able to find // arbitrary bogus pointers. I could have written a thingy that had peek types // and all, but that seemed like overkill. - HashSet<CodeBlock*> m_oldCodeBlocks; - HashSet<CodeBlock*> m_newCodeBlocks; + HashSet<CodeBlock* > m_set; Vector<CodeBlock*> m_currentlyExecuting; }; diff --git a/Source/JavaScriptCore/heap/ConservativeRoots.cpp b/Source/JavaScriptCore/heap/ConservativeRoots.cpp index 620208175..7fc8eee3f 100644 --- a/Source/JavaScriptCore/heap/ConservativeRoots.cpp +++ b/Source/JavaScriptCore/heap/ConservativeRoots.cpp @@ -32,7 +32,6 @@ #include "CopiedSpaceInlines.h" #include "JSCell.h" #include "JSObject.h" -#include "JSCInlines.h" #include "Structure.h" namespace JSC { @@ -92,7 +91,6 @@ inline void ConservativeRoots::genericAddPointer(void* p, TinyBloomFilter filter } template<typename MarkHook> -SUPPRESS_ASAN void ConservativeRoots::genericAddSpan(void* begin, void* end, MarkHook& markHook) { if (begin > end) { @@ -101,8 +99,9 @@ void ConservativeRoots::genericAddSpan(void* begin, void* end, MarkHook& markHoo end = swapTemp; } - RELEASE_ASSERT(isPointerAligned(begin)); - RELEASE_ASSERT(isPointerAligned(end)); + ASSERT((static_cast<char*>(end) - static_cast<char*>(begin)) < 0x1000000); + ASSERT(isPointerAligned(begin)); + ASSERT(isPointerAligned(end)); TinyBloomFilter filter = m_blocks->filter(); // Make a local copy of filter to show the compiler it won't alias, and can be register-allocated. for (char** it = static_cast<char**>(begin); it != static_cast<char**>(end); ++it) diff --git a/Source/JavaScriptCore/heap/CopiedBlock.h b/Source/JavaScriptCore/heap/CopiedBlock.h index 3be585c93..6d59aa6bc 100644 --- a/Source/JavaScriptCore/heap/CopiedBlock.h +++ b/Source/JavaScriptCore/heap/CopiedBlock.h @@ -26,25 +26,25 @@ #ifndef CopiedBlock_h #define CopiedBlock_h +#include "BlockAllocator.h" #include "CopyWorkList.h" +#include "HeapBlock.h" #include "JSCJSValue.h" #include "Options.h" #include <wtf/Atomics.h> -#include <wtf/DoublyLinkedList.h> -#include <wtf/Lock.h> +#include <wtf/OwnPtr.h> +#include <wtf/PassOwnPtr.h> namespace JSC { class CopiedSpace; -class CopiedBlock : public DoublyLinkedListNode<CopiedBlock> { - friend class WTF::DoublyLinkedListNode<CopiedBlock>; +class CopiedBlock : public HeapBlock<CopiedBlock> { friend class CopiedSpace; friend class CopiedAllocator; public: - static CopiedBlock* create(size_t = blockSize); - static CopiedBlock* createNoZeroFill(size_t = blockSize); - static void destroy(CopiedBlock*); + static CopiedBlock* create(DeadBlock*); + static CopiedBlock* createNoZeroFill(DeadBlock*); void pin(); bool isPinned(); @@ -54,8 +54,8 @@ public: void didPromote(); unsigned liveBytes(); - bool shouldReportLiveBytes(LockHolder&, JSCell* owner); - void reportLiveBytes(LockHolder&, JSCell*, CopyToken, unsigned); + bool shouldReportLiveBytes(SpinLockHolder&, JSCell* owner); + void reportLiveBytes(SpinLockHolder&, JSCell*, CopyToken, unsigned); void reportLiveBytesDuringCopying(unsigned); void didSurviveGC(); void didEvacuateBytes(unsigned); @@ -85,21 +85,16 @@ public: bool hasWorkList(); CopyWorkList& workList(); - Lock& workListLock() { return m_workListLock; } + SpinLock& workListLock() { return m_workListLock; } private: - CopiedBlock(size_t); + CopiedBlock(Region*); void zeroFillWilderness(); // Can be called at any time to zero-fill to the end of the block. void checkConsistency(); - CopiedBlock* m_prev; - CopiedBlock* m_next; - - size_t m_capacity; - - Lock m_workListLock; - std::unique_ptr<CopyWorkList> m_workList; + SpinLock m_workListLock; + OwnPtr<CopyWorkList> m_workList; size_t m_remaining; bool m_isPinned : 1; @@ -110,20 +105,15 @@ private: #endif }; -inline CopiedBlock* CopiedBlock::createNoZeroFill(size_t capacity) -{ - return new(NotNull, fastAlignedMalloc(CopiedBlock::blockSize, capacity)) CopiedBlock(capacity); -} - -inline void CopiedBlock::destroy(CopiedBlock* copiedBlock) +inline CopiedBlock* CopiedBlock::createNoZeroFill(DeadBlock* block) { - copiedBlock->~CopiedBlock(); - fastAlignedFree(copiedBlock); + Region* region = block->region(); + return new(NotNull, block) CopiedBlock(region); } -inline CopiedBlock* CopiedBlock::create(size_t capacity) +inline CopiedBlock* CopiedBlock::create(DeadBlock* block) { - CopiedBlock* newBlock = createNoZeroFill(capacity); + CopiedBlock* newBlock = createNoZeroFill(block); newBlock->zeroFillWilderness(); return newBlock; } @@ -140,9 +130,8 @@ inline void CopiedBlock::zeroFillWilderness() #endif } -inline CopiedBlock::CopiedBlock(size_t capacity) - : DoublyLinkedListNode<CopiedBlock>() - , m_capacity(capacity) +inline CopiedBlock::CopiedBlock(Region* region) + : HeapBlock<CopiedBlock>(region) , m_remaining(payloadCapacity()) , m_isPinned(false) , m_isOld(false) @@ -151,20 +140,20 @@ inline CopiedBlock::CopiedBlock(size_t capacity) , m_liveObjects(0) #endif { + m_workListLock.Init(); ASSERT(is8ByteAligned(reinterpret_cast<void*>(m_remaining))); } inline void CopiedBlock::didSurviveGC() { checkConsistency(); - ASSERT(isOld()); m_liveBytes = 0; #ifndef NDEBUG m_liveObjects = 0; #endif m_isPinned = false; if (m_workList) - m_workList = nullptr; + m_workList.clear(); } inline void CopiedBlock::didEvacuateBytes(unsigned bytes) @@ -195,7 +184,7 @@ inline void CopiedBlock::pin() { m_isPinned = true; if (m_workList) - m_workList = nullptr; + m_workList.clear(); } inline bool CopiedBlock::isPinned() @@ -215,7 +204,7 @@ inline void CopiedBlock::didPromote() inline bool CopiedBlock::isOversize() { - return m_capacity != blockSize; + return region()->isCustomSize(); } inline unsigned CopiedBlock::liveBytes() @@ -226,12 +215,12 @@ inline unsigned CopiedBlock::liveBytes() inline char* CopiedBlock::payload() { - return reinterpret_cast<char*>(this) + WTF::roundUpToMultipleOf<sizeof(double)>(sizeof(CopiedBlock)); + return reinterpret_cast<char*>(this) + ((sizeof(CopiedBlock) + 7) & ~7); } inline char* CopiedBlock::payloadEnd() { - return reinterpret_cast<char*>(this) + m_capacity; + return reinterpret_cast<char*>(this) + region()->blockSize(); } inline size_t CopiedBlock::payloadCapacity() @@ -276,7 +265,7 @@ inline size_t CopiedBlock::size() inline size_t CopiedBlock::capacity() { - return m_capacity; + return region()->blockSize(); } inline bool CopiedBlock::hasWorkList() diff --git a/Source/JavaScriptCore/heap/CopiedBlockInlines.h b/Source/JavaScriptCore/heap/CopiedBlockInlines.h index 3d1133413..8bf831cfc 100644 --- a/Source/JavaScriptCore/heap/CopiedBlockInlines.h +++ b/Source/JavaScriptCore/heap/CopiedBlockInlines.h @@ -33,17 +33,18 @@ namespace JSC { -inline bool CopiedBlock::shouldReportLiveBytes(LockHolder&, JSCell* owner) +inline bool CopiedBlock::shouldReportLiveBytes(SpinLockHolder&, JSCell* owner) { // We want to add to live bytes if the owner isn't part of the remembered set or // if this block was allocated during the last cycle. // If we always added live bytes we would double count for elements in the remembered // set across collections. // If we didn't always add live bytes to new blocks, we'd get too few. - return !Heap::isRemembered(owner) || !m_isOld; + bool ownerIsRemembered = MarkedBlock::blockFor(owner)->isRemembered(owner); + return !ownerIsRemembered || !m_isOld; } -inline void CopiedBlock::reportLiveBytes(LockHolder&, JSCell* owner, CopyToken token, unsigned bytes) +inline void CopiedBlock::reportLiveBytes(SpinLockHolder&, JSCell* owner, CopyToken token, unsigned bytes) { checkConsistency(); #ifndef NDEBUG @@ -51,7 +52,7 @@ inline void CopiedBlock::reportLiveBytes(LockHolder&, JSCell* owner, CopyToken t #endif m_liveBytes += bytes; checkConsistency(); - ASSERT(m_liveBytes <= m_capacity); + ASSERT(m_liveBytes <= CopiedBlock::blockSize); if (isPinned()) return; @@ -62,7 +63,7 @@ inline void CopiedBlock::reportLiveBytes(LockHolder&, JSCell* owner, CopyToken t } if (!m_workList) - m_workList = std::make_unique<CopyWorkList>(); + m_workList = adoptPtr(new CopyWorkList(Heap::heap(owner)->blockAllocator())); m_workList->append(CopyWorklistItem(owner, token)); } diff --git a/Source/JavaScriptCore/heap/CopiedSpace.cpp b/Source/JavaScriptCore/heap/CopiedSpace.cpp index a3874994f..eb294214f 100644 --- a/Source/JavaScriptCore/heap/CopiedSpace.cpp +++ b/Source/JavaScriptCore/heap/CopiedSpace.cpp @@ -28,7 +28,7 @@ #include "CopiedSpaceInlines.h" #include "GCActivityCallback.h" -#include "JSCInlines.h" +#include "Operations.h" #include "Options.h" namespace JSC { @@ -38,29 +38,29 @@ CopiedSpace::CopiedSpace(Heap* heap) , m_inCopyingPhase(false) , m_shouldDoCopyPhase(false) , m_numberOfLoanedBlocks(0) - , m_bytesRemovedFromOldSpaceDueToReallocation(0) { + m_toSpaceLock.Init(); } CopiedSpace::~CopiedSpace() { while (!m_oldGen.toSpace->isEmpty()) - CopiedBlock::destroy(m_oldGen.toSpace->removeHead()); + m_heap->blockAllocator().deallocate(CopiedBlock::destroy(m_oldGen.toSpace->removeHead())); while (!m_oldGen.fromSpace->isEmpty()) - CopiedBlock::destroy(m_oldGen.fromSpace->removeHead()); + m_heap->blockAllocator().deallocate(CopiedBlock::destroy(m_oldGen.fromSpace->removeHead())); while (!m_oldGen.oversizeBlocks.isEmpty()) - CopiedBlock::destroy(m_oldGen.oversizeBlocks.removeHead()); + m_heap->blockAllocator().deallocateCustomSize(CopiedBlock::destroy(m_oldGen.oversizeBlocks.removeHead())); while (!m_newGen.toSpace->isEmpty()) - CopiedBlock::destroy(m_newGen.toSpace->removeHead()); + m_heap->blockAllocator().deallocate(CopiedBlock::destroy(m_newGen.toSpace->removeHead())); while (!m_newGen.fromSpace->isEmpty()) - CopiedBlock::destroy(m_newGen.fromSpace->removeHead()); + m_heap->blockAllocator().deallocate(CopiedBlock::destroy(m_newGen.fromSpace->removeHead())); while (!m_newGen.oversizeBlocks.isEmpty()) - CopiedBlock::destroy(m_newGen.oversizeBlocks.removeHead()); + m_heap->blockAllocator().deallocateCustomSize(CopiedBlock::destroy(m_newGen.oversizeBlocks.removeHead())); ASSERT(m_oldGen.toSpace->isEmpty()); ASSERT(m_oldGen.fromSpace->isEmpty()); @@ -99,7 +99,7 @@ CheckedBoolean CopiedSpace::tryAllocateOversize(size_t bytes, void** outPtr) { ASSERT(isOversize(bytes)); - CopiedBlock* block = CopiedBlock::create(WTF::roundUpToMultipleOf<sizeof(double)>(sizeof(CopiedBlock) + bytes)); + CopiedBlock* block = CopiedBlock::create(m_heap->blockAllocator().allocateCustomSize(sizeof(CopiedBlock) + bytes, CopiedBlock::blockSize)); m_newGen.oversizeBlocks.push(block); m_newGen.blockFilter.add(reinterpret_cast<Bits>(block)); m_blockSet.add(block); @@ -110,7 +110,7 @@ CheckedBoolean CopiedSpace::tryAllocateOversize(size_t bytes, void** outPtr) *outPtr = allocator.forceAllocate(bytes); allocator.resetCurrentBlock(); - m_heap->didAllocate(block->capacity()); + m_heap->didAllocate(block->region()->blockSize()); return true; } @@ -156,16 +156,12 @@ CheckedBoolean CopiedSpace::tryReallocateOversize(void** ptr, size_t oldSize, si CopiedBlock* oldBlock = CopiedSpace::blockFor(oldPtr); if (oldBlock->isOversize()) { - // FIXME: Eagerly deallocating the old space block probably buys more confusion than - // value. - // https://bugs.webkit.org/show_bug.cgi?id=144750 - if (oldBlock->isOld()) { - m_bytesRemovedFromOldSpaceDueToReallocation += oldBlock->size(); + if (oldBlock->isOld()) m_oldGen.oversizeBlocks.remove(oldBlock); - } else + else m_newGen.oversizeBlocks.remove(oldBlock); m_blockSet.remove(oldBlock); - CopiedBlock::destroy(oldBlock); + m_heap->blockAllocator().deallocateCustomSize(CopiedBlock::destroy(oldBlock)); } *ptr = newPtr; @@ -191,19 +187,19 @@ void CopiedSpace::doneFillingBlock(CopiedBlock* block, CopiedBlock** exchange) { // Always put the block into the old gen because it's being promoted! - LockHolder locker(&m_toSpaceLock); + SpinLockHolder locker(&m_toSpaceLock); m_oldGen.toSpace->push(block); m_blockSet.add(block); m_oldGen.blockFilter.add(reinterpret_cast<Bits>(block)); } { - LockHolder locker(m_loanedBlocksLock); + MutexLocker locker(m_loanedBlocksLock); ASSERT(m_numberOfLoanedBlocks > 0); ASSERT(m_inCopyingPhase); m_numberOfLoanedBlocks--; if (!m_numberOfLoanedBlocks) - m_loanedBlocksCondition.notifyOne(); + m_loanedBlocksCondition.signal(); } } @@ -231,7 +227,7 @@ void CopiedSpace::didStartFullCollection() void CopiedSpace::doneCopying() { { - LockHolder locker(m_loanedBlocksLock); + MutexLocker locker(m_loanedBlocksLock); while (m_numberOfLoanedBlocks > 0) m_loanedBlocksCondition.wait(m_loanedBlocksLock); } @@ -257,7 +253,6 @@ void CopiedSpace::doneCopying() // We don't add the block to the blockSet because it was never removed. ASSERT(m_blockSet.contains(block)); blockFilter->add(reinterpret_cast<Bits>(block)); - block->didSurviveGC(); toSpace->push(block); } diff --git a/Source/JavaScriptCore/heap/CopiedSpace.h b/Source/JavaScriptCore/heap/CopiedSpace.h index ed2982da0..c0a59a27e 100644 --- a/Source/JavaScriptCore/heap/CopiedSpace.h +++ b/Source/JavaScriptCore/heap/CopiedSpace.h @@ -27,17 +27,19 @@ #define CopiedSpace_h #include "CopiedAllocator.h" +#include "HeapBlock.h" #include "HeapOperation.h" #include "TinyBloomFilter.h" #include <wtf/Assertions.h> #include <wtf/CheckedBoolean.h> -#include <wtf/Condition.h> #include <wtf/DoublyLinkedList.h> #include <wtf/HashSet.h> -#include <wtf/Lock.h> #include <wtf/OSAllocator.h> +#include <wtf/PageAllocationAligned.h> #include <wtf/PageBlock.h> #include <wtf/StdLibExtras.h> +#include <wtf/TCSpinLock.h> +#include <wtf/ThreadingPrimitives.h> namespace JSC { @@ -85,13 +87,6 @@ public: static CopiedBlock* blockFor(void*); Heap* heap() const { return m_heap; } - - size_t takeBytesRemovedFromOldSpaceDueToReallocation() - { - size_t result = 0; - std::swap(m_bytesRemovedFromOldSpaceDueToReallocation, result); - return result; - } private: static bool isOversize(size_t); @@ -113,7 +108,7 @@ private: HashSet<CopiedBlock*> m_blockSet; - Lock m_toSpaceLock; + SpinLock m_toSpaceLock; struct CopiedGeneration { CopiedGeneration() @@ -138,11 +133,9 @@ private: bool m_inCopyingPhase; bool m_shouldDoCopyPhase; - Lock m_loanedBlocksLock; - Condition m_loanedBlocksCondition; + Mutex m_loanedBlocksLock; + ThreadCondition m_loanedBlocksCondition; size_t m_numberOfLoanedBlocks; - - size_t m_bytesRemovedFromOldSpaceDueToReallocation; static const size_t s_maxAllocationSize = CopiedBlock::blockSize / 2; static const size_t s_initialBlockNum = 16; diff --git a/Source/JavaScriptCore/heap/CopiedSpaceInlines.h b/Source/JavaScriptCore/heap/CopiedSpaceInlines.h index aa4eeff42..ec33f582f 100644 --- a/Source/JavaScriptCore/heap/CopiedSpaceInlines.h +++ b/Source/JavaScriptCore/heap/CopiedSpaceInlines.h @@ -29,6 +29,7 @@ #include "CopiedBlock.h" #include "CopiedSpace.h" #include "Heap.h" +#include "HeapBlock.h" #include "VM.h" #include <wtf/CheckedBoolean.h> @@ -98,37 +99,37 @@ inline void CopiedSpace::recycleEvacuatedBlock(CopiedBlock* block, HeapOperation ASSERT(block->canBeRecycled()); ASSERT(!block->m_isPinned); { - LockHolder locker(&m_toSpaceLock); + SpinLockHolder locker(&m_toSpaceLock); m_blockSet.remove(block); if (collectionType == EdenCollection) m_newGen.fromSpace->remove(block); else m_oldGen.fromSpace->remove(block); } - CopiedBlock::destroy(block); + m_heap->blockAllocator().deallocate(CopiedBlock::destroy(block)); } inline void CopiedSpace::recycleBorrowedBlock(CopiedBlock* block) { - CopiedBlock::destroy(block); + m_heap->blockAllocator().deallocate(CopiedBlock::destroy(block)); { - LockHolder locker(m_loanedBlocksLock); + MutexLocker locker(m_loanedBlocksLock); ASSERT(m_numberOfLoanedBlocks > 0); ASSERT(m_inCopyingPhase); m_numberOfLoanedBlocks--; if (!m_numberOfLoanedBlocks) - m_loanedBlocksCondition.notifyOne(); + m_loanedBlocksCondition.signal(); } } inline CopiedBlock* CopiedSpace::allocateBlockForCopyingPhase() { ASSERT(m_inCopyingPhase); - CopiedBlock* block = CopiedBlock::createNoZeroFill(); + CopiedBlock* block = CopiedBlock::createNoZeroFill(m_heap->blockAllocator().allocate<CopiedBlock>()); { - LockHolder locker(m_loanedBlocksLock); + MutexLocker locker(m_loanedBlocksLock); m_numberOfLoanedBlocks++; } @@ -142,7 +143,7 @@ inline void CopiedSpace::allocateBlock() m_allocator.resetCurrentBlock(); - CopiedBlock* block = CopiedBlock::create(); + CopiedBlock* block = CopiedBlock::create(m_heap->blockAllocator().allocate<CopiedBlock>()); m_newGen.toSpace->push(block); m_newGen.blockFilter.add(reinterpret_cast<Bits>(block)); @@ -234,7 +235,7 @@ inline void CopiedSpace::startedCopying() } else { oversizeBlocks->remove(block); m_blockSet.remove(block); - CopiedBlock::destroy(block); + m_heap->blockAllocator().deallocateCustomSize(CopiedBlock::destroy(block)); } block = next; } diff --git a/Source/JavaScriptCore/heap/CopyToken.h b/Source/JavaScriptCore/heap/CopyToken.h index 5aceb5ba7..35e0e67e6 100644 --- a/Source/JavaScriptCore/heap/CopyToken.h +++ b/Source/JavaScriptCore/heap/CopyToken.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013, 2015 Apple Inc. All rights reserved. + * Copyright (C) 2013 Apple Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -31,8 +31,7 @@ namespace JSC { enum CopyToken { ButterflyCopyToken, TypedArrayVectorCopyToken, - MapBackingStoreCopyToken, - DirectArgumentsOverridesCopyToken + MapBackingStoreCopyToken }; } // namespace JSC diff --git a/Source/JavaScriptCore/heap/CopyVisitor.cpp b/Source/JavaScriptCore/heap/CopyVisitor.cpp index 5dd841923..3d18936ec 100644 --- a/Source/JavaScriptCore/heap/CopyVisitor.cpp +++ b/Source/JavaScriptCore/heap/CopyVisitor.cpp @@ -32,7 +32,7 @@ #include "GCThreadSharedData.h" #include "JSCell.h" #include "JSObject.h" -#include "JSCInlines.h" +#include "Operations.h" #include <wtf/Threading.h> namespace JSC { diff --git a/Source/JavaScriptCore/heap/CopyWorkList.h b/Source/JavaScriptCore/heap/CopyWorkList.h index 8308667c5..c79063b97 100644 --- a/Source/JavaScriptCore/heap/CopyWorkList.h +++ b/Source/JavaScriptCore/heap/CopyWorkList.h @@ -27,7 +27,6 @@ #define CopyWorkList_h #include "CopyToken.h" -#include <wtf/DoublyLinkedList.h> #include <wtf/Vector.h> namespace JSC { @@ -58,18 +57,11 @@ private: uintptr_t m_value; }; -class CopyWorkListSegment : public DoublyLinkedListNode<CopyWorkListSegment> { - friend class WTF::DoublyLinkedListNode<CopyWorkListSegment>; +class CopyWorkListSegment : public HeapBlock<CopyWorkListSegment> { public: - static CopyWorkListSegment* create() + static CopyWorkListSegment* create(DeadBlock* block) { - return new (NotNull, fastMalloc(blockSize)) CopyWorkListSegment(); - } - - static void destroy(CopyWorkListSegment* segment) - { - segment->~CopyWorkListSegment(); - fastFree(segment); + return new (NotNull, block) CopyWorkListSegment(block->region()); } size_t size() { return m_size; } @@ -86,8 +78,8 @@ public: static const size_t blockSize = 512; private: - CopyWorkListSegment() - : DoublyLinkedListNode<CopyWorkListSegment>() + CopyWorkListSegment(Region* region) + : HeapBlock<CopyWorkListSegment>(region) , m_size(0) { } @@ -95,8 +87,6 @@ private: CopyWorklistItem* data() { return reinterpret_cast<CopyWorklistItem*>(this + 1); } char* endOfBlock() { return reinterpret_cast<char*>(this) + blockSize; } - CopyWorkListSegment* m_prev; - CopyWorkListSegment* m_next; size_t m_size; }; @@ -149,11 +139,10 @@ private: }; class CopyWorkList { - WTF_MAKE_FAST_ALLOCATED; public: typedef CopyWorkListIterator iterator; - CopyWorkList(); + CopyWorkList(BlockAllocator&); ~CopyWorkList(); void append(CopyWorklistItem); @@ -162,22 +151,24 @@ public: private: DoublyLinkedList<CopyWorkListSegment> m_segments; + BlockAllocator& m_blockAllocator; }; -inline CopyWorkList::CopyWorkList() +inline CopyWorkList::CopyWorkList(BlockAllocator& blockAllocator) + : m_blockAllocator(blockAllocator) { } inline CopyWorkList::~CopyWorkList() { while (!m_segments.isEmpty()) - CopyWorkListSegment::destroy(m_segments.removeHead()); + m_blockAllocator.deallocate(CopyWorkListSegment::destroy(m_segments.removeHead())); } inline void CopyWorkList::append(CopyWorklistItem item) { if (m_segments.isEmpty() || m_segments.tail()->isFull()) - m_segments.append(CopyWorkListSegment::create()); + m_segments.append(CopyWorkListSegment::create(m_blockAllocator.allocate<CopyWorkListSegment>())); ASSERT(!m_segments.tail()->isFull()); diff --git a/Source/JavaScriptCore/heap/CopyWriteBarrier.h b/Source/JavaScriptCore/heap/CopyWriteBarrier.h index b014fcdf5..847712666 100644 --- a/Source/JavaScriptCore/heap/CopyWriteBarrier.h +++ b/Source/JavaScriptCore/heap/CopyWriteBarrier.h @@ -50,7 +50,8 @@ public: bool operator!() const { return !m_value; } - explicit operator bool() const { return m_value; } + typedef T* (CopyWriteBarrier::*UnspecifiedBoolType); + operator UnspecifiedBoolType*() const { return m_value ? reinterpret_cast<UnspecifiedBoolType*>(1) : 0; } T* get() const { @@ -67,10 +68,10 @@ public: return get(); } - void set(VM& vm, const JSCell* owner, T* value) + void set(VM&, const JSCell* owner, T* value) { this->m_value = value; - vm.heap.writeBarrier(owner); + Heap::writeBarrier(owner); } void setWithoutWriteBarrier(T* value) diff --git a/Source/JavaScriptCore/heap/DeferGC.cpp b/Source/JavaScriptCore/heap/DeferGC.cpp index dd66c6384..72b3dc934 100644 --- a/Source/JavaScriptCore/heap/DeferGC.cpp +++ b/Source/JavaScriptCore/heap/DeferGC.cpp @@ -26,8 +26,6 @@ #include "config.h" #include "DeferGC.h" -#include "JSCInlines.h" - namespace JSC { #ifndef NDEBUG diff --git a/Source/JavaScriptCore/heap/DelayedReleaseScope.h b/Source/JavaScriptCore/heap/DelayedReleaseScope.h new file mode 100644 index 000000000..bbb4724b9 --- /dev/null +++ b/Source/JavaScriptCore/heap/DelayedReleaseScope.h @@ -0,0 +1,100 @@ +/* + * Copyright (C) 2013 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DelayedReleaseScope_h +#define DelayedReleaseScope_h + +#include "APIShims.h" +#include "MarkedSpace.h" + +namespace JSC { + +#if USE(CF) + +class DelayedReleaseScope { +public: + DelayedReleaseScope(MarkedSpace& markedSpace) + : m_markedSpace(markedSpace) + { + ASSERT(!m_markedSpace.m_currentDelayedReleaseScope); + m_markedSpace.m_currentDelayedReleaseScope = this; + } + + ~DelayedReleaseScope() + { + ASSERT(m_markedSpace.m_currentDelayedReleaseScope == this); + m_markedSpace.m_currentDelayedReleaseScope = nullptr; + + HeapOperation operationInProgress = NoOperation; + std::swap(operationInProgress, m_markedSpace.m_heap->m_operationInProgress); + + APICallbackShim callbackShim(*m_markedSpace.m_heap->vm()); + m_delayedReleaseObjects.clear(); + + std::swap(operationInProgress, m_markedSpace.m_heap->m_operationInProgress); + } + + template <typename T> + void releaseSoon(RetainPtr<T>&& object) + { + m_delayedReleaseObjects.append(std::move(object)); + } + + static bool isInEffectFor(MarkedSpace& markedSpace) + { + return markedSpace.m_currentDelayedReleaseScope; + } + +private: + MarkedSpace& m_markedSpace; + Vector<RetainPtr<CFTypeRef>> m_delayedReleaseObjects; +}; + +template <typename T> +inline void MarkedSpace::releaseSoon(RetainPtr<T>&& object) +{ + ASSERT(m_currentDelayedReleaseScope); + m_currentDelayedReleaseScope->releaseSoon(std::move(object)); +} + +#else // USE(CF) + +class DelayedReleaseScope { +public: + DelayedReleaseScope(MarkedSpace&) + { + } + + static bool isInEffectFor(MarkedSpace&) + { + return true; + } +}; + +#endif // USE(CF) + +} // namespace JSC + +#endif // DelayedReleaseScope_h diff --git a/Source/JavaScriptCore/heap/EdenGCActivityCallback.cpp b/Source/JavaScriptCore/heap/EdenGCActivityCallback.cpp deleted file mode 100644 index 500ad55a4..000000000 --- a/Source/JavaScriptCore/heap/EdenGCActivityCallback.cpp +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright (C) 2014 Apple Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS - * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF - * THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include "config.h" -#include "EdenGCActivityCallback.h" - -#include "VM.h" - -namespace JSC { - -#if USE(CF) || PLATFORM(EFL) - -EdenGCActivityCallback::EdenGCActivityCallback(Heap* heap) - : GCActivityCallback(heap) -{ -} - -void EdenGCActivityCallback::doCollection() -{ - m_vm->heap.collect(EdenCollection); -} - -double EdenGCActivityCallback::lastGCLength() -{ - return m_vm->heap.lastEdenGCLength(); -} - -double EdenGCActivityCallback::deathRate() -{ - Heap* heap = &m_vm->heap; - size_t sizeBefore = heap->sizeBeforeLastEdenCollection(); - size_t sizeAfter = heap->sizeAfterLastEdenCollection(); - if (!sizeBefore) - return 1.0; - return static_cast<double>(sizeBefore - sizeAfter) / static_cast<double>(sizeBefore); -} - -double EdenGCActivityCallback::gcTimeSlice(size_t bytes) -{ - return std::min((static_cast<double>(bytes) / MB) * Options::percentCPUPerMBForEdenTimer(), Options::collectionTimerMaxPercentCPU()); -} - -#else - -EdenGCActivityCallback::EdenGCActivityCallback(Heap* heap) - : GCActivityCallback(heap->vm()) -{ -} - -void EdenGCActivityCallback::doCollection() -{ -} - -double EdenGCActivityCallback::lastGCLength() -{ - return 0; -} - -double EdenGCActivityCallback::deathRate() -{ - return 0; -} - -double EdenGCActivityCallback::gcTimeSlice(size_t) -{ - return 0; -} - -#endif // USE(CF) || PLATFORM(EFL) - -} // namespace JSC diff --git a/Source/JavaScriptCore/heap/EdenGCActivityCallback.h b/Source/JavaScriptCore/heap/EdenGCActivityCallback.h deleted file mode 100644 index 214ab43dc..000000000 --- a/Source/JavaScriptCore/heap/EdenGCActivityCallback.h +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (C) 2014 Apple Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS - * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF - * THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef EdenGCActivityCallback_h -#define EdenGCActivityCallback_h - -#include "GCActivityCallback.h" - -namespace JSC { - -class JS_EXPORT_PRIVATE EdenGCActivityCallback : public GCActivityCallback { -public: - EdenGCActivityCallback(Heap*); - - virtual void doCollection() override; - -protected: -#if USE(CF) - EdenGCActivityCallback(Heap* heap, CFRunLoopRef runLoop) - : GCActivityCallback(heap, runLoop) - { - } -#endif - - virtual double lastGCLength() override; - virtual double gcTimeSlice(size_t bytes) override; - virtual double deathRate() override; -}; - -inline RefPtr<GCActivityCallback> GCActivityCallback::createEdenTimer(Heap* heap) -{ - return s_shouldCreateGCTimer ? adoptRef(new EdenGCActivityCallback(heap)) : nullptr; -} - -} // namespace JSC - -#endif // EdenGCActivityCallback_h diff --git a/Source/JavaScriptCore/heap/FullGCActivityCallback.cpp b/Source/JavaScriptCore/heap/FullGCActivityCallback.cpp deleted file mode 100644 index 07ebbbd22..000000000 --- a/Source/JavaScriptCore/heap/FullGCActivityCallback.cpp +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright (C) 2014 Apple Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS - * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF - * THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include "config.h" -#include "FullGCActivityCallback.h" - -#include "VM.h" - -namespace JSC { - -#if USE(CF) || PLATFORM(EFL) - -#if !PLATFORM(IOS) -const double pagingTimeOut = 0.1; // Time in seconds to allow opportunistic timer to iterate over all blocks to see if the Heap is paged out. -#endif - -FullGCActivityCallback::FullGCActivityCallback(Heap* heap) - : GCActivityCallback(heap) -{ -} - -void FullGCActivityCallback::doCollection() -{ - Heap& heap = m_vm->heap; - m_didSyncGCRecently = false; - -#if !PLATFORM(IOS) - double startTime = WTF::monotonicallyIncreasingTime(); - if (heap.isPagedOut(startTime + pagingTimeOut)) { - cancel(); - heap.increaseLastFullGCLength(pagingTimeOut); - return; - } -#endif - - heap.collect(FullCollection); -} - -double FullGCActivityCallback::lastGCLength() -{ - return m_vm->heap.lastFullGCLength(); -} - -double FullGCActivityCallback::deathRate() -{ - Heap* heap = &m_vm->heap; - size_t sizeBefore = heap->sizeBeforeLastFullCollection(); - size_t sizeAfter = heap->sizeAfterLastFullCollection(); - if (!sizeBefore) - return 1.0; - return static_cast<double>(sizeBefore - sizeAfter) / static_cast<double>(sizeBefore); -} - -double FullGCActivityCallback::gcTimeSlice(size_t bytes) -{ - return std::min((static_cast<double>(bytes) / MB) * Options::percentCPUPerMBForFullTimer(), Options::collectionTimerMaxPercentCPU()); -} - -#else - -FullGCActivityCallback::FullGCActivityCallback(Heap* heap) - : GCActivityCallback(heap) -{ -} - -void FullGCActivityCallback::doCollection() -{ -} - -double FullGCActivityCallback::lastGCLength() -{ - return 0; -} - -double FullGCActivityCallback::deathRate() -{ - return 0; -} - -double FullGCActivityCallback::gcTimeSlice(size_t) -{ - return 0; -} - -#endif // USE(CF) || PLATFORM(EFL) - -} // namespace JSC diff --git a/Source/JavaScriptCore/heap/FullGCActivityCallback.h b/Source/JavaScriptCore/heap/FullGCActivityCallback.h deleted file mode 100644 index e727592e2..000000000 --- a/Source/JavaScriptCore/heap/FullGCActivityCallback.h +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (C) 2014 Apple Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS - * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF - * THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef FullGCActivityCallback_h -#define FullGCActivityCallback_h - -#include "GCActivityCallback.h" - -namespace JSC { - -class JS_EXPORT_PRIVATE FullGCActivityCallback : public GCActivityCallback { -public: - FullGCActivityCallback(Heap*); - - virtual void doCollection() override; - - bool didSyncGCRecently() const { return m_didSyncGCRecently; } - void setDidSyncGCRecently() { m_didSyncGCRecently = true; } - -protected: -#if USE(CF) - FullGCActivityCallback(Heap* heap, CFRunLoopRef runLoop) - : GCActivityCallback(heap, runLoop) - { - } -#endif - - virtual double lastGCLength() override; - virtual double gcTimeSlice(size_t bytes) override; - virtual double deathRate() override; - - bool m_didSyncGCRecently { false }; -}; - -inline RefPtr<FullGCActivityCallback> GCActivityCallback::createFullTimer(Heap* heap) -{ - return s_shouldCreateGCTimer ? adoptRef(new FullGCActivityCallback(heap)) : nullptr; -} - -} // namespace JSC - -#endif // FullGCActivityCallback_h diff --git a/Source/JavaScriptCore/heap/GCActivityCallback.cpp b/Source/JavaScriptCore/heap/GCActivityCallback.cpp deleted file mode 100644 index d324b3731..000000000 --- a/Source/JavaScriptCore/heap/GCActivityCallback.cpp +++ /dev/null @@ -1,173 +0,0 @@ -/* - * Copyright (C) 2010 Apple Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. Neither the name of Apple Inc. ("Apple") nor the names of - * its contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include "config.h" -#include "GCActivityCallback.h" - -#include "Heap.h" -#include "JSLock.h" -#include "JSObject.h" -#include "VM.h" - -#include <wtf/RetainPtr.h> -#include <wtf/WTFThreadData.h> - -#if PLATFORM(EFL) -#include <wtf/MainThread.h> -#endif - -namespace JSC { - -bool GCActivityCallback::s_shouldCreateGCTimer = true; - -#if USE(CF) || PLATFORM(EFL) - -const double timerSlop = 2.0; // Fudge factor to avoid performance cost of resetting timer. - -#if USE(CF) -GCActivityCallback::GCActivityCallback(Heap* heap) - : GCActivityCallback(heap->vm(), CFRunLoopGetCurrent()) -{ -} - -GCActivityCallback::GCActivityCallback(Heap* heap, CFRunLoopRef runLoop) - : GCActivityCallback(heap->vm(), runLoop) -{ -} -#elif PLATFORM(EFL) -GCActivityCallback::GCActivityCallback(Heap* heap) - : GCActivityCallback(heap->vm(), WTF::isMainThread()) -{ -} -#endif - -void GCActivityCallback::doWork() -{ - Heap* heap = &m_vm->heap; - if (!isEnabled()) - return; - - JSLockHolder locker(m_vm); - if (heap->isDeferred()) { - scheduleTimer(0); - return; - } - - doCollection(); -} - -#if USE(CF) -void GCActivityCallback::scheduleTimer(double newDelay) -{ - if (newDelay * timerSlop > m_delay) - return; - double delta = m_delay - newDelay; - m_delay = newDelay; - CFRunLoopTimerSetNextFireDate(m_timer.get(), CFRunLoopTimerGetNextFireDate(m_timer.get()) - delta); -} - -void GCActivityCallback::cancelTimer() -{ - m_delay = s_decade; - CFRunLoopTimerSetNextFireDate(m_timer.get(), CFAbsoluteTimeGetCurrent() + s_decade); -} -#elif PLATFORM(EFL) -void GCActivityCallback::scheduleTimer(double newDelay) -{ - if (newDelay * timerSlop > m_delay) - return; - - stop(); - m_delay = newDelay; - - ASSERT(!m_timer); - m_timer = add(newDelay, this); -} - -void GCActivityCallback::cancelTimer() -{ - m_delay = s_hour; - stop(); -} -#endif - -void GCActivityCallback::didAllocate(size_t bytes) -{ -#if PLATFORM(EFL) - if (!isEnabled()) - return; - - ASSERT(WTF::isMainThread()); -#endif - - // The first byte allocated in an allocation cycle will report 0 bytes to didAllocate. - // We pretend it's one byte so that we don't ignore this allocation entirely. - if (!bytes) - bytes = 1; - double bytesExpectedToReclaim = static_cast<double>(bytes) * deathRate(); - double newDelay = lastGCLength() / gcTimeSlice(bytesExpectedToReclaim); - scheduleTimer(newDelay); -} - -void GCActivityCallback::willCollect() -{ - cancelTimer(); -} - -void GCActivityCallback::cancel() -{ - cancelTimer(); -} - -#else - -GCActivityCallback::GCActivityCallback(Heap* heap) - : GCActivityCallback(heap->vm()) -{ -} - -void GCActivityCallback::doWork() -{ -} - -void GCActivityCallback::didAllocate(size_t) -{ -} - -void GCActivityCallback::willCollect() -{ -} - -void GCActivityCallback::cancel() -{ -} - -#endif - -} - diff --git a/Source/JavaScriptCore/heap/GCActivityCallback.h b/Source/JavaScriptCore/heap/GCActivityCallback.h deleted file mode 100644 index 73d0e8907..000000000 --- a/Source/JavaScriptCore/heap/GCActivityCallback.h +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright (C) 2010 Apple Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. Neither the name of Apple Inc. ("Apple") nor the names of - * its contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef GCActivityCallback_h -#define GCActivityCallback_h - -#include "HeapTimer.h" -#include <wtf/RefPtr.h> - -#if USE(CF) -#include <CoreFoundation/CoreFoundation.h> -#endif - -namespace JSC { - -class FullGCActivityCallback; -class Heap; - -class JS_EXPORT_PRIVATE GCActivityCallback : public HeapTimer, public ThreadSafeRefCounted<GCActivityCallback> { - WTF_MAKE_FAST_ALLOCATED; -public: - static RefPtr<FullGCActivityCallback> createFullTimer(Heap*); - static RefPtr<GCActivityCallback> createEdenTimer(Heap*); - - GCActivityCallback(Heap*); - - virtual void doWork() override; - - virtual void doCollection() = 0; - - virtual void didAllocate(size_t); - virtual void willCollect(); - virtual void cancel(); - bool isEnabled() const { return m_enabled; } - void setEnabled(bool enabled) { m_enabled = enabled; } - - static bool s_shouldCreateGCTimer; - -protected: - virtual double lastGCLength() = 0; - virtual double gcTimeSlice(size_t bytes) = 0; - virtual double deathRate() = 0; - -#if USE(CF) - GCActivityCallback(VM* vm, CFRunLoopRef runLoop) - : HeapTimer(vm, runLoop) - , m_enabled(true) - , m_delay(s_decade) - { - } -#elif PLATFORM(EFL) - static constexpr double s_hour = 3600; - GCActivityCallback(VM* vm, bool flag) - : HeapTimer(vm) - , m_enabled(flag) - , m_delay(s_hour) - { - } -#else - GCActivityCallback(VM* vm) - : HeapTimer(vm) - , m_enabled(true) - { - } -#endif - - bool m_enabled; - -#if USE(CF) -protected: - GCActivityCallback(Heap*, CFRunLoopRef); -#endif -#if USE(CF) || PLATFORM(EFL) -protected: - void cancelTimer(); - void scheduleTimer(double); - -private: - double m_delay; -#endif -}; - -} // namespace JSC - -#endif diff --git a/Source/JavaScriptCore/heap/GCAssertions.h b/Source/JavaScriptCore/heap/GCAssertions.h index fcb135ac5..b0676bfee 100644 --- a/Source/JavaScriptCore/heap/GCAssertions.h +++ b/Source/JavaScriptCore/heap/GCAssertions.h @@ -32,7 +32,7 @@ #if ENABLE(GC_VALIDATION) #define ASSERT_GC_OBJECT_LOOKS_VALID(cell) do { \ RELEASE_ASSERT(cell);\ - RELEASE_ASSERT(cell->structure()->structure() == cell->structure()->structure()->structure()); \ + RELEASE_ASSERT(cell->unvalidatedStructure()->unvalidatedStructure() == cell->unvalidatedStructure()->unvalidatedStructure()->unvalidatedStructure()); \ } while (0) #define ASSERT_GC_OBJECT_INHERITS(object, classInfo) do {\ @@ -45,6 +45,14 @@ #define ASSERT_GC_OBJECT_INHERITS(object, classInfo) do { (void)object; (void)classInfo; } while (0) #endif +#if COMPILER(CLANG) #define STATIC_ASSERT_IS_TRIVIALLY_DESTRUCTIBLE(klass) static_assert(std::is_trivially_destructible<klass>::value, #klass " must have a trivial destructor") +#elif COMPILER(MSVC) +// An earlier verison of the C++11 spec used to call this type trait std::has_trivial_destructor, and that's what MSVC uses. +#define STATIC_ASSERT_IS_TRIVIALLY_DESTRUCTIBLE(klass) static_assert(std::has_trivial_destructor<klass>::value, #klass " must have a trivial destructor") +#else +// This is not enabled on GCC due to http://gcc.gnu.org/bugzilla/show_bug.cgi?id=52702 +#define STATIC_ASSERT_IS_TRIVIALLY_DESTRUCTIBLE(klass) +#endif #endif // GCAssertions_h diff --git a/Source/JavaScriptCore/heap/GCLogging.cpp b/Source/JavaScriptCore/heap/GCLogging.cpp deleted file mode 100644 index b8ed121aa..000000000 --- a/Source/JavaScriptCore/heap/GCLogging.cpp +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Copyright (C) 2014, 2015 Apple Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS - * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF - * THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include "config.h" -#include "GCLogging.h" - -#include "ClassInfo.h" -#include "Heap.h" -#include "HeapIterationScope.h" -#include "JSCell.h" -#include "JSCellInlines.h" -#include <wtf/PrintStream.h> - -namespace JSC { - -const char* GCLogging::levelAsString(Level level) -{ - switch (level) { - case None: - return "None"; - case Basic: - return "Basic"; - case Verbose: - return "Verbose"; - default: - RELEASE_ASSERT_NOT_REACHED(); - return ""; - } -} - -class LoggingFunctor { -public: - LoggingFunctor(SlotVisitor& slotVisitor) - : m_slotVisitor(slotVisitor) - { - m_savedMarkStack.resize(m_slotVisitor.markStack().size()); - m_slotVisitor.markStack().fillVector(m_savedMarkStack); - } - - ~LoggingFunctor() - { - reviveCells(); - } - - IterationStatus operator()(JSCell* cell) - { - m_liveCells.append(cell); - MarkedBlock::blockFor(cell)->clearMarked(cell); - return IterationStatus::Continue; - } - - void log() - { - m_slotVisitor.clearMarkStack(); - for (JSCell* cell : m_liveCells) { - cell->methodTable()->visitChildren(cell, m_slotVisitor); - dataLog("\n", *cell, ":\n", m_slotVisitor); - for (const JSCell* neighbor : m_slotVisitor.markStack()) - MarkedBlock::blockFor(neighbor)->clearMarked(neighbor); - m_slotVisitor.clearMarkStack(); - } - m_slotVisitor.reset(); - } - - void reviveCells() - { - for (JSCell* cell : m_liveCells) - MarkedBlock::blockFor(cell)->setMarked(cell); - - for (const JSCell* cell : m_savedMarkStack) { - m_slotVisitor.markStack().append(cell); - const_cast<JSCell*>(cell)->setRemembered(true); - } - } - - typedef void ReturnType; - - void returnValue() { }; - -private: - Vector<const JSCell*> m_savedMarkStack; - Vector<JSCell*> m_liveCells; - SlotVisitor& m_slotVisitor; -}; - -void GCLogging::dumpObjectGraph(Heap* heap) -{ - LoggingFunctor loggingFunctor(heap->m_slotVisitor); - HeapIterationScope iterationScope(*heap); - heap->objectSpace().forEachLiveCell(iterationScope, loggingFunctor); - loggingFunctor.log(); -} - -} // namespace JSC - -namespace WTF { - -void printInternal(PrintStream& out, JSC::GCLogging::Level level) -{ - switch (level) { - case JSC::GCLogging::Level::None: - out.print("None"); - return; - case JSC::GCLogging::Level::Basic: - out.print("Basic"); - return; - case JSC::GCLogging::Level::Verbose: - out.print("Verbose"); - return; - default: - out.print("Level=", level - JSC::GCLogging::Level::None); - return; - } -} - -} // namespace WTF - diff --git a/Source/JavaScriptCore/heap/GCSegmentedArray.h b/Source/JavaScriptCore/heap/GCSegmentedArray.h deleted file mode 100644 index 8aeba1025..000000000 --- a/Source/JavaScriptCore/heap/GCSegmentedArray.h +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Copyright (C) 2014 Apple Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS - * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF - * THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef GCSegmentedArray_h -#define GCSegmentedArray_h - -#include <wtf/DoublyLinkedList.h> -#include <wtf/Vector.h> - -namespace JSC { - -template <typename T> -class GCArraySegment : public DoublyLinkedListNode<GCArraySegment<T>> { - friend class WTF::DoublyLinkedListNode<GCArraySegment<T>>; -public: - GCArraySegment() - : DoublyLinkedListNode<GCArraySegment<T>>() -#if !ASSERT_DISABLED - , m_top(0) -#endif - { - } - - static GCArraySegment* create(); - static void destroy(GCArraySegment*); - - T* data() - { - return bitwise_cast<T*>(this + 1); - } - - static const size_t blockSize = 4 * KB; - - GCArraySegment* m_prev; - GCArraySegment* m_next; -#if !ASSERT_DISABLED - size_t m_top; -#endif -}; - -template <typename T> class GCSegmentedArrayIterator; - -template <typename T> -class GCSegmentedArray { - friend class GCSegmentedArrayIterator<T>; - friend class GCSegmentedArrayIterator<const T>; -public: - GCSegmentedArray(); - ~GCSegmentedArray(); - - void append(T); - - bool canRemoveLast(); - const T removeLast(); - bool refill(); - - size_t size(); - bool isEmpty(); - - void fillVector(Vector<T>&); - void clear(); - - typedef GCSegmentedArrayIterator<T> iterator; - iterator begin() const { return GCSegmentedArrayIterator<T>(m_segments.head(), m_top); } - iterator end() const { return GCSegmentedArrayIterator<T>(); } - -protected: - template <size_t size> struct CapacityFromSize { - static const size_t value = (size - sizeof(GCArraySegment<T>)) / sizeof(T); - }; - - void expand(); - - size_t postIncTop(); - size_t preDecTop(); - void setTopForFullSegment(); - void setTopForEmptySegment(); - size_t top(); - - void validatePrevious(); - - DoublyLinkedList<GCArraySegment<T>> m_segments; - - JS_EXPORT_PRIVATE static const size_t s_segmentCapacity = CapacityFromSize<GCArraySegment<T>::blockSize>::value; - size_t m_top; - size_t m_numberOfSegments; -}; - -template <typename T> -class GCSegmentedArrayIterator { - friend class GCSegmentedArray<T>; -public: - GCSegmentedArrayIterator() - : m_currentSegment(0) - , m_currentOffset(0) - { - } - - T& get() { return m_currentSegment->data()[m_currentOffset]; } - T& operator*() { return get(); } - T& operator->() { return get(); } - - bool operator==(const GCSegmentedArrayIterator& other) - { - return m_currentSegment == other.m_currentSegment && m_currentOffset == other.m_currentOffset; - } - - bool operator!=(const GCSegmentedArrayIterator& other) - { - return !(*this == other); - } - - GCSegmentedArrayIterator& operator++() - { - ASSERT(m_currentSegment); - - m_currentOffset++; - - if (m_currentOffset >= m_offsetLimit) { - m_currentSegment = m_currentSegment->next(); - m_currentOffset = 0; - m_offsetLimit = GCSegmentedArray<T>::s_segmentCapacity; - } - - return *this; - } - -private: - GCSegmentedArrayIterator(GCArraySegment<T>* start, size_t top) - : m_currentSegment(start) - , m_currentOffset(0) - , m_offsetLimit(top) - { - if (!m_offsetLimit) - m_currentSegment = nullptr; - } - - GCArraySegment<T>* m_currentSegment; - size_t m_currentOffset; - size_t m_offsetLimit; -}; - -} // namespace JSC - -#endif // GCSegmentedArray_h diff --git a/Source/JavaScriptCore/heap/GCSegmentedArrayInlines.h b/Source/JavaScriptCore/heap/GCSegmentedArrayInlines.h deleted file mode 100644 index 88e43cc9b..000000000 --- a/Source/JavaScriptCore/heap/GCSegmentedArrayInlines.h +++ /dev/null @@ -1,230 +0,0 @@ -/* - * Copyright (C) 2014 Apple Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS - * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF - * THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef GCSegmentedArrayInlines_h -#define GCSegmentedArrayInlines_h - -#include "GCSegmentedArray.h" - -namespace JSC { - -template <typename T> -GCSegmentedArray<T>::GCSegmentedArray() - : m_top(0) - , m_numberOfSegments(0) -{ - m_segments.push(GCArraySegment<T>::create()); - m_numberOfSegments++; -} - -template <typename T> -GCSegmentedArray<T>::~GCSegmentedArray() -{ - ASSERT(m_numberOfSegments == 1); - ASSERT(m_segments.size() == 1); - GCArraySegment<T>::destroy(m_segments.removeHead()); - m_numberOfSegments--; - ASSERT(!m_numberOfSegments); - ASSERT(!m_segments.size()); -} - -template <typename T> -void GCSegmentedArray<T>::clear() -{ - if (!m_segments.head()) - return; - GCArraySegment<T>* next; - for (GCArraySegment<T>* current = m_segments.head(); current->next(); current = next) { - next = current->next(); - m_segments.remove(current); - GCArraySegment<T>::destroy(current); - } - m_top = 0; - m_numberOfSegments = 1; -#if !ASSERT_DISABLED - m_segments.head()->m_top = 0; -#endif -} - -template <typename T> -void GCSegmentedArray<T>::expand() -{ - ASSERT(m_segments.head()->m_top == s_segmentCapacity); - - GCArraySegment<T>* nextSegment = GCArraySegment<T>::create(); - m_numberOfSegments++; - -#if !ASSERT_DISABLED - nextSegment->m_top = 0; -#endif - - m_segments.push(nextSegment); - setTopForEmptySegment(); - validatePrevious(); -} - -template <typename T> -bool GCSegmentedArray<T>::refill() -{ - validatePrevious(); - if (top()) - return true; - GCArraySegment<T>::destroy(m_segments.removeHead()); - ASSERT(m_numberOfSegments > 1); - m_numberOfSegments--; - setTopForFullSegment(); - validatePrevious(); - return true; -} - -template <typename T> -void GCSegmentedArray<T>::fillVector(Vector<T>& vector) -{ - ASSERT(vector.size() == size()); - - GCArraySegment<T>* currentSegment = m_segments.head(); - if (!currentSegment) - return; - - unsigned count = 0; - for (unsigned i = 0; i < m_top; ++i) { - ASSERT(currentSegment->data()[i]); - vector[count++] = currentSegment->data()[i]; - } - - currentSegment = currentSegment->next(); - while (currentSegment) { - for (unsigned i = 0; i < s_segmentCapacity; ++i) { - ASSERT(currentSegment->data()[i]); - vector[count++] = currentSegment->data()[i]; - } - currentSegment = currentSegment->next(); - } -} - -template <typename T> -inline GCArraySegment<T>* GCArraySegment<T>::create() -{ - return new (NotNull, fastMalloc(blockSize)) GCArraySegment<T>(); -} - -template <typename T> -inline void GCArraySegment<T>::destroy(GCArraySegment* segment) -{ - segment->~GCArraySegment(); - fastFree(segment); -} - -template <typename T> -inline size_t GCSegmentedArray<T>::postIncTop() -{ - size_t result = m_top++; - ASSERT(result == m_segments.head()->m_top++); - return result; -} - -template <typename T> -inline size_t GCSegmentedArray<T>::preDecTop() -{ - size_t result = --m_top; - ASSERT(result == --m_segments.head()->m_top); - return result; -} - -template <typename T> -inline void GCSegmentedArray<T>::setTopForFullSegment() -{ - ASSERT(m_segments.head()->m_top == s_segmentCapacity); - m_top = s_segmentCapacity; -} - -template <typename T> -inline void GCSegmentedArray<T>::setTopForEmptySegment() -{ - ASSERT(!m_segments.head()->m_top); - m_top = 0; -} - -template <typename T> -inline size_t GCSegmentedArray<T>::top() -{ - ASSERT(m_top == m_segments.head()->m_top); - return m_top; -} - -template <typename T> -#if ASSERT_DISABLED -inline void GCSegmentedArray<T>::validatePrevious() { } -#else -inline void GCSegmentedArray<T>::validatePrevious() -{ - unsigned count = 0; - for (GCArraySegment<T>* current = m_segments.head(); current; current = current->next()) - count++; - ASSERT(m_segments.size() == m_numberOfSegments); -} -#endif - -template <typename T> -inline void GCSegmentedArray<T>::append(T value) -{ - if (m_top == s_segmentCapacity) - expand(); - m_segments.head()->data()[postIncTop()] = value; -} - -template <typename T> -inline bool GCSegmentedArray<T>::canRemoveLast() -{ - return !!m_top; -} - -template <typename T> -inline const T GCSegmentedArray<T>::removeLast() -{ - return m_segments.head()->data()[preDecTop()]; -} - -template <typename T> -inline bool GCSegmentedArray<T>::isEmpty() -{ - if (m_top) - return false; - if (m_segments.head()->next()) { - ASSERT(m_segments.head()->next()->m_top == s_segmentCapacity); - return false; - } - return true; -} - -template <typename T> -inline size_t GCSegmentedArray<T>::size() -{ - return m_top + s_segmentCapacity * (m_numberOfSegments - 1); -} - -} // namespace JSC - -#endif // GCSegmentedArrayInlines_h diff --git a/Source/JavaScriptCore/heap/GCThread.cpp b/Source/JavaScriptCore/heap/GCThread.cpp index bf562b5cf..50f02ce19 100644 --- a/Source/JavaScriptCore/heap/GCThread.cpp +++ b/Source/JavaScriptCore/heap/GCThread.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012, 2015 Apple Inc. All rights reserved. + * Copyright (C) 2012 Apple Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -29,17 +29,17 @@ #include "CopyVisitor.h" #include "CopyVisitorInlines.h" #include "GCThreadSharedData.h" -#include "JSCInlines.h" #include "SlotVisitor.h" #include <wtf/MainThread.h> +#include <wtf/PassOwnPtr.h> namespace JSC { -GCThread::GCThread(GCThreadSharedData& shared, std::unique_ptr<SlotVisitor> slotVisitor, std::unique_ptr<CopyVisitor> copyVisitor) +GCThread::GCThread(GCThreadSharedData& shared, SlotVisitor* slotVisitor, CopyVisitor* copyVisitor) : m_threadID(0) , m_shared(shared) - , m_slotVisitor(WTF::move(slotVisitor)) - , m_copyVisitor(WTF::move(copyVisitor)) + , m_slotVisitor(WTF::adoptPtr(slotVisitor)) + , m_copyVisitor(WTF::adoptPtr(copyVisitor)) { } @@ -69,12 +69,12 @@ CopyVisitor* GCThread::copyVisitor() GCPhase GCThread::waitForNextPhase() { - std::unique_lock<Lock> lock(m_shared.m_phaseMutex); + std::unique_lock<std::mutex> lock(m_shared.m_phaseMutex); m_shared.m_phaseConditionVariable.wait(lock, [this] { return !m_shared.m_gcThreadsShouldWait; }); m_shared.m_numberOfActiveGCThreads--; if (!m_shared.m_numberOfActiveGCThreads) - m_shared.m_activityConditionVariable.notifyOne(); + m_shared.m_activityConditionVariable.notify_one(); m_shared.m_phaseConditionVariable.wait(lock, [this] { return m_shared.m_currentPhase != NoPhase; }); m_shared.m_numberOfActiveGCThreads++; @@ -90,7 +90,7 @@ void GCThread::gcThreadMain() // Wait for the main thread to finish creating and initializing us. The main thread grabs this lock before // creating this thread. We aren't guaranteed to have a valid threadID until the main thread releases this lock. { - std::lock_guard<Lock> lock(m_shared.m_phaseMutex); + std::lock_guard<std::mutex> lock(m_shared.m_phaseMutex); } { ParallelModeEnabler enabler(*m_slotVisitor); @@ -115,9 +115,6 @@ void GCThread::gcThreadMain() // all of the blocks that the GCThreads borrowed have been returned. doneCopying() // returns our borrowed CopiedBlock, allowing the copying phase to finish. m_copyVisitor->doneCopying(); - - WTF::releaseFastMallocFreeMemoryForThisThread(); - break; case NoPhase: RELEASE_ASSERT_NOT_REACHED(); diff --git a/Source/JavaScriptCore/heap/GCThread.h b/Source/JavaScriptCore/heap/GCThread.h index b7277992c..0d218f975 100644 --- a/Source/JavaScriptCore/heap/GCThread.h +++ b/Source/JavaScriptCore/heap/GCThread.h @@ -28,6 +28,7 @@ #include <GCThreadSharedData.h> #include <wtf/Deque.h> +#include <wtf/OwnPtr.h> #include <wtf/Threading.h> namespace JSC { @@ -38,7 +39,7 @@ class SlotVisitor; class GCThread { public: - GCThread(GCThreadSharedData&, std::unique_ptr<SlotVisitor>, std::unique_ptr<CopyVisitor>); + GCThread(GCThreadSharedData&, SlotVisitor*, CopyVisitor*); SlotVisitor* slotVisitor(); CopyVisitor* copyVisitor(); @@ -53,8 +54,8 @@ private: ThreadIdentifier m_threadID; GCThreadSharedData& m_shared; - std::unique_ptr<SlotVisitor> m_slotVisitor; - std::unique_ptr<CopyVisitor> m_copyVisitor; + OwnPtr<SlotVisitor> m_slotVisitor; + OwnPtr<CopyVisitor> m_copyVisitor; }; } // namespace JSC diff --git a/Source/JavaScriptCore/heap/GCThreadSharedData.cpp b/Source/JavaScriptCore/heap/GCThreadSharedData.cpp index 3fad8734b..09143a15f 100644 --- a/Source/JavaScriptCore/heap/GCThreadSharedData.cpp +++ b/Source/JavaScriptCore/heap/GCThreadSharedData.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2009, 2011, 2015 Apple Inc. All rights reserved. + * Copyright (C) 2009, 2011 Apple Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -29,11 +29,10 @@ #include "CopyVisitor.h" #include "CopyVisitorInlines.h" #include "GCThread.h" +#include "VM.h" #include "MarkStack.h" -#include "JSCInlines.h" #include "SlotVisitor.h" #include "SlotVisitorInlines.h" -#include "VM.h" namespace JSC { @@ -73,7 +72,7 @@ GCThreadSharedData::GCThreadSharedData(VM* vm) : m_vm(vm) , m_copiedSpace(&vm->heap.m_storageSpace) , m_shouldHashCons(false) - , m_sharedMarkStack() + , m_sharedMarkStack(vm->heap.blockAllocator()) , m_numberOfActiveParallelMarkers(0) , m_parallelMarkersShouldExit(false) , m_copyIndex(0) @@ -81,12 +80,15 @@ GCThreadSharedData::GCThreadSharedData(VM* vm) , m_gcThreadsShouldWait(false) , m_currentPhase(NoPhase) { + m_copyLock.Init(); #if ENABLE(PARALLEL_GC) // Grab the lock so the new GC threads can be properly initialized before they start running. - std::unique_lock<Lock> lock(m_phaseMutex); + std::unique_lock<std::mutex> lock(m_phaseMutex); for (unsigned i = 1; i < Options::numberOfGCMarkers(); ++i) { m_numberOfActiveGCThreads++; - GCThread* newThread = new GCThread(*this, std::make_unique<SlotVisitor>(*this), std::make_unique<CopyVisitor>(*this)); + SlotVisitor* slotVisitor = new SlotVisitor(*this); + CopyVisitor* copyVisitor = new CopyVisitor(*this); + GCThread* newThread = new GCThread(*this, slotVisitor, copyVisitor); ThreadIdentifier threadID = createThread(GCThread::gcThreadStartFunc, newThread, "JavaScriptCore::Marking"); newThread->initializeThreadID(threadID); m_gcThreads.append(newThread); @@ -102,13 +104,13 @@ GCThreadSharedData::~GCThreadSharedData() #if ENABLE(PARALLEL_GC) // Destroy our marking threads. { - std::lock_guard<Lock> markingLock(m_markingMutex); - std::lock_guard<Lock> phaseLock(m_phaseMutex); + std::lock_guard<std::mutex> markingLock(m_markingMutex); + std::lock_guard<std::mutex> phaseLock(m_phaseMutex); ASSERT(m_currentPhase == NoPhase); m_parallelMarkersShouldExit = true; m_gcThreadsShouldWait = false; m_currentPhase = Exit; - m_phaseConditionVariable.notifyAll(); + m_phaseConditionVariable.notify_all(); } for (unsigned i = 0; i < m_gcThreads.size(); ++i) { waitForThreadCompletion(m_gcThreads[i]->threadID()); @@ -116,11 +118,16 @@ GCThreadSharedData::~GCThreadSharedData() } #endif } - + void GCThreadSharedData::reset() { ASSERT(m_sharedMarkStack.isEmpty()); +#if ENABLE(PARALLEL_GC) + m_opaqueRoots.clear(); +#else + ASSERT(m_opaqueRoots.isEmpty()); +#endif m_weakReferenceHarvesters.removeAll(); if (m_shouldHashCons) { @@ -131,34 +138,27 @@ void GCThreadSharedData::reset() void GCThreadSharedData::startNextPhase(GCPhase phase) { - std::lock_guard<Lock> lock(m_phaseMutex); + std::lock_guard<std::mutex> lock(m_phaseMutex); ASSERT(!m_gcThreadsShouldWait); ASSERT(m_currentPhase == NoPhase); m_gcThreadsShouldWait = true; m_currentPhase = phase; - m_phaseConditionVariable.notifyAll(); + m_phaseConditionVariable.notify_all(); } void GCThreadSharedData::endCurrentPhase() { ASSERT(m_gcThreadsShouldWait); - std::unique_lock<Lock> lock(m_phaseMutex); + std::unique_lock<std::mutex> lock(m_phaseMutex); m_currentPhase = NoPhase; m_gcThreadsShouldWait = false; - m_phaseConditionVariable.notifyAll(); + m_phaseConditionVariable.notify_all(); m_activityConditionVariable.wait(lock, [this] { return !m_numberOfActiveGCThreads; }); } void GCThreadSharedData::didStartMarking() { - if (m_vm->heap.operationInProgress() == FullCollection) { -#if ENABLE(PARALLEL_GC) - m_opaqueRoots.clear(); -#else - ASSERT(m_opaqueRoots.isEmpty()); -#endif -} - std::lock_guard<Lock> lock(m_markingMutex); + std::lock_guard<std::mutex> lock(m_markingMutex); m_parallelMarkersShouldExit = false; startNextPhase(Mark); } @@ -166,9 +166,9 @@ void GCThreadSharedData::didStartMarking() void GCThreadSharedData::didFinishMarking() { { - std::lock_guard<Lock> lock(m_markingMutex); + std::lock_guard<std::mutex> lock(m_markingMutex); m_parallelMarkersShouldExit = true; - m_markingConditionVariable.notifyAll(); + m_markingConditionVariable.notify_all(); } ASSERT(m_currentPhase == Mark); @@ -178,7 +178,7 @@ void GCThreadSharedData::didFinishMarking() void GCThreadSharedData::didStartCopying() { { - LockHolder locker(&m_copyLock); + SpinLockHolder locker(&m_copyLock); if (m_vm->heap.operationInProgress() == EdenCollection) { // Reset the vector to be empty, but don't throw away the backing store. m_blocksToCopy.shrink(0); diff --git a/Source/JavaScriptCore/heap/GCThreadSharedData.h b/Source/JavaScriptCore/heap/GCThreadSharedData.h index ae0cdfb2e..915c2c991 100644 --- a/Source/JavaScriptCore/heap/GCThreadSharedData.h +++ b/Source/JavaScriptCore/heap/GCThreadSharedData.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2009, 2011, 2015 Apple Inc. All rights reserved. + * Copyright (C) 2009, 2011 Apple Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -32,18 +32,16 @@ #include "UnconditionalFinalizer.h" #include "WeakReferenceHarvester.h" #include <condition_variable> -#include <wtf/Condition.h> #include <wtf/HashSet.h> -#include <wtf/Lock.h> +#include <wtf/TCSpinLock.h> #include <wtf/Vector.h> namespace JSC { -class CopiedBlock; -class CopiedSpace; -class CopyVisitor; class GCThread; class VM; +class CopiedSpace; +class CopyVisitor; enum GCPhase { NoPhase, @@ -53,7 +51,6 @@ enum GCPhase { }; class GCThreadSharedData { - WTF_MAKE_FAST_ALLOCATED; public: GCThreadSharedData(VM*); ~GCThreadSharedData(); @@ -89,23 +86,23 @@ private: Vector<GCThread*> m_gcThreads; - Lock m_markingMutex; - Condition m_markingConditionVariable; + std::mutex m_markingMutex; + std::condition_variable m_markingConditionVariable; MarkStackArray m_sharedMarkStack; unsigned m_numberOfActiveParallelMarkers; bool m_parallelMarkersShouldExit; - Lock m_opaqueRootsMutex; + Mutex m_opaqueRootsLock; HashSet<void*> m_opaqueRoots; - Lock m_copyLock; + SpinLock m_copyLock; Vector<CopiedBlock*> m_blocksToCopy; size_t m_copyIndex; static const size_t s_blockFragmentLength = 32; - Lock m_phaseMutex; - Condition m_phaseConditionVariable; - Condition m_activityConditionVariable; + std::mutex m_phaseMutex; + std::condition_variable m_phaseConditionVariable; + std::condition_variable m_activityConditionVariable; unsigned m_numberOfActiveGCThreads; bool m_gcThreadsShouldWait; GCPhase m_currentPhase; @@ -116,7 +113,7 @@ private: inline void GCThreadSharedData::getNextBlocksToCopy(size_t& start, size_t& end) { - LockHolder locker(&m_copyLock); + SpinLockHolder locker(&m_copyLock); start = m_copyIndex; end = std::min(m_blocksToCopy.size(), m_copyIndex + s_blockFragmentLength); m_copyIndex = end; diff --git a/Source/JavaScriptCore/heap/Handle.h b/Source/JavaScriptCore/heap/Handle.h index c924e041d..28ac30cd9 100644 --- a/Source/JavaScriptCore/heap/Handle.h +++ b/Source/JavaScriptCore/heap/Handle.h @@ -52,7 +52,9 @@ class HandleBase { public: bool operator!() const { return !m_slot || !*m_slot; } - explicit operator bool() const { return m_slot && *m_slot; } + // This conversion operator allows implicit conversion to bool but not to other integer types. + typedef JSValue (HandleBase::*UnspecifiedBoolType); + operator UnspecifiedBoolType*() const { return (m_slot && *m_slot) ? reinterpret_cast<UnspecifiedBoolType*>(1) : 0; } HandleSlot slot() const { return m_slot; } diff --git a/Source/JavaScriptCore/heap/HandleBlock.h b/Source/JavaScriptCore/heap/HandleBlock.h index fb8c10511..962d37c5e 100644 --- a/Source/JavaScriptCore/heap/HandleBlock.h +++ b/Source/JavaScriptCore/heap/HandleBlock.h @@ -26,7 +26,7 @@ #ifndef HandleBlock_h #define HandleBlock_h -#include <wtf/DoublyLinkedList.h> +#include "HeapBlock.h" namespace JSC { @@ -34,11 +34,9 @@ class DeadBlock; class HandleSet; class HandleNode; -class HandleBlock : public DoublyLinkedListNode<HandleBlock> { - friend class WTF::DoublyLinkedListNode<HandleBlock>; +class HandleBlock : public HeapBlock<HandleBlock> { public: - static HandleBlock* create(HandleSet*); - static void destroy(HandleBlock*); + static HandleBlock* create(DeadBlock*, HandleSet*); static HandleBlock* blockFor(HandleNode*); static const size_t blockSize = 4 * KB; @@ -50,15 +48,13 @@ public: unsigned nodeCapacity(); private: - HandleBlock(HandleSet*); + HandleBlock(Region*, HandleSet*); char* payload(); char* payloadEnd(); static const size_t s_blockMask = ~(blockSize - 1); - HandleBlock* m_prev; - HandleBlock* m_next; HandleSet* m_handleSet; }; diff --git a/Source/JavaScriptCore/heap/HandleBlockInlines.h b/Source/JavaScriptCore/heap/HandleBlockInlines.h index 9e29bffd1..7c771935e 100644 --- a/Source/JavaScriptCore/heap/HandleBlockInlines.h +++ b/Source/JavaScriptCore/heap/HandleBlockInlines.h @@ -26,31 +26,26 @@ #ifndef HandleBlockInlines_h #define HandleBlockInlines_h +#include "BlockAllocator.h" #include "HandleBlock.h" -#include <wtf/FastMalloc.h> namespace JSC { -inline HandleBlock* HandleBlock::create(HandleSet* handleSet) +inline HandleBlock* HandleBlock::create(DeadBlock* block, HandleSet* handleSet) { - return new (NotNull, fastAlignedMalloc(blockSize, blockSize)) HandleBlock(handleSet); + Region* region = block->region(); + return new (NotNull, block) HandleBlock(region, handleSet); } -inline void HandleBlock::destroy(HandleBlock* block) -{ - block->~HandleBlock(); - fastAlignedFree(block); -} - -inline HandleBlock::HandleBlock(HandleSet* handleSet) - : DoublyLinkedListNode<HandleBlock>() +inline HandleBlock::HandleBlock(Region* region, HandleSet* handleSet) + : HeapBlock<HandleBlock>(region) , m_handleSet(handleSet) { } inline char* HandleBlock::payloadEnd() { - return reinterpret_cast<char*>(this) + blockSize; + return reinterpret_cast<char*>(this) + region()->blockSize(); } inline char* HandleBlock::payload() diff --git a/Source/JavaScriptCore/heap/HandleSet.cpp b/Source/JavaScriptCore/heap/HandleSet.cpp index dec8370eb..fdb554448 100644 --- a/Source/JavaScriptCore/heap/HandleSet.cpp +++ b/Source/JavaScriptCore/heap/HandleSet.cpp @@ -30,13 +30,14 @@ #include "HandleBlockInlines.h" #include "HeapRootVisitor.h" #include "JSObject.h" -#include "JSCInlines.h" +#include "Operations.h" #include <wtf/DataLog.h> namespace JSC { HandleSet::HandleSet(VM* vm) : m_vm(vm) + , m_nextToFinalize(0) { grow(); } @@ -44,12 +45,12 @@ HandleSet::HandleSet(VM* vm) HandleSet::~HandleSet() { while (!m_blockList.isEmpty()) - HandleBlock::destroy(m_blockList.removeHead()); + m_vm->heap.blockAllocator().deallocate(HandleBlock::destroy(m_blockList.removeHead())); } void HandleSet::grow() { - HandleBlock* newBlock = HandleBlock::create(this); + HandleBlock* newBlock = HandleBlock::create(m_vm->heap.blockAllocator().allocate<HandleBlock>(), this); m_blockList.append(newBlock); for (int i = newBlock->nodeCapacity() - 1; i >= 0; --i) { @@ -72,6 +73,10 @@ void HandleSet::visitStrongHandles(HeapRootVisitor& heapRootVisitor) void HandleSet::writeBarrier(HandleSlot slot, const JSValue& value) { + // Forbid assignment to handles during the finalization phase, since it would violate many GC invariants. + // File a bug with stack trace if you hit this. + RELEASE_ASSERT(!m_nextToFinalize); + if (!value == !*slot && slot->isCell() == value.isCell()) return; diff --git a/Source/JavaScriptCore/heap/HandleSet.h b/Source/JavaScriptCore/heap/HandleSet.h index c4caf9de9..58251f66a 100644 --- a/Source/JavaScriptCore/heap/HandleSet.h +++ b/Source/JavaScriptCore/heap/HandleSet.h @@ -100,6 +100,7 @@ private: SentinelLinkedList<Node> m_strongList; SentinelLinkedList<Node> m_immediateList; SinglyLinkedList<Node> m_freeList; + Node* m_nextToFinalize; }; inline HandleSet* HandleSet::heapFor(HandleSlot handle) @@ -124,6 +125,10 @@ inline HandleSet::Node* HandleSet::toNode(HandleSlot handle) inline HandleSlot HandleSet::allocate() { + // Forbid assignment to handles during the finalization phase, since it would violate many GC invariants. + // File a bug with stack trace if you hit this. + RELEASE_ASSERT(!m_nextToFinalize); + if (m_freeList.isEmpty()) grow(); @@ -136,6 +141,11 @@ inline HandleSlot HandleSet::allocate() inline void HandleSet::deallocate(HandleSlot handle) { HandleSet::Node* node = toNode(handle); + if (node == m_nextToFinalize) { + ASSERT(m_nextToFinalize->next()); + m_nextToFinalize = m_nextToFinalize->next(); + } + SentinelLinkedList<HandleSet::Node>::remove(node); m_freeList.push(node); } diff --git a/Source/JavaScriptCore/heap/HandleStack.cpp b/Source/JavaScriptCore/heap/HandleStack.cpp index 178bbccf3..41b2ada5f 100644 --- a/Source/JavaScriptCore/heap/HandleStack.cpp +++ b/Source/JavaScriptCore/heap/HandleStack.cpp @@ -28,7 +28,7 @@ #include "HeapRootVisitor.h" #include "JSObject.h" -#include "JSCInlines.h" +#include "Operations.h" namespace JSC { diff --git a/Source/JavaScriptCore/heap/HandleStack.h b/Source/JavaScriptCore/heap/HandleStack.h index 8df8684ec..a7ce97650 100644 --- a/Source/JavaScriptCore/heap/HandleStack.h +++ b/Source/JavaScriptCore/heap/HandleStack.h @@ -53,7 +53,7 @@ public: void visit(HeapRootVisitor&); private: - JS_EXPORT_PRIVATE void grow(); + void grow(); void zapTo(Frame&); HandleSlot findFirstAfter(HandleSlot); diff --git a/Source/JavaScriptCore/heap/Heap.cpp b/Source/JavaScriptCore/heap/Heap.cpp index 5a805ae2c..26ec23980 100644 --- a/Source/JavaScriptCore/heap/Heap.cpp +++ b/Source/JavaScriptCore/heap/Heap.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2011, 2013, 2014 Apple Inc. All rights reserved. + * Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2011, 2013 Apple Inc. All rights reserved. * Copyright (C) 2007 Eric Seidel <eric@webkit.org> * * This library is free software; you can redistribute it and/or @@ -27,32 +27,26 @@ #include "CopiedSpaceInlines.h" #include "CopyVisitorInlines.h" #include "DFGWorklist.h" -#include "EdenGCActivityCallback.h" -#include "FullGCActivityCallback.h" +#include "DelayedReleaseScope.h" #include "GCActivityCallback.h" #include "GCIncomingRefCountedSetInlines.h" #include "HeapIterationScope.h" #include "HeapRootVisitor.h" #include "HeapStatistics.h" -#include "HeapVerifier.h" #include "IncrementalSweeper.h" #include "Interpreter.h" #include "JSGlobalObject.h" #include "JSLock.h" #include "JSONObject.h" -#include "JSCInlines.h" -#include "JSVirtualMachineInternal.h" +#include "Operations.h" #include "RecursiveAllocationScope.h" -#include "RegExpCache.h" #include "Tracing.h" -#include "TypeProfilerLog.h" #include "UnlinkedCodeBlock.h" #include "VM.h" #include "WeakSetInlines.h" #include <algorithm> #include <wtf/RAMSize.h> #include <wtf/CurrentTime.h> -#include <wtf/ProcessID.h> using namespace std; using namespace JSC; @@ -81,129 +75,82 @@ static type name arguments; struct GCTimer { GCTimer(const char* name) - : name(name) + : m_time(0) + , m_min(100000000) + , m_max(0) + , m_count(0) + , m_name(name) { } ~GCTimer() { - logData(allCollectionData, "(All)"); - logData(edenCollectionData, "(Eden)"); - logData(fullCollectionData, "(Full)"); + dataLogF("%s: %.2lfms (avg. %.2lf, min. %.2lf, max. %.2lf)\n", m_name, m_time * 1000, m_time * 1000 / m_count, m_min*1000, m_max*1000); } - - struct TimeRecord { - TimeRecord() - : time(0) - , min(std::numeric_limits<double>::infinity()) - , max(0) - , count(0) - { - } - - double time; - double min; - double max; - size_t count; - }; - - void logData(const TimeRecord& data, const char* extra) - { - dataLogF("[%d] %s (Parent: %s) %s: %.2lfms (avg. %.2lf, min. %.2lf, max. %.2lf, count %lu)\n", - getCurrentProcessID(), - name, - parent ? parent->name : "nullptr", - extra, - data.time * 1000, - data.time * 1000 / data.count, - data.min * 1000, - data.max * 1000, - data.count); - } - - void updateData(TimeRecord& data, double duration) - { - if (duration < data.min) - data.min = duration; - if (duration > data.max) - data.max = duration; - data.count++; - data.time += duration; - } - - void didFinishPhase(HeapOperation collectionType, double duration) - { - TimeRecord& data = collectionType == EdenCollection ? edenCollectionData : fullCollectionData; - updateData(data, duration); - updateData(allCollectionData, duration); - } - - static GCTimer* s_currentGlobalTimer; - - TimeRecord allCollectionData; - TimeRecord fullCollectionData; - TimeRecord edenCollectionData; - const char* name; - GCTimer* parent { nullptr }; + double m_time; + double m_min; + double m_max; + size_t m_count; + const char* m_name; }; -GCTimer* GCTimer::s_currentGlobalTimer = nullptr; - struct GCTimerScope { - GCTimerScope(GCTimer& timer, HeapOperation collectionType) - : timer(timer) - , start(WTF::monotonicallyIncreasingTime()) - , collectionType(collectionType) + GCTimerScope(GCTimer* timer) + : m_timer(timer) + , m_start(WTF::monotonicallyIncreasingTime()) { - timer.parent = GCTimer::s_currentGlobalTimer; - GCTimer::s_currentGlobalTimer = &timer; } ~GCTimerScope() { - double delta = WTF::monotonicallyIncreasingTime() - start; - timer.didFinishPhase(collectionType, delta); - GCTimer::s_currentGlobalTimer = timer.parent; + double delta = WTF::monotonicallyIncreasingTime() - m_start; + if (delta < m_timer->m_min) + m_timer->m_min = delta; + if (delta > m_timer->m_max) + m_timer->m_max = delta; + m_timer->m_count++; + m_timer->m_time += delta; } - GCTimer& timer; - double start; - HeapOperation collectionType; + GCTimer* m_timer; + double m_start; }; struct GCCounter { GCCounter(const char* name) - : name(name) - , count(0) - , total(0) - , min(10000000) - , max(0) + : m_name(name) + , m_count(0) + , m_total(0) + , m_min(10000000) + , m_max(0) { } - void add(size_t amount) + void count(size_t amount) { - count++; - total += amount; - if (amount < min) - min = amount; - if (amount > max) - max = amount; + m_count++; + m_total += amount; + if (amount < m_min) + m_min = amount; + if (amount > m_max) + m_max = amount; } ~GCCounter() { - dataLogF("[%d] %s: %zu values (avg. %zu, min. %zu, max. %zu)\n", getCurrentProcessID(), name, total, total / count, min, max); + dataLogF("%s: %zu values (avg. %zu, min. %zu, max. %zu)\n", m_name, m_total, m_total / m_count, m_min, m_max); } - const char* name; - size_t count; - size_t total; - size_t min; - size_t max; + const char* m_name; + size_t m_count; + size_t m_total; + size_t m_min; + size_t m_max; }; -#define GCPHASE(name) DEFINE_GC_LOGGING_GLOBAL(GCTimer, name##Timer, (#name)); GCTimerScope name##TimerScope(name##Timer, m_operationInProgress) -#define GCCOUNTER(name, value) do { DEFINE_GC_LOGGING_GLOBAL(GCCounter, name##Counter, (#name)); name##Counter.add(value); } while (false) +#define GCPHASE(name) DEFINE_GC_LOGGING_GLOBAL(GCTimer, name##Timer, (#name)); GCTimerScope name##TimerScope(&name##Timer) +#define COND_GCPHASE(cond, name1, name2) DEFINE_GC_LOGGING_GLOBAL(GCTimer, name1##Timer, (#name1)); DEFINE_GC_LOGGING_GLOBAL(GCTimer, name2##Timer, (#name2)); GCTimerScope name1##CondTimerScope(cond ? &name1##Timer : &name2##Timer) +#define GCCOUNTER(name, value) do { DEFINE_GC_LOGGING_GLOBAL(GCCounter, name##Counter, (#name)); name##Counter.count(value); } while (false) #else #define GCPHASE(name) do { } while (false) +#define COND_GCPHASE(cond, name1, name2) do { } while (false) #define GCCOUNTER(name, value) do { } while (false) #endif @@ -231,7 +178,7 @@ static inline bool isValidSharedInstanceThreadState(VM* vm) static inline bool isValidThreadState(VM* vm) { - if (vm->atomicStringTable() != wtfThreadData().atomicStringTable()) + if (vm->identifierTable != wtfThreadData().currentIdentifierTable()) return false; if (vm->isSharedInstance() && !isValidSharedInstanceThreadState(vm)) @@ -241,17 +188,12 @@ static inline bool isValidThreadState(VM* vm) } struct MarkObject : public MarkedBlock::VoidFunctor { - inline void visit(JSCell* cell) + void operator()(JSCell* cell) { if (cell->isZapped()) return; Heap::heap(cell)->setMarked(cell); } - IterationStatus operator()(JSCell* cell) - { - visit(cell); - return IterationStatus::Continue; - } }; struct Count : public MarkedBlock::CountFunctor { @@ -259,36 +201,30 @@ struct Count : public MarkedBlock::CountFunctor { }; struct CountIfGlobalObject : MarkedBlock::CountFunctor { - inline void visit(JSCell* cell) - { + void operator()(JSCell* cell) { if (!cell->isObject()) return; if (!asObject(cell)->isGlobalObject()) return; count(1); } - IterationStatus operator()(JSCell* cell) - { - visit(cell); - return IterationStatus::Continue; - } }; class RecordType { public: - typedef std::unique_ptr<TypeCountSet> ReturnType; + typedef PassOwnPtr<TypeCountSet> ReturnType; RecordType(); - IterationStatus operator()(JSCell*); + void operator()(JSCell*); ReturnType returnValue(); private: const char* typeName(JSCell*); - std::unique_ptr<TypeCountSet> m_typeCountSet; + OwnPtr<TypeCountSet> m_typeCountSet; }; inline RecordType::RecordType() - : m_typeCountSet(std::make_unique<TypeCountSet>()) + : m_typeCountSet(adoptPtr(new TypeCountSet)) { } @@ -300,40 +236,35 @@ inline const char* RecordType::typeName(JSCell* cell) return info->className; } -inline IterationStatus RecordType::operator()(JSCell* cell) +inline void RecordType::operator()(JSCell* cell) { m_typeCountSet->add(typeName(cell)); - return IterationStatus::Continue; } -inline std::unique_ptr<TypeCountSet> RecordType::returnValue() +inline PassOwnPtr<TypeCountSet> RecordType::returnValue() { - return WTF::move(m_typeCountSet); + return m_typeCountSet.release(); } } // anonymous namespace Heap::Heap(VM* vm, HeapType heapType) : m_heapType(heapType) - , m_ramSize(Options::forceRAMSize() ? Options::forceRAMSize() : ramSize()) + , m_ramSize(ramSize()) , m_minBytesPerCycle(minHeapSize(m_heapType, m_ramSize)) , m_sizeAfterLastCollect(0) - , m_sizeAfterLastFullCollect(0) - , m_sizeBeforeLastFullCollect(0) - , m_sizeAfterLastEdenCollect(0) - , m_sizeBeforeLastEdenCollect(0) , m_bytesAllocatedThisCycle(0) - , m_bytesAbandonedSinceLastFullCollect(0) + , m_bytesAbandonedThisCycle(0) , m_maxEdenSize(m_minBytesPerCycle) , m_maxHeapSize(m_minBytesPerCycle) , m_shouldDoFullCollection(false) , m_totalBytesVisited(0) , m_totalBytesCopied(0) , m_operationInProgress(NoOperation) + , m_blockAllocator() , m_objectSpace(this) , m_storageSpace(this) - , m_extraMemorySize(0) - , m_deprecatedExtraMemorySize(0) + , m_extraMemoryUsage(0) , m_machineThreads(this) , m_sharedData(vm) , m_slotVisitor(m_sharedData) @@ -342,36 +273,17 @@ Heap::Heap(VM* vm, HeapType heapType) , m_isSafeToCollect(false) , m_writeBarrierBuffer(256) , m_vm(vm) - // We seed with 10ms so that GCActivityCallback::didAllocate doesn't continuously - // schedule the timer if we've never done a collection. - , m_lastFullGCLength(0.01) - , m_lastEdenGCLength(0.01) + , m_lastGCLength(0) , m_lastCodeDiscardTime(WTF::monotonicallyIncreasingTime()) - , m_fullActivityCallback(GCActivityCallback::createFullTimer(this)) -#if ENABLE(GGC) - , m_edenActivityCallback(GCActivityCallback::createEdenTimer(this)) -#else - , m_edenActivityCallback(m_fullActivityCallback) -#endif -#if USE(CF) - , m_sweeper(std::make_unique<IncrementalSweeper>(this, CFRunLoopGetCurrent())) -#else - , m_sweeper(std::make_unique<IncrementalSweeper>(this->vm())) -#endif + , m_activityCallback(DefaultGCActivityCallback::create(this)) + , m_sweeper(IncrementalSweeper::create(this)) , m_deferralDepth(0) -#if USE(CF) - , m_delayedReleaseRecursionCount(0) -#endif { m_storageSpace.init(); - if (Options::verifyHeap()) - m_verifier = std::make_unique<HeapVerifier>(this, Options::numberOfGCCyclesToRecordForVerification()); } Heap::~Heap() { - for (WeakBlock* block : m_logicallyEmptyWeakBlocks) - WeakBlock::destroy(block); } bool Heap::isPagedOut(double deadline) @@ -387,56 +299,30 @@ void Heap::lastChanceToFinalize() RELEASE_ASSERT(m_operationInProgress == NoOperation); m_objectSpace.lastChanceToFinalize(); - releaseDelayedReleasedObjects(); - - sweepAllLogicallyEmptyWeakBlocks(); } -void Heap::releaseDelayedReleasedObjects() +void Heap::reportExtraMemoryCostSlowCase(size_t cost) { -#if USE(CF) - // We need to guard against the case that releasing an object can create more objects due to the - // release calling into JS. When those JS call(s) exit and all locks are being dropped we end up - // back here and could try to recursively release objects. We guard that with a recursive entry - // count. Only the initial call will release objects, recursive calls simple return and let the - // the initial call to the function take care of any objects created during release time. - // This also means that we need to loop until there are no objects in m_delayedReleaseObjects - // and use a temp Vector for the actual releasing. - if (!m_delayedReleaseRecursionCount++) { - while (!m_delayedReleaseObjects.isEmpty()) { - ASSERT(m_vm->currentThreadIsHoldingAPILock()); - - Vector<RetainPtr<CFTypeRef>> objectsToRelease = WTF::move(m_delayedReleaseObjects); - - { - // We need to drop locks before calling out to arbitrary code. - JSLock::DropAllLocks dropAllLocks(m_vm); - - objectsToRelease.clear(); - } - } - } - m_delayedReleaseRecursionCount--; -#endif -} + // Our frequency of garbage collection tries to balance memory use against speed + // by collecting based on the number of newly created values. However, for values + // that hold on to a great deal of memory that's not in the form of other JS values, + // that is not good enough - in some cases a lot of those objects can pile up and + // use crazy amounts of memory without a GC happening. So we track these extra + // memory costs. Only unusually large objects are noted, and we only keep track + // of this extra cost until the next GC. In garbage collected languages, most values + // are either very short lived temporaries, or have extremely long lifetimes. So + // if a large value survives one garbage collection, there is not much point to + // collecting more frequently as long as it stays alive. -void Heap::reportExtraMemoryAllocatedSlowCase(size_t size) -{ - didAllocate(size); + didAllocate(cost); collectIfNecessaryOrDefer(); } -void Heap::deprecatedReportExtraMemorySlowCase(size_t size) -{ - m_deprecatedExtraMemorySize += size; - reportExtraMemoryAllocatedSlowCase(size); -} - void Heap::reportAbandonedObjectGraph() { // Our clients don't know exactly how much memory they // are abandoning so we just guess for them. - double abandonedBytes = 0.1 * m_sizeAfterLastCollect; + double abandonedBytes = 0.10 * m_sizeAfterLastCollect; // We want to accelerate the next collection. Because memory has just // been abandoned, the next collection has the potential to @@ -447,11 +333,9 @@ void Heap::reportAbandonedObjectGraph() void Heap::didAbandon(size_t bytes) { - if (m_fullActivityCallback) { - m_fullActivityCallback->didAllocate( - m_sizeAfterLastCollect - m_sizeAfterLastFullCollect + m_bytesAllocatedThisCycle + m_bytesAbandonedSinceLastFullCollect); - } - m_bytesAbandonedSinceLastFullCollect += bytes; + if (m_activityCallback) + m_activityCallback->didAllocate(m_bytesAllocatedThisCycle + m_bytesAbandonedThisCycle); + m_bytesAbandonedThisCycle += bytes; } void Heap::protect(JSValue k) @@ -484,6 +368,40 @@ void Heap::addReference(JSCell* cell, ArrayBuffer* buffer) } } +void Heap::markProtectedObjects(HeapRootVisitor& heapRootVisitor) +{ + ProtectCountSet::iterator end = m_protectedValues.end(); + for (ProtectCountSet::iterator it = m_protectedValues.begin(); it != end; ++it) + heapRootVisitor.visit(&it->key); +} + +void Heap::pushTempSortVector(Vector<ValueStringPair, 0, UnsafeVectorOverflow>* tempVector) +{ + m_tempSortingVectors.append(tempVector); +} + +void Heap::popTempSortVector(Vector<ValueStringPair, 0, UnsafeVectorOverflow>* tempVector) +{ + ASSERT_UNUSED(tempVector, tempVector == m_tempSortingVectors.last()); + m_tempSortingVectors.removeLast(); +} + +void Heap::markTempSortVectors(HeapRootVisitor& heapRootVisitor) +{ + typedef Vector<Vector<ValueStringPair, 0, UnsafeVectorOverflow>* > VectorOfValueStringVectors; + + VectorOfValueStringVectors::iterator end = m_tempSortingVectors.end(); + for (VectorOfValueStringVectors::iterator it = m_tempSortingVectors.begin(); it != end; ++it) { + Vector<ValueStringPair, 0, UnsafeVectorOverflow>* tempSortingVector = *it; + + Vector<ValueStringPair>::iterator vectorEnd = tempSortingVector->end(); + for (Vector<ValueStringPair>::iterator vectorIt = tempSortingVector->begin(); vectorIt != vectorEnd; ++vectorIt) { + if (vectorIt->first) + heapRootVisitor.visit(&vectorIt->first); + } + } +} + void Heap::harvestWeakReferences() { m_slotVisitor.harvestWeakReferences(); @@ -491,7 +409,6 @@ void Heap::harvestWeakReferences() void Heap::finalizeUnconditionalFinalizers() { - GCPHASE(FinalizeUnconditionalFinalizers); m_slotVisitor.finalizeUnconditionalFinalizers(); } @@ -519,350 +436,242 @@ void Heap::getConservativeRegisterRoots(HashSet<JSCell*>& roots) JSCell** registerRoots = stackRoots.roots(); for (size_t i = 0; i < stackRootCount; i++) { setMarked(registerRoots[i]); - registerRoots[i]->setMarked(); roots.add(registerRoots[i]); } } -void Heap::markRoots(double gcStartTime, void* stackOrigin, void* stackTop, MachineThreads::RegisterState& calleeSavedRegisters) +void Heap::markRoots() { - SamplingRegion samplingRegion("Garbage Collection: Marking"); + SamplingRegion samplingRegion("Garbage Collection: Tracing"); GCPHASE(MarkRoots); ASSERT(isValidThreadState(m_vm)); -#if ENABLE(GGC) - Vector<const JSCell*> rememberedSet(m_slotVisitor.markStack().size()); - m_slotVisitor.markStack().fillVector(rememberedSet); -#else - Vector<const JSCell*> rememberedSet; +#if ENABLE(OBJECT_MARK_LOGGING) + double gcStartTime = WTF::monotonicallyIncreasingTime(); #endif - if (m_operationInProgress == EdenCollection) - m_codeBlocks.clearMarksForEdenCollection(rememberedSet); - else - m_codeBlocks.clearMarksForFullCollection(); - + void* dummy; + // We gather conservative roots before clearing mark bits because conservative // gathering uses the mark bits to determine whether a reference is valid. - ConservativeRoots conservativeRoots(&m_objectSpace.blocks(), &m_storageSpace); - gatherStackRoots(conservativeRoots, stackOrigin, stackTop, calleeSavedRegisters); - gatherJSStackRoots(conservativeRoots); - gatherScratchBufferRoots(conservativeRoots); - - clearLivenessData(); - - m_sharedData.didStartMarking(); - m_slotVisitor.didStartMarking(); - HeapRootVisitor heapRootVisitor(m_slotVisitor); - + ConservativeRoots machineThreadRoots(&m_objectSpace.blocks(), &m_storageSpace); + m_jitStubRoutines.clearMarks(); { - ParallelModeEnabler enabler(m_slotVisitor); - - visitExternalRememberedSet(); - visitSmallStrings(); - visitConservativeRoots(conservativeRoots); - visitProtectedObjects(heapRootVisitor); - visitArgumentBuffers(heapRootVisitor); - visitException(heapRootVisitor); - visitStrongHandles(heapRootVisitor); - visitHandleStack(heapRootVisitor); - traceCodeBlocksAndJITStubRoutines(); - converge(); + GCPHASE(GatherConservativeRoots); + m_machineThreads.gatherConservativeRoots(machineThreadRoots, &dummy); } - // Weak references must be marked last because their liveness depends on - // the liveness of the rest of the object graph. - visitWeakHandles(heapRootVisitor); - - clearRememberedSet(rememberedSet); - m_sharedData.didFinishMarking(); - updateObjectCounts(gcStartTime); - resetVisitors(); -} - -void Heap::copyBackingStores() -{ - GCPHASE(CopyBackingStores); - if (m_operationInProgress == EdenCollection) - m_storageSpace.startedCopying<EdenCollection>(); - else { - ASSERT(m_operationInProgress == FullCollection); - m_storageSpace.startedCopying<FullCollection>(); + ConservativeRoots stackRoots(&m_objectSpace.blocks(), &m_storageSpace); + m_codeBlocks.clearMarks(); + { + GCPHASE(GatherStackRoots); + stack().gatherConservativeRoots(stackRoots, m_jitStubRoutines, m_codeBlocks); } - if (m_storageSpace.shouldDoCopyPhase()) { - m_sharedData.didStartCopying(); - m_copyVisitor.startCopying(); - m_copyVisitor.copyFromShared(); - m_copyVisitor.doneCopying(); - // We need to wait for everybody to finish and return their CopiedBlocks - // before signaling that the phase is complete. - m_storageSpace.doneCopying(); - m_sharedData.didFinishCopying(); - } else - m_storageSpace.doneCopying(); -} - -void Heap::gatherStackRoots(ConservativeRoots& roots, void* stackOrigin, void* stackTop, MachineThreads::RegisterState& calleeSavedRegisters) -{ - GCPHASE(GatherStackRoots); - m_jitStubRoutines.clearMarks(); - m_machineThreads.gatherConservativeRoots(roots, m_jitStubRoutines, m_codeBlocks, stackOrigin, stackTop, calleeSavedRegisters); -} - -void Heap::gatherJSStackRoots(ConservativeRoots& roots) -{ -#if !ENABLE(JIT) - GCPHASE(GatherJSStackRoots); - stack().gatherConservativeRoots(roots, m_jitStubRoutines, m_codeBlocks); -#else - UNUSED_PARAM(roots); -#endif -} - -void Heap::gatherScratchBufferRoots(ConservativeRoots& roots) -{ #if ENABLE(DFG_JIT) - GCPHASE(GatherScratchBufferRoots); - m_vm->gatherConservativeRoots(roots); -#else - UNUSED_PARAM(roots); -#endif -} - -void Heap::clearLivenessData() -{ - GCPHASE(ClearLivenessData); - m_objectSpace.clearNewlyAllocated(); - m_objectSpace.clearMarks(); -} - -void Heap::visitExternalRememberedSet() -{ -#if JSC_OBJC_API_ENABLED - scanExternalRememberedSet(*m_vm, m_slotVisitor); + ConservativeRoots scratchBufferRoots(&m_objectSpace.blocks(), &m_storageSpace); + { + GCPHASE(GatherScratchBufferRoots); + m_vm->gatherConservativeRoots(scratchBufferRoots); + } #endif -} - -void Heap::visitSmallStrings() -{ - GCPHASE(VisitSmallStrings); - if (!m_vm->smallStrings.needsToBeVisited(m_operationInProgress)) - return; - - m_vm->smallStrings.visitStrongReferences(m_slotVisitor); - if (Options::logGC() == GCLogging::Verbose) - dataLog("Small strings:\n", m_slotVisitor); - m_slotVisitor.donateAndDrain(); -} -void Heap::visitConservativeRoots(ConservativeRoots& roots) -{ - GCPHASE(VisitConservativeRoots); - m_slotVisitor.append(roots); + { + GCPHASE(ClearLivenessData); + m_objectSpace.clearNewlyAllocated(); + m_objectSpace.clearMarks(); + } - if (Options::logGC() == GCLogging::Verbose) - dataLog("Conservative Roots:\n", m_slotVisitor); + m_sharedData.didStartMarking(); + SlotVisitor& visitor = m_slotVisitor; + visitor.setup(); + HeapRootVisitor heapRootVisitor(visitor); - m_slotVisitor.donateAndDrain(); -} +#if ENABLE(GGC) + Vector<const JSCell*> rememberedSet(m_slotVisitor.markStack().size()); + m_slotVisitor.markStack().fillVector(rememberedSet); +#endif -void Heap::visitCompilerWorklistWeakReferences() -{ -#if ENABLE(DFG_JIT) - for (auto worklist : m_suspendedCompilerWorklists) - worklist->visitWeakReferences(m_slotVisitor, m_codeBlocks); + { + ParallelModeEnabler enabler(visitor); - if (Options::logGC() == GCLogging::Verbose) - dataLog("DFG Worklists:\n", m_slotVisitor); -#endif -} + m_vm->smallStrings.visitStrongReferences(visitor); -void Heap::removeDeadCompilerWorklistEntries() -{ + { + GCPHASE(VisitMachineRoots); + MARK_LOG_ROOT(visitor, "C++ Stack"); + visitor.append(machineThreadRoots); + visitor.donateAndDrain(); + } + { + GCPHASE(VisitStackRoots); + MARK_LOG_ROOT(visitor, "Stack"); + visitor.append(stackRoots); + visitor.donateAndDrain(); + } #if ENABLE(DFG_JIT) - GCPHASE(FinalizeDFGWorklists); - for (auto worklist : m_suspendedCompilerWorklists) - worklist->removeDeadPlans(*m_vm); + { + GCPHASE(VisitScratchBufferRoots); + MARK_LOG_ROOT(visitor, "Scratch Buffers"); + visitor.append(scratchBufferRoots); + visitor.donateAndDrain(); + } #endif -} - -void Heap::visitProtectedObjects(HeapRootVisitor& heapRootVisitor) -{ - GCPHASE(VisitProtectedObjects); - - for (auto& pair : m_protectedValues) - heapRootVisitor.visit(&pair.key); - - if (Options::logGC() == GCLogging::Verbose) - dataLog("Protected Objects:\n", m_slotVisitor); - - m_slotVisitor.donateAndDrain(); -} - -void Heap::visitArgumentBuffers(HeapRootVisitor& visitor) -{ - GCPHASE(MarkingArgumentBuffers); - if (!m_markListSet || !m_markListSet->size()) - return; - - MarkedArgumentBuffer::markLists(visitor, *m_markListSet); - - if (Options::logGC() == GCLogging::Verbose) - dataLog("Argument Buffers:\n", m_slotVisitor); - - m_slotVisitor.donateAndDrain(); -} - -void Heap::visitException(HeapRootVisitor& visitor) -{ - GCPHASE(MarkingException); - if (!m_vm->exception() && !m_vm->lastException()) - return; - - visitor.visit(m_vm->addressOfException()); - visitor.visit(m_vm->addressOfLastException()); - - if (Options::logGC() == GCLogging::Verbose) - dataLog("Exceptions:\n", m_slotVisitor); - - m_slotVisitor.donateAndDrain(); -} - -void Heap::visitStrongHandles(HeapRootVisitor& visitor) -{ - GCPHASE(VisitStrongHandles); - m_handleSet.visitStrongHandles(visitor); - - if (Options::logGC() == GCLogging::Verbose) - dataLog("Strong Handles:\n", m_slotVisitor); - - m_slotVisitor.donateAndDrain(); -} - -void Heap::visitHandleStack(HeapRootVisitor& visitor) -{ - GCPHASE(VisitHandleStack); - m_handleStack.visit(visitor); - - if (Options::logGC() == GCLogging::Verbose) - dataLog("Handle Stack:\n", m_slotVisitor); - - m_slotVisitor.donateAndDrain(); -} - -void Heap::traceCodeBlocksAndJITStubRoutines() -{ - GCPHASE(TraceCodeBlocksAndJITStubRoutines); - m_codeBlocks.traceMarked(m_slotVisitor); - m_jitStubRoutines.traceMarkedStubRoutines(m_slotVisitor); - - if (Options::logGC() == GCLogging::Verbose) - dataLog("Code Blocks and JIT Stub Routines:\n", m_slotVisitor); - - m_slotVisitor.donateAndDrain(); -} + { + GCPHASE(VisitProtectedObjects); + MARK_LOG_ROOT(visitor, "Protected Objects"); + markProtectedObjects(heapRootVisitor); + visitor.donateAndDrain(); + } + { + GCPHASE(VisitTempSortVectors); + MARK_LOG_ROOT(visitor, "Temp Sort Vectors"); + markTempSortVectors(heapRootVisitor); + visitor.donateAndDrain(); + } -void Heap::converge() -{ + { + GCPHASE(MarkingArgumentBuffers); + if (m_markListSet && m_markListSet->size()) { + MARK_LOG_ROOT(visitor, "Argument Buffers"); + MarkedArgumentBuffer::markLists(heapRootVisitor, *m_markListSet); + visitor.donateAndDrain(); + } + } + if (m_vm->exception()) { + GCPHASE(MarkingException); + MARK_LOG_ROOT(visitor, "Exceptions"); + heapRootVisitor.visit(m_vm->addressOfException()); + visitor.donateAndDrain(); + } + + { + GCPHASE(VisitStrongHandles); + MARK_LOG_ROOT(visitor, "Strong Handles"); + m_handleSet.visitStrongHandles(heapRootVisitor); + visitor.donateAndDrain(); + } + + { + GCPHASE(HandleStack); + MARK_LOG_ROOT(visitor, "Handle Stack"); + m_handleStack.visit(heapRootVisitor); + visitor.donateAndDrain(); + } + + { + GCPHASE(TraceCodeBlocksAndJITStubRoutines); + MARK_LOG_ROOT(visitor, "Trace Code Blocks and JIT Stub Routines"); + m_codeBlocks.traceMarked(visitor); + m_jitStubRoutines.traceMarkedStubRoutines(visitor); + visitor.donateAndDrain(); + } + #if ENABLE(PARALLEL_GC) - GCPHASE(Convergence); - m_slotVisitor.drainFromShared(SlotVisitor::MasterDrain); + { + GCPHASE(Convergence); + visitor.drainFromShared(SlotVisitor::MasterDrain); + } #endif -} - -void Heap::visitWeakHandles(HeapRootVisitor& visitor) -{ - GCPHASE(VisitingLiveWeakHandles); - while (true) { - m_objectSpace.visitWeakSets(visitor); - harvestWeakReferences(); - visitCompilerWorklistWeakReferences(); - m_codeBlocks.traceMarked(m_slotVisitor); // New "executing" code blocks may be discovered. - if (m_slotVisitor.isEmpty()) - break; - - if (Options::logGC() == GCLogging::Verbose) - dataLog("Live Weak Handles:\n", m_slotVisitor); + } - { - ParallelModeEnabler enabler(m_slotVisitor); - m_slotVisitor.donateAndDrain(); + // Weak references must be marked last because their liveness depends on + // the liveness of the rest of the object graph. + { + GCPHASE(VisitingLiveWeakHandles); + MARK_LOG_ROOT(visitor, "Live Weak Handles"); + while (true) { + m_objectSpace.visitWeakSets(heapRootVisitor); + harvestWeakReferences(); + if (visitor.isEmpty()) + break; + { + ParallelModeEnabler enabler(visitor); + visitor.donateAndDrain(); #if ENABLE(PARALLEL_GC) - m_slotVisitor.drainFromShared(SlotVisitor::MasterDrain); + visitor.drainFromShared(SlotVisitor::MasterDrain); #endif + } } } -} -void Heap::clearRememberedSet(Vector<const JSCell*>& rememberedSet) -{ #if ENABLE(GGC) - GCPHASE(ClearRememberedSet); - for (auto* cell : rememberedSet) - const_cast<JSCell*>(cell)->setRemembered(false); -#else - UNUSED_PARAM(rememberedSet); + { + GCPHASE(ClearRememberedSet); + for (unsigned i = 0; i < rememberedSet.size(); ++i) { + const JSCell* cell = rememberedSet[i]; + MarkedBlock::blockFor(cell)->clearRemembered(cell); + } + } #endif -} -void Heap::updateObjectCounts(double gcStartTime) -{ - GCCOUNTER(VisitedValueCount, m_slotVisitor.visitCount()); + GCCOUNTER(VisitedValueCount, visitor.visitCount()); - if (Options::logGC() == GCLogging::Verbose) { - size_t visitCount = m_slotVisitor.visitCount(); + m_sharedData.didFinishMarking(); +#if ENABLE(OBJECT_MARK_LOGGING) + size_t visitCount = visitor.visitCount(); #if ENABLE(PARALLEL_GC) - visitCount += m_sharedData.childVisitCount(); + visitCount += m_sharedData.childVisitCount(); +#endif + MARK_LOG_MESSAGE2("\nNumber of live Objects after full GC %lu, took %.6f secs\n", visitCount, WTF::monotonicallyIncreasingTime() - gcStartTime); #endif - dataLogF("\nNumber of live Objects after GC %lu, took %.6f secs\n", static_cast<unsigned long>(visitCount), WTF::monotonicallyIncreasingTime() - gcStartTime); + + if (m_operationInProgress == EdenCollection) { + m_totalBytesVisited += visitor.bytesVisited(); + m_totalBytesCopied += visitor.bytesCopied(); + } else { + ASSERT(m_operationInProgress == FullCollection); + m_totalBytesVisited = visitor.bytesVisited(); + m_totalBytesCopied = visitor.bytesCopied(); } - - size_t bytesRemovedFromOldSpaceDueToReallocation = - m_storageSpace.takeBytesRemovedFromOldSpaceDueToReallocation(); - - if (m_operationInProgress == FullCollection) { - m_totalBytesVisited = 0; - m_totalBytesCopied = 0; - } else - m_totalBytesCopied -= bytesRemovedFromOldSpaceDueToReallocation; - - m_totalBytesVisited += m_slotVisitor.bytesVisited(); - m_totalBytesCopied += m_slotVisitor.bytesCopied(); #if ENABLE(PARALLEL_GC) m_totalBytesVisited += m_sharedData.childBytesVisited(); m_totalBytesCopied += m_sharedData.childBytesCopied(); #endif -} -void Heap::resetVisitors() -{ - m_slotVisitor.reset(); + visitor.reset(); #if ENABLE(PARALLEL_GC) m_sharedData.resetChildren(); #endif m_sharedData.reset(); } +template <HeapOperation collectionType> +void Heap::copyBackingStores() +{ + m_storageSpace.startedCopying<collectionType>(); + if (m_storageSpace.shouldDoCopyPhase()) { + m_sharedData.didStartCopying(); + m_copyVisitor.startCopying(); + m_copyVisitor.copyFromShared(); + m_copyVisitor.doneCopying(); + // We need to wait for everybody to finish and return their CopiedBlocks + // before signaling that the phase is complete. + m_storageSpace.doneCopying(); + m_sharedData.didFinishCopying(); + } else + m_storageSpace.doneCopying(); +} + size_t Heap::objectCount() { return m_objectSpace.objectCount(); } -size_t Heap::extraMemorySize() +size_t Heap::extraSize() { - return m_extraMemorySize + m_deprecatedExtraMemorySize + m_arrayBuffers.size(); + return m_extraMemoryUsage + m_arrayBuffers.size(); } size_t Heap::size() { - return m_objectSpace.size() + m_storageSpace.size() + extraMemorySize(); + return m_objectSpace.size() + m_storageSpace.size() + extraSize(); } size_t Heap::capacity() { - return m_objectSpace.capacity() + m_storageSpace.capacity() + extraMemorySize(); + return m_objectSpace.capacity() + m_storageSpace.capacity() + extraSize(); } size_t Heap::sizeAfterCollect() @@ -872,7 +681,7 @@ size_t Heap::sizeAfterCollect() // rather than all used (including dead) copied bytes, thus it's // always the case that m_totalBytesCopied <= m_storageSpace.size(). ASSERT(m_totalBytesCopied <= m_storageSpace.size()); - return m_totalBytesVisited + m_totalBytesCopied + extraMemorySize(); + return m_totalBytesVisited + m_totalBytesCopied + extraSize(); } size_t Heap::protectedGlobalObjectCount() @@ -891,12 +700,12 @@ size_t Heap::protectedObjectCount() return forEachProtectedCell<Count>(); } -std::unique_ptr<TypeCountSet> Heap::protectedObjectTypeCounts() +PassOwnPtr<TypeCountSet> Heap::protectedObjectTypeCounts() { return forEachProtectedCell<RecordType>(); } -std::unique_ptr<TypeCountSet> Heap::objectTypeCounts() +PassOwnPtr<TypeCountSet> Heap::objectTypeCounts() { HeapIterationScope iterationScope(*this); return m_objectSpace.forEachLiveCell<RecordType>(iterationScope); @@ -904,68 +713,36 @@ std::unique_ptr<TypeCountSet> Heap::objectTypeCounts() void Heap::deleteAllCompiledCode() { - // If JavaScript is running, it's not safe to delete JavaScript code, since - // we'll end up returning to deleted code. + // If JavaScript is running, it's not safe to delete code, since we'll end + // up deleting code that is live on the stack. if (m_vm->entryScope) return; - - // If we have things on any worklist, then don't delete code. This is kind of - // a weird heuristic. It's definitely not safe to throw away code that is on - // the worklist. But this change was made in a hurry so we just avoid throwing - // away any code if there is any code on any worklist. I suspect that this - // might not actually be too dumb: if there is code on worklists then that - // means that we are running some hot JS code right now. Maybe causing - // recompilations isn't a good idea. -#if ENABLE(DFG_JIT) - for (unsigned i = DFG::numberOfWorklists(); i--;) { - if (DFG::Worklist* worklist = DFG::worklistForIndexOrNull(i)) { - if (worklist->isActiveForVM(*vm())) - return; - } - } -#endif // ENABLE(DFG_JIT) - for (ExecutableBase* current : m_compiledCode) { + for (ExecutableBase* current = m_compiledCode.head(); current; current = current->next()) { if (!current->isFunctionExecutable()) continue; - static_cast<FunctionExecutable*>(current)->clearCode(); + static_cast<FunctionExecutable*>(current)->clearCodeIfNotCompiling(); } - ASSERT(m_operationInProgress == FullCollection || m_operationInProgress == NoOperation); - m_codeBlocks.clearMarksForFullCollection(); - m_codeBlocks.deleteUnmarkedAndUnreferenced(FullCollection); -} - -void Heap::deleteAllUnlinkedFunctionCode() -{ - for (ExecutableBase* current : m_compiledCode) { - if (!current->isFunctionExecutable()) - continue; - static_cast<FunctionExecutable*>(current)->clearUnlinkedCodeForRecompilation(); - } + m_codeBlocks.clearMarks(); + m_codeBlocks.deleteUnmarkedAndUnreferenced(); } -void Heap::clearUnmarkedExecutables() +void Heap::deleteUnmarkedCompiledCode() { - GCPHASE(ClearUnmarkedExecutables); - for (unsigned i = m_compiledCode.size(); i--;) { - ExecutableBase* current = m_compiledCode[i]; + ExecutableBase* next; + for (ExecutableBase* current = m_compiledCode.head(); current; current = next) { + next = current->next(); if (isMarked(current)) continue; // We do this because executable memory is limited on some platforms and because // CodeBlock requires eager finalization. ExecutableBase::clearCodeVirtual(current); - std::swap(m_compiledCode[i], m_compiledCode.last()); - m_compiledCode.removeLast(); + m_compiledCode.remove(current); } -} -void Heap::deleteUnmarkedCompiledCode() -{ - GCPHASE(DeleteCodeBlocks); - clearUnmarkedExecutables(); - m_codeBlocks.deleteUnmarkedAndUnreferenced(m_operationInProgress); + m_codeBlocks.deleteUnmarkedAndUnreferenced(); m_jitStubRoutines.deleteUnmarkedJettisonedStubRoutines(); } @@ -973,41 +750,29 @@ void Heap::addToRememberedSet(const JSCell* cell) { ASSERT(cell); ASSERT(!Options::enableConcurrentJIT() || !isCompilationThread()); - if (isRemembered(cell)) + if (isInRememberedSet(cell)) return; - const_cast<JSCell*>(cell)->setRemembered(true); + MarkedBlock::blockFor(cell)->setRemembered(cell); m_slotVisitor.unconditionallyAppend(const_cast<JSCell*>(cell)); } -void Heap::collectAndSweep(HeapOperation collectionType) +void Heap::collectAllGarbage() { if (!m_isSafeToCollect) return; - collect(collectionType); + m_shouldDoFullCollection = true; + collect(); SamplingRegion samplingRegion("Garbage Collection: Sweeping"); - - DeferGCForAWhile deferGC(*this); + DelayedReleaseScope delayedReleaseScope(m_objectSpace); m_objectSpace.sweep(); m_objectSpace.shrink(); - - sweepAllLogicallyEmptyWeakBlocks(); } static double minute = 60.0; -NEVER_INLINE void Heap::collect(HeapOperation collectionType) -{ - void* stackTop; - ALLOCATE_AND_GET_REGISTER_STATE(registers); - - collectImpl(collectionType, wtfThreadData().stack().origin(), &stackTop, registers); - - sanitizeStackForVM(m_vm); -} - -NEVER_INLINE void Heap::collectImpl(HeapOperation collectionType, void* stackOrigin, void* stackTop, MachineThreads::RegisterState& calleeSavedRegisters) +void Heap::collect() { #if ENABLE(ALLOCATION_LOGGING) dataLogF("JSC GC starting collection.\n"); @@ -1021,242 +786,120 @@ NEVER_INLINE void Heap::collectImpl(HeapOperation collectionType, void* stackOri SamplingRegion samplingRegion("Garbage Collection"); - if (vm()->typeProfiler()) { - DeferGCForAWhile awhile(*this); - vm()->typeProfilerLog()->processLogEntries(ASCIILiteral("GC")); - } - RELEASE_ASSERT(!m_deferralDepth); + GCPHASE(Collect); ASSERT(vm()->currentThreadIsHoldingAPILock()); - RELEASE_ASSERT(vm()->atomicStringTable() == wtfThreadData().atomicStringTable()); + RELEASE_ASSERT(vm()->identifierTable == wtfThreadData().currentIdentifierTable()); ASSERT(m_isSafeToCollect); JAVASCRIPTCORE_GC_BEGIN(); RELEASE_ASSERT(m_operationInProgress == NoOperation); - - suspendCompilerThreads(); - willStartCollection(collectionType); - GCPHASE(Collect); - - double gcStartTime = WTF::monotonicallyIncreasingTime(); - if (m_verifier) { - // Verify that live objects from the last GC cycle haven't been corrupted by - // mutators before we begin this new GC cycle. - m_verifier->verify(HeapVerifier::Phase::BeforeGC); - - m_verifier->initializeGCCycle(); - m_verifier->gatherLiveObjects(HeapVerifier::Phase::BeforeMarking); - } - - deleteOldCode(gcStartTime); - flushOldStructureIDTables(); - stopAllocation(); - flushWriteBarrierBuffer(); - - markRoots(gcStartTime, stackOrigin, stackTop, calleeSavedRegisters); - - if (m_verifier) { - m_verifier->gatherLiveObjects(HeapVerifier::Phase::AfterMarking); - m_verifier->verify(HeapVerifier::Phase::AfterMarking); - } - JAVASCRIPTCORE_GC_MARKED(); - - if (vm()->typeProfiler()) - vm()->typeProfiler()->invalidateTypeSetCache(); - - reapWeakHandles(); - pruneStaleEntriesFromWeakGCMaps(); - sweepArrayBuffers(); - snapshotMarkedSpace(); - - copyBackingStores(); - - finalizeUnconditionalFinalizers(); - removeDeadCompilerWorklistEntries(); - deleteUnmarkedCompiledCode(); - deleteSourceProviderCaches(); - notifyIncrementalSweeper(); - rememberCurrentlyExecutingCodeBlocks(); - - resetAllocators(); - updateAllocationLimits(); - didFinishCollection(gcStartTime); - resumeCompilerThreads(); - - if (m_verifier) { - m_verifier->trimDeadObjects(); - m_verifier->verify(HeapVerifier::Phase::AfterGC); - } - - if (Options::logGC()) { - double after = currentTimeMS(); - dataLog(after - before, " ms]\n"); - } -} - -void Heap::suspendCompilerThreads() -{ -#if ENABLE(DFG_JIT) - GCPHASE(SuspendCompilerThreads); - ASSERT(m_suspendedCompilerWorklists.isEmpty()); - for (unsigned i = DFG::numberOfWorklists(); i--;) { - if (DFG::Worklist* worklist = DFG::worklistForIndexOrNull(i)) { - m_suspendedCompilerWorklists.append(worklist); - worklist->suspendAllThreads(); - } + + { + RecursiveAllocationScope scope(*this); + m_vm->prepareToDiscardCode(); } -#endif -} -void Heap::willStartCollection(HeapOperation collectionType) -{ - GCPHASE(StartingCollection); - if (shouldDoFullCollection(collectionType)) { + bool isFullCollection = m_shouldDoFullCollection; + if (isFullCollection) { m_operationInProgress = FullCollection; m_slotVisitor.clearMarkStack(); m_shouldDoFullCollection = false; if (Options::logGC()) dataLog("FullCollection, "); } else { +#if ENABLE(GGC) m_operationInProgress = EdenCollection; if (Options::logGC()) dataLog("EdenCollection, "); +#else + m_operationInProgress = FullCollection; + m_slotVisitor.clearMarkStack(); + if (Options::logGC()) + dataLog("FullCollection, "); +#endif } - if (m_operationInProgress == FullCollection) { - m_sizeBeforeLastFullCollect = m_sizeAfterLastCollect + m_bytesAllocatedThisCycle; - m_extraMemorySize = 0; - m_deprecatedExtraMemorySize = 0; - - if (m_fullActivityCallback) - m_fullActivityCallback->willCollect(); - } else { - ASSERT(m_operationInProgress == EdenCollection); - m_sizeBeforeLastEdenCollect = m_sizeAfterLastCollect + m_bytesAllocatedThisCycle; - } - - if (m_edenActivityCallback) - m_edenActivityCallback->willCollect(); -} + if (m_operationInProgress == FullCollection) + m_extraMemoryUsage = 0; -void Heap::deleteOldCode(double gcStartTime) -{ - if (m_operationInProgress == EdenCollection) - return; + if (m_activityCallback) + m_activityCallback->willCollect(); - GCPHASE(DeleteOldCode); - if (gcStartTime - m_lastCodeDiscardTime > minute) { - m_vm->regExpCache()->deleteAllCode(); + double lastGCStartTime = WTF::monotonicallyIncreasingTime(); + if (lastGCStartTime - m_lastCodeDiscardTime > minute) { deleteAllCompiledCode(); m_lastCodeDiscardTime = WTF::monotonicallyIncreasingTime(); } -} - -void Heap::flushOldStructureIDTables() -{ - GCPHASE(FlushOldStructureIDTables); - m_structureIDTable.flushOldTables(); -} -void Heap::flushWriteBarrierBuffer() -{ - GCPHASE(FlushWriteBarrierBuffer); - if (m_operationInProgress == EdenCollection) { - m_writeBarrierBuffer.flush(*this); - return; + { + GCPHASE(StopAllocation); + m_objectSpace.stopAllocating(); + if (m_operationInProgress == FullCollection) + m_storageSpace.didStartFullCollection(); } - m_writeBarrierBuffer.reset(); -} -void Heap::stopAllocation() -{ - GCPHASE(StopAllocation); - m_objectSpace.stopAllocating(); - if (m_operationInProgress == FullCollection) - m_storageSpace.didStartFullCollection(); -} - -void Heap::reapWeakHandles() -{ - GCPHASE(ReapingWeakHandles); - m_objectSpace.reapWeakSets(); -} - -void Heap::pruneStaleEntriesFromWeakGCMaps() -{ - GCPHASE(PruningStaleEntriesFromWeakGCMaps); - if (m_operationInProgress != FullCollection) - return; - for (auto& pruneCallback : m_weakGCMaps.values()) - pruneCallback(); -} - -void Heap::sweepArrayBuffers() -{ - GCPHASE(SweepingArrayBuffers); - m_arrayBuffers.sweep(); -} - -struct MarkedBlockSnapshotFunctor : public MarkedBlock::VoidFunctor { - MarkedBlockSnapshotFunctor(Vector<MarkedBlock*>& blocks) - : m_index(0) - , m_blocks(blocks) { + GCPHASE(FlushWriteBarrierBuffer); + if (m_operationInProgress == EdenCollection) + m_writeBarrierBuffer.flush(*this); + else + m_writeBarrierBuffer.reset(); } - void operator()(MarkedBlock* block) { m_blocks[m_index++] = block; } - - size_t m_index; - Vector<MarkedBlock*>& m_blocks; -}; + markRoots(); + + { + GCPHASE(ReapingWeakHandles); + m_objectSpace.reapWeakSets(); + } -void Heap::snapshotMarkedSpace() -{ - GCPHASE(SnapshotMarkedSpace); + JAVASCRIPTCORE_GC_MARKED(); + + { + GCPHASE(SweepingArrayBuffers); + m_arrayBuffers.sweep(); + } - if (m_operationInProgress == EdenCollection) { - m_blockSnapshot.appendVector(m_objectSpace.blocksWithNewObjects()); - // Sort and deduplicate the block snapshot since we might be appending to an unfinished work list. - std::sort(m_blockSnapshot.begin(), m_blockSnapshot.end()); - m_blockSnapshot.shrink(std::unique(m_blockSnapshot.begin(), m_blockSnapshot.end()) - m_blockSnapshot.begin()); - } else { - m_blockSnapshot.resizeToFit(m_objectSpace.blocks().set().size()); + if (m_operationInProgress == FullCollection) { + m_blockSnapshot.resize(m_objectSpace.blocks().set().size()); MarkedBlockSnapshotFunctor functor(m_blockSnapshot); m_objectSpace.forEachBlock(functor); } -} -void Heap::deleteSourceProviderCaches() -{ - GCPHASE(DeleteSourceProviderCaches); - m_vm->clearSourceProviderCaches(); -} + if (m_operationInProgress == FullCollection) + copyBackingStores<FullCollection>(); + else + copyBackingStores<EdenCollection>(); -void Heap::notifyIncrementalSweeper() -{ - GCPHASE(NotifyIncrementalSweeper); + { + GCPHASE(FinalizeUnconditionalFinalizers); + finalizeUnconditionalFinalizers(); + } - if (m_operationInProgress == FullCollection) { - if (!m_logicallyEmptyWeakBlocks.isEmpty()) - m_indexOfNextLogicallyEmptyWeakBlockToSweep = 0; + { + GCPHASE(DeleteCodeBlocks); + deleteUnmarkedCompiledCode(); } - m_sweeper->startSweeping(); -} + { + GCPHASE(DeleteSourceProviderCaches); + m_vm->clearSourceProviderCaches(); + } -void Heap::rememberCurrentlyExecutingCodeBlocks() -{ - GCPHASE(RememberCurrentlyExecutingCodeBlocks); - m_codeBlocks.rememberCurrentlyExecutingCodeBlocks(this); -} + if (m_operationInProgress == FullCollection) + m_sweeper->startSweeping(m_blockSnapshot); -void Heap::resetAllocators() -{ - GCPHASE(ResetAllocators); - m_objectSpace.resetAllocators(); -} + { + GCPHASE(AddCurrentlyExecutingCodeBlocksToRememberedSet); + m_codeBlocks.rememberCurrentlyExecutingCodeBlocks(this); + } -void Heap::updateAllocationLimits() -{ - GCPHASE(UpdateAllocationLimits); + m_bytesAbandonedThisCycle = 0; + + { + GCPHASE(ResetAllocators); + m_objectSpace.resetAllocators(); + } + size_t currentHeapSize = sizeAfterCollect(); if (Options::gcMaxHeapSize() && currentHeapSize > Options::gcMaxHeapSize()) HeapStatistics::exitWithFailure(); @@ -1267,42 +910,29 @@ void Heap::updateAllocationLimits() // fixed minimum. m_maxHeapSize = max(minHeapSize(m_heapType, m_ramSize), proportionalHeapSize(currentHeapSize, m_ramSize)); m_maxEdenSize = m_maxHeapSize - currentHeapSize; - m_sizeAfterLastFullCollect = currentHeapSize; - m_bytesAbandonedSinceLastFullCollect = 0; } else { ASSERT(currentHeapSize >= m_sizeAfterLastCollect); m_maxEdenSize = m_maxHeapSize - currentHeapSize; - m_sizeAfterLastEdenCollect = currentHeapSize; double edenToOldGenerationRatio = (double)m_maxEdenSize / (double)m_maxHeapSize; double minEdenToOldGenerationRatio = 1.0 / 3.0; if (edenToOldGenerationRatio < minEdenToOldGenerationRatio) m_shouldDoFullCollection = true; m_maxHeapSize += currentHeapSize - m_sizeAfterLastCollect; m_maxEdenSize = m_maxHeapSize - currentHeapSize; - if (m_fullActivityCallback) { - ASSERT(currentHeapSize >= m_sizeAfterLastFullCollect); - m_fullActivityCallback->didAllocate(currentHeapSize - m_sizeAfterLastFullCollect); - } } m_sizeAfterLastCollect = currentHeapSize; - m_bytesAllocatedThisCycle = 0; - - if (Options::logGC()) - dataLog(currentHeapSize / 1024, " kb, "); -} -void Heap::didFinishCollection(double gcStartTime) -{ - GCPHASE(FinishingCollection); - double gcEndTime = WTF::monotonicallyIncreasingTime(); - if (m_operationInProgress == FullCollection) - m_lastFullGCLength = gcEndTime - gcStartTime; - else - m_lastEdenGCLength = gcEndTime - gcStartTime; + m_bytesAllocatedThisCycle = 0; + double lastGCEndTime = WTF::monotonicallyIncreasingTime(); + m_lastGCLength = lastGCEndTime - lastGCStartTime; if (Options::recordGCPauseTimes()) - HeapStatistics::recordGCPauseTime(gcStartTime, gcEndTime); + HeapStatistics::recordGCPauseTime(lastGCStartTime, lastGCEndTime); + RELEASE_ASSERT(m_operationInProgress == EdenCollection || m_operationInProgress == FullCollection); + + m_operationInProgress = NoOperation; + JAVASCRIPTCORE_GC_END(); if (Options::useZombieMode()) zombifyDeadObjects(); @@ -1312,23 +942,23 @@ void Heap::didFinishCollection(double gcStartTime) if (Options::showObjectStatistics()) HeapStatistics::showObjectStatistics(this); - - if (Options::logGC() == GCLogging::Verbose) - GCLogging::dumpObjectGraph(this); - - RELEASE_ASSERT(m_operationInProgress == EdenCollection || m_operationInProgress == FullCollection); - m_operationInProgress = NoOperation; - JAVASCRIPTCORE_GC_END(); + + if (Options::logGC()) { + double after = currentTimeMS(); + dataLog(after - before, " ms, ", currentHeapSize / 1024, " kb]\n"); + } } -void Heap::resumeCompilerThreads() +bool Heap::collectIfNecessaryOrDefer() { -#if ENABLE(DFG_JIT) - GCPHASE(ResumeCompilerThreads); - for (auto worklist : m_suspendedCompilerWorklists) - worklist->resumeAllThreads(); - m_suspendedCompilerWorklists.clear(); -#endif + if (m_deferralDepth) + return false; + + if (!shouldCollect()) + return false; + + collect(); + return true; } void Heap::markDeadObjects() @@ -1337,29 +967,19 @@ void Heap::markDeadObjects() m_objectSpace.forEachDeadCell<MarkObject>(iterationScope); } -void Heap::setFullActivityCallback(PassRefPtr<FullGCActivityCallback> activityCallback) +void Heap::setActivityCallback(PassOwnPtr<GCActivityCallback> activityCallback) { - m_fullActivityCallback = activityCallback; + m_activityCallback = activityCallback; } -void Heap::setEdenActivityCallback(PassRefPtr<EdenGCActivityCallback> activityCallback) +GCActivityCallback* Heap::activityCallback() { - m_edenActivityCallback = activityCallback; + return m_activityCallback.get(); } -GCActivityCallback* Heap::fullActivityCallback() +void Heap::setIncrementalSweeper(PassOwnPtr<IncrementalSweeper> sweeper) { - return m_fullActivityCallback.get(); -} - -GCActivityCallback* Heap::edenActivityCallback() -{ - return m_edenActivityCallback.get(); -} - -void Heap::setIncrementalSweeper(std::unique_ptr<IncrementalSweeper> sweeper) -{ - m_sweeper = WTF::move(sweeper); + m_sweeper = sweeper; } IncrementalSweeper* Heap::sweeper() @@ -1369,16 +989,14 @@ IncrementalSweeper* Heap::sweeper() void Heap::setGarbageCollectionTimerEnabled(bool enable) { - if (m_fullActivityCallback) - m_fullActivityCallback->setEnabled(enable); - if (m_edenActivityCallback) - m_edenActivityCallback->setEnabled(enable); + if (m_activityCallback) + m_activityCallback->setEnabled(enable); } void Heap::didAllocate(size_t bytes) { - if (m_edenActivityCallback) - m_edenActivityCallback->didAllocate(m_bytesAllocatedThisCycle + m_bytesAbandonedSinceLastFullCollect); + if (m_activityCallback) + m_activityCallback->didAllocate(m_bytesAllocatedThisCycle + m_bytesAbandonedThisCycle); m_bytesAllocatedThisCycle += bytes; } @@ -1411,26 +1029,9 @@ void Heap::addCompiledCode(ExecutableBase* executable) m_compiledCode.append(executable); } -void Heap::collectAllGarbageIfNotDoneRecently() -{ - if (!m_fullActivityCallback) { - collectAllGarbage(); - return; - } - - if (m_fullActivityCallback->didSyncGCRecently()) { - // A synchronous GC was already requested recently so we merely accelerate next collection. - reportAbandonedObjectGraph(); - return; - } - - m_fullActivityCallback->setDidSyncGCRecently(); - collectAllGarbage(); -} - class Zombify : public MarkedBlock::VoidFunctor { public: - inline void visit(JSCell* cell) + void operator()(JSCell* cell) { void** current = reinterpret_cast<void**>(cell); @@ -1441,96 +1042,59 @@ public: void* limit = static_cast<void*>(reinterpret_cast<char*>(cell) + MarkedBlock::blockFor(cell)->cellSize()); for (; current < limit; current++) - *current = zombifiedBits; - } - IterationStatus operator()(JSCell* cell) - { - visit(cell); - return IterationStatus::Continue; + *current = reinterpret_cast<void*>(0xbbadbeef); } }; void Heap::zombifyDeadObjects() { // Sweep now because destructors will crash once we're zombified. - { - SamplingRegion samplingRegion("Garbage Collection: Sweeping"); - m_objectSpace.zombifySweep(); - } + m_objectSpace.sweep(); HeapIterationScope iterationScope(*this); m_objectSpace.forEachDeadCell<Zombify>(iterationScope); } -void Heap::flushWriteBarrierBuffer(JSCell* cell) +void Heap::incrementDeferralDepth() { -#if ENABLE(GGC) - m_writeBarrierBuffer.flush(*this); - m_writeBarrierBuffer.add(cell); -#else - UNUSED_PARAM(cell); -#endif + RELEASE_ASSERT(m_deferralDepth < 100); // Sanity check to make sure this doesn't get ridiculous. + + m_deferralDepth++; } -bool Heap::shouldDoFullCollection(HeapOperation requestedCollectionType) const +void Heap::decrementDeferralDepth() { -#if ENABLE(GGC) - if (Options::alwaysDoFullCollection()) - return true; - - switch (requestedCollectionType) { - case EdenCollection: - return false; - case FullCollection: - return true; - case AnyCollection: - return m_shouldDoFullCollection; - default: - RELEASE_ASSERT_NOT_REACHED(); - return false; - } - RELEASE_ASSERT_NOT_REACHED(); - return false; -#else - UNUSED_PARAM(requestedCollectionType); - return true; -#endif + RELEASE_ASSERT(m_deferralDepth >= 1); + + m_deferralDepth--; } -void Heap::addLogicallyEmptyWeakBlock(WeakBlock* block) +void Heap::decrementDeferralDepthAndGCIfNeeded() { - m_logicallyEmptyWeakBlocks.append(block); + decrementDeferralDepth(); + collectIfNecessaryOrDefer(); } -void Heap::sweepAllLogicallyEmptyWeakBlocks() +void Heap::writeBarrier(const JSCell* from) { - if (m_logicallyEmptyWeakBlocks.isEmpty()) +#if ENABLE(GGC) + ASSERT_GC_OBJECT_LOOKS_VALID(const_cast<JSCell*>(from)); + if (!from || !isMarked(from)) return; - - m_indexOfNextLogicallyEmptyWeakBlockToSweep = 0; - while (sweepNextLogicallyEmptyWeakBlock()) { } + Heap* heap = Heap::heap(from); + heap->addToRememberedSet(from); +#else + UNUSED_PARAM(from); +#endif } -bool Heap::sweepNextLogicallyEmptyWeakBlock() +void Heap::flushWriteBarrierBuffer(JSCell* cell) { - if (m_indexOfNextLogicallyEmptyWeakBlockToSweep == WTF::notFound) - return false; - - WeakBlock* block = m_logicallyEmptyWeakBlocks[m_indexOfNextLogicallyEmptyWeakBlockToSweep]; - - block->sweep(); - if (block->isEmpty()) { - std::swap(m_logicallyEmptyWeakBlocks[m_indexOfNextLogicallyEmptyWeakBlockToSweep], m_logicallyEmptyWeakBlocks.last()); - m_logicallyEmptyWeakBlocks.removeLast(); - WeakBlock::destroy(block); - } else - m_indexOfNextLogicallyEmptyWeakBlockToSweep++; - - if (m_indexOfNextLogicallyEmptyWeakBlockToSweep >= m_logicallyEmptyWeakBlocks.size()) { - m_indexOfNextLogicallyEmptyWeakBlockToSweep = WTF::notFound; - return false; - } - - return true; +#if ENABLE(GGC) + m_writeBarrierBuffer.flush(*this); + m_writeBarrierBuffer.add(cell); +#else + UNUSED_PARAM(cell); +#endif } } // namespace JSC diff --git a/Source/JavaScriptCore/heap/Heap.h b/Source/JavaScriptCore/heap/Heap.h index 9cbebdcb5..90988a3fc 100644 --- a/Source/JavaScriptCore/heap/Heap.h +++ b/Source/JavaScriptCore/heap/Heap.h @@ -1,7 +1,7 @@ /* * Copyright (C) 1999-2000 Harri Porten (porten@kde.org) * Copyright (C) 2001 Peter Kelly (pmk@post.com) - * Copyright (C) 2003-2009, 2013-2014 Apple Inc. All rights reserved. + * Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2013 Apple Inc. All rights reserved. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -23,6 +23,7 @@ #define Heap_h #include "ArrayBuffer.h" +#include "BlockAllocator.h" #include "CodeBlockSet.h" #include "CopyVisitor.h" #include "GCIncomingRefCountedSet.h" @@ -37,381 +38,482 @@ #include "MarkedSpace.h" #include "Options.h" #include "SlotVisitor.h" -#include "StructureIDTable.h" #include "WeakHandleOwner.h" #include "WriteBarrierBuffer.h" #include "WriteBarrierSupport.h" #include <wtf/HashCountedSet.h> #include <wtf/HashSet.h> +#define COLLECT_ON_EVERY_ALLOCATION 0 + namespace JSC { -class CopiedSpace; -class CodeBlock; -class ExecutableBase; -class EdenGCActivityCallback; -class FullGCActivityCallback; -class GCActivityCallback; -class GCAwareJITStubRoutine; -class GlobalCodeBlock; -class Heap; -class HeapRootVisitor; -class HeapVerifier; -class IncrementalSweeper; -class JITStubRoutine; -class JSCell; -class VM; -class JSStack; -class JSValue; -class LiveObjectIterator; -class LLIntOffsetsExtractor; -class MarkedArgumentBuffer; -class WeakGCHandlePool; -class SlotVisitor; - -namespace DFG { -class Worklist; -} - -static void* const zombifiedBits = reinterpret_cast<void*>(static_cast<uintptr_t>(0xdeadbeef)); - -typedef HashCountedSet<JSCell*> ProtectCountSet; -typedef HashCountedSet<const char*> TypeCountSet; - -enum HeapType { SmallHeap, LargeHeap }; - -class Heap { - WTF_MAKE_NONCOPYABLE(Heap); -public: - friend class JIT; - friend class DFG::SpeculativeJIT; - friend class GCThreadSharedData; - static Heap* heap(const JSValue); // 0 for immediate values - static Heap* heap(const JSCell*); - - // This constant determines how many blocks we iterate between checks of our - // deadline when calling Heap::isPagedOut. Decreasing it will cause us to detect - // overstepping our deadline more quickly, while increasing it will cause - // our scan to run faster. - static const unsigned s_timeCheckResolution = 16; - - static bool isLive(const void*); - static bool isMarked(const void*); - static bool testAndSetMarked(const void*); - static void setMarked(const void*); - static bool isRemembered(const void*); - - JS_EXPORT_PRIVATE void addToRememberedSet(const JSCell*); - static bool isWriteBarrierEnabled(); - void writeBarrier(const JSCell*); - void writeBarrier(const JSCell*, JSValue); - void writeBarrier(const JSCell*, JSCell*); - - WriteBarrierBuffer& writeBarrierBuffer() { return m_writeBarrierBuffer; } - void flushWriteBarrierBuffer(JSCell*); - - Heap(VM*, HeapType); - ~Heap(); - JS_EXPORT_PRIVATE void lastChanceToFinalize(); - void releaseDelayedReleasedObjects(); - - VM* vm() const { return m_vm; } - MarkedSpace& objectSpace() { return m_objectSpace; } - CopiedSpace& storageSpace() { return m_storageSpace; } - MachineThreads& machineThreads() { return m_machineThreads; } - - const SlotVisitor& slotVisitor() const { return m_slotVisitor; } - - JS_EXPORT_PRIVATE GCActivityCallback* fullActivityCallback(); - JS_EXPORT_PRIVATE GCActivityCallback* edenActivityCallback(); - JS_EXPORT_PRIVATE void setFullActivityCallback(PassRefPtr<FullGCActivityCallback>); - JS_EXPORT_PRIVATE void setEdenActivityCallback(PassRefPtr<EdenGCActivityCallback>); - JS_EXPORT_PRIVATE void setGarbageCollectionTimerEnabled(bool); - - JS_EXPORT_PRIVATE IncrementalSweeper* sweeper(); - JS_EXPORT_PRIVATE void setIncrementalSweeper(std::unique_ptr<IncrementalSweeper>); - - // true if collection is in progress - bool isCollecting(); - HeapOperation operationInProgress() { return m_operationInProgress; } - // true if an allocation or collection is in progress - bool isBusy(); - MarkedSpace::Subspace& subspaceForObjectWithoutDestructor() { return m_objectSpace.subspaceForObjectsWithoutDestructor(); } - MarkedSpace::Subspace& subspaceForObjectDestructor() { return m_objectSpace.subspaceForObjectsWithDestructor(); } - template<typename ClassType> MarkedSpace::Subspace& subspaceForObjectOfType(); - MarkedAllocator& allocatorForObjectWithoutDestructor(size_t bytes) { return m_objectSpace.allocatorFor(bytes); } - MarkedAllocator& allocatorForObjectWithDestructor(size_t bytes) { return m_objectSpace.destructorAllocatorFor(bytes); } - template<typename ClassType> MarkedAllocator& allocatorForObjectOfType(size_t bytes); - CopiedAllocator& storageAllocator() { return m_storageSpace.allocator(); } - CheckedBoolean tryAllocateStorage(JSCell* intendedOwner, size_t, void**); - CheckedBoolean tryReallocateStorage(JSCell* intendedOwner, void**, size_t, size_t); - void ascribeOwner(JSCell* intendedOwner, void*); - - typedef void (*Finalizer)(JSCell*); - JS_EXPORT_PRIVATE void addFinalizer(JSCell*, Finalizer); - void addCompiledCode(ExecutableBase*); - - void notifyIsSafeToCollect() { m_isSafeToCollect = true; } - bool isSafeToCollect() const { return m_isSafeToCollect; } - - JS_EXPORT_PRIVATE void collectAllGarbageIfNotDoneRecently(); - void collectAllGarbage() { collectAndSweep(FullCollection); } - JS_EXPORT_PRIVATE void collectAndSweep(HeapOperation collectionType = AnyCollection); - bool shouldCollect(); - JS_EXPORT_PRIVATE void collect(HeapOperation collectionType = AnyCollection); - bool collectIfNecessaryOrDefer(); // Returns true if it did collect. - - // Use this API to report non-GC memory referenced by GC objects. Be sure to - // call both of these functions: Calling only one may trigger catastropic - // memory growth. - void reportExtraMemoryAllocated(size_t); - void reportExtraMemoryVisited(JSCell*, size_t); - - // Use this API to report non-GC memory if you can't use the better API above. - void deprecatedReportExtraMemory(size_t); - - JS_EXPORT_PRIVATE void reportAbandonedObjectGraph(); - - JS_EXPORT_PRIVATE void protect(JSValue); - JS_EXPORT_PRIVATE bool unprotect(JSValue); // True when the protect count drops to 0. - - size_t extraMemorySize(); // Non-GC memory referenced by GC objects. - JS_EXPORT_PRIVATE size_t size(); - JS_EXPORT_PRIVATE size_t capacity(); - JS_EXPORT_PRIVATE size_t objectCount(); - JS_EXPORT_PRIVATE size_t globalObjectCount(); - JS_EXPORT_PRIVATE size_t protectedObjectCount(); - JS_EXPORT_PRIVATE size_t protectedGlobalObjectCount(); - JS_EXPORT_PRIVATE std::unique_ptr<TypeCountSet> protectedObjectTypeCounts(); - JS_EXPORT_PRIVATE std::unique_ptr<TypeCountSet> objectTypeCounts(); - void showStatistics(); - - HashSet<MarkedArgumentBuffer*>& markListSet(); + class CopiedSpace; + class CodeBlock; + class ExecutableBase; + class GCActivityCallback; + class GCAwareJITStubRoutine; + class GlobalCodeBlock; + class Heap; + class HeapRootVisitor; + class IncrementalSweeper; + class JITStubRoutine; + class JSCell; + class VM; + class JSStack; + class JSValue; + class LiveObjectIterator; + class LLIntOffsetsExtractor; + class MarkedArgumentBuffer; + class WeakGCHandlePool; + class SlotVisitor; + + typedef std::pair<JSValue, WTF::String> ValueStringPair; + typedef HashCountedSet<JSCell*> ProtectCountSet; + typedef HashCountedSet<const char*> TypeCountSet; + + enum HeapType { SmallHeap, LargeHeap }; + + class Heap { + WTF_MAKE_NONCOPYABLE(Heap); + public: + friend class JIT; + friend class DFG::SpeculativeJIT; + friend class GCThreadSharedData; + static Heap* heap(const JSValue); // 0 for immediate values + static Heap* heap(const JSCell*); + + // This constant determines how many blocks we iterate between checks of our + // deadline when calling Heap::isPagedOut. Decreasing it will cause us to detect + // overstepping our deadline more quickly, while increasing it will cause + // our scan to run faster. + static const unsigned s_timeCheckResolution = 16; + + static bool isLive(const void*); + static bool isMarked(const void*); + static bool testAndSetMarked(const void*); + static void setMarked(const void*); + + JS_EXPORT_PRIVATE void addToRememberedSet(const JSCell*); + bool isInRememberedSet(const JSCell* cell) const + { + ASSERT(cell); + ASSERT(!Options::enableConcurrentJIT() || !isCompilationThread()); + return MarkedBlock::blockFor(cell)->isRemembered(cell); + } + static bool isWriteBarrierEnabled(); + JS_EXPORT_PRIVATE static void writeBarrier(const JSCell*); + static void writeBarrier(const JSCell*, JSValue); + static void writeBarrier(const JSCell*, JSCell*); + + WriteBarrierBuffer& writeBarrierBuffer() { return m_writeBarrierBuffer; } + void flushWriteBarrierBuffer(JSCell*); + + Heap(VM*, HeapType); + ~Heap(); + JS_EXPORT_PRIVATE void lastChanceToFinalize(); + + VM* vm() const { return m_vm; } + MarkedSpace& objectSpace() { return m_objectSpace; } + MachineThreads& machineThreads() { return m_machineThreads; } + + JS_EXPORT_PRIVATE GCActivityCallback* activityCallback(); + JS_EXPORT_PRIVATE void setActivityCallback(PassOwnPtr<GCActivityCallback>); + JS_EXPORT_PRIVATE void setGarbageCollectionTimerEnabled(bool); + + JS_EXPORT_PRIVATE IncrementalSweeper* sweeper(); + JS_EXPORT_PRIVATE void setIncrementalSweeper(PassOwnPtr<IncrementalSweeper>); + + // true if collection is in progress + inline bool isCollecting(); + inline HeapOperation operationInProgress() { return m_operationInProgress; } + // true if an allocation or collection is in progress + inline bool isBusy(); + + MarkedAllocator& allocatorForObjectWithoutDestructor(size_t bytes) { return m_objectSpace.allocatorFor(bytes); } + MarkedAllocator& allocatorForObjectWithNormalDestructor(size_t bytes) { return m_objectSpace.normalDestructorAllocatorFor(bytes); } + MarkedAllocator& allocatorForObjectWithImmortalStructureDestructor(size_t bytes) { return m_objectSpace.immortalStructureDestructorAllocatorFor(bytes); } + CopiedAllocator& storageAllocator() { return m_storageSpace.allocator(); } + CheckedBoolean tryAllocateStorage(JSCell* intendedOwner, size_t, void**); + CheckedBoolean tryReallocateStorage(JSCell* intendedOwner, void**, size_t, size_t); + void ascribeOwner(JSCell* intendedOwner, void*); + + typedef void (*Finalizer)(JSCell*); + JS_EXPORT_PRIVATE void addFinalizer(JSCell*, Finalizer); + void addCompiledCode(ExecutableBase*); + + void notifyIsSafeToCollect() { m_isSafeToCollect = true; } + bool isSafeToCollect() const { return m_isSafeToCollect; } + + JS_EXPORT_PRIVATE void collectAllGarbage(); + bool shouldCollect(); + void collect(); + bool collectIfNecessaryOrDefer(); // Returns true if it did collect. + + void reportExtraMemoryCost(size_t cost); + JS_EXPORT_PRIVATE void reportAbandonedObjectGraph(); + + JS_EXPORT_PRIVATE void protect(JSValue); + JS_EXPORT_PRIVATE bool unprotect(JSValue); // True when the protect count drops to 0. + + size_t extraSize(); // extra memory usage outside of pages allocated by the heap + JS_EXPORT_PRIVATE size_t size(); + JS_EXPORT_PRIVATE size_t capacity(); + JS_EXPORT_PRIVATE size_t objectCount(); + JS_EXPORT_PRIVATE size_t globalObjectCount(); + JS_EXPORT_PRIVATE size_t protectedObjectCount(); + JS_EXPORT_PRIVATE size_t protectedGlobalObjectCount(); + JS_EXPORT_PRIVATE PassOwnPtr<TypeCountSet> protectedObjectTypeCounts(); + JS_EXPORT_PRIVATE PassOwnPtr<TypeCountSet> objectTypeCounts(); + void showStatistics(); + + void pushTempSortVector(Vector<ValueStringPair, 0, UnsafeVectorOverflow>*); + void popTempSortVector(Vector<ValueStringPair, 0, UnsafeVectorOverflow>*); - template<typename Functor> typename Functor::ReturnType forEachProtectedCell(Functor&); - template<typename Functor> typename Functor::ReturnType forEachProtectedCell(); - template<typename Functor> void forEachCodeBlock(Functor&); - - HandleSet* handleSet() { return &m_handleSet; } - HandleStack* handleStack() { return &m_handleStack; } - - void willStartIterating(); - void didFinishIterating(); - void getConservativeRegisterRoots(HashSet<JSCell*>& roots); + HashSet<MarkedArgumentBuffer*>& markListSet() { if (!m_markListSet) m_markListSet = adoptPtr(new HashSet<MarkedArgumentBuffer*>); return *m_markListSet; } + + template<typename Functor> typename Functor::ReturnType forEachProtectedCell(Functor&); + template<typename Functor> typename Functor::ReturnType forEachProtectedCell(); + template<typename Functor> inline void forEachCodeBlock(Functor&); - double lastFullGCLength() const { return m_lastFullGCLength; } - double lastEdenGCLength() const { return m_lastEdenGCLength; } - void increaseLastFullGCLength(double amount) { m_lastFullGCLength += amount; } + HandleSet* handleSet() { return &m_handleSet; } + HandleStack* handleStack() { return &m_handleStack; } - size_t sizeBeforeLastEdenCollection() const { return m_sizeBeforeLastEdenCollect; } - size_t sizeAfterLastEdenCollection() const { return m_sizeAfterLastEdenCollect; } - size_t sizeBeforeLastFullCollection() const { return m_sizeBeforeLastFullCollect; } - size_t sizeAfterLastFullCollection() const { return m_sizeAfterLastFullCollect; } + void willStartIterating(); + void didFinishIterating(); + void getConservativeRegisterRoots(HashSet<JSCell*>& roots); - JS_EXPORT_PRIVATE void deleteAllCompiledCode(); - void deleteAllUnlinkedFunctionCode(); + double lastGCLength() { return m_lastGCLength; } + void increaseLastGCLength(double amount) { m_lastGCLength += amount; } - void didAllocate(size_t); - void didAbandon(size_t); + JS_EXPORT_PRIVATE void deleteAllCompiledCode(); - bool isPagedOut(double deadline); - - const JITStubRoutineSet& jitStubRoutines() { return m_jitStubRoutines; } - - void addReference(JSCell*, ArrayBuffer*); - - bool isDeferred() const { return !!m_deferralDepth || Options::disableGC(); } + void didAllocate(size_t); + void didAbandon(size_t); - StructureIDTable& structureIDTable() { return m_structureIDTable; } + bool isPagedOut(double deadline); + + const JITStubRoutineSet& jitStubRoutines() { return m_jitStubRoutines; } + + void addReference(JSCell*, ArrayBuffer*); + + bool isDeferred() const { return !!m_deferralDepth; } #if USE(CF) template<typename T> void releaseSoon(RetainPtr<T>&&); #endif - void removeCodeBlock(CodeBlock* cb) { m_codeBlocks.remove(cb); } - - static bool isZombified(JSCell* cell) { return *(void**)cell == zombifiedBits; } - - void registerWeakGCMap(void* weakGCMap, std::function<void()> pruningCallback); - void unregisterWeakGCMap(void* weakGCMap); - - void addLogicallyEmptyWeakBlock(WeakBlock*); - -private: - friend class CodeBlock; - friend class CopiedBlock; - friend class DeferGC; - friend class DeferGCForAWhile; - friend class GCAwareJITStubRoutine; - friend class GCLogging; - friend class HandleSet; - friend class HeapVerifier; - friend class JITStubRoutine; - friend class LLIntOffsetsExtractor; - friend class MarkedSpace; - friend class MarkedAllocator; - friend class MarkedBlock; - friend class CopiedSpace; - friend class CopyVisitor; - friend class RecursiveAllocationScope; - friend class SlotVisitor; - friend class SuperRegion; - friend class IncrementalSweeper; - friend class HeapStatistics; - friend class VM; - friend class WeakSet; - template<typename T> friend void* allocateCell(Heap&); - template<typename T> friend void* allocateCell(Heap&, size_t); - - void* allocateWithDestructor(size_t); // For use with objects with destructors. - void* allocateWithoutDestructor(size_t); // For use with objects without destructors. - template<typename ClassType> void* allocateObjectOfType(size_t); // Chooses one of the methods above based on type. - - static const size_t minExtraMemory = 256; - - class FinalizerOwner : public WeakHandleOwner { - virtual void finalize(Handle<Unknown>, void* context) override; + private: + friend class CodeBlock; + friend class CopiedBlock; + friend class DeferGC; + friend class DeferGCForAWhile; + friend class DelayedReleaseScope; + friend class GCAwareJITStubRoutine; + friend class HandleSet; + friend class JITStubRoutine; + friend class LLIntOffsetsExtractor; + friend class MarkedSpace; + friend class MarkedAllocator; + friend class MarkedBlock; + friend class CopiedSpace; + friend class CopyVisitor; + friend class RecursiveAllocationScope; + friend class SlotVisitor; + friend class SuperRegion; + friend class IncrementalSweeper; + friend class HeapStatistics; + friend class VM; + friend class WeakSet; + template<typename T> friend void* allocateCell(Heap&); + template<typename T> friend void* allocateCell(Heap&, size_t); + + void* allocateWithImmortalStructureDestructor(size_t); // For use with special objects whose Structures never die. + void* allocateWithNormalDestructor(size_t); // For use with objects that inherit directly or indirectly from JSDestructibleObject. + void* allocateWithoutDestructor(size_t); // For use with objects without destructors. + + static const size_t minExtraCost = 256; + static const size_t maxExtraCost = 1024 * 1024; + + class FinalizerOwner : public WeakHandleOwner { + virtual void finalize(Handle<Unknown>, void* context) override; + }; + + JS_EXPORT_PRIVATE bool isValidAllocation(size_t); + JS_EXPORT_PRIVATE void reportExtraMemoryCostSlowCase(size_t); + + void markRoots(); + void markProtectedObjects(HeapRootVisitor&); + void markTempSortVectors(HeapRootVisitor&); + template <HeapOperation collectionType> + void copyBackingStores(); + void harvestWeakReferences(); + void finalizeUnconditionalFinalizers(); + void deleteUnmarkedCompiledCode(); + void zombifyDeadObjects(); + void markDeadObjects(); + + size_t sizeAfterCollect(); + + JSStack& stack(); + BlockAllocator& blockAllocator(); + + JS_EXPORT_PRIVATE void incrementDeferralDepth(); + void decrementDeferralDepth(); + JS_EXPORT_PRIVATE void decrementDeferralDepthAndGCIfNeeded(); + + const HeapType m_heapType; + const size_t m_ramSize; + const size_t m_minBytesPerCycle; + size_t m_sizeAfterLastCollect; + + size_t m_bytesAllocatedThisCycle; + size_t m_bytesAbandonedThisCycle; + size_t m_maxEdenSize; + size_t m_maxHeapSize; + bool m_shouldDoFullCollection; + size_t m_totalBytesVisited; + size_t m_totalBytesCopied; + + HeapOperation m_operationInProgress; + BlockAllocator m_blockAllocator; + MarkedSpace m_objectSpace; + CopiedSpace m_storageSpace; + GCIncomingRefCountedSet<ArrayBuffer> m_arrayBuffers; + size_t m_extraMemoryUsage; + + HashSet<const JSCell*> m_copyingRememberedSet; + + ProtectCountSet m_protectedValues; + Vector<Vector<ValueStringPair, 0, UnsafeVectorOverflow>* > m_tempSortingVectors; + OwnPtr<HashSet<MarkedArgumentBuffer*>> m_markListSet; + + MachineThreads m_machineThreads; + + GCThreadSharedData m_sharedData; + SlotVisitor m_slotVisitor; + CopyVisitor m_copyVisitor; + + HandleSet m_handleSet; + HandleStack m_handleStack; + CodeBlockSet m_codeBlocks; + JITStubRoutineSet m_jitStubRoutines; + FinalizerOwner m_finalizerOwner; + + bool m_isSafeToCollect; + + WriteBarrierBuffer m_writeBarrierBuffer; + + VM* m_vm; + double m_lastGCLength; + double m_lastCodeDiscardTime; + + DoublyLinkedList<ExecutableBase> m_compiledCode; + + OwnPtr<GCActivityCallback> m_activityCallback; + OwnPtr<IncrementalSweeper> m_sweeper; + Vector<MarkedBlock*> m_blockSnapshot; + + unsigned m_deferralDepth; }; - JS_EXPORT_PRIVATE bool isValidAllocation(size_t); - JS_EXPORT_PRIVATE void reportExtraMemoryAllocatedSlowCase(size_t); - JS_EXPORT_PRIVATE void deprecatedReportExtraMemorySlowCase(size_t); - - void collectImpl(HeapOperation, void* stackOrigin, void* stackTop, MachineThreads::RegisterState&); - - void suspendCompilerThreads(); - void willStartCollection(HeapOperation collectionType); - void deleteOldCode(double gcStartTime); - void flushOldStructureIDTables(); - void flushWriteBarrierBuffer(); - void stopAllocation(); - - void markRoots(double gcStartTime, void* stackOrigin, void* stackTop, MachineThreads::RegisterState&); - void gatherStackRoots(ConservativeRoots&, void* stackOrigin, void* stackTop, MachineThreads::RegisterState&); - void gatherJSStackRoots(ConservativeRoots&); - void gatherScratchBufferRoots(ConservativeRoots&); - void clearLivenessData(); - void visitExternalRememberedSet(); - void visitSmallStrings(); - void visitConservativeRoots(ConservativeRoots&); - void visitCompilerWorklistWeakReferences(); - void removeDeadCompilerWorklistEntries(); - void visitProtectedObjects(HeapRootVisitor&); - void visitArgumentBuffers(HeapRootVisitor&); - void visitException(HeapRootVisitor&); - void visitStrongHandles(HeapRootVisitor&); - void visitHandleStack(HeapRootVisitor&); - void traceCodeBlocksAndJITStubRoutines(); - void converge(); - void visitWeakHandles(HeapRootVisitor&); - void clearRememberedSet(Vector<const JSCell*>&); - void updateObjectCounts(double gcStartTime); - void resetVisitors(); - - void reapWeakHandles(); - void pruneStaleEntriesFromWeakGCMaps(); - void sweepArrayBuffers(); - void snapshotMarkedSpace(); - void deleteSourceProviderCaches(); - void notifyIncrementalSweeper(); - void rememberCurrentlyExecutingCodeBlocks(); - void resetAllocators(); - void copyBackingStores(); - void harvestWeakReferences(); - void finalizeUnconditionalFinalizers(); - void clearUnmarkedExecutables(); - void deleteUnmarkedCompiledCode(); - void updateAllocationLimits(); - void didFinishCollection(double gcStartTime); - void resumeCompilerThreads(); - void zombifyDeadObjects(); - void markDeadObjects(); - - void sweepAllLogicallyEmptyWeakBlocks(); - bool sweepNextLogicallyEmptyWeakBlock(); - - bool shouldDoFullCollection(HeapOperation requestedCollectionType) const; - size_t sizeAfterCollect(); - - JSStack& stack(); + struct MarkedBlockSnapshotFunctor : public MarkedBlock::VoidFunctor { + MarkedBlockSnapshotFunctor(Vector<MarkedBlock*>& blocks) + : m_index(0) + , m_blocks(blocks) + { + } - void incrementDeferralDepth(); - void decrementDeferralDepth(); - void decrementDeferralDepthAndGCIfNeeded(); - - const HeapType m_heapType; - const size_t m_ramSize; - const size_t m_minBytesPerCycle; - size_t m_sizeAfterLastCollect; - size_t m_sizeAfterLastFullCollect; - size_t m_sizeBeforeLastFullCollect; - size_t m_sizeAfterLastEdenCollect; - size_t m_sizeBeforeLastEdenCollect; - - size_t m_bytesAllocatedThisCycle; - size_t m_bytesAbandonedSinceLastFullCollect; - size_t m_maxEdenSize; - size_t m_maxHeapSize; - bool m_shouldDoFullCollection; - size_t m_totalBytesVisited; - size_t m_totalBytesCopied; + void operator()(MarkedBlock* block) { m_blocks[m_index++] = block; } - HeapOperation m_operationInProgress; - StructureIDTable m_structureIDTable; - MarkedSpace m_objectSpace; - CopiedSpace m_storageSpace; - GCIncomingRefCountedSet<ArrayBuffer> m_arrayBuffers; - size_t m_extraMemorySize; - size_t m_deprecatedExtraMemorySize; - - HashSet<const JSCell*> m_copyingRememberedSet; + size_t m_index; + Vector<MarkedBlock*>& m_blocks; + }; - ProtectCountSet m_protectedValues; - std::unique_ptr<HashSet<MarkedArgumentBuffer*>> m_markListSet; + inline bool Heap::shouldCollect() + { + if (isDeferred()) + return false; + if (Options::gcMaxHeapSize()) + return m_bytesAllocatedThisCycle > Options::gcMaxHeapSize() && m_isSafeToCollect && m_operationInProgress == NoOperation; + return m_bytesAllocatedThisCycle > m_maxEdenSize && m_isSafeToCollect && m_operationInProgress == NoOperation; + } + + bool Heap::isBusy() + { + return m_operationInProgress != NoOperation; + } + + bool Heap::isCollecting() + { + return m_operationInProgress == FullCollection || m_operationInProgress == EdenCollection; + } + + inline Heap* Heap::heap(const JSCell* cell) + { + return MarkedBlock::blockFor(cell)->heap(); + } + + inline Heap* Heap::heap(const JSValue v) + { + if (!v.isCell()) + return 0; + return heap(v.asCell()); + } + + inline bool Heap::isLive(const void* cell) + { + return MarkedBlock::blockFor(cell)->isLiveCell(cell); + } + + inline bool Heap::isMarked(const void* cell) + { + return MarkedBlock::blockFor(cell)->isMarked(cell); + } + + inline bool Heap::testAndSetMarked(const void* cell) + { + return MarkedBlock::blockFor(cell)->testAndSetMarked(cell); + } + + inline void Heap::setMarked(const void* cell) + { + MarkedBlock::blockFor(cell)->setMarked(cell); + } + + inline bool Heap::isWriteBarrierEnabled() + { +#if ENABLE(WRITE_BARRIER_PROFILING) || ENABLE(GGC) + return true; +#else + return false; +#endif + } - MachineThreads m_machineThreads; - - GCThreadSharedData m_sharedData; - SlotVisitor m_slotVisitor; - CopyVisitor m_copyVisitor; - - HandleSet m_handleSet; - HandleStack m_handleStack; - CodeBlockSet m_codeBlocks; - JITStubRoutineSet m_jitStubRoutines; - FinalizerOwner m_finalizerOwner; + inline void Heap::writeBarrier(const JSCell* from, JSCell* to) + { +#if ENABLE(WRITE_BARRIER_PROFILING) + WriteBarrierCounters::countWriteBarrier(); +#endif + if (!from || !isMarked(from)) + return; + if (!to || isMarked(to)) + return; + Heap::heap(from)->addToRememberedSet(from); + } + + inline void Heap::writeBarrier(const JSCell* from, JSValue to) + { +#if ENABLE(WRITE_BARRIER_PROFILING) + WriteBarrierCounters::countWriteBarrier(); +#endif + if (!to.isCell()) + return; + writeBarrier(from, to.asCell()); + } + + inline void Heap::reportExtraMemoryCost(size_t cost) + { + if (cost > minExtraCost) + reportExtraMemoryCostSlowCase(cost); + } + + template<typename Functor> inline typename Functor::ReturnType Heap::forEachProtectedCell(Functor& functor) + { + ProtectCountSet::iterator end = m_protectedValues.end(); + for (ProtectCountSet::iterator it = m_protectedValues.begin(); it != end; ++it) + functor(it->key); + m_handleSet.forEachStrongHandle(functor, m_protectedValues); + + return functor.returnValue(); + } + + template<typename Functor> inline typename Functor::ReturnType Heap::forEachProtectedCell() + { + Functor functor; + return forEachProtectedCell(functor); + } + + template<typename Functor> inline void Heap::forEachCodeBlock(Functor& functor) + { + return m_codeBlocks.iterate<Functor>(functor); + } + + inline void* Heap::allocateWithNormalDestructor(size_t bytes) + { +#if ENABLE(ALLOCATION_LOGGING) + dataLogF("JSC GC allocating %lu bytes with normal destructor.\n", bytes); +#endif + ASSERT(isValidAllocation(bytes)); + return m_objectSpace.allocateWithNormalDestructor(bytes); + } - bool m_isSafeToCollect; - - WriteBarrierBuffer m_writeBarrierBuffer; - - VM* m_vm; - double m_lastFullGCLength; - double m_lastEdenGCLength; - double m_lastCodeDiscardTime; - - Vector<ExecutableBase*> m_compiledCode; - - Vector<WeakBlock*> m_logicallyEmptyWeakBlocks; - size_t m_indexOfNextLogicallyEmptyWeakBlockToSweep { WTF::notFound }; + inline void* Heap::allocateWithImmortalStructureDestructor(size_t bytes) + { +#if ENABLE(ALLOCATION_LOGGING) + dataLogF("JSC GC allocating %lu bytes with immortal structure destructor.\n", bytes); +#endif + ASSERT(isValidAllocation(bytes)); + return m_objectSpace.allocateWithImmortalStructureDestructor(bytes); + } - RefPtr<FullGCActivityCallback> m_fullActivityCallback; - RefPtr<GCActivityCallback> m_edenActivityCallback; - std::unique_ptr<IncrementalSweeper> m_sweeper; - Vector<MarkedBlock*> m_blockSnapshot; + inline void* Heap::allocateWithoutDestructor(size_t bytes) + { +#if ENABLE(ALLOCATION_LOGGING) + dataLogF("JSC GC allocating %lu bytes without destructor.\n", bytes); +#endif + ASSERT(isValidAllocation(bytes)); + return m_objectSpace.allocateWithoutDestructor(bytes); + } + + inline CheckedBoolean Heap::tryAllocateStorage(JSCell* intendedOwner, size_t bytes, void** outPtr) + { + CheckedBoolean result = m_storageSpace.tryAllocate(bytes, outPtr); +#if ENABLE(ALLOCATION_LOGGING) + dataLogF("JSC GC allocating %lu bytes of storage for %p: %p.\n", bytes, intendedOwner, *outPtr); +#else + UNUSED_PARAM(intendedOwner); +#endif + return result; + } - unsigned m_deferralDepth; - Vector<DFG::Worklist*> m_suspendedCompilerWorklists; + inline CheckedBoolean Heap::tryReallocateStorage(JSCell* intendedOwner, void** ptr, size_t oldSize, size_t newSize) + { +#if ENABLE(ALLOCATION_LOGGING) + void* oldPtr = *ptr; +#endif + CheckedBoolean result = m_storageSpace.tryReallocate(ptr, oldSize, newSize); +#if ENABLE(ALLOCATION_LOGGING) + dataLogF("JSC GC reallocating %lu -> %lu bytes of storage for %p: %p -> %p.\n", oldSize, newSize, intendedOwner, oldPtr, *ptr); +#else + UNUSED_PARAM(intendedOwner); +#endif + return result; + } + + inline void Heap::ascribeOwner(JSCell* intendedOwner, void* storage) + { +#if ENABLE(ALLOCATION_LOGGING) + dataLogF("JSC GC ascribing %p as owner of storage %p.\n", intendedOwner, storage); +#else + UNUSED_PARAM(intendedOwner); + UNUSED_PARAM(storage); +#endif + } + + inline BlockAllocator& Heap::blockAllocator() + { + return m_blockAllocator; + } - std::unique_ptr<HeapVerifier> m_verifier; #if USE(CF) - Vector<RetainPtr<CFTypeRef>> m_delayedReleaseObjects; - unsigned m_delayedReleaseRecursionCount; + template <typename T> + inline void Heap::releaseSoon(RetainPtr<T>&& object) + { + m_objectSpace.releaseSoon(std::move(object)); + } #endif - HashMap<void*, std::function<void()>> m_weakGCMaps; -}; - } // namespace JSC #endif // Heap_h diff --git a/Source/JavaScriptCore/heap/LiveObjectList.h b/Source/JavaScriptCore/heap/HeapBlock.h index 4d2874570..6f2a74c08 100644 --- a/Source/JavaScriptCore/heap/LiveObjectList.h +++ b/Source/JavaScriptCore/heap/HeapBlock.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2015 Apple Inc. All rights reserved. + * Copyright (C) 2011 Apple Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -20,38 +20,54 @@ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#ifndef LiveObjectList_h -#define LiveObjectList_h +#ifndef HeapBlock_h +#define HeapBlock_h -#include "LiveObjectData.h" -#include <wtf/Vector.h> +#include <wtf/DoublyLinkedList.h> +#include <wtf/StdLibExtras.h> namespace JSC { -struct LiveObjectList { - LiveObjectList(const char* name) - : name(name) - , hasLiveObjects(true) +enum AllocationEffort { AllocationCanFail, AllocationMustSucceed }; + +class Region; + +#if COMPILER(GCC) +#define CLASS_IF_GCC class +#else +#define CLASS_IF_GCC +#endif + +template<typename T> +class HeapBlock : public DoublyLinkedListNode<T> { + friend CLASS_IF_GCC DoublyLinkedListNode<T>; +public: + static HeapBlock* destroy(HeapBlock* block) WARN_UNUSED_RETURN { + static_cast<T*>(block)->~T(); + return block; } - - void reset() + + HeapBlock(Region* region) + : DoublyLinkedListNode<T>() + , m_region(region) + , m_prev(0) + , m_next(0) { - liveObjects.clear(); - hasLiveObjects = true; // Presume to have live objects until the list is trimmed. + ASSERT(m_region); } - - LiveObjectData* findObject(JSObject*); - - const char* name; - Vector<LiveObjectData> liveObjects; - bool hasLiveObjects; + + Region* region() const { return m_region; } + +private: + Region* m_region; + T* m_prev; + T* m_next; }; - -} // namespace JSC -#endif // LiveObjectList_h +} // namespace JSC +#endif diff --git a/Source/JavaScriptCore/heap/HeapInlines.h b/Source/JavaScriptCore/heap/HeapInlines.h deleted file mode 100644 index c7e9735ce..000000000 --- a/Source/JavaScriptCore/heap/HeapInlines.h +++ /dev/null @@ -1,354 +0,0 @@ -/* - * Copyright (C) 2014 Apple Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS - * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF - * THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef HeapInlines_h -#define HeapInlines_h - -#include "Heap.h" -#include "JSCell.h" -#include "Structure.h" -#include <type_traits> -#include <wtf/Assertions.h> - -namespace JSC { - -inline bool Heap::shouldCollect() -{ - if (isDeferred()) - return false; - if (Options::gcMaxHeapSize()) - return m_bytesAllocatedThisCycle > Options::gcMaxHeapSize() && m_isSafeToCollect && m_operationInProgress == NoOperation; - return m_bytesAllocatedThisCycle > m_maxEdenSize && m_isSafeToCollect && m_operationInProgress == NoOperation; -} - -inline bool Heap::isBusy() -{ - return m_operationInProgress != NoOperation; -} - -inline bool Heap::isCollecting() -{ - return m_operationInProgress == FullCollection || m_operationInProgress == EdenCollection; -} - -inline Heap* Heap::heap(const JSCell* cell) -{ - return MarkedBlock::blockFor(cell)->heap(); -} - -inline Heap* Heap::heap(const JSValue v) -{ - if (!v.isCell()) - return 0; - return heap(v.asCell()); -} - -inline bool Heap::isLive(const void* cell) -{ - return MarkedBlock::blockFor(cell)->isLiveCell(cell); -} - -inline bool Heap::isRemembered(const void* ptr) -{ - const JSCell* cell = static_cast<const JSCell*>(ptr); - ASSERT(cell); - ASSERT(!Options::enableConcurrentJIT() || !isCompilationThread()); - return cell->isRemembered(); -} - -inline bool Heap::isMarked(const void* cell) -{ - return MarkedBlock::blockFor(cell)->isMarked(cell); -} - -inline bool Heap::testAndSetMarked(const void* cell) -{ - return MarkedBlock::blockFor(cell)->testAndSetMarked(cell); -} - -inline void Heap::setMarked(const void* cell) -{ - MarkedBlock::blockFor(cell)->setMarked(cell); -} - -inline bool Heap::isWriteBarrierEnabled() -{ -#if ENABLE(WRITE_BARRIER_PROFILING) || ENABLE(GGC) - return true; -#else - return false; -#endif -} - -inline void Heap::writeBarrier(const JSCell* from, JSValue to) -{ -#if ENABLE(WRITE_BARRIER_PROFILING) - WriteBarrierCounters::countWriteBarrier(); -#endif -#if ENABLE(GGC) - if (!to.isCell()) - return; - writeBarrier(from, to.asCell()); -#else - UNUSED_PARAM(from); - UNUSED_PARAM(to); -#endif -} - -inline void Heap::writeBarrier(const JSCell* from, JSCell* to) -{ -#if ENABLE(WRITE_BARRIER_PROFILING) - WriteBarrierCounters::countWriteBarrier(); -#endif -#if ENABLE(GGC) - if (!from || !from->isMarked()) { - ASSERT(!from || !isMarked(from)); - return; - } - if (!to || to->isMarked()) { - ASSERT(!to || isMarked(to)); - return; - } - addToRememberedSet(from); -#else - UNUSED_PARAM(from); - UNUSED_PARAM(to); -#endif -} - -inline void Heap::writeBarrier(const JSCell* from) -{ -#if ENABLE(GGC) - ASSERT_GC_OBJECT_LOOKS_VALID(const_cast<JSCell*>(from)); - if (!from || !from->isMarked()) { - ASSERT(!from || !isMarked(from)); - return; - } - ASSERT(isMarked(from)); - addToRememberedSet(from); -#else - UNUSED_PARAM(from); -#endif -} - -inline void Heap::reportExtraMemoryAllocated(size_t size) -{ - if (size > minExtraMemory) - reportExtraMemoryAllocatedSlowCase(size); -} - -inline void Heap::reportExtraMemoryVisited(JSCell* owner, size_t size) -{ -#if ENABLE(GGC) - // We don't want to double-count the extra memory that was reported in previous collections. - if (operationInProgress() == EdenCollection && Heap::isRemembered(owner)) - return; -#else - UNUSED_PARAM(owner); -#endif - - size_t* counter = &m_extraMemorySize; - -#if ENABLE(COMPARE_AND_SWAP) - for (;;) { - size_t oldSize = *counter; - if (WTF::weakCompareAndSwapSize(counter, oldSize, oldSize + size)) - return; - } -#else - (*counter) += size; -#endif -} - -inline void Heap::deprecatedReportExtraMemory(size_t size) -{ - if (size > minExtraMemory) - deprecatedReportExtraMemorySlowCase(size); -} - -template<typename Functor> inline typename Functor::ReturnType Heap::forEachProtectedCell(Functor& functor) -{ - for (auto& pair : m_protectedValues) - functor(pair.key); - m_handleSet.forEachStrongHandle(functor, m_protectedValues); - - return functor.returnValue(); -} - -template<typename Functor> inline typename Functor::ReturnType Heap::forEachProtectedCell() -{ - Functor functor; - return forEachProtectedCell(functor); -} - -template<typename Functor> inline void Heap::forEachCodeBlock(Functor& functor) -{ - return m_codeBlocks.iterate<Functor>(functor); -} - -inline void* Heap::allocateWithDestructor(size_t bytes) -{ -#if ENABLE(ALLOCATION_LOGGING) - dataLogF("JSC GC allocating %lu bytes with normal destructor.\n", bytes); -#endif - ASSERT(isValidAllocation(bytes)); - return m_objectSpace.allocateWithDestructor(bytes); -} - -inline void* Heap::allocateWithoutDestructor(size_t bytes) -{ -#if ENABLE(ALLOCATION_LOGGING) - dataLogF("JSC GC allocating %lu bytes without destructor.\n", bytes); -#endif - ASSERT(isValidAllocation(bytes)); - return m_objectSpace.allocateWithoutDestructor(bytes); -} - -template<typename ClassType> -void* Heap::allocateObjectOfType(size_t bytes) -{ - // JSCell::classInfo() expects objects allocated with normal destructor to derive from JSDestructibleObject. - ASSERT((!ClassType::needsDestruction || (ClassType::StructureFlags & StructureIsImmortal) || std::is_convertible<ClassType, JSDestructibleObject>::value)); - - if (ClassType::needsDestruction) - return allocateWithDestructor(bytes); - return allocateWithoutDestructor(bytes); -} - -template<typename ClassType> -MarkedSpace::Subspace& Heap::subspaceForObjectOfType() -{ - // JSCell::classInfo() expects objects allocated with normal destructor to derive from JSDestructibleObject. - ASSERT((!ClassType::needsDestruction || (ClassType::StructureFlags & StructureIsImmortal) || std::is_convertible<ClassType, JSDestructibleObject>::value)); - - if (ClassType::needsDestruction) - return subspaceForObjectDestructor(); - return subspaceForObjectWithoutDestructor(); -} - -template<typename ClassType> -MarkedAllocator& Heap::allocatorForObjectOfType(size_t bytes) -{ - // JSCell::classInfo() expects objects allocated with normal destructor to derive from JSDestructibleObject. - ASSERT((!ClassType::needsDestruction || (ClassType::StructureFlags & StructureIsImmortal) || std::is_convertible<ClassType, JSDestructibleObject>::value)); - - if (ClassType::needsDestruction) - return allocatorForObjectWithDestructor(bytes); - return allocatorForObjectWithoutDestructor(bytes); -} - -inline CheckedBoolean Heap::tryAllocateStorage(JSCell* intendedOwner, size_t bytes, void** outPtr) -{ - CheckedBoolean result = m_storageSpace.tryAllocate(bytes, outPtr); -#if ENABLE(ALLOCATION_LOGGING) - dataLogF("JSC GC allocating %lu bytes of storage for %p: %p.\n", bytes, intendedOwner, *outPtr); -#else - UNUSED_PARAM(intendedOwner); -#endif - return result; -} - -inline CheckedBoolean Heap::tryReallocateStorage(JSCell* intendedOwner, void** ptr, size_t oldSize, size_t newSize) -{ -#if ENABLE(ALLOCATION_LOGGING) - void* oldPtr = *ptr; -#endif - CheckedBoolean result = m_storageSpace.tryReallocate(ptr, oldSize, newSize); -#if ENABLE(ALLOCATION_LOGGING) - dataLogF("JSC GC reallocating %lu -> %lu bytes of storage for %p: %p -> %p.\n", oldSize, newSize, intendedOwner, oldPtr, *ptr); -#else - UNUSED_PARAM(intendedOwner); -#endif - return result; -} - -inline void Heap::ascribeOwner(JSCell* intendedOwner, void* storage) -{ -#if ENABLE(ALLOCATION_LOGGING) - dataLogF("JSC GC ascribing %p as owner of storage %p.\n", intendedOwner, storage); -#else - UNUSED_PARAM(intendedOwner); - UNUSED_PARAM(storage); -#endif -} - -#if USE(CF) -template <typename T> -inline void Heap::releaseSoon(RetainPtr<T>&& object) -{ - m_delayedReleaseObjects.append(WTF::move(object)); -} -#endif - -inline void Heap::incrementDeferralDepth() -{ - RELEASE_ASSERT(m_deferralDepth < 100); // Sanity check to make sure this doesn't get ridiculous. - m_deferralDepth++; -} - -inline void Heap::decrementDeferralDepth() -{ - RELEASE_ASSERT(m_deferralDepth >= 1); - m_deferralDepth--; -} - -inline bool Heap::collectIfNecessaryOrDefer() -{ - if (isDeferred()) - return false; - - if (!shouldCollect()) - return false; - - collect(); - return true; -} - -inline void Heap::decrementDeferralDepthAndGCIfNeeded() -{ - decrementDeferralDepth(); - collectIfNecessaryOrDefer(); -} - -inline HashSet<MarkedArgumentBuffer*>& Heap::markListSet() -{ - if (!m_markListSet) - m_markListSet = std::make_unique<HashSet<MarkedArgumentBuffer*>>(); - return *m_markListSet; -} - -inline void Heap::registerWeakGCMap(void* weakGCMap, std::function<void()> pruningCallback) -{ - m_weakGCMaps.add(weakGCMap, WTF::move(pruningCallback)); -} - -inline void Heap::unregisterWeakGCMap(void* weakGCMap) -{ - m_weakGCMaps.remove(weakGCMap); -} - -} // namespace JSC - -#endif // HeapInlines_h diff --git a/Source/JavaScriptCore/heap/HeapOperation.h b/Source/JavaScriptCore/heap/HeapOperation.h index 272e3c068..769127e89 100644 --- a/Source/JavaScriptCore/heap/HeapOperation.h +++ b/Source/JavaScriptCore/heap/HeapOperation.h @@ -28,7 +28,7 @@ namespace JSC { -enum HeapOperation { NoOperation, Allocation, FullCollection, EdenCollection, AnyCollection }; +enum HeapOperation { NoOperation, Allocation, FullCollection, EdenCollection }; } // namespace JSC diff --git a/Source/JavaScriptCore/heap/HeapStatistics.cpp b/Source/JavaScriptCore/heap/HeapStatistics.cpp index bc5465fb9..f23def711 100644 --- a/Source/JavaScriptCore/heap/HeapStatistics.cpp +++ b/Source/JavaScriptCore/heap/HeapStatistics.cpp @@ -28,8 +28,8 @@ #include "Heap.h" #include "HeapIterationScope.h" -#include "JSCInlines.h" #include "JSObject.h" +#include "Operations.h" #include "Options.h" #include <stdlib.h> #if OS(UNIX) @@ -132,7 +132,6 @@ void HeapStatistics::logStatistics() void HeapStatistics::exitWithFailure() { - exit(-1); } void HeapStatistics::reportSuccess() @@ -167,7 +166,7 @@ class StorageStatistics : public MarkedBlock::VoidFunctor { public: StorageStatistics(); - IterationStatus operator()(JSCell*); + void operator()(JSCell*); size_t objectWithOutOfLineStorageCount(); size_t objectCount(); @@ -176,8 +175,6 @@ public: size_t storageCapacity(); private: - void visit(JSCell*); - size_t m_objectWithOutOfLineStorageCount; size_t m_objectCount; size_t m_storageSize; @@ -192,13 +189,13 @@ inline StorageStatistics::StorageStatistics() { } -inline void StorageStatistics::visit(JSCell* cell) +inline void StorageStatistics::operator()(JSCell* cell) { if (!cell->isObject()) return; JSObject* object = jsCast<JSObject*>(cell); - if (hasIndexedProperties(object->indexingType())) + if (hasIndexedProperties(object->structure()->indexingType())) return; if (object->structure()->isUncacheableDictionary()) @@ -211,12 +208,6 @@ inline void StorageStatistics::visit(JSCell* cell) m_storageCapacity += object->structure()->totalStorageCapacity() * sizeof(WriteBarrierBase<Unknown>); } -inline IterationStatus StorageStatistics::operator()(JSCell* cell) -{ - visit(cell); - return IterationStatus::Continue; -} - inline size_t StorageStatistics::objectWithOutOfLineStorageCount() { return m_objectWithOutOfLineStorageCount; @@ -242,26 +233,25 @@ void HeapStatistics::showObjectStatistics(Heap* heap) dataLogF("\n=== Heap Statistics: ===\n"); dataLogF("size: %ldkB\n", static_cast<long>(heap->m_sizeAfterLastCollect / KB)); dataLogF("capacity: %ldkB\n", static_cast<long>(heap->capacity() / KB)); - dataLogF("pause time: %lfs\n\n", heap->m_lastFullGCLength); + dataLogF("pause time: %lfms\n\n", heap->m_lastGCLength); StorageStatistics storageStatistics; { HeapIterationScope iterationScope(*heap); heap->m_objectSpace.forEachLiveCell(iterationScope, storageStatistics); } - long wastedPropertyStorageBytes = 0; - long wastedPropertyStoragePercent = 0; - long objectWithOutOfLineStorageCount = 0; - long objectsWithOutOfLineStoragePercent = 0; - if ((storageStatistics.storageCapacity() > 0) && (storageStatistics.objectCount() > 0)) { - wastedPropertyStorageBytes = static_cast<long>((storageStatistics.storageCapacity() - storageStatistics.storageSize()) / KB); - wastedPropertyStoragePercent = static_cast<long>( - (storageStatistics.storageCapacity() - storageStatistics.storageSize()) * 100 / storageStatistics.storageCapacity()); - objectWithOutOfLineStorageCount = static_cast<long>(storageStatistics.objectWithOutOfLineStorageCount()); - objectsWithOutOfLineStoragePercent = objectWithOutOfLineStorageCount * 100 / storageStatistics.objectCount(); - } - dataLogF("wasted .property storage: %ldkB (%ld%%)\n", wastedPropertyStorageBytes, wastedPropertyStoragePercent); - dataLogF("objects with out-of-line .property storage: %ld (%ld%%)\n", objectWithOutOfLineStorageCount, objectsWithOutOfLineStoragePercent); + dataLogF("wasted .property storage: %ldkB (%ld%%)\n", + static_cast<long>( + (storageStatistics.storageCapacity() - storageStatistics.storageSize()) / KB), + static_cast<long>( + (storageStatistics.storageCapacity() - storageStatistics.storageSize()) * 100 + / storageStatistics.storageCapacity())); + dataLogF("objects with out-of-line .property storage: %ld (%ld%%)\n", + static_cast<long>( + storageStatistics.objectWithOutOfLineStorageCount()), + static_cast<long>( + storageStatistics.objectWithOutOfLineStorageCount() * 100 + / storageStatistics.objectCount())); } } // namespace JSC diff --git a/Source/JavaScriptCore/heap/HeapTimer.cpp b/Source/JavaScriptCore/heap/HeapTimer.cpp index 15e5484db..1331b0ac0 100644 --- a/Source/JavaScriptCore/heap/HeapTimer.cpp +++ b/Source/JavaScriptCore/heap/HeapTimer.cpp @@ -26,11 +26,10 @@ #include "config.h" #include "HeapTimer.h" -#include "GCActivityCallback.h" -#include "IncrementalSweeper.h" +#include "APIShims.h" #include "JSObject.h" #include "JSString.h" -#include "JSCInlines.h" + #include <wtf/MainThread.h> #include <wtf/Threading.h> @@ -63,7 +62,7 @@ HeapTimer::HeapTimer(VM* vm, CFRunLoopRef runLoop) m_context.info = &vm->apiLock(); m_context.retain = retainAPILock; m_context.release = releaseAPILock; - m_timer = adoptCF(CFRunLoopTimerCreate(kCFAllocatorDefault, s_decade, s_decade, 0, 0, HeapTimer::timerDidFire, &m_context)); + m_timer = adoptCF(CFRunLoopTimerCreate(0, s_decade, s_decade, 0, 0, HeapTimer::timerDidFire, &m_context)); CFRunLoopAddTimer(m_runLoop.get(), m_timer.get(), kCFRunLoopCommonModes); } @@ -86,17 +85,15 @@ void HeapTimer::timerDidFire(CFRunLoopTimerRef timer, void* context) } HeapTimer* heapTimer = 0; - if (vm->heap.fullActivityCallback() && vm->heap.fullActivityCallback()->m_timer.get() == timer) - heapTimer = vm->heap.fullActivityCallback(); - else if (vm->heap.edenActivityCallback() && vm->heap.edenActivityCallback()->m_timer.get() == timer) - heapTimer = vm->heap.edenActivityCallback(); + if (vm->heap.activityCallback() && vm->heap.activityCallback()->m_timer.get() == timer) + heapTimer = vm->heap.activityCallback(); else if (vm->heap.sweeper()->m_timer.get() == timer) heapTimer = vm->heap.sweeper(); else RELEASE_ASSERT_NOT_REACHED(); { - JSLockHolder locker(vm); + APIEntryShim shim(vm); heapTimer->doWork(); } @@ -134,7 +131,7 @@ bool HeapTimer::timerEvent(void* info) { HeapTimer* agent = static_cast<HeapTimer*>(info); - JSLockHolder locker(agent->m_vm); + APIEntryShim shim(agent->m_vm); agent->doWork(); agent->m_timer = 0; diff --git a/Source/JavaScriptCore/heap/HeapTimer.h b/Source/JavaScriptCore/heap/HeapTimer.h index bbce5f90f..760405c79 100644 --- a/Source/JavaScriptCore/heap/HeapTimer.h +++ b/Source/JavaScriptCore/heap/HeapTimer.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012, 2015 Apple Inc. All rights reserved. + * Copyright (C) 2012 Apple Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -26,12 +26,17 @@ #ifndef HeapTimer_h #define HeapTimer_h -#include <wtf/Lock.h> #include <wtf/RetainPtr.h> #include <wtf/Threading.h> #if USE(CF) #include <CoreFoundation/CoreFoundation.h> +#elif PLATFORM(EFL) +#if USE(EO) +typedef struct _Eo_Opaque Ecore_Timer; +#else +typedef struct _Ecore_Timer Ecore_Timer; +#endif #endif namespace JSC { @@ -60,7 +65,7 @@ protected: RetainPtr<CFRunLoopRef> m_runLoop; CFRunLoopTimerContext m_context; - Lock m_shutdownMutex; + Mutex m_shutdownMutex; #elif PLATFORM(EFL) static bool timerEvent(void*); Ecore_Timer* add(double delay, void* agent); diff --git a/Source/JavaScriptCore/heap/HeapVerifier.cpp b/Source/JavaScriptCore/heap/HeapVerifier.cpp deleted file mode 100644 index 0f4e28277..000000000 --- a/Source/JavaScriptCore/heap/HeapVerifier.cpp +++ /dev/null @@ -1,290 +0,0 @@ -/* - * Copyright (C) 2014 Apple Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY - * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include "config.h" -#include "HeapVerifier.h" - -#include "ButterflyInlines.h" -#include "CopiedSpaceInlines.h" -#include "HeapIterationScope.h" -#include "JSCInlines.h" -#include "JSObject.h" - -namespace JSC { - -HeapVerifier::HeapVerifier(Heap* heap, unsigned numberOfGCCyclesToRecord) - : m_heap(heap) - , m_currentCycle(0) - , m_numberOfCycles(numberOfGCCyclesToRecord) -{ - RELEASE_ASSERT(m_numberOfCycles > 0); - m_cycles = std::make_unique<GCCycle[]>(m_numberOfCycles); -} - -const char* HeapVerifier::collectionTypeName(HeapOperation type) -{ - switch (type) { - case NoOperation: - return "NoOperation"; - case AnyCollection: - return "AnyCollection"; - case Allocation: - return "Allocation"; - case EdenCollection: - return "EdenCollection"; - case FullCollection: - return "FullCollection"; - } - RELEASE_ASSERT_NOT_REACHED(); - return nullptr; // Silencing a compiler warning. -} - -const char* HeapVerifier::phaseName(HeapVerifier::Phase phase) -{ - switch (phase) { - case Phase::BeforeGC: - return "BeforeGC"; - case Phase::BeforeMarking: - return "BeforeMarking"; - case Phase::AfterMarking: - return "AfterMarking"; - case Phase::AfterGC: - return "AfterGC"; - } - RELEASE_ASSERT_NOT_REACHED(); - return nullptr; // Silencing a compiler warning. -} - -static void getButterflyDetails(JSObject* obj, void*& butterflyBase, size_t& butterflyCapacityInBytes, CopiedBlock*& butterflyBlock) -{ - Structure* structure = obj->structure(); - Butterfly* butterfly = obj->butterfly(); - butterflyBase = butterfly->base(structure); - butterflyBlock = CopiedSpace::blockFor(butterflyBase); - - size_t propertyCapacity = structure->outOfLineCapacity(); - size_t preCapacity; - size_t indexingPayloadSizeInBytes; - bool hasIndexingHeader = obj->hasIndexingHeader(); - if (UNLIKELY(hasIndexingHeader)) { - preCapacity = butterfly->indexingHeader()->preCapacity(structure); - indexingPayloadSizeInBytes = butterfly->indexingHeader()->indexingPayloadSizeInBytes(structure); - } else { - preCapacity = 0; - indexingPayloadSizeInBytes = 0; - } - butterflyCapacityInBytes = Butterfly::totalSize(preCapacity, propertyCapacity, hasIndexingHeader, indexingPayloadSizeInBytes); -} - -void HeapVerifier::initializeGCCycle() -{ - Heap* heap = m_heap; - incrementCycle(); - currentCycle().collectionType = heap->operationInProgress(); -} - -struct GatherLiveObjFunctor : MarkedBlock::CountFunctor { - GatherLiveObjFunctor(LiveObjectList& list) - : m_list(list) - { - ASSERT(!list.liveObjects.size()); - } - - inline void visit(JSCell* cell) - { - if (!cell->isObject()) - return; - LiveObjectData data(asObject(cell)); - m_list.liveObjects.append(data); - } - - IterationStatus operator()(JSCell* cell) - { - visit(cell); - return IterationStatus::Continue; - } - - LiveObjectList& m_list; -}; - -void HeapVerifier::gatherLiveObjects(HeapVerifier::Phase phase) -{ - Heap* heap = m_heap; - LiveObjectList& list = *liveObjectListForGathering(phase); - - HeapIterationScope iterationScope(*heap); - list.reset(); - GatherLiveObjFunctor functor(list); - heap->m_objectSpace.forEachLiveCell(iterationScope, functor); -} - -LiveObjectList* HeapVerifier::liveObjectListForGathering(HeapVerifier::Phase phase) -{ - switch (phase) { - case Phase::BeforeMarking: - return ¤tCycle().before; - case Phase::AfterMarking: - return ¤tCycle().after; - case Phase::BeforeGC: - case Phase::AfterGC: - // We should not be gathering live objects during these phases. - break; - } - RELEASE_ASSERT_NOT_REACHED(); - return nullptr; // Silencing a compiler warning. -} - -static void trimDeadObjectsFromList(HashSet<JSObject*>& knownLiveSet, LiveObjectList& list) -{ - if (!list.hasLiveObjects) - return; - - size_t liveObjectsFound = 0; - for (size_t i = 0; i < list.liveObjects.size(); i++) { - LiveObjectData& objData = list.liveObjects[i]; - if (objData.isConfirmedDead) - continue; // Don't "resurrect" known dead objects. - if (!knownLiveSet.contains(objData.obj)) { - objData.isConfirmedDead = true; - continue; - } - liveObjectsFound++; - } - list.hasLiveObjects = !!liveObjectsFound; -} - -void HeapVerifier::trimDeadObjects() -{ - HashSet<JSObject*> knownLiveSet; - - LiveObjectList& after = currentCycle().after; - for (size_t i = 0; i < after.liveObjects.size(); i++) { - LiveObjectData& objData = after.liveObjects[i]; - knownLiveSet.add(objData.obj); - } - - trimDeadObjectsFromList(knownLiveSet, currentCycle().before); - - for (int i = -1; i > -m_numberOfCycles; i--) { - trimDeadObjectsFromList(knownLiveSet, cycleForIndex(i).before); - trimDeadObjectsFromList(knownLiveSet, cycleForIndex(i).after); - } -} - -bool HeapVerifier::verifyButterflyIsInStorageSpace(Phase phase, LiveObjectList& list) -{ - auto& liveObjects = list.liveObjects; - - CopiedSpace& storageSpace = m_heap->m_storageSpace; - bool listNamePrinted = false; - bool success = true; - for (size_t i = 0; i < liveObjects.size(); i++) { - LiveObjectData& objectData = liveObjects[i]; - if (objectData.isConfirmedDead) - continue; - - JSObject* obj = objectData.obj; - Butterfly* butterfly = obj->butterfly(); - if (butterfly) { - void* butterflyBase; - size_t butterflyCapacityInBytes; - CopiedBlock* butterflyBlock; - getButterflyDetails(obj, butterflyBase, butterflyCapacityInBytes, butterflyBlock); - - if (!storageSpace.contains(butterflyBlock)) { - if (!listNamePrinted) { - dataLogF("Verification @ phase %s FAILED in object list '%s' (size %zu)\n", - phaseName(phase), list.name, liveObjects.size()); - listNamePrinted = true; - } - - Structure* structure = obj->structure(); - const char* structureClassName = structure->classInfo()->className; - dataLogF(" butterfly %p (base %p size %zu block %p) NOT in StorageSpace | obj %p type '%s'\n", - butterfly, butterflyBase, butterflyCapacityInBytes, butterflyBlock, obj, structureClassName); - success = false; - } - } - } - return success; -} - -void HeapVerifier::verify(HeapVerifier::Phase phase) -{ - bool beforeVerified = verifyButterflyIsInStorageSpace(phase, currentCycle().before); - bool afterVerified = verifyButterflyIsInStorageSpace(phase, currentCycle().after); - RELEASE_ASSERT(beforeVerified && afterVerified); -} - -void HeapVerifier::reportObject(LiveObjectData& objData, int cycleIndex, HeapVerifier::GCCycle& cycle, LiveObjectList& list) -{ - JSObject* obj = objData.obj; - - if (objData.isConfirmedDead) { - dataLogF("FOUND dead obj %p in GC[%d] %s list '%s'\n", - obj, cycleIndex, cycle.collectionTypeName(), list.name); - return; - } - - Structure* structure = obj->structure(); - Butterfly* butterfly = obj->butterfly(); - void* butterflyBase; - size_t butterflyCapacityInBytes; - CopiedBlock* butterflyBlock; - getButterflyDetails(obj, butterflyBase, butterflyCapacityInBytes, butterflyBlock); - - dataLogF("FOUND obj %p type '%s' butterfly %p (base %p size %zu block %p) in GC[%d] %s list '%s'\n", - obj, structure->classInfo()->className, - butterfly, butterflyBase, butterflyCapacityInBytes, butterflyBlock, - cycleIndex, cycle.collectionTypeName(), list.name); -} - -void HeapVerifier::checkIfRecorded(JSObject* obj) -{ - bool found = false; - - for (int cycleIndex = 0; cycleIndex > -m_numberOfCycles; cycleIndex--) { - GCCycle& cycle = cycleForIndex(cycleIndex); - LiveObjectList& beforeList = cycle.before; - LiveObjectList& afterList = cycle.after; - - LiveObjectData* objData; - objData = beforeList.findObject(obj); - if (objData) { - reportObject(*objData, cycleIndex, cycle, beforeList); - found = true; - } - objData = afterList.findObject(obj); - if (objData) { - reportObject(*objData, cycleIndex, cycle, afterList); - found = true; - } - } - - if (!found) - dataLogF("obj %p NOT FOUND\n", obj); -} - -} // namespace JSC diff --git a/Source/JavaScriptCore/heap/HeapVerifier.h b/Source/JavaScriptCore/heap/HeapVerifier.h deleted file mode 100644 index d55ec4a89..000000000 --- a/Source/JavaScriptCore/heap/HeapVerifier.h +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright (C) 2014-2015 Apple Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY - * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef HeapVerifier_h -#define HeapVerifier_h - -#include "Heap.h" -#include "LiveObjectList.h" - -namespace JSC { - -class JSObject; -class MarkedBlock; - -class HeapVerifier { - WTF_MAKE_FAST_ALLOCATED; -public: - enum class Phase { - BeforeGC, - BeforeMarking, - AfterMarking, - AfterGC - }; - - HeapVerifier(Heap*, unsigned numberOfGCCyclesToRecord); - - void initializeGCCycle(); - void gatherLiveObjects(Phase); - void trimDeadObjects(); - void verify(Phase); - - // Scans all previously recorded LiveObjectLists and checks if the specified - // object was in any of those lists. - JS_EXPORT_PRIVATE void checkIfRecorded(JSObject*); - - static const char* collectionTypeName(HeapOperation); - static const char* phaseName(Phase); - -private: - struct GCCycle { - GCCycle() - : before("Before Marking") - , after("After Marking") - { - } - - HeapOperation collectionType; - LiveObjectList before; - LiveObjectList after; - - const char* collectionTypeName() const - { - return HeapVerifier::collectionTypeName(collectionType); - } - }; - - void incrementCycle() { m_currentCycle = (m_currentCycle + 1) % m_numberOfCycles; } - GCCycle& currentCycle() { return m_cycles[m_currentCycle]; } - GCCycle& cycleForIndex(int cycleIndex) - { - ASSERT(cycleIndex <= 0 && cycleIndex > -m_numberOfCycles); - cycleIndex += m_currentCycle; - if (cycleIndex < 0) - cycleIndex += m_numberOfCycles; - ASSERT(cycleIndex < m_numberOfCycles); - return m_cycles[cycleIndex]; - } - - LiveObjectList* liveObjectListForGathering(Phase); - bool verifyButterflyIsInStorageSpace(Phase, LiveObjectList&); - - static void reportObject(LiveObjectData&, int cycleIndex, HeapVerifier::GCCycle&, LiveObjectList&); - - Heap* m_heap; - int m_currentCycle; - int m_numberOfCycles; - std::unique_ptr<GCCycle[]> m_cycles; -}; - -} // namespace JSC - -#endif // HeapVerifier diff --git a/Source/JavaScriptCore/heap/IncrementalSweeper.cpp b/Source/JavaScriptCore/heap/IncrementalSweeper.cpp index e0783d615..2852266aa 100644 --- a/Source/JavaScriptCore/heap/IncrementalSweeper.cpp +++ b/Source/JavaScriptCore/heap/IncrementalSweeper.cpp @@ -26,11 +26,12 @@ #include "config.h" #include "IncrementalSweeper.h" +#include "APIShims.h" +#include "DelayedReleaseScope.h" #include "Heap.h" #include "JSObject.h" #include "JSString.h" #include "MarkedBlock.h" -#include "JSCInlines.h" #include <wtf/HashSet.h> #include <wtf/WTFThreadData.h> @@ -45,10 +46,16 @@ static const double sweepTimeMultiplier = 1.0 / sweepTimeTotal; IncrementalSweeper::IncrementalSweeper(Heap* heap, CFRunLoopRef runLoop) : HeapTimer(heap->vm(), runLoop) + , m_currentBlockToSweepIndex(0) , m_blocksToSweep(heap->m_blockSnapshot) { } +PassOwnPtr<IncrementalSweeper> IncrementalSweeper::create(Heap* heap) +{ + return adoptPtr(new IncrementalSweeper(heap, CFRunLoopGetCurrent())); +} + void IncrementalSweeper::scheduleTimer() { CFRunLoopTimerSetNextFireDate(m_timer.get(), CFAbsoluteTimeGetCurrent() + (sweepTimeSlice * sweepTimeMultiplier)); @@ -66,7 +73,10 @@ void IncrementalSweeper::doWork() void IncrementalSweeper::doSweep(double sweepBeginTime) { - while (sweepNextBlock()) { + DelayedReleaseScope scope(m_vm->heap.m_objectSpace); + while (m_currentBlockToSweepIndex < m_blocksToSweep.size()) { + sweepNextBlock(); + double elapsedTime = WTF::monotonicallyIncreasingTime() - sweepBeginTime; if (elapsedTime < sweepTimeSlice) continue; @@ -79,30 +89,30 @@ void IncrementalSweeper::doSweep(double sweepBeginTime) cancelTimer(); } -bool IncrementalSweeper::sweepNextBlock() +void IncrementalSweeper::sweepNextBlock() { - while (!m_blocksToSweep.isEmpty()) { - MarkedBlock* block = m_blocksToSweep.takeLast(); + while (m_currentBlockToSweepIndex < m_blocksToSweep.size()) { + MarkedBlock* block = m_blocksToSweep[m_currentBlockToSweepIndex++]; if (!block->needsSweeping()) continue; - DeferGCForAWhile deferGC(m_vm->heap); block->sweep(); m_vm->heap.objectSpace().freeOrShrinkBlock(block); - return true; + return; } - - return m_vm->heap.sweepNextLogicallyEmptyWeakBlock(); } -void IncrementalSweeper::startSweeping() +void IncrementalSweeper::startSweeping(Vector<MarkedBlock*>& blockSnapshot) { + m_blocksToSweep = blockSnapshot; + m_currentBlockToSweepIndex = 0; scheduleTimer(); } void IncrementalSweeper::willFinishSweeping() { + m_currentBlockToSweepIndex = 0; m_blocksToSweep.clear(); if (m_vm) cancelTimer(); @@ -119,7 +129,12 @@ void IncrementalSweeper::doWork() { } -void IncrementalSweeper::startSweeping() +PassOwnPtr<IncrementalSweeper> IncrementalSweeper::create(Heap* heap) +{ + return adoptPtr(new IncrementalSweeper(heap->vm())); +} + +void IncrementalSweeper::startSweeping(Vector<MarkedBlock*>&) { } @@ -127,9 +142,8 @@ void IncrementalSweeper::willFinishSweeping() { } -bool IncrementalSweeper::sweepNextBlock() +void IncrementalSweeper::sweepNextBlock() { - return false; } #endif diff --git a/Source/JavaScriptCore/heap/IncrementalSweeper.h b/Source/JavaScriptCore/heap/IncrementalSweeper.h index a86fd1ce4..0ac145cbd 100644 --- a/Source/JavaScriptCore/heap/IncrementalSweeper.h +++ b/Source/JavaScriptCore/heap/IncrementalSweeper.h @@ -27,6 +27,7 @@ #define IncrementalSweeper_h #include "HeapTimer.h" +#include <wtf/PassOwnPtr.h> #include <wtf/Vector.h> namespace JSC { @@ -35,26 +36,27 @@ class Heap; class MarkedBlock; class IncrementalSweeper : public HeapTimer { - WTF_MAKE_FAST_ALLOCATED; public: + static PassOwnPtr<IncrementalSweeper> create(Heap*); + void startSweeping(Vector<MarkedBlock*>&); + JS_EXPORT_PRIVATE virtual void doWork() override; + void sweepNextBlock(); + void willFinishSweeping(); + +protected: #if USE(CF) JS_EXPORT_PRIVATE IncrementalSweeper(Heap*, CFRunLoopRef); #else - explicit IncrementalSweeper(VM*); + IncrementalSweeper(VM*); #endif - void startSweeping(); - - JS_EXPORT_PRIVATE virtual void doWork() override; - bool sweepNextBlock(); - void willFinishSweeping(); - #if USE(CF) private: void doSweep(double startTime); void scheduleTimer(); void cancelTimer(); + unsigned m_currentBlockToSweepIndex; Vector<MarkedBlock*>& m_blocksToSweep; #endif }; diff --git a/Source/JavaScriptCore/heap/JITStubRoutineSet.cpp b/Source/JavaScriptCore/heap/JITStubRoutineSet.cpp index ae8059532..a37dc6f5c 100644 --- a/Source/JavaScriptCore/heap/JITStubRoutineSet.cpp +++ b/Source/JavaScriptCore/heap/JITStubRoutineSet.cpp @@ -29,7 +29,7 @@ #if ENABLE(JIT) #include "GCAwareJITStubRoutine.h" -#include "JSCInlines.h" + #include "SlotVisitor.h" namespace JSC { diff --git a/Source/JavaScriptCore/heap/JITStubRoutineSet.h b/Source/JavaScriptCore/heap/JITStubRoutineSet.h index 25ec44c13..29f0a4bff 100644 --- a/Source/JavaScriptCore/heap/JITStubRoutineSet.h +++ b/Source/JavaScriptCore/heap/JITStubRoutineSet.h @@ -26,6 +26,8 @@ #ifndef JITStubRoutineSet_h #define JITStubRoutineSet_h +#include <wtf/Platform.h> + #include "JITStubRoutine.h" #include <wtf/FastMalloc.h> #include <wtf/HashMap.h> diff --git a/Source/JavaScriptCore/heap/ListableHandler.h b/Source/JavaScriptCore/heap/ListableHandler.h index d653f0513..16c34146c 100644 --- a/Source/JavaScriptCore/heap/ListableHandler.h +++ b/Source/JavaScriptCore/heap/ListableHandler.h @@ -21,10 +21,10 @@ #define ListableHandler_h #include <stdint.h> -#include <wtf/Lock.h> #include <wtf/Locker.h> #include <wtf/Noncopyable.h> #include <wtf/ThreadingPrimitives.h> +#include <wtf/TCSpinLock.h> namespace JSC { @@ -61,11 +61,12 @@ private: List() : m_first(0) { + m_lock.Init(); } void addThreadSafe(T* handler) { - LockHolder locker(&m_lock); + SpinLockHolder locker(&m_lock); addNotThreadSafe(handler); } @@ -103,7 +104,7 @@ private: m_first = handler; } - Lock m_lock; + SpinLock m_lock; T* m_first; }; diff --git a/Source/JavaScriptCore/heap/LiveObjectData.h b/Source/JavaScriptCore/heap/LiveObjectData.h deleted file mode 100644 index 6953266e0..000000000 --- a/Source/JavaScriptCore/heap/LiveObjectData.h +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (C) 2015 Apple Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY - * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef LiveObjectData_h -#define LiveObjectData_h - -namespace JSC { - -class JSObject; - -struct LiveObjectData { - LiveObjectData(JSObject* obj, bool isConfirmedDead = false) - : obj(obj) - , isConfirmedDead(isConfirmedDead) - { - } - - JSObject* obj; - bool isConfirmedDead; -}; - -} // namespace JSC - -#endif // LiveObjectData_h - diff --git a/Source/JavaScriptCore/heap/LiveObjectList.cpp b/Source/JavaScriptCore/heap/LiveObjectList.cpp deleted file mode 100644 index af0367dc3..000000000 --- a/Source/JavaScriptCore/heap/LiveObjectList.cpp +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (C) 2015 Apple Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY - * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include "config.h" -#include "LiveObjectList.h" - -namespace JSC { - -LiveObjectData* LiveObjectList::findObject(JSObject* obj) -{ - for (size_t i = 0; i < liveObjects.size(); i++) { - LiveObjectData& data = liveObjects[i]; - if (obj == data.obj) - return &data; - } - return nullptr; -} - -} // namespace JSC diff --git a/Source/JavaScriptCore/heap/MachineStackMarker.cpp b/Source/JavaScriptCore/heap/MachineStackMarker.cpp index 5156c54ca..f546cb38b 100644 --- a/Source/JavaScriptCore/heap/MachineStackMarker.cpp +++ b/Source/JavaScriptCore/heap/MachineStackMarker.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2003-2009, 2015 Apple Inc. All rights reserved. + * Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009 Apple Inc. All rights reserved. * Copyright (C) 2007 Eric Seidel <eric@webkit.org> * Copyright (C) 2009 Acision BV. All rights reserved. * @@ -25,7 +25,6 @@ #include "ConservativeRoots.h" #include "Heap.h" #include "JSArray.h" -#include "JSCInlines.h" #include "VM.h" #include <setjmp.h> #include <stdlib.h> @@ -69,10 +68,22 @@ using namespace WTF; namespace JSC { +static inline void swapIfBackwards(void*& begin, void*& end) +{ +#if OS(WINCE) + if (begin <= end) + return; + std::swap(begin, end); +#else +UNUSED_PARAM(begin); +UNUSED_PARAM(end); +#endif +} + #if OS(DARWIN) typedef mach_port_t PlatformThread; #elif OS(WINDOWS) -typedef DWORD PlatformThread; +typedef HANDLE PlatformThread; #elif USE(PTHREADS) typedef pthread_t PlatformThread; static const int SigThreadSuspendResume = SIGUSR2; @@ -88,78 +99,9 @@ static void pthreadSignalHandlerSuspendResume(int) #endif #endif -class ActiveMachineThreadsManager; -static ActiveMachineThreadsManager& activeMachineThreadsManager(); - -class ActiveMachineThreadsManager { - WTF_MAKE_NONCOPYABLE(ActiveMachineThreadsManager); -public: - - class Locker { - public: - Locker(ActiveMachineThreadsManager& manager) - : m_locker(manager.m_lock) - { - } - - private: - LockHolder m_locker; - }; - - void add(MachineThreads* machineThreads) - { - LockHolder managerLock(m_lock); - m_set.add(machineThreads); - } - - void remove(MachineThreads* machineThreads) - { - LockHolder managerLock(m_lock); - auto recordedMachineThreads = m_set.take(machineThreads); - RELEASE_ASSERT(recordedMachineThreads = machineThreads); - } - - bool contains(MachineThreads* machineThreads) - { - return m_set.contains(machineThreads); - } - -private: - typedef HashSet<MachineThreads*> MachineThreadsSet; - - ActiveMachineThreadsManager() { } - - Lock m_lock; - MachineThreadsSet m_set; - - friend ActiveMachineThreadsManager& activeMachineThreadsManager(); -}; - -static ActiveMachineThreadsManager& activeMachineThreadsManager() -{ - static std::once_flag initializeManagerOnceFlag; - static ActiveMachineThreadsManager* manager = nullptr; - - std::call_once(initializeManagerOnceFlag, [] { - manager = new ActiveMachineThreadsManager(); - }); - return *manager; -} - -static inline PlatformThread getCurrentPlatformThread() -{ -#if OS(DARWIN) - return pthread_mach_thread_np(pthread_self()); -#elif OS(WINDOWS) - return GetCurrentThreadId(); -#elif USE(PTHREADS) - return pthread_self(); -#endif -} - class MachineThreads::Thread { WTF_MAKE_FAST_ALLOCATED; - +public: Thread(const PlatformThread& platThread, void* base) : platformThread(platThread) , stackBase(base) @@ -176,74 +118,12 @@ class MachineThreads::Thread { sigemptyset(&mask); sigaddset(&mask, SigThreadSuspendResume); pthread_sigmask(SIG_UNBLOCK, &mask, 0); -#elif OS(WINDOWS) - ASSERT(platformThread == GetCurrentThreadId()); - bool isSuccessful = - DuplicateHandle(GetCurrentProcess(), GetCurrentThread(), GetCurrentProcess(), - &platformThreadHandle, 0, FALSE, DUPLICATE_SAME_ACCESS); - RELEASE_ASSERT(isSuccessful); -#endif - } - -public: - ~Thread() - { -#if OS(WINDOWS) - CloseHandle(platformThreadHandle); #endif } - static Thread* createForCurrentThread() - { - return new Thread(getCurrentPlatformThread(), wtfThreadData().stack().origin()); - } - - struct Registers { - inline void* stackPointer() const; - -#if OS(DARWIN) -#if CPU(X86) - typedef i386_thread_state_t PlatformRegisters; -#elif CPU(X86_64) - typedef x86_thread_state64_t PlatformRegisters; -#elif CPU(PPC) - typedef ppc_thread_state_t PlatformRegisters; -#elif CPU(PPC64) - typedef ppc_thread_state64_t PlatformRegisters; -#elif CPU(ARM) - typedef arm_thread_state_t PlatformRegisters; -#elif CPU(ARM64) - typedef arm_thread_state64_t PlatformRegisters; -#else -#error Unknown Architecture -#endif - -#elif OS(WINDOWS) - typedef CONTEXT PlatformRegisters; -#elif USE(PTHREADS) - typedef pthread_attr_t PlatformRegisters; -#else -#error Need a thread register struct for this platform -#endif - - PlatformRegisters regs; - }; - - inline bool operator==(const PlatformThread& other) const; - inline bool operator!=(const PlatformThread& other) const { return !(*this == other); } - - inline bool suspend(); - inline void resume(); - size_t getRegisters(Registers&); - void freeRegisters(Registers&); - std::pair<void*, size_t> captureStack(void* stackTop); - Thread* next; PlatformThread platformThread; void* stackBase; -#if OS(WINDOWS) - HANDLE platformThreadHandle; -#endif }; MachineThreads::MachineThreads(Heap* heap) @@ -254,16 +134,14 @@ MachineThreads::MachineThreads(Heap* heap) #endif { UNUSED_PARAM(heap); - threadSpecificKeyCreate(&m_threadSpecific, removeThread); - activeMachineThreadsManager().add(this); } MachineThreads::~MachineThreads() { - activeMachineThreadsManager().remove(this); - threadSpecificKeyDelete(m_threadSpecific); + if (m_threadSpecific) + threadSpecificKeyDelete(m_threadSpecific); - LockHolder registeredThreadsLock(m_registeredThreadsMutex); + MutexLocker registeredThreadsLock(m_registeredThreadsMutex); for (Thread* t = m_registeredThreads; t;) { Thread* next = t->next; delete t; @@ -271,30 +149,47 @@ MachineThreads::~MachineThreads() } } -inline bool MachineThreads::Thread::operator==(const PlatformThread& other) const +static inline PlatformThread getCurrentPlatformThread() +{ +#if OS(DARWIN) + return pthread_mach_thread_np(pthread_self()); +#elif OS(WINDOWS) + return GetCurrentThread(); +#elif USE(PTHREADS) + return pthread_self(); +#endif +} + +static inline bool equalThread(const PlatformThread& first, const PlatformThread& second) { #if OS(DARWIN) || OS(WINDOWS) - return platformThread == other; + return first == second; #elif USE(PTHREADS) - return !!pthread_equal(platformThread, other); + return !!pthread_equal(first, second); #else #error Need a way to compare threads on this platform #endif } +void MachineThreads::makeUsableFromMultipleThreads() +{ + if (m_threadSpecific) + return; + + threadSpecificKeyCreate(&m_threadSpecific, removeThread); +} + void MachineThreads::addCurrentThread() { - ASSERT(!m_heap->vm()->hasExclusiveThread() || m_heap->vm()->exclusiveThread() == std::this_thread::get_id()); + ASSERT(!m_heap->vm()->exclusiveThread || m_heap->vm()->exclusiveThread == currentThread()); - if (threadSpecificGet(m_threadSpecific)) { - ASSERT(threadSpecificGet(m_threadSpecific) == this); + if (!m_threadSpecific || threadSpecificGet(m_threadSpecific)) return; - } threadSpecificSet(m_threadSpecific, this); - Thread* thread = Thread::createForCurrentThread(); + Thread* thread = new Thread(getCurrentPlatformThread(), wtfThreadData().stack().origin()); - LockHolder lock(m_registeredThreadsMutex); + MutexLocker lock(m_registeredThreadsMutex); thread->next = m_registeredThreads; m_registeredThreads = thread; @@ -302,73 +197,84 @@ void MachineThreads::addCurrentThread() void MachineThreads::removeThread(void* p) { - auto& manager = activeMachineThreadsManager(); - ActiveMachineThreadsManager::Locker lock(manager); - auto machineThreads = static_cast<MachineThreads*>(p); - if (manager.contains(machineThreads)) { - // There's a chance that the MachineThreads registry that this thread - // was registered with was already destructed, and another one happened - // to be instantiated at the same address. Hence, this thread may or - // may not be found in this MachineThreads registry. We only need to - // do a removal if this thread is found in it. - machineThreads->removeThreadIfFound(getCurrentPlatformThread()); - } + if (p) + static_cast<MachineThreads*>(p)->removeCurrentThread(); } -template<typename PlatformThread> -void MachineThreads::removeThreadIfFound(PlatformThread platformThread) +void MachineThreads::removeCurrentThread() { - LockHolder lock(m_registeredThreadsMutex); - Thread* t = m_registeredThreads; - if (*t == platformThread) { + PlatformThread currentPlatformThread = getCurrentPlatformThread(); + + MutexLocker lock(m_registeredThreadsMutex); + + if (equalThread(currentPlatformThread, m_registeredThreads->platformThread)) { + Thread* t = m_registeredThreads; m_registeredThreads = m_registeredThreads->next; delete t; } else { Thread* last = m_registeredThreads; + Thread* t; for (t = m_registeredThreads->next; t; t = t->next) { - if (*t == platformThread) { + if (equalThread(t->platformThread, currentPlatformThread)) { last->next = t->next; break; } last = t; } + ASSERT(t); // If t is NULL, we never found ourselves in the list. delete t; } } -SUPPRESS_ASAN -void MachineThreads::gatherFromCurrentThread(ConservativeRoots& conservativeRoots, JITStubRoutineSet& jitStubRoutines, CodeBlockSet& codeBlocks, void* stackOrigin, void* stackTop, RegisterState& calleeSavedRegisters) +#if COMPILER(GCC) +#define REGISTER_BUFFER_ALIGNMENT __attribute__ ((aligned (sizeof(void*)))) +#else +#define REGISTER_BUFFER_ALIGNMENT +#endif + +void MachineThreads::gatherFromCurrentThread(ConservativeRoots& conservativeRoots, void* stackCurrent) { - void* registersBegin = &calleeSavedRegisters; - void* registersEnd = reinterpret_cast<void*>(roundUpToMultipleOf<sizeof(void*)>(reinterpret_cast<uintptr_t>(&calleeSavedRegisters + 1))); - conservativeRoots.add(registersBegin, registersEnd, jitStubRoutines, codeBlocks); + // setjmp forces volatile registers onto the stack + jmp_buf registers REGISTER_BUFFER_ALIGNMENT; +#if COMPILER(MSVC) +#pragma warning(push) +#pragma warning(disable: 4611) +#endif + setjmp(registers); +#if COMPILER(MSVC) +#pragma warning(pop) +#endif + + void* registersBegin = ®isters; + void* registersEnd = reinterpret_cast<void*>(roundUpToMultipleOf<sizeof(void*)>(reinterpret_cast<uintptr_t>(®isters + 1))); + swapIfBackwards(registersBegin, registersEnd); + conservativeRoots.add(registersBegin, registersEnd); - conservativeRoots.add(stackTop, stackOrigin, jitStubRoutines, codeBlocks); + void* stackBegin = stackCurrent; + void* stackEnd = wtfThreadData().stack().origin(); + swapIfBackwards(stackBegin, stackEnd); + conservativeRoots.add(stackBegin, stackEnd); } -inline bool MachineThreads::Thread::suspend() +static inline void suspendThread(const PlatformThread& platformThread) { #if OS(DARWIN) - kern_return_t result = thread_suspend(platformThread); - return result == KERN_SUCCESS; + thread_suspend(platformThread); #elif OS(WINDOWS) - bool threadIsSuspended = (SuspendThread(platformThreadHandle) != (DWORD)-1); - ASSERT(threadIsSuspended); - return threadIsSuspended; + SuspendThread(platformThread); #elif USE(PTHREADS) pthread_kill(platformThread, SigThreadSuspendResume); - return true; #else #error Need a way to suspend threads on this platform #endif } -inline void MachineThreads::Thread::resume() +static inline void resumeThread(const PlatformThread& platformThread) { #if OS(DARWIN) thread_resume(platformThread); #elif OS(WINDOWS) - ResumeThread(platformThreadHandle); + ResumeThread(platformThread); #elif USE(PTHREADS) pthread_kill(platformThread, SigThreadSuspendResume); #else @@ -376,10 +282,38 @@ inline void MachineThreads::Thread::resume() #endif } -size_t MachineThreads::Thread::getRegisters(MachineThreads::Thread::Registers& registers) +typedef unsigned long usword_t; // word size, assumed to be either 32 or 64 bit + +#if OS(DARWIN) + +#if CPU(X86) +typedef i386_thread_state_t PlatformThreadRegisters; +#elif CPU(X86_64) +typedef x86_thread_state64_t PlatformThreadRegisters; +#elif CPU(PPC) +typedef ppc_thread_state_t PlatformThreadRegisters; +#elif CPU(PPC64) +typedef ppc_thread_state64_t PlatformThreadRegisters; +#elif CPU(ARM) +typedef arm_thread_state_t PlatformThreadRegisters; +#elif CPU(ARM64) +typedef arm_thread_state64_t PlatformThreadRegisters; +#else +#error Unknown Architecture +#endif + +#elif OS(WINDOWS) +typedef CONTEXT PlatformThreadRegisters; +#elif USE(PTHREADS) +typedef pthread_attr_t PlatformThreadRegisters; +#else +#error Need a thread register struct for this platform +#endif + +static size_t getPlatformThreadRegisters(const PlatformThread& platformThread, PlatformThreadRegisters& regs) { - Thread::Registers::PlatformRegisters& regs = registers.regs; #if OS(DARWIN) + #if CPU(X86) unsigned user_count = sizeof(regs)/sizeof(int); thread_state_flavor_t flavor = i386_THREAD_STATE; @@ -408,20 +342,18 @@ size_t MachineThreads::Thread::getRegisters(MachineThreads::Thread::Registers& r "JavaScript garbage collection failed because thread_get_state returned an error (%d). This is probably the result of running inside Rosetta, which is not supported.", result); CRASH(); } - return user_count * sizeof(uintptr_t); + return user_count * sizeof(usword_t); // end OS(DARWIN) #elif OS(WINDOWS) regs.ContextFlags = CONTEXT_INTEGER | CONTEXT_CONTROL; - GetThreadContext(platformThreadHandle, ®s); + GetThreadContext(platformThread, ®s); return sizeof(CONTEXT); #elif USE(PTHREADS) pthread_attr_init(®s); #if HAVE(PTHREAD_NP_H) || OS(NETBSD) -#if !OS(OPENBSD) // e.g. on FreeBSD 5.4, neundorf@kde.org pthread_attr_get_np(platformThread, ®s); -#endif #else // FIXME: this function is non-portable; other POSIX systems may have different np alternatives pthread_getattr_np(platformThread, ®s); @@ -432,7 +364,7 @@ size_t MachineThreads::Thread::getRegisters(MachineThreads::Thread::Registers& r #endif } -inline void* MachineThreads::Thread::Registers::stackPointer() const +static inline void* otherThreadStackPointer(const PlatformThreadRegisters& regs) { #if OS(DARWIN) @@ -484,14 +416,7 @@ inline void* MachineThreads::Thread::Registers::stackPointer() const #elif USE(PTHREADS) void* stackBase = 0; size_t stackSize = 0; -#if OS(OPENBSD) - stack_t ss; - int rc = pthread_stackseg_np(pthread_self(), &ss); - stackBase = (void*)((size_t) ss.ss_sp - ss.ss_size); - stackSize = ss.ss_size; -#else int rc = pthread_attr_getstack(®s, &stackBase, &stackSize); -#endif (void)rc; // FIXME: Deal with error code somehow? Seems fatal. ASSERT(stackBase); return static_cast<char*>(stackBase) + stackSize; @@ -500,9 +425,8 @@ inline void* MachineThreads::Thread::Registers::stackPointer() const #endif } -void MachineThreads::Thread::freeRegisters(MachineThreads::Thread::Registers& registers) +static void freePlatformThreadRegisters(PlatformThreadRegisters& regs) { - Thread::Registers::PlatformRegisters& regs = registers.regs; #if USE(PTHREADS) && !OS(WINDOWS) && !OS(DARWIN) pthread_attr_destroy(®s); #else @@ -510,165 +434,57 @@ void MachineThreads::Thread::freeRegisters(MachineThreads::Thread::Registers& re #endif } -std::pair<void*, size_t> MachineThreads::Thread::captureStack(void* stackTop) -{ - void* begin = stackBase; - void* end = reinterpret_cast<void*>( - WTF::roundUpToMultipleOf<sizeof(void*)>(reinterpret_cast<uintptr_t>(stackTop))); - if (begin > end) - std::swap(begin, end); - return std::make_pair(begin, static_cast<char*>(end) - static_cast<char*>(begin)); -} - -SUPPRESS_ASAN -static void copyMemory(void* dst, const void* src, size_t size) +void MachineThreads::gatherFromOtherThread(ConservativeRoots& conservativeRoots, Thread* thread) { - size_t dstAsSize = reinterpret_cast<size_t>(dst); - size_t srcAsSize = reinterpret_cast<size_t>(src); - RELEASE_ASSERT(dstAsSize == WTF::roundUpToMultipleOf<sizeof(intptr_t)>(dstAsSize)); - RELEASE_ASSERT(srcAsSize == WTF::roundUpToMultipleOf<sizeof(intptr_t)>(srcAsSize)); - RELEASE_ASSERT(size == WTF::roundUpToMultipleOf<sizeof(intptr_t)>(size)); - - intptr_t* dstPtr = reinterpret_cast<intptr_t*>(dst); - const intptr_t* srcPtr = reinterpret_cast<const intptr_t*>(src); - size /= sizeof(intptr_t); - while (size--) - *dstPtr++ = *srcPtr++; -} - - - -// This function must not call malloc(), free(), or any other function that might -// acquire a lock. Since 'thread' is suspended, trying to acquire a lock -// will deadlock if 'thread' holds that lock. -// This function, specifically the memory copying, was causing problems with Address Sanitizer in -// apps. Since we cannot blacklist the system memcpy we must use our own naive implementation, -// copyMemory, for ASan to work on either instrumented or non-instrumented builds. This is not a -// significant performance loss as tryCopyOtherThreadStack is only called as part of an O(heapsize) -// operation. As the heap is generally much larger than the stack the performance hit is minimal. -// See: https://bugs.webkit.org/show_bug.cgi?id=146297 -void MachineThreads::tryCopyOtherThreadStack(Thread* thread, void* buffer, size_t capacity, size_t* size) -{ - Thread::Registers registers; - size_t registersSize = thread->getRegisters(registers); - std::pair<void*, size_t> stack = thread->captureStack(registers.stackPointer()); - - bool canCopy = *size + registersSize + stack.second <= capacity; + PlatformThreadRegisters regs; + size_t regSize = getPlatformThreadRegisters(thread->platformThread, regs); - if (canCopy) - copyMemory(static_cast<char*>(buffer) + *size, ®isters, registersSize); - *size += registersSize; + conservativeRoots.add(static_cast<void*>(®s), static_cast<void*>(reinterpret_cast<char*>(®s) + regSize)); - if (canCopy) - copyMemory(static_cast<char*>(buffer) + *size, stack.first, stack.second); - *size += stack.second; + void* stackPointer = otherThreadStackPointer(regs); + void* stackBase = thread->stackBase; + swapIfBackwards(stackPointer, stackBase); + conservativeRoots.add(stackPointer, stackBase); - thread->freeRegisters(registers); + freePlatformThreadRegisters(regs); } -bool MachineThreads::tryCopyOtherThreadStacks(LockHolder&, void* buffer, size_t capacity, size_t* size) +void MachineThreads::gatherConservativeRoots(ConservativeRoots& conservativeRoots, void* stackCurrent) { - // Prevent two VMs from suspending each other's threads at the same time, - // which can cause deadlock: <rdar://problem/20300842>. - static StaticLock mutex; - std::lock_guard<StaticLock> lock(mutex); + gatherFromCurrentThread(conservativeRoots, stackCurrent); - *size = 0; + if (m_threadSpecific) { + PlatformThread currentPlatformThread = getCurrentPlatformThread(); - PlatformThread currentPlatformThread = getCurrentPlatformThread(); - int numberOfThreads = 0; // Using 0 to denote that we haven't counted the number of threads yet. - int index = 1; - Thread* threadsToBeDeleted = nullptr; - - Thread* previousThread = nullptr; - for (Thread* thread = m_registeredThreads; thread; index++) { - if (*thread != currentPlatformThread) { - bool success = thread->suspend(); -#if OS(DARWIN) - if (!success) { - if (!numberOfThreads) { - for (Thread* countedThread = m_registeredThreads; countedThread; countedThread = countedThread->next) - numberOfThreads++; - } - - // Re-do the suspension to get the actual failure result for logging. - kern_return_t error = thread_suspend(thread->platformThread); - ASSERT(error != KERN_SUCCESS); - - WTFReportError(__FILE__, __LINE__, WTF_PRETTY_FUNCTION, - "JavaScript garbage collection encountered an invalid thread (err 0x%x): Thread [%d/%d: %p] platformThread %p.", - error, index, numberOfThreads, thread, reinterpret_cast<void*>(thread->platformThread)); - - // Put the invalid thread on the threadsToBeDeleted list. - // We can't just delete it here because we have suspended other - // threads, and they may still be holding the C heap lock which - // we need for deleting the invalid thread. Hence, we need to - // defer the deletion till after we have resumed all threads. - Thread* nextThread = thread->next; - thread->next = threadsToBeDeleted; - threadsToBeDeleted = thread; - - if (previousThread) - previousThread->next = nextThread; - else - m_registeredThreads = nextThread; - thread = nextThread; - continue; - } -#else - UNUSED_PARAM(numberOfThreads); - UNUSED_PARAM(previousThread); - ASSERT_UNUSED(success, success); + MutexLocker lock(m_registeredThreadsMutex); + +#ifndef NDEBUG + // Forbid malloc during the gather phase. The gather phase suspends + // threads, so a malloc during gather would risk a deadlock with a + // thread that had been suspended while holding the malloc lock. + fastMallocForbid(); #endif + for (Thread* thread = m_registeredThreads; thread; thread = thread->next) { + if (!equalThread(thread->platformThread, currentPlatformThread)) + suspendThread(thread->platformThread); } - previousThread = thread; - thread = thread->next; - } - for (Thread* thread = m_registeredThreads; thread; thread = thread->next) { - if (*thread != currentPlatformThread) - tryCopyOtherThreadStack(thread, buffer, capacity, size); - } + // It is safe to access the registeredThreads list, because we earlier asserted that locks are being held, + // and since this is a shared heap, they are real locks. + for (Thread* thread = m_registeredThreads; thread; thread = thread->next) { + if (!equalThread(thread->platformThread, currentPlatformThread)) + gatherFromOtherThread(conservativeRoots, thread); + } - for (Thread* thread = m_registeredThreads; thread; thread = thread->next) { - if (*thread != currentPlatformThread) - thread->resume(); - } + for (Thread* thread = m_registeredThreads; thread; thread = thread->next) { + if (!equalThread(thread->platformThread, currentPlatformThread)) + resumeThread(thread->platformThread); + } - for (Thread* thread = threadsToBeDeleted; thread; ) { - Thread* nextThread = thread->next; - delete thread; - thread = nextThread; +#ifndef NDEBUG + fastMallocAllow(); +#endif } - - return *size <= capacity; -} - -static void growBuffer(size_t size, void** buffer, size_t* capacity) -{ - if (*buffer) - fastFree(*buffer); - - *capacity = WTF::roundUpToMultipleOf(WTF::pageSize(), size * 2); - *buffer = fastMalloc(*capacity); -} - -void MachineThreads::gatherConservativeRoots(ConservativeRoots& conservativeRoots, JITStubRoutineSet& jitStubRoutines, CodeBlockSet& codeBlocks, void* stackOrigin, void* stackTop, RegisterState& calleeSavedRegisters) -{ - gatherFromCurrentThread(conservativeRoots, jitStubRoutines, codeBlocks, stackOrigin, stackTop, calleeSavedRegisters); - - size_t size; - size_t capacity = 0; - void* buffer = nullptr; - LockHolder lock(m_registeredThreadsMutex); - while (!tryCopyOtherThreadStacks(lock, buffer, capacity, &size)) - growBuffer(size, &buffer, &capacity); - - if (!buffer) - return; - - conservativeRoots.add(buffer, static_cast<char*>(buffer) + size, jitStubRoutines, codeBlocks); - fastFree(buffer); } } // namespace JSC diff --git a/Source/JavaScriptCore/heap/MachineStackMarker.h b/Source/JavaScriptCore/heap/MachineStackMarker.h index 3080d2b74..49823d43e 100644 --- a/Source/JavaScriptCore/heap/MachineStackMarker.h +++ b/Source/JavaScriptCore/heap/MachineStackMarker.h @@ -1,7 +1,7 @@ /* * Copyright (C) 1999-2000 Harri Porten (porten@kde.org) * Copyright (C) 2001 Peter Kelly (pmk@post.com) - * Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2015 Apple Inc. All rights reserved. + * Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009 Apple Inc. All rights reserved. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -22,44 +22,37 @@ #ifndef MachineThreads_h #define MachineThreads_h -#include <setjmp.h> -#include <wtf/Lock.h> #include <wtf/Noncopyable.h> #include <wtf/ThreadSpecific.h> +#include <wtf/ThreadingPrimitives.h> namespace JSC { - class CodeBlockSet; class ConservativeRoots; class Heap; - class JITStubRoutineSet; class MachineThreads { WTF_MAKE_NONCOPYABLE(MachineThreads); public: - typedef jmp_buf RegisterState; - MachineThreads(Heap*); ~MachineThreads(); - void gatherConservativeRoots(ConservativeRoots&, JITStubRoutineSet&, CodeBlockSet&, void* stackOrigin, void* stackTop, RegisterState& calleeSavedRegisters); + void gatherConservativeRoots(ConservativeRoots&, void* stackCurrent); + JS_EXPORT_PRIVATE void makeUsableFromMultipleThreads(); JS_EXPORT_PRIVATE void addCurrentThread(); // Only needs to be called by clients that can use the same heap from multiple threads. private: - class Thread; + void gatherFromCurrentThread(ConservativeRoots&, void* stackCurrent); - void gatherFromCurrentThread(ConservativeRoots&, JITStubRoutineSet&, CodeBlockSet&, void* stackOrigin, void* stackTop, RegisterState& calleeSavedRegisters); - - void tryCopyOtherThreadStack(Thread*, void*, size_t capacity, size_t*); - bool tryCopyOtherThreadStacks(LockHolder&, void*, size_t capacity, size_t*); + class Thread; static void removeThread(void*); + void removeCurrentThread(); - template<typename PlatformThread> - void removeThreadIfFound(PlatformThread); + void gatherFromOtherThread(ConservativeRoots&, Thread*); - Lock m_registeredThreadsMutex; + Mutex m_registeredThreadsMutex; Thread* m_registeredThreads; WTF::ThreadSpecificKey m_threadSpecific; #if !ASSERT_DISABLED @@ -69,24 +62,4 @@ namespace JSC { } // namespace JSC -#if COMPILER(GCC_OR_CLANG) -#define REGISTER_BUFFER_ALIGNMENT __attribute__ ((aligned (sizeof(void*)))) -#else -#define REGISTER_BUFFER_ALIGNMENT -#endif - -// ALLOCATE_AND_GET_REGISTER_STATE() is a macro so that it is always "inlined" even in debug builds. -#if COMPILER(MSVC) -#pragma warning(push) -#pragma warning(disable: 4611) -#define ALLOCATE_AND_GET_REGISTER_STATE(registers) \ - MachineThreads::RegisterState registers REGISTER_BUFFER_ALIGNMENT; \ - setjmp(registers) -#pragma warning(pop) -#else -#define ALLOCATE_AND_GET_REGISTER_STATE(registers) \ - MachineThreads::RegisterState registers REGISTER_BUFFER_ALIGNMENT; \ - setjmp(registers) -#endif - #endif // MachineThreads_h diff --git a/Source/JavaScriptCore/heap/MarkStack.cpp b/Source/JavaScriptCore/heap/MarkStack.cpp index da6ef9472..688de42e3 100644 --- a/Source/JavaScriptCore/heap/MarkStack.cpp +++ b/Source/JavaScriptCore/heap/MarkStack.cpp @@ -25,14 +25,90 @@ #include "config.h" #include "MarkStack.h" - -#include "JSCInlines.h" +#include "MarkStackInlines.h" + +#include "ConservativeRoots.h" +#include "CopiedSpace.h" +#include "CopiedSpaceInlines.h" +#include "Heap.h" +#include "JSArray.h" +#include "JSCell.h" +#include "JSObject.h" + +#include "SlotVisitorInlines.h" +#include "Structure.h" +#include "WriteBarrier.h" +#include <wtf/Atomics.h> +#include <wtf/DataLog.h> +#include <wtf/MainThread.h> namespace JSC { -MarkStackArray::MarkStackArray() - : GCSegmentedArray<const JSCell*>() +COMPILE_ASSERT(MarkStackSegment::blockSize == WeakBlock::blockSize, blockSizeMatch); + +MarkStackArray::MarkStackArray(BlockAllocator& blockAllocator) + : m_blockAllocator(blockAllocator) + , m_top(0) + , m_numberOfSegments(0) +{ + m_segments.push(MarkStackSegment::create(m_blockAllocator.allocate<MarkStackSegment>())); + m_numberOfSegments++; +} + +MarkStackArray::~MarkStackArray() +{ + ASSERT(m_numberOfSegments == 1); + ASSERT(m_segments.size() == 1); + m_blockAllocator.deallocate(MarkStackSegment::destroy(m_segments.removeHead())); + m_numberOfSegments--; + ASSERT(!m_numberOfSegments); + ASSERT(!m_segments.size()); +} + +void MarkStackArray::clear() +{ + if (!m_segments.head()) + return; + MarkStackSegment* next; + for (MarkStackSegment* current = m_segments.head(); current->next(); current = next) { + next = current->next(); + m_segments.remove(current); + m_blockAllocator.deallocate(MarkStackSegment::destroy(current)); + } + m_top = 0; + m_numberOfSegments = 1; +#if !ASSERT_DISABLED + m_segments.head()->m_top = 0; +#endif +} + +void MarkStackArray::expand() +{ + ASSERT(m_segments.head()->m_top == s_segmentCapacity); + + MarkStackSegment* nextSegment = MarkStackSegment::create(m_blockAllocator.allocate<MarkStackSegment>()); + m_numberOfSegments++; + +#if !ASSERT_DISABLED + nextSegment->m_top = 0; +#endif + + m_segments.push(nextSegment); + setTopForEmptySegment(); + validatePrevious(); +} + +bool MarkStackArray::refill() { + validatePrevious(); + if (top()) + return true; + m_blockAllocator.deallocate(MarkStackSegment::destroy(m_segments.removeHead())); + ASSERT(m_numberOfSegments > 1); + m_numberOfSegments--; + setTopForFullSegment(); + validatePrevious(); + return true; } void MarkStackArray::donateSomeCellsTo(MarkStackArray& other) @@ -57,11 +133,11 @@ void MarkStackArray::donateSomeCellsTo(MarkStackArray& other) // Remove our head and the head of the other list before we start moving segments around. // We'll add them back on once we're done donating. - GCArraySegment<const JSCell*>* myHead = m_segments.removeHead(); - GCArraySegment<const JSCell*>* otherHead = other.m_segments.removeHead(); + MarkStackSegment* myHead = m_segments.removeHead(); + MarkStackSegment* otherHead = other.m_segments.removeHead(); while (segmentsToDonate--) { - GCArraySegment<const JSCell*>* current = m_segments.removeHead(); + MarkStackSegment* current = m_segments.removeHead(); ASSERT(current); ASSERT(m_numberOfSegments > 1); other.m_segments.push(current); @@ -89,8 +165,8 @@ void MarkStackArray::stealSomeCellsFrom(MarkStackArray& other, size_t idleThread // If other has an entire segment, steal it and return. if (other.m_numberOfSegments > 1) { // Move the heads of the lists aside. We'll push them back on after. - GCArraySegment<const JSCell*>* otherHead = other.m_segments.removeHead(); - GCArraySegment<const JSCell*>* myHead = m_segments.removeHead(); + MarkStackSegment* otherHead = other.m_segments.removeHead(); + MarkStackSegment* myHead = m_segments.removeHead(); ASSERT(other.m_segments.head()->m_top == s_segmentCapacity); @@ -112,4 +188,28 @@ void MarkStackArray::stealSomeCellsFrom(MarkStackArray& other, size_t idleThread append(other.removeLast()); } +void MarkStackArray::fillVector(Vector<const JSCell*>& vector) +{ + ASSERT(vector.size() == size()); + + MarkStackSegment* currentSegment = m_segments.head(); + if (!currentSegment) + return; + + unsigned count = 0; + for (unsigned i = 0; i < m_top; ++i) { + ASSERT(currentSegment->data()[i]); + vector[count++] = currentSegment->data()[i]; + } + + currentSegment = currentSegment->next(); + while (currentSegment) { + for (unsigned i = 0; i < s_segmentCapacity; ++i) { + ASSERT(currentSegment->data()[i]); + vector[count++] = currentSegment->data()[i]; + } + currentSegment = currentSegment->next(); + } +} + } // namespace JSC diff --git a/Source/JavaScriptCore/heap/MarkStack.h b/Source/JavaScriptCore/heap/MarkStack.h index 04f19c62c..6729bad22 100644 --- a/Source/JavaScriptCore/heap/MarkStack.h +++ b/Source/JavaScriptCore/heap/MarkStack.h @@ -26,18 +26,106 @@ #ifndef MarkStack_h #define MarkStack_h -#include "GCSegmentedArrayInlines.h" +#if ENABLE(OBJECT_MARK_LOGGING) +#define MARK_LOG_MESSAGE0(message) dataLogF(message) +#define MARK_LOG_MESSAGE1(message, arg1) dataLogF(message, arg1) +#define MARK_LOG_MESSAGE2(message, arg1, arg2) dataLogF(message, arg1, arg2) +#define MARK_LOG_ROOT(visitor, rootName) \ + dataLogF("\n%s: ", rootName); \ + (visitor).resetChildCount() +#define MARK_LOG_PARENT(visitor, parent) \ + dataLogF("\n%p (%s): ", parent, parent->className() ? parent->className() : "unknown"); \ + (visitor).resetChildCount() +#define MARK_LOG_CHILD(visitor, child) \ + if ((visitor).childCount()) \ + dataLogFString(", "); \ + dataLogF("%p", child); \ + (visitor).incrementChildCount() +#else +#define MARK_LOG_MESSAGE0(message) do { } while (false) +#define MARK_LOG_MESSAGE1(message, arg1) do { } while (false) +#define MARK_LOG_MESSAGE2(message, arg1, arg2) do { } while (false) +#define MARK_LOG_ROOT(visitor, rootName) do { } while (false) +#define MARK_LOG_PARENT(visitor, parent) do { } while (false) +#define MARK_LOG_CHILD(visitor, child) do { } while (false) +#endif + +#include "HeapBlock.h" +#include <wtf/StdLibExtras.h> +#include <wtf/Vector.h> namespace JSC { +class BlockAllocator; +class DeadBlock; class JSCell; -class MarkStackArray : public GCSegmentedArray<const JSCell*> { +class MarkStackSegment : public HeapBlock<MarkStackSegment> { +public: + MarkStackSegment(Region* region) + : HeapBlock<MarkStackSegment>(region) +#if !ASSERT_DISABLED + , m_top(0) +#endif + { + } + + static MarkStackSegment* create(DeadBlock*); + + const JSCell** data() + { + return bitwise_cast<const JSCell**>(this + 1); + } + + static const size_t blockSize = 4 * KB; + +#if !ASSERT_DISABLED + size_t m_top; +#endif +}; + +class MarkStackArray { public: - MarkStackArray(); + MarkStackArray(BlockAllocator&); + ~MarkStackArray(); + + void append(const JSCell*); + bool canRemoveLast(); + const JSCell* removeLast(); + bool refill(); + void donateSomeCellsTo(MarkStackArray& other); void stealSomeCellsFrom(MarkStackArray& other, size_t idleThreadCount); + + size_t size(); + bool isEmpty(); + + void fillVector(Vector<const JSCell*>&); + void clear(); + +private: + template <size_t size> struct CapacityFromSize { + static const size_t value = (size - sizeof(MarkStackSegment)) / sizeof(const JSCell*); + }; + + JS_EXPORT_PRIVATE void expand(); + + size_t postIncTop(); + size_t preDecTop(); + void setTopForFullSegment(); + void setTopForEmptySegment(); + size_t top(); + + void validatePrevious(); + + DoublyLinkedList<MarkStackSegment> m_segments; + BlockAllocator& m_blockAllocator; + + JS_EXPORT_PRIVATE static const size_t s_segmentCapacity = CapacityFromSize<MarkStackSegment::blockSize>::value; + size_t m_top; + size_t m_numberOfSegments; + }; } // namespace JSC diff --git a/Source/JavaScriptCore/heap/MarkStackInlines.h b/Source/JavaScriptCore/heap/MarkStackInlines.h new file mode 100644 index 000000000..c577de602 --- /dev/null +++ b/Source/JavaScriptCore/heap/MarkStackInlines.h @@ -0,0 +1,119 @@ +/* + * Copyright (C) 2009, 2011 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef MarkStackInlines_h +#define MarkStackInlines_h + +#include "GCThreadSharedData.h" +#include "MarkStack.h" + +namespace JSC { + +inline MarkStackSegment* MarkStackSegment::create(DeadBlock* block) +{ + return new (NotNull, block) MarkStackSegment(block->region()); +} + +inline size_t MarkStackArray::postIncTop() +{ + size_t result = m_top++; + ASSERT(result == m_segments.head()->m_top++); + return result; +} + +inline size_t MarkStackArray::preDecTop() +{ + size_t result = --m_top; + ASSERT(result == --m_segments.head()->m_top); + return result; +} + +inline void MarkStackArray::setTopForFullSegment() +{ + ASSERT(m_segments.head()->m_top == s_segmentCapacity); + m_top = s_segmentCapacity; +} + +inline void MarkStackArray::setTopForEmptySegment() +{ + ASSERT(!m_segments.head()->m_top); + m_top = 0; +} + +inline size_t MarkStackArray::top() +{ + ASSERT(m_top == m_segments.head()->m_top); + return m_top; +} + +#if ASSERT_DISABLED +inline void MarkStackArray::validatePrevious() { } +#else +inline void MarkStackArray::validatePrevious() +{ + unsigned count = 0; + for (MarkStackSegment* current = m_segments.head(); current; current = current->next()) + count++; + ASSERT(m_segments.size() == m_numberOfSegments); +} +#endif + +inline void MarkStackArray::append(const JSCell* cell) +{ + if (m_top == s_segmentCapacity) + expand(); + m_segments.head()->data()[postIncTop()] = cell; +} + +inline bool MarkStackArray::canRemoveLast() +{ + return !!m_top; +} + +inline const JSCell* MarkStackArray::removeLast() +{ + return m_segments.head()->data()[preDecTop()]; +} + +inline bool MarkStackArray::isEmpty() +{ + if (m_top) + return false; + if (m_segments.head()->next()) { + ASSERT(m_segments.head()->next()->m_top == s_segmentCapacity); + return false; + } + return true; +} + +inline size_t MarkStackArray::size() +{ + return m_top + s_segmentCapacity * (m_numberOfSegments - 1); +} + +} // namespace JSC + +#endif // MarkStackInlines_h + diff --git a/Source/JavaScriptCore/heap/MarkedAllocator.cpp b/Source/JavaScriptCore/heap/MarkedAllocator.cpp index 281fac4ed..c2b0f72de 100644 --- a/Source/JavaScriptCore/heap/MarkedAllocator.cpp +++ b/Source/JavaScriptCore/heap/MarkedAllocator.cpp @@ -1,35 +1,10 @@ -/* - * Copyright (C) 2012, 2013 Apple Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY - * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - #include "config.h" #include "MarkedAllocator.h" +#include "DelayedReleaseScope.h" #include "GCActivityCallback.h" #include "Heap.h" #include "IncrementalSweeper.h" -#include "JSCInlines.h" #include "VM.h" #include <wtf/CurrentTime.h> @@ -61,91 +36,74 @@ bool MarkedAllocator::isPagedOut(double deadline) inline void* MarkedAllocator::tryAllocateHelper(size_t bytes) { - if (m_currentBlock) { - ASSERT(m_currentBlock == m_nextBlockToSweep); - m_currentBlock->didConsumeFreeList(); - m_nextBlockToSweep = m_currentBlock->next(); - } - - MarkedBlock* next; - for (MarkedBlock*& block = m_nextBlockToSweep; block; block = next) { - next = block->next(); - - MarkedBlock::FreeList freeList = block->sweep(MarkedBlock::SweepToFreeList); - - double utilization = ((double)MarkedBlock::blockSize - (double)freeList.bytes) / (double)MarkedBlock::blockSize; - if (utilization >= Options::minMarkedBlockUtilization()) { - ASSERT(freeList.bytes || !freeList.head); - m_blockList.remove(block); - m_retiredBlocks.push(block); - block->didRetireBlock(freeList); - continue; + // We need a while loop to check the free list because the DelayedReleaseScope + // could cause arbitrary code to execute and exhaust the free list that we + // thought had elements in it. + while (!m_freeList.head) { + DelayedReleaseScope delayedReleaseScope(*m_markedSpace); + if (m_currentBlock) { + ASSERT(m_currentBlock == m_nextBlockToSweep); + m_currentBlock->didConsumeFreeList(); + m_nextBlockToSweep = m_currentBlock->next(); } - if (bytes > block->cellSize()) { - block->stopAllocating(freeList); - continue; + MarkedBlock* next; + for (MarkedBlock*& block = m_nextBlockToSweep; block; block = next) { + next = block->next(); + + MarkedBlock::FreeList freeList = block->sweep(MarkedBlock::SweepToFreeList); + + if (!freeList.head) { + block->didConsumeEmptyFreeList(); + m_blockList.remove(block); + m_blockList.push(block); + if (!m_lastFullBlock) + m_lastFullBlock = block; + continue; + } + + if (bytes > block->cellSize()) { + block->stopAllocating(freeList); + continue; + } + + m_currentBlock = block; + m_freeList = freeList; + break; + } + + if (!m_freeList.head) { + m_currentBlock = 0; + return 0; } - - m_currentBlock = block; - m_freeList = freeList; - break; - } - - if (!m_freeList.head) { - m_currentBlock = 0; - return 0; } ASSERT(m_freeList.head); - void* head = tryPopFreeList(bytes); - ASSERT(head); - m_markedSpace->didAllocateInBlock(m_currentBlock); - return head; -} - -inline void* MarkedAllocator::tryPopFreeList(size_t bytes) -{ - ASSERT(m_currentBlock); - if (bytes > m_currentBlock->cellSize()) - return 0; - MarkedBlock::FreeCell* head = m_freeList.head; m_freeList.head = head->next; + ASSERT(head); + m_markedSpace->didAllocateInBlock(m_currentBlock); return head; } - + inline void* MarkedAllocator::tryAllocate(size_t bytes) { ASSERT(!m_heap->isBusy()); m_heap->m_operationInProgress = Allocation; void* result = tryAllocateHelper(bytes); - m_heap->m_operationInProgress = NoOperation; - ASSERT(result || !m_currentBlock); return result; } - -ALWAYS_INLINE void MarkedAllocator::doTestCollectionsIfNeeded() -{ - if (!Options::slowPathAllocsBetweenGCs()) - return; - - static unsigned allocationCount = 0; - if (!allocationCount) { - if (!m_heap->isDeferred()) - m_heap->collectAllGarbage(); - ASSERT(m_heap->m_operationInProgress == NoOperation); - } - if (++allocationCount >= Options::slowPathAllocsBetweenGCs()) - allocationCount = 0; -} - + void* MarkedAllocator::allocateSlowCase(size_t bytes) { ASSERT(m_heap->vm()->currentThreadIsHoldingAPILock()); - doTestCollectionsIfNeeded(); - +#if COLLECT_ON_EVERY_ALLOCATION + if (!m_heap->isDeferred()) + m_heap->collectAllGarbage(); + ASSERT(m_heap->m_operationInProgress == NoOperation); +#endif + ASSERT(!m_markedSpace->isIterating()); ASSERT(!m_freeList.head); m_heap->didAllocate(m_freeList.bytes); @@ -175,22 +133,26 @@ void* MarkedAllocator::allocateSlowCase(size_t bytes) MarkedBlock* MarkedAllocator::allocateBlock(size_t bytes) { size_t minBlockSize = MarkedBlock::blockSize; - size_t minAllocationSize = WTF::roundUpToMultipleOf<MarkedBlock::atomSize>(sizeof(MarkedBlock)) + WTF::roundUpToMultipleOf<MarkedBlock::atomSize>(bytes); - minAllocationSize = WTF::roundUpToMultipleOf(WTF::pageSize(), minAllocationSize); + size_t minAllocationSize = WTF::roundUpToMultipleOf(WTF::pageSize(), sizeof(MarkedBlock) + bytes); size_t blockSize = std::max(minBlockSize, minAllocationSize); size_t cellSize = m_cellSize ? m_cellSize : WTF::roundUpToMultipleOf<MarkedBlock::atomSize>(bytes); - return MarkedBlock::create(this, blockSize, cellSize, m_needsDestruction); + if (blockSize == MarkedBlock::blockSize) + return MarkedBlock::create(m_heap->blockAllocator().allocate<MarkedBlock>(), this, cellSize, m_destructorType); + return MarkedBlock::create(m_heap->blockAllocator().allocateCustomSize(blockSize, MarkedBlock::blockSize), this, cellSize, m_destructorType); } void MarkedAllocator::addBlock(MarkedBlock* block) { + // Satisfy the ASSERT in MarkedBlock::sweep. + DelayedReleaseScope delayedReleaseScope(*m_markedSpace); ASSERT(!m_currentBlock); ASSERT(!m_freeList.head); m_blockList.append(block); - m_nextBlockToSweep = block; + m_nextBlockToSweep = m_currentBlock = block; + m_freeList = block->sweep(MarkedBlock::SweepToFreeList); m_markedSpace->didAddBlock(block); } @@ -203,7 +165,9 @@ void MarkedAllocator::removeBlock(MarkedBlock* block) if (m_nextBlockToSweep == block) m_nextBlockToSweep = m_nextBlockToSweep->next(); - block->willRemoveBlock(); + if (block == m_lastFullBlock) + m_lastFullBlock = m_lastFullBlock->prev(); + m_blockList.remove(block); } @@ -213,20 +177,12 @@ void MarkedAllocator::reset() m_currentBlock = 0; m_freeList = MarkedBlock::FreeList(); if (m_heap->operationInProgress() == FullCollection) - m_blockList.append(m_retiredBlocks); + m_lastFullBlock = 0; - m_nextBlockToSweep = m_blockList.head(); -} - -struct LastChanceToFinalize : MarkedBlock::VoidFunctor { - void operator()(MarkedBlock* block) { block->lastChanceToFinalize(); } -}; - -void MarkedAllocator::lastChanceToFinalize() -{ - m_blockList.append(m_retiredBlocks); - LastChanceToFinalize functor; - forEachBlock(functor); + if (m_lastFullBlock) + m_nextBlockToSweep = m_lastFullBlock->next() ? m_lastFullBlock->next() : m_lastFullBlock; + else + m_nextBlockToSweep = m_blockList.head(); } } // namespace JSC diff --git a/Source/JavaScriptCore/heap/MarkedAllocator.h b/Source/JavaScriptCore/heap/MarkedAllocator.h index 161af480f..e0d3e8902 100644 --- a/Source/JavaScriptCore/heap/MarkedAllocator.h +++ b/Source/JavaScriptCore/heap/MarkedAllocator.h @@ -21,12 +21,11 @@ public: static ptrdiff_t offsetOfFreeListHead(); MarkedAllocator(); - void lastChanceToFinalize(); void reset(); void stopAllocating(); void resumeAllocating(); size_t cellSize() { return m_cellSize; } - bool needsDestruction() { return m_needsDestruction; } + MarkedBlock::DestructorType destructorType() { return m_destructorType; } void* allocate(size_t); Heap* heap() { return m_heap; } MarkedBlock* takeLastActiveBlock() @@ -40,7 +39,7 @@ public: void addBlock(MarkedBlock*); void removeBlock(MarkedBlock*); - void init(Heap*, MarkedSpace*, size_t cellSize, bool needsDestruction); + void init(Heap*, MarkedSpace*, size_t cellSize, MarkedBlock::DestructorType); bool isPagedOut(double deadline); @@ -48,18 +47,16 @@ private: JS_EXPORT_PRIVATE void* allocateSlowCase(size_t); void* tryAllocate(size_t); void* tryAllocateHelper(size_t); - void* tryPopFreeList(size_t); MarkedBlock* allocateBlock(size_t); - ALWAYS_INLINE void doTestCollectionsIfNeeded(); MarkedBlock::FreeList m_freeList; MarkedBlock* m_currentBlock; MarkedBlock* m_lastActiveBlock; MarkedBlock* m_nextBlockToSweep; + MarkedBlock* m_lastFullBlock; DoublyLinkedList<MarkedBlock> m_blockList; - DoublyLinkedList<MarkedBlock> m_retiredBlocks; size_t m_cellSize; - bool m_needsDestruction { false }; + MarkedBlock::DestructorType m_destructorType; Heap* m_heap; MarkedSpace* m_markedSpace; }; @@ -73,18 +70,20 @@ inline MarkedAllocator::MarkedAllocator() : m_currentBlock(0) , m_lastActiveBlock(0) , m_nextBlockToSweep(0) + , m_lastFullBlock(0) , m_cellSize(0) + , m_destructorType(MarkedBlock::None) , m_heap(0) , m_markedSpace(0) { } -inline void MarkedAllocator::init(Heap* heap, MarkedSpace* markedSpace, size_t cellSize, bool needsDestruction) +inline void MarkedAllocator::init(Heap* heap, MarkedSpace* markedSpace, size_t cellSize, MarkedBlock::DestructorType destructorType) { m_heap = heap; m_markedSpace = markedSpace; m_cellSize = cellSize; - m_needsDestruction = needsDestruction; + m_destructorType = destructorType; } inline void* MarkedAllocator::allocate(size_t bytes) @@ -136,11 +135,6 @@ template <typename Functor> inline void MarkedAllocator::forEachBlock(Functor& f next = block->next(); functor(block); } - - for (MarkedBlock* block = m_retiredBlocks.head(); block; block = next) { - next = block->next(); - functor(block); - } } } // namespace JSC diff --git a/Source/JavaScriptCore/heap/MarkedBlock.cpp b/Source/JavaScriptCore/heap/MarkedBlock.cpp index b9c3f9ff9..674f45636 100644 --- a/Source/JavaScriptCore/heap/MarkedBlock.cpp +++ b/Source/JavaScriptCore/heap/MarkedBlock.cpp @@ -26,33 +26,29 @@ #include "config.h" #include "MarkedBlock.h" +#include "DelayedReleaseScope.h" #include "IncrementalSweeper.h" #include "JSCell.h" #include "JSDestructibleObject.h" -#include "JSCInlines.h" +#include "Operations.h" namespace JSC { -MarkedBlock* MarkedBlock::create(MarkedAllocator* allocator, size_t capacity, size_t cellSize, bool needsDestruction) +MarkedBlock* MarkedBlock::create(DeadBlock* block, MarkedAllocator* allocator, size_t cellSize, DestructorType destructorType) { - return new (NotNull, fastAlignedMalloc(blockSize, capacity)) MarkedBlock(allocator, capacity, cellSize, needsDestruction); + ASSERT(reinterpret_cast<size_t>(block) == (reinterpret_cast<size_t>(block) & blockMask)); + Region* region = block->region(); + return new (NotNull, block) MarkedBlock(region, allocator, cellSize, destructorType); } -void MarkedBlock::destroy(MarkedBlock* block) -{ - block->~MarkedBlock(); - fastAlignedFree(block); -} - -MarkedBlock::MarkedBlock(MarkedAllocator* allocator, size_t capacity, size_t cellSize, bool needsDestruction) - : DoublyLinkedListNode<MarkedBlock>() +MarkedBlock::MarkedBlock(Region* region, MarkedAllocator* allocator, size_t cellSize, DestructorType destructorType) + : HeapBlock<MarkedBlock>(region) , m_atomsPerCell((cellSize + atomSize - 1) / atomSize) - , m_endAtom((allocator->cellSize() ? atomsPerBlock - m_atomsPerCell : firstAtom()) + 1) - , m_capacity(capacity) - , m_needsDestruction(needsDestruction) + , m_endAtom((allocator->cellSize() ? atomsPerBlock : region->blockSize() / atomSize) - m_atomsPerCell + 1) + , m_destructorType(destructorType) , m_allocator(allocator) , m_state(New) // All cells start out unmarked. - , m_weakSet(allocator->heap()->vm(), *this) + , m_weakSet(allocator->heap()->vm()) { ASSERT(allocator); HEAP_LOG_BLOCK_STATE_TRANSITION(this); @@ -64,22 +60,16 @@ inline void MarkedBlock::callDestructor(JSCell* cell) if (cell->isZapped()) return; - ASSERT(cell->structureID()); - if (cell->inlineTypeFlags() & StructureIsImmortal) - cell->structure(*vm())->classInfo()->methodTable.destroy(cell); - else - jsCast<JSDestructibleObject*>(cell)->classInfo()->methodTable.destroy(cell); + cell->methodTableForDestruction()->destroy(cell); cell->zap(); } -template<MarkedBlock::BlockState blockState, MarkedBlock::SweepMode sweepMode, bool callDestructors> +template<MarkedBlock::BlockState blockState, MarkedBlock::SweepMode sweepMode, MarkedBlock::DestructorType dtorType> MarkedBlock::FreeList MarkedBlock::specializedSweep() { ASSERT(blockState != Allocated && blockState != FreeListed); - ASSERT(!(!callDestructors && sweepMode == SweepOnly)); + ASSERT(!(dtorType == MarkedBlock::None && sweepMode == SweepOnly)); - SamplingRegion samplingRegion((!callDestructors && blockState != New) ? "Calling destructors" : "sweeping"); - // This produces a free list that is ordered in reverse through the block. // This is fine, since the allocation code makes no assumptions about the // order of the free list. @@ -91,7 +81,7 @@ MarkedBlock::FreeList MarkedBlock::specializedSweep() JSCell* cell = reinterpret_cast_ptr<JSCell*>(&atoms()[i]); - if (callDestructors && blockState != New) + if (dtorType != MarkedBlock::None && blockState != New) callDestructor(cell); if (sweepMode == SweepToFreeList) { @@ -105,7 +95,7 @@ MarkedBlock::FreeList MarkedBlock::specializedSweep() // We only want to discard the newlyAllocated bits if we're creating a FreeList, // otherwise we would lose information on what's currently alive. if (sweepMode == SweepToFreeList && m_newlyAllocated) - m_newlyAllocated = nullptr; + m_newlyAllocated.clear(); m_state = ((sweepMode == SweepToFreeList) ? FreeListed : Marked); return FreeList(head, count * cellSize()); @@ -113,37 +103,39 @@ MarkedBlock::FreeList MarkedBlock::specializedSweep() MarkedBlock::FreeList MarkedBlock::sweep(SweepMode sweepMode) { + ASSERT(DelayedReleaseScope::isInEffectFor(heap()->m_objectSpace)); HEAP_LOG_BLOCK_STATE_TRANSITION(this); m_weakSet.sweep(); - if (sweepMode == SweepOnly && !m_needsDestruction) + if (sweepMode == SweepOnly && m_destructorType == MarkedBlock::None) return FreeList(); - if (m_needsDestruction) - return sweepHelper<true>(sweepMode); - return sweepHelper<false>(sweepMode); + if (m_destructorType == MarkedBlock::ImmortalStructure) + return sweepHelper<MarkedBlock::ImmortalStructure>(sweepMode); + if (m_destructorType == MarkedBlock::Normal) + return sweepHelper<MarkedBlock::Normal>(sweepMode); + return sweepHelper<MarkedBlock::None>(sweepMode); } -template<bool callDestructors> +template<MarkedBlock::DestructorType dtorType> MarkedBlock::FreeList MarkedBlock::sweepHelper(SweepMode sweepMode) { switch (m_state) { case New: ASSERT(sweepMode == SweepToFreeList); - return specializedSweep<New, SweepToFreeList, callDestructors>(); + return specializedSweep<New, SweepToFreeList, dtorType>(); case FreeListed: // Happens when a block transitions to fully allocated. ASSERT(sweepMode == SweepToFreeList); return FreeList(); - case Retired: case Allocated: RELEASE_ASSERT_NOT_REACHED(); return FreeList(); case Marked: return sweepMode == SweepToFreeList - ? specializedSweep<Marked, SweepToFreeList, callDestructors>() - : specializedSweep<Marked, SweepOnly, callDestructors>(); + ? specializedSweep<Marked, SweepToFreeList, dtorType>() + : specializedSweep<Marked, SweepOnly, dtorType>(); } RELEASE_ASSERT_NOT_REACHED(); @@ -157,11 +149,10 @@ public: { } - IterationStatus operator()(JSCell* cell) + void operator()(JSCell* cell) { ASSERT(MarkedBlock::blockFor(cell) == m_block); m_block->setNewlyAllocated(cell); - return IterationStatus::Continue; } private: @@ -191,7 +182,7 @@ void MarkedBlock::stopAllocating(const FreeList& freeList) // way to tell what's live vs dead. ASSERT(!m_newlyAllocated); - m_newlyAllocated = std::make_unique<WTF::Bitmap<atomsPerBlock>>(); + m_newlyAllocated = adoptPtr(new WTF::Bitmap<atomsPerBlock>()); SetNewlyAllocatedFunctor functor(this); forEachCell(functor); @@ -218,6 +209,11 @@ void MarkedBlock::clearMarks() #endif } +void MarkedBlock::clearRememberedSet() +{ + m_rememberedSet.clearAll(); +} + template <HeapOperation collectionType> void MarkedBlock::clearMarksWithCollectionType() { @@ -227,16 +223,14 @@ void MarkedBlock::clearMarksWithCollectionType() ASSERT(m_state != New && m_state != FreeListed); if (collectionType == FullCollection) { m_marks.clearAll(); - // This will become true at the end of the mark phase. We set it now to - // avoid an extra pass to do so later. - m_state = Marked; - return; +#if ENABLE(GGC) + m_rememberedSet.clearAll(); +#endif } - ASSERT(collectionType == EdenCollection); - // If a block was retired then there's no way an EdenCollection can un-retire it. - if (m_state != Retired) - m_state = Marked; + // This will become true at the end of the mark phase. We set it now to + // avoid an extra pass to do so later. + m_state = Marked; } void MarkedBlock::lastChanceToFinalize() @@ -264,27 +258,4 @@ MarkedBlock::FreeList MarkedBlock::resumeAllocating() return sweep(SweepToFreeList); } -void MarkedBlock::didRetireBlock(const FreeList& freeList) -{ - HEAP_LOG_BLOCK_STATE_TRANSITION(this); - FreeCell* head = freeList.head; - - // Currently we don't notify the Heap that we're giving up on this block. - // The Heap might be able to make a better decision about how many bytes should - // be allocated before the next collection if it knew about this retired block. - // On the other hand we'll waste at most 10% of our Heap space between FullCollections - // and only under heavy fragmentation. - - // We need to zap the free list when retiring a block so that we don't try to destroy - // previously destroyed objects when we re-sweep the block in the future. - FreeCell* next; - for (FreeCell* current = head; current; current = next) { - next = current->next; - reinterpret_cast<JSCell*>(current)->zap(); - } - - ASSERT(m_state == FreeListed); - m_state = Retired; -} - } // namespace JSC diff --git a/Source/JavaScriptCore/heap/MarkedBlock.h b/Source/JavaScriptCore/heap/MarkedBlock.h index 839099da0..73f56cd72 100644 --- a/Source/JavaScriptCore/heap/MarkedBlock.h +++ b/Source/JavaScriptCore/heap/MarkedBlock.h @@ -22,13 +22,16 @@ #ifndef MarkedBlock_h #define MarkedBlock_h +#include "BlockAllocator.h" +#include "HeapBlock.h" + #include "HeapOperation.h" -#include "IterationStatus.h" #include "WeakSet.h" #include <wtf/Bitmap.h> #include <wtf/DataLog.h> #include <wtf/DoublyLinkedList.h> #include <wtf/HashFunctions.h> +#include <wtf/PageAllocationAligned.h> #include <wtf/StdLibExtras.h> #include <wtf/Vector.h> @@ -66,14 +69,13 @@ namespace JSC { // size is equal to the difference between the cell size and the object // size. - class MarkedBlock : public DoublyLinkedListNode<MarkedBlock> { - friend class WTF::DoublyLinkedListNode<MarkedBlock>; + class MarkedBlock : public HeapBlock<MarkedBlock> { friend class LLIntOffsetsExtractor; - friend struct VerifyMarkedOrRetired; + public: static const size_t atomSize = 16; // bytes static const size_t atomShiftAmount = 4; // log_2(atomSize) FIXME: Change atomSize to 16. - static const size_t blockSize = 16 * KB; + static const size_t blockSize = 64 * KB; static const size_t blockMask = ~(blockSize - 1); // blockSize must be a power of two. static const size_t atomsPerBlock = blockSize / atomSize; @@ -110,8 +112,8 @@ namespace JSC { ReturnType m_count; }; - static MarkedBlock* create(MarkedAllocator*, size_t capacity, size_t cellSize, bool needsDestruction); - static void destroy(MarkedBlock*); + enum DestructorType { None, ImmortalStructure, Normal }; + static MarkedBlock* create(DeadBlock*, MarkedAllocator*, size_t cellSize, DestructorType); static bool isAtomAligned(const void*); static MarkedBlock* blockFor(const void*); @@ -145,6 +147,7 @@ namespace JSC { // and was successfully cleared and false otherwise. bool clearNewlyAllocated(); void clearMarks(); + void clearRememberedSet(); template <HeapOperation collectionType> void clearMarksWithCollectionType(); @@ -152,7 +155,7 @@ namespace JSC { bool isEmpty(); size_t cellSize(); - bool needsDestruction() const; + DestructorType destructorType(); size_t size(); size_t capacity(); @@ -161,7 +164,6 @@ namespace JSC { bool testAndSetMarked(const void*); bool isLive(const JSCell*); bool isLiveCell(const void*); - bool isMarkedOrNewlyAllocated(const JSCell*); void setMarked(const void*); void clearMarked(const void*); @@ -174,45 +176,40 @@ namespace JSC { void setNewlyAllocated(const void*); void clearNewlyAllocated(const void*); - bool isAllocated() const; bool needsSweeping(); - void didRetireBlock(const FreeList&); - void willRemoveBlock(); - template <typename Functor> IterationStatus forEachCell(Functor&); - template <typename Functor> IterationStatus forEachLiveCell(Functor&); - template <typename Functor> IterationStatus forEachDeadCell(Functor&); + template <typename Functor> void forEachCell(Functor&); + template <typename Functor> void forEachLiveCell(Functor&); + template <typename Functor> void forEachDeadCell(Functor&); static ptrdiff_t offsetOfMarks() { return OBJECT_OFFSETOF(MarkedBlock, m_marks); } private: static const size_t atomAlignmentMask = atomSize - 1; // atomSize must be a power of two. - enum BlockState { New, FreeListed, Allocated, Marked, Retired }; - template<bool callDestructors> FreeList sweepHelper(SweepMode = SweepOnly); + enum BlockState { New, FreeListed, Allocated, Marked }; + template<DestructorType> FreeList sweepHelper(SweepMode = SweepOnly); typedef char Atom[atomSize]; - MarkedBlock(MarkedAllocator*, size_t capacity, size_t cellSize, bool needsDestruction); + MarkedBlock(Region*, MarkedAllocator*, size_t cellSize, DestructorType); Atom* atoms(); size_t atomNumber(const void*); void callDestructor(JSCell*); - template<BlockState, SweepMode, bool callDestructors> FreeList specializedSweep(); + template<BlockState, SweepMode, DestructorType> FreeList specializedSweep(); - MarkedBlock* m_prev; - MarkedBlock* m_next; - size_t m_atomsPerCell; size_t m_endAtom; // This is a fuzzy end. Always test for < m_endAtom. #if ENABLE(PARALLEL_GC) WTF::Bitmap<atomsPerBlock, WTF::BitmapAtomic, uint8_t> m_marks; + WTF::Bitmap<atomsPerBlock, WTF::BitmapAtomic, uint8_t> m_rememberedSet; #else WTF::Bitmap<atomsPerBlock, WTF::BitmapNotAtomic, uint8_t> m_marks; + WTF::Bitmap<atomsPerBlock, WTF::BitmapNotAtomic, uint8_t> m_rememberedSet; #endif - std::unique_ptr<WTF::Bitmap<atomsPerBlock>> m_newlyAllocated; + OwnPtr<WTF::Bitmap<atomsPerBlock>> m_newlyAllocated; - size_t m_capacity; - bool m_needsDestruction; + DestructorType m_destructorType; MarkedAllocator* m_allocator; BlockState m_state; WeakSet m_weakSet; @@ -285,11 +282,6 @@ namespace JSC { m_weakSet.reap(); } - inline void MarkedBlock::willRemoveBlock() - { - ASSERT(m_state != Retired); - } - inline void MarkedBlock::didConsumeFreeList() { HEAP_LOG_BLOCK_STATE_TRANSITION(this); @@ -322,9 +314,9 @@ namespace JSC { return m_atomsPerCell * atomSize; } - inline bool MarkedBlock::needsDestruction() const + inline MarkedBlock::DestructorType MarkedBlock::destructorType() { - return m_needsDestruction; + return m_destructorType; } inline size_t MarkedBlock::size() @@ -334,7 +326,7 @@ namespace JSC { inline size_t MarkedBlock::capacity() { - return m_capacity; + return region()->blockSize(); } inline size_t MarkedBlock::atomNumber(const void* p) @@ -342,6 +334,26 @@ namespace JSC { return (reinterpret_cast<Bits>(p) - reinterpret_cast<Bits>(this)) / atomSize; } + inline void MarkedBlock::setRemembered(const void* p) + { + m_rememberedSet.set(atomNumber(p)); + } + + inline void MarkedBlock::clearRemembered(const void* p) + { + m_rememberedSet.clear(atomNumber(p)); + } + + inline void MarkedBlock::atomicClearRemembered(const void* p) + { + m_rememberedSet.concurrentTestAndClear(atomNumber(p)); + } + + inline bool MarkedBlock::isRemembered(const void* p) + { + return m_rememberedSet.get(atomNumber(p)); + } + inline bool MarkedBlock::isMarked(const void* p) { return m_marks.get(atomNumber(p)); @@ -381,27 +393,20 @@ namespace JSC { inline bool MarkedBlock::clearNewlyAllocated() { if (m_newlyAllocated) { - m_newlyAllocated = nullptr; + m_newlyAllocated.clear(); return true; } return false; } - inline bool MarkedBlock::isMarkedOrNewlyAllocated(const JSCell* cell) - { - ASSERT(m_state == Retired || m_state == Marked); - return m_marks.get(atomNumber(cell)) || (m_newlyAllocated && isNewlyAllocated(cell)); - } - inline bool MarkedBlock::isLive(const JSCell* cell) { switch (m_state) { case Allocated: return true; - case Retired: case Marked: - return isMarkedOrNewlyAllocated(cell); + return m_marks.get(atomNumber(cell)) || (m_newlyAllocated && isNewlyAllocated(cell)); case New: case FreeListed: @@ -428,40 +433,34 @@ namespace JSC { return isLive(static_cast<const JSCell*>(p)); } - template <typename Functor> inline IterationStatus MarkedBlock::forEachCell(Functor& functor) + template <typename Functor> inline void MarkedBlock::forEachCell(Functor& functor) { for (size_t i = firstAtom(); i < m_endAtom; i += m_atomsPerCell) { JSCell* cell = reinterpret_cast_ptr<JSCell*>(&atoms()[i]); - if (functor(cell) == IterationStatus::Done) - return IterationStatus::Done; + functor(cell); } - return IterationStatus::Continue; } - template <typename Functor> inline IterationStatus MarkedBlock::forEachLiveCell(Functor& functor) + template <typename Functor> inline void MarkedBlock::forEachLiveCell(Functor& functor) { for (size_t i = firstAtom(); i < m_endAtom; i += m_atomsPerCell) { JSCell* cell = reinterpret_cast_ptr<JSCell*>(&atoms()[i]); if (!isLive(cell)) continue; - if (functor(cell) == IterationStatus::Done) - return IterationStatus::Done; + functor(cell); } - return IterationStatus::Continue; } - template <typename Functor> inline IterationStatus MarkedBlock::forEachDeadCell(Functor& functor) + template <typename Functor> inline void MarkedBlock::forEachDeadCell(Functor& functor) { for (size_t i = firstAtom(); i < m_endAtom; i += m_atomsPerCell) { JSCell* cell = reinterpret_cast_ptr<JSCell*>(&atoms()[i]); if (isLive(cell)) continue; - if (functor(cell) == IterationStatus::Done) - return IterationStatus::Done; + functor(cell); } - return IterationStatus::Continue; } inline bool MarkedBlock::needsSweeping() @@ -469,11 +468,6 @@ namespace JSC { return m_state == Marked; } - inline bool MarkedBlock::isAllocated() const - { - return m_state == Allocated; - } - } // namespace JSC namespace WTF { diff --git a/Source/JavaScriptCore/heap/MarkedBlockSet.h b/Source/JavaScriptCore/heap/MarkedBlockSet.h index 9cf19088c..022a17389 100644 --- a/Source/JavaScriptCore/heap/MarkedBlockSet.h +++ b/Source/JavaScriptCore/heap/MarkedBlockSet.h @@ -57,7 +57,7 @@ inline void MarkedBlockSet::add(MarkedBlock* block) inline void MarkedBlockSet::remove(MarkedBlock* block) { - unsigned oldCapacity = m_set.capacity(); + int oldCapacity = m_set.capacity(); m_set.remove(block); if (m_set.capacity() != oldCapacity) // Indicates we've removed a lot of blocks. recomputeFilter(); diff --git a/Source/JavaScriptCore/heap/MarkedSpace.cpp b/Source/JavaScriptCore/heap/MarkedSpace.cpp index 4f308900f..e005337a6 100644 --- a/Source/JavaScriptCore/heap/MarkedSpace.cpp +++ b/Source/JavaScriptCore/heap/MarkedSpace.cpp @@ -21,11 +21,12 @@ #include "config.h" #include "MarkedSpace.h" +#include "DelayedReleaseScope.h" #include "IncrementalSweeper.h" #include "JSGlobalObject.h" #include "JSLock.h" #include "JSObject.h" -#include "JSCInlines.h" + namespace JSC { @@ -81,19 +82,23 @@ MarkedSpace::MarkedSpace(Heap* heap) : m_heap(heap) , m_capacity(0) , m_isIterating(false) + , m_currentDelayedReleaseScope(nullptr) { for (size_t cellSize = preciseStep; cellSize <= preciseCutoff; cellSize += preciseStep) { - allocatorFor(cellSize).init(heap, this, cellSize, false); - destructorAllocatorFor(cellSize).init(heap, this, cellSize, true); + allocatorFor(cellSize).init(heap, this, cellSize, MarkedBlock::None); + normalDestructorAllocatorFor(cellSize).init(heap, this, cellSize, MarkedBlock::Normal); + immortalStructureDestructorAllocatorFor(cellSize).init(heap, this, cellSize, MarkedBlock::ImmortalStructure); } for (size_t cellSize = impreciseStep; cellSize <= impreciseCutoff; cellSize += impreciseStep) { - allocatorFor(cellSize).init(heap, this, cellSize, false); - destructorAllocatorFor(cellSize).init(heap, this, cellSize, true); + allocatorFor(cellSize).init(heap, this, cellSize, MarkedBlock::None); + normalDestructorAllocatorFor(cellSize).init(heap, this, cellSize, MarkedBlock::Normal); + immortalStructureDestructorAllocatorFor(cellSize).init(heap, this, cellSize, MarkedBlock::ImmortalStructure); } - m_normalSpace.largeAllocator.init(heap, this, 0, false); - m_destructorSpace.largeAllocator.init(heap, this, 0, true); + m_normalSpace.largeAllocator.init(heap, this, 0, MarkedBlock::None); + m_normalDestructorSpace.largeAllocator.init(heap, this, 0, MarkedBlock::Normal); + m_immortalStructureDestructorSpace.largeAllocator.init(heap, this, 0, MarkedBlock::ImmortalStructure); } MarkedSpace::~MarkedSpace() @@ -103,44 +108,42 @@ MarkedSpace::~MarkedSpace() ASSERT(!m_blocks.set().size()); } -struct LastChanceToFinalize { - void operator()(MarkedAllocator& allocator) { allocator.lastChanceToFinalize(); } +struct LastChanceToFinalize : MarkedBlock::VoidFunctor { + void operator()(MarkedBlock* block) { block->lastChanceToFinalize(); } }; void MarkedSpace::lastChanceToFinalize() { + DelayedReleaseScope delayedReleaseScope(*this); stopAllocating(); - forEachAllocator<LastChanceToFinalize>(); + forEachBlock<LastChanceToFinalize>(); } void MarkedSpace::sweep() { - m_heap->sweeper()->willFinishSweeping(); - forEachBlock<Sweep>(); -} - -void MarkedSpace::zombifySweep() -{ if (Options::logGC()) - dataLog("Zombifying sweep..."); + dataLog("Eagerly sweeping..."); m_heap->sweeper()->willFinishSweeping(); - forEachBlock<ZombifySweep>(); + forEachBlock<Sweep>(); } void MarkedSpace::resetAllocators() { for (size_t cellSize = preciseStep; cellSize <= preciseCutoff; cellSize += preciseStep) { allocatorFor(cellSize).reset(); - destructorAllocatorFor(cellSize).reset(); + normalDestructorAllocatorFor(cellSize).reset(); + immortalStructureDestructorAllocatorFor(cellSize).reset(); } for (size_t cellSize = impreciseStep; cellSize <= impreciseCutoff; cellSize += impreciseStep) { allocatorFor(cellSize).reset(); - destructorAllocatorFor(cellSize).reset(); + normalDestructorAllocatorFor(cellSize).reset(); + immortalStructureDestructorAllocatorFor(cellSize).reset(); } m_normalSpace.largeAllocator.reset(); - m_destructorSpace.largeAllocator.reset(); + m_normalDestructorSpace.largeAllocator.reset(); + m_immortalStructureDestructorSpace.largeAllocator.reset(); #if ENABLE(GGC) m_blocksWithNewObjects.clear(); @@ -178,16 +181,19 @@ void MarkedSpace::forEachAllocator(Functor& functor) { for (size_t cellSize = preciseStep; cellSize <= preciseCutoff; cellSize += preciseStep) { functor(allocatorFor(cellSize)); - functor(destructorAllocatorFor(cellSize)); + functor(normalDestructorAllocatorFor(cellSize)); + functor(immortalStructureDestructorAllocatorFor(cellSize)); } for (size_t cellSize = impreciseStep; cellSize <= impreciseCutoff; cellSize += impreciseStep) { functor(allocatorFor(cellSize)); - functor(destructorAllocatorFor(cellSize)); + functor(normalDestructorAllocatorFor(cellSize)); + functor(immortalStructureDestructorAllocatorFor(cellSize)); } functor(m_normalSpace.largeAllocator); - functor(m_destructorSpace.largeAllocator); + functor(m_normalDestructorSpace.largeAllocator); + functor(m_immortalStructureDestructorSpace.largeAllocator); } struct StopAllocatingFunctor { @@ -207,6 +213,7 @@ struct ResumeAllocatingFunctor { void MarkedSpace::resumeAllocating() { ASSERT(isIterating()); + DelayedReleaseScope scope(*this); forEachAllocator<ResumeAllocatingFunctor>(); } @@ -214,18 +221,21 @@ bool MarkedSpace::isPagedOut(double deadline) { for (size_t cellSize = preciseStep; cellSize <= preciseCutoff; cellSize += preciseStep) { if (allocatorFor(cellSize).isPagedOut(deadline) - || destructorAllocatorFor(cellSize).isPagedOut(deadline)) + || normalDestructorAllocatorFor(cellSize).isPagedOut(deadline) + || immortalStructureDestructorAllocatorFor(cellSize).isPagedOut(deadline)) return true; } for (size_t cellSize = impreciseStep; cellSize <= impreciseCutoff; cellSize += impreciseStep) { if (allocatorFor(cellSize).isPagedOut(deadline) - || destructorAllocatorFor(cellSize).isPagedOut(deadline)) + || normalDestructorAllocatorFor(cellSize).isPagedOut(deadline) + || immortalStructureDestructorAllocatorFor(cellSize).isPagedOut(deadline)) return true; } if (m_normalSpace.largeAllocator.isPagedOut(deadline) - || m_destructorSpace.largeAllocator.isPagedOut(deadline)) + || m_normalDestructorSpace.largeAllocator.isPagedOut(deadline) + || m_immortalStructureDestructorSpace.largeAllocator.isPagedOut(deadline)) return true; return false; @@ -236,7 +246,11 @@ void MarkedSpace::freeBlock(MarkedBlock* block) block->allocator()->removeBlock(block); m_capacity -= block->capacity(); m_blocks.remove(block); - MarkedBlock::destroy(block); + if (block->capacity() == MarkedBlock::blockSize) { + m_heap->blockAllocator().deallocate(MarkedBlock::destroy(block)); + return; + } + m_heap->blockAllocator().deallocateCustomSize(MarkedBlock::destroy(block)); } void MarkedSpace::freeOrShrinkBlock(MarkedBlock* block) @@ -280,12 +294,14 @@ void MarkedSpace::clearNewlyAllocated() { for (size_t i = 0; i < preciseCount; ++i) { clearNewlyAllocatedInBlock(m_normalSpace.preciseAllocators[i].takeLastActiveBlock()); - clearNewlyAllocatedInBlock(m_destructorSpace.preciseAllocators[i].takeLastActiveBlock()); + clearNewlyAllocatedInBlock(m_normalDestructorSpace.preciseAllocators[i].takeLastActiveBlock()); + clearNewlyAllocatedInBlock(m_immortalStructureDestructorSpace.preciseAllocators[i].takeLastActiveBlock()); } for (size_t i = 0; i < impreciseCount; ++i) { clearNewlyAllocatedInBlock(m_normalSpace.impreciseAllocators[i].takeLastActiveBlock()); - clearNewlyAllocatedInBlock(m_destructorSpace.impreciseAllocators[i].takeLastActiveBlock()); + clearNewlyAllocatedInBlock(m_normalDestructorSpace.impreciseAllocators[i].takeLastActiveBlock()); + clearNewlyAllocatedInBlock(m_immortalStructureDestructorSpace.impreciseAllocators[i].takeLastActiveBlock()); } // We have to iterate all of the blocks in the large allocators because they are @@ -293,7 +309,8 @@ void MarkedSpace::clearNewlyAllocated() // which creates the m_newlyAllocated bitmap. ClearNewlyAllocated functor; m_normalSpace.largeAllocator.forEachBlock(functor); - m_destructorSpace.largeAllocator.forEachBlock(functor); + m_normalDestructorSpace.largeAllocator.forEachBlock(functor); + m_immortalStructureDestructorSpace.largeAllocator.forEachBlock(functor); #ifndef NDEBUG VerifyNewlyAllocated verifyFunctor; @@ -301,20 +318,11 @@ void MarkedSpace::clearNewlyAllocated() #endif } -#ifndef NDEBUG -struct VerifyMarkedOrRetired : MarkedBlock::VoidFunctor { - void operator()(MarkedBlock* block) - { - switch (block->m_state) { - case MarkedBlock::Marked: - case MarkedBlock::Retired: - return; - default: - RELEASE_ASSERT_NOT_REACHED(); - } - } -}; -#endif +#ifndef NDEBUG +struct VerifyMarked : MarkedBlock::VoidFunctor { + void operator()(MarkedBlock* block) { ASSERT(block->needsSweeping()); } +}; +#endif void MarkedSpace::clearMarks() { @@ -323,10 +331,8 @@ void MarkedSpace::clearMarks() m_blocksWithNewObjects[i]->clearMarks(); } else forEachBlock<ClearMarks>(); - #ifndef NDEBUG - VerifyMarkedOrRetired verifyFunctor; - forEachBlock(verifyFunctor); + forEachBlock<VerifyMarked>(); #endif } diff --git a/Source/JavaScriptCore/heap/MarkedSpace.h b/Source/JavaScriptCore/heap/MarkedSpace.h index bb388dd70..e853d6674 100644 --- a/Source/JavaScriptCore/heap/MarkedSpace.h +++ b/Source/JavaScriptCore/heap/MarkedSpace.h @@ -27,15 +27,16 @@ #include "MarkedBlock.h" #include "MarkedBlockSet.h" #include <array> +#include <wtf/PageAllocationAligned.h> #include <wtf/Bitmap.h> #include <wtf/DoublyLinkedList.h> #include <wtf/HashSet.h> #include <wtf/Noncopyable.h> -#include <wtf/RetainPtr.h> #include <wtf/Vector.h> namespace JSC { +class DelayedReleaseScope; class Heap; class HeapIterationScope; class JSCell; @@ -51,18 +52,17 @@ struct ClearMarks : MarkedBlock::VoidFunctor { } }; -struct Sweep : MarkedBlock::VoidFunctor { - void operator()(MarkedBlock* block) { block->sweep(); } -}; - -struct ZombifySweep : MarkedBlock::VoidFunctor { +struct ClearRememberedSet : MarkedBlock::VoidFunctor { void operator()(MarkedBlock* block) { - if (block->needsSweeping()) - block->sweep(); + block->clearRememberedSet(); } }; +struct Sweep : MarkedBlock::VoidFunctor { + void operator()(MarkedBlock* block) { block->sweep(); } +}; + struct MarkCount : MarkedBlock::CountFunctor { void operator()(MarkedBlock* block) { count(block->markCount()); } }; @@ -74,35 +74,18 @@ struct Size : MarkedBlock::CountFunctor { class MarkedSpace { WTF_MAKE_NONCOPYABLE(MarkedSpace); public: - // [ 32... 128 ] - static const size_t preciseStep = MarkedBlock::atomSize; - static const size_t preciseCutoff = 128; - static const size_t preciseCount = preciseCutoff / preciseStep; - - // [ 1024... blockSize ] - static const size_t impreciseStep = 2 * preciseCutoff; - static const size_t impreciseCutoff = MarkedBlock::blockSize / 2; - static const size_t impreciseCount = impreciseCutoff / impreciseStep; - - struct Subspace { - std::array<MarkedAllocator, preciseCount> preciseAllocators; - std::array<MarkedAllocator, impreciseCount> impreciseAllocators; - MarkedAllocator largeAllocator; - }; - MarkedSpace(Heap*); ~MarkedSpace(); void lastChanceToFinalize(); MarkedAllocator& firstAllocator(); MarkedAllocator& allocatorFor(size_t); - MarkedAllocator& destructorAllocatorFor(size_t); - void* allocateWithDestructor(size_t); + MarkedAllocator& immortalStructureDestructorAllocatorFor(size_t); + MarkedAllocator& normalDestructorAllocatorFor(size_t); + void* allocateWithNormalDestructor(size_t); + void* allocateWithImmortalStructureDestructor(size_t); void* allocateWithoutDestructor(size_t); - - Subspace& subspaceForObjectsWithDestructor() { return m_destructorSpace; } - Subspace& subspaceForObjectsWithoutDestructor() { return m_normalSpace; } - + void resetAllocators(); void visitWeakSets(HeapRootVisitor&); @@ -135,9 +118,9 @@ public: void didAllocateInBlock(MarkedBlock*); void clearMarks(); + void clearRememberedSet(); void clearNewlyAllocated(); void sweep(); - void zombifySweep(); size_t objectCount(); size_t size(); size_t capacity(); @@ -148,16 +131,31 @@ public: template<typename T> void releaseSoon(RetainPtr<T>&&); #endif - const Vector<MarkedBlock*>& blocksWithNewObjects() const { return m_blocksWithNewObjects; } - private: + friend class DelayedReleaseScope; friend class LLIntOffsetsExtractor; - friend class JIT; template<typename Functor> void forEachAllocator(Functor&); template<typename Functor> void forEachAllocator(); - Subspace m_destructorSpace; + // [ 32... 128 ] + static const size_t preciseStep = MarkedBlock::atomSize; + static const size_t preciseCutoff = 128; + static const size_t preciseCount = preciseCutoff / preciseStep; + + // [ 1024... blockSize ] + static const size_t impreciseStep = 2 * preciseCutoff; + static const size_t impreciseCutoff = MarkedBlock::blockSize / 2; + static const size_t impreciseCount = impreciseCutoff / impreciseStep; + + struct Subspace { + std::array<MarkedAllocator, preciseCount> preciseAllocators; + std::array<MarkedAllocator, impreciseCount> impreciseAllocators; + MarkedAllocator largeAllocator; + }; + + Subspace m_normalDestructorSpace; + Subspace m_immortalStructureDestructorSpace; Subspace m_normalSpace; Heap* m_heap; @@ -165,16 +163,16 @@ private: bool m_isIterating; MarkedBlockSet m_blocks; Vector<MarkedBlock*> m_blocksWithNewObjects; + + DelayedReleaseScope* m_currentDelayedReleaseScope; }; template<typename Functor> inline typename Functor::ReturnType MarkedSpace::forEachLiveCell(HeapIterationScope&, Functor& functor) { ASSERT(isIterating()); BlockIterator end = m_blocks.set().end(); - for (BlockIterator it = m_blocks.set().begin(); it != end; ++it) { - if ((*it)->forEachLiveCell(functor) == IterationStatus::Done) - break; - } + for (BlockIterator it = m_blocks.set().begin(); it != end; ++it) + (*it)->forEachLiveCell(functor); return functor.returnValue(); } @@ -188,10 +186,8 @@ template<typename Functor> inline typename Functor::ReturnType MarkedSpace::forE { ASSERT(isIterating()); BlockIterator end = m_blocks.set().end(); - for (BlockIterator it = m_blocks.set().begin(); it != end; ++it) { - if ((*it)->forEachDeadCell(functor) == IterationStatus::Done) - break; - } + for (BlockIterator it = m_blocks.set().begin(); it != end; ++it) + (*it)->forEachDeadCell(functor); return functor.returnValue(); } @@ -211,14 +207,24 @@ inline MarkedAllocator& MarkedSpace::allocatorFor(size_t bytes) return m_normalSpace.largeAllocator; } -inline MarkedAllocator& MarkedSpace::destructorAllocatorFor(size_t bytes) +inline MarkedAllocator& MarkedSpace::immortalStructureDestructorAllocatorFor(size_t bytes) { ASSERT(bytes); if (bytes <= preciseCutoff) - return m_destructorSpace.preciseAllocators[(bytes - 1) / preciseStep]; + return m_immortalStructureDestructorSpace.preciseAllocators[(bytes - 1) / preciseStep]; if (bytes <= impreciseCutoff) - return m_destructorSpace.impreciseAllocators[(bytes - 1) / impreciseStep]; - return m_destructorSpace.largeAllocator; + return m_immortalStructureDestructorSpace.impreciseAllocators[(bytes - 1) / impreciseStep]; + return m_immortalStructureDestructorSpace.largeAllocator; +} + +inline MarkedAllocator& MarkedSpace::normalDestructorAllocatorFor(size_t bytes) +{ + ASSERT(bytes); + if (bytes <= preciseCutoff) + return m_normalDestructorSpace.preciseAllocators[(bytes - 1) / preciseStep]; + if (bytes <= impreciseCutoff) + return m_normalDestructorSpace.impreciseAllocators[(bytes - 1) / impreciseStep]; + return m_normalDestructorSpace.largeAllocator; } inline void* MarkedSpace::allocateWithoutDestructor(size_t bytes) @@ -226,24 +232,33 @@ inline void* MarkedSpace::allocateWithoutDestructor(size_t bytes) return allocatorFor(bytes).allocate(bytes); } -inline void* MarkedSpace::allocateWithDestructor(size_t bytes) +inline void* MarkedSpace::allocateWithImmortalStructureDestructor(size_t bytes) { - return destructorAllocatorFor(bytes).allocate(bytes); + return immortalStructureDestructorAllocatorFor(bytes).allocate(bytes); +} + +inline void* MarkedSpace::allocateWithNormalDestructor(size_t bytes) +{ + return normalDestructorAllocatorFor(bytes).allocate(bytes); } template <typename Functor> inline typename Functor::ReturnType MarkedSpace::forEachBlock(Functor& functor) { - for (size_t i = 0; i < preciseCount; ++i) + for (size_t i = 0; i < preciseCount; ++i) { m_normalSpace.preciseAllocators[i].forEachBlock(functor); - for (size_t i = 0; i < impreciseCount; ++i) + m_normalDestructorSpace.preciseAllocators[i].forEachBlock(functor); + m_immortalStructureDestructorSpace.preciseAllocators[i].forEachBlock(functor); + } + + for (size_t i = 0; i < impreciseCount; ++i) { m_normalSpace.impreciseAllocators[i].forEachBlock(functor); - m_normalSpace.largeAllocator.forEachBlock(functor); + m_normalDestructorSpace.impreciseAllocators[i].forEachBlock(functor); + m_immortalStructureDestructorSpace.impreciseAllocators[i].forEachBlock(functor); + } - for (size_t i = 0; i < preciseCount; ++i) - m_destructorSpace.preciseAllocators[i].forEachBlock(functor); - for (size_t i = 0; i < impreciseCount; ++i) - m_destructorSpace.impreciseAllocators[i].forEachBlock(functor); - m_destructorSpace.largeAllocator.forEachBlock(functor); + m_normalSpace.largeAllocator.forEachBlock(functor); + m_normalDestructorSpace.largeAllocator.forEachBlock(functor); + m_immortalStructureDestructorSpace.largeAllocator.forEachBlock(functor); return functor.returnValue(); } @@ -269,6 +284,11 @@ inline void MarkedSpace::didAllocateInBlock(MarkedBlock* block) #endif } +inline void MarkedSpace::clearRememberedSet() +{ + forEachBlock<ClearRememberedSet>(); +} + inline size_t MarkedSpace::objectCount() { return forEachBlock<MarkCount>(); diff --git a/Source/JavaScriptCore/heap/OpaqueRootSet.h b/Source/JavaScriptCore/heap/OpaqueRootSet.h deleted file mode 100644 index a08bdec04..000000000 --- a/Source/JavaScriptCore/heap/OpaqueRootSet.h +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright (C) 2014 Apple Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS - * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF - * THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef OpaqueRootSet_h -#define OpaqueRootSet_h - -#include <wtf/HashSet.h> - -namespace JSC { - -class OpaqueRootSet { - WTF_MAKE_NONCOPYABLE(OpaqueRootSet); -public: - OpaqueRootSet() - : m_lastQueriedRoot(nullptr) - , m_containsLastQueriedRoot(false) - { - } - - bool contains(void* root) const - { - if (root != m_lastQueriedRoot) { - m_lastQueriedRoot = root; - m_containsLastQueriedRoot = m_roots.contains(root); - } - return m_containsLastQueriedRoot; - } - - bool isEmpty() const - { - return m_roots.isEmpty(); - } - - void clear() - { - m_roots.clear(); - m_lastQueriedRoot = nullptr; - m_containsLastQueriedRoot = false; - } - - void add(void* root) - { - if (root == m_lastQueriedRoot) - m_containsLastQueriedRoot = true; - m_roots.add(root); - } - - int size() const - { - return m_roots.size(); - } - - HashSet<void*>::const_iterator begin() const - { - return m_roots.begin(); - } - - HashSet<void*>::const_iterator end() const - { - return m_roots.end(); - } - - -private: - HashSet<void*> m_roots; - mutable void* m_lastQueriedRoot; - mutable bool m_containsLastQueriedRoot; -}; - -} // namespace JSC - -#endif // OpaqueRootSet_h diff --git a/Source/JavaScriptCore/heap/Region.h b/Source/JavaScriptCore/heap/Region.h new file mode 100644 index 000000000..c638059b6 --- /dev/null +++ b/Source/JavaScriptCore/heap/Region.h @@ -0,0 +1,319 @@ +/* + * Copyright (C) 2013 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef JSC_Region_h +#define JSC_Region_h + +#include "HeapBlock.h" +#include "SuperRegion.h" +#include <wtf/DoublyLinkedList.h> +#include <wtf/MetaAllocatorHandle.h> +#include <wtf/PageAllocationAligned.h> + +#define HEAP_MEMORY_ID reinterpret_cast<void*>(static_cast<intptr_t>(-3)) + +#define ENABLE_SUPER_REGION 0 + +#ifndef ENABLE_SUPER_REGION +#if USE(JSVALUE64) && !CPU(ARM64) +#define ENABLE_SUPER_REGION 1 +#else +#define ENABLE_SUPER_REGION 0 +#endif +#endif + +namespace JSC { + +class DeadBlock : public HeapBlock<DeadBlock> { +public: + DeadBlock(Region*); +}; + +inline DeadBlock::DeadBlock(Region* region) + : HeapBlock<DeadBlock>(region) +{ +} + +class Region : public DoublyLinkedListNode<Region> { + friend CLASS_IF_GCC DoublyLinkedListNode<Region>; + friend class BlockAllocator; +public: + ~Region(); + static Region* create(SuperRegion*, size_t blockSize); + static Region* createCustomSize(SuperRegion*, size_t blockSize, size_t blockAlignment); + Region* reset(size_t blockSize); + void destroy(); + + size_t blockSize() const { return m_blockSize; } + bool isFull() const { return m_blocksInUse == m_totalBlocks; } + bool isEmpty() const { return !m_blocksInUse; } + bool isCustomSize() const { return m_isCustomSize; } + + DeadBlock* allocate(); + void deallocate(void*); + + static const size_t s_regionSize = 64 * KB; + static const size_t s_regionMask = ~(s_regionSize - 1); + +protected: + Region(size_t blockSize, size_t totalBlocks, bool isExcess); + void initializeBlockList(); + + bool m_isExcess; + +private: + void* base(); + size_t size(); + + size_t m_totalBlocks; + size_t m_blocksInUse; + size_t m_blockSize; + bool m_isCustomSize; + Region* m_prev; + Region* m_next; + DoublyLinkedList<DeadBlock> m_deadBlocks; +}; + + +class NormalRegion : public Region { + friend class Region; +private: + NormalRegion(PassRefPtr<WTF::MetaAllocatorHandle>, size_t blockSize, size_t totalBlocks); + + static NormalRegion* tryCreate(SuperRegion*, size_t blockSize); + static NormalRegion* tryCreateCustomSize(SuperRegion*, size_t blockSize, size_t blockAlignment); + + void* base() { return m_allocation->start(); } + size_t size() { return m_allocation->sizeInBytes(); } + + NormalRegion* reset(size_t blockSize); + + RefPtr<WTF::MetaAllocatorHandle> m_allocation; +}; + +class ExcessRegion : public Region { + friend class Region; +private: + ExcessRegion(PageAllocationAligned&, size_t blockSize, size_t totalBlocks); + + ~ExcessRegion(); + + static ExcessRegion* create(size_t blockSize); + static ExcessRegion* createCustomSize(size_t blockSize, size_t blockAlignment); + + void* base() { return m_allocation.base(); } + size_t size() { return m_allocation.size(); } + + ExcessRegion* reset(size_t blockSize); + + PageAllocationAligned m_allocation; +}; + +inline NormalRegion::NormalRegion(PassRefPtr<WTF::MetaAllocatorHandle> allocation, size_t blockSize, size_t totalBlocks) + : Region(blockSize, totalBlocks, false) + , m_allocation(allocation) +{ + initializeBlockList(); +} + +inline NormalRegion* NormalRegion::tryCreate(SuperRegion* superRegion, size_t blockSize) +{ + RefPtr<WTF::MetaAllocatorHandle> allocation = superRegion->allocate(s_regionSize, HEAP_MEMORY_ID); + if (!allocation) + return 0; + return new NormalRegion(allocation, blockSize, s_regionSize / blockSize); +} + +inline NormalRegion* NormalRegion::tryCreateCustomSize(SuperRegion* superRegion, size_t blockSize, size_t blockAlignment) +{ + ASSERT_UNUSED(blockAlignment, blockAlignment <= s_regionSize); + RefPtr<WTF::MetaAllocatorHandle> allocation = superRegion->allocate(blockSize, HEAP_MEMORY_ID); + if (!allocation) + return 0; + return new NormalRegion(allocation, blockSize, 1); +} + +inline NormalRegion* NormalRegion::reset(size_t blockSize) +{ + ASSERT(!m_isExcess); + RefPtr<WTF::MetaAllocatorHandle> allocation = m_allocation.release(); + return new (NotNull, this) NormalRegion(allocation.release(), blockSize, s_regionSize / blockSize); +} + +inline ExcessRegion::ExcessRegion(PageAllocationAligned& allocation, size_t blockSize, size_t totalBlocks) + : Region(blockSize, totalBlocks, true) + , m_allocation(allocation) +{ + initializeBlockList(); +} + +inline ExcessRegion::~ExcessRegion() +{ + m_allocation.deallocate(); +} + +inline ExcessRegion* ExcessRegion::create(size_t blockSize) +{ + PageAllocationAligned allocation = PageAllocationAligned::allocate(s_regionSize, s_regionSize, OSAllocator::JSGCHeapPages); + ASSERT(static_cast<bool>(allocation)); + return new ExcessRegion(allocation, blockSize, s_regionSize / blockSize); +} + +inline ExcessRegion* ExcessRegion::createCustomSize(size_t blockSize, size_t blockAlignment) +{ + PageAllocationAligned allocation = PageAllocationAligned::allocate(blockSize, blockAlignment, OSAllocator::JSGCHeapPages); + ASSERT(static_cast<bool>(allocation)); + return new ExcessRegion(allocation, blockSize, 1); +} + +inline ExcessRegion* ExcessRegion::reset(size_t blockSize) +{ + ASSERT(m_isExcess); + PageAllocationAligned allocation = m_allocation; + return new (NotNull, this) ExcessRegion(allocation, blockSize, s_regionSize / blockSize); +} + +inline Region::Region(size_t blockSize, size_t totalBlocks, bool isExcess) + : DoublyLinkedListNode<Region>() + , m_isExcess(isExcess) + , m_totalBlocks(totalBlocks) + , m_blocksInUse(0) + , m_blockSize(blockSize) + , m_isCustomSize(false) + , m_prev(0) + , m_next(0) +{ +} + +inline void Region::initializeBlockList() +{ + char* start = static_cast<char*>(base()); + char* current = start; + for (size_t i = 0; i < m_totalBlocks; i++) { + ASSERT(current < start + size()); + m_deadBlocks.append(new (NotNull, current) DeadBlock(this)); + current += m_blockSize; + } +} + +inline Region* Region::create(SuperRegion* superRegion, size_t blockSize) +{ +#if ENABLE(SUPER_REGION) + ASSERT(blockSize <= s_regionSize); + ASSERT(!(s_regionSize % blockSize)); + Region* region = NormalRegion::tryCreate(superRegion, blockSize); + if (LIKELY(!!region)) + return region; +#else + UNUSED_PARAM(superRegion); +#endif + return ExcessRegion::create(blockSize); +} + +inline Region* Region::createCustomSize(SuperRegion* superRegion, size_t blockSize, size_t blockAlignment) +{ +#if ENABLE(SUPER_REGION) + Region* region = NormalRegion::tryCreateCustomSize(superRegion, blockSize, blockAlignment); + if (UNLIKELY(!region)) + region = ExcessRegion::createCustomSize(blockSize, blockAlignment); +#else + UNUSED_PARAM(superRegion); + Region* region = ExcessRegion::createCustomSize(blockSize, blockAlignment); +#endif + region->m_isCustomSize = true; + return region; +} + +inline Region::~Region() +{ + ASSERT(isEmpty()); +} + +inline void Region::destroy() +{ +#if ENABLE(SUPER_REGION) + if (UNLIKELY(m_isExcess)) + delete static_cast<ExcessRegion*>(this); + else + delete static_cast<NormalRegion*>(this); +#else + delete static_cast<ExcessRegion*>(this); +#endif +} + +inline Region* Region::reset(size_t blockSize) +{ +#if ENABLE(SUPER_REGION) + ASSERT(isEmpty()); + if (UNLIKELY(m_isExcess)) + return static_cast<ExcessRegion*>(this)->reset(blockSize); + return static_cast<NormalRegion*>(this)->reset(blockSize); +#else + return static_cast<ExcessRegion*>(this)->reset(blockSize); +#endif +} + +inline DeadBlock* Region::allocate() +{ + ASSERT(!isFull()); + m_blocksInUse++; + return m_deadBlocks.removeHead(); +} + +inline void Region::deallocate(void* base) +{ + ASSERT(base); + ASSERT(m_blocksInUse); + ASSERT(base >= this->base() && base < static_cast<char*>(this->base()) + size()); + DeadBlock* block = new (NotNull, base) DeadBlock(this); + m_deadBlocks.push(block); + m_blocksInUse--; +} + +inline void* Region::base() +{ +#if ENABLE(SUPER_REGION) + if (UNLIKELY(m_isExcess)) + return static_cast<ExcessRegion*>(this)->ExcessRegion::base(); + return static_cast<NormalRegion*>(this)->NormalRegion::base(); +#else + return static_cast<ExcessRegion*>(this)->ExcessRegion::base(); +#endif +} + +inline size_t Region::size() +{ +#if ENABLE(SUPER_REGION) + if (UNLIKELY(m_isExcess)) + return static_cast<ExcessRegion*>(this)->ExcessRegion::size(); + return static_cast<NormalRegion*>(this)->NormalRegion::size(); +#else + return static_cast<ExcessRegion*>(this)->ExcessRegion::size(); +#endif +} + +} // namespace JSC + +#endif // JSC_Region_h diff --git a/Source/JavaScriptCore/heap/SlotVisitor.cpp b/Source/JavaScriptCore/heap/SlotVisitor.cpp index a22538661..4fd0da725 100644 --- a/Source/JavaScriptCore/heap/SlotVisitor.cpp +++ b/Source/JavaScriptCore/heap/SlotVisitor.cpp @@ -1,28 +1,3 @@ -/* - * Copyright (C) 2012, 2015 Apple Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS - * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF - * THE POSSIBILITY OF SUCH DAMAGE. - */ - #include "config.h" #include "SlotVisitor.h" #include "SlotVisitorInlines.h" @@ -36,14 +11,13 @@ #include "VM.h" #include "JSObject.h" #include "JSString.h" -#include "JSCInlines.h" -#include <wtf/Lock.h> +#include "Operations.h" #include <wtf/StackStats.h> namespace JSC { SlotVisitor::SlotVisitor(GCThreadSharedData& shared) - : m_stack() + : m_stack(shared.m_vm->heap.blockAllocator()) , m_bytesVisited(0) , m_bytesCopied(0) , m_visitCount(0) @@ -62,16 +36,8 @@ SlotVisitor::~SlotVisitor() clearMarkStack(); } -void SlotVisitor::didStartMarking() +void SlotVisitor::setup() { - if (heap()->operationInProgress() == FullCollection) { -#if ENABLE(PARALLEL_GC) - ASSERT(m_opaqueRoots.isEmpty()); // Should have merged by now. -#else - m_opaqueRoots.clear(); -#endif - } - m_shared.m_shouldHashCons = m_shared.m_vm->haveEnoughNewStringsToHashCons(); m_shouldHashCons = m_shared.m_shouldHashCons; #if ENABLE(PARALLEL_GC) @@ -86,6 +52,11 @@ void SlotVisitor::reset() m_bytesCopied = 0; m_visitCount = 0; ASSERT(m_stack.isEmpty()); +#if ENABLE(PARALLEL_GC) + ASSERT(m_opaqueRoots.isEmpty()); // Should have merged by now. +#else + m_opaqueRoots.clear(); +#endif if (m_shouldHashCons) { m_uniqueStrings.clear(); m_shouldHashCons = false; @@ -147,7 +118,7 @@ void SlotVisitor::donateKnownParallel() // If we're contending on the lock, be conservative and assume that another // thread is already donating. - std::unique_lock<Lock> lock(m_shared.m_markingMutex, std::try_to_lock); + std::unique_lock<std::mutex> lock(m_shared.m_markingMutex, std::try_to_lock); if (!lock.owns_lock()) return; @@ -155,7 +126,7 @@ void SlotVisitor::donateKnownParallel() m_stack.donateSomeCellsTo(m_shared.m_sharedMarkStack); if (m_shared.m_numberOfActiveParallelMarkers < Options::numberOfGCMarkers()) - m_shared.m_markingConditionVariable.notifyAll(); + m_shared.m_markingConditionVariable.notify_all(); } void SlotVisitor::drain() @@ -210,12 +181,12 @@ void SlotVisitor::drainFromShared(SharedDrainMode sharedDrainMode) #if ENABLE(PARALLEL_GC) { - std::lock_guard<Lock> lock(m_shared.m_markingMutex); + std::lock_guard<std::mutex> lock(m_shared.m_markingMutex); m_shared.m_numberOfActiveParallelMarkers++; } while (true) { { - std::unique_lock<Lock> lock(m_shared.m_markingMutex); + std::unique_lock<std::mutex> lock(m_shared.m_markingMutex); m_shared.m_numberOfActiveParallelMarkers--; // How we wait differs depending on drain mode. @@ -226,7 +197,7 @@ void SlotVisitor::drainFromShared(SharedDrainMode sharedDrainMode) // Did we reach termination? if (!m_shared.m_numberOfActiveParallelMarkers && m_shared.m_sharedMarkStack.isEmpty()) { // Let any sleeping slaves know it's time for them to return; - m_shared.m_markingConditionVariable.notifyAll(); + m_shared.m_markingConditionVariable.notify_all(); return; } @@ -242,7 +213,7 @@ void SlotVisitor::drainFromShared(SharedDrainMode sharedDrainMode) // Did we detect termination? If so, let the master know. if (!m_shared.m_numberOfActiveParallelMarkers && m_shared.m_sharedMarkStack.isEmpty()) - m_shared.m_markingConditionVariable.notifyAll(); + m_shared.m_markingConditionVariable.notify_all(); m_shared.m_markingConditionVariable.wait(lock, [this] { return !m_shared.m_sharedMarkStack.isEmpty() || m_shared.m_parallelMarkersShouldExit; }); @@ -266,9 +237,11 @@ void SlotVisitor::mergeOpaqueRoots() StackStats::probe(); ASSERT(!m_opaqueRoots.isEmpty()); // Should only be called when opaque roots are non-empty. { - std::lock_guard<Lock> lock(m_shared.m_opaqueRootsMutex); - for (auto* root : m_opaqueRoots) - m_shared.m_opaqueRoots.add(root); + MutexLocker locker(m_shared.m_opaqueRootsLock); + HashSet<void*>::iterator begin = m_opaqueRoots.begin(); + HashSet<void*>::iterator end = m_opaqueRoots.end(); + for (HashSet<void*>::iterator iter = begin; iter != end; ++iter) + m_shared.m_opaqueRoots.add(*iter); } m_opaqueRoots.clear(); } @@ -399,10 +372,4 @@ void SlotVisitor::validate(JSCell*) } #endif -void SlotVisitor::dump(PrintStream&) const -{ - for (const JSCell* cell : markStack()) - dataLog(*cell, "\n"); -} - } // namespace JSC diff --git a/Source/JavaScriptCore/heap/SlotVisitor.h b/Source/JavaScriptCore/heap/SlotVisitor.h index b8295ed2f..4a8dc3e97 100644 --- a/Source/JavaScriptCore/heap/SlotVisitor.h +++ b/Source/JavaScriptCore/heap/SlotVisitor.h @@ -28,10 +28,8 @@ #include "CopyToken.h" #include "HandleTypes.h" -#include "MarkStack.h" -#include "OpaqueRootSet.h" +#include "MarkStackInlines.h" -#include <wtf/HashSet.h> #include <wtf/text/StringHash.h> namespace JSC { @@ -39,14 +37,12 @@ namespace JSC { class ConservativeRoots; class GCThreadSharedData; class Heap; -template<typename T> class JITWriteBarrier; -class UnconditionalFinalizer; template<typename T> class Weak; -class WeakReferenceHarvester; template<typename T> class WriteBarrierBase; +template<typename T> class JITWriteBarrier; class SlotVisitor { - WTF_MAKE_NONCOPYABLE(SlotVisitor); WTF_MAKE_FAST_ALLOCATED; + WTF_MAKE_NONCOPYABLE(SlotVisitor); friend class HeapRootVisitor; // Allowed to mark a JSValue* or JSCell** directly. public: @@ -54,10 +50,7 @@ public: ~SlotVisitor(); MarkStackArray& markStack() { return m_stack; } - const MarkStackArray& markStack() const { return m_stack; } - VM& vm(); - const VM& vm() const; Heap* heap() const; void append(ConservativeRoots&); @@ -72,20 +65,17 @@ public: void appendUnbarrieredValue(JSValue*); template<typename T> void appendUnbarrieredWeak(Weak<T>*); - template<typename T> - void appendUnbarrieredReadOnlyPointer(T*); - void appendUnbarrieredReadOnlyValue(JSValue); void unconditionallyAppend(JSCell*); void addOpaqueRoot(void*); - bool containsOpaqueRoot(void*) const; - TriState containsOpaqueRootTriState(void*) const; + bool containsOpaqueRoot(void*); + TriState containsOpaqueRootTriState(void*); int opaqueRootCount(); GCThreadSharedData& sharedData() const { return m_shared; } bool isEmpty() { return m_stack.isEmpty(); } - void didStartMarking(); + void setup(); void reset(); void clearMarkStack(); @@ -105,16 +95,16 @@ public: void copyLater(JSCell*, CopyToken, void*, size_t); - void reportExtraMemoryVisited(JSCell* owner, size_t); + void reportExtraMemoryUsage(JSCell* owner, size_t); void addWeakReferenceHarvester(WeakReferenceHarvester*); void addUnconditionalFinalizer(UnconditionalFinalizer*); +#if ENABLE(OBJECT_MARK_LOGGING) inline void resetChildCount() { m_logChildCount = 0; } inline unsigned childCount() { return m_logChildCount; } inline void incrementChildCount() { m_logChildCount++; } - - void dump(PrintStream&) const; +#endif private: friend class ParallelModeEnabler; @@ -136,7 +126,7 @@ private: void donateKnownParallel(); MarkStackArray m_stack; - OpaqueRootSet m_opaqueRoots; // Handle-owning data structures not visible to the garbage collector. + HashSet<void*> m_opaqueRoots; // Handle-owning data structures not visible to the garbage collector. size_t m_bytesVisited; size_t m_bytesCopied; @@ -149,7 +139,9 @@ private: typedef HashMap<StringImpl*, JSValue> UniqueStringMap; UniqueStringMap m_uniqueStrings; +#if ENABLE(OBJECT_MARK_LOGGING) unsigned m_logChildCount; +#endif public: #if !ASSERT_DISABLED diff --git a/Source/JavaScriptCore/heap/SlotVisitorInlines.h b/Source/JavaScriptCore/heap/SlotVisitorInlines.h index 3338b1c3d..ccd2e4ae1 100644 --- a/Source/JavaScriptCore/heap/SlotVisitorInlines.h +++ b/Source/JavaScriptCore/heap/SlotVisitorInlines.h @@ -51,12 +51,6 @@ inline void SlotVisitor::appendUnbarrieredPointer(T** slot) internalAppend(slot, cell); } -template<typename T> -inline void SlotVisitor::appendUnbarrieredReadOnlyPointer(T* cell) -{ - internalAppend(0, cell); -} - ALWAYS_INLINE void SlotVisitor::append(JSValue* slot) { ASSERT(slot); @@ -69,11 +63,6 @@ ALWAYS_INLINE void SlotVisitor::appendUnbarrieredValue(JSValue* slot) internalAppend(slot, *slot); } -ALWAYS_INLINE void SlotVisitor::appendUnbarrieredReadOnlyValue(JSValue value) -{ - internalAppend(0, value); -} - ALWAYS_INLINE void SlotVisitor::append(JSCell** slot) { ASSERT(slot); @@ -108,14 +97,13 @@ ALWAYS_INLINE void SlotVisitor::internalAppend(void* from, JSCell* cell) #if ENABLE(GC_VALIDATION) validate(cell); #endif - if (Heap::testAndSetMarked(cell) || !cell->structure()) { - ASSERT(cell->structure()); + if (Heap::testAndSetMarked(cell) || !cell->structure()) return; - } - cell->setMarked(); m_bytesVisited += MarkedBlock::blockFor(cell)->cellSize(); + MARK_LOG_CHILD(*this, cell); + unconditionallyAppend(cell); } @@ -173,7 +161,7 @@ inline void SlotVisitor::addOpaqueRoot(void* root) #endif } -inline bool SlotVisitor::containsOpaqueRoot(void* root) const +inline bool SlotVisitor::containsOpaqueRoot(void* root) { ASSERT(!m_isInParallelMode); #if ENABLE(PARALLEL_GC) @@ -184,11 +172,11 @@ inline bool SlotVisitor::containsOpaqueRoot(void* root) const #endif } -inline TriState SlotVisitor::containsOpaqueRootTriState(void* root) const +inline TriState SlotVisitor::containsOpaqueRootTriState(void* root) { if (m_opaqueRoots.contains(root)) return TrueTriState; - std::lock_guard<Lock> lock(m_shared.m_opaqueRootsMutex); + MutexLocker locker(m_shared.m_opaqueRootsLock); if (m_shared.m_opaqueRoots.contains(root)) return TrueTriState; return MixedTriState; @@ -239,27 +227,38 @@ inline void SlotVisitor::copyLater(JSCell* owner, CopyToken token, void* ptr, si ASSERT(bytes); CopiedBlock* block = CopiedSpace::blockFor(ptr); if (block->isOversize()) { - ASSERT(bytes <= block->size()); - // FIXME: We should be able to shrink the allocation if bytes went below the block size. - // For now, we just make sure that our accounting of how much memory we are actually using - // is correct. - // https://bugs.webkit.org/show_bug.cgi?id=144749 - bytes = block->size(); m_shared.m_copiedSpace->pin(block); + return; } - ASSERT(heap()->m_storageSpace.contains(block)); - - LockHolder locker(&block->workListLock()); + SpinLockHolder locker(&block->workListLock()); if (heap()->operationInProgress() == FullCollection || block->shouldReportLiveBytes(locker, owner)) { m_bytesCopied += bytes; block->reportLiveBytes(locker, owner, token, bytes); } } -inline void SlotVisitor::reportExtraMemoryVisited(JSCell* owner, size_t size) +inline void SlotVisitor::reportExtraMemoryUsage(JSCell* owner, size_t size) { - heap()->reportExtraMemoryVisited(owner, size); +#if ENABLE(GGC) + // We don't want to double-count the extra memory that was reported in previous collections. + if (heap()->operationInProgress() == EdenCollection && MarkedBlock::blockFor(owner)->isRemembered(owner)) + return; +#else + UNUSED_PARAM(owner); +#endif + + size_t* counter = &m_shared.m_vm->heap.m_extraMemoryUsage; + +#if ENABLE(COMPARE_AND_SWAP) + for (;;) { + size_t oldSize = *counter; + if (WTF::weakCompareAndSwapSize(counter, oldSize, oldSize + size)) + return; + } +#else + (*counter) += size; +#endif } inline Heap* SlotVisitor::heap() const @@ -267,16 +266,6 @@ inline Heap* SlotVisitor::heap() const return &sharedData().m_vm->heap; } -inline VM& SlotVisitor::vm() -{ - return *sharedData().m_vm; -} - -inline const VM& SlotVisitor::vm() const -{ - return *sharedData().m_vm; -} - } // namespace JSC #endif // SlotVisitorInlines_h diff --git a/Source/JavaScriptCore/heap/Strong.h b/Source/JavaScriptCore/heap/Strong.h index 5c0f83267..27ab5d31f 100644 --- a/Source/JavaScriptCore/heap/Strong.h +++ b/Source/JavaScriptCore/heap/Strong.h @@ -84,7 +84,9 @@ public: bool operator!() const { return !slot() || !*slot(); } - explicit operator bool() const { return !!*this; } + // This conversion operator allows implicit conversion to bool but not to other integer types. + typedef JSValue (HandleBase::*UnspecifiedBoolType); + operator UnspecifiedBoolType*() const { return !!*this ? reinterpret_cast<UnspecifiedBoolType*>(1) : 0; } void swap(Strong& other) { diff --git a/Source/JavaScriptCore/heap/SuperRegion.cpp b/Source/JavaScriptCore/heap/SuperRegion.cpp new file mode 100644 index 000000000..d58f600b5 --- /dev/null +++ b/Source/JavaScriptCore/heap/SuperRegion.cpp @@ -0,0 +1,82 @@ +/* + * Copyright (C) 2013 Apple Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "SuperRegion.h" + +#include "Region.h" + +namespace JSC { + +const uint64_t SuperRegion::s_fixedHeapMemoryPoolSize = 4 * 1024 * static_cast<uint64_t>(MB); + +SuperRegion::SuperRegion() + : MetaAllocator(Region::s_regionSize, Region::s_regionSize) + , m_reservationBase(0) +{ +#if ENABLE(SUPER_REGION) + // Over-allocate so that we can make sure that we're aligned to the size of Regions. + m_reservation = PageReservation::reserve(s_fixedHeapMemoryPoolSize + Region::s_regionSize, OSAllocator::JSGCHeapPages); + m_reservationBase = getAlignedBase(m_reservation); + addFreshFreeSpace(m_reservationBase, s_fixedHeapMemoryPoolSize); +#else + UNUSED_PARAM(m_reservation); + UNUSED_PARAM(m_reservationBase); +#endif +} + +SuperRegion::~SuperRegion() +{ +#if ENABLE(SUPER_REGION) + m_reservation.deallocate(); +#endif +} + +void* SuperRegion::getAlignedBase(PageReservation& reservation) +{ + for (char* current = static_cast<char*>(reservation.base()); current < static_cast<char*>(reservation.base()) + Region::s_regionSize; current += pageSize()) { + if (!(reinterpret_cast<size_t>(current) & ~Region::s_regionMask)) + return current; + } + ASSERT_NOT_REACHED(); + return 0; +} + +void* SuperRegion::allocateNewSpace(size_t&) +{ + return 0; +} + +void SuperRegion::notifyNeedPage(void* page) +{ + m_reservation.commit(page, Region::s_regionSize); +} + +void SuperRegion::notifyPageIsFree(void* page) +{ + m_reservation.decommit(page, Region::s_regionSize); +} + +} // namespace JSC diff --git a/Source/JavaScriptCore/heap/GCLogging.h b/Source/JavaScriptCore/heap/SuperRegion.h index 650d3fc04..e21526b7a 100644 --- a/Source/JavaScriptCore/heap/GCLogging.h +++ b/Source/JavaScriptCore/heap/SuperRegion.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2014, 2015 Apple Inc. All rights reserved. + * Copyright (C) 2013 Apple Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -23,37 +23,36 @@ * THE POSSIBILITY OF SUCH DAMAGE. */ -#ifndef GCLogging_h -#define GCLogging_h +#ifndef SuperRegion_h +#define SuperRegion_h -#include <wtf/Assertions.h> +#include <wtf/MetaAllocator.h> +#include <wtf/PageBlock.h> +#include <wtf/PageReservation.h> namespace JSC { -class Heap; +class VM; -class GCLogging { +class SuperRegion : public WTF::MetaAllocator { public: - enum Level : uint8_t { - None = 0, - Basic, - Verbose - }; - - static const char* levelAsString(Level); - static void dumpObjectGraph(Heap*); -}; - -typedef GCLogging::Level gcLogLevel; + SuperRegion(); + virtual ~SuperRegion(); -} // namespace JSC +protected: + virtual void* allocateNewSpace(size_t&) override; + virtual void notifyNeedPage(void*) override; + virtual void notifyPageIsFree(void*) override; -namespace WTF { +private: + static const uint64_t s_fixedHeapMemoryPoolSize; -class PrintStream; + static void* getAlignedBase(PageReservation&); -void printInternal(PrintStream&, JSC::GCLogging::Level); + PageReservation m_reservation; + void* m_reservationBase; +}; -} // namespace WTF +} // namespace JSC -#endif // GCLogging_h +#endif // SuperRegion_h diff --git a/Source/JavaScriptCore/heap/UnconditionalFinalizer.h b/Source/JavaScriptCore/heap/UnconditionalFinalizer.h index 1cd7c7bfb..26029d046 100644 --- a/Source/JavaScriptCore/heap/UnconditionalFinalizer.h +++ b/Source/JavaScriptCore/heap/UnconditionalFinalizer.h @@ -35,8 +35,7 @@ namespace JSC { // associated with each CodeBlock. class UnconditionalFinalizer : public ListableHandler<UnconditionalFinalizer> { - WTF_MAKE_FAST_ALLOCATED; -public: +public: virtual void finalizeUnconditionally() = 0; protected: diff --git a/Source/JavaScriptCore/heap/Weak.cpp b/Source/JavaScriptCore/heap/Weak.cpp index a30b7c085..3857b60d2 100644 --- a/Source/JavaScriptCore/heap/Weak.cpp +++ b/Source/JavaScriptCore/heap/Weak.cpp @@ -26,7 +26,6 @@ #include "config.h" #include "Weak.h" -#include "JSCInlines.h" #include "WeakSetInlines.h" namespace JSC { diff --git a/Source/JavaScriptCore/heap/Weak.h b/Source/JavaScriptCore/heap/Weak.h index 2d82f67ae..80cdbd82c 100644 --- a/Source/JavaScriptCore/heap/Weak.h +++ b/Source/JavaScriptCore/heap/Weak.h @@ -26,7 +26,6 @@ #ifndef Weak_h #define Weak_h -#include "JSExportMacros.h" #include <cstddef> #include <wtf/Noncopyable.h> @@ -75,7 +74,9 @@ public: bool was(T*) const; - explicit operator bool() const; + // This conversion operator allows implicit conversion to bool but not to other integer types. + typedef void* (Weak::*UnspecifiedBoolType); + operator UnspecifiedBoolType*() const; WeakImpl* leakImpl() WARN_UNUSED_RETURN; void clear() diff --git a/Source/JavaScriptCore/heap/WeakBlock.cpp b/Source/JavaScriptCore/heap/WeakBlock.cpp index 7c7d86c59..957090569 100644 --- a/Source/JavaScriptCore/heap/WeakBlock.cpp +++ b/Source/JavaScriptCore/heap/WeakBlock.cpp @@ -29,25 +29,19 @@ #include "Heap.h" #include "HeapRootVisitor.h" #include "JSObject.h" -#include "JSCInlines.h" +#include "Operations.h" #include "Structure.h" namespace JSC { -WeakBlock* WeakBlock::create(MarkedBlock& markedBlock) +WeakBlock* WeakBlock::create(DeadBlock* block) { - return new (NotNull, fastMalloc(blockSize)) WeakBlock(markedBlock); + Region* region = block->region(); + return new (NotNull, block) WeakBlock(region); } -void WeakBlock::destroy(WeakBlock* block) -{ - block->~WeakBlock(); - fastFree(block); -} - -WeakBlock::WeakBlock(MarkedBlock& markedBlock) - : DoublyLinkedListNode<WeakBlock>() - , m_markedBlock(&markedBlock) +WeakBlock::WeakBlock(Region* region) + : HeapBlock<WeakBlock>(region) { for (size_t i = 0; i < weakImplCount(); ++i) { WeakImpl* weakImpl = &weakImpls()[i]; @@ -82,11 +76,8 @@ void WeakBlock::sweep() finalize(weakImpl); if (weakImpl->state() == WeakImpl::Deallocated) addToFreeList(&sweepResult.freeList, weakImpl); - else { + else sweepResult.blockIsFree = false; - if (weakImpl->state() == WeakImpl::Live) - sweepResult.blockIsLogicallyEmpty = false; - } } m_sweepResult = sweepResult; @@ -99,12 +90,6 @@ void WeakBlock::visit(HeapRootVisitor& heapRootVisitor) if (isEmpty()) return; - // If this WeakBlock doesn't belong to a MarkedBlock, we won't even be here. - ASSERT(m_markedBlock); - - if (m_markedBlock->isAllocated()) - return; - SlotVisitor& visitor = heapRootVisitor.visitor(); for (size_t i = 0; i < weakImplCount(); ++i) { @@ -113,7 +98,7 @@ void WeakBlock::visit(HeapRootVisitor& heapRootVisitor) continue; const JSValue& jsValue = weakImpl->jsValue(); - if (m_markedBlock->isMarkedOrNewlyAllocated(jsValue.asCell())) + if (Heap::isLive(jsValue.asCell())) continue; WeakHandleOwner* weakHandleOwner = weakImpl->weakHandleOwner(); @@ -133,18 +118,12 @@ void WeakBlock::reap() if (isEmpty()) return; - // If this WeakBlock doesn't belong to a MarkedBlock, we won't even be here. - ASSERT(m_markedBlock); - - if (m_markedBlock->isAllocated()) - return; - for (size_t i = 0; i < weakImplCount(); ++i) { WeakImpl* weakImpl = &weakImpls()[i]; if (weakImpl->state() > WeakImpl::Dead) continue; - if (m_markedBlock->isMarkedOrNewlyAllocated(weakImpl->jsValue().asCell())) { + if (Heap::isLive(weakImpl->jsValue().asCell())) { ASSERT(weakImpl->state() == WeakImpl::Live); continue; } diff --git a/Source/JavaScriptCore/heap/WeakBlock.h b/Source/JavaScriptCore/heap/WeakBlock.h index b829e9dbe..b6b631e27 100644 --- a/Source/JavaScriptCore/heap/WeakBlock.h +++ b/Source/JavaScriptCore/heap/WeakBlock.h @@ -26,7 +26,7 @@ #ifndef WeakBlock_h #define WeakBlock_h -#include <wtf/DoublyLinkedList.h> +#include "HeapBlock.h" #include "WeakHandleOwner.h" #include "WeakImpl.h" #include <wtf/DoublyLinkedList.h> @@ -34,35 +34,33 @@ namespace JSC { +class DeadBlock; class HeapRootVisitor; class JSValue; -class MarkedBlock; class WeakHandleOwner; -class WeakBlock : public DoublyLinkedListNode<WeakBlock> { +class WeakBlock : public HeapBlock<WeakBlock> { public: friend class WTF::DoublyLinkedListNode<WeakBlock>; - static const size_t blockSize = 1 * KB; // 1/16 of MarkedBlock size + static const size_t blockSize = 4 * KB; // 5% of MarkedBlock size struct FreeCell { FreeCell* next; }; struct SweepResult { + SweepResult(); bool isNull() const; - bool blockIsFree { true }; - bool blockIsLogicallyEmpty { true }; - FreeCell* freeList { nullptr }; + bool blockIsFree; + FreeCell* freeList; }; - static WeakBlock* create(MarkedBlock&); - static void destroy(WeakBlock*); + static WeakBlock* create(DeadBlock*); static WeakImpl* asWeakImpl(FreeCell*); bool isEmpty(); - bool isLogicallyEmptyButNotFree() const; void sweep(); SweepResult takeSweepResult(); @@ -71,24 +69,27 @@ public: void reap(); void lastChanceToFinalize(); - void disconnectMarkedBlock() { m_markedBlock = nullptr; } private: static FreeCell* asFreeCell(WeakImpl*); - explicit WeakBlock(MarkedBlock&); + WeakBlock(Region*); WeakImpl* firstWeakImpl(); void finalize(WeakImpl*); WeakImpl* weakImpls(); size_t weakImplCount(); void addToFreeList(FreeCell**, WeakImpl*); - MarkedBlock* m_markedBlock; - WeakBlock* m_prev; - WeakBlock* m_next; SweepResult m_sweepResult; }; +inline WeakBlock::SweepResult::SweepResult() + : blockIsFree(true) + , freeList(0) +{ + ASSERT(isNull()); +} + inline bool WeakBlock::SweepResult::isNull() const { return blockIsFree && !freeList; // This state is impossible, so we can use it to mean null. @@ -137,11 +138,6 @@ inline bool WeakBlock::isEmpty() return !m_sweepResult.isNull() && m_sweepResult.blockIsFree; } -inline bool WeakBlock::isLogicallyEmptyButNotFree() const -{ - return !m_sweepResult.isNull() && !m_sweepResult.blockIsFree && m_sweepResult.blockIsLogicallyEmpty; -} - } // namespace JSC #endif // WeakBlock_h diff --git a/Source/JavaScriptCore/heap/WeakHandleOwner.cpp b/Source/JavaScriptCore/heap/WeakHandleOwner.cpp index 044518f7a..67e1774df 100644 --- a/Source/JavaScriptCore/heap/WeakHandleOwner.cpp +++ b/Source/JavaScriptCore/heap/WeakHandleOwner.cpp @@ -26,8 +26,6 @@ #include "config.h" #include "WeakHandleOwner.h" -#include "JSCInlines.h" - namespace JSC { class SlotVisitor; diff --git a/Source/JavaScriptCore/heap/WeakInlines.h b/Source/JavaScriptCore/heap/WeakInlines.h index fbd352013..8cfd50153 100644 --- a/Source/JavaScriptCore/heap/WeakInlines.h +++ b/Source/JavaScriptCore/heap/WeakInlines.h @@ -65,7 +65,7 @@ template<typename T> inline void Weak<T>::swap(Weak& other) template<typename T> inline auto Weak<T>::operator=(Weak&& other) -> Weak& { - Weak weak = WTF::move(other); + Weak weak = std::move(other); swap(weak); return *this; } @@ -99,9 +99,9 @@ template<typename T> inline bool Weak<T>::operator!() const return !m_impl || !m_impl->jsValue() || m_impl->state() != WeakImpl::Live; } -template<typename T> inline Weak<T>::operator bool() const +template<typename T> inline Weak<T>::operator UnspecifiedBoolType*() const { - return !!*this; + return reinterpret_cast<UnspecifiedBoolType*>(!!*this); } template<typename T> inline WeakImpl* Weak<T>::leakImpl() diff --git a/Source/JavaScriptCore/heap/WeakSet.cpp b/Source/JavaScriptCore/heap/WeakSet.cpp index 1eed8c37a..e62e66eae 100644 --- a/Source/JavaScriptCore/heap/WeakSet.cpp +++ b/Source/JavaScriptCore/heap/WeakSet.cpp @@ -27,7 +27,6 @@ #include "WeakSet.h" #include "Heap.h" -#include "JSCInlines.h" #include "VM.h" namespace JSC { @@ -37,26 +36,15 @@ WeakSet::~WeakSet() WeakBlock* next = 0; for (WeakBlock* block = m_blocks.head(); block; block = next) { next = block->next(); - WeakBlock::destroy(block); + heap()->blockAllocator().deallocate(WeakBlock::destroy(block)); } m_blocks.clear(); } void WeakSet::sweep() { - for (WeakBlock* block = m_blocks.head(); block;) { - WeakBlock* nextBlock = block->next(); + for (WeakBlock* block = m_blocks.head(); block; block = block->next()) block->sweep(); - if (block->isLogicallyEmptyButNotFree()) { - // If this WeakBlock is logically empty, but still has Weaks pointing into it, - // we can't destroy it just yet. Detach it from the WeakSet and hand ownership - // to the Heap so we don't pin down the entire 64kB MarkedBlock. - m_blocks.remove(block); - heap()->addLogicallyEmptyWeakBlock(block); - block->disconnectMarkedBlock(); - } - block = nextBlock; - } resetAllocator(); } @@ -85,7 +73,7 @@ WeakBlock::FreeCell* WeakSet::tryFindAllocator() WeakBlock::FreeCell* WeakSet::addAllocator() { - WeakBlock* block = WeakBlock::create(m_markedBlock); + WeakBlock* block = WeakBlock::create(heap()->blockAllocator().allocate<WeakBlock>()); heap()->didAllocate(WeakBlock::blockSize); m_blocks.append(block); WeakBlock::SweepResult sweepResult = block->takeSweepResult(); @@ -96,7 +84,7 @@ WeakBlock::FreeCell* WeakSet::addAllocator() void WeakSet::removeAllocator(WeakBlock* block) { m_blocks.remove(block); - WeakBlock::destroy(block); + heap()->blockAllocator().deallocate(WeakBlock::destroy(block)); } } // namespace JSC diff --git a/Source/JavaScriptCore/heap/WeakSet.h b/Source/JavaScriptCore/heap/WeakSet.h index dbde5108b..a5ddcaffa 100644 --- a/Source/JavaScriptCore/heap/WeakSet.h +++ b/Source/JavaScriptCore/heap/WeakSet.h @@ -31,7 +31,6 @@ namespace JSC { class Heap; -class MarkedBlock; class WeakImpl; class WeakSet { @@ -41,7 +40,7 @@ public: static WeakImpl* allocate(JSValue, WeakHandleOwner* = 0, void* context = 0); static void deallocate(WeakImpl*); - WeakSet(VM*, MarkedBlock&); + WeakSet(VM*); ~WeakSet(); void lastChanceToFinalize(); @@ -66,14 +65,12 @@ private: WeakBlock* m_nextAllocator; DoublyLinkedList<WeakBlock> m_blocks; VM* m_vm; - MarkedBlock& m_markedBlock; }; -inline WeakSet::WeakSet(VM* vm, MarkedBlock& markedBlock) +inline WeakSet::WeakSet(VM* vm) : m_allocator(0) , m_nextAllocator(0) , m_vm(vm) - , m_markedBlock(markedBlock) { } diff --git a/Source/JavaScriptCore/heap/WriteBarrierBuffer.cpp b/Source/JavaScriptCore/heap/WriteBarrierBuffer.cpp index 10b7430ec..5314b5112 100644 --- a/Source/JavaScriptCore/heap/WriteBarrierBuffer.cpp +++ b/Source/JavaScriptCore/heap/WriteBarrierBuffer.cpp @@ -29,7 +29,6 @@ #include "GCAssertions.h" #include "Heap.h" #include "JSCell.h" -#include "JSCInlines.h" #include "Structure.h" namespace JSC { @@ -44,6 +43,7 @@ WriteBarrierBuffer::WriteBarrierBuffer(unsigned capacity) WriteBarrierBuffer::~WriteBarrierBuffer() { fastFree(m_buffer); + m_buffer = 0; } void WriteBarrierBuffer::flush(Heap& heap) diff --git a/Source/JavaScriptCore/heap/WriteBarrierBuffer.h b/Source/JavaScriptCore/heap/WriteBarrierBuffer.h index 7359083cd..9126bdbe9 100644 --- a/Source/JavaScriptCore/heap/WriteBarrierBuffer.h +++ b/Source/JavaScriptCore/heap/WriteBarrierBuffer.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013, 2015 Apple Inc. All rights reserved. + * Copyright (C) 2013 Apple Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -33,7 +33,10 @@ namespace JSC { class Heap; class JSCell; +namespace FTL { class LowerDFGToLLVM; } + class WriteBarrierBuffer { + friend class FTL::LowerDFGToLLVM; public: WriteBarrierBuffer(unsigned capacity); ~WriteBarrierBuffer(); @@ -42,25 +45,25 @@ public: void flush(Heap&); void reset(); - unsigned* currentIndexAddress() + static ptrdiff_t currentIndexOffset() { - return &m_currentIndex; + return OBJECT_OFFSETOF(WriteBarrierBuffer, m_currentIndex); } - unsigned capacity() const + static ptrdiff_t capacityOffset() { - return m_capacity; + return OBJECT_OFFSETOF(WriteBarrierBuffer, m_capacity); } - JSCell** buffer() + static ptrdiff_t bufferOffset() { - return m_buffer; + return OBJECT_OFFSETOF(WriteBarrierBuffer, m_buffer); } private: unsigned m_currentIndex; - const unsigned m_capacity; - JSCell** const m_buffer; + unsigned m_capacity; + JSCell** m_buffer; }; } // namespace JSC diff --git a/Source/JavaScriptCore/heap/WriteBarrierSupport.cpp b/Source/JavaScriptCore/heap/WriteBarrierSupport.cpp index 984f0044b..5ca33c861 100644 --- a/Source/JavaScriptCore/heap/WriteBarrierSupport.cpp +++ b/Source/JavaScriptCore/heap/WriteBarrierSupport.cpp @@ -26,8 +26,6 @@ #include "config.h" #include "WriteBarrierSupport.h" -#include "JSCInlines.h" - namespace JSC { #if ENABLE(WRITE_BARRIER_PROFILING) |