diff options
Diffstat (limited to 'Source/JavaScriptCore/dfg/DFGSpeculativeJIT.cpp')
| -rw-r--r-- | Source/JavaScriptCore/dfg/DFGSpeculativeJIT.cpp | 3199 |
1 files changed, 2056 insertions, 1143 deletions
diff --git a/Source/JavaScriptCore/dfg/DFGSpeculativeJIT.cpp b/Source/JavaScriptCore/dfg/DFGSpeculativeJIT.cpp index 4f2889b8f..325a876a2 100644 --- a/Source/JavaScriptCore/dfg/DFGSpeculativeJIT.cpp +++ b/Source/JavaScriptCore/dfg/DFGSpeculativeJIT.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2011 Apple Inc. All rights reserved. + * Copyright (C) 2011, 2012, 2013 Apple Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -29,8 +29,10 @@ #if ENABLE(DFG_JIT) #include "Arguments.h" +#include "DFGArrayifySlowPathGenerator.h" #include "DFGCallArrayAllocatorSlowPathGenerator.h" #include "DFGSlowPathGenerator.h" +#include "JSCJSValueInlines.h" #include "LinkBuffer.h" namespace JSC { namespace DFG { @@ -38,7 +40,7 @@ namespace JSC { namespace DFG { SpeculativeJIT::SpeculativeJIT(JITCompiler& jit) : m_compileOkay(true) , m_jit(jit) - , m_compileIndex(0) + , m_currentNode(0) , m_indexInBlock(0) , m_generationInfo(m_jit.codeBlock()->m_numCalleeRegisters) , m_blockHeads(jit.graph().m_blocks.size()) @@ -54,15 +56,16 @@ SpeculativeJIT::SpeculativeJIT(JITCompiler& jit) SpeculativeJIT::~SpeculativeJIT() { - WTF::deleteAllValues(m_slowPathGenerators); } -void SpeculativeJIT::emitAllocateJSArray(Structure* structure, GPRReg resultGPR, GPRReg storageGPR, unsigned numElements) +void SpeculativeJIT::emitAllocateJSArray(GPRReg resultGPR, Structure* structure, GPRReg storageGPR, unsigned numElements) { ASSERT(hasUndecided(structure->indexingType()) || hasInt32(structure->indexingType()) || hasDouble(structure->indexingType()) || hasContiguous(structure->indexingType())); GPRTemporary scratch(this); + GPRTemporary scratch2(this); GPRReg scratchGPR = scratch.gpr(); + GPRReg scratch2GPR = scratch2.gpr(); unsigned vectorLength = std::max(BASE_VECTOR_LEN, numElements); @@ -71,12 +74,8 @@ void SpeculativeJIT::emitAllocateJSArray(Structure* structure, GPRReg resultGPR, slowCases.append( emitAllocateBasicStorage(TrustedImm32(vectorLength * sizeof(JSValue) + sizeof(IndexingHeader)), storageGPR)); m_jit.subPtr(TrustedImm32(vectorLength * sizeof(JSValue)), storageGPR); - emitAllocateBasicJSObject<JSArray, MarkedBlock::None>( - TrustedImmPtr(structure), resultGPR, scratchGPR, - storageGPR, sizeof(JSArray), slowCases); + emitAllocateJSObject<JSArray>(resultGPR, TrustedImmPtr(structure), storageGPR, scratchGPR, scratch2GPR, slowCases); - // I'm assuming that two 32-bit stores are better than a 64-bit store. - // I have no idea if that's true. And it probably doesn't matter anyway. m_jit.store32(TrustedImm32(numElements), MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength())); m_jit.store32(TrustedImm32(vectorLength), MacroAssembler::Address(storageGPR, Butterfly::offsetOfVectorLength())); @@ -104,74 +103,122 @@ void SpeculativeJIT::emitAllocateJSArray(Structure* structure, GPRReg resultGPR, structure, numElements))); } -void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, NodeIndex nodeIndex, MacroAssembler::Jump jumpToFail) +void SpeculativeJIT::backwardSpeculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node, MacroAssembler::Jump jumpToFail) { if (!m_compileOkay) return; - ASSERT(at(m_compileIndex).canExit() || m_isCheckingArgumentTypes); - m_jit.codeBlock()->appendOSRExit(OSRExit(kind, jsValueSource, m_jit.graph().methodOfGettingAValueProfileFor(nodeIndex), jumpToFail, this, m_stream->size())); + ASSERT(m_isCheckingArgumentTypes || m_canExit); + m_jit.appendExitInfo(jumpToFail); + m_jit.codeBlock()->appendOSRExit(OSRExit(kind, jsValueSource, m_jit.graph().methodOfGettingAValueProfileFor(node), this, m_stream->size())); +} + +void SpeculativeJIT::backwardSpeculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node, const MacroAssembler::JumpList& jumpsToFail) +{ + if (!m_compileOkay) + return; + ASSERT(m_isCheckingArgumentTypes || m_canExit); + m_jit.appendExitInfo(jumpsToFail); + m_jit.codeBlock()->appendOSRExit(OSRExit(kind, jsValueSource, m_jit.graph().methodOfGettingAValueProfileFor(node), this, m_stream->size())); +} + +void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node, MacroAssembler::Jump jumpToFail) +{ + if (!m_compileOkay) + return; + backwardSpeculationCheck(kind, jsValueSource, node, jumpToFail); + if (m_speculationDirection == ForwardSpeculation) + convertLastOSRExitToForward(); } void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Edge nodeUse, MacroAssembler::Jump jumpToFail) { - ASSERT(at(m_compileIndex).canExit() || m_isCheckingArgumentTypes); - speculationCheck(kind, jsValueSource, nodeUse.index(), jumpToFail); + ASSERT(m_isCheckingArgumentTypes || m_canExit); + speculationCheck(kind, jsValueSource, nodeUse.node(), jumpToFail); } -void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, NodeIndex nodeIndex, const MacroAssembler::JumpList& jumpsToFail) +OSRExitJumpPlaceholder SpeculativeJIT::backwardSpeculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node) { - ASSERT(at(m_compileIndex).canExit() || m_isCheckingArgumentTypes); - Vector<MacroAssembler::Jump, 16> jumpVector = jumpsToFail.jumps(); - for (unsigned i = 0; i < jumpVector.size(); ++i) - speculationCheck(kind, jsValueSource, nodeIndex, jumpVector[i]); + if (!m_compileOkay) + return OSRExitJumpPlaceholder(); + ASSERT(m_isCheckingArgumentTypes || m_canExit); + unsigned index = m_jit.codeBlock()->numberOfOSRExits(); + m_jit.appendExitInfo(); + m_jit.codeBlock()->appendOSRExit(OSRExit(kind, jsValueSource, m_jit.graph().methodOfGettingAValueProfileFor(node), this, m_stream->size())); + return OSRExitJumpPlaceholder(index); +} + +OSRExitJumpPlaceholder SpeculativeJIT::backwardSpeculationCheck(ExitKind kind, JSValueSource jsValueSource, Edge nodeUse) +{ + ASSERT(m_isCheckingArgumentTypes || m_canExit); + return backwardSpeculationCheck(kind, jsValueSource, nodeUse.node()); +} + +void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node, const MacroAssembler::JumpList& jumpsToFail) +{ + if (!m_compileOkay) + return; + backwardSpeculationCheck(kind, jsValueSource, node, jumpsToFail); + if (m_speculationDirection == ForwardSpeculation) + convertLastOSRExitToForward(); } void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Edge nodeUse, const MacroAssembler::JumpList& jumpsToFail) { - ASSERT(at(m_compileIndex).canExit() || m_isCheckingArgumentTypes); - speculationCheck(kind, jsValueSource, nodeUse.index(), jumpsToFail); + ASSERT(m_isCheckingArgumentTypes || m_canExit); + speculationCheck(kind, jsValueSource, nodeUse.node(), jumpsToFail); } -void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, NodeIndex nodeIndex, MacroAssembler::Jump jumpToFail, const SpeculationRecovery& recovery) +void SpeculativeJIT::backwardSpeculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node, MacroAssembler::Jump jumpToFail, const SpeculationRecovery& recovery) { if (!m_compileOkay) return; - ASSERT(at(m_compileIndex).canExit() || m_isCheckingArgumentTypes); + ASSERT(m_isCheckingArgumentTypes || m_canExit); m_jit.codeBlock()->appendSpeculationRecovery(recovery); - m_jit.codeBlock()->appendOSRExit(OSRExit(kind, jsValueSource, m_jit.graph().methodOfGettingAValueProfileFor(nodeIndex), jumpToFail, this, m_stream->size(), m_jit.codeBlock()->numberOfSpeculationRecoveries())); + m_jit.appendExitInfo(jumpToFail); + m_jit.codeBlock()->appendOSRExit(OSRExit(kind, jsValueSource, m_jit.graph().methodOfGettingAValueProfileFor(node), this, m_stream->size(), m_jit.codeBlock()->numberOfSpeculationRecoveries())); } -void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Edge nodeUse, MacroAssembler::Jump jumpToFail, const SpeculationRecovery& recovery) +void SpeculativeJIT::backwardSpeculationCheck(ExitKind kind, JSValueSource jsValueSource, Edge nodeUse, MacroAssembler::Jump jumpToFail, const SpeculationRecovery& recovery) { - ASSERT(at(m_compileIndex).canExit() || m_isCheckingArgumentTypes); - speculationCheck(kind, jsValueSource, nodeUse.index(), jumpToFail, recovery); + ASSERT(m_isCheckingArgumentTypes || m_canExit); + backwardSpeculationCheck(kind, jsValueSource, nodeUse.node(), jumpToFail, recovery); } -void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, NodeIndex nodeIndex, MacroAssembler::Jump jumpToFail, const SpeculationRecovery& recovery, SpeculationDirection direction) +void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node, MacroAssembler::Jump jumpToFail, const SpeculationRecovery& recovery) { - speculationCheck(kind, jsValueSource, nodeIndex, jumpToFail, recovery); - if (direction == ForwardSpeculation) + if (!m_compileOkay) + return; + backwardSpeculationCheck(kind, jsValueSource, node, jumpToFail, recovery); + if (m_speculationDirection == ForwardSpeculation) convertLastOSRExitToForward(); } -JumpReplacementWatchpoint* SpeculativeJIT::speculationWatchpoint(ExitKind kind, JSValueSource jsValueSource, NodeIndex nodeIndex) +void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Edge edge, MacroAssembler::Jump jumpToFail, const SpeculationRecovery& recovery) +{ + speculationCheck(kind, jsValueSource, edge.node(), jumpToFail, recovery); +} + +JumpReplacementWatchpoint* SpeculativeJIT::speculationWatchpoint(ExitKind kind, JSValueSource jsValueSource, Node* node) { if (!m_compileOkay) return 0; - ASSERT(at(m_compileIndex).canExit() || m_isCheckingArgumentTypes); + ASSERT(m_isCheckingArgumentTypes || m_canExit); + m_jit.appendExitInfo(JITCompiler::JumpList()); OSRExit& exit = m_jit.codeBlock()->osrExit( - m_jit.codeBlock()->appendOSRExit( - OSRExit(kind, jsValueSource, - m_jit.graph().methodOfGettingAValueProfileFor(nodeIndex), - JITCompiler::Jump(), this, m_stream->size()))); + m_jit.codeBlock()->appendOSRExit(OSRExit( + kind, jsValueSource, + m_jit.graph().methodOfGettingAValueProfileFor(node), + this, m_stream->size()))); exit.m_watchpointIndex = m_jit.codeBlock()->appendWatchpoint( JumpReplacementWatchpoint(m_jit.watchpointLabel())); + if (m_speculationDirection == ForwardSpeculation) + convertLastOSRExitToForward(); return &m_jit.codeBlock()->watchpoint(exit.m_watchpointIndex); } JumpReplacementWatchpoint* SpeculativeJIT::speculationWatchpoint(ExitKind kind) { - return speculationWatchpoint(kind, JSValueSource(), NoNode); + return speculationWatchpoint(kind, JSValueSource(), 0); } void SpeculativeJIT::convertLastOSRExitToForward(const ValueRecovery& valueRecovery) @@ -179,10 +226,10 @@ void SpeculativeJIT::convertLastOSRExitToForward(const ValueRecovery& valueRecov if (!valueRecovery) { // Check that either the current node is a SetLocal, or the preceding node was a // SetLocal with the same code origin. - if (at(m_compileIndex).op() != SetLocal) { - Node* setLocal = &at(m_jit.graph().m_blocks[m_block]->at(m_indexInBlock - 1)); - ASSERT_UNUSED(setLocal, setLocal->op() == SetLocal); - ASSERT_UNUSED(setLocal, setLocal->codeOrigin == at(m_compileIndex).codeOrigin); + if (!m_currentNode->containsMovHint()) { + Node* setLocal = m_jit.graph().m_blocks[m_block]->at(m_indexInBlock - 1); + ASSERT_UNUSED(setLocal, setLocal->containsMovHint()); + ASSERT_UNUSED(setLocal, setLocal->codeOrigin == m_currentNode->codeOrigin); } // Find the next node. @@ -196,13 +243,13 @@ void SpeculativeJIT::convertLastOSRExitToForward(const ValueRecovery& valueRecov ASSERT(node->op() == Jump); return; } - node = &at(m_jit.graph().m_blocks[m_block]->at(indexInBlock)); - if (node->codeOrigin != at(m_compileIndex).codeOrigin) + node = m_jit.graph().m_blocks[m_block]->at(indexInBlock); + if (node->codeOrigin != m_currentNode->codeOrigin) break; indexInBlock++; } - ASSERT(node->codeOrigin != at(m_compileIndex).codeOrigin); + ASSERT(node->codeOrigin != m_currentNode->codeOrigin); OSRExit& exit = m_jit.codeBlock()->lastOSRExit(); exit.m_codeOrigin = node->codeOrigin; return; @@ -210,29 +257,29 @@ void SpeculativeJIT::convertLastOSRExitToForward(const ValueRecovery& valueRecov unsigned setLocalIndexInBlock = m_indexInBlock + 1; - Node* setLocal = &at(m_jit.graph().m_blocks[m_block]->at(setLocalIndexInBlock)); + Node* setLocal = m_jit.graph().m_blocks[m_block]->at(setLocalIndexInBlock); bool hadInt32ToDouble = false; - if (setLocal->op() == Int32ToDouble) { - setLocal = &at(m_jit.graph().m_blocks[m_block]->at(++setLocalIndexInBlock)); + if (setLocal->op() == ForwardInt32ToDouble) { + setLocal = m_jit.graph().m_blocks[m_block]->at(++setLocalIndexInBlock); hadInt32ToDouble = true; } if (setLocal->op() == Flush || setLocal->op() == Phantom) - setLocal = &at(m_jit.graph().m_blocks[m_block]->at(++setLocalIndexInBlock)); + setLocal = m_jit.graph().m_blocks[m_block]->at(++setLocalIndexInBlock); if (hadInt32ToDouble) - ASSERT(at(setLocal->child1()).child1() == m_compileIndex); + ASSERT(setLocal->child1()->child1() == m_currentNode); else - ASSERT(setLocal->child1() == m_compileIndex); - ASSERT(setLocal->op() == SetLocal); - ASSERT(setLocal->codeOrigin == at(m_compileIndex).codeOrigin); + ASSERT(setLocal->child1() == m_currentNode); + ASSERT(setLocal->containsMovHint()); + ASSERT(setLocal->codeOrigin == m_currentNode->codeOrigin); - Node* nextNode = &at(m_jit.graph().m_blocks[m_block]->at(setLocalIndexInBlock + 1)); - if (nextNode->op() == Jump && nextNode->codeOrigin == at(m_compileIndex).codeOrigin) { + Node* nextNode = m_jit.graph().m_blocks[m_block]->at(setLocalIndexInBlock + 1); + if (nextNode->op() == Jump && nextNode->codeOrigin == m_currentNode->codeOrigin) { // We're at an inlined return. Use a backward speculation instead. return; } - ASSERT(nextNode->codeOrigin != at(m_compileIndex).codeOrigin); + ASSERT(nextNode->codeOrigin != m_currentNode->codeOrigin); OSRExit& exit = m_jit.codeBlock()->lastOSRExit(); exit.m_codeOrigin = nextNode->codeOrigin; @@ -242,77 +289,61 @@ void SpeculativeJIT::convertLastOSRExitToForward(const ValueRecovery& valueRecov new ValueRecoveryOverride(setLocal->local(), valueRecovery)); } -JumpReplacementWatchpoint* SpeculativeJIT::forwardSpeculationWatchpoint(ExitKind kind) -{ - JumpReplacementWatchpoint* result = speculationWatchpoint(kind); - convertLastOSRExitToForward(); - return result; -} - -JumpReplacementWatchpoint* SpeculativeJIT::speculationWatchpoint(ExitKind kind, SpeculationDirection direction) +void SpeculativeJIT::forwardSpeculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node, MacroAssembler::Jump jumpToFail, const ValueRecovery& valueRecovery) { - JumpReplacementWatchpoint* result = speculationWatchpoint(kind); - if (direction == ForwardSpeculation) - convertLastOSRExitToForward(); - return result; -} - -void SpeculativeJIT::forwardSpeculationCheck(ExitKind kind, JSValueSource jsValueSource, NodeIndex nodeIndex, MacroAssembler::Jump jumpToFail, const ValueRecovery& valueRecovery) -{ - ASSERT(at(m_compileIndex).canExit() || m_isCheckingArgumentTypes); - speculationCheck(kind, jsValueSource, nodeIndex, jumpToFail); + ASSERT(m_isCheckingArgumentTypes || m_canExit); + backwardSpeculationCheck(kind, jsValueSource, node, jumpToFail); convertLastOSRExitToForward(valueRecovery); } -void SpeculativeJIT::forwardSpeculationCheck(ExitKind kind, JSValueSource jsValueSource, NodeIndex nodeIndex, const MacroAssembler::JumpList& jumpsToFail, const ValueRecovery& valueRecovery) +void SpeculativeJIT::forwardSpeculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node, const MacroAssembler::JumpList& jumpsToFail, const ValueRecovery& valueRecovery) { - ASSERT(at(m_compileIndex).canExit() || m_isCheckingArgumentTypes); - Vector<MacroAssembler::Jump, 16> jumpVector = jumpsToFail.jumps(); - for (unsigned i = 0; i < jumpVector.size(); ++i) - forwardSpeculationCheck(kind, jsValueSource, nodeIndex, jumpVector[i], valueRecovery); -} - -void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, NodeIndex nodeIndex, MacroAssembler::Jump jumpToFail, SpeculationDirection direction) -{ - if (direction == ForwardSpeculation) - forwardSpeculationCheck(kind, jsValueSource, nodeIndex, jumpToFail); - else - speculationCheck(kind, jsValueSource, nodeIndex, jumpToFail); + ASSERT(m_isCheckingArgumentTypes || m_canExit); + backwardSpeculationCheck(kind, jsValueSource, node, jumpsToFail); + convertLastOSRExitToForward(valueRecovery); } -void SpeculativeJIT::terminateSpeculativeExecution(ExitKind kind, JSValueRegs jsValueRegs, NodeIndex nodeIndex) +void SpeculativeJIT::terminateSpeculativeExecution(ExitKind kind, JSValueRegs jsValueRegs, Node* node) { - ASSERT(at(m_compileIndex).canExit() || m_isCheckingArgumentTypes); + ASSERT(m_isCheckingArgumentTypes || m_canExit); #if DFG_ENABLE(DEBUG_VERBOSE) dataLogF("SpeculativeJIT was terminated.\n"); #endif if (!m_compileOkay) return; - speculationCheck(kind, jsValueRegs, nodeIndex, m_jit.jump()); + speculationCheck(kind, jsValueRegs, node, m_jit.jump()); m_compileOkay = false; } void SpeculativeJIT::terminateSpeculativeExecution(ExitKind kind, JSValueRegs jsValueRegs, Edge nodeUse) { - ASSERT(at(m_compileIndex).canExit() || m_isCheckingArgumentTypes); - terminateSpeculativeExecution(kind, jsValueRegs, nodeUse.index()); + ASSERT(m_isCheckingArgumentTypes || m_canExit); + terminateSpeculativeExecution(kind, jsValueRegs, nodeUse.node()); } -void SpeculativeJIT::terminateSpeculativeExecution(ExitKind kind, JSValueRegs jsValueRegs, NodeIndex nodeIndex, SpeculationDirection direction) +void SpeculativeJIT::backwardTypeCheck(JSValueSource source, Edge edge, SpeculatedType typesPassedThrough, MacroAssembler::Jump jumpToFail) { - ASSERT(at(m_compileIndex).canExit() || m_isCheckingArgumentTypes); -#if DFG_ENABLE(DEBUG_VERBOSE) - dataLogF("SpeculativeJIT was terminated.\n"); -#endif - if (!m_compileOkay) - return; - speculationCheck(kind, jsValueRegs, nodeIndex, m_jit.jump(), direction); - m_compileOkay = false; + ASSERT(needsTypeCheck(edge, typesPassedThrough)); + m_state.forNode(edge).filter(typesPassedThrough); + backwardSpeculationCheck(BadType, source, edge.node(), jumpToFail); +} + +void SpeculativeJIT::typeCheck(JSValueSource source, Edge edge, SpeculatedType typesPassedThrough, MacroAssembler::Jump jumpToFail) +{ + backwardTypeCheck(source, edge, typesPassedThrough, jumpToFail); + if (m_speculationDirection == ForwardSpeculation) + convertLastOSRExitToForward(); +} + +void SpeculativeJIT::forwardTypeCheck(JSValueSource source, Edge edge, SpeculatedType typesPassedThrough, MacroAssembler::Jump jumpToFail, const ValueRecovery& valueRecovery) +{ + backwardTypeCheck(source, edge, typesPassedThrough, jumpToFail); + convertLastOSRExitToForward(valueRecovery); } void SpeculativeJIT::addSlowPathGenerator(PassOwnPtr<SlowPathGenerator> slowPathGenerator) { - m_slowPathGenerators.append(slowPathGenerator.leakPtr()); + m_slowPathGenerators.append(slowPathGenerator); } void SpeculativeJIT::runSlowPathGenerators() @@ -343,33 +374,301 @@ void SpeculativeJIT::clearGenerationInfo() m_fprs = RegisterBank<FPRInfo>(); } +SilentRegisterSavePlan SpeculativeJIT::silentSavePlanForGPR(VirtualRegister spillMe, GPRReg source) +{ + GenerationInfo& info = m_generationInfo[spillMe]; + Node* node = info.node(); + DataFormat registerFormat = info.registerFormat(); + ASSERT(registerFormat != DataFormatNone); + ASSERT(registerFormat != DataFormatDouble); + + SilentSpillAction spillAction; + SilentFillAction fillAction; + + if (!info.needsSpill()) + spillAction = DoNothingForSpill; + else { +#if USE(JSVALUE64) + ASSERT(info.gpr() == source); + if (registerFormat == DataFormatInteger) + spillAction = Store32Payload; + else if (registerFormat == DataFormatCell || registerFormat == DataFormatStorage) + spillAction = StorePtr; + else { + ASSERT(registerFormat & DataFormatJS); + spillAction = Store64; + } +#elif USE(JSVALUE32_64) + if (registerFormat & DataFormatJS) { + ASSERT(info.tagGPR() == source || info.payloadGPR() == source); + spillAction = source == info.tagGPR() ? Store32Tag : Store32Payload; + } else { + ASSERT(info.gpr() == source); + spillAction = Store32Payload; + } +#endif + } + + if (registerFormat == DataFormatInteger) { + ASSERT(info.gpr() == source); + ASSERT(isJSInteger(info.registerFormat())); + if (node->hasConstant()) { + ASSERT(isInt32Constant(node)); + fillAction = SetInt32Constant; + } else + fillAction = Load32Payload; + } else if (registerFormat == DataFormatBoolean) { +#if USE(JSVALUE64) + RELEASE_ASSERT_NOT_REACHED(); + fillAction = DoNothingForFill; +#elif USE(JSVALUE32_64) + ASSERT(info.gpr() == source); + if (node->hasConstant()) { + ASSERT(isBooleanConstant(node)); + fillAction = SetBooleanConstant; + } else + fillAction = Load32Payload; +#endif + } else if (registerFormat == DataFormatCell) { + ASSERT(info.gpr() == source); + if (node->hasConstant()) { + JSValue value = valueOfJSConstant(node); + ASSERT_UNUSED(value, value.isCell()); + fillAction = SetCellConstant; + } else { +#if USE(JSVALUE64) + fillAction = LoadPtr; +#else + fillAction = Load32Payload; +#endif + } + } else if (registerFormat == DataFormatStorage) { + ASSERT(info.gpr() == source); + fillAction = LoadPtr; + } else { + ASSERT(registerFormat & DataFormatJS); +#if USE(JSVALUE64) + ASSERT(info.gpr() == source); + if (node->hasConstant()) { + if (valueOfJSConstant(node).isCell()) + fillAction = SetTrustedJSConstant; + else + fillAction = SetJSConstant; + } else if (info.spillFormat() == DataFormatInteger) { + ASSERT(registerFormat == DataFormatJSInteger); + fillAction = Load32PayloadBoxInt; + } else if (info.spillFormat() == DataFormatDouble) { + ASSERT(registerFormat == DataFormatJSDouble); + fillAction = LoadDoubleBoxDouble; + } else + fillAction = Load64; +#else + ASSERT(info.tagGPR() == source || info.payloadGPR() == source); + if (node->hasConstant()) + fillAction = info.tagGPR() == source ? SetJSConstantTag : SetJSConstantPayload; + else if (info.payloadGPR() == source) + fillAction = Load32Payload; + else { // Fill the Tag + switch (info.spillFormat()) { + case DataFormatInteger: + ASSERT(registerFormat == DataFormatJSInteger); + fillAction = SetInt32Tag; + break; + case DataFormatCell: + ASSERT(registerFormat == DataFormatJSCell); + fillAction = SetCellTag; + break; + case DataFormatBoolean: + ASSERT(registerFormat == DataFormatJSBoolean); + fillAction = SetBooleanTag; + break; + default: + fillAction = Load32Tag; + break; + } + } +#endif + } + + return SilentRegisterSavePlan(spillAction, fillAction, node, source); +} + +SilentRegisterSavePlan SpeculativeJIT::silentSavePlanForFPR(VirtualRegister spillMe, FPRReg source) +{ + GenerationInfo& info = m_generationInfo[spillMe]; + Node* node = info.node(); + ASSERT(info.registerFormat() == DataFormatDouble); + + SilentSpillAction spillAction; + SilentFillAction fillAction; + + if (!info.needsSpill()) + spillAction = DoNothingForSpill; + else { + ASSERT(!node->hasConstant()); + ASSERT(info.spillFormat() == DataFormatNone); + ASSERT(info.fpr() == source); + spillAction = StoreDouble; + } + +#if USE(JSVALUE64) + if (node->hasConstant()) { + ASSERT(isNumberConstant(node)); + fillAction = SetDoubleConstant; + } else if (info.spillFormat() != DataFormatNone && info.spillFormat() != DataFormatDouble) { + // it was already spilled previously and not as a double, which means we need unboxing. + ASSERT(info.spillFormat() & DataFormatJS); + fillAction = LoadJSUnboxDouble; + } else + fillAction = LoadDouble; +#elif USE(JSVALUE32_64) + ASSERT(info.registerFormat() == DataFormatDouble || info.registerFormat() == DataFormatJSDouble); + if (node->hasConstant()) { + ASSERT(isNumberConstant(node)); + fillAction = SetDoubleConstant; + } else + fillAction = LoadDouble; +#endif + + return SilentRegisterSavePlan(spillAction, fillAction, node, source); +} + +void SpeculativeJIT::silentSpill(const SilentRegisterSavePlan& plan) +{ + switch (plan.spillAction()) { + case DoNothingForSpill: + break; + case Store32Tag: + m_jit.store32(plan.gpr(), JITCompiler::tagFor(plan.node()->virtualRegister())); + break; + case Store32Payload: + m_jit.store32(plan.gpr(), JITCompiler::payloadFor(plan.node()->virtualRegister())); + break; + case StorePtr: + m_jit.storePtr(plan.gpr(), JITCompiler::addressFor(plan.node()->virtualRegister())); + break; +#if USE(JSVALUE64) + case Store64: + m_jit.store64(plan.gpr(), JITCompiler::addressFor(plan.node()->virtualRegister())); + break; +#endif + case StoreDouble: + m_jit.storeDouble(plan.fpr(), JITCompiler::addressFor(plan.node()->virtualRegister())); + break; + default: + RELEASE_ASSERT_NOT_REACHED(); + } +} + +void SpeculativeJIT::silentFill(const SilentRegisterSavePlan& plan, GPRReg canTrample) +{ +#if USE(JSVALUE32_64) + UNUSED_PARAM(canTrample); +#endif + switch (plan.fillAction()) { + case DoNothingForFill: + break; + case SetInt32Constant: + m_jit.move(Imm32(valueOfInt32Constant(plan.node())), plan.gpr()); + break; + case SetBooleanConstant: + m_jit.move(TrustedImm32(valueOfBooleanConstant(plan.node())), plan.gpr()); + break; + case SetCellConstant: + m_jit.move(TrustedImmPtr(valueOfJSConstant(plan.node()).asCell()), plan.gpr()); + break; +#if USE(JSVALUE64) + case SetTrustedJSConstant: + m_jit.move(valueOfJSConstantAsImm64(plan.node()).asTrustedImm64(), plan.gpr()); + break; + case SetJSConstant: + m_jit.move(valueOfJSConstantAsImm64(plan.node()), plan.gpr()); + break; + case SetDoubleConstant: + m_jit.move(Imm64(reinterpretDoubleToInt64(valueOfNumberConstant(plan.node()))), canTrample); + m_jit.move64ToDouble(canTrample, plan.fpr()); + break; + case Load32PayloadBoxInt: + m_jit.load32(JITCompiler::payloadFor(plan.node()->virtualRegister()), plan.gpr()); + m_jit.or64(GPRInfo::tagTypeNumberRegister, plan.gpr()); + break; + case LoadDoubleBoxDouble: + m_jit.load64(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.gpr()); + m_jit.sub64(GPRInfo::tagTypeNumberRegister, plan.gpr()); + break; + case LoadJSUnboxDouble: + m_jit.load64(JITCompiler::addressFor(plan.node()->virtualRegister()), canTrample); + unboxDouble(canTrample, plan.fpr()); + break; +#else + case SetJSConstantTag: + m_jit.move(Imm32(valueOfJSConstant(plan.node()).tag()), plan.gpr()); + break; + case SetJSConstantPayload: + m_jit.move(Imm32(valueOfJSConstant(plan.node()).payload()), plan.gpr()); + break; + case SetInt32Tag: + m_jit.move(TrustedImm32(JSValue::Int32Tag), plan.gpr()); + break; + case SetCellTag: + m_jit.move(TrustedImm32(JSValue::CellTag), plan.gpr()); + break; + case SetBooleanTag: + m_jit.move(TrustedImm32(JSValue::BooleanTag), plan.gpr()); + break; + case SetDoubleConstant: + m_jit.loadDouble(addressOfDoubleConstant(plan.node()), plan.fpr()); + break; +#endif + case Load32Tag: + m_jit.load32(JITCompiler::tagFor(plan.node()->virtualRegister()), plan.gpr()); + break; + case Load32Payload: + m_jit.load32(JITCompiler::payloadFor(plan.node()->virtualRegister()), plan.gpr()); + break; + case LoadPtr: + m_jit.loadPtr(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.gpr()); + break; +#if USE(JSVALUE64) + case Load64: + m_jit.load64(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.gpr()); + break; +#endif + case LoadDouble: + m_jit.loadDouble(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.fpr()); + break; + default: + RELEASE_ASSERT_NOT_REACHED(); + } +} + const TypedArrayDescriptor* SpeculativeJIT::typedArrayDescriptor(ArrayMode arrayMode) { switch (arrayMode.type()) { case Array::Int8Array: - return &m_jit.globalData()->int8ArrayDescriptor(); + return &m_jit.vm()->int8ArrayDescriptor(); case Array::Int16Array: - return &m_jit.globalData()->int16ArrayDescriptor(); + return &m_jit.vm()->int16ArrayDescriptor(); case Array::Int32Array: - return &m_jit.globalData()->int32ArrayDescriptor(); + return &m_jit.vm()->int32ArrayDescriptor(); case Array::Uint8Array: - return &m_jit.globalData()->uint8ArrayDescriptor(); + return &m_jit.vm()->uint8ArrayDescriptor(); case Array::Uint8ClampedArray: - return &m_jit.globalData()->uint8ClampedArrayDescriptor(); + return &m_jit.vm()->uint8ClampedArrayDescriptor(); case Array::Uint16Array: - return &m_jit.globalData()->uint16ArrayDescriptor(); + return &m_jit.vm()->uint16ArrayDescriptor(); case Array::Uint32Array: - return &m_jit.globalData()->uint32ArrayDescriptor(); + return &m_jit.vm()->uint32ArrayDescriptor(); case Array::Float32Array: - return &m_jit.globalData()->float32ArrayDescriptor(); + return &m_jit.vm()->float32ArrayDescriptor(); case Array::Float64Array: - return &m_jit.globalData()->float64ArrayDescriptor(); + return &m_jit.vm()->float64ArrayDescriptor(); default: return 0; } } -JITCompiler::Jump SpeculativeJIT::jumpSlowForUnwantedArrayMode(GPRReg tempGPR, ArrayMode arrayMode, IndexingType shape, bool invert) +JITCompiler::Jump SpeculativeJIT::jumpSlowForUnwantedArrayMode(GPRReg tempGPR, ArrayMode arrayMode, IndexingType shape) { switch (arrayMode.arrayClass()) { case Array::OriginalArray: { @@ -381,27 +680,27 @@ JITCompiler::Jump SpeculativeJIT::jumpSlowForUnwantedArrayMode(GPRReg tempGPR, A case Array::Array: m_jit.and32(TrustedImm32(IsArray | IndexingShapeMask), tempGPR); return m_jit.branch32( - invert ? MacroAssembler::Equal : MacroAssembler::NotEqual, tempGPR, TrustedImm32(IsArray | shape)); + MacroAssembler::NotEqual, tempGPR, TrustedImm32(IsArray | shape)); default: m_jit.and32(TrustedImm32(IndexingShapeMask), tempGPR); - return m_jit.branch32(invert ? MacroAssembler::Equal : MacroAssembler::NotEqual, tempGPR, TrustedImm32(shape)); + return m_jit.branch32(MacroAssembler::NotEqual, tempGPR, TrustedImm32(shape)); } } -JITCompiler::JumpList SpeculativeJIT::jumpSlowForUnwantedArrayMode(GPRReg tempGPR, ArrayMode arrayMode, bool invert) +JITCompiler::JumpList SpeculativeJIT::jumpSlowForUnwantedArrayMode(GPRReg tempGPR, ArrayMode arrayMode) { JITCompiler::JumpList result; switch (arrayMode.type()) { case Array::Int32: - return jumpSlowForUnwantedArrayMode(tempGPR, arrayMode, Int32Shape, invert); + return jumpSlowForUnwantedArrayMode(tempGPR, arrayMode, Int32Shape); case Array::Double: - return jumpSlowForUnwantedArrayMode(tempGPR, arrayMode, DoubleShape, invert); + return jumpSlowForUnwantedArrayMode(tempGPR, arrayMode, DoubleShape); case Array::Contiguous: - return jumpSlowForUnwantedArrayMode(tempGPR, arrayMode, ContiguousShape, invert); + return jumpSlowForUnwantedArrayMode(tempGPR, arrayMode, ContiguousShape); case Array::ArrayStorage: case Array::SlowPutArrayStorage: { @@ -409,19 +708,6 @@ JITCompiler::JumpList SpeculativeJIT::jumpSlowForUnwantedArrayMode(GPRReg tempGP if (arrayMode.isJSArray()) { if (arrayMode.isSlowPut()) { - if (invert) { - JITCompiler::Jump slow = m_jit.branchTest32( - MacroAssembler::Zero, tempGPR, MacroAssembler::TrustedImm32(IsArray)); - m_jit.and32(TrustedImm32(IndexingShapeMask), tempGPR); - m_jit.sub32(TrustedImm32(ArrayStorageShape), tempGPR); - result.append( - m_jit.branch32( - MacroAssembler::BelowOrEqual, tempGPR, - TrustedImm32(SlowPutArrayStorageShape - ArrayStorageShape))); - - slow.link(&m_jit); - } - result.append( m_jit.branchTest32( MacroAssembler::Zero, tempGPR, MacroAssembler::TrustedImm32(IsArray))); @@ -435,7 +721,7 @@ JITCompiler::JumpList SpeculativeJIT::jumpSlowForUnwantedArrayMode(GPRReg tempGP } m_jit.and32(TrustedImm32(IsArray | IndexingShapeMask), tempGPR); result.append( - m_jit.branch32(invert ? MacroAssembler::Equal : MacroAssembler::NotEqual, tempGPR, TrustedImm32(ArrayStorageShape))); + m_jit.branch32(MacroAssembler::NotEqual, tempGPR, TrustedImm32(IsArray | ArrayStorageShape))); break; } m_jit.and32(TrustedImm32(IndexingShapeMask), tempGPR); @@ -443,12 +729,12 @@ JITCompiler::JumpList SpeculativeJIT::jumpSlowForUnwantedArrayMode(GPRReg tempGP m_jit.sub32(TrustedImm32(ArrayStorageShape), tempGPR); result.append( m_jit.branch32( - invert ? MacroAssembler::BelowOrEqual : MacroAssembler::Above, tempGPR, + MacroAssembler::Above, tempGPR, TrustedImm32(SlowPutArrayStorageShape - ArrayStorageShape))); break; } result.append( - m_jit.branch32(invert ? MacroAssembler::Equal : MacroAssembler::NotEqual, tempGPR, TrustedImm32(ArrayStorageShape))); + m_jit.branch32(MacroAssembler::NotEqual, tempGPR, TrustedImm32(ArrayStorageShape))); break; } default: @@ -459,24 +745,24 @@ JITCompiler::JumpList SpeculativeJIT::jumpSlowForUnwantedArrayMode(GPRReg tempGP return result; } -void SpeculativeJIT::checkArray(Node& node) +void SpeculativeJIT::checkArray(Node* node) { - ASSERT(node.arrayMode().isSpecific()); - ASSERT(!node.arrayMode().doesConversion()); + ASSERT(node->arrayMode().isSpecific()); + ASSERT(!node->arrayMode().doesConversion()); - SpeculateCellOperand base(this, node.child1()); + SpeculateCellOperand base(this, node->child1()); GPRReg baseReg = base.gpr(); - const TypedArrayDescriptor* result = typedArrayDescriptor(node.arrayMode()); + const TypedArrayDescriptor* result = typedArrayDescriptor(node->arrayMode()); - if (node.arrayMode().alreadyChecked(m_jit.graph(), node, m_state.forNode(node.child1()))) { - noResult(m_compileIndex); + if (node->arrayMode().alreadyChecked(m_jit.graph(), node, m_state.forNode(node->child1()))) { + noResult(m_currentNode); return; } const ClassInfo* expectedClassInfo = 0; - switch (node.arrayMode().type()) { + switch (node->arrayMode().type()) { case Array::String: expectedClassInfo = &JSString::s_info; break; @@ -491,10 +777,10 @@ void SpeculativeJIT::checkArray(Node& node) MacroAssembler::Address(baseReg, JSCell::structureOffset()), tempGPR); m_jit.load8(MacroAssembler::Address(tempGPR, Structure::indexingTypeOffset()), tempGPR); speculationCheck( - BadIndexingType, JSValueSource::unboxedCell(baseReg), NoNode, - jumpSlowForUnwantedArrayMode(tempGPR, node.arrayMode())); + BadIndexingType, JSValueSource::unboxedCell(baseReg), 0, + jumpSlowForUnwantedArrayMode(tempGPR, node->arrayMode())); - noResult(m_compileIndex); + noResult(m_currentNode); return; } case Array::Arguments: @@ -512,7 +798,7 @@ void SpeculativeJIT::checkArray(Node& node) expectedClassInfo = result->m_classInfo; break; default: - ASSERT_NOT_REACHED(); + RELEASE_ASSERT_NOT_REACHED(); break; } @@ -520,38 +806,38 @@ void SpeculativeJIT::checkArray(Node& node) m_jit.loadPtr( MacroAssembler::Address(baseReg, JSCell::structureOffset()), temp.gpr()); speculationCheck( - Uncountable, JSValueRegs(), NoNode, + Uncountable, JSValueRegs(), 0, m_jit.branchPtr( MacroAssembler::NotEqual, MacroAssembler::Address(temp.gpr(), Structure::classInfoOffset()), MacroAssembler::TrustedImmPtr(expectedClassInfo))); - noResult(m_compileIndex); + noResult(m_currentNode); } -void SpeculativeJIT::arrayify(Node& node, GPRReg baseReg, GPRReg propertyReg) +void SpeculativeJIT::arrayify(Node* node, GPRReg baseReg, GPRReg propertyReg) { - ASSERT(node.arrayMode().doesConversion()); + ASSERT(node->arrayMode().doesConversion()); GPRTemporary temp(this); GPRTemporary structure; GPRReg tempGPR = temp.gpr(); GPRReg structureGPR = InvalidGPRReg; - if (node.op() != ArrayifyToStructure) { + if (node->op() != ArrayifyToStructure) { GPRTemporary realStructure(this); structure.adopt(realStructure); structureGPR = structure.gpr(); } // We can skip all that comes next if we already have array storage. - MacroAssembler::JumpList done; + MacroAssembler::JumpList slowPath; - if (node.op() == ArrayifyToStructure) { - done.append(m_jit.branchWeakPtr( - JITCompiler::Equal, + if (node->op() == ArrayifyToStructure) { + slowPath.append(m_jit.branchWeakPtr( + JITCompiler::NotEqual, JITCompiler::Address(baseReg, JSCell::structureOffset()), - node.structure())); + node->structure())); } else { m_jit.loadPtr( MacroAssembler::Address(baseReg, JSCell::structureOffset()), structureGPR); @@ -559,106 +845,34 @@ void SpeculativeJIT::arrayify(Node& node, GPRReg baseReg, GPRReg propertyReg) m_jit.load8( MacroAssembler::Address(structureGPR, Structure::indexingTypeOffset()), tempGPR); - done = jumpSlowForUnwantedArrayMode(tempGPR, node.arrayMode(), true); - - // Next check that the object does not intercept indexed accesses. If it does, - // then this mode won't work. - speculationCheck( - BadIndexingType, JSValueSource::unboxedCell(baseReg), NoNode, - m_jit.branchTest8( - MacroAssembler::NonZero, - MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()), - MacroAssembler::TrustedImm32(InterceptsGetOwnPropertySlotByIndexEvenWhenLengthIsNotZero))); - } - - // If we're allegedly creating contiguous storage and the index is bogus, then - // just don't. - if (propertyReg != InvalidGPRReg) { - switch (node.arrayMode().type()) { - case Array::Int32: - case Array::Double: - case Array::Contiguous: - speculationCheck( - Uncountable, JSValueRegs(), NoNode, - m_jit.branch32( - MacroAssembler::AboveOrEqual, propertyReg, TrustedImm32(MIN_SPARSE_ARRAY_INDEX))); - break; - default: - break; - } + slowPath.append(jumpSlowForUnwantedArrayMode(tempGPR, node->arrayMode())); } - // Now call out to create the array storage. - silentSpillAllRegisters(tempGPR); - switch (node.arrayMode().type()) { - case Array::Int32: - callOperation(operationEnsureInt32, tempGPR, baseReg); - break; - case Array::Double: - callOperation(operationEnsureDouble, tempGPR, baseReg); - break; - case Array::Contiguous: - callOperation(operationEnsureContiguous, tempGPR, baseReg); - break; - case Array::ArrayStorage: - case Array::SlowPutArrayStorage: - callOperation(operationEnsureArrayStorage, tempGPR, baseReg); - break; - default: - CRASH(); - break; - } - silentFillAllRegisters(tempGPR); - - if (node.op() == ArrayifyToStructure) { - speculationCheck( - BadIndexingType, JSValueSource::unboxedCell(baseReg), NoNode, - m_jit.branchWeakPtr( - JITCompiler::NotEqual, - JITCompiler::Address(baseReg, JSCell::structureOffset()), - node.structure())); - } else { - // Alas, we need to reload the structure because silent spilling does not save - // temporaries. Nor would it be useful for it to do so. Either way we're talking - // about a load. - m_jit.loadPtr( - MacroAssembler::Address(baseReg, JSCell::structureOffset()), structureGPR); - - // Finally, check that we have the kind of array storage that we wanted to get. - // Note that this is a backwards speculation check, which will result in the - // bytecode operation corresponding to this arrayification being reexecuted. - // That's fine, since arrayification is not user-visible. - m_jit.load8( - MacroAssembler::Address(structureGPR, Structure::indexingTypeOffset()), structureGPR); - speculationCheck( - BadIndexingType, JSValueSource::unboxedCell(baseReg), NoNode, - jumpSlowForUnwantedArrayMode(structureGPR, node.arrayMode())); - } + addSlowPathGenerator(adoptPtr(new ArrayifySlowPathGenerator( + slowPath, this, node, baseReg, propertyReg, tempGPR, structureGPR))); - done.link(&m_jit); - noResult(m_compileIndex); + noResult(m_currentNode); } -void SpeculativeJIT::arrayify(Node& node) +void SpeculativeJIT::arrayify(Node* node) { - ASSERT(node.arrayMode().isSpecific()); + ASSERT(node->arrayMode().isSpecific()); - SpeculateCellOperand base(this, node.child1()); + SpeculateCellOperand base(this, node->child1()); - if (!node.child2()) { + if (!node->child2()) { arrayify(node, base.gpr(), InvalidGPRReg); return; } - SpeculateIntegerOperand property(this, node.child2()); + SpeculateIntegerOperand property(this, node->child2()); arrayify(node, base.gpr(), property.gpr()); } -GPRReg SpeculativeJIT::fillStorage(NodeIndex nodeIndex) +GPRReg SpeculativeJIT::fillStorage(Edge edge) { - Node& node = m_jit.graph()[nodeIndex]; - VirtualRegister virtualRegister = node.virtualRegister(); + VirtualRegister virtualRegister = edge->virtualRegister(); GenerationInfo& info = m_generationInfo[virtualRegister]; switch (info.registerFormat()) { @@ -672,7 +886,7 @@ GPRReg SpeculativeJIT::fillStorage(NodeIndex nodeIndex) } // Must be a cell; fill it as a cell and then return the pointer. - return fillSpeculateCell(nodeIndex, BackwardSpeculation); + return fillSpeculateCell(edge); } case DataFormatStorage: { @@ -682,115 +896,39 @@ GPRReg SpeculativeJIT::fillStorage(NodeIndex nodeIndex) } default: - return fillSpeculateCell(nodeIndex, BackwardSpeculation); + return fillSpeculateCell(edge); } } -void SpeculativeJIT::useChildren(Node& node) +void SpeculativeJIT::useChildren(Node* node) { - if (node.flags() & NodeHasVarArgs) { - for (unsigned childIdx = node.firstChild(); childIdx < node.firstChild() + node.numChildren(); childIdx++) { + if (node->flags() & NodeHasVarArgs) { + for (unsigned childIdx = node->firstChild(); childIdx < node->firstChild() + node->numChildren(); childIdx++) { if (!!m_jit.graph().m_varArgChildren[childIdx]) use(m_jit.graph().m_varArgChildren[childIdx]); } } else { - Edge child1 = node.child1(); + Edge child1 = node->child1(); if (!child1) { - ASSERT(!node.child2() && !node.child3()); + ASSERT(!node->child2() && !node->child3()); return; } use(child1); - Edge child2 = node.child2(); + Edge child2 = node->child2(); if (!child2) { - ASSERT(!node.child3()); + ASSERT(!node->child3()); return; } use(child2); - Edge child3 = node.child3(); + Edge child3 = node->child3(); if (!child3) return; use(child3); } } -bool SpeculativeJIT::isStrictInt32(NodeIndex nodeIndex) -{ - if (isInt32Constant(nodeIndex)) - return true; - - Node& node = m_jit.graph()[nodeIndex]; - GenerationInfo& info = m_generationInfo[node.virtualRegister()]; - - return info.registerFormat() == DataFormatInteger; -} - -bool SpeculativeJIT::isKnownInteger(NodeIndex nodeIndex) -{ - if (isInt32Constant(nodeIndex)) - return true; - - Node& node = m_jit.graph()[nodeIndex]; - - if (node.hasInt32Result()) - return true; - - GenerationInfo& info = m_generationInfo[node.virtualRegister()]; - - return info.isJSInteger(); -} - -bool SpeculativeJIT::isKnownNumeric(NodeIndex nodeIndex) -{ - if (isInt32Constant(nodeIndex) || isNumberConstant(nodeIndex)) - return true; - - Node& node = m_jit.graph()[nodeIndex]; - - if (node.hasNumberResult()) - return true; - - GenerationInfo& info = m_generationInfo[node.virtualRegister()]; - - return info.isJSInteger() || info.isJSDouble(); -} - -bool SpeculativeJIT::isKnownCell(NodeIndex nodeIndex) -{ - return m_generationInfo[m_jit.graph()[nodeIndex].virtualRegister()].isJSCell(); -} - -bool SpeculativeJIT::isKnownNotCell(NodeIndex nodeIndex) -{ - Node& node = m_jit.graph()[nodeIndex]; - VirtualRegister virtualRegister = node.virtualRegister(); - GenerationInfo& info = m_generationInfo[virtualRegister]; - if (node.hasConstant() && !valueOfJSConstant(nodeIndex).isCell()) - return true; - return !(info.isJSCell() || info.isUnknownJS()); -} - -bool SpeculativeJIT::isKnownNotInteger(NodeIndex nodeIndex) -{ - Node& node = m_jit.graph()[nodeIndex]; - VirtualRegister virtualRegister = node.virtualRegister(); - GenerationInfo& info = m_generationInfo[virtualRegister]; - - return info.isJSDouble() || info.isJSCell() || info.isJSBoolean() - || (node.hasConstant() && !valueOfJSConstant(nodeIndex).isInt32()); -} - -bool SpeculativeJIT::isKnownNotNumber(NodeIndex nodeIndex) -{ - Node& node = m_jit.graph()[nodeIndex]; - VirtualRegister virtualRegister = node.virtualRegister(); - GenerationInfo& info = m_generationInfo[virtualRegister]; - - return (!info.isJSDouble() && !info.isJSInteger() && !info.isUnknownJS()) - || (node.hasConstant() && !isNumberConstant(nodeIndex)); -} - void SpeculativeJIT::writeBarrier(MacroAssembler& jit, GPRReg owner, GPRReg scratch1, GPRReg scratch2, WriteBarrierUseKind useKind) { UNUSED_PARAM(jit); @@ -805,30 +943,6 @@ void SpeculativeJIT::writeBarrier(MacroAssembler& jit, GPRReg owner, GPRReg scra #if ENABLE(WRITE_BARRIER_PROFILING) JITCompiler::emitCount(jit, WriteBarrierCounters::jitCounterFor(useKind)); #endif - markCellCard(jit, owner, scratch1, scratch2); -} - -void SpeculativeJIT::markCellCard(MacroAssembler& jit, GPRReg owner, GPRReg scratch1, GPRReg scratch2) -{ - UNUSED_PARAM(jit); - UNUSED_PARAM(owner); - UNUSED_PARAM(scratch1); - UNUSED_PARAM(scratch2); - -#if ENABLE(GGC) - jit.move(owner, scratch1); - jit.andPtr(TrustedImm32(static_cast<int32_t>(MarkedBlock::blockMask)), scratch1); - jit.move(owner, scratch2); - // consume additional 8 bits as we're using an approximate filter - jit.rshift32(TrustedImm32(MarkedBlock::atomShift + 8), scratch2); - jit.andPtr(TrustedImm32(MarkedBlock::atomMask >> 8), scratch2); - MacroAssembler::Jump filter = jit.branchTest8(MacroAssembler::Zero, MacroAssembler::BaseIndex(scratch1, scratch2, MacroAssembler::TimesOne, MarkedBlock::offsetOfMarks())); - jit.move(owner, scratch2); - jit.rshift32(TrustedImm32(MarkedBlock::cardShift), scratch2); - jit.andPtr(TrustedImm32(MarkedBlock::cardMask), scratch2); - jit.store8(TrustedImm32(1), MacroAssembler::BaseIndex(scratch1, scratch2, MacroAssembler::TimesOne, MarkedBlock::offsetOfCards())); - filter.link(&jit); -#endif } void SpeculativeJIT::writeBarrier(GPRReg ownerGPR, GPRReg valueGPR, Edge valueUse, WriteBarrierUseKind useKind, GPRReg scratch1, GPRReg scratch2) @@ -839,39 +953,12 @@ void SpeculativeJIT::writeBarrier(GPRReg ownerGPR, GPRReg valueGPR, Edge valueUs UNUSED_PARAM(scratch2); UNUSED_PARAM(useKind); - if (isKnownNotCell(valueUse.index())) + if (isKnownNotCell(valueUse.node())) return; #if ENABLE(WRITE_BARRIER_PROFILING) JITCompiler::emitCount(m_jit, WriteBarrierCounters::jitCounterFor(useKind)); #endif - -#if ENABLE(GGC) - GPRTemporary temp1; - GPRTemporary temp2; - if (scratch1 == InvalidGPRReg) { - GPRTemporary scratchGPR(this); - temp1.adopt(scratchGPR); - scratch1 = temp1.gpr(); - } - if (scratch2 == InvalidGPRReg) { - GPRTemporary scratchGPR(this); - temp2.adopt(scratchGPR); - scratch2 = temp2.gpr(); - } - - JITCompiler::Jump rhsNotCell; - bool hadCellCheck = false; - if (!isKnownCell(valueUse.index()) && !isCellSpeculation(m_jit.getSpeculation(valueUse.index()))) { - hadCellCheck = true; - rhsNotCell = m_jit.branchIfNotCell(valueGPR); - } - - markCellCard(m_jit, ownerGPR, scratch1, scratch2); - - if (hadCellCheck) - rhsNotCell.link(&m_jit); -#endif } void SpeculativeJIT::writeBarrier(GPRReg ownerGPR, JSCell* value, WriteBarrierUseKind useKind, GPRReg scratch1, GPRReg scratch2) @@ -888,23 +975,6 @@ void SpeculativeJIT::writeBarrier(GPRReg ownerGPR, JSCell* value, WriteBarrierUs #if ENABLE(WRITE_BARRIER_PROFILING) JITCompiler::emitCount(m_jit, WriteBarrierCounters::jitCounterFor(useKind)); #endif - -#if ENABLE(GGC) - GPRTemporary temp1; - GPRTemporary temp2; - if (scratch1 == InvalidGPRReg) { - GPRTemporary scratchGPR(this); - temp1.adopt(scratchGPR); - scratch1 = temp1.gpr(); - } - if (scratch2 == InvalidGPRReg) { - GPRTemporary scratchGPR(this); - temp2.adopt(scratchGPR); - scratch2 = temp2.gpr(); - } - - markCellCard(m_jit, ownerGPR, scratch1, scratch2); -#endif } void SpeculativeJIT::writeBarrier(JSCell* owner, GPRReg valueGPR, Edge valueUse, WriteBarrierUseKind useKind, GPRReg scratch) @@ -914,49 +984,26 @@ void SpeculativeJIT::writeBarrier(JSCell* owner, GPRReg valueGPR, Edge valueUse, UNUSED_PARAM(scratch); UNUSED_PARAM(useKind); - if (isKnownNotCell(valueUse.index())) + if (isKnownNotCell(valueUse.node())) return; #if ENABLE(WRITE_BARRIER_PROFILING) JITCompiler::emitCount(m_jit, WriteBarrierCounters::jitCounterFor(useKind)); #endif - -#if ENABLE(GGC) - JITCompiler::Jump rhsNotCell; - bool hadCellCheck = false; - if (!isKnownCell(valueUse.index()) && !isCellSpeculation(m_jit.getSpeculation(valueUse.index()))) { - hadCellCheck = true; - rhsNotCell = m_jit.branchIfNotCell(valueGPR); - } - - GPRTemporary temp; - if (scratch == InvalidGPRReg) { - GPRTemporary scratchGPR(this); - temp.adopt(scratchGPR); - scratch = temp.gpr(); - } - - uint8_t* cardAddress = Heap::addressOfCardFor(owner); - m_jit.move(JITCompiler::TrustedImmPtr(cardAddress), scratch); - m_jit.store8(JITCompiler::TrustedImm32(1), JITCompiler::Address(scratch)); - - if (hadCellCheck) - rhsNotCell.link(&m_jit); -#endif } -bool SpeculativeJIT::nonSpeculativeCompare(Node& node, MacroAssembler::RelationalCondition cond, S_DFGOperation_EJJ helperFunction) +bool SpeculativeJIT::nonSpeculativeCompare(Node* node, MacroAssembler::RelationalCondition cond, S_DFGOperation_EJJ helperFunction) { unsigned branchIndexInBlock = detectPeepHoleBranch(); if (branchIndexInBlock != UINT_MAX) { - NodeIndex branchNodeIndex = m_jit.graph().m_blocks[m_block]->at(branchIndexInBlock); + Node* branchNode = m_jit.graph().m_blocks[m_block]->at(branchIndexInBlock); - ASSERT(node.adjustedRefCount() == 1); + ASSERT(node->adjustedRefCount() == 1); - nonSpeculativePeepholeBranch(node, branchNodeIndex, cond, helperFunction); + nonSpeculativePeepholeBranch(node, branchNode, cond, helperFunction); m_indexInBlock = branchIndexInBlock; - m_compileIndex = branchNodeIndex; + m_currentNode = branchNode; return true; } @@ -966,18 +1013,18 @@ bool SpeculativeJIT::nonSpeculativeCompare(Node& node, MacroAssembler::Relationa return false; } -bool SpeculativeJIT::nonSpeculativeStrictEq(Node& node, bool invert) +bool SpeculativeJIT::nonSpeculativeStrictEq(Node* node, bool invert) { unsigned branchIndexInBlock = detectPeepHoleBranch(); if (branchIndexInBlock != UINT_MAX) { - NodeIndex branchNodeIndex = m_jit.graph().m_blocks[m_block]->at(branchIndexInBlock); + Node* branchNode = m_jit.graph().m_blocks[m_block]->at(branchIndexInBlock); - ASSERT(node.adjustedRefCount() == 1); + ASSERT(node->adjustedRefCount() == 1); - nonSpeculativePeepholeStrictEq(node, branchNodeIndex, invert); + nonSpeculativePeepholeStrictEq(node, branchNode, invert); m_indexInBlock = branchIndexInBlock; - m_compileIndex = branchNodeIndex; + m_currentNode = branchNode; return true; } @@ -1101,6 +1148,11 @@ void SpeculativeJIT::checkConsistency() } break; } + case DataFormatOSRMarker: + case DataFormatDead: + case DataFormatArguments: + RELEASE_ASSERT_NOT_REACHED(); + break; } } @@ -1173,7 +1225,7 @@ GPRTemporary::GPRTemporary(SpeculativeJIT* jit, SpeculateIntegerOperand& op1) : m_jit(jit) , m_gpr(InvalidGPRReg) { - if (m_jit->canReuse(op1.index())) + if (m_jit->canReuse(op1.node())) m_gpr = m_jit->reuse(op1.gpr()); else m_gpr = m_jit->allocate(); @@ -1183,9 +1235,9 @@ GPRTemporary::GPRTemporary(SpeculativeJIT* jit, SpeculateIntegerOperand& op1, Sp : m_jit(jit) , m_gpr(InvalidGPRReg) { - if (m_jit->canReuse(op1.index())) + if (m_jit->canReuse(op1.node())) m_gpr = m_jit->reuse(op1.gpr()); - else if (m_jit->canReuse(op2.index())) + else if (m_jit->canReuse(op2.node())) m_gpr = m_jit->reuse(op2.gpr()); else m_gpr = m_jit->allocate(); @@ -1195,7 +1247,7 @@ GPRTemporary::GPRTemporary(SpeculativeJIT* jit, SpeculateStrictInt32Operand& op1 : m_jit(jit) , m_gpr(InvalidGPRReg) { - if (m_jit->canReuse(op1.index())) + if (m_jit->canReuse(op1.node())) m_gpr = m_jit->reuse(op1.gpr()); else m_gpr = m_jit->allocate(); @@ -1205,7 +1257,7 @@ GPRTemporary::GPRTemporary(SpeculativeJIT* jit, IntegerOperand& op1) : m_jit(jit) , m_gpr(InvalidGPRReg) { - if (m_jit->canReuse(op1.index())) + if (m_jit->canReuse(op1.node())) m_gpr = m_jit->reuse(op1.gpr()); else m_gpr = m_jit->allocate(); @@ -1215,9 +1267,9 @@ GPRTemporary::GPRTemporary(SpeculativeJIT* jit, IntegerOperand& op1, IntegerOper : m_jit(jit) , m_gpr(InvalidGPRReg) { - if (m_jit->canReuse(op1.index())) + if (m_jit->canReuse(op1.node())) m_gpr = m_jit->reuse(op1.gpr()); - else if (m_jit->canReuse(op2.index())) + else if (m_jit->canReuse(op2.node())) m_gpr = m_jit->reuse(op2.gpr()); else m_gpr = m_jit->allocate(); @@ -1227,7 +1279,7 @@ GPRTemporary::GPRTemporary(SpeculativeJIT* jit, SpeculateCellOperand& op1) : m_jit(jit) , m_gpr(InvalidGPRReg) { - if (m_jit->canReuse(op1.index())) + if (m_jit->canReuse(op1.node())) m_gpr = m_jit->reuse(op1.gpr()); else m_gpr = m_jit->allocate(); @@ -1237,7 +1289,7 @@ GPRTemporary::GPRTemporary(SpeculativeJIT* jit, SpeculateBooleanOperand& op1) : m_jit(jit) , m_gpr(InvalidGPRReg) { - if (m_jit->canReuse(op1.index())) + if (m_jit->canReuse(op1.node())) m_gpr = m_jit->reuse(op1.gpr()); else m_gpr = m_jit->allocate(); @@ -1248,7 +1300,7 @@ GPRTemporary::GPRTemporary(SpeculativeJIT* jit, JSValueOperand& op1) : m_jit(jit) , m_gpr(InvalidGPRReg) { - if (m_jit->canReuse(op1.index())) + if (m_jit->canReuse(op1.node())) m_gpr = m_jit->reuse(op1.gpr()); else m_gpr = m_jit->allocate(); @@ -1258,7 +1310,7 @@ GPRTemporary::GPRTemporary(SpeculativeJIT* jit, JSValueOperand& op1, bool tag) : m_jit(jit) , m_gpr(InvalidGPRReg) { - if (!op1.isDouble() && m_jit->canReuse(op1.index())) + if (!op1.isDouble() && m_jit->canReuse(op1.node())) m_gpr = m_jit->reuse(tag ? op1.tagGPR() : op1.payloadGPR()); else m_gpr = m_jit->allocate(); @@ -1269,7 +1321,7 @@ GPRTemporary::GPRTemporary(SpeculativeJIT* jit, StorageOperand& op1) : m_jit(jit) , m_gpr(InvalidGPRReg) { - if (m_jit->canReuse(op1.index())) + if (m_jit->canReuse(op1.node())) m_gpr = m_jit->reuse(op1.gpr()); else m_gpr = m_jit->allocate(); @@ -1294,33 +1346,11 @@ FPRTemporary::FPRTemporary(SpeculativeJIT* jit) m_fpr = m_jit->fprAllocate(); } -FPRTemporary::FPRTemporary(SpeculativeJIT* jit, DoubleOperand& op1) - : m_jit(jit) - , m_fpr(InvalidFPRReg) -{ - if (m_jit->canReuse(op1.index())) - m_fpr = m_jit->reuse(op1.fpr()); - else - m_fpr = m_jit->fprAllocate(); -} - -FPRTemporary::FPRTemporary(SpeculativeJIT* jit, DoubleOperand& op1, DoubleOperand& op2) - : m_jit(jit) - , m_fpr(InvalidFPRReg) -{ - if (m_jit->canReuse(op1.index())) - m_fpr = m_jit->reuse(op1.fpr()); - else if (m_jit->canReuse(op2.index())) - m_fpr = m_jit->reuse(op2.fpr()); - else - m_fpr = m_jit->fprAllocate(); -} - FPRTemporary::FPRTemporary(SpeculativeJIT* jit, SpeculateDoubleOperand& op1) : m_jit(jit) , m_fpr(InvalidFPRReg) { - if (m_jit->canReuse(op1.index())) + if (m_jit->canReuse(op1.node())) m_fpr = m_jit->reuse(op1.fpr()); else m_fpr = m_jit->fprAllocate(); @@ -1330,9 +1360,9 @@ FPRTemporary::FPRTemporary(SpeculativeJIT* jit, SpeculateDoubleOperand& op1, Spe : m_jit(jit) , m_fpr(InvalidFPRReg) { - if (m_jit->canReuse(op1.index())) + if (m_jit->canReuse(op1.node())) m_fpr = m_jit->reuse(op1.fpr()); - else if (m_jit->canReuse(op2.index())) + else if (m_jit->canReuse(op2.node())) m_fpr = m_jit->reuse(op2.fpr()); else m_fpr = m_jit->fprAllocate(); @@ -1343,31 +1373,29 @@ FPRTemporary::FPRTemporary(SpeculativeJIT* jit, JSValueOperand& op1) : m_jit(jit) , m_fpr(InvalidFPRReg) { - if (op1.isDouble() && m_jit->canReuse(op1.index())) + if (op1.isDouble() && m_jit->canReuse(op1.node())) m_fpr = m_jit->reuse(op1.fpr()); else m_fpr = m_jit->fprAllocate(); } #endif -void SpeculativeJIT::compilePeepHoleDoubleBranch(Node& node, NodeIndex branchNodeIndex, JITCompiler::DoubleCondition condition) +void SpeculativeJIT::compilePeepHoleDoubleBranch(Node* node, Node* branchNode, JITCompiler::DoubleCondition condition) { - Node& branchNode = at(branchNodeIndex); - BlockIndex taken = branchNode.takenBlockIndex(); - BlockIndex notTaken = branchNode.notTakenBlockIndex(); + BlockIndex taken = branchNode->takenBlockIndex(); + BlockIndex notTaken = branchNode->notTakenBlockIndex(); - SpeculateDoubleOperand op1(this, node.child1()); - SpeculateDoubleOperand op2(this, node.child2()); + SpeculateDoubleOperand op1(this, node->child1()); + SpeculateDoubleOperand op2(this, node->child2()); branchDouble(condition, op1.fpr(), op2.fpr(), taken); jump(notTaken); } -void SpeculativeJIT::compilePeepHoleObjectEquality(Node& node, NodeIndex branchNodeIndex) +void SpeculativeJIT::compilePeepHoleObjectEquality(Node* node, Node* branchNode) { - Node& branchNode = at(branchNodeIndex); - BlockIndex taken = branchNode.takenBlockIndex(); - BlockIndex notTaken = branchNode.notTakenBlockIndex(); + BlockIndex taken = branchNode->takenBlockIndex(); + BlockIndex notTaken = branchNode->notTakenBlockIndex(); MacroAssembler::RelationalCondition condition = MacroAssembler::Equal; @@ -1378,47 +1406,59 @@ void SpeculativeJIT::compilePeepHoleObjectEquality(Node& node, NodeIndex branchN notTaken = tmp; } - SpeculateCellOperand op1(this, node.child1()); - SpeculateCellOperand op2(this, node.child2()); + SpeculateCellOperand op1(this, node->child1()); + SpeculateCellOperand op2(this, node->child2()); GPRReg op1GPR = op1.gpr(); GPRReg op2GPR = op2.gpr(); - if (m_jit.graph().globalObjectFor(node.codeOrigin)->masqueradesAsUndefinedWatchpoint()->isStillValid()) { - m_jit.graph().globalObjectFor(node.codeOrigin)->masqueradesAsUndefinedWatchpoint()->add(speculationWatchpoint()); - speculationCheck(BadType, JSValueSource::unboxedCell(op1GPR), node.child1().index(), - m_jit.branchPtr( - MacroAssembler::Equal, - MacroAssembler::Address(op1GPR, JSCell::structureOffset()), - MacroAssembler::TrustedImmPtr(m_jit.globalData()->stringStructure.get()))); - speculationCheck(BadType, JSValueSource::unboxedCell(op2GPR), node.child2().index(), - m_jit.branchPtr( - MacroAssembler::Equal, - MacroAssembler::Address(op2GPR, JSCell::structureOffset()), - MacroAssembler::TrustedImmPtr(m_jit.globalData()->stringStructure.get()))); + if (m_jit.graph().globalObjectFor(node->codeOrigin)->masqueradesAsUndefinedWatchpoint()->isStillValid()) { + m_jit.graph().globalObjectFor(node->codeOrigin)->masqueradesAsUndefinedWatchpoint()->add(speculationWatchpoint()); + if (m_state.forNode(node->child1()).m_type & ~SpecObject) { + speculationCheck( + BadType, JSValueSource::unboxedCell(op1GPR), node->child1(), + m_jit.branchPtr( + MacroAssembler::Equal, + MacroAssembler::Address(op1GPR, JSCell::structureOffset()), + MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + } + if (m_state.forNode(node->child2()).m_type & ~SpecObject) { + speculationCheck( + BadType, JSValueSource::unboxedCell(op2GPR), node->child2(), + m_jit.branchPtr( + MacroAssembler::Equal, + MacroAssembler::Address(op2GPR, JSCell::structureOffset()), + MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + } } else { GPRTemporary structure(this); GPRReg structureGPR = structure.gpr(); m_jit.loadPtr(MacroAssembler::Address(op1GPR, JSCell::structureOffset()), structureGPR); - speculationCheck(BadType, JSValueSource::unboxedCell(op1GPR), node.child1().index(), - m_jit.branchPtr( - MacroAssembler::Equal, - structureGPR, - MacroAssembler::TrustedImmPtr(m_jit.globalData()->stringStructure.get()))); - speculationCheck(BadType, JSValueSource::unboxedCell(op1GPR), node.child1().index(), + if (m_state.forNode(node->child1()).m_type & ~SpecObject) { + speculationCheck( + BadType, JSValueSource::unboxedCell(op1GPR), node->child1(), + m_jit.branchPtr( + MacroAssembler::Equal, + structureGPR, + MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + } + speculationCheck(BadType, JSValueSource::unboxedCell(op1GPR), node->child1(), m_jit.branchTest8( MacroAssembler::NonZero, MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()), MacroAssembler::TrustedImm32(MasqueradesAsUndefined))); m_jit.loadPtr(MacroAssembler::Address(op2GPR, JSCell::structureOffset()), structureGPR); - speculationCheck(BadType, JSValueSource::unboxedCell(op2GPR), node.child2().index(), - m_jit.branchPtr( - MacroAssembler::Equal, - structureGPR, - MacroAssembler::TrustedImmPtr(m_jit.globalData()->stringStructure.get()))); - speculationCheck(BadType, JSValueSource::unboxedCell(op2GPR), node.child2().index(), + if (m_state.forNode(node->child2()).m_type & ~SpecObject) { + speculationCheck( + BadType, JSValueSource::unboxedCell(op2GPR), node->child2(), + m_jit.branchPtr( + MacroAssembler::Equal, + structureGPR, + MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + } + speculationCheck(BadType, JSValueSource::unboxedCell(op2GPR), node->child2(), m_jit.branchTest8( MacroAssembler::NonZero, MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()), @@ -1429,11 +1469,41 @@ void SpeculativeJIT::compilePeepHoleObjectEquality(Node& node, NodeIndex branchN jump(notTaken); } -void SpeculativeJIT::compilePeepHoleIntegerBranch(Node& node, NodeIndex branchNodeIndex, JITCompiler::RelationalCondition condition) +void SpeculativeJIT::compilePeepHoleBooleanBranch(Node* node, Node* branchNode, JITCompiler::RelationalCondition condition) +{ + BlockIndex taken = branchNode->takenBlockIndex(); + BlockIndex notTaken = branchNode->notTakenBlockIndex(); + + // The branch instruction will branch to the taken block. + // If taken is next, switch taken with notTaken & invert the branch condition so we can fall through. + if (taken == nextBlock()) { + condition = JITCompiler::invert(condition); + BlockIndex tmp = taken; + taken = notTaken; + notTaken = tmp; + } + + if (isBooleanConstant(node->child1().node())) { + bool imm = valueOfBooleanConstant(node->child1().node()); + SpeculateBooleanOperand op2(this, node->child2()); + branch32(condition, JITCompiler::Imm32(static_cast<int32_t>(JSValue::encode(jsBoolean(imm)))), op2.gpr(), taken); + } else if (isBooleanConstant(node->child2().node())) { + SpeculateBooleanOperand op1(this, node->child1()); + bool imm = valueOfBooleanConstant(node->child2().node()); + branch32(condition, op1.gpr(), JITCompiler::Imm32(static_cast<int32_t>(JSValue::encode(jsBoolean(imm)))), taken); + } else { + SpeculateBooleanOperand op1(this, node->child1()); + SpeculateBooleanOperand op2(this, node->child2()); + branch32(condition, op1.gpr(), op2.gpr(), taken); + } + + jump(notTaken); +} + +void SpeculativeJIT::compilePeepHoleIntegerBranch(Node* node, Node* branchNode, JITCompiler::RelationalCondition condition) { - Node& branchNode = at(branchNodeIndex); - BlockIndex taken = branchNode.takenBlockIndex(); - BlockIndex notTaken = branchNode.notTakenBlockIndex(); + BlockIndex taken = branchNode->takenBlockIndex(); + BlockIndex notTaken = branchNode->notTakenBlockIndex(); // The branch instruction will branch to the taken block. // If taken is next, switch taken with notTaken & invert the branch condition so we can fall through. @@ -1444,17 +1514,17 @@ void SpeculativeJIT::compilePeepHoleIntegerBranch(Node& node, NodeIndex branchNo notTaken = tmp; } - if (isInt32Constant(node.child1().index())) { - int32_t imm = valueOfInt32Constant(node.child1().index()); - SpeculateIntegerOperand op2(this, node.child2()); + if (isInt32Constant(node->child1().node())) { + int32_t imm = valueOfInt32Constant(node->child1().node()); + SpeculateIntegerOperand op2(this, node->child2()); branch32(condition, JITCompiler::Imm32(imm), op2.gpr(), taken); - } else if (isInt32Constant(node.child2().index())) { - SpeculateIntegerOperand op1(this, node.child1()); - int32_t imm = valueOfInt32Constant(node.child2().index()); + } else if (isInt32Constant(node->child2().node())) { + SpeculateIntegerOperand op1(this, node->child1()); + int32_t imm = valueOfInt32Constant(node->child2().node()); branch32(condition, op1.gpr(), JITCompiler::Imm32(imm), taken); } else { - SpeculateIntegerOperand op1(this, node.child1()); - SpeculateIntegerOperand op2(this, node.child2()); + SpeculateIntegerOperand op1(this, node->child1()); + SpeculateIntegerOperand op2(this, node->child2()); branch32(condition, op1.gpr(), op2.gpr(), taken); } @@ -1462,74 +1532,124 @@ void SpeculativeJIT::compilePeepHoleIntegerBranch(Node& node, NodeIndex branchNo } // Returns true if the compare is fused with a subsequent branch. -bool SpeculativeJIT::compilePeepHoleBranch(Node& node, MacroAssembler::RelationalCondition condition, MacroAssembler::DoubleCondition doubleCondition, S_DFGOperation_EJJ operation) +bool SpeculativeJIT::compilePeepHoleBranch(Node* node, MacroAssembler::RelationalCondition condition, MacroAssembler::DoubleCondition doubleCondition, S_DFGOperation_EJJ operation) { // Fused compare & branch. unsigned branchIndexInBlock = detectPeepHoleBranch(); if (branchIndexInBlock != UINT_MAX) { - NodeIndex branchNodeIndex = m_jit.graph().m_blocks[m_block]->at(branchIndexInBlock); + Node* branchNode = m_jit.graph().m_blocks[m_block]->at(branchIndexInBlock); // detectPeepHoleBranch currently only permits the branch to be the very next node, // so can be no intervening nodes to also reference the compare. - ASSERT(node.adjustedRefCount() == 1); - - if (Node::shouldSpeculateInteger(at(node.child1()), at(node.child2()))) - compilePeepHoleIntegerBranch(node, branchNodeIndex, condition); - else if (Node::shouldSpeculateNumber(at(node.child1()), at(node.child2()))) - compilePeepHoleDoubleBranch(node, branchNodeIndex, doubleCondition); - else if (node.op() == CompareEq) { - if (at(node.child1()).shouldSpeculateString() || at(node.child2()).shouldSpeculateString()) { - nonSpeculativePeepholeBranch(node, branchNodeIndex, condition, operation); - return true; + ASSERT(node->adjustedRefCount() == 1); + + if (node->isBinaryUseKind(Int32Use)) + compilePeepHoleIntegerBranch(node, branchNode, condition); + else if (node->isBinaryUseKind(NumberUse)) + compilePeepHoleDoubleBranch(node, branchNode, doubleCondition); + else if (node->op() == CompareEq) { + if (node->isBinaryUseKind(StringUse)) { + // Use non-peephole comparison, for now. + return false; } - if (at(node.child1()).shouldSpeculateNonStringCell() && at(node.child2()).shouldSpeculateNonStringCellOrOther()) - compilePeepHoleObjectToObjectOrOtherEquality(node.child1(), node.child2(), branchNodeIndex); - else if (at(node.child1()).shouldSpeculateNonStringCellOrOther() && at(node.child2()).shouldSpeculateNonStringCell()) - compilePeepHoleObjectToObjectOrOtherEquality(node.child2(), node.child1(), branchNodeIndex); - else if (at(node.child1()).shouldSpeculateNonStringCell() && at(node.child2()).shouldSpeculateNonStringCell()) - compilePeepHoleObjectEquality(node, branchNodeIndex); + if (node->isBinaryUseKind(BooleanUse)) + compilePeepHoleBooleanBranch(node, branchNode, condition); + else if (node->isBinaryUseKind(ObjectUse)) + compilePeepHoleObjectEquality(node, branchNode); + else if (node->child1().useKind() == ObjectUse && node->child2().useKind() == ObjectOrOtherUse) + compilePeepHoleObjectToObjectOrOtherEquality(node->child1(), node->child2(), branchNode); + else if (node->child1().useKind() == ObjectOrOtherUse && node->child2().useKind() == ObjectUse) + compilePeepHoleObjectToObjectOrOtherEquality(node->child2(), node->child1(), branchNode); else { - nonSpeculativePeepholeBranch(node, branchNodeIndex, condition, operation); + nonSpeculativePeepholeBranch(node, branchNode, condition, operation); return true; } } else { - nonSpeculativePeepholeBranch(node, branchNodeIndex, condition, operation); + nonSpeculativePeepholeBranch(node, branchNode, condition, operation); return true; } - use(node.child1()); - use(node.child2()); + use(node->child1()); + use(node->child2()); m_indexInBlock = branchIndexInBlock; - m_compileIndex = branchNodeIndex; + m_currentNode = branchNode; return true; } return false; } -void SpeculativeJIT::noticeOSRBirth(NodeIndex nodeIndex, Node& node) +void SpeculativeJIT::noticeOSRBirth(Node* node) { - if (!node.hasVirtualRegister()) + if (!node->hasVirtualRegister()) return; - VirtualRegister virtualRegister = node.virtualRegister(); + VirtualRegister virtualRegister = node->virtualRegister(); GenerationInfo& info = m_generationInfo[virtualRegister]; - info.noticeOSRBirth(*m_stream, nodeIndex, virtualRegister); + info.noticeOSRBirth(*m_stream, node, virtualRegister); } -void SpeculativeJIT::compileMovHint(Node& node) +void SpeculativeJIT::compileMovHint(Node* node) { - ASSERT(node.op() == SetLocal); - - m_lastSetOperand = node.local(); + ASSERT(node->containsMovHint() && node->op() != ZombieHint); - Node& child = at(node.child1()); - noticeOSRBirth(node.child1().index(), child); + m_lastSetOperand = node->local(); + + Node* child = node->child1().node(); + noticeOSRBirth(child); - if (child.op() == UInt32ToNumber) - noticeOSRBirth(child.child1().index(), at(child.child1())); + if (child->op() == UInt32ToNumber) + noticeOSRBirth(child->child1().node()); - m_stream->appendAndLog(VariableEvent::movHint(node.child1().index(), node.local())); + m_stream->appendAndLog(VariableEvent::movHint(MinifiedID(child), node->local())); +} + +void SpeculativeJIT::compileMovHintAndCheck(Node* node) +{ + compileMovHint(node); + speculate(node, node->child1()); + noResult(node); +} + +void SpeculativeJIT::compileInlineStart(Node* node) +{ + InlineCallFrame* inlineCallFrame = node->codeOrigin.inlineCallFrame; + int argumentCountIncludingThis = inlineCallFrame->arguments.size(); + unsigned argumentPositionStart = node->argumentPositionStart(); + CodeBlock* codeBlock = baselineCodeBlockForInlineCallFrame(inlineCallFrame); + for (int i = 0; i < argumentCountIncludingThis; ++i) { + ValueRecovery recovery; + if (codeBlock->isCaptured(argumentToOperand(i))) + recovery = ValueRecovery::alreadyInJSStack(); + else { + ArgumentPosition& argumentPosition = + m_jit.graph().m_argumentPositions[argumentPositionStart + i]; + ValueSource valueSource; + if (!argumentPosition.shouldUnboxIfPossible()) + valueSource = ValueSource(ValueInJSStack); + else if (argumentPosition.shouldUseDoubleFormat()) + valueSource = ValueSource(DoubleInJSStack); + else if (isInt32Speculation(argumentPosition.prediction())) + valueSource = ValueSource(Int32InJSStack); + else if (isCellSpeculation(argumentPosition.prediction())) + valueSource = ValueSource(CellInJSStack); + else if (isBooleanSpeculation(argumentPosition.prediction())) + valueSource = ValueSource(BooleanInJSStack); + else + valueSource = ValueSource(ValueInJSStack); + recovery = computeValueRecoveryFor(valueSource); + } + // The recovery should refer either to something that has already been + // stored into the stack at the right place, or to a constant, + // since the Arguments code isn't smart enough to handle anything else. + // The exception is the this argument, which we don't really need to be + // able to recover. +#if DFG_ENABLE(DEBUG_VERBOSE) + dataLogF("\nRecovery for argument %d: ", i); + recovery.dump(WTF::dataFile()); +#endif + inlineCallFrame->arguments[i] = recovery; + } } void SpeculativeJIT::compile(BasicBlock& block) @@ -1574,21 +1694,22 @@ void SpeculativeJIT::compile(BasicBlock& block) ASSERT(m_variables.size() == block.variablesAtHead.numberOfLocals()); for (size_t i = 0; i < m_variables.size(); ++i) { - NodeIndex nodeIndex = block.variablesAtHead.local(i); + Node* node = block.variablesAtHead.local(i); ValueSource valueSource; - if (nodeIndex == NoNode) + if (!node) valueSource = ValueSource(SourceIsDead); - else if (at(nodeIndex).variableAccessData()->isArgumentsAlias()) + else if (node->variableAccessData()->isArgumentsAlias()) valueSource = ValueSource(ArgumentsSource); - else if (at(nodeIndex).variableAccessData()->isCaptured()) - valueSource = ValueSource(ValueInJSStack); - else if (!at(nodeIndex).refCount()) + else if (!node->refCount()) valueSource = ValueSource(SourceIsDead); - else if (at(nodeIndex).variableAccessData()->shouldUseDoubleFormat()) + else if (!node->variableAccessData()->shouldUnboxIfPossible()) + valueSource = ValueSource(ValueInJSStack); + else if (node->variableAccessData()->shouldUseDoubleFormat()) valueSource = ValueSource(DoubleInJSStack); else - valueSource = ValueSource::forSpeculation(at(nodeIndex).variableAccessData()->argumentAwarePrediction()); + valueSource = ValueSource::forSpeculation(node->variableAccessData()->argumentAwarePrediction()); m_variables[i] = valueSource; + // FIXME: Don't emit SetLocal(Dead). https://bugs.webkit.org/show_bug.cgi?id=108019 m_stream->appendAndLog(VariableEvent::setLocal(i, valueSource.dataFormat())); } @@ -1607,101 +1728,86 @@ void SpeculativeJIT::compile(BasicBlock& block) #endif for (m_indexInBlock = 0; m_indexInBlock < block.size(); ++m_indexInBlock) { - m_compileIndex = block[m_indexInBlock]; - m_jit.setForNode(m_compileIndex); - Node& node = at(m_compileIndex); - m_codeOriginForOSR = node.codeOrigin; - if (!node.shouldGenerate()) { + m_currentNode = block[m_indexInBlock]; +#if !ASSERT_DISABLED + m_canExit = m_currentNode->canExit(); +#endif + bool shouldExecuteEffects = m_state.startExecuting(m_currentNode); + m_jit.setForNode(m_currentNode); + m_codeOriginForOSR = m_currentNode->codeOrigin; + if (!m_currentNode->shouldGenerate()) { #if DFG_ENABLE(DEBUG_VERBOSE) - dataLogF("SpeculativeJIT skipping Node @%d (bc#%u) at JIT offset 0x%x ", (int)m_compileIndex, node.codeOrigin.bytecodeIndex, m_jit.debugOffset()); + dataLogF("SpeculativeJIT skipping Node @%d (bc#%u) at JIT offset 0x%x ", m_currentNode->index(), m_currentNode->codeOrigin.bytecodeIndex, m_jit.debugOffset()); #endif - switch (node.op()) { + switch (m_currentNode->op()) { case JSConstant: - m_minifiedGraph->append(MinifiedNode::fromNode(m_compileIndex, node)); + m_minifiedGraph->append(MinifiedNode::fromNode(m_currentNode)); break; case WeakJSConstant: - m_jit.addWeakReference(node.weakConstant()); - m_minifiedGraph->append(MinifiedNode::fromNode(m_compileIndex, node)); + m_jit.addWeakReference(m_currentNode->weakConstant()); + m_minifiedGraph->append(MinifiedNode::fromNode(m_currentNode)); break; case SetLocal: - compileMovHint(node); + RELEASE_ASSERT_NOT_REACHED(); break; - - case InlineStart: { - InlineCallFrame* inlineCallFrame = node.codeOrigin.inlineCallFrame; - int argumentCountIncludingThis = inlineCallFrame->arguments.size(); - unsigned argumentPositionStart = node.argumentPositionStart(); - CodeBlock* codeBlock = baselineCodeBlockForInlineCallFrame(inlineCallFrame); - for (int i = 0; i < argumentCountIncludingThis; ++i) { - ValueRecovery recovery; - if (codeBlock->isCaptured(argumentToOperand(i))) - recovery = ValueRecovery::alreadyInJSStack(); - else { - ArgumentPosition& argumentPosition = - m_jit.graph().m_argumentPositions[argumentPositionStart + i]; - ValueSource valueSource; - if (argumentPosition.shouldUseDoubleFormat()) - valueSource = ValueSource(DoubleInJSStack); - else if (isInt32Speculation(argumentPosition.prediction())) - valueSource = ValueSource(Int32InJSStack); - else if (isCellSpeculation(argumentPosition.prediction())) - valueSource = ValueSource(CellInJSStack); - else if (isBooleanSpeculation(argumentPosition.prediction())) - valueSource = ValueSource(BooleanInJSStack); - else - valueSource = ValueSource(ValueInJSStack); - recovery = computeValueRecoveryFor(valueSource); - } - // The recovery should refer either to something that has already been - // stored into the stack at the right place, or to a constant, - // since the Arguments code isn't smart enough to handle anything else. - // The exception is the this argument, which we don't really need to be - // able to recover. -#if DFG_ENABLE(DEBUG_VERBOSE) - dataLogF("\nRecovery for argument %d: ", i); - recovery.dump(WTF::dataFile()); -#endif - inlineCallFrame->arguments[i] = recovery; - } + + case MovHint: + compileMovHint(m_currentNode); break; - } + case ZombieHint: { + m_lastSetOperand = m_currentNode->local(); + m_stream->appendAndLog(VariableEvent::setLocal(m_currentNode->local(), DataFormatDead)); + break; + } + default: - if (belongsInMinifiedGraph(node.op())) - m_minifiedGraph->append(MinifiedNode::fromNode(m_compileIndex, node)); + if (belongsInMinifiedGraph(m_currentNode->op())) + m_minifiedGraph->append(MinifiedNode::fromNode(m_currentNode)); break; } } else { + if (verboseCompilationEnabled()) { + dataLogF( + "SpeculativeJIT generating Node @%d (bc#%u) at JIT offset 0x%x", + (int)m_currentNode->index(), + m_currentNode->codeOrigin.bytecodeIndex, m_jit.debugOffset()); #if DFG_ENABLE(DEBUG_VERBOSE) - dataLogF("SpeculativeJIT generating Node @%d (bc#%u) at JIT offset 0x%x ", (int)m_compileIndex, node.codeOrigin.bytecodeIndex, m_jit.debugOffset()); + dataLog(" "); +#else + dataLog("\n"); #endif + } #if DFG_ENABLE(JIT_BREAK_ON_EVERY_NODE) m_jit.breakpoint(); #endif #if DFG_ENABLE(XOR_DEBUG_AID) - m_jit.xorPtr(JITCompiler::TrustedImm32(m_compileIndex), GPRInfo::regT0); - m_jit.xorPtr(JITCompiler::TrustedImm32(m_compileIndex), GPRInfo::regT0); + m_jit.xorPtr(JITCompiler::TrustedImm32(m_currentNode->index()), GPRInfo::regT0); + m_jit.xorPtr(JITCompiler::TrustedImm32(m_currentNode->index()), GPRInfo::regT0); #endif checkConsistency(); - compile(node); + + m_speculationDirection = (m_currentNode->flags() & NodeExitsForward) ? ForwardSpeculation : BackwardSpeculation; + + compile(m_currentNode); if (!m_compileOkay) { m_compileOkay = true; clearGenerationInfo(); return; } - if (belongsInMinifiedGraph(node.op())) { - m_minifiedGraph->append(MinifiedNode::fromNode(m_compileIndex, node)); - noticeOSRBirth(m_compileIndex, node); + if (belongsInMinifiedGraph(m_currentNode->op())) { + m_minifiedGraph->append(MinifiedNode::fromNode(m_currentNode)); + noticeOSRBirth(m_currentNode); } #if DFG_ENABLE(DEBUG_VERBOSE) - if (node.hasResult()) { - GenerationInfo& info = m_generationInfo[node.virtualRegister()]; - dataLogF("-> %s, vr#%d", dataFormatToString(info.registerFormat()), (int)node.virtualRegister()); + if (m_currentNode->hasResult()) { + GenerationInfo& info = m_generationInfo[m_currentNode->virtualRegister()]; + dataLogF("-> %s, vr#%d", dataFormatToString(info.registerFormat()), (int)m_currentNode->virtualRegister()); if (info.registerFormat() != DataFormatNone) { if (info.registerFormat() == DataFormatDouble) dataLogF(", %s", FPRInfo::debugName(info.fpr())); @@ -1723,9 +1829,10 @@ void SpeculativeJIT::compile(BasicBlock& block) #endif // Make sure that the abstract state is rematerialized for the next node. - m_state.execute(m_indexInBlock); + if (shouldExecuteEffects) + m_state.executeEffects(m_indexInBlock); - if (node.shouldGenerate()) + if (m_currentNode->shouldGenerate()) checkConsistency(); } @@ -1742,8 +1849,9 @@ void SpeculativeJIT::compile(BasicBlock& block) // we need to check that they are correct on function entry. void SpeculativeJIT::checkArgumentTypes() { - ASSERT(!m_compileIndex); + ASSERT(!m_currentNode); m_isCheckingArgumentTypes = true; + m_speculationDirection = BackwardSpeculation; m_codeOriginForOSR = CodeOrigin(0); for (size_t i = 0; i < m_arguments.size(); ++i) @@ -1752,15 +1860,17 @@ void SpeculativeJIT::checkArgumentTypes() m_variables[i] = ValueSource(ValueInJSStack); for (int i = 0; i < m_jit.codeBlock()->numParameters(); ++i) { - NodeIndex nodeIndex = m_jit.graph().m_arguments[i]; - Node& node = at(nodeIndex); - ASSERT(node.op() == SetArgument); - if (!node.shouldGenerate()) { + Node* node = m_jit.graph().m_arguments[i]; + ASSERT(node->op() == SetArgument); + if (!node->shouldGenerate()) { // The argument is dead. We don't do any checks for such arguments. continue; } - VariableAccessData* variableAccessData = node.variableAccessData(); + VariableAccessData* variableAccessData = node->variableAccessData(); + if (!variableAccessData->isProfitableToUnbox()) + continue; + VirtualRegister virtualRegister = variableAccessData->local(); SpeculatedType predictedType = variableAccessData->prediction(); @@ -1768,21 +1878,21 @@ void SpeculativeJIT::checkArgumentTypes() #if USE(JSVALUE64) if (isInt32Speculation(predictedType)) - speculationCheck(BadType, valueSource, nodeIndex, m_jit.branch64(MacroAssembler::Below, JITCompiler::addressFor(virtualRegister), GPRInfo::tagTypeNumberRegister)); + speculationCheck(BadType, valueSource, node, m_jit.branch64(MacroAssembler::Below, JITCompiler::addressFor(virtualRegister), GPRInfo::tagTypeNumberRegister)); else if (isBooleanSpeculation(predictedType)) { GPRTemporary temp(this); m_jit.load64(JITCompiler::addressFor(virtualRegister), temp.gpr()); m_jit.xor64(TrustedImm32(static_cast<int32_t>(ValueFalse)), temp.gpr()); - speculationCheck(BadType, valueSource, nodeIndex, m_jit.branchTest64(MacroAssembler::NonZero, temp.gpr(), TrustedImm32(static_cast<int32_t>(~1)))); + speculationCheck(BadType, valueSource, node, m_jit.branchTest64(MacroAssembler::NonZero, temp.gpr(), TrustedImm32(static_cast<int32_t>(~1)))); } else if (isCellSpeculation(predictedType)) - speculationCheck(BadType, valueSource, nodeIndex, m_jit.branchTest64(MacroAssembler::NonZero, JITCompiler::addressFor(virtualRegister), GPRInfo::tagMaskRegister)); + speculationCheck(BadType, valueSource, node, m_jit.branchTest64(MacroAssembler::NonZero, JITCompiler::addressFor(virtualRegister), GPRInfo::tagMaskRegister)); #else if (isInt32Speculation(predictedType)) - speculationCheck(BadType, valueSource, nodeIndex, m_jit.branch32(MacroAssembler::NotEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::Int32Tag))); + speculationCheck(BadType, valueSource, node, m_jit.branch32(MacroAssembler::NotEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::Int32Tag))); else if (isBooleanSpeculation(predictedType)) - speculationCheck(BadType, valueSource, nodeIndex, m_jit.branch32(MacroAssembler::NotEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::BooleanTag))); + speculationCheck(BadType, valueSource, node, m_jit.branch32(MacroAssembler::NotEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::BooleanTag))); else if (isCellSpeculation(predictedType)) - speculationCheck(BadType, valueSource, nodeIndex, m_jit.branch32(MacroAssembler::NotEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::CellTag))); + speculationCheck(BadType, valueSource, node, m_jit.branch32(MacroAssembler::NotEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::CellTag))); #endif } m_isCheckingArgumentTypes = false; @@ -1795,7 +1905,7 @@ bool SpeculativeJIT::compile() if (DFG_ENABLE_EDGE_CODE_VERIFICATION) m_jit.move(TrustedImm32(0), GPRInfo::regT0); - ASSERT(!m_compileIndex); + ASSERT(!m_currentNode); for (m_block = 0; m_block < m_jit.graph().m_blocks.size(); ++m_block) { m_jit.setForBlock(m_block); BasicBlock* block = m_jit.graph().m_blocks[m_block].get(); @@ -1850,18 +1960,19 @@ ValueRecovery SpeculativeJIT::computeValueRecoveryFor(const ValueSource& valueSo return valueSource.valueRecovery(); ASSERT(valueSource.kind() == HaveNode); - if (isConstant(valueSource.nodeIndex())) - return ValueRecovery::constant(valueOfJSConstant(valueSource.nodeIndex())); + Node* node = valueSource.id().node(m_jit.graph()); + if (isConstant(node)) + return ValueRecovery::constant(valueOfJSConstant(node)); return ValueRecovery(); } -void SpeculativeJIT::compileDoublePutByVal(Node& node, SpeculateCellOperand& base, SpeculateStrictInt32Operand& property) +void SpeculativeJIT::compileDoublePutByVal(Node* node, SpeculateCellOperand& base, SpeculateStrictInt32Operand& property) { Edge child3 = m_jit.graph().varArgChild(node, 2); Edge child4 = m_jit.graph().varArgChild(node, 3); - ArrayMode arrayMode = node.arrayMode(); + ArrayMode arrayMode = node->arrayMode(); GPRReg baseReg = base.gpr(); GPRReg propertyReg = property.gpr(); @@ -1870,13 +1981,10 @@ void SpeculativeJIT::compileDoublePutByVal(Node& node, SpeculateCellOperand& bas FPRReg valueReg = value.fpr(); - if (!isRealNumberSpeculation(m_state.forNode(child3).m_type)) { - // FIXME: We need a way of profiling these, and we need to hoist them into - // SpeculateDoubleOperand. - speculationCheck( - BadType, JSValueRegs(), NoNode, - m_jit.branchDouble(MacroAssembler::DoubleNotEqualOrUnordered, valueReg, valueReg)); - } + DFG_TYPE_CHECK( + JSValueRegs(), child3, SpecRealNumber, + m_jit.branchDouble( + MacroAssembler::DoubleNotEqualOrUnordered, valueReg, valueReg)); if (!m_compileOkay) return; @@ -1884,32 +1992,32 @@ void SpeculativeJIT::compileDoublePutByVal(Node& node, SpeculateCellOperand& bas StorageOperand storage(this, child4); GPRReg storageReg = storage.gpr(); - if (node.op() == PutByValAlias) { + if (node->op() == PutByValAlias) { // Store the value to the array. GPRReg propertyReg = property.gpr(); FPRReg valueReg = value.fpr(); m_jit.storeDouble(valueReg, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight)); - noResult(m_compileIndex); + noResult(m_currentNode); return; } GPRTemporary temporary; GPRReg temporaryReg = temporaryRegisterForPutByVal(temporary, node); - MacroAssembler::JumpList slowCases; + MacroAssembler::Jump slowCase; if (arrayMode.isInBounds()) { speculationCheck( - Uncountable, JSValueRegs(), NoNode, + StoreToHoleOrOutOfBounds, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength()))); } else { MacroAssembler::Jump inBounds = m_jit.branch32(MacroAssembler::Below, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength())); - slowCases.append(m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfVectorLength()))); + slowCase = m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfVectorLength())); if (!arrayMode.isOutOfBounds()) - speculationCheck(Uncountable, JSValueRegs(), NoNode, slowCases); + speculationCheck(OutOfBounds, JSValueRegs(), 0, slowCase); m_jit.add32(TrustedImm32(1), propertyReg, temporaryReg); m_jit.store32(temporaryReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength())); @@ -1927,28 +2035,28 @@ void SpeculativeJIT::compileDoublePutByVal(Node& node, SpeculateCellOperand& bas if (arrayMode.isOutOfBounds()) { addSlowPathGenerator( slowPathCall( - slowCases, this, + slowCase, this, m_jit.codeBlock()->isStrictMode() ? operationPutDoubleByValBeyondArrayBoundsStrict : operationPutDoubleByValBeyondArrayBoundsNonStrict, NoResult, baseReg, propertyReg, valueReg)); } - noResult(m_compileIndex, UseChildrenCalledExplicitly); + noResult(m_currentNode, UseChildrenCalledExplicitly); } -void SpeculativeJIT::compileGetCharCodeAt(Node& node) +void SpeculativeJIT::compileGetCharCodeAt(Node* node) { - SpeculateCellOperand string(this, node.child1()); - SpeculateStrictInt32Operand index(this, node.child2()); - StorageOperand storage(this, node.child3()); + SpeculateCellOperand string(this, node->child1()); + SpeculateStrictInt32Operand index(this, node->child2()); + StorageOperand storage(this, node->child3()); GPRReg stringReg = string.gpr(); GPRReg indexReg = index.gpr(); GPRReg storageReg = storage.gpr(); - ASSERT(speculationChecked(m_state.forNode(node.child1()).m_type, SpecString)); + ASSERT(speculationChecked(m_state.forNode(node->child1()).m_type, SpecString)); // unsigned comparison so we can filter out negative indices and indices that are too large - speculationCheck(Uncountable, JSValueRegs(), NoNode, m_jit.branch32(MacroAssembler::AboveOrEqual, indexReg, MacroAssembler::Address(stringReg, JSString::offsetOfLength()))); + speculationCheck(Uncountable, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::AboveOrEqual, indexReg, MacroAssembler::Address(stringReg, JSString::offsetOfLength()))); GPRTemporary scratch(this); GPRReg scratchReg = scratch.gpr(); @@ -1967,22 +2075,22 @@ void SpeculativeJIT::compileGetCharCodeAt(Node& node) cont8Bit.link(&m_jit); - integerResult(scratchReg, m_compileIndex); + integerResult(scratchReg, m_currentNode); } -void SpeculativeJIT::compileGetByValOnString(Node& node) +void SpeculativeJIT::compileGetByValOnString(Node* node) { - SpeculateCellOperand base(this, node.child1()); - SpeculateStrictInt32Operand property(this, node.child2()); - StorageOperand storage(this, node.child3()); + SpeculateCellOperand base(this, node->child1()); + SpeculateStrictInt32Operand property(this, node->child2()); + StorageOperand storage(this, node->child3()); GPRReg baseReg = base.gpr(); GPRReg propertyReg = property.gpr(); GPRReg storageReg = storage.gpr(); - ASSERT(ArrayMode(Array::String).alreadyChecked(m_jit.graph(), node, m_state.forNode(node.child1()))); + ASSERT(ArrayMode(Array::String).alreadyChecked(m_jit.graph(), node, m_state.forNode(node->child1()))); // unsigned comparison so we can filter out negative indices and indices that are too large - speculationCheck(Uncountable, JSValueRegs(), NoNode, m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(baseReg, JSString::offsetOfLength()))); + speculationCheck(Uncountable, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(baseReg, JSString::offsetOfLength()))); GPRTemporary scratch(this); GPRReg scratchReg = scratch.gpr(); @@ -2000,51 +2108,53 @@ void SpeculativeJIT::compileGetByValOnString(Node& node) m_jit.load16(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesTwo, 0), scratchReg); // We only support ascii characters - speculationCheck(Uncountable, JSValueRegs(), NoNode, m_jit.branch32(MacroAssembler::AboveOrEqual, scratchReg, TrustedImm32(0x100))); + speculationCheck(Uncountable, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::AboveOrEqual, scratchReg, TrustedImm32(0x100))); // 8 bit string values don't need the isASCII check. cont8Bit.link(&m_jit); GPRTemporary smallStrings(this); GPRReg smallStringsReg = smallStrings.gpr(); - m_jit.move(MacroAssembler::TrustedImmPtr(m_jit.globalData()->smallStrings.singleCharacterStrings()), smallStringsReg); + m_jit.move(MacroAssembler::TrustedImmPtr(m_jit.vm()->smallStrings.singleCharacterStrings()), smallStringsReg); m_jit.loadPtr(MacroAssembler::BaseIndex(smallStringsReg, scratchReg, MacroAssembler::ScalePtr, 0), scratchReg); - speculationCheck(Uncountable, JSValueRegs(), NoNode, m_jit.branchTest32(MacroAssembler::Zero, scratchReg)); - cellResult(scratchReg, m_compileIndex); + speculationCheck(Uncountable, JSValueRegs(), 0, m_jit.branchTest32(MacroAssembler::Zero, scratchReg)); + cellResult(scratchReg, m_currentNode); } -GeneratedOperandType SpeculativeJIT::checkGeneratedTypeForToInt32(NodeIndex nodeIndex) +void SpeculativeJIT::compileFromCharCode(Node* node) { -#if DFG_ENABLE(DEBUG_VERBOSE) - dataLogF("checkGeneratedTypeForToInt32@%d ", nodeIndex); -#endif - Node& node = at(nodeIndex); - VirtualRegister virtualRegister = node.virtualRegister(); - GenerationInfo& info = m_generationInfo[virtualRegister]; - - if (info.registerFormat() == DataFormatNone) { - if (node.hasConstant()) { - if (isInt32Constant(nodeIndex)) - return GeneratedOperandInteger; + SpeculateStrictInt32Operand property(this, node->child1()); + GPRReg propertyReg = property.gpr(); + GPRTemporary smallStrings(this); + GPRTemporary scratch(this); + GPRReg scratchReg = scratch.gpr(); + GPRReg smallStringsReg = smallStrings.gpr(); - if (isNumberConstant(nodeIndex)) - return GeneratedOperandDouble; + JITCompiler::JumpList slowCases; + slowCases.append(m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, TrustedImm32(0xff))); + m_jit.move(MacroAssembler::TrustedImmPtr(m_jit.vm()->smallStrings.singleCharacterStrings()), smallStringsReg); + m_jit.loadPtr(MacroAssembler::BaseIndex(smallStringsReg, propertyReg, MacroAssembler::ScalePtr, 0), scratchReg); - terminateSpeculativeExecution(Uncountable, JSValueRegs(), NoNode); - return GeneratedOperandTypeUnknown; - } + slowCases.append(m_jit.branchTest32(MacroAssembler::Zero, scratchReg)); + addSlowPathGenerator(slowPathCall(slowCases, this, operationStringFromCharCode, scratchReg, propertyReg)); + cellResult(scratchReg, m_currentNode); +} - if (info.spillFormat() == DataFormatDouble) - return GeneratedOperandDouble; - } +GeneratedOperandType SpeculativeJIT::checkGeneratedTypeForToInt32(Node* node) +{ +#if DFG_ENABLE(DEBUG_VERBOSE) + dataLogF("checkGeneratedTypeForToInt32@%d ", node->index()); +#endif + VirtualRegister virtualRegister = node->virtualRegister(); + GenerationInfo& info = m_generationInfo[virtualRegister]; switch (info.registerFormat()) { - case DataFormatBoolean: // This type never occurs. case DataFormatStorage: - ASSERT_NOT_REACHED(); + RELEASE_ASSERT_NOT_REACHED(); + case DataFormatBoolean: case DataFormatCell: - terminateSpeculativeExecution(Uncountable, JSValueRegs(), NoNode); + terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); return GeneratedOperandTypeUnknown; case DataFormatNone: @@ -2062,46 +2172,48 @@ GeneratedOperandType SpeculativeJIT::checkGeneratedTypeForToInt32(NodeIndex node return GeneratedOperandDouble; default: - ASSERT_NOT_REACHED(); + RELEASE_ASSERT_NOT_REACHED(); return GeneratedOperandTypeUnknown; } } -void SpeculativeJIT::compileValueToInt32(Node& node) +void SpeculativeJIT::compileValueToInt32(Node* node) { - if (at(node.child1()).shouldSpeculateInteger()) { - SpeculateIntegerOperand op1(this, node.child1()); + switch (node->child1().useKind()) { + case Int32Use: { + SpeculateIntegerOperand op1(this, node->child1()); GPRTemporary result(this, op1); m_jit.move(op1.gpr(), result.gpr()); - integerResult(result.gpr(), m_compileIndex, op1.format()); + integerResult(result.gpr(), node, op1.format()); return; } - if (at(node.child1()).shouldSpeculateNumber()) { - switch (checkGeneratedTypeForToInt32(node.child1().index())) { + case NumberUse: + case NotCellUse: { + switch (checkGeneratedTypeForToInt32(node->child1().node())) { case GeneratedOperandInteger: { - SpeculateIntegerOperand op1(this, node.child1()); + SpeculateIntegerOperand op1(this, node->child1(), ManualOperandSpeculation); GPRTemporary result(this, op1); m_jit.move(op1.gpr(), result.gpr()); - integerResult(result.gpr(), m_compileIndex, op1.format()); + integerResult(result.gpr(), node, op1.format()); return; } case GeneratedOperandDouble: { GPRTemporary result(this); - DoubleOperand op1(this, node.child1()); + SpeculateDoubleOperand op1(this, node->child1(), ManualOperandSpeculation); FPRReg fpr = op1.fpr(); GPRReg gpr = result.gpr(); JITCompiler::Jump notTruncatedToInteger = m_jit.branchTruncateDoubleToInt32(fpr, gpr, JITCompiler::BranchIfTruncateFailed); addSlowPathGenerator(slowPathCall(notTruncatedToInteger, this, toInt32, gpr, fpr)); - integerResult(gpr, m_compileIndex); + integerResult(gpr, node); return; } case GeneratedOperandJSValue: { GPRTemporary result(this); #if USE(JSVALUE64) - JSValueOperand op1(this, node.child1()); + JSValueOperand op1(this, node->child1(), ManualOperandSpeculation); GPRReg gpr = op1.gpr(); GPRReg resultGpr = result.gpr(); @@ -2109,9 +2221,27 @@ void SpeculativeJIT::compileValueToInt32(Node& node) FPRReg fpr = tempFpr.fpr(); JITCompiler::Jump isInteger = m_jit.branch64(MacroAssembler::AboveOrEqual, gpr, GPRInfo::tagTypeNumberRegister); + JITCompiler::JumpList converted; - if (!isNumberSpeculation(m_state.forNode(node.child1()).m_type)) - speculationCheck(BadType, JSValueRegs(gpr), node.child1().index(), m_jit.branchTest64(MacroAssembler::Zero, gpr, GPRInfo::tagTypeNumberRegister)); + if (node->child1().useKind() == NumberUse) { + DFG_TYPE_CHECK( + JSValueRegs(gpr), node->child1(), SpecNumber, + m_jit.branchTest64( + MacroAssembler::Zero, gpr, GPRInfo::tagTypeNumberRegister)); + } else { + JITCompiler::Jump isNumber = m_jit.branchTest64(MacroAssembler::NonZero, gpr, GPRInfo::tagTypeNumberRegister); + + DFG_TYPE_CHECK( + JSValueRegs(gpr), node->child1(), ~SpecCell, + m_jit.branchTest64( + JITCompiler::Zero, gpr, GPRInfo::tagMaskRegister)); + + // It's not a cell: so true turns into 1 and all else turns into 0. + m_jit.compare64(JITCompiler::Equal, gpr, TrustedImm32(ValueTrue), resultGpr); + converted.append(m_jit.jump()); + + isNumber.link(&m_jit); + } // First, if we get here we have a double encoded as a JSValue m_jit.move(gpr, resultGpr); @@ -2121,21 +2251,23 @@ void SpeculativeJIT::compileValueToInt32(Node& node) callOperation(toInt32, resultGpr, fpr); silentFillAllRegisters(resultGpr); - JITCompiler::Jump converted = m_jit.jump(); + converted.append(m_jit.jump()); isInteger.link(&m_jit); m_jit.zeroExtend32ToPtr(gpr, resultGpr); converted.link(&m_jit); #else - Node& childNode = at(node.child1().index()); - VirtualRegister virtualRegister = childNode.virtualRegister(); + Node* childNode = node->child1().node(); + VirtualRegister virtualRegister = childNode->virtualRegister(); GenerationInfo& info = m_generationInfo[virtualRegister]; - JSValueOperand op1(this, node.child1()); + JSValueOperand op1(this, node->child1(), ManualOperandSpeculation); GPRReg payloadGPR = op1.payloadGPR(); GPRReg resultGpr = result.gpr(); + + JITCompiler::JumpList converted; if (info.registerFormat() == DataFormatJSInteger) m_jit.move(payloadGPR, resultGpr); @@ -2147,8 +2279,31 @@ void SpeculativeJIT::compileValueToInt32(Node& node) JITCompiler::Jump isInteger = m_jit.branch32(MacroAssembler::Equal, tagGPR, TrustedImm32(JSValue::Int32Tag)); - if (!isNumberSpeculation(m_state.forNode(node.child1()).m_type)) - speculationCheck(BadType, JSValueRegs(tagGPR, payloadGPR), node.child1().index(), m_jit.branch32(MacroAssembler::AboveOrEqual, tagGPR, TrustedImm32(JSValue::LowestTag))); + if (node->child1().useKind() == NumberUse) { + DFG_TYPE_CHECK( + JSValueRegs(tagGPR, payloadGPR), node->child1(), SpecNumber, + m_jit.branch32( + MacroAssembler::AboveOrEqual, tagGPR, + TrustedImm32(JSValue::LowestTag))); + } else { + JITCompiler::Jump isNumber = m_jit.branch32(MacroAssembler::Below, tagGPR, TrustedImm32(JSValue::LowestTag)); + + DFG_TYPE_CHECK( + JSValueRegs(tagGPR, payloadGPR), node->child1(), ~SpecCell, + m_jit.branch32( + JITCompiler::Equal, tagGPR, TrustedImm32(JSValue::CellTag))); + + // It's not a cell: so true turns into 1 and all else turns into 0. + JITCompiler::Jump isBoolean = m_jit.branch32(JITCompiler::Equal, tagGPR, TrustedImm32(JSValue::BooleanTag)); + m_jit.move(TrustedImm32(0), resultGpr); + converted.append(m_jit.jump()); + + isBoolean.link(&m_jit); + m_jit.move(payloadGPR, resultGpr); + converted.append(m_jit.jump()); + + isNumber.link(&m_jit); + } unboxDouble(tagGPR, payloadGPR, fpr, scratch.fpr()); @@ -2156,7 +2311,7 @@ void SpeculativeJIT::compileValueToInt32(Node& node) callOperation(toInt32, resultGpr, fpr); silentFillAllRegisters(resultGpr); - JITCompiler::Jump converted = m_jit.jump(); + converted.append(m_jit.jump()); isInteger.link(&m_jit); m_jit.move(payloadGPR, resultGpr); @@ -2164,38 +2319,41 @@ void SpeculativeJIT::compileValueToInt32(Node& node) converted.link(&m_jit); } #endif - integerResult(resultGpr, m_compileIndex); + integerResult(resultGpr, node); return; } case GeneratedOperandTypeUnknown: - ASSERT_NOT_REACHED(); - break; + RELEASE_ASSERT(!m_compileOkay); + return; } + RELEASE_ASSERT_NOT_REACHED(); + return; } - if (at(node.child1()).shouldSpeculateBoolean()) { - SpeculateBooleanOperand op1(this, node.child1()); + case BooleanUse: { + SpeculateBooleanOperand op1(this, node->child1()); GPRTemporary result(this, op1); m_jit.move(op1.gpr(), result.gpr()); m_jit.and32(JITCompiler::TrustedImm32(1), result.gpr()); - integerResult(result.gpr(), m_compileIndex); + integerResult(result.gpr(), node); + return; + } + + default: + ASSERT(!m_compileOkay); return; } - - // Do it the safe way. - nonSpeculativeValueToInt32(node); - return; } -void SpeculativeJIT::compileUInt32ToNumber(Node& node) +void SpeculativeJIT::compileUInt32ToNumber(Node* node) { - if (!nodeCanSpeculateInteger(node.arithNodeFlags())) { + if (!nodeCanSpeculateInteger(node->arithNodeFlags())) { // We know that this sometimes produces doubles. So produce a double every // time. This at least allows subsequent code to not have weird conditionals. - IntegerOperand op1(this, node.child1()); + IntegerOperand op1(this, node->child1()); FPRTemporary result(this); GPRReg inputGPR = op1.gpr(); @@ -2207,11 +2365,11 @@ void SpeculativeJIT::compileUInt32ToNumber(Node& node) m_jit.addDouble(JITCompiler::AbsoluteAddress(&AssemblyHelpers::twoToThe32), outputFPR); positive.link(&m_jit); - doubleResult(outputFPR, m_compileIndex); + doubleResult(outputFPR, node); return; } - IntegerOperand op1(this, node.child1()); + IntegerOperand op1(this, node->child1()); GPRTemporary result(this); // For the benefit of OSR exit, force these to be in different registers. In reality the OSR exit compiler could find cases where you have uint32(%r1) followed by int32(%r1) and then use different registers, but that seems like too much effort. m_jit.move(op1.gpr(), result.gpr()); @@ -2221,14 +2379,14 @@ void SpeculativeJIT::compileUInt32ToNumber(Node& node) // instruction that follows us, rather than the one we're executing right now. We have // to do this because by this point, the original values necessary to compile whatever // operation the UInt32ToNumber originated from might be dead. - forwardSpeculationCheck(Overflow, JSValueRegs(), NoNode, m_jit.branch32(MacroAssembler::LessThan, result.gpr(), TrustedImm32(0)), ValueRecovery::uint32InGPR(result.gpr())); + forwardSpeculationCheck(Overflow, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::LessThan, result.gpr(), TrustedImm32(0)), ValueRecovery::uint32InGPR(result.gpr())); - integerResult(result.gpr(), m_compileIndex, op1.format()); + integerResult(result.gpr(), node, op1.format()); } -void SpeculativeJIT::compileDoubleAsInt32(Node& node) +void SpeculativeJIT::compileDoubleAsInt32(Node* node) { - SpeculateDoubleOperand op1(this, node.child1()); + SpeculateDoubleOperand op1(this, node->child1()); FPRTemporary scratch(this); GPRTemporary result(this); @@ -2237,38 +2395,26 @@ void SpeculativeJIT::compileDoubleAsInt32(Node& node) GPRReg resultGPR = result.gpr(); JITCompiler::JumpList failureCases; - m_jit.branchConvertDoubleToInt32(valueFPR, resultGPR, failureCases, scratchFPR); - forwardSpeculationCheck(Overflow, JSValueRegs(), NoNode, failureCases, ValueRecovery::inFPR(valueFPR)); + bool negZeroCheck = !nodeCanIgnoreNegativeZero(node->arithNodeFlags()); + m_jit.branchConvertDoubleToInt32(valueFPR, resultGPR, failureCases, scratchFPR, negZeroCheck); + forwardSpeculationCheck(Overflow, JSValueRegs(), 0, failureCases, ValueRecovery::inFPR(valueFPR)); - integerResult(resultGPR, m_compileIndex); + integerResult(resultGPR, node); } -void SpeculativeJIT::compileInt32ToDouble(Node& node) +void SpeculativeJIT::compileInt32ToDouble(Node* node) { -#if USE(JSVALUE64) - // On JSVALUE64 we have a way of loading double constants in a more direct manner - // than a int->double conversion. On 32_64, unfortunately, we currently don't have - // any such mechanism - though we could have it, if we just provisioned some memory - // in CodeBlock for the double form of integer constants. - if (isInt32Constant(node.child1().index())) { - FPRTemporary result(this); - GPRTemporary temp(this); - m_jit.move(MacroAssembler::Imm64(reinterpretDoubleToInt64(valueOfNumberConstant(node.child1().index()))), temp.gpr()); - m_jit.move64ToDouble(temp.gpr(), result.fpr()); - doubleResult(result.fpr(), m_compileIndex); - return; - } -#endif + ASSERT(!isInt32Constant(node->child1().node())); // This should have been constant folded. - if (isInt32Speculation(m_state.forNode(node.child1()).m_type)) { - SpeculateIntegerOperand op1(this, node.child1()); + if (isInt32Speculation(m_state.forNode(node->child1()).m_type)) { + SpeculateIntegerOperand op1(this, node->child1(), ManualOperandSpeculation); FPRTemporary result(this); m_jit.convertInt32ToDouble(op1.gpr(), result.fpr()); - doubleResult(result.fpr(), m_compileIndex); + doubleResult(result.fpr(), node); return; } - JSValueOperand op1(this, node.child1()); + JSValueOperand op1(this, node->child1(), ManualOperandSpeculation); FPRTemporary result(this); #if USE(JSVALUE64) @@ -2281,10 +2427,17 @@ void SpeculativeJIT::compileInt32ToDouble(Node& node) JITCompiler::Jump isInteger = m_jit.branch64( MacroAssembler::AboveOrEqual, op1GPR, GPRInfo::tagTypeNumberRegister); - if (!isNumberSpeculation(m_state.forNode(node.child1()).m_type)) { - speculationCheck( - BadType, JSValueRegs(op1GPR), node.child1(), - m_jit.branchTest64(MacroAssembler::Zero, op1GPR, GPRInfo::tagTypeNumberRegister)); + if (needsTypeCheck(node->child1(), SpecNumber)) { + if (node->op() == ForwardInt32ToDouble) { + forwardTypeCheck( + JSValueRegs(op1GPR), node->child1(), SpecNumber, + m_jit.branchTest64(MacroAssembler::Zero, op1GPR, GPRInfo::tagTypeNumberRegister), + ValueRecovery::inGPR(op1GPR, DataFormatJS)); + } else { + backwardTypeCheck( + JSValueRegs(op1GPR), node->child1(), SpecNumber, + m_jit.branchTest64(MacroAssembler::Zero, op1GPR, GPRInfo::tagTypeNumberRegister)); + } } m_jit.move(op1GPR, tempGPR); @@ -2305,10 +2458,17 @@ void SpeculativeJIT::compileInt32ToDouble(Node& node) JITCompiler::Jump isInteger = m_jit.branch32( MacroAssembler::Equal, op1TagGPR, TrustedImm32(JSValue::Int32Tag)); - if (!isNumberSpeculation(m_state.forNode(node.child1()).m_type)) { - speculationCheck( - BadType, JSValueRegs(op1TagGPR, op1PayloadGPR), node.child1(), - m_jit.branch32(MacroAssembler::AboveOrEqual, op1TagGPR, TrustedImm32(JSValue::LowestTag))); + if (needsTypeCheck(node->child1(), SpecNumber)) { + if (node->op() == ForwardInt32ToDouble) { + forwardTypeCheck( + JSValueRegs(op1TagGPR, op1PayloadGPR), node->child1(), SpecNumber, + m_jit.branch32(MacroAssembler::AboveOrEqual, op1TagGPR, TrustedImm32(JSValue::LowestTag)), + ValueRecovery::inPair(op1TagGPR, op1PayloadGPR)); + } else { + backwardTypeCheck( + JSValueRegs(op1TagGPR, op1PayloadGPR), node->child1(), SpecNumber, + m_jit.branch32(MacroAssembler::AboveOrEqual, op1TagGPR, TrustedImm32(JSValue::LowestTag))); + } } unboxDouble(op1TagGPR, op1PayloadGPR, resultFPR, tempFPR); @@ -2319,7 +2479,7 @@ void SpeculativeJIT::compileInt32ToDouble(Node& node) done.link(&m_jit); #endif - doubleResult(resultFPR, m_compileIndex); + doubleResult(resultFPR, node); } static double clampDoubleToByte(double d) @@ -2374,11 +2534,11 @@ static void compileClampDoubleToByte(JITCompiler& jit, GPRReg result, FPRReg sou } -void SpeculativeJIT::compileGetByValOnIntTypedArray(const TypedArrayDescriptor& descriptor, Node& node, size_t elementSize, TypedArraySignedness signedness) +void SpeculativeJIT::compileGetByValOnIntTypedArray(const TypedArrayDescriptor& descriptor, Node* node, size_t elementSize, TypedArraySignedness signedness) { - SpeculateCellOperand base(this, node.child1()); - SpeculateStrictInt32Operand property(this, node.child2()); - StorageOperand storage(this, node.child3()); + SpeculateCellOperand base(this, node->child1()); + SpeculateStrictInt32Operand property(this, node->child2()); + StorageOperand storage(this, node->child3()); GPRReg baseReg = base.gpr(); GPRReg propertyReg = property.gpr(); @@ -2387,10 +2547,10 @@ void SpeculativeJIT::compileGetByValOnIntTypedArray(const TypedArrayDescriptor& GPRTemporary result(this); GPRReg resultReg = result.gpr(); - ASSERT(node.arrayMode().alreadyChecked(m_jit.graph(), node, m_state.forNode(node.child1()))); + ASSERT(node->arrayMode().alreadyChecked(m_jit.graph(), node, m_state.forNode(node->child1()))); speculationCheck( - Uncountable, JSValueRegs(), NoNode, + Uncountable, JSValueRegs(), 0, m_jit.branch32( MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(baseReg, descriptor.m_lengthOffset))); switch (elementSize) { @@ -2413,14 +2573,14 @@ void SpeculativeJIT::compileGetByValOnIntTypedArray(const TypedArrayDescriptor& CRASH(); } if (elementSize < 4 || signedness == SignedTypedArray) { - integerResult(resultReg, m_compileIndex); + integerResult(resultReg, node); return; } ASSERT(elementSize == 4 && signedness == UnsignedTypedArray); - if (node.shouldSpeculateInteger()) { - forwardSpeculationCheck(Overflow, JSValueRegs(), NoNode, m_jit.branch32(MacroAssembler::LessThan, resultReg, TrustedImm32(0)), ValueRecovery::uint32InGPR(resultReg)); - integerResult(resultReg, m_compileIndex); + if (node->shouldSpeculateInteger()) { + forwardSpeculationCheck(Overflow, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::LessThan, resultReg, TrustedImm32(0)), ValueRecovery::uint32InGPR(resultReg)); + integerResult(resultReg, node); return; } @@ -2429,10 +2589,10 @@ void SpeculativeJIT::compileGetByValOnIntTypedArray(const TypedArrayDescriptor& JITCompiler::Jump positive = m_jit.branch32(MacroAssembler::GreaterThanOrEqual, resultReg, TrustedImm32(0)); m_jit.addDouble(JITCompiler::AbsoluteAddress(&AssemblyHelpers::twoToThe32), fresult.fpr()); positive.link(&m_jit); - doubleResult(fresult.fpr(), m_compileIndex); + doubleResult(fresult.fpr(), node); } -void SpeculativeJIT::compilePutByValForIntTypedArray(const TypedArrayDescriptor& descriptor, GPRReg base, GPRReg property, Node& node, size_t elementSize, TypedArraySignedness signedness, TypedArrayRounding rounding) +void SpeculativeJIT::compilePutByValForIntTypedArray(const TypedArrayDescriptor& descriptor, GPRReg base, GPRReg property, Node* node, size_t elementSize, TypedArraySignedness signedness, TypedArrayRounding rounding) { StorageOperand storage(this, m_jit.graph().varArgChild(node, 3)); GPRReg storageReg = storage.gpr(); @@ -2440,13 +2600,13 @@ void SpeculativeJIT::compilePutByValForIntTypedArray(const TypedArrayDescriptor& Edge valueUse = m_jit.graph().varArgChild(node, 2); GPRTemporary value; - GPRReg valueGPR; + GPRReg valueGPR = InvalidGPRReg; - if (at(valueUse).isConstant()) { - JSValue jsValue = valueOfJSConstant(valueUse.index()); + if (valueUse->isConstant()) { + JSValue jsValue = valueOfJSConstant(valueUse.node()); if (!jsValue.isNumber()) { - terminateSpeculativeExecution(Uncountable, JSValueRegs(), NoNode); - noResult(m_compileIndex); + terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); + noResult(node); return; } double d = jsValue.asNumber(); @@ -2459,54 +2619,69 @@ void SpeculativeJIT::compilePutByValForIntTypedArray(const TypedArrayDescriptor& m_jit.move(Imm32(toInt32(d)), scratchReg); value.adopt(scratch); valueGPR = scratchReg; - } else if (at(valueUse).shouldSpeculateInteger()) { - SpeculateIntegerOperand valueOp(this, valueUse); - GPRTemporary scratch(this); - GPRReg scratchReg = scratch.gpr(); - m_jit.move(valueOp.gpr(), scratchReg); - if (rounding == ClampRounding) { - ASSERT(elementSize == 1); - compileClampIntegerToByte(m_jit, scratchReg); - } - value.adopt(scratch); - valueGPR = scratchReg; - } else if (rounding == ClampRounding) { - ASSERT(elementSize == 1); - SpeculateDoubleOperand valueOp(this, valueUse); - GPRTemporary result(this); - FPRTemporary floatScratch(this); - FPRReg fpr = valueOp.fpr(); - GPRReg gpr = result.gpr(); - compileClampDoubleToByte(m_jit, gpr, fpr, floatScratch.fpr()); - value.adopt(result); - valueGPR = gpr; } else { - SpeculateDoubleOperand valueOp(this, valueUse); - GPRTemporary result(this); - FPRReg fpr = valueOp.fpr(); - GPRReg gpr = result.gpr(); - MacroAssembler::Jump notNaN = m_jit.branchDouble(MacroAssembler::DoubleEqual, fpr, fpr); - m_jit.xorPtr(gpr, gpr); - MacroAssembler::Jump fixed = m_jit.jump(); - notNaN.link(&m_jit); - - MacroAssembler::Jump failed; - if (signedness == SignedTypedArray) - failed = m_jit.branchTruncateDoubleToInt32(fpr, gpr, MacroAssembler::BranchIfTruncateFailed); - else - failed = m_jit.branchTruncateDoubleToUint32(fpr, gpr, MacroAssembler::BranchIfTruncateFailed); - - addSlowPathGenerator(slowPathCall(failed, this, toInt32, gpr, fpr)); - - fixed.link(&m_jit); - value.adopt(result); - valueGPR = gpr; + switch (valueUse.useKind()) { + case Int32Use: { + SpeculateIntegerOperand valueOp(this, valueUse); + GPRTemporary scratch(this); + GPRReg scratchReg = scratch.gpr(); + m_jit.move(valueOp.gpr(), scratchReg); + if (rounding == ClampRounding) { + ASSERT(elementSize == 1); + compileClampIntegerToByte(m_jit, scratchReg); + } + value.adopt(scratch); + valueGPR = scratchReg; + break; + } + + case NumberUse: { + if (rounding == ClampRounding) { + ASSERT(elementSize == 1); + SpeculateDoubleOperand valueOp(this, valueUse); + GPRTemporary result(this); + FPRTemporary floatScratch(this); + FPRReg fpr = valueOp.fpr(); + GPRReg gpr = result.gpr(); + compileClampDoubleToByte(m_jit, gpr, fpr, floatScratch.fpr()); + value.adopt(result); + valueGPR = gpr; + } else { + SpeculateDoubleOperand valueOp(this, valueUse); + GPRTemporary result(this); + FPRReg fpr = valueOp.fpr(); + GPRReg gpr = result.gpr(); + MacroAssembler::Jump notNaN = m_jit.branchDouble(MacroAssembler::DoubleEqual, fpr, fpr); + m_jit.xorPtr(gpr, gpr); + MacroAssembler::Jump fixed = m_jit.jump(); + notNaN.link(&m_jit); + + MacroAssembler::Jump failed; + if (signedness == SignedTypedArray) + failed = m_jit.branchTruncateDoubleToInt32(fpr, gpr, MacroAssembler::BranchIfTruncateFailed); + else + failed = m_jit.branchTruncateDoubleToUint32(fpr, gpr, MacroAssembler::BranchIfTruncateFailed); + + addSlowPathGenerator(slowPathCall(failed, this, toInt32, gpr, fpr)); + + fixed.link(&m_jit); + value.adopt(result); + valueGPR = gpr; + } + break; + } + + default: + RELEASE_ASSERT_NOT_REACHED(); + break; + } } + ASSERT_UNUSED(valueGPR, valueGPR != property); ASSERT(valueGPR != base); ASSERT(valueGPR != storageReg); MacroAssembler::Jump outOfBounds; - if (node.op() == PutByVal) + if (node->op() == PutByVal) outOfBounds = m_jit.branch32(MacroAssembler::AboveOrEqual, property, MacroAssembler::Address(base, descriptor.m_lengthOffset)); switch (elementSize) { @@ -2522,27 +2697,27 @@ void SpeculativeJIT::compilePutByValForIntTypedArray(const TypedArrayDescriptor& default: CRASH(); } - if (node.op() == PutByVal) + if (node->op() == PutByVal) outOfBounds.link(&m_jit); - noResult(m_compileIndex); + noResult(node); } -void SpeculativeJIT::compileGetByValOnFloatTypedArray(const TypedArrayDescriptor& descriptor, Node& node, size_t elementSize) +void SpeculativeJIT::compileGetByValOnFloatTypedArray(const TypedArrayDescriptor& descriptor, Node* node, size_t elementSize) { - SpeculateCellOperand base(this, node.child1()); - SpeculateStrictInt32Operand property(this, node.child2()); - StorageOperand storage(this, node.child3()); + SpeculateCellOperand base(this, node->child1()); + SpeculateStrictInt32Operand property(this, node->child2()); + StorageOperand storage(this, node->child3()); GPRReg baseReg = base.gpr(); GPRReg propertyReg = property.gpr(); GPRReg storageReg = storage.gpr(); - ASSERT(node.arrayMode().alreadyChecked(m_jit.graph(), node, m_state.forNode(node.child1()))); + ASSERT(node->arrayMode().alreadyChecked(m_jit.graph(), node, m_state.forNode(node->child1()))); FPRTemporary result(this); FPRReg resultReg = result.fpr(); speculationCheck( - Uncountable, JSValueRegs(), NoNode, + Uncountable, JSValueRegs(), 0, m_jit.branch32( MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(baseReg, descriptor.m_lengthOffset))); switch (elementSize) { @@ -2552,60 +2727,62 @@ void SpeculativeJIT::compileGetByValOnFloatTypedArray(const TypedArrayDescriptor break; case 8: { m_jit.loadDouble(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight), resultReg); - MacroAssembler::Jump notNaN = m_jit.branchDouble(MacroAssembler::DoubleEqual, resultReg, resultReg); - static const double NaN = QNaN; - m_jit.loadDouble(&NaN, resultReg); - notNaN.link(&m_jit); break; } default: - ASSERT_NOT_REACHED(); + RELEASE_ASSERT_NOT_REACHED(); } - doubleResult(resultReg, m_compileIndex); + + MacroAssembler::Jump notNaN = m_jit.branchDouble(MacroAssembler::DoubleEqual, resultReg, resultReg); + static const double NaN = QNaN; + m_jit.loadDouble(&NaN, resultReg); + notNaN.link(&m_jit); + + doubleResult(resultReg, node); } -void SpeculativeJIT::compilePutByValForFloatTypedArray(const TypedArrayDescriptor& descriptor, GPRReg base, GPRReg property, Node& node, size_t elementSize) +void SpeculativeJIT::compilePutByValForFloatTypedArray(const TypedArrayDescriptor& descriptor, GPRReg base, GPRReg property, Node* node, size_t elementSize) { StorageOperand storage(this, m_jit.graph().varArgChild(node, 3)); GPRReg storageReg = storage.gpr(); Edge baseUse = m_jit.graph().varArgChild(node, 0); Edge valueUse = m_jit.graph().varArgChild(node, 2); - + SpeculateDoubleOperand valueOp(this, valueUse); - - ASSERT_UNUSED(baseUse, node.arrayMode().alreadyChecked(m_jit.graph(), m_jit.graph()[m_compileIndex], m_state.forNode(baseUse))); - - GPRTemporary result(this); + FPRTemporary scratch(this); + FPRReg valueFPR = valueOp.fpr(); + FPRReg scratchFPR = scratch.fpr(); + + ASSERT_UNUSED(baseUse, node->arrayMode().alreadyChecked(m_jit.graph(), node, m_state.forNode(baseUse))); MacroAssembler::Jump outOfBounds; - if (node.op() == PutByVal) + if (node->op() == PutByVal) outOfBounds = m_jit.branch32(MacroAssembler::AboveOrEqual, property, MacroAssembler::Address(base, descriptor.m_lengthOffset)); switch (elementSize) { case 4: { - FPRTemporary scratch(this); - m_jit.moveDouble(valueOp.fpr(), scratch.fpr()); - m_jit.convertDoubleToFloat(valueOp.fpr(), scratch.fpr()); - m_jit.storeFloat(scratch.fpr(), MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesFour)); + m_jit.moveDouble(valueFPR, scratchFPR); + m_jit.convertDoubleToFloat(valueFPR, scratchFPR); + m_jit.storeFloat(scratchFPR, MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesFour)); break; } case 8: - m_jit.storeDouble(valueOp.fpr(), MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesEight)); + m_jit.storeDouble(valueFPR, MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesEight)); break; default: - ASSERT_NOT_REACHED(); + RELEASE_ASSERT_NOT_REACHED(); } - if (node.op() == PutByVal) + if (node->op() == PutByVal) outOfBounds.link(&m_jit); - noResult(m_compileIndex); + noResult(node); } -void SpeculativeJIT::compileInstanceOfForObject(Node&, GPRReg valueReg, GPRReg prototypeReg, GPRReg scratchReg) +void SpeculativeJIT::compileInstanceOfForObject(Node*, GPRReg valueReg, GPRReg prototypeReg, GPRReg scratchReg) { // Check that prototype is an object. m_jit.loadPtr(MacroAssembler::Address(prototypeReg, JSCell::structureOffset()), scratchReg); - speculationCheck(BadType, JSValueRegs(), NoNode, m_jit.branchIfNotObject(scratchReg)); + speculationCheck(BadType, JSValueRegs(), 0, m_jit.branchIfNotObject(scratchReg)); // Initialize scratchReg with the value being checked. m_jit.move(valueReg, scratchReg); @@ -2641,17 +2818,15 @@ void SpeculativeJIT::compileInstanceOfForObject(Node&, GPRReg valueReg, GPRReg p putResult.link(&m_jit); } -void SpeculativeJIT::compileInstanceOf(Node& node) +void SpeculativeJIT::compileInstanceOf(Node* node) { - if ((!!(at(node.child1()).prediction() & ~SpecCell) - && !!(m_state.forNode(node.child1()).m_type & ~SpecCell)) - || at(node.child1()).adjustedRefCount() == 1) { + if (node->child1().useKind() == UntypedUse) { // It might not be a cell. Speculate less aggressively. // Or: it might only be used once (i.e. by us), so we get zero benefit // from speculating any more aggressively than we absolutely need to. - JSValueOperand value(this, node.child1()); - SpeculateCellOperand prototype(this, node.child2()); + JSValueOperand value(this, node->child1()); + SpeculateCellOperand prototype(this, node->child2()); GPRTemporary scratch(this); GPRReg prototypeReg = prototype.gpr(); @@ -2677,15 +2852,15 @@ void SpeculativeJIT::compileInstanceOf(Node& node) done.link(&m_jit); #if USE(JSVALUE64) - jsValueResult(scratchReg, m_compileIndex, DataFormatJSBoolean); + jsValueResult(scratchReg, node, DataFormatJSBoolean); #else - booleanResult(scratchReg, m_compileIndex); + booleanResult(scratchReg, node); #endif return; } - SpeculateCellOperand value(this, node.child1()); - SpeculateCellOperand prototype(this, node.child2()); + SpeculateCellOperand value(this, node->child1()); + SpeculateCellOperand prototype(this, node->child2()); GPRTemporary scratch(this); @@ -2696,20 +2871,20 @@ void SpeculativeJIT::compileInstanceOf(Node& node) compileInstanceOfForObject(node, valueReg, prototypeReg, scratchReg); #if USE(JSVALUE64) - jsValueResult(scratchReg, m_compileIndex, DataFormatJSBoolean); + jsValueResult(scratchReg, node, DataFormatJSBoolean); #else - booleanResult(scratchReg, m_compileIndex); + booleanResult(scratchReg, node); #endif } -void SpeculativeJIT::compileSoftModulo(Node& node) +void SpeculativeJIT::compileSoftModulo(Node* node) { // In the fast path, the dividend value could be the final result // (in case of |dividend| < |divisor|), so we speculate it as strict int32. - SpeculateStrictInt32Operand op1(this, node.child1()); + SpeculateStrictInt32Operand op1(this, node->child1()); #if CPU(X86) || CPU(X86_64) - if (isInt32Constant(node.child2().index())) { - int32_t divisor = valueOfInt32Constant(node.child2().index()); + if (isInt32Constant(node->child2().node())) { + int32_t divisor = valueOfInt32Constant(node->child2().node()); if (divisor) { GPRReg op1Gpr = op1.gpr(); @@ -2731,25 +2906,51 @@ void SpeculativeJIT::compileSoftModulo(Node& node) m_jit.move(op1Gpr, eax.gpr()); m_jit.move(TrustedImm32(divisor), scratchGPR); if (divisor == -1) - speculationCheck(Overflow, JSValueRegs(), NoNode, m_jit.branch32(JITCompiler::Equal, eax.gpr(), TrustedImm32(-2147483647-1))); + speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branch32(JITCompiler::Equal, eax.gpr(), TrustedImm32(-2147483647-1))); m_jit.assembler().cdq(); m_jit.assembler().idivl_r(scratchGPR); - // Check that we're not about to create negative zero. - // FIXME: if the node use doesn't care about neg zero, we can do this more easily. - JITCompiler::Jump numeratorPositive = m_jit.branch32(JITCompiler::GreaterThanOrEqual, op1SaveGPR, TrustedImm32(0)); - speculationCheck(Overflow, JSValueRegs(), NoNode, m_jit.branchTest32(JITCompiler::Zero, edx.gpr())); - numeratorPositive.link(&m_jit); - + if (!nodeCanIgnoreNegativeZero(node->arithNodeFlags())) { + // Check that we're not about to create negative zero. + JITCompiler::Jump numeratorPositive = m_jit.branch32(JITCompiler::GreaterThanOrEqual, op1SaveGPR, TrustedImm32(0)); + speculationCheck(NegativeZero, JSValueRegs(), 0, m_jit.branchTest32(JITCompiler::Zero, edx.gpr())); + numeratorPositive.link(&m_jit); + } if (op1SaveGPR != op1Gpr) unlock(op1SaveGPR); - integerResult(edx.gpr(), m_compileIndex); + integerResult(edx.gpr(), node); + return; + } + } +#elif CPU(APPLE_ARMV7S) || CPU(ARM_THUMB2) + if (isInt32Constant(node->child2().node())) { + int32_t divisor = valueOfInt32Constant(node->child2().node()); + if (divisor > 0 && hasOneBitSet(divisor)) { // If power of 2 then just mask + GPRReg dividendGPR = op1.gpr(); + GPRTemporary result(this); + GPRReg resultGPR = result.gpr(); + + m_jit.assembler().cmp(dividendGPR, ARMThumbImmediate::makeEncodedImm(0)); + m_jit.assembler().it(ARMv7Assembler::ConditionLT, false); + m_jit.assembler().neg(resultGPR, dividendGPR); + m_jit.assembler().mov(resultGPR, dividendGPR); + m_jit.and32(TrustedImm32(divisor - 1), resultGPR); + m_jit.assembler().it(ARMv7Assembler::ConditionLT); + m_jit.assembler().neg(resultGPR, resultGPR); + + if (!nodeCanIgnoreNegativeZero(node->arithNodeFlags())) { + // Check that we're not about to create negative zero. + JITCompiler::Jump numeratorPositive = m_jit.branch32(JITCompiler::GreaterThanOrEqual, dividendGPR, TrustedImm32(0)); + speculationCheck(NegativeZero, JSValueRegs(), 0, m_jit.branchTest32(JITCompiler::Zero, resultGPR)); + numeratorPositive.link(&m_jit); + } + integerResult(resultGPR, node); return; } } #endif - SpeculateIntegerOperand op2(this, node.child2()); + SpeculateIntegerOperand op2(this, node->child2()); #if CPU(X86) || CPU(X86_64) GPRTemporary eax(this, X86Registers::eax); GPRTemporary edx(this, X86Registers::edx); @@ -2789,8 +2990,8 @@ void SpeculativeJIT::compileSoftModulo(Node& node) JITCompiler::Jump done; // FIXME: if the node is not used as number then we can do this more easily. - speculationCheck(Overflow, JSValueRegs(), NoNode, m_jit.branchTest32(JITCompiler::Zero, op2GPR)); - speculationCheck(Overflow, JSValueRegs(), NoNode, m_jit.branch32(JITCompiler::Equal, op1GPR, TrustedImm32(-2147483647-1))); + speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branchTest32(JITCompiler::Zero, op2GPR)); + speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branch32(JITCompiler::Equal, op1GPR, TrustedImm32(-2147483647-1))); safeDenominator.link(&m_jit); @@ -2806,17 +3007,42 @@ void SpeculativeJIT::compileSoftModulo(Node& node) if (op2TempGPR != InvalidGPRReg) unlock(op2TempGPR); - // Check that we're not about to create negative zero. - // FIXME: if the node use doesn't care about neg zero, we can do this more easily. - JITCompiler::Jump numeratorPositive = m_jit.branch32(JITCompiler::GreaterThanOrEqual, op1SaveGPR, TrustedImm32(0)); - speculationCheck(Overflow, JSValueRegs(), NoNode, m_jit.branchTest32(JITCompiler::Zero, edx.gpr())); - numeratorPositive.link(&m_jit); + if (!nodeCanIgnoreNegativeZero(node->arithNodeFlags())) { + // Check that we're not about to create negative zero. + JITCompiler::Jump numeratorPositive = m_jit.branch32(JITCompiler::GreaterThanOrEqual, op1SaveGPR, TrustedImm32(0)); + speculationCheck(NegativeZero, JSValueRegs(), 0, m_jit.branchTest32(JITCompiler::Zero, edx.gpr())); + numeratorPositive.link(&m_jit); + } if (op1SaveGPR != op1GPR) unlock(op1SaveGPR); - integerResult(edx.gpr(), m_compileIndex); -#else // CPU(X86) || CPU(X86_64) --> so not X86 + integerResult(edx.gpr(), node); + +#elif CPU(APPLE_ARMV7S) + GPRTemporary temp(this); + GPRTemporary quotientThenRemainder(this); + GPRTemporary multiplyAnswer(this); + GPRReg dividendGPR = op1.gpr(); + GPRReg divisorGPR = op2.gpr(); + GPRReg quotientThenRemainderGPR = quotientThenRemainder.gpr(); + GPRReg multiplyAnswerGPR = multiplyAnswer.gpr(); + + m_jit.assembler().sdiv(quotientThenRemainderGPR, dividendGPR, divisorGPR); + speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branchMul32(JITCompiler::Overflow, quotientThenRemainderGPR, divisorGPR, multiplyAnswerGPR)); + m_jit.assembler().sub(quotientThenRemainderGPR, dividendGPR, multiplyAnswerGPR); + + // If the user cares about negative zero, then speculate that we're not about + // to produce negative zero. + if (!nodeCanIgnoreNegativeZero(node->arithNodeFlags())) { + // Check that we're not about to create negative zero. + JITCompiler::Jump numeratorPositive = m_jit.branch32(JITCompiler::GreaterThanOrEqual, dividendGPR, TrustedImm32(0)); + speculationCheck(NegativeZero, JSValueRegs(), 0, m_jit.branchTest32(JITCompiler::Zero, quotientThenRemainderGPR)); + numeratorPositive.link(&m_jit); + } + + integerResult(quotientThenRemainderGPR, node); +#else // not architecture that can do integer division // Do this the *safest* way possible: call out to a C function that will do the modulo, // and then attempt to convert back. GPRReg op1GPR = op1.gpr(); @@ -2830,55 +3056,62 @@ void SpeculativeJIT::compileSoftModulo(Node& node) FPRTemporary scratch(this); GPRTemporary intResult(this); JITCompiler::JumpList failureCases; - m_jit.branchConvertDoubleToInt32(result.fpr(), intResult.gpr(), failureCases, scratch.fpr()); - speculationCheck(Overflow, JSValueRegs(), NoNode, failureCases); + m_jit.branchConvertDoubleToInt32(result.fpr(), intResult.gpr(), failureCases, scratch.fpr(), false); + speculationCheck(Overflow, JSValueRegs(), 0, failureCases); + if (!nodeCanIgnoreNegativeZero(node->arithNodeFlags())) { + // Check that we're not about to create negative zero. + JITCompiler::Jump numeratorPositive = m_jit.branch32(JITCompiler::GreaterThanOrEqual, op1GPR, TrustedImm32(0)); + speculationCheck(NegativeZero, JSValueRegs(), 0, m_jit.branchTest32(JITCompiler::Zero, intResult.gpr())); + numeratorPositive.link(&m_jit); + } - integerResult(intResult.gpr(), m_compileIndex); + integerResult(intResult.gpr(), node); #endif // CPU(X86) || CPU(X86_64) } -void SpeculativeJIT::compileAdd(Node& node) +void SpeculativeJIT::compileAdd(Node* node) { - if (m_jit.graph().addShouldSpeculateInteger(node)) { - if (isNumberConstant(node.child1().index())) { - int32_t imm1 = valueOfNumberConstantAsInt32(node.child1().index()); - SpeculateIntegerOperand op2(this, node.child2()); + switch (node->binaryUseKind()) { + case Int32Use: { + if (isNumberConstant(node->child1().node())) { + int32_t imm1 = valueOfInt32Constant(node->child1().node()); + SpeculateIntegerOperand op2(this, node->child2()); GPRTemporary result(this); - if (nodeCanTruncateInteger(node.arithNodeFlags())) { + if (nodeCanTruncateInteger(node->arithNodeFlags())) { m_jit.move(op2.gpr(), result.gpr()); m_jit.add32(Imm32(imm1), result.gpr()); } else - speculationCheck(Overflow, JSValueRegs(), NoNode, m_jit.branchAdd32(MacroAssembler::Overflow, op2.gpr(), Imm32(imm1), result.gpr())); + speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branchAdd32(MacroAssembler::Overflow, op2.gpr(), Imm32(imm1), result.gpr())); - integerResult(result.gpr(), m_compileIndex); + integerResult(result.gpr(), node); return; } - if (isNumberConstant(node.child2().index())) { - SpeculateIntegerOperand op1(this, node.child1()); - int32_t imm2 = valueOfNumberConstantAsInt32(node.child2().index()); + if (isNumberConstant(node->child2().node())) { + SpeculateIntegerOperand op1(this, node->child1()); + int32_t imm2 = valueOfInt32Constant(node->child2().node()); GPRTemporary result(this); - if (nodeCanTruncateInteger(node.arithNodeFlags())) { + if (nodeCanTruncateInteger(node->arithNodeFlags())) { m_jit.move(op1.gpr(), result.gpr()); m_jit.add32(Imm32(imm2), result.gpr()); } else - speculationCheck(Overflow, JSValueRegs(), NoNode, m_jit.branchAdd32(MacroAssembler::Overflow, op1.gpr(), Imm32(imm2), result.gpr())); + speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branchAdd32(MacroAssembler::Overflow, op1.gpr(), Imm32(imm2), result.gpr())); - integerResult(result.gpr(), m_compileIndex); + integerResult(result.gpr(), node); return; } - SpeculateIntegerOperand op1(this, node.child1()); - SpeculateIntegerOperand op2(this, node.child2()); + SpeculateIntegerOperand op1(this, node->child1()); + SpeculateIntegerOperand op2(this, node->child2()); GPRTemporary result(this, op1, op2); GPRReg gpr1 = op1.gpr(); GPRReg gpr2 = op2.gpr(); GPRReg gprResult = result.gpr(); - if (nodeCanTruncateInteger(node.arithNodeFlags())) { + if (nodeCanTruncateInteger(node->arithNodeFlags())) { if (gpr1 == gprResult) m_jit.add32(gpr2, gprResult); else { @@ -2889,136 +3122,237 @@ void SpeculativeJIT::compileAdd(Node& node) MacroAssembler::Jump check = m_jit.branchAdd32(MacroAssembler::Overflow, gpr1, gpr2, gprResult); if (gpr1 == gprResult) - speculationCheck(Overflow, JSValueRegs(), NoNode, check, SpeculationRecovery(SpeculativeAdd, gprResult, gpr2)); + speculationCheck(Overflow, JSValueRegs(), 0, check, SpeculationRecovery(SpeculativeAdd, gprResult, gpr2)); else if (gpr2 == gprResult) - speculationCheck(Overflow, JSValueRegs(), NoNode, check, SpeculationRecovery(SpeculativeAdd, gprResult, gpr1)); + speculationCheck(Overflow, JSValueRegs(), 0, check, SpeculationRecovery(SpeculativeAdd, gprResult, gpr1)); else - speculationCheck(Overflow, JSValueRegs(), NoNode, check); + speculationCheck(Overflow, JSValueRegs(), 0, check); } - integerResult(gprResult, m_compileIndex); + integerResult(gprResult, node); return; } - - if (Node::shouldSpeculateNumberExpectingDefined(at(node.child1()), at(node.child2()))) { - SpeculateDoubleOperand op1(this, node.child1()); - SpeculateDoubleOperand op2(this, node.child2()); + + case NumberUse: { + SpeculateDoubleOperand op1(this, node->child1()); + SpeculateDoubleOperand op2(this, node->child2()); FPRTemporary result(this, op1, op2); FPRReg reg1 = op1.fpr(); FPRReg reg2 = op2.fpr(); m_jit.addDouble(reg1, reg2, result.fpr()); - doubleResult(result.fpr(), m_compileIndex); + doubleResult(result.fpr(), node); return; } - - if (node.op() == ValueAdd) { + + case UntypedUse: { + RELEASE_ASSERT(node->op() == ValueAdd); compileValueAdd(node); return; } + + default: + RELEASE_ASSERT_NOT_REACHED(); + break; + } +} + +void SpeculativeJIT::compileMakeRope(Node* node) +{ + ASSERT(node->child1().useKind() == KnownStringUse); + ASSERT(node->child2().useKind() == KnownStringUse); + ASSERT(!node->child3() || node->child3().useKind() == KnownStringUse); + + SpeculateCellOperand op1(this, node->child1()); + SpeculateCellOperand op2(this, node->child2()); + SpeculateCellOperand op3(this, node->child3()); + GPRTemporary result(this); + GPRTemporary allocator(this); + GPRTemporary scratch(this); - // We don't handle this yet. :-( - terminateSpeculativeExecution(Uncountable, JSValueRegs(), NoNode); + GPRReg opGPRs[3]; + unsigned numOpGPRs; + opGPRs[0] = op1.gpr(); + opGPRs[1] = op2.gpr(); + if (node->child3()) { + opGPRs[2] = op3.gpr(); + numOpGPRs = 3; + } else { + opGPRs[2] = InvalidGPRReg; + numOpGPRs = 2; + } + GPRReg resultGPR = result.gpr(); + GPRReg allocatorGPR = allocator.gpr(); + GPRReg scratchGPR = scratch.gpr(); + + JITCompiler::JumpList slowPath; + MarkedAllocator& markedAllocator = m_jit.vm()->heap.allocatorForObjectWithImmortalStructureDestructor(sizeof(JSRopeString)); + m_jit.move(TrustedImmPtr(&markedAllocator), allocatorGPR); + emitAllocateJSCell(resultGPR, allocatorGPR, TrustedImmPtr(m_jit.vm()->stringStructure.get()), scratchGPR, slowPath); + + m_jit.storePtr(TrustedImmPtr(0), JITCompiler::Address(resultGPR, JSString::offsetOfValue())); + for (unsigned i = 0; i < numOpGPRs; ++i) + m_jit.storePtr(opGPRs[i], JITCompiler::Address(resultGPR, JSRopeString::offsetOfFibers() + sizeof(WriteBarrier<JSString>) * i)); + for (unsigned i = numOpGPRs; i < JSRopeString::s_maxInternalRopeLength; ++i) + m_jit.storePtr(TrustedImmPtr(0), JITCompiler::Address(resultGPR, JSRopeString::offsetOfFibers() + sizeof(WriteBarrier<JSString>) * i)); + m_jit.load32(JITCompiler::Address(opGPRs[0], JSString::offsetOfFlags()), scratchGPR); + m_jit.load32(JITCompiler::Address(opGPRs[0], JSString::offsetOfLength()), allocatorGPR); + for (unsigned i = 1; i < numOpGPRs; ++i) { + m_jit.and32(JITCompiler::Address(opGPRs[i], JSString::offsetOfFlags()), scratchGPR); + m_jit.add32(JITCompiler::Address(opGPRs[i], JSString::offsetOfLength()), allocatorGPR); + } + m_jit.and32(JITCompiler::TrustedImm32(JSString::Is8Bit), scratchGPR); + m_jit.store32(scratchGPR, JITCompiler::Address(resultGPR, JSString::offsetOfFlags())); + m_jit.store32(allocatorGPR, JITCompiler::Address(resultGPR, JSString::offsetOfLength())); + + switch (numOpGPRs) { + case 2: + addSlowPathGenerator(slowPathCall( + slowPath, this, operationMakeRope2, resultGPR, opGPRs[0], opGPRs[1])); + break; + case 3: + addSlowPathGenerator(slowPathCall( + slowPath, this, operationMakeRope3, resultGPR, opGPRs[0], opGPRs[1], opGPRs[2])); + break; + default: + RELEASE_ASSERT_NOT_REACHED(); + break; + } + + cellResult(resultGPR, node); } -void SpeculativeJIT::compileArithSub(Node& node) +void SpeculativeJIT::compileArithSub(Node* node) { - if (m_jit.graph().addShouldSpeculateInteger(node)) { - if (isNumberConstant(node.child2().index())) { - SpeculateIntegerOperand op1(this, node.child1()); - int32_t imm2 = valueOfNumberConstantAsInt32(node.child2().index()); + switch (node->binaryUseKind()) { + case Int32Use: { + if (isNumberConstant(node->child2().node())) { + SpeculateIntegerOperand op1(this, node->child1()); + int32_t imm2 = valueOfInt32Constant(node->child2().node()); GPRTemporary result(this); - if (nodeCanTruncateInteger(node.arithNodeFlags())) { + if (nodeCanTruncateInteger(node->arithNodeFlags())) { m_jit.move(op1.gpr(), result.gpr()); m_jit.sub32(Imm32(imm2), result.gpr()); } else { #if ENABLE(JIT_CONSTANT_BLINDING) GPRTemporary scratch(this); - speculationCheck(Overflow, JSValueRegs(), NoNode, m_jit.branchSub32(MacroAssembler::Overflow, op1.gpr(), Imm32(imm2), result.gpr(), scratch.gpr())); + speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branchSub32(MacroAssembler::Overflow, op1.gpr(), Imm32(imm2), result.gpr(), scratch.gpr())); #else - speculationCheck(Overflow, JSValueRegs(), NoNode, m_jit.branchSub32(MacroAssembler::Overflow, op1.gpr(), Imm32(imm2), result.gpr())); + speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branchSub32(MacroAssembler::Overflow, op1.gpr(), Imm32(imm2), result.gpr())); #endif } - integerResult(result.gpr(), m_compileIndex); + integerResult(result.gpr(), node); return; } - if (isNumberConstant(node.child1().index())) { - int32_t imm1 = valueOfNumberConstantAsInt32(node.child1().index()); - SpeculateIntegerOperand op2(this, node.child2()); + if (isNumberConstant(node->child1().node())) { + int32_t imm1 = valueOfInt32Constant(node->child1().node()); + SpeculateIntegerOperand op2(this, node->child2()); GPRTemporary result(this); m_jit.move(Imm32(imm1), result.gpr()); - if (nodeCanTruncateInteger(node.arithNodeFlags())) + if (nodeCanTruncateInteger(node->arithNodeFlags())) m_jit.sub32(op2.gpr(), result.gpr()); else - speculationCheck(Overflow, JSValueRegs(), NoNode, m_jit.branchSub32(MacroAssembler::Overflow, op2.gpr(), result.gpr())); + speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branchSub32(MacroAssembler::Overflow, op2.gpr(), result.gpr())); - integerResult(result.gpr(), m_compileIndex); + integerResult(result.gpr(), node); return; } - SpeculateIntegerOperand op1(this, node.child1()); - SpeculateIntegerOperand op2(this, node.child2()); + SpeculateIntegerOperand op1(this, node->child1()); + SpeculateIntegerOperand op2(this, node->child2()); GPRTemporary result(this); - if (nodeCanTruncateInteger(node.arithNodeFlags())) { + if (nodeCanTruncateInteger(node->arithNodeFlags())) { m_jit.move(op1.gpr(), result.gpr()); m_jit.sub32(op2.gpr(), result.gpr()); } else - speculationCheck(Overflow, JSValueRegs(), NoNode, m_jit.branchSub32(MacroAssembler::Overflow, op1.gpr(), op2.gpr(), result.gpr())); + speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branchSub32(MacroAssembler::Overflow, op1.gpr(), op2.gpr(), result.gpr())); - integerResult(result.gpr(), m_compileIndex); + integerResult(result.gpr(), node); return; } - SpeculateDoubleOperand op1(this, node.child1()); - SpeculateDoubleOperand op2(this, node.child2()); - FPRTemporary result(this, op1); + case NumberUse: { + SpeculateDoubleOperand op1(this, node->child1()); + SpeculateDoubleOperand op2(this, node->child2()); + FPRTemporary result(this, op1); - FPRReg reg1 = op1.fpr(); - FPRReg reg2 = op2.fpr(); - m_jit.subDouble(reg1, reg2, result.fpr()); + FPRReg reg1 = op1.fpr(); + FPRReg reg2 = op2.fpr(); + m_jit.subDouble(reg1, reg2, result.fpr()); - doubleResult(result.fpr(), m_compileIndex); + doubleResult(result.fpr(), node); + return; + } + + default: + RELEASE_ASSERT_NOT_REACHED(); + return; + } } -void SpeculativeJIT::compileArithNegate(Node& node) +void SpeculativeJIT::compileArithNegate(Node* node) { - if (m_jit.graph().negateShouldSpeculateInteger(node)) { - SpeculateIntegerOperand op1(this, node.child1()); + switch (node->child1().useKind()) { + case Int32Use: { + SpeculateIntegerOperand op1(this, node->child1()); GPRTemporary result(this); m_jit.move(op1.gpr(), result.gpr()); - if (nodeCanTruncateInteger(node.arithNodeFlags())) + if (nodeCanTruncateInteger(node->arithNodeFlags())) m_jit.neg32(result.gpr()); else { - speculationCheck(Overflow, JSValueRegs(), NoNode, m_jit.branchNeg32(MacroAssembler::Overflow, result.gpr())); - if (!nodeCanIgnoreNegativeZero(node.arithNodeFlags())) - speculationCheck(Overflow, JSValueRegs(), NoNode, m_jit.branchTest32(MacroAssembler::Zero, result.gpr())); + speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branchNeg32(MacroAssembler::Overflow, result.gpr())); + if (!nodeCanIgnoreNegativeZero(node->arithNodeFlags())) + speculationCheck(NegativeZero, JSValueRegs(), 0, m_jit.branchTest32(MacroAssembler::Zero, result.gpr())); } - integerResult(result.gpr(), m_compileIndex); + integerResult(result.gpr(), node); return; } - SpeculateDoubleOperand op1(this, node.child1()); - FPRTemporary result(this); + case NumberUse: { + SpeculateDoubleOperand op1(this, node->child1()); + FPRTemporary result(this); + + m_jit.negateDouble(op1.fpr(), result.fpr()); + + doubleResult(result.fpr(), node); + return; + } + + default: + RELEASE_ASSERT_NOT_REACHED(); + return; + } +} +void SpeculativeJIT::compileArithIMul(Node* node) +{ + SpeculateIntegerOperand op1(this, node->child1()); + SpeculateIntegerOperand op2(this, node->child2()); + GPRTemporary result(this); - m_jit.negateDouble(op1.fpr(), result.fpr()); + GPRReg reg1 = op1.gpr(); + GPRReg reg2 = op2.gpr(); - doubleResult(result.fpr(), m_compileIndex); + m_jit.move(reg1, result.gpr()); + m_jit.mul32(reg2, result.gpr()); + integerResult(result.gpr(), node); + return; } -void SpeculativeJIT::compileArithMul(Node& node) +void SpeculativeJIT::compileArithMul(Node* node) { - if (m_jit.graph().mulShouldSpeculateInteger(node)) { - SpeculateIntegerOperand op1(this, node.child1()); - SpeculateIntegerOperand op2(this, node.child2()); + switch (node->binaryUseKind()) { + case Int32Use: { + SpeculateIntegerOperand op1(this, node->child1()); + SpeculateIntegerOperand op2(this, node->child2()); GPRTemporary result(this); GPRReg reg1 = op1.gpr(); @@ -3027,44 +3361,52 @@ void SpeculativeJIT::compileArithMul(Node& node) // We can perform truncated multiplications if we get to this point, because if the // fixup phase could not prove that it would be safe, it would have turned us into // a double multiplication. - if (nodeCanTruncateInteger(node.arithNodeFlags())) { + if (nodeCanTruncateInteger(node->arithNodeFlags())) { m_jit.move(reg1, result.gpr()); m_jit.mul32(reg2, result.gpr()); } else { speculationCheck( - Overflow, JSValueRegs(), NoNode, + Overflow, JSValueRegs(), 0, m_jit.branchMul32(MacroAssembler::Overflow, reg1, reg2, result.gpr())); } // Check for negative zero, if the users of this node care about such things. - if (!nodeCanIgnoreNegativeZero(node.arithNodeFlags())) { + if (!nodeCanIgnoreNegativeZero(node->arithNodeFlags())) { MacroAssembler::Jump resultNonZero = m_jit.branchTest32(MacroAssembler::NonZero, result.gpr()); - speculationCheck(NegativeZero, JSValueRegs(), NoNode, m_jit.branch32(MacroAssembler::LessThan, reg1, TrustedImm32(0))); - speculationCheck(NegativeZero, JSValueRegs(), NoNode, m_jit.branch32(MacroAssembler::LessThan, reg2, TrustedImm32(0))); + speculationCheck(NegativeZero, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::LessThan, reg1, TrustedImm32(0))); + speculationCheck(NegativeZero, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::LessThan, reg2, TrustedImm32(0))); resultNonZero.link(&m_jit); } - integerResult(result.gpr(), m_compileIndex); + integerResult(result.gpr(), node); return; } - - SpeculateDoubleOperand op1(this, node.child1()); - SpeculateDoubleOperand op2(this, node.child2()); - FPRTemporary result(this, op1, op2); - - FPRReg reg1 = op1.fpr(); - FPRReg reg2 = op2.fpr(); - m_jit.mulDouble(reg1, reg2, result.fpr()); + case NumberUse: { + SpeculateDoubleOperand op1(this, node->child1()); + SpeculateDoubleOperand op2(this, node->child2()); + FPRTemporary result(this, op1, op2); + + FPRReg reg1 = op1.fpr(); + FPRReg reg2 = op2.fpr(); + + m_jit.mulDouble(reg1, reg2, result.fpr()); - doubleResult(result.fpr(), m_compileIndex); + doubleResult(result.fpr(), node); + return; + } + + default: + RELEASE_ASSERT_NOT_REACHED(); + return; + } } #if CPU(X86) || CPU(X86_64) -void SpeculativeJIT::compileIntegerArithDivForX86(Node& node) +void SpeculativeJIT::compileIntegerArithDivForX86(Node* node) { - SpeculateIntegerOperand op1(this, node.child1()); - SpeculateIntegerOperand op2(this, node.child2()); + SpeculateIntegerOperand op1(this, node->child1()); + SpeculateIntegerOperand op2(this, node->child2()); GPRTemporary eax(this, X86Registers::eax); GPRTemporary edx(this, X86Registers::edx); GPRReg op1GPR = op1.gpr(); @@ -3090,26 +3432,38 @@ void SpeculativeJIT::compileIntegerArithDivForX86(Node& node) JITCompiler::Jump safeDenominator = m_jit.branch32(JITCompiler::Above, temp, JITCompiler::TrustedImm32(1)); - JITCompiler::Jump done; - if (nodeUsedAsNumber(node.arithNodeFlags())) { - speculationCheck(Overflow, JSValueRegs(), NoNode, m_jit.branchTest32(JITCompiler::Zero, op2GPR)); - speculationCheck(Overflow, JSValueRegs(), NoNode, m_jit.branch32(JITCompiler::Equal, op1GPR, TrustedImm32(-2147483647-1))); + JITCompiler::JumpList done; + if (nodeUsedAsNumber(node->arithNodeFlags())) { + speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branchTest32(JITCompiler::Zero, op2GPR)); + speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branch32(JITCompiler::Equal, op1GPR, TrustedImm32(-2147483647-1))); } else { - JITCompiler::Jump zero = m_jit.branchTest32(JITCompiler::Zero, op2GPR); - JITCompiler::Jump isNeg2ToThe31 = m_jit.branch32(JITCompiler::Equal, op1GPR, TrustedImm32(-2147483647-1)); - zero.link(&m_jit); + // This is the case where we convert the result to an int after we're done, and we + // already know that the denominator is either -1 or 0. So, if the denominator is + // zero, then the result should be zero. If the denominator is not zero (i.e. it's + // -1) and the numerator is -2^31 then the result should be -2^31. Otherwise we + // are happy to fall through to a normal division, since we're just dividing + // something by negative 1. + + JITCompiler::Jump notZero = m_jit.branchTest32(JITCompiler::NonZero, op2GPR); m_jit.move(TrustedImm32(0), eax.gpr()); - isNeg2ToThe31.link(&m_jit); - done = m_jit.jump(); + done.append(m_jit.jump()); + + notZero.link(&m_jit); + JITCompiler::Jump notNeg2ToThe31 = + m_jit.branch32(JITCompiler::NotEqual, op1GPR, TrustedImm32(-2147483647-1)); + m_jit.move(op1GPR, eax.gpr()); + done.append(m_jit.jump()); + + notNeg2ToThe31.link(&m_jit); } safeDenominator.link(&m_jit); - + // If the user cares about negative zero, then speculate that we're not about // to produce negative zero. - if (!nodeCanIgnoreNegativeZero(node.arithNodeFlags())) { + if (!nodeCanIgnoreNegativeZero(node->arithNodeFlags())) { MacroAssembler::Jump numeratorNonZero = m_jit.branchTest32(MacroAssembler::NonZero, op1GPR); - speculationCheck(NegativeZero, JSValueRegs(), NoNode, m_jit.branch32(MacroAssembler::LessThan, op2GPR, TrustedImm32(0))); + speculationCheck(NegativeZero, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::LessThan, op2GPR, TrustedImm32(0))); numeratorNonZero.link(&m_jit); } @@ -3127,72 +3481,114 @@ void SpeculativeJIT::compileIntegerArithDivForX86(Node& node) // Check that there was no remainder. If there had been, then we'd be obligated to // produce a double result instead. - if (nodeUsedAsNumber(node.arithNodeFlags())) - speculationCheck(Overflow, JSValueRegs(), NoNode, m_jit.branchTest32(JITCompiler::NonZero, edx.gpr())); + if (nodeUsedAsNumber(node->arithNodeFlags())) + speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branchTest32(JITCompiler::NonZero, edx.gpr())); else done.link(&m_jit); - integerResult(eax.gpr(), m_compileIndex); + integerResult(eax.gpr(), node); } -#endif // CPU(X86) || CPU(X86_64) +#elif CPU(APPLE_ARMV7S) +void SpeculativeJIT::compileIntegerArithDivForARMv7s(Node* node) +{ + SpeculateIntegerOperand op1(this, node->child1()); + SpeculateIntegerOperand op2(this, node->child2()); + GPRReg op1GPR = op1.gpr(); + GPRReg op2GPR = op2.gpr(); + GPRTemporary quotient(this); + GPRTemporary multiplyAnswer(this); + + // If the user cares about negative zero, then speculate that we're not about + // to produce negative zero. + if (!nodeCanIgnoreNegativeZero(node->arithNodeFlags())) { + MacroAssembler::Jump numeratorNonZero = m_jit.branchTest32(MacroAssembler::NonZero, op1GPR); + speculationCheck(NegativeZero, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::LessThan, op2GPR, TrustedImm32(0))); + numeratorNonZero.link(&m_jit); + } -void SpeculativeJIT::compileArithMod(Node& node) + m_jit.assembler().sdiv(quotient.gpr(), op1GPR, op2GPR); + + // Check that there was no remainder. If there had been, then we'd be obligated to + // produce a double result instead. + if (nodeUsedAsNumber(node->arithNodeFlags())) { + speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branchMul32(JITCompiler::Overflow, quotient.gpr(), op2GPR, multiplyAnswer.gpr())); + speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branch32(JITCompiler::NotEqual, multiplyAnswer.gpr(), op1GPR)); + } + + integerResult(quotient.gpr(), node); +} +#endif + +void SpeculativeJIT::compileArithMod(Node* node) { - if (Node::shouldSpeculateIntegerForArithmetic(at(node.child1()), at(node.child2())) - && node.canSpeculateInteger()) { + switch (node->binaryUseKind()) { + case Int32Use: { compileSoftModulo(node); return; } - SpeculateDoubleOperand op1(this, node.child1()); - SpeculateDoubleOperand op2(this, node.child2()); + case NumberUse: { + SpeculateDoubleOperand op1(this, node->child1()); + SpeculateDoubleOperand op2(this, node->child2()); - FPRReg op1FPR = op1.fpr(); - FPRReg op2FPR = op2.fpr(); + FPRReg op1FPR = op1.fpr(); + FPRReg op2FPR = op2.fpr(); - flushRegisters(); + flushRegisters(); - FPRResult result(this); - - callOperation(fmodAsDFGOperation, result.fpr(), op1FPR, op2FPR); + FPRResult result(this); + + callOperation(fmodAsDFGOperation, result.fpr(), op1FPR, op2FPR); + + doubleResult(result.fpr(), node); + return; + } - doubleResult(result.fpr(), m_compileIndex); + default: + RELEASE_ASSERT_NOT_REACHED(); + return; + } } // Returns true if the compare is fused with a subsequent branch. -bool SpeculativeJIT::compare(Node& node, MacroAssembler::RelationalCondition condition, MacroAssembler::DoubleCondition doubleCondition, S_DFGOperation_EJJ operation) +bool SpeculativeJIT::compare(Node* node, MacroAssembler::RelationalCondition condition, MacroAssembler::DoubleCondition doubleCondition, S_DFGOperation_EJJ operation) { if (compilePeepHoleBranch(node, condition, doubleCondition, operation)) return true; - if (Node::shouldSpeculateInteger(at(node.child1()), at(node.child2()))) { + if (node->isBinaryUseKind(Int32Use)) { compileIntegerCompare(node, condition); return false; } - - if (Node::shouldSpeculateNumber(at(node.child1()), at(node.child2()))) { + + if (node->isBinaryUseKind(NumberUse)) { compileDoubleCompare(node, doubleCondition); return false; } - if (node.op() == CompareEq) { - if (at(node.child1()).shouldSpeculateString() || at(node.child2()).shouldSpeculateString()) { - nonSpeculativeNonPeepholeCompare(node, condition, operation); + if (node->op() == CompareEq) { + if (node->isBinaryUseKind(StringUse)) { + compileStringEquality(node); return false; } - if (at(node.child1()).shouldSpeculateNonStringCell() && at(node.child2()).shouldSpeculateNonStringCellOrOther()) { - compileObjectToObjectOrOtherEquality(node.child1(), node.child2()); + if (node->isBinaryUseKind(BooleanUse)) { + compileBooleanCompare(node, condition); + return false; + } + + if (node->isBinaryUseKind(ObjectUse)) { + compileObjectEquality(node); return false; } - if (at(node.child1()).shouldSpeculateNonStringCellOrOther() && at(node.child2()).shouldSpeculateNonStringCell()) { - compileObjectToObjectOrOtherEquality(node.child2(), node.child1()); + if (node->child1().useKind() == ObjectUse && node->child2().useKind() == ObjectOrOtherUse) { + compileObjectToObjectOrOtherEquality(node->child1(), node->child2()); return false; } - - if (at(node.child1()).shouldSpeculateNonStringCell() && at(node.child2()).shouldSpeculateNonStringCell()) { - compileObjectEquality(node); + + if (node->child1().useKind() == ObjectOrOtherUse && node->child2().useKind() == ObjectUse) { + compileObjectToObjectOrOtherEquality(node->child2(), node->child1()); return false; } } @@ -3201,16 +3597,19 @@ bool SpeculativeJIT::compare(Node& node, MacroAssembler::RelationalCondition con return false; } -bool SpeculativeJIT::compileStrictEqForConstant(Node& node, Edge value, JSValue constant) +bool SpeculativeJIT::compileStrictEqForConstant(Node* node, Edge value, JSValue constant) { JSValueOperand op1(this, value); + // FIXME: This code is wrong for the case that the constant is null or undefined, + // and the value is an object that MasqueradesAsUndefined. + // https://bugs.webkit.org/show_bug.cgi?id=109487 + unsigned branchIndexInBlock = detectPeepHoleBranch(); if (branchIndexInBlock != UINT_MAX) { - NodeIndex branchNodeIndex = m_jit.graph().m_blocks[m_block]->at(branchIndexInBlock); - Node& branchNode = at(branchNodeIndex); - BlockIndex taken = branchNode.takenBlockIndex(); - BlockIndex notTaken = branchNode.notTakenBlockIndex(); + Node* branchNode = m_jit.graph().m_blocks[m_block]->at(branchIndexInBlock); + BlockIndex taken = branchNode->takenBlockIndex(); + BlockIndex notTaken = branchNode->notTakenBlockIndex(); MacroAssembler::RelationalCondition condition = MacroAssembler::Equal; // The branch instruction will branch to the taken block. @@ -3241,10 +3640,10 @@ bool SpeculativeJIT::compileStrictEqForConstant(Node& node, Edge value, JSValue jump(notTaken); - use(node.child1()); - use(node.child2()); + use(node->child1()); + use(node->child2()); m_indexInBlock = branchIndexInBlock; - m_compileIndex = branchNodeIndex; + m_currentNode = branchNode; return true; } @@ -3257,7 +3656,7 @@ bool SpeculativeJIT::compileStrictEqForConstant(Node& node, Edge value, JSValue MacroAssembler::Jump notEqual = m_jit.branch64(MacroAssembler::NotEqual, op1GPR, MacroAssembler::TrustedImm64(JSValue::encode(constant))); m_jit.or32(MacroAssembler::TrustedImm32(1), resultGPR); notEqual.link(&m_jit); - jsValueResult(resultGPR, m_compileIndex, DataFormatJSBoolean); + jsValueResult(resultGPR, node, DataFormatJSBoolean); #else GPRReg op1PayloadGPR = op1.payloadGPR(); GPRReg op1TagGPR = op1.tagGPR(); @@ -3268,101 +3667,230 @@ bool SpeculativeJIT::compileStrictEqForConstant(Node& node, Edge value, JSValue notEqual.append(m_jit.branch32(MacroAssembler::NotEqual, op1PayloadGPR, MacroAssembler::Imm32(constant.payload()))); m_jit.move(TrustedImm32(1), resultGPR); notEqual.link(&m_jit); - booleanResult(resultGPR, m_compileIndex); + booleanResult(resultGPR, node); #endif return false; } -bool SpeculativeJIT::compileStrictEq(Node& node) +bool SpeculativeJIT::compileStrictEq(Node* node) { - // 1) If either operand is a constant and that constant is not a double, integer, - // or string, then do a JSValue comparison. - - if (isJSConstant(node.child1().index())) { - JSValue value = valueOfJSConstant(node.child1().index()); - if (!value.isNumber() && !value.isString()) - return compileStrictEqForConstant(node, node.child2(), value); - } - - if (isJSConstant(node.child2().index())) { - JSValue value = valueOfJSConstant(node.child2().index()); - if (!value.isNumber() && !value.isString()) - return compileStrictEqForConstant(node, node.child1(), value); + switch (node->binaryUseKind()) { + case BooleanUse: { + unsigned branchIndexInBlock = detectPeepHoleBranch(); + if (branchIndexInBlock != UINT_MAX) { + Node* branchNode = m_jit.graph().m_blocks[m_block]->at(branchIndexInBlock); + compilePeepHoleBooleanBranch(node, branchNode, MacroAssembler::Equal); + use(node->child1()); + use(node->child2()); + m_indexInBlock = branchIndexInBlock; + m_currentNode = branchNode; + return true; + } + compileBooleanCompare(node, MacroAssembler::Equal); + return false; } - - // 2) If the operands are predicted integer, do an integer comparison. - - if (Node::shouldSpeculateInteger(at(node.child1()), at(node.child2()))) { + + case Int32Use: { unsigned branchIndexInBlock = detectPeepHoleBranch(); if (branchIndexInBlock != UINT_MAX) { - NodeIndex branchNodeIndex = m_jit.graph().m_blocks[m_block]->at(branchIndexInBlock); - compilePeepHoleIntegerBranch(node, branchNodeIndex, MacroAssembler::Equal); - use(node.child1()); - use(node.child2()); + Node* branchNode = m_jit.graph().m_blocks[m_block]->at(branchIndexInBlock); + compilePeepHoleIntegerBranch(node, branchNode, MacroAssembler::Equal); + use(node->child1()); + use(node->child2()); m_indexInBlock = branchIndexInBlock; - m_compileIndex = branchNodeIndex; + m_currentNode = branchNode; return true; } compileIntegerCompare(node, MacroAssembler::Equal); return false; } - - // 3) If the operands are predicted double, do a double comparison. - - if (Node::shouldSpeculateNumber(at(node.child1()), at(node.child2()))) { + + case NumberUse: { unsigned branchIndexInBlock = detectPeepHoleBranch(); if (branchIndexInBlock != UINT_MAX) { - NodeIndex branchNodeIndex = m_jit.graph().m_blocks[m_block]->at(branchIndexInBlock); - compilePeepHoleDoubleBranch(node, branchNodeIndex, MacroAssembler::DoubleEqual); - use(node.child1()); - use(node.child2()); + Node* branchNode = m_jit.graph().m_blocks[m_block]->at(branchIndexInBlock); + compilePeepHoleDoubleBranch(node, branchNode, MacroAssembler::DoubleEqual); + use(node->child1()); + use(node->child2()); m_indexInBlock = branchIndexInBlock; - m_compileIndex = branchNodeIndex; + m_currentNode = branchNode; return true; } compileDoubleCompare(node, MacroAssembler::DoubleEqual); return false; } - - if (at(node.child1()).shouldSpeculateString() || at(node.child2()).shouldSpeculateString()) - return nonSpeculativeStrictEq(node); - if (at(node.child1()).shouldSpeculateNonStringCell() && at(node.child2()).shouldSpeculateNonStringCell()) { + + case StringUse: { + compileStringEquality(node); + return false; + } + + case ObjectUse: { unsigned branchIndexInBlock = detectPeepHoleBranch(); if (branchIndexInBlock != UINT_MAX) { - NodeIndex branchNodeIndex = m_jit.graph().m_blocks[m_block]->at(branchIndexInBlock); - compilePeepHoleObjectEquality(node, branchNodeIndex); - use(node.child1()); - use(node.child2()); + Node* branchNode = m_jit.graph().m_blocks[m_block]->at(branchIndexInBlock); + compilePeepHoleObjectEquality(node, branchNode); + use(node->child1()); + use(node->child2()); m_indexInBlock = branchIndexInBlock; - m_compileIndex = branchNodeIndex; + m_currentNode = branchNode; return true; } compileObjectEquality(node); return false; } + + case UntypedUse: { + return nonSpeculativeStrictEq(node); + } + + default: + RELEASE_ASSERT_NOT_REACHED(); + return false; + } +} + +void SpeculativeJIT::compileBooleanCompare(Node* node, MacroAssembler::RelationalCondition condition) +{ + SpeculateBooleanOperand op1(this, node->child1()); + SpeculateBooleanOperand op2(this, node->child2()); + GPRTemporary result(this); - // 5) Fall back to non-speculative strict equality. + m_jit.compare32(condition, op1.gpr(), op2.gpr(), result.gpr()); - return nonSpeculativeStrictEq(node); + // If we add a DataFormatBool, we should use it here. +#if USE(JSVALUE32_64) + booleanResult(result.gpr(), node); +#else + m_jit.or32(TrustedImm32(ValueFalse), result.gpr()); + jsValueResult(result.gpr(), m_currentNode, DataFormatJSBoolean); +#endif } -void SpeculativeJIT::compileGetIndexedPropertyStorage(Node& node) +void SpeculativeJIT::compileStringEquality(Node* node) { - SpeculateCellOperand base(this, node.child1()); + SpeculateCellOperand left(this, node->child1()); + SpeculateCellOperand right(this, node->child2()); + GPRTemporary length(this); + GPRTemporary leftTemp(this); + GPRTemporary rightTemp(this); + GPRTemporary leftTemp2(this, left); + GPRTemporary rightTemp2(this, right); + + GPRReg leftGPR = left.gpr(); + GPRReg rightGPR = right.gpr(); + GPRReg lengthGPR = length.gpr(); + GPRReg leftTempGPR = leftTemp.gpr(); + GPRReg rightTempGPR = rightTemp.gpr(); + GPRReg leftTemp2GPR = leftTemp2.gpr(); + GPRReg rightTemp2GPR = rightTemp2.gpr(); + + JITCompiler::JumpList trueCase; + JITCompiler::JumpList falseCase; + JITCompiler::JumpList slowCase; + + DFG_TYPE_CHECK( + JSValueSource::unboxedCell(leftGPR), node->child1(), SpecString, m_jit.branchPtr( + MacroAssembler::NotEqual, + MacroAssembler::Address(leftGPR, JSCell::structureOffset()), + MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + + // It's safe to branch around the type check below, since proving that the values are + // equal does indeed prove that the right value is a string. + trueCase.append(m_jit.branchPtr(MacroAssembler::Equal, leftGPR, rightGPR)); + + DFG_TYPE_CHECK( + JSValueSource::unboxedCell(rightGPR), node->child2(), SpecString, m_jit.branchPtr( + MacroAssembler::NotEqual, + MacroAssembler::Address(rightGPR, JSCell::structureOffset()), + MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + + m_jit.load32(MacroAssembler::Address(leftGPR, JSString::offsetOfLength()), lengthGPR); + + falseCase.append(m_jit.branch32( + MacroAssembler::NotEqual, + MacroAssembler::Address(rightGPR, JSString::offsetOfLength()), + lengthGPR)); + + trueCase.append(m_jit.branchTest32(MacroAssembler::Zero, lengthGPR)); + + m_jit.loadPtr(MacroAssembler::Address(leftGPR, JSString::offsetOfValue()), leftTempGPR); + m_jit.loadPtr(MacroAssembler::Address(rightGPR, JSString::offsetOfValue()), rightTempGPR); + + slowCase.append(m_jit.branchTestPtr(MacroAssembler::Zero, leftTempGPR)); + slowCase.append(m_jit.branchTestPtr(MacroAssembler::Zero, rightTempGPR)); + + slowCase.append(m_jit.branchTest32( + MacroAssembler::Zero, + MacroAssembler::Address(leftTempGPR, StringImpl::flagsOffset()), + TrustedImm32(StringImpl::flagIs8Bit()))); + slowCase.append(m_jit.branchTest32( + MacroAssembler::Zero, + MacroAssembler::Address(rightTempGPR, StringImpl::flagsOffset()), + TrustedImm32(StringImpl::flagIs8Bit()))); + + m_jit.loadPtr(MacroAssembler::Address(leftTempGPR, StringImpl::dataOffset()), leftTempGPR); + m_jit.loadPtr(MacroAssembler::Address(rightTempGPR, StringImpl::dataOffset()), rightTempGPR); + + MacroAssembler::Label loop = m_jit.label(); + + m_jit.sub32(TrustedImm32(1), lengthGPR); + + // This isn't going to generate the best code on x86. But that's OK, it's still better + // than not inlining. + m_jit.load8(MacroAssembler::BaseIndex(leftTempGPR, lengthGPR, MacroAssembler::TimesOne), leftTemp2GPR); + m_jit.load8(MacroAssembler::BaseIndex(rightTempGPR, lengthGPR, MacroAssembler::TimesOne), rightTemp2GPR); + falseCase.append(m_jit.branch32(MacroAssembler::NotEqual, leftTemp2GPR, rightTemp2GPR)); + + m_jit.branchTest32(MacroAssembler::NonZero, lengthGPR).linkTo(loop, &m_jit); + + trueCase.link(&m_jit); +#if USE(JSVALUE64) + m_jit.move(TrustedImm64(ValueTrue), leftTempGPR); +#else + m_jit.move(TrustedImm32(true), leftTempGPR); +#endif + + JITCompiler::Jump done = m_jit.jump(); + + falseCase.link(&m_jit); +#if USE(JSVALUE64) + m_jit.move(TrustedImm64(ValueFalse), leftTempGPR); +#else + m_jit.move(TrustedImm32(false), leftTempGPR); +#endif + + done.link(&m_jit); + addSlowPathGenerator( + slowPathCall( + slowCase, this, operationCompareStringEq, leftTempGPR, leftGPR, rightGPR)); + +#if USE(JSVALUE64) + jsValueResult(leftTempGPR, node, DataFormatJSBoolean); +#else + booleanResult(leftTempGPR, node); +#endif +} + +void SpeculativeJIT::compileGetIndexedPropertyStorage(Node* node) +{ + SpeculateCellOperand base(this, node->child1()); GPRReg baseReg = base.gpr(); GPRTemporary storage(this); GPRReg storageReg = storage.gpr(); - const TypedArrayDescriptor* descriptor = typedArrayDescriptor(node.arrayMode()); + const TypedArrayDescriptor* descriptor = typedArrayDescriptor(node->arrayMode()); - switch (node.arrayMode().type()) { + switch (node->arrayMode().type()) { case Array::String: m_jit.loadPtr(MacroAssembler::Address(baseReg, JSString::offsetOfValue()), storageReg); - // Speculate that we're not accessing a rope - speculationCheck(Uncountable, JSValueRegs(), NoNode, m_jit.branchTest32(MacroAssembler::Zero, storageReg)); + addSlowPathGenerator( + slowPathCall( + m_jit.branchTest32(MacroAssembler::Zero, storageReg), + this, operationResolveRope, storageReg, baseReg)); m_jit.loadPtr(MacroAssembler::Address(storageReg, StringImpl::dataOffset()), storageReg); break; @@ -3373,13 +3901,13 @@ void SpeculativeJIT::compileGetIndexedPropertyStorage(Node& node) break; } - storageResult(storageReg, m_compileIndex); + storageResult(storageReg, node); } -void SpeculativeJIT::compileGetByValOnArguments(Node& node) +void SpeculativeJIT::compileGetByValOnArguments(Node* node) { - SpeculateCellOperand base(this, node.child1()); - SpeculateStrictInt32Operand property(this, node.child2()); + SpeculateCellOperand base(this, node->child1()); + SpeculateStrictInt32Operand property(this, node->child2()); GPRTemporary result(this); #if USE(JSVALUE32_64) GPRTemporary resultTag(this); @@ -3397,16 +3925,16 @@ void SpeculativeJIT::compileGetByValOnArguments(Node& node) if (!m_compileOkay) return; - ASSERT(ArrayMode(Array::Arguments).alreadyChecked(m_jit.graph(), node, m_state.forNode(node.child1()))); + ASSERT(ArrayMode(Array::Arguments).alreadyChecked(m_jit.graph(), node, m_state.forNode(node->child1()))); // Two really lame checks. speculationCheck( - Uncountable, JSValueSource(), NoNode, - m_jit.branchPtr( + Uncountable, JSValueSource(), 0, + m_jit.branch32( MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(baseReg, OBJECT_OFFSETOF(Arguments, m_numArguments)))); speculationCheck( - Uncountable, JSValueSource(), NoNode, + Uncountable, JSValueSource(), 0, m_jit.branchTestPtr( MacroAssembler::NonZero, MacroAssembler::Address( @@ -3432,20 +3960,20 @@ void SpeculativeJIT::compileGetByValOnArguments(Node& node) CallFrame::thisArgumentOffset() * sizeof(Register) - sizeof(Register) + OBJECT_OFFSETOF(JSValue, u.asBits.payload)), resultReg); - jsValueResult(resultTagReg, resultReg, m_compileIndex); + jsValueResult(resultTagReg, resultReg, node); #else m_jit.load64( MacroAssembler::BaseIndex( scratchReg, resultReg, MacroAssembler::TimesEight, CallFrame::thisArgumentOffset() * sizeof(Register) - sizeof(Register)), resultReg); - jsValueResult(resultReg, m_compileIndex); + jsValueResult(resultReg, node); #endif } -void SpeculativeJIT::compileGetArgumentsLength(Node& node) +void SpeculativeJIT::compileGetArgumentsLength(Node* node) { - SpeculateCellOperand base(this, node.child1()); + SpeculateCellOperand base(this, node->child1()); GPRTemporary result(this, base); GPRReg baseReg = base.gpr(); @@ -3454,10 +3982,10 @@ void SpeculativeJIT::compileGetArgumentsLength(Node& node) if (!m_compileOkay) return; - ASSERT(ArrayMode(Array::Arguments).alreadyChecked(m_jit.graph(), node, m_state.forNode(node.child1()))); + ASSERT(ArrayMode(Array::Arguments).alreadyChecked(m_jit.graph(), node, m_state.forNode(node->child1()))); speculationCheck( - Uncountable, JSValueSource(), NoNode, + Uncountable, JSValueSource(), 0, m_jit.branchTest8( MacroAssembler::NonZero, MacroAssembler::Address(baseReg, OBJECT_OFFSETOF(Arguments, m_overrodeLength)))); @@ -3465,46 +3993,46 @@ void SpeculativeJIT::compileGetArgumentsLength(Node& node) m_jit.load32( MacroAssembler::Address(baseReg, OBJECT_OFFSETOF(Arguments, m_numArguments)), resultReg); - integerResult(resultReg, m_compileIndex); + integerResult(resultReg, node); } -void SpeculativeJIT::compileGetArrayLength(Node& node) +void SpeculativeJIT::compileGetArrayLength(Node* node) { - const TypedArrayDescriptor* descriptor = typedArrayDescriptor(node.arrayMode()); + const TypedArrayDescriptor* descriptor = typedArrayDescriptor(node->arrayMode()); - switch (node.arrayMode().type()) { + switch (node->arrayMode().type()) { case Array::Int32: case Array::Double: case Array::Contiguous: { - StorageOperand storage(this, node.child2()); + StorageOperand storage(this, node->child2()); GPRTemporary result(this, storage); GPRReg storageReg = storage.gpr(); GPRReg resultReg = result.gpr(); m_jit.load32(MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength()), resultReg); - integerResult(resultReg, m_compileIndex); + integerResult(resultReg, node); break; } case Array::ArrayStorage: case Array::SlowPutArrayStorage: { - StorageOperand storage(this, node.child2()); + StorageOperand storage(this, node->child2()); GPRTemporary result(this, storage); GPRReg storageReg = storage.gpr(); GPRReg resultReg = result.gpr(); m_jit.load32(MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength()), resultReg); - speculationCheck(Uncountable, JSValueRegs(), NoNode, m_jit.branch32(MacroAssembler::LessThan, resultReg, MacroAssembler::TrustedImm32(0))); + speculationCheck(Uncountable, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::LessThan, resultReg, MacroAssembler::TrustedImm32(0))); - integerResult(resultReg, m_compileIndex); + integerResult(resultReg, node); break; } case Array::String: { - SpeculateCellOperand base(this, node.child1()); + SpeculateCellOperand base(this, node->child1()); GPRTemporary result(this, base); GPRReg baseGPR = base.gpr(); GPRReg resultGPR = result.gpr(); m_jit.load32(MacroAssembler::Address(baseGPR, JSString::offsetOfLength()), resultGPR); - integerResult(resultGPR, m_compileIndex); + integerResult(resultGPR, node); break; } case Array::Arguments: { @@ -3512,28 +4040,28 @@ void SpeculativeJIT::compileGetArrayLength(Node& node) break; } default: - SpeculateCellOperand base(this, node.child1()); + SpeculateCellOperand base(this, node->child1()); GPRTemporary result(this, base); GPRReg baseGPR = base.gpr(); GPRReg resultGPR = result.gpr(); ASSERT(descriptor); m_jit.load32(MacroAssembler::Address(baseGPR, descriptor->m_lengthOffset), resultGPR); - integerResult(resultGPR, m_compileIndex); + integerResult(resultGPR, node); break; } } -void SpeculativeJIT::compileNewFunctionNoCheck(Node& node) +void SpeculativeJIT::compileNewFunctionNoCheck(Node* node) { GPRResult result(this); GPRReg resultGPR = result.gpr(); flushRegisters(); callOperation( - operationNewFunction, resultGPR, m_jit.codeBlock()->functionDecl(node.functionDeclIndex())); - cellResult(resultGPR, m_compileIndex); + operationNewFunctionNoCheck, resultGPR, m_jit.codeBlock()->functionDecl(node->functionDeclIndex())); + cellResult(resultGPR, node); } -void SpeculativeJIT::compileNewFunctionExpression(Node& node) +void SpeculativeJIT::compileNewFunctionExpression(Node* node) { GPRResult result(this); GPRReg resultGPR = result.gpr(); @@ -3541,21 +4069,20 @@ void SpeculativeJIT::compileNewFunctionExpression(Node& node) callOperation( operationNewFunctionExpression, resultGPR, - m_jit.codeBlock()->functionExpr(node.functionExprIndex())); - cellResult(resultGPR, m_compileIndex); + m_jit.codeBlock()->functionExpr(node->functionExprIndex())); + cellResult(resultGPR, node); } -bool SpeculativeJIT::compileRegExpExec(Node& node) +bool SpeculativeJIT::compileRegExpExec(Node* node) { unsigned branchIndexInBlock = detectPeepHoleBranch(); if (branchIndexInBlock == UINT_MAX) return false; - NodeIndex branchNodeIndex = m_jit.graph().m_blocks[m_block]->at(branchIndexInBlock); - ASSERT(node.adjustedRefCount() == 1); + Node* branchNode = m_jit.graph().m_blocks[m_block]->at(branchIndexInBlock); + ASSERT(node->adjustedRefCount() == 1); - Node& branchNode = at(branchNodeIndex); - BlockIndex taken = branchNode.takenBlockIndex(); - BlockIndex notTaken = branchNode.notTakenBlockIndex(); + BlockIndex taken = branchNode->takenBlockIndex(); + BlockIndex notTaken = branchNode->notTakenBlockIndex(); bool invert = false; if (taken == nextBlock()) { @@ -3565,8 +4092,8 @@ bool SpeculativeJIT::compileRegExpExec(Node& node) notTaken = tmp; } - SpeculateCellOperand base(this, node.child1()); - SpeculateCellOperand argument(this, node.child2()); + SpeculateCellOperand base(this, node->child1()); + SpeculateCellOperand argument(this, node->child2()); GPRReg baseGPR = base.gpr(); GPRReg argumentGPR = argument.gpr(); @@ -3577,18 +4104,18 @@ bool SpeculativeJIT::compileRegExpExec(Node& node) branchTest32(invert ? JITCompiler::Zero : JITCompiler::NonZero, result.gpr(), taken); jump(notTaken); - use(node.child1()); - use(node.child2()); + use(node->child1()); + use(node->child2()); m_indexInBlock = branchIndexInBlock; - m_compileIndex = branchNodeIndex; + m_currentNode = branchNode; return true; } -void SpeculativeJIT::compileAllocatePropertyStorage(Node& node) +void SpeculativeJIT::compileAllocatePropertyStorage(Node* node) { - if (hasIndexingHeader(node.structureTransitionData().previousStructure->indexingType())) { - SpeculateCellOperand base(this, node.child1()); + if (hasIndexingHeader(node->structureTransitionData().previousStructure->indexingType())) { + SpeculateCellOperand base(this, node->child1()); GPRReg baseGPR = base.gpr(); @@ -3597,18 +4124,18 @@ void SpeculativeJIT::compileAllocatePropertyStorage(Node& node) GPRResult result(this); callOperation(operationReallocateButterflyToHavePropertyStorageWithInitialCapacity, result.gpr(), baseGPR); - storageResult(result.gpr(), m_compileIndex); + storageResult(result.gpr(), node); return; } - SpeculateCellOperand base(this, node.child1()); + SpeculateCellOperand base(this, node->child1()); GPRTemporary scratch(this); GPRReg baseGPR = base.gpr(); GPRReg scratchGPR = scratch.gpr(); - ASSERT(!node.structureTransitionData().previousStructure->outOfLineCapacity()); - ASSERT(initialOutOfLineCapacity == node.structureTransitionData().newStructure->outOfLineCapacity()); + ASSERT(!node->structureTransitionData().previousStructure->outOfLineCapacity()); + ASSERT(initialOutOfLineCapacity == node->structureTransitionData().newStructure->outOfLineCapacity()); JITCompiler::Jump slowPath = emitAllocateBasicStorage( @@ -3621,17 +4148,17 @@ void SpeculativeJIT::compileAllocatePropertyStorage(Node& node) m_jit.storePtr(scratchGPR, JITCompiler::Address(baseGPR, JSObject::butterflyOffset())); - storageResult(scratchGPR, m_compileIndex); + storageResult(scratchGPR, node); } -void SpeculativeJIT::compileReallocatePropertyStorage(Node& node) +void SpeculativeJIT::compileReallocatePropertyStorage(Node* node) { - size_t oldSize = node.structureTransitionData().previousStructure->outOfLineCapacity() * sizeof(JSValue); + size_t oldSize = node->structureTransitionData().previousStructure->outOfLineCapacity() * sizeof(JSValue); size_t newSize = oldSize * outOfLineGrowthFactor; - ASSERT(newSize == node.structureTransitionData().newStructure->outOfLineCapacity() * sizeof(JSValue)); + ASSERT(newSize == node->structureTransitionData().newStructure->outOfLineCapacity() * sizeof(JSValue)); - if (hasIndexingHeader(node.structureTransitionData().previousStructure->indexingType())) { - SpeculateCellOperand base(this, node.child1()); + if (hasIndexingHeader(node->structureTransitionData().previousStructure->indexingType())) { + SpeculateCellOperand base(this, node->child1()); GPRReg baseGPR = base.gpr(); @@ -3640,12 +4167,12 @@ void SpeculativeJIT::compileReallocatePropertyStorage(Node& node) GPRResult result(this); callOperation(operationReallocateButterflyToGrowPropertyStorage, result.gpr(), baseGPR, newSize / sizeof(JSValue)); - storageResult(result.gpr(), m_compileIndex); + storageResult(result.gpr(), node); return; } - SpeculateCellOperand base(this, node.child1()); - StorageOperand oldStorage(this, node.child2()); + SpeculateCellOperand base(this, node->child1()); + StorageOperand oldStorage(this, node->child2()); GPRTemporary scratch1(this); GPRTemporary scratch2(this); @@ -3668,7 +4195,7 @@ void SpeculativeJIT::compileReallocatePropertyStorage(Node& node) } m_jit.storePtr(scratchGPR2, JITCompiler::Address(baseGPR, JSObject::butterflyOffset())); - storageResult(scratchGPR2, m_compileIndex); + storageResult(scratchGPR2, node); } GPRReg SpeculativeJIT::temporaryRegisterForPutByVal(GPRTemporary& temporary, ArrayMode arrayMode) @@ -3681,6 +4208,392 @@ GPRReg SpeculativeJIT::temporaryRegisterForPutByVal(GPRTemporary& temporary, Arr return temporary.gpr(); } +void SpeculativeJIT::compileToStringOnCell(Node* node) +{ + SpeculateCellOperand op1(this, node->child1()); + GPRReg op1GPR = op1.gpr(); + + switch (node->child1().useKind()) { + case StringObjectUse: { + GPRTemporary result(this); + GPRReg resultGPR = result.gpr(); + + speculateStringObject(node->child1(), op1GPR); + m_state.forNode(node->child1()).filter(SpecStringObject); + m_jit.loadPtr(JITCompiler::Address(op1GPR, JSWrapperObject::internalValueCellOffset()), resultGPR); + cellResult(resultGPR, node); + break; + } + + case StringOrStringObjectUse: { + GPRTemporary result(this); + GPRReg resultGPR = result.gpr(); + + m_jit.loadPtr(JITCompiler::Address(op1GPR, JSCell::structureOffset()), resultGPR); + JITCompiler::Jump isString = m_jit.branchPtr( + JITCompiler::Equal, resultGPR, TrustedImmPtr(m_jit.vm()->stringStructure.get())); + + speculateStringObjectForStructure(node->child1(), resultGPR); + + m_jit.loadPtr(JITCompiler::Address(op1GPR, JSWrapperObject::internalValueCellOffset()), resultGPR); + + JITCompiler::Jump done = m_jit.jump(); + isString.link(&m_jit); + m_jit.move(op1GPR, resultGPR); + done.link(&m_jit); + + m_state.forNode(node->child1()).filter(SpecString | SpecStringObject); + + cellResult(resultGPR, node); + break; + } + + case CellUse: { + GPRResult result(this); + GPRReg resultGPR = result.gpr(); + + // We flush registers instead of silent spill/fill because in this mode we + // believe that most likely the input is not a string, and we need to take + // slow path. + flushRegisters(); + JITCompiler::Jump done; + if (node->child1()->prediction() & SpecString) { + JITCompiler::Jump needCall = m_jit.branchPtr( + JITCompiler::NotEqual, + JITCompiler::Address(op1GPR, JSCell::structureOffset()), + TrustedImmPtr(m_jit.vm()->stringStructure.get())); + m_jit.move(op1GPR, resultGPR); + done = m_jit.jump(); + needCall.link(&m_jit); + } + callOperation(operationToStringOnCell, resultGPR, op1GPR); + if (done.isSet()) + done.link(&m_jit); + cellResult(resultGPR, node); + break; + } + + default: + RELEASE_ASSERT_NOT_REACHED(); + } +} + +void SpeculativeJIT::compileNewStringObject(Node* node) +{ + SpeculateCellOperand operand(this, node->child1()); + + GPRTemporary result(this); + GPRTemporary scratch1(this); + GPRTemporary scratch2(this); + + GPRReg operandGPR = operand.gpr(); + GPRReg resultGPR = result.gpr(); + GPRReg scratch1GPR = scratch1.gpr(); + GPRReg scratch2GPR = scratch2.gpr(); + + JITCompiler::JumpList slowPath; + + emitAllocateJSObject<StringObject>( + resultGPR, TrustedImmPtr(node->structure()), TrustedImmPtr(0), scratch1GPR, scratch2GPR, + slowPath); + + m_jit.storePtr( + TrustedImmPtr(&StringObject::s_info), + JITCompiler::Address(resultGPR, JSDestructibleObject::classInfoOffset())); +#if USE(JSVALUE64) + m_jit.store64( + operandGPR, JITCompiler::Address(resultGPR, JSWrapperObject::internalValueOffset())); +#else + m_jit.store32( + TrustedImm32(JSValue::CellTag), + JITCompiler::Address(resultGPR, JSWrapperObject::internalValueOffset() + OBJECT_OFFSETOF(JSValue, u.asBits.tag))); + m_jit.store32( + operandGPR, + JITCompiler::Address(resultGPR, JSWrapperObject::internalValueOffset() + OBJECT_OFFSETOF(JSValue, u.asBits.payload))); +#endif + + addSlowPathGenerator(slowPathCall( + slowPath, this, operationNewStringObject, resultGPR, operandGPR, node->structure())); + + cellResult(resultGPR, node); +} + +void SpeculativeJIT::speculateInt32(Edge edge) +{ + if (!needsTypeCheck(edge, SpecInt32)) + return; + + (SpeculateIntegerOperand(this, edge)).gpr(); +} + +void SpeculativeJIT::speculateNumber(Edge edge) +{ + if (!needsTypeCheck(edge, SpecNumber)) + return; + + (SpeculateDoubleOperand(this, edge)).fpr(); +} + +void SpeculativeJIT::speculateRealNumber(Edge edge) +{ + if (!needsTypeCheck(edge, SpecRealNumber)) + return; + + SpeculateDoubleOperand operand(this, edge); + FPRReg fpr = operand.fpr(); + DFG_TYPE_CHECK( + JSValueRegs(), edge, SpecRealNumber, + m_jit.branchDouble( + MacroAssembler::DoubleNotEqualOrUnordered, fpr, fpr)); +} + +void SpeculativeJIT::speculateBoolean(Edge edge) +{ + if (!needsTypeCheck(edge, SpecBoolean)) + return; + + (SpeculateBooleanOperand(this, edge)).gpr(); +} + +void SpeculativeJIT::speculateCell(Edge edge) +{ + if (!needsTypeCheck(edge, SpecCell)) + return; + + (SpeculateCellOperand(this, edge)).gpr(); +} + +void SpeculativeJIT::speculateObject(Edge edge) +{ + if (!needsTypeCheck(edge, SpecObject)) + return; + + SpeculateCellOperand operand(this, edge); + GPRReg gpr = operand.gpr(); + DFG_TYPE_CHECK( + JSValueSource::unboxedCell(gpr), edge, SpecObject, m_jit.branchPtr( + MacroAssembler::Equal, + MacroAssembler::Address(gpr, JSCell::structureOffset()), + MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); +} + +void SpeculativeJIT::speculateObjectOrOther(Edge edge) +{ + if (!needsTypeCheck(edge, SpecObject | SpecOther)) + return; + + JSValueOperand operand(this, edge, ManualOperandSpeculation); + GPRTemporary temp(this); + GPRReg tempGPR = temp.gpr(); +#if USE(JSVALUE64) + GPRReg gpr = operand.gpr(); + MacroAssembler::Jump notCell = m_jit.branchTest64( + MacroAssembler::NonZero, gpr, GPRInfo::tagMaskRegister); + DFG_TYPE_CHECK( + JSValueRegs(gpr), edge, (~SpecCell) | SpecObject, m_jit.branchPtr( + MacroAssembler::Equal, + MacroAssembler::Address(gpr, JSCell::structureOffset()), + MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + MacroAssembler::Jump done = m_jit.jump(); + notCell.link(&m_jit); + if (needsTypeCheck(edge, SpecCell | SpecOther)) { + m_jit.move(gpr, tempGPR); + m_jit.and64(MacroAssembler::TrustedImm32(~TagBitUndefined), tempGPR); + + typeCheck( + JSValueRegs(gpr), edge, SpecCell | SpecOther, + m_jit.branch64( + MacroAssembler::NotEqual, tempGPR, + MacroAssembler::TrustedImm64(ValueNull))); + } + done.link(&m_jit); +#else + GPRReg tagGPR = operand.tagGPR(); + GPRReg payloadGPR = operand.payloadGPR(); + MacroAssembler::Jump notCell = + m_jit.branch32(MacroAssembler::NotEqual, tagGPR, TrustedImm32(JSValue::CellTag)); + DFG_TYPE_CHECK( + JSValueRegs(tagGPR, payloadGPR), edge, (~SpecCell) | SpecObject, m_jit.branchPtr( + MacroAssembler::Equal, + MacroAssembler::Address(payloadGPR, JSCell::structureOffset()), + MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + MacroAssembler::Jump done = m_jit.jump(); + notCell.link(&m_jit); + if (needsTypeCheck(edge, SpecCell | SpecOther)) { + m_jit.move(tagGPR, tempGPR); + m_jit.or32(TrustedImm32(1), tempGPR); + + typeCheck( + JSValueRegs(tagGPR, payloadGPR), edge, SpecCell | SpecOther, + m_jit.branch32( + MacroAssembler::NotEqual, tempGPR, + MacroAssembler::TrustedImm32(JSValue::NullTag))); + } + done.link(&m_jit); +#endif +} + +void SpeculativeJIT::speculateString(Edge edge) +{ + if (!needsTypeCheck(edge, SpecString)) + return; + + SpeculateCellOperand operand(this, edge); + GPRReg gpr = operand.gpr(); + DFG_TYPE_CHECK( + JSValueSource::unboxedCell(gpr), edge, SpecString, m_jit.branchPtr( + MacroAssembler::NotEqual, + MacroAssembler::Address(gpr, JSCell::structureOffset()), + MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); +} + +void SpeculativeJIT::speculateStringObject(Edge edge, GPRReg gpr) +{ + speculateStringObjectForStructure(edge, JITCompiler::Address(gpr, JSCell::structureOffset())); +} + +void SpeculativeJIT::speculateStringObject(Edge edge) +{ + if (!needsTypeCheck(edge, SpecStringObject)) + return; + + SpeculateCellOperand operand(this, edge); + GPRReg gpr = operand.gpr(); + if (!needsTypeCheck(edge, SpecStringObject)) + return; + + speculateStringObject(edge, gpr); + m_state.forNode(edge).filter(SpecStringObject); +} + +void SpeculativeJIT::speculateStringOrStringObject(Edge edge) +{ + if (!needsTypeCheck(edge, SpecString | SpecStringObject)) + return; + + SpeculateCellOperand operand(this, edge); + GPRReg gpr = operand.gpr(); + if (!needsTypeCheck(edge, SpecString | SpecStringObject)) + return; + + GPRTemporary structure(this); + GPRReg structureGPR = structure.gpr(); + + m_jit.loadPtr(JITCompiler::Address(gpr, JSCell::structureOffset()), structureGPR); + + JITCompiler::Jump isString = m_jit.branchPtr( + JITCompiler::Equal, structureGPR, TrustedImmPtr(m_jit.vm()->stringStructure.get())); + + speculateStringObjectForStructure(edge, structureGPR); + + isString.link(&m_jit); + + m_state.forNode(edge).filter(SpecString | SpecStringObject); +} + +void SpeculativeJIT::speculateNotCell(Edge edge) +{ + if (!needsTypeCheck(edge, ~SpecCell)) + return; + + JSValueOperand operand(this, edge, ManualOperandSpeculation); +#if USE(JSVALUE64) + typeCheck( + JSValueRegs(operand.gpr()), edge, ~SpecCell, + m_jit.branchTest64( + JITCompiler::Zero, operand.gpr(), GPRInfo::tagMaskRegister)); +#else + typeCheck( + JSValueRegs(operand.tagGPR(), operand.payloadGPR()), edge, ~SpecCell, + m_jit.branch32( + JITCompiler::Equal, operand.tagGPR(), TrustedImm32(JSValue::CellTag))); +#endif +} + +void SpeculativeJIT::speculateOther(Edge edge) +{ + if (!needsTypeCheck(edge, SpecOther)) + return; + + JSValueOperand operand(this, edge, ManualOperandSpeculation); + GPRTemporary temp(this); + GPRReg tempGPR = temp.gpr(); +#if USE(JSVALUE64) + m_jit.move(operand.gpr(), tempGPR); + m_jit.and64(MacroAssembler::TrustedImm32(~TagBitUndefined), tempGPR); + typeCheck( + JSValueRegs(operand.gpr()), edge, SpecOther, + m_jit.branch64( + MacroAssembler::NotEqual, tempGPR, + MacroAssembler::TrustedImm64(ValueNull))); +#else + m_jit.move(operand.tagGPR(), tempGPR); + m_jit.or32(TrustedImm32(1), tempGPR); + typeCheck( + JSValueRegs(operand.tagGPR(), operand.payloadGPR()), edge, SpecOther, + m_jit.branch32(MacroAssembler::NotEqual, tempGPR, TrustedImm32(JSValue::NullTag))); +#endif +} + +void SpeculativeJIT::speculate(Node*, Edge edge) +{ + switch (edge.useKind()) { + case UntypedUse: + break; + case KnownInt32Use: + ASSERT(!needsTypeCheck(edge, SpecInt32)); + break; + case KnownNumberUse: + ASSERT(!needsTypeCheck(edge, SpecNumber)); + break; + case KnownCellUse: + ASSERT(!needsTypeCheck(edge, SpecCell)); + break; + case KnownStringUse: + ASSERT(!needsTypeCheck(edge, SpecString)); + break; + case Int32Use: + speculateInt32(edge); + break; + case RealNumberUse: + speculateRealNumber(edge); + break; + case NumberUse: + speculateNumber(edge); + break; + case BooleanUse: + speculateBoolean(edge); + break; + case CellUse: + speculateCell(edge); + break; + case ObjectUse: + speculateObject(edge); + break; + case ObjectOrOtherUse: + speculateObjectOrOther(edge); + break; + case StringUse: + speculateString(edge); + break; + case StringObjectUse: + speculateStringObject(edge); + break; + case StringOrStringObjectUse: + speculateStringOrStringObject(edge); + break; + case NotCellUse: + speculateNotCell(edge); + break; + case OtherUse: + speculateOther(edge); + break; + default: + RELEASE_ASSERT_NOT_REACHED(); + break; + } +} + } } // namespace JSC::DFG #endif |