diff options
author | Lorry Tar Creator <lorry-tar-importer@lorry> | 2015-05-20 09:56:07 +0000 |
---|---|---|
committer | Lorry Tar Creator <lorry-tar-importer@lorry> | 2015-05-20 09:56:07 +0000 |
commit | 41386e9cb918eed93b3f13648cbef387e371e451 (patch) | |
tree | a97f9d7bd1d9d091833286085f72da9d83fd0606 /Source/JavaScriptCore/ftl/FTLLowerDFGToLLVM.cpp | |
parent | e15dd966d523731101f70ccf768bba12435a0208 (diff) | |
download | WebKitGtk-tarball-41386e9cb918eed93b3f13648cbef387e371e451.tar.gz |
webkitgtk-2.4.9webkitgtk-2.4.9
Diffstat (limited to 'Source/JavaScriptCore/ftl/FTLLowerDFGToLLVM.cpp')
-rw-r--r-- | Source/JavaScriptCore/ftl/FTLLowerDFGToLLVM.cpp | 6421 |
1 files changed, 1186 insertions, 5235 deletions
diff --git a/Source/JavaScriptCore/ftl/FTLLowerDFGToLLVM.cpp b/Source/JavaScriptCore/ftl/FTLLowerDFGToLLVM.cpp index 678629fd9..be2a40c7d 100644 --- a/Source/JavaScriptCore/ftl/FTLLowerDFGToLLVM.cpp +++ b/Source/JavaScriptCore/ftl/FTLLowerDFGToLLVM.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2015 Apple Inc. All rights reserved. + * Copyright (C) 2013 Apple Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -31,55 +31,25 @@ #include "CodeBlockWithJITType.h" #include "DFGAbstractInterpreterInlines.h" #include "DFGInPlaceAbstractState.h" -#include "DFGOSRAvailabilityAnalysisPhase.h" -#include "DFGOSRExitFuzz.h" -#include "DirectArguments.h" #include "FTLAbstractHeapRepository.h" -#include "FTLAvailableRecovery.h" #include "FTLForOSREntryJITCode.h" #include "FTLFormattedValue.h" #include "FTLInlineCacheSize.h" #include "FTLLoweredNodeValue.h" -#include "FTLOperations.h" #include "FTLOutput.h" #include "FTLThunks.h" -#include "FTLWeightedTarget.h" -#include "JSCInlines.h" -#include "JSLexicalEnvironment.h" +#include "LinkBuffer.h" #include "OperandsInlines.h" -#include "ScopedArguments.h" -#include "ScopedArgumentsTable.h" +#include "Operations.h" #include "VirtualRegister.h" #include <atomic> -#include <dlfcn.h> -#include <llvm/InitializeLLVM.h> -#include <unordered_set> #include <wtf/ProcessID.h> namespace JSC { namespace FTL { using namespace DFG; -namespace { - -std::atomic<int> compileCounter; - -#if ASSERT_DISABLED -NO_RETURN_DUE_TO_CRASH static void ftlUnreachable() -{ - CRASH(); -} -#else -NO_RETURN_DUE_TO_CRASH static void ftlUnreachable( - CodeBlock* codeBlock, BlockIndex blockIndex, unsigned nodeIndex) -{ - dataLog("Crashing in thought-to-be-unreachable FTL-generated code for ", pointerDump(codeBlock), " at basic block #", blockIndex); - if (nodeIndex != UINT_MAX) - dataLog(", node @", nodeIndex); - dataLog(".\n"); - CRASH(); -} -#endif +static std::atomic<int> compileCounter; // Using this instead of typeCheck() helps to reduce the load on LLVM, by creating // significantly less dead code. @@ -99,11 +69,10 @@ public: , m_ftlState(state) , m_heaps(state.context) , m_out(state.context) + , m_availability(OperandsLike, state.graph.block(0)->variablesAtHead) , m_state(state.graph) , m_interpreter(state.graph, m_state) , m_stackmapIDs(0) - , m_tbaaKind(mdKindID(state.context, "tbaa")) - , m_tbaaStructKind(mdKindID(state.context, "tbaa.struct")) { } @@ -120,28 +89,22 @@ public: m_graph.m_dominators.computeIfNecessary(m_graph); m_ftlState.module = - moduleCreateWithNameInContext(name.data(), m_ftlState.context); + llvm->ModuleCreateWithNameInContext(name.data(), m_ftlState.context); m_ftlState.function = addFunction( - m_ftlState.module, name.data(), functionType(m_out.int64)); + m_ftlState.module, name.data(), functionType(m_out.int64, m_out.intPtr)); setFunctionCallingConv(m_ftlState.function, LLVMCCallConv); - if (isX86() && Options::llvmDisallowAVX()) { - // AVX makes V8/raytrace 80% slower. It makes Kraken/audio-oscillator 4.5x - // slower. It should be disabled. - addTargetDependentFunctionAttr(m_ftlState.function, "target-features", "-avx"); - } - - if (verboseCompilationEnabled()) - dataLog("Function ready, beginning lowering.\n"); m_out.initialize(m_ftlState.module, m_ftlState.function, m_heaps); - m_prologue = FTL_NEW_BLOCK(m_out, ("Prologue")); - LBasicBlock stackOverflow = FTL_NEW_BLOCK(m_out, ("Stack overflow")); - m_handleExceptions = FTL_NEW_BLOCK(m_out, ("Handle Exceptions")); + m_prologue = appendBasicBlock(m_ftlState.context, m_ftlState.function); + m_out.appendTo(m_prologue); + createPhiVariables(); + + m_callFrame = m_out.param(0); + m_tagTypeNumber = m_out.constInt64(TagTypeNumber); + m_tagMask = m_out.constInt64(TagMask); - LBasicBlock checkArguments = FTL_NEW_BLOCK(m_out, ("Check arguments")); - for (BlockIndex blockIndex = 0; blockIndex < m_graph.numBlocks(); ++blockIndex) { m_highBlock = m_graph.block(blockIndex); if (!m_highBlock) @@ -149,122 +112,13 @@ public: m_blocks.add(m_highBlock, FTL_NEW_BLOCK(m_out, ("Block ", *m_highBlock))); } - m_out.appendTo(m_prologue, stackOverflow); - createPhiVariables(); - - auto preOrder = m_graph.blocksInPreOrder(); - - LValue capturedAlloca = m_out.alloca(arrayType(m_out.int64, m_graph.m_nextMachineLocal)); - - m_captured = m_out.add( - m_out.ptrToInt(capturedAlloca, m_out.intPtr), - m_out.constIntPtr(m_graph.m_nextMachineLocal * sizeof(Register))); - - m_ftlState.capturedStackmapID = m_stackmapIDs++; - m_out.call( - m_out.stackmapIntrinsic(), m_out.constInt64(m_ftlState.capturedStackmapID), - m_out.int32Zero, capturedAlloca); - - // If we have any CallVarargs then we nee to have a spill slot for it. - bool hasVarargs = false; - for (BasicBlock* block : preOrder) { - for (Node* node : *block) { - switch (node->op()) { - case CallVarargs: - case CallForwardVarargs: - case ConstructVarargs: - case ConstructForwardVarargs: - hasVarargs = true; - break; - default: - break; - } - } - } - if (hasVarargs) { - LValue varargsSpillSlots = m_out.alloca( - arrayType(m_out.int64, JSCallVarargs::numSpillSlotsNeeded())); - m_ftlState.varargsSpillSlotsStackmapID = m_stackmapIDs++; - m_out.call( - m_out.stackmapIntrinsic(), m_out.constInt64(m_ftlState.varargsSpillSlotsStackmapID), - m_out.int32Zero, varargsSpillSlots); - } - - // We should not create any alloca's after this point, since they will cease to - // be mem2reg candidates. - - m_callFrame = m_out.ptrToInt( - m_out.call(m_out.frameAddressIntrinsic(), m_out.int32Zero), m_out.intPtr); - m_tagTypeNumber = m_out.constInt64(TagTypeNumber); - m_tagMask = m_out.constInt64(TagMask); - - m_out.storePtr(m_out.constIntPtr(codeBlock()), addressFor(JSStack::CodeBlock)); - - m_out.branch( - didOverflowStack(), rarely(stackOverflow), usually(checkArguments)); - - m_out.appendTo(stackOverflow, m_handleExceptions); - m_out.call(m_out.operation(operationThrowStackOverflowError), m_callFrame, m_out.constIntPtr(codeBlock())); - m_ftlState.handleStackOverflowExceptionStackmapID = m_stackmapIDs++; - m_out.call( - m_out.stackmapIntrinsic(), m_out.constInt64(m_ftlState.handleStackOverflowExceptionStackmapID), - m_out.constInt32(MacroAssembler::maxJumpReplacementSize())); - m_out.unreachable(); - - m_out.appendTo(m_handleExceptions, checkArguments); - m_ftlState.handleExceptionStackmapID = m_stackmapIDs++; - m_out.call( - m_out.stackmapIntrinsic(), m_out.constInt64(m_ftlState.handleExceptionStackmapID), - m_out.constInt32(MacroAssembler::maxJumpReplacementSize())); - m_out.unreachable(); - - m_out.appendTo(checkArguments, lowBlock(m_graph.block(0))); - availabilityMap().clear(); - availabilityMap().m_locals = Operands<Availability>(codeBlock()->numParameters(), 0); - for (unsigned i = codeBlock()->numParameters(); i--;) { - availabilityMap().m_locals.argument(i) = - Availability(FlushedAt(FlushedJSValue, virtualRegisterForArgument(i))); - } - m_codeOriginForExitTarget = CodeOrigin(0); - m_codeOriginForExitProfile = CodeOrigin(0); - m_node = nullptr; - for (unsigned i = codeBlock()->numParameters(); i--;) { - Node* node = m_graph.m_arguments[i]; - VirtualRegister operand = virtualRegisterForArgument(i); - - LValue jsValue = m_out.load64(addressFor(operand)); - - if (node) { - DFG_ASSERT(m_graph, node, operand == node->stackAccessData()->machineLocal); - - // This is a hack, but it's an effective one. It allows us to do CSE on the - // primordial load of arguments. This assumes that the GetLocal that got put in - // place of the original SetArgument doesn't have any effects before it. This - // should hold true. - m_loadedArgumentValues.add(node, jsValue); - } - - switch (m_graph.m_argumentFormats[i]) { - case FlushedInt32: - speculate(BadType, jsValueValue(jsValue), node, isNotInt32(jsValue)); - break; - case FlushedBoolean: - speculate(BadType, jsValueValue(jsValue), node, isNotBoolean(jsValue)); - break; - case FlushedCell: - speculate(BadType, jsValueValue(jsValue), node, isNotCell(jsValue)); - break; - case FlushedJSValue: - break; - default: - DFG_CRASH(m_graph, node, "Bad flush format for argument"); - break; - } - } + m_out.appendTo(m_prologue); m_out.jump(lowBlock(m_graph.block(0))); - for (BasicBlock* block : preOrder) - compileBlock(block); + Vector<BasicBlock*> depthFirst; + m_graph.getBlocksInDepthFirstOrder(depthFirst); + for (unsigned i = 0; i < depthFirst.size(); ++i) + compileBlock(depthFirst[i]); if (Options::dumpLLVMIR()) dumpModule(m_ftlState.module); @@ -289,7 +143,7 @@ private: continue; LType type; switch (node->flags() & NodeResultMask) { - case NodeResultDouble: + case NodeResultNumber: type = m_out.doubleType; break; case NodeResultInt32: @@ -305,7 +159,7 @@ private: type = m_out.int64; break; default: - DFG_CRASH(m_graph, node, "Bad Phi node result type"); + RELEASE_ASSERT_NOT_REACHED(); break; } m_phis.add(node, buildAlloca(m_out.m_builder, type)); @@ -340,16 +194,14 @@ private: m_out.appendTo(lowBlock, m_nextLowBlock); if (Options::ftlCrashes()) - m_out.trap(); + m_out.crashNonTerminal(); if (!m_highBlock->cfaHasVisited) { - if (verboseCompilationEnabled()) - dataLog("Bailing because CFA didn't reach.\n"); - crash(m_highBlock->index, UINT_MAX); + m_out.crash(); return; } - m_availabilityCalculator.beginBlock(m_highBlock); + initializeOSRExitStateForBlock(); m_state.reset(); m_state.beginBasicBlock(m_highBlock); @@ -359,47 +211,22 @@ private: break; } } - - void safelyInvalidateAfterTermination() - { - if (verboseCompilationEnabled()) - dataLog("Bailing.\n"); - crash(); - - // Invalidate dominated blocks. Under normal circumstances we would expect - // them to be invalidated already. But you can have the CFA become more - // precise over time because the structures of objects change on the main - // thread. Failing to do this would result in weird crashes due to a value - // being used but not defined. Race conditions FTW! - for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) { - BasicBlock* target = m_graph.block(blockIndex); - if (!target) - continue; - if (m_graph.m_dominators.dominates(m_highBlock, target)) { - if (verboseCompilationEnabled()) - dataLog("Block ", *target, " will bail also.\n"); - target->cfaHasVisited = false; - } - } - } - + bool compileNode(unsigned nodeIndex) { if (!m_state.isValid()) { - safelyInvalidateAfterTermination(); + m_out.unreachable(); return false; } m_node = m_highBlock->at(nodeIndex); - m_codeOriginForExitProfile = m_node->origin.semantic; - m_codeOriginForExitTarget = m_node->origin.forExit; + m_codeOriginForExitProfile = m_node->codeOrigin; + m_codeOriginForExitTarget = m_node->codeOriginForExitTarget; if (verboseCompilationEnabled()) dataLog("Lowering ", m_node, "\n"); - m_availableRecoveries.resize(0); - - m_interpreter.startExecuting(); + bool shouldExecuteEffects = m_interpreter.startExecuting(m_node); switch (m_node->op()) { case Upsilon: @@ -410,63 +237,47 @@ private: break; case JSConstant: break; - case DoubleConstant: - compileDoubleConstant(); + case WeakJSConstant: + compileWeakJSConstant(); break; - case Int52Constant: - compileInt52Constant(); - break; - case DoubleRep: - compileDoubleRep(); - break; - case DoubleAsInt32: - compileDoubleAsInt32(); - break; - case ValueRep: - compileValueRep(); - break; - case Int52Rep: - compileInt52Rep(); - break; - case ValueToInt32: - compileValueToInt32(); - break; - case BooleanToNumber: - compileBooleanToNumber(); + case GetArgument: + compileGetArgument(); break; case ExtractOSREntryLocal: compileExtractOSREntryLocal(); break; - case GetStack: - compileGetStack(); + case GetLocal: + compileGetLocal(); + break; + case SetLocal: + compileSetLocal(); break; - case PutStack: - compilePutStack(); + case MovHint: + compileMovHint(); break; - case Check: - compileNoOp(); + case ZombieHint: + compileZombieHint(); break; - case ToThis: - compileToThis(); + case Phantom: + compilePhantom(); break; case ValueAdd: compileValueAdd(); break; case ArithAdd: - case ArithSub: - compileArithAddOrSub(); + compileAddSub(); break; - case ArithClz32: - compileArithClz32(); + case ArithSub: + compileAddSub(); break; case ArithMul: compileArithMul(); break; case ArithDiv: - compileArithDiv(); + compileArithDivMod(); break; case ArithMod: - compileArithMod(); + compileArithDivMod(); break; case ArithMin: case ArithMax: @@ -475,27 +286,6 @@ private: case ArithAbs: compileArithAbs(); break; - case ArithSin: - compileArithSin(); - break; - case ArithCos: - compileArithCos(); - break; - case ArithPow: - compileArithPow(); - break; - case ArithRound: - compileArithRound(); - break; - case ArithSqrt: - compileArithSqrt(); - break; - case ArithLog: - compileArithLog(); - break; - case ArithFRound: - compileArithFRound(); - break; case ArithNegate: compileArithNegate(); break; @@ -520,23 +310,17 @@ private: case UInt32ToNumber: compileUInt32ToNumber(); break; + case Int32ToDouble: + compileInt32ToDouble(); + break; case CheckStructure: compileCheckStructure(); break; - case CheckCell: - compileCheckCell(); - break; - case CheckNotEmpty: - compileCheckNotEmpty(); - break; - case CheckBadCell: - compileCheckBadCell(); - break; - case CheckIdent: - compileCheckIdent(); + case StructureTransitionWatchpoint: + compileStructureTransitionWatchpoint(); break; - case GetExecutable: - compileGetExecutable(); + case CheckFunction: + compileCheckFunction(); break; case ArrayifyToStructure: compileArrayifyToStructure(); @@ -544,14 +328,13 @@ private: case PutStructure: compilePutStructure(); break; + case PhantomPutStructure: + compilePhantomPutStructure(); + break; case GetById: compileGetById(); break; - case In: - compileIn(); - break; case PutById: - case PutByIdDirect: compilePutById(); break; case GetButterfly: @@ -575,35 +358,11 @@ private: case GetByVal: compileGetByVal(); break; - case GetMyArgumentByVal: - compileGetMyArgumentByVal(); - break; case PutByVal: case PutByValAlias: case PutByValDirect: compilePutByVal(); break; - case ArrayPush: - compileArrayPush(); - break; - case ArrayPop: - compileArrayPop(); - break; - case CreateActivation: - compileCreateActivation(); - break; - case NewFunction: - compileNewFunction(); - break; - case CreateDirectArguments: - compileCreateDirectArguments(); - break; - case CreateScopedArguments: - compileCreateScopedArguments(); - break; - case CreateClonedArguments: - compileCreateClonedArguments(); - break; case NewObject: compileNewObject(); break; @@ -613,28 +372,9 @@ private: case NewArrayBuffer: compileNewArrayBuffer(); break; - case NewArrayWithSize: - compileNewArrayWithSize(); - break; - case GetTypedArrayByteOffset: - compileGetTypedArrayByteOffset(); - break; case AllocatePropertyStorage: compileAllocatePropertyStorage(); break; - case ReallocatePropertyStorage: - compileReallocatePropertyStorage(); - break; - case ToString: - case CallStringConstructor: - compileToStringOrCallStringConstructor(); - break; - case ToPrimitive: - compileToPrimitive(); - break; - case MakeRope: - compileMakeRope(); - break; case StringCharAt: compileStringCharAt(); break; @@ -642,24 +382,11 @@ private: compileStringCharCodeAt(); break; case GetByOffset: - case GetGetterSetterByOffset: compileGetByOffset(); break; - case GetGetter: - compileGetGetter(); - break; - case GetSetter: - compileGetSetter(); - break; - case MultiGetByOffset: - compileMultiGetByOffset(); - break; case PutByOffset: compilePutByOffset(); break; - case MultiPutByOffset: - compileMultiPutByOffset(); - break; case GetGlobalVar: compileGetGlobalVar(); break; @@ -669,30 +396,21 @@ private: case NotifyWrite: compileNotifyWrite(); break; - case GetCallee: - compileGetCallee(); - break; - case GetArgumentCount: - compileGetArgumentCount(); - break; - case GetScope: - compileGetScope(); + case GetMyScope: + compileGetMyScope(); break; case SkipScope: compileSkipScope(); break; + case GetClosureRegisters: + compileGetClosureRegisters(); + break; case GetClosureVar: compileGetClosureVar(); break; case PutClosureVar: compilePutClosureVar(); break; - case GetFromArguments: - compileGetFromArguments(); - break; - case PutToArguments: - compilePutToArguments(); - break; case CompareEq: compileCompareEq(); break; @@ -702,6 +420,9 @@ private: case CompareStrictEq: compileCompareStrictEq(); break; + case CompareStrictEqConstant: + compileCompareStrictEqConstant(); + break; case CompareLess: compileCompareLess(); break; @@ -721,18 +442,6 @@ private: case Construct: compileCallOrConstruct(); break; - case CallVarargs: - case CallForwardVarargs: - case ConstructVarargs: - case ConstructForwardVarargs: - compileCallOrConstructVarargs(); - break; - case LoadVarargs: - compileLoadVarargs(); - break; - case ForwardVarargs: - compileForwardVarargs(); - break; case Jump: compileJump(); break; @@ -748,116 +457,168 @@ private: case ForceOSRExit: compileForceOSRExit(); break; - case Throw: - case ThrowReferenceError: - compileThrow(); - break; case InvalidationPoint: compileInvalidationPoint(); break; - case IsUndefined: - compileIsUndefined(); - break; - case IsBoolean: - compileIsBoolean(); - break; - case IsNumber: - compileIsNumber(); - break; - case IsString: - compileIsString(); - break; - case IsObject: - compileIsObject(); - break; - case IsObjectOrNull: - compileIsObjectOrNull(); - break; - case IsFunction: - compileIsFunction(); - break; - case TypeOf: - compileTypeOf(); - break; - case CheckHasInstance: - compileCheckHasInstance(); - break; - case InstanceOf: - compileInstanceOf(); + case ValueToInt32: + compileValueToInt32(); break; - case CountExecution: - compileCountExecution(); + case Int52ToValue: + compileInt52ToValue(); break; case StoreBarrier: compileStoreBarrier(); break; - case HasIndexedProperty: - compileHasIndexedProperty(); - break; - case HasGenericProperty: - compileHasGenericProperty(); - break; - case HasStructureProperty: - compileHasStructureProperty(); + case ConditionalStoreBarrier: + compileConditionalStoreBarrier(); break; - case GetDirectPname: - compileGetDirectPname(); + case StoreBarrierWithNullCheck: + compileStoreBarrierWithNullCheck(); break; - case GetEnumerableLength: - compileGetEnumerableLength(); - break; - case GetPropertyEnumerator: - compileGetPropertyEnumerator(); - break; - case GetEnumeratorStructurePname: - compileGetEnumeratorStructurePname(); - break; - case GetEnumeratorGenericPname: - compileGetEnumeratorGenericPname(); - break; - case ToIndexString: - compileToIndexString(); - break; - case CheckStructureImmediate: - compileCheckStructureImmediate(); - break; - case MaterializeNewObject: - compileMaterializeNewObject(); - break; - case MaterializeCreateActivation: - compileMaterializeCreateActivation(); - break; - + case Flush: case PhantomLocal: + case SetArgument: case LoopHint: - case MovHint: - case ZombieHint: - case PhantomNewObject: - case PhantomNewFunction: - case PhantomCreateActivation: - case PhantomDirectArguments: - case PhantomClonedArguments: - case PutHint: - case BottomValue: - case KillStack: + case VariableWatchpoint: + case FunctionReentryWatchpoint: + case TypedArrayWatchpoint: break; default: - DFG_CRASH(m_graph, m_node, "Unrecognized node in FTL backend"); + RELEASE_ASSERT_NOT_REACHED(); break; } - if (m_node->isTerminal()) - return false; + if (shouldExecuteEffects) + m_interpreter.executeEffects(nodeIndex); - if (!m_state.isValid()) { - safelyInvalidateAfterTermination(); - return false; + return true; + } + + void compileValueToInt32() + { + switch (m_node->child1().useKind()) { + case Int32Use: + setInt32(lowInt32(m_node->child1())); + break; + + case MachineIntUse: + setInt32(m_out.castToInt32(lowStrictInt52(m_node->child1()))); + break; + + case NumberUse: + case NotCellUse: { + LoweredNodeValue value = m_int32Values.get(m_node->child1().node()); + if (isValid(value)) { + setInt32(value.value()); + break; + } + + value = m_jsValueValues.get(m_node->child1().node()); + if (isValid(value)) { + LBasicBlock intCase = FTL_NEW_BLOCK(m_out, ("ValueToInt32 int case")); + LBasicBlock notIntCase = FTL_NEW_BLOCK(m_out, ("ValueToInt32 not int case")); + LBasicBlock doubleCase = 0; + LBasicBlock notNumberCase = 0; + if (m_node->child1().useKind() == NotCellUse) { + doubleCase = FTL_NEW_BLOCK(m_out, ("ValueToInt32 double case")); + notNumberCase = FTL_NEW_BLOCK(m_out, ("ValueToInt32 not number case")); + } + LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ValueToInt32 continuation")); + + Vector<ValueFromBlock> results; + + m_out.branch(isNotInt32(value.value()), notIntCase, intCase); + + LBasicBlock lastNext = m_out.appendTo(intCase, notIntCase); + results.append(m_out.anchor(unboxInt32(value.value()))); + m_out.jump(continuation); + + if (m_node->child1().useKind() == NumberUse) { + m_out.appendTo(notIntCase, continuation); + FTL_TYPE_CHECK( + jsValueValue(value.value()), m_node->child1(), SpecFullNumber, + isCellOrMisc(value.value())); + results.append(m_out.anchor(doubleToInt32(unboxDouble(value.value())))); + m_out.jump(continuation); + } else { + m_out.appendTo(notIntCase, doubleCase); + m_out.branch(isCellOrMisc(value.value()), notNumberCase, doubleCase); + + m_out.appendTo(doubleCase, notNumberCase); + results.append(m_out.anchor(doubleToInt32(unboxDouble(value.value())))); + m_out.jump(continuation); + + m_out.appendTo(notNumberCase, continuation); + + FTL_TYPE_CHECK( + jsValueValue(value.value()), m_node->child1(), ~SpecCell, + isCell(value.value())); + + LValue specialResult = m_out.select( + m_out.equal( + value.value(), + m_out.constInt64(JSValue::encode(jsBoolean(true)))), + m_out.int32One, m_out.int32Zero); + results.append(m_out.anchor(specialResult)); + m_out.jump(continuation); + } + + m_out.appendTo(continuation, lastNext); + setInt32(m_out.phi(m_out.int32, results)); + break; + } + + value = m_doubleValues.get(m_node->child1().node()); + if (isValid(value)) { + setInt32(doubleToInt32(value.value())); + break; + } + + terminate(Uncountable); + break; } + + case BooleanUse: + setInt32(m_out.zeroExt(lowBoolean(m_node->child1()), m_out.int32)); + break; + + default: + RELEASE_ASSERT_NOT_REACHED(); + break; + } + } - m_availabilityCalculator.executeNode(m_node); - m_interpreter.executeEffects(nodeIndex); - - return true; + void compileInt52ToValue() + { + setJSValue(lowJSValue(m_node->child1())); + } + + void compileStoreBarrier() + { + emitStoreBarrier(lowCell(m_node->child1())); + } + + void compileConditionalStoreBarrier() + { + LValue base = lowCell(m_node->child1()); + LValue value = lowJSValue(m_node->child2()); + emitStoreBarrier(base, value, m_node->child2()); + } + + void compileStoreBarrierWithNullCheck() + { +#if ENABLE(GGC) + LBasicBlock isNotNull = FTL_NEW_BLOCK(m_out, ("Store barrier with null check value not null")); + LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Store barrier continuation")); + + LValue base = lowJSValue(m_node->child1()); + m_out.branch(m_out.isZero64(base), continuation, isNotNull); + LBasicBlock lastNext = m_out.appendTo(isNotNull, continuation); + emitStoreBarrier(base); + m_out.appendTo(continuation, lastNext); +#else + speculate(m_node->child1()); +#endif } void compileUpsilon() @@ -865,13 +626,13 @@ private: LValue destination = m_phis.get(m_node->phi()); switch (m_node->child1().useKind()) { - case DoubleRepUse: + case NumberUse: m_out.set(lowDouble(m_node->child1()), destination); break; case Int32Use: m_out.set(lowInt32(m_node->child1()), destination); break; - case Int52RepUse: + case MachineIntUse: m_out.set(lowInt52(m_node->child1()), destination); break; case BooleanUse: @@ -884,7 +645,7 @@ private: m_out.set(lowJSValue(m_node->child1()), destination); break; default: - DFG_CRASH(m_graph, m_node, "Bad use kind"); + RELEASE_ASSERT_NOT_REACHED(); break; } } @@ -894,7 +655,7 @@ private: LValue source = m_phis.get(m_node); switch (m_node->flags() & NodeResultMask) { - case NodeResultDouble: + case NodeResultNumber: setDouble(m_out.get(source)); break; case NodeResultInt32: @@ -910,282 +671,46 @@ private: setJSValue(m_out.get(source)); break; default: - DFG_CRASH(m_graph, m_node, "Bad use kind"); + RELEASE_ASSERT_NOT_REACHED(); break; } } - void compileDoubleConstant() + void compileWeakJSConstant() { - setDouble(m_out.constDouble(m_node->asNumber())); + setJSValue(weakPointer(m_node->weakConstant())); } - void compileInt52Constant() + void compileGetArgument() { - int64_t value = m_node->asMachineInt(); - - setInt52(m_out.constInt64(value << JSValue::int52ShiftAmount)); - setStrictInt52(m_out.constInt64(value)); - } + VariableAccessData* variable = m_node->variableAccessData(); + VirtualRegister operand = variable->machineLocal(); + RELEASE_ASSERT(operand.isArgument()); - void compileDoubleRep() - { - switch (m_node->child1().useKind()) { - case RealNumberUse: { - LValue value = lowJSValue(m_node->child1(), ManualOperandSpeculation); - - LValue doubleValue = unboxDouble(value); - - LBasicBlock intCase = FTL_NEW_BLOCK(m_out, ("DoubleRep RealNumberUse int case")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("DoubleRep continuation")); - - ValueFromBlock fastResult = m_out.anchor(doubleValue); - m_out.branch( - m_out.doubleEqual(doubleValue, doubleValue), - usually(continuation), rarely(intCase)); - - LBasicBlock lastNext = m_out.appendTo(intCase, continuation); - - FTL_TYPE_CHECK( - jsValueValue(value), m_node->child1(), SpecBytecodeRealNumber, - isNotInt32(value, provenType(m_node->child1()) & ~SpecFullDouble)); - ValueFromBlock slowResult = m_out.anchor(m_out.intToDouble(unboxInt32(value))); - m_out.jump(continuation); - - m_out.appendTo(continuation, lastNext); - - setDouble(m_out.phi(m_out.doubleType, fastResult, slowResult)); - return; - } - - case NotCellUse: - case NumberUse: { - bool shouldConvertNonNumber = m_node->child1().useKind() == NotCellUse; - - LValue value = lowJSValue(m_node->child1(), ManualOperandSpeculation); - - LBasicBlock intCase = FTL_NEW_BLOCK(m_out, ("jsValueToDouble unboxing int case")); - LBasicBlock doubleTesting = FTL_NEW_BLOCK(m_out, ("jsValueToDouble testing double case")); - LBasicBlock doubleCase = FTL_NEW_BLOCK(m_out, ("jsValueToDouble unboxing double case")); - LBasicBlock nonDoubleCase = FTL_NEW_BLOCK(m_out, ("jsValueToDouble testing undefined case")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("jsValueToDouble unboxing continuation")); - - m_out.branch( - isNotInt32(value, provenType(m_node->child1())), - unsure(doubleTesting), unsure(intCase)); - - LBasicBlock lastNext = m_out.appendTo(intCase, doubleTesting); - - ValueFromBlock intToDouble = m_out.anchor( - m_out.intToDouble(unboxInt32(value))); - m_out.jump(continuation); - - m_out.appendTo(doubleTesting, doubleCase); - LValue valueIsNumber = isNumber(value, provenType(m_node->child1())); - m_out.branch(valueIsNumber, usually(doubleCase), rarely(nonDoubleCase)); - - m_out.appendTo(doubleCase, nonDoubleCase); - ValueFromBlock unboxedDouble = m_out.anchor(unboxDouble(value)); - m_out.jump(continuation); - - if (shouldConvertNonNumber) { - LBasicBlock undefinedCase = FTL_NEW_BLOCK(m_out, ("jsValueToDouble converting undefined case")); - LBasicBlock testNullCase = FTL_NEW_BLOCK(m_out, ("jsValueToDouble testing null case")); - LBasicBlock nullCase = FTL_NEW_BLOCK(m_out, ("jsValueToDouble converting null case")); - LBasicBlock testBooleanTrueCase = FTL_NEW_BLOCK(m_out, ("jsValueToDouble testing boolean true case")); - LBasicBlock convertBooleanTrueCase = FTL_NEW_BLOCK(m_out, ("jsValueToDouble convert boolean true case")); - LBasicBlock convertBooleanFalseCase = FTL_NEW_BLOCK(m_out, ("jsValueToDouble convert boolean false case")); - - m_out.appendTo(nonDoubleCase, undefinedCase); - LValue valueIsUndefined = m_out.equal(value, m_out.constInt64(ValueUndefined)); - m_out.branch(valueIsUndefined, unsure(undefinedCase), unsure(testNullCase)); - - m_out.appendTo(undefinedCase, testNullCase); - ValueFromBlock convertedUndefined = m_out.anchor(m_out.constDouble(PNaN)); - m_out.jump(continuation); + LValue jsValue = m_out.load64(addressFor(operand)); - m_out.appendTo(testNullCase, nullCase); - LValue valueIsNull = m_out.equal(value, m_out.constInt64(ValueNull)); - m_out.branch(valueIsNull, unsure(nullCase), unsure(testBooleanTrueCase)); - - m_out.appendTo(nullCase, testBooleanTrueCase); - ValueFromBlock convertedNull = m_out.anchor(m_out.constDouble(0)); - m_out.jump(continuation); - - m_out.appendTo(testBooleanTrueCase, convertBooleanTrueCase); - LValue valueIsBooleanTrue = m_out.equal(value, m_out.constInt64(ValueTrue)); - m_out.branch(valueIsBooleanTrue, unsure(convertBooleanTrueCase), unsure(convertBooleanFalseCase)); - - m_out.appendTo(convertBooleanTrueCase, convertBooleanFalseCase); - ValueFromBlock convertedTrue = m_out.anchor(m_out.constDouble(1)); - m_out.jump(continuation); - - m_out.appendTo(convertBooleanFalseCase, continuation); - - LValue valueIsNotBooleanFalse = m_out.notEqual(value, m_out.constInt64(ValueFalse)); - FTL_TYPE_CHECK(jsValueValue(value), m_node->child1(), ~SpecCell, valueIsNotBooleanFalse); - ValueFromBlock convertedFalse = m_out.anchor(m_out.constDouble(0)); - m_out.jump(continuation); - - m_out.appendTo(continuation, lastNext); - setDouble(m_out.phi(m_out.doubleType, intToDouble, unboxedDouble, convertedUndefined, convertedNull, convertedTrue, convertedFalse)); - return; - } - m_out.appendTo(nonDoubleCase, continuation); - FTL_TYPE_CHECK(jsValueValue(value), m_node->child1(), SpecBytecodeNumber, m_out.booleanTrue); - m_out.unreachable(); - - m_out.appendTo(continuation, lastNext); - - setDouble(m_out.phi(m_out.doubleType, intToDouble, unboxedDouble)); - return; - } - - case Int52RepUse: { - setDouble(strictInt52ToDouble(lowStrictInt52(m_node->child1()))); - return; - } - - default: - DFG_CRASH(m_graph, m_node, "Bad use kind"); - } - } - - void compileDoubleAsInt32() - { - LValue integerValue = convertDoubleToInt32(lowDouble(m_node->child1()), shouldCheckNegativeZero(m_node->arithMode())); - setInt32(integerValue); - } - - void compileValueRep() - { - switch (m_node->child1().useKind()) { - case DoubleRepUse: { - LValue value = lowDouble(m_node->child1()); - - if (m_interpreter.needsTypeCheck(m_node->child1(), ~SpecDoubleImpureNaN)) { - value = m_out.select( - m_out.doubleEqual(value, value), value, m_out.constDouble(PNaN)); - } - - setJSValue(boxDouble(value)); - return; - } - - case Int52RepUse: { - setJSValue(strictInt52ToJSValue(lowStrictInt52(m_node->child1()))); - return; - } - - default: - DFG_CRASH(m_graph, m_node, "Bad use kind"); - } - } - - void compileInt52Rep() - { - switch (m_node->child1().useKind()) { + switch (useKindFor(variable->flushFormat())) { case Int32Use: - setStrictInt52(m_out.signExt(lowInt32(m_node->child1()), m_out.int64)); - return; - - case MachineIntUse: - setStrictInt52( - jsValueToStrictInt52( - m_node->child1(), lowJSValue(m_node->child1(), ManualOperandSpeculation))); - return; - - case DoubleRepMachineIntUse: - setStrictInt52( - doubleToStrictInt52( - m_node->child1(), lowDouble(m_node->child1()))); - return; - - default: - RELEASE_ASSERT_NOT_REACHED(); - } - } - - void compileValueToInt32() - { - switch (m_node->child1().useKind()) { - case Int52RepUse: - setInt32(m_out.castToInt32(lowStrictInt52(m_node->child1()))); + speculate(BadType, jsValueValue(jsValue), m_node, isNotInt32(jsValue)); + setInt32(unboxInt32(jsValue)); break; - - case DoubleRepUse: - setInt32(doubleToInt32(lowDouble(m_node->child1()))); + case CellUse: + speculate(BadType, jsValueValue(jsValue), m_node, isNotCell(jsValue)); + setJSValue(jsValue); break; - - case NumberUse: - case NotCellUse: { - LoweredNodeValue value = m_int32Values.get(m_node->child1().node()); - if (isValid(value)) { - setInt32(value.value()); - break; - } - - value = m_jsValueValues.get(m_node->child1().node()); - if (isValid(value)) { - setInt32(numberOrNotCellToInt32(m_node->child1(), value.value())); - break; - } - - // We'll basically just get here for constants. But it's good to have this - // catch-all since we often add new representations into the mix. - setInt32( - numberOrNotCellToInt32( - m_node->child1(), - lowJSValue(m_node->child1(), ManualOperandSpeculation))); + case BooleanUse: + speculate(BadType, jsValueValue(jsValue), m_node, isNotBoolean(jsValue)); + setBoolean(unboxBoolean(jsValue)); break; - } - - default: - DFG_CRASH(m_graph, m_node, "Bad use kind"); + case UntypedUse: + setJSValue(jsValue); break; - } - } - - void compileBooleanToNumber() - { - switch (m_node->child1().useKind()) { - case BooleanUse: { - setInt32(m_out.zeroExt(lowBoolean(m_node->child1()), m_out.int32)); - return; - } - - case UntypedUse: { - LValue value = lowJSValue(m_node->child1()); - - if (!m_interpreter.needsTypeCheck(m_node->child1(), SpecBoolInt32 | SpecBoolean)) { - setInt32(m_out.bitAnd(m_out.castToInt32(value), m_out.int32One)); - return; - } - - LBasicBlock booleanCase = FTL_NEW_BLOCK(m_out, ("BooleanToNumber boolean case")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("BooleanToNumber continuation")); - - ValueFromBlock notBooleanResult = m_out.anchor(value); - m_out.branch( - isBoolean(value, provenType(m_node->child1())), - unsure(booleanCase), unsure(continuation)); - - LBasicBlock lastNext = m_out.appendTo(booleanCase, continuation); - ValueFromBlock booleanResult = m_out.anchor(m_out.bitOr( - m_out.zeroExt(unboxBoolean(value), m_out.int64), m_tagTypeNumber)); - m_out.jump(continuation); - - m_out.appendTo(continuation, lastNext); - setJSValue(m_out.phi(m_out.int64, booleanResult, notBooleanResult)); - return; - } - default: RELEASE_ASSERT_NOT_REACHED(); - return; + break; } } - + void compileExtractOSREntryLocal() { EncodedJSValue* buffer = static_cast<EncodedJSValue*>( @@ -1193,58 +718,52 @@ private: setJSValue(m_out.load64(m_out.absolute(buffer + m_node->unlinkedLocal().toLocal()))); } - void compileGetStack() + void compileGetLocal() { - // GetLocals arise only for captured variables and arguments. For arguments, we might have - // already loaded it. - if (LValue value = m_loadedArgumentValues.get(m_node)) { - setJSValue(value); - return; - } + // GetLocals arise only for captured variables. - StackAccessData* data = m_node->stackAccessData(); - AbstractValue& value = m_state.variables().operand(data->local); + VariableAccessData* variable = m_node->variableAccessData(); + AbstractValue& value = m_state.variables().operand(variable->local()); - DFG_ASSERT(m_graph, m_node, isConcrete(data->format)); - DFG_ASSERT(m_graph, m_node, data->format != FlushedDouble); // This just happens to not arise for GetStacks, right now. It would be trivial to support. + RELEASE_ASSERT(variable->isCaptured()); if (isInt32Speculation(value.m_type)) - setInt32(m_out.load32(payloadFor(data->machineLocal))); + setInt32(m_out.load32(payloadFor(variable->machineLocal()))); else - setJSValue(m_out.load64(addressFor(data->machineLocal))); + setJSValue(m_out.load64(addressFor(variable->machineLocal()))); } - void compilePutStack() + void compileSetLocal() { - StackAccessData* data = m_node->stackAccessData(); - switch (data->format) { + VariableAccessData* variable = m_node->variableAccessData(); + switch (variable->flushFormat()) { case FlushedJSValue: { LValue value = lowJSValue(m_node->child1()); - m_out.store64(value, addressFor(data->machineLocal)); + m_out.store64(value, addressFor(variable->machineLocal())); break; } case FlushedDouble: { LValue value = lowDouble(m_node->child1()); - m_out.storeDouble(value, addressFor(data->machineLocal)); + m_out.storeDouble(value, addressFor(variable->machineLocal())); break; } case FlushedInt32: { LValue value = lowInt32(m_node->child1()); - m_out.store32(value, payloadFor(data->machineLocal)); + m_out.store32(value, payloadFor(variable->machineLocal())); break; } case FlushedInt52: { LValue value = lowInt52(m_node->child1()); - m_out.store64(value, addressFor(data->machineLocal)); + m_out.store64(value, addressFor(variable->machineLocal())); break; } case FlushedCell: { LValue value = lowCell(m_node->child1()); - m_out.store64(value, addressFor(data->machineLocal)); + m_out.store64(value, addressFor(variable->machineLocal())); break; } @@ -1252,55 +771,42 @@ private: speculateBoolean(m_node->child1()); m_out.store64( lowJSValue(m_node->child1(), ManualOperandSpeculation), - addressFor(data->machineLocal)); + addressFor(variable->machineLocal())); break; } default: - DFG_CRASH(m_graph, m_node, "Bad flush format"); + RELEASE_ASSERT_NOT_REACHED(); break; } + + m_availability.operand(variable->local()) = Availability(variable->flushedAt()); } - void compileNoOp() + void compileMovHint() { - DFG_NODE_DO_TO_CHILDREN(m_graph, m_node, speculate); + ASSERT(m_node->containsMovHint()); + ASSERT(m_node->op() != ZombieHint); + + VirtualRegister operand = m_node->unlinkedLocal(); + m_availability.operand(operand) = Availability(m_node->child1().node()); } - void compileToThis() + void compileZombieHint() { - LValue value = lowJSValue(m_node->child1()); - - LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("ToThis is cell case")); - LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("ToThis slow case")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ToThis continuation")); - - m_out.branch( - isCell(value, provenType(m_node->child1())), usually(isCellCase), rarely(slowCase)); - - LBasicBlock lastNext = m_out.appendTo(isCellCase, slowCase); - ValueFromBlock fastResult = m_out.anchor(value); - m_out.branch(isType(value, FinalObjectType), usually(continuation), rarely(slowCase)); - - m_out.appendTo(slowCase, continuation); - J_JITOperation_EJ function; - if (m_graph.isStrictModeFor(m_node->origin.semantic)) - function = operationToThisStrict; - else - function = operationToThis; - ValueFromBlock slowResult = m_out.anchor( - vmCall(m_out.operation(function), m_callFrame, value)); - m_out.jump(continuation); - - m_out.appendTo(continuation, lastNext); - setJSValue(m_out.phi(m_out.int64, fastResult, slowResult)); + m_availability.operand(m_node->unlinkedLocal()) = Availability::unavailable(); + } + + void compilePhantom() + { + DFG_NODE_DO_TO_CHILDREN(m_graph, m_node, speculate); } void compileValueAdd() { J_JITOperation_EJJ operation; - if (!(provenType(m_node->child1()) & SpecFullNumber) - && !(provenType(m_node->child2()) & SpecFullNumber)) + if (!(m_state.forNode(m_node->child1()).m_type & SpecFullNumber) + && !(m_state.forNode(m_node->child2()).m_type & SpecFullNumber)) operation = operationValueAddNotNumber; else operation = operationValueAdd; @@ -1309,59 +815,30 @@ private: lowJSValue(m_node->child1()), lowJSValue(m_node->child2()))); } - void compileArithAddOrSub() + void compileAddSub() { bool isSub = m_node->op() == ArithSub; switch (m_node->binaryUseKind()) { case Int32Use: { LValue left = lowInt32(m_node->child1()); LValue right = lowInt32(m_node->child2()); + LValue result = isSub ? m_out.sub(left, right) : m_out.add(left, right); if (!shouldCheckOverflow(m_node->arithMode())) { - setInt32(isSub ? m_out.sub(left, right) : m_out.add(left, right)); + setInt32(result); break; } - LValue result; - if (!isSub) { - result = m_out.addWithOverflow32(left, right); - - if (doesKill(m_node->child2())) { - addAvailableRecovery( - m_node->child2(), SubRecovery, - m_out.extractValue(result, 0), left, ValueFormatInt32); - } else if (doesKill(m_node->child1())) { - addAvailableRecovery( - m_node->child1(), SubRecovery, - m_out.extractValue(result, 0), right, ValueFormatInt32); - } - } else { - result = m_out.subWithOverflow32(left, right); - - if (doesKill(m_node->child2())) { - // result = left - right - // result - left = -right - // right = left - result - addAvailableRecovery( - m_node->child2(), SubRecovery, - left, m_out.extractValue(result, 0), ValueFormatInt32); - } else if (doesKill(m_node->child1())) { - // result = left - right - // result + right = left - addAvailableRecovery( - m_node->child1(), AddRecovery, - m_out.extractValue(result, 0), right, ValueFormatInt32); - } - } + LValue overflow = isSub ? m_out.subWithOverflow32(left, right) : m_out.addWithOverflow32(left, right); - speculate(Overflow, noValue(), 0, m_out.extractValue(result, 1)); - setInt32(m_out.extractValue(result, 0)); + speculate(Overflow, noValue(), 0, m_out.extractValue(overflow, 1)); + setInt32(result); break; } - case Int52RepUse: { - if (!abstractValue(m_node->child1()).couldBeType(SpecInt52) - && !abstractValue(m_node->child2()).couldBeType(SpecInt52)) { + case MachineIntUse: { + if (!m_state.forNode(m_node->child1()).couldBeType(SpecInt52) + && !m_state.forNode(m_node->child2()).couldBeType(SpecInt52)) { Int52Kind kind; LValue left = lowWhicheverInt52(m_node->child1(), kind); LValue right = lowInt52(m_node->child2(), kind); @@ -1371,45 +848,15 @@ private: LValue left = lowInt52(m_node->child1()); LValue right = lowInt52(m_node->child2()); + LValue result = isSub ? m_out.sub(left, right) : m_out.add(left, right); - LValue result; - if (!isSub) { - result = m_out.addWithOverflow64(left, right); - - if (doesKill(m_node->child2())) { - addAvailableRecovery( - m_node->child2(), SubRecovery, - m_out.extractValue(result, 0), left, ValueFormatInt52); - } else if (doesKill(m_node->child1())) { - addAvailableRecovery( - m_node->child1(), SubRecovery, - m_out.extractValue(result, 0), right, ValueFormatInt52); - } - } else { - result = m_out.subWithOverflow64(left, right); - - if (doesKill(m_node->child2())) { - // result = left - right - // result - left = -right - // right = left - result - addAvailableRecovery( - m_node->child2(), SubRecovery, - left, m_out.extractValue(result, 0), ValueFormatInt52); - } else if (doesKill(m_node->child1())) { - // result = left - right - // result + right = left - addAvailableRecovery( - m_node->child1(), AddRecovery, - m_out.extractValue(result, 0), right, ValueFormatInt52); - } - } - - speculate(Int52Overflow, noValue(), 0, m_out.extractValue(result, 1)); - setInt52(m_out.extractValue(result, 0)); + LValue overflow = isSub ? m_out.subWithOverflow64(left, right) : m_out.addWithOverflow64(left, right); + speculate(Int52Overflow, noValue(), 0, m_out.extractValue(overflow, 1)); + setInt52(result); break; } - case DoubleRepUse: { + case NumberUse: { LValue C1 = lowDouble(m_node->child1()); LValue C2 = lowDouble(m_node->child2()); @@ -1418,17 +865,10 @@ private: } default: - DFG_CRASH(m_graph, m_node, "Bad use kind"); + RELEASE_ASSERT_NOT_REACHED(); break; } } - - void compileArithClz32() - { - LValue operand = lowInt32(m_node->child1()); - LValue isZeroUndef = m_out.booleanFalse; - setInt32(m_out.ctlz32(operand, isZeroUndef)); - } void compileArithMul() { @@ -1436,23 +876,18 @@ private: case Int32Use: { LValue left = lowInt32(m_node->child1()); LValue right = lowInt32(m_node->child2()); - - LValue result; + LValue result = m_out.mul(left, right); - if (!shouldCheckOverflow(m_node->arithMode())) - result = m_out.mul(left, right); - else { + if (shouldCheckOverflow(m_node->arithMode())) { LValue overflowResult = m_out.mulWithOverflow32(left, right); speculate(Overflow, noValue(), 0, m_out.extractValue(overflowResult, 1)); - result = m_out.extractValue(overflowResult, 0); } if (shouldCheckNegativeZero(m_node->arithMode())) { LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("ArithMul slow case")); LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMul continuation")); - m_out.branch( - m_out.notZero32(result), usually(continuation), rarely(slowCase)); + m_out.branch(m_out.notZero32(result), continuation, slowCase); LBasicBlock lastNext = m_out.appendTo(slowCase, continuation); LValue cond = m_out.bitOr(m_out.lessThan(left, m_out.int32Zero), m_out.lessThan(right, m_out.int32Zero)); @@ -1465,21 +900,21 @@ private: break; } - case Int52RepUse: { + case MachineIntUse: { Int52Kind kind; LValue left = lowWhicheverInt52(m_node->child1(), kind); LValue right = lowInt52(m_node->child2(), opposite(kind)); + LValue result = m_out.mul(left, right); + LValue overflowResult = m_out.mulWithOverflow64(left, right); speculate(Int52Overflow, noValue(), 0, m_out.extractValue(overflowResult, 1)); - LValue result = m_out.extractValue(overflowResult, 0); if (shouldCheckNegativeZero(m_node->arithMode())) { LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("ArithMul slow case")); LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMul continuation")); - m_out.branch( - m_out.notZero64(result), usually(continuation), rarely(slowCase)); + m_out.branch(m_out.notZero64(result), continuation, slowCase); LBasicBlock lastNext = m_out.appendTo(slowCase, continuation); LValue cond = m_out.bitOr(m_out.lessThan(left, m_out.int64Zero), m_out.lessThan(right, m_out.int64Zero)); @@ -1492,36 +927,34 @@ private: break; } - case DoubleRepUse: { + case NumberUse: { setDouble( m_out.doubleMul(lowDouble(m_node->child1()), lowDouble(m_node->child2()))); break; } default: - DFG_CRASH(m_graph, m_node, "Bad use kind"); + RELEASE_ASSERT_NOT_REACHED(); break; } } - void compileArithDiv() + void compileArithDivMod() { switch (m_node->binaryUseKind()) { case Int32Use: { LValue numerator = lowInt32(m_node->child1()); LValue denominator = lowInt32(m_node->child2()); - LBasicBlock unsafeDenominator = FTL_NEW_BLOCK(m_out, ("ArithDiv unsafe denominator")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithDiv continuation")); - LBasicBlock done = FTL_NEW_BLOCK(m_out, ("ArithDiv done")); + LBasicBlock unsafeDenominator = FTL_NEW_BLOCK(m_out, ("ArithDivMod unsafe denominator")); + LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithDivMod continuation")); + LBasicBlock done = FTL_NEW_BLOCK(m_out, ("ArithDivMod done")); Vector<ValueFromBlock, 3> results; LValue adjustedDenominator = m_out.add(denominator, m_out.int32One); - m_out.branch( - m_out.above(adjustedDenominator, m_out.int32One), - usually(continuation), rarely(unsafeDenominator)); + m_out.branch(m_out.above(adjustedDenominator, m_out.int32One), continuation, unsafeDenominator); LBasicBlock lastNext = m_out.appendTo(unsafeDenominator, continuation); @@ -1541,17 +974,14 @@ private: LBasicBlock notDivByZero = FTL_NEW_BLOCK(m_out, ("ArithDiv not divide by zero")); LBasicBlock neg2ToThe31ByNeg1 = FTL_NEW_BLOCK(m_out, ("ArithDiv -2^31/-1")); - m_out.branch( - m_out.isZero32(denominator), rarely(divByZero), usually(notDivByZero)); + m_out.branch(m_out.isZero32(denominator), divByZero, notDivByZero); m_out.appendTo(divByZero, notDivByZero); results.append(m_out.anchor(m_out.int32Zero)); m_out.jump(done); m_out.appendTo(notDivByZero, neg2ToThe31ByNeg1); - m_out.branch( - m_out.equal(numerator, neg2ToThe31), - rarely(neg2ToThe31ByNeg1), usually(continuation)); + m_out.branch(m_out.equal(numerator, neg2ToThe31), neg2ToThe31ByNeg1, continuation); m_out.appendTo(neg2ToThe31ByNeg1, continuation); results.append(m_out.anchor(neg2ToThe31)); @@ -1561,12 +991,10 @@ private: m_out.appendTo(continuation, done); if (shouldCheckNegativeZero(m_node->arithMode())) { - LBasicBlock zeroNumerator = FTL_NEW_BLOCK(m_out, ("ArithDiv zero numerator")); - LBasicBlock numeratorContinuation = FTL_NEW_BLOCK(m_out, ("ArithDiv numerator continuation")); + LBasicBlock zeroNumerator = FTL_NEW_BLOCK(m_out, ("ArithDivMod zero numerator")); + LBasicBlock numeratorContinuation = FTL_NEW_BLOCK(m_out, ("ArithDivMod numerator continuation")); - m_out.branch( - m_out.isZero32(numerator), - rarely(zeroNumerator), usually(numeratorContinuation)); + m_out.branch(m_out.isZero32(numerator), zeroNumerator, numeratorContinuation); LBasicBlock innerLastNext = m_out.appendTo(zeroNumerator, numeratorContinuation); @@ -1578,15 +1006,17 @@ private: m_out.appendTo(numeratorContinuation, innerLastNext); } - LValue result = m_out.div(numerator, denominator); + LValue divModResult = m_node->op() == ArithDiv + ? m_out.div(numerator, denominator) + : m_out.rem(numerator, denominator); if (shouldCheckOverflow(m_node->arithMode())) { speculate( Overflow, noValue(), 0, - m_out.notEqual(m_out.mul(result, denominator), numerator)); + m_out.notEqual(m_out.mul(divModResult, denominator), numerator)); } - results.append(m_out.anchor(result)); + results.append(m_out.anchor(divModResult)); m_out.jump(done); m_out.appendTo(done, lastNext); @@ -1595,116 +1025,19 @@ private: break; } - case DoubleRepUse: { - setDouble(m_out.doubleDiv( - lowDouble(m_node->child1()), lowDouble(m_node->child2()))); + case NumberUse: { + LValue C1 = lowDouble(m_node->child1()); + LValue C2 = lowDouble(m_node->child2()); + setDouble(m_node->op() == ArithDiv ? m_out.doubleDiv(C1, C2) : m_out.doubleRem(C1, C2)); break; } default: - DFG_CRASH(m_graph, m_node, "Bad use kind"); + RELEASE_ASSERT_NOT_REACHED(); break; } } - void compileArithMod() - { - switch (m_node->binaryUseKind()) { - case Int32Use: { - LValue numerator = lowInt32(m_node->child1()); - LValue denominator = lowInt32(m_node->child2()); - - LBasicBlock unsafeDenominator = FTL_NEW_BLOCK(m_out, ("ArithMod unsafe denominator")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMod continuation")); - LBasicBlock done = FTL_NEW_BLOCK(m_out, ("ArithMod done")); - - Vector<ValueFromBlock, 3> results; - - LValue adjustedDenominator = m_out.add(denominator, m_out.int32One); - - m_out.branch( - m_out.above(adjustedDenominator, m_out.int32One), - usually(continuation), rarely(unsafeDenominator)); - - LBasicBlock lastNext = m_out.appendTo(unsafeDenominator, continuation); - - LValue neg2ToThe31 = m_out.constInt32(-2147483647-1); - - // FIXME: -2^31 / -1 will actually yield negative zero, so we could have a - // separate case for that. But it probably doesn't matter so much. - if (shouldCheckOverflow(m_node->arithMode())) { - LValue cond = m_out.bitOr(m_out.isZero32(denominator), m_out.equal(numerator, neg2ToThe31)); - speculate(Overflow, noValue(), 0, cond); - m_out.jump(continuation); - } else { - // This is the case where we convert the result to an int after we're done. So, - // if the denominator is zero, then the result should be result should be zero. - // If the denominator is not zero (i.e. it's -1 because we're guarded by the - // check above) and the numerator is -2^31 then the result should be -2^31. - - LBasicBlock modByZero = FTL_NEW_BLOCK(m_out, ("ArithMod modulo by zero")); - LBasicBlock notModByZero = FTL_NEW_BLOCK(m_out, ("ArithMod not modulo by zero")); - LBasicBlock neg2ToThe31ByNeg1 = FTL_NEW_BLOCK(m_out, ("ArithMod -2^31/-1")); - - m_out.branch( - m_out.isZero32(denominator), rarely(modByZero), usually(notModByZero)); - - m_out.appendTo(modByZero, notModByZero); - results.append(m_out.anchor(m_out.int32Zero)); - m_out.jump(done); - - m_out.appendTo(notModByZero, neg2ToThe31ByNeg1); - m_out.branch( - m_out.equal(numerator, neg2ToThe31), - rarely(neg2ToThe31ByNeg1), usually(continuation)); - - m_out.appendTo(neg2ToThe31ByNeg1, continuation); - results.append(m_out.anchor(m_out.int32Zero)); - m_out.jump(done); - } - - m_out.appendTo(continuation, done); - - LValue remainder = m_out.rem(numerator, denominator); - - if (shouldCheckNegativeZero(m_node->arithMode())) { - LBasicBlock negativeNumerator = FTL_NEW_BLOCK(m_out, ("ArithMod negative numerator")); - LBasicBlock numeratorContinuation = FTL_NEW_BLOCK(m_out, ("ArithMod numerator continuation")); - - m_out.branch( - m_out.lessThan(numerator, m_out.int32Zero), - unsure(negativeNumerator), unsure(numeratorContinuation)); - - LBasicBlock innerLastNext = m_out.appendTo(negativeNumerator, numeratorContinuation); - - speculate(NegativeZero, noValue(), 0, m_out.isZero32(remainder)); - - m_out.jump(numeratorContinuation); - - m_out.appendTo(numeratorContinuation, innerLastNext); - } - - results.append(m_out.anchor(remainder)); - m_out.jump(done); - - m_out.appendTo(done, lastNext); - - setInt32(m_out.phi(m_out.int32, results)); - break; - } - - case DoubleRepUse: { - setDouble( - m_out.doubleRem(lowDouble(m_node->child1()), lowDouble(m_node->child2()))); - break; - } - - default: - DFG_CRASH(m_graph, m_node, "Bad use kind"); - break; - } - } - void compileArithMinOrMax() { switch (m_node->binaryUseKind()) { @@ -1721,7 +1054,7 @@ private: break; } - case DoubleRepUse: { + case NumberUse: { LValue left = lowDouble(m_node->child1()); LValue right = lowDouble(m_node->child2()); @@ -1735,14 +1068,14 @@ private: m_node->op() == ArithMin ? m_out.doubleLessThan(left, right) : m_out.doubleGreaterThan(left, right), - unsure(continuation), unsure(notLessThan)); + continuation, notLessThan); LBasicBlock lastNext = m_out.appendTo(notLessThan, continuation); results.append(m_out.anchor(m_out.select( m_node->op() == ArithMin ? m_out.doubleGreaterThanOrEqual(left, right) : m_out.doubleLessThanOrEqual(left, right), - right, m_out.constDouble(PNaN)))); + right, m_out.constDouble(0.0 / 0.0)))); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); @@ -1751,7 +1084,7 @@ private: } default: - DFG_CRASH(m_graph, m_node, "Bad use kind"); + RELEASE_ASSERT_NOT_REACHED(); break; } } @@ -1771,124 +1104,16 @@ private: break; } - case DoubleRepUse: { + case NumberUse: { setDouble(m_out.doubleAbs(lowDouble(m_node->child1()))); break; } default: - DFG_CRASH(m_graph, m_node, "Bad use kind"); + RELEASE_ASSERT_NOT_REACHED(); break; } } - - void compileArithSin() { setDouble(m_out.doubleSin(lowDouble(m_node->child1()))); } - - void compileArithCos() { setDouble(m_out.doubleCos(lowDouble(m_node->child1()))); } - - void compileArithPow() - { - // FIXME: investigate llvm.powi to better understand its performance characteristics. - // It might be better to have the inline loop in DFG too. - if (m_node->child2().useKind() == Int32Use) - setDouble(m_out.doublePowi(lowDouble(m_node->child1()), lowInt32(m_node->child2()))); - else { - LValue base = lowDouble(m_node->child1()); - LValue exponent = lowDouble(m_node->child2()); - - LBasicBlock integerExponentIsSmallBlock = FTL_NEW_BLOCK(m_out, ("ArithPow test integer exponent is small.")); - LBasicBlock integerExponentPowBlock = FTL_NEW_BLOCK(m_out, ("ArithPow pow(double, (int)double).")); - LBasicBlock doubleExponentPowBlockEntry = FTL_NEW_BLOCK(m_out, ("ArithPow pow(double, double).")); - LBasicBlock nanExceptionExponentIsInfinity = FTL_NEW_BLOCK(m_out, ("ArithPow NaN Exception, check exponent is infinity.")); - LBasicBlock nanExceptionBaseIsOne = FTL_NEW_BLOCK(m_out, ("ArithPow NaN Exception, check base is one.")); - LBasicBlock powBlock = FTL_NEW_BLOCK(m_out, ("ArithPow regular pow")); - LBasicBlock nanExceptionResultIsNaN = FTL_NEW_BLOCK(m_out, ("ArithPow NaN Exception, result is NaN.")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithPow continuation")); - - LValue integerExponent = m_out.fpToInt32(exponent); - LValue integerExponentConvertedToDouble = m_out.intToDouble(integerExponent); - LValue exponentIsInteger = m_out.doubleEqual(exponent, integerExponentConvertedToDouble); - m_out.branch(exponentIsInteger, unsure(integerExponentIsSmallBlock), unsure(doubleExponentPowBlockEntry)); - - LBasicBlock lastNext = m_out.appendTo(integerExponentIsSmallBlock, integerExponentPowBlock); - LValue integerExponentBelow1000 = m_out.below(integerExponent, m_out.constInt32(1000)); - m_out.branch(integerExponentBelow1000, usually(integerExponentPowBlock), rarely(doubleExponentPowBlockEntry)); - - m_out.appendTo(integerExponentPowBlock, doubleExponentPowBlockEntry); - ValueFromBlock powDoubleIntResult = m_out.anchor(m_out.doublePowi(base, integerExponent)); - m_out.jump(continuation); - - // If y is NaN, the result is NaN. - m_out.appendTo(doubleExponentPowBlockEntry, nanExceptionExponentIsInfinity); - LValue exponentIsNaN; - if (provenType(m_node->child2()) & SpecDoubleNaN) - exponentIsNaN = m_out.doubleNotEqualOrUnordered(exponent, exponent); - else - exponentIsNaN = m_out.booleanFalse; - m_out.branch(exponentIsNaN, rarely(nanExceptionResultIsNaN), usually(nanExceptionExponentIsInfinity)); - - // If abs(x) is 1 and y is +infinity, the result is NaN. - // If abs(x) is 1 and y is -infinity, the result is NaN. - m_out.appendTo(nanExceptionExponentIsInfinity, nanExceptionBaseIsOne); - LValue absoluteExponent = m_out.doubleAbs(exponent); - LValue absoluteExponentIsInfinity = m_out.doubleEqual(absoluteExponent, m_out.constDouble(std::numeric_limits<double>::infinity())); - m_out.branch(absoluteExponentIsInfinity, rarely(nanExceptionBaseIsOne), usually(powBlock)); - - m_out.appendTo(nanExceptionBaseIsOne, powBlock); - LValue absoluteBase = m_out.doubleAbs(base); - LValue absoluteBaseIsOne = m_out.doubleEqual(absoluteBase, m_out.constDouble(1)); - m_out.branch(absoluteBaseIsOne, unsure(nanExceptionResultIsNaN), unsure(powBlock)); - - m_out.appendTo(powBlock, nanExceptionResultIsNaN); - ValueFromBlock powResult = m_out.anchor(m_out.doublePow(base, exponent)); - m_out.jump(continuation); - - m_out.appendTo(nanExceptionResultIsNaN, continuation); - ValueFromBlock pureNan = m_out.anchor(m_out.constDouble(PNaN)); - m_out.jump(continuation); - - m_out.appendTo(continuation, lastNext); - setDouble(m_out.phi(m_out.doubleType, powDoubleIntResult, powResult, pureNan)); - } - } - - void compileArithRound() - { - LBasicBlock realPartIsMoreThanHalf = FTL_NEW_BLOCK(m_out, ("ArithRound should round down")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithRound continuation")); - - LValue value = lowDouble(m_node->child1()); - LValue integerValue = m_out.ceil64(value); - ValueFromBlock integerValueResult = m_out.anchor(integerValue); - - LValue realPart = m_out.doubleSub(integerValue, value); - - m_out.branch(m_out.doubleGreaterThanOrUnordered(realPart, m_out.constDouble(0.5)), unsure(realPartIsMoreThanHalf), unsure(continuation)); - - LBasicBlock lastNext = m_out.appendTo(realPartIsMoreThanHalf, continuation); - LValue integerValueRoundedDown = m_out.doubleSub(integerValue, m_out.constDouble(1)); - ValueFromBlock integerValueRoundedDownResult = m_out.anchor(integerValueRoundedDown); - m_out.jump(continuation); - m_out.appendTo(continuation, lastNext); - - LValue result = m_out.phi(m_out.doubleType, integerValueResult, integerValueRoundedDownResult); - - if (producesInteger(m_node->arithRoundingMode())) { - LValue integerValue = convertDoubleToInt32(result, shouldCheckNegativeZero(m_node->arithRoundingMode())); - setInt32(integerValue); - } else - setDouble(result); - } - - void compileArithSqrt() { setDouble(m_out.doubleSqrt(lowDouble(m_node->child1()))); } - - void compileArithLog() { setDouble(m_out.doubleLog(lowDouble(m_node->child1()))); } - - void compileArithFRound() - { - LValue floatValue = m_out.fpCast(lowDouble(m_node->child1()), m_out.floatType); - setDouble(m_out.fpCast(floatValue, m_out.doubleType)); - } void compileArithNegate() { @@ -1896,26 +1121,24 @@ private: case Int32Use: { LValue value = lowInt32(m_node->child1()); - LValue result; - if (!shouldCheckOverflow(m_node->arithMode())) - result = m_out.neg(value); - else if (!shouldCheckNegativeZero(m_node->arithMode())) { - // We don't have a negate-with-overflow intrinsic. Hopefully this - // does the trick, though. - LValue overflowResult = m_out.subWithOverflow32(m_out.int32Zero, value); - speculate(Overflow, noValue(), 0, m_out.extractValue(overflowResult, 1)); - result = m_out.extractValue(overflowResult, 0); - } else { - speculate(Overflow, noValue(), 0, m_out.testIsZero32(value, m_out.constInt32(0x7fffffff))); - result = m_out.neg(value); + LValue result = m_out.neg(value); + if (shouldCheckOverflow(m_node->arithMode())) { + if (!shouldCheckNegativeZero(m_node->arithMode())) { + // We don't have a negate-with-overflow intrinsic. Hopefully this + // does the trick, though. + LValue overflowResult = m_out.subWithOverflow32(m_out.int32Zero, value); + speculate(Overflow, noValue(), 0, m_out.extractValue(overflowResult, 1)); + } else + speculate(Overflow, noValue(), 0, m_out.testIsZero32(value, m_out.constInt32(0x7fffffff))); + } setInt32(result); break; } - case Int52RepUse: { - if (!abstractValue(m_node->child1()).couldBeType(SpecInt52)) { + case MachineIntUse: { + if (!m_state.forNode(m_node->child1()).couldBeType(SpecInt52)) { Int52Kind kind; LValue value = lowWhicheverInt52(m_node->child1(), kind); LValue result = m_out.neg(value); @@ -1928,19 +1151,19 @@ private: LValue value = lowInt52(m_node->child1()); LValue overflowResult = m_out.subWithOverflow64(m_out.int64Zero, value); speculate(Int52Overflow, noValue(), 0, m_out.extractValue(overflowResult, 1)); - LValue result = m_out.extractValue(overflowResult, 0); + LValue result = m_out.neg(value); speculate(NegativeZero, noValue(), 0, m_out.isZero64(result)); setInt52(result); break; } - case DoubleRepUse: { + case NumberUse: { setDouble(m_out.doubleNeg(lowDouble(m_node->child1()))); break; } default: - DFG_CRASH(m_graph, m_node, "Bad use kind"); + RELEASE_ASSERT_NOT_REACHED(); break; } } @@ -1994,63 +1217,62 @@ private: setInt32(value); } + void compileInt32ToDouble() + { + setDouble(lowDouble(m_node->child1())); + } + void compileCheckStructure() { LValue cell = lowCell(m_node->child1()); ExitKind exitKind; - if (m_node->child1()->hasConstant()) - exitKind = BadConstantCache; + if (m_node->child1()->op() == WeakJSConstant) + exitKind = BadWeakConstantCache; else exitKind = BadCache; - LValue structureID = m_out.load32(cell, m_heaps.JSCell_structureID); + LValue structure = m_out.loadPtr(cell, m_heaps.JSCell_structure); - checkStructure( - structureID, jsValueValue(cell), exitKind, m_node->structureSet(), - [this] (Structure* structure) { - return weakStructureID(structure); - }); - } - - void compileCheckCell() - { - LValue cell = lowCell(m_node->child1()); + if (m_node->structureSet().size() == 1) { + speculate( + exitKind, jsValueValue(cell), 0, + m_out.notEqual(structure, weakPointer(m_node->structureSet()[0]))); + return; + } + + LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("CheckStructure continuation")); + + LBasicBlock lastNext = m_out.insertNewBlocksBefore(continuation); + for (unsigned i = 0; i < m_node->structureSet().size() - 1; ++i) { + LBasicBlock nextStructure = FTL_NEW_BLOCK(m_out, ("CheckStructure nextStructure")); + m_out.branch( + m_out.equal(structure, weakPointer(m_node->structureSet()[i])), + continuation, nextStructure); + m_out.appendTo(nextStructure); + } speculate( - BadCell, jsValueValue(cell), m_node->child1().node(), - m_out.notEqual(cell, weakPointer(m_node->cellOperand()->cell()))); + exitKind, jsValueValue(cell), 0, + m_out.notEqual(structure, weakPointer(m_node->structureSet().last()))); + + m_out.jump(continuation); + m_out.appendTo(continuation, lastNext); } - void compileCheckBadCell() + void compileStructureTransitionWatchpoint() { - terminate(BadCell); + addWeakReference(m_node->structure()); + speculateCell(m_node->child1()); } - - void compileCheckNotEmpty() - { - speculate(TDZFailure, noValue(), nullptr, m_out.isZero64(lowJSValue(m_node->child1()))); - } - - void compileCheckIdent() - { - UniquedStringImpl* uid = m_node->uidOperand(); - if (uid->isSymbol()) { - LValue symbol = lowSymbol(m_node->child1()); - LValue stringImpl = m_out.loadPtr(symbol, m_heaps.Symbol_privateName); - speculate(BadIdent, noValue(), nullptr, m_out.notEqual(stringImpl, m_out.constIntPtr(uid))); - } else { - LValue string = lowStringIdent(m_node->child1()); - LValue stringImpl = m_out.loadPtr(string, m_heaps.JSString_value); - speculate(BadIdent, noValue(), nullptr, m_out.notEqual(stringImpl, m_out.constIntPtr(uid))); - } - } - - void compileGetExecutable() + + void compileCheckFunction() { LValue cell = lowCell(m_node->child1()); - speculateFunction(m_node->child1(), cell); - setJSValue(m_out.loadPtr(cell, m_heaps.JSFunction_executable)); + + speculate( + BadFunction, jsValueValue(cell), m_node->child1().node(), + m_out.notEqual(cell, weakPointer(m_node->function()))); } void compileArrayifyToStructure() @@ -2061,11 +1283,11 @@ private: LBasicBlock unexpectedStructure = FTL_NEW_BLOCK(m_out, ("ArrayifyToStructure unexpected structure")); LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArrayifyToStructure continuation")); - LValue structureID = m_out.load32(cell, m_heaps.JSCell_structureID); + LValue structure = m_out.loadPtr(cell, m_heaps.JSCell_structure); m_out.branch( - m_out.notEqual(structureID, weakStructureID(m_node->structure())), - rarely(unexpectedStructure), usually(continuation)); + m_out.notEqual(structure, weakPointer(m_node->structure())), + unexpectedStructure, continuation); LBasicBlock lastNext = m_out.appendTo(unexpectedStructure, continuation); @@ -2091,21 +1313,24 @@ private: vmCall(m_out.operation(operationEnsureDouble), m_callFrame, cell); break; case Array::Contiguous: - vmCall(m_out.operation(operationEnsureContiguous), m_callFrame, cell); + if (m_node->arrayMode().conversion() == Array::RageConvert) + vmCall(m_out.operation(operationRageEnsureContiguous), m_callFrame, cell); + else + vmCall(m_out.operation(operationEnsureContiguous), m_callFrame, cell); break; case Array::ArrayStorage: case Array::SlowPutArrayStorage: vmCall(m_out.operation(operationEnsureArrayStorage), m_callFrame, cell); break; default: - DFG_CRASH(m_graph, m_node, "Bad array type"); + RELEASE_ASSERT_NOT_REACHED(); break; } - structureID = m_out.load32(cell, m_heaps.JSCell_structureID); + structure = m_out.loadPtr(cell, m_heaps.JSCell_structure); speculate( BadIndexingType, jsValueValue(cell), 0, - m_out.notEqual(structureID, weakStructureID(m_node->structure()))); + m_out.notEqual(structure, weakPointer(m_node->structure()))); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); @@ -2114,63 +1339,43 @@ private: void compilePutStructure() { m_ftlState.jitCode->common.notifyCompilingStructureTransition(m_graph.m_plan, codeBlock(), m_node); - - Structure* oldStructure = m_node->transition()->previous; - Structure* newStructure = m_node->transition()->next; - ASSERT_UNUSED(oldStructure, oldStructure->indexingType() == newStructure->indexingType()); - ASSERT(oldStructure->typeInfo().inlineTypeFlags() == newStructure->typeInfo().inlineTypeFlags()); - ASSERT(oldStructure->typeInfo().type() == newStructure->typeInfo().type()); - - LValue cell = lowCell(m_node->child1()); - m_out.store32( - weakStructureID(newStructure), - cell, m_heaps.JSCell_structureID); + + m_out.store64( + m_out.constIntPtr(m_node->structureTransitionData().newStructure), + lowCell(m_node->child1()), m_heaps.JSCell_structure); + } + + void compilePhantomPutStructure() + { + m_ftlState.jitCode->common.notifyCompilingStructureTransition(m_graph.m_plan, codeBlock(), m_node); } void compileGetById() { - // Pretty much the only reason why we don't also support GetByIdFlush is because: - // https://bugs.webkit.org/show_bug.cgi?id=125711 + // UntypedUse is a bit harder to reason about and I'm not sure how best to do it, yet. + // Basically we need to emit a cell branch that takes you to the slow path, but the slow + // path is generated by the IC generator so we can't jump to it from here. And the IC + // generator currently doesn't know how to emit such a branch. So, for now, we just + // restrict this to CellUse. + ASSERT(m_node->child1().useKind() == CellUse); + + LValue base = lowCell(m_node->child1()); + StringImpl* uid = m_graph.identifiers()[m_node->identifierNumber()]; + + // Arguments: id, bytes, target, numArgs, args... + unsigned stackmapID = m_stackmapIDs++; - switch (m_node->child1().useKind()) { - case CellUse: { - setJSValue(getById(lowCell(m_node->child1()))); - return; - } - - case UntypedUse: { - // This is pretty weird, since we duplicate the slow path both here and in the - // code generated by the IC. We should investigate making this less bad. - // https://bugs.webkit.org/show_bug.cgi?id=127830 - LValue value = lowJSValue(m_node->child1()); - - LBasicBlock cellCase = FTL_NEW_BLOCK(m_out, ("GetById untyped cell case")); - LBasicBlock notCellCase = FTL_NEW_BLOCK(m_out, ("GetById untyped not cell case")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("GetById untyped continuation")); - - m_out.branch( - isCell(value, provenType(m_node->child1())), unsure(cellCase), unsure(notCellCase)); - - LBasicBlock lastNext = m_out.appendTo(cellCase, notCellCase); - ValueFromBlock cellResult = m_out.anchor(getById(value)); - m_out.jump(continuation); - - m_out.appendTo(notCellCase, continuation); - ValueFromBlock notCellResult = m_out.anchor(vmCall( - m_out.operation(operationGetByIdGeneric), - m_callFrame, value, - m_out.constIntPtr(m_graph.identifiers()[m_node->identifierNumber()]))); - m_out.jump(continuation); - - m_out.appendTo(continuation, lastNext); - setJSValue(m_out.phi(m_out.int64, cellResult, notCellResult)); - return; - } - - default: - DFG_CRASH(m_graph, m_node, "Bad use kind"); - return; - } + if (Options::verboseCompilation()) + dataLog(" Emitting GetById patchpoint with stackmap #", stackmapID, "\n"); + + LValue call = m_out.call( + m_out.patchpointInt64Intrinsic(), + m_out.constInt32(stackmapID), m_out.constInt32(sizeOfGetById()), + constNull(m_out.ref8), m_out.constInt32(2), m_callFrame, base); + setInstructionCallingConvention(call, LLVMAnyRegCallConv); + setJSValue(call); + + m_ftlState.getByIds.append(GetByIdDescriptor(stackmapID, m_node->codeOrigin, uid)); } void compilePutById() @@ -2180,23 +1385,23 @@ private: LValue base = lowCell(m_node->child1()); LValue value = lowJSValue(m_node->child2()); - auto uid = m_graph.identifiers()[m_node->identifierNumber()]; + StringImpl* uid = m_graph.identifiers()[m_node->identifierNumber()]; // Arguments: id, bytes, target, numArgs, args... unsigned stackmapID = m_stackmapIDs++; - if (verboseCompilationEnabled()) + if (Options::verboseCompilation()) dataLog(" Emitting PutById patchpoint with stackmap #", stackmapID, "\n"); LValue call = m_out.call( m_out.patchpointVoidIntrinsic(), - m_out.constInt64(stackmapID), m_out.constInt32(sizeOfPutById()), - constNull(m_out.ref8), m_out.constInt32(2), base, value); + m_out.constInt32(stackmapID), m_out.constInt32(sizeOfPutById()), + constNull(m_out.ref8), m_out.constInt32(3), m_callFrame, base, value); setInstructionCallingConvention(call, LLVMAnyRegCallConv); m_ftlState.putByIds.append(PutByIdDescriptor( - stackmapID, m_node->origin.semantic, uid, - m_graph.executableFor(m_node->origin.semantic)->ecmaMode(), + stackmapID, m_node->codeOrigin, uid, + m_graph.executableFor(m_node->codeOrigin)->ecmaMode(), m_node->op() == PutByIdDirect ? Direct : NotDirect)); } @@ -2221,8 +1426,7 @@ private: ValueFromBlock fastResult = m_out.anchor( m_out.loadPtr(cell, m_heaps.JSString_value)); - m_out.branch( - m_out.notNull(fastResult.value()), usually(continuation), rarely(slowPath)); + m_out.branch(m_out.notNull(fastResult.value()), continuation, slowPath); LBasicBlock lastNext = m_out.appendTo(slowPath, continuation); @@ -2245,50 +1449,13 @@ private: Edge edge = m_node->child1(); LValue cell = lowCell(edge); - if (m_node->arrayMode().alreadyChecked(m_graph, m_node, abstractValue(edge))) + if (m_node->arrayMode().alreadyChecked(m_graph, m_node, m_state.forNode(edge))) return; speculate( BadIndexingType, jsValueValue(cell), 0, m_out.bitNot(isArrayType(cell, m_node->arrayMode()))); } - - void compileGetTypedArrayByteOffset() - { - LValue basePtr = lowCell(m_node->child1()); - - LBasicBlock simpleCase = FTL_NEW_BLOCK(m_out, ("wasteless typed array")); - LBasicBlock wastefulCase = FTL_NEW_BLOCK(m_out, ("wasteful typed array")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("continuation branch")); - - LValue mode = m_out.load32(basePtr, m_heaps.JSArrayBufferView_mode); - m_out.branch( - m_out.notEqual(mode, m_out.constInt32(WastefulTypedArray)), - unsure(simpleCase), unsure(wastefulCase)); - - // begin simple case - LBasicBlock lastNext = m_out.appendTo(simpleCase, wastefulCase); - - ValueFromBlock simpleOut = m_out.anchor(m_out.constIntPtr(0)); - - m_out.jump(continuation); - - // begin wasteful case - m_out.appendTo(wastefulCase, continuation); - - LValue vectorPtr = m_out.loadPtr(basePtr, m_heaps.JSArrayBufferView_vector); - LValue butterflyPtr = m_out.loadPtr(basePtr, m_heaps.JSObject_butterfly); - LValue arrayBufferPtr = m_out.loadPtr(butterflyPtr, m_heaps.Butterfly_arrayBuffer); - LValue dataPtr = m_out.loadPtr(arrayBufferPtr, m_heaps.ArrayBuffer_data); - - ValueFromBlock wastefulOut = m_out.anchor(m_out.sub(vectorPtr, dataPtr)); - - m_out.jump(continuation); - m_out.appendTo(continuation, lastNext); - - // output - setInt32(m_out.castToInt32(m_out.phi(m_out.intPtr, simpleOut, wastefulOut))); - } void compileGetArrayLength() { @@ -2296,42 +1463,24 @@ private: case Array::Int32: case Array::Double: case Array::Contiguous: { - setInt32(m_out.load32NonNegative(lowStorage(m_node->child2()), m_heaps.Butterfly_publicLength)); + setInt32(m_out.load32(lowStorage(m_node->child2()), m_heaps.Butterfly_publicLength)); return; } case Array::String: { LValue string = lowCell(m_node->child1()); - setInt32(m_out.load32NonNegative(string, m_heaps.JSString_length)); - return; - } - - case Array::DirectArguments: { - LValue arguments = lowCell(m_node->child1()); - speculate( - ExoticObjectMode, noValue(), nullptr, - m_out.notNull(m_out.loadPtr(arguments, m_heaps.DirectArguments_overrides))); - setInt32(m_out.load32NonNegative(arguments, m_heaps.DirectArguments_length)); - return; - } - - case Array::ScopedArguments: { - LValue arguments = lowCell(m_node->child1()); - speculate( - ExoticObjectMode, noValue(), nullptr, - m_out.notZero8(m_out.load8(arguments, m_heaps.ScopedArguments_overrodeThings))); - setInt32(m_out.load32NonNegative(arguments, m_heaps.ScopedArguments_totalLength)); + setInt32(m_out.load32(string, m_heaps.JSString_length)); return; } default: if (isTypedView(m_node->arrayMode().typedArrayType())) { setInt32( - m_out.load32NonNegative(lowCell(m_node->child1()), m_heaps.JSArrayBufferView_length)); + m_out.load32(lowCell(m_node->child1()), m_heaps.JSArrayBufferView_length)); return; } - DFG_CRASH(m_graph, m_node, "Bad array type"); + RELEASE_ASSERT_NOT_REACHED(); return; } } @@ -2356,14 +1505,7 @@ private: if (m_node->arrayMode().isInBounds()) { LValue result = m_out.load64(baseIndex(heap, storage, index, m_node->child2())); - LValue isHole = m_out.isZero64(result); - if (m_node->arrayMode().isSaneChain()) { - DFG_ASSERT( - m_graph, m_node, m_node->arrayMode().type() == Array::Contiguous); - result = m_out.select( - isHole, m_out.constInt64(JSValue::encode(jsUndefined())), result); - } else - speculate(LoadFromHole, noValue(), 0, isHole); + speculate(LoadFromHole, noValue(), 0, m_out.isZero64(result)); setJSValue(result); return; } @@ -2376,15 +1518,14 @@ private: m_out.branch( m_out.aboveOrEqual( - index, m_out.load32NonNegative(storage, m_heaps.Butterfly_publicLength)), - rarely(slowCase), usually(fastCase)); + index, m_out.load32(storage, m_heaps.Butterfly_publicLength)), + slowCase, fastCase); LBasicBlock lastNext = m_out.appendTo(fastCase, slowCase); ValueFromBlock fastResult = m_out.anchor( m_out.load64(baseIndex(heap, storage, index, m_node->child2()))); - m_out.branch( - m_out.isZero64(fastResult.value()), rarely(slowCase), usually(continuation)); + m_out.branch(m_out.isZero64(fastResult.value()), slowCase, continuation); m_out.appendTo(slowCase, continuation); ValueFromBlock slowResult = m_out.anchor( @@ -2424,15 +1565,14 @@ private: m_out.branch( m_out.aboveOrEqual( - index, m_out.load32NonNegative(storage, m_heaps.Butterfly_publicLength)), - rarely(slowCase), usually(inBounds)); + index, m_out.load32(storage, m_heaps.Butterfly_publicLength)), + slowCase, inBounds); LBasicBlock lastNext = m_out.appendTo(inBounds, boxPath); LValue doubleValue = m_out.loadDouble( baseIndex(heap, storage, index, m_node->child2())); m_out.branch( - m_out.doubleNotEqualOrUnordered(doubleValue, doubleValue), - rarely(slowCase), usually(boxPath)); + m_out.doubleNotEqualOrUnordered(doubleValue, doubleValue), slowCase, boxPath); m_out.appendTo(boxPath, slowCase); ValueFromBlock fastResult = m_out.anchor(boxDouble(doubleValue)); @@ -2447,86 +1587,6 @@ private: setJSValue(m_out.phi(m_out.int64, fastResult, slowResult)); return; } - - case Array::Undecided: { - LValue index = lowInt32(m_node->child2()); - - speculate(OutOfBounds, noValue(), m_node, m_out.lessThan(index, m_out.int32Zero)); - setJSValue(m_out.constInt64(ValueUndefined)); - return; - } - - case Array::DirectArguments: { - LValue base = lowCell(m_node->child1()); - LValue index = lowInt32(m_node->child2()); - - speculate( - ExoticObjectMode, noValue(), nullptr, - m_out.notNull(m_out.loadPtr(base, m_heaps.DirectArguments_overrides))); - speculate( - ExoticObjectMode, noValue(), nullptr, - m_out.aboveOrEqual( - index, - m_out.load32NonNegative(base, m_heaps.DirectArguments_length))); - - TypedPointer address = m_out.baseIndex( - m_heaps.DirectArguments_storage, base, m_out.zeroExtPtr(index)); - setJSValue(m_out.load64(address)); - return; - } - - case Array::ScopedArguments: { - LValue base = lowCell(m_node->child1()); - LValue index = lowInt32(m_node->child2()); - - speculate( - ExoticObjectMode, noValue(), nullptr, - m_out.aboveOrEqual( - index, - m_out.load32NonNegative(base, m_heaps.ScopedArguments_totalLength))); - - LValue table = m_out.loadPtr(base, m_heaps.ScopedArguments_table); - LValue namedLength = m_out.load32(table, m_heaps.ScopedArgumentsTable_length); - - LBasicBlock namedCase = FTL_NEW_BLOCK(m_out, ("GetByVal ScopedArguments named case")); - LBasicBlock overflowCase = FTL_NEW_BLOCK(m_out, ("GetByVal ScopedArguments overflow case")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("GetByVal ScopedArguments continuation")); - - m_out.branch( - m_out.aboveOrEqual(index, namedLength), unsure(overflowCase), unsure(namedCase)); - - LBasicBlock lastNext = m_out.appendTo(namedCase, overflowCase); - - LValue scope = m_out.loadPtr(base, m_heaps.ScopedArguments_scope); - LValue arguments = m_out.loadPtr(table, m_heaps.ScopedArgumentsTable_arguments); - - TypedPointer address = m_out.baseIndex( - m_heaps.scopedArgumentsTableArguments, arguments, m_out.zeroExtPtr(index)); - LValue scopeOffset = m_out.load32(address); - - speculate( - ExoticObjectMode, noValue(), nullptr, - m_out.equal(scopeOffset, m_out.constInt32(ScopeOffset::invalidOffset))); - - address = m_out.baseIndex( - m_heaps.JSEnvironmentRecord_variables, scope, m_out.zeroExtPtr(scopeOffset)); - ValueFromBlock namedResult = m_out.anchor(m_out.load64(address)); - m_out.jump(continuation); - - m_out.appendTo(overflowCase, continuation); - - address = m_out.baseIndex( - m_heaps.ScopedArguments_overflowStorage, base, - m_out.zeroExtPtr(m_out.sub(index, namedLength))); - LValue overflowValue = m_out.load64(address); - speculate(ExoticObjectMode, noValue(), nullptr, m_out.isZero64(overflowValue)); - ValueFromBlock overflowResult = m_out.anchor(overflowValue); - m_out.jump(continuation); - - m_out.appendTo(continuation, lastNext); - setJSValue(m_out.phi(m_out.int64, namedResult, overflowResult)); - return; - } case Array::Generic: { setJSValue(vmCall( @@ -2552,7 +1612,7 @@ private: m_out.add( storage, m_out.shl( - m_out.zeroExtPtr(index), + m_out.zeroExt(index, m_out.intPtr), m_out.constIntPtr(logElementSize(type))))); if (isInt(type)) { @@ -2568,7 +1628,7 @@ private: result = m_out.load32(pointer); break; default: - DFG_CRASH(m_graph, m_node, "Bad element size"); + RELEASE_ASSERT_NOT_REACHED(); } if (elementSize(type) < 4) { @@ -2592,11 +1652,6 @@ private: return; } - if (m_node->shouldSpeculateMachineInt()) { - setStrictInt52(m_out.zeroExt(result, m_out.int64)); - return; - } - setDouble(m_out.unsignedToFP(result, m_out.doubleType)); return; } @@ -2612,76 +1667,37 @@ private: result = m_out.loadDouble(pointer); break; default: - DFG_CRASH(m_graph, m_node, "Bad typed array type"); + RELEASE_ASSERT_NOT_REACHED(); } + result = m_out.select( + m_out.doubleEqual(result, result), result, m_out.constDouble(QNaN)); setDouble(result); return; } - DFG_CRASH(m_graph, m_node, "Bad array type"); + RELEASE_ASSERT_NOT_REACHED(); return; } } } - void compileGetMyArgumentByVal() - { - InlineCallFrame* inlineCallFrame = m_node->child1()->origin.semantic.inlineCallFrame; - - LValue index = lowInt32(m_node->child2()); - - LValue limit; - if (inlineCallFrame && !inlineCallFrame->isVarargs()) - limit = m_out.constInt32(inlineCallFrame->arguments.size() - 1); - else { - VirtualRegister argumentCountRegister; - if (!inlineCallFrame) - argumentCountRegister = VirtualRegister(JSStack::ArgumentCount); - else - argumentCountRegister = inlineCallFrame->argumentCountRegister; - limit = m_out.sub(m_out.load32(payloadFor(argumentCountRegister)), m_out.int32One); - } - - speculate(ExoticObjectMode, noValue(), 0, m_out.aboveOrEqual(index, limit)); - - TypedPointer base; - if (inlineCallFrame) { - if (inlineCallFrame->arguments.size() <= 1) { - // We should have already exited due to the bounds check, above. Just tell the - // compiler that anything dominated by this instruction is not reachable, so - // that we don't waste time generating such code. This will also plant some - // kind of crashing instruction so that if by some fluke the bounds check didn't - // work, we'll crash in an easy-to-see way. - didAlreadyTerminate(); - return; - } - base = addressFor(inlineCallFrame->arguments[1].virtualRegister()); - } else - base = addressFor(virtualRegisterForArgument(1)); - - LValue pointer = m_out.baseIndex( - base.value(), m_out.zeroExt(index, m_out.intPtr), ScaleEight); - setJSValue(m_out.load64(TypedPointer(m_heaps.variables.atAnyIndex(), pointer))); - } - void compilePutByVal() { Edge child1 = m_graph.varArgChild(m_node, 0); Edge child2 = m_graph.varArgChild(m_node, 1); Edge child3 = m_graph.varArgChild(m_node, 2); Edge child4 = m_graph.varArgChild(m_node, 3); - Edge child5 = m_graph.varArgChild(m_node, 4); switch (m_node->arrayMode().type()) { case Array::Generic: { V_JITOperation_EJJJ operation; if (m_node->op() == PutByValDirect) { - if (m_graph.isStrictModeFor(m_node->origin.semantic)) + if (m_graph.isStrictModeFor(m_node->codeOrigin)) operation = operationPutByValDirectStrict; else operation = operationPutByValDirectNonStrict; } else { - if (m_graph.isStrictModeFor(m_node->origin.semantic)) + if (m_graph.isStrictModeFor(m_node->codeOrigin)) operation = operationPutByValStrict; else operation = operationPutByValNonStrict; @@ -2719,7 +1735,8 @@ private: TypedPointer elementPointer = m_out.baseIndex( m_node->arrayMode().type() == Array::Int32 ? m_heaps.indexedInt32Properties : m_heaps.indexedContiguousProperties, - storage, m_out.zeroExtPtr(index), provenValue(child2)); + storage, m_out.zeroExt(index, m_out.intPtr), + m_state.forNode(child2).m_value); if (m_node->op() == PutByValAlias) { m_out.store64(value, elementPointer); @@ -2740,12 +1757,13 @@ private: LValue value = lowDouble(child3); FTL_TYPE_CHECK( - doubleValue(value), child3, SpecDoubleReal, + doubleValue(value), child3, SpecFullRealNumber, m_out.doubleNotEqualOrUnordered(value, value)); TypedPointer elementPointer = m_out.baseIndex( - m_heaps.indexedDoubleProperties, storage, m_out.zeroExtPtr(index), - provenValue(child2)); + m_heaps.indexedDoubleProperties, + storage, m_out.zeroExt(index, m_out.intPtr), + m_state.forNode(child2).m_value); if (m_node->op() == PutByValAlias) { m_out.storeDouble(value, elementPointer); @@ -2763,7 +1781,7 @@ private: } default: - DFG_CRASH(m_graph, m_node, "Bad array type"); + RELEASE_ASSERT_NOT_REACHED(); } m_out.jump(continuation); @@ -2783,13 +1801,10 @@ private: m_out.zeroExt(index, m_out.intPtr), m_out.constIntPtr(logElementSize(type))))); - LType refType; - LValue valueToStore; - if (isInt(type)) { LValue intValue; switch (child3.useKind()) { - case Int52RepUse: + case MachineIntUse: case Int32Use: { if (child3.useKind() == Int32Use) intValue = lowInt32(child3); @@ -2806,7 +1821,7 @@ private: intValues.append(m_out.anchor(m_out.int32Zero)); m_out.branch( m_out.lessThan(intValue, m_out.int32Zero), - unsure(continuation), unsure(atLeastZero)); + continuation, atLeastZero); LBasicBlock lastNext = m_out.appendTo(atLeastZero, continuation); @@ -2822,7 +1837,7 @@ private: break; } - case DoubleRepUse: { + case NumberUse: { LValue doubleValue = lowDouble(child3); if (isClamped(type)) { @@ -2836,13 +1851,13 @@ private: intValues.append(m_out.anchor(m_out.int32Zero)); m_out.branch( m_out.doubleLessThanOrUnordered(doubleValue, m_out.doubleZero), - unsure(continuation), unsure(atLeastZero)); + continuation, atLeastZero); LBasicBlock lastNext = m_out.appendTo(atLeastZero, withinRange); intValues.append(m_out.anchor(m_out.constInt32(255))); m_out.branch( m_out.doubleGreaterThan(doubleValue, m_out.constDouble(255)), - unsure(continuation), unsure(withinRange)); + continuation, withinRange); m_out.appendTo(withinRange, continuation); intValues.append(m_out.anchor(m_out.fpToInt32(doubleValue))); @@ -2856,408 +1871,73 @@ private: } default: - DFG_CRASH(m_graph, m_node, "Bad use kind"); + RELEASE_ASSERT_NOT_REACHED(); } switch (elementSize(type)) { case 1: - valueToStore = m_out.intCast(intValue, m_out.int8); - refType = m_out.ref8; + m_out.store8(m_out.intCast(intValue, m_out.int8), pointer); break; case 2: - valueToStore = m_out.intCast(intValue, m_out.int16); - refType = m_out.ref16; + m_out.store16(m_out.intCast(intValue, m_out.int16), pointer); break; case 4: - valueToStore = intValue; - refType = m_out.ref32; + m_out.store32(intValue, pointer); break; default: - DFG_CRASH(m_graph, m_node, "Bad element size"); + RELEASE_ASSERT_NOT_REACHED(); } - } else /* !isInt(type) */ { - LValue value = lowDouble(child3); - switch (type) { - case TypeFloat32: - valueToStore = m_out.fpCast(value, m_out.floatType); - refType = m_out.refFloat; - break; - case TypeFloat64: - valueToStore = value; - refType = m_out.refDouble; - break; - default: - DFG_CRASH(m_graph, m_node, "Bad typed array type"); - } - } - - if (m_node->arrayMode().isInBounds() || m_node->op() == PutByValAlias) - m_out.store(valueToStore, pointer, refType); - else { - LBasicBlock isInBounds = FTL_NEW_BLOCK(m_out, ("PutByVal typed array in bounds case")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("PutByVal typed array continuation")); - - m_out.branch( - m_out.aboveOrEqual(index, lowInt32(child5)), - unsure(continuation), unsure(isInBounds)); - LBasicBlock lastNext = m_out.appendTo(isInBounds, continuation); - m_out.store(valueToStore, pointer, refType); - m_out.jump(continuation); - - m_out.appendTo(continuation, lastNext); + return; } + ASSERT(isFloat(type)); + + LValue value = lowDouble(child3); + switch (type) { + case TypeFloat32: + m_out.storeFloat(m_out.fpCast(value, m_out.floatType), pointer); + break; + case TypeFloat64: + m_out.storeDouble(value, pointer); + break; + default: + RELEASE_ASSERT_NOT_REACHED(); + } return; } - DFG_CRASH(m_graph, m_node, "Bad array type"); + RELEASE_ASSERT_NOT_REACHED(); break; } } - void compileArrayPush() - { - LValue base = lowCell(m_node->child1()); - LValue storage = lowStorage(m_node->child3()); - - switch (m_node->arrayMode().type()) { - case Array::Int32: - case Array::Contiguous: - case Array::Double: { - LValue value; - LType refType; - - if (m_node->arrayMode().type() != Array::Double) { - value = lowJSValue(m_node->child2(), ManualOperandSpeculation); - if (m_node->arrayMode().type() == Array::Int32) { - FTL_TYPE_CHECK( - jsValueValue(value), m_node->child2(), SpecInt32, isNotInt32(value)); - } - refType = m_out.ref64; - } else { - value = lowDouble(m_node->child2()); - FTL_TYPE_CHECK( - doubleValue(value), m_node->child2(), SpecDoubleReal, - m_out.doubleNotEqualOrUnordered(value, value)); - refType = m_out.refDouble; - } - - IndexedAbstractHeap& heap = m_heaps.forArrayType(m_node->arrayMode().type()); - - LValue prevLength = m_out.load32(storage, m_heaps.Butterfly_publicLength); - - LBasicBlock fastPath = FTL_NEW_BLOCK(m_out, ("ArrayPush fast path")); - LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("ArrayPush slow path")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArrayPush continuation")); - - m_out.branch( - m_out.aboveOrEqual( - prevLength, m_out.load32(storage, m_heaps.Butterfly_vectorLength)), - rarely(slowPath), usually(fastPath)); - - LBasicBlock lastNext = m_out.appendTo(fastPath, slowPath); - m_out.store( - value, m_out.baseIndex(heap, storage, m_out.zeroExtPtr(prevLength)), refType); - LValue newLength = m_out.add(prevLength, m_out.int32One); - m_out.store32(newLength, storage, m_heaps.Butterfly_publicLength); - - ValueFromBlock fastResult = m_out.anchor(boxInt32(newLength)); - m_out.jump(continuation); - - m_out.appendTo(slowPath, continuation); - LValue operation; - if (m_node->arrayMode().type() != Array::Double) - operation = m_out.operation(operationArrayPush); - else - operation = m_out.operation(operationArrayPushDouble); - ValueFromBlock slowResult = m_out.anchor( - vmCall(operation, m_callFrame, value, base)); - m_out.jump(continuation); - - m_out.appendTo(continuation, lastNext); - setJSValue(m_out.phi(m_out.int64, fastResult, slowResult)); - return; - } - - default: - DFG_CRASH(m_graph, m_node, "Bad array type"); - return; - } - } - - void compileArrayPop() - { - LValue base = lowCell(m_node->child1()); - LValue storage = lowStorage(m_node->child2()); - - switch (m_node->arrayMode().type()) { - case Array::Int32: - case Array::Double: - case Array::Contiguous: { - IndexedAbstractHeap& heap = m_heaps.forArrayType(m_node->arrayMode().type()); - - LBasicBlock fastCase = FTL_NEW_BLOCK(m_out, ("ArrayPop fast case")); - LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("ArrayPop slow case")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArrayPop continuation")); - - LValue prevLength = m_out.load32(storage, m_heaps.Butterfly_publicLength); - - Vector<ValueFromBlock, 3> results; - results.append(m_out.anchor(m_out.constInt64(JSValue::encode(jsUndefined())))); - m_out.branch( - m_out.isZero32(prevLength), rarely(continuation), usually(fastCase)); - - LBasicBlock lastNext = m_out.appendTo(fastCase, slowCase); - LValue newLength = m_out.sub(prevLength, m_out.int32One); - m_out.store32(newLength, storage, m_heaps.Butterfly_publicLength); - TypedPointer pointer = m_out.baseIndex(heap, storage, m_out.zeroExtPtr(newLength)); - if (m_node->arrayMode().type() != Array::Double) { - LValue result = m_out.load64(pointer); - m_out.store64(m_out.int64Zero, pointer); - results.append(m_out.anchor(result)); - m_out.branch( - m_out.notZero64(result), usually(continuation), rarely(slowCase)); - } else { - LValue result = m_out.loadDouble(pointer); - m_out.store64(m_out.constInt64(bitwise_cast<int64_t>(PNaN)), pointer); - results.append(m_out.anchor(boxDouble(result))); - m_out.branch( - m_out.doubleEqual(result, result), - usually(continuation), rarely(slowCase)); - } - - m_out.appendTo(slowCase, continuation); - results.append(m_out.anchor(vmCall( - m_out.operation(operationArrayPopAndRecoverLength), m_callFrame, base))); - m_out.jump(continuation); - - m_out.appendTo(continuation, lastNext); - setJSValue(m_out.phi(m_out.int64, results)); - return; - } - - default: - DFG_CRASH(m_graph, m_node, "Bad array type"); - return; - } - } - - void compileCreateActivation() + void compileNewObject() { - LValue scope = lowCell(m_node->child1()); - SymbolTable* table = m_node->castOperand<SymbolTable*>(); - Structure* structure = m_graph.globalObjectFor(m_node->origin.semantic)->activationStructure(); - JSValue initializationValue = m_node->initializationValueForActivation(); - ASSERT(initializationValue.isUndefined() || initializationValue == jsTDZValue()); - if (table->singletonScope()->isStillValid()) { - LValue callResult = vmCall( - m_out.operation(operationCreateActivationDirect), m_callFrame, weakPointer(structure), - scope, weakPointer(table), m_out.constInt64(JSValue::encode(initializationValue))); - setJSValue(callResult); - return; - } + Structure* structure = m_node->structure(); + size_t allocationSize = JSFinalObject::allocationSize(structure->inlineCapacity()); + MarkedAllocator* allocator = &vm().heap.allocatorForObjectWithoutDestructor(allocationSize); - LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("CreateActivation slow path")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("CreateActivation continuation")); + LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("NewObject slow path")); + LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("NewObject continuation")); LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath); - LValue fastObject = allocateObject<JSLexicalEnvironment>( - JSLexicalEnvironment::allocationSize(table), structure, m_out.intPtrZero, slowPath); - - // We don't need memory barriers since we just fast-created the activation, so the - // activation must be young. - m_out.storePtr(scope, fastObject, m_heaps.JSScope_next); - m_out.storePtr(weakPointer(table), fastObject, m_heaps.JSSymbolTableObject_symbolTable); - - for (unsigned i = 0; i < table->scopeSize(); ++i) { - m_out.store64( - m_out.constInt64(JSValue::encode(initializationValue)), - fastObject, m_heaps.JSEnvironmentRecord_variables[i]); - } + ValueFromBlock fastResult = m_out.anchor(allocateObject( + m_out.constIntPtr(allocator), m_out.constIntPtr(structure), m_out.intPtrZero, slowPath)); - ValueFromBlock fastResult = m_out.anchor(fastObject); m_out.jump(continuation); m_out.appendTo(slowPath, continuation); - LValue callResult = vmCall( - m_out.operation(operationCreateActivationDirect), m_callFrame, weakPointer(structure), - scope, weakPointer(table), m_out.constInt64(JSValue::encode(initializationValue))); - ValueFromBlock slowResult = m_out.anchor(callResult); - m_out.jump(continuation); - m_out.appendTo(continuation, lastNext); - setJSValue(m_out.phi(m_out.intPtr, fastResult, slowResult)); - } - - void compileNewFunction() - { - LValue scope = lowCell(m_node->child1()); - FunctionExecutable* executable = m_node->castOperand<FunctionExecutable*>(); - if (executable->singletonFunction()->isStillValid()) { - LValue callResult = vmCall( - m_out.operation(operationNewFunction), m_callFrame, scope, weakPointer(executable)); - setJSValue(callResult); - return; - } - - Structure* structure = m_graph.globalObjectFor(m_node->origin.semantic)->functionStructure(); - - LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("NewFunction slow path")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("NewFunction continuation")); - - LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath); - - LValue fastObject = allocateObject<JSFunction>( - structure, m_out.intPtrZero, slowPath); - - // We don't need memory barriers since we just fast-created the function, so it - // must be young. - m_out.storePtr(scope, fastObject, m_heaps.JSFunction_scope); - m_out.storePtr(weakPointer(executable), fastObject, m_heaps.JSFunction_executable); - m_out.storePtr(m_out.intPtrZero, fastObject, m_heaps.JSFunction_rareData); - - ValueFromBlock fastResult = m_out.anchor(fastObject); - m_out.jump(continuation); - - m_out.appendTo(slowPath, continuation); - LValue callResult = vmCall( - m_out.operation(operationNewFunctionWithInvalidatedReallocationWatchpoint), - m_callFrame, scope, weakPointer(executable)); - ValueFromBlock slowResult = m_out.anchor(callResult); + ValueFromBlock slowResult = m_out.anchor(vmCall( + m_out.operation(operationNewObject), m_callFrame, m_out.constIntPtr(structure))); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setJSValue(m_out.phi(m_out.intPtr, fastResult, slowResult)); } - void compileCreateDirectArguments() - { - // FIXME: A more effective way of dealing with the argument count and callee is to have - // them be explicit arguments to this node. - // https://bugs.webkit.org/show_bug.cgi?id=142207 - - Structure* structure = - m_graph.globalObjectFor(m_node->origin.semantic)->directArgumentsStructure(); - - unsigned minCapacity = m_graph.baselineCodeBlockFor(m_node->origin.semantic)->numParameters() - 1; - - LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("CreateDirectArguments slow path")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("CreateDirectArguments continuation")); - - LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath); - - ArgumentsLength length = getArgumentsLength(); - - LValue fastObject; - if (length.isKnown) { - fastObject = allocateObject<DirectArguments>( - DirectArguments::allocationSize(std::max(length.known, minCapacity)), structure, - m_out.intPtrZero, slowPath); - } else { - LValue size = m_out.add( - m_out.shl(length.value, m_out.constInt32(3)), - m_out.constInt32(DirectArguments::storageOffset())); - - size = m_out.select( - m_out.aboveOrEqual(length.value, m_out.constInt32(minCapacity)), - size, m_out.constInt32(DirectArguments::allocationSize(minCapacity))); - - fastObject = allocateVariableSizedObject<DirectArguments>( - size, structure, m_out.intPtrZero, slowPath); - } - - m_out.store32(length.value, fastObject, m_heaps.DirectArguments_length); - m_out.store32(m_out.constInt32(minCapacity), fastObject, m_heaps.DirectArguments_minCapacity); - m_out.storePtr(m_out.intPtrZero, fastObject, m_heaps.DirectArguments_overrides); - - ValueFromBlock fastResult = m_out.anchor(fastObject); - m_out.jump(continuation); - - m_out.appendTo(slowPath, continuation); - LValue callResult = vmCall( - m_out.operation(operationCreateDirectArguments), m_callFrame, weakPointer(structure), - length.value, m_out.constInt32(minCapacity)); - ValueFromBlock slowResult = m_out.anchor(callResult); - m_out.jump(continuation); - - m_out.appendTo(continuation, lastNext); - LValue result = m_out.phi(m_out.intPtr, fastResult, slowResult); - - m_out.storePtr(getCurrentCallee(), result, m_heaps.DirectArguments_callee); - - if (length.isKnown) { - VirtualRegister start = AssemblyHelpers::argumentsStart(m_node->origin.semantic); - for (unsigned i = 0; i < std::max(length.known, minCapacity); ++i) { - m_out.store64( - m_out.load64(addressFor(start + i)), - result, m_heaps.DirectArguments_storage[i]); - } - } else { - LValue stackBase = getArgumentsStart(); - - LBasicBlock loop = FTL_NEW_BLOCK(m_out, ("CreateDirectArguments loop body")); - LBasicBlock end = FTL_NEW_BLOCK(m_out, ("CreateDirectArguments loop end")); - - ValueFromBlock originalLength; - if (minCapacity) { - LValue capacity = m_out.select( - m_out.aboveOrEqual(length.value, m_out.constInt32(minCapacity)), - length.value, - m_out.constInt32(minCapacity)); - originalLength = m_out.anchor(m_out.zeroExtPtr(capacity)); - m_out.jump(loop); - } else { - originalLength = m_out.anchor(m_out.zeroExtPtr(length.value)); - m_out.branch(m_out.isNull(originalLength.value()), unsure(end), unsure(loop)); - } - - lastNext = m_out.appendTo(loop, end); - LValue previousIndex = m_out.phi(m_out.intPtr, originalLength); - LValue index = m_out.sub(previousIndex, m_out.intPtrOne); - m_out.store64( - m_out.load64(m_out.baseIndex(m_heaps.variables, stackBase, index)), - m_out.baseIndex(m_heaps.DirectArguments_storage, result, index)); - ValueFromBlock nextIndex = m_out.anchor(index); - addIncoming(previousIndex, nextIndex); - m_out.branch(m_out.isNull(index), unsure(end), unsure(loop)); - - m_out.appendTo(end, lastNext); - } - - setJSValue(result); - } - - void compileCreateScopedArguments() - { - LValue scope = lowCell(m_node->child1()); - - LValue result = vmCall( - m_out.operation(operationCreateScopedArguments), m_callFrame, - weakPointer( - m_graph.globalObjectFor(m_node->origin.semantic)->scopedArgumentsStructure()), - getArgumentsStart(), getArgumentsLength().value, getCurrentCallee(), scope); - - setJSValue(result); - } - - void compileCreateClonedArguments() - { - LValue result = vmCall( - m_out.operation(operationCreateClonedArguments), m_callFrame, - weakPointer( - m_graph.globalObjectFor(m_node->origin.semantic)->outOfBandArgumentsStructure()), - getArgumentsStart(), getArgumentsLength().value, getCurrentCallee()); - - setJSValue(result); - } - - void compileNewObject() - { - setJSValue(allocateObject(m_node->structure())); - } - void compileNewArray() { // First speculate appropriately on all of the children. Do this unconditionally up here @@ -3268,13 +1948,13 @@ private: for (unsigned operandIndex = 0; operandIndex < m_node->numChildren(); ++operandIndex) speculate(m_graph.varArgChild(m_node, operandIndex)); - JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic); + JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->codeOrigin); Structure* structure = globalObject->arrayStructureForIndexingTypeDuringAllocation( m_node->indexingType()); - DFG_ASSERT(m_graph, m_node, structure->indexingType() == m_node->indexingType()); + RELEASE_ASSERT(structure->indexingType() == m_node->indexingType()); - if (!globalObject->isHavingABadTime() && !hasAnyArrayStorage(m_node->indexingType())) { + if (!globalObject->isHavingABadTime() && !hasArrayStorage(m_node->indexingType())) { unsigned numElements = m_node->numChildren(); ArrayValues arrayValues = allocateJSArray(structure, numElements); @@ -3285,7 +1965,7 @@ private: switch (m_node->indexingType()) { case ALL_BLANK_INDEXING_TYPES: case ALL_UNDECIDED_INDEXING_TYPES: - DFG_CRASH(m_graph, m_node, "Bad indexing type"); + CRASH(); break; case ALL_DOUBLE_INDEXING_TYPES: @@ -3303,8 +1983,7 @@ private: break; default: - DFG_CRASH(m_graph, m_node, "Corrupt indexing type"); - break; + CRASH(); } } @@ -3346,13 +2025,13 @@ private: void compileNewArrayBuffer() { - JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic); + JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->codeOrigin); Structure* structure = globalObject->arrayStructureForIndexingTypeDuringAllocation( m_node->indexingType()); - DFG_ASSERT(m_graph, m_node, structure->indexingType() == m_node->indexingType()); + RELEASE_ASSERT(structure->indexingType() == m_node->indexingType()); - if (!globalObject->isHavingABadTime() && !hasAnyArrayStorage(m_node->indexingType())) { + if (!globalObject->isHavingABadTime() && !hasArrayStorage(m_node->indexingType())) { unsigned numElements = m_node->numConstants(); ArrayValues arrayValues = allocateJSArray(structure, numElements); @@ -3381,310 +2060,46 @@ private: m_out.constIntPtr(m_node->numConstants()))); } - void compileNewArrayWithSize() - { - LValue publicLength = lowInt32(m_node->child1()); - - JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic); - Structure* structure = globalObject->arrayStructureForIndexingTypeDuringAllocation( - m_node->indexingType()); - - if (!globalObject->isHavingABadTime() && !hasAnyArrayStorage(m_node->indexingType())) { - ASSERT( - hasUndecided(structure->indexingType()) - || hasInt32(structure->indexingType()) - || hasDouble(structure->indexingType()) - || hasContiguous(structure->indexingType())); - - LBasicBlock fastCase = FTL_NEW_BLOCK(m_out, ("NewArrayWithSize fast case")); - LBasicBlock largeCase = FTL_NEW_BLOCK(m_out, ("NewArrayWithSize large case")); - LBasicBlock failCase = FTL_NEW_BLOCK(m_out, ("NewArrayWithSize fail case")); - LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("NewArrayWithSize slow case")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("NewArrayWithSize continuation")); - - m_out.branch( - m_out.aboveOrEqual(publicLength, m_out.constInt32(MIN_ARRAY_STORAGE_CONSTRUCTION_LENGTH)), - rarely(largeCase), usually(fastCase)); - - LBasicBlock lastNext = m_out.appendTo(fastCase, largeCase); - - // We don't round up to BASE_VECTOR_LEN for new Array(blah). - LValue vectorLength = publicLength; - - LValue payloadSize = - m_out.shl(m_out.zeroExt(vectorLength, m_out.intPtr), m_out.constIntPtr(3)); - - LValue butterflySize = m_out.add( - payloadSize, m_out.constIntPtr(sizeof(IndexingHeader))); - - LValue endOfStorage = allocateBasicStorageAndGetEnd(butterflySize, failCase); - - LValue butterfly = m_out.sub(endOfStorage, payloadSize); - - LValue object = allocateObject<JSArray>( - structure, butterfly, failCase); - - m_out.store32(publicLength, butterfly, m_heaps.Butterfly_publicLength); - m_out.store32(vectorLength, butterfly, m_heaps.Butterfly_vectorLength); - - if (hasDouble(m_node->indexingType())) { - LBasicBlock initLoop = FTL_NEW_BLOCK(m_out, ("NewArrayWithSize double init loop")); - LBasicBlock initDone = FTL_NEW_BLOCK(m_out, ("NewArrayWithSize double init done")); - - ValueFromBlock originalIndex = m_out.anchor(vectorLength); - ValueFromBlock originalPointer = m_out.anchor(butterfly); - m_out.branch( - m_out.notZero32(vectorLength), unsure(initLoop), unsure(initDone)); - - LBasicBlock initLastNext = m_out.appendTo(initLoop, initDone); - LValue index = m_out.phi(m_out.int32, originalIndex); - LValue pointer = m_out.phi(m_out.intPtr, originalPointer); - - m_out.store64( - m_out.constInt64(bitwise_cast<int64_t>(PNaN)), - TypedPointer(m_heaps.indexedDoubleProperties.atAnyIndex(), pointer)); - - LValue nextIndex = m_out.sub(index, m_out.int32One); - addIncoming(index, m_out.anchor(nextIndex)); - addIncoming(pointer, m_out.anchor(m_out.add(pointer, m_out.intPtrEight))); - m_out.branch( - m_out.notZero32(nextIndex), unsure(initLoop), unsure(initDone)); - - m_out.appendTo(initDone, initLastNext); - } - - ValueFromBlock fastResult = m_out.anchor(object); - m_out.jump(continuation); - - m_out.appendTo(largeCase, failCase); - ValueFromBlock largeStructure = m_out.anchor(m_out.constIntPtr( - globalObject->arrayStructureForIndexingTypeDuringAllocation(ArrayWithArrayStorage))); - m_out.jump(slowCase); - - m_out.appendTo(failCase, slowCase); - ValueFromBlock failStructure = m_out.anchor(m_out.constIntPtr(structure)); - m_out.jump(slowCase); - - m_out.appendTo(slowCase, continuation); - LValue structureValue = m_out.phi( - m_out.intPtr, largeStructure, failStructure); - ValueFromBlock slowResult = m_out.anchor(vmCall( - m_out.operation(operationNewArrayWithSize), - m_callFrame, structureValue, publicLength)); - m_out.jump(continuation); - - m_out.appendTo(continuation, lastNext); - setJSValue(m_out.phi(m_out.intPtr, fastResult, slowResult)); - return; - } - - LValue structureValue = m_out.select( - m_out.aboveOrEqual(publicLength, m_out.constInt32(MIN_ARRAY_STORAGE_CONSTRUCTION_LENGTH)), - m_out.constIntPtr( - globalObject->arrayStructureForIndexingTypeDuringAllocation(ArrayWithArrayStorage)), - m_out.constIntPtr(structure)); - setJSValue(vmCall(m_out.operation(operationNewArrayWithSize), m_callFrame, structureValue, publicLength)); - } - void compileAllocatePropertyStorage() { + StructureTransitionData& data = m_node->structureTransitionData(); + LValue object = lowCell(m_node->child1()); - setStorage(allocatePropertyStorage(object, m_node->transition()->previous)); - } - - void compileReallocatePropertyStorage() - { - Transition* transition = m_node->transition(); - LValue object = lowCell(m_node->child1()); - LValue oldStorage = lowStorage(m_node->child2()); - setStorage( - reallocatePropertyStorage( - object, oldStorage, transition->previous, transition->next)); - } - - void compileToStringOrCallStringConstructor() - { - switch (m_node->child1().useKind()) { - case StringObjectUse: { - LValue cell = lowCell(m_node->child1()); - speculateStringObjectForCell(m_node->child1(), cell); - m_interpreter.filter(m_node->child1(), SpecStringObject); - - setJSValue(m_out.loadPtr(cell, m_heaps.JSWrapperObject_internalValue)); - return; - } - - case StringOrStringObjectUse: { - LValue cell = lowCell(m_node->child1()); - LValue structureID = m_out.load32(cell, m_heaps.JSCell_structureID); - - LBasicBlock notString = FTL_NEW_BLOCK(m_out, ("ToString StringOrStringObject not string case")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ToString StringOrStringObject continuation")); - - ValueFromBlock simpleResult = m_out.anchor(cell); - m_out.branch( - m_out.equal(structureID, m_out.constInt32(vm().stringStructure->id())), - unsure(continuation), unsure(notString)); - - LBasicBlock lastNext = m_out.appendTo(notString, continuation); - speculateStringObjectForStructureID(m_node->child1(), structureID); - ValueFromBlock unboxedResult = m_out.anchor( - m_out.loadPtr(cell, m_heaps.JSWrapperObject_internalValue)); - m_out.jump(continuation); - - m_out.appendTo(continuation, lastNext); - setJSValue(m_out.phi(m_out.int64, simpleResult, unboxedResult)); - - m_interpreter.filter(m_node->child1(), SpecString | SpecStringObject); - return; - } - - case CellUse: - case UntypedUse: { - LValue value; - if (m_node->child1().useKind() == CellUse) - value = lowCell(m_node->child1()); - else - value = lowJSValue(m_node->child1()); - - LBasicBlock isCell = FTL_NEW_BLOCK(m_out, ("ToString CellUse/UntypedUse is cell")); - LBasicBlock notString = FTL_NEW_BLOCK(m_out, ("ToString CellUse/UntypedUse not string")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ToString CellUse/UntypedUse continuation")); - - LValue isCellPredicate; - if (m_node->child1().useKind() == CellUse) - isCellPredicate = m_out.booleanTrue; - else - isCellPredicate = this->isCell(value, provenType(m_node->child1())); - m_out.branch(isCellPredicate, unsure(isCell), unsure(notString)); - - LBasicBlock lastNext = m_out.appendTo(isCell, notString); - ValueFromBlock simpleResult = m_out.anchor(value); - LValue isStringPredicate; - if (m_node->child1()->prediction() & SpecString) { - isStringPredicate = isString(value, provenType(m_node->child1())); - } else - isStringPredicate = m_out.booleanFalse; - m_out.branch(isStringPredicate, unsure(continuation), unsure(notString)); - - m_out.appendTo(notString, continuation); - LValue operation; - if (m_node->child1().useKind() == CellUse) - operation = m_out.operation(m_node->op() == ToString ? operationToStringOnCell : operationCallStringConstructorOnCell); - else - operation = m_out.operation(m_node->op() == ToString ? operationToString : operationCallStringConstructor); - ValueFromBlock convertedResult = m_out.anchor(vmCall(operation, m_callFrame, value)); - m_out.jump(continuation); - - m_out.appendTo(continuation, lastNext); - setJSValue(m_out.phi(m_out.int64, simpleResult, convertedResult)); + if (data.previousStructure->couldHaveIndexingHeader()) { + setStorage(vmCall( + m_out.operation( + operationReallocateButterflyToHavePropertyStorageWithInitialCapacity), + m_callFrame, object)); return; } - - default: - DFG_CRASH(m_graph, m_node, "Bad use kind"); - break; - } - } - - void compileToPrimitive() - { - LValue value = lowJSValue(m_node->child1()); - - LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("ToPrimitive cell case")); - LBasicBlock isObjectCase = FTL_NEW_BLOCK(m_out, ("ToPrimitive object case")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ToPrimitive continuation")); - - Vector<ValueFromBlock, 3> results; - - results.append(m_out.anchor(value)); - m_out.branch( - isCell(value, provenType(m_node->child1())), unsure(isCellCase), unsure(continuation)); - - LBasicBlock lastNext = m_out.appendTo(isCellCase, isObjectCase); - results.append(m_out.anchor(value)); - m_out.branch( - isObject(value, provenType(m_node->child1())), - unsure(isObjectCase), unsure(continuation)); - - m_out.appendTo(isObjectCase, continuation); - results.append(m_out.anchor(vmCall( - m_out.operation(operationToPrimitive), m_callFrame, value))); - m_out.jump(continuation); - - m_out.appendTo(continuation, lastNext); - setJSValue(m_out.phi(m_out.int64, results)); - } - - void compileMakeRope() - { - LValue kids[3]; - unsigned numKids; - kids[0] = lowCell(m_node->child1()); - kids[1] = lowCell(m_node->child2()); - if (m_node->child3()) { - kids[2] = lowCell(m_node->child3()); - numKids = 3; - } else { - kids[2] = 0; - numKids = 2; - } - LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("MakeRope slow path")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("MakeRope continuation")); + LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("AllocatePropertyStorage slow path")); + LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("AllocatePropertyStorage continuation")); LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath); - MarkedAllocator& allocator = - vm().heap.allocatorForObjectWithDestructor(sizeof(JSRopeString)); - - LValue result = allocateCell( - m_out.constIntPtr(&allocator), - vm().stringStructure.get(), - slowPath); + LValue endOfStorage = allocateBasicStorageAndGetEnd( + m_out.constIntPtr(initialOutOfLineCapacity * sizeof(JSValue)), slowPath); - m_out.storePtr(m_out.intPtrZero, result, m_heaps.JSString_value); - for (unsigned i = 0; i < numKids; ++i) - m_out.storePtr(kids[i], result, m_heaps.JSRopeString_fibers[i]); - for (unsigned i = numKids; i < JSRopeString::s_maxInternalRopeLength; ++i) - m_out.storePtr(m_out.intPtrZero, result, m_heaps.JSRopeString_fibers[i]); - LValue flags = m_out.load32(kids[0], m_heaps.JSString_flags); - LValue length = m_out.load32(kids[0], m_heaps.JSString_length); - for (unsigned i = 1; i < numKids; ++i) { - flags = m_out.bitAnd(flags, m_out.load32(kids[i], m_heaps.JSString_flags)); - LValue lengthAndOverflow = m_out.addWithOverflow32( - length, m_out.load32(kids[i], m_heaps.JSString_length)); - speculate(Uncountable, noValue(), 0, m_out.extractValue(lengthAndOverflow, 1)); - length = m_out.extractValue(lengthAndOverflow, 0); - } - m_out.store32( - m_out.bitAnd(m_out.constInt32(JSString::Is8Bit), flags), - result, m_heaps.JSString_flags); - m_out.store32(length, result, m_heaps.JSString_length); + ValueFromBlock fastButterfly = m_out.anchor( + m_out.add(m_out.constIntPtr(sizeof(IndexingHeader)), endOfStorage)); - ValueFromBlock fastResult = m_out.anchor(result); m_out.jump(continuation); m_out.appendTo(slowPath, continuation); - ValueFromBlock slowResult; - switch (numKids) { - case 2: - slowResult = m_out.anchor(vmCall( - m_out.operation(operationMakeRope2), m_callFrame, kids[0], kids[1])); - break; - case 3: - slowResult = m_out.anchor(vmCall( - m_out.operation(operationMakeRope3), m_callFrame, kids[0], kids[1], kids[2])); - break; - default: - DFG_CRASH(m_graph, m_node, "Bad number of children"); - break; - } + + ValueFromBlock slowButterfly = m_out.anchor(vmCall( + m_out.operation(operationAllocatePropertyStorageWithInitialCapacity), m_callFrame)); + m_out.jump(continuation); m_out.appendTo(continuation, lastNext); - setJSValue(m_out.phi(m_out.int64, fastResult, slowResult)); + + LValue result = m_out.phi(m_out.intPtr, fastButterfly, slowButterfly); + m_out.storePtr(result, object, m_heaps.JSObject_butterfly); + + setStorage(result); } void compileStringCharAt() @@ -3699,8 +2114,8 @@ private: m_out.branch( m_out.aboveOrEqual( - index, m_out.load32NonNegative(base, m_heaps.JSString_length)), - rarely(slowPath), usually(fastPath)); + index, m_out.load32(base, m_heaps.JSString_length)), + slowPath, fastPath); LBasicBlock lastNext = m_out.appendTo(fastPath, slowPath); @@ -3715,14 +2130,15 @@ private: m_out.testIsZero32( m_out.load32(stringImpl, m_heaps.StringImpl_hashAndFlags), m_out.constInt32(StringImpl::flagIs8Bit())), - unsure(is16Bit), unsure(is8Bit)); + is16Bit, is8Bit); m_out.appendTo(is8Bit, is16Bit); ValueFromBlock char8Bit = m_out.anchor(m_out.zeroExt( m_out.load8(m_out.baseIndex( - m_heaps.characters8, storage, m_out.zeroExtPtr(index), - provenValue(m_node->child2()))), + m_heaps.characters8, + storage, m_out.zeroExt(index, m_out.intPtr), + m_state.forNode(m_node->child2()).m_value)), m_out.int32)); m_out.jump(bitsContinuation); @@ -3730,12 +2146,11 @@ private: ValueFromBlock char16Bit = m_out.anchor(m_out.zeroExt( m_out.load16(m_out.baseIndex( - m_heaps.characters16, storage, m_out.zeroExtPtr(index), - provenValue(m_node->child2()))), + m_heaps.characters16, + storage, m_out.zeroExt(index, m_out.intPtr), + m_state.forNode(m_node->child2()).m_value)), m_out.int32)); - m_out.branch( - m_out.aboveOrEqual(char16Bit.value(), m_out.constInt32(0x100)), - rarely(bigCharacter), usually(bitsContinuation)); + m_out.branch(m_out.aboveOrEqual(char16Bit.value(), m_out.constInt32(0x100)), bigCharacter, bitsContinuation); m_out.appendTo(bigCharacter, bitsContinuation); @@ -3752,7 +2167,8 @@ private: LValue smallStrings = m_out.constIntPtr(vm().smallStrings.singleCharacterStrings()); results.append(m_out.anchor(m_out.loadPtr(m_out.baseIndex( - m_heaps.singleCharacterStrings, smallStrings, m_out.zeroExtPtr(character))))); + m_heaps.singleCharacterStrings, smallStrings, + m_out.zeroExt(character, m_out.intPtr))))); m_out.jump(continuation); m_out.appendTo(slowPath, continuation); @@ -3761,23 +2177,13 @@ private: speculate(OutOfBounds, noValue(), 0, m_out.booleanTrue); results.append(m_out.anchor(m_out.intPtrZero)); } else { - JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic); + JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->codeOrigin); if (globalObject->stringPrototypeChainIsSane()) { - // FIXME: This could be captured using a Speculation mode that means - // "out-of-bounds loads return a trivial value", something like - // SaneChainOutOfBounds. - // https://bugs.webkit.org/show_bug.cgi?id=144668 - - m_graph.watchpoints().addLazily(globalObject->stringPrototype()->structure()->transitionWatchpointSet()); - m_graph.watchpoints().addLazily(globalObject->objectPrototype()->structure()->transitionWatchpointSet()); - LBasicBlock negativeIndex = FTL_NEW_BLOCK(m_out, ("GetByVal String negative index")); results.append(m_out.anchor(m_out.constInt64(JSValue::encode(jsUndefined())))); - m_out.branch( - m_out.lessThan(index, m_out.int32Zero), - rarely(negativeIndex), usually(continuation)); + m_out.branch(m_out.lessThan(index, m_out.int32Zero), negativeIndex, continuation); m_out.appendTo(negativeIndex, continuation); } @@ -3804,8 +2210,7 @@ private: speculate( Uncountable, noValue(), 0, - m_out.aboveOrEqual( - index, m_out.load32NonNegative(base, m_heaps.JSString_length))); + m_out.aboveOrEqual(index, m_out.load32(base, m_heaps.JSString_length))); LValue stringImpl = m_out.loadPtr(base, m_heaps.JSString_value); @@ -3813,14 +2218,15 @@ private: m_out.testIsZero32( m_out.load32(stringImpl, m_heaps.StringImpl_hashAndFlags), m_out.constInt32(StringImpl::flagIs8Bit())), - unsure(is16Bit), unsure(is8Bit)); + is16Bit, is8Bit); LBasicBlock lastNext = m_out.appendTo(is8Bit, is16Bit); ValueFromBlock char8Bit = m_out.anchor(m_out.zeroExt( m_out.load8(m_out.baseIndex( - m_heaps.characters8, storage, m_out.zeroExtPtr(index), - provenValue(m_node->child2()))), + m_heaps.characters8, + storage, m_out.zeroExt(index, m_out.intPtr), + m_state.forNode(m_node->child2()).m_value)), m_out.int32)); m_out.jump(continuation); @@ -3828,8 +2234,9 @@ private: ValueFromBlock char16Bit = m_out.anchor(m_out.zeroExt( m_out.load16(m_out.baseIndex( - m_heaps.characters16, storage, m_out.zeroExtPtr(index), - provenValue(m_node->child2()))), + m_heaps.characters16, + storage, m_out.zeroExt(index, m_out.intPtr), + m_state.forNode(m_node->child2()).m_value)), m_out.int32)); m_out.jump(continuation); @@ -3840,220 +2247,108 @@ private: void compileGetByOffset() { - StorageAccessData& data = m_node->storageAccessData(); + StorageAccessData& data = + m_graph.m_storageAccessData[m_node->storageAccessDataIndex()]; - setJSValue(loadProperty( - lowStorage(m_node->child1()), data.identifierNumber, data.offset)); - } - - void compileGetGetter() - { - setJSValue(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.GetterSetter_getter)); - } - - void compileGetSetter() - { - setJSValue(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.GetterSetter_setter)); - } - - void compileMultiGetByOffset() - { - LValue base = lowCell(m_node->child1()); - - MultiGetByOffsetData& data = m_node->multiGetByOffsetData(); - - if (data.cases.isEmpty()) { - // Protect against creating a Phi function with zero inputs. LLVM doesn't like that. - terminate(BadCache); - return; - } - - Vector<LBasicBlock, 2> blocks(data.cases.size()); - for (unsigned i = data.cases.size(); i--;) - blocks[i] = FTL_NEW_BLOCK(m_out, ("MultiGetByOffset case ", i)); - LBasicBlock exit = FTL_NEW_BLOCK(m_out, ("MultiGetByOffset fail")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("MultiGetByOffset continuation")); - - Vector<SwitchCase, 2> cases; - StructureSet baseSet; - for (unsigned i = data.cases.size(); i--;) { - MultiGetByOffsetCase getCase = data.cases[i]; - for (unsigned j = getCase.set().size(); j--;) { - Structure* structure = getCase.set()[j]; - baseSet.add(structure); - cases.append(SwitchCase(weakStructureID(structure), blocks[i], Weight(1))); - } - } - m_out.switchInstruction( - m_out.load32(base, m_heaps.JSCell_structureID), cases, exit, Weight(0)); - - LBasicBlock lastNext = m_out.m_nextBlock; - - Vector<ValueFromBlock, 2> results; - for (unsigned i = data.cases.size(); i--;) { - MultiGetByOffsetCase getCase = data.cases[i]; - GetByOffsetMethod method = getCase.method(); - - m_out.appendTo(blocks[i], i + 1 < data.cases.size() ? blocks[i + 1] : exit); - - LValue result; - - switch (method.kind()) { - case GetByOffsetMethod::Invalid: - RELEASE_ASSERT_NOT_REACHED(); - break; - - case GetByOffsetMethod::Constant: - result = m_out.constInt64(JSValue::encode(method.constant()->value())); - break; - - case GetByOffsetMethod::Load: - case GetByOffsetMethod::LoadFromPrototype: { - LValue propertyBase; - if (method.kind() == GetByOffsetMethod::Load) - propertyBase = base; - else - propertyBase = weakPointer(method.prototype()->value().asCell()); - if (!isInlineOffset(method.offset())) - propertyBase = m_out.loadPtr(propertyBase, m_heaps.JSObject_butterfly); - result = loadProperty( - propertyBase, data.identifierNumber, method.offset()); - break; - } } - - results.append(m_out.anchor(result)); - m_out.jump(continuation); - } - - m_out.appendTo(exit, continuation); - if (!m_interpreter.forNode(m_node->child1()).m_structure.isSubsetOf(baseSet)) - speculate(BadCache, noValue(), nullptr, m_out.booleanTrue); - m_out.unreachable(); - - m_out.appendTo(continuation, lastNext); - setJSValue(m_out.phi(m_out.int64, results)); + setJSValue( + m_out.load64( + m_out.address( + m_heaps.properties[data.identifierNumber], + lowStorage(m_node->child1()), + offsetRelativeToBase(data.offset)))); } void compilePutByOffset() { - StorageAccessData& data = m_node->storageAccessData(); + StorageAccessData& data = + m_graph.m_storageAccessData[m_node->storageAccessDataIndex()]; - storeProperty( + m_out.store64( lowJSValue(m_node->child3()), - lowStorage(m_node->child1()), data.identifierNumber, data.offset); - } - - void compileMultiPutByOffset() - { - LValue base = lowCell(m_node->child1()); - LValue value = lowJSValue(m_node->child2()); - - MultiPutByOffsetData& data = m_node->multiPutByOffsetData(); - - Vector<LBasicBlock, 2> blocks(data.variants.size()); - for (unsigned i = data.variants.size(); i--;) - blocks[i] = FTL_NEW_BLOCK(m_out, ("MultiPutByOffset case ", i)); - LBasicBlock exit = FTL_NEW_BLOCK(m_out, ("MultiPutByOffset fail")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("MultiPutByOffset continuation")); - - Vector<SwitchCase, 2> cases; - StructureSet baseSet; - for (unsigned i = data.variants.size(); i--;) { - PutByIdVariant variant = data.variants[i]; - for (unsigned j = variant.oldStructure().size(); j--;) { - Structure* structure = variant.oldStructure()[j]; - baseSet.add(structure); - cases.append(SwitchCase(weakStructureID(structure), blocks[i], Weight(1))); - } - } - m_out.switchInstruction( - m_out.load32(base, m_heaps.JSCell_structureID), cases, exit, Weight(0)); - - LBasicBlock lastNext = m_out.m_nextBlock; - - for (unsigned i = data.variants.size(); i--;) { - m_out.appendTo(blocks[i], i + 1 < data.variants.size() ? blocks[i + 1] : exit); - - PutByIdVariant variant = data.variants[i]; - - LValue storage; - if (variant.kind() == PutByIdVariant::Replace) { - if (isInlineOffset(variant.offset())) - storage = base; - else - storage = m_out.loadPtr(base, m_heaps.JSObject_butterfly); - } else { - m_graph.m_plan.transitions.addLazily( - codeBlock(), m_node->origin.semantic.codeOriginOwner(), - variant.oldStructureForTransition(), variant.newStructure()); - - storage = storageForTransition( - base, variant.offset(), - variant.oldStructureForTransition(), variant.newStructure()); - - ASSERT(variant.oldStructureForTransition()->indexingType() == variant.newStructure()->indexingType()); - ASSERT(variant.oldStructureForTransition()->typeInfo().inlineTypeFlags() == variant.newStructure()->typeInfo().inlineTypeFlags()); - ASSERT(variant.oldStructureForTransition()->typeInfo().type() == variant.newStructure()->typeInfo().type()); - m_out.store32( - weakStructureID(variant.newStructure()), base, m_heaps.JSCell_structureID); - } - - storeProperty(value, storage, data.identifierNumber, variant.offset()); - m_out.jump(continuation); - } - - m_out.appendTo(exit, continuation); - if (!m_interpreter.forNode(m_node->child1()).m_structure.isSubsetOf(baseSet)) - speculate(BadCache, noValue(), nullptr, m_out.booleanTrue); - m_out.unreachable(); - - m_out.appendTo(continuation, lastNext); + m_out.address( + m_heaps.properties[data.identifierNumber], + lowStorage(m_node->child1()), + offsetRelativeToBase(data.offset))); } void compileGetGlobalVar() { - setJSValue(m_out.load64(m_out.absolute(m_node->variablePointer()))); + setJSValue(m_out.load64(m_out.absolute(m_node->registerPointer()))); } void compilePutGlobalVar() { m_out.store64( - lowJSValue(m_node->child2()), m_out.absolute(m_node->variablePointer())); + lowJSValue(m_node->child1()), m_out.absolute(m_node->registerPointer())); } void compileNotifyWrite() { - WatchpointSet* set = m_node->watchpointSet(); + VariableWatchpointSet* set = m_node->variableWatchpointSet(); + + LValue value = lowJSValue(m_node->child1()); LBasicBlock isNotInvalidated = FTL_NEW_BLOCK(m_out, ("NotifyWrite not invalidated case")); + LBasicBlock isClear = FTL_NEW_BLOCK(m_out, ("NotifyWrite clear case")); + LBasicBlock isWatched = FTL_NEW_BLOCK(m_out, ("NotifyWrite watched case")); + LBasicBlock invalidate = FTL_NEW_BLOCK(m_out, ("NotifyWrite invalidate case")); + LBasicBlock invalidateFast = FTL_NEW_BLOCK(m_out, ("NotifyWrite invalidate fast case")); + LBasicBlock invalidateSlow = FTL_NEW_BLOCK(m_out, ("NotifyWrite invalidate slow case")); LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("NotifyWrite continuation")); LValue state = m_out.load8(m_out.absolute(set->addressOfState())); + m_out.branch( m_out.equal(state, m_out.constInt8(IsInvalidated)), - usually(continuation), rarely(isNotInvalidated)); + continuation, isNotInvalidated); - LBasicBlock lastNext = m_out.appendTo(isNotInvalidated, continuation); + LBasicBlock lastNext = m_out.appendTo(isNotInvalidated, isClear); - vmCall(m_out.operation(operationNotifyWrite), m_callFrame, m_out.constIntPtr(set)); + LValue isClearValue; + if (set->state() == ClearWatchpoint) + isClearValue = m_out.equal(state, m_out.constInt8(ClearWatchpoint)); + else + isClearValue = m_out.booleanFalse; + m_out.branch(isClearValue, isClear, isWatched); + + m_out.appendTo(isClear, isWatched); + + m_out.store64(value, m_out.absolute(set->addressOfInferredValue())); + m_out.store8(m_out.constInt8(IsWatched), m_out.absolute(set->addressOfState())); + m_out.jump(continuation); + + m_out.appendTo(isWatched, invalidate); + + m_out.branch( + m_out.equal(value, m_out.load64(m_out.absolute(set->addressOfInferredValue()))), + continuation, invalidate); + + m_out.appendTo(invalidate, invalidateFast); + + m_out.branch( + m_out.notZero8(m_out.load8(m_out.absolute(set->addressOfSetIsNotEmpty()))), + invalidateSlow, invalidateFast); + + m_out.appendTo(invalidateFast, invalidateSlow); + + m_out.store64( + m_out.constInt64(JSValue::encode(JSValue())), + m_out.absolute(set->addressOfInferredValue())); + m_out.store8(m_out.constInt8(IsInvalidated), m_out.absolute(set->addressOfState())); + m_out.jump(continuation); + + m_out.appendTo(invalidateSlow, continuation); + + vmCall(m_out.operation(operationInvalidate), m_callFrame, m_out.constIntPtr(set)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); } - void compileGetCallee() + void compileGetMyScope() { - setJSValue(m_out.loadPtr(addressFor(JSStack::Callee))); - } - - void compileGetArgumentCount() - { - setInt32(m_out.load32(payloadFor(JSStack::ArgumentCount))); - } - - void compileGetScope() - { - setJSValue(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.JSFunction_scope)); + setJSValue(m_out.loadPtr(addressFor( + m_node->codeOrigin.stackOffset() + JSStack::ScopeChain))); } void compileSkipScope() @@ -4061,71 +2356,51 @@ private: setJSValue(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.JSScope_next)); } - void compileGetClosureVar() + void compileGetClosureRegisters() { - setJSValue( - m_out.load64( - lowCell(m_node->child1()), - m_heaps.JSEnvironmentRecord_variables[m_node->scopeOffset().offset()])); + if (WriteBarrierBase<Unknown>* registers = m_graph.tryGetRegisters(m_node->child1().node())) { + setStorage(m_out.constIntPtr(registers)); + return; + } + + setStorage(m_out.loadPtr( + lowCell(m_node->child1()), m_heaps.JSVariableObject_registers)); } - void compilePutClosureVar() + void compileGetClosureVar() { - m_out.store64( - lowJSValue(m_node->child2()), - lowCell(m_node->child1()), - m_heaps.JSEnvironmentRecord_variables[m_node->scopeOffset().offset()]); + setJSValue(m_out.load64( + addressFor(lowStorage(m_node->child1()), m_node->varNumber()))); } - void compileGetFromArguments() - { - setJSValue( - m_out.load64( - lowCell(m_node->child1()), - m_heaps.DirectArguments_storage[m_node->capturedArgumentsOffset().offset()])); - } - - void compilePutToArguments() + void compilePutClosureVar() { m_out.store64( - lowJSValue(m_node->child2()), - lowCell(m_node->child1()), - m_heaps.DirectArguments_storage[m_node->capturedArgumentsOffset().offset()]); + lowJSValue(m_node->child3()), + addressFor(lowStorage(m_node->child2()), m_node->varNumber())); } void compileCompareEq() { if (m_node->isBinaryUseKind(Int32Use) - || m_node->isBinaryUseKind(Int52RepUse) - || m_node->isBinaryUseKind(DoubleRepUse) - || m_node->isBinaryUseKind(ObjectUse) - || m_node->isBinaryUseKind(BooleanUse) - || m_node->isBinaryUseKind(StringIdentUse)) { + || m_node->isBinaryUseKind(MachineIntUse) + || m_node->isBinaryUseKind(NumberUse) + || m_node->isBinaryUseKind(ObjectUse)) { compileCompareStrictEq(); return; } - if (m_node->isBinaryUseKind(ObjectUse, ObjectOrOtherUse)) { - compareEqObjectOrOtherToObject(m_node->child2(), m_node->child1()); - return; - } - - if (m_node->isBinaryUseKind(ObjectOrOtherUse, ObjectUse)) { - compareEqObjectOrOtherToObject(m_node->child1(), m_node->child2()); - return; - } - if (m_node->isBinaryUseKind(UntypedUse)) { nonSpeculativeCompare(LLVMIntEQ, operationCompareEq); return; } - DFG_CRASH(m_graph, m_node, "Bad use kinds"); + RELEASE_ASSERT_NOT_REACHED(); } void compileCompareEqConstant() { - ASSERT(m_node->child2()->asJSValue().isNull()); + ASSERT(m_graph.valueOfJSConstant(m_node->child2().node()).isNull()); setBoolean( equalNullOrUndefined( m_node->child1(), AllCellsAreFalse, EqualNullOrUndefined)); @@ -4139,7 +2414,7 @@ private: return; } - if (m_node->isBinaryUseKind(Int52RepUse)) { + if (m_node->isBinaryUseKind(MachineIntUse)) { Int52Kind kind; LValue left = lowWhicheverInt52(m_node->child1(), kind); LValue right = lowInt52(m_node->child2(), kind); @@ -4147,34 +2422,12 @@ private: return; } - if (m_node->isBinaryUseKind(DoubleRepUse)) { + if (m_node->isBinaryUseKind(NumberUse)) { setBoolean( m_out.doubleEqual(lowDouble(m_node->child1()), lowDouble(m_node->child2()))); return; } - if (m_node->isBinaryUseKind(StringIdentUse)) { - setBoolean( - m_out.equal(lowStringIdent(m_node->child1()), lowStringIdent(m_node->child2()))); - return; - } - - if (m_node->isBinaryUseKind(ObjectUse, UntypedUse)) { - setBoolean( - m_out.equal( - lowNonNullObject(m_node->child1()), - lowJSValue(m_node->child2()))); - return; - } - - if (m_node->isBinaryUseKind(UntypedUse, ObjectUse)) { - setBoolean( - m_out.equal( - lowNonNullObject(m_node->child2()), - lowJSValue(m_node->child1()))); - return; - } - if (m_node->isBinaryUseKind(ObjectUse)) { setBoolean( m_out.equal( @@ -4183,63 +2436,25 @@ private: return; } - if (m_node->isBinaryUseKind(BooleanUse)) { - setBoolean( - m_out.equal(lowBoolean(m_node->child1()), lowBoolean(m_node->child2()))); - return; - } - - if (m_node->isBinaryUseKind(MiscUse, UntypedUse) - || m_node->isBinaryUseKind(UntypedUse, MiscUse)) { - speculate(m_node->child1()); - speculate(m_node->child2()); - LValue left = lowJSValue(m_node->child1(), ManualOperandSpeculation); - LValue right = lowJSValue(m_node->child2(), ManualOperandSpeculation); - setBoolean(m_out.equal(left, right)); - return; - } - - if (m_node->isBinaryUseKind(StringIdentUse, NotStringVarUse) - || m_node->isBinaryUseKind(NotStringVarUse, StringIdentUse)) { - Edge leftEdge = m_node->childFor(StringIdentUse); - Edge rightEdge = m_node->childFor(NotStringVarUse); - - LValue left = lowStringIdent(leftEdge); - LValue rightValue = lowJSValue(rightEdge, ManualOperandSpeculation); - - LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("CompareStrictEq StringIdent to NotStringVar is cell case")); - LBasicBlock isStringCase = FTL_NEW_BLOCK(m_out, ("CompareStrictEq StringIdent to NotStringVar is string case")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("CompareStrictEq StringIdent to NotStringVar continuation")); - - ValueFromBlock notCellResult = m_out.anchor(m_out.booleanFalse); - m_out.branch( - isCell(rightValue, provenType(rightEdge)), - unsure(isCellCase), unsure(continuation)); - - LBasicBlock lastNext = m_out.appendTo(isCellCase, isStringCase); - ValueFromBlock notStringResult = m_out.anchor(m_out.booleanFalse); - m_out.branch( - isString(rightValue, provenType(rightEdge)), - unsure(isStringCase), unsure(continuation)); - - m_out.appendTo(isStringCase, continuation); - LValue right = m_out.loadPtr(rightValue, m_heaps.JSString_value); - speculateStringIdent(rightEdge, rightValue, right); - ValueFromBlock isStringResult = m_out.anchor(m_out.equal(left, right)); - m_out.jump(continuation); - - m_out.appendTo(continuation, lastNext); - setBoolean(m_out.phi(m_out.boolean, notCellResult, notStringResult, isStringResult)); - return; - } - - DFG_CRASH(m_graph, m_node, "Bad use kinds"); + RELEASE_ASSERT_NOT_REACHED(); } void compileCompareStrictEqConstant() { - JSValue constant = m_node->child2()->asJSValue(); + JSValue constant = m_graph.valueOfJSConstant(m_node->child2().node()); + if (constant.isUndefinedOrNull() + && !masqueradesAsUndefinedWatchpointIsStillValid()) { + if (constant.isNull()) { + setBoolean(equalNullOrUndefined(m_node->child1(), AllCellsAreFalse, EqualNull)); + return; + } + + ASSERT(constant.isUndefined()); + setBoolean(equalNullOrUndefined(m_node->child1(), AllCellsAreFalse, EqualUndefined)); + return; + } + setBoolean( m_out.equal( lowJSValue(m_node->child1()), @@ -4270,188 +2485,51 @@ private: { setBoolean(m_out.bitNot(boolify(m_node->child1()))); } - + void compileCallOrConstruct() { - int numPassedArgs = m_node->numChildren() - 1; - int numArgs = numPassedArgs; - - LValue jsCallee = lowJSValue(m_graph.varArgChild(m_node, 0)); - - unsigned stackmapID = m_stackmapIDs++; + // FIXME: This is unacceptably slow. + // https://bugs.webkit.org/show_bug.cgi?id=113621 - Vector<LValue> arguments; - arguments.append(m_out.constInt64(stackmapID)); - arguments.append(m_out.constInt32(sizeOfCall())); - arguments.append(constNull(m_out.ref8)); - arguments.append(m_out.constInt32(1 + JSStack::CallFrameHeaderSize - JSStack::CallerFrameAndPCSize + numArgs)); - arguments.append(jsCallee); // callee -> %rax - arguments.append(getUndef(m_out.int64)); // code block - arguments.append(jsCallee); // callee -> stack - arguments.append(m_out.constInt64(numArgs)); // argument count and zeros for the tag - for (int i = 0; i < numPassedArgs; ++i) - arguments.append(lowJSValue(m_graph.varArgChild(m_node, 1 + i))); + J_JITOperation_E function = + m_node->op() == Call ? operationFTLCall : operationFTLConstruct; - callPreflight(); + int dummyThisArgument = m_node->op() == Call ? 0 : 1; - LValue call = m_out.call(m_out.patchpointInt64Intrinsic(), arguments); - setInstructionCallingConvention(call, LLVMWebKitJSCallConv); + int numPassedArgs = m_node->numChildren() - 1; - m_ftlState.jsCalls.append(JSCall(stackmapID, m_node)); + LValue calleeFrame = m_out.add( + m_callFrame, + m_out.constIntPtr(sizeof(Register) * virtualRegisterForLocal(m_graph.frameRegisterCount()).offset())); - setJSValue(call); - } - - void compileCallOrConstructVarargs() - { - LValue jsCallee = lowJSValue(m_node->child1()); - LValue thisArg = lowJSValue(m_node->child3()); - - LValue jsArguments = nullptr; + m_out.store32( + m_out.constInt32(numPassedArgs + dummyThisArgument), + payloadFor(calleeFrame, JSStack::ArgumentCount)); + m_out.store64(m_callFrame, calleeFrame, m_heaps.CallFrame_callerFrame); + m_out.store64( + lowJSValue(m_graph.varArgChild(m_node, 0)), + addressFor(calleeFrame, JSStack::Callee)); - switch (m_node->op()) { - case CallVarargs: - case ConstructVarargs: - jsArguments = lowJSValue(m_node->child2()); - break; - case CallForwardVarargs: - case ConstructForwardVarargs: - break; - default: - DFG_CRASH(m_graph, m_node, "bad node type"); - break; + for (int i = 0; i < numPassedArgs; ++i) { + m_out.store64( + lowJSValue(m_graph.varArgChild(m_node, 1 + i)), + addressFor(calleeFrame, virtualRegisterForArgument(i + dummyThisArgument).offset())); } - unsigned stackmapID = m_stackmapIDs++; - - Vector<LValue> arguments; - arguments.append(m_out.constInt64(stackmapID)); - arguments.append(m_out.constInt32(sizeOfICFor(m_node))); - arguments.append(constNull(m_out.ref8)); - arguments.append(m_out.constInt32(2 + !!jsArguments)); - arguments.append(jsCallee); - if (jsArguments) - arguments.append(jsArguments); - ASSERT(thisArg); - arguments.append(thisArg); - - callPreflight(); - - LValue call = m_out.call(m_out.patchpointInt64Intrinsic(), arguments); - setInstructionCallingConvention(call, LLVMCCallConv); - - m_ftlState.jsCallVarargses.append(JSCallVarargs(stackmapID, m_node)); - - setJSValue(call); - } - - void compileLoadVarargs() - { - LoadVarargsData* data = m_node->loadVarargsData(); - LValue jsArguments = lowJSValue(m_node->child1()); - - LValue length = vmCall( - m_out.operation(operationSizeOfVarargs), m_callFrame, jsArguments, - m_out.constInt32(data->offset)); - - // FIXME: There is a chance that we will call an effectful length property twice. This is safe - // from the standpoint of the VM's integrity, but it's subtly wrong from a spec compliance - // standpoint. The best solution would be one where we can exit *into* the op_call_varargs right - // past the sizing. - // https://bugs.webkit.org/show_bug.cgi?id=141448 - - LValue lengthIncludingThis = m_out.add(length, m_out.int32One); - speculate( - VarargsOverflow, noValue(), nullptr, - m_out.above(lengthIncludingThis, m_out.constInt32(data->limit))); - - m_out.store32(lengthIncludingThis, payloadFor(data->machineCount)); - - // FIXME: This computation is rather silly. If operationLaodVarargs just took a pointer instead - // of a VirtualRegister, we wouldn't have to do this. - // https://bugs.webkit.org/show_bug.cgi?id=141660 - LValue machineStart = m_out.lShr( - m_out.sub(addressFor(data->machineStart.offset()).value(), m_callFrame), - m_out.constIntPtr(3)); - - vmCall( - m_out.operation(operationLoadVarargs), m_callFrame, - m_out.castToInt32(machineStart), jsArguments, m_out.constInt32(data->offset), - length, m_out.constInt32(data->mandatoryMinimum)); + setJSValue(vmCall(m_out.operation(function), calleeFrame)); } - void compileForwardVarargs() - { - LoadVarargsData* data = m_node->loadVarargsData(); - InlineCallFrame* inlineCallFrame = m_node->child1()->origin.semantic.inlineCallFrame; - - LValue length = getArgumentsLength(inlineCallFrame).value; - LValue lengthIncludingThis = m_out.add(length, m_out.constInt32(1 - data->offset)); - - speculate( - VarargsOverflow, noValue(), nullptr, - m_out.above(lengthIncludingThis, m_out.constInt32(data->limit))); - - m_out.store32(lengthIncludingThis, payloadFor(data->machineCount)); - - LValue sourceStart = getArgumentsStart(inlineCallFrame); - LValue targetStart = addressFor(data->machineStart).value(); - - LBasicBlock undefinedLoop = FTL_NEW_BLOCK(m_out, ("ForwardVarargs undefined loop body")); - LBasicBlock mainLoopEntry = FTL_NEW_BLOCK(m_out, ("ForwardVarargs main loop entry")); - LBasicBlock mainLoop = FTL_NEW_BLOCK(m_out, ("ForwardVarargs main loop body")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ForwardVarargs continuation")); - - LValue lengthAsPtr = m_out.zeroExtPtr(length); - ValueFromBlock loopBound = m_out.anchor(m_out.constIntPtr(data->mandatoryMinimum)); - m_out.branch( - m_out.above(loopBound.value(), lengthAsPtr), unsure(undefinedLoop), unsure(mainLoopEntry)); - - LBasicBlock lastNext = m_out.appendTo(undefinedLoop, mainLoopEntry); - LValue previousIndex = m_out.phi(m_out.intPtr, loopBound); - LValue currentIndex = m_out.sub(previousIndex, m_out.intPtrOne); - m_out.store64( - m_out.constInt64(JSValue::encode(jsUndefined())), - m_out.baseIndex(m_heaps.variables, targetStart, currentIndex)); - ValueFromBlock nextIndex = m_out.anchor(currentIndex); - addIncoming(previousIndex, nextIndex); - m_out.branch( - m_out.above(currentIndex, lengthAsPtr), unsure(undefinedLoop), unsure(mainLoopEntry)); - - m_out.appendTo(mainLoopEntry, mainLoop); - loopBound = m_out.anchor(lengthAsPtr); - m_out.branch(m_out.notNull(loopBound.value()), unsure(mainLoop), unsure(continuation)); - - m_out.appendTo(mainLoop, continuation); - previousIndex = m_out.phi(m_out.intPtr, loopBound); - currentIndex = m_out.sub(previousIndex, m_out.intPtrOne); - LValue value = m_out.load64( - m_out.baseIndex( - m_heaps.variables, sourceStart, - m_out.add(currentIndex, m_out.constIntPtr(data->offset)))); - m_out.store64(value, m_out.baseIndex(m_heaps.variables, targetStart, currentIndex)); - nextIndex = m_out.anchor(currentIndex); - addIncoming(previousIndex, nextIndex); - m_out.branch(m_out.isNull(currentIndex), unsure(continuation), unsure(mainLoop)); - - m_out.appendTo(continuation, lastNext); - } - void compileJump() { - m_out.jump(lowBlock(m_node->targetBlock())); + m_out.jump(lowBlock(m_node->takenBlock())); } void compileBranch() { m_out.branch( boolify(m_node->child1()), - WeightedTarget( - lowBlock(m_node->branchData()->taken.block), - m_node->branchData()->taken.count), - WeightedTarget( - lowBlock(m_node->branchData()->notTaken.block), - m_node->branchData()->notTaken.count)); + lowBlock(m_node->takenBlock()), + lowBlock(m_node->notTakenBlock())); } void compileSwitch() @@ -4477,7 +2555,7 @@ private: LBasicBlock isDouble = FTL_NEW_BLOCK(m_out, ("Switch/SwitchImm is double")); LValue boxedValue = lowJSValue(m_node->child1()); - m_out.branch(isNotInt32(boxedValue), unsure(isNotInt), unsure(isInt)); + m_out.branch(isNotInt32(boxedValue), isNotInt, isInt); LBasicBlock innerLastNext = m_out.appendTo(isInt, isNotInt); @@ -4486,8 +2564,7 @@ private: m_out.appendTo(isNotInt, isDouble); m_out.branch( - isCellOrMisc(boxedValue, provenType(m_node->child1())), - usually(lowBlock(data->fallThrough.block)), rarely(isDouble)); + isCellOrMisc(boxedValue), lowBlock(data->fallThrough), isDouble); m_out.appendTo(isDouble, innerLastNext); LValue doubleValue = unboxDouble(boxedValue); @@ -4495,12 +2572,12 @@ private: intValues.append(m_out.anchor(intInDouble)); m_out.branch( m_out.doubleEqual(m_out.intToDouble(intInDouble), doubleValue), - unsure(switchOnInts), unsure(lowBlock(data->fallThrough.block))); + switchOnInts, lowBlock(data->fallThrough)); break; } default: - DFG_CRASH(m_graph, m_node, "Bad use kind"); + RELEASE_ASSERT_NOT_REACHED(); break; } @@ -4512,12 +2589,6 @@ private: case SwitchChar: { LValue stringValue; - // FIXME: We should use something other than unsure() for the branch weight - // of the fallThrough block. The main challenge is just that we have multiple - // branches to fallThrough but a single count, so we would need to divvy it up - // among the different lowered branches. - // https://bugs.webkit.org/show_bug.cgi?id=129082 - switch (m_node->child1().useKind()) { case StringUse: { stringValue = lowString(m_node->child1()); @@ -4531,14 +2602,11 @@ private: LBasicBlock isStringCase = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar is string")); m_out.branch( - isNotCell(unboxedValue, provenType(m_node->child1())), - unsure(lowBlock(data->fallThrough.block)), unsure(isCellCase)); + isNotCell(unboxedValue), lowBlock(data->fallThrough), isCellCase); LBasicBlock lastNext = m_out.appendTo(isCellCase, isStringCase); LValue cellValue = unboxedValue; - m_out.branch( - isNotString(cellValue, provenType(m_node->child1())), - unsure(lowBlock(data->fallThrough.block)), unsure(isStringCase)); + m_out.branch(isNotString(cellValue), lowBlock(data->fallThrough), isStringCase); m_out.appendTo(isStringCase, lastNext); stringValue = cellValue; @@ -4546,7 +2614,7 @@ private: } default: - DFG_CRASH(m_graph, m_node, "Bad use kind"); + RELEASE_ASSERT_NOT_REACHED(); break; } @@ -4559,15 +2627,15 @@ private: m_out.branch( m_out.notEqual( - m_out.load32NonNegative(stringValue, m_heaps.JSString_length), + m_out.load32(stringValue, m_heaps.JSString_length), m_out.int32One), - unsure(lowBlock(data->fallThrough.block)), unsure(lengthIs1)); + lowBlock(data->fallThrough), lengthIs1); LBasicBlock lastNext = m_out.appendTo(lengthIs1, needResolution); Vector<ValueFromBlock, 2> values; LValue fastValue = m_out.loadPtr(stringValue, m_heaps.JSString_value); values.append(m_out.anchor(fastValue)); - m_out.branch(m_out.isNull(fastValue), rarely(needResolution), usually(resolved)); + m_out.branch(m_out.isNull(fastValue), needResolution, resolved); m_out.appendTo(needResolution, resolved); values.append(m_out.anchor( @@ -4581,7 +2649,7 @@ private: m_out.testNonZero32( m_out.load32(value, m_heaps.StringImpl_hashAndFlags), m_out.constInt32(StringImpl::flagIs8Bit())), - unsure(is8Bit), unsure(is16Bit)); + is8Bit, is16Bit); Vector<ValueFromBlock, 2> characters; m_out.appendTo(is8Bit, is16Bit); @@ -4598,92 +2666,18 @@ private: return; } - case SwitchString: { - switch (m_node->child1().useKind()) { - case StringIdentUse: { - LValue stringImpl = lowStringIdent(m_node->child1()); - - Vector<SwitchCase> cases; - for (unsigned i = 0; i < data->cases.size(); ++i) { - LValue value = m_out.constIntPtr(data->cases[i].value.stringImpl()); - LBasicBlock block = lowBlock(data->cases[i].target.block); - Weight weight = Weight(data->cases[i].target.count); - cases.append(SwitchCase(value, block, weight)); - } - - m_out.switchInstruction( - stringImpl, cases, lowBlock(data->fallThrough.block), - Weight(data->fallThrough.count)); - return; - } - - case StringUse: { - switchString(data, lowString(m_node->child1())); - return; - } - - case UntypedUse: { - LValue value = lowJSValue(m_node->child1()); - - LBasicBlock isCellBlock = FTL_NEW_BLOCK(m_out, ("Switch/SwitchString Untyped cell case")); - LBasicBlock isStringBlock = FTL_NEW_BLOCK(m_out, ("Switch/SwitchString Untyped string case")); - - m_out.branch( - isCell(value, provenType(m_node->child1())), - unsure(isCellBlock), unsure(lowBlock(data->fallThrough.block))); - - LBasicBlock lastNext = m_out.appendTo(isCellBlock, isStringBlock); - - m_out.branch( - isString(value, provenType(m_node->child1())), - unsure(isStringBlock), unsure(lowBlock(data->fallThrough.block))); - - m_out.appendTo(isStringBlock, lastNext); - - switchString(data, value); - return; - } - - default: - DFG_CRASH(m_graph, m_node, "Bad use kind"); - return; - } - return; + case SwitchString: + RELEASE_ASSERT_NOT_REACHED(); + break; } - - case SwitchCell: { - LValue cell; - switch (m_node->child1().useKind()) { - case CellUse: { - cell = lowCell(m_node->child1()); - break; - } - - case UntypedUse: { - LValue value = lowJSValue(m_node->child1()); - LBasicBlock cellCase = FTL_NEW_BLOCK(m_out, ("Switch/SwitchCell cell case")); - m_out.branch( - isCell(value, provenType(m_node->child1())), - unsure(cellCase), unsure(lowBlock(data->fallThrough.block))); - m_out.appendTo(cellCase); - cell = value; - break; - } - - default: - DFG_CRASH(m_graph, m_node, "Bad use kind"); - return; - } - - buildSwitch(m_node->switchData(), m_out.intPtr, cell); - return; - } } - DFG_CRASH(m_graph, m_node, "Bad switch kind"); + RELEASE_ASSERT_NOT_REACHED(); } void compileReturn() { + // FIXME: have a real epilogue when we switch to using our calling convention. + // https://bugs.webkit.org/show_bug.cgi?id=113621 m_out.ret(lowJSValue(m_node->child1())); } @@ -4692,21 +2686,15 @@ private: terminate(InadequateCoverage); } - void compileThrow() - { - terminate(Uncountable); - } - void compileInvalidationPoint() { if (verboseCompilationEnabled()) - dataLog(" Invalidation point with availability: ", availabilityMap(), "\n"); + dataLog(" Invalidation point with availability: ", m_availability, "\n"); m_ftlState.jitCode->osrExit.append(OSRExit( UncountableInvalidation, InvalidValueFormat, MethodOfGettingAValueProfile(), m_codeOriginForExitTarget, m_codeOriginForExitProfile, - availabilityMap().m_locals.numberOfArguments(), - availabilityMap().m_locals.numberOfLocals())); + m_availability.numberOfArguments(), m_availability.numberOfLocals())); m_ftlState.finalizer->osrExit.append(OSRExitCompilationInfo()); OSRExit& exit = m_ftlState.jitCode->osrExit.last(); @@ -4720,1022 +2708,11 @@ private: info.m_isInvalidationPoint = true; } - void compileIsUndefined() - { - setBoolean(equalNullOrUndefined(m_node->child1(), AllCellsAreFalse, EqualUndefined)); - } - - void compileIsBoolean() - { - setBoolean(isBoolean(lowJSValue(m_node->child1()), provenType(m_node->child1()))); - } - - void compileIsNumber() - { - setBoolean(isNumber(lowJSValue(m_node->child1()), provenType(m_node->child1()))); - } - - void compileIsString() - { - LValue value = lowJSValue(m_node->child1()); - - LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("IsString cell case")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("IsString continuation")); - - ValueFromBlock notCellResult = m_out.anchor(m_out.booleanFalse); - m_out.branch( - isCell(value, provenType(m_node->child1())), unsure(isCellCase), unsure(continuation)); - - LBasicBlock lastNext = m_out.appendTo(isCellCase, continuation); - ValueFromBlock cellResult = m_out.anchor(isString(value, provenType(m_node->child1()))); - m_out.jump(continuation); - - m_out.appendTo(continuation, lastNext); - setBoolean(m_out.phi(m_out.boolean, notCellResult, cellResult)); - } - - void compileIsObject() - { - LValue value = lowJSValue(m_node->child1()); - - LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("IsObject cell case")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("IsObject continuation")); - - ValueFromBlock notCellResult = m_out.anchor(m_out.booleanFalse); - m_out.branch( - isCell(value, provenType(m_node->child1())), unsure(isCellCase), unsure(continuation)); - - LBasicBlock lastNext = m_out.appendTo(isCellCase, continuation); - ValueFromBlock cellResult = m_out.anchor(isObject(value, provenType(m_node->child1()))); - m_out.jump(continuation); - - m_out.appendTo(continuation, lastNext); - setBoolean(m_out.phi(m_out.boolean, notCellResult, cellResult)); - } - - void compileIsObjectOrNull() - { - JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic); - - Edge child = m_node->child1(); - LValue value = lowJSValue(child); - - LBasicBlock cellCase = FTL_NEW_BLOCK(m_out, ("IsObjectOrNull cell case")); - LBasicBlock notFunctionCase = FTL_NEW_BLOCK(m_out, ("IsObjectOrNull not function case")); - LBasicBlock objectCase = FTL_NEW_BLOCK(m_out, ("IsObjectOrNull object case")); - LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("IsObjectOrNull slow path")); - LBasicBlock notCellCase = FTL_NEW_BLOCK(m_out, ("IsObjectOrNull not cell case")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("IsObjectOrNull continuation")); - - m_out.branch(isCell(value, provenType(child)), unsure(cellCase), unsure(notCellCase)); - - LBasicBlock lastNext = m_out.appendTo(cellCase, notFunctionCase); - ValueFromBlock isFunctionResult = m_out.anchor(m_out.booleanFalse); - m_out.branch( - isFunction(value, provenType(child)), - unsure(continuation), unsure(notFunctionCase)); - - m_out.appendTo(notFunctionCase, objectCase); - ValueFromBlock notObjectResult = m_out.anchor(m_out.booleanFalse); - m_out.branch( - isObject(value, provenType(child)), - unsure(objectCase), unsure(continuation)); - - m_out.appendTo(objectCase, slowPath); - ValueFromBlock objectResult = m_out.anchor(m_out.booleanTrue); - m_out.branch( - isExoticForTypeof(value, provenType(child)), - rarely(slowPath), usually(continuation)); - - m_out.appendTo(slowPath, notCellCase); - LValue slowResultValue = vmCall( - m_out.operation(operationObjectIsObject), m_callFrame, weakPointer(globalObject), - value); - ValueFromBlock slowResult = m_out.anchor(m_out.notNull(slowResultValue)); - m_out.jump(continuation); - - m_out.appendTo(notCellCase, continuation); - LValue notCellResultValue = m_out.equal(value, m_out.constInt64(JSValue::encode(jsNull()))); - ValueFromBlock notCellResult = m_out.anchor(notCellResultValue); - m_out.jump(continuation); - - m_out.appendTo(continuation, lastNext); - LValue result = m_out.phi( - m_out.boolean, - isFunctionResult, notObjectResult, objectResult, slowResult, notCellResult); - setBoolean(result); - } - - void compileIsFunction() - { - JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic); - - Edge child = m_node->child1(); - LValue value = lowJSValue(child); - - LBasicBlock cellCase = FTL_NEW_BLOCK(m_out, ("IsFunction cell case")); - LBasicBlock notFunctionCase = FTL_NEW_BLOCK(m_out, ("IsFunction not function case")); - LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("IsFunction slow path")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("IsFunction continuation")); - - ValueFromBlock notCellResult = m_out.anchor(m_out.booleanFalse); - m_out.branch( - isCell(value, provenType(child)), unsure(cellCase), unsure(continuation)); - - LBasicBlock lastNext = m_out.appendTo(cellCase, notFunctionCase); - ValueFromBlock functionResult = m_out.anchor(m_out.booleanTrue); - m_out.branch( - isFunction(value, provenType(child)), - unsure(continuation), unsure(notFunctionCase)); - - m_out.appendTo(notFunctionCase, slowPath); - ValueFromBlock objectResult = m_out.anchor(m_out.booleanFalse); - m_out.branch( - isExoticForTypeof(value, provenType(child)), - rarely(slowPath), usually(continuation)); - - m_out.appendTo(slowPath, continuation); - LValue slowResultValue = vmCall( - m_out.operation(operationObjectIsFunction), m_callFrame, weakPointer(globalObject), - value); - ValueFromBlock slowResult = m_out.anchor(m_out.notNull(slowResultValue)); - m_out.jump(continuation); - - m_out.appendTo(continuation, lastNext); - LValue result = m_out.phi( - m_out.boolean, notCellResult, functionResult, objectResult, slowResult); - setBoolean(result); - } - - void compileTypeOf() - { - Edge child = m_node->child1(); - LValue value = lowJSValue(child); - - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("TypeOf continuation")); - LBasicBlock lastNext = m_out.insertNewBlocksBefore(continuation); - - Vector<ValueFromBlock> results; - - buildTypeOf( - child, value, - [&] (TypeofType type) { - results.append(m_out.anchor(weakPointer(vm().smallStrings.typeString(type)))); - m_out.jump(continuation); - }); - - m_out.appendTo(continuation, lastNext); - setJSValue(m_out.phi(m_out.int64, results)); - } - - void compileIn() - { - Edge base = m_node->child2(); - LValue cell = lowCell(base); - speculateObject(base, cell); - if (JSString* string = m_node->child1()->dynamicCastConstant<JSString*>()) { - if (string->tryGetValueImpl() && string->tryGetValueImpl()->isAtomic()) { - - const auto str = static_cast<const AtomicStringImpl*>(string->tryGetValueImpl()); - unsigned stackmapID = m_stackmapIDs++; - - LValue call = m_out.call( - m_out.patchpointInt64Intrinsic(), - m_out.constInt64(stackmapID), m_out.constInt32(sizeOfIn()), - constNull(m_out.ref8), m_out.constInt32(1), cell); - - setInstructionCallingConvention(call, LLVMAnyRegCallConv); - - m_ftlState.checkIns.append(CheckInDescriptor(stackmapID, m_node->origin.semantic, str)); - setJSValue(call); - return; - } - } - - setJSValue(vmCall(m_out.operation(operationGenericIn), m_callFrame, cell, lowJSValue(m_node->child1()))); - } - - void compileCheckHasInstance() - { - speculate( - Uncountable, noValue(), 0, - m_out.testIsZero8( - m_out.load8(lowCell(m_node->child1()), m_heaps.JSCell_typeInfoFlags), - m_out.constInt8(ImplementsDefaultHasInstance))); - } - - void compileInstanceOf() - { - LValue cell; - - if (m_node->child1().useKind() == UntypedUse) - cell = lowJSValue(m_node->child1()); - else - cell = lowCell(m_node->child1()); - - LValue prototype = lowCell(m_node->child2()); - - LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("InstanceOf cell case")); - LBasicBlock loop = FTL_NEW_BLOCK(m_out, ("InstanceOf loop")); - LBasicBlock notYetInstance = FTL_NEW_BLOCK(m_out, ("InstanceOf not yet instance")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("InstanceOf continuation")); - - LValue condition; - if (m_node->child1().useKind() == UntypedUse) - condition = isCell(cell, provenType(m_node->child1())); - else - condition = m_out.booleanTrue; - - ValueFromBlock notCellResult = m_out.anchor(m_out.booleanFalse); - m_out.branch(condition, unsure(isCellCase), unsure(continuation)); - - LBasicBlock lastNext = m_out.appendTo(isCellCase, loop); - - speculate(BadType, noValue(), 0, isNotObject(prototype, provenType(m_node->child2()))); - - ValueFromBlock originalValue = m_out.anchor(cell); - m_out.jump(loop); - - m_out.appendTo(loop, notYetInstance); - LValue value = m_out.phi(m_out.int64, originalValue); - LValue structure = loadStructure(value); - LValue currentPrototype = m_out.load64(structure, m_heaps.Structure_prototype); - ValueFromBlock isInstanceResult = m_out.anchor(m_out.booleanTrue); - m_out.branch( - m_out.equal(currentPrototype, prototype), - unsure(continuation), unsure(notYetInstance)); - - m_out.appendTo(notYetInstance, continuation); - ValueFromBlock notInstanceResult = m_out.anchor(m_out.booleanFalse); - addIncoming(value, m_out.anchor(currentPrototype)); - m_out.branch(isCell(currentPrototype), unsure(loop), unsure(continuation)); - - m_out.appendTo(continuation, lastNext); - setBoolean( - m_out.phi(m_out.boolean, notCellResult, isInstanceResult, notInstanceResult)); - } - - void compileCountExecution() - { - TypedPointer counter = m_out.absolute(m_node->executionCounter()->address()); - m_out.store64(m_out.add(m_out.load64(counter), m_out.constInt64(1)), counter); - } - - void compileStoreBarrier() - { - emitStoreBarrier(lowCell(m_node->child1())); - } - - void compileHasIndexedProperty() - { - switch (m_node->arrayMode().type()) { - case Array::Int32: - case Array::Contiguous: { - LValue base = lowCell(m_node->child1()); - LValue index = lowInt32(m_node->child2()); - LValue storage = lowStorage(m_node->child3()); - - IndexedAbstractHeap& heap = m_node->arrayMode().type() == Array::Int32 ? - m_heaps.indexedInt32Properties : m_heaps.indexedContiguousProperties; - - LBasicBlock checkHole = FTL_NEW_BLOCK(m_out, ("HasIndexedProperty int/contiguous check hole")); - LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("HasIndexedProperty int/contiguous slow case")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("HasIndexedProperty int/contiguous continuation")); - - if (!m_node->arrayMode().isInBounds()) { - m_out.branch( - m_out.aboveOrEqual( - index, m_out.load32NonNegative(storage, m_heaps.Butterfly_publicLength)), - rarely(slowCase), usually(checkHole)); - } else - m_out.jump(checkHole); - - LBasicBlock lastNext = m_out.appendTo(checkHole, slowCase); - ValueFromBlock checkHoleResult = m_out.anchor( - m_out.notZero64(m_out.load64(baseIndex(heap, storage, index, m_node->child2())))); - m_out.branch(checkHoleResult.value(), usually(continuation), rarely(slowCase)); - - m_out.appendTo(slowCase, continuation); - ValueFromBlock slowResult = m_out.anchor(m_out.equal( - m_out.constInt64(JSValue::encode(jsBoolean(true))), - vmCall(m_out.operation(operationHasIndexedProperty), m_callFrame, base, index))); - m_out.jump(continuation); - - m_out.appendTo(continuation, lastNext); - setBoolean(m_out.phi(m_out.boolean, checkHoleResult, slowResult)); - return; - } - case Array::Double: { - LValue base = lowCell(m_node->child1()); - LValue index = lowInt32(m_node->child2()); - LValue storage = lowStorage(m_node->child3()); - - IndexedAbstractHeap& heap = m_heaps.indexedDoubleProperties; - - LBasicBlock checkHole = FTL_NEW_BLOCK(m_out, ("HasIndexedProperty double check hole")); - LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("HasIndexedProperty double slow case")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("HasIndexedProperty double continuation")); - - if (!m_node->arrayMode().isInBounds()) { - m_out.branch( - m_out.aboveOrEqual( - index, m_out.load32NonNegative(storage, m_heaps.Butterfly_publicLength)), - rarely(slowCase), usually(checkHole)); - } else - m_out.jump(checkHole); - - LBasicBlock lastNext = m_out.appendTo(checkHole, slowCase); - LValue doubleValue = m_out.loadDouble(baseIndex(heap, storage, index, m_node->child2())); - ValueFromBlock checkHoleResult = m_out.anchor(m_out.doubleEqual(doubleValue, doubleValue)); - m_out.branch(checkHoleResult.value(), usually(continuation), rarely(slowCase)); - - m_out.appendTo(slowCase, continuation); - ValueFromBlock slowResult = m_out.anchor(m_out.equal( - m_out.constInt64(JSValue::encode(jsBoolean(true))), - vmCall(m_out.operation(operationHasIndexedProperty), m_callFrame, base, index))); - m_out.jump(continuation); - - m_out.appendTo(continuation, lastNext); - setBoolean(m_out.phi(m_out.boolean, checkHoleResult, slowResult)); - return; - } - - default: - RELEASE_ASSERT_NOT_REACHED(); - return; - } - } - - void compileHasGenericProperty() - { - LValue base = lowJSValue(m_node->child1()); - LValue property = lowCell(m_node->child2()); - setJSValue(vmCall(m_out.operation(operationHasGenericProperty), m_callFrame, base, property)); - } - - void compileHasStructureProperty() - { - LValue base = lowJSValue(m_node->child1()); - LValue property = lowString(m_node->child2()); - LValue enumerator = lowCell(m_node->child3()); - - LBasicBlock correctStructure = FTL_NEW_BLOCK(m_out, ("HasStructureProperty correct structure")); - LBasicBlock wrongStructure = FTL_NEW_BLOCK(m_out, ("HasStructureProperty wrong structure")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("HasStructureProperty continuation")); - - m_out.branch(m_out.notEqual( - m_out.load32(base, m_heaps.JSCell_structureID), - m_out.load32(enumerator, m_heaps.JSPropertyNameEnumerator_cachedStructureID)), - rarely(wrongStructure), usually(correctStructure)); - - LBasicBlock lastNext = m_out.appendTo(correctStructure, wrongStructure); - ValueFromBlock correctStructureResult = m_out.anchor(m_out.booleanTrue); - m_out.jump(continuation); - - m_out.appendTo(wrongStructure, continuation); - ValueFromBlock wrongStructureResult = m_out.anchor( - m_out.equal( - m_out.constInt64(JSValue::encode(jsBoolean(true))), - vmCall(m_out.operation(operationHasGenericProperty), m_callFrame, base, property))); - m_out.jump(continuation); - - m_out.appendTo(continuation, lastNext); - setBoolean(m_out.phi(m_out.boolean, correctStructureResult, wrongStructureResult)); - } - - void compileGetDirectPname() - { - LValue base = lowCell(m_graph.varArgChild(m_node, 0)); - LValue property = lowCell(m_graph.varArgChild(m_node, 1)); - LValue index = lowInt32(m_graph.varArgChild(m_node, 2)); - LValue enumerator = lowCell(m_graph.varArgChild(m_node, 3)); - - LBasicBlock checkOffset = FTL_NEW_BLOCK(m_out, ("GetDirectPname check offset")); - LBasicBlock inlineLoad = FTL_NEW_BLOCK(m_out, ("GetDirectPname inline load")); - LBasicBlock outOfLineLoad = FTL_NEW_BLOCK(m_out, ("GetDirectPname out-of-line load")); - LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("GetDirectPname slow case")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("GetDirectPname continuation")); - - m_out.branch(m_out.notEqual( - m_out.load32(base, m_heaps.JSCell_structureID), - m_out.load32(enumerator, m_heaps.JSPropertyNameEnumerator_cachedStructureID)), - rarely(slowCase), usually(checkOffset)); - - LBasicBlock lastNext = m_out.appendTo(checkOffset, inlineLoad); - m_out.branch(m_out.aboveOrEqual(index, m_out.load32(enumerator, m_heaps.JSPropertyNameEnumerator_cachedInlineCapacity)), - unsure(outOfLineLoad), unsure(inlineLoad)); - - m_out.appendTo(inlineLoad, outOfLineLoad); - ValueFromBlock inlineResult = m_out.anchor( - m_out.load64(m_out.baseIndex(m_heaps.properties.atAnyNumber(), - base, m_out.zeroExt(index, m_out.int64), ScaleEight, JSObject::offsetOfInlineStorage()))); - m_out.jump(continuation); - - m_out.appendTo(outOfLineLoad, slowCase); - LValue storage = m_out.loadPtr(base, m_heaps.JSObject_butterfly); - LValue realIndex = m_out.signExt( - m_out.neg(m_out.sub(index, m_out.load32(enumerator, m_heaps.JSPropertyNameEnumerator_cachedInlineCapacity))), - m_out.int64); - int32_t offsetOfFirstProperty = static_cast<int32_t>(offsetInButterfly(firstOutOfLineOffset)) * sizeof(EncodedJSValue); - ValueFromBlock outOfLineResult = m_out.anchor( - m_out.load64(m_out.baseIndex(m_heaps.properties.atAnyNumber(), storage, realIndex, ScaleEight, offsetOfFirstProperty))); - m_out.jump(continuation); - - m_out.appendTo(slowCase, continuation); - ValueFromBlock slowCaseResult = m_out.anchor( - vmCall(m_out.operation(operationGetByVal), m_callFrame, base, property)); - m_out.jump(continuation); - - m_out.appendTo(continuation, lastNext); - setJSValue(m_out.phi(m_out.int64, inlineResult, outOfLineResult, slowCaseResult)); - } - - void compileGetEnumerableLength() - { - LValue enumerator = lowCell(m_node->child1()); - setInt32(m_out.load32(enumerator, m_heaps.JSPropertyNameEnumerator_indexLength)); - } - - void compileGetPropertyEnumerator() - { - LValue base = lowCell(m_node->child1()); - setJSValue(vmCall(m_out.operation(operationGetPropertyEnumerator), m_callFrame, base)); - } - - void compileGetEnumeratorStructurePname() - { - LValue enumerator = lowCell(m_node->child1()); - LValue index = lowInt32(m_node->child2()); - - LBasicBlock inBounds = FTL_NEW_BLOCK(m_out, ("GetEnumeratorStructurePname in bounds")); - LBasicBlock outOfBounds = FTL_NEW_BLOCK(m_out, ("GetEnumeratorStructurePname out of bounds")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("GetEnumeratorStructurePname continuation")); - - m_out.branch(m_out.below(index, m_out.load32(enumerator, m_heaps.JSPropertyNameEnumerator_endStructurePropertyIndex)), - usually(inBounds), rarely(outOfBounds)); - - LBasicBlock lastNext = m_out.appendTo(inBounds, outOfBounds); - LValue storage = m_out.loadPtr(enumerator, m_heaps.JSPropertyNameEnumerator_cachedPropertyNamesVector); - ValueFromBlock inBoundsResult = m_out.anchor( - m_out.loadPtr(m_out.baseIndex(m_heaps.JSPropertyNameEnumerator_cachedPropertyNamesVectorContents, storage, m_out.zeroExtPtr(index)))); - m_out.jump(continuation); - - m_out.appendTo(outOfBounds, continuation); - ValueFromBlock outOfBoundsResult = m_out.anchor(m_out.constInt64(ValueNull)); - m_out.jump(continuation); - - m_out.appendTo(continuation, lastNext); - setJSValue(m_out.phi(m_out.int64, inBoundsResult, outOfBoundsResult)); - } - - void compileGetEnumeratorGenericPname() - { - LValue enumerator = lowCell(m_node->child1()); - LValue index = lowInt32(m_node->child2()); - - LBasicBlock inBounds = FTL_NEW_BLOCK(m_out, ("GetEnumeratorGenericPname in bounds")); - LBasicBlock outOfBounds = FTL_NEW_BLOCK(m_out, ("GetEnumeratorGenericPname out of bounds")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("GetEnumeratorGenericPname continuation")); - - m_out.branch(m_out.below(index, m_out.load32(enumerator, m_heaps.JSPropertyNameEnumerator_endGenericPropertyIndex)), - usually(inBounds), rarely(outOfBounds)); - - LBasicBlock lastNext = m_out.appendTo(inBounds, outOfBounds); - LValue storage = m_out.loadPtr(enumerator, m_heaps.JSPropertyNameEnumerator_cachedPropertyNamesVector); - ValueFromBlock inBoundsResult = m_out.anchor( - m_out.loadPtr(m_out.baseIndex(m_heaps.JSPropertyNameEnumerator_cachedPropertyNamesVectorContents, storage, m_out.zeroExtPtr(index)))); - m_out.jump(continuation); - - m_out.appendTo(outOfBounds, continuation); - ValueFromBlock outOfBoundsResult = m_out.anchor(m_out.constInt64(ValueNull)); - m_out.jump(continuation); - - m_out.appendTo(continuation, lastNext); - setJSValue(m_out.phi(m_out.int64, inBoundsResult, outOfBoundsResult)); - } - - void compileToIndexString() - { - LValue index = lowInt32(m_node->child1()); - setJSValue(vmCall(m_out.operation(operationToIndexString), m_callFrame, index)); - } - - void compileCheckStructureImmediate() - { - LValue structure = lowCell(m_node->child1()); - checkStructure( - structure, noValue(), BadCache, m_node->structureSet(), - [this] (Structure* structure) { - return weakStructure(structure); - }); - } - - void compileMaterializeNewObject() - { - ObjectMaterializationData& data = m_node->objectMaterializationData(); - - // Lower the values first, to avoid creating values inside a control flow diamond. - - Vector<LValue, 8> values; - for (unsigned i = 0; i < data.m_properties.size(); ++i) - values.append(lowJSValue(m_graph.varArgChild(m_node, 1 + i))); - - const StructureSet& set = m_node->structureSet(); - - Vector<LBasicBlock, 1> blocks(set.size()); - for (unsigned i = set.size(); i--;) - blocks[i] = FTL_NEW_BLOCK(m_out, ("MaterializeNewObject case ", i)); - LBasicBlock dummyDefault = FTL_NEW_BLOCK(m_out, ("MaterializeNewObject default case")); - LBasicBlock outerContinuation = FTL_NEW_BLOCK(m_out, ("MaterializeNewObject continuation")); - - Vector<SwitchCase, 1> cases(set.size()); - for (unsigned i = set.size(); i--;) - cases[i] = SwitchCase(weakStructure(set[i]), blocks[i], Weight(1)); - m_out.switchInstruction( - lowCell(m_graph.varArgChild(m_node, 0)), cases, dummyDefault, Weight(0)); - - LBasicBlock outerLastNext = m_out.m_nextBlock; - - Vector<ValueFromBlock, 1> results; - - for (unsigned i = set.size(); i--;) { - m_out.appendTo(blocks[i], i + 1 < set.size() ? blocks[i + 1] : dummyDefault); - - Structure* structure = set[i]; - - LValue object; - LValue butterfly; - - if (structure->outOfLineCapacity()) { - size_t allocationSize = JSFinalObject::allocationSize(structure->inlineCapacity()); - MarkedAllocator* allocator = &vm().heap.allocatorForObjectWithoutDestructor(allocationSize); - - LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("MaterializeNewObject complex object allocation slow path")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("MaterializeNewObject complex object allocation continuation")); - - LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath); - - LValue endOfStorage = allocateBasicStorageAndGetEnd( - m_out.constIntPtr(structure->outOfLineCapacity() * sizeof(JSValue)), - slowPath); - - LValue fastButterflyValue = m_out.add( - m_out.constIntPtr(sizeof(IndexingHeader)), endOfStorage); - - LValue fastObjectValue = allocateObject( - m_out.constIntPtr(allocator), structure, fastButterflyValue, slowPath); - - ValueFromBlock fastObject = m_out.anchor(fastObjectValue); - ValueFromBlock fastButterfly = m_out.anchor(fastButterflyValue); - m_out.jump(continuation); - - m_out.appendTo(slowPath, continuation); - - ValueFromBlock slowObject = m_out.anchor(vmCall( - m_out.operation(operationNewObjectWithButterfly), - m_callFrame, m_out.constIntPtr(structure))); - ValueFromBlock slowButterfly = m_out.anchor( - m_out.loadPtr(slowObject.value(), m_heaps.JSObject_butterfly)); - - m_out.jump(continuation); - - m_out.appendTo(continuation, lastNext); - - object = m_out.phi(m_out.intPtr, fastObject, slowObject); - butterfly = m_out.phi(m_out.intPtr, fastButterfly, slowButterfly); - } else { - // In the easy case where we can do a one-shot allocation, we simply allocate the - // object to directly have the desired structure. - object = allocateObject(structure); - butterfly = nullptr; // Don't have one, don't need one. - } - - for (PropertyMapEntry entry : structure->getPropertiesConcurrently()) { - for (unsigned i = data.m_properties.size(); i--;) { - PhantomPropertyValue value = data.m_properties[i]; - if (m_graph.identifiers()[value.m_identifierNumber] != entry.key) - continue; - - LValue base = isInlineOffset(entry.offset) ? object : butterfly; - storeProperty(values[i], base, value.m_identifierNumber, entry.offset); - break; - } - } - - results.append(m_out.anchor(object)); - m_out.jump(outerContinuation); - } - - m_out.appendTo(dummyDefault, outerContinuation); - m_out.unreachable(); - - m_out.appendTo(outerContinuation, outerLastNext); - setJSValue(m_out.phi(m_out.intPtr, results)); - } - - void compileMaterializeCreateActivation() - { - ObjectMaterializationData& data = m_node->objectMaterializationData(); - - Vector<LValue, 8> values; - for (unsigned i = 0; i < data.m_properties.size(); ++i) - values.append(lowJSValue(m_graph.varArgChild(m_node, 2 + i))); - - LValue scope = lowCell(m_graph.varArgChild(m_node, 1)); - SymbolTable* table = m_node->castOperand<SymbolTable*>(); - ASSERT(table == m_graph.varArgChild(m_node, 0)->castConstant<SymbolTable*>()); - Structure* structure = m_graph.globalObjectFor(m_node->origin.semantic)->activationStructure(); - - LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("MaterializeCreateActivation slow path")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("MaterializeCreateActivation continuation")); - - LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath); - - LValue fastObject = allocateObject<JSLexicalEnvironment>( - JSLexicalEnvironment::allocationSize(table), structure, m_out.intPtrZero, slowPath); - - m_out.storePtr(scope, fastObject, m_heaps.JSScope_next); - m_out.storePtr(weakPointer(table), fastObject, m_heaps.JSSymbolTableObject_symbolTable); - - - ValueFromBlock fastResult = m_out.anchor(fastObject); - m_out.jump(continuation); - - m_out.appendTo(slowPath, continuation); - // We ensure allocation sinking explictly sets bottom values for all field members. - // Therefore, it doesn't matter what JSValue we pass in as the initialization value - // because all fields will be overwritten. - // FIXME: It may be worth creating an operation that calls a constructor on JSLexicalEnvironment that - // doesn't initialize every slot because we are guaranteed to do that here. - LValue callResult = vmCall( - m_out.operation(operationCreateActivationDirect), m_callFrame, weakPointer(structure), - scope, weakPointer(table), m_out.constInt64(JSValue::encode(jsUndefined()))); - ValueFromBlock slowResult = m_out.anchor(callResult); - m_out.jump(continuation); - - m_out.appendTo(continuation, lastNext); - LValue activation = m_out.phi(m_out.intPtr, fastResult, slowResult); - RELEASE_ASSERT(data.m_properties.size() == table->scopeSize()); - for (unsigned i = 0; i < data.m_properties.size(); ++i) { - m_out.store64(values[i], - activation, - m_heaps.JSEnvironmentRecord_variables[data.m_properties[i].m_identifierNumber]); - } - - if (validationEnabled()) { - // Validate to make sure every slot in the scope has one value. - ConcurrentJITLocker locker(table->m_lock); - for (auto iter = table->begin(locker), end = table->end(locker); iter != end; ++iter) { - bool found = false; - for (unsigned i = 0; i < data.m_properties.size(); ++i) { - if (iter->value.scopeOffset().offset() == data.m_properties[i].m_identifierNumber) { - found = true; - break; - } - } - ASSERT_UNUSED(found, found); - } - } - - setJSValue(activation); - } - - bool isInlinableSize(LValue function) - { - size_t instructionCount = 0; - size_t maxSize = Options::maximumLLVMInstructionCountForNativeInlining(); - for (LBasicBlock basicBlock = getFirstBasicBlock(function); basicBlock; basicBlock = getNextBasicBlock(basicBlock)) { - for (LValue instruction = getFirstInstruction(basicBlock); instruction; instruction = getNextInstruction(instruction)) { - if (++instructionCount >= maxSize) - return false; - } - } - return true; - } - - LValue didOverflowStack() - { - // This does a very simple leaf function analysis. The invariant of FTL call - // frames is that the caller had already done enough of a stack check to - // prove that this call frame has enough stack to run, and also enough stack - // to make runtime calls. So, we only need to stack check when making calls - // to other JS functions. If we don't find such calls then we don't need to - // do any stack checks. - - for (BlockIndex blockIndex = 0; blockIndex < m_graph.numBlocks(); ++blockIndex) { - BasicBlock* block = m_graph.block(blockIndex); - if (!block) - continue; - - for (unsigned nodeIndex = block->size(); nodeIndex--;) { - Node* node = block->at(nodeIndex); - - switch (node->op()) { - case GetById: - case PutById: - case Call: - case Construct: - return m_out.below( - m_callFrame, - m_out.loadPtr( - m_out.absolute(vm().addressOfFTLStackLimit()))); - - default: - break; - } - } - } - - return m_out.booleanFalse; - } - - struct ArgumentsLength { - ArgumentsLength() - : isKnown(false) - , known(UINT_MAX) - , value(nullptr) - { - } - - bool isKnown; - unsigned known; - LValue value; - }; - ArgumentsLength getArgumentsLength(InlineCallFrame* inlineCallFrame) - { - ArgumentsLength length; - - if (inlineCallFrame && !inlineCallFrame->isVarargs()) { - length.known = inlineCallFrame->arguments.size() - 1; - length.isKnown = true; - length.value = m_out.constInt32(length.known); - } else { - length.known = UINT_MAX; - length.isKnown = false; - - VirtualRegister argumentCountRegister; - if (!inlineCallFrame) - argumentCountRegister = VirtualRegister(JSStack::ArgumentCount); - else - argumentCountRegister = inlineCallFrame->argumentCountRegister; - length.value = m_out.sub(m_out.load32(payloadFor(argumentCountRegister)), m_out.int32One); - } - - return length; - } - - ArgumentsLength getArgumentsLength() - { - return getArgumentsLength(m_node->origin.semantic.inlineCallFrame); - } - - LValue getCurrentCallee() - { - if (InlineCallFrame* frame = m_node->origin.semantic.inlineCallFrame) { - if (frame->isClosureCall) - return m_out.loadPtr(addressFor(frame->calleeRecovery.virtualRegister())); - return weakPointer(frame->calleeRecovery.constant().asCell()); - } - return m_out.loadPtr(addressFor(JSStack::Callee)); - } - - LValue getArgumentsStart(InlineCallFrame* inlineCallFrame) - { - VirtualRegister start = AssemblyHelpers::argumentsStart(inlineCallFrame); - return addressFor(start).value(); - } - - LValue getArgumentsStart() - { - return getArgumentsStart(m_node->origin.semantic.inlineCallFrame); - } - - template<typename Functor> - void checkStructure( - LValue structureDiscriminant, const FormattedValue& formattedValue, ExitKind exitKind, - const StructureSet& set, const Functor& weakStructureDiscriminant) - { - if (set.size() == 1) { - speculate( - exitKind, formattedValue, 0, - m_out.notEqual(structureDiscriminant, weakStructureDiscriminant(set[0]))); - return; - } - - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("checkStructure continuation")); - - LBasicBlock lastNext = m_out.insertNewBlocksBefore(continuation); - for (unsigned i = 0; i < set.size() - 1; ++i) { - LBasicBlock nextStructure = FTL_NEW_BLOCK(m_out, ("checkStructure nextStructure")); - m_out.branch( - m_out.equal(structureDiscriminant, weakStructureDiscriminant(set[i])), - unsure(continuation), unsure(nextStructure)); - m_out.appendTo(nextStructure); - } - - speculate( - exitKind, formattedValue, 0, - m_out.notEqual(structureDiscriminant, weakStructureDiscriminant(set.last()))); - - m_out.jump(continuation); - m_out.appendTo(continuation, lastNext); - } - - LValue numberOrNotCellToInt32(Edge edge, LValue value) - { - LBasicBlock intCase = FTL_NEW_BLOCK(m_out, ("ValueToInt32 int case")); - LBasicBlock notIntCase = FTL_NEW_BLOCK(m_out, ("ValueToInt32 not int case")); - LBasicBlock doubleCase = 0; - LBasicBlock notNumberCase = 0; - if (edge.useKind() == NotCellUse) { - doubleCase = FTL_NEW_BLOCK(m_out, ("ValueToInt32 double case")); - notNumberCase = FTL_NEW_BLOCK(m_out, ("ValueToInt32 not number case")); - } - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ValueToInt32 continuation")); - - Vector<ValueFromBlock> results; - - m_out.branch(isNotInt32(value), unsure(notIntCase), unsure(intCase)); - - LBasicBlock lastNext = m_out.appendTo(intCase, notIntCase); - results.append(m_out.anchor(unboxInt32(value))); - m_out.jump(continuation); - - if (edge.useKind() == NumberUse) { - m_out.appendTo(notIntCase, continuation); - FTL_TYPE_CHECK(jsValueValue(value), edge, SpecBytecodeNumber, isCellOrMisc(value)); - results.append(m_out.anchor(doubleToInt32(unboxDouble(value)))); - m_out.jump(continuation); - } else { - m_out.appendTo(notIntCase, doubleCase); - m_out.branch( - isCellOrMisc(value, provenType(edge)), unsure(notNumberCase), unsure(doubleCase)); - - m_out.appendTo(doubleCase, notNumberCase); - results.append(m_out.anchor(doubleToInt32(unboxDouble(value)))); - m_out.jump(continuation); - - m_out.appendTo(notNumberCase, continuation); - - FTL_TYPE_CHECK(jsValueValue(value), edge, ~SpecCell, isCell(value)); - - LValue specialResult = m_out.select( - m_out.equal(value, m_out.constInt64(JSValue::encode(jsBoolean(true)))), - m_out.int32One, m_out.int32Zero); - results.append(m_out.anchor(specialResult)); - m_out.jump(continuation); - } - - m_out.appendTo(continuation, lastNext); - return m_out.phi(m_out.int32, results); - } - - LValue loadProperty(LValue storage, unsigned identifierNumber, PropertyOffset offset) - { - return m_out.load64(addressOfProperty(storage, identifierNumber, offset)); - } - - void storeProperty( - LValue value, LValue storage, unsigned identifierNumber, PropertyOffset offset) - { - m_out.store64(value, addressOfProperty(storage, identifierNumber, offset)); - } - - TypedPointer addressOfProperty( - LValue storage, unsigned identifierNumber, PropertyOffset offset) - { - return m_out.address( - m_heaps.properties[identifierNumber], storage, offsetRelativeToBase(offset)); - } - - LValue storageForTransition( - LValue object, PropertyOffset offset, - Structure* previousStructure, Structure* nextStructure) - { - if (isInlineOffset(offset)) - return object; - - if (previousStructure->outOfLineCapacity() == nextStructure->outOfLineCapacity()) - return m_out.loadPtr(object, m_heaps.JSObject_butterfly); - - LValue result; - if (!previousStructure->outOfLineCapacity()) - result = allocatePropertyStorage(object, previousStructure); - else { - result = reallocatePropertyStorage( - object, m_out.loadPtr(object, m_heaps.JSObject_butterfly), - previousStructure, nextStructure); - } - - emitStoreBarrier(object); - - return result; - } - - LValue allocatePropertyStorage(LValue object, Structure* previousStructure) - { - if (previousStructure->couldHaveIndexingHeader()) { - return vmCall( - m_out.operation( - operationReallocateButterflyToHavePropertyStorageWithInitialCapacity), - m_callFrame, object); - } - - LValue result = allocatePropertyStorageWithSizeImpl(initialOutOfLineCapacity); - m_out.storePtr(result, object, m_heaps.JSObject_butterfly); - return result; - } - - LValue reallocatePropertyStorage( - LValue object, LValue oldStorage, Structure* previous, Structure* next) - { - size_t oldSize = previous->outOfLineCapacity(); - size_t newSize = oldSize * outOfLineGrowthFactor; - - ASSERT_UNUSED(next, newSize == next->outOfLineCapacity()); - - if (previous->couldHaveIndexingHeader()) { - LValue newAllocSize = m_out.constIntPtr(newSize); - return vmCall(m_out.operation(operationReallocateButterflyToGrowPropertyStorage), m_callFrame, object, newAllocSize); - } - - LValue result = allocatePropertyStorageWithSizeImpl(newSize); - - ptrdiff_t headerSize = -sizeof(IndexingHeader) - sizeof(void*); - ptrdiff_t endStorage = headerSize - static_cast<ptrdiff_t>(oldSize * sizeof(JSValue)); - - for (ptrdiff_t offset = headerSize; offset > endStorage; offset -= sizeof(void*)) { - LValue loaded = - m_out.loadPtr(m_out.address(m_heaps.properties.atAnyNumber(), oldStorage, offset)); - m_out.storePtr(loaded, m_out.address(m_heaps.properties.atAnyNumber(), result, offset)); - } - - m_out.storePtr(result, m_out.address(object, m_heaps.JSObject_butterfly)); - - return result; - } - - LValue allocatePropertyStorageWithSizeImpl(size_t sizeInValues) - { - LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("allocatePropertyStorageWithSizeImpl slow path")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("allocatePropertyStorageWithSizeImpl continuation")); - - LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath); - - LValue endOfStorage = allocateBasicStorageAndGetEnd( - m_out.constIntPtr(sizeInValues * sizeof(JSValue)), slowPath); - - ValueFromBlock fastButterfly = m_out.anchor( - m_out.add(m_out.constIntPtr(sizeof(IndexingHeader)), endOfStorage)); - - m_out.jump(continuation); - - m_out.appendTo(slowPath, continuation); - - LValue slowButterflyValue; - if (sizeInValues == initialOutOfLineCapacity) { - slowButterflyValue = vmCall( - m_out.operation(operationAllocatePropertyStorageWithInitialCapacity), - m_callFrame); - } else { - slowButterflyValue = vmCall( - m_out.operation(operationAllocatePropertyStorage), - m_callFrame, m_out.constIntPtr(sizeInValues)); - } - ValueFromBlock slowButterfly = m_out.anchor(slowButterflyValue); - - m_out.jump(continuation); - - m_out.appendTo(continuation, lastNext); - - return m_out.phi(m_out.intPtr, fastButterfly, slowButterfly); - } - - LValue getById(LValue base) - { - auto uid = m_graph.identifiers()[m_node->identifierNumber()]; - - // Arguments: id, bytes, target, numArgs, args... - unsigned stackmapID = m_stackmapIDs++; - - if (Options::verboseCompilation()) - dataLog(" Emitting GetById patchpoint with stackmap #", stackmapID, "\n"); - - LValue call = m_out.call( - m_out.patchpointInt64Intrinsic(), - m_out.constInt64(stackmapID), m_out.constInt32(sizeOfGetById()), - constNull(m_out.ref8), m_out.constInt32(1), base); - setInstructionCallingConvention(call, LLVMAnyRegCallConv); - - m_ftlState.getByIds.append(GetByIdDescriptor(stackmapID, m_node->origin.semantic, uid)); - - return call; - } - - TypedPointer baseIndex(IndexedAbstractHeap& heap, LValue storage, LValue index, Edge edge, ptrdiff_t offset = 0) + TypedPointer baseIndex(IndexedAbstractHeap& heap, LValue storage, LValue index, Edge edge) { return m_out.baseIndex( - heap, storage, m_out.zeroExtPtr(index), provenValue(edge), offset); + heap, storage, m_out.zeroExt(index, m_out.intPtr), + m_state.forNode(edge).m_value); } void compare( @@ -5749,7 +2726,7 @@ private: return; } - if (m_node->isBinaryUseKind(Int52RepUse)) { + if (m_node->isBinaryUseKind(MachineIntUse)) { Int52Kind kind; LValue left = lowWhicheverInt52(m_node->child1(), kind); LValue right = lowInt52(m_node->child2(), kind); @@ -5757,7 +2734,7 @@ private: return; } - if (m_node->isBinaryUseKind(DoubleRepUse)) { + if (m_node->isBinaryUseKind(NumberUse)) { LValue left = lowDouble(m_node->child1()); LValue right = lowDouble(m_node->child2()); setBoolean(m_out.fcmp(realCondition, left, right)); @@ -5769,52 +2746,7 @@ private: return; } - DFG_CRASH(m_graph, m_node, "Bad use kinds"); - } - - void compareEqObjectOrOtherToObject(Edge leftChild, Edge rightChild) - { - LValue rightCell = lowCell(rightChild); - LValue leftValue = lowJSValue(leftChild, ManualOperandSpeculation); - - speculateTruthyObject(rightChild, rightCell, SpecObject); - - LBasicBlock leftCellCase = FTL_NEW_BLOCK(m_out, ("CompareEqObjectOrOtherToObject left cell case")); - LBasicBlock leftNotCellCase = FTL_NEW_BLOCK(m_out, ("CompareEqObjectOrOtherToObject left not cell case")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("CompareEqObjectOrOtherToObject continuation")); - - m_out.branch( - isCell(leftValue, provenType(leftChild)), - unsure(leftCellCase), unsure(leftNotCellCase)); - - LBasicBlock lastNext = m_out.appendTo(leftCellCase, leftNotCellCase); - speculateTruthyObject(leftChild, leftValue, SpecObject | (~SpecCell)); - ValueFromBlock cellResult = m_out.anchor(m_out.equal(rightCell, leftValue)); - m_out.jump(continuation); - - m_out.appendTo(leftNotCellCase, continuation); - FTL_TYPE_CHECK( - jsValueValue(leftValue), leftChild, SpecOther | SpecCell, isNotOther(leftValue)); - ValueFromBlock notCellResult = m_out.anchor(m_out.booleanFalse); - m_out.jump(continuation); - - m_out.appendTo(continuation, lastNext); - setBoolean(m_out.phi(m_out.boolean, cellResult, notCellResult)); - } - - void speculateTruthyObject(Edge edge, LValue cell, SpeculatedType filter) - { - if (masqueradesAsUndefinedWatchpointIsStillValid()) { - FTL_TYPE_CHECK(jsValueValue(cell), edge, filter, isNotObject(cell)); - return; - } - - FTL_TYPE_CHECK(jsValueValue(cell), edge, filter, isNotObject(cell)); - speculate( - BadType, jsValueValue(cell), edge.node(), - m_out.testNonZero8( - m_out.load8(cell, m_heaps.JSCell_typeInfoFlags), - m_out.constInt8(MasqueradesAsUndefined))); + RELEASE_ASSERT_NOT_REACHED(); } void nonSpeculativeCompare(LIntPredicate intCondition, S_JITOperation_EJJ helperFunction) @@ -5827,10 +2759,10 @@ private: LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("CompareEq untyped slow path")); LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("CompareEq untyped continuation")); - m_out.branch(isNotInt32(left), rarely(slowPath), usually(leftIsInt)); + m_out.branch(isNotInt32(left), slowPath, leftIsInt); LBasicBlock lastNext = m_out.appendTo(leftIsInt, fastPath); - m_out.branch(isNotInt32(right), rarely(slowPath), usually(fastPath)); + m_out.branch(isNotInt32(right), slowPath, fastPath); m_out.appendTo(fastPath, slowPath); ValueFromBlock fastResult = m_out.anchor( @@ -5845,15 +2777,15 @@ private: m_out.appendTo(continuation, lastNext); setBoolean(m_out.phi(m_out.boolean, fastResult, slowResult)); } - - LValue allocateCell(LValue allocator, LBasicBlock slowPath) + + LValue allocateCell(LValue allocator, LValue structure, LBasicBlock slowPath) { LBasicBlock success = FTL_NEW_BLOCK(m_out, ("object allocation success")); LValue result = m_out.loadPtr( allocator, m_heaps.MarkedAllocator_freeListHead); - m_out.branch(m_out.notNull(result), usually(success), rarely(slowPath)); + m_out.branch(m_out.notNull(result), success, slowPath); m_out.appendTo(success); @@ -5861,26 +2793,13 @@ private: m_out.loadPtr(result, m_heaps.JSCell_freeListNext), allocator, m_heaps.MarkedAllocator_freeListHead); + m_out.storePtr(structure, result, m_heaps.JSCell_structure); + return result; } - void storeStructure(LValue object, Structure* structure) - { - m_out.store32(m_out.constInt32(structure->id()), object, m_heaps.JSCell_structureID); - m_out.store32( - m_out.constInt32(structure->objectInitializationBlob()), - object, m_heaps.JSCell_usefulBytes); - } - - LValue allocateCell(LValue allocator, Structure* structure, LBasicBlock slowPath) - { - LValue result = allocateCell(allocator, slowPath); - storeStructure(result, structure); - return result; - } - LValue allocateObject( - LValue allocator, Structure* structure, LValue butterfly, LBasicBlock slowPath) + LValue allocator, LValue structure, LValue butterfly, LBasicBlock slowPath) { LValue result = allocateCell(allocator, structure, slowPath); m_out.storePtr(butterfly, result, m_heaps.JSObject_butterfly); @@ -5888,63 +2807,19 @@ private: } template<typename ClassType> - LValue allocateObject( - size_t size, Structure* structure, LValue butterfly, LBasicBlock slowPath) - { - MarkedAllocator* allocator = &vm().heap.allocatorForObjectOfType<ClassType>(size); + LValue allocateObject(LValue structure, LValue butterfly, LBasicBlock slowPath) + { + MarkedAllocator* allocator; + size_t size = ClassType::allocationSize(0); + if (ClassType::needsDestruction && ClassType::hasImmortalStructure) + allocator = &vm().heap.allocatorForObjectWithImmortalStructureDestructor(size); + else if (ClassType::needsDestruction) + allocator = &vm().heap.allocatorForObjectWithNormalDestructor(size); + else + allocator = &vm().heap.allocatorForObjectWithoutDestructor(size); return allocateObject(m_out.constIntPtr(allocator), structure, butterfly, slowPath); } - template<typename ClassType> - LValue allocateObject(Structure* structure, LValue butterfly, LBasicBlock slowPath) - { - return allocateObject<ClassType>( - ClassType::allocationSize(0), structure, butterfly, slowPath); - } - - template<typename ClassType> - LValue allocateVariableSizedObject( - LValue size, Structure* structure, LValue butterfly, LBasicBlock slowPath) - { - static_assert(!(MarkedSpace::preciseStep & (MarkedSpace::preciseStep - 1)), "MarkedSpace::preciseStep must be a power of two."); - static_assert(!(MarkedSpace::impreciseStep & (MarkedSpace::impreciseStep - 1)), "MarkedSpace::impreciseStep must be a power of two."); - - LValue subspace = m_out.constIntPtr(&vm().heap.subspaceForObjectOfType<ClassType>()); - - LBasicBlock smallCaseBlock = FTL_NEW_BLOCK(m_out, ("allocateVariableSizedObject small case")); - LBasicBlock largeOrOversizeCaseBlock = FTL_NEW_BLOCK(m_out, ("allocateVariableSizedObject large or oversize case")); - LBasicBlock largeCaseBlock = FTL_NEW_BLOCK(m_out, ("allocateVariableSizedObject large case")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("allocateVariableSizedObject continuation")); - - LValue uproundedSize = m_out.add(size, m_out.constInt32(MarkedSpace::preciseStep - 1)); - LValue isSmall = m_out.below(uproundedSize, m_out.constInt32(MarkedSpace::preciseCutoff)); - m_out.branch(isSmall, unsure(smallCaseBlock), unsure(largeOrOversizeCaseBlock)); - - LBasicBlock lastNext = m_out.appendTo(smallCaseBlock, largeOrOversizeCaseBlock); - TypedPointer address = m_out.baseIndex( - m_heaps.MarkedSpace_Subspace_preciseAllocators, subspace, - m_out.zeroExtPtr(m_out.lShr(uproundedSize, m_out.constInt32(getLSBSet(MarkedSpace::preciseStep))))); - ValueFromBlock smallAllocator = m_out.anchor(address.value()); - m_out.jump(continuation); - - m_out.appendTo(largeOrOversizeCaseBlock, largeCaseBlock); - m_out.branch( - m_out.below(uproundedSize, m_out.constInt32(MarkedSpace::impreciseCutoff)), - usually(largeCaseBlock), rarely(slowPath)); - - m_out.appendTo(largeCaseBlock, continuation); - address = m_out.baseIndex( - m_heaps.MarkedSpace_Subspace_impreciseAllocators, subspace, - m_out.zeroExtPtr(m_out.lShr(uproundedSize, m_out.constInt32(getLSBSet(MarkedSpace::impreciseStep))))); - ValueFromBlock largeAllocator = m_out.anchor(address.value()); - m_out.jump(continuation); - - m_out.appendTo(continuation, lastNext); - LValue allocator = m_out.phi(m_out.intPtr, smallAllocator, largeAllocator); - - return allocateObject(allocator, structure, butterfly, slowPath); - } - // Returns a pointer to the end of the allocation. LValue allocateBasicStorageAndGetEnd(LValue size, LBasicBlock slowPath) { @@ -5955,9 +2830,7 @@ private: LValue remaining = m_out.loadPtr(m_out.absolute(&allocator.m_currentRemaining)); LValue newRemaining = m_out.sub(remaining, size); - m_out.branch( - m_out.lessThan(newRemaining, m_out.intPtrZero), - rarely(slowPath), usually(success)); + m_out.branch(m_out.lessThan(newRemaining, m_out.intPtrZero), slowPath, success); m_out.appendTo(success); @@ -5966,31 +2839,6 @@ private: m_out.loadPtr(m_out.absolute(&allocator.m_currentPayloadEnd)), newRemaining); } - LValue allocateObject(Structure* structure) - { - size_t allocationSize = JSFinalObject::allocationSize(structure->inlineCapacity()); - MarkedAllocator* allocator = &vm().heap.allocatorForObjectWithoutDestructor(allocationSize); - - LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("allocateObject slow path")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("allocateObject continuation")); - - LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath); - - ValueFromBlock fastResult = m_out.anchor(allocateObject( - m_out.constIntPtr(allocator), structure, m_out.intPtrZero, slowPath)); - - m_out.jump(continuation); - - m_out.appendTo(slowPath, continuation); - - ValueFromBlock slowResult = m_out.anchor(vmCall( - m_out.operation(operationNewObject), m_callFrame, m_out.constIntPtr(structure))); - m_out.jump(continuation); - - m_out.appendTo(continuation, lastNext); - return m_out.phi(m_out.intPtr, fastResult, slowResult); - } - struct ArrayValues { ArrayValues() : array(0) @@ -6026,7 +2874,7 @@ private: endOfStorage, m_out.constIntPtr(sizeof(JSValue) * vectorLength)); LValue object = allocateObject<JSArray>( - structure, butterfly, slowPath); + m_out.constIntPtr(structure), butterfly, slowPath); m_out.store32(m_out.constInt32(numElements), butterfly, m_heaps.Butterfly_publicLength); m_out.store32(m_out.constInt32(vectorLength), butterfly, m_heaps.Butterfly_vectorLength); @@ -6034,7 +2882,7 @@ private: if (hasDouble(structure->indexingType())) { for (unsigned i = numElements; i < vectorLength; ++i) { m_out.store64( - m_out.constInt64(bitwise_cast<int64_t>(PNaN)), + m_out.constInt64(bitwise_cast<int64_t>(QNaN)), butterfly, m_heaps.indexedDoubleProperties[i]); } } @@ -6072,15 +2920,26 @@ private: m_out.phi(m_out.intPtr, fastButterfly, slowButterfly)); } + LValue typedArrayLength(Edge baseEdge, ArrayMode arrayMode, LValue base) + { + if (JSArrayBufferView* view = m_graph.tryGetFoldableView(baseEdge.node(), arrayMode)) + return m_out.constInt32(view->length()); + return m_out.load32(base, m_heaps.JSArrayBufferView_length); + } + + LValue typedArrayLength(Edge baseEdge, ArrayMode arrayMode) + { + return typedArrayLength(baseEdge, arrayMode, lowCell(baseEdge)); + } + LValue boolify(Edge edge) { switch (edge.useKind()) { case BooleanUse: - case KnownBooleanUse: - return lowBoolean(edge); + return lowBoolean(m_node->child1()); case Int32Use: - return m_out.notZero32(lowInt32(edge)); - case DoubleRepUse: + return m_out.notZero32(lowInt32(m_node->child1())); + case NumberUse: return m_out.doubleNotEqual(lowDouble(edge), m_out.doubleZero); case ObjectOrOtherUse: return m_out.bitNot( @@ -6088,111 +2947,33 @@ private: edge, CellCaseSpeculatesObject, SpeculateNullOrUndefined, ManualOperandSpeculation)); case StringUse: { - LValue stringValue = lowString(edge); - LValue length = m_out.load32NonNegative(stringValue, m_heaps.JSString_length); + LValue stringValue = lowString(m_node->child1()); + LValue length = m_out.load32(stringValue, m_heaps.JSString_length); return m_out.notEqual(length, m_out.int32Zero); } case UntypedUse: { - LValue value = lowJSValue(edge); - - // Implements the following control flow structure: - // if (value is cell) { - // if (value is string) - // result = !!value->length - // else { - // do evil things for masquerades-as-undefined - // result = true - // } - // } else if (value is int32) { - // result = !!unboxInt32(value) - // } else if (value is number) { - // result = !!unboxDouble(value) - // } else { - // result = value == jsTrue - // } - - LBasicBlock cellCase = FTL_NEW_BLOCK(m_out, ("Boolify untyped cell case")); - LBasicBlock stringCase = FTL_NEW_BLOCK(m_out, ("Boolify untyped string case")); - LBasicBlock notStringCase = FTL_NEW_BLOCK(m_out, ("Boolify untyped not string case")); - LBasicBlock notCellCase = FTL_NEW_BLOCK(m_out, ("Boolify untyped not cell case")); - LBasicBlock int32Case = FTL_NEW_BLOCK(m_out, ("Boolify untyped int32 case")); - LBasicBlock notInt32Case = FTL_NEW_BLOCK(m_out, ("Boolify untyped not int32 case")); - LBasicBlock doubleCase = FTL_NEW_BLOCK(m_out, ("Boolify untyped double case")); - LBasicBlock notDoubleCase = FTL_NEW_BLOCK(m_out, ("Boolify untyped not double case")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Boolify untyped continuation")); - - Vector<ValueFromBlock> results; - - m_out.branch(isCell(value, provenType(edge)), unsure(cellCase), unsure(notCellCase)); - - LBasicBlock lastNext = m_out.appendTo(cellCase, stringCase); - m_out.branch( - isString(value, provenType(edge) & SpecCell), - unsure(stringCase), unsure(notStringCase)); - - m_out.appendTo(stringCase, notStringCase); - LValue nonEmptyString = m_out.notZero32( - m_out.load32NonNegative(value, m_heaps.JSString_length)); - results.append(m_out.anchor(nonEmptyString)); - m_out.jump(continuation); - - m_out.appendTo(notStringCase, notCellCase); - LValue isTruthyObject; - if (masqueradesAsUndefinedWatchpointIsStillValid()) - isTruthyObject = m_out.booleanTrue; - else { - LBasicBlock masqueradesCase = FTL_NEW_BLOCK(m_out, ("Boolify untyped masquerades case")); - - results.append(m_out.anchor(m_out.booleanTrue)); - - m_out.branch( - m_out.testIsZero8( - m_out.load8(value, m_heaps.JSCell_typeInfoFlags), - m_out.constInt8(MasqueradesAsUndefined)), - usually(continuation), rarely(masqueradesCase)); - - m_out.appendTo(masqueradesCase); - - isTruthyObject = m_out.notEqual( - m_out.constIntPtr(m_graph.globalObjectFor(m_node->origin.semantic)), - m_out.loadPtr(loadStructure(value), m_heaps.Structure_globalObject)); - } - results.append(m_out.anchor(isTruthyObject)); - m_out.jump(continuation); + LValue value = lowJSValue(m_node->child1()); - m_out.appendTo(notCellCase, int32Case); - m_out.branch( - isInt32(value, provenType(edge) & ~SpecCell), - unsure(int32Case), unsure(notInt32Case)); + LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("Boolify untyped slow case")); + LBasicBlock fastCase = FTL_NEW_BLOCK(m_out, ("Boolify untyped fast case")); + LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Boolify untyped continuation")); - m_out.appendTo(int32Case, notInt32Case); - results.append(m_out.anchor(m_out.notZero32(unboxInt32(value)))); - m_out.jump(continuation); + m_out.branch(isNotBoolean(value), slowCase, fastCase); - m_out.appendTo(notInt32Case, doubleCase); - m_out.branch( - isNumber(value, provenType(edge) & ~SpecCell), - unsure(doubleCase), unsure(notDoubleCase)); - - m_out.appendTo(doubleCase, notDoubleCase); - // Note that doubleNotEqual() really means not-equal-and-ordered. It will return false - // if value is NaN. - LValue doubleIsTruthy = m_out.doubleNotEqual( - unboxDouble(value), m_out.constDouble(0)); - results.append(m_out.anchor(doubleIsTruthy)); + LBasicBlock lastNext = m_out.appendTo(fastCase, slowCase); + ValueFromBlock fastResult = m_out.anchor(unboxBoolean(value)); m_out.jump(continuation); - m_out.appendTo(notDoubleCase, continuation); - LValue miscIsTruthy = m_out.equal( - value, m_out.constInt64(JSValue::encode(jsBoolean(true)))); - results.append(m_out.anchor(miscIsTruthy)); + m_out.appendTo(slowCase, continuation); + ValueFromBlock slowResult = m_out.anchor(m_out.notNull(vmCall( + m_out.operation(operationConvertJSValueToBoolean), m_callFrame, value))); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); - return m_out.phi(m_out.boolean, results); + return m_out.phi(m_out.boolean, fastResult, slowResult); } default: - DFG_CRASH(m_graph, m_node, "Bad use kind"); + RELEASE_ASSERT_NOT_REACHED(); return 0; } } @@ -6219,7 +3000,7 @@ private: LBasicBlock primitiveCase = FTL_NEW_BLOCK(m_out, ("EqualNullOrUndefined primitive case")); LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("EqualNullOrUndefined continuation")); - m_out.branch(isNotCell(value, provenType(edge)), unsure(primitiveCase), unsure(cellCase)); + m_out.branch(isNotCell(value), primitiveCase, cellCase); LBasicBlock lastNext = m_out.appendTo(cellCase, primitiveCase); @@ -6230,7 +3011,10 @@ private: break; case CellCaseSpeculatesObject: FTL_TYPE_CHECK( - jsValueValue(value), edge, (~SpecCell) | SpecObject, isNotObject(value)); + jsValueValue(value), edge, (~SpecCell) | SpecObject, + m_out.equal( + m_out.loadPtr(value, m_heaps.JSCell_structure), + m_out.constIntPtr(vm().stringStructure.get()))); break; } @@ -6241,21 +3025,21 @@ private: LBasicBlock masqueradesCase = FTL_NEW_BLOCK(m_out, ("EqualNullOrUndefined masquerades case")); + LValue structure = m_out.loadPtr(value, m_heaps.JSCell_structure); + results.append(m_out.anchor(m_out.booleanFalse)); m_out.branch( m_out.testNonZero8( - m_out.load8(value, m_heaps.JSCell_typeInfoFlags), + m_out.load8(structure, m_heaps.Structure_typeInfoFlags), m_out.constInt8(MasqueradesAsUndefined)), - rarely(masqueradesCase), usually(continuation)); + masqueradesCase, continuation); m_out.appendTo(masqueradesCase, primitiveCase); - LValue structure = loadStructure(value); - results.append(m_out.anchor( m_out.equal( - m_out.constIntPtr(m_graph.globalObjectFor(m_node->origin.semantic)), + m_out.constIntPtr(m_graph.globalObjectFor(m_node->codeOrigin)), m_out.loadPtr(structure, m_heaps.Structure_globalObject)))); m_out.jump(continuation); } @@ -6271,11 +3055,16 @@ private: primitiveResult = m_out.equal(value, m_out.constInt64(ValueUndefined)); break; case EqualNullOrUndefined: - primitiveResult = isOther(value, provenType(edge)); + primitiveResult = m_out.equal( + m_out.bitAnd(value, m_out.constInt64(~TagBitUndefined)), + m_out.constInt64(ValueNull)); break; case SpeculateNullOrUndefined: FTL_TYPE_CHECK( - jsValueValue(value), edge, SpecCell | SpecOther, isNotOther(value)); + jsValueValue(value), edge, SpecCell | SpecOther, + m_out.notEqual( + m_out.bitAnd(value, m_out.constInt64(~TagBitUndefined)), + m_out.constInt64(ValueNull))); primitiveResult = m_out.booleanTrue; break; } @@ -6293,19 +3082,19 @@ private: LBasicBlock continuation) { LValue isNotInBounds = m_out.aboveOrEqual( - index, m_out.load32NonNegative(storage, m_heaps.Butterfly_publicLength)); + index, m_out.load32(storage, m_heaps.Butterfly_publicLength)); if (!m_node->arrayMode().isInBounds()) { LBasicBlock notInBoundsCase = FTL_NEW_BLOCK(m_out, ("PutByVal not in bounds")); LBasicBlock performStore = FTL_NEW_BLOCK(m_out, ("PutByVal perform store")); - m_out.branch(isNotInBounds, unsure(notInBoundsCase), unsure(performStore)); + m_out.branch(isNotInBounds, notInBoundsCase, performStore); LBasicBlock lastNext = m_out.appendTo(notInBoundsCase, performStore); LValue isOutOfBounds = m_out.aboveOrEqual( - index, m_out.load32NonNegative(storage, m_heaps.Butterfly_vectorLength)); + index, m_out.load32(storage, m_heaps.Butterfly_vectorLength)); if (!m_node->arrayMode().isOutOfBounds()) speculate(OutOfBounds, noValue(), 0, isOutOfBounds); @@ -6315,7 +3104,7 @@ private: LBasicBlock holeCase = FTL_NEW_BLOCK(m_out, ("PutByVal hole case")); - m_out.branch(isOutOfBounds, unsure(outOfBoundsCase), unsure(holeCase)); + m_out.branch(isOutOfBounds, outOfBoundsCase, holeCase); LBasicBlock innerLastNext = m_out.appendTo(outOfBoundsCase, holeCase); @@ -6342,445 +3131,18 @@ private: Vector<SwitchCase> cases; for (unsigned i = 0; i < data->cases.size(); ++i) { cases.append(SwitchCase( - constInt(type, data->cases[i].value.switchLookupValue(data->kind)), - lowBlock(data->cases[i].target.block), Weight(data->cases[i].target.count))); - } - - m_out.switchInstruction( - switchValue, cases, - lowBlock(data->fallThrough.block), Weight(data->fallThrough.count)); - } - - void switchString(SwitchData* data, LValue string) - { - bool canDoBinarySwitch = true; - unsigned totalLength = 0; - - for (DFG::SwitchCase myCase : data->cases) { - StringImpl* string = myCase.value.stringImpl(); - if (!string->is8Bit()) { - canDoBinarySwitch = false; - break; - } - if (string->length() > Options::maximumBinaryStringSwitchCaseLength()) { - canDoBinarySwitch = false; - break; - } - totalLength += string->length(); - } - - if (!canDoBinarySwitch || totalLength > Options::maximumBinaryStringSwitchTotalLength()) { - switchStringSlow(data, string); - return; - } - - LValue stringImpl = m_out.loadPtr(string, m_heaps.JSString_value); - LValue length = m_out.load32(string, m_heaps.JSString_length); - - LBasicBlock hasImplBlock = FTL_NEW_BLOCK(m_out, ("Switch/SwitchString has impl case")); - LBasicBlock is8BitBlock = FTL_NEW_BLOCK(m_out, ("Switch/SwitchString is 8 bit case")); - LBasicBlock slowBlock = FTL_NEW_BLOCK(m_out, ("Switch/SwitchString slow case")); - - m_out.branch(m_out.isNull(stringImpl), unsure(slowBlock), unsure(hasImplBlock)); - - LBasicBlock lastNext = m_out.appendTo(hasImplBlock, is8BitBlock); - - m_out.branch( - m_out.testIsZero32( - m_out.load32(stringImpl, m_heaps.StringImpl_hashAndFlags), - m_out.constInt32(StringImpl::flagIs8Bit())), - unsure(slowBlock), unsure(is8BitBlock)); - - m_out.appendTo(is8BitBlock, slowBlock); - - LValue buffer = m_out.loadPtr(stringImpl, m_heaps.StringImpl_data); - - // FIXME: We should propagate branch weight data to the cases of this switch. - // https://bugs.webkit.org/show_bug.cgi?id=144368 - - Vector<StringSwitchCase> cases; - for (DFG::SwitchCase myCase : data->cases) - cases.append(StringSwitchCase(myCase.value.stringImpl(), lowBlock(myCase.target.block))); - std::sort(cases.begin(), cases.end()); - switchStringRecurse(data, buffer, length, cases, 0, 0, cases.size(), 0, false); - - m_out.appendTo(slowBlock, lastNext); - switchStringSlow(data, string); - } - - // The code for string switching is based closely on the same code in the DFG backend. While it - // would be nice to reduce the amount of similar-looking code, it seems like this is one of - // those algorithms where factoring out the common bits would result in more code than just - // duplicating. - - struct StringSwitchCase { - StringSwitchCase() { } - - StringSwitchCase(StringImpl* string, LBasicBlock target) - : string(string) - , target(target) - { - } - - bool operator<(const StringSwitchCase& other) const - { - return stringLessThan(*string, *other.string); - } - - StringImpl* string; - LBasicBlock target; - }; - - struct CharacterCase { - CharacterCase() - : character(0) - , begin(0) - , end(0) - { - } - - CharacterCase(LChar character, unsigned begin, unsigned end) - : character(character) - , begin(begin) - , end(end) - { - } - - bool operator<(const CharacterCase& other) const - { - return character < other.character; - } - - LChar character; - unsigned begin; - unsigned end; - }; - - void switchStringRecurse( - SwitchData* data, LValue buffer, LValue length, const Vector<StringSwitchCase>& cases, - unsigned numChecked, unsigned begin, unsigned end, unsigned alreadyCheckedLength, - unsigned checkedExactLength) - { - LBasicBlock fallThrough = lowBlock(data->fallThrough.block); - - if (begin == end) { - m_out.jump(fallThrough); - return; - } - - unsigned minLength = cases[begin].string->length(); - unsigned commonChars = minLength; - bool allLengthsEqual = true; - for (unsigned i = begin + 1; i < end; ++i) { - unsigned myCommonChars = numChecked; - unsigned limit = std::min(cases[begin].string->length(), cases[i].string->length()); - for (unsigned j = numChecked; j < limit; ++j) { - if (cases[begin].string->at(j) != cases[i].string->at(j)) - break; - myCommonChars++; - } - commonChars = std::min(commonChars, myCommonChars); - if (minLength != cases[i].string->length()) - allLengthsEqual = false; - minLength = std::min(minLength, cases[i].string->length()); - } - - if (checkedExactLength) { - DFG_ASSERT(m_graph, m_node, alreadyCheckedLength == minLength); - DFG_ASSERT(m_graph, m_node, allLengthsEqual); - } - - DFG_ASSERT(m_graph, m_node, minLength >= commonChars); - - if (!allLengthsEqual && alreadyCheckedLength < minLength) - m_out.check(m_out.below(length, m_out.constInt32(minLength)), unsure(fallThrough)); - if (allLengthsEqual && (alreadyCheckedLength < minLength || !checkedExactLength)) - m_out.check(m_out.notEqual(length, m_out.constInt32(minLength)), unsure(fallThrough)); - - for (unsigned i = numChecked; i < commonChars; ++i) { - m_out.check( - m_out.notEqual( - m_out.load8(buffer, m_heaps.characters8[i]), - m_out.constInt8(cases[begin].string->at(i))), - unsure(fallThrough)); - } - - if (minLength == commonChars) { - // This is the case where one of the cases is a prefix of all of the other cases. - // We've already checked that the input string is a prefix of all of the cases, - // so we just check length to jump to that case. - - DFG_ASSERT(m_graph, m_node, cases[begin].string->length() == commonChars); - for (unsigned i = begin + 1; i < end; ++i) - DFG_ASSERT(m_graph, m_node, cases[i].string->length() > commonChars); - - if (allLengthsEqual) { - DFG_ASSERT(m_graph, m_node, end == begin + 1); - m_out.jump(cases[begin].target); - return; - } - - m_out.check( - m_out.equal(length, m_out.constInt32(commonChars)), - unsure(cases[begin].target)); - - // We've checked if the length is >= minLength, and then we checked if the length is - // == commonChars. We get to this point if it is >= minLength but not == commonChars. - // Hence we know that it now must be > minLength, i.e. that it's >= minLength + 1. - switchStringRecurse( - data, buffer, length, cases, commonChars, begin + 1, end, minLength + 1, false); - return; - } - - // At this point we know that the string is longer than commonChars, and we've only verified - // commonChars. Use a binary switch on the next unchecked character, i.e. - // string[commonChars]. - - DFG_ASSERT(m_graph, m_node, end >= begin + 2); - - LValue uncheckedChar = m_out.load8(buffer, m_heaps.characters8[commonChars]); - - Vector<CharacterCase> characterCases; - CharacterCase currentCase(cases[begin].string->at(commonChars), begin, begin + 1); - for (unsigned i = begin + 1; i < end; ++i) { - LChar currentChar = cases[i].string->at(commonChars); - if (currentChar != currentCase.character) { - currentCase.end = i; - characterCases.append(currentCase); - currentCase = CharacterCase(currentChar, i, i + 1); - } else - currentCase.end = i + 1; - } - characterCases.append(currentCase); - - Vector<LBasicBlock> characterBlocks; - for (CharacterCase& myCase : characterCases) - characterBlocks.append(FTL_NEW_BLOCK(m_out, ("Switch/SwitchString case for ", myCase.character, " at index ", commonChars))); - - Vector<SwitchCase> switchCases; - for (unsigned i = 0; i < characterCases.size(); ++i) { - if (i) - DFG_ASSERT(m_graph, m_node, characterCases[i - 1].character < characterCases[i].character); - switchCases.append(SwitchCase( - m_out.constInt8(characterCases[i].character), characterBlocks[i], Weight())); - } - m_out.switchInstruction(uncheckedChar, switchCases, fallThrough, Weight()); - - LBasicBlock lastNext = m_out.m_nextBlock; - characterBlocks.append(lastNext); // Makes it convenient to set nextBlock. - for (unsigned i = 0; i < characterCases.size(); ++i) { - m_out.appendTo(characterBlocks[i], characterBlocks[i + 1]); - switchStringRecurse( - data, buffer, length, cases, commonChars + 1, - characterCases[i].begin, characterCases[i].end, minLength, allLengthsEqual); - } - - DFG_ASSERT(m_graph, m_node, m_out.m_nextBlock == lastNext); - } - - void switchStringSlow(SwitchData* data, LValue string) - { - // FIXME: We ought to be able to use computed gotos here. We would save the labels of the - // blocks we want to jump to, and then request their addresses after compilation completes. - // https://bugs.webkit.org/show_bug.cgi?id=144369 - - LValue branchOffset = vmCall( - m_out.operation(operationSwitchStringAndGetBranchOffset), - m_callFrame, m_out.constIntPtr(data->switchTableIndex), string); - - StringJumpTable& table = codeBlock()->stringSwitchJumpTable(data->switchTableIndex); - - Vector<SwitchCase> cases; - std::unordered_set<int32_t> alreadyHandled; // These may be negative, or zero, or probably other stuff, too. We don't want to mess with HashSet's corner cases and we don't really care about throughput here. - for (unsigned i = 0; i < data->cases.size(); ++i) { - // FIXME: The fact that we're using the bytecode's switch table means that the - // following DFG IR transformation would be invalid. - // - // Original code: - // switch (v) { - // case "foo": - // case "bar": - // things(); - // break; - // default: - // break; - // } - // - // New code: - // switch (v) { - // case "foo": - // instrumentFoo(); - // goto _things; - // case "bar": - // instrumentBar(); - // _things: - // things(); - // break; - // default: - // break; - // } - // - // Luckily, we don't currently do any such transformation. But it's kind of silly that - // this is an issue. - // https://bugs.webkit.org/show_bug.cgi?id=144635 - - DFG::SwitchCase myCase = data->cases[i]; - StringJumpTable::StringOffsetTable::iterator iter = - table.offsetTable.find(myCase.value.stringImpl()); - DFG_ASSERT(m_graph, m_node, iter != table.offsetTable.end()); - - if (!alreadyHandled.insert(iter->value.branchOffset).second) - continue; - - cases.append(SwitchCase( - m_out.constInt32(iter->value.branchOffset), - lowBlock(myCase.target.block), Weight(myCase.target.count))); + constInt(type, data->cases[i].value.switchLookupValue()), + lowBlock(data->cases[i].target))); } - m_out.switchInstruction( - branchOffset, cases, lowBlock(data->fallThrough.block), - Weight(data->fallThrough.count)); - } - - // Calls the functor at the point of code generation where we know what the result type is. - // You can emit whatever code you like at that point. Expects you to terminate the basic block. - // When buildTypeOf() returns, it will have terminated all basic blocks that it created. So, if - // you aren't using this as the terminator of a high-level block, you should create your own - // contination and set it as the nextBlock (m_out.insertNewBlocksBefore(continuation)) before - // calling this. For example: - // - // LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("My continuation")); - // LBasicBlock lastNext = m_out.insertNewBlocksBefore(continuation); - // buildTypeOf( - // child, value, - // [&] (TypeofType type) { - // do things; - // m_out.jump(continuation); - // }); - // m_out.appendTo(continuation, lastNext); - template<typename Functor> - void buildTypeOf(Edge child, LValue value, const Functor& functor) - { - JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic); - - // Implements the following branching structure: - // - // if (is cell) { - // if (is object) { - // if (is function) { - // return function; - // } else if (doesn't have call trap and doesn't masquerade as undefined) { - // return object - // } else { - // return slowPath(); - // } - // } else if (is string) { - // return string - // } else { - // return symbol - // } - // } else if (is number) { - // return number - // } else if (is null) { - // return object - // } else if (is boolean) { - // return boolean - // } else { - // return undefined - // } - - LBasicBlock cellCase = FTL_NEW_BLOCK(m_out, ("buildTypeOf cell case")); - LBasicBlock objectCase = FTL_NEW_BLOCK(m_out, ("buildTypeOf object case")); - LBasicBlock functionCase = FTL_NEW_BLOCK(m_out, ("buildTypeOf function case")); - LBasicBlock notFunctionCase = FTL_NEW_BLOCK(m_out, ("buildTypeOf not function case")); - LBasicBlock reallyObjectCase = FTL_NEW_BLOCK(m_out, ("buildTypeOf really object case")); - LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("buildTypeOf slow path")); - LBasicBlock unreachable = FTL_NEW_BLOCK(m_out, ("buildTypeOf unreachable")); - LBasicBlock notObjectCase = FTL_NEW_BLOCK(m_out, ("buildTypeOf not object case")); - LBasicBlock stringCase = FTL_NEW_BLOCK(m_out, ("buildTypeOf string case")); - LBasicBlock symbolCase = FTL_NEW_BLOCK(m_out, ("buildTypeOf symbol case")); - LBasicBlock notCellCase = FTL_NEW_BLOCK(m_out, ("buildTypeOf not cell case")); - LBasicBlock numberCase = FTL_NEW_BLOCK(m_out, ("buildTypeOf number case")); - LBasicBlock notNumberCase = FTL_NEW_BLOCK(m_out, ("buildTypeOf not number case")); - LBasicBlock notNullCase = FTL_NEW_BLOCK(m_out, ("buildTypeOf not null case")); - LBasicBlock booleanCase = FTL_NEW_BLOCK(m_out, ("buildTypeOf boolean case")); - LBasicBlock undefinedCase = FTL_NEW_BLOCK(m_out, ("buildTypeOf undefined case")); - - m_out.branch(isCell(value, provenType(child)), unsure(cellCase), unsure(notCellCase)); - - LBasicBlock lastNext = m_out.appendTo(cellCase, objectCase); - m_out.branch(isObject(value, provenType(child)), unsure(objectCase), unsure(notObjectCase)); - - m_out.appendTo(objectCase, functionCase); - m_out.branch( - isFunction(value, provenType(child) & SpecObject), - unsure(functionCase), unsure(notFunctionCase)); - - m_out.appendTo(functionCase, notFunctionCase); - functor(TypeofType::Function); - - m_out.appendTo(notFunctionCase, reallyObjectCase); - m_out.branch( - isExoticForTypeof(value, provenType(child) & (SpecObject - SpecFunction)), - rarely(slowPath), usually(reallyObjectCase)); - - m_out.appendTo(reallyObjectCase, slowPath); - functor(TypeofType::Object); - - m_out.appendTo(slowPath, unreachable); - LValue result = vmCall( - m_out.operation(operationTypeOfObjectAsTypeofType), m_callFrame, - weakPointer(globalObject), value); - Vector<SwitchCase, 3> cases; - cases.append(SwitchCase(m_out.constInt32(static_cast<int32_t>(TypeofType::Undefined)), undefinedCase)); - cases.append(SwitchCase(m_out.constInt32(static_cast<int32_t>(TypeofType::Object)), reallyObjectCase)); - cases.append(SwitchCase(m_out.constInt32(static_cast<int32_t>(TypeofType::Function)), functionCase)); - m_out.switchInstruction(result, cases, unreachable, Weight()); - - m_out.appendTo(unreachable, notObjectCase); - m_out.unreachable(); - - m_out.appendTo(notObjectCase, stringCase); - m_out.branch( - isString(value, provenType(child) & (SpecCell - SpecObject)), - unsure(stringCase), unsure(symbolCase)); - - m_out.appendTo(stringCase, symbolCase); - functor(TypeofType::String); - - m_out.appendTo(symbolCase, notCellCase); - functor(TypeofType::Symbol); - - m_out.appendTo(notCellCase, numberCase); - m_out.branch( - isNumber(value, provenType(child) & ~SpecCell), - unsure(numberCase), unsure(notNumberCase)); - - m_out.appendTo(numberCase, notNumberCase); - functor(TypeofType::Number); - - m_out.appendTo(notNumberCase, notNullCase); - LValue isNull; - if (provenType(child) & SpecOther) - isNull = m_out.equal(value, m_out.constInt64(ValueNull)); - else - isNull = m_out.booleanFalse; - m_out.branch(isNull, unsure(reallyObjectCase), unsure(notNullCase)); - - m_out.appendTo(notNullCase, booleanCase); - m_out.branch( - isBoolean(value, provenType(child) & ~(SpecCell | SpecFullNumber)), - unsure(booleanCase), unsure(undefinedCase)); - - m_out.appendTo(booleanCase, undefinedCase); - functor(TypeofType::Boolean); - - m_out.appendTo(undefinedCase, lastNext); - functor(TypeofType::Undefined); + m_out.switchInstruction(switchValue, cases, lowBlock(data->fallThrough)); } LValue doubleToInt32(LValue doubleValue, double low, double high, bool isSigned = true) { + // FIXME: Optimize double-to-int conversions. + // <rdar://problem/14938465> + LBasicBlock greatEnough = FTL_NEW_BLOCK(m_out, ("doubleToInt32 greatEnough")); LBasicBlock withinRange = FTL_NEW_BLOCK(m_out, ("doubleToInt32 withinRange")); LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("doubleToInt32 slowPath")); @@ -6790,12 +3152,12 @@ private: m_out.branch( m_out.doubleGreaterThanOrEqual(doubleValue, m_out.constDouble(low)), - unsure(greatEnough), unsure(slowPath)); + greatEnough, slowPath); LBasicBlock lastNext = m_out.appendTo(greatEnough, withinRange); m_out.branch( m_out.doubleLessThanOrEqual(doubleValue, m_out.constDouble(high)), - unsure(withinRange), unsure(slowPath)); + withinRange, slowPath); m_out.appendTo(withinRange, slowPath); LValue fastResult; @@ -6832,7 +3194,7 @@ private: m_out.sensibleDoubleToInt(doubleValue)); m_out.branch( m_out.equal(fastResult.value(), m_out.constInt32(0x80000000)), - rarely(slowPath), usually(continuation)); + slowPath, continuation); LBasicBlock lastNext = m_out.appendTo(slowPath, continuation); ValueFromBlock slowResult = m_out.anchor( @@ -6851,13 +3213,7 @@ private: void terminate(ExitKind kind) { - speculate(kind, noValue(), nullptr, m_out.booleanTrue); - didAlreadyTerminate(); - } - - void didAlreadyTerminate() - { - m_state.setIsValid(false); + speculate(kind, noValue(), 0, m_out.booleanTrue); } void typeCheck( @@ -6883,7 +3239,7 @@ private: ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || (edge.useKind() == Int32Use || edge.useKind() == KnownInt32Use)); if (edge->hasConstant()) { - JSValue value = edge->asJSValue(); + JSValue value = m_graph.valueOfJSConstant(edge.node()); if (!value.isInt32()) { terminate(Uncountable); return m_out.int32Zero; @@ -6913,15 +3269,27 @@ private: return result; } - DFG_ASSERT(m_graph, m_node, !(provenType(edge) & SpecInt32)); + RELEASE_ASSERT(!(m_state.forNode(edge).m_type & SpecInt32)); terminate(Uncountable); return m_out.int32Zero; } enum Int52Kind { StrictInt52, Int52 }; - LValue lowInt52(Edge edge, Int52Kind kind) + LValue lowInt52(Edge edge, Int52Kind kind, OperandSpeculationMode mode = AutomaticOperandSpeculation) { - DFG_ASSERT(m_graph, m_node, edge.useKind() == Int52RepUse); + ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == MachineIntUse); + + if (edge->hasConstant()) { + JSValue value = m_graph.valueOfJSConstant(edge.node()); + if (!value.isMachineInt()) { + terminate(Uncountable); + return m_out.int64Zero; + } + int64_t result = value.asMachineInt(); + if (kind == Int52) + result <<= JSValue::int52ShiftAmount; + return m_out.constInt64(result); + } LoweredNodeValue value; @@ -6946,20 +3314,37 @@ private: return int52ToStrictInt52(value.value()); break; } - - DFG_ASSERT(m_graph, m_node, !provenType(edge)); + + value = m_int32Values.get(edge.node()); + if (isValid(value)) { + return setInt52WithStrictValue( + edge.node(), m_out.signExt(value.value(), m_out.int64), kind); + } + + RELEASE_ASSERT(!(m_state.forNode(edge).m_type & SpecInt52)); + + value = m_jsValueValues.get(edge.node()); + if (isValid(value)) { + LValue boxedResult = value.value(); + FTL_TYPE_CHECK( + jsValueValue(boxedResult), edge, SpecMachineInt, isNotInt32(boxedResult)); + return setInt52WithStrictValue( + edge.node(), m_out.signExt(unboxInt32(boxedResult), m_out.int64), kind); + } + + RELEASE_ASSERT(!(m_state.forNode(edge).m_type & SpecMachineInt)); terminate(Uncountable); return m_out.int64Zero; } - LValue lowInt52(Edge edge) + LValue lowInt52(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation) { - return lowInt52(edge, Int52); + return lowInt52(edge, Int52, mode); } - LValue lowStrictInt52(Edge edge) + LValue lowStrictInt52(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation) { - return lowInt52(edge, StrictInt52); + return lowInt52(edge, StrictInt52, mode); } bool betterUseStrictInt52(Node* node) @@ -6983,22 +3368,21 @@ private: case StrictInt52: return Int52; } - DFG_CRASH(m_graph, m_node, "Bad use kind"); - return Int52; + RELEASE_ASSERT_NOT_REACHED(); } - LValue lowWhicheverInt52(Edge edge, Int52Kind& kind) + LValue lowWhicheverInt52(Edge edge, Int52Kind& kind, OperandSpeculationMode mode = AutomaticOperandSpeculation) { kind = bestInt52Kind(edge); - return lowInt52(edge, kind); + return lowInt52(edge, kind, mode); } LValue lowCell(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation) { - DFG_ASSERT(m_graph, m_node, mode == ManualOperandSpeculation || DFG::isCell(edge.useKind())); + ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || DFG::isCell(edge.useKind())); if (edge->op() == JSConstant) { - JSValue value = edge->asJSValue(); + JSValue value = m_graph.valueOfJSConstant(edge.node()); if (!value.isCell()) { terminate(Uncountable); return m_out.intPtrZero; @@ -7014,7 +3398,7 @@ private: return uncheckedValue; } - DFG_ASSERT(m_graph, m_node, !(provenType(edge) & SpecCell)); + RELEASE_ASSERT(!(m_state.forNode(edge).m_type & SpecCell)); terminate(Uncountable); return m_out.intPtrZero; } @@ -7030,32 +3414,13 @@ private: LValue lowString(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation) { - ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == StringUse || edge.useKind() == KnownStringUse || edge.useKind() == StringIdentUse); + ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == StringUse || edge.useKind() == KnownStringUse); LValue result = lowCell(edge, mode); speculateString(edge, result); return result; } - LValue lowStringIdent(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation) - { - ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == StringIdentUse); - - LValue string = lowString(edge, mode); - LValue stringImpl = m_out.loadPtr(string, m_heaps.JSString_value); - speculateStringIdent(edge, string, stringImpl); - return stringImpl; - } - - LValue lowSymbol(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation) - { - ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == SymbolUse); - - LValue result = lowCell(edge, mode); - speculateSymbol(edge, result); - return result; - } - LValue lowNonNullObject(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation) { ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == ObjectUse); @@ -7067,10 +3432,10 @@ private: LValue lowBoolean(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation) { - ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == BooleanUse || edge.useKind() == KnownBooleanUse); + ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == BooleanUse); if (edge->hasConstant()) { - JSValue value = edge->asJSValue(); + JSValue value = m_graph.valueOfJSConstant(edge.node()); if (!value.isBoolean()) { terminate(Uncountable); return m_out.booleanFalse; @@ -7092,32 +3457,87 @@ private: return result; } - DFG_ASSERT(m_graph, m_node, !(provenType(edge) & SpecBoolean)); + RELEASE_ASSERT(!(m_state.forNode(edge).m_type & SpecBoolean)); terminate(Uncountable); return m_out.booleanFalse; } - LValue lowDouble(Edge edge) + LValue lowDouble(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation) { - DFG_ASSERT(m_graph, m_node, isDouble(edge.useKind())); + ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || isDouble(edge.useKind())); + + if (edge->hasConstant()) { + JSValue value = m_graph.valueOfJSConstant(edge.node()); + if (!value.isNumber()) { + terminate(Uncountable); + return m_out.doubleZero; + } + return m_out.constDouble(value.asNumber()); + } LoweredNodeValue value = m_doubleValues.get(edge.node()); if (isValid(value)) return value.value(); - DFG_ASSERT(m_graph, m_node, !provenType(edge)); + + value = m_int32Values.get(edge.node()); + if (isValid(value)) { + LValue result = m_out.intToDouble(value.value()); + setDouble(edge.node(), result); + return result; + } + + value = m_strictInt52Values.get(edge.node()); + if (isValid(value)) + return strictInt52ToDouble(edge, value.value()); + + value = m_int52Values.get(edge.node()); + if (isValid(value)) + return strictInt52ToDouble(edge, int52ToStrictInt52(value.value())); + + value = m_jsValueValues.get(edge.node()); + if (isValid(value)) { + LValue boxedResult = value.value(); + + LBasicBlock intCase = FTL_NEW_BLOCK(m_out, ("Double unboxing int case")); + LBasicBlock doubleCase = FTL_NEW_BLOCK(m_out, ("Double unboxing double case")); + LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Double unboxing continuation")); + + m_out.branch(isNotInt32(boxedResult), doubleCase, intCase); + + LBasicBlock lastNext = m_out.appendTo(intCase, doubleCase); + + ValueFromBlock intToDouble = m_out.anchor( + m_out.intToDouble(unboxInt32(boxedResult))); + m_out.jump(continuation); + + m_out.appendTo(doubleCase, continuation); + + FTL_TYPE_CHECK( + jsValueValue(boxedResult), edge, SpecFullNumber, isCellOrMisc(boxedResult)); + + ValueFromBlock unboxedDouble = m_out.anchor(unboxDouble(boxedResult)); + m_out.jump(continuation); + + m_out.appendTo(continuation, lastNext); + + LValue result = m_out.phi(m_out.doubleType, intToDouble, unboxedDouble); + + setDouble(edge.node(), result); + return result; + } + + RELEASE_ASSERT(!(m_state.forNode(edge).m_type & SpecFullNumber)); terminate(Uncountable); return m_out.doubleZero; } LValue lowJSValue(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation) { - DFG_ASSERT(m_graph, m_node, mode == ManualOperandSpeculation || edge.useKind() == UntypedUse); - DFG_ASSERT(m_graph, m_node, !isDouble(edge.useKind())); - DFG_ASSERT(m_graph, m_node, edge.useKind() != Int52RepUse); + ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == UntypedUse); if (edge->hasConstant()) - return m_out.constInt64(JSValue::encode(edge->asJSValue())); - + return m_out.constInt64(JSValue::encode(m_graph.valueOfJSConstant(edge.node()))); + LoweredNodeValue value = m_jsValueValues.get(edge.node()); if (isValid(value)) return value.value(); @@ -7129,6 +3549,14 @@ private: return result; } + value = m_strictInt52Values.get(edge.node()); + if (isValid(value)) + return strictInt52ToJSValue(value.value()); + + value = m_int52Values.get(edge.node()); + if (isValid(value)) + return strictInt52ToJSValue(int52ToStrictInt52(value.value())); + value = m_booleanValues.get(edge.node()); if (isValid(value)) { LValue result = boxBoolean(value.value()); @@ -7136,7 +3564,14 @@ private: return result; } - DFG_CRASH(m_graph, m_node, "Value not defined"); + value = m_doubleValues.get(edge.node()); + if (isValid(value)) { + LValue result = boxDouble(value.value()); + setJSValue(edge.node(), result); + return result; + } + + RELEASE_ASSERT_NOT_REACHED(); return 0; } @@ -7161,9 +3596,11 @@ private: return result; } - LValue strictInt52ToDouble(LValue value) + LValue strictInt52ToDouble(Edge edge, LValue value) { - return m_out.intToDouble(value); + LValue result = m_out.intToDouble(value); + setDouble(edge.node(), result); + return result; } LValue strictInt52ToJSValue(LValue value) @@ -7177,7 +3614,7 @@ private: LValue int32Value = m_out.castToInt32(value); m_out.branch( m_out.equal(m_out.signExt(int32Value, m_out.int64), value), - unsure(isInt32), unsure(isDouble)); + isInt32, isDouble); LBasicBlock lastNext = m_out.appendTo(isInt32, isDouble); @@ -7193,6 +3630,23 @@ private: return m_out.phi(m_out.int64, results); } + LValue setInt52WithStrictValue(Node* node, LValue value, Int52Kind kind) + { + switch (kind) { + case StrictInt52: + setStrictInt52(node, value); + return value; + + case Int52: + value = strictInt52ToInt52(value); + setInt52(node, value); + return value; + } + + RELEASE_ASSERT_NOT_REACHED(); + return 0; + } + LValue strictInt52ToInt52(LValue value) { return m_out.shl(value, m_out.constInt64(JSValue::int52ShiftAmount)); @@ -7203,16 +3657,8 @@ private: return m_out.aShr(value, m_out.constInt64(JSValue::int52ShiftAmount)); } - LValue isInt32(LValue jsValue, SpeculatedType type = SpecFullTop) + LValue isNotInt32(LValue jsValue) { - if (LValue proven = isProvenValue(type, SpecInt32)) - return proven; - return m_out.aboveOrEqual(jsValue, m_tagTypeNumber); - } - LValue isNotInt32(LValue jsValue, SpeculatedType type = SpecFullTop) - { - if (LValue proven = isProvenValue(type, ~SpecInt32)) - return proven; return m_out.below(jsValue, m_tagTypeNumber); } LValue unboxInt32(LValue jsValue) @@ -7224,19 +3670,10 @@ private: return m_out.add(m_out.zeroExt(value, m_out.int64), m_tagTypeNumber); } - LValue isCellOrMisc(LValue jsValue, SpeculatedType type = SpecFullTop) + LValue isCellOrMisc(LValue jsValue) { - if (LValue proven = isProvenValue(type, SpecCell | SpecMisc)) - return proven; return m_out.testIsZero64(jsValue, m_tagTypeNumber); } - LValue isNotCellOrMisc(LValue jsValue, SpeculatedType type = SpecFullTop) - { - if (LValue proven = isProvenValue(type, ~(SpecCell | SpecMisc))) - return proven; - return m_out.testNonZero64(jsValue, m_tagTypeNumber); - } - LValue unboxDouble(LValue jsValue) { return m_out.bitCast(m_out.add(jsValue, m_tagTypeNumber), m_out.doubleType); @@ -7246,133 +3683,22 @@ private: return m_out.sub(m_out.bitCast(doubleValue, m_out.int64), m_tagTypeNumber); } - LValue jsValueToStrictInt52(Edge edge, LValue boxedValue) - { - LBasicBlock intCase = FTL_NEW_BLOCK(m_out, ("jsValueToInt52 unboxing int case")); - LBasicBlock doubleCase = FTL_NEW_BLOCK(m_out, ("jsValueToInt52 unboxing double case")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("jsValueToInt52 unboxing continuation")); - - LValue isNotInt32; - if (!m_interpreter.needsTypeCheck(edge, SpecInt32)) - isNotInt32 = m_out.booleanFalse; - else if (!m_interpreter.needsTypeCheck(edge, ~SpecInt32)) - isNotInt32 = m_out.booleanTrue; - else - isNotInt32 = this->isNotInt32(boxedValue); - m_out.branch(isNotInt32, unsure(doubleCase), unsure(intCase)); - - LBasicBlock lastNext = m_out.appendTo(intCase, doubleCase); - - ValueFromBlock intToInt52 = m_out.anchor( - m_out.signExt(unboxInt32(boxedValue), m_out.int64)); - m_out.jump(continuation); - - m_out.appendTo(doubleCase, continuation); - - LValue possibleResult = m_out.call( - m_out.operation(operationConvertBoxedDoubleToInt52), boxedValue); - FTL_TYPE_CHECK( - jsValueValue(boxedValue), edge, SpecInt32 | SpecInt52AsDouble, - m_out.equal(possibleResult, m_out.constInt64(JSValue::notInt52))); - - ValueFromBlock doubleToInt52 = m_out.anchor(possibleResult); - m_out.jump(continuation); - - m_out.appendTo(continuation, lastNext); - - return m_out.phi(m_out.int64, intToInt52, doubleToInt52); - } - - LValue doubleToStrictInt52(Edge edge, LValue value) - { - LValue possibleResult = m_out.call( - m_out.operation(operationConvertDoubleToInt52), value); - FTL_TYPE_CHECK( - doubleValue(value), edge, SpecInt52AsDouble, - m_out.equal(possibleResult, m_out.constInt64(JSValue::notInt52))); - - return possibleResult; - } - - LValue convertDoubleToInt32(LValue value, bool shouldCheckNegativeZero) + LValue isNotCell(LValue jsValue) { - LValue integerValue = m_out.fpToInt32(value); - LValue integerValueConvertedToDouble = m_out.intToDouble(integerValue); - LValue valueNotConvertibleToInteger = m_out.doubleNotEqualOrUnordered(value, integerValueConvertedToDouble); - speculate(Overflow, FormattedValue(ValueFormatDouble, value), m_node, valueNotConvertibleToInteger); - - if (shouldCheckNegativeZero) { - LBasicBlock valueIsZero = FTL_NEW_BLOCK(m_out, ("ConvertDoubleToInt32 on zero")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ConvertDoubleToInt32 continuation")); - m_out.branch(m_out.isZero32(integerValue), unsure(valueIsZero), unsure(continuation)); - - LBasicBlock lastNext = m_out.appendTo(valueIsZero, continuation); - - LValue doubleBitcastToInt64 = m_out.bitCast(value, m_out.int64); - LValue signBitSet = m_out.lessThan(doubleBitcastToInt64, m_out.constInt64(0)); - - speculate(NegativeZero, FormattedValue(ValueFormatDouble, value), m_node, signBitSet); - m_out.jump(continuation); - m_out.appendTo(continuation, lastNext); - } - return integerValue; - } - - LValue isNumber(LValue jsValue, SpeculatedType type = SpecFullTop) - { - if (LValue proven = isProvenValue(type, SpecFullNumber)) - return proven; - return isNotCellOrMisc(jsValue); - } - LValue isNotNumber(LValue jsValue, SpeculatedType type = SpecFullTop) - { - if (LValue proven = isProvenValue(type, ~SpecFullNumber)) - return proven; - return isCellOrMisc(jsValue); - } - - LValue isNotCell(LValue jsValue, SpeculatedType type = SpecFullTop) - { - if (LValue proven = isProvenValue(type, ~SpecCell)) - return proven; return m_out.testNonZero64(jsValue, m_tagMask); } - LValue isCell(LValue jsValue, SpeculatedType type = SpecFullTop) + LValue isCell(LValue jsValue) { - if (LValue proven = isProvenValue(type, SpecCell)) - return proven; return m_out.testIsZero64(jsValue, m_tagMask); } - LValue isNotMisc(LValue value, SpeculatedType type = SpecFullTop) - { - if (LValue proven = isProvenValue(type, ~SpecMisc)) - return proven; - return m_out.above(value, m_out.constInt64(TagBitTypeOther | TagBitBool | TagBitUndefined)); - } - - LValue isMisc(LValue value, SpeculatedType type = SpecFullTop) + LValue isNotBoolean(LValue jsValue) { - if (LValue proven = isProvenValue(type, SpecMisc)) - return proven; - return m_out.bitNot(isNotMisc(value)); - } - - LValue isNotBoolean(LValue jsValue, SpeculatedType type = SpecFullTop) - { - if (LValue proven = isProvenValue(type, ~SpecBoolean)) - return proven; return m_out.testNonZero64( m_out.bitXor(jsValue, m_out.constInt64(ValueFalse)), m_out.constInt64(~1)); } - LValue isBoolean(LValue jsValue, SpeculatedType type = SpecFullTop) - { - if (LValue proven = isProvenValue(type, SpecBoolean)) - return proven; - return m_out.bitNot(isNotBoolean(jsValue)); - } LValue unboxBoolean(LValue jsValue) { // We want to use a cast that guarantees that LLVM knows that even the integer @@ -7384,32 +3710,6 @@ private: return m_out.select( value, m_out.constInt64(ValueTrue), m_out.constInt64(ValueFalse)); } - - LValue isNotOther(LValue value, SpeculatedType type = SpecFullTop) - { - if (LValue proven = isProvenValue(type, ~SpecOther)) - return proven; - return m_out.notEqual( - m_out.bitAnd(value, m_out.constInt64(~TagBitUndefined)), - m_out.constInt64(ValueNull)); - } - LValue isOther(LValue value, SpeculatedType type = SpecFullTop) - { - if (LValue proven = isProvenValue(type, SpecOther)) - return proven; - return m_out.equal( - m_out.bitAnd(value, m_out.constInt64(~TagBitUndefined)), - m_out.constInt64(ValueNull)); - } - - LValue isProvenValue(SpeculatedType provenType, SpeculatedType wantedType) - { - if (!(provenType & ~wantedType)) - return m_out.booleanTrue; - if (!(provenType & wantedType)) - return m_out.booleanFalse; - return nullptr; - } void speculate(Edge edge) { @@ -7417,9 +3717,7 @@ private: case UntypedUse: break; case KnownInt32Use: - case KnownStringUse: - case DoubleRepUse: - case Int52RepUse: + case KnownNumberUse: ASSERT(!m_interpreter.needsTypeCheck(edge)); break; case Int32Use: @@ -7431,15 +3729,9 @@ private: case KnownCellUse: ASSERT(!m_interpreter.needsTypeCheck(edge)); break; - case MachineIntUse: - speculateMachineInt(edge); - break; case ObjectUse: speculateObject(edge); break; - case FunctionUse: - speculateFunction(edge); - break; case ObjectOrOtherUse: speculateObjectOrOther(edge); break; @@ -7449,47 +3741,21 @@ private: case StringUse: speculateString(edge); break; - case StringIdentUse: - speculateStringIdent(edge); - break; - case SymbolUse: - speculateSymbol(edge); - break; - case StringObjectUse: - speculateStringObject(edge); - break; - case StringOrStringObjectUse: - speculateStringOrStringObject(edge); - break; - case NumberUse: - speculateNumber(edge); - break; case RealNumberUse: speculateRealNumber(edge); break; - case DoubleRepRealUse: - speculateDoubleRepReal(edge); + case NumberUse: + speculateNumber(edge); break; - case DoubleRepMachineIntUse: - speculateDoubleRepMachineInt(edge); + case MachineIntUse: + speculateMachineInt(edge); break; case BooleanUse: speculateBoolean(edge); break; - case NotStringVarUse: - speculateNotStringVar(edge); - break; - case NotCellUse: - speculateNotCell(edge); - break; - case OtherUse: - speculateOther(edge); - break; - case MiscUse: - speculateMisc(edge); - break; default: - DFG_CRASH(m_graph, m_node, "Unsupported speculation use kind"); + dataLog("Unsupported speculation use kind: ", edge.useKind(), "\n"); + RELEASE_ASSERT_NOT_REACHED(); } } @@ -7508,70 +3774,43 @@ private: lowCell(edge); } - void speculateMachineInt(Edge edge) + LValue isObject(LValue cell) { - if (!m_interpreter.needsTypeCheck(edge)) - return; - - jsValueToStrictInt52(edge, lowJSValue(edge, ManualOperandSpeculation)); + return m_out.notEqual( + m_out.loadPtr(cell, m_heaps.JSCell_structure), + m_out.constIntPtr(vm().stringStructure.get())); } - LValue isObject(LValue cell, SpeculatedType type = SpecFullTop) + LValue isNotString(LValue cell) { - if (LValue proven = isProvenValue(type & SpecCell, SpecObject)) - return proven; - return m_out.aboveOrEqual( - m_out.load8(cell, m_heaps.JSCell_typeInfoType), - m_out.constInt8(ObjectType)); - } - - LValue isNotObject(LValue cell, SpeculatedType type = SpecFullTop) - { - if (LValue proven = isProvenValue(type & SpecCell, ~SpecObject)) - return proven; - return m_out.below( - m_out.load8(cell, m_heaps.JSCell_typeInfoType), - m_out.constInt8(ObjectType)); - } - - LValue isNotString(LValue cell, SpeculatedType type = SpecFullTop) - { - if (LValue proven = isProvenValue(type & SpecCell, ~SpecString)) - return proven; - return m_out.notEqual( - m_out.load32(cell, m_heaps.JSCell_structureID), - m_out.constInt32(vm().stringStructure->id())); + return isObject(cell); } - LValue isString(LValue cell, SpeculatedType type = SpecFullTop) + LValue isString(LValue cell) { - if (LValue proven = isProvenValue(type & SpecCell, SpecString)) - return proven; return m_out.equal( - m_out.load32(cell, m_heaps.JSCell_structureID), - m_out.constInt32(vm().stringStructure->id())); + m_out.loadPtr(cell, m_heaps.JSCell_structure), + m_out.constIntPtr(vm().stringStructure.get())); } - - LValue isNotSymbol(LValue cell, SpeculatedType type = SpecFullTop) + + LValue isNotObject(LValue cell) { - if (LValue proven = isProvenValue(type & SpecCell, ~SpecSymbol)) - return proven; - return m_out.notEqual( - m_out.load32(cell, m_heaps.JSCell_structureID), - m_out.constInt32(vm().symbolStructure->id())); + return isString(cell); } - + LValue isArrayType(LValue cell, ArrayMode arrayMode) { switch (arrayMode.type()) { case Array::Int32: case Array::Double: case Array::Contiguous: { - LValue indexingType = m_out.load8(cell, m_heaps.JSCell_indexingType); + LValue indexingType = m_out.load8( + m_out.loadPtr(cell, m_heaps.JSCell_structure), + m_heaps.Structure_indexingType); switch (arrayMode.arrayClass()) { case Array::OriginalArray: - DFG_CRASH(m_graph, m_node, "Unexpected original array"); + RELEASE_ASSERT_NOT_REACHED(); return 0; case Array::Array: @@ -7591,52 +3830,29 @@ private: m_out.constInt8(arrayMode.shapeMask())); } - DFG_CRASH(m_graph, m_node, "Corrupt array class"); + RELEASE_ASSERT_NOT_REACHED(); } - case Array::DirectArguments: - return m_out.equal( - m_out.load8(cell, m_heaps.JSCell_typeInfoType), - m_out.constInt8(DirectArgumentsType)); - - case Array::ScopedArguments: - return m_out.equal( - m_out.load8(cell, m_heaps.JSCell_typeInfoType), - m_out.constInt8(ScopedArgumentsType)); - default: - return m_out.equal( - m_out.load8(cell, m_heaps.JSCell_typeInfoType), - m_out.constInt8(typeForTypedArrayType(arrayMode.typedArrayType()))); + return hasClassInfo(cell, classInfoForType(arrayMode.typedArrayType())); } } - LValue isFunction(LValue cell, SpeculatedType type = SpecFullTop) - { - if (LValue proven = isProvenValue(type & SpecCell, SpecFunction)) - return proven; - return isType(cell, JSFunctionType); - } - LValue isNotFunction(LValue cell, SpeculatedType type = SpecFullTop) + LValue hasClassInfo(LValue cell, const ClassInfo* classInfo) { - if (LValue proven = isProvenValue(type & SpecCell, ~SpecFunction)) - return proven; - return isNotType(cell, JSFunctionType); - } - - LValue isExoticForTypeof(LValue cell, SpeculatedType type = SpecFullTop) - { - if (!(type & SpecObjectOther)) - return m_out.booleanFalse; - return m_out.testNonZero8( - m_out.load8(cell, m_heaps.JSCell_typeInfoFlags), - m_out.constInt8(MasqueradesAsUndefined | TypeOfShouldCallGetCallData)); + return m_out.equal( + m_out.loadPtr( + m_out.loadPtr(cell, m_heaps.JSCell_structure), + m_heaps.Structure_classInfo), + m_out.constIntPtr(classInfo)); } LValue isType(LValue cell, JSType type) { return m_out.equal( - m_out.load8(cell, m_heaps.JSCell_typeInfoType), + m_out.load8( + m_out.loadPtr(cell, m_heaps.JSCell_structure), + m_heaps.Structure_typeInfoType), m_out.constInt8(type)); } @@ -7655,40 +3871,36 @@ private: speculateObject(edge, lowCell(edge)); } - void speculateFunction(Edge edge, LValue cell) - { - FTL_TYPE_CHECK(jsValueValue(cell), edge, SpecFunction, isNotFunction(cell)); - } - - void speculateFunction(Edge edge) - { - speculateFunction(edge, lowCell(edge)); - } - void speculateObjectOrOther(Edge edge) { if (!m_interpreter.needsTypeCheck(edge)) return; - LValue value = lowJSValue(edge, ManualOperandSpeculation); + LValue value = lowJSValue(edge); LBasicBlock cellCase = FTL_NEW_BLOCK(m_out, ("speculateObjectOrOther cell case")); LBasicBlock primitiveCase = FTL_NEW_BLOCK(m_out, ("speculateObjectOrOther primitive case")); LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("speculateObjectOrOther continuation")); - m_out.branch(isNotCell(value, provenType(edge)), unsure(primitiveCase), unsure(cellCase)); + m_out.branch(isNotCell(value), primitiveCase, cellCase); LBasicBlock lastNext = m_out.appendTo(cellCase, primitiveCase); FTL_TYPE_CHECK( - jsValueValue(value), edge, (~SpecCell) | SpecObject, isNotObject(value)); + jsValueValue(value), edge, (~SpecCell) | SpecObject, + m_out.equal( + m_out.loadPtr(value, m_heaps.JSCell_structure), + m_out.constIntPtr(vm().stringStructure.get()))); m_out.jump(continuation); m_out.appendTo(primitiveCase, continuation); FTL_TYPE_CHECK( - jsValueValue(value), edge, SpecCell | SpecOther, isNotOther(value)); + jsValueValue(value), edge, SpecCell | SpecOther, + m_out.notEqual( + m_out.bitAnd(value, m_out.constInt64(~TagBitUndefined)), + m_out.constInt64(ValueNull))); m_out.jump(continuation); @@ -7708,7 +3920,7 @@ private: void speculateString(Edge edge, LValue cell) { - FTL_TYPE_CHECK(jsValueValue(cell), edge, SpecString | ~SpecCell, isNotString(cell)); + FTL_TYPE_CHECK(jsValueValue(cell), edge, SpecString, isNotString(cell)); } void speculateString(Edge edge) @@ -7716,130 +3928,32 @@ private: speculateString(edge, lowCell(edge)); } - void speculateStringIdent(Edge edge, LValue string, LValue stringImpl) - { - if (!m_interpreter.needsTypeCheck(edge, SpecStringIdent | ~SpecString)) - return; - - speculate(BadType, jsValueValue(string), edge.node(), m_out.isNull(stringImpl)); - speculate( - BadType, jsValueValue(string), edge.node(), - m_out.testIsZero32( - m_out.load32(stringImpl, m_heaps.StringImpl_hashAndFlags), - m_out.constInt32(StringImpl::flagIsAtomic()))); - m_interpreter.filter(edge, SpecStringIdent | ~SpecString); - } - - void speculateStringIdent(Edge edge) - { - lowStringIdent(edge); - } - - void speculateStringObject(Edge edge) - { - if (!m_interpreter.needsTypeCheck(edge, SpecStringObject)) - return; - - speculateStringObjectForCell(edge, lowCell(edge)); - m_interpreter.filter(edge, SpecStringObject); - } - - void speculateStringOrStringObject(Edge edge) - { - if (!m_interpreter.needsTypeCheck(edge, SpecString | SpecStringObject)) - return; - - LBasicBlock notString = FTL_NEW_BLOCK(m_out, ("Speculate StringOrStringObject not string case")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Speculate StringOrStringObject continuation")); - - LValue structureID = m_out.load32(lowCell(edge), m_heaps.JSCell_structureID); - m_out.branch( - m_out.equal(structureID, m_out.constInt32(vm().stringStructure->id())), - unsure(continuation), unsure(notString)); - - LBasicBlock lastNext = m_out.appendTo(notString, continuation); - speculateStringObjectForStructureID(edge, structureID); - m_out.jump(continuation); - - m_out.appendTo(continuation, lastNext); - - m_interpreter.filter(edge, SpecString | SpecStringObject); - } - - void speculateStringObjectForCell(Edge edge, LValue cell) - { - speculateStringObjectForStructureID(edge, m_out.load32(cell, m_heaps.JSCell_structureID)); - } - - void speculateStringObjectForStructureID(Edge edge, LValue structureID) - { - Structure* stringObjectStructure = - m_graph.globalObjectFor(m_node->origin.semantic)->stringObjectStructure(); - - if (abstractStructure(edge).isSubsetOf(StructureSet(stringObjectStructure))) - return; - - speculate( - NotStringObject, noValue(), 0, - m_out.notEqual(structureID, weakStructureID(stringObjectStructure))); - } - - void speculateSymbol(Edge edge, LValue cell) - { - FTL_TYPE_CHECK(jsValueValue(cell), edge, SpecSymbol | ~SpecCell, isNotSymbol(cell)); - } - - void speculateSymbol(Edge edge) - { - speculateSymbol(edge, lowCell(edge)); - } - void speculateNonNullObject(Edge edge, LValue cell) { - FTL_TYPE_CHECK(jsValueValue(cell), edge, SpecObject, isNotObject(cell)); + LValue structure = m_out.loadPtr(cell, m_heaps.JSCell_structure); + FTL_TYPE_CHECK( + jsValueValue(cell), edge, SpecObject, + m_out.equal(structure, m_out.constIntPtr(vm().stringStructure.get()))); if (masqueradesAsUndefinedWatchpointIsStillValid()) return; speculate( BadType, jsValueValue(cell), edge.node(), m_out.testNonZero8( - m_out.load8(cell, m_heaps.JSCell_typeInfoFlags), + m_out.load8(structure, m_heaps.Structure_typeInfoFlags), m_out.constInt8(MasqueradesAsUndefined))); } void speculateNumber(Edge edge) { - LValue value = lowJSValue(edge, ManualOperandSpeculation); - FTL_TYPE_CHECK(jsValueValue(value), edge, SpecBytecodeNumber, isNotNumber(value)); - } - - void speculateRealNumber(Edge edge) - { // Do an early return here because lowDouble() can create a lot of control flow. if (!m_interpreter.needsTypeCheck(edge)) return; - LValue value = lowJSValue(edge, ManualOperandSpeculation); - LValue doubleValue = unboxDouble(value); - - LBasicBlock intCase = FTL_NEW_BLOCK(m_out, ("speculateRealNumber int case")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("speculateRealNumber continuation")); - - m_out.branch( - m_out.doubleEqual(doubleValue, doubleValue), - usually(continuation), rarely(intCase)); - - LBasicBlock lastNext = m_out.appendTo(intCase, continuation); - - typeCheck( - jsValueValue(value), m_node->child1(), SpecBytecodeRealNumber, - isNotInt32(value, provenType(m_node->child1()) & ~SpecFullDouble)); - m_out.jump(continuation); - - m_out.appendTo(continuation, lastNext); + lowDouble(edge); } - void speculateDoubleRepReal(Edge edge) + void speculateRealNumber(Edge edge) { // Do an early return here because lowDouble() can create a lot of control flow. if (!m_interpreter.needsTypeCheck(edge)) @@ -7847,16 +3961,17 @@ private: LValue value = lowDouble(edge); FTL_TYPE_CHECK( - doubleValue(value), edge, SpecDoubleReal, + doubleValue(value), edge, SpecFullRealNumber, m_out.doubleNotEqualOrUnordered(value, value)); } - void speculateDoubleRepMachineInt(Edge edge) + void speculateMachineInt(Edge edge) { if (!m_interpreter.needsTypeCheck(edge)) return; - doubleToStrictInt52(edge, lowDouble(edge)); + Int52Kind kind; + lowWhicheverInt52(edge, kind); } void speculateBoolean(Edge edge) @@ -7864,96 +3979,69 @@ private: lowBoolean(edge); } - void speculateNotStringVar(Edge edge) - { - if (!m_interpreter.needsTypeCheck(edge, ~SpecStringVar)) - return; - - LValue value = lowJSValue(edge, ManualOperandSpeculation); - - LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("Speculate NotStringVar is cell case")); - LBasicBlock isStringCase = FTL_NEW_BLOCK(m_out, ("Speculate NotStringVar is string case")); - LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Speculate NotStringVar continuation")); - - m_out.branch(isCell(value, provenType(edge)), unsure(isCellCase), unsure(continuation)); - - LBasicBlock lastNext = m_out.appendTo(isCellCase, isStringCase); - m_out.branch(isString(value, provenType(edge)), unsure(isStringCase), unsure(continuation)); - - m_out.appendTo(isStringCase, continuation); - speculateStringIdent(edge, value, m_out.loadPtr(value, m_heaps.JSString_value)); - m_out.jump(continuation); - - m_out.appendTo(continuation, lastNext); - } - - void speculateNotCell(Edge edge) - { - if (!m_interpreter.needsTypeCheck(edge)) - return; - - LValue value = lowJSValue(edge, ManualOperandSpeculation); - typeCheck(jsValueValue(value), edge, ~SpecCell, isCell(value)); - } - - void speculateOther(Edge edge) - { - if (!m_interpreter.needsTypeCheck(edge)) - return; - - LValue value = lowJSValue(edge, ManualOperandSpeculation); - typeCheck(jsValueValue(value), edge, SpecOther, isNotOther(value)); - } - - void speculateMisc(Edge edge) - { - if (!m_interpreter.needsTypeCheck(edge)) - return; - - LValue value = lowJSValue(edge, ManualOperandSpeculation); - typeCheck(jsValueValue(value), edge, SpecMisc, isNotMisc(value)); - } - bool masqueradesAsUndefinedWatchpointIsStillValid() { - return m_graph.masqueradesAsUndefinedWatchpointIsStillValid(m_node->origin.semantic); + return m_graph.masqueradesAsUndefinedWatchpointIsStillValid(m_node->codeOrigin); } LValue loadMarkByte(LValue base) { - return m_out.load8(base, m_heaps.JSCell_gcData); + LValue markedBlock = m_out.bitAnd(base, m_out.constInt64(MarkedBlock::blockMask)); + LValue baseOffset = m_out.bitAnd(base, m_out.constInt64(~MarkedBlock::blockMask)); + LValue markByteIndex = m_out.lShr(baseOffset, m_out.constInt64(MarkedBlock::atomShiftAmount + MarkedBlock::markByteShiftAmount)); + return m_out.load8(m_out.baseIndex(m_heaps.MarkedBlock_markBits, markedBlock, markByteIndex, ScaleOne, MarkedBlock::offsetOfMarks())); + } + + void emitStoreBarrier(LValue base, LValue value, Edge& valueEdge) + { +#if ENABLE(GGC) + LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Store barrier continuation")); + LBasicBlock isCell = FTL_NEW_BLOCK(m_out, ("Store barrier is cell block")); + + if (m_state.forNode(valueEdge.node()).couldBeType(SpecCell)) + m_out.branch(isNotCell(value), continuation, isCell); + else + m_out.jump(isCell); + + LBasicBlock lastNext = m_out.appendTo(isCell, continuation); + emitStoreBarrier(base); + m_out.jump(continuation); + + m_out.appendTo(continuation, lastNext); +#else + UNUSED_PARAM(base); + UNUSED_PARAM(value); + UNUSED_PARAM(valueEdge); +#endif } void emitStoreBarrier(LValue base) { #if ENABLE(GGC) - LBasicBlock isMarkedAndNotRemembered = FTL_NEW_BLOCK(m_out, ("Store barrier is marked block")); - LBasicBlock bufferHasSpace = FTL_NEW_BLOCK(m_out, ("Store barrier buffer has space")); - LBasicBlock bufferIsFull = FTL_NEW_BLOCK(m_out, ("Store barrier buffer is full")); LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Store barrier continuation")); + LBasicBlock isMarked = FTL_NEW_BLOCK(m_out, ("Store barrier is marked block")); + LBasicBlock bufferHasSpace = FTL_NEW_BLOCK(m_out, ("Store barrier buffer is full")); + LBasicBlock bufferIsFull = FTL_NEW_BLOCK(m_out, ("Store barrier buffer is full")); // Check the mark byte. - m_out.branch( - m_out.notZero8(loadMarkByte(base)), usually(continuation), rarely(isMarkedAndNotRemembered)); + m_out.branch(m_out.isZero8(loadMarkByte(base)), continuation, isMarked); // Append to the write barrier buffer. - LBasicBlock lastNext = m_out.appendTo(isMarkedAndNotRemembered, bufferHasSpace); - LValue currentBufferIndex = m_out.load32(m_out.absolute(vm().heap.writeBarrierBuffer().currentIndexAddress())); - LValue bufferCapacity = m_out.constInt32(vm().heap.writeBarrierBuffer().capacity()); - m_out.branch( - m_out.lessThan(currentBufferIndex, bufferCapacity), - usually(bufferHasSpace), rarely(bufferIsFull)); + LBasicBlock lastNext = m_out.appendTo(isMarked, bufferHasSpace); + LValue currentBufferIndex = m_out.load32(m_out.absolute(&vm().heap.writeBarrierBuffer().m_currentIndex)); + LValue bufferCapacity = m_out.load32(m_out.absolute(&vm().heap.writeBarrierBuffer().m_capacity)); + m_out.branch(m_out.lessThan(currentBufferIndex, bufferCapacity), bufferHasSpace, bufferIsFull); // Buffer has space, store to it. m_out.appendTo(bufferHasSpace, bufferIsFull); - LValue writeBarrierBufferBase = m_out.constIntPtr(vm().heap.writeBarrierBuffer().buffer()); - m_out.storePtr(base, m_out.baseIndex(m_heaps.WriteBarrierBuffer_bufferContents, writeBarrierBufferBase, m_out.zeroExtPtr(currentBufferIndex))); - m_out.store32(m_out.add(currentBufferIndex, m_out.constInt32(1)), m_out.absolute(vm().heap.writeBarrierBuffer().currentIndexAddress())); + LValue writeBarrierBufferBase = m_out.loadPtr(m_out.absolute(&vm().heap.writeBarrierBuffer().m_buffer)); + m_out.storePtr(base, m_out.baseIndex(m_heaps.WriteBarrierBuffer_bufferContents, writeBarrierBufferBase, m_out.zeroExt(currentBufferIndex, m_out.intPtr), ScalePtr)); + m_out.store32(m_out.add(currentBufferIndex, m_out.constInt32(1)), m_out.absolute(&vm().heap.writeBarrierBuffer().m_currentIndex)); m_out.jump(continuation); // Buffer is out of space, flush it. m_out.appendTo(bufferIsFull, continuation); - vmCallNoExceptions(m_out.operation(operationFlushWriteBarrierBuffer), m_callFrame, base); + vmCall(m_out.operation(operationFlushWriteBarrierBuffer), m_callFrame, base); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); @@ -7962,23 +4050,44 @@ private: #endif } - template<typename... Args> - LValue vmCall(LValue function, Args... args) + enum ExceptionCheckMode { NoExceptions, CheckExceptions }; + + LValue vmCall(LValue function, ExceptionCheckMode mode = CheckExceptions) { callPreflight(); - LValue result = m_out.call(function, args...); - callCheck(); + LValue result = m_out.call(function); + callCheck(mode); return result; } - - template<typename... Args> - LValue vmCallNoExceptions(LValue function, Args... args) + LValue vmCall(LValue function, LValue arg1, ExceptionCheckMode mode = CheckExceptions) { callPreflight(); - LValue result = m_out.call(function, args...); + LValue result = m_out.call(function, arg1); + callCheck(mode); return result; } - + LValue vmCall(LValue function, LValue arg1, LValue arg2, ExceptionCheckMode mode = CheckExceptions) + { + callPreflight(); + LValue result = m_out.call(function, arg1, arg2); + callCheck(mode); + return result; + } + LValue vmCall(LValue function, LValue arg1, LValue arg2, LValue arg3, ExceptionCheckMode mode = CheckExceptions) + { + callPreflight(); + LValue result = m_out.call(function, arg1, arg2, arg3); + callCheck(mode); + return result; + } + LValue vmCall(LValue function, LValue arg1, LValue arg2, LValue arg3, LValue arg4, ExceptionCheckMode mode = CheckExceptions) + { + callPreflight(); + LValue result = m_out.call(function, arg1, arg2, arg3, arg4); + callCheck(mode); + return result; + } + void callPreflight(CodeOrigin codeOrigin) { m_out.store32( @@ -7989,22 +4098,26 @@ private: } void callPreflight() { - callPreflight(m_node->origin.semantic); + callPreflight(m_node->codeOrigin); } - void callCheck() + void callCheck(ExceptionCheckMode mode = CheckExceptions) { - if (Options::enableExceptionFuzz()) - m_out.call(m_out.operation(operationExceptionFuzz)); + if (mode == NoExceptions) + return; + LBasicBlock didHaveException = FTL_NEW_BLOCK(m_out, ("Did have exception")); LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Exception check continuation")); - LValue exception = m_out.load64(m_out.absolute(vm().addressOfException())); - m_out.branch( - m_out.notZero64(exception), rarely(m_handleExceptions), usually(continuation)); + m_out.notZero64(m_out.load64(m_out.absolute(vm().addressOfException()))), + didHaveException, continuation); + + LBasicBlock lastNext = m_out.appendTo(didHaveException, continuation); + // FIXME: Handle exceptions. https://bugs.webkit.org/show_bug.cgi?id=113622 + m_out.crash(); - m_out.appendTo(continuation); + m_out.appendTo(continuation, lastNext); } LBasicBlock lowBlock(BasicBlock* block) @@ -8012,52 +4125,34 @@ private: return m_blocks.get(block); } + void initializeOSRExitStateForBlock() + { + m_availability = m_highBlock->ssa->availabilityAtHead; + } + void appendOSRExit( ExitKind kind, FormattedValue lowValue, Node* highValue, LValue failCondition) { - if (verboseCompilationEnabled()) { - dataLog(" OSR exit #", m_ftlState.jitCode->osrExit.size(), " with availability: ", availabilityMap(), "\n"); - if (!m_availableRecoveries.isEmpty()) - dataLog(" Available recoveries: ", listDump(m_availableRecoveries), "\n"); - } - - if (doOSRExitFuzzing()) { - LValue numberOfFuzzChecks = m_out.add( - m_out.load32(m_out.absolute(&g_numberOfOSRExitFuzzChecks)), - m_out.int32One); - - m_out.store32(numberOfFuzzChecks, m_out.absolute(&g_numberOfOSRExitFuzzChecks)); - - if (unsigned atOrAfter = Options::fireOSRExitFuzzAtOrAfter()) { - failCondition = m_out.bitOr( - failCondition, - m_out.aboveOrEqual(numberOfFuzzChecks, m_out.constInt32(atOrAfter))); - } - if (unsigned at = Options::fireOSRExitFuzzAt()) { - failCondition = m_out.bitOr( - failCondition, - m_out.equal(numberOfFuzzChecks, m_out.constInt32(at))); - } - } + if (verboseCompilationEnabled()) + dataLog(" OSR exit #", m_ftlState.jitCode->osrExit.size(), " with availability: ", m_availability, "\n"); ASSERT(m_ftlState.jitCode->osrExit.size() == m_ftlState.finalizer->osrExit.size()); m_ftlState.jitCode->osrExit.append(OSRExit( kind, lowValue.format(), m_graph.methodOfGettingAValueProfileFor(highValue), m_codeOriginForExitTarget, m_codeOriginForExitProfile, - availabilityMap().m_locals.numberOfArguments(), - availabilityMap().m_locals.numberOfLocals())); + m_availability.numberOfArguments(), m_availability.numberOfLocals())); m_ftlState.finalizer->osrExit.append(OSRExitCompilationInfo()); OSRExit& exit = m_ftlState.jitCode->osrExit.last(); - LBasicBlock lastNext = nullptr; - LBasicBlock continuation = nullptr; + LBasicBlock lastNext = 0; + LBasicBlock continuation = 0; LBasicBlock failCase = FTL_NEW_BLOCK(m_out, ("OSR exit failCase for ", m_node)); continuation = FTL_NEW_BLOCK(m_out, ("OSR exit continuation for ", m_node)); - m_out.branch(failCondition, rarely(failCase), usually(continuation)); + m_out.branch(failCondition, failCase, continuation); lastNext = m_out.appendTo(failCase, continuation); @@ -8083,202 +4178,155 @@ private: OSRExit& exit, ExitArgumentList& arguments, FormattedValue lowValue, CodeOrigin codeOrigin) { + arguments.append(m_callFrame); if (!!lowValue) arguments.append(lowValue.value()); - AvailabilityMap availabilityMap = this->availabilityMap(); - availabilityMap.pruneByLiveness(m_graph, codeOrigin); - - HashMap<Node*, ExitTimeObjectMaterialization*> map; - availabilityMap.forEachAvailability( - [&] (Availability availability) { - if (!availability.shouldUseNode()) - return; - - Node* node = availability.node(); - if (!node->isPhantomAllocation()) - return; - - auto result = map.add(node, nullptr); - if (result.isNewEntry) { - result.iterator->value = - exit.m_materializations.add(node->op(), node->origin.semantic); - } - }); - for (unsigned i = 0; i < exit.m_values.size(); ++i) { int operand = exit.m_values.operandForIndex(i); - - Availability availability = availabilityMap.m_locals[i]; - - if (Options::validateFTLOSRExitLiveness()) { - DFG_ASSERT( - m_graph, m_node, - (!(availability.isDead() && m_graph.isLiveInBytecode(VirtualRegister(operand), codeOrigin))) || m_graph.m_plan.mode == FTLForOSREntryMode); + bool isLive = m_graph.isLiveInBytecode(VirtualRegister(operand), codeOrigin); + if (!isLive) { + exit.m_values[i] = ExitValue::dead(); + continue; } - exit.m_values[i] = exitValueForAvailability(arguments, map, availability); - } - - for (auto heapPair : availabilityMap.m_heap) { - Node* node = heapPair.key.base(); - ExitTimeObjectMaterialization* materialization = map.get(node); - materialization->add( - heapPair.key.descriptor(), - exitValueForAvailability(arguments, map, heapPair.value)); + Availability availability = m_availability[i]; + FlushedAt flush = availability.flushedAt(); + switch (flush.format()) { + case DeadFlush: + case ConflictingFlush: + if (availability.hasNode()) { + addExitArgumentForNode(exit, arguments, i, availability.node()); + break; + } + + if (Options::validateFTLOSRExitLiveness()) { + dataLog("Expected r", operand, " to be available but it wasn't.\n"); + RELEASE_ASSERT_NOT_REACHED(); + } + + // This means that the DFG's DCE proved that the value is dead in bytecode + // even though the bytecode liveness analysis thinks it's live. This is + // acceptable since the DFG's DCE is by design more aggressive while still + // being sound. + exit.m_values[i] = ExitValue::dead(); + break; + + case FlushedJSValue: + case FlushedCell: + case FlushedBoolean: + exit.m_values[i] = ExitValue::inJSStack(flush.virtualRegister()); + break; + + case FlushedInt32: + exit.m_values[i] = ExitValue::inJSStackAsInt32(flush.virtualRegister()); + break; + + case FlushedInt52: + exit.m_values[i] = ExitValue::inJSStackAsInt52(flush.virtualRegister()); + break; + + case FlushedDouble: + exit.m_values[i] = ExitValue::inJSStackAsDouble(flush.virtualRegister()); + break; + + case FlushedArguments: + // FIXME: implement PhantomArguments. + // https://bugs.webkit.org/show_bug.cgi?id=113986 + RELEASE_ASSERT_NOT_REACHED(); + break; + } } - if (verboseCompilationEnabled()) { + if (verboseCompilationEnabled()) dataLog(" Exit values: ", exit.m_values, "\n"); - if (!exit.m_materializations.isEmpty()) { - dataLog(" Materializations: \n"); - for (ExitTimeObjectMaterialization* materialization : exit.m_materializations) - dataLog(" ", pointerDump(materialization), "\n"); - } - } } void callStackmap(OSRExit& exit, ExitArgumentList& arguments) { exit.m_stackmapID = m_stackmapIDs++; arguments.insert(0, m_out.constInt32(MacroAssembler::maxJumpReplacementSize())); - arguments.insert(0, m_out.constInt64(exit.m_stackmapID)); + arguments.insert(0, m_out.constInt32(exit.m_stackmapID)); m_out.call(m_out.stackmapIntrinsic(), arguments); } - ExitValue exitValueForAvailability( - ExitArgumentList& arguments, const HashMap<Node*, ExitTimeObjectMaterialization*>& map, - Availability availability) - { - FlushedAt flush = availability.flushedAt(); - switch (flush.format()) { - case DeadFlush: - case ConflictingFlush: - if (availability.hasNode()) - return exitValueForNode(arguments, map, availability.node()); - - // This means that the value is dead. It could be dead in bytecode or it could have - // been killed by our DCE, which can sometimes kill things even if they were live in - // bytecode. - return ExitValue::dead(); - - case FlushedJSValue: - case FlushedCell: - case FlushedBoolean: - return ExitValue::inJSStack(flush.virtualRegister()); - - case FlushedInt32: - return ExitValue::inJSStackAsInt32(flush.virtualRegister()); - - case FlushedInt52: - return ExitValue::inJSStackAsInt52(flush.virtualRegister()); - - case FlushedDouble: - return ExitValue::inJSStackAsDouble(flush.virtualRegister()); - } - - DFG_CRASH(m_graph, m_node, "Invalid flush format"); - return ExitValue::dead(); - } - - ExitValue exitValueForNode( - ExitArgumentList& arguments, const HashMap<Node*, ExitTimeObjectMaterialization*>& map, - Node* node) + void addExitArgumentForNode( + OSRExit& exit, ExitArgumentList& arguments, unsigned index, Node* node) { ASSERT(node->shouldGenerate()); ASSERT(node->hasResult()); - if (node) { - switch (node->op()) { - case BottomValue: - // This might arise in object materializations. I actually doubt that it would, - // but it seems worthwhile to be conservative. - return ExitValue::dead(); - - case JSConstant: - case Int52Constant: - case DoubleConstant: - return ExitValue::constant(node->asJSValue()); - - default: - if (node->isPhantomAllocation()) - return ExitValue::materializeNewObject(map.get(node)); - break; - } - } - - for (unsigned i = 0; i < m_availableRecoveries.size(); ++i) { - AvailableRecovery recovery = m_availableRecoveries[i]; - if (recovery.node() != node) - continue; - - ExitValue result = ExitValue::recovery( - recovery.opcode(), arguments.size(), arguments.size() + 1, - recovery.format()); - arguments.append(recovery.left()); - arguments.append(recovery.right()); - return result; - } + if (tryToSetConstantExitArgument(exit, index, node)) + return; LoweredNodeValue value = m_int32Values.get(node); - if (isValid(value)) - return exitArgument(arguments, ValueFormatInt32, value.value()); + if (isValid(value)) { + addExitArgument(exit, arguments, index, ValueFormatInt32, value.value()); + return; + } value = m_int52Values.get(node); - if (isValid(value)) - return exitArgument(arguments, ValueFormatInt52, value.value()); + if (isValid(value)) { + addExitArgument(exit, arguments, index, ValueFormatInt52, value.value()); + return; + } value = m_strictInt52Values.get(node); - if (isValid(value)) - return exitArgument(arguments, ValueFormatStrictInt52, value.value()); + if (isValid(value)) { + addExitArgument(exit, arguments, index, ValueFormatStrictInt52, value.value()); + return; + } value = m_booleanValues.get(node); if (isValid(value)) { LValue valueToPass = m_out.zeroExt(value.value(), m_out.int32); - return exitArgument(arguments, ValueFormatBoolean, valueToPass); + addExitArgument(exit, arguments, index, ValueFormatBoolean, valueToPass); + return; } value = m_jsValueValues.get(node); - if (isValid(value)) - return exitArgument(arguments, ValueFormatJSValue, value.value()); + if (isValid(value)) { + addExitArgument(exit, arguments, index, ValueFormatJSValue, value.value()); + return; + } value = m_doubleValues.get(node); - if (isValid(value)) - return exitArgument(arguments, ValueFormatDouble, value.value()); + if (isValid(value)) { + addExitArgument(exit, arguments, index, ValueFormatDouble, value.value()); + return; + } - DFG_CRASH(m_graph, m_node, toCString("Cannot find value for node: ", node).data()); - return ExitValue::dead(); - } - - ExitValue exitArgument(ExitArgumentList& arguments, ValueFormat format, LValue value) - { - ExitValue result = ExitValue::exitArgument(ExitArgument(format, arguments.size())); - arguments.append(value); - return result; + dataLog("Cannot find value for node: ", node, "\n"); + RELEASE_ASSERT_NOT_REACHED(); } - bool doesKill(Edge edge) + bool tryToSetConstantExitArgument(OSRExit& exit, unsigned index, Node* node) { - if (edge.doesNotKill()) + if (!node) return false; - if (edge->hasConstant()) + switch (node->op()) { + case JSConstant: + case WeakJSConstant: + exit.m_values[index] = ExitValue::constant(m_graph.valueOfJSConstant(node)); + return true; + case PhantomArguments: + // FIXME: implement PhantomArguments. + // https://bugs.webkit.org/show_bug.cgi?id=113986 + RELEASE_ASSERT_NOT_REACHED(); + return true; + default: return false; - - return true; - } - - void addAvailableRecovery( - Node* node, RecoveryOpcode opcode, LValue left, LValue right, ValueFormat format) - { - m_availableRecoveries.append(AvailableRecovery(node, opcode, left, right, format)); + } } - void addAvailableRecovery( - Edge edge, RecoveryOpcode opcode, LValue left, LValue right, ValueFormat format) + void addExitArgument( + OSRExit& exit, ExitArgumentList& arguments, unsigned index, ValueFormat format, + LValue value) { - addAvailableRecovery(edge.node(), opcode, left, right, format); + exit.m_values[index] = ExitValue::exitArgument(ExitArgument(format, arguments.size())); + arguments.append(value); } void setInt32(Node* node, LValue value) @@ -8305,7 +4353,7 @@ private: return; } - DFG_CRASH(m_graph, m_node, "Corrupt int52 kind"); + RELEASE_ASSERT_NOT_REACHED(); } void setJSValue(Node* node, LValue value) { @@ -8370,33 +4418,12 @@ private: { m_graph.m_plan.weakReferences.addLazily(target); } - - LValue loadStructure(LValue value) - { - LValue tableIndex = m_out.load32(value, m_heaps.JSCell_structureID); - LValue tableBase = m_out.loadPtr( - m_out.absolute(vm().heap.structureIDTable().base())); - TypedPointer address = m_out.baseIndex( - m_heaps.structureTable, tableBase, m_out.zeroExtPtr(tableIndex)); - return m_out.loadPtr(address); - } - + LValue weakPointer(JSCell* pointer) { addWeakReference(pointer); return m_out.constIntPtr(pointer); } - - LValue weakStructureID(Structure* structure) - { - addWeakReference(structure); - return m_out.constInt32(structure->id()); - } - - LValue weakStructure(Structure* structure) - { - return weakPointer(structure); - } TypedPointer addressFor(LValue base, int operand, ptrdiff_t offset = 0) { @@ -8404,100 +4431,37 @@ private: } TypedPointer payloadFor(LValue base, int operand) { - return addressFor(base, operand, PayloadOffset); + return addressFor(base, operand, OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)); } TypedPointer tagFor(LValue base, int operand) { - return addressFor(base, operand, TagOffset); + return addressFor(base, operand, OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)); } - TypedPointer addressFor(int operand, ptrdiff_t offset = 0) + TypedPointer addressFor(int operand) { - return addressFor(VirtualRegister(operand), offset); + return addressFor(m_callFrame, operand); } - TypedPointer addressFor(VirtualRegister operand, ptrdiff_t offset = 0) + TypedPointer addressFor(VirtualRegister operand) { - if (operand.isLocal()) - return addressFor(m_captured, operand.offset(), offset); - return addressFor(m_callFrame, operand.offset(), offset); + return addressFor(m_callFrame, operand.offset()); } TypedPointer payloadFor(int operand) { - return payloadFor(VirtualRegister(operand)); + return payloadFor(m_callFrame, operand); } TypedPointer payloadFor(VirtualRegister operand) { - return addressFor(operand, PayloadOffset); + return payloadFor(m_callFrame, operand.offset()); } TypedPointer tagFor(int operand) { - return tagFor(VirtualRegister(operand)); + return tagFor(m_callFrame, operand); } TypedPointer tagFor(VirtualRegister operand) { - return addressFor(operand, TagOffset); - } - - AbstractValue abstractValue(Node* node) - { - return m_state.forNode(node); - } - AbstractValue abstractValue(Edge edge) - { - return abstractValue(edge.node()); - } - - SpeculatedType provenType(Node* node) - { - return abstractValue(node).m_type; - } - SpeculatedType provenType(Edge edge) - { - return provenType(edge.node()); - } - - JSValue provenValue(Node* node) - { - return abstractValue(node).m_value; - } - JSValue provenValue(Edge edge) - { - return provenValue(edge.node()); - } - - StructureAbstractValue abstractStructure(Node* node) - { - return abstractValue(node).m_structure; - } - StructureAbstractValue abstractStructure(Edge edge) - { - return abstractStructure(edge.node()); + return tagFor(m_callFrame, operand.offset()); } - void crash() - { - crash(m_highBlock->index, m_node->index()); - } - void crash(BlockIndex blockIndex, unsigned nodeIndex) - { -#if ASSERT_DISABLED - m_out.call(m_out.operation(ftlUnreachable)); - UNUSED_PARAM(blockIndex); - UNUSED_PARAM(nodeIndex); -#else - m_out.call( - m_out.intToPtr( - m_out.constIntPtr(ftlUnreachable), - pointerType( - functionType( - m_out.voidType, m_out.intPtr, m_out.int32, m_out.int32))), - m_out.constIntPtr(codeBlock()), m_out.constInt32(blockIndex), - m_out.constInt32(nodeIndex)); -#endif - m_out.unreachable(); - } - - AvailabilityMap& availabilityMap() { return m_availabilityCalculator.m_availability; } - VM& vm() { return m_graph.m_vm; } CodeBlock* codeBlock() { return m_graph.m_codeBlock; } @@ -8507,11 +4471,9 @@ private: Output m_out; LBasicBlock m_prologue; - LBasicBlock m_handleExceptions; HashMap<BasicBlock*, LBasicBlock> m_blocks; LValue m_callFrame; - LValue m_captured; LValue m_tagTypeNumber; LValue m_tagMask; @@ -8523,16 +4485,9 @@ private: HashMap<Node*, LoweredNodeValue> m_storageValues; HashMap<Node*, LoweredNodeValue> m_doubleValues; - // This is a bit of a hack. It prevents LLVM from having to do CSE on loading of arguments. - // It's nice to have these optimizations on our end because we can guarantee them a bit better. - // Probably also saves LLVM compile time. - HashMap<Node*, LValue> m_loadedArgumentValues; - HashMap<Node*, LValue> m_phis; - LocalOSRAvailabilityCalculator m_availabilityCalculator; - - Vector<AvailableRecovery, 3> m_availableRecoveries; + Operands<Availability> m_availability; InPlaceAbstractState m_state; AbstractInterpreter<InPlaceAbstractState> m_interpreter; @@ -8546,12 +4501,8 @@ private: Node* m_node; uint32_t m_stackmapIDs; - unsigned m_tbaaKind; - unsigned m_tbaaStructKind; }; -} // anonymous namespace - void lowerDFGToLLVM(State& state) { LowerDFGToLLVM lowering(state); |