Imported WebKit commit 2ea9d364d0f6efa8fa64acf19f451504c59be0e4 (http://svn.webkit.org/repository/webkit/trunk@104285)

1 files changed, 633 insertions, 0 deletions

diff --git a/Source/JavaScriptCore/dfg/DFGOSRExitCompiler64.cpp b/Source/JavaScriptCore/dfg/DFGOSRExitCompiler64.cpp
new file mode 100644
index 000000000..c6f4a9ed4
--- /dev/null
+++ b/Source/JavaScriptCore/dfg/DFGOSRExitCompiler64.cpp
@@ -0,0 +1,633 @@
+/*
+ * Copyright (C) 2011 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
+ */
+
+#include "config.h"
+#include "DFGOSRExitCompiler.h"
+
+#if ENABLE(DFG_JIT) && USE(JSVALUE64)
+
+#include "DFGOperations.h"
+
+namespace JSC { namespace DFG {
+
+void OSRExitCompiler::compileExit(const OSRExit& exit, SpeculationRecovery* recovery)
+{
+    // 1) Pro-forma stuff.
+#if DFG_ENABLE(DEBUG_VERBOSE)
+    fprintf(stderr, "OSR exit for Node @%d (", (int)exit.m_nodeIndex);
+    for (CodeOrigin codeOrigin = exit.m_codeOrigin; ; codeOrigin = codeOrigin.inlineCallFrame->caller) {
+        fprintf(stderr, "bc#%u", codeOrigin.bytecodeIndex);
+        if (!codeOrigin.inlineCallFrame)
+            break;
+        fprintf(stderr, " -> %p ", codeOrigin.inlineCallFrame->executable.get());
+    }
+    fprintf(stderr, ")  ");
+    exit.dump(stderr);
+#endif
+#if DFG_ENABLE(VERBOSE_SPECULATION_FAILURE)
+    SpeculationFailureDebugInfo* debugInfo = new SpeculationFailureDebugInfo;
+    debugInfo->codeBlock = m_jit.codeBlock();
+    debugInfo->nodeIndex = exit.m_nodeIndex;
+    
+    m_jit.debugCall(debugOperationPrintSpeculationFailure, debugInfo);
+#endif
+    
+#if DFG_ENABLE(JIT_BREAK_ON_SPECULATION_FAILURE)
+    m_jit.breakpoint();
+#endif
+    
+#if DFG_ENABLE(SUCCESS_STATS)
+    static SamplingCounter counter("SpeculationFailure");
+    m_jit.emitCount(counter);
+#endif
+    
+    // 2) Perform speculation recovery. This only comes into play when an operation
+    //    starts mutating state before verifying the speculation it has already made.
+    
+    GPRReg alreadyBoxed = InvalidGPRReg;
+    
+    if (recovery) {
+        switch (recovery->type()) {
+        case SpeculativeAdd:
+            m_jit.sub32(recovery->src(), recovery->dest());
+            m_jit.orPtr(GPRInfo::tagTypeNumberRegister, recovery->dest());
+            alreadyBoxed = recovery->dest();
+            break;
+            
+        case BooleanSpeculationCheck:
+            m_jit.xorPtr(AssemblyHelpers::TrustedImm32(static_cast<int32_t>(ValueFalse)), recovery->dest());
+            break;
+            
+        default:
+            break;
+        }
+    }
+
+    // 3) Refine some value profile, if appropriate.
+    
+    if (!!exit.m_jsValueSource && !!exit.m_valueProfile) {
+        if (exit.m_jsValueSource.isAddress()) {
+            // We can't be sure that we have a spare register. So use the tagTypeNumberRegister,
+            // since we know how to restore it.
+            m_jit.loadPtr(AssemblyHelpers::Address(exit.m_jsValueSource.asAddress()), GPRInfo::tagTypeNumberRegister);
+            m_jit.storePtr(GPRInfo::tagTypeNumberRegister, exit.m_valueProfile->specFailBucket(0));
+            m_jit.move(AssemblyHelpers::TrustedImmPtr(bitwise_cast<void*>(TagTypeNumber)), GPRInfo::tagTypeNumberRegister);
+        } else
+            m_jit.storePtr(exit.m_jsValueSource.gpr(), exit.m_valueProfile->specFailBucket(0));
+    }
+
+    // 4) Figure out how many scratch slots we'll need. We need one for every GPR/FPR
+    //    whose destination is now occupied by a DFG virtual register, and we need
+    //    one for every displaced virtual register if there are more than
+    //    GPRInfo::numberOfRegisters of them. Also see if there are any constants,
+    //    any undefined slots, any FPR slots, and any unboxed ints.
+            
+    Vector<bool> poisonedVirtualRegisters(exit.m_variables.size());
+    for (unsigned i = 0; i < poisonedVirtualRegisters.size(); ++i)
+        poisonedVirtualRegisters[i] = false;
+
+    unsigned numberOfPoisonedVirtualRegisters = 0;
+    unsigned numberOfDisplacedVirtualRegisters = 0;
+    
+    // Booleans for fast checks. We expect that most OSR exits do not have to rebox
+    // Int32s, have no FPRs, and have no constants. If there are constants, we
+    // expect most of them to be jsUndefined(); if that's true then we handle that
+    // specially to minimize code size and execution time.
+    bool haveUnboxedInt32s = false;
+    bool haveUnboxedDoubles = false;
+    bool haveFPRs = false;
+    bool haveConstants = false;
+    bool haveUndefined = false;
+    bool haveUInt32s = false;
+    
+    for (int index = 0; index < exit.numberOfRecoveries(); ++index) {
+        const ValueRecovery& recovery = exit.valueRecovery(index);
+        switch (recovery.technique()) {
+        case Int32DisplacedInRegisterFile:
+        case DoubleDisplacedInRegisterFile:
+        case DisplacedInRegisterFile:
+            numberOfDisplacedVirtualRegisters++;
+            ASSERT((int)recovery.virtualRegister() >= 0);
+            
+            // See if we might like to store to this virtual register before doing
+            // virtual register shuffling. If so, we say that the virtual register
+            // is poisoned: it cannot be stored to until after displaced virtual
+            // registers are handled. We track poisoned virtual register carefully
+            // to ensure this happens efficiently. Note that we expect this case
+            // to be rare, so the handling of it is optimized for the cases in
+            // which it does not happen.
+            if (recovery.virtualRegister() < (int)exit.m_variables.size()) {
+                switch (exit.m_variables[recovery.virtualRegister()].technique()) {
+                case InGPR:
+                case UnboxedInt32InGPR:
+                case UInt32InGPR:
+                case InFPR:
+                    if (!poisonedVirtualRegisters[recovery.virtualRegister()]) {
+                        poisonedVirtualRegisters[recovery.virtualRegister()] = true;
+                        numberOfPoisonedVirtualRegisters++;
+                    }
+                    break;
+                default:
+                    break;
+                }
+            }
+            break;
+            
+        case UnboxedInt32InGPR:
+        case AlreadyInRegisterFileAsUnboxedInt32:
+            haveUnboxedInt32s = true;
+            break;
+            
+        case AlreadyInRegisterFileAsUnboxedDouble:
+            haveUnboxedDoubles = true;
+            break;
+            
+        case UInt32InGPR:
+            haveUInt32s = true;
+            break;
+            
+        case InFPR:
+            haveFPRs = true;
+            break;
+            
+        case Constant:
+            haveConstants = true;
+            if (recovery.constant().isUndefined())
+                haveUndefined = true;
+            break;
+            
+        default:
+            break;
+        }
+    }
+    
+#if DFG_ENABLE(DEBUG_VERBOSE)
+    fprintf(stderr, "  ");
+    if (numberOfPoisonedVirtualRegisters)
+        fprintf(stderr, "Poisoned=%u ", numberOfPoisonedVirtualRegisters);
+    if (numberOfDisplacedVirtualRegisters)
+        fprintf(stderr, "Displaced=%u ", numberOfDisplacedVirtualRegisters);
+    if (haveUnboxedInt32s)
+        fprintf(stderr, "UnboxedInt32 ");
+    if (haveUnboxedDoubles)
+        fprintf(stderr, "UnboxedDoubles ");
+    if (haveUInt32s)
+        fprintf(stderr, "UInt32 ");
+    if (haveFPRs)
+        fprintf(stderr, "FPR ");
+    if (haveConstants)
+        fprintf(stderr, "Constants ");
+    if (haveUndefined)
+        fprintf(stderr, "Undefined ");
+    fprintf(stderr, " ");
+#endif
+    
+    EncodedJSValue* scratchBuffer = static_cast<EncodedJSValue*>(m_jit.globalData()->scratchBufferForSize(sizeof(EncodedJSValue) * std::max(haveUInt32s ? 2u : 0u, numberOfPoisonedVirtualRegisters + (numberOfDisplacedVirtualRegisters <= GPRInfo::numberOfRegisters ? 0 : numberOfDisplacedVirtualRegisters))));
+
+    // From here on, the code assumes that it is profitable to maximize the distance
+    // between when something is computed and when it is stored.
+    
+    // 5) Perform all reboxing of integers.
+    
+    if (haveUnboxedInt32s || haveUInt32s) {
+        for (int index = 0; index < exit.numberOfRecoveries(); ++index) {
+            const ValueRecovery& recovery = exit.valueRecovery(index);
+            switch (recovery.technique()) {
+            case UnboxedInt32InGPR:
+                if (recovery.gpr() != alreadyBoxed)
+                    m_jit.orPtr(GPRInfo::tagTypeNumberRegister, recovery.gpr());
+                break;
+                
+            case AlreadyInRegisterFileAsUnboxedInt32:
+                m_jit.store32(AssemblyHelpers::Imm32(static_cast<uint32_t>(TagTypeNumber >> 32)), AssemblyHelpers::tagFor(static_cast<VirtualRegister>(exit.operandForIndex(index))));
+                break;
+                
+            case UInt32InGPR: {
+                // This occurs when the speculative JIT left an unsigned 32-bit integer
+                // in a GPR. If it's positive, we can just box the int. Otherwise we
+                // need to turn it into a boxed double.
+                
+                // We don't try to be clever with register allocation here; we assume
+                // that the program is using FPRs and we don't try to figure out which
+                // ones it is using. Instead just temporarily save fpRegT0 and then
+                // restore it. This makes sense because this path is not cheap to begin
+                // with, and should happen very rarely.
+                
+                GPRReg addressGPR = GPRInfo::regT0;
+                if (addressGPR == recovery.gpr())
+                    addressGPR = GPRInfo::regT1;
+                
+                m_jit.storePtr(addressGPR, scratchBuffer);
+                m_jit.move(AssemblyHelpers::TrustedImmPtr(scratchBuffer + 1), addressGPR);
+                m_jit.storeDouble(FPRInfo::fpRegT0, addressGPR);
+                
+                AssemblyHelpers::Jump positive = m_jit.branch32(AssemblyHelpers::GreaterThanOrEqual, recovery.gpr(), AssemblyHelpers::TrustedImm32(0));
+
+                m_jit.convertInt32ToDouble(recovery.gpr(), FPRInfo::fpRegT0);
+                m_jit.addDouble(AssemblyHelpers::AbsoluteAddress(&AssemblyHelpers::twoToThe32), FPRInfo::fpRegT0);
+                m_jit.boxDouble(FPRInfo::fpRegT0, recovery.gpr());
+                
+                AssemblyHelpers::Jump done = m_jit.jump();
+                
+                positive.link(&m_jit);
+                
+                m_jit.orPtr(GPRInfo::tagTypeNumberRegister, recovery.gpr());
+                
+                done.link(&m_jit);
+                
+                m_jit.loadDouble(addressGPR, FPRInfo::fpRegT0);
+                m_jit.loadPtr(scratchBuffer, addressGPR);
+                break;
+            }
+                
+            default:
+                break;
+            }
+        }
+    }
+    
+    // 6) Dump all non-poisoned GPRs. For poisoned GPRs, save them into the scratch storage.
+    //    Note that GPRs do not have a fast change (like haveFPRs) because we expect that
+    //    most OSR failure points will have at least one GPR that needs to be dumped.
+    
+    initializePoisoned(exit.m_variables.size());
+    unsigned currentPoisonIndex = 0;
+    
+    for (int index = 0; index < exit.numberOfRecoveries(); ++index) {
+        const ValueRecovery& recovery = exit.valueRecovery(index);
+        int operand = exit.operandForIndex(index);
+        switch (recovery.technique()) {
+        case InGPR:
+        case UnboxedInt32InGPR:
+        case UInt32InGPR:
+            if (exit.isVariable(index) && poisonedVirtualRegisters[exit.variableForIndex(index)]) {
+                m_jit.storePtr(recovery.gpr(), scratchBuffer + currentPoisonIndex);
+                m_poisonScratchIndices[exit.variableForIndex(index)] = currentPoisonIndex;
+                currentPoisonIndex++;
+            } else
+                m_jit.storePtr(recovery.gpr(), AssemblyHelpers::addressFor((VirtualRegister)operand));
+            break;
+        default:
+            break;
+        }
+    }
+    
+    // At this point all GPRs are available for scratch use.
+    
+    if (haveFPRs) {
+        // 7) Box all doubles (relies on there being more GPRs than FPRs)
+        
+        for (int index = 0; index < exit.numberOfRecoveries(); ++index) {
+            const ValueRecovery& recovery = exit.valueRecovery(index);
+            if (recovery.technique() != InFPR)
+                continue;
+            FPRReg fpr = recovery.fpr();
+            GPRReg gpr = GPRInfo::toRegister(FPRInfo::toIndex(fpr));
+            m_jit.boxDouble(fpr, gpr);
+        }
+        
+        // 8) Dump all doubles into the register file, or to the scratch storage if
+        //    the destination virtual register is poisoned.
+        
+        for (int index = 0; index < exit.numberOfRecoveries(); ++index) {
+            const ValueRecovery& recovery = exit.valueRecovery(index);
+            if (recovery.technique() != InFPR)
+                continue;
+            GPRReg gpr = GPRInfo::toRegister(FPRInfo::toIndex(recovery.fpr()));
+            if (exit.isVariable(index) && poisonedVirtualRegisters[exit.variableForIndex(index)]) {
+                m_jit.storePtr(gpr, scratchBuffer + currentPoisonIndex);
+                m_poisonScratchIndices[exit.variableForIndex(index)] = currentPoisonIndex;
+                currentPoisonIndex++;
+            } else
+                m_jit.storePtr(gpr, AssemblyHelpers::addressFor((VirtualRegister)exit.operandForIndex(index)));
+        }
+    }
+    
+    // At this point all GPRs and FPRs are available for scratch use.
+    
+    // 9) Box all unboxed doubles in the register file.
+    if (haveUnboxedDoubles) {
+        for (int index = 0; index < exit.numberOfRecoveries(); ++index) {
+            const ValueRecovery& recovery = exit.valueRecovery(index);
+            if (recovery.technique() != AlreadyInRegisterFileAsUnboxedDouble)
+                continue;
+            m_jit.loadDouble(AssemblyHelpers::addressFor((VirtualRegister)exit.operandForIndex(index)), FPRInfo::fpRegT0);
+            m_jit.boxDouble(FPRInfo::fpRegT0, GPRInfo::regT0);
+            m_jit.storePtr(GPRInfo::regT0, AssemblyHelpers::addressFor((VirtualRegister)exit.operandForIndex(index)));
+        }
+    }
+    
+    ASSERT(currentPoisonIndex == numberOfPoisonedVirtualRegisters);
+    
+    // 10) Reshuffle displaced virtual registers. Optimize for the case that
+    //    the number of displaced virtual registers is not more than the number
+    //    of available physical registers.
+    
+    if (numberOfDisplacedVirtualRegisters) {
+        if (numberOfDisplacedVirtualRegisters <= GPRInfo::numberOfRegisters) {
+            // So far this appears to be the case that triggers all the time, but
+            // that is far from guaranteed.
+        
+            unsigned displacementIndex = 0;
+            for (int index = 0; index < exit.numberOfRecoveries(); ++index) {
+                const ValueRecovery& recovery = exit.valueRecovery(index);
+                switch (recovery.technique()) {
+                case DisplacedInRegisterFile:
+                    m_jit.loadPtr(AssemblyHelpers::addressFor(recovery.virtualRegister()), GPRInfo::toRegister(displacementIndex++));
+                    break;
+                    
+                case Int32DisplacedInRegisterFile: {
+                    GPRReg gpr = GPRInfo::toRegister(displacementIndex++);
+                    m_jit.load32(AssemblyHelpers::addressFor(recovery.virtualRegister()), gpr);
+                    m_jit.orPtr(GPRInfo::tagTypeNumberRegister, gpr);
+                    break;
+                }
+                    
+                case DoubleDisplacedInRegisterFile: {
+                    GPRReg gpr = GPRInfo::toRegister(displacementIndex++);
+                    m_jit.loadPtr(AssemblyHelpers::addressFor(recovery.virtualRegister()), gpr);
+                    m_jit.subPtr(GPRInfo::tagTypeNumberRegister, gpr);
+                    break;
+                }
+                    
+                default:
+                    break;
+                }
+            }
+        
+            displacementIndex = 0;
+            for (int index = 0; index < exit.numberOfRecoveries(); ++index) {
+                const ValueRecovery& recovery = exit.valueRecovery(index);
+                switch (recovery.technique()) {
+                case DisplacedInRegisterFile:
+                case Int32DisplacedInRegisterFile:
+                case DoubleDisplacedInRegisterFile:
+                    m_jit.storePtr(GPRInfo::toRegister(displacementIndex++), AssemblyHelpers::addressFor((VirtualRegister)exit.operandForIndex(index)));
+                    break;
+                    
+                default:
+                    break;
+                }
+            }
+        } else {
+            // FIXME: This should use the shuffling algorithm that we use
+            // for speculative->non-speculative jumps, if we ever discover that
+            // some hot code with lots of live values that get displaced and
+            // spilled really enjoys frequently failing speculation.
+        
+            // For now this code is engineered to be correct but probably not
+            // super. In particular, it correctly handles cases where for example
+            // the displacements are a permutation of the destination values, like
+            //
+            // 1 -> 2
+            // 2 -> 1
+            //
+            // It accomplishes this by simply lifting all of the virtual registers
+            // from their old (DFG JIT) locations and dropping them in a scratch
+            // location in memory, and then transferring from that scratch location
+            // to their new (old JIT) locations.
+        
+            unsigned scratchIndex = numberOfPoisonedVirtualRegisters;
+            for (int index = 0; index < exit.numberOfRecoveries(); ++index) {
+                const ValueRecovery& recovery = exit.valueRecovery(index);
+                
+                switch (recovery.technique()) {
+                case DisplacedInRegisterFile:
+                    m_jit.loadPtr(AssemblyHelpers::addressFor(recovery.virtualRegister()), GPRInfo::regT0);
+                    m_jit.storePtr(GPRInfo::regT0, scratchBuffer + scratchIndex++);
+                    break;
+                    
+                case Int32DisplacedInRegisterFile: {
+                    m_jit.load32(AssemblyHelpers::addressFor(recovery.virtualRegister()), GPRInfo::regT0);
+                    m_jit.orPtr(GPRInfo::tagTypeNumberRegister, GPRInfo::regT0);
+                    m_jit.storePtr(GPRInfo::regT0, scratchBuffer + scratchIndex++);
+                    break;
+                }
+                    
+                case DoubleDisplacedInRegisterFile: {
+                    m_jit.loadPtr(AssemblyHelpers::addressFor(recovery.virtualRegister()), GPRInfo::regT0);
+                    m_jit.subPtr(GPRInfo::tagTypeNumberRegister, GPRInfo::regT0);
+                    m_jit.storePtr(GPRInfo::regT0, scratchBuffer + scratchIndex++);
+                    break;
+                }
+                    
+                default:
+                    break;
+                }
+            }
+        
+            scratchIndex = numberOfPoisonedVirtualRegisters;
+            for (int index = 0; index < exit.numberOfRecoveries(); ++index) {
+                const ValueRecovery& recovery = exit.valueRecovery(index);
+                switch (recovery.technique()) {
+                case DisplacedInRegisterFile:
+                case Int32DisplacedInRegisterFile:
+                case DoubleDisplacedInRegisterFile:
+                    m_jit.loadPtr(scratchBuffer + scratchIndex++, GPRInfo::regT0);
+                    m_jit.storePtr(GPRInfo::regT0, AssemblyHelpers::addressFor((VirtualRegister)exit.operandForIndex(index)));
+                    break;
+                    
+                default:
+                    break;
+                }
+            }
+        
+            ASSERT(scratchIndex == numberOfPoisonedVirtualRegisters + numberOfDisplacedVirtualRegisters);
+        }
+    }
+    
+    // 11) Dump all poisoned virtual registers.
+    
+    if (numberOfPoisonedVirtualRegisters) {
+        for (int virtualRegister = 0; virtualRegister < (int)exit.m_variables.size(); ++virtualRegister) {
+            if (!poisonedVirtualRegisters[virtualRegister])
+                continue;
+            
+            const ValueRecovery& recovery = exit.m_variables[virtualRegister];
+            switch (recovery.technique()) {
+            case InGPR:
+            case UnboxedInt32InGPR:
+            case UInt32InGPR:
+            case InFPR:
+                m_jit.loadPtr(scratchBuffer + poisonIndex(virtualRegister), GPRInfo::regT0);
+                m_jit.storePtr(GPRInfo::regT0, AssemblyHelpers::addressFor((VirtualRegister)virtualRegister));
+                break;
+                
+            default:
+                break;
+            }
+        }
+    }
+    
+    // 12) Dump all constants. Optimize for Undefined, since that's a constant we see
+    //     often.
+
+    if (haveConstants) {
+        if (haveUndefined)
+            m_jit.move(AssemblyHelpers::TrustedImmPtr(JSValue::encode(jsUndefined())), GPRInfo::regT0);
+        
+        for (int index = 0; index < exit.numberOfRecoveries(); ++index) {
+            const ValueRecovery& recovery = exit.valueRecovery(index);
+            if (recovery.technique() != Constant)
+                continue;
+            if (recovery.constant().isUndefined())
+                m_jit.storePtr(GPRInfo::regT0, AssemblyHelpers::addressFor((VirtualRegister)exit.operandForIndex(index)));
+            else
+                m_jit.storePtr(AssemblyHelpers::TrustedImmPtr(JSValue::encode(recovery.constant())), AssemblyHelpers::addressFor((VirtualRegister)exit.operandForIndex(index)));
+        }
+    }
+    
+    // 13) Adjust the old JIT's execute counter. Since we are exiting OSR, we know
+    //     that all new calls into this code will go to the new JIT, so the execute
+    //     counter only affects call frames that performed OSR exit and call frames
+    //     that were still executing the old JIT at the time of another call frame's
+    //     OSR exit. We want to ensure that the following is true:
+    //
+    //     (a) Code the performs an OSR exit gets a chance to reenter optimized
+    //         code eventually, since optimized code is faster. But we don't
+    //         want to do such reentery too aggressively (see (c) below).
+    //
+    //     (b) If there is code on the call stack that is still running the old
+    //         JIT's code and has never OSR'd, then it should get a chance to
+    //         perform OSR entry despite the fact that we've exited.
+    //
+    //     (c) Code the performs an OSR exit should not immediately retry OSR
+    //         entry, since both forms of OSR are expensive. OSR entry is
+    //         particularly expensive.
+    //
+    //     (d) Frequent OSR failures, even those that do not result in the code
+    //         running in a hot loop, result in recompilation getting triggered.
+    //
+    //     To ensure (c), we'd like to set the execute counter to
+    //     counterValueForOptimizeAfterWarmUp(). This seems like it would endanger
+    //     (a) and (b), since then every OSR exit would delay the opportunity for
+    //     every call frame to perform OSR entry. Essentially, if OSR exit happens
+    //     frequently and the function has few loops, then the counter will never
+    //     become non-negative and OSR entry will never be triggered. OSR entry
+    //     will only happen if a loop gets hot in the old JIT, which does a pretty
+    //     good job of ensuring (a) and (b). But that doesn't take care of (d),
+    //     since each speculation failure would reset the execute counter.
+    //     So we check here if the number of speculation failures is significantly
+    //     larger than the number of successes (we want 90% success rate), and if
+    //     there have been a large enough number of failures. If so, we set the
+    //     counter to 0; otherwise we set the counter to
+    //     counterValueForOptimizeAfterWarmUp().
+    
+    m_jit.add32(AssemblyHelpers::Imm32(1), AssemblyHelpers::AbsoluteAddress(&exit.m_count));
+    
+    m_jit.move(AssemblyHelpers::TrustedImmPtr(m_jit.codeBlock()), GPRInfo::regT0);
+    
+    m_jit.load32(AssemblyHelpers::Address(GPRInfo::regT0, CodeBlock::offsetOfSpeculativeFailCounter()), GPRInfo::regT2);
+    m_jit.load32(AssemblyHelpers::Address(GPRInfo::regT0, CodeBlock::offsetOfSpeculativeSuccessCounter()), GPRInfo::regT1);
+    m_jit.add32(AssemblyHelpers::Imm32(1), GPRInfo::regT2);
+    m_jit.add32(AssemblyHelpers::Imm32(-1), GPRInfo::regT1);
+    m_jit.store32(GPRInfo::regT2, AssemblyHelpers::Address(GPRInfo::regT0, CodeBlock::offsetOfSpeculativeFailCounter()));
+    m_jit.store32(GPRInfo::regT1, AssemblyHelpers::Address(GPRInfo::regT0, CodeBlock::offsetOfSpeculativeSuccessCounter()));
+    
+    m_jit.move(AssemblyHelpers::TrustedImmPtr(m_jit.baselineCodeBlock()), GPRInfo::regT0);
+    
+    AssemblyHelpers::Jump fewFails = m_jit.branch32(AssemblyHelpers::BelowOrEqual, GPRInfo::regT2, AssemblyHelpers::Imm32(m_jit.codeBlock()->largeFailCountThreshold()));
+    m_jit.mul32(AssemblyHelpers::Imm32(Options::desiredSpeculativeSuccessFailRatio), GPRInfo::regT2, GPRInfo::regT2);
+    
+    AssemblyHelpers::Jump lowFailRate = m_jit.branch32(AssemblyHelpers::BelowOrEqual, GPRInfo::regT2, GPRInfo::regT1);
+    
+    // Reoptimize as soon as possible.
+    m_jit.store32(AssemblyHelpers::Imm32(Options::executionCounterValueForOptimizeNextInvocation), AssemblyHelpers::Address(GPRInfo::regT0, CodeBlock::offsetOfExecuteCounter()));
+    AssemblyHelpers::Jump doneAdjusting = m_jit.jump();
+    
+    fewFails.link(&m_jit);
+    lowFailRate.link(&m_jit);
+    
+    m_jit.store32(AssemblyHelpers::Imm32(m_jit.baselineCodeBlock()->counterValueForOptimizeAfterLongWarmUp()), AssemblyHelpers::Address(GPRInfo::regT0, CodeBlock::offsetOfExecuteCounter()));
+    
+    doneAdjusting.link(&m_jit);
+    
+    // 14) Load the result of the last bytecode operation into regT0.
+    
+    if (exit.m_lastSetOperand != std::numeric_limits<int>::max())
+        m_jit.loadPtr(AssemblyHelpers::addressFor((VirtualRegister)exit.m_lastSetOperand), GPRInfo::cachedResultRegister);
+    
+    // 15) Fix call frame(s).
+    
+    ASSERT(m_jit.baselineCodeBlock()->getJITType() == JITCode::BaselineJIT);
+    m_jit.storePtr(AssemblyHelpers::TrustedImmPtr(m_jit.baselineCodeBlock()), AssemblyHelpers::addressFor((VirtualRegister)RegisterFile::CodeBlock));
+    
+    for (CodeOrigin codeOrigin = exit.m_codeOrigin; codeOrigin.inlineCallFrame; codeOrigin = codeOrigin.inlineCallFrame->caller) {
+        InlineCallFrame* inlineCallFrame = codeOrigin.inlineCallFrame;
+        CodeBlock* baselineCodeBlock = m_jit.baselineCodeBlockFor(codeOrigin);
+        CodeBlock* baselineCodeBlockForCaller = m_jit.baselineCodeBlockFor(inlineCallFrame->caller);
+        Vector<BytecodeAndMachineOffset>& decodedCodeMap = m_jit.decodedCodeMapFor(baselineCodeBlockForCaller);
+        unsigned returnBytecodeIndex = inlineCallFrame->caller.bytecodeIndex + OPCODE_LENGTH(op_call);
+        BytecodeAndMachineOffset* mapping = binarySearch<BytecodeAndMachineOffset, unsigned, BytecodeAndMachineOffset::getBytecodeIndex>(decodedCodeMap.begin(), decodedCodeMap.size(), returnBytecodeIndex);
+        
+        ASSERT(mapping);
+        ASSERT(mapping->m_bytecodeIndex == returnBytecodeIndex);
+        
+        void* jumpTarget = baselineCodeBlockForCaller->getJITCode().executableAddressAtOffset(mapping->m_machineCodeOffset);
+
+        GPRReg callerFrameGPR;
+        if (inlineCallFrame->caller.inlineCallFrame) {
+            m_jit.addPtr(AssemblyHelpers::Imm32(inlineCallFrame->caller.inlineCallFrame->stackOffset * sizeof(EncodedJSValue)), GPRInfo::callFrameRegister, GPRInfo::regT3);
+            callerFrameGPR = GPRInfo::regT3;
+        } else
+            callerFrameGPR = GPRInfo::callFrameRegister;
+        
+        m_jit.storePtr(AssemblyHelpers::TrustedImmPtr(baselineCodeBlock), AssemblyHelpers::addressFor((VirtualRegister)(inlineCallFrame->stackOffset + RegisterFile::CodeBlock)));
+        m_jit.storePtr(AssemblyHelpers::TrustedImmPtr(inlineCallFrame->callee->scope()), AssemblyHelpers::addressFor((VirtualRegister)(inlineCallFrame->stackOffset + RegisterFile::ScopeChain)));
+        m_jit.storePtr(callerFrameGPR, AssemblyHelpers::addressFor((VirtualRegister)(inlineCallFrame->stackOffset + RegisterFile::CallerFrame)));
+        m_jit.storePtr(AssemblyHelpers::TrustedImmPtr(jumpTarget), AssemblyHelpers::addressFor((VirtualRegister)(inlineCallFrame->stackOffset + RegisterFile::ReturnPC)));
+        m_jit.store32(AssemblyHelpers::TrustedImm32(inlineCallFrame->arguments.size()), AssemblyHelpers::payloadFor((VirtualRegister)(inlineCallFrame->stackOffset + RegisterFile::ArgumentCount)));
+        m_jit.storePtr(AssemblyHelpers::TrustedImmPtr(inlineCallFrame->callee.get()), AssemblyHelpers::addressFor((VirtualRegister)(inlineCallFrame->stackOffset + RegisterFile::Callee)));
+    }
+    
+    if (exit.m_codeOrigin.inlineCallFrame)
+        m_jit.addPtr(AssemblyHelpers::Imm32(exit.m_codeOrigin.inlineCallFrame->stackOffset * sizeof(EncodedJSValue)), GPRInfo::callFrameRegister);
+    
+    // 16) Jump into the corresponding baseline JIT code.
+    
+    CodeBlock* baselineCodeBlock = m_jit.baselineCodeBlockFor(exit.m_codeOrigin);
+    Vector<BytecodeAndMachineOffset>& decodedCodeMap = m_jit.decodedCodeMapFor(baselineCodeBlock);
+    
+    BytecodeAndMachineOffset* mapping = binarySearch<BytecodeAndMachineOffset, unsigned, BytecodeAndMachineOffset::getBytecodeIndex>(decodedCodeMap.begin(), decodedCodeMap.size(), exit.m_codeOrigin.bytecodeIndex);
+    
+    ASSERT(mapping);
+    ASSERT(mapping->m_bytecodeIndex == exit.m_codeOrigin.bytecodeIndex);
+    
+    void* jumpTarget = baselineCodeBlock->getJITCode().executableAddressAtOffset(mapping->m_machineCodeOffset);
+    
+    ASSERT(GPRInfo::regT1 != GPRInfo::cachedResultRegister);
+    
+    m_jit.move(AssemblyHelpers::TrustedImmPtr(jumpTarget), GPRInfo::regT1);
+    
+    m_jit.jump(GPRInfo::regT1);
+
+#if DFG_ENABLE(DEBUG_VERBOSE)
+    fprintf(stderr, "-> %p\n", jumpTarget);
+#endif
+}
+
+} } // namespace JSC::DFG
+
+#endif // ENABLE(DFG_JIT) && USE(JSVALUE64)