diff options
author | Lorry Tar Creator <lorry-tar-importer@lorry> | 2016-04-10 09:28:39 +0000 |
---|---|---|
committer | Lorry Tar Creator <lorry-tar-importer@lorry> | 2016-04-10 09:28:39 +0000 |
commit | 32761a6cee1d0dee366b885b7b9c777e67885688 (patch) | |
tree | d6bec92bebfb216f4126356e55518842c2f476a1 /Source/JavaScriptCore/assembler/MacroAssemblerARM64.h | |
parent | a4e969f4965059196ca948db781e52f7cfebf19e (diff) | |
download | WebKitGtk-tarball-32761a6cee1d0dee366b885b7b9c777e67885688.tar.gz |
webkitgtk-2.4.11webkitgtk-2.4.11
Diffstat (limited to 'Source/JavaScriptCore/assembler/MacroAssemblerARM64.h')
-rw-r--r-- | Source/JavaScriptCore/assembler/MacroAssemblerARM64.h | 1172 |
1 files changed, 132 insertions, 1040 deletions
diff --git a/Source/JavaScriptCore/assembler/MacroAssemblerARM64.h b/Source/JavaScriptCore/assembler/MacroAssemblerARM64.h index 42ac400fc..a128923fc 100644 --- a/Source/JavaScriptCore/assembler/MacroAssemblerARM64.h +++ b/Source/JavaScriptCore/assembler/MacroAssemblerARM64.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012, 2014, 2015 Apple Inc. All rights reserved. + * Copyright (C) 2012 Apple Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -31,22 +31,12 @@ #include "ARM64Assembler.h" #include "AbstractMacroAssembler.h" #include <wtf/MathExtras.h> -#include <wtf/Optional.h> namespace JSC { -class MacroAssemblerARM64 : public AbstractMacroAssembler<ARM64Assembler, MacroAssemblerARM64> { -public: +class MacroAssemblerARM64 : public AbstractMacroAssembler<ARM64Assembler> { static const RegisterID dataTempRegister = ARM64Registers::ip0; static const RegisterID memoryTempRegister = ARM64Registers::ip1; - - RegisterID scratchRegister() - { - RELEASE_ASSERT(m_allowScratchRegister); - return getCachedDataTempRegisterIDAndInvalidate(); - } - -private: static const ARM64Registers::FPRegisterID fpTempRegister = ARM64Registers::q31; static const ARM64Assembler::SetFlags S = ARM64Assembler::S; static const intptr_t maskHalfWord0 = 0xffffl; @@ -74,11 +64,13 @@ public: Vector<LinkRecord, 0, UnsafeVectorOverflow>& jumpsToLink() { return m_assembler.jumpsToLink(); } void* unlinkedCode() { return m_assembler.unlinkedCode(); } - static bool canCompact(JumpType jumpType) { return ARM64Assembler::canCompact(jumpType); } - static JumpLinkType computeJumpType(JumpType jumpType, const uint8_t* from, const uint8_t* to) { return ARM64Assembler::computeJumpType(jumpType, from, to); } - static JumpLinkType computeJumpType(LinkRecord& record, const uint8_t* from, const uint8_t* to) { return ARM64Assembler::computeJumpType(record, from, to); } - static int jumpSizeDelta(JumpType jumpType, JumpLinkType jumpLinkType) { return ARM64Assembler::jumpSizeDelta(jumpType, jumpLinkType); } - static void link(LinkRecord& record, uint8_t* from, uint8_t* to) { return ARM64Assembler::link(record, from, to); } + bool canCompact(JumpType jumpType) { return m_assembler.canCompact(jumpType); } + JumpLinkType computeJumpType(JumpType jumpType, const uint8_t* from, const uint8_t* to) { return m_assembler.computeJumpType(jumpType, from, to); } + JumpLinkType computeJumpType(LinkRecord& record, const uint8_t* from, const uint8_t* to) { return m_assembler.computeJumpType(record, from, to); } + void recordLinkOffsets(int32_t regionStart, int32_t regionEnd, int32_t offset) {return m_assembler.recordLinkOffsets(regionStart, regionEnd, offset); } + int jumpSizeDelta(JumpType jumpType, JumpLinkType jumpLinkType) { return m_assembler.jumpSizeDelta(jumpType, jumpLinkType); } + void link(LinkRecord& record, uint8_t* from, uint8_t* to) { return m_assembler.link(record, from, to); } + int executableOffsetFor(int location) { return m_assembler.executableOffsetFor(location); } static const Scale ScalePtr = TimesEight; @@ -138,15 +130,10 @@ public: // FIXME: Get reasonable implementations for these static bool shouldBlindForSpecificArch(uint32_t value) { return value >= 0x00ffffff; } static bool shouldBlindForSpecificArch(uint64_t value) { return value >= 0x00ffffff; } + static bool shouldBlindForSpecificArch(uintptr_t value) { return value >= 0x00ffffff; } // Integer operations: - void add32(RegisterID a, RegisterID b, RegisterID dest) - { - ASSERT(a != ARM64Registers::sp && b != ARM64Registers::sp); - m_assembler.add<32>(dest, a, b); - } - void add32(RegisterID src, RegisterID dest) { m_assembler.add<32>(dest, dest, src); @@ -212,20 +199,9 @@ public: add32(dataTempRegister, dest); } - void add64(RegisterID a, RegisterID b, RegisterID dest) - { - ASSERT(a != ARM64Registers::sp || b != ARM64Registers::sp); - if (b == ARM64Registers::sp) - std::swap(a, b); - m_assembler.add<64>(dest, a, b); - } - void add64(RegisterID src, RegisterID dest) { - if (src == ARM64Registers::sp) - m_assembler.add<64>(dest, src, dest); - else - m_assembler.add<64>(dest, dest, src); + m_assembler.add<64>(dest, dest, src); } void add64(TrustedImm32 imm, RegisterID dest) @@ -312,11 +288,6 @@ public: store64(dataTempRegister, address.m_ptr); } - void addPtrNoFlags(TrustedImm32 imm, RegisterID srcDest) - { - add64(imm, srcDest); - } - void add64(Address src, RegisterID dest) { load64(src, getCachedDataTempRegisterIDAndInvalidate()); @@ -363,24 +334,6 @@ public: and32(dataTempRegister, dest); } - void and64(RegisterID src1, RegisterID src2, RegisterID dest) - { - m_assembler.and_<64>(dest, src1, src2); - } - - void and64(TrustedImm64 imm, RegisterID src, RegisterID dest) - { - LogicalImmediate logicalImm = LogicalImmediate::create64(imm.m_value); - - if (logicalImm.isValid()) { - m_assembler.and_<64>(dest, src, logicalImm); - return; - } - - move(imm, getCachedDataTempRegisterIDAndInvalidate()); - m_assembler.and_<64>(dest, src, dataTempRegister); - } - void and64(RegisterID src, RegisterID dest) { m_assembler.and_<64>(dest, dest, src); @@ -417,11 +370,6 @@ public: m_assembler.clz<32>(dest, src); } - void countLeadingZeros64(RegisterID src, RegisterID dest) - { - m_assembler.clz<64>(dest, src); - } - void lshift32(RegisterID src, RegisterID shiftAmount, RegisterID dest) { m_assembler.lsl<32>(dest, src, shiftAmount); @@ -461,71 +409,21 @@ public: { lshift64(dest, imm, dest); } - - void mul32(RegisterID left, RegisterID right, RegisterID dest) - { - m_assembler.mul<32>(dest, left, right); - } void mul32(RegisterID src, RegisterID dest) { m_assembler.mul<32>(dest, dest, src); } - - void mul32(TrustedImm32 imm, RegisterID src, RegisterID dest) - { - move(imm, getCachedDataTempRegisterIDAndInvalidate()); - m_assembler.mul<32>(dest, src, dataTempRegister); - } - + void mul64(RegisterID src, RegisterID dest) { m_assembler.mul<64>(dest, dest, src); } - void mul64(RegisterID left, RegisterID right, RegisterID dest) - { - m_assembler.mul<64>(dest, left, right); - } - - void multiplyAdd32(RegisterID mulLeft, RegisterID mulRight, RegisterID summand, RegisterID dest) - { - m_assembler.madd<32>(dest, mulLeft, mulRight, summand); - } - - void multiplySub32(RegisterID mulLeft, RegisterID mulRight, RegisterID minuend, RegisterID dest) - { - m_assembler.msub<32>(dest, mulLeft, mulRight, minuend); - } - - void multiplyNeg32(RegisterID mulLeft, RegisterID mulRight, RegisterID dest) - { - m_assembler.msub<32>(dest, mulLeft, mulRight, ARM64Registers::zr); - } - - void multiplyAdd64(RegisterID mulLeft, RegisterID mulRight, RegisterID summand, RegisterID dest) - { - m_assembler.madd<64>(dest, mulLeft, mulRight, summand); - } - - void multiplySub64(RegisterID mulLeft, RegisterID mulRight, RegisterID minuend, RegisterID dest) - { - m_assembler.msub<64>(dest, mulLeft, mulRight, minuend); - } - - void multiplyNeg64(RegisterID mulLeft, RegisterID mulRight, RegisterID dest) - { - m_assembler.msub<64>(dest, mulLeft, mulRight, ARM64Registers::zr); - } - - void div32(RegisterID dividend, RegisterID divisor, RegisterID dest) - { - m_assembler.sdiv<32>(dest, dividend, divisor); - } - - void div64(RegisterID dividend, RegisterID divisor, RegisterID dest) + void mul32(TrustedImm32 imm, RegisterID src, RegisterID dest) { - m_assembler.sdiv<64>(dest, dividend, divisor); + move(imm, getCachedDataTempRegisterIDAndInvalidate()); + m_assembler.mul<32>(dest, src, dataTempRegister); } void neg32(RegisterID dest) @@ -562,7 +460,6 @@ public: return; } - ASSERT(src != dataTempRegister); move(imm, getCachedDataTempRegisterIDAndInvalidate()); m_assembler.orr<32>(dest, src, dataTempRegister); } @@ -574,27 +471,6 @@ public: store32(dataTempRegister, address.m_ptr); } - void or32(TrustedImm32 imm, AbsoluteAddress address) - { - LogicalImmediate logicalImm = LogicalImmediate::create32(imm.m_value); - if (logicalImm.isValid()) { - load32(address.m_ptr, getCachedDataTempRegisterIDAndInvalidate()); - m_assembler.orr<32>(dataTempRegister, dataTempRegister, logicalImm); - store32(dataTempRegister, address.m_ptr); - } else { - load32(address.m_ptr, getCachedMemoryTempRegisterIDAndInvalidate()); - or32(imm, memoryTempRegister, getCachedDataTempRegisterIDAndInvalidate()); - store32(dataTempRegister, address.m_ptr); - } - } - - void or32(TrustedImm32 imm, Address address) - { - load32(address, getCachedDataTempRegisterIDAndInvalidate()); - or32(imm, dataTempRegister, dataTempRegister); - store32(dataTempRegister, address); - } - void or64(RegisterID src, RegisterID dest) { or64(dest, src, dest); @@ -615,27 +491,14 @@ public: LogicalImmediate logicalImm = LogicalImmediate::create64(static_cast<intptr_t>(static_cast<int64_t>(imm.m_value))); if (logicalImm.isValid()) { - m_assembler.orr<64>(dest, src, logicalImm); + m_assembler.orr<64>(dest, dest, logicalImm); return; } signExtend32ToPtr(imm, getCachedDataTempRegisterIDAndInvalidate()); m_assembler.orr<64>(dest, src, dataTempRegister); } - - void or64(TrustedImm64 imm, RegisterID src, RegisterID dest) - { - LogicalImmediate logicalImm = LogicalImmediate::create64(imm.m_value); - - if (logicalImm.isValid()) { - m_assembler.orr<64>(dest, src, logicalImm); - return; - } - - move(imm, getCachedDataTempRegisterIDAndInvalidate()); - m_assembler.orr<64>(dest, src, dataTempRegister); - } - + void or64(TrustedImm64 imm, RegisterID dest) { LogicalImmediate logicalImm = LogicalImmediate::create64(static_cast<intptr_t>(static_cast<int64_t>(imm.m_value))); @@ -676,12 +539,12 @@ public: void rshift64(RegisterID src, RegisterID shiftAmount, RegisterID dest) { - m_assembler.asr<64>(dest, src, shiftAmount); + m_assembler.lsr<64>(dest, src, shiftAmount); } void rshift64(RegisterID src, TrustedImm32 imm, RegisterID dest) { - m_assembler.asr<64>(dest, src, imm.m_value & 0x3f); + m_assembler.lsr<64>(dest, src, imm.m_value & 0x3f); } void rshift64(RegisterID shiftAmount, RegisterID dest) @@ -814,26 +677,6 @@ public: urshift32(dest, imm, dest); } - void urshift64(RegisterID src, RegisterID shiftAmount, RegisterID dest) - { - m_assembler.lsr<64>(dest, src, shiftAmount); - } - - void urshift64(RegisterID src, TrustedImm32 imm, RegisterID dest) - { - m_assembler.lsr<64>(dest, src, imm.m_value & 0x3f); - } - - void urshift64(RegisterID shiftAmount, RegisterID dest) - { - urshift64(dest, shiftAmount, dest); - } - - void urshift64(TrustedImm32 imm, RegisterID dest) - { - urshift64(dest, imm, dest); - } - void xor32(RegisterID src, RegisterID dest) { xor32(dest, src, dest); @@ -857,7 +700,7 @@ public: LogicalImmediate logicalImm = LogicalImmediate::create32(imm.m_value); if (logicalImm.isValid()) { - m_assembler.eor<32>(dest, src, logicalImm); + m_assembler.eor<32>(dest, dest, logicalImm); return; } @@ -888,23 +731,6 @@ public: xor64(imm, dest, dest); } - void xor64(TrustedImm64 imm, RegisterID src, RegisterID dest) - { - if (imm.m_value == -1) - m_assembler.mvn<64>(dest, src); - else { - LogicalImmediate logicalImm = LogicalImmediate::create64(imm.m_value); - - if (logicalImm.isValid()) { - m_assembler.eor<64>(dest, src, logicalImm); - return; - } - - move(imm, getCachedDataTempRegisterIDAndInvalidate()); - m_assembler.eor<64>(dest, src, dataTempRegister); - } - } - void xor64(TrustedImm32 imm, RegisterID src, RegisterID dest) { if (imm.m_value == -1) @@ -913,7 +739,7 @@ public: LogicalImmediate logicalImm = LogicalImmediate::create64(static_cast<intptr_t>(static_cast<int64_t>(imm.m_value))); if (logicalImm.isValid()) { - m_assembler.eor<64>(dest, src, logicalImm); + m_assembler.eor<64>(dest, dest, logicalImm); return; } @@ -922,15 +748,6 @@ public: } } - void not32(RegisterID src, RegisterID dest) - { - m_assembler.mvn<32>(dest, src); - } - - void not64(RegisterID src, RegisterID dest) - { - m_assembler.mvn<64>(dest, src); - } // Memory access operations: @@ -976,18 +793,6 @@ public: return label; } - void abortWithReason(AbortReason reason) - { - move(TrustedImm32(reason), dataTempRegister); - breakpoint(); - } - - void abortWithReason(AbortReason reason, intptr_t misc) - { - move(TrustedImm64(misc), memoryTempRegister); - abortWithReason(reason); - } - ConvertibleLoadLabel convertibleLoadPtr(Address address, RegisterID dest) { ConvertibleLoadLabel result(this); @@ -1069,35 +874,16 @@ public: load16(address, dest); } - void load16SignedExtendTo32(ImplicitAddress address, RegisterID dest) - { - if (tryLoadSignedWithOffset<16>(dest, address.base, address.offset)) - return; - - signExtend32ToPtr(TrustedImm32(address.offset), getCachedMemoryTempRegisterIDAndInvalidate()); - m_assembler.ldrsh<32>(dest, address.base, memoryTempRegister); - } - - void load16SignedExtendTo32(BaseIndex address, RegisterID dest) + void load16Signed(BaseIndex address, RegisterID dest) { if (!address.offset && (!address.scale || address.scale == 1)) { - m_assembler.ldrsh<32>(dest, address.base, address.index, ARM64Assembler::UXTX, address.scale); + m_assembler.ldrsh<64>(dest, address.base, address.index, ARM64Assembler::UXTX, address.scale); return; } signExtend32ToPtr(TrustedImm32(address.offset), getCachedMemoryTempRegisterIDAndInvalidate()); m_assembler.add<64>(memoryTempRegister, memoryTempRegister, address.index, ARM64Assembler::UXTX, address.scale); - m_assembler.ldrsh<32>(dest, address.base, memoryTempRegister); - } - - void zeroExtend16To32(RegisterID src, RegisterID dest) - { - m_assembler.uxth<64>(dest, src); - } - - void signExtend16To32(RegisterID src, RegisterID dest) - { - m_assembler.sxth<64>(dest, src); + m_assembler.ldrsh<64>(dest, address.base, memoryTempRegister); } void load8(ImplicitAddress address, RegisterID dest) @@ -1123,41 +909,22 @@ public: void load8(const void* address, RegisterID dest) { - moveToCachedReg(TrustedImmPtr(address), cachedMemoryTempRegister()); + moveToCachedReg(TrustedImmPtr(address), m_cachedMemoryTempRegister); m_assembler.ldrb(dest, memoryTempRegister, ARM64Registers::zr); if (dest == memoryTempRegister) - cachedMemoryTempRegister().invalidate(); - } - - void load8SignedExtendTo32(ImplicitAddress address, RegisterID dest) - { - if (tryLoadSignedWithOffset<8>(dest, address.base, address.offset)) - return; - - signExtend32ToPtr(TrustedImm32(address.offset), getCachedMemoryTempRegisterIDAndInvalidate()); - m_assembler.ldrsb<32>(dest, address.base, memoryTempRegister); + m_cachedMemoryTempRegister.invalidate(); } - void load8SignedExtendTo32(BaseIndex address, RegisterID dest) + void load8Signed(BaseIndex address, RegisterID dest) { if (!address.offset && !address.scale) { - m_assembler.ldrsb<32>(dest, address.base, address.index, ARM64Assembler::UXTX, address.scale); + m_assembler.ldrsb<64>(dest, address.base, address.index, ARM64Assembler::UXTX, address.scale); return; } signExtend32ToPtr(TrustedImm32(address.offset), getCachedMemoryTempRegisterIDAndInvalidate()); m_assembler.add<64>(memoryTempRegister, memoryTempRegister, address.index, ARM64Assembler::UXTX, address.scale); - m_assembler.ldrsb<32>(dest, address.base, memoryTempRegister); - } - - void zeroExtend8To32(RegisterID src, RegisterID dest) - { - m_assembler.uxtb<64>(dest, src); - } - - void signExtend8To32(RegisterID src, RegisterID dest) - { - m_assembler.sxtb<64>(dest, src); + m_assembler.ldrsb<64>(dest, address.base, memoryTempRegister); } void store64(RegisterID src, ImplicitAddress address) @@ -1186,11 +953,6 @@ public: store<64>(src, address); } - void store64(TrustedImm32 imm, ImplicitAddress address) - { - store64(TrustedImm64(imm.m_value), address); - } - void store64(TrustedImm64 imm, ImplicitAddress address) { if (!imm.m_value) { @@ -1198,7 +960,7 @@ public: return; } - moveToCachedReg(imm, dataMemoryTempRegister()); + moveToCachedReg(imm, m_dataMemoryTempRegister); store64(dataTempRegister, address); } @@ -1209,7 +971,7 @@ public: return; } - moveToCachedReg(imm, dataMemoryTempRegister()); + moveToCachedReg(imm, m_dataMemoryTempRegister); store64(dataTempRegister, address); } @@ -1254,7 +1016,7 @@ public: return; } - moveToCachedReg(imm, dataMemoryTempRegister()); + moveToCachedReg(imm, m_dataMemoryTempRegister); store32(dataTempRegister, address); } @@ -1265,7 +1027,7 @@ public: return; } - moveToCachedReg(imm, dataMemoryTempRegister()); + moveToCachedReg(imm, m_dataMemoryTempRegister); store32(dataTempRegister, address); } @@ -1276,20 +1038,10 @@ public: return; } - moveToCachedReg(imm, dataMemoryTempRegister()); + moveToCachedReg(imm, m_dataMemoryTempRegister); store32(dataTempRegister, address); } - void storeZero32(ImplicitAddress address) - { - store32(ARM64Registers::zr, address); - } - - void storeZero32(BaseIndex address) - { - store32(ARM64Registers::zr, address); - } - DataLabel32 store32WithAddressOffsetPatch(RegisterID src, Address address) { DataLabel32 label(this); @@ -1298,15 +1050,6 @@ public: return label; } - void store16(RegisterID src, ImplicitAddress address) - { - if (tryStoreWithOffset<16>(src, address.base, address.offset)) - return; - - signExtend32ToPtr(TrustedImm32(address.offset), getCachedMemoryTempRegisterIDAndInvalidate()); - m_assembler.str<16>(src, address.base, memoryTempRegister); - } - void store16(RegisterID src, BaseIndex address) { if (!address.offset && (!address.scale || address.scale == 1)) { @@ -1337,15 +1080,6 @@ public: m_assembler.strb(src, memoryTempRegister, 0); } - void store8(RegisterID src, ImplicitAddress address) - { - if (tryStoreWithOffset<8>(src, address.base, address.offset)) - return; - - signExtend32ToPtr(TrustedImm32(address.offset), getCachedMemoryTempRegisterIDAndInvalidate()); - m_assembler.str<8>(src, address.base, memoryTempRegister); - } - void store8(TrustedImm32 imm, void* address) { if (!imm.m_value) { @@ -1357,16 +1091,6 @@ public: store8(dataTempRegister, address); } - void store8(TrustedImm32 imm, ImplicitAddress address) - { - if (!imm.m_value) { - store8(ARM64Registers::zr, address); - return; - } - - move(imm, getCachedDataTempRegisterIDAndInvalidate()); - store8(dataTempRegister, address); - } // Floating-point operations: @@ -1374,7 +1098,6 @@ public: static bool supportsFloatingPointTruncate() { return true; } static bool supportsFloatingPointSqrt() { return true; } static bool supportsFloatingPointAbs() { return true; } - static bool supportsFloatingPointRounding() { return true; } enum BranchTruncateType { BranchIfTruncateFailed, BranchIfTruncateSuccessful }; @@ -1383,11 +1106,6 @@ public: m_assembler.fabs<64>(dest, src); } - void absFloat(FPRegisterID src, FPRegisterID dest) - { - m_assembler.fabs<32>(dest, src); - } - void addDouble(FPRegisterID src, FPRegisterID dest) { addDouble(dest, src, dest); @@ -1406,35 +1124,20 @@ public: void addDouble(AbsoluteAddress address, FPRegisterID dest) { - loadDouble(TrustedImmPtr(address.m_ptr), fpTempRegister); + loadDouble(address.m_ptr, fpTempRegister); addDouble(fpTempRegister, dest); } - void addFloat(FPRegisterID op1, FPRegisterID op2, FPRegisterID dest) - { - m_assembler.fadd<32>(dest, op1, op2); - } - void ceilDouble(FPRegisterID src, FPRegisterID dest) { m_assembler.frintp<64>(dest, src); } - void ceilFloat(FPRegisterID src, FPRegisterID dest) - { - m_assembler.frintp<32>(dest, src); - } - void floorDouble(FPRegisterID src, FPRegisterID dest) { m_assembler.frintm<64>(dest, src); } - void floorFloat(FPRegisterID src, FPRegisterID dest) - { - m_assembler.frintm<32>(dest, src); - } - // Convert 'src' to an integer, and places the resulting 'dest'. // If the result is not representable as a 32 bit value, branch. // May also branch for some values that are representable in 32 bits @@ -1447,26 +1150,32 @@ public: m_assembler.scvtf<64, 32>(fpTempRegister, dest); failureCases.append(branchDouble(DoubleNotEqualOrUnordered, src, fpTempRegister)); - // Test for negative zero. - if (negZeroCheck) { - Jump valueIsNonZero = branchTest32(NonZero, dest); - RegisterID scratch = getCachedMemoryTempRegisterIDAndInvalidate(); - m_assembler.fmov<64>(scratch, src); - failureCases.append(makeTestBitAndBranch(scratch, 63, IsNonZero)); - valueIsNonZero.link(this); - } + // If the result is zero, it might have been -0.0, and the double comparison won't catch this! + if (negZeroCheck) + failureCases.append(branchTest32(Zero, dest)); } Jump branchDouble(DoubleCondition cond, FPRegisterID left, FPRegisterID right) { m_assembler.fcmp<64>(left, right); - return jumpAfterFloatingPointCompare(cond); - } - Jump branchFloat(DoubleCondition cond, FPRegisterID left, FPRegisterID right) - { - m_assembler.fcmp<32>(left, right); - return jumpAfterFloatingPointCompare(cond); + if (cond == DoubleNotEqual) { + // ConditionNE jumps if NotEqual *or* unordered - force the unordered cases not to jump. + Jump unordered = makeBranch(ARM64Assembler::ConditionVS); + Jump result = makeBranch(ARM64Assembler::ConditionNE); + unordered.link(this); + return result; + } + if (cond == DoubleEqualOrUnordered) { + Jump unordered = makeBranch(ARM64Assembler::ConditionVS); + Jump notEqual = makeBranch(ARM64Assembler::ConditionNE); + unordered.link(this); + // We get here if either unordered or equal. + Jump result = jump(); + notEqual.link(this); + return result; + } + return makeBranch(cond); } Jump branchDoubleNonZero(FPRegisterID reg, FPRegisterID) @@ -1548,11 +1257,6 @@ public: m_assembler.fdiv<64>(dest, op1, op2); } - void divFloat(FPRegisterID op1, FPRegisterID op2, FPRegisterID dest) - { - m_assembler.fdiv<32>(dest, op1, op2); - } - void loadDouble(ImplicitAddress address, FPRegisterID dest) { if (tryLoadWithOffset<64>(dest, address.base, address.offset)) @@ -1574,21 +1278,12 @@ public: m_assembler.ldr<64>(dest, address.base, memoryTempRegister); } - void loadDouble(TrustedImmPtr address, FPRegisterID dest) + void loadDouble(const void* address, FPRegisterID dest) { - moveToCachedReg(address, cachedMemoryTempRegister()); + moveToCachedReg(TrustedImmPtr(address), m_cachedMemoryTempRegister); m_assembler.ldr<64>(dest, memoryTempRegister, ARM64Registers::zr); } - void loadFloat(ImplicitAddress address, FPRegisterID dest) - { - if (tryLoadWithOffset<32>(dest, address.base, address.offset)) - return; - - signExtend32ToPtr(TrustedImm32(address.offset), getCachedMemoryTempRegisterIDAndInvalidate()); - m_assembler.ldr<32>(dest, address.base, memoryTempRegister); - } - void loadFloat(BaseIndex address, FPRegisterID dest) { if (!address.offset && (!address.scale || address.scale == 2)) { @@ -1606,130 +1301,16 @@ public: m_assembler.fmov<64>(dest, src); } - void moveZeroToDouble(FPRegisterID reg) - { - m_assembler.fmov<64>(reg, ARM64Registers::zr); - } - void moveDoubleTo64(FPRegisterID src, RegisterID dest) { m_assembler.fmov<64>(dest, src); } - void moveFloatTo32(FPRegisterID src, RegisterID dest) - { - m_assembler.fmov<32>(dest, src); - } - void move64ToDouble(RegisterID src, FPRegisterID dest) { m_assembler.fmov<64>(dest, src); } - void move32ToFloat(RegisterID src, FPRegisterID dest) - { - m_assembler.fmov<32>(dest, src); - } - - void moveConditionallyDouble(DoubleCondition cond, FPRegisterID left, FPRegisterID right, RegisterID src, RegisterID dest) - { - m_assembler.fcmp<64>(left, right); - moveConditionallyAfterFloatingPointCompare<64>(cond, src, dest); - } - - void moveConditionallyDouble(DoubleCondition cond, FPRegisterID left, FPRegisterID right, RegisterID thenCase, RegisterID elseCase, RegisterID dest) - { - m_assembler.fcmp<64>(left, right); - moveConditionallyAfterFloatingPointCompare<64>(cond, thenCase, elseCase, dest); - } - - void moveConditionallyFloat(DoubleCondition cond, FPRegisterID left, FPRegisterID right, RegisterID src, RegisterID dest) - { - m_assembler.fcmp<32>(left, right); - moveConditionallyAfterFloatingPointCompare<64>(cond, src, dest); - } - - void moveConditionallyFloat(DoubleCondition cond, FPRegisterID left, FPRegisterID right, RegisterID thenCase, RegisterID elseCase, RegisterID dest) - { - m_assembler.fcmp<32>(left, right); - moveConditionallyAfterFloatingPointCompare<64>(cond, thenCase, elseCase, dest); - } - - template<int datasize> - void moveConditionallyAfterFloatingPointCompare(DoubleCondition cond, RegisterID src, RegisterID dest) - { - if (cond == DoubleNotEqual) { - Jump unordered = makeBranch(ARM64Assembler::ConditionVS); - m_assembler.csel<datasize>(dest, src, dest, ARM64Assembler::ConditionNE); - unordered.link(this); - return; - } - if (cond == DoubleEqualOrUnordered) { - // If the compare is unordered, src is copied to dest and the - // next csel has all arguments equal to src. - // If the compare is ordered, dest is unchanged and EQ decides - // what value to set. - m_assembler.csel<datasize>(dest, src, dest, ARM64Assembler::ConditionVS); - m_assembler.csel<datasize>(dest, src, dest, ARM64Assembler::ConditionEQ); - return; - } - m_assembler.csel<datasize>(dest, src, dest, ARM64Condition(cond)); - } - - template<int datasize> - void moveConditionallyAfterFloatingPointCompare(DoubleCondition cond, RegisterID thenCase, RegisterID elseCase, RegisterID dest) - { - if (cond == DoubleNotEqual) { - Jump unordered = makeBranch(ARM64Assembler::ConditionVS); - m_assembler.csel<datasize>(dest, thenCase, elseCase, ARM64Assembler::ConditionNE); - unordered.link(this); - return; - } - if (cond == DoubleEqualOrUnordered) { - // If the compare is unordered, thenCase is copied to elseCase and the - // next csel has all arguments equal to thenCase. - // If the compare is ordered, dest is unchanged and EQ decides - // what value to set. - m_assembler.csel<datasize>(elseCase, thenCase, elseCase, ARM64Assembler::ConditionVS); - m_assembler.csel<datasize>(dest, thenCase, elseCase, ARM64Assembler::ConditionEQ); - return; - } - m_assembler.csel<datasize>(dest, thenCase, elseCase, ARM64Condition(cond)); - } - - template<int datasize> - void moveDoubleConditionallyAfterFloatingPointCompare(DoubleCondition cond, FPRegisterID thenCase, FPRegisterID elseCase, FPRegisterID dest) - { - if (cond == DoubleNotEqual) { - Jump unordered = makeBranch(ARM64Assembler::ConditionVS); - m_assembler.fcsel<datasize>(dest, thenCase, elseCase, ARM64Assembler::ConditionNE); - unordered.link(this); - return; - } - if (cond == DoubleEqualOrUnordered) { - // If the compare is unordered, thenCase is copied to elseCase and the - // next csel has all arguments equal to thenCase. - // If the compare is ordered, dest is unchanged and EQ decides - // what value to set. - m_assembler.fcsel<datasize>(elseCase, thenCase, elseCase, ARM64Assembler::ConditionVS); - m_assembler.fcsel<datasize>(dest, thenCase, elseCase, ARM64Assembler::ConditionEQ); - return; - } - m_assembler.fcsel<datasize>(dest, thenCase, elseCase, ARM64Condition(cond)); - } - - void moveDoubleConditionallyDouble(DoubleCondition cond, FPRegisterID left, FPRegisterID right, FPRegisterID thenCase, FPRegisterID elseCase, FPRegisterID dest) - { - m_assembler.fcmp<64>(left, right); - moveDoubleConditionallyAfterFloatingPointCompare<64>(cond, thenCase, elseCase, dest); - } - - void moveDoubleConditionallyFloat(DoubleCondition cond, FPRegisterID left, FPRegisterID right, FPRegisterID thenCase, FPRegisterID elseCase, FPRegisterID dest) - { - m_assembler.fcmp<32>(left, right); - moveDoubleConditionallyAfterFloatingPointCompare<64>(cond, thenCase, elseCase, dest); - } - void mulDouble(FPRegisterID src, FPRegisterID dest) { mulDouble(dest, src, dest); @@ -1746,21 +1327,6 @@ public: mulDouble(fpTempRegister, dest); } - void mulFloat(FPRegisterID op1, FPRegisterID op2, FPRegisterID dest) - { - m_assembler.fmul<32>(dest, op1, op2); - } - - void andDouble(FPRegisterID op1, FPRegisterID op2, FPRegisterID dest) - { - m_assembler.vand<64>(dest, op1, op2); - } - - void andFloat(FPRegisterID op1, FPRegisterID op2, FPRegisterID dest) - { - andDouble(op1, op2, dest); - } - void negateDouble(FPRegisterID src, FPRegisterID dest) { m_assembler.fneg<64>(dest, src); @@ -1771,11 +1337,6 @@ public: m_assembler.fsqrt<64>(dest, src); } - void sqrtFloat(FPRegisterID src, FPRegisterID dest) - { - m_assembler.fsqrt<32>(dest, src); - } - void storeDouble(FPRegisterID src, ImplicitAddress address) { if (tryStoreWithOffset<64>(src, address.base, address.offset)) @@ -1785,9 +1346,9 @@ public: m_assembler.str<64>(src, address.base, memoryTempRegister); } - void storeDouble(FPRegisterID src, TrustedImmPtr address) + void storeDouble(FPRegisterID src, const void* address) { - moveToCachedReg(address, cachedMemoryTempRegister()); + moveToCachedReg(TrustedImmPtr(address), m_cachedMemoryTempRegister); m_assembler.str<64>(src, memoryTempRegister, ARM64Registers::zr); } @@ -1802,15 +1363,6 @@ public: m_assembler.add<64>(memoryTempRegister, memoryTempRegister, address.index, ARM64Assembler::UXTX, address.scale); m_assembler.str<64>(src, address.base, memoryTempRegister); } - - void storeFloat(FPRegisterID src, ImplicitAddress address) - { - if (tryStoreWithOffset<32>(src, address.base, address.offset)) - return; - - signExtend32ToPtr(TrustedImm32(address.offset), getCachedMemoryTempRegisterIDAndInvalidate()); - m_assembler.str<32>(src, address.base, memoryTempRegister); - } void storeFloat(FPRegisterID src, BaseIndex address) { @@ -1840,11 +1392,6 @@ public: subDouble(fpTempRegister, dest); } - void subFloat(FPRegisterID op1, FPRegisterID op2, FPRegisterID dest) - { - m_assembler.fsub<32>(dest, op1, op2); - } - // Result is undefined if the value is outside of the integer range. void truncateDoubleToInt32(FPRegisterID src, RegisterID dest) { @@ -1890,16 +1437,6 @@ public: CRASH(); } - void popPair(RegisterID dest1, RegisterID dest2) - { - m_assembler.ldp<64>(dest1, dest2, ARM64Registers::sp, PairPostIndex(16)); - } - - void pushPair(RegisterID src1, RegisterID src2) - { - m_assembler.stp<64>(src1, src2, ARM64Registers::sp, PairPreIndex(-16)); - } - void popToRestore(RegisterID dest) { m_assembler.ldr<64>(dest, ARM64Registers::sp, PostIndex(16)); @@ -1909,15 +1446,6 @@ public: { m_assembler.str<64>(src, ARM64Registers::sp, PreIndex(-16)); } - - void pushToSaveImmediateWithoutTouchingRegisters(TrustedImm32 imm) - { - RegisterID reg = dataTempRegister; - pushPair(reg, reg); - move(imm, reg); - store64(reg, stackPointerRegister); - load64(Address(stackPointerRegister, 8), reg); - } void pushToSave(Address address) { @@ -1943,7 +1471,6 @@ public: storeDouble(src, stackPointerRegister); } - static ptrdiff_t pushToSaveByteOffset() { return 16; } // Register move operations: @@ -1974,11 +1501,6 @@ public: move(reg2, reg1); move(dataTempRegister, reg2); } - - void signExtend32ToPtr(TrustedImm32 imm, RegisterID dest) - { - move(TrustedImmPtr(reinterpret_cast<void*>(static_cast<intptr_t>(imm.m_value))), dest); - } void signExtend32ToPtr(RegisterID src, RegisterID dest) { @@ -1990,169 +1512,6 @@ public: m_assembler.uxtw(dest, src); } - void moveConditionally32(RelationalCondition cond, RegisterID left, RegisterID right, RegisterID src, RegisterID dest) - { - m_assembler.cmp<32>(left, right); - m_assembler.csel<32>(dest, src, dest, ARM64Condition(cond)); - } - - void moveConditionally32(RelationalCondition cond, RegisterID left, RegisterID right, RegisterID thenCase, RegisterID elseCase, RegisterID dest) - { - m_assembler.cmp<32>(left, right); - m_assembler.csel<32>(dest, thenCase, elseCase, ARM64Condition(cond)); - } - - void moveConditionally32(RelationalCondition cond, RegisterID left, TrustedImm32 right, RegisterID thenCase, RegisterID elseCase, RegisterID dest) - { - if (!right.m_value) { - if (auto resultCondition = commuteCompareToZeroIntoTest(cond)) { - moveConditionallyTest32(*resultCondition, left, left, thenCase, elseCase, dest); - return; - } - } - - if (isUInt12(right.m_value)) - m_assembler.cmp<32>(left, UInt12(right.m_value)); - else if (isUInt12(-right.m_value)) - m_assembler.cmn<32>(left, UInt12(-right.m_value)); - else { - moveToCachedReg(right, dataMemoryTempRegister()); - m_assembler.cmp<32>(left, dataTempRegister); - } - m_assembler.csel<64>(dest, thenCase, elseCase, ARM64Condition(cond)); - } - - void moveConditionally64(RelationalCondition cond, RegisterID left, RegisterID right, RegisterID src, RegisterID dest) - { - m_assembler.cmp<64>(left, right); - m_assembler.csel<64>(dest, src, dest, ARM64Condition(cond)); - } - - void moveConditionally64(RelationalCondition cond, RegisterID left, RegisterID right, RegisterID thenCase, RegisterID elseCase, RegisterID dest) - { - m_assembler.cmp<64>(left, right); - m_assembler.csel<64>(dest, thenCase, elseCase, ARM64Condition(cond)); - } - - void moveConditionally64(RelationalCondition cond, RegisterID left, TrustedImm32 right, RegisterID thenCase, RegisterID elseCase, RegisterID dest) - { - if (!right.m_value) { - if (auto resultCondition = commuteCompareToZeroIntoTest(cond)) { - moveConditionallyTest64(*resultCondition, left, left, thenCase, elseCase, dest); - return; - } - } - - if (isUInt12(right.m_value)) - m_assembler.cmp<64>(left, UInt12(right.m_value)); - else if (isUInt12(-right.m_value)) - m_assembler.cmn<64>(left, UInt12(-right.m_value)); - else { - moveToCachedReg(right, dataMemoryTempRegister()); - m_assembler.cmp<64>(left, dataTempRegister); - } - m_assembler.csel<64>(dest, thenCase, elseCase, ARM64Condition(cond)); - } - - void moveConditionallyTest32(ResultCondition cond, RegisterID testReg, RegisterID mask, RegisterID src, RegisterID dest) - { - m_assembler.tst<32>(testReg, mask); - m_assembler.csel<32>(dest, src, dest, ARM64Condition(cond)); - } - - void moveConditionallyTest32(ResultCondition cond, RegisterID left, RegisterID right, RegisterID thenCase, RegisterID elseCase, RegisterID dest) - { - m_assembler.tst<32>(left, right); - m_assembler.csel<32>(dest, thenCase, elseCase, ARM64Condition(cond)); - } - - void moveConditionallyTest32(ResultCondition cond, RegisterID left, TrustedImm32 right, RegisterID thenCase, RegisterID elseCase, RegisterID dest) - { - test32(left, right); - m_assembler.csel<32>(dest, thenCase, elseCase, ARM64Condition(cond)); - } - - void moveConditionallyTest64(ResultCondition cond, RegisterID testReg, RegisterID mask, RegisterID src, RegisterID dest) - { - m_assembler.tst<64>(testReg, mask); - m_assembler.csel<64>(dest, src, dest, ARM64Condition(cond)); - } - - void moveConditionallyTest64(ResultCondition cond, RegisterID left, RegisterID right, RegisterID thenCase, RegisterID elseCase, RegisterID dest) - { - m_assembler.tst<64>(left, right); - m_assembler.csel<64>(dest, thenCase, elseCase, ARM64Condition(cond)); - } - - void moveDoubleConditionally32(RelationalCondition cond, RegisterID left, RegisterID right, FPRegisterID thenCase, FPRegisterID elseCase, FPRegisterID dest) - { - m_assembler.cmp<32>(left, right); - m_assembler.fcsel<32>(dest, thenCase, elseCase, ARM64Condition(cond)); - } - - void moveDoubleConditionally32(RelationalCondition cond, RegisterID left, TrustedImm32 right, FPRegisterID thenCase, FPRegisterID elseCase, FPRegisterID dest) - { - if (!right.m_value) { - if (auto resultCondition = commuteCompareToZeroIntoTest(cond)) { - moveDoubleConditionallyTest32(*resultCondition, left, left, thenCase, elseCase, dest); - return; - } - } - - if (isUInt12(right.m_value)) - m_assembler.cmp<32>(left, UInt12(right.m_value)); - else if (isUInt12(-right.m_value)) - m_assembler.cmn<32>(left, UInt12(-right.m_value)); - else { - moveToCachedReg(right, dataMemoryTempRegister()); - m_assembler.cmp<32>(left, dataTempRegister); - } - m_assembler.fcsel<64>(dest, thenCase, elseCase, ARM64Condition(cond)); - } - - void moveDoubleConditionally64(RelationalCondition cond, RegisterID left, RegisterID right, FPRegisterID thenCase, FPRegisterID elseCase, FPRegisterID dest) - { - m_assembler.cmp<64>(left, right); - m_assembler.fcsel<64>(dest, thenCase, elseCase, ARM64Condition(cond)); - } - - void moveDoubleConditionally64(RelationalCondition cond, RegisterID left, TrustedImm32 right, FPRegisterID thenCase, FPRegisterID elseCase, FPRegisterID dest) - { - if (!right.m_value) { - if (auto resultCondition = commuteCompareToZeroIntoTest(cond)) { - moveDoubleConditionallyTest64(*resultCondition, left, left, thenCase, elseCase, dest); - return; - } - } - - if (isUInt12(right.m_value)) - m_assembler.cmp<64>(left, UInt12(right.m_value)); - else if (isUInt12(-right.m_value)) - m_assembler.cmn<64>(left, UInt12(-right.m_value)); - else { - moveToCachedReg(right, dataMemoryTempRegister()); - m_assembler.cmp<64>(left, dataTempRegister); - } - m_assembler.fcsel<64>(dest, thenCase, elseCase, ARM64Condition(cond)); - } - - void moveDoubleConditionallyTest32(ResultCondition cond, RegisterID left, RegisterID right, FPRegisterID thenCase, FPRegisterID elseCase, FPRegisterID dest) - { - m_assembler.tst<32>(left, right); - m_assembler.fcsel<64>(dest, thenCase, elseCase, ARM64Condition(cond)); - } - - void moveDoubleConditionallyTest32(ResultCondition cond, RegisterID left, TrustedImm32 right, FPRegisterID thenCase, FPRegisterID elseCase, FPRegisterID dest) - { - test32(left, right); - m_assembler.fcsel<64>(dest, thenCase, elseCase, ARM64Condition(cond)); - } - - void moveDoubleConditionallyTest64(ResultCondition cond, RegisterID left, RegisterID right, FPRegisterID thenCase, FPRegisterID elseCase, FPRegisterID dest) - { - m_assembler.tst<64>(left, right); - m_assembler.fcsel<64>(dest, thenCase, elseCase, ARM64Condition(cond)); - } // Forwards / external control flow operations: // @@ -2180,17 +1539,12 @@ public: Jump branch32(RelationalCondition cond, RegisterID left, TrustedImm32 right) { - if (!right.m_value) { - if (auto resultCondition = commuteCompareToZeroIntoTest(cond)) - return branchTest32(*resultCondition, left, left); - } - if (isUInt12(right.m_value)) m_assembler.cmp<32>(left, UInt12(right.m_value)); else if (isUInt12(-right.m_value)) m_assembler.cmn<32>(left, UInt12(-right.m_value)); else { - moveToCachedReg(right, dataMemoryTempRegister()); + moveToCachedReg(right, m_dataMemoryTempRegister); m_assembler.cmp<32>(left, dataTempRegister); } return Jump(makeBranch(cond)); @@ -2234,52 +1588,19 @@ public: Jump branch64(RelationalCondition cond, RegisterID left, RegisterID right) { - if (right == ARM64Registers::sp) { - if (cond == Equal && left != ARM64Registers::sp) { - // CMP can only use SP for the left argument, since we are testing for equality, the order - // does not matter here. - std::swap(left, right); - } else { - move(right, getCachedDataTempRegisterIDAndInvalidate()); - right = dataTempRegister; - } - } m_assembler.cmp<64>(left, right); return Jump(makeBranch(cond)); } - Jump branch64(RelationalCondition cond, RegisterID left, TrustedImm32 right) - { - if (!right.m_value) { - if (auto resultCondition = commuteCompareToZeroIntoTest(cond)) - return branchTest64(*resultCondition, left, left); - } - - if (isUInt12(right.m_value)) - m_assembler.cmp<64>(left, UInt12(right.m_value)); - else if (isUInt12(-right.m_value)) - m_assembler.cmn<64>(left, UInt12(-right.m_value)); - else { - moveToCachedReg(right, dataMemoryTempRegister()); - m_assembler.cmp<64>(left, dataTempRegister); - } - return Jump(makeBranch(cond)); - } - Jump branch64(RelationalCondition cond, RegisterID left, TrustedImm64 right) { intptr_t immediate = right.m_value; - if (!immediate) { - if (auto resultCondition = commuteCompareToZeroIntoTest(cond)) - return branchTest64(*resultCondition, left, left); - } - if (isUInt12(immediate)) m_assembler.cmp<64>(left, UInt12(static_cast<int32_t>(immediate))); else if (isUInt12(-immediate)) m_assembler.cmn<64>(left, UInt12(static_cast<int32_t>(-immediate))); else { - moveToCachedReg(right, dataMemoryTempRegister()); + moveToCachedReg(right, m_dataMemoryTempRegister); m_assembler.cmp<64>(left, dataTempRegister); } return Jump(makeBranch(cond)); @@ -2309,12 +1630,6 @@ public: return branch64(cond, memoryTempRegister, right); } - Jump branchPtr(RelationalCondition cond, BaseIndex left, RegisterID right) - { - load64(left, getCachedMemoryTempRegisterIDAndInvalidate()); - return branch64(cond, memoryTempRegister, right); - } - Jump branch8(RelationalCondition cond, Address left, TrustedImm32 right) { ASSERT(!(0xffffff00 & right.m_value)); @@ -2338,33 +1653,10 @@ public: Jump branchTest32(ResultCondition cond, RegisterID reg, RegisterID mask) { - if (reg == mask && (cond == Zero || cond == NonZero)) - return Jump(makeCompareAndBranch<32>(static_cast<ZeroCondition>(cond), reg)); m_assembler.tst<32>(reg, mask); return Jump(makeBranch(cond)); } - void test32(RegisterID reg, TrustedImm32 mask = TrustedImm32(-1)) - { - if (mask.m_value == -1) - m_assembler.tst<32>(reg, reg); - else { - LogicalImmediate logicalImm = LogicalImmediate::create32(mask.m_value); - - if (logicalImm.isValid()) - m_assembler.tst<32>(reg, logicalImm); - else { - move(mask, getCachedDataTempRegisterIDAndInvalidate()); - m_assembler.tst<32>(reg, dataTempRegister); - } - } - } - - Jump branch(ResultCondition cond) - { - return Jump(makeBranch(cond)); - } - Jump branchTest32(ResultCondition cond, RegisterID reg, TrustedImm32 mask = TrustedImm32(-1)) { if (mask.m_value == -1) { @@ -2374,10 +1666,13 @@ public: } else if (hasOneBitSet(mask.m_value) && ((cond == Zero) || (cond == NonZero))) return Jump(makeTestBitAndBranch(reg, getLSBSet(mask.m_value), static_cast<ZeroCondition>(cond))); else { - LogicalImmediate logicalImm = LogicalImmediate::create32(mask.m_value); - if (logicalImm.isValid()) { - m_assembler.tst<32>(reg, logicalImm); - return Jump(makeBranch(cond)); + if ((cond == Zero) || (cond == NonZero)) { + LogicalImmediate logicalImm = LogicalImmediate::create32(mask.m_value); + + if (logicalImm.isValid()) { + m_assembler.tst<32>(reg, logicalImm); + return Jump(makeBranch(cond)); + } } move(mask, getCachedDataTempRegisterIDAndInvalidate()); @@ -2400,8 +1695,6 @@ public: Jump branchTest64(ResultCondition cond, RegisterID reg, RegisterID mask) { - if (reg == mask && (cond == Zero || cond == NonZero)) - return Jump(makeCompareAndBranch<64>(static_cast<ZeroCondition>(cond), reg)); m_assembler.tst<64>(reg, mask); return Jump(makeBranch(cond)); } @@ -2415,11 +1708,13 @@ public: } else if (hasOneBitSet(mask.m_value) && ((cond == Zero) || (cond == NonZero))) return Jump(makeTestBitAndBranch(reg, getLSBSet(mask.m_value), static_cast<ZeroCondition>(cond))); else { - LogicalImmediate logicalImm = LogicalImmediate::create64(mask.m_value); + if ((cond == Zero) || (cond == NonZero)) { + LogicalImmediate logicalImm = LogicalImmediate::create64(mask.m_value); - if (logicalImm.isValid()) { - m_assembler.tst<64>(reg, logicalImm); - return Jump(makeBranch(cond)); + if (logicalImm.isValid()) { + m_assembler.tst<64>(reg, logicalImm); + return Jump(makeBranch(cond)); + } } signExtend32ToPtr(mask, getCachedDataTempRegisterIDAndInvalidate()); @@ -2428,12 +1723,6 @@ public: return Jump(makeBranch(cond)); } - Jump branchTest64(ResultCondition cond, RegisterID reg, TrustedImm64 mask) - { - move(mask, getCachedDataTempRegisterIDAndInvalidate()); - return branchTest64(cond, reg, dataTempRegister); - } - Jump branchTest64(ResultCondition cond, Address address, RegisterID mask) { load64(address, getCachedDataTempRegisterIDAndInvalidate()); @@ -2586,7 +1875,7 @@ public: return branchAdd64(cond, dest, imm, dest); } - Jump branchMul32(ResultCondition cond, RegisterID src1, RegisterID src2, RegisterID scratch1, RegisterID scratch2, RegisterID dest) + Jump branchMul32(ResultCondition cond, RegisterID src1, RegisterID src2, RegisterID dest) { ASSERT(cond != Signed); @@ -2597,19 +1886,14 @@ public: // This is a signed multiple of two 32-bit values, producing a 64-bit result. m_assembler.smull(dest, src1, src2); - // Copy bits 63..32 of the result to bits 31..0 of scratch1. - m_assembler.asr<64>(scratch1, dest, 32); - // Splat bit 31 of the result to bits 31..0 of scratch2. - m_assembler.asr<32>(scratch2, dest, 31); + // Copy bits 63..32 of the result to bits 31..0 of dataTempRegister. + m_assembler.asr<64>(getCachedDataTempRegisterIDAndInvalidate(), dest, 32); + // Splat bit 31 of the result to bits 31..0 of memoryTempRegister. + m_assembler.asr<32>(getCachedMemoryTempRegisterIDAndInvalidate(), dest, 31); // After a mul32 the top 32 bits of the register should be clear. zeroExtend32ToPtr(dest, dest); // Check that bits 31..63 of the original result were all equal. - return branch32(NotEqual, scratch2, scratch1); - } - - Jump branchMul32(ResultCondition cond, RegisterID src1, RegisterID src2, RegisterID dest) - { - return branchMul32(cond, src1, src2, getCachedDataTempRegisterIDAndInvalidate(), getCachedMemoryTempRegisterIDAndInvalidate(), dest); + return branch32(NotEqual, memoryTempRegister, dataTempRegister); } Jump branchMul32(ResultCondition cond, RegisterID src, RegisterID dest) @@ -2617,13 +1901,13 @@ public: return branchMul32(cond, dest, src, dest); } - Jump branchMul32(ResultCondition cond, RegisterID src, TrustedImm32 imm, RegisterID dest) + Jump branchMul32(ResultCondition cond, TrustedImm32 imm, RegisterID src, RegisterID dest) { move(imm, getCachedDataTempRegisterIDAndInvalidate()); return branchMul32(cond, dataTempRegister, src, dest); } - Jump branchMul64(ResultCondition cond, RegisterID src1, RegisterID src2, RegisterID scratch1, RegisterID scratch2, RegisterID dest) + Jump branchMul64(ResultCondition cond, RegisterID src1, RegisterID src2, RegisterID dest) { ASSERT(cond != Signed); @@ -2633,17 +1917,12 @@ public: if (cond != Overflow) return branchTest64(cond, dest); - // Compute bits 127..64 of the result into scratch1. - m_assembler.smulh(scratch1, src1, src2); - // Splat bit 63 of the result to bits 63..0 of scratch2. - m_assembler.asr<64>(scratch2, dest, 63); + // Compute bits 127..64 of the result into dataTempRegister. + m_assembler.smulh(getCachedDataTempRegisterIDAndInvalidate(), src1, src2); + // Splat bit 63 of the result to bits 63..0 of memoryTempRegister. + m_assembler.asr<64>(getCachedMemoryTempRegisterIDAndInvalidate(), dest, 63); // Check that bits 31..63 of the original result were all equal. - return branch64(NotEqual, scratch2, scratch1); - } - - Jump branchMul64(ResultCondition cond, RegisterID src1, RegisterID src2, RegisterID dest) - { - return branchMul64(cond, src1, src2, getCachedDataTempRegisterIDAndInvalidate(), getCachedMemoryTempRegisterIDAndInvalidate(), dest); + return branch64(NotEqual, memoryTempRegister, dataTempRegister); } Jump branchMul64(ResultCondition cond, RegisterID src, RegisterID dest) @@ -2795,13 +2074,6 @@ public: return Call(m_assembler.label(), Call::LinkableNear); } - ALWAYS_INLINE Call nearTailCall() - { - AssemblerLabel label = m_assembler.label(); - m_assembler.b(); - return Call(label, Call::LinkableNearTail); - } - ALWAYS_INLINE void ret() { m_assembler.ret(); @@ -2836,21 +2108,8 @@ public: void compare32(RelationalCondition cond, RegisterID left, TrustedImm32 right, RegisterID dest) { - if (!right.m_value) { - if (auto resultCondition = commuteCompareToZeroIntoTest(cond)) { - test32(*resultCondition, left, left, dest); - return; - } - } - - if (isUInt12(right.m_value)) - m_assembler.cmp<32>(left, UInt12(right.m_value)); - else if (isUInt12(-right.m_value)) - m_assembler.cmn<32>(left, UInt12(-right.m_value)); - else { - move(right, getCachedDataTempRegisterIDAndInvalidate()); - m_assembler.cmp<32>(left, dataTempRegister); - } + move(right, getCachedDataTempRegisterIDAndInvalidate()); + m_assembler.cmp<32>(left, dataTempRegister); m_assembler.cset<32>(dest, ARM64Condition(cond)); } @@ -2862,13 +2121,6 @@ public: void compare64(RelationalCondition cond, RegisterID left, TrustedImm32 right, RegisterID dest) { - if (!right.m_value) { - if (auto resultCondition = commuteCompareToZeroIntoTest(cond)) { - test64(*resultCondition, left, left, dest); - return; - } - } - signExtend32ToPtr(right, getCachedDataTempRegisterIDAndInvalidate()); m_assembler.cmp<64>(left, dataTempRegister); m_assembler.cset<32>(dest, ARM64Condition(cond)); @@ -2880,29 +2132,28 @@ public: move(right, getCachedDataTempRegisterIDAndInvalidate()); compare32(cond, memoryTempRegister, dataTempRegister, dest); } - - void test32(ResultCondition cond, RegisterID src, RegisterID mask, RegisterID dest) - { - m_assembler.tst<32>(src, mask); - m_assembler.cset<32>(dest, ARM64Condition(cond)); - } - + void test32(ResultCondition cond, RegisterID src, TrustedImm32 mask, RegisterID dest) { - test32(src, mask); + if (mask.m_value == -1) + m_assembler.tst<32>(src, src); + else { + signExtend32ToPtr(mask, getCachedDataTempRegisterIDAndInvalidate()); + m_assembler.tst<32>(src, dataTempRegister); + } m_assembler.cset<32>(dest, ARM64Condition(cond)); } void test32(ResultCondition cond, Address address, TrustedImm32 mask, RegisterID dest) { - load32(address, getCachedMemoryTempRegisterIDAndInvalidate()); - test32(cond, memoryTempRegister, mask, dest); + load32(address, getCachedDataTempRegisterIDAndInvalidate()); + test32(cond, dataTempRegister, mask, dest); } void test8(ResultCondition cond, Address address, TrustedImm32 mask, RegisterID dest) { - load8(address, getCachedMemoryTempRegisterIDAndInvalidate()); - test32(cond, memoryTempRegister, mask, dest); + load8(address, getCachedDataTempRegisterIDAndInvalidate()); + test32(cond, dataTempRegister, mask, dest); } void test64(ResultCondition cond, RegisterID op1, RegisterID op2, RegisterID dest) @@ -2922,10 +2173,6 @@ public: m_assembler.cset<32>(dest, ARM64Condition(cond)); } - void setCarry(RegisterID dest) - { - m_assembler.cset<32>(dest, ARM64Assembler::ConditionCS); - } // Patchable operations @@ -2957,17 +2204,10 @@ public: return branch64(cond, left, dataTempRegister); } - ALWAYS_INLINE Jump branch32WithPatch(RelationalCondition cond, Address left, DataLabel32& dataLabel, TrustedImm32 initialRightValue = TrustedImm32(0)) - { - dataLabel = DataLabel32(this); - moveWithPatch(initialRightValue, getCachedDataTempRegisterIDAndInvalidate()); - return branch32(cond, left, dataTempRegister); - } - - PatchableJump patchableBranchPtr(RelationalCondition cond, Address left, TrustedImmPtr right) + PatchableJump patchableBranchPtr(RelationalCondition cond, Address left, TrustedImmPtr right = TrustedImmPtr(0)) { m_makeJumpPatchable = true; - Jump result = branch64(cond, left, TrustedImm64(right)); + Jump result = branch32(cond, left, TrustedImm32(right)); m_makeJumpPatchable = false; return PatchableJump(result); } @@ -2988,22 +2228,6 @@ public: return PatchableJump(result); } - PatchableJump patchableBranch64(RelationalCondition cond, RegisterID reg, TrustedImm64 imm) - { - m_makeJumpPatchable = true; - Jump result = branch64(cond, reg, imm); - m_makeJumpPatchable = false; - return PatchableJump(result); - } - - PatchableJump patchableBranch64(RelationalCondition cond, RegisterID left, RegisterID right) - { - m_makeJumpPatchable = true; - Jump result = branch64(cond, left, right); - m_makeJumpPatchable = false; - return PatchableJump(result); - } - PatchableJump patchableBranchPtrWithPatch(RelationalCondition cond, Address left, DataLabelPtr& dataLabel, TrustedImmPtr initialRightValue = TrustedImmPtr(0)) { m_makeJumpPatchable = true; @@ -3012,14 +2236,6 @@ public: return PatchableJump(result); } - PatchableJump patchableBranch32WithPatch(RelationalCondition cond, Address left, DataLabel32& dataLabel, TrustedImm32 initialRightValue = TrustedImm32(0)) - { - m_makeJumpPatchable = true; - Jump result = branch32WithPatch(cond, left, dataLabel, initialRightValue); - m_makeJumpPatchable = false; - return PatchableJump(result); - } - PatchableJump patchableJump() { m_makeJumpPatchable = true; @@ -3072,23 +2288,6 @@ public: return static_cast<RelationalCondition>(ARM64Assembler::invert(static_cast<ARM64Assembler::Condition>(cond))); } - static Optional<ResultCondition> commuteCompareToZeroIntoTest(RelationalCondition cond) - { - switch (cond) { - case Equal: - return Zero; - case NotEqual: - return NonZero; - case LessThan: - return Signed; - case GreaterThanOrEqual: - return PositiveOrZero; - break; - default: - return Nullopt; - } - } - static FunctionPtr readCallTarget(CodeLocationCall call) { return FunctionPtr(reinterpret_cast<void(*)()>(ARM64Assembler::readCallTarget(call.dataLocation()))); @@ -3104,15 +2303,9 @@ public: return ARM64Assembler::maxJumpReplacementSize(); } - RegisterID scratchRegisterForBlinding() - { - // We *do not* have a scratch register for blinding. - RELEASE_ASSERT_NOT_REACHED(); - return getCachedDataTempRegisterIDAndInvalidate(); - } + RegisterID scratchRegisterForBlinding() { return getCachedDataTempRegisterIDAndInvalidate(); } static bool canJumpReplacePatchableBranchPtrWithPatch() { return false; } - static bool canJumpReplacePatchableBranch32WithPatch() { return false; } static CodeLocationLabel startOfBranchPtrWithPatchOnRegister(CodeLocationDataLabelPtr label) { @@ -3125,12 +2318,6 @@ public: return CodeLocationLabel(); } - static CodeLocationLabel startOfPatchableBranch32WithPatchOnAddress(CodeLocationDataLabel32) - { - UNREACHABLE_FOR_PLATFORM(); - return CodeLocationLabel(); - } - static void revertJumpReplacementToBranchPtrWithPatch(CodeLocationLabel instructionStart, RegisterID, void* initialValue) { reemitInitialMoveWithPatch(instructionStart.dataLocation(), initialValue); @@ -3141,25 +2328,6 @@ public: UNREACHABLE_FOR_PLATFORM(); } - static void revertJumpReplacementToPatchableBranch32WithPatch(CodeLocationLabel, Address, int32_t) - { - UNREACHABLE_FOR_PLATFORM(); - } - - static void repatchCall(CodeLocationCall call, CodeLocationLabel destination) - { - ARM64Assembler::repatchPointer(call.dataLabelPtrAtOffset(REPATCH_OFFSET_CALL_TO_POINTER).dataLocation(), destination.executableAddress()); - } - - static void repatchCall(CodeLocationCall call, FunctionPtr destination) - { - ARM64Assembler::repatchPointer(call.dataLabelPtrAtOffset(REPATCH_OFFSET_CALL_TO_POINTER).dataLocation(), destination.executableAddress()); - } - -#if ENABLE(MASM_PROBE) - void probe(ProbeFunction, void* arg1, void* arg2); -#endif // ENABLE(MASM_PROBE) - protected: ALWAYS_INLINE Jump makeBranch(ARM64Assembler::Condition cond) { @@ -3213,26 +2381,8 @@ protected: } private: - ALWAYS_INLINE RegisterID getCachedDataTempRegisterIDAndInvalidate() - { - RELEASE_ASSERT(m_allowScratchRegister); - return dataMemoryTempRegister().registerIDInvalidate(); - } - ALWAYS_INLINE RegisterID getCachedMemoryTempRegisterIDAndInvalidate() - { - RELEASE_ASSERT(m_allowScratchRegister); - return cachedMemoryTempRegister().registerIDInvalidate(); - } - ALWAYS_INLINE CachedTempRegister& dataMemoryTempRegister() - { - RELEASE_ASSERT(m_allowScratchRegister); - return m_dataMemoryTempRegister; - } - ALWAYS_INLINE CachedTempRegister& cachedMemoryTempRegister() - { - RELEASE_ASSERT(m_allowScratchRegister); - return m_cachedMemoryTempRegister; - } + ALWAYS_INLINE RegisterID getCachedDataTempRegisterIDAndInvalidate() { return m_dataMemoryTempRegister.registerIDInvalidate(); } + ALWAYS_INLINE RegisterID getCachedMemoryTempRegisterIDAndInvalidate() { return m_cachedMemoryTempRegister.registerIDInvalidate(); } ALWAYS_INLINE bool isInIntRange(intptr_t value) { @@ -3311,18 +2461,6 @@ private: } template<int datasize> - ALWAYS_INLINE void loadSignedAddressedByUnsignedImmediate(RegisterID rt, RegisterID rn, unsigned pimm) - { - loadUnsignedImmediate<datasize>(rt, rn, pimm); - } - - template<int datasize> - ALWAYS_INLINE void loadSignedAddressedByUnscaledImmediate(RegisterID rt, RegisterID rn, int simm) - { - loadUnscaledImmediate<datasize>(rt, rn, simm); - } - - template<int datasize> ALWAYS_INLINE void storeUnsignedImmediate(RegisterID rt, RegisterID rn, unsigned pimm) { m_assembler.str<datasize>(rt, rn, pimm); @@ -3360,16 +2498,21 @@ private: } } + void signExtend32ToPtr(TrustedImm32 imm, RegisterID dest) + { + move(TrustedImmPtr(reinterpret_cast<void*>(static_cast<intptr_t>(imm.m_value))), dest); + } + template<int datasize> ALWAYS_INLINE void load(const void* address, RegisterID dest) { intptr_t currentRegisterContents; - if (cachedMemoryTempRegister().value(currentRegisterContents)) { + if (m_cachedMemoryTempRegister.value(currentRegisterContents)) { intptr_t addressAsInt = reinterpret_cast<intptr_t>(address); intptr_t addressDelta = addressAsInt - currentRegisterContents; if (dest == memoryTempRegister) - cachedMemoryTempRegister().invalidate(); + m_cachedMemoryTempRegister.invalidate(); if (isInIntRange(addressDelta)) { if (ARM64Assembler::canEncodeSImmOffset(addressDelta)) { @@ -3385,7 +2528,7 @@ private: if ((addressAsInt & (~maskHalfWord0)) == (currentRegisterContents & (~maskHalfWord0))) { m_assembler.movk<64>(memoryTempRegister, addressAsInt & maskHalfWord0, 0); - cachedMemoryTempRegister().setValue(reinterpret_cast<intptr_t>(address)); + m_cachedMemoryTempRegister.setValue(reinterpret_cast<intptr_t>(address)); m_assembler.ldr<datasize>(dest, memoryTempRegister, ARM64Registers::zr); return; } @@ -3393,18 +2536,17 @@ private: move(TrustedImmPtr(address), memoryTempRegister); if (dest == memoryTempRegister) - cachedMemoryTempRegister().invalidate(); + m_cachedMemoryTempRegister.invalidate(); else - cachedMemoryTempRegister().setValue(reinterpret_cast<intptr_t>(address)); + m_cachedMemoryTempRegister.setValue(reinterpret_cast<intptr_t>(address)); m_assembler.ldr<datasize>(dest, memoryTempRegister, ARM64Registers::zr); } template<int datasize> ALWAYS_INLINE void store(RegisterID src, const void* address) { - ASSERT(src != memoryTempRegister); intptr_t currentRegisterContents; - if (cachedMemoryTempRegister().value(currentRegisterContents)) { + if (m_cachedMemoryTempRegister.value(currentRegisterContents)) { intptr_t addressAsInt = reinterpret_cast<intptr_t>(address); intptr_t addressDelta = addressAsInt - currentRegisterContents; @@ -3422,14 +2564,14 @@ private: if ((addressAsInt & (~maskHalfWord0)) == (currentRegisterContents & (~maskHalfWord0))) { m_assembler.movk<64>(memoryTempRegister, addressAsInt & maskHalfWord0, 0); - cachedMemoryTempRegister().setValue(reinterpret_cast<intptr_t>(address)); + m_cachedMemoryTempRegister.setValue(reinterpret_cast<intptr_t>(address)); m_assembler.str<datasize>(src, memoryTempRegister, ARM64Registers::zr); return; } } move(TrustedImmPtr(address), memoryTempRegister); - cachedMemoryTempRegister().setValue(reinterpret_cast<intptr_t>(address)); + m_cachedMemoryTempRegister.setValue(reinterpret_cast<intptr_t>(address)); m_assembler.str<datasize>(src, memoryTempRegister, ARM64Registers::zr); } @@ -3506,20 +2648,6 @@ private: } template<int datasize> - ALWAYS_INLINE bool tryLoadSignedWithOffset(RegisterID rt, RegisterID rn, int32_t offset) - { - if (ARM64Assembler::canEncodeSImmOffset(offset)) { - loadSignedAddressedByUnscaledImmediate<datasize>(rt, rn, offset); - return true; - } - if (ARM64Assembler::canEncodePImmOffset<datasize>(offset)) { - loadSignedAddressedByUnsignedImmediate<datasize>(rt, rn, static_cast<unsigned>(offset)); - return true; - } - return false; - } - - template<int datasize> ALWAYS_INLINE bool tryLoadWithOffset(FPRegisterID rt, RegisterID rn, int32_t offset) { if (ARM64Assembler::canEncodeSImmOffset(offset)) { @@ -3561,37 +2689,25 @@ private: return false; } - Jump jumpAfterFloatingPointCompare(DoubleCondition cond) - { - if (cond == DoubleNotEqual) { - // ConditionNE jumps if NotEqual *or* unordered - force the unordered cases not to jump. - Jump unordered = makeBranch(ARM64Assembler::ConditionVS); - Jump result = makeBranch(ARM64Assembler::ConditionNE); - unordered.link(this); - return result; - } - if (cond == DoubleEqualOrUnordered) { - Jump unordered = makeBranch(ARM64Assembler::ConditionVS); - Jump notEqual = makeBranch(ARM64Assembler::ConditionNE); - unordered.link(this); - // We get here if either unordered or equal. - Jump result = jump(); - notEqual.link(this); - return result; - } - return makeBranch(cond); - } - friend class LinkBuffer; + friend class RepatchBuffer; static void linkCall(void* code, Call call, FunctionPtr function) { - if (!call.isFlagSet(Call::Near)) - ARM64Assembler::linkPointer(code, call.m_label.labelAtOffset(REPATCH_OFFSET_CALL_TO_POINTER), function.value()); - else if (call.isFlagSet(Call::Tail)) - ARM64Assembler::linkJump(code, call.m_label, function.value()); - else + if (call.isFlagSet(Call::Near)) ARM64Assembler::linkCall(code, call.m_label, function.value()); + else + ARM64Assembler::linkPointer(code, call.m_label.labelAtOffset(REPATCH_OFFSET_CALL_TO_POINTER), function.value()); + } + + static void repatchCall(CodeLocationCall call, CodeLocationLabel destination) + { + ARM64Assembler::repatchPointer(call.dataLabelPtrAtOffset(REPATCH_OFFSET_CALL_TO_POINTER).dataLocation(), destination.executableAddress()); + } + + static void repatchCall(CodeLocationCall call, FunctionPtr destination) + { + ARM64Assembler::repatchPointer(call.dataLabelPtrAtOffset(REPATCH_OFFSET_CALL_TO_POINTER).dataLocation(), destination.executableAddress()); } CachedTempRegister m_dataMemoryTempRegister; @@ -3613,18 +2729,6 @@ ALWAYS_INLINE void MacroAssemblerARM64::loadUnsignedImmediate<16>(RegisterID rt, } template<> -ALWAYS_INLINE void MacroAssemblerARM64::loadSignedAddressedByUnsignedImmediate<8>(RegisterID rt, RegisterID rn, unsigned pimm) -{ - m_assembler.ldrsb<64>(rt, rn, pimm); -} - -template<> -ALWAYS_INLINE void MacroAssemblerARM64::loadSignedAddressedByUnsignedImmediate<16>(RegisterID rt, RegisterID rn, unsigned pimm) -{ - m_assembler.ldrsh<64>(rt, rn, pimm); -} - -template<> ALWAYS_INLINE void MacroAssemblerARM64::loadUnscaledImmediate<8>(RegisterID rt, RegisterID rn, int simm) { m_assembler.ldurb(rt, rn, simm); @@ -3637,18 +2741,6 @@ ALWAYS_INLINE void MacroAssemblerARM64::loadUnscaledImmediate<16>(RegisterID rt, } template<> -ALWAYS_INLINE void MacroAssemblerARM64::loadSignedAddressedByUnscaledImmediate<8>(RegisterID rt, RegisterID rn, int simm) -{ - m_assembler.ldursb<64>(rt, rn, simm); -} - -template<> -ALWAYS_INLINE void MacroAssemblerARM64::loadSignedAddressedByUnscaledImmediate<16>(RegisterID rt, RegisterID rn, int simm) -{ - m_assembler.ldursh<64>(rt, rn, simm); -} - -template<> ALWAYS_INLINE void MacroAssemblerARM64::storeUnsignedImmediate<8>(RegisterID rt, RegisterID rn, unsigned pimm) { m_assembler.strb(rt, rn, pimm); |