diff options
Diffstat (limited to 'Source/JavaScriptCore/assembler/MacroAssemblerX86Common.h')
| -rw-r--r-- | Source/JavaScriptCore/assembler/MacroAssemblerX86Common.h | 1475 |
1 files changed, 1339 insertions, 136 deletions
diff --git a/Source/JavaScriptCore/assembler/MacroAssemblerX86Common.h b/Source/JavaScriptCore/assembler/MacroAssemblerX86Common.h index f171dc408..f502a2551 100644 --- a/Source/JavaScriptCore/assembler/MacroAssemblerX86Common.h +++ b/Source/JavaScriptCore/assembler/MacroAssemblerX86Common.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2008 Apple Inc. All rights reserved. + * Copyright (C) 2008, 2014-2016 Apple Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -30,21 +30,35 @@ #include "X86Assembler.h" #include "AbstractMacroAssembler.h" +#include <wtf/Optional.h> + +#if COMPILER(MSVC) +#include <intrin.h> +#endif namespace JSC { -class MacroAssemblerX86Common : public AbstractMacroAssembler<X86Assembler> { -protected: +class MacroAssemblerX86Common : public AbstractMacroAssembler<X86Assembler, MacroAssemblerX86Common> { +public: #if CPU(X86_64) - static const X86Registers::RegisterID scratchRegister = X86Registers::r11; + // Use this directly only if you're not generating code with it. + static const X86Registers::RegisterID s_scratchRegister = X86Registers::r11; + + // Use this when generating code so that we get enforcement of the disallowing of scratch register + // usage. + X86Registers::RegisterID scratchRegister() + { + RELEASE_ASSERT(m_allowScratchRegister); + return s_scratchRegister; + } #endif +protected: static const int DoubleConditionBitInvert = 0x10; static const int DoubleConditionBitSpecial = 0x20; static const int DoubleConditionBits = DoubleConditionBitInvert | DoubleConditionBitSpecial; public: - typedef X86Assembler::FPRegisterID FPRegisterID; typedef X86Assembler::XMMRegisterID XMMRegisterID; static bool isCompactPtrAlignedAddressOffset(ptrdiff_t value) @@ -73,6 +87,7 @@ public: NonZero = X86Assembler::ConditionNE }; + // FIXME: it would be neat to rename this to FloatingPointCondition in every assembler. enum DoubleCondition { // These conditions will only evaluate to true if the comparison is ordered - i.e. neither operand is NaN. DoubleEqual = X86Assembler::ConditionE | DoubleConditionBitSpecial, @@ -94,16 +109,11 @@ public: DoubleConditionBits_should_not_interfere_with_X86Assembler_Condition_codes); static const RegisterID stackPointerRegister = X86Registers::esp; - -#if ENABLE(JIT_CONSTANT_BLINDING) + static const RegisterID framePointerRegister = X86Registers::ebp; + + static bool canBlind() { return true; } static bool shouldBlindForSpecificArch(uint32_t value) { return value >= 0x00ffffff; } -#if CPU(X86_64) static bool shouldBlindForSpecificArch(uint64_t value) { return value >= 0x00ffffff; } -#if OS(DARWIN) // On 64-bit systems other than DARWIN uint64_t and uintptr_t are the same type so overload is prohibited. - static bool shouldBlindForSpecificArch(uintptr_t value) { return value >= 0x00ffffff; } -#endif -#endif -#endif // Integer arithmetic operations: // @@ -122,9 +132,37 @@ public: m_assembler.addl_im(imm.m_value, address.offset, address.base); } + void add32(TrustedImm32 imm, BaseIndex address) + { + m_assembler.addl_im(imm.m_value, address.offset, address.base, address.index, address.scale); + } + + void add8(TrustedImm32 imm, Address address) + { + m_assembler.addb_im(imm.m_value, address.offset, address.base); + } + + void add8(TrustedImm32 imm, BaseIndex address) + { + m_assembler.addb_im(imm.m_value, address.offset, address.base, address.index, address.scale); + } + + void add16(TrustedImm32 imm, Address address) + { + m_assembler.addw_im(imm.m_value, address.offset, address.base); + } + + void add16(TrustedImm32 imm, BaseIndex address) + { + m_assembler.addw_im(imm.m_value, address.offset, address.base, address.index, address.scale); + } + void add32(TrustedImm32 imm, RegisterID dest) { - m_assembler.addl_ir(imm.m_value, dest); + if (imm.m_value == 1) + m_assembler.inc_r(dest); + else + m_assembler.addl_ir(imm.m_value, dest); } void add32(Address src, RegisterID dest) @@ -137,10 +175,65 @@ public: m_assembler.addl_rm(src, dest.offset, dest.base); } + void add32(RegisterID src, BaseIndex dest) + { + m_assembler.addl_rm(src, dest.offset, dest.base, dest.index, dest.scale); + } + + void add8(RegisterID src, Address dest) + { + m_assembler.addb_rm(src, dest.offset, dest.base); + } + + void add8(RegisterID src, BaseIndex dest) + { + m_assembler.addb_rm(src, dest.offset, dest.base, dest.index, dest.scale); + } + + void add16(RegisterID src, Address dest) + { + m_assembler.addw_rm(src, dest.offset, dest.base); + } + + void add16(RegisterID src, BaseIndex dest) + { + m_assembler.addw_rm(src, dest.offset, dest.base, dest.index, dest.scale); + } + void add32(TrustedImm32 imm, RegisterID src, RegisterID dest) { + if (!imm.m_value) { + zeroExtend32ToPtr(src, dest); + return; + } + + if (src == dest) { + add32(imm, dest); + return; + } + m_assembler.leal_mr(imm.m_value, src, dest); } + + void add32(RegisterID a, RegisterID b, RegisterID dest) + { + x86Lea32(BaseIndex(a, b, TimesOne), dest); + } + + void x86Lea32(BaseIndex index, RegisterID dest) + { + if (!index.scale && !index.offset) { + if (index.base == dest) { + add32(index.index, dest); + return; + } + if (index.index == dest) { + add32(index.base, dest); + return; + } + } + m_assembler.leal_mr(index.offset, index.base, index.index, index.scale, dest); + } void and32(RegisterID src, RegisterID dest) { @@ -174,24 +267,55 @@ public: else if (op1 == dest) and32(op2, dest); else { - move(op2, dest); + move32IfNeeded(op2, dest); and32(op1, dest); } } + void and32(Address op1, RegisterID op2, RegisterID dest) + { + move32IfNeeded(op2, dest); + and32(op1, dest); + } + + void and32(RegisterID op1, Address op2, RegisterID dest) + { + move32IfNeeded(op1, dest); + and32(op2, dest); + } + void and32(TrustedImm32 imm, RegisterID src, RegisterID dest) { - move(src, dest); + move32IfNeeded(src, dest); and32(imm, dest); } - void lshift32(RegisterID shift_amount, RegisterID dest) + void countLeadingZeros32(RegisterID src, RegisterID dst) { - ASSERT(shift_amount != dest); + if (supportsLZCNT()) { + m_assembler.lzcnt_rr(src, dst); + return; + } + m_assembler.bsr_rr(src, dst); + clz32AfterBsr(dst); + } + void countLeadingZeros32(Address src, RegisterID dst) + { + if (supportsLZCNT()) { + m_assembler.lzcnt_mr(src.offset, src.base, dst); + return; + } + m_assembler.bsr_mr(src.offset, src.base, dst); + clz32AfterBsr(dst); + } + + void lshift32(RegisterID shift_amount, RegisterID dest) + { if (shift_amount == X86Registers::ecx) m_assembler.shll_CLr(dest); else { + ASSERT(shift_amount != dest); // On x86 we can only shift by ecx; if asked to shift by another register we'll // need rejig the shift amount into ecx first, and restore the registers afterwards. // If we dest is ecx, then shift the swapped register! @@ -205,8 +329,7 @@ public: { ASSERT(shift_amount != dest); - if (src != dest) - move(src, dest); + move32IfNeeded(src, dest); lshift32(shift_amount, dest); } @@ -217,8 +340,7 @@ public: void lshift32(RegisterID src, TrustedImm32 imm, RegisterID dest) { - if (src != dest) - move(src, dest); + move32IfNeeded(src, dest); lshift32(imm, dest); } @@ -227,16 +349,62 @@ public: m_assembler.imull_rr(src, dest); } + void mul32(RegisterID src1, RegisterID src2, RegisterID dest) + { + if (src2 == dest) { + m_assembler.imull_rr(src1, dest); + return; + } + move32IfNeeded(src1, dest); + m_assembler.imull_rr(src2, dest); + } + void mul32(Address src, RegisterID dest) { m_assembler.imull_mr(src.offset, src.base, dest); } + + void mul32(Address src1, RegisterID src2, RegisterID dest) + { + move32IfNeeded(src2, dest); + mul32(src1, dest); + } + + void mul32(RegisterID src1, Address src2, RegisterID dest) + { + move32IfNeeded(src1, dest); + mul32(src2, dest); + } void mul32(TrustedImm32 imm, RegisterID src, RegisterID dest) { m_assembler.imull_i32r(src, imm.m_value, dest); } + void x86ConvertToDoubleWord32() + { + m_assembler.cdq(); + } + + void x86ConvertToDoubleWord32(RegisterID eax, RegisterID edx) + { + ASSERT_UNUSED(eax, eax == X86Registers::eax); + ASSERT_UNUSED(edx, edx == X86Registers::edx); + x86ConvertToDoubleWord32(); + } + + void x86Div32(RegisterID denominator) + { + m_assembler.idivl_r(denominator); + } + + void x86Div32(RegisterID eax, RegisterID edx, RegisterID denominator) + { + ASSERT_UNUSED(eax, eax == X86Registers::eax); + ASSERT_UNUSED(edx, edx == X86Registers::edx); + x86Div32(denominator); + } + void neg32(RegisterID srcDest) { m_assembler.negl_r(srcDest); @@ -279,24 +447,36 @@ public: else if (op1 == dest) or32(op2, dest); else { - move(op2, dest); + move32IfNeeded(op2, dest); or32(op1, dest); } } + void or32(Address op1, RegisterID op2, RegisterID dest) + { + move32IfNeeded(op2, dest); + or32(op1, dest); + } + + void or32(RegisterID op1, Address op2, RegisterID dest) + { + move32IfNeeded(op1, dest); + or32(op2, dest); + } + void or32(TrustedImm32 imm, RegisterID src, RegisterID dest) { - move(src, dest); + move32IfNeeded(src, dest); or32(imm, dest); } void rshift32(RegisterID shift_amount, RegisterID dest) { - ASSERT(shift_amount != dest); - if (shift_amount == X86Registers::ecx) m_assembler.sarl_CLr(dest); else { + ASSERT(shift_amount != dest); + // On x86 we can only shift by ecx; if asked to shift by another register we'll // need rejig the shift amount into ecx first, and restore the registers afterwards. // If we dest is ecx, then shift the swapped register! @@ -310,8 +490,7 @@ public: { ASSERT(shift_amount != dest); - if (src != dest) - move(src, dest); + move32IfNeeded(src, dest); rshift32(shift_amount, dest); } @@ -322,18 +501,17 @@ public: void rshift32(RegisterID src, TrustedImm32 imm, RegisterID dest) { - if (src != dest) - move(src, dest); + move32IfNeeded(src, dest); rshift32(imm, dest); } void urshift32(RegisterID shift_amount, RegisterID dest) { - ASSERT(shift_amount != dest); - if (shift_amount == X86Registers::ecx) m_assembler.shrl_CLr(dest); else { + ASSERT(shift_amount != dest); + // On x86 we can only shift by ecx; if asked to shift by another register we'll // need rejig the shift amount into ecx first, and restore the registers afterwards. // If we dest is ecx, then shift the swapped register! @@ -347,8 +525,7 @@ public: { ASSERT(shift_amount != dest); - if (src != dest) - move(src, dest); + move32IfNeeded(src, dest); urshift32(shift_amount, dest); } @@ -359,8 +536,7 @@ public: void urshift32(RegisterID src, TrustedImm32 imm, RegisterID dest) { - if (src != dest) - move(src, dest); + move32IfNeeded(src, dest); urshift32(imm, dest); } @@ -371,7 +547,10 @@ public: void sub32(TrustedImm32 imm, RegisterID dest) { - m_assembler.subl_ir(imm.m_value, dest); + if (imm.m_value == 1) + m_assembler.dec_r(dest); + else + m_assembler.subl_ir(imm.m_value, dest); } void sub32(TrustedImm32 imm, Address address) @@ -405,9 +584,9 @@ public: void xor32(TrustedImm32 imm, RegisterID dest) { if (imm.m_value == -1) - m_assembler.notl_r(dest); + m_assembler.notl_r(dest); else - m_assembler.xorl_ir(imm.m_value, dest); + m_assembler.xorl_ir(imm.m_value, dest); } void xor32(RegisterID src, Address dest) @@ -427,27 +606,64 @@ public: else if (op1 == dest) xor32(op2, dest); else { - move(op2, dest); + move32IfNeeded(op2, dest); xor32(op1, dest); } } + void xor32(Address op1, RegisterID op2, RegisterID dest) + { + move32IfNeeded(op2, dest); + xor32(op1, dest); + } + + void xor32(RegisterID op1, Address op2, RegisterID dest) + { + move32IfNeeded(op1, dest); + xor32(op2, dest); + } + void xor32(TrustedImm32 imm, RegisterID src, RegisterID dest) { - move(src, dest); + move32IfNeeded(src, dest); xor32(imm, dest); } + void not32(RegisterID srcDest) + { + m_assembler.notl_r(srcDest); + } + + void not32(Address dest) + { + m_assembler.notl_m(dest.offset, dest.base); + } + void sqrtDouble(FPRegisterID src, FPRegisterID dst) { m_assembler.sqrtsd_rr(src, dst); } + void sqrtDouble(Address src, FPRegisterID dst) + { + m_assembler.sqrtsd_mr(src.offset, src.base, dst); + } + + void sqrtFloat(FPRegisterID src, FPRegisterID dst) + { + m_assembler.sqrtss_rr(src, dst); + } + + void sqrtFloat(Address src, FPRegisterID dst) + { + m_assembler.sqrtss_mr(src.offset, src.base, dst); + } + void absDouble(FPRegisterID src, FPRegisterID dst) { ASSERT(src != dst); static const double negativeZeroConstant = -0.0; - loadDouble(&negativeZeroConstant, dst); + loadDouble(TrustedImmPtr(&negativeZeroConstant), dst); m_assembler.andnpd_rr(src, dst); } @@ -455,10 +671,49 @@ public: { ASSERT(src != dst); static const double negativeZeroConstant = -0.0; - loadDouble(&negativeZeroConstant, dst); + loadDouble(TrustedImmPtr(&negativeZeroConstant), dst); m_assembler.xorpd_rr(src, dst); } + void ceilDouble(FPRegisterID src, FPRegisterID dst) + { + m_assembler.roundsd_rr(src, dst, X86Assembler::RoundingType::TowardInfiniti); + } + + void ceilDouble(Address src, FPRegisterID dst) + { + m_assembler.roundsd_mr(src.offset, src.base, dst, X86Assembler::RoundingType::TowardInfiniti); + } + + void ceilFloat(FPRegisterID src, FPRegisterID dst) + { + m_assembler.roundss_rr(src, dst, X86Assembler::RoundingType::TowardInfiniti); + } + + void ceilFloat(Address src, FPRegisterID dst) + { + m_assembler.roundss_mr(src.offset, src.base, dst, X86Assembler::RoundingType::TowardInfiniti); + } + + void floorDouble(FPRegisterID src, FPRegisterID dst) + { + m_assembler.roundsd_rr(src, dst, X86Assembler::RoundingType::TowardNegativeInfiniti); + } + + void floorDouble(Address src, FPRegisterID dst) + { + m_assembler.roundsd_mr(src.offset, src.base, dst, X86Assembler::RoundingType::TowardNegativeInfiniti); + } + + void floorFloat(FPRegisterID src, FPRegisterID dst) + { + m_assembler.roundss_rr(src, dst, X86Assembler::RoundingType::TowardNegativeInfiniti); + } + + void floorFloat(Address src, FPRegisterID dst) + { + m_assembler.roundss_mr(src.offset, src.base, dst, X86Assembler::RoundingType::TowardNegativeInfiniti); + } // Memory access operations: // @@ -524,15 +779,25 @@ public: m_assembler.movzbl_mr(address.offset, address.base, dest); } - void load8Signed(BaseIndex address, RegisterID dest) + void load8SignedExtendTo32(BaseIndex address, RegisterID dest) { m_assembler.movsbl_mr(address.offset, address.base, address.index, address.scale, dest); } - void load8Signed(ImplicitAddress address, RegisterID dest) + void load8SignedExtendTo32(ImplicitAddress address, RegisterID dest) { m_assembler.movsbl_mr(address.offset, address.base, dest); } + + void zeroExtend8To32(RegisterID src, RegisterID dest) + { + m_assembler.movzbl_rr(src, dest); + } + + void signExtend8To32(RegisterID src, RegisterID dest) + { + m_assembler.movsbl_rr(src, dest); + } void load16(BaseIndex address, RegisterID dest) { @@ -544,16 +809,26 @@ public: m_assembler.movzwl_mr(address.offset, address.base, dest); } - void load16Signed(BaseIndex address, RegisterID dest) + void load16SignedExtendTo32(BaseIndex address, RegisterID dest) { m_assembler.movswl_mr(address.offset, address.base, address.index, address.scale, dest); } - void load16Signed(Address address, RegisterID dest) + void load16SignedExtendTo32(Address address, RegisterID dest) { m_assembler.movswl_mr(address.offset, address.base, dest); } + void zeroExtend16To32(RegisterID src, RegisterID dest) + { + m_assembler.movzwl_rr(src, dest); + } + + void signExtend16To32(RegisterID src, RegisterID dest) + { + m_assembler.movswl_rr(src, dest); + } + DataLabel32 store32WithAddressOffsetPatch(RegisterID src, Address address) { padBeforePatch(); @@ -581,18 +856,47 @@ public: m_assembler.movl_i32m(imm.m_value, address.offset, address.base, address.index, address.scale); } + void storeZero32(ImplicitAddress address) + { + store32(TrustedImm32(0), address); + } + + void storeZero32(BaseIndex address) + { + store32(TrustedImm32(0), address); + } + void store8(TrustedImm32 imm, Address address) { - ASSERT(-128 <= imm.m_value && imm.m_value < 128); - m_assembler.movb_i8m(imm.m_value, address.offset, address.base); + m_assembler.movb_i8m(static_cast<int8_t>(imm.m_value), address.offset, address.base); } void store8(TrustedImm32 imm, BaseIndex address) { - ASSERT(-128 <= imm.m_value && imm.m_value < 128); - m_assembler.movb_i8m(imm.m_value, address.offset, address.base, address.index, address.scale); + m_assembler.movb_i8m(static_cast<int8_t>(imm.m_value), address.offset, address.base, address.index, address.scale); } - + + static ALWAYS_INLINE RegisterID getUnusedRegister(BaseIndex address) + { + if (address.base != X86Registers::eax && address.index != X86Registers::eax) + return X86Registers::eax; + + if (address.base != X86Registers::ebx && address.index != X86Registers::ebx) + return X86Registers::ebx; + + ASSERT(address.base != X86Registers::ecx && address.index != X86Registers::ecx); + return X86Registers::ecx; + } + + static ALWAYS_INLINE RegisterID getUnusedRegister(Address address) + { + if (address.base != X86Registers::eax) + return X86Registers::eax; + + ASSERT(address.base != X86Registers::edx); + return X86Registers::edx; + } + void store8(RegisterID src, BaseIndex address) { #if CPU(X86) @@ -600,15 +904,7 @@ public: // esp..edi are mapped to the 'h' registers! if (src >= 4) { // Pick a temporary register. - RegisterID temp; - if (address.base != X86Registers::eax && address.index != X86Registers::eax) - temp = X86Registers::eax; - else if (address.base != X86Registers::ebx && address.index != X86Registers::ebx) - temp = X86Registers::ebx; - else { - ASSERT(address.base != X86Registers::ecx && address.index != X86Registers::ecx); - temp = X86Registers::ecx; - } + RegisterID temp = getUnusedRegister(address); // Swap to the temporary register to perform the store. swap(src, temp); @@ -619,6 +915,25 @@ public: #endif m_assembler.movb_rm(src, address.offset, address.base, address.index, address.scale); } + + void store8(RegisterID src, Address address) + { +#if CPU(X86) + // On 32-bit x86 we can only store from the first 4 registers; + // esp..edi are mapped to the 'h' registers! + if (src >= 4) { + // Pick a temporary register. + RegisterID temp = getUnusedRegister(address); + + // Swap to the temporary register to perform the store. + swap(src, temp); + m_assembler.movb_rm(temp, address.offset, address.base); + swap(src, temp); + return; + } +#endif + m_assembler.movb_rm(src, address.offset, address.base); + } void store16(RegisterID src, BaseIndex address) { @@ -627,16 +942,8 @@ public: // esp..edi are mapped to the 'h' registers! if (src >= 4) { // Pick a temporary register. - RegisterID temp; - if (address.base != X86Registers::eax && address.index != X86Registers::eax) - temp = X86Registers::eax; - else if (address.base != X86Registers::ebx && address.index != X86Registers::ebx) - temp = X86Registers::ebx; - else { - ASSERT(address.base != X86Registers::ecx && address.index != X86Registers::ecx); - temp = X86Registers::ecx; - } - + RegisterID temp = getUnusedRegister(address); + // Swap to the temporary register to perform the store. swap(src, temp); m_assembler.movw_rm(temp, address.offset, address.base, address.index, address.scale); @@ -647,6 +954,25 @@ public: m_assembler.movw_rm(src, address.offset, address.base, address.index, address.scale); } + void store16(RegisterID src, Address address) + { +#if CPU(X86) + // On 32-bit x86 we can only store from the first 4 registers; + // esp..edi are mapped to the 'h' registers! + if (src >= 4) { + // Pick a temporary register. + RegisterID temp = getUnusedRegister(address); + + // Swap to the temporary register to perform the store. + swap(src, temp); + m_assembler.movw_rm(temp, address.offset, address.base); + swap(src, temp); + return; + } +#endif + m_assembler.movw_rm(src, address.offset, address.base); + } + // Floating-point operation: // @@ -656,17 +982,17 @@ public: { ASSERT(isSSE2Present()); if (src != dest) - m_assembler.movsd_rr(src, dest); + m_assembler.movaps_rr(src, dest); } - void loadDouble(const void* address, FPRegisterID dest) + void loadDouble(TrustedImmPtr address, FPRegisterID dest) { #if CPU(X86) ASSERT(isSSE2Present()); - m_assembler.movsd_mr(address, dest); + m_assembler.movsd_mr(address.m_value, dest); #else - move(TrustedImmPtr(address), scratchRegister); - loadDouble(scratchRegister, dest); + move(address, scratchRegister()); + loadDouble(scratchRegister(), dest); #endif } @@ -675,12 +1001,19 @@ public: ASSERT(isSSE2Present()); m_assembler.movsd_mr(address.offset, address.base, dest); } - + void loadDouble(BaseIndex address, FPRegisterID dest) { ASSERT(isSSE2Present()); m_assembler.movsd_mr(address.offset, address.base, address.index, address.scale, dest); } + + void loadFloat(ImplicitAddress address, FPRegisterID dest) + { + ASSERT(isSSE2Present()); + m_assembler.movss_mr(address.offset, address.base, dest); + } + void loadFloat(BaseIndex address, FPRegisterID dest) { ASSERT(isSSE2Present()); @@ -698,7 +1031,13 @@ public: ASSERT(isSSE2Present()); m_assembler.movsd_rm(src, address.offset, address.base, address.index, address.scale); } - + + void storeFloat(FPRegisterID src, ImplicitAddress address) + { + ASSERT(isSSE2Present()); + m_assembler.movss_rm(src, address.offset, address.base); + } + void storeFloat(FPRegisterID src, BaseIndex address) { ASSERT(isSSE2Present()); @@ -711,12 +1050,24 @@ public: m_assembler.cvtsd2ss_rr(src, dst); } + void convertDoubleToFloat(Address address, FPRegisterID dst) + { + ASSERT(isSSE2Present()); + m_assembler.cvtsd2ss_mr(address.offset, address.base, dst); + } + void convertFloatToDouble(FPRegisterID src, FPRegisterID dst) { ASSERT(isSSE2Present()); m_assembler.cvtss2sd_rr(src, dst); } + void convertFloatToDouble(Address address, FPRegisterID dst) + { + ASSERT(isSSE2Present()); + m_assembler.cvtss2sd_mr(address.offset, address.base, dst); + } + void addDouble(FPRegisterID src, FPRegisterID dest) { ASSERT(isSSE2Present()); @@ -740,6 +1091,77 @@ public: m_assembler.addsd_mr(src.offset, src.base, dest); } + void addDouble(Address op1, FPRegisterID op2, FPRegisterID dest) + { + ASSERT(isSSE2Present()); + if (op2 == dest) { + addDouble(op1, dest); + return; + } + + loadDouble(op1, dest); + addDouble(op2, dest); + } + + void addDouble(FPRegisterID op1, Address op2, FPRegisterID dest) + { + ASSERT(isSSE2Present()); + if (op1 == dest) { + addDouble(op2, dest); + return; + } + + loadDouble(op2, dest); + addDouble(op1, dest); + } + + void addFloat(FPRegisterID src, FPRegisterID dest) + { + ASSERT(isSSE2Present()); + m_assembler.addss_rr(src, dest); + } + + void addFloat(Address src, FPRegisterID dest) + { + ASSERT(isSSE2Present()); + m_assembler.addss_mr(src.offset, src.base, dest); + } + + void addFloat(FPRegisterID op1, FPRegisterID op2, FPRegisterID dest) + { + ASSERT(isSSE2Present()); + if (op1 == dest) + addFloat(op2, dest); + else { + moveDouble(op2, dest); + addFloat(op1, dest); + } + } + + void addFloat(Address op1, FPRegisterID op2, FPRegisterID dest) + { + ASSERT(isSSE2Present()); + if (op2 == dest) { + addFloat(op1, dest); + return; + } + + loadFloat(op1, dest); + addFloat(op2, dest); + } + + void addFloat(FPRegisterID op1, Address op2, FPRegisterID dest) + { + ASSERT(isSSE2Present()); + if (op1 == dest) { + addFloat(op2, dest); + return; + } + + loadFloat(op2, dest); + addFloat(op1, dest); + } + void divDouble(FPRegisterID src, FPRegisterID dest) { ASSERT(isSSE2Present()); @@ -761,6 +1183,18 @@ public: m_assembler.divsd_mr(src.offset, src.base, dest); } + void divFloat(FPRegisterID src, FPRegisterID dest) + { + ASSERT(isSSE2Present()); + m_assembler.divss_rr(src, dest); + } + + void divFloat(Address src, FPRegisterID dest) + { + ASSERT(isSSE2Present()); + m_assembler.divss_mr(src.offset, src.base, dest); + } + void subDouble(FPRegisterID src, FPRegisterID dest) { ASSERT(isSSE2Present()); @@ -782,6 +1216,18 @@ public: m_assembler.subsd_mr(src.offset, src.base, dest); } + void subFloat(FPRegisterID src, FPRegisterID dest) + { + ASSERT(isSSE2Present()); + m_assembler.subss_rr(src, dest); + } + + void subFloat(Address src, FPRegisterID dest) + { + ASSERT(isSSE2Present()); + m_assembler.subss_mr(src.offset, src.base, dest); + } + void mulDouble(FPRegisterID src, FPRegisterID dest) { ASSERT(isSSE2Present()); @@ -805,6 +1251,134 @@ public: m_assembler.mulsd_mr(src.offset, src.base, dest); } + void mulDouble(Address op1, FPRegisterID op2, FPRegisterID dest) + { + ASSERT(isSSE2Present()); + if (op2 == dest) { + mulDouble(op1, dest); + return; + } + loadDouble(op1, dest); + mulDouble(op2, dest); + } + + void mulDouble(FPRegisterID op1, Address op2, FPRegisterID dest) + { + ASSERT(isSSE2Present()); + if (op1 == dest) { + mulDouble(op2, dest); + return; + } + loadDouble(op2, dest); + mulDouble(op1, dest); + } + + void mulFloat(FPRegisterID src, FPRegisterID dest) + { + ASSERT(isSSE2Present()); + m_assembler.mulss_rr(src, dest); + } + + void mulFloat(Address src, FPRegisterID dest) + { + ASSERT(isSSE2Present()); + m_assembler.mulss_mr(src.offset, src.base, dest); + } + + void mulFloat(FPRegisterID op1, FPRegisterID op2, FPRegisterID dest) + { + ASSERT(isSSE2Present()); + if (op1 == dest) + mulFloat(op2, dest); + else { + moveDouble(op2, dest); + mulFloat(op1, dest); + } + } + + void mulFloat(Address op1, FPRegisterID op2, FPRegisterID dest) + { + ASSERT(isSSE2Present()); + if (op2 == dest) { + mulFloat(op1, dest); + return; + } + loadFloat(op1, dest); + mulFloat(op2, dest); + } + + void mulFloat(FPRegisterID op1, Address op2, FPRegisterID dest) + { + ASSERT(isSSE2Present()); + if (op1 == dest) { + mulFloat(op2, dest); + return; + } + loadFloat(op2, dest); + mulFloat(op1, dest); + } + + void andDouble(FPRegisterID src, FPRegisterID dst) + { + // ANDPS is defined on 128bits and is shorter than ANDPD. + m_assembler.andps_rr(src, dst); + } + + void andDouble(FPRegisterID src1, FPRegisterID src2, FPRegisterID dst) + { + if (src1 == dst) + andDouble(src2, dst); + else { + moveDouble(src2, dst); + andDouble(src1, dst); + } + } + + void andFloat(FPRegisterID src, FPRegisterID dst) + { + m_assembler.andps_rr(src, dst); + } + + void andFloat(FPRegisterID src1, FPRegisterID src2, FPRegisterID dst) + { + if (src1 == dst) + andFloat(src2, dst); + else { + moveDouble(src2, dst); + andFloat(src1, dst); + } + } + + void xorDouble(FPRegisterID src, FPRegisterID dst) + { + m_assembler.xorps_rr(src, dst); + } + + void xorDouble(FPRegisterID src1, FPRegisterID src2, FPRegisterID dst) + { + if (src1 == dst) + xorDouble(src2, dst); + else { + moveDouble(src2, dst); + xorDouble(src1, dst); + } + } + + void xorFloat(FPRegisterID src, FPRegisterID dst) + { + m_assembler.xorps_rr(src, dst); + } + + void xorFloat(FPRegisterID src1, FPRegisterID src2, FPRegisterID dst) + { + if (src1 == dst) + xorFloat(src2, dst); + else { + moveDouble(src2, dst); + xorFloat(src1, dst); + } + } + void convertInt32ToDouble(RegisterID src, FPRegisterID dest) { ASSERT(isSSE2Present()); @@ -825,27 +1399,18 @@ public: m_assembler.ucomisd_rr(left, right); else m_assembler.ucomisd_rr(right, left); + return jumpAfterFloatingPointCompare(cond, left, right); + } - if (cond == DoubleEqual) { - if (left == right) - return Jump(m_assembler.jnp()); - Jump isUnordered(m_assembler.jp()); - Jump result = Jump(m_assembler.je()); - isUnordered.link(this); - return result; - } else if (cond == DoubleNotEqualOrUnordered) { - if (left == right) - return Jump(m_assembler.jp()); - Jump isUnordered(m_assembler.jp()); - Jump isEqual(m_assembler.je()); - isUnordered.link(this); - Jump result = jump(); - isEqual.link(this); - return result; - } + Jump branchFloat(DoubleCondition cond, FPRegisterID left, FPRegisterID right) + { + ASSERT(isSSE2Present()); - ASSERT(!(cond & DoubleConditionBitSpecial)); - return Jump(m_assembler.jCC(static_cast<X86Assembler::Condition>(cond & ~DoubleConditionBits))); + if (cond & DoubleConditionBitInvert) + m_assembler.ucomiss_rr(left, right); + else + m_assembler.ucomiss_rr(right, left); + return jumpAfterFloatingPointCompare(cond, left, right); } // Truncates 'src' to an integer, and places the resulting 'dest'. @@ -860,13 +1425,6 @@ public: return branch32(branchType ? NotEqual : Equal, dest, TrustedImm32(0x80000000)); } - Jump branchTruncateDoubleToUint32(FPRegisterID src, RegisterID dest, BranchTruncateType branchType = BranchIfTruncateFailed) - { - ASSERT(isSSE2Present()); - m_assembler.cvttsd2si_rr(src, dest); - return branch32(branchType ? GreaterThanOrEqual : LessThan, dest, TrustedImm32(0)); - } - void truncateDoubleToInt32(FPRegisterID src, RegisterID dest) { ASSERT(isSSE2Present()); @@ -891,8 +1449,17 @@ public: m_assembler.cvttsd2si_rr(src, dest); // If the result is zero, it might have been -0.0, and the double comparison won't catch this! +#if CPU(X86_64) + if (negZeroCheck) { + Jump valueIsNonZero = branchTest32(NonZero, dest); + m_assembler.movmskpd_rr(src, scratchRegister()); + failureCases.append(branchTest32(NonZero, scratchRegister(), TrustedImm32(1))); + valueIsNonZero.link(this); + } +#else if (negZeroCheck) failureCases.append(branchTest32(Zero, dest)); +#endif // Convert the integer result back to float & compare to the original value - if not equal or unordered (NaN) then jump. convertInt32ToDouble(dest, fpTemp); @@ -901,6 +1468,11 @@ public: failureCases.append(m_assembler.jne()); } + void moveZeroToDouble(FPRegisterID reg) + { + m_assembler.xorps_rr(reg, reg); + } + Jump branchDoubleNonZero(FPRegisterID reg, FPRegisterID scratch) { ASSERT(isSSE2Present()); @@ -933,13 +1505,13 @@ public: m_assembler.por_rr(src, dst); } - void moveInt32ToPacked(RegisterID src, XMMRegisterID dst) + void move32ToFloat(RegisterID src, XMMRegisterID dst) { ASSERT(isSSE2Present()); m_assembler.movd_rr(src, dst); } - void movePackedToInt32(XMMRegisterID src, RegisterID dst) + void moveFloatTo32(XMMRegisterID src, RegisterID dst) { ASSERT(isSSE2Present()); m_assembler.movd_rr(src, dst); @@ -999,20 +1571,104 @@ public: void move(TrustedImmPtr imm, RegisterID dest) { - m_assembler.movq_i64r(imm.asIntptr(), dest); + if (!imm.m_value) + m_assembler.xorq_rr(dest, dest); + else + m_assembler.movq_i64r(imm.asIntptr(), dest); } void move(TrustedImm64 imm, RegisterID dest) { - m_assembler.movq_i64r(imm.m_value, dest); + if (!imm.m_value) + m_assembler.xorq_rr(dest, dest); + else + m_assembler.movq_i64r(imm.m_value, dest); } + void moveConditionallyDouble(DoubleCondition cond, FPRegisterID left, FPRegisterID right, RegisterID src, RegisterID dest) + { + ASSERT(isSSE2Present()); + + if (cond & DoubleConditionBitInvert) + m_assembler.ucomisd_rr(left, right); + else + m_assembler.ucomisd_rr(right, left); + moveConditionallyAfterFloatingPointCompare(cond, left, right, src, dest); + } + + void moveConditionallyDouble(DoubleCondition cond, FPRegisterID left, FPRegisterID right, RegisterID thenCase, RegisterID elseCase, RegisterID dest) + { + ASSERT(isSSE2Present()); + + if (thenCase != dest && elseCase != dest) { + move(elseCase, dest); + elseCase = dest; + } + + RegisterID src; + if (elseCase == dest) + src = thenCase; + else { + cond = invert(cond); + src = elseCase; + } + + if (cond & DoubleConditionBitInvert) + m_assembler.ucomisd_rr(left, right); + else + m_assembler.ucomisd_rr(right, left); + moveConditionallyAfterFloatingPointCompare(cond, left, right, src, dest); + } + + void moveConditionallyFloat(DoubleCondition cond, FPRegisterID left, FPRegisterID right, RegisterID src, RegisterID dest) + { + ASSERT(isSSE2Present()); + + if (cond & DoubleConditionBitInvert) + m_assembler.ucomiss_rr(left, right); + else + m_assembler.ucomiss_rr(right, left); + moveConditionallyAfterFloatingPointCompare(cond, left, right, src, dest); + } + + void moveConditionallyFloat(DoubleCondition cond, FPRegisterID left, FPRegisterID right, RegisterID thenCase, RegisterID elseCase, RegisterID dest) + { + ASSERT(isSSE2Present()); + + if (thenCase != dest && elseCase != dest) { + move(elseCase, dest); + elseCase = dest; + } + + RegisterID src; + if (elseCase == dest) + src = thenCase; + else { + cond = invert(cond); + src = elseCase; + } + + if (cond & DoubleConditionBitInvert) + m_assembler.ucomiss_rr(left, right); + else + m_assembler.ucomiss_rr(right, left); + moveConditionallyAfterFloatingPointCompare(cond, left, right, src, dest); + } + void swap(RegisterID reg1, RegisterID reg2) { if (reg1 != reg2) m_assembler.xchgq_rr(reg1, reg2); } + void signExtend32ToPtr(TrustedImm32 imm, RegisterID dest) + { + if (!imm.m_value) + m_assembler.xorq_rr(dest, dest); + else + m_assembler.mov_i32r(imm.m_value, dest); + } + void signExtend32ToPtr(RegisterID src, RegisterID dest) { m_assembler.movsxd_rr(src, dest); @@ -1022,6 +1678,11 @@ public: { m_assembler.movl_rr(src, dest); } + + void zeroExtend32ToPtr(TrustedImm32 src, RegisterID dest) + { + m_assembler.movl_i32r(src.m_value, dest); + } #else void move(RegisterID src, RegisterID dest) { @@ -1031,7 +1692,46 @@ public: void move(TrustedImmPtr imm, RegisterID dest) { - m_assembler.movl_i32r(imm.asIntptr(), dest); + if (!imm.m_value) + m_assembler.xorl_rr(dest, dest); + else + m_assembler.movl_i32r(imm.asIntptr(), dest); + } + + void moveConditionallyDouble(DoubleCondition cond, FPRegisterID left, FPRegisterID right, RegisterID src, RegisterID dest) + { + ASSERT(isSSE2Present()); + + if (cond & DoubleConditionBitInvert) + m_assembler.ucomisd_rr(left, right); + else + m_assembler.ucomisd_rr(right, left); + + if (cond == DoubleEqual) { + if (left == right) { + m_assembler.cmovnpl_rr(src, dest); + return; + } + + Jump isUnordered(m_assembler.jp()); + m_assembler.cmovel_rr(src, dest); + isUnordered.link(this); + return; + } + + if (cond == DoubleNotEqualOrUnordered) { + if (left == right) { + m_assembler.cmovpl_rr(src, dest); + return; + } + + m_assembler.cmovpl_rr(src, dest); + m_assembler.cmovnel_rr(src, dest); + return; + } + + ASSERT(!(cond & DoubleConditionBitSpecial)); + m_assembler.cmovl_rr(static_cast<X86Assembler::Condition>(cond & ~DoubleConditionBits), src, dest); } void swap(RegisterID reg1, RegisterID reg2) @@ -1051,6 +1751,190 @@ public: } #endif + void swap32(RegisterID src, RegisterID dest) + { + m_assembler.xchgl_rr(src, dest); + } + + void swap32(RegisterID src, Address dest) + { + m_assembler.xchgl_rm(src, dest.offset, dest.base); + } + + void moveConditionally32(RelationalCondition cond, RegisterID left, RegisterID right, RegisterID src, RegisterID dest) + { + m_assembler.cmpl_rr(right, left); + cmov(x86Condition(cond), src, dest); + } + + void moveConditionally32(RelationalCondition cond, RegisterID left, RegisterID right, RegisterID thenCase, RegisterID elseCase, RegisterID dest) + { + m_assembler.cmpl_rr(right, left); + + if (thenCase != dest && elseCase != dest) { + move(elseCase, dest); + elseCase = dest; + } + + if (elseCase == dest) + cmov(x86Condition(cond), thenCase, dest); + else + cmov(x86Condition(invert(cond)), elseCase, dest); + } + + void moveConditionally32(RelationalCondition cond, RegisterID left, TrustedImm32 right, RegisterID thenCase, RegisterID elseCase, RegisterID dest) + { + if (!right.m_value) { + if (auto resultCondition = commuteCompareToZeroIntoTest(cond)) { + moveConditionallyTest32(*resultCondition, left, left, thenCase, elseCase, dest); + return; + } + } + + m_assembler.cmpl_ir(right.m_value, left); + + if (thenCase != dest && elseCase != dest) { + move(elseCase, dest); + elseCase = dest; + } + + if (elseCase == dest) + cmov(x86Condition(cond), thenCase, dest); + else + cmov(x86Condition(invert(cond)), elseCase, dest); + } + + void moveConditionallyTest32(ResultCondition cond, RegisterID testReg, RegisterID mask, RegisterID src, RegisterID dest) + { + m_assembler.testl_rr(testReg, mask); + cmov(x86Condition(cond), src, dest); + } + + void moveConditionallyTest32(ResultCondition cond, RegisterID left, RegisterID right, RegisterID thenCase, RegisterID elseCase, RegisterID dest) + { + ASSERT(isInvertible(cond)); + ASSERT_WITH_MESSAGE(cond != Overflow, "TEST does not set the Overflow Flag."); + + m_assembler.testl_rr(right, left); + + if (thenCase != dest && elseCase != dest) { + move(elseCase, dest); + elseCase = dest; + } + + if (elseCase == dest) + cmov(x86Condition(cond), thenCase, dest); + else + cmov(x86Condition(invert(cond)), elseCase, dest); + } + + void moveConditionallyTest32(ResultCondition cond, RegisterID testReg, TrustedImm32 mask, RegisterID src, RegisterID dest) + { + test32(testReg, mask); + cmov(x86Condition(cond), src, dest); + } + + void moveConditionallyTest32(ResultCondition cond, RegisterID testReg, TrustedImm32 mask, RegisterID thenCase, RegisterID elseCase, RegisterID dest) + { + ASSERT(isInvertible(cond)); + ASSERT_WITH_MESSAGE(cond != Overflow, "TEST does not set the Overflow Flag."); + + test32(testReg, mask); + + if (thenCase != dest && elseCase != dest) { + move(elseCase, dest); + elseCase = dest; + } + + if (elseCase == dest) + cmov(x86Condition(cond), thenCase, dest); + else + cmov(x86Condition(invert(cond)), elseCase, dest); + } + + template<typename LeftType, typename RightType> + void moveDoubleConditionally32(RelationalCondition cond, LeftType left, RightType right, FPRegisterID thenCase, FPRegisterID elseCase, FPRegisterID dest) + { + static_assert(!std::is_same<LeftType, FPRegisterID>::value && !std::is_same<RightType, FPRegisterID>::value, "One of the tested argument could be aliased on dest. Use moveDoubleConditionallyDouble()."); + + if (thenCase != dest && elseCase != dest) { + moveDouble(elseCase, dest); + elseCase = dest; + } + + if (elseCase == dest) { + Jump falseCase = branch32(invert(cond), left, right); + moveDouble(thenCase, dest); + falseCase.link(this); + } else { + Jump trueCase = branch32(cond, left, right); + moveDouble(elseCase, dest); + trueCase.link(this); + } + } + + template<typename TestType, typename MaskType> + void moveDoubleConditionallyTest32(ResultCondition cond, TestType test, MaskType mask, FPRegisterID thenCase, FPRegisterID elseCase, FPRegisterID dest) + { + static_assert(!std::is_same<TestType, FPRegisterID>::value && !std::is_same<MaskType, FPRegisterID>::value, "One of the tested argument could be aliased on dest. Use moveDoubleConditionallyDouble()."); + + if (elseCase == dest && isInvertible(cond)) { + Jump falseCase = branchTest32(invert(cond), test, mask); + moveDouble(thenCase, dest); + falseCase.link(this); + } else if (thenCase == dest) { + Jump trueCase = branchTest32(cond, test, mask); + moveDouble(elseCase, dest); + trueCase.link(this); + } + + Jump trueCase = branchTest32(cond, test, mask); + moveDouble(elseCase, dest); + Jump falseCase = jump(); + trueCase.link(this); + moveDouble(thenCase, dest); + falseCase.link(this); + } + + void moveDoubleConditionallyDouble(DoubleCondition cond, FPRegisterID left, FPRegisterID right, FPRegisterID thenCase, FPRegisterID elseCase, FPRegisterID dest) + { + if (elseCase == dest) { + Jump falseCase = branchDouble(invert(cond), left, right); + moveDouble(thenCase, dest); + falseCase.link(this); + } else if (thenCase == dest) { + Jump trueCase = branchDouble(cond, left, right); + moveDouble(elseCase, dest); + trueCase.link(this); + } else { + Jump trueCase = branchDouble(cond, left, right); + moveDouble(elseCase, dest); + Jump falseCase = jump(); + trueCase.link(this); + moveDouble(thenCase, dest); + falseCase.link(this); + } + } + + void moveDoubleConditionallyFloat(DoubleCondition cond, FPRegisterID left, FPRegisterID right, FPRegisterID thenCase, FPRegisterID elseCase, FPRegisterID dest) + { + if (elseCase == dest) { + Jump falseCase = branchFloat(invert(cond), left, right); + moveDouble(thenCase, dest); + falseCase.link(this); + } else if (thenCase == dest) { + Jump trueCase = branchFloat(cond, left, right); + moveDouble(elseCase, dest); + trueCase.link(this); + } else { + Jump trueCase = branchFloat(cond, left, right); + moveDouble(elseCase, dest); + Jump falseCase = jump(); + trueCase.link(this); + moveDouble(thenCase, dest); + falseCase.link(this); + } + } // Forwards / external control flow operations: // @@ -1085,10 +1969,12 @@ public: Jump branch32(RelationalCondition cond, RegisterID left, TrustedImm32 right) { - if (((cond == Equal) || (cond == NotEqual)) && !right.m_value) - m_assembler.testl_rr(left, left); - else - m_assembler.cmpl_ir(right.m_value, left); + if (!right.m_value) { + if (auto resultCondition = commuteCompareToZeroIntoTest(cond)) + return branchTest32(*resultCondition, left, left); + } + + m_assembler.cmpl_ir(right.m_value, left); return Jump(m_assembler.jCC(x86Condition(cond))); } @@ -1127,22 +2013,33 @@ public: return Jump(m_assembler.jCC(x86Condition(cond))); } - Jump branchTest32(ResultCondition cond, RegisterID reg, TrustedImm32 mask = TrustedImm32(-1)) + void test32(RegisterID reg, TrustedImm32 mask = TrustedImm32(-1)) { - // if we are only interested in the low seven bits, this can be tested with a testb if (mask.m_value == -1) m_assembler.testl_rr(reg, reg); - else + else if (!(mask.m_value & ~0xff) && reg < X86Registers::esp) { // Using esp and greater as a byte register yields the upper half of the 16 bit registers ax, cx, dx and bx, e.g. esp, register 4, is actually ah. + if (mask.m_value == 0xff) + m_assembler.testb_rr(reg, reg); + else + m_assembler.testb_i8r(mask.m_value, reg); + } else m_assembler.testl_i32r(mask.m_value, reg); + } + + Jump branch(ResultCondition cond) + { return Jump(m_assembler.jCC(x86Condition(cond))); } + Jump branchTest32(ResultCondition cond, RegisterID reg, TrustedImm32 mask = TrustedImm32(-1)) + { + test32(reg, mask); + return branch(cond); + } + Jump branchTest32(ResultCondition cond, Address address, TrustedImm32 mask = TrustedImm32(-1)) { - if (mask.m_value == -1) - m_assembler.cmpl_im(0, address.offset, address.base); - else - m_assembler.testl_i32m(mask.m_value, address.offset, address.base); + generateTest32(address, mask); return Jump(m_assembler.jCC(x86Condition(cond))); } @@ -1179,7 +2076,7 @@ public: Jump branch8(RelationalCondition cond, BaseIndex left, TrustedImm32 right) { - ASSERT(!(right.m_value & 0xFFFFFF00)); + ASSERT(std::numeric_limits<int8_t>::min() <= right.m_value && right.m_value <= std::numeric_limits<int8_t>::max()); m_assembler.cmpb_im(right.m_value, left.offset, left.base, left.index, left.scale); return Jump(m_assembler.jCC(x86Condition(cond))); @@ -1246,13 +2143,25 @@ public: { if (src1 == dest) return branchAdd32(cond, src2, dest); - move(src2, dest); + move32IfNeeded(src2, dest); + return branchAdd32(cond, src1, dest); + } + + Jump branchAdd32(ResultCondition cond, Address src1, RegisterID src2, RegisterID dest) + { + move32IfNeeded(src2, dest); return branchAdd32(cond, src1, dest); } + Jump branchAdd32(ResultCondition cond, RegisterID src1, Address src2, RegisterID dest) + { + move32IfNeeded(src1, dest); + return branchAdd32(cond, src2, dest); + } + Jump branchAdd32(ResultCondition cond, RegisterID src, TrustedImm32 imm, RegisterID dest) { - move(src, dest); + move32IfNeeded(src, dest); return branchAdd32(cond, imm, dest); } @@ -1272,7 +2181,7 @@ public: return Jump(m_assembler.jCC(x86Condition(cond))); } - Jump branchMul32(ResultCondition cond, TrustedImm32 imm, RegisterID src, RegisterID dest) + Jump branchMul32(ResultCondition cond, RegisterID src, TrustedImm32 imm, RegisterID dest) { mul32(imm, src, dest); if (cond != Overflow) @@ -1284,7 +2193,7 @@ public: { if (src1 == dest) return branchMul32(cond, src2, dest); - move(src2, dest); + move32IfNeeded(src2, dest); return branchMul32(cond, src1, dest); } @@ -1323,13 +2232,13 @@ public: // B := A - B is invalid. ASSERT(src1 == dest || src2 != dest); - move(src1, dest); + move32IfNeeded(src1, dest); return branchSub32(cond, src2, dest); } Jump branchSub32(ResultCondition cond, RegisterID src1, TrustedImm32 src2, RegisterID dest) { - move(src1, dest); + move32IfNeeded(src1, dest); return branchSub32(cond, src2, dest); } @@ -1353,6 +2262,11 @@ public: m_assembler.int3(); } + Call nearTailCall() + { + return Call(m_assembler.jmp(), Call::LinkableNearTail); + } + Call nearCall() { return Call(m_assembler.call(), Call::LinkableNear); @@ -1387,10 +2301,14 @@ public: void compare32(RelationalCondition cond, RegisterID left, TrustedImm32 right, RegisterID dest) { - if (((cond == Equal) || (cond == NotEqual)) && !right.m_value) - m_assembler.testl_rr(left, left); - else - m_assembler.cmpl_ir(right.m_value, left); + if (!right.m_value) { + if (auto resultCondition = commuteCompareToZeroIntoTest(cond)) { + test32(*resultCondition, left, left, dest); + return; + } + } + + m_assembler.cmpl_ir(right.m_value, left); set32(x86Condition(cond), dest); } @@ -1410,23 +2328,121 @@ public: void test32(ResultCondition cond, Address address, TrustedImm32 mask, RegisterID dest) { - if (mask.m_value == -1) - m_assembler.cmpl_im(0, address.offset, address.base); - else - m_assembler.testl_i32m(mask.m_value, address.offset, address.base); + generateTest32(address, mask); set32(x86Condition(cond), dest); } + void test32(ResultCondition cond, RegisterID reg, RegisterID mask, RegisterID dest) + { + m_assembler.testl_rr(reg, mask); + set32(x86Condition(cond), dest); + } + + void test32(ResultCondition cond, RegisterID reg, TrustedImm32 mask, RegisterID dest) + { + test32(reg, mask); + set32(x86Condition(cond), dest); + } + + void setCarry(RegisterID dest) + { + set32(X86Assembler::ConditionC, dest); + } + // Invert a relational condition, e.g. == becomes !=, < becomes >=, etc. static RelationalCondition invert(RelationalCondition cond) { return static_cast<RelationalCondition>(cond ^ 1); } + static DoubleCondition invert(DoubleCondition cond) + { + switch (cond) { + case DoubleEqual: + return DoubleNotEqualOrUnordered; + case DoubleNotEqual: + return DoubleEqualOrUnordered; + case DoubleGreaterThan: + return DoubleLessThanOrEqualOrUnordered; + case DoubleGreaterThanOrEqual: + return DoubleLessThanOrUnordered; + case DoubleLessThan: + return DoubleGreaterThanOrEqualOrUnordered; + case DoubleLessThanOrEqual: + return DoubleGreaterThanOrUnordered; + case DoubleEqualOrUnordered: + return DoubleNotEqual; + case DoubleNotEqualOrUnordered: + return DoubleEqual; + case DoubleGreaterThanOrUnordered: + return DoubleLessThanOrEqual; + case DoubleGreaterThanOrEqualOrUnordered: + return DoubleLessThan; + case DoubleLessThanOrUnordered: + return DoubleGreaterThanOrEqual; + case DoubleLessThanOrEqualOrUnordered: + return DoubleGreaterThan; + } + RELEASE_ASSERT_NOT_REACHED(); + return DoubleEqual; // make compiler happy + } + + static bool isInvertible(ResultCondition cond) + { + switch (cond) { + case Zero: + case NonZero: + case Signed: + case PositiveOrZero: + return true; + default: + return false; + } + } + + static ResultCondition invert(ResultCondition cond) + { + switch (cond) { + case Zero: + return NonZero; + case NonZero: + return Zero; + case Signed: + return PositiveOrZero; + case PositiveOrZero: + return Signed; + default: + RELEASE_ASSERT_NOT_REACHED(); + return Zero; // Make compiler happy for release builds. + } + } + + static Optional<ResultCondition> commuteCompareToZeroIntoTest(RelationalCondition cond) + { + switch (cond) { + case Equal: + return Zero; + case NotEqual: + return NonZero; + case LessThan: + return Signed; + case GreaterThanOrEqual: + return PositiveOrZero; + break; + default: + return Nullopt; + } + } + void nop() { m_assembler.nop(); } + + void memoryFence() + { + m_assembler.mfence(); + } static void replaceWithJump(CodeLocationLabel instructionStart, CodeLocationLabel destination) { @@ -1438,6 +2454,46 @@ public: return X86Assembler::maxJumpReplacementSize(); } + static bool supportsFloatingPointRounding() + { + if (s_sse4_1CheckState == CPUIDCheckState::NotChecked) { + int flags = 0; +#if COMPILER(MSVC) + int cpuInfo[4]; + __cpuid(cpuInfo, 0x1); + flags = cpuInfo[2]; +#elif COMPILER(GCC_OR_CLANG) +#if CPU(X86_64) + asm ( + "movl $0x1, %%eax;" + "cpuid;" + "movl %%ecx, %0;" + : "=g" (flags) + : + : "%eax", "%ebx", "%ecx", "%edx" + ); +#else + asm ( + "movl $0x1, %%eax;" + "pushl %%ebx;" + "cpuid;" + "popl %%ebx;" + "movl %%ecx, %0;" + : "=g" (flags) + : + : "%eax", "%ecx", "%edx" + ); +#endif +#endif // COMPILER(GCC_OR_CLANG) + s_sse4_1CheckState = (flags & (1 << 19)) ? CPUIDCheckState::Set : CPUIDCheckState::Clear; + } + return s_sse4_1CheckState == CPUIDCheckState::Set; + } + +#if ENABLE(MASM_PROBE) + void probe(ProbeFunction, void* arg1, void* arg2); +#endif // ENABLE(MASM_PROBE) + protected: X86Assembler::Condition x86Condition(RelationalCondition cond) { @@ -1466,11 +2522,150 @@ protected: m_assembler.movzbl_rr(dest, dest); } + void cmov(X86Assembler::Condition cond, RegisterID src, RegisterID dest) + { +#if CPU(X86_64) + m_assembler.cmovq_rr(cond, src, dest); +#else + m_assembler.cmovl_rr(cond, src, dest); +#endif + } + + static bool supportsLZCNT() + { + if (s_lzcntCheckState == CPUIDCheckState::NotChecked) { + int flags = 0; +#if COMPILER(MSVC) + int cpuInfo[4]; + __cpuid(cpuInfo, 0x80000001); + flags = cpuInfo[2]; +#elif COMPILER(GCC_OR_CLANG) +#if CPU(X86_64) + asm ( + "movl $0x80000001, %%eax;" + "cpuid;" + "movl %%ecx, %0;" + : "=g" (flags) + : + : "%eax", "%ebx", "%ecx", "%edx" + ); +#else + asm ( + "movl $0x80000001, %%eax;" + "pushl %%ebx;" + "cpuid;" + "popl %%ebx;" + "movl %%ecx, %0;" + : "=g" (flags) + : + : "%eax", "%ecx", "%edx" + ); +#endif +#endif // COMPILER(GCC_OR_CLANG) + s_lzcntCheckState = (flags & 0x20) ? CPUIDCheckState::Set : CPUIDCheckState::Clear; + } + return s_lzcntCheckState == CPUIDCheckState::Set; + } + private: // Only MacroAssemblerX86 should be using the following method; SSE2 is always available on // x86_64, and clients & subclasses of MacroAssembler should be using 'supportsFloatingPoint()'. friend class MacroAssemblerX86; + ALWAYS_INLINE void generateTest32(Address address, TrustedImm32 mask = TrustedImm32(-1)) + { + if (mask.m_value == -1) + m_assembler.cmpl_im(0, address.offset, address.base); + else if (!(mask.m_value & ~0xff)) + m_assembler.testb_im(mask.m_value, address.offset, address.base); + else if (!(mask.m_value & ~0xff00)) + m_assembler.testb_im(mask.m_value >> 8, address.offset + 1, address.base); + else if (!(mask.m_value & ~0xff0000)) + m_assembler.testb_im(mask.m_value >> 16, address.offset + 2, address.base); + else if (!(mask.m_value & ~0xff000000)) + m_assembler.testb_im(mask.m_value >> 24, address.offset + 3, address.base); + else + m_assembler.testl_i32m(mask.m_value, address.offset, address.base); + } + + // If lzcnt is not available, use this after BSR + // to count the leading zeros. + void clz32AfterBsr(RegisterID dst) + { + Jump srcIsNonZero = m_assembler.jCC(x86Condition(NonZero)); + move(TrustedImm32(32), dst); + + Jump skipNonZeroCase = jump(); + srcIsNonZero.link(this); + xor32(TrustedImm32(0x1f), dst); + skipNonZeroCase.link(this); + } + + Jump jumpAfterFloatingPointCompare(DoubleCondition cond, FPRegisterID left, FPRegisterID right) + { + if (cond == DoubleEqual) { + if (left == right) + return Jump(m_assembler.jnp()); + Jump isUnordered(m_assembler.jp()); + Jump result = Jump(m_assembler.je()); + isUnordered.link(this); + return result; + } + if (cond == DoubleNotEqualOrUnordered) { + if (left == right) + return Jump(m_assembler.jp()); + Jump isUnordered(m_assembler.jp()); + Jump isEqual(m_assembler.je()); + isUnordered.link(this); + Jump result = jump(); + isEqual.link(this); + return result; + } + + ASSERT(!(cond & DoubleConditionBitSpecial)); + return Jump(m_assembler.jCC(static_cast<X86Assembler::Condition>(cond & ~DoubleConditionBits))); + } + + // The 32bit Move does not need the REX byte for low registers, making it shorter. + // Use this if the top bits are irrelevant because they will be reset by the next instruction. + void move32IfNeeded(RegisterID src, RegisterID dest) + { + if (src == dest) + return; + m_assembler.movl_rr(src, dest); + } + +#if CPU(X86_64) + void moveConditionallyAfterFloatingPointCompare(DoubleCondition cond, FPRegisterID left, FPRegisterID right, RegisterID src, RegisterID dest) + { + if (cond == DoubleEqual) { + if (left == right) { + m_assembler.cmovnpq_rr(src, dest); + return; + } + + Jump isUnordered(m_assembler.jp()); + m_assembler.cmoveq_rr(src, dest); + isUnordered.link(this); + return; + } + + if (cond == DoubleNotEqualOrUnordered) { + if (left == right) { + m_assembler.cmovpq_rr(src, dest); + return; + } + + m_assembler.cmovpq_rr(src, dest); + m_assembler.cmovneq_rr(src, dest); + return; + } + + ASSERT(!(cond & DoubleConditionBitSpecial)); + cmov(static_cast<X86Assembler::Condition>(cond & ~DoubleConditionBits), src, dest); + } +#endif + #if CPU(X86) #if OS(MAC_OS_X) @@ -1500,7 +2695,7 @@ private: cpuid; mov flags, edx; } -#elif COMPILER(GCC) +#elif COMPILER(GCC_OR_CLANG) asm ( "movl $0x1, %%eax;" "pushl %%ebx;" @@ -1534,6 +2729,14 @@ private: } #endif + + enum class CPUIDCheckState { + NotChecked, + Clear, + Set + }; + static CPUIDCheckState s_sse4_1CheckState; + static CPUIDCheckState s_lzcntCheckState; }; } // namespace JSC |