[dev.link] all: merge branch 'master' into dev.linkdev.link

Clean merge.

Change-Id: Ia7b2808bc649790198d34c226a61d9e569084dc5

1 files changed, 812 insertions, 231 deletions

diff --git a/src/cmd/compile/internal/ssa/rewriteRISCV64.go b/src/cmd/compile/internal/ssa/rewriteRISCV64.go
index c178290343..c337ffbfe7 100644
--- a/src/cmd/compile/internal/ssa/rewriteRISCV64.go
+++ b/src/cmd/compile/internal/ssa/rewriteRISCV64.go
@@ -420,8 +420,12 @@ func rewriteValueRISCV64(v *Value) bool {
 		return rewriteValueRISCV64_OpRISCV64AND(v)
 	case OpRISCV64MOVBUload:
 		return rewriteValueRISCV64_OpRISCV64MOVBUload(v)
+	case OpRISCV64MOVBUreg:
+		return rewriteValueRISCV64_OpRISCV64MOVBUreg(v)
 	case OpRISCV64MOVBload:
 		return rewriteValueRISCV64_OpRISCV64MOVBload(v)
+	case OpRISCV64MOVBreg:
+		return rewriteValueRISCV64_OpRISCV64MOVBreg(v)
 	case OpRISCV64MOVBstore:
 		return rewriteValueRISCV64_OpRISCV64MOVBstore(v)
 	case OpRISCV64MOVBstorezero:
@@ -430,22 +434,32 @@ func rewriteValueRISCV64(v *Value) bool {
 		return rewriteValueRISCV64_OpRISCV64MOVDconst(v)
 	case OpRISCV64MOVDload:
 		return rewriteValueRISCV64_OpRISCV64MOVDload(v)
+	case OpRISCV64MOVDreg:
+		return rewriteValueRISCV64_OpRISCV64MOVDreg(v)
 	case OpRISCV64MOVDstore:
 		return rewriteValueRISCV64_OpRISCV64MOVDstore(v)
 	case OpRISCV64MOVDstorezero:
 		return rewriteValueRISCV64_OpRISCV64MOVDstorezero(v)
 	case OpRISCV64MOVHUload:
 		return rewriteValueRISCV64_OpRISCV64MOVHUload(v)
+	case OpRISCV64MOVHUreg:
+		return rewriteValueRISCV64_OpRISCV64MOVHUreg(v)
 	case OpRISCV64MOVHload:
 		return rewriteValueRISCV64_OpRISCV64MOVHload(v)
+	case OpRISCV64MOVHreg:
+		return rewriteValueRISCV64_OpRISCV64MOVHreg(v)
 	case OpRISCV64MOVHstore:
 		return rewriteValueRISCV64_OpRISCV64MOVHstore(v)
 	case OpRISCV64MOVHstorezero:
 		return rewriteValueRISCV64_OpRISCV64MOVHstorezero(v)
 	case OpRISCV64MOVWUload:
 		return rewriteValueRISCV64_OpRISCV64MOVWUload(v)
+	case OpRISCV64MOVWUreg:
+		return rewriteValueRISCV64_OpRISCV64MOVWUreg(v)
 	case OpRISCV64MOVWload:
 		return rewriteValueRISCV64_OpRISCV64MOVWload(v)
+	case OpRISCV64MOVWreg:
+		return rewriteValueRISCV64_OpRISCV64MOVWreg(v)
 	case OpRISCV64MOVWstore:
 		return rewriteValueRISCV64_OpRISCV64MOVWstore(v)
 	case OpRISCV64MOVWstorezero:
@@ -543,17 +557,23 @@ func rewriteValueRISCV64(v *Value) bool {
 	case OpRsh8x8:
 		return rewriteValueRISCV64_OpRsh8x8(v)
 	case OpSignExt16to32:
-		return rewriteValueRISCV64_OpSignExt16to32(v)
+		v.Op = OpRISCV64MOVHreg
+		return true
 	case OpSignExt16to64:
-		return rewriteValueRISCV64_OpSignExt16to64(v)
+		v.Op = OpRISCV64MOVHreg
+		return true
 	case OpSignExt32to64:
-		return rewriteValueRISCV64_OpSignExt32to64(v)
+		v.Op = OpRISCV64MOVWreg
+		return true
 	case OpSignExt8to16:
-		return rewriteValueRISCV64_OpSignExt8to16(v)
+		v.Op = OpRISCV64MOVBreg
+		return true
 	case OpSignExt8to32:
-		return rewriteValueRISCV64_OpSignExt8to32(v)
+		v.Op = OpRISCV64MOVBreg
+		return true
 	case OpSignExt8to64:
-		return rewriteValueRISCV64_OpSignExt8to64(v)
+		v.Op = OpRISCV64MOVBreg
+		return true
 	case OpSlicemask:
 		return rewriteValueRISCV64_OpSlicemask(v)
 	case OpSqrt:
@@ -621,17 +641,23 @@ func rewriteValueRISCV64(v *Value) bool {
 	case OpZero:
 		return rewriteValueRISCV64_OpZero(v)
 	case OpZeroExt16to32:
-		return rewriteValueRISCV64_OpZeroExt16to32(v)
+		v.Op = OpRISCV64MOVHUreg
+		return true
 	case OpZeroExt16to64:
-		return rewriteValueRISCV64_OpZeroExt16to64(v)
+		v.Op = OpRISCV64MOVHUreg
+		return true
 	case OpZeroExt32to64:
-		return rewriteValueRISCV64_OpZeroExt32to64(v)
+		v.Op = OpRISCV64MOVWUreg
+		return true
 	case OpZeroExt8to16:
-		return rewriteValueRISCV64_OpZeroExt8to16(v)
+		v.Op = OpRISCV64MOVBUreg
+		return true
 	case OpZeroExt8to32:
-		return rewriteValueRISCV64_OpZeroExt8to32(v)
+		v.Op = OpRISCV64MOVBUreg
+		return true
 	case OpZeroExt8to64:
-		return rewriteValueRISCV64_OpZeroExt8to64(v)
+		v.Op = OpRISCV64MOVBUreg
+		return true
 	}
 	return false
 }
@@ -844,15 +870,17 @@ func rewriteValueRISCV64_OpEq16(v *Value) bool {
 	b := v.Block
 	typ := &b.Func.Config.Types
 	// match: (Eq16 x y)
-	// result: (SEQZ (ZeroExt16to64 (SUB <x.Type> x y)))
+	// result: (SEQZ (SUB <x.Type> (ZeroExt16to64 x) (ZeroExt16to64 y)))
 	for {
 		x := v_0
 		y := v_1
 		v.reset(OpRISCV64SEQZ)
-		v0 := b.NewValue0(v.Pos, OpZeroExt16to64, typ.UInt64)
-		v1 := b.NewValue0(v.Pos, OpRISCV64SUB, x.Type)
-		v1.AddArg2(x, y)
-		v0.AddArg(v1)
+		v0 := b.NewValue0(v.Pos, OpRISCV64SUB, x.Type)
+		v1 := b.NewValue0(v.Pos, OpZeroExt16to64, typ.UInt64)
+		v1.AddArg(x)
+		v2 := b.NewValue0(v.Pos, OpZeroExt16to64, typ.UInt64)
+		v2.AddArg(y)
+		v0.AddArg2(v1, v2)
 		v.AddArg(v0)
 		return true
 	}
@@ -895,15 +923,17 @@ func rewriteValueRISCV64_OpEq8(v *Value) bool {
 	b := v.Block
 	typ := &b.Func.Config.Types
 	// match: (Eq8 x y)
-	// result: (SEQZ (ZeroExt8to64 (SUB <x.Type> x y)))
+	// result: (SEQZ (SUB <x.Type> (ZeroExt8to64 x) (ZeroExt8to64 y)))
 	for {
 		x := v_0
 		y := v_1
 		v.reset(OpRISCV64SEQZ)
-		v0 := b.NewValue0(v.Pos, OpZeroExt8to64, typ.UInt64)
-		v1 := b.NewValue0(v.Pos, OpRISCV64SUB, x.Type)
-		v1.AddArg2(x, y)
-		v0.AddArg(v1)
+		v0 := b.NewValue0(v.Pos, OpRISCV64SUB, x.Type)
+		v1 := b.NewValue0(v.Pos, OpZeroExt8to64, typ.UInt64)
+		v1.AddArg(x)
+		v2 := b.NewValue0(v.Pos, OpZeroExt8to64, typ.UInt64)
+		v2.AddArg(y)
+		v0.AddArg2(v1, v2)
 		v.AddArg(v0)
 		return true
 	}
@@ -1987,6 +2017,23 @@ func rewriteValueRISCV64_OpMove(v *Value) bool {
 		return true
 	}
 	// match: (Move [s] {t} dst src mem)
+	// cond: s%8 == 0 && s >= 16 && s <= 8*128 && t.Alignment()%8 == 0 && !config.noDuffDevice && logLargeCopy(v, s)
+	// result: (DUFFCOPY [16 * (128 - s/8)] dst src mem)
+	for {
+		s := auxIntToInt64(v.AuxInt)
+		t := auxToType(v.Aux)
+		dst := v_0
+		src := v_1
+		mem := v_2
+		if !(s%8 == 0 && s >= 16 && s <= 8*128 && t.Alignment()%8 == 0 && !config.noDuffDevice && logLargeCopy(v, s)) {
+			break
+		}
+		v.reset(OpRISCV64DUFFCOPY)
+		v.AuxInt = int64ToAuxInt(16 * (128 - s/8))
+		v.AddArg3(dst, src, mem)
+		return true
+	}
+	// match: (Move [s] {t} dst src mem)
 	// cond: (s <= 16 || logLargeCopy(v, s))
 	// result: (LoweredMove [t.Alignment()] dst src (ADDI <src.Type> [s-moveSize(t.Alignment(), config)] src) mem)
 	for {
@@ -2052,15 +2099,17 @@ func rewriteValueRISCV64_OpNeq16(v *Value) bool {
 	b := v.Block
 	typ := &b.Func.Config.Types
 	// match: (Neq16 x y)
-	// result: (SNEZ (ZeroExt16to64 (SUB <x.Type> x y)))
+	// result: (SNEZ (SUB <x.Type> (ZeroExt16to64 x) (ZeroExt16to64 y)))
 	for {
 		x := v_0
 		y := v_1
 		v.reset(OpRISCV64SNEZ)
-		v0 := b.NewValue0(v.Pos, OpZeroExt16to64, typ.UInt64)
-		v1 := b.NewValue0(v.Pos, OpRISCV64SUB, x.Type)
-		v1.AddArg2(x, y)
-		v0.AddArg(v1)
+		v0 := b.NewValue0(v.Pos, OpRISCV64SUB, x.Type)
+		v1 := b.NewValue0(v.Pos, OpZeroExt16to64, typ.UInt64)
+		v1.AddArg(x)
+		v2 := b.NewValue0(v.Pos, OpZeroExt16to64, typ.UInt64)
+		v2.AddArg(y)
+		v0.AddArg2(v1, v2)
 		v.AddArg(v0)
 		return true
 	}
@@ -2103,15 +2152,17 @@ func rewriteValueRISCV64_OpNeq8(v *Value) bool {
 	b := v.Block
 	typ := &b.Func.Config.Types
 	// match: (Neq8 x y)
-	// result: (SNEZ (ZeroExt8to64 (SUB <x.Type> x y)))
+	// result: (SNEZ (SUB <x.Type> (ZeroExt8to64 x) (ZeroExt8to64 y)))
 	for {
 		x := v_0
 		y := v_1
 		v.reset(OpRISCV64SNEZ)
-		v0 := b.NewValue0(v.Pos, OpZeroExt8to64, typ.UInt64)
-		v1 := b.NewValue0(v.Pos, OpRISCV64SUB, x.Type)
-		v1.AddArg2(x, y)
-		v0.AddArg(v1)
+		v0 := b.NewValue0(v.Pos, OpRISCV64SUB, x.Type)
+		v1 := b.NewValue0(v.Pos, OpZeroExt8to64, typ.UInt64)
+		v1.AddArg(x)
+		v2 := b.NewValue0(v.Pos, OpZeroExt8to64, typ.UInt64)
+		v2.AddArg(y)
+		v0.AddArg2(v1, v2)
 		v.AddArg(v0)
 		return true
 	}
@@ -2458,6 +2509,57 @@ func rewriteValueRISCV64_OpRISCV64MOVBUload(v *Value) bool {
 	}
 	return false
 }
+func rewriteValueRISCV64_OpRISCV64MOVBUreg(v *Value) bool {
+	v_0 := v.Args[0]
+	b := v.Block
+	// match: (MOVBUreg x:(MOVBUload _ _))
+	// result: (MOVDreg x)
+	for {
+		x := v_0
+		if x.Op != OpRISCV64MOVBUload {
+			break
+		}
+		v.reset(OpRISCV64MOVDreg)
+		v.AddArg(x)
+		return true
+	}
+	// match: (MOVBUreg x:(MOVBUreg _))
+	// result: (MOVDreg x)
+	for {
+		x := v_0
+		if x.Op != OpRISCV64MOVBUreg {
+			break
+		}
+		v.reset(OpRISCV64MOVDreg)
+		v.AddArg(x)
+		return true
+	}
+	// match: (MOVBUreg <t> x:(MOVBload [off] {sym} ptr mem))
+	// cond: x.Uses == 1 && clobber(x)
+	// result: @x.Block (MOVBUload <t> [off] {sym} ptr mem)
+	for {
+		t := v.Type
+		x := v_0
+		if x.Op != OpRISCV64MOVBload {
+			break
+		}
+		off := auxIntToInt32(x.AuxInt)
+		sym := auxToSym(x.Aux)
+		mem := x.Args[1]
+		ptr := x.Args[0]
+		if !(x.Uses == 1 && clobber(x)) {
+			break
+		}
+		b = x.Block
+		v0 := b.NewValue0(x.Pos, OpRISCV64MOVBUload, t)
+		v.copyOf(v0)
+		v0.AuxInt = int32ToAuxInt(off)
+		v0.Aux = symToAux(sym)
+		v0.AddArg2(ptr, mem)
+		return true
+	}
+	return false
+}
 func rewriteValueRISCV64_OpRISCV64MOVBload(v *Value) bool {
 	v_1 := v.Args[1]
 	v_0 := v.Args[0]
@@ -2506,6 +2608,57 @@ func rewriteValueRISCV64_OpRISCV64MOVBload(v *Value) bool {
 	}
 	return false
 }
+func rewriteValueRISCV64_OpRISCV64MOVBreg(v *Value) bool {
+	v_0 := v.Args[0]
+	b := v.Block
+	// match: (MOVBreg x:(MOVBload _ _))
+	// result: (MOVDreg x)
+	for {
+		x := v_0
+		if x.Op != OpRISCV64MOVBload {
+			break
+		}
+		v.reset(OpRISCV64MOVDreg)
+		v.AddArg(x)
+		return true
+	}
+	// match: (MOVBreg x:(MOVBreg _))
+	// result: (MOVDreg x)
+	for {
+		x := v_0
+		if x.Op != OpRISCV64MOVBreg {
+			break
+		}
+		v.reset(OpRISCV64MOVDreg)
+		v.AddArg(x)
+		return true
+	}
+	// match: (MOVBreg <t> x:(MOVBUload [off] {sym} ptr mem))
+	// cond: x.Uses == 1 && clobber(x)
+	// result: @x.Block (MOVBload <t> [off] {sym} ptr mem)
+	for {
+		t := v.Type
+		x := v_0
+		if x.Op != OpRISCV64MOVBUload {
+			break
+		}
+		off := auxIntToInt32(x.AuxInt)
+		sym := auxToSym(x.Aux)
+		mem := x.Args[1]
+		ptr := x.Args[0]
+		if !(x.Uses == 1 && clobber(x)) {
+			break
+		}
+		b = x.Block
+		v0 := b.NewValue0(x.Pos, OpRISCV64MOVBload, t)
+		v.copyOf(v0)
+		v0.AuxInt = int32ToAuxInt(off)
+		v0.Aux = symToAux(sym)
+		v0.AddArg2(ptr, mem)
+		return true
+	}
+	return false
+}
 func rewriteValueRISCV64_OpRISCV64MOVBstore(v *Value) bool {
 	v_2 := v.Args[2]
 	v_1 := v.Args[1]
@@ -2571,6 +2724,108 @@ func rewriteValueRISCV64_OpRISCV64MOVBstore(v *Value) bool {
 		v.AddArg2(ptr, mem)
 		return true
 	}
+	// match: (MOVBstore [off] {sym} ptr (MOVBreg x) mem)
+	// result: (MOVBstore [off] {sym} ptr x mem)
+	for {
+		off := auxIntToInt32(v.AuxInt)
+		sym := auxToSym(v.Aux)
+		ptr := v_0
+		if v_1.Op != OpRISCV64MOVBreg {
+			break
+		}
+		x := v_1.Args[0]
+		mem := v_2
+		v.reset(OpRISCV64MOVBstore)
+		v.AuxInt = int32ToAuxInt(off)
+		v.Aux = symToAux(sym)
+		v.AddArg3(ptr, x, mem)
+		return true
+	}
+	// match: (MOVBstore [off] {sym} ptr (MOVHreg x) mem)
+	// result: (MOVBstore [off] {sym} ptr x mem)
+	for {
+		off := auxIntToInt32(v.AuxInt)
+		sym := auxToSym(v.Aux)
+		ptr := v_0
+		if v_1.Op != OpRISCV64MOVHreg {
+			break
+		}
+		x := v_1.Args[0]
+		mem := v_2
+		v.reset(OpRISCV64MOVBstore)
+		v.AuxInt = int32ToAuxInt(off)
+		v.Aux = symToAux(sym)
+		v.AddArg3(ptr, x, mem)
+		return true
+	}
+	// match: (MOVBstore [off] {sym} ptr (MOVWreg x) mem)
+	// result: (MOVBstore [off] {sym} ptr x mem)
+	for {
+		off := auxIntToInt32(v.AuxInt)
+		sym := auxToSym(v.Aux)
+		ptr := v_0
+		if v_1.Op != OpRISCV64MOVWreg {
+			break
+		}
+		x := v_1.Args[0]
+		mem := v_2
+		v.reset(OpRISCV64MOVBstore)
+		v.AuxInt = int32ToAuxInt(off)
+		v.Aux = symToAux(sym)
+		v.AddArg3(ptr, x, mem)
+		return true
+	}
+	// match: (MOVBstore [off] {sym} ptr (MOVBUreg x) mem)
+	// result: (MOVBstore [off] {sym} ptr x mem)
+	for {
+		off := auxIntToInt32(v.AuxInt)
+		sym := auxToSym(v.Aux)
+		ptr := v_0
+		if v_1.Op != OpRISCV64MOVBUreg {
+			break
+		}
+		x := v_1.Args[0]
+		mem := v_2
+		v.reset(OpRISCV64MOVBstore)
+		v.AuxInt = int32ToAuxInt(off)
+		v.Aux = symToAux(sym)
+		v.AddArg3(ptr, x, mem)
+		return true
+	}
+	// match: (MOVBstore [off] {sym} ptr (MOVHUreg x) mem)
+	// result: (MOVBstore [off] {sym} ptr x mem)
+	for {
+		off := auxIntToInt32(v.AuxInt)
+		sym := auxToSym(v.Aux)
+		ptr := v_0
+		if v_1.Op != OpRISCV64MOVHUreg {
+			break
+		}
+		x := v_1.Args[0]
+		mem := v_2
+		v.reset(OpRISCV64MOVBstore)
+		v.AuxInt = int32ToAuxInt(off)
+		v.Aux = symToAux(sym)
+		v.AddArg3(ptr, x, mem)
+		return true
+	}
+	// match: (MOVBstore [off] {sym} ptr (MOVWUreg x) mem)
+	// result: (MOVBstore [off] {sym} ptr x mem)
+	for {
+		off := auxIntToInt32(v.AuxInt)
+		sym := auxToSym(v.Aux)
+		ptr := v_0
+		if v_1.Op != OpRISCV64MOVWUreg {
+			break
+		}
+		x := v_1.Args[0]
+		mem := v_2
+		v.reset(OpRISCV64MOVBstore)
+		v.AuxInt = int32ToAuxInt(off)
+		v.Aux = symToAux(sym)
+		v.AddArg3(ptr, x, mem)
+		return true
+	}
 	return false
 }
 func rewriteValueRISCV64_OpRISCV64MOVBstorezero(v *Value) bool {
@@ -2714,6 +2969,22 @@ func rewriteValueRISCV64_OpRISCV64MOVDload(v *Value) bool {
 	}
 	return false
 }
+func rewriteValueRISCV64_OpRISCV64MOVDreg(v *Value) bool {
+	v_0 := v.Args[0]
+	// match: (MOVDreg x)
+	// cond: x.Uses == 1
+	// result: (MOVDnop x)
+	for {
+		x := v_0
+		if !(x.Uses == 1) {
+			break
+		}
+		v.reset(OpRISCV64MOVDnop)
+		v.AddArg(x)
+		return true
+	}
+	return false
+}
 func rewriteValueRISCV64_OpRISCV64MOVDstore(v *Value) bool {
 	v_2 := v.Args[2]
 	v_1 := v.Args[1]
@@ -2877,6 +3148,79 @@ func rewriteValueRISCV64_OpRISCV64MOVHUload(v *Value) bool {
 	}
 	return false
 }
+func rewriteValueRISCV64_OpRISCV64MOVHUreg(v *Value) bool {
+	v_0 := v.Args[0]
+	b := v.Block
+	// match: (MOVHUreg x:(MOVBUload _ _))
+	// result: (MOVDreg x)
+	for {
+		x := v_0
+		if x.Op != OpRISCV64MOVBUload {
+			break
+		}
+		v.reset(OpRISCV64MOVDreg)
+		v.AddArg(x)
+		return true
+	}
+	// match: (MOVHUreg x:(MOVHUload _ _))
+	// result: (MOVDreg x)
+	for {
+		x := v_0
+		if x.Op != OpRISCV64MOVHUload {
+			break
+		}
+		v.reset(OpRISCV64MOVDreg)
+		v.AddArg(x)
+		return true
+	}
+	// match: (MOVHUreg x:(MOVBUreg _))
+	// result: (MOVDreg x)
+	for {
+		x := v_0
+		if x.Op != OpRISCV64MOVBUreg {
+			break
+		}
+		v.reset(OpRISCV64MOVDreg)
+		v.AddArg(x)
+		return true
+	}
+	// match: (MOVHUreg x:(MOVHUreg _))
+	// result: (MOVDreg x)
+	for {
+		x := v_0
+		if x.Op != OpRISCV64MOVHUreg {
+			break
+		}
+		v.reset(OpRISCV64MOVDreg)
+		v.AddArg(x)
+		return true
+	}
+	// match: (MOVHUreg <t> x:(MOVHload [off] {sym} ptr mem))
+	// cond: x.Uses == 1 && clobber(x)
+	// result: @x.Block (MOVHUload <t> [off] {sym} ptr mem)
+	for {
+		t := v.Type
+		x := v_0
+		if x.Op != OpRISCV64MOVHload {
+			break
+		}
+		off := auxIntToInt32(x.AuxInt)
+		sym := auxToSym(x.Aux)
+		mem := x.Args[1]
+		ptr := x.Args[0]
+		if !(x.Uses == 1 && clobber(x)) {
+			break
+		}
+		b = x.Block
+		v0 := b.NewValue0(x.Pos, OpRISCV64MOVHUload, t)
+		v.copyOf(v0)
+		v0.AuxInt = int32ToAuxInt(off)
+		v0.Aux = symToAux(sym)
+		v0.AddArg2(ptr, mem)
+		return true
+	}
+	return false
+}
 func rewriteValueRISCV64_OpRISCV64MOVHload(v *Value) bool {
 	v_1 := v.Args[1]
 	v_0 := v.Args[0]
@@ -2925,6 +3269,101 @@ func rewriteValueRISCV64_OpRISCV64MOVHload(v *Value) bool {
 	}
 	return false
 }
+func rewriteValueRISCV64_OpRISCV64MOVHreg(v *Value) bool {
+	v_0 := v.Args[0]
+	b := v.Block
+	// match: (MOVHreg x:(MOVBload _ _))
+	// result: (MOVDreg x)
+	for {
+		x := v_0
+		if x.Op != OpRISCV64MOVBload {
+			break
+		}
+		v.reset(OpRISCV64MOVDreg)
+		v.AddArg(x)
+		return true
+	}
+	// match: (MOVHreg x:(MOVBUload _ _))
+	// result: (MOVDreg x)
+	for {
+		x := v_0
+		if x.Op != OpRISCV64MOVBUload {
+			break
+		}
+		v.reset(OpRISCV64MOVDreg)
+		v.AddArg(x)
+		return true
+	}
+	// match: (MOVHreg x:(MOVHload _ _))
+	// result: (MOVDreg x)
+	for {
+		x := v_0
+		if x.Op != OpRISCV64MOVHload {
+			break
+		}
+		v.reset(OpRISCV64MOVDreg)
+		v.AddArg(x)
+		return true
+	}
+	// match: (MOVHreg x:(MOVBreg _))
+	// result: (MOVDreg x)
+	for {
+		x := v_0
+		if x.Op != OpRISCV64MOVBreg {
+			break
+		}
+		v.reset(OpRISCV64MOVDreg)
+		v.AddArg(x)
+		return true
+	}
+	// match: (MOVHreg x:(MOVBUreg _))
+	// result: (MOVDreg x)
+	for {
+		x := v_0
+		if x.Op != OpRISCV64MOVBUreg {
+			break
+		}
+		v.reset(OpRISCV64MOVDreg)
+		v.AddArg(x)
+		return true
+	}
+	// match: (MOVHreg x:(MOVHreg _))
+	// result: (MOVDreg x)
+	for {
+		x := v_0
+		if x.Op != OpRISCV64MOVHreg {
+			break
+		}
+		v.reset(OpRISCV64MOVDreg)
+		v.AddArg(x)
+		return true
+	}
+	// match: (MOVHreg <t> x:(MOVHUload [off] {sym} ptr mem))
+	// cond: x.Uses == 1 && clobber(x)
+	// result: @x.Block (MOVHload <t> [off] {sym} ptr mem)
+	for {
+		t := v.Type
+		x := v_0
+		if x.Op != OpRISCV64MOVHUload {
+			break
+		}
+		off := auxIntToInt32(x.AuxInt)
+		sym := auxToSym(x.Aux)
+		mem := x.Args[1]
+		ptr := x.Args[0]
+		if !(x.Uses == 1 && clobber(x)) {
+			break
+		}
+		b = x.Block
+		v0 := b.NewValue0(x.Pos, OpRISCV64MOVHload, t)
+		v.copyOf(v0)
+		v0.AuxInt = int32ToAuxInt(off)
+		v0.Aux = symToAux(sym)
+		v0.AddArg2(ptr, mem)
+		return true
+	}
+	return false
+}
 func rewriteValueRISCV64_OpRISCV64MOVHstore(v *Value) bool {
 	v_2 := v.Args[2]
 	v_1 := v.Args[1]
@@ -2990,6 +3429,74 @@ func rewriteValueRISCV64_OpRISCV64MOVHstore(v *Value) bool {
 		v.AddArg2(ptr, mem)
 		return true
 	}
+	// match: (MOVHstore [off] {sym} ptr (MOVHreg x) mem)
+	// result: (MOVHstore [off] {sym} ptr x mem)
+	for {
+		off := auxIntToInt32(v.AuxInt)
+		sym := auxToSym(v.Aux)
+		ptr := v_0
+		if v_1.Op != OpRISCV64MOVHreg {
+			break
+		}
+		x := v_1.Args[0]
+		mem := v_2
+		v.reset(OpRISCV64MOVHstore)
+		v.AuxInt = int32ToAuxInt(off)
+		v.Aux = symToAux(sym)
+		v.AddArg3(ptr, x, mem)
+		return true
+	}
+	// match: (MOVHstore [off] {sym} ptr (MOVWreg x) mem)
+	// result: (MOVHstore [off] {sym} ptr x mem)
+	for {
+		off := auxIntToInt32(v.AuxInt)
+		sym := auxToSym(v.Aux)
+		ptr := v_0
+		if v_1.Op != OpRISCV64MOVWreg {
+			break
+		}
+		x := v_1.Args[0]
+		mem := v_2
+		v.reset(OpRISCV64MOVHstore)
+		v.AuxInt = int32ToAuxInt(off)
+		v.Aux = symToAux(sym)
+		v.AddArg3(ptr, x, mem)
+		return true
+	}
+	// match: (MOVHstore [off] {sym} ptr (MOVHUreg x) mem)
+	// result: (MOVHstore [off] {sym} ptr x mem)
+	for {
+		off := auxIntToInt32(v.AuxInt)
+		sym := auxToSym(v.Aux)
+		ptr := v_0
+		if v_1.Op != OpRISCV64MOVHUreg {
+			break
+		}
+		x := v_1.Args[0]
+		mem := v_2
+		v.reset(OpRISCV64MOVHstore)
+		v.AuxInt = int32ToAuxInt(off)
+		v.Aux = symToAux(sym)
+		v.AddArg3(ptr, x, mem)
+		return true
+	}
+	// match: (MOVHstore [off] {sym} ptr (MOVWUreg x) mem)
+	// result: (MOVHstore [off] {sym} ptr x mem)
+	for {
+		off := auxIntToInt32(v.AuxInt)
+		sym := auxToSym(v.Aux)
+		ptr := v_0
+		if v_1.Op != OpRISCV64MOVWUreg {
+			break
+		}
+		x := v_1.Args[0]
+		mem := v_2
+		v.reset(OpRISCV64MOVHstore)
+		v.AuxInt = int32ToAuxInt(off)
+		v.Aux = symToAux(sym)
+		v.AddArg3(ptr, x, mem)
+		return true
+	}
 	return false
 }
 func rewriteValueRISCV64_OpRISCV64MOVHstorezero(v *Value) bool {
@@ -3088,6 +3595,101 @@ func rewriteValueRISCV64_OpRISCV64MOVWUload(v *Value) bool {
 	}
 	return false
 }
+func rewriteValueRISCV64_OpRISCV64MOVWUreg(v *Value) bool {
+	v_0 := v.Args[0]
+	b := v.Block
+	// match: (MOVWUreg x:(MOVBUload _ _))
+	// result: (MOVDreg x)
+	for {
+		x := v_0
+		if x.Op != OpRISCV64MOVBUload {
+			break
+		}
+		v.reset(OpRISCV64MOVDreg)
+		v.AddArg(x)
+		return true
+	}
+	// match: (MOVWUreg x:(MOVHUload _ _))
+	// result: (MOVDreg x)
+	for {
+		x := v_0
+		if x.Op != OpRISCV64MOVHUload {
+			break
+		}
+		v.reset(OpRISCV64MOVDreg)
+		v.AddArg(x)
+		return true
+	}
+	// match: (MOVWUreg x:(MOVWUload _ _))
+	// result: (MOVDreg x)
+	for {
+		x := v_0
+		if x.Op != OpRISCV64MOVWUload {
+			break
+		}
+		v.reset(OpRISCV64MOVDreg)
+		v.AddArg(x)
+		return true
+	}
+	// match: (MOVWUreg x:(MOVBUreg _))
+	// result: (MOVDreg x)
+	for {
+		x := v_0
+		if x.Op != OpRISCV64MOVBUreg {
+			break
+		}
+		v.reset(OpRISCV64MOVDreg)
+		v.AddArg(x)
+		return true
+	}
+	// match: (MOVWUreg x:(MOVHUreg _))
+	// result: (MOVDreg x)
+	for {
+		x := v_0
+		if x.Op != OpRISCV64MOVHUreg {
+			break
+		}
+		v.reset(OpRISCV64MOVDreg)
+		v.AddArg(x)
+		return true
+	}
+	// match: (MOVWUreg x:(MOVWUreg _))
+	// result: (MOVDreg x)
+	for {
+		x := v_0
+		if x.Op != OpRISCV64MOVWUreg {
+			break
+		}
+		v.reset(OpRISCV64MOVDreg)
+		v.AddArg(x)
+		return true
+	}
+	// match: (MOVWUreg <t> x:(MOVWload [off] {sym} ptr mem))
+	// cond: x.Uses == 1 && clobber(x)
+	// result: @x.Block (MOVWUload <t> [off] {sym} ptr mem)
+	for {
+		t := v.Type
+		x := v_0
+		if x.Op != OpRISCV64MOVWload {
+			break
+		}
+		off := auxIntToInt32(x.AuxInt)
+		sym := auxToSym(x.Aux)
+		mem := x.Args[1]
+		ptr := x.Args[0]
+		if !(x.Uses == 1 && clobber(x)) {
+			break
+		}
+		b = x.Block
+		v0 := b.NewValue0(x.Pos, OpRISCV64MOVWUload, t)
+		v.copyOf(v0)
+		v0.AuxInt = int32ToAuxInt(off)
+		v0.Aux = symToAux(sym)
+		v0.AddArg2(ptr, mem)
+		return true
+	}
+	return false
+}
 func rewriteValueRISCV64_OpRISCV64MOVWload(v *Value) bool {
 	v_1 := v.Args[1]
 	v_0 := v.Args[0]
@@ -3136,6 +3738,134 @@ func rewriteValueRISCV64_OpRISCV64MOVWload(v *Value) bool {
 	}
 	return false
 }
+func rewriteValueRISCV64_OpRISCV64MOVWreg(v *Value) bool {
+	v_0 := v.Args[0]
+	b := v.Block
+	// match: (MOVWreg x:(MOVBload _ _))
+	// result: (MOVDreg x)
+	for {
+		x := v_0
+		if x.Op != OpRISCV64MOVBload {
+			break
+		}
+		v.reset(OpRISCV64MOVDreg)
+		v.AddArg(x)
+		return true
+	}
+	// match: (MOVWreg x:(MOVBUload _ _))
+	// result: (MOVDreg x)
+	for {
+		x := v_0
+		if x.Op != OpRISCV64MOVBUload {
+			break
+		}
+		v.reset(OpRISCV64MOVDreg)
+		v.AddArg(x)
+		return true
+	}
+	// match: (MOVWreg x:(MOVHload _ _))
+	// result: (MOVDreg x)
+	for {
+		x := v_0
+		if x.Op != OpRISCV64MOVHload {
+			break
+		}
+		v.reset(OpRISCV64MOVDreg)
+		v.AddArg(x)
+		return true
+	}
+	// match: (MOVWreg x:(MOVHUload _ _))
+	// result: (MOVDreg x)
+	for {
+		x := v_0
+		if x.Op != OpRISCV64MOVHUload {
+			break
+		}
+		v.reset(OpRISCV64MOVDreg)
+		v.AddArg(x)
+		return true
+	}
+	// match: (MOVWreg x:(MOVWload _ _))
+	// result: (MOVDreg x)
+	for {
+		x := v_0
+		if x.Op != OpRISCV64MOVWload {
+			break
+		}
+		v.reset(OpRISCV64MOVDreg)
+		v.AddArg(x)
+		return true
+	}
+	// match: (MOVWreg x:(MOVBreg _))
+	// result: (MOVDreg x)
+	for {
+		x := v_0
+		if x.Op != OpRISCV64MOVBreg {
+			break
+		}
+		v.reset(OpRISCV64MOVDreg)
+		v.AddArg(x)
+		return true
+	}
+	// match: (MOVWreg x:(MOVBUreg _))
+	// result: (MOVDreg x)
+	for {
+		x := v_0
+		if x.Op != OpRISCV64MOVBUreg {
+			break
+		}
+		v.reset(OpRISCV64MOVDreg)
+		v.AddArg(x)
+		return true
+	}
+	// match: (MOVWreg x:(MOVHreg _))
+	// result: (MOVDreg x)
+	for {
+		x := v_0
+		if x.Op != OpRISCV64MOVHreg {
+			break
+		}
+		v.reset(OpRISCV64MOVDreg)
+		v.AddArg(x)
+		return true
+	}
+	// match: (MOVWreg x:(MOVWreg _))
+	// result: (MOVDreg x)
+	for {
+		x := v_0
+		if x.Op != OpRISCV64MOVWreg {
+			break
+		}
+		v.reset(OpRISCV64MOVDreg)
+		v.AddArg(x)
+		return true
+	}
+	// match: (MOVWreg <t> x:(MOVWUload [off] {sym} ptr mem))
+	// cond: x.Uses == 1 && clobber(x)
+	// result: @x.Block (MOVWload <t> [off] {sym} ptr mem)
+	for {
+		t := v.Type
+		x := v_0
+		if x.Op != OpRISCV64MOVWUload {
+			break
+		}
+		off := auxIntToInt32(x.AuxInt)
+		sym := auxToSym(x.Aux)
+		mem := x.Args[1]
+		ptr := x.Args[0]
+		if !(x.Uses == 1 && clobber(x)) {
+			break
+		}
+		b = x.Block
+		v0 := b.NewValue0(x.Pos, OpRISCV64MOVWload, t)
+		v.copyOf(v0)
+		v0.AuxInt = int32ToAuxInt(off)
+		v0.Aux = symToAux(sym)
+		v0.AddArg2(ptr, mem)
+		return true
+	}
+	return false
+}
 func rewriteValueRISCV64_OpRISCV64MOVWstore(v *Value) bool {
 	v_2 := v.Args[2]
 	v_1 := v.Args[1]
@@ -3201,6 +3931,40 @@ func rewriteValueRISCV64_OpRISCV64MOVWstore(v *Value) bool {
 		v.AddArg2(ptr, mem)
 		return true
 	}
+	// match: (MOVWstore [off] {sym} ptr (MOVWreg x) mem)
+	// result: (MOVWstore [off] {sym} ptr x mem)
+	for {
+		off := auxIntToInt32(v.AuxInt)
+		sym := auxToSym(v.Aux)
+		ptr := v_0
+		if v_1.Op != OpRISCV64MOVWreg {
+			break
+		}
+		x := v_1.Args[0]
+		mem := v_2
+		v.reset(OpRISCV64MOVWstore)
+		v.AuxInt = int32ToAuxInt(off)
+		v.Aux = symToAux(sym)
+		v.AddArg3(ptr, x, mem)
+		return true
+	}
+	// match: (MOVWstore [off] {sym} ptr (MOVWUreg x) mem)
+	// result: (MOVWstore [off] {sym} ptr x mem)
+	for {
+		off := auxIntToInt32(v.AuxInt)
+		sym := auxToSym(v.Aux)
+		ptr := v_0
+		if v_1.Op != OpRISCV64MOVWUreg {
+			break
+		}
+		x := v_1.Args[0]
+		mem := v_2
+		v.reset(OpRISCV64MOVWstore)
+		v.AuxInt = int32ToAuxInt(off)
+		v.Aux = symToAux(sym)
+		v.AddArg3(ptr, x, mem)
+		return true
+	}
 	return false
 }
 func rewriteValueRISCV64_OpRISCV64MOVWstorezero(v *Value) bool {
@@ -4719,103 +5483,6 @@ func rewriteValueRISCV64_OpRsh8x8(v *Value) bool {
 		return true
 	}
 }
-func rewriteValueRISCV64_OpSignExt16to32(v *Value) bool {
-	v_0 := v.Args[0]
-	b := v.Block
-	// match: (SignExt16to32 <t> x)
-	// result: (SRAI [48] (SLLI <t> [48] x))
-	for {
-		t := v.Type
-		x := v_0
-		v.reset(OpRISCV64SRAI)
-		v.AuxInt = int64ToAuxInt(48)
-		v0 := b.NewValue0(v.Pos, OpRISCV64SLLI, t)
-		v0.AuxInt = int64ToAuxInt(48)
-		v0.AddArg(x)
-		v.AddArg(v0)
-		return true
-	}
-}
-func rewriteValueRISCV64_OpSignExt16to64(v *Value) bool {
-	v_0 := v.Args[0]
-	b := v.Block
-	// match: (SignExt16to64 <t> x)
-	// result: (SRAI [48] (SLLI <t> [48] x))
-	for {
-		t := v.Type
-		x := v_0
-		v.reset(OpRISCV64SRAI)
-		v.AuxInt = int64ToAuxInt(48)
-		v0 := b.NewValue0(v.Pos, OpRISCV64SLLI, t)
-		v0.AuxInt = int64ToAuxInt(48)
-		v0.AddArg(x)
-		v.AddArg(v0)
-		return true
-	}
-}
-func rewriteValueRISCV64_OpSignExt32to64(v *Value) bool {
-	v_0 := v.Args[0]
-	// match: (SignExt32to64 <t> x)
-	// result: (ADDIW [0] x)
-	for {
-		x := v_0
-		v.reset(OpRISCV64ADDIW)
-		v.AuxInt = int64ToAuxInt(0)
-		v.AddArg(x)
-		return true
-	}
-}
-func rewriteValueRISCV64_OpSignExt8to16(v *Value) bool {
-	v_0 := v.Args[0]
-	b := v.Block
-	// match: (SignExt8to16 <t> x)
-	// result: (SRAI [56] (SLLI <t> [56] x))
-	for {
-		t := v.Type
-		x := v_0
-		v.reset(OpRISCV64SRAI)
-		v.AuxInt = int64ToAuxInt(56)
-		v0 := b.NewValue0(v.Pos, OpRISCV64SLLI, t)
-		v0.AuxInt = int64ToAuxInt(56)
-		v0.AddArg(x)
-		v.AddArg(v0)
-		return true
-	}
-}
-func rewriteValueRISCV64_OpSignExt8to32(v *Value) bool {
-	v_0 := v.Args[0]
-	b := v.Block
-	// match: (SignExt8to32 <t> x)
-	// result: (SRAI [56] (SLLI <t> [56] x))
-	for {
-		t := v.Type
-		x := v_0
-		v.reset(OpRISCV64SRAI)
-		v.AuxInt = int64ToAuxInt(56)
-		v0 := b.NewValue0(v.Pos, OpRISCV64SLLI, t)
-		v0.AuxInt = int64ToAuxInt(56)
-		v0.AddArg(x)
-		v.AddArg(v0)
-		return true
-	}
-}
-func rewriteValueRISCV64_OpSignExt8to64(v *Value) bool {
-	v_0 := v.Args[0]
-	b := v.Block
-	// match: (SignExt8to64 <t> x)
-	// result: (SRAI [56] (SLLI <t> [56] x))
-	for {
-		t := v.Type
-		x := v_0
-		v.reset(OpRISCV64SRAI)
-		v.AuxInt = int64ToAuxInt(56)
-		v0 := b.NewValue0(v.Pos, OpRISCV64SLLI, t)
-		v0.AuxInt = int64ToAuxInt(56)
-		v0.AddArg(x)
-		v.AddArg(v0)
-		return true
-	}
-}
 func rewriteValueRISCV64_OpSlicemask(v *Value) bool {
 	v_0 := v.Args[0]
 	b := v.Block
@@ -5000,6 +5667,22 @@ func rewriteValueRISCV64_OpZero(v *Value) bool {
 		return true
 	}
 	// match: (Zero [s] {t} ptr mem)
+	// cond: s%8 == 0 && s >= 16 && s <= 8*128 && t.Alignment()%8 == 0 && !config.noDuffDevice
+	// result: (DUFFZERO [8 * (128 - s/8)] ptr mem)
+	for {
+		s := auxIntToInt64(v.AuxInt)
+		t := auxToType(v.Aux)
+		ptr := v_0
+		mem := v_1
+		if !(s%8 == 0 && s >= 16 && s <= 8*128 && t.Alignment()%8 == 0 && !config.noDuffDevice) {
+			break
+		}
+		v.reset(OpRISCV64DUFFZERO)
+		v.AuxInt = int64ToAuxInt(8 * (128 - s/8))
+		v.AddArg2(ptr, mem)
+		return true
+	}
+	// match: (Zero [s] {t} ptr mem)
 	// result: (LoweredZero [t.Alignment()] ptr (ADD <ptr.Type> ptr (MOVDconst [s-moveSize(t.Alignment(), config)])) mem)
 	for {
 		s := auxIntToInt64(v.AuxInt)
@@ -5016,108 +5699,6 @@ func rewriteValueRISCV64_OpZero(v *Value) bool {
 		return true
 	}
 }
-func rewriteValueRISCV64_OpZeroExt16to32(v *Value) bool {
-	v_0 := v.Args[0]
-	b := v.Block
-	// match: (ZeroExt16to32 <t> x)
-	// result: (SRLI [48] (SLLI <t> [48] x))
-	for {
-		t := v.Type
-		x := v_0
-		v.reset(OpRISCV64SRLI)
-		v.AuxInt = int64ToAuxInt(48)
-		v0 := b.NewValue0(v.Pos, OpRISCV64SLLI, t)
-		v0.AuxInt = int64ToAuxInt(48)
-		v0.AddArg(x)
-		v.AddArg(v0)
-		return true
-	}
-}
-func rewriteValueRISCV64_OpZeroExt16to64(v *Value) bool {
-	v_0 := v.Args[0]
-	b := v.Block
-	// match: (ZeroExt16to64 <t> x)
-	// result: (SRLI [48] (SLLI <t> [48] x))
-	for {
-		t := v.Type
-		x := v_0
-		v.reset(OpRISCV64SRLI)
-		v.AuxInt = int64ToAuxInt(48)
-		v0 := b.NewValue0(v.Pos, OpRISCV64SLLI, t)
-		v0.AuxInt = int64ToAuxInt(48)
-		v0.AddArg(x)
-		v.AddArg(v0)
-		return true
-	}
-}
-func rewriteValueRISCV64_OpZeroExt32to64(v *Value) bool {
-	v_0 := v.Args[0]
-	b := v.Block
-	// match: (ZeroExt32to64 <t> x)
-	// result: (SRLI [32] (SLLI <t> [32] x))
-	for {
-		t := v.Type
-		x := v_0
-		v.reset(OpRISCV64SRLI)
-		v.AuxInt = int64ToAuxInt(32)
-		v0 := b.NewValue0(v.Pos, OpRISCV64SLLI, t)
-		v0.AuxInt = int64ToAuxInt(32)
-		v0.AddArg(x)
-		v.AddArg(v0)
-		return true
-	}
-}
-func rewriteValueRISCV64_OpZeroExt8to16(v *Value) bool {
-	v_0 := v.Args[0]
-	b := v.Block
-	// match: (ZeroExt8to16 <t> x)
-	// result: (SRLI [56] (SLLI <t> [56] x))
-	for {
-		t := v.Type
-		x := v_0
-		v.reset(OpRISCV64SRLI)
-		v.AuxInt = int64ToAuxInt(56)
-		v0 := b.NewValue0(v.Pos, OpRISCV64SLLI, t)
-		v0.AuxInt = int64ToAuxInt(56)
-		v0.AddArg(x)
-		v.AddArg(v0)
-		return true
-	}
-}
-func rewriteValueRISCV64_OpZeroExt8to32(v *Value) bool {
-	v_0 := v.Args[0]
-	b := v.Block
-	// match: (ZeroExt8to32 <t> x)
-	// result: (SRLI [56] (SLLI <t> [56] x))
-	for {
-		t := v.Type
-		x := v_0
-		v.reset(OpRISCV64SRLI)
-		v.AuxInt = int64ToAuxInt(56)
-		v0 := b.NewValue0(v.Pos, OpRISCV64SLLI, t)
-		v0.AuxInt = int64ToAuxInt(56)
-		v0.AddArg(x)
-		v.AddArg(v0)
-		return true
-	}
-}
-func rewriteValueRISCV64_OpZeroExt8to64(v *Value) bool {
-	v_0 := v.Args[0]
-	b := v.Block
-	// match: (ZeroExt8to64 <t> x)
-	// result: (SRLI [56] (SLLI <t> [56] x))
-	for {
-		t := v.Type
-		x := v_0
-		v.reset(OpRISCV64SRLI)
-		v.AuxInt = int64ToAuxInt(56)
-		v0 := b.NewValue0(v.Pos, OpRISCV64SLLI, t)
-		v0.AuxInt = int64ToAuxInt(56)
-		v0.AddArg(x)
-		v.AddArg(v0)
-		return true
-	}
-}
 func rewriteBlockRISCV64(b *Block) bool {
 	switch b.Kind {
 	case BlockRISCV64BEQ: