runtime: add wasm bulk memory operations

The existing implementation uses loops to implement bulk memory
operations such as memcpy and memclr. Now that bulk memory operations
have been standardized and are implemented in all major browsers and
engines (see https://webassembly.org/roadmap/), we should use them
to improve performance.

Updates golang#28360

Change-Id: I28df0e0350287d5e7e1d1c09a4064ea1054e7575
Reviewed-on: https://go-review.googlesource.com/c/go/+/444935
Reviewed-by: Cherry Mui <[email protected]>
Reviewed-by: Keith Randall <[email protected]>
TryBot-Result: Gopher Robot <[email protected]>
Run-TryBot: Richard Musiol <[email protected]>
Reviewed-by: David Chase <[email protected]>
Auto-Submit: Richard Musiol <[email protected]>
Reviewed-by: Richard Musiol <[email protected]>

Author: garet90

src/cmd/compile/internal/ir/symtab.go

@@ -58,10 +58,6 @@ var Syms struct {
 	// Wasm
 	WasmDiv *obj.LSym
 	// Wasm
-	WasmMove *obj.LSym
-	// Wasm
-	WasmZero *obj.LSym
-	// Wasm
 	WasmTruncS *obj.LSym
 	// Wasm
 	WasmTruncU *obj.LSym

src/cmd/compile/internal/ssa/_gen/Wasm.rules

@@ -234,24 +234,9 @@
 	(I64Store [s-8] dst (I64Load [s-8] src mem)
 		(I64Store dst (I64Load src mem) mem))
 
-// Adjust moves to be a multiple of 16 bytes.
-(Move [s] dst src mem)
-	&& s > 16 && s%16 != 0 && s%16 <= 8 =>
-	(Move [s-s%16]
-		(OffPtr <dst.Type> dst [s%16])
-		(OffPtr <src.Type> src [s%16])
-		(I64Store dst (I64Load src mem) mem))
-(Move [s] dst src mem)
-	&& s > 16 && s%16 != 0 && s%16 > 8 =>
-	(Move [s-s%16]
-		(OffPtr <dst.Type> dst [s%16])
-		(OffPtr <src.Type> src [s%16])
-		(I64Store [8] dst (I64Load [8] src mem)
-			(I64Store dst (I64Load src mem) mem)))
-
 // Large copying uses helper.
-(Move [s] dst src mem) && s%8 == 0 && logLargeCopy(v, s) =>
-	(LoweredMove [s/8] dst src mem)
+(Move [s] dst src mem) && logLargeCopy(v, s) =>
+	(LoweredMove [s] dst src mem)
 
 // Lowering Zero instructions
 (Zero [0] _ mem) => mem
@@ -274,7 +259,7 @@
 		(I64Store32 destptr (I64Const [0]) mem))
 
 // Strip off any fractional word zeroing.
-(Zero [s] destptr mem) && s%8 != 0 && s > 8 =>
+(Zero [s] destptr mem) && s%8 != 0 && s > 8 && s < 32 =>
 	(Zero [s-s%8] (OffPtr <destptr.Type> destptr [s%8])
 		(I64Store destptr (I64Const [0]) mem))
 
@@ -293,8 +278,8 @@
 				(I64Store destptr (I64Const [0]) mem))))
 
 // Large zeroing uses helper.
-(Zero [s] destptr mem) && s%8 == 0 && s > 32 =>
-	(LoweredZero [s/8] destptr mem)
+(Zero [s] destptr mem) =>
+	(LoweredZero [s] destptr mem)
 
 // Lowering constants
 (Const64 ...) => (I64Const ...)

src/cmd/compile/internal/ssa/_gen/WasmOps.go

@@ -126,8 +126,8 @@ func init() {
 		{name: "LoweredInterCall", argLength: 2, reg: regInfo{inputs: []regMask{gp}, clobbers: callerSave}, aux: "CallOff", call: true},          // call fn by pointer. arg0=codeptr, arg1=mem, auxint=argsize, returns mem
 
 		{name: "LoweredAddr", argLength: 1, reg: gp11, aux: "SymOff", rematerializeable: true, symEffect: "Addr"}, // returns base+aux+auxint, arg0=base
-		{name: "LoweredMove", argLength: 3, reg: regInfo{inputs: []regMask{gp, gp}}, aux: "Int64"},                // large move. arg0=dst, arg1=src, arg2=mem, auxint=len/8, returns mem
-		{name: "LoweredZero", argLength: 2, reg: regInfo{inputs: []regMask{gp}}, aux: "Int64"},                    // large zeroing. arg0=start, arg1=mem, auxint=len/8, returns mem
+		{name: "LoweredMove", argLength: 3, reg: regInfo{inputs: []regMask{gp, gp}}, aux: "Int64"},                // large move. arg0=dst, arg1=src, arg2=mem, auxint=len, returns mem
+		{name: "LoweredZero", argLength: 2, reg: regInfo{inputs: []regMask{gp}}, aux: "Int64"},                    // large zeroing. arg0=start, arg1=mem, auxint=len, returns mem
 
 		{name: "LoweredGetClosurePtr", reg: gp01},                                                                          // returns wasm.REG_CTXT, the closure pointer
 		{name: "LoweredGetCallerPC", reg: gp01, rematerializeable: true},                                                   // returns the PC of the caller of the current function

src/cmd/compile/internal/ssa/rewriteWasm.go

@@ -206,8 +206,6 @@ func InitConfig() {
 	}
 
 	// Wasm (all asm funcs with special ABIs)
-	ir.Syms.WasmMove = typecheck.LookupRuntimeVar("wasmMove")
-	ir.Syms.WasmZero = typecheck.LookupRuntimeVar("wasmZero")
 	ir.Syms.WasmDiv = typecheck.LookupRuntimeVar("wasmDiv")
 	ir.Syms.WasmTruncS = typecheck.LookupRuntimeVar("wasmTruncS")
 	ir.Syms.WasmTruncU = typecheck.LookupRuntimeVar("wasmTruncU")

src/cmd/compile/internal/ssagen/ssa.go

@@ -149,14 +149,13 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
 		getValue32(s, v.Args[0])
 		getValue32(s, v.Args[1])
 		i32Const(s, int32(v.AuxInt))
-		p := s.Prog(wasm.ACall)
-		p.To = obj.Addr{Type: obj.TYPE_MEM, Name: obj.NAME_EXTERN, Sym: ir.Syms.WasmMove}
+		s.Prog(wasm.AMemoryCopy)
 
 	case ssa.OpWasmLoweredZero:
 		getValue32(s, v.Args[0])
+		i32Const(s, 0)
 		i32Const(s, int32(v.AuxInt))
-		p := s.Prog(wasm.ACall)
-		p.To = obj.Addr{Type: obj.TYPE_MEM, Name: obj.NAME_EXTERN, Sym: ir.Syms.WasmZero}
+		s.Prog(wasm.AMemoryFill)
 
 	case ssa.OpWasmLoweredNilCheck:
 		getValue64(s, v.Args[0])

src/cmd/compile/internal/wasm/ssa.go

@@ -231,6 +231,17 @@ const (
 	AI64TruncSatF64S
 	AI64TruncSatF64U
 
+	AMemoryInit
+	ADataDrop
+	AMemoryCopy
+	AMemoryFill
+	ATableInit
+	AElemDrop
+	ATableCopy
+	ATableGrow
+	ATableSize
+	ATableFill
+
 	ALast // Sentinel: End of low-level WebAssembly instructions.
 
 	ARESUMEPOINT

src/cmd/internal/obj/wasm/a.out.go

@@ -799,8 +799,6 @@ var notUsePC_B = map[string]bool{
 	"wasm_export_resume":     true,
 	"wasm_export_getsp":      true,
 	"wasm_pc_f_loop":         true,
-	"runtime.wasmMove":       true,
-	"runtime.wasmZero":       true,
 	"runtime.wasmDiv":        true,
 	"runtime.wasmTruncS":     true,
 	"runtime.wasmTruncU":     true,
@@ -844,7 +842,7 @@ func assemble(ctxt *obj.Link, s *obj.LSym, newprog obj.ProgAlloc) {
 	// Some functions use a special calling convention.
 	switch s.Name {
 	case "_rt0_wasm_js", "wasm_export_run", "wasm_export_resume", "wasm_export_getsp", "wasm_pc_f_loop",
-		"runtime.wasmMove", "runtime.wasmZero", "runtime.wasmDiv", "runtime.wasmTruncS", "runtime.wasmTruncU", "memeqbody":
+		"runtime.wasmDiv", "runtime.wasmTruncS", "runtime.wasmTruncU", "memeqbody":
 		varDecls = []*varDecl{}
 		useAssemblyRegMap()
 	case "memchr", "memcmp":
@@ -1088,7 +1086,11 @@ func assemble(ctxt *obj.Link, s *obj.LSym, newprog obj.ProgAlloc) {
 			writeUleb128(w, align(p.As))
 			writeUleb128(w, uint64(p.To.Offset))
 
-		case ACurrentMemory, AGrowMemory:
+		case ACurrentMemory, AGrowMemory, AMemoryFill:
+			w.WriteByte(0x00)
+
+		case AMemoryCopy:
+			w.WriteByte(0x00)
 			w.WriteByte(0x00)
 
 		}

src/cmd/internal/obj/wasm/anames.go

@@ -60,8 +60,6 @@ var wasmFuncTypes = map[string]*wasmFuncType{
 	"wasm_export_resume":     {Params: []byte{}},                                         //
 	"wasm_export_getsp":      {Results: []byte{I32}},                                     // sp
 	"wasm_pc_f_loop":         {Params: []byte{}},                                         //
-	"runtime.wasmMove":       {Params: []byte{I32, I32, I32}},                            // dst, src, len
-	"runtime.wasmZero":       {Params: []byte{I32, I32}},                                 // ptr, len
 	"runtime.wasmDiv":        {Params: []byte{I64, I64}, Results: []byte{I64}},           // x, y -> x/y
 	"runtime.wasmTruncS":     {Params: []byte{F64}, Results: []byte{I64}},                // x -> int(x)
 	"runtime.wasmTruncU":     {Params: []byte{F64}, Results: []byte{I64}},                // x -> uint(x)