cmd/compile: loop invariant code motion

src/cmd/compile/internal/ssa/block.go

@@ -257,6 +257,38 @@ func (b *Block) resetWithControl2(kind BlockKind, v, w *Value) {
 	w.Uses++
 }
 
+// ReplaceSucc replaces b->oldSucc to b->newSucc, n indicates which predecessor
+// index of newSucc refers to b. It is the responsibility of the caller to clear
+// the corresponding predecessor of oldSucc.
+func (b *Block) ReplaceSucc(oldSucc, newSucc *Block, n int) {
+	for i := 0; i < len(b.Succs); i++ {
+		succ := &b.Succs[i]
+		if succ.b == oldSucc {
+			succ.b = newSucc
+			succ.i = n
+			newSucc.Preds[n] = Edge{b, i}
+			return
+		}
+	}
+	panic(fmt.Sprintf("Can not found %v->%v", b, oldSucc))
+}
+
+// ReplacePred replaces oldPred->b to newPred->b, n indicates which successor
+// index of newPred refers to b. It is the responsibility of the caller to clear
+// the corresponding successor of oldPred.
+func (b *Block) ReplacePred(oldPred, newPred *Block, n int) {
+	for i := 0; i < len(b.Preds); i++ {
+		pred := &b.Preds[i]
+		if pred.b == oldPred {
+			pred.b = newPred
+			pred.i = n
+			newPred.Succs[n] = Edge{b, i}
+			return
+		}
+	}
+	panic(fmt.Sprintf("Can not found %v->%v", oldPred, b))
+}
+
 // truncateValues truncates b.Values at the ith element, zeroing subsequent elements.
 // The values in b.Values after i must already have had their args reset,
 // to maintain correct value uses counts.

src/cmd/compile/internal/ssa/branchelim.go

@@ -424,6 +424,14 @@ func shouldElimIfElse(no, yes, post *Block, arch string) bool {
 	}
 }
 
+func isAccessMemory(v *Value) bool {
+	if v.Op == OpPhi || v.Type.IsMemory() ||
+		v.MemoryArg() != nil || opcodeTable[v.Op].hasSideEffects {
+		return true
+	}
+	return false
+}
+
 // canSpeculativelyExecute reports whether every value in the block can
 // be evaluated without causing any observable side effects (memory
 // accesses, panics and so on) except for execution time changes. It
@@ -436,8 +444,8 @@ func canSpeculativelyExecute(b *Block) bool {
 	// don't fuse memory ops, Phi ops, divides (can panic),
 	// or anything else with side-effects
 	for _, v := range b.Values {
-		if v.Op == OpPhi || isDivMod(v.Op) || isPtrArithmetic(v.Op) || v.Type.IsMemory() ||
-			v.MemoryArg() != nil || opcodeTable[v.Op].hasSideEffects {
+		if v.Op == OpPhi || isDivMod(v.Op) || isPtrArithmetic(v.Op) ||
+			isAccessMemory(v) {
 			return false
 		}
 	}

src/cmd/compile/internal/ssa/check.go

@@ -460,33 +460,8 @@ func checkFunc(f *Func) {
 	memCheck(f)
 }
 
-func memCheck(f *Func) {
-	// Check that if a tuple has a memory type, it is second.
-	for _, b := range f.Blocks {
-		for _, v := range b.Values {
-			if v.Type.IsTuple() && v.Type.FieldType(0).IsMemory() {
-				f.Fatalf("memory is first in a tuple: %s\n", v.LongString())
-			}
-		}
-	}
-
-	// Single live memory checks.
-	// These checks only work if there are no memory copies.
-	// (Memory copies introduce ambiguity about which mem value is really live.
-	// probably fixable, but it's easier to avoid the problem.)
-	// For the same reason, disable this check if some memory ops are unused.
-	for _, b := range f.Blocks {
-		for _, v := range b.Values {
-			if (v.Op == OpCopy || v.Uses == 0) && v.Type.IsMemory() {
-				return
-			}
-		}
-		if b != f.Entry && len(b.Preds) == 0 {
-			return
-		}
-	}
-
-	// Compute live memory at the end of each block.
+// computeLastMem compute live memory at the end of each block.
+func computeLastMem(f *Func) []*Value {
 	lastmem := make([]*Value, f.NumBlocks())
 	ss := newSparseSet(f.NumValues())
 	for _, b := range f.Blocks {
@@ -552,6 +527,36 @@ func memCheck(f *Func) {
 			break
 		}
 	}
+	return lastmem
+}
+
+func memCheck(f *Func) {
+	// Check that if a tuple has a memory type, it is second.
+	for _, b := range f.Blocks {
+		for _, v := range b.Values {
+			if v.Type.IsTuple() && v.Type.FieldType(0).IsMemory() {
+				f.Fatalf("memory is first in a tuple: %s\n", v.LongString())
+			}
+		}
+	}
+
+	// Single live memory checks.
+	// These checks only work if there are no memory copies.
+	// (Memory copies introduce ambiguity about which mem value is really live.
+	// probably fixable, but it's easier to avoid the problem.)
+	// For the same reason, disable this check if some memory ops are unused.
+	for _, b := range f.Blocks {
+		for _, v := range b.Values {
+			if (v.Op == OpCopy || v.Uses == 0) && v.Type.IsMemory() {
+				return
+			}
+		}
+		if b != f.Entry && len(b.Preds) == 0 {
+			return
+		}
+	}
+
+	lastmem := computeLastMem(f)
 	// Check merge points.
 	for _, b := range f.Blocks {
 		for _, v := range b.Values {

src/cmd/compile/internal/ssa/compile.go

@@ -453,12 +453,13 @@ commas. For example:
 	return fmt.Sprintf("Did not find a phase matching %s in -d=ssa/... debug option", phase)
 }
 
+var EnableLoopOpts = buildcfg.Experiment.LoopOpts
+
 // list of passes for the compiler
 var passes = [...]pass{
-	// TODO: combine phielim and copyelim into a single pass?
+	// Generic Optimizations
 	{name: "number lines", fn: numberLines, required: true},
 	{name: "early phielim", fn: phielim},
-	{name: "early copyelim", fn: copyelim},
 	{name: "early deadcode", fn: deadcode}, // remove generated dead code to avoid doing pointless work during opt
 	{name: "short circuit", fn: shortcircuit},
 	{name: "decompose user", fn: decomposeUser, required: true},
@@ -484,9 +485,18 @@ var passes = [...]pass{
 	{name: "late fuse", fn: fuseLate},
 	{name: "dse", fn: dse},
 	{name: "memcombine", fn: memcombine},
-	{name: "writebarrier", fn: writebarrier, required: true}, // expand write barrier ops
+	// Loop Optimizations
+	{name: "loop deadcode", fn: deadcode, disabled: !EnableLoopOpts},          // remove dead blocks before loop opts to avoid extra work
+	{name: "loop invariant code motion", fn: licm, disabled: !EnableLoopOpts}, // hoist loop invariant code out of loops
+	{name: "lcssa destruct", fn: phielim, disabled: !EnableLoopOpts},          // eliminate LCSSA proxy phi to restore general SSA form
+	{name: "loop sccp", fn: sccp, disabled: !EnableLoopOpts},                  // optimize loop guard conditional test
+	{name: "loop opt", fn: opt, disabled: !EnableLoopOpts},                    // further optimize loop guard conditional test
+	{name: "loop deadcode late", fn: deadcode, disabled: !EnableLoopOpts},     // remove dead loop guard to simplify cfg
+	{name: "loop nilcheckelim", fn: nilcheckelim, disabled: !EnableLoopOpts},  // remove duplicated nil check in loop guard
+	{name: "writebarrier", fn: writebarrier, required: true},                  // expand write barrier ops
 	{name: "insert resched checks", fn: insertLoopReschedChecks,
 		disabled: !buildcfg.Experiment.PreemptibleLoops}, // insert resched checks in loops.
+	// Code Generation
 	{name: "lower", fn: lower, required: true},
 	{name: "addressing modes", fn: addressingModes, required: false},
 	{name: "late lower", fn: lateLower, required: true},
@@ -497,7 +507,6 @@ var passes = [...]pass{
 	{name: "lowered deadcode", fn: deadcode, required: true},
 	{name: "checkLower", fn: checkLower, required: true},
 	{name: "late phielim", fn: phielim},
-	{name: "late copyelim", fn: copyelim},
 	{name: "tighten", fn: tighten, required: true}, // move values closer to their uses
 	{name: "late deadcode", fn: deadcode},
 	{name: "critical", fn: critical, required: true}, // remove critical edges
@@ -508,7 +517,7 @@ var passes = [...]pass{
 	{name: "late nilcheck", fn: nilcheckelim2},
 	{name: "flagalloc", fn: flagalloc, required: true}, // allocate flags register
 	{name: "regalloc", fn: regalloc, required: true},   // allocate int & float registers + stack slots
-	{name: "loop rotate", fn: loopRotate},
+	{name: "layout loop", fn: layoutLoop},
 	{name: "trim", fn: trim}, // remove empty blocks
 }
 
@@ -577,8 +586,8 @@ var passOrder = [...]constraint{
 	{"schedule", "flagalloc"},
 	// regalloc needs flags to be allocated first.
 	{"flagalloc", "regalloc"},
-	// loopRotate will confuse regalloc.
-	{"regalloc", "loop rotate"},
+	// layout loop will confuse regalloc.
+	{"regalloc", "layout loop"},
 	// trim needs regalloc to be done first.
 	{"regalloc", "trim"},
 	// memcombine works better if fuse happens first, to help merge stores.

src/cmd/compile/internal/ssa/func.go

@@ -86,6 +86,13 @@ type LocalSlotSplitKey struct {
 	Type   *types.Type // type of slot
 }
 
+// assert is used for development sanity check
+func assert(cond bool, fx string, msg ...interface{}) {
+	if !cond {
+		panic(fmt.Sprintf(fx, msg...))
+	}
+}
+
 // NewFunc returns a new, empty function object.
 // Caller must reset cache before calling NewFunc.
 func (c *Config) NewFunc(fe Frontend, cache *Cache) *Func {
@@ -298,7 +305,7 @@ func (f *Func) newValue(op Op, t *types.Type, b *Block, pos src.XPos) *Value {
 // newValueNoBlock allocates a new Value with the given fields.
 // The returned value is not placed in any block.  Once the caller
 // decides on a block b, it must set b.Block and append
-// the returned value to b.Values.
+// the returned value to b.Values or simply use placeValue.
 func (f *Func) newValueNoBlock(op Op, t *types.Type, pos src.XPos) *Value {
 	var v *Value
 	if f.freeValues != nil {
@@ -324,6 +331,12 @@ func (f *Func) newValueNoBlock(op Op, t *types.Type, pos src.XPos) *Value {
 	return v
 }
 
+// placeValue places new Value that not placed yet into given block.
+func (block *Block) placeValue(v *Value) {
+	v.Block = block
+	block.Values = append(block.Values, v)
+}
+
 // LogStat writes a string key and int value as a warning in a
 // tab-separated format easily handled by spreadsheets or awk.
 // file names, lines, and function names are included to provide enough (?)

src/cmd/compile/internal/ssa/layout.go

@@ -88,6 +88,7 @@ func layoutOrder(f *Func) []*Block {
 	}
 
 	bid := f.Entry.ID
+	blockTrace := false
 blockloop:
 	for {
 		// add block to schedule
@@ -120,7 +121,6 @@ blockloop:
 		}
 
 		// Pick the next block to schedule
-		// Pick among the successor blocks that have not been scheduled yet.
 
 		// Use likely direction if we have it.
 		var likely *Block
@@ -131,10 +131,27 @@ blockloop:
 			likely = b.Succs[1].b
 		}
 		if likely != nil && !scheduled[likely.ID] {
+			blockTrace = true
 			bid = likely.ID
 			continue
 		}
 
+		// Pick the next block in the path trace if possible, trace starts with
+		// statically predicted branch, e.g.
+		//   b0: ... If -> b1(likely),b2
+		//   b1: ... Plain -> b3
+		// schedule the path trace b0->b1->b3 sequentially
+		if blockTrace {
+			if len(b.Succs) == 1 {
+				s := b.Succs[0].b
+				if !scheduled[s.ID] {
+					bid = s.ID
+					continue blockloop
+				}
+			}
+			blockTrace = false
+		}
+
 		// Use degree for now.
 		bid = 0
 		// TODO: improve this part