some basic dag optimizations

commands.go

@@ -318,12 +318,6 @@ func initCommands(config *cliconfig.Config, services *disco.Disco, providerSrc g
 			}, nil
 		},
 
-		"debug json2dot": func() (cli.Command, error) {
-			return &command.DebugJSON2DotCommand{
-				Meta: meta,
-			}, nil
-		},
-
 		"force-unlock": func() (cli.Command, error) {
 			return &command.UnlockCommand{
 				Meta: meta,

dag/dag.go

@@ -29,15 +29,14 @@ func (g *AcyclicGraph) DirectedGraph() Grapher {
 
 // Returns a Set that includes every Vertex yielded by walking down from the
 // provided starting Vertex v.
-func (g *AcyclicGraph) Ancestors(v Vertex) (*Set, error) {
-	s := new(Set)
-	start := AsVertexList(g.DownEdges(v))
+func (g *AcyclicGraph) Ancestors(v Vertex) (Set, error) {
+	s := make(Set)
 	memoFunc := func(v Vertex, d int) error {
 		s.Add(v)
 		return nil
 	}
 
-	if err := g.DepthFirstWalk(start, memoFunc); err != nil {
+	if err := g.DepthFirstWalk(g.DownEdges(v), memoFunc); err != nil {
 		return nil, err
 	}
 
@@ -46,15 +45,14 @@ func (g *AcyclicGraph) Ancestors(v Vertex) (*Set, error) {
 
 // Returns a Set that includes every Vertex yielded by walking up from the
 // provided starting Vertex v.
-func (g *AcyclicGraph) Descendents(v Vertex) (*Set, error) {
-	s := new(Set)
-	start := AsVertexList(g.UpEdges(v))
+func (g *AcyclicGraph) Descendents(v Vertex) (Set, error) {
+	s := make(Set)
 	memoFunc := func(v Vertex, d int) error {
 		s.Add(v)
 		return nil
 	}
 
-	if err := g.ReverseDepthFirstWalk(start, memoFunc); err != nil {
+	if err := g.ReverseDepthFirstWalk(g.UpEdges(v), memoFunc); err != nil {
 		return nil, err
 	}
 
@@ -102,15 +100,12 @@ func (g *AcyclicGraph) TransitiveReduction() {
 	// v such that the edge (u,v) exists (v is a direct descendant of u).
 	//
 	// For each v-prime reachable from v, remove the edge (u, v-prime).
-	defer g.debug.BeginOperation("TransitiveReduction", "").End("")
-
 	for _, u := range g.Vertices() {
 		uTargets := g.DownEdges(u)
-		vs := AsVertexList(g.DownEdges(u))
 
-		g.depthFirstWalk(vs, false, func(v Vertex, d int) error {
+		g.DepthFirstWalk(g.DownEdges(u), func(v Vertex, d int) error {
 			shared := uTargets.Intersection(g.DownEdges(v))
-			for _, vPrime := range AsVertexList(shared) {
+			for _, vPrime := range shared {
 				g.RemoveEdge(BasicEdge(u, vPrime))
 			}
 
@@ -166,19 +161,16 @@ func (g *AcyclicGraph) Cycles() [][]Vertex {
 // This will walk nodes in parallel if it can. The resulting diagnostics
 // contains problems from all graphs visited, in no particular order.
 func (g *AcyclicGraph) Walk(cb WalkFunc) tfdiags.Diagnostics {
-	defer g.debug.BeginOperation(typeWalk, "").End("")
-
 	w := &Walker{Callback: cb, Reverse: true}
 	w.Update(g)
 	return w.Wait()
 }
 
 // simple convenience helper for converting a dag.Set to a []Vertex
-func AsVertexList(s *Set) []Vertex {
-	rawList := s.List()
-	vertexList := make([]Vertex, len(rawList))
-	for i, raw := range rawList {
-		vertexList[i] = raw.(Vertex)
+func AsVertexList(s Set) []Vertex {
+	vertexList := make([]Vertex, 0, len(s))
+	for _, raw := range s {
+		vertexList = append(vertexList, raw.(Vertex))
 	}
 	return vertexList
 }
@@ -188,21 +180,48 @@ type vertexAtDepth struct {
 	Depth  int
 }
 
-// depthFirstWalk does a depth-first walk of the graph starting from
+// DepthFirstWalk does a depth-first walk of the graph starting from
 // the vertices in start.
-func (g *AcyclicGraph) DepthFirstWalk(start []Vertex, f DepthWalkFunc) error {
-	return g.depthFirstWalk(start, true, f)
-}
+func (g *AcyclicGraph) DepthFirstWalk(start Set, f DepthWalkFunc) error {
+	seen := make(map[Vertex]struct{})
+	frontier := make([]*vertexAtDepth, 0, len(start))
+	for _, v := range start {
+		frontier = append(frontier, &vertexAtDepth{
+			Vertex: v,
+			Depth:  0,
+		})
+	}
+	for len(frontier) > 0 {
+		// Pop the current vertex
+		n := len(frontier)
+		current := frontier[n-1]
+		frontier = frontier[:n-1]
 
-// This internal method provides the option of not sorting the vertices during
-// the walk, which we use for the Transitive reduction.
-// Some configurations can lead to fully-connected subgraphs, which makes our
-// transitive reduction algorithm O(n^3). This is still passable for the size
-// of our graphs, but the additional n^2 sort operations would make this
-// uncomputable in a reasonable amount of time.
-func (g *AcyclicGraph) depthFirstWalk(start []Vertex, sorted bool, f DepthWalkFunc) error {
-	defer g.debug.BeginOperation(typeDepthFirstWalk, "").End("")
+		// Check if we've seen this already and return...
+		if _, ok := seen[current.Vertex]; ok {
+			continue
+		}
+		seen[current.Vertex] = struct{}{}
+
+		// Visit the current node
+		if err := f(current.Vertex, current.Depth); err != nil {
+			return err
+		}
+
+		for _, v := range g.DownEdges(current.Vertex) {
+			frontier = append(frontier, &vertexAtDepth{
+				Vertex: v,
+				Depth:  current.Depth + 1,
+			})
+		}
+	}
 
+	return nil
+}
+
+// SortedDepthFirstWalk does a depth-first walk of the graph starting from
+// the vertices in start, always iterating the nodes in a consistent order.
+func (g *AcyclicGraph) SortedDepthFirstWalk(start []Vertex, f DepthWalkFunc) error {
 	seen := make(map[Vertex]struct{})
 	frontier := make([]*vertexAtDepth, len(start))
 	for i, v := range start {
@@ -230,10 +249,7 @@ func (g *AcyclicGraph) depthFirstWalk(start []Vertex, sorted bool, f DepthWalkFu
 
 		// Visit targets of this in a consistent order.
 		targets := AsVertexList(g.DownEdges(current.Vertex))
-
-		if sorted {
-			sort.Sort(byVertexName(targets))
-		}
+		sort.Sort(byVertexName(targets))
 
 		for _, t := range targets {
 			frontier = append(frontier, &vertexAtDepth{
@@ -246,11 +262,48 @@ func (g *AcyclicGraph) depthFirstWalk(start []Vertex, sorted bool, f DepthWalkFu
 	return nil
 }
 
-// reverseDepthFirstWalk does a depth-first walk _up_ the graph starting from
+// ReverseDepthFirstWalk does a depth-first walk _up_ the graph starting from
 // the vertices in start.
-func (g *AcyclicGraph) ReverseDepthFirstWalk(start []Vertex, f DepthWalkFunc) error {
-	defer g.debug.BeginOperation(typeReverseDepthFirstWalk, "").End("")
+func (g *AcyclicGraph) ReverseDepthFirstWalk(start Set, f DepthWalkFunc) error {
+	seen := make(map[Vertex]struct{})
+	frontier := make([]*vertexAtDepth, 0, len(start))
+	for _, v := range start {
+		frontier = append(frontier, &vertexAtDepth{
+			Vertex: v,
+			Depth:  0,
+		})
+	}
+	for len(frontier) > 0 {
+		// Pop the current vertex
+		n := len(frontier)
+		current := frontier[n-1]
+		frontier = frontier[:n-1]
+
+		// Check if we've seen this already and return...
+		if _, ok := seen[current.Vertex]; ok {
+			continue
+		}
+		seen[current.Vertex] = struct{}{}
+
+		for _, t := range g.UpEdges(current.Vertex) {
+			frontier = append(frontier, &vertexAtDepth{
+				Vertex: t,
+				Depth:  current.Depth + 1,
+			})
+		}
+
+		// Visit the current node
+		if err := f(current.Vertex, current.Depth); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
 
+// SortedReverseDepthFirstWalk does a depth-first walk _up_ the graph starting from
+// the vertices in start, always iterating the nodes in a consistent order.
+func (g *AcyclicGraph) SortedReverseDepthFirstWalk(start []Vertex, f DepthWalkFunc) error {
 	seen := make(map[Vertex]struct{})
 	frontier := make([]*vertexAtDepth, len(start))
 	for i, v := range start {