Merge pull request kata-containers#148 from egernst/update-routes

grpc: network: add UpdateRoutes

Author: WeiZhang555

grpc.go

@@ -728,16 +728,8 @@ func (a *agentGRPC) RemoveInterface(ctx context.Context, req *pb.RemoveInterface
 	return a.sandbox.removeInterface(nil, req.Interface)
 }
 
-func (a *agentGRPC) AddRoute(ctx context.Context, req *pb.AddRouteRequest) (*gpb.Empty, error) {
-	return emptyResp, a.sandbox.addRoute(nil, req.Route)
-}
-
-func (a *agentGRPC) UpdateRoute(ctx context.Context, req *pb.UpdateRouteRequest) (*gpb.Empty, error) {
-	return emptyResp, nil
-}
-
-func (a *agentGRPC) RemoveRoute(ctx context.Context, req *pb.RemoveRouteRequest) (*gpb.Empty, error) {
-	return emptyResp, a.sandbox.removeRoute(nil, req.Route)
+func (a *agentGRPC) UpdateRoutes(ctx context.Context, req *pb.UpdateRoutesRequest) (*pb.Routes, error) {
+	return a.sandbox.updateRoutes(nil, req.Routes)
 }
 
 func (a *agentGRPC) OnlineCPUMem(ctx context.Context, req *pb.OnlineCPUMemRequest) (*gpb.Empty, error) {

network.go

@@ -17,6 +17,10 @@ import (
 	"github.com/vishvananda/netlink"
 )
 
+const (
+	emptyRouteAddr = "<nil>"
+)
+
 // Network fully describes a sandbox network with its interfaces, routes and dns
 // related information.
 type network struct {
@@ -273,8 +277,130 @@ func getInterface(netHandle *netlink.Handle, link netlink.Link) *pb.Interface {
 // Routes //
 ////////////
 
-func (s *sandbox) addRoute(netHandle *netlink.Handle, route *pb.Route) error {
-	return s.updateRoute(netHandle, route, true)
+//updateRoutes will take requestedRoutes and create netlink routes, with a goal of creating a final
+// state which matches the requested routes.  In doing this, preesxisting non-loopback routes will be
+// removed from the network.  If an error occurs, this function returns the list of routes in
+// gRPC-route format at the time of failure
+func (s *sandbox) updateRoutes(netHandle *netlink.Handle, requestedRoutes *pb.Routes) (resultingRoutes *pb.Routes, err error) {
+
+	if netHandle == nil {
+		netHandle, err = netlink.NewHandle()
+		if err != nil {
+			return nil, err
+		}
+		defer netHandle.Delete()
+	}
+
+	//If we are returning an error, return the current routes on the system
+	defer func() {
+		if err != nil {
+			resultingRoutes, _ = getCurrentRoutes(netHandle)
+		}
+	}()
+
+	//
+	// First things first, let's blow away all the existing routes.  The updateRoutes function
+	// is designed to be declarative, so we will attempt to create state matching what is
+	// requested, and in the event that we fail to do so, will return the error and final state.
+	//
+
+	initRouteList, err := netHandle.RouteList(nil, netlink.FAMILY_ALL)
+	if err != nil {
+		return nil, err
+	}
+	for _, initRoute := range initRouteList {
+		// don't delete routes associated with lo:
+		link, _ := netHandle.LinkByIndex(initRoute.LinkIndex)
+		if link.Attrs().Name == "lo" || link.Attrs().Name == "::1" {
+			continue
+		}
+
+		err = netHandle.RouteDel(&initRoute)
+		if err != nil {
+			//If there was an error deleting some of the initial routes,
+			//return the error and the current routes on the system via
+			//the defer function
+			return
+		}
+	}
+
+	//
+	// Set each of the requested routes
+	//
+	// First make sure we set the interfaces initial routes, as otherwise we
+	// won't be able to access the gateway
+	for _, reqRoute := range requestedRoutes.Routes {
+		if reqRoute.Gateway == "" {
+			err = s.updateRoute(netHandle, reqRoute, true)
+			if err != nil {
+				agentLog.WithError(err).Error("update Route failed")
+				//If there was an error setting the route, return the error
+				//and the current routes on the system via the defer func
+				return
+			}
+
+		}
+	}
+	// Take a second pass and apply the routes which include a gateway
+	for _, reqRoute := range requestedRoutes.Routes {
+		if reqRoute.Gateway != "" {
+			err = s.updateRoute(netHandle, reqRoute, true)
+			if err != nil {
+				agentLog.WithError(err).Error("update Route failed")
+				//If there was an error setting the route, return the
+				//error and the current routes on the system via defer
+				return
+			}
+		}
+	}
+
+	return requestedRoutes, err
+}
+
+//getCurrentRoutes is a helper to gather existing routes in gRPC protocol format
+func getCurrentRoutes(netHandle *netlink.Handle) (*pb.Routes, error) {
+
+	if netHandle == nil {
+		netHandle, err := netlink.NewHandle()
+		if err != nil {
+			return nil, err
+		}
+		defer netHandle.Delete()
+	}
+
+	var routes pb.Routes
+
+	finalRouteList, err := netHandle.RouteList(nil, netlink.FAMILY_ALL)
+	if err != nil {
+		return &routes, err
+	}
+
+	for _, route := range finalRouteList {
+		var r pb.Route
+		if route.Dst != nil {
+			r.Dest = route.Dst.String()
+		}
+
+		if route.Gw != nil {
+			r.Gateway = route.Gw.String()
+		}
+
+		if route.Src != nil {
+			r.Source = route.Src.String()
+		}
+
+		r.Scope = uint32(route.Scope)
+
+		link, err := netHandle.LinkByIndex(route.LinkIndex)
+		if err != nil {
+			return &routes, err
+		}
+		r.Device = link.Attrs().Name
+
+		routes.Routes = append(routes.Routes, &r)
+	}
+
+	return &routes, nil
 }
 
 func (s *sandbox) removeRoute(netHandle *netlink.Handle, route *pb.Route) error {
@@ -321,7 +447,9 @@ func (s *sandbox) updateRoute(netHandle *netlink.Handle, route *pb.Route, add bo
 	netRoute := &netlink.Route{
 		LinkIndex: linkAttrs.Index,
 		Dst:       dst,
+		Src:       net.ParseIP(route.Source),
 		Gw:        net.ParseIP(route.Gateway),
+		Scope:     netlink.Scope(route.Scope),
 	}
 
 	if add {

network_test.go

@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2017 Intel Corporation
+// Copyright (c) 2018 Intel Corporation
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -8,12 +8,15 @@ package main
 
 import (
 	"net"
+	"os"
 	"reflect"
+	"runtime"
 	"testing"
 
 	pb "github.com/kata-containers/agent/protocols/grpc"
 	"github.com/stretchr/testify/assert"
 	"github.com/vishvananda/netlink"
+	"github.com/vishvananda/netns"
 )
 
 func TestUpdateRemoveInterface(t *testing.T) {
@@ -74,7 +77,6 @@ func TestUpdateRemoveInterface(t *testing.T) {
 		"Interface created didn't match: got %+v, expecting %+v", resultingIfc, ifc)
 
 	// Try with garbage:
-	//
 	ifc.Mtu = 999999999999
 	resultingIfc, err = s.updateInterface(netHandle, &ifc)
 	// expecting this failed
@@ -84,30 +86,104 @@ func TestUpdateRemoveInterface(t *testing.T) {
 	assert.True(t, reflect.DeepEqual(resultingIfc, &ifc),
 		"Resulting inteface should have been unchanged: got %+v, expecting %+v", resultingIfc, ifc)
 
-	//
-	// Test adding routes:
-	//
-	route := pb.Route{
-		Dest:    "192.168.3.0/24",
-		Gateway: "192.168.0.1",
-		Device:  "enoNumber",
-	}
-	err = s.addRoute(netHandle, &route)
-	assert.Nil(t, err, "add route failed: %v", err)
-
-	//
-	// Test remove routes:
-	//
-	err = s.removeRoute(netHandle, &route)
-	assert.Nil(t, err, "remove route failed: %v", err)
-
-	//
 	// Exercise the removeInterface code:
-	//
 	_, err = s.removeInterface(netHandle, &ifc)
 	assert.Nil(t, err, "remove interface failed: %v", err)
 
 	// Try to remove non existent interface:
 	_, err = s.removeInterface(netHandle, &ifc)
 	assert.NotNil(t, err, "Expected failed removal: %v", err)
 }
+
+type teardownNetworkTest func()
+
+func skipUnlessRoot(t *testing.T) {
+	if os.Getuid() != 0 {
+		t.Skip("")
+	}
+}
+
+func setupNetworkTest(t *testing.T) teardownNetworkTest {
+	skipUnlessRoot(t)
+
+	// new temporary namespace so we don't pollute the host
+	// lock thread since the namespace is thread local
+	runtime.LockOSThread()
+	var err error
+	ns, err := netns.New()
+	if err != nil {
+		t.Fatal("Failed to create newns", ns)
+	}
+
+	return func() {
+		ns.Close()
+		runtime.UnlockOSThread()
+	}
+}
+
+func TestUpdateRoutes(t *testing.T) {
+	tearDown := setupNetworkTest(t)
+	defer tearDown()
+
+	s := sandbox{}
+
+	// create a dummy link which we'll play with
+	macAddr := net.HardwareAddr{0x02, 0x00, 0xCA, 0xFE, 0x00, 0x48}
+	link := &netlink.Dummy{
+		LinkAttrs: netlink.LinkAttrs{
+			MTU:          1500,
+			TxQLen:       -1,
+			Name:         "ifc-name",
+			HardwareAddr: macAddr,
+		},
+	}
+	netHandle, _ := netlink.NewHandle()
+	defer netHandle.Delete()
+
+	netHandle.LinkAdd(link)
+	if err := netHandle.LinkSetUp(link); err != nil {
+		t.Fatal(err)
+	}
+	netlinkAddr, _ := netlink.ParseAddr("192.168.0.2/16")
+	netHandle.AddrAdd(link, netlinkAddr)
+
+	//Test a simple route setup:
+	inputRoutesSimple := []*pb.Route{
+		{Dest: "", Gateway: "192.168.0.1", Source: "", Scope: 0, Device: "ifc-name"},
+		{Dest: "192.168.0.0/16", Gateway: "", Source: "192.168.0.2", Scope: 253, Device: "ifc-name"},
+	}
+
+	testRoutes := &pb.Routes{
+		Routes: inputRoutesSimple,
+	}
+
+	results, err := s.updateRoutes(netHandle, testRoutes)
+	assert.Nil(t, err, "Unexpected update interface failure: %v", err)
+	assert.True(t, reflect.DeepEqual(results, testRoutes),
+		"Interface created didn't match: got %+v, expecting %+v", results, testRoutes)
+
+	//Test a route setup mimicking what could be provided by PTP CNI plugin:
+	inputRoutesPTPExample := []*pb.Route{
+		{Dest: "", Gateway: "192.168.0.1", Source: "", Scope: 0, Device: "ifc-name"},
+		{Dest: "192.168.0.144/16", Gateway: "192.168.0.1", Source: "192.168.0.2", Scope: 0, Device: "ifc-name"},
+		{Dest: "192.168.0.1/32", Gateway: "", Source: "192.168.0.2", Scope: 254, Device: "ifc-name"},
+	}
+	testRoutes.Routes = inputRoutesPTPExample
+
+	results, err = s.updateRoutes(netHandle, testRoutes)
+	assert.Nil(t, err, "Unexpected update interface failure: %v", err)
+	assert.True(t, reflect.DeepEqual(results, testRoutes),
+		"Interface created didn't match: got %+v, expecting %+v", results, testRoutes)
+
+	//Test unreachable example (no scope provided for initial link route)
+	inputRoutesNoScope := []*pb.Route{
+		{Dest: "", Gateway: "192.168.0.1", Source: "", Scope: 0, Device: "ifc-name"},
+		{Dest: "192.168.0.0/16", Gateway: "", Source: "192.168.0.2", Scope: 0, Device: "ifc-name"},
+	}
+	testRoutes.Routes = inputRoutesNoScope
+	results, err = s.updateRoutes(netHandle, testRoutes)
+	assert.NotNil(t, err, "Expected to observe unreachable route failure")
+
+	assert.True(t, reflect.DeepEqual(results.Routes[0], testRoutes.Routes[1]),
+		"Interface created didn't match: got %+v, expecting %+v", results.Routes[0], testRoutes.Routes[1])
+}