[flexible-ipam] Use uplink interface name for host interface internal port

Add a suffix to uplink when bridging to OVS bridge.

Signed-off-by: gran <gran@vmware.com>

Author: gran-vmv

pkg/agent/agent.go

@@ -606,7 +606,7 @@ func (i *Initializer) setupGatewayInterface() error {
 		externalIDs := map[string]interface{}{
 			interfacestore.AntreaInterfaceTypeKey: interfacestore.AntreaGateway,
 		}
-		gwPortUUID, err := i.ovsBridgeClient.CreateInternalPort(i.hostGateway, config.HostGatewayOFPort, externalIDs)
+		gwPortUUID, err := i.ovsBridgeClient.CreateInternalPort(i.hostGateway, config.HostGatewayOFPort, "", externalIDs)
 		if err != nil {
 			klog.ErrorS(err, "Failed to create gateway port on OVS bridge", "port", i.hostGateway)
 			return err
@@ -1144,8 +1144,11 @@ func (i *Initializer) getNodeInterfaceFromIP(nodeIPs *utilip.DualStackIPs) (v4IP
 // getFreeOFPort returns an OpenFlow port number which is not used by any existing OVS port. Note that, the returned port
 // is not saved in OVSDB yet before the real port is created, so it might introduce an issue for the same return value
 // if it is called multiple times before OVS port creation.
-func (i *Initializer) getFreeOFPort(startPort int) (int32, error) {
+func (i *Initializer) getFreeOFPort(startPort int, reservedPorts []int32) (int32, error) {
 	existingOFPorts := sets.NewInt32()
+	for _, reservedPort := range reservedPorts {
+		existingOFPorts.Insert(reservedPort)
+	}
 	ports, err := i.ovsBridgeClient.GetPortList()
 	if err != nil {
 		return 0, err

pkg/agent/agent_linux.go

@@ -30,7 +30,6 @@ import (
 	"antrea.io/antrea/pkg/agent/config"
 	"antrea.io/antrea/pkg/agent/interfacestore"
 	"antrea.io/antrea/pkg/agent/util"
-	"antrea.io/antrea/pkg/ovs/ovsconfig"
 	utilip "antrea.io/antrea/pkg/util/ip"
 )
 
@@ -78,63 +77,39 @@ func (i *Initializer) prepareOVSBridge() error {
 		return fmt.Errorf("failed to set datapath_id %s: err=%w", datapathID, err)
 	}
 
-	// Create local port.
-	brName := i.ovsBridgeClient.GetBridgeName()
-	if _, err = i.ovsBridgeClient.GetOFPort(brName, false); err == nil {
-		klog.Infof("OVS bridge local port %s already exists, skip the configuration", brName)
-		// If uplink is internal port, get the real uplink interface.
+	if hostOFPort, err := i.ovsBridgeClient.GetOFPort(uplinkNetConfig.Name, false); err == nil {
+		klog.Infof("OVS bridge local port %s already exists", uplinkNetConfig.Name)
+		i.nodeConfig.HostInterfaceOFPort = uint32(hostOFPort)
+		// If local port exists, get the real uplink interface.
 		// This branch is used when antrea-agent get a hard restart (e.g. SIGKILL)
-		if uplinkNetConfig.Name == brName {
-			ports, err := i.ovsBridgeClient.GetPortList()
-			if err != nil {
-				return err
-			}
-			klog.V(2).Infof("Found ports from OVS bridge: %+v", ports)
-			var uplinkPort *ovsconfig.OVSPortData
-			for index := range ports {
-				antreaIfaceType, _ := ports[index].ExternalIDs[interfacestore.AntreaInterfaceTypeKey]
-				if antreaIfaceType == interfacestore.AntreaUplink {
-					uplinkPort = &ports[index]
-					break
-				}
-			}
-			if uplinkPort == nil {
-				return fmt.Errorf("cannot find uplink port from OVS bridge %s", brName)
-			}
-			adapter, err2 := net.InterfaceByName(uplinkPort.Name)
-			if err2 != nil {
-				return fmt.Errorf("cannot find uplink port %s: err=%w", uplinkPort.Name, err2)
-			}
-			klog.Infof("Found uplink device %s", adapter.Name)
-			uplinkNetConfig.OFPort = uint32(uplinkPort.OFPort)
-			uplinkNetConfig.Name = adapter.Name
+		bridgedUplinkName := util.GetBridgedUplinkName(uplinkNetConfig.Name)
+		if uplinkOFPort, err := i.ovsBridgeClient.GetOFPort(bridgedUplinkName, false); err != nil {
+			return fmt.Errorf("cannot find uplink port %s: err=%w", bridgedUplinkName, err)
+		} else {
+			uplinkNetConfig.OFPort = uint32(uplinkOFPort)
+		}
+		if adapter, err := net.InterfaceByName(bridgedUplinkName); err != nil {
+			return fmt.Errorf("cannot find uplink port %s: err=%w", bridgedUplinkName, err)
+		} else {
 			uplinkNetConfig.Index = adapter.Index
-			uplinkInterface := interfacestore.NewUplinkInterface(uplinkPort.Name)
-			uplinkInterface.OVSPortConfig = &interfacestore.OVSPortConfig{uplinkPort.UUID, uplinkPort.OFPort} //nolint: govet
-			i.ifaceStore.AddInterface(uplinkInterface)
 		}
+		klog.InfoS("Found uplink", "Name", adapter.Name, "Index", uplinkNetConfig.Index, "OFPort", uplinkNetConfig.OFPort)
 	} else {
-		// OVS does not receive "ofport_request" param when creating local port, so here use config.AutoAssignedOFPort=0
-		// to ignore this param.
-		externalIDs := map[string]interface{}{
-			interfacestore.AntreaInterfaceTypeKey: interfacestore.AntreaHost,
-		}
-		if _, err = i.ovsBridgeClient.CreateInternalPort(brName, config.AutoAssignedOFPort, externalIDs); err != nil {
+		freePort, err := i.getFreeOFPort(config.UplinkOFPort, nil)
+		if err != nil {
+			klog.ErrorS(err, "Failed to find a free port on OVS")
 			return err
 		}
-	}
-
-	i.nodeConfig.HostInterfaceOFPort = config.BridgeOFPort
-	if uplinkNetConfig.OFPort == 0 {
-		freePort, err := i.getFreeOFPort(config.UplinkOFPort)
+		uplinkNetConfig.OFPort = uint32(freePort)
+		klog.InfoS("Set OpenFlow port in UplinkNetConfig", "ofport", uplinkNetConfig.OFPort)
+		freePort, err = i.getFreeOFPort(config.UplinkOFPort, []int32{freePort})
 		if err != nil {
 			klog.ErrorS(err, "Failed to find a free port on OVS")
 			return err
 		}
-		uplinkNetConfig.OFPort = uint32(freePort)
-		klog.InfoS("Set OpenFlow port in UplinkNetConfig", "ofport", freePort)
+		i.nodeConfig.HostInterfaceOFPort = uint32(freePort)
+		klog.InfoS("Set host interface", "ofport", i.nodeConfig.HostInterfaceOFPort)
 	}
-
 	return nil
 }
 
@@ -179,7 +154,7 @@ func (i *Initializer) saveHostRoutes() error {
 // the antrea network initialize stage.
 // The backup routes are restored after the IP configuration change.
 func (i *Initializer) restoreHostRoutes() error {
-	return i.restoreHostRoutesToInterface(i.ovsBridge)
+	return i.restoreHostRoutesToInterface(i.nodeConfig.UplinkNetConfig.Name)
 }
 
 func (i *Initializer) restoreHostRoutesToInterface(ifaceName string) error {
@@ -206,53 +181,65 @@ func (i *Initializer) ConnectUplinkToOVSBridge() error {
 	klog.Infof("Bridging uplink to OVS bridge")
 	var err error
 	uplinkNetConfig := i.nodeConfig.UplinkNetConfig
-	brName := i.ovsBridgeClient.GetBridgeName()
+	uplinkName := uplinkNetConfig.Name
+	bridgedUplinkName := util.GetBridgedUplinkName(uplinkNetConfig.Name)
 
 	// If uplink is already exists, return.
-	uplink := uplinkNetConfig.Name
-	if uplinkOFPort, err := i.ovsBridgeClient.GetOFPort(uplink, false); err == nil {
-		klog.InfoS("Uplink already exists, skip the configuration", "uplink", uplink, "port", uplinkOFPort)
+	if uplinkOFPort, err := i.ovsBridgeClient.GetOFPort(bridgedUplinkName, false); err == nil {
+		klog.InfoS("Uplink already exists, skip the configuration", "uplink", bridgedUplinkName, "port", uplinkOFPort)
 		return nil
 	}
+	uplinkIPs, err := getTransportIPNetDeviceByNameWithExcludedList(uplinkName, nil)
+	if err != nil {
+		return fmt.Errorf("failed to get uplink IPs: err=%w", err)
+	}
+	if err := util.ChangeAdapterName(uplinkName, bridgedUplinkName, true); err != nil {
+		return fmt.Errorf("failed to change uplink interface name: err=%w", err)
+	}
+
 	// Create uplink port.
-	uplinkPortUUID, err := i.ovsBridgeClient.CreateUplinkPort(uplink, int32(i.nodeConfig.UplinkNetConfig.OFPort), map[string]interface{}{interfacestore.AntreaInterfaceTypeKey: interfacestore.AntreaUplink})
+	uplinkPortUUID, err := i.ovsBridgeClient.CreateUplinkPort(bridgedUplinkName, int32(uplinkNetConfig.OFPort), map[string]interface{}{interfacestore.AntreaInterfaceTypeKey: interfacestore.AntreaUplink})
 	if err != nil {
-		return fmt.Errorf("failed to add uplink port %s: err=%w", uplink, err)
+		return fmt.Errorf("failed to add uplink port %s: err=%w", bridgedUplinkName, err)
 	}
-	// Add newly created uplinkInterface to interface cache. This will be overwritten by initInterfaceStore.
-	uplinkInterface := interfacestore.NewUplinkInterface(uplink)
-	uplinkOFPort, err := i.ovsBridgeClient.GetOFPort(uplink, false)
+	// Add newly created uplinkInterface to interface cache.
+	uplinkInterface := interfacestore.NewUplinkInterface(bridgedUplinkName)
+	uplinkOFPort, err := i.ovsBridgeClient.GetOFPort(bridgedUplinkName, false)
 	if err != nil {
-		return fmt.Errorf("failed to get uplink ofport %s: err=%w", uplink, err)
+		return fmt.Errorf("failed to get uplink ofport %s: err=%w", bridgedUplinkName, err)
 	}
-	klog.InfoS("Allocated OpenFlow port for uplink interface", "port", uplink, "ofPort", uplinkOFPort)
+	klog.InfoS("Allocated OpenFlow port for uplink interface", "port", bridgedUplinkName, "ofPort", uplinkOFPort)
 	uplinkInterface.OVSPortConfig = &interfacestore.OVSPortConfig{uplinkPortUUID, uplinkOFPort} //nolint: govet
 	i.ifaceStore.AddInterface(uplinkInterface)
 
+	// Create local port.
+	externalIDs := map[string]interface{}{
+		interfacestore.AntreaInterfaceTypeKey: interfacestore.AntreaHost,
+	}
+	if _, err = i.ovsBridgeClient.CreateInternalPort(uplinkName, int32(i.nodeConfig.HostInterfaceOFPort), uplinkNetConfig.MAC.String(), externalIDs); err != nil {
+		return fmt.Errorf("cannot create host interface port %s: err=%w", uplinkName, err)
+	}
+
 	// Move network configuration of uplink interface to OVS bridge local interface.
 	// The net configuration of uplink will be restored by RestoreOVSBridge when shutting down.
 	wait.PollImmediate(100*time.Millisecond, 10000*time.Millisecond, func() (bool, error) {
 		// Wait a few seconds for OVS bridge local port.
-		link, err := netlink.LinkByName(brName)
+		link, err := netlink.LinkByName(uplinkName)
 		if err != nil {
-			klog.V(4).InfoS("OVS bridge local port is not ready", "port", brName, "err", err)
+			klog.V(4).InfoS("OVS bridge local port is not ready", "port", uplinkName, "err", err)
 			return false, nil
 		}
 		klog.InfoS("OVS bridge local port is ready", "type", link.Type(), "attrs", link.Attrs())
 		return true, nil
 	})
-	brLink, err := netlink.LinkByName(brName)
+	localLink, err := netlink.LinkByName(uplinkName)
 	if err != nil {
 		return err
 	}
-	if _, _, err = util.SetLinkUp(brName); err != nil {
-		return err
-	}
-	if err = util.SetAdapterMACAddress(brName, &uplinkNetConfig.MAC); err != nil {
+	if _, _, err = util.SetLinkUp(uplinkName); err != nil {
 		return err
 	}
-	// TODO(gran): support IPv6
-	if err = util.ConfigureLinkAddresses(brLink.Attrs().Index, []*net.IPNet{uplinkNetConfig.IP}); err != nil {
+	if err = util.ConfigureLinkAddresses(localLink.Attrs().Index, uplinkIPs); err != nil {
 		return err
 	}
 	if err = util.ConfigureLinkAddresses(uplinkNetConfig.Index, nil); err != nil {
@@ -274,36 +261,33 @@ func (i *Initializer) RestoreOVSBridge() {
 	}
 	klog.Infof("Restoring bridge config to uplink...")
 	uplinkNetConfig := i.nodeConfig.UplinkNetConfig
-	uplink := ""
+	uplinkName := ""
+	bridgedUplinkName := ""
 	if uplinkNetConfig != nil {
-		uplink = uplinkNetConfig.Name
+		uplinkName = uplinkNetConfig.Name
+		bridgedUplinkName = util.GetBridgedUplinkName(uplinkName)
 	}
+	brName := i.ovsBridge
 
-	brName := i.ovsBridgeClient.GetBridgeName()
-	brLink, err := netlink.LinkByName(brName)
+	uplinkIPs, err := getTransportIPNetDeviceByNameWithExcludedList(uplinkName, nil)
 	if err != nil {
-		klog.Warningf("OVS bridge local port not found: %+v", err)
+		klog.ErrorS(err, "Failed to get uplink IPs")
 	}
-
-	if uplink != "" {
-		if err := util.DeleteOVSPort(brName, uplink); err != nil {
-			klog.ErrorS(err, "Removing uplink port from bridge failed", "uplink", uplink, "bridge", brName)
+	if uplinkName != "" {
+		if err := util.DeleteOVSPort(brName, uplinkName); err != nil {
+			klog.ErrorS(err, "Delete OVS port failed", "port", uplinkName)
 		}
-		if err := util.ConfigureLinkAddresses(uplinkNetConfig.Index, []*net.IPNet{uplinkNetConfig.IP}); err != nil {
-			klog.ErrorS(err, "Configure IP to uplink failed", "uplink", uplink)
+		if err := util.DeleteOVSPort(brName, bridgedUplinkName); err != nil {
+			klog.ErrorS(err, "Delete OVS port failed", "port", bridgedUplinkName)
 		}
-	}
-	if brLink != nil {
-		if err := util.ConfigureLinkAddresses(brLink.Attrs().Index, nil); err != nil {
-			klog.ErrorS(err, "Remove IP from bridge interface failed", "interface", brName)
+		if err := util.ChangeAdapterName(bridgedUplinkName, uplinkName, true); err != nil {
+			klog.ErrorS(err, "Restore uplink name failed", "uplink", bridgedUplinkName)
 		}
-		if err := netlink.LinkSetDown(brLink); err != nil {
-			klog.ErrorS(err, "Disable bridge interface failed", "interface", brName)
+		if err := util.ConfigureLinkAddresses(uplinkNetConfig.Index, uplinkIPs); err != nil {
+			klog.ErrorS(err, "Configure IP to uplink failed", "uplink", uplinkName)
 		}
-	}
-	if uplink != "" {
-		if err := i.restoreHostRoutesToInterface(uplink); err != nil {
-			klog.ErrorS(err, "Configure route to uplink interface failed", "interface", uplink)
+		if err := i.restoreHostRoutesToInterface(uplinkName); err != nil {
+			klog.ErrorS(err, "Configure route to uplink interface failed", "uplink", uplinkName)
 		}
 	}
 	klog.Infof("Finished to restore bridge config to uplink...")
@@ -312,3 +296,32 @@ func (i *Initializer) RestoreOVSBridge() {
 func (i *Initializer) setInterfaceMTU(iface string, mtu int) error {
 	return i.ovsBridgeClient.SetInterfaceMTU(iface, mtu)
 }
+
+func getTransportIPNetDeviceByNameWithExcludedList(interfaceName string, excludedIPs []*net.IPNet) ([]*net.IPNet, error) {
+	IPs := []*net.IPNet{}
+	_, _, adapter, err := getTransportIPNetDeviceByName(interfaceName, "")
+	if err != nil {
+		return nil, err
+	}
+	addrs, _ := adapter.Addrs()
+	for _, addr := range addrs {
+		if ip, ipNet, err := net.ParseCIDR(addr.String()); err != nil {
+			klog.Warningf("Unable to parse addr %+v, err=%+v", addr, err)
+		} else if !ip.IsLinkLocalUnicast() {
+			ipNet.IP = ip
+			ipNetString := ipNet.String()
+			isExcludedIP := false
+			for _, excludedIP := range excludedIPs {
+				if ipNetString == excludedIP.String() {
+					isExcludedIP = true
+					break
+				}
+			}
+			if !isExcludedIP {
+				IPs = append(IPs, ipNet)
+			}
+		}
+	}
+	klog.InfoS("Found IPs on interface", "IPs", IPs, "interface", interfaceName)
+	return IPs, nil
+}

pkg/agent/agent_windows.go

@@ -150,7 +150,7 @@ func (i *Initializer) prepareOVSBridge() error {
 		externalIDs := map[string]interface{}{
 			interfacestore.AntreaInterfaceTypeKey: interfacestore.AntreaHost,
 		}
-		if _, err = i.ovsBridgeClient.CreateInternalPort(brName, config.AutoAssignedOFPort, externalIDs); err != nil {
+		if _, err = i.ovsBridgeClient.CreateInternalPort(brName, config.AutoAssignedOFPort, "", externalIDs); err != nil {
 			return err
 		}
 	}
@@ -165,7 +165,7 @@ func (i *Initializer) prepareOVSBridge() error {
 		return nil
 	}
 	// Create uplink port.
-	freePort, err := i.getFreeOFPort(config.UplinkOFPort)
+	freePort, err := i.getFreeOFPort(config.UplinkOFPort, nil)
 	if err != nil {
 		klog.ErrorS(err, "Failed to find a free port on OVS")
 		return err

pkg/agent/cniserver/pod_configuration.go

@@ -271,7 +271,7 @@ func (pc *podConfigurator) createOVSPort(ovsPortName string, ovsAttachInfo map[s
 	var err error
 	switch pc.ifConfigurator.getOVSInterfaceType(ovsPortName) {
 	case internalOVSInterfaceType:
-		portUUID, err = pc.ovsBridgeClient.CreateInternalPort(ovsPortName, 0, ovsAttachInfo)
+		portUUID, err = pc.ovsBridgeClient.CreateInternalPort(ovsPortName, 0, "", ovsAttachInfo)
 	default:
 		if vlanID == 0 {
 			portUUID, err = pc.ovsBridgeClient.CreatePort(ovsPortName, ovsPortName, ovsAttachInfo)

pkg/agent/controller/trafficcontrol/controller.go

@@ -585,7 +585,7 @@ func ParseTrafficControlInterfaceConfig(portData *ovsconfig.OVSPortData, portCon
 // createOVSInternalPort creates an OVS internal port on OVS and corresponding interface on host. Note that, host interface
 // might not be available immediately after creating OVS internal port.
 func (c *Controller) createOVSInternalPort(portName string) (string, error) {
-	portUUID, err := c.ovsBridgeClient.CreateInternalPort(portName, 0, trafficControlPortExternalIDs)
+	portUUID, err := c.ovsBridgeClient.CreateInternalPort(portName, 0, "", trafficControlPortExternalIDs)
 	if err != nil {
 		return "", err
 	}

pkg/agent/util/net.go

@@ -38,6 +38,8 @@ const (
 
 	FamilyIPv4 uint8 = 4
 	FamilyIPv6 uint8 = 6
+
+	BridgedUplinkPrefix = "b-"
 )
 
 func generateInterfaceName(key string, name string, useHead bool) string {
@@ -378,3 +380,8 @@ func PortToUint16(port int) uint16 {
 	klog.Errorf("Port value %d out-of-bounds", port)
 	return 0
 }
+
+// GetBridgedUplinkName generates the uplink interface name after bridged to OVS
+func GetBridgedUplinkName(name string) string {
+	return BridgedUplinkPrefix + name
+}

pkg/agent/util/net_linux.go

@@ -237,3 +237,19 @@ func DeleteOVSPort(brName, portName string) error {
 	cmd := exec.Command("ovs-vsctl", "--if-exists", "del-port", brName, portName)
 	return cmd.Run()
 }
+
+// ChangeAdapterName changes adapter name.
+func ChangeAdapterName(adapterName string, adapterNewName string, isUp bool) error {
+	klog.InfoS("Change adapter name", "oldName", adapterName, "newName", adapterNewName)
+	link, err := netlink.LinkByName(adapterName)
+	if err != nil {
+		return err
+	}
+	if isUp {
+		defer netlink.LinkSetUp(link)
+	}
+	if err := netlink.LinkSetDown(link); err != nil {
+		return err
+	}
+	return netlink.LinkSetName(link, adapterNewName)
+}

pkg/ovs/ovsconfig/interfaces.go

@@ -39,7 +39,7 @@ type OVSBridgeClient interface {
 	SetInterfaceOptions(name string, options map[string]interface{}) Error
 	CreatePort(name, ifDev string, externalIDs map[string]interface{}) (string, Error)
 	CreateAccessPort(name, ifDev string, externalIDs map[string]interface{}, vlanID uint16) (string, Error)
-	CreateInternalPort(name string, ofPortRequest int32, externalIDs map[string]interface{}) (string, Error)
+	CreateInternalPort(name string, ofPortRequest int32, mac string, externalIDs map[string]interface{}) (string, Error)
 	CreateTunnelPort(name string, tunnelType TunnelType, ofPortRequest int32) (string, Error)
 	CreateTunnelPortExt(name string, tunnelType TunnelType, ofPortRequest int32, csum bool, localIP string, remoteIP string, remoteName string, psk string, extraOptions, externalIDs map[string]interface{}) (string, Error)
 	CreateUplinkPort(name string, ofPortRequest int32, externalIDs map[string]interface{}) (string, Error)