Support for lima usernet network

Signed-off-by: Balaji Vijayakumar <[email protected]>

Author: balajiv113

cmd/limactl/main.go

@@ -100,6 +100,7 @@ func newApp() *cobra.Command {
 		newEditCommand(),
 		newFactoryResetCommand(),
 		newDiskCommand(),
+		newUsernetCommand(),
 	)
 	return rootCmd
 }

cmd/limactl/usernet.go

@@ -0,0 +1,72 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"strconv"
+
+	"github.com/lima-vm/lima/pkg/networks/usernet"
+	"github.com/spf13/cobra"
+)
+
+func newUsernetCommand() *cobra.Command {
+	var hostagentCommand = &cobra.Command{
+		Use:    "usernet",
+		Short:  "run usernet",
+		Args:   cobra.ExactArgs(0),
+		RunE:   usernetAction,
+		Hidden: true,
+	}
+	hostagentCommand.Flags().StringP("pidfile", "p", "", "write pid to file")
+	hostagentCommand.Flags().StringP("endpoint", "e", "", "exposes usernet api(s) on this endpoint")
+	hostagentCommand.Flags().String("listen-qemu", "", "listen for qemu connections")
+	hostagentCommand.Flags().String("listen-fd", "", "listen for fd connections")
+	hostagentCommand.Flags().Int("mtu", 1500, "mtu")
+	return hostagentCommand
+}
+
+func usernetAction(cmd *cobra.Command, args []string) error {
+
+	pidfile, err := cmd.Flags().GetString("pidfile")
+	if err != nil {
+		return err
+	}
+	if pidfile != "" {
+		if _, err := os.Stat(pidfile); !errors.Is(err, os.ErrNotExist) {
+			return fmt.Errorf("pidfile %q already exists", pidfile)
+		}
+		if err := os.WriteFile(pidfile, []byte(strconv.Itoa(os.Getpid())+"\n"), 0644); err != nil {
+			return err
+		}
+		defer os.RemoveAll(pidfile)
+	}
+	endpoint, err := cmd.Flags().GetString("endpoint")
+	if err != nil {
+		return err
+	}
+	qemuSocket, err := cmd.Flags().GetString("listen-qemu")
+	if err != nil {
+		return err
+	}
+	fdSocket, err := cmd.Flags().GetString("listen-fd")
+	if err != nil {
+		return err
+	}
+
+	mtu, err := cmd.Flags().GetInt("mtu")
+	if err != nil {
+		return err
+	}
+
+	os.RemoveAll(endpoint)
+	os.RemoveAll(qemuSocket)
+	os.RemoveAll(fdSocket)
+
+	return usernet.StartGVisorNetstack(cmd.Context(), &usernet.GVisorNetstackOpts{
+		MTU:        mtu,
+		Endpoint:   endpoint,
+		QemuSocket: qemuSocket,
+		FdSocket:   fdSocket,
+	})
+}

examples/experimental/usernet.yaml

@@ -0,0 +1,15 @@
+# Example to run vz instance with lima usernet enabled
+vmType: "vz"
+images:
+- location: "https://cloud-images.ubuntu.com/releases/22.04/release/ubuntu-22.04-server-cloudimg-amd64.img"
+  arch: "x86_64"
+- location: "https://cloud-images.ubuntu.com/releases/22.04/release/ubuntu-22.04-server-cloudimg-arm64.img"
+  arch: "aarch64"
+
+mounts:
+- location: "~"
+- location: "/tmp/lima"
+  writable: true
+mountType: "virtiofs"
+networks:
+- lima: default

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/cheggaaa/pb/v3 v3.1.2
 	github.com/containerd/containerd v1.6.19
 	github.com/containerd/continuity v0.3.0
-	github.com/containers/gvisor-tap-vsock v0.5.0
+	github.com/containers/gvisor-tap-vsock v0.5.1-0.20230220110902-2732d3a55dc7
 	github.com/coreos/go-semver v0.3.1
 	github.com/cyphar/filepath-securejoin v0.2.3
 	github.com/digitalocean/go-qemu v0.0.0-20210326154740-ac9e0b687001
@@ -78,10 +78,13 @@ require (
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
 	github.com/kr/fs v0.1.0 // indirect
+	github.com/linuxkit/virtsock v0.0.0-20220523201153-1a23e78aa7a2 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mailru/easyjson v0.7.6 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-runewidth v0.0.12 // indirect
+	github.com/mdlayher/socket v0.4.0 // indirect
+	github.com/mdlayher/vsock v1.2.0 // indirect
 	github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect

go.sum

@@ -111,8 +111,8 @@ github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDG
 github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y=
 github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc=
 github.com/containerd/typeurl v0.0.0-20200205145503-b45ef1f1f737/go.mod h1:TB1hUtrpaiO88KEK56ijojHS1+NeF0izUACaJW2mdXg=
-github.com/containers/gvisor-tap-vsock v0.5.0 h1:hoCkrfQ96tjek2BtiW1BHy50zAQCzkqeiAQY96y6NLk=
-github.com/containers/gvisor-tap-vsock v0.5.0/go.mod h1:jrnI5plQtmys5LEKpXcCCrLqZlrHsozQg0V2Jw1UG74=
+github.com/containers/gvisor-tap-vsock v0.5.1-0.20230220110902-2732d3a55dc7 h1:Ej2OTGLHMjcpnGApet9gmEfboe9aTT2eU/civ1Imm6g=
+github.com/containers/gvisor-tap-vsock v0.5.1-0.20230220110902-2732d3a55dc7/go.mod h1:RfDVvmr+DHvp6whJVEwrMn+hfTYIPLWlUPJhWXNUsCM=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
@@ -376,6 +376,8 @@ github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/lima-vm/sshocker v0.3.2 h1:o0WqVzcpt6mzVCuqtS3N3O8kwTx6X4SLr4h7YaRISuE=
 github.com/lima-vm/sshocker v0.3.2/go.mod h1:9SWN6wob210VM6oJkkzvWQOlHSp/rQLB+0fSEc92zig=
+github.com/linuxkit/virtsock v0.0.0-20220523201153-1a23e78aa7a2 h1:DZMFueDbfz6PNc1GwDRA8+6lBx1TB9UnxDQliCqR73Y=
+github.com/linuxkit/virtsock v0.0.0-20220523201153-1a23e78aa7a2/go.mod h1:SWzULI85WerrFt3u+nIm5F9l7EvxZTKQvd0InF3nmgM=
 github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY=
 github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
@@ -409,6 +411,10 @@ github.com/mdlayher/netlink v1.1.1/go.mod h1:WTYpFb/WTvlRJAyKhZL5/uy69TDDpHHu2VZ
 github.com/mdlayher/raw v0.0.0-20190606142536-fef19f00fc18/go.mod h1:7EpbotpCmVZcu+KCX4g9WaRNuu11uyhiW7+Le1dKawg=
 github.com/mdlayher/raw v0.0.0-20191009151244-50f2db8cc065 h1:aFkJ6lx4FPip+S+Uw4aTegFMct9shDvP+79PsSxpm3w=
 github.com/mdlayher/raw v0.0.0-20191009151244-50f2db8cc065/go.mod h1:7EpbotpCmVZcu+KCX4g9WaRNuu11uyhiW7+Le1dKawg=
+github.com/mdlayher/socket v0.4.0 h1:280wsy40IC9M9q1uPGcLBwXpcTQDtoGwVt+BNoITxIw=
+github.com/mdlayher/socket v0.4.0/go.mod h1:xxFqz5GRCUN3UEOm9CZqEJsAbe1C8OwSK46NlmWuVoc=
+github.com/mdlayher/vsock v1.2.0 h1:klRY9lndjmg6k/QWbX/ucQ3e2JFRm1M7vfG9hijbQ0A=
+github.com/mdlayher/vsock v1.2.0/go.mod h1:w4kdSTQB9p1l/WwGmAs0V62qQ869qRYoongwgN+Y1HE=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
 github.com/miekg/dns v1.1.25/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
@@ -444,7 +450,7 @@ github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
 github.com/onsi/ginkgo/v2 v2.4.0 h1:+Ig9nvqgS5OBSACXNk15PLdp0U9XPYROt9CFzVdFGIs=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/onsi/gomega v1.24.2 h1:J/tulyYK6JwBldPViHJReihxxZ+22FHs0piGjQAvoUE=
+github.com/onsi/gomega v1.27.1 h1:rfztXRbg6nv/5f+Raen9RcGoSecHIFgBBLQK3Wdj754=
 github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=

pkg/cidata/cidata.go

@@ -211,7 +211,11 @@ func GenerateISO9660(instDir, name string, y *limayaml.LimaYAML, udpDNSLocalPort
 
 	slirpMACAddress := limayaml.MACAddress(instDir)
 	args.Networks = append(args.Networks, Network{MACAddress: slirpMACAddress, Interface: networks.SlirpNICName})
-	for _, nw := range y.Networks {
+	firstUsernetIndex := limayaml.GetFirstUsernetIndex(y)
+	for i, nw := range y.Networks {
+		if i == firstUsernetIndex {
+			continue
+		}
 		args.Networks = append(args.Networks, Network{MACAddress: nw.MACAddress, Interface: nw.Interface})
 	}
 

pkg/fd/fd.go

@@ -0,0 +1,109 @@
+//go:build !windows
+// +build !windows
+
+// This file has been adapted from https://github.com/ftrvxmtrx/fd/blob/master/fd.go
+
+// Package fd provides a simple API to pass file descriptors
+// between different OS processes.
+//
+// It can be useful if you want to inherit network connections
+// from another process without closing them.
+//
+// Example scenario:
+//
+//  1. Running server receives a "let's upgrade" message
+//  2. Server opens a Unix domain socket for the "upgrade"
+//  3. Server starts a new copy of itself and passes Unix domain socket name
+//  4. New copy starts reading for the socket
+//  5. Server sends its state over the socket, also sending the number
+//     of network connections to inherit, then it sends those connections
+//     using fd.Put()
+//  6. New copy reads the state and inherits connections using fd.Get(),
+//     checks that everything is OK and sends the "OK" message to the socket
+//  7. Server receives "OK" message and kills itself
+package fd
+
+import (
+	"net"
+	"os"
+	"syscall"
+)
+
+// Get receives single file descriptors from a Unix domain socket.
+//
+// Num specifies the expected number of file descriptors in one message.
+// Internal files' names to be assigned are specified via optional filenames
+// argument.
+//
+// You need to close all files in the returned slice. The slice can be
+// non-empty even if this function returns an error.
+//
+// Use net.FileConn() if you're receiving a network connection.
+func Get(via *net.UnixConn, num int, filenames []string) ([]*os.File, error) {
+	if num < 1 {
+		return nil, nil
+	}
+
+	// get the underlying socket
+	viaf, err := via.File()
+	if err != nil {
+		return nil, err
+	}
+	socket := int(viaf.Fd())
+	defer viaf.Close()
+
+	// recvmsg
+	buf := make([]byte, syscall.CmsgSpace(num*4))
+	_, _, _, _, err = syscall.Recvmsg(socket, nil, buf, 0)
+	if err != nil {
+		return nil, err
+	}
+
+	// parse control msgs
+	var msgs []syscall.SocketControlMessage
+	msgs, err = syscall.ParseSocketControlMessage(buf)
+
+	// convert fds to files
+	res := make([]*os.File, 0, len(msgs))
+	for i := 0; i < len(msgs) && err == nil; i++ {
+		var fds []int
+		fds, err = syscall.ParseUnixRights(&msgs[i])
+
+		for fi, fd := range fds {
+			var filename string
+			if fi < len(filenames) {
+				filename = filenames[fi]
+			}
+
+			res = append(res, os.NewFile(uintptr(fd), filename))
+		}
+	}
+
+	return res, err
+}
+
+// Put sends file descriptors to Unix domain socket.
+//
+// Please note that the number of descriptors in one message is limited
+// and is rather small.
+// Use conn.File() to get a file if you want to put a network connection.
+func Put(via *net.UnixConn, files ...*os.File) error {
+	if len(files) == 0 {
+		return nil
+	}
+
+	viaf, err := via.File()
+	if err != nil {
+		return err
+	}
+	socket := int(viaf.Fd())
+	defer viaf.Close()
+
+	fds := make([]int, len(files))
+	for i := range files {
+		fds[i] = int(files[i].Fd())
+	}
+
+	rights := syscall.UnixRights(fds...)
+	return syscall.Sendmsg(socket, nil, rights, nil, 0)
+}

pkg/fd/fd_test.go

@@ -0,0 +1,87 @@
+//go:build linux && darwin
+// +build linux,darwin
+
+// This file has been adapted from https://github.com/ftrvxmtrx/fd/blob/master/fd_test.go
+
+package fd
+
+import (
+	"errors"
+	"net"
+	"os"
+	"sync"
+	"testing"
+)
+
+var SockFilename = "/tmp/sendfd.sock"
+
+func getFD(w *sync.WaitGroup) error {
+	defer w.Done()
+
+	c, err := net.Dial("unix", SockFilename)
+	if err != nil {
+		return err
+	}
+	defer c.Close()
+	sendfdConn := c.(*net.UnixConn)
+
+	var fs []*os.File
+	fs, err = Get(sendfdConn, 1, []string{"a file"})
+	if err != nil {
+		return err
+	}
+	f := fs[0]
+	defer f.Close()
+
+	b := make([]byte, 64)
+	var n int
+	n, err = f.Read(b)
+	if err != nil {
+		return err
+	}
+
+	if n < 1 {
+		return errors.New("failed to read the data")
+	}
+	return nil
+}
+
+func TestPutGet(t *testing.T) {
+	f, err := os.Open("/dev/urandom")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+
+	os.Remove(SockFilename)
+	l, err := net.Listen("unix", SockFilename)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer l.Close()
+
+	var w sync.WaitGroup
+	w.Add(1)
+
+	go func() {
+		err := getFD(&w)
+		if err != nil {
+			t.Error(err)
+		}
+	}()
+
+	var a net.Conn
+	a, err = l.Accept()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer a.Close()
+
+	listenConn := a.(*net.UnixConn)
+
+	if err = Put(listenConn, f); err != nil {
+		t.Fatal(err)
+	}
+
+	w.Wait()
+}

pkg/fd/fd_windows.go

@@ -0,0 +1,18 @@
+//go:build windows
+// +build windows
+
+package fd
+
+import (
+	"errors"
+	"net"
+	"os"
+)
+
+func Get(via *net.UnixConn, num int, filenames []string) ([]*os.File, error) {
+	return nil, errors.New("Not supported")
+}
+
+func Put(via *net.UnixConn, files ...*os.File) error {
+	return errors.New("Not supported")
+}