diff --git a/.cirrus.yml b/.cirrus.yml index a6264d01e38..803b5ec6f47 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -23,6 +23,7 @@ task: EXAMPLE: fedora.yaml EXAMPLE: archlinux.yaml EXAMPLE: opensuse.yaml + EXAMPLE: experimental/net-user-v2.yaml info_script: - uname -a - df -T diff --git a/cmd/limactl/main.go b/cmd/limactl/main.go index 646b8a3c0dd..d8f3afec889 100644 --- a/cmd/limactl/main.go +++ b/cmd/limactl/main.go @@ -109,6 +109,7 @@ func newApp() *cobra.Command { newEditCommand(), newFactoryResetCommand(), newDiskCommand(), + newUsernetCommand(), ) return rootCmd } diff --git a/cmd/limactl/usernet.go b/cmd/limactl/usernet.go new file mode 100644 index 00000000000..54f0dc9873f --- /dev/null +++ b/cmd/limactl/usernet.go @@ -0,0 +1,71 @@ +package main + +import ( + "errors" + "fmt" + "os" + "strconv" + + "github.com/lima-vm/lima/pkg/networks/usernet" + "github.com/spf13/cobra" +) + +func newUsernetCommand() *cobra.Command { + var hostagentCommand = &cobra.Command{ + Use: "usernet", + Short: "run usernet", + Args: cobra.ExactArgs(0), + RunE: usernetAction, + Hidden: true, + } + hostagentCommand.Flags().StringP("pidfile", "p", "", "write pid to file") + hostagentCommand.Flags().StringP("endpoint", "e", "", "exposes usernet api(s) on this endpoint") + hostagentCommand.Flags().String("listen-qemu", "", "listen for qemu connections") + hostagentCommand.Flags().String("listen", "", "listen on a Unix socket and receive Bess-compatible FDs as SCM_RIGHTS messages") + hostagentCommand.Flags().Int("mtu", 1500, "mtu") + return hostagentCommand +} + +func usernetAction(cmd *cobra.Command, args []string) error { + pidfile, err := cmd.Flags().GetString("pidfile") + if err != nil { + return err + } + if pidfile != "" { + if _, err := os.Stat(pidfile); !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("pidfile %q already exists", pidfile) + } + if err := os.WriteFile(pidfile, []byte(strconv.Itoa(os.Getpid())+"\n"), 0644); err != nil { + return err + } + defer os.RemoveAll(pidfile) + } + endpoint, err := cmd.Flags().GetString("endpoint") + if err != nil { + return err + } + qemuSocket, err := cmd.Flags().GetString("listen-qemu") + if err != nil { + return err + } + fdSocket, err := cmd.Flags().GetString("listen") + if err != nil { + return err + } + + mtu, err := cmd.Flags().GetInt("mtu") + if err != nil { + return err + } + + os.RemoveAll(endpoint) + os.RemoveAll(qemuSocket) + os.RemoveAll(fdSocket) + + return usernet.StartGVisorNetstack(cmd.Context(), &usernet.GVisorNetstackOpts{ + MTU: mtu, + Endpoint: endpoint, + QemuSocket: qemuSocket, + FdSocket: fdSocket, + }) +} diff --git a/docs/experimental.md b/docs/experimental.md index 19c8d2d47a5..da46859d864 100644 --- a/docs/experimental.md +++ b/docs/experimental.md @@ -6,6 +6,7 @@ The following features are experimental and subject to change: - `vmType: vz` and relevant configurations (`mountType: virtiofs`, `rosetta`, `[]networks.vzNAT`) - `arch: riscv64` - `video.display: vnc` and relevant configuration (`video.vnc.display`) +- `mode: user-v2` in `networks.yml` and relevant configuration in `lima.yaml` The following flags are experimental and subject to change: diff --git a/docs/network.md b/docs/network.md index 7f19924c66c..aaf70fc5f9c 100644 --- a/docs/network.md +++ b/docs/network.md @@ -193,3 +193,33 @@ networks: The range of the IP address is not specifiable. The "vzNAT" network does not need the `socket_vmnet` binary and the `sudoers` file. + +## Lima user-v2 network + +user-v2 network provides a user-mode networking similar to the [default user-mode network](#user-mode-network--1921685024-) and also provides support for `vm -> vm` communication. + +> **Warning** +> This network mode is experimental + +To enable this network mode, define a network with `mode: user-v2` in networks.yaml + +```yaml +... +networks: + example-user-v2: + mode: user-v2 +... +``` + +Instances can then reference these networks from their `lima.yaml` file: + +```yaml +networks: + - lima: example-user-v2 +``` + +_Note_ + +- Enabling this network will disable the [default user-mode network](#user-mode-network--1921685024-) +- Subnet used for this network is 192.168.5.0/24 with 192.168.5.2 used for host connection and 192.168.5.3 used for DNS resolution + diff --git a/examples/experimental/net-user-v2.yaml b/examples/experimental/net-user-v2.yaml new file mode 100644 index 00000000000..6692b922415 --- /dev/null +++ b/examples/experimental/net-user-v2.yaml @@ -0,0 +1,13 @@ +# Example to run lima instance with experimental user-v2 network enabled +images: +- location: "https://cloud-images.ubuntu.com/releases/22.04/release/ubuntu-22.04-server-cloudimg-amd64.img" + arch: "x86_64" +- location: "https://cloud-images.ubuntu.com/releases/22.04/release/ubuntu-22.04-server-cloudimg-arm64.img" + arch: "aarch64" + +mounts: +- location: "~" +- location: "/tmp/lima" + writable: true +networks: +- lima: user-v2 diff --git a/go.mod b/go.mod index 14cb62736bd..1eb33a2f260 100644 --- a/go.mod +++ b/go.mod @@ -52,6 +52,7 @@ require ( github.com/a8m/envsubst v1.4.2 // indirect github.com/alecthomas/participle/v2 v2.0.0 // indirect github.com/apparentlymart/go-cidr v1.1.0 // indirect + github.com/balajiv113/fd v0.0.0-20230330094840-143eec500f3e // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/digitalocean/go-libvirt v0.0.0-20201209184759-e2a69bcd5bd1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect @@ -80,10 +81,13 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/kr/fs v0.1.0 // indirect + github.com/linuxkit/virtsock v0.0.0-20220523201153-1a23e78aa7a2 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.6 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-runewidth v0.0.12 // indirect + github.com/mdlayher/socket v0.4.0 // indirect + github.com/mdlayher/vsock v1.2.0 // indirect github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect diff --git a/go.sum b/go.sum index b3ad90b0bd4..b0af3a39f2e 100644 --- a/go.sum +++ b/go.sum @@ -73,6 +73,12 @@ github.com/apparentlymart/go-cidr v1.1.0 h1:2mAhrMoF+nhXqxTzSZMUzDHkLjmIHC+Zzn4t github.com/apparentlymart/go-cidr v1.1.0/go.mod h1:EBcsNrHc3zQeuaeCeCtQruQm+n9/YjEn/vI25Lg7Gwc= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-proxyproto v0.0.0-20210323213023-7e956b284f0a/go.mod h1:QmP9hvJ91BbJmGVGSbutW19IC0Q9phDCLGaomwTJbgU= +github.com/balajiv113/fd v0.0.0-20150925145434-c6d800382fff h1:igcsiQjkP5E7AfbHNG73RNYQq+MZt0MezdYyW/zqOcg= +github.com/balajiv113/fd v0.0.0-20150925145434-c6d800382fff/go.mod h1:aXGMJsd3XrnUFTuyf/pTGg5jG6CY8JMZ5juywvShjgQ= +github.com/balajiv113/fd v0.0.0-20230330061141-18b1d0daf6e7 h1:6Y2nNPVsEoUA3fmWCouMVLlezu++LP6y9CSX1J8VtVg= +github.com/balajiv113/fd v0.0.0-20230330061141-18b1d0daf6e7/go.mod h1:aXGMJsd3XrnUFTuyf/pTGg5jG6CY8JMZ5juywvShjgQ= +github.com/balajiv113/fd v0.0.0-20230330094840-143eec500f3e h1:IdMhFPEfTZQU971tIHx3UhY4l+yCeynprnINrDTSrOc= +github.com/balajiv113/fd v0.0.0-20230330094840-143eec500f3e/go.mod h1:aXGMJsd3XrnUFTuyf/pTGg5jG6CY8JMZ5juywvShjgQ= github.com/cenkalti/backoff v1.1.1-0.20190506075156-2146c9339422/go.mod h1:b6Nc7NRH5C4aCISLry0tLnTjcuTEvoiqcWDdsU0sOGM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cheggaaa/pb/v3 v3.1.2 h1:FIxT3ZjOj9XJl0U4o2XbEhjFfZl7jCVCDOGq1ZAB7wQ= @@ -362,6 +368,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/lima-vm/sshocker v0.3.2 h1:o0WqVzcpt6mzVCuqtS3N3O8kwTx6X4SLr4h7YaRISuE= github.com/lima-vm/sshocker v0.3.2/go.mod h1:9SWN6wob210VM6oJkkzvWQOlHSp/rQLB+0fSEc92zig= +github.com/linuxkit/virtsock v0.0.0-20220523201153-1a23e78aa7a2 h1:DZMFueDbfz6PNc1GwDRA8+6lBx1TB9UnxDQliCqR73Y= +github.com/linuxkit/virtsock v0.0.0-20220523201153-1a23e78aa7a2/go.mod h1:SWzULI85WerrFt3u+nIm5F9l7EvxZTKQvd0InF3nmgM= github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY= github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -394,6 +402,10 @@ github.com/mdlayher/netlink v1.1.1/go.mod h1:WTYpFb/WTvlRJAyKhZL5/uy69TDDpHHu2VZ github.com/mdlayher/raw v0.0.0-20190606142536-fef19f00fc18/go.mod h1:7EpbotpCmVZcu+KCX4g9WaRNuu11uyhiW7+Le1dKawg= github.com/mdlayher/raw v0.0.0-20191009151244-50f2db8cc065 h1:aFkJ6lx4FPip+S+Uw4aTegFMct9shDvP+79PsSxpm3w= github.com/mdlayher/raw v0.0.0-20191009151244-50f2db8cc065/go.mod h1:7EpbotpCmVZcu+KCX4g9WaRNuu11uyhiW7+Le1dKawg= +github.com/mdlayher/socket v0.4.0 h1:280wsy40IC9M9q1uPGcLBwXpcTQDtoGwVt+BNoITxIw= +github.com/mdlayher/socket v0.4.0/go.mod h1:xxFqz5GRCUN3UEOm9CZqEJsAbe1C8OwSK46NlmWuVoc= +github.com/mdlayher/vsock v1.2.0 h1:klRY9lndjmg6k/QWbX/ucQ3e2JFRm1M7vfG9hijbQ0A= +github.com/mdlayher/vsock v1.2.0/go.mod h1:w4kdSTQB9p1l/WwGmAs0V62qQ869qRYoongwgN+Y1HE= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/dns v1.1.25/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= diff --git a/hack/test-example.sh b/hack/test-example.sh index 540db8f264b..d3a7ce11158 100755 --- a/hack/test-example.sh +++ b/hack/test-example.sh @@ -25,6 +25,7 @@ declare -A CHECKS=( ["port-forwards"]="1" ["vmnet"]="" ["disk"]="" + ["user-v2"]="" ) case "$NAME" in @@ -49,6 +50,10 @@ case "$NAME" in "test-misc") CHECKS["disk"]=1 ;; +"net-user-v2") + CHECKS["port-forwards"]="" + CHECKS["user-v2"]=1 + ;; esac if limactl ls -q | grep -q "$NAME"; then @@ -303,6 +308,28 @@ if [[ -n ${CHECKS["restart"]} ]]; then fi fi +if [[ -n ${CHECKS["user-v2"]} ]]; then + INFO "Testing user-v2 network" + secondvm="$NAME-1" + limactl start "$FILE" --name "$secondvm" --tty=false + guestNewip="$(limactl shell "$secondvm" ip -4 -j addr show dev eth0 | jq -r '.[0].addr_info[0].local')" + INFO "IP of $secondvm is $guestNewip" + set -x + if ! limactl shell "$NAME" ping -c 1 "$guestNewip"; then + ERROR "Failed to do vm->vm communication via user-v2" + INFO "Stopping \"$secondvm\"" + limactl stop "$secondvm" + INFO "Deleting \"$secondvm\"" + limactl delete "$secondvm" + exit 1 + fi + INFO "Stopping \"$secondvm\"" + limactl stop "$secondvm" + INFO "Deleting \"$secondvm\"" + limactl delete "$secondvm" + set +x +fi + INFO "Stopping \"$NAME\"" limactl stop "$NAME" sleep 3 diff --git a/pkg/cidata/cidata.go b/pkg/cidata/cidata.go index 8e52d1bae6b..0873a53c7ec 100644 --- a/pkg/cidata/cidata.go +++ b/pkg/cidata/cidata.go @@ -211,7 +211,11 @@ func GenerateISO9660(instDir, name string, y *limayaml.LimaYAML, udpDNSLocalPort slirpMACAddress := limayaml.MACAddress(instDir) args.Networks = append(args.Networks, Network{MACAddress: slirpMACAddress, Interface: networks.SlirpNICName}) - for _, nw := range y.Networks { + firstUsernetIndex := limayaml.FirstUsernetIndex(y) + for i, nw := range y.Networks { + if i == firstUsernetIndex { + continue + } args.Networks = append(args.Networks, Network{MACAddress: nw.MACAddress, Interface: nw.Interface}) } diff --git a/pkg/limayaml/defaults.go b/pkg/limayaml/defaults.go index 9852c0cd01b..f358f70d883 100644 --- a/pkg/limayaml/defaults.go +++ b/pkg/limayaml/defaults.go @@ -11,6 +11,8 @@ import ( "strconv" "text/template" + "github.com/lima-vm/lima/pkg/networks" + "github.com/lima-vm/lima/pkg/guestagent/api" "github.com/lima-vm/lima/pkg/osutil" "github.com/lima-vm/lima/pkg/store/dirnames" @@ -49,6 +51,18 @@ func defaultContainerdArchives() []File { } } +// FirstUsernetIndex gets the index of first usernet network under l.Network[]. Returns -1 if no usernet network found +func FirstUsernetIndex(l *LimaYAML) int { + for i := range l.Networks { + nwName := l.Networks[i].Lima + isUsernet, _ := networks.Usernet(nwName) + if isUsernet { + return i + } + } + return -1 +} + func MACAddress(uniqueID string) string { sha := sha256.Sum256([]byte(osutil.MachineID() + uniqueID)) // "5" is the magic number in the Lima ecosystem. diff --git a/pkg/limayaml/validate.go b/pkg/limayaml/validate.go index 77091589268..4ef2b2065cf 100644 --- a/pkg/limayaml/validate.go +++ b/pkg/limayaml/validate.go @@ -281,7 +281,18 @@ func validateNetwork(y LimaYAML, warn bool) error { for i, nw := range y.Networks { field := fmt.Sprintf("networks[%d]", i) if nw.Lima != "" { - if runtime.GOOS != "darwin" { + config, err := networks.Config() + if err != nil { + return err + } + if config.Check(nw.Lima) != nil { + return fmt.Errorf("field `%s.lima` references network %q which is not defined in networks.yaml", field, nw.Lima) + } + usernet, err := config.Usernet(nw.Lima) + if err != nil { + return err + } + if !usernet && runtime.GOOS != "darwin" { return fmt.Errorf("field `%s.lima` is only supported on macOS right now", field) } if nw.Socket != "" { @@ -296,13 +307,6 @@ func validateNetwork(y LimaYAML, warn bool) error { if nw.SwitchPortDeprecated != 0 { return fmt.Errorf("field `%s.switchPort` cannot be used with field `%s.lima`", field, field) } - config, err := networks.Config() - if err != nil { - return err - } - if config.Check(nw.Lima) != nil { - return fmt.Errorf("field `%s.lima` references network %q which is not defined in networks.yaml", field, nw.Lima) - } } else if nw.Socket != "" { if nw.VZNAT != nil && *nw.VZNAT { return fmt.Errorf("field `%s.socket` and field `%s.vzNAT` are mutually exclusive", field, field) diff --git a/pkg/networks/commands.go b/pkg/networks/commands.go index 87180b287da..b1e6b8e61db 100644 --- a/pkg/networks/commands.go +++ b/pkg/networks/commands.go @@ -28,6 +28,14 @@ func (config *YAML) Check(name string) error { return fmt.Errorf("network %q is not defined", name) } +// Usernet Returns true if the mode of given network is ModeUserV2 +func (config *YAML) Usernet(name string) (bool, error) { + if nw, ok := config.Networks[name]; ok { + return nw.Mode == ModeUserV2, nil + } + return false, fmt.Errorf("network %q is not defined", name) +} + // DaemonPath returns the daemon path. func (config *YAML) DaemonPath(daemon string) (string, error) { switch daemon { diff --git a/pkg/networks/config.go b/pkg/networks/config.go index e9072413051..15a03c60a77 100644 --- a/pkg/networks/config.go +++ b/pkg/networks/config.go @@ -7,7 +7,6 @@ import ( "os" "os/exec" "path/filepath" - "runtime" "sync" "github.com/goccy/go-yaml" @@ -122,9 +121,6 @@ func loadCache() { // Config returns the network config from the _config/networks.yaml file. func Config() (YAML, error) { - if runtime.GOOS != "darwin" { - return YAML{}, errors.New("networks.yaml configuration is only supported on macOS right now") - } loadCache() return cache.config, cache.err } @@ -144,6 +140,15 @@ func Sock(name string) (string, error) { return cache.config.Sock(name), nil } +// Usernet Returns true if the given network name is usernet network +func Usernet(name string) (bool, error) { + loadCache() + if cache.err != nil { + return false, cache.err + } + return cache.config.Usernet(name) +} + // VDESock returns a vde socket. // // Deprecated. Use Sock. diff --git a/pkg/networks/gvisor.go b/pkg/networks/gvisor.go deleted file mode 100644 index 557d5c50426..00000000000 --- a/pkg/networks/gvisor.go +++ /dev/null @@ -1,122 +0,0 @@ -package networks - -import ( - "bufio" - "context" - "fmt" - "net" - "os" - "runtime" - "strings" - - "github.com/containers/gvisor-tap-vsock/pkg/types" - "github.com/containers/gvisor-tap-vsock/pkg/virtualnetwork" - "github.com/sirupsen/logrus" - "golang.org/x/sync/errgroup" -) - -type GVisorNetstackOpts struct { - MTU int - Stream bool - - Conn net.Conn - - MacAddress string - SSHLocalPort int -} - -var ( - opts *GVisorNetstackOpts -) - -func StartGVisorNetstack(ctx context.Context, gVisorOpts *GVisorNetstackOpts) { - opts = gVisorOpts - - var protocol types.Protocol - if opts.Stream == false { - protocol = types.BessProtocol - } else { - protocol = types.QemuProtocol - } - - // The way gvisor-tap-vsock implemented slirp is different from tradition SLIRP, - // - GatewayIP handling all request, also answers DNS queries - // - based on NAT configuration, gateway forwards and translates calls to host - // Comparing this with QEMU SLIRP, - // - DNS is equivalent to GatewayIP - // - GatewayIP is equivalent to NAT configuration - config := types.Configuration{ - Debug: false, - MTU: opts.MTU, - Subnet: SlirpNetwork, - GatewayIP: SlirpDNS, - GatewayMacAddress: "5a:94:ef:e4:0c:dd", - DHCPStaticLeases: map[string]string{ - SlirpIPAddress: opts.MacAddress, - }, - Forwards: map[string]string{ - fmt.Sprintf("127.0.0.1:%d", opts.SSHLocalPort): net.JoinHostPort(SlirpIPAddress, "22"), - }, - DNS: []types.Zone{}, - DNSSearchDomains: searchDomains(), - NAT: map[string]string{ - SlirpGateway: "127.0.0.1", - }, - GatewayVirtualIPs: []string{SlirpGateway}, - Protocol: protocol, - } - - groupErrs, ctx := errgroup.WithContext(ctx) - groupErrs.Go(func() error { - return run(ctx, groupErrs, &config) - }) - go func() { - err := groupErrs.Wait() - if err != nil { - logrus.Errorf("virtual network error: %q", err) - } - }() -} - -func run(ctx context.Context, g *errgroup.Group, configuration *types.Configuration) error { - vn, err := virtualnetwork.New(configuration) - if err != nil { - return err - } - - if opts.Conn != nil { - g.Go(func() error { - if opts.Stream == false { - return vn.AcceptBess(ctx, opts.Conn) - } - return vn.AcceptQemu(ctx, opts.Conn) - }) - } - - return nil -} - -func searchDomains() []string { - if runtime.GOOS != "windows" { - f, err := os.Open("/etc/resolv.conf") - if err != nil { - logrus.Errorf("open file error: %v", err) - return nil - } - defer f.Close() - sc := bufio.NewScanner(f) - searchPrefix := "search " - for sc.Scan() { - if strings.HasPrefix(sc.Text(), searchPrefix) { - searchDomains := strings.Split(strings.TrimPrefix(sc.Text(), searchPrefix), " ") - logrus.Debugf("Using search domains: %v", searchDomains) - return searchDomains - } - } - if err := sc.Err(); err != nil { - logrus.Errorf("scan file error: %v", err) - return nil - } - } - return nil -} diff --git a/pkg/networks/networks.TEMPLATE.yaml b/pkg/networks/networks.TEMPLATE.yaml index 0032e005910..0916473fa5c 100644 --- a/pkg/networks/networks.TEMPLATE.yaml +++ b/pkg/networks/networks.TEMPLATE.yaml @@ -22,6 +22,10 @@ paths: group: everyone networks: + user-v2: + mode: user-v2 + # user-v2 network is experimental network mode which supports all functionalities of default usernet network and also allows vm -> vm communication. + # Doesn't support configuration of custom gateway; hardcoded to 192.168.5.0/24 shared: mode: shared gateway: 192.168.105.1 diff --git a/pkg/networks/networks.go b/pkg/networks/networks.go index 69026512119..679bebb201e 100644 --- a/pkg/networks/networks.go +++ b/pkg/networks/networks.go @@ -17,6 +17,7 @@ type Paths struct { } const ( + ModeUserV2 = "user-v2" ModeHost = "host" ModeShared = "shared" ModeBridged = "bridged" diff --git a/pkg/networks/reconcile/reconcile.go b/pkg/networks/reconcile/reconcile.go index 19ad9d5bf6e..36c556ef5f8 100644 --- a/pkg/networks/reconcile/reconcile.go +++ b/pkg/networks/reconcile/reconcile.go @@ -12,6 +12,7 @@ import ( "time" "github.com/lima-vm/lima/pkg/networks" + "github.com/lima-vm/lima/pkg/networks/usernet" "github.com/lima-vm/lima/pkg/osutil" "github.com/lima-vm/lima/pkg/store" "github.com/lima-vm/lima/pkg/store/dirnames" @@ -19,9 +20,6 @@ import ( ) func Reconcile(ctx context.Context, newInst string) error { - if runtime.GOOS != "darwin" { - return nil - } config, err := networks.Config() if err != nil { return err @@ -175,6 +173,23 @@ func validateConfig(config *networks.YAML) error { func startNetwork(ctx context.Context, config *networks.YAML, name string) error { logrus.Debugf("Make sure %q network is running", name) + + //Handle usernet first without sudo requirements + isUsernet, err := config.Usernet(name) + if err != nil { + return err + } + if isUsernet { + if err := usernet.Start(ctx, name); err != nil { + return err + } + return nil + } + + if runtime.GOOS != "darwin" { + return nil + } + if err := validateConfig(config); err != nil { return err } @@ -205,6 +220,22 @@ func startNetwork(ctx context.Context, config *networks.YAML, name string) error func stopNetwork(config *networks.YAML, name string) error { logrus.Debugf("Make sure %q network is stopped", name) + //Handle usernet first without sudo requirements + isUsernet, err := config.Usernet(name) + if err != nil { + return err + } + if isUsernet { + if err := usernet.Stop(name); err != nil { + return err + } + return nil + } + + if runtime.GOOS != "darwin" { + return nil + } + // Don't call validateConfig() until we actually need to stop a daemon because // stopNetwork() may be called even when the vde daemons are not installed. for _, daemon := range []string{networks.SocketVMNet, networks.VDEVMNet, networks.VDESwitch} { diff --git a/pkg/networks/usernet/client.go b/pkg/networks/usernet/client.go new file mode 100644 index 00000000000..e6989880c4a --- /dev/null +++ b/pkg/networks/usernet/client.go @@ -0,0 +1,96 @@ +package usernet + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net" + "net/http" + "time" + + gvproxyclient "github.com/containers/gvisor-tap-vsock/pkg/client" + "github.com/containers/gvisor-tap-vsock/pkg/types" +) + +type Client struct { + Directory string + + client *http.Client + delegate *gvproxyclient.Client + base string +} + +func (c *Client) UnExposeSSH(sshPort int) error { + return c.delegate.Unexpose(&types.UnexposeRequest{ + Local: fmt.Sprintf("127.0.0.1:%d", sshPort), + Protocol: "tcp", + }) +} + +func (c *Client) ResolveAndForwardSSH(vmMacAddr string, sshPort int) error { + timeout := time.After(1 * time.Minute) + tick := time.Tick(500 * time.Millisecond) + for { + select { + case <-timeout: + return errors.New("usernet unable to resolve IP for SSH forwarding") + case <-tick: + leases, err := c.leases() + if err != nil { + return err + } + + for ipAddr, leaseAddr := range leases { + if vmMacAddr == leaseAddr { + err = c.delegate.Expose(&types.ExposeRequest{ + Local: fmt.Sprintf("127.0.0.1:%d", sshPort), + Remote: fmt.Sprintf("%s:22", ipAddr), + Protocol: "tcp", + }) + if err != nil { + return err + } + return nil + } + } + } + } +} + +func (c *Client) leases() (map[string]string, error) { + res, err := c.client.Get(fmt.Sprintf("%s%s", c.base, "/services/dhcp/leases")) + if err != nil { + return nil, err + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected status: %d", res.StatusCode) + } + dec := json.NewDecoder(res.Body) + var leases map[string]string + if err := dec.Decode(&leases); err != nil { + return nil, err + } + return leases, nil +} + +func NewClient(endpointSock string) *Client { + return create(endpointSock, "http://lima") +} + +func create(sock string, base string) *Client { + client := &http.Client{ + Transport: &http.Transport{ + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + return net.Dial("unix", sock) + }, + }, + } + delegate := gvproxyclient.New(client, "http://lima") + return &Client{ + client: client, + delegate: delegate, + base: base, + } +} diff --git a/pkg/networks/usernet/config.go b/pkg/networks/usernet/config.go new file mode 100644 index 00000000000..d97cc18d8a8 --- /dev/null +++ b/pkg/networks/usernet/config.go @@ -0,0 +1,39 @@ +package usernet + +import ( + "fmt" + "path/filepath" + + "github.com/lima-vm/lima/pkg/store/dirnames" +) + +type SockType = string + +const ( + FDSock = "fd" + QEMUSock = "qemu" + EndpointSock = "endpoint" +) + +// Sock returns a usernet socket based on name and sockType. +func Sock(name string, sockType SockType) (string, error) { + dir, err := dirnames.LimaNetworksDir() + if err != nil { + return "", err + } + return SockWithDirectory(filepath.Join(dir, name), name, sockType), nil +} + +// SockWithDirectory return a usernet socket based on dir, name and sockType +func SockWithDirectory(dir string, name string, sockType SockType) string { + return filepath.Join(dir, fmt.Sprintf("usernet_%s_%s.sock", name, sockType)) +} + +// PIDFile returns a path for usernet PID file +func PIDFile(name string) (string, error) { + dir, err := dirnames.LimaNetworksDir() + if err != nil { + return "", err + } + return filepath.Join(dir, name, fmt.Sprintf("usernet_%s.pid", name)), nil +} diff --git a/pkg/networks/usernet/gvproxy.go b/pkg/networks/usernet/gvproxy.go new file mode 100644 index 00000000000..1683630f4cd --- /dev/null +++ b/pkg/networks/usernet/gvproxy.go @@ -0,0 +1,234 @@ +package usernet + +import ( + "bufio" + "context" + "fmt" + "net" + "net/http" + "os" + "runtime" + "strings" + "time" + + "github.com/lima-vm/lima/pkg/networks" + + "github.com/balajiv113/fd" + "github.com/containers/gvisor-tap-vsock/pkg/transport" + "github.com/containers/gvisor-tap-vsock/pkg/types" + "github.com/containers/gvisor-tap-vsock/pkg/virtualnetwork" + "github.com/sirupsen/logrus" + "golang.org/x/sync/errgroup" +) + +type GVisorNetstackOpts struct { + MTU int + + QemuSocket string + FdSocket string + Endpoint string + + Async bool +} + +var ( + opts *GVisorNetstackOpts +) + +func StartGVisorNetstack(ctx context.Context, gVisorOpts *GVisorNetstackOpts) error { + opts = gVisorOpts + + // The way gvisor-tap-vsock implemented slirp is different from tradition SLIRP, + // - GatewayIP handling all request, also answers DNS queries + // - based on NAT configuration, gateway forwards and translates calls to host + // Comparing this with QEMU SLIRP, + // - DNS is equivalent to GatewayIP + // - GatewayIP is equivalent to NAT configuration + config := types.Configuration{ + Debug: false, + MTU: opts.MTU, + Subnet: networks.SlirpNetwork, + GatewayIP: networks.SlirpDNS, + GatewayMacAddress: "5a:94:ef:e4:0c:dd", + DHCPStaticLeases: map[string]string{ + networks.SlirpGateway: "5a:94:ef:e4:0c:df", + }, + Forwards: map[string]string{}, + DNS: []types.Zone{}, + DNSSearchDomains: searchDomains(), + NAT: map[string]string{ + networks.SlirpGateway: "127.0.0.1", + }, + GatewayVirtualIPs: []string{networks.SlirpGateway}, + } + + groupErrs, ctx := errgroup.WithContext(ctx) + err := run(ctx, groupErrs, &config) + if err != nil { + return err + } + if opts.Async { + return err + } + return groupErrs.Wait() +} + +func run(ctx context.Context, g *errgroup.Group, configuration *types.Configuration) error { + vn, err := virtualnetwork.New(configuration) + if err != nil { + return err + } + + ln, err := transport.Listen(fmt.Sprintf("unix://%s", opts.Endpoint)) + if err != nil { + return err + } + httpServe(ctx, g, ln, vn.Mux()) + + if opts.QemuSocket != "" { + err = listenQEMU(ctx, vn) + if err != nil { + return err + } + } + if opts.FdSocket != "" { + err = listenFD(ctx, vn) + if err != nil { + return err + } + } + return nil +} + +func listenQEMU(ctx context.Context, vn *virtualnetwork.VirtualNetwork) error { + listener, err := net.Listen("unix", opts.QemuSocket) + if err != nil { + return err + } + + go func() { + defer listener.Close() + for { + conn, err := listener.Accept() + if err != nil { + logrus.Error("QEMU accept failed", err) + } + + go func() { + err = vn.AcceptQemu(ctx, conn) + if err != nil { + logrus.Error("QEMU connection closed with error", err) + } + conn.Close() + }() + select { + case <-ctx.Done(): + return + default: + continue + } + } + + }() + + return nil +} + +func listenFD(ctx context.Context, vn *virtualnetwork.VirtualNetwork) error { + listener, err := net.Listen("unix", opts.FdSocket) + if err != nil { + return err + } + + go func() { + defer listener.Close() + for { + conn, err := listener.Accept() + if err != nil { + logrus.Error("FD accept failed", err) + } + + files, err := fd.Get(conn.(*net.UnixConn), 1, []string{"client"}) + if err != nil { + logrus.Error("Failed to get FD via socket", err) + } + + if len(files) != 1 { + logrus.Error("Invalid number of fd in response", err) + } + fileConn, err := net.FileConn(files[0]) + if err != nil { + logrus.Error("Error in FD Socket", err) + } + files[0].Close() + + go func() { + err = vn.AcceptBess(ctx, fileConn) + if err != nil { + logrus.Error("FD connection closed with error", err) + } + defer fileConn.Close() + }() + select { + case <-ctx.Done(): + return + default: + continue + } + } + }() + + return nil +} + +func httpServe(ctx context.Context, g *errgroup.Group, ln net.Listener, mux http.Handler) { + g.Go(func() error { + <-ctx.Done() + return ln.Close() + }) + g.Go(func() error { + s := &http.Server{ + Handler: mux, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + } + err := s.Serve(ln) + if err != nil { + if err != http.ErrServerClosed { + return err + } + return err + } + return nil + }) +} + +func searchDomains() []string { + if runtime.GOOS != "windows" { + return resolveSearchDomain("/etc/resolv.conf") + } + return nil +} + +func resolveSearchDomain(file string) []string { + f, err := os.Open(file) + if err != nil { + logrus.Errorf("open file error: %v", err) + return nil + } + defer f.Close() + sc := bufio.NewScanner(f) + searchPrefix := "search " + for sc.Scan() { + if strings.HasPrefix(sc.Text(), searchPrefix) { + searchDomains := strings.Split(strings.TrimPrefix(sc.Text(), searchPrefix), " ") + logrus.Debugf("Using search domains: %v", searchDomains) + return searchDomains + } + } + if err := sc.Err(); err != nil { + logrus.Errorf("scan file error: %v", err) + return nil + } + return nil +} diff --git a/pkg/networks/usernet/gvproxy_test.go b/pkg/networks/usernet/gvproxy_test.go new file mode 100644 index 00000000000..c2b52048962 --- /dev/null +++ b/pkg/networks/usernet/gvproxy_test.go @@ -0,0 +1,57 @@ +package usernet + +import ( + "bufio" + "os" + "path" + "runtime" + "testing" + + "gotest.tools/v3/assert" +) + +func TestSearchDomain(t *testing.T) { + + if runtime.GOOS == "windows" { + // FIXME: `TempDir RemoveAll cleanup: remove C:\users\runner\Temp\TestDownloadLocalwithout_digest2738386858\002\test-file: Sharing violation.` + t.Skip("Skipping on windows") + } + + t.Run("search domain", func(t *testing.T) { + resolvFile := path.Join(t.TempDir(), "resolv.conf") + createResolveFile(t, resolvFile, ` +search test.com lima.net +nameserver 192.168.0.100 +nameserver 8.8.8.8`) + + dns := resolveSearchDomain(resolvFile) + assert.DeepEqual(t, dns, []string{"test.com", "lima.net"}) + }) + + t.Run("empty search domain", func(t *testing.T) { + resolvFile := path.Join(t.TempDir(), "resolv.conf") + createResolveFile(t, resolvFile, ` +nameserver 192.168.0.100 +nameserver 8.8.8.8`) + + dns := resolveSearchDomain(resolvFile) + var expected []string + assert.DeepEqual(t, dns, expected) + }) +} + +func createResolveFile(t *testing.T, file string, content string) { + f, err := os.Create(file) + if err != nil { + t.Fatal(err) + } + writer := bufio.NewWriter(f) + _, err = writer.Write([]byte(content)) + if err != nil { + t.Fatal(err) + } + err = writer.Flush() + if err != nil { + t.Fatal(err) + } +} diff --git a/pkg/networks/usernet/recoincile.go b/pkg/networks/usernet/recoincile.go new file mode 100644 index 00000000000..6fc725c949e --- /dev/null +++ b/pkg/networks/usernet/recoincile.go @@ -0,0 +1,142 @@ +package usernet + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "os/exec" + "path" + "path/filepath" + "time" + + "github.com/lima-vm/lima/pkg/lockutil" + + "github.com/lima-vm/lima/pkg/store" + "github.com/lima-vm/lima/pkg/store/dirnames" + "github.com/sirupsen/logrus" +) + +// Start starts a instance a usernet network with the given name. +// The name parameter must point to a valid network configuration name under /_config/networks.yaml with `mode: user-v2` +func Start(ctx context.Context, name string) error { + logrus.Debugf("Make sure usernet network is started") + networksDir, err := dirnames.LimaNetworksDir() + if err != nil { + return err + } + //usernet files contents are stored under {LIMA_HOME}/_networks/user-v2/ + usernetDir := path.Join(networksDir, name) + if err := os.MkdirAll(usernetDir, 0755); err != nil { + return err + } + + pidFile, err := PIDFile(name) + if err != nil { + return err + } + pid, _ := store.ReadPIDFile(pidFile) + if pid == 0 { + qemuSock, err := Sock(name, QEMUSock) + if err != nil { + return err + } + + fdSock, err := Sock(name, FDSock) + if err != nil { + return err + } + + endpointSock, err := Sock(name, EndpointSock) + if err != nil { + return err + } + + err = lockutil.WithDirLock(usernetDir, func() error { + self, err := os.Executable() + if err != nil { + return err + } + args := []string{"usernet", "-p", pidFile, + "-e", endpointSock, + "--listen-qemu", qemuSock, + "--listen", fdSock} + cmd := exec.CommandContext(ctx, self, args...) + + stdoutPath := filepath.Join(usernetDir, fmt.Sprintf("%s.%s.%s.log", "usernet", name, "stdout")) + stderrPath := filepath.Join(usernetDir, fmt.Sprintf("%s.%s.%s.log", "usernet", name, "stderr")) + if err := os.RemoveAll(stdoutPath); err != nil { + return err + } + if err := os.RemoveAll(stderrPath); err != nil { + return err + } + + cmd.Stdout, err = os.Create(stdoutPath) + if err != nil { + return err + } + cmd.Stderr, err = os.Create(stderrPath) + if err != nil { + return err + } + + logrus.Debugf("Starting usernet network: %v", cmd.Args) + if err := cmd.Start(); err != nil { + return err + } + return nil + }) + if err != nil { + return err + } + for { + if _, err := os.Stat(fdSock); !errors.Is(err, os.ErrNotExist) { + break + } + time.Sleep(500 * time.Millisecond) + } + } + return nil +} + +// Stop stops running instance a usernet network with the given name. +// The name parameter must point to a valid network configuration name under /_config/networks.yaml with `mode: user-v2` +func Stop(name string) error { + logrus.Debugf("Make sure usernet network is stopped") + pidFile, err := PIDFile(name) + if err != nil { + return err + } + pid, _ := store.ReadPIDFile(pidFile) + + if pid != 0 { + logrus.Debugf("Stopping usernet daemon") + + var stdout, stderr bytes.Buffer + cmd := exec.Command("/usr/bin/pkill", "-F", pidFile) + cmd.Stdout = &stdout + cmd.Stderr = &stderr + logrus.Debugf("Running: %v", cmd.Args) + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to run %v: stdout=%q, stderr=%q: %w", + cmd.Args, stdout.String(), stderr.String(), err) + } + } + + // wait for daemons to terminate (up to 5s) before stopping, otherwise the sockets may not get deleted which + // will cause subsequent start commands to fail. + startWaiting := time.Now() + for { + if pid, _ := store.ReadPIDFile(pidFile); pid == 0 { + break + } + if time.Since(startWaiting) > 5*time.Second { + logrus.Infof("usernet network still running after 5 seconds") + break + } + time.Sleep(500 * time.Millisecond) + } + return nil +} diff --git a/pkg/qemu/qemu.go b/pkg/qemu/qemu.go index 182d2ce5859..d2de1e9aee1 100644 --- a/pkg/qemu/qemu.go +++ b/pkg/qemu/qemu.go @@ -13,6 +13,8 @@ import ( "strconv" "strings" + "github.com/lima-vm/lima/pkg/networks/usernet" + "github.com/coreos/go-semver/semver" "github.com/docker/go-units" "github.com/lima-vm/lima/pkg/fileutils" @@ -485,9 +487,20 @@ func Cmdline(cfg Config) (string, []string, error) { } // Network - args = append(args, "-netdev", fmt.Sprintf("user,id=net0,net=%s,dhcpstart=%s,hostfwd=tcp:127.0.0.1:%d-:22", - networks.SlirpNetwork, networks.SlirpIPAddress, cfg.SSHLocalPort)) + //Configure default usernetwork with limayaml.MACAddress(driver.Instance.Dir) for eth0 interface + firstUsernetIndex := limayaml.FirstUsernetIndex(y) + if firstUsernetIndex == -1 { + args = append(args, "-netdev", fmt.Sprintf("user,id=net0,net=%s,dhcpstart=%s,hostfwd=tcp:127.0.0.1:%d-:22", + networks.SlirpNetwork, networks.SlirpIPAddress, cfg.SSHLocalPort)) + } else { + qemuSock, err := usernet.Sock(y.Networks[firstUsernetIndex].Lima, usernet.QEMUSock) + if err != nil { + return "", nil, err + } + args = append(args, "-netdev", fmt.Sprintf("socket,id=net0,fd={{ fd_connect %q }}", qemuSock)) + } args = append(args, "-device", "virtio-net-pci,netdev=net0,mac="+limayaml.MACAddress(cfg.InstanceDir)) + for i, nw := range y.Networks { var vdeSock string if nw.Lima != "" { @@ -495,29 +508,50 @@ func Cmdline(cfg Config) (string, []string, error) { if err != nil { return "", nil, err } - socketVMNetOk, err := nwCfg.IsDaemonInstalled(networks.SocketVMNet) + + //Handle usernet connections + isUsernet, err := nwCfg.Usernet(nw.Lima) if err != nil { return "", nil, err } - if socketVMNetOk { - logrus.Debugf("Using socketVMNet (%q)", nwCfg.Paths.SocketVMNet) - if vdeVMNetOk, _ := nwCfg.IsDaemonInstalled(networks.VDEVMNet); vdeVMNetOk { - logrus.Debugf("Ignoring vdeVMNet (%q), as socketVMNet (%q) is available and has higher precedence", nwCfg.Paths.VDEVMNet, nwCfg.Paths.SocketVMNet) + if isUsernet { + if i == firstUsernetIndex { + continue } - sock, err := networks.Sock(nw.Lima) + qemuSock, err := usernet.Sock(nw.Lima, usernet.QEMUSock) if err != nil { return "", nil, err } - args = append(args, "-netdev", fmt.Sprintf("socket,id=net%d,fd={{ fd_connect %q }}", i+1, sock)) - } else if nwCfg.Paths.VDEVMNet != "" { - logrus.Warn("vdeVMNet is deprecated, use socketVMNet instead (See docs/network.md)") - vdeSock, err = networks.VDESock(nw.Lima) + args = append(args, "-netdev", fmt.Sprintf("socket,id=net%d,fd={{ fd_connect %q }}", i+1, qemuSock)) + args = append(args, "-device", fmt.Sprintf("virtio-net-pci,netdev=net%d,mac=%s", i+1, nw.MACAddress)) + } else { + if runtime.GOOS != "darwin" { + return "", nil, fmt.Errorf("networks.yaml '%s' configuration is only supported on macOS right now", nw.Lima) + } + socketVMNetOk, err := nwCfg.IsDaemonInstalled(networks.SocketVMNet) if err != nil { return "", nil, err } + if socketVMNetOk { + logrus.Debugf("Using socketVMNet (%q)", nwCfg.Paths.SocketVMNet) + if vdeVMNetOk, _ := nwCfg.IsDaemonInstalled(networks.VDEVMNet); vdeVMNetOk { + logrus.Debugf("Ignoring vdeVMNet (%q), as socketVMNet (%q) is available and has higher precedence", nwCfg.Paths.VDEVMNet, nwCfg.Paths.SocketVMNet) + } + sock, err := networks.Sock(nw.Lima) + if err != nil { + return "", nil, err + } + args = append(args, "-netdev", fmt.Sprintf("socket,id=net%d,fd={{ fd_connect %q }}", i+1, sock)) + } else if nwCfg.Paths.VDEVMNet != "" { + logrus.Warn("vdeVMNet is deprecated, use socketVMNet instead (See docs/network.md)") + vdeSock, err = networks.VDESock(nw.Lima) + if err != nil { + return "", nil, err + } + } + // TODO: should we also validate that the socket exists, or do we rely on the + // networks reconciler to throw an error when the network cannot start? } - // TODO: should we also validate that the socket exists, or do we rely on the - // networks reconciler to throw an error when the network cannot start? } else if nw.Socket != "" { args = append(args, "-netdev", fmt.Sprintf("socket,id=net%d,fd={{ fd_connect %q }}", i+1, nw.Socket)) } else if nw.VNLDeprecated != "" { diff --git a/pkg/qemu/qemu_driver.go b/pkg/qemu/qemu_driver.go index 1604b96aaf6..bede6183608 100644 --- a/pkg/qemu/qemu_driver.go +++ b/pkg/qemu/qemu_driver.go @@ -20,6 +20,7 @@ import ( "github.com/digitalocean/go-qemu/qmp/raw" "github.com/lima-vm/lima/pkg/driver" "github.com/lima-vm/lima/pkg/limayaml" + "github.com/lima-vm/lima/pkg/networks/usernet" "github.com/lima-vm/lima/pkg/store/filenames" "github.com/sirupsen/logrus" ) @@ -100,6 +101,12 @@ func (l *LimaQemuDriver) Start(ctx context.Context) (chan error, error) { go func() { l.qWaitCh <- qCmd.Wait() }() + go func() { + if usernetIndex := limayaml.FirstUsernetIndex(l.Yaml); usernetIndex != -1 { + client := newUsernetClient(l.Yaml.Networks[usernetIndex].Lima) + err = client.ResolveAndForwardSSH(limayaml.MACAddress(l.Instance.Dir), l.SSHLocalPort) + } + }() return l.qWaitCh, nil } @@ -189,6 +196,13 @@ func (l *LimaQemuDriver) removeVNCFiles() error { func (l *LimaQemuDriver) shutdownQEMU(ctx context.Context, timeout time.Duration, qCmd *exec.Cmd, qWaitCh <-chan error) error { logrus.Info("Shutting down QEMU with ACPI") + if usernetIndex := limayaml.FirstUsernetIndex(l.Yaml); usernetIndex != -1 { + client := newUsernetClient(l.Yaml.Networks[usernetIndex].Lima) + err := client.UnExposeSSH(l.SSHLocalPort) + if err != nil { + logrus.Warnf("Failed to remove SSH binding for port %d", l.SSHLocalPort) + } + } qmpSockPath := filepath.Join(l.Instance.Dir, filenames.QMPSock) qmpClient, err := qmp.NewSocketMonitor("unix", qmpSockPath, 5*time.Second) if err != nil { @@ -235,6 +249,14 @@ func (l *LimaQemuDriver) killQEMU(_ context.Context, _ time.Duration, qCmd *exec return qWaitErr } +func newUsernetClient(nwName string) *usernet.Client { + endpointSock, err := usernet.Sock(nwName, usernet.EndpointSock) + if err != nil { + return nil + } + return usernet.NewClient(endpointSock) +} + func logPipeRoutine(r io.Reader, header string) { scanner := bufio.NewScanner(r) for scanner.Scan() { diff --git a/pkg/vz/network_darwin.go b/pkg/vz/network_darwin.go index 84f72d843fb..6cd71363e56 100644 --- a/pkg/vz/network_darwin.go +++ b/pkg/vz/network_darwin.go @@ -11,10 +11,29 @@ import ( "os" "time" + "github.com/balajiv113/fd" + "github.com/sirupsen/logrus" "inet.af/tcpproxy" ) +func PassFDToUnix(unixSock string) (*os.File, error) { + unixConn, err := net.Dial("unix", unixSock) + if err != nil { + return nil, err + } + + server, client, err := createSockPair() + if err != nil { + return nil, err + } + err = fd.Put(unixConn.(*net.UnixConn), server) + if err != nil { + return nil, err + } + return client, nil +} + // DialQemu support connecting to QEMU supported network stack via unix socket // Returns os.File, connected dgram connection to be used for vz func DialQemu(unixSock string) (*os.File, error) { diff --git a/pkg/vz/vm_darwin.go b/pkg/vz/vm_darwin.go index c5a0691a985..cd46e247499 100644 --- a/pkg/vz/vm_darwin.go +++ b/pkg/vz/vm_darwin.go @@ -10,6 +10,7 @@ import ( "net" "os" "path/filepath" + "runtime" "strconv" "strings" "sync" @@ -22,6 +23,7 @@ import ( "github.com/lima-vm/lima/pkg/limayaml" "github.com/lima-vm/lima/pkg/localpathutil" "github.com/lima-vm/lima/pkg/networks" + "github.com/lima-vm/lima/pkg/networks/usernet" "github.com/lima-vm/lima/pkg/qemu/imgutil" "github.com/lima-vm/lima/pkg/store" "github.com/lima-vm/lima/pkg/store/filenames" @@ -35,29 +37,17 @@ type virtualMachineWrapper struct { } func startVM(ctx context.Context, driver *driver.BaseDriver) (*virtualMachineWrapper, chan error, error) { - server, client, err := createSockPair() - if err != nil { - return nil, nil, err - } - machine, err := createVM(driver, client) + usernetClient, err := startUsernet(ctx, driver) if err != nil { return nil, nil, err } - fileConn, err := net.FileConn(server) + machine, err := createVM(driver) if err != nil { return nil, nil, err } err = machine.Start() - - networks.StartGVisorNetstack(ctx, &networks.GVisorNetstackOpts{ - Conn: fileConn, - MTU: 1500, - SSHLocalPort: driver.SSHLocalPort, - MacAddress: limayaml.MACAddress(driver.Instance.Dir), - Stream: false, - }) if err != nil { return nil, nil, err } @@ -90,11 +80,17 @@ func startVM(ctx context.Context, driver *driver.BaseDriver) (*virtualMachineWra } defer os.RemoveAll(pidFile) logrus.Info("[VZ] - vm state change: running") + + err := usernetClient.ResolveAndForwardSSH(limayaml.MACAddress(driver.Instance.Dir), driver.SSHLocalPort) + if err != nil { + errCh <- err + } case vz.VirtualMachineStateStopped: logrus.Info("[VZ] - vm state change: stopped") wrapper.mu.Lock() wrapper.stopped = true wrapper.mu.Unlock() + usernetClient.UnExposeSSH(driver.SSHLocalPort) errCh <- errors.New("vz driver state stopped") default: logrus.Debugf("[VZ] - vm state change: %q", newState) @@ -106,7 +102,33 @@ func startVM(ctx context.Context, driver *driver.BaseDriver) (*virtualMachineWra return wrapper, errCh, err } -func createVM(driver *driver.BaseDriver, networkConn *os.File) (*vz.VirtualMachine, error) { +func startUsernet(ctx context.Context, driver *driver.BaseDriver) (*usernet.Client, error) { + firstUsernetIndex := limayaml.FirstUsernetIndex(driver.Yaml) + if firstUsernetIndex == -1 { + //Start a in-process gvisor-tap-vsock + endpointSock := usernet.SockWithDirectory(driver.Instance.Dir, "", usernet.EndpointSock) + vzSock := usernet.SockWithDirectory(driver.Instance.Dir, "", usernet.FDSock) + os.RemoveAll(endpointSock) + os.RemoveAll(vzSock) + err := usernet.StartGVisorNetstack(ctx, &usernet.GVisorNetstackOpts{ + MTU: 1500, + Endpoint: endpointSock, + FdSocket: vzSock, + Async: true, + }) + if err != nil { + return nil, err + } + return usernet.NewClient(endpointSock), nil + } + endpointSock, err := usernet.Sock(driver.Yaml.Networks[firstUsernetIndex].Lima, usernet.EndpointSock) + if err != nil { + return nil, err + } + return usernet.NewClient(endpointSock), nil +} + +func createVM(driver *driver.BaseDriver) (*vz.VirtualMachine, error) { vmConfig, err := createInitialConfig(driver) if err != nil { return nil, err @@ -120,7 +142,7 @@ func createVM(driver *driver.BaseDriver, networkConn *os.File) (*vz.VirtualMachi return nil, err } - if err = attachNetwork(driver, vmConfig, networkConn); err != nil { + if err = attachNetwork(driver, vmConfig); err != nil { return nil, err } @@ -202,6 +224,14 @@ func attachSerialPort(driver *driver.BaseDriver, config *vz.VirtualMachineConfig return err } +func newVirtioFileNetworkDeviceConfiguration(file *os.File, macStr string) (*vz.VirtioNetworkDeviceConfiguration, error) { + fileAttachment, err := vz.NewFileHandleNetworkDeviceAttachment(file) + if err != nil { + return nil, err + } + return newVirtioNetworkDeviceConfiguration(fileAttachment, macStr) +} + func newVirtioNetworkDeviceConfiguration(attachment vz.NetworkDeviceAttachment, macStr string) (*vz.VirtioNetworkDeviceConfiguration, error) { networkConfig, err := vz.NewVirtioNetworkDeviceConfiguration(attachment) if err != nil { @@ -219,76 +249,108 @@ func newVirtioNetworkDeviceConfiguration(attachment vz.NetworkDeviceAttachment, return networkConfig, nil } -func attachNetwork(driver *driver.BaseDriver, vmConfig *vz.VirtualMachineConfiguration, networkConn *os.File) error { - //slirp network using gvisor netstack - fileAttachment, err := vz.NewFileHandleNetworkDeviceAttachment(networkConn) - if err != nil { - return err - } - err = fileAttachment.SetMaximumTransmissionUnit(1500) - if err != nil { - return err - } - networkConfig, err := newVirtioNetworkDeviceConfiguration(fileAttachment, limayaml.MACAddress(driver.Instance.Dir)) - if err != nil { - return err - } - configurations := []*vz.VirtioNetworkDeviceConfiguration{ - networkConfig, +func attachNetwork(driver *driver.BaseDriver, vmConfig *vz.VirtualMachineConfiguration) error { + var configurations []*vz.VirtioNetworkDeviceConfiguration + + //Configure default usernetwork with limayaml.MACAddress(driver.Instance.Dir) for eth0 interface + firstUsernetIndex := limayaml.FirstUsernetIndex(driver.Yaml) + if firstUsernetIndex == -1 { + //slirp network using gvisor netstack + vzSock := usernet.SockWithDirectory(driver.Instance.Dir, "", usernet.FDSock) + networkConn, err := PassFDToUnix(vzSock) + if err != nil { + return err + } + networkConfig, err := newVirtioFileNetworkDeviceConfiguration(networkConn, limayaml.MACAddress(driver.Instance.Dir)) + if err != nil { + return err + } + configurations = append(configurations, networkConfig) + } else { + vzSock, err := usernet.Sock(driver.Yaml.Networks[firstUsernetIndex].Lima, usernet.FDSock) + if err != nil { + return err + } + networkConn, err := PassFDToUnix(vzSock) + if err != nil { + return err + } + networkConfig, err := newVirtioFileNetworkDeviceConfiguration(networkConn, limayaml.MACAddress(driver.Instance.Dir)) + if err != nil { + return err + } + configurations = append(configurations, networkConfig) } - for _, nw := range driver.Instance.Networks { + + for i, nw := range driver.Instance.Networks { if nw.VZNAT != nil && *nw.VZNAT { attachment, err := vz.NewNATNetworkDeviceAttachment() if err != nil { return err } - networkConfig, err = newVirtioNetworkDeviceConfiguration(attachment, nw.MACAddress) + networkConfig, err := newVirtioNetworkDeviceConfiguration(attachment, nw.MACAddress) if err != nil { return err } configurations = append(configurations, networkConfig) - } - - if nw.Lima != "" { + } else if nw.Lima != "" { nwCfg, err := networks.Config() if err != nil { return err } - socketVMNetOk, err := nwCfg.IsDaemonInstalled(networks.SocketVMNet) + isUsernet, err := nwCfg.Usernet(nw.Lima) if err != nil { return err } - if socketVMNetOk { - logrus.Debugf("Using socketVMNet (%q)", nwCfg.Paths.SocketVMNet) - sock, err := networks.Sock(nw.Lima) + if isUsernet { + if i == firstUsernetIndex { + continue + } + vzSock, err := usernet.Sock(nw.Lima, usernet.FDSock) if err != nil { return err } - - clientFile, err := DialQemu(sock) + clientFile, err := PassFDToUnix(vzSock) if err != nil { return err } - attachment, err := vz.NewFileHandleNetworkDeviceAttachment(clientFile) + networkConfig, err := newVirtioFileNetworkDeviceConfiguration(clientFile, nw.MACAddress) if err != nil { return err } - networkConfig, err = newVirtioNetworkDeviceConfiguration(attachment, nw.MACAddress) + configurations = append(configurations, networkConfig) + } else { + if runtime.GOOS != "darwin" { + return fmt.Errorf("networks.yaml '%s' configuration is only supported on macOS right now", nw.Lima) + } + socketVMNetOk, err := nwCfg.IsDaemonInstalled(networks.SocketVMNet) if err != nil { return err } - configurations = append(configurations, networkConfig) + if socketVMNetOk { + logrus.Debugf("Using socketVMNet (%q)", nwCfg.Paths.SocketVMNet) + sock, err := networks.Sock(nw.Lima) + if err != nil { + return err + } + + clientFile, err := DialQemu(sock) + if err != nil { + return err + } + networkConfig, err := newVirtioFileNetworkDeviceConfiguration(clientFile, nw.MACAddress) + if err != nil { + return err + } + configurations = append(configurations, networkConfig) + } } } else if nw.Socket != "" { clientFile, err := DialQemu(nw.Socket) if err != nil { return err } - attachment, err := vz.NewFileHandleNetworkDeviceAttachment(clientFile) - if err != nil { - return err - } - networkConfig, err = newVirtioNetworkDeviceConfiguration(attachment, nw.MACAddress) + networkConfig, err := newVirtioFileNetworkDeviceConfiguration(clientFile, nw.MACAddress) if err != nil { return err }