cmd/makemac: add start of command/package to create OS X builder VMs

Currently it's just a package main. It might split in two later. It
might stop shelling out to govc and use the underlying API directly
too, but it's hairy.

Easier to work like this for now.

Updates golang/go#9495

Change-Id: I0d2e19abcb5114ab7fe2e2c543d14e50897d4cbb
Reviewed-on: https://go-review.googlesource.com/28584
Reviewed-by: Quentin Smith <[email protected]>

Author: bradfitz

cmd/makemac/makemac.go

@@ -0,0 +1,315 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+/*
+The makemac command starts OS X VMs for the builders.
+It is currently just a thin wrapper around govc.
+
+See https://github.com/vmware/govmomi/tree/master/govc
+
+Usage:
+
+  $ makemac <osx_minor_version>  # e.g, 8, 9, 10, 11
+
+*/
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+)
+
+func usage() {
+	fmt.Fprintf(os.Stderr, "Usage: makemac <osx_minor_version>\n")
+	os.Exit(1)
+}
+
+var flagStatus = flag.Bool("status", false, "print status only")
+
+func main() {
+	flag.Parse()
+	if !(flag.NArg() == 1 || (*flagStatus && flag.NArg() == 0)) {
+		usage()
+	}
+	minor, err := strconv.Atoi(flag.Arg(0))
+	if err != nil && !*flagStatus {
+		usage()
+	}
+
+	ctx := context.Background()
+	state, err := getState(ctx)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if *flagStatus {
+		stj, _ := json.MarshalIndent(state, "", "  ")
+		fmt.Printf("%s\n", stj)
+		return
+	}
+
+	err = state.CreateMac(ctx, minor)
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+// State is the state of the world.
+type State struct {
+	mu sync.Mutex `json:"-"`
+
+	Hosts  map[string]int    // IP address -> running Mac VM count (including 0)
+	VMHost map[string]string // "mac_10_8_host2b" => "10.0.0.0"
+	HostIP map[string]string // "host-5" -> "10.0.0.0"
+}
+
+// CreateMac creates an Mac VM running OS X 10.<minor>.
+func (st *State) CreateMac(ctx context.Context, minor int) (err error) {
+	// TODO(bradfitz): return VM name, update state, etc.
+
+	st.mu.Lock()
+	defer st.mu.Unlock()
+
+	var guestType string
+	switch minor {
+	case 8:
+		guestType = "darwin12_64Guest"
+	case 9:
+		guestType = "darwin13_64Guest"
+	case 10, 11:
+		guestType = "darwin14_64Guest"
+	default:
+		return fmt.Errorf("unsupported makemac minor OS X version %d", minor)
+	}
+
+	builderType := fmt.Sprintf("darwin-amd64-10_%d", minor)
+	key, err := ioutil.ReadFile(filepath.Join(os.Getenv("HOME"), "keys", builderType))
+	if err != nil {
+		return err
+	}
+
+	// Find the top-level datastore directory hosting the vmdk COW disk for
+	// the linked clone. This is usually named "osx_9_frozen", but may be named
+	// with a "_1", "_2", etc suffix. Search for it.
+	netAppDir, err := findFrozenDir(ctx, minor)
+	if err != nil {
+		return fmt.Errorf("failed to find osx_%d_frozen base directory: %v", minor, err)
+	}
+
+	hostNum, hostWhich, err := st.pickHost()
+	if err != nil {
+		return err
+	}
+	name := fmt.Sprintf("mac_10_%v_host%d%s", minor, hostNum, hostWhich)
+
+	if err := govc(ctx, "vm.create",
+		"-m", "4096",
+		"-c", "6",
+		"-on=false",
+		"-net", "dvPortGroup-Private", // 10.50.0.0/16
+		"-g", guestType,
+		// Put the config on the host's datastore, which
+		// forces the VM to run on that host:
+		"-ds", fmt.Sprintf("BOOT_%d", hostNum),
+		name,
+	); err != nil {
+		return err
+	}
+	defer func() {
+		if err != nil {
+			err := govc(ctx, "vm.destroy", name)
+			if err != nil {
+				log.Printf("failed to destroy %v: %v", name, err)
+			}
+		}
+	}()
+
+	if err := govc(ctx, "vm.change",
+		"-e", "smc.present=TRUE",
+		"-e", "ich7m.present=TRUE",
+		"-e", "firmware=efi",
+		"-e", fmt.Sprintf("guestinfo.key-%s=%s", builderType, strings.TrimSpace(string(key))),
+		"-e", "guestinfo.name="+name,
+		"-vm", name,
+	); err != nil {
+		return err
+	}
+
+	if err := govc(ctx, "device.usb.add", "-vm", name); err != nil {
+		return err
+	}
+
+	if err := govc(ctx, "vm.disk.attach",
+		"-vm", name,
+		"-link=true",
+		"-persist=false",
+		"-ds=NetApp-1",
+		"-disk", fmt.Sprintf("%s/osx_%d_frozen.vmdk", netAppDir, minor),
+	); err != nil {
+		return err
+	}
+
+	if err := govc(ctx, "vm.power", "-on", name); err != nil {
+		return err
+	}
+	log.Printf("Success.")
+	return nil
+}
+
+// govc runs "govc <args...>" and ignores its output, unless there's an error.
+func govc(ctx context.Context, args ...string) error {
+	fmt.Fprintf(os.Stderr, "$ govc %v\n", strings.Join(args, " "))
+	out, err := exec.CommandContext(ctx, "govc", args...).CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("govc %s ...: %v, %s", args[0], err, out)
+	}
+	return nil
+}
+
+const hostIPPrefix = "10.88.203." // with fourth octet starting at 10
+
+// st.mu must be held.
+func (st *State) pickHost() (hostNum int, hostWhich string, err error) {
+	for ip, inUse := range st.Hosts {
+		if !strings.HasPrefix(ip, hostIPPrefix) {
+			continue
+		}
+		if inUse >= 2 {
+			// Apple policy.
+			continue
+		}
+		hostNum, err = strconv.Atoi(strings.TrimPrefix(ip, hostIPPrefix))
+		if err != nil {
+			return 0, "", err
+		}
+		hostNum -= 10   // 10.88.203.11 is "BOOT_1" datastore.
+		hostWhich = "a" // unless in use
+		if st.whichAInUse(hostNum) {
+			hostWhich = "b"
+		}
+		return
+	}
+	return 0, "", errors.New("no usable host found")
+}
+
+// whichAInUse reports whether a VM is running on the provided hostNum named
+// with suffix "_host<n>a".
+//
+// st.mu must be held
+func (st *State) whichAInUse(hostNum int) bool {
+	suffix := fmt.Sprintf("_host%da", hostNum)
+	for name := range st.VMHost {
+		if strings.HasSuffix(name, suffix) {
+			return true
+		}
+	}
+	return false
+}
+
+// getStat queries govc to find the current state of the hosts and VMs.
+func getState(ctx context.Context) (*State, error) {
+	st := &State{
+		VMHost: make(map[string]string),
+		Hosts:  make(map[string]int),
+		HostIP: make(map[string]string),
+	}
+
+	var hosts elementList
+	if err := govcJSONDecode(ctx, &hosts, "ls", "-json", "/MacStadium-ATL/host/MacMini_Cluster"); err != nil {
+		return nil, fmt.Errorf("Reading /MacStadium-ATL/host/MacMini_Cluster: %v", err)
+	}
+	for _, h := range hosts.Elements {
+		if h.Object.Self.Type == "HostSystem" {
+			ip := path.Base(h.Path)
+			st.Hosts[ip] = 0
+			st.HostIP[h.Object.Self.Value] = ip
+		}
+	}
+
+	var vms elementList
+	if err := govcJSONDecode(ctx, &vms, "ls", "-json", "/MacStadium-ATL/vm"); err != nil {
+		return nil, fmt.Errorf("Reading /MacStadium-ATL/vm: %v", err)
+	}
+	for _, h := range vms.Elements {
+		if h.Object.Self.Type == "VirtualMachine" {
+			name := path.Base(h.Path)
+			hostID := h.Object.Runtime.Host.Value
+			hostIP := st.HostIP[hostID]
+			st.VMHost[name] = hostIP
+			if hostIP != "" && strings.HasPrefix(name, "mac_10_") {
+				st.Hosts[hostIP]++
+			}
+		}
+	}
+
+	return st, nil
+}
+
+// objRef is a VMWare "Managed Object Reference".
+type objRef struct {
+	Type  string // e.g. "VirtualMachine"
+	Value string // e.g. "host-12"
+}
+
+type elementList struct {
+	Elements []*elementJSON `json:"elements"`
+}
+
+type elementJSON struct {
+	Path   string
+	Object struct {
+		Self    objRef
+		Runtime struct {
+			Host objRef // for VMs; not present otherwise
+		}
+	}
+}
+
+// govcJSONDecode runs "govc <args...>" and decodes its JSON output into dst.
+func govcJSONDecode(ctx context.Context, dst interface{}, args ...string) error {
+	cmd := exec.CommandContext(ctx, "govc", args...)
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return err
+	}
+	if err := cmd.Start(); err != nil {
+		return err
+	}
+	err = json.NewDecoder(stdout).Decode(dst)
+	cmd.Process.Kill() // usually unnecessary
+	if werr := cmd.Wait(); werr != nil && err == nil {
+		err = werr
+	}
+	return err
+}
+
+// findFrozenDir returns the name of the top-level directory on the
+// NetApp-1 shared datastore containing a directory starting with
+// "osx_<minor>_frozen". It might be that just that, or have a suffix
+// like "_1" or "_2".
+func findFrozenDir(ctx context.Context, minor int) (string, error) {
+	out, err := exec.CommandContext(ctx, "govc", "datastore.ls", "-ds=NetApp-1").Output()
+	if err != nil {
+		return "", err
+	}
+	prefix := fmt.Sprintf("osx_%d_frozen", minor)
+	for _, dir := range strings.Fields(string(out)) {
+		if strings.HasPrefix(dir, prefix) {
+			return dir, nil
+		}
+	}
+	return "", os.ErrNotExist
+}