devapp, status: start of status handler for monitoring

Also bump Go from 1.8 to 1.10, and change how the static binary is
built to avoid warnings during link.

Updates golang/go#21315
Updates golang/go#22603

Change-Id: I426491d48f787a77cb3eea4dff4d11f474236548
Reviewed-on: https://go-review.googlesource.com/96416
Reviewed-by: Andrew Bonventre <[email protected]>

Author: bradfitz

devapp/Dockerfile.0

@@ -1,9 +1,12 @@
 # Copyright 2017 The Go Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style
 # license that can be found in the LICENSE file.
-FROM golang:1.8
+
+FROM golang:1.10
 LABEL maintainer "[email protected]"
 
+ENV CGO_ENABLED=0
+
 # BEGIN deps (run `make update-deps` to update)
 
 # Repo cloud.google.com/go at ba25346 (2017-10-13)
@@ -152,4 +155,4 @@ RUN go install cloud.google.com/go/compute/metadata \
 
 COPY . /go/src/golang.org/x/build/
 
-RUN go install -ldflags "-linkmode=external -extldflags '-static -pthread'" golang.org/x/build/devapp
+RUN go install golang.org/x/build/devapp

devapp/devapp.go

@@ -62,7 +62,9 @@ func main() {
 		go func() {
 			handler := http.Handler(s)
 			if *autocertBucket != "" {
-				handler = http.HandlerFunc(redirectHTTP)
+				handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+					redirectHTTP(w, r, s)
+				})
 			}
 			errc <- fmt.Errorf("http.Serve = %v", http.Serve(ln, handler))
 		}()
@@ -77,11 +79,19 @@ func main() {
 	log.Fatal(<-errc)
 }
 
-func redirectHTTP(w http.ResponseWriter, r *http.Request) {
+func redirectHTTP(w http.ResponseWriter, r *http.Request, h http.Handler) {
 	if r.TLS != nil || r.Host == "" {
 		http.NotFound(w, r)
 		return
 	}
+
+	// Serve /status directly, without a forced redirect.
+	// This lets monitoring avoid a dependency on LetsEncrypt being up.
+	if strings.HasPrefix(r.RequestURI, "/status") {
+		h.ServeHTTP(w, r)
+		return
+	}
+
 	http.Redirect(w, r, "https://"+r.Host+r.RequestURI, http.StatusFound)
 }
 

devapp/server.go

@@ -19,6 +19,7 @@ import (
 
 	"golang.org/x/build/maintner"
 	"golang.org/x/build/maintner/godata"
+	"golang.org/x/build/status/statusserver"
 )
 
 // A server is an http.Handler that serves content within staticDir at root and
@@ -62,6 +63,10 @@ func newServer(mux *http.ServeMux, staticDir, templateDir string) *server {
 	s.mux.HandleFunc("/release", s.withTemplate("/release.tmpl", s.handleRelease))
 	s.mux.HandleFunc("/reviews", s.withTemplate("/reviews.tmpl", s.handleReviews))
 	s.mux.HandleFunc("/dir/", handleDirRedirect)
+
+	ss := statusserver.NewHandler()
+	ss.Register(s.mux) // at /status
+
 	for _, p := range []string{"/imfeelinghelpful", "/imfeelinglucky"} {
 		s.mux.HandleFunc(p, s.handleRandomHelpWantedIssue)
 	}

status/status.go

@@ -0,0 +1,50 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package status contains code for monitoring the golang.org infrastructure.
+// It is not intended for use outside of the Go project.
+package status
+
+import (
+	"os"
+	"sync"
+	"time"
+
+	"cloud.google.com/go/compute/metadata"
+)
+
+var (
+	statusTokenMu  sync.Mutex
+	statusTokenVal string
+	statusTokenErr error
+	lastCheck      time.Time
+)
+
+// UpdateToken returns the value of "monitor-probe-token" from the GCE
+// metadata server, or from the environment variable
+// $DEV_STATUS_UPDATE_TOKEN. This is the the shared secret that must
+// be sent to dev.golang.org in prober status update requests.
+// Most probers will not use this directly.
+func UpdateToken() (string, error) {
+	statusTokenMu.Lock()
+	defer statusTokenMu.Unlock()
+	if statusTokenVal != "" {
+		return statusTokenVal, nil
+	}
+	if lastCheck.After(time.Now().Add(-10 * time.Second)) {
+		return "", statusTokenErr
+	}
+	if v := os.Getenv("DEV_STATUS_UPDATE_TOKEN"); v != "" {
+		statusTokenVal = v
+		return v, nil
+	}
+	v, err := metadata.Get("monitor-probe-token")
+	if err == nil {
+		statusTokenVal = v
+		return v, nil
+	}
+	lastCheck = time.Now()
+	statusTokenErr = err
+	return "", err
+}

status/statusserver/statusserver.go

@@ -0,0 +1,136 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package statusserver contains the status server for the golang.org
+// properties and build infrastructure.
+package statusserver
+
+import (
+	"io"
+	"log"
+	"net/http"
+	"sort"
+	"sync"
+	"time"
+
+	"golang.org/x/build/status"
+)
+
+// Handler is the status server handler.
+// It expects to be mounted at /status.
+type Handler struct {
+	mu          sync.Mutex
+	items       map[string]*monItem
+	sortedItems []string // sorted keys of the items map
+}
+
+func NewHandler() *Handler {
+	h := &Handler{
+		items: map[string]*monItem{
+			"pubsubhelper": &monItem{
+				goodFor: 10 * time.Minute,
+			},
+		},
+	}
+	h.updateSorted()
+	return h
+}
+
+// Register registers h in mux at /status.
+func (h *Handler) Register(mux *http.ServeMux) {
+	mux.Handle("/status", mux)
+}
+
+// requires h.mu is held
+func (h *Handler) getItemLocked(name string) *monItem {
+	mi, ok := h.items[name]
+	if !ok {
+		mi = new(monItem)
+		mi.goodFor = 10 * time.Minute // default
+		h.items[name] = mi
+		h.updateSorted()
+	}
+	return mi
+}
+
+// requires h.mu is held
+func (h *Handler) updateSorted() {
+	h.sortedItems = nil
+	for name := range h.items {
+		h.sortedItems = append(h.sortedItems, name)
+	}
+	sort.Strings(h.sortedItems)
+}
+
+// monItem is an item that is monitored. It can be monitored
+// externally from the server hosting this handler, or it can be
+// self-reported, in which case it'll come in via /status/update.
+//
+// All state is guarded by h.mu.
+type monItem struct {
+	goodFor time.Duration
+
+	lastUpdate time.Time
+	ok         bool
+	errText    string
+	warn       string
+}
+
+func (mi *monItem) updateFromRequest(r *http.Request) {
+	// TODO: record state changes over time, so we have a history of the last
+	// N transitions.
+
+	mi.lastUpdate = time.Now()
+	if s := r.FormValue("state"); s == "ok" {
+		mi.ok = true
+		mi.errText = ""
+	} else {
+		mi.ok = false
+		mi.errText = s
+	}
+	mi.warn = r.FormValue("warn")
+}
+
+func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	path := r.URL.Path
+	if path == "/status/update" {
+		h.handleUpdate(w, r)
+		return
+	}
+	io.WriteString(w, "ok\n")
+}
+
+func (h *Handler) handleUpdate(w http.ResponseWriter, r *http.Request) {
+	switch {
+	case r.TLS == nil:
+		http.Error(w, "request requires TLS", http.StatusBadRequest)
+		return
+	case r.Method != "POST":
+		http.Error(w, "request requires POST", http.StatusBadRequest)
+		return
+	case r.Header.Get("Content-Type") != "application/x-www-form-urlencoded":
+		http.Error(w, "request requires Content-Type x-www-form-urlencoded", http.StatusBadRequest)
+		return
+	case r.ContentLength > 0 && r.ContentLength <= 1<<20:
+		http.Error(w, "request requires explicit Content-Length under 1MB", http.StatusBadRequest)
+		return
+	}
+	wantToken, err := status.UpdateToken()
+	if err != nil {
+		log.Printf("error: status.UpdateToken: %v", err)
+		http.Error(w, "failed to get status update token to validate against", 500)
+		return
+	}
+	if r.Header.Get("X-Status-Update-Token") != wantToken {
+		http.Error(w, "invalid X-Status-Update-Token value", http.StatusUnauthorized)
+		return
+	}
+	name := r.FormValue("name")
+
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	mi := h.getItemLocked(name)
+	mi.updateFromRequest(r)
+	w.WriteHeader(http.StatusNoContent)
+}