Skip to content

Commit c7f36d0

Browse files
bradfitzbradfitz
committed
dashboard, env, cmd/buildlet/testssh: fix gomote ssh for a number of buidlers
And add a testssh tool to validate that SSH is working. Updates golang/go#32430 Change-Id: I5182419fa4db31b598f7a02412fb4ecc4060a796 Reviewed-on: https://go-review.googlesource.com/c/build/+/207459 Reviewed-by: Bryan C. Mills <[email protected]>
1 parent f0d0eff commit c7f36d0

File tree

10 files changed

+242
-30
lines changed

10 files changed

+242
-30
lines changed

‎cmd/buildlet/buildlet.go

Lines changed: 38 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1337,14 +1337,27 @@ func handleConnectSSH(w http.ResponseWriter, r *http.Request) {
13371337
}
13381338
}
13391339

1340-
sshConn, err := net.Dial("tcp", "localhost:"+sshPort())
1341-
if err != nil {
1342-
sshServerOnce.Do(startSSHServer)
1340+
sshServerOnce.Do(startSSHServer)
1341+
1342+
var sshConn net.Conn
1343+
var err error
1344+
1345+
// In theory we shouldn't need retries here at all, but the
1346+
// startSSHServerLinux's use of sshd -D is kinda sketchy and
1347+
// restarts the process whenever we connect to it, so in case
1348+
// it's just down between restarts, try a few times. 5 tries
1349+
// and 5 seconds seems plenty.
1350+
const maxTries = 5
1351+
for try := 1; try <= maxTries; try++ {
13431352
sshConn, err = net.Dial("tcp", "localhost:"+sshPort())
1344-
if err != nil {
1353+
if err == nil {
1354+
break
1355+
}
1356+
if try == maxTries {
13451357
http.Error(w, err.Error(), http.StatusBadGateway)
13461358
return
13471359
}
1360+
time.Sleep(time.Second)
13481361
}
13491362
defer sshConn.Close()
13501363
hj, ok := w.(http.Hijacker)
@@ -1450,13 +1463,27 @@ func startSSHServerLinux() {
14501463
}
14511464
}
14521465

1453-
cmd := exec.Command("/usr/sbin/sshd", "-D", "-p", sshPort())
1454-
err := cmd.Start()
1455-
if err != nil {
1456-
log.Printf("starting sshd: %v", err)
1457-
return
1458-
}
1459-
log.Printf("sshd started.")
1466+
go func() {
1467+
for {
1468+
// TODO: using sshd -D isn't great as it only
1469+
// handles a single connection and exits.
1470+
// Maybe run in sshd -i (inetd) mode instead,
1471+
// and hook that up to the buildlet directly?
1472+
t0 := time.Now()
1473+
cmd := exec.Command("/usr/sbin/sshd", "-D", "-p", sshPort(), "-d", "-d")
1474+
cmd.Stderr = os.Stderr
1475+
err := cmd.Start()
1476+
if err != nil {
1477+
log.Printf("starting sshd: %v", err)
1478+
return
1479+
}
1480+
log.Printf("sshd started.")
1481+
log.Printf("sshd exited: %v; restarting", cmd.Wait())
1482+
if d := time.Since(t0); d < time.Second {
1483+
time.Sleep(time.Second - d)
1484+
}
1485+
}
1486+
}()
14601487
waitLocalSSH()
14611488
}
14621489

‎cmd/buildlet/stage0/Dockerfile

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
# Use of this source code is governed by a BSD-style
33
# license that can be found in the LICENSE file.
44

5-
FROM golang:1.12 AS build
5+
FROM golang:1.13 AS build
66
LABEL maintainer "[email protected]"
77

88
ENV GO111MODULE=on
@@ -24,6 +24,8 @@ COPY . /go/src/golang.org/x/build/
2424

2525
# Install binary to /go/bin/stage0
2626
RUN go install golang.org/x/build/cmd/buildlet/stage0
27+
RUN CGO_ENABLED=0 go build -o /go/bin/stage0.static golang.org/x/build/cmd/buildlet/stage0
2728

28-
FROM golang:1.12
29+
FROM golang:1.13
2930
COPY --from=build /go/bin/stage0 /go/bin/stage0
31+
COPY --from=build /go/bin/stage0.static /go/bin/stage0.static

‎cmd/buildlet/testssh/testssh.go

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
// Copyright 2019 The Go Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style
3+
// license that can be found in the LICENSE file.
4+
5+
// The testssh binary exists to verify that a buildlet container's
6+
// ssh works, without running the whole coordinator binary in the
7+
// staging environment.
8+
package main
9+
10+
import (
11+
"bytes"
12+
"flag"
13+
"fmt"
14+
"io"
15+
"io/ioutil"
16+
"log"
17+
"net"
18+
"os"
19+
"os/exec"
20+
"path/filepath"
21+
"strings"
22+
"time"
23+
24+
"golang.org/x/build/buildenv"
25+
"golang.org/x/build/buildlet"
26+
)
27+
28+
var (
29+
container = flag.String("container", "", "if non-empty, the ID of a running docker container")
30+
startImage = flag.String("start-image", "", "if non-empty, the Docker image to start a buildlet of locally, and use its container ID for the -container value")
31+
user = flag.String("user", "root", "SSH user")
32+
)
33+
34+
func main() {
35+
flag.Parse()
36+
ipPort := getIPPort()
37+
defer cleanContainer()
38+
39+
bc := buildlet.NewClient(ipPort, buildlet.NoKeyPair)
40+
for {
41+
c, err := net.Dial("tcp", ipPort)
42+
if err == nil {
43+
c.Close()
44+
break
45+
}
46+
log.Printf("waiting for %v to come up...", ipPort)
47+
time.Sleep(time.Second)
48+
}
49+
50+
pubKey, privPath := genKey()
51+
52+
log.Printf("hitting buildlet's /connect-ssh ...")
53+
buildletConn, err := bc.ConnectSSH(*user, pubKey)
54+
if err != nil {
55+
var out []byte
56+
if *container != "" {
57+
var err error
58+
out, err = exec.Command("docker", "logs", *container).CombinedOutput()
59+
if err != nil {
60+
log.Printf("failed to fetch docker logs: %v", err)
61+
}
62+
}
63+
cleanContainer()
64+
log.Printf("image logs: %s", out)
65+
log.Fatalf("ConnectSSH: %v (logs above)", err)
66+
}
67+
defer buildletConn.Close()
68+
log.Printf("ConnectSSH succeeded; testing connection...")
69+
70+
ln, err := net.Listen("tcp", "localhost:0")
71+
if err != nil {
72+
log.Fatal(err)
73+
}
74+
go func() {
75+
c, err := ln.Accept()
76+
if err != nil {
77+
log.Fatal(err)
78+
}
79+
go io.Copy(buildletConn, c)
80+
go io.Copy(c, buildletConn)
81+
}()
82+
ip, port, err := net.SplitHostPort(ln.Addr().String())
83+
if err != nil {
84+
log.Fatal(err)
85+
}
86+
87+
cmd := exec.Command("ssh",
88+
"-v",
89+
"-i", privPath,
90+
"-o", "StrictHostKeyChecking=no",
91+
"-o", "UserKnownHostsFile=/dev/null",
92+
"-o", "LogLevel=ERROR",
93+
"-p", port,
94+
*user+"@"+ip,
95+
"echo", "SSH works")
96+
stdout := new(bytes.Buffer)
97+
stderr := new(bytes.Buffer)
98+
cmd.Stderr = stderr
99+
cmd.Stdout = stdout
100+
if err := cmd.Run(); err != nil {
101+
cleanContainer()
102+
log.Fatalf("ssh client: %v, %s", err, stderr)
103+
}
104+
fmt.Print(stdout.String())
105+
}
106+
107+
func cleanContainer() {
108+
if *startImage == "" {
109+
return
110+
}
111+
out, err := exec.Command("docker", "rm", "-f", *container).CombinedOutput()
112+
if err != nil {
113+
log.Printf("docker rm: %v, %s", err, out)
114+
}
115+
}
116+
117+
func genKey() (pubKey, privateKeyPath string) {
118+
cache, err := os.UserCacheDir()
119+
if err != nil {
120+
log.Fatal(err)
121+
}
122+
cache = filepath.Join(cache, "testssh")
123+
os.MkdirAll(cache, 0755)
124+
privateKeyPath = filepath.Join(cache, "testkey")
125+
pubKeyPath := filepath.Join(cache, "testkey.pub")
126+
if _, err := os.Stat(pubKeyPath); err != nil {
127+
out, err := exec.Command("ssh-keygen", "-t", "ed25519", "-f", privateKeyPath, "-N", "").CombinedOutput()
128+
if err != nil {
129+
log.Fatalf("ssh-keygen: %v, %s", err, out)
130+
}
131+
}
132+
slurp, err := ioutil.ReadFile(pubKeyPath)
133+
if err != nil {
134+
log.Fatal(err)
135+
}
136+
return strings.TrimSpace(string(slurp)), privateKeyPath
137+
}
138+
139+
func getIPPort() string {
140+
if *startImage != "" {
141+
buildlet := "buildlet.linux-amd64"
142+
if strings.Contains(*startImage, "linux-x86-alpine") {
143+
buildlet = "buildlet.linux-amd64-static"
144+
}
145+
log.Printf("creating container with image %s ...", *startImage)
146+
out, err := exec.Command("docker", "run", "-d",
147+
"--stop-timeout=300",
148+
"-e", "META_BUILDLET_BINARY_URL=https://storage.googleapis.com/"+buildenv.Production.BuildletBucket+"/"+buildlet,
149+
*startImage).CombinedOutput()
150+
if err != nil {
151+
log.Fatalf("docker run: %v, %s", err, out)
152+
}
153+
*container = strings.TrimSpace(string(out))
154+
log.Printf("created container %s ...", *container)
155+
}
156+
if *container != "" {
157+
out, err := exec.Command("bash", "-c", "docker inspect "+*container+" | jq -r '.[0].NetworkSettings.IPAddress'").CombinedOutput()
158+
if err != nil {
159+
log.Fatalf("%v: %s", err, out)
160+
}
161+
return strings.TrimSpace(string(out)) + ":80"
162+
}
163+
log.Fatalf("no address specified")
164+
return ""
165+
}

‎cmd/xb/xb.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,7 @@ func runDocker() {
130130
for _, layer := range layers {
131131
if strings.HasPrefix(layer, "golang:") ||
132132
strings.HasPrefix(layer, "debian:") ||
133+
strings.HasPrefix(layer, "alpine:") ||
133134
strings.HasPrefix(layer, "fedora:") {
134135
continue
135136
}

‎dashboard/builders.go

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -167,6 +167,7 @@ var Hosts = map[string]*HostConfig{
167167
ContainerImage: "js-wasm:latest",
168168
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.linux-amd64",
169169
env: []string{"GOROOT_BOOTSTRAP=/go1.4"},
170+
SSHUsername: "root",
170171
},
171172
"host-s390x-cross-kube": &HostConfig{
172173
Notes: "Container with s390x cross-compiler.",
@@ -177,26 +178,30 @@ var Hosts = map[string]*HostConfig{
177178
"host-linux-x86-alpine": &HostConfig{
178179
Notes: "Alpine container",
179180
ContainerImage: "linux-x86-alpine:latest",
180-
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.linux-amd64-static",
181+
buildletURLTmpl: "https://storage.googleapis.com/$BUCKET/buildlet.linux-amd64-static",
181182
env: []string{"GOROOT_BOOTSTRAP=/usr/lib/go"},
183+
SSHUsername: "root",
182184
},
183185
"host-linux-clang": &HostConfig{
184186
Notes: "Container with clang.",
185187
ContainerImage: "linux-x86-clang:latest",
186188
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.linux-amd64",
187189
env: []string{"GOROOT_BOOTSTRAP=/go1.4"},
190+
SSHUsername: "root",
188191
},
189192
"host-linux-sid": &HostConfig{
190193
Notes: "Debian sid, updated occasionally.",
191194
ContainerImage: "linux-x86-sid:latest",
192195
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.linux-amd64",
193196
env: []string{"GOROOT_BOOTSTRAP=/go1.4"},
197+
SSHUsername: "root",
194198
},
195199
"host-linux-fedora": &HostConfig{
196200
Notes: "Fedora 30",
197201
ContainerImage: "linux-x86-fedora:latest",
198202
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.linux-amd64",
199203
env: []string{"GOROOT_BOOTSTRAP=/goboot"},
204+
SSHUsername: "root",
200205
},
201206
"host-linux-arm-scaleway": &HostConfig{
202207
IsReverse: true,

‎env/linux-x86-alpine/Dockerfile

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88

99
FROM golang/buildlet-stage0 AS stage0
1010

11-
FROM alpine:3.5
11+
FROM alpine:3.10
1212
MAINTAINER golang-dev <[email protected]>
1313

1414
RUN apk add --no-cache \
@@ -24,10 +24,15 @@ RUN apk add --no-cache \
2424
go \
2525
libc-dev \
2626
lsof \
27+
openssh \
2728
procps \
2829
strace
2930

30-
COPY --from=stage0 /go/bin/stage0 /usr/local/bin/stage0
31+
RUN ssh-keygen -A
32+
RUN bash -c "(echo ChallengeResponseAuthentication no; echo PasswordAuthentication no; echo PermitRootLogin yes) > /etc/ssh/sshd_config"
33+
RUN passwd -u root
34+
35+
COPY --from=stage0 /go/bin/stage0.static /usr/local/bin/stage0
3136

3237
ENV GOROOT_BOOTSTRAP=/usr/lib/go
3338

‎env/linux-x86-clang/Dockerfile

Lines changed: 7 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,39 +2,35 @@
22
# Use of this source code is governed by a BSD-style
33
# license that can be found in the LICENSE file.
44

5-
# Linux builder VM with clang instead of gccc.
5+
# Linux builder VM with clang instead of gcc.
66
# Docker tag gobuilders/linux-x86-clang
77

88
FROM golang/buildlet-stage0 AS stage0
99

10-
FROM debian:jessie
10+
FROM debian:buster
1111
MAINTAINER golang-dev <[email protected]>
1212

1313
ENV DEBIAN_FRONTEND noninteractive
1414

15-
COPY sources/clang-deps.list /etc/apt/sources.list.d/
16-
COPY llvm-snapshot.gpg.key /tmp/
17-
18-
RUN apt-key add /tmp/llvm-snapshot.gpg.key
19-
2015
# strace: optionally used by some net/http tests
2116
# libc6-dev-i386 gcc-multilib: for 32-bit builds
2217
# procps lsof psmisc: misc basic tools
2318
RUN apt-get update && apt-get install -y \
2419
--no-install-recommends \
2520
ca-certificates \
2621
curl \
27-
clang-3.9 \
22+
clang \
2823
strace \
2924
libc6-dev-i386 \
3025
gcc-multilib \
3126
procps \
3227
lsof \
3328
psmisc \
29+
openssh-server \
3430
&& rm -rf /var/lib/apt/lists/* \
35-
&& rm -f /usr/bin/gcc \
36-
&& ln -snf /usr/bin/clang-3.9 /usr/bin/clang \
37-
&& ln -snf /usr/bin/clang++-3.9 /usr/bin/clang++
31+
&& rm -f /usr/bin/gcc
32+
33+
RUN mkdir --mode=0700 /run/sshd
3834

3935
RUN mkdir -p /go1.4-amd64 \
4036
&& ( \

‎env/linux-x86-fedora/Dockerfile

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,14 @@ RUN yum -y update && yum -y install \
1515
patch \
1616
strace \
1717
which \
18+
openssh-server \
1819
&& true
1920

21+
RUN ssh-keygen -A
22+
2023
RUN mkdir -p /goboot-amd64 \
2124
&& ( \
22-
curl --silent https://storage.googleapis.com/golang/go1.12.5.linux-amd64.tar.gz | tar -C /goboot-amd64 -zxv \
25+
curl --silent https://storage.googleapis.com/golang/go1.13.4.linux-amd64.tar.gz | tar -C /goboot-amd64 -zxv \
2326
) \
2427
&& mv /goboot-amd64/go /goboot \
2528
&& rm -rf /goboot-amd64 \

0 commit comments

Comments
 (0)