Skip to content

Commit 1635205

Browse files
committed
cmd/dist: add -distpack flag to build distribution archives
We want to enable others to reproduce the exact distribution archives we are serving on go.dev/dl. Today the code for building those archives lives in golang.org/x/build, which is fundamentally tied to running on the Go team build infrastructure and not easy for others to run. This CL adds a new flag -distpack to cmd/dist, usually invoked as make.bash -distpack, to build the distribution archives using code in the main repository that anyone can run. Starting in Go 1.21, the Go team build infrastructure will run this instead of its current custom code to build those archives. The current builds are not reproducible even given identical infrastructure, because the archives are stamped with the current time. It is helpful to have a timestamp in the archives indicating when the code is from, but that time needs to be reproducible. To ensure this, the new -distpack flag extends the VERSION file to include a time stamp, which it uses as the modification time for all files in the archive. The new -distpack flag is implemented by a separate program, cmd/distpack, instead of being in cmd/dist, so that it can be compiled by the toolchain being distributed and not the bootstrap toolchain. Otherwise details like the exact compression algorithms might vary from one bootstrap toolchain to another and produce non-reproducible builds. So there is a new 'go tool distpack', but it's omitted from the distributions themselves, just as 'go tool dist' is. make.bash already accepts any flags for cmd/dist, including -distpack. make.bat is less sophisticated and looks for each known flag, so this CL adds an update to look for -distpack. The CL also changes make.bat to accept the idiomatic Go -flagname in addition to the non-idiomatic (for Go) --flagname. Previously it insisted on the --flag form. I have confirmed that using make.bash -distpack produces the identical distribution archives for windows/amd64, linux/amd64, darwin/amd64, and darwin/arm64 whether it is run on windows/amd64, linux/amd64, or darwin/amd64 hosts. For #24904. Change-Id: Ie6d69365ee3d7294d05b4f96ffb9159b41918074 Reviewed-on: https://go-review.googlesource.com/c/go/+/470676 TryBot-Result: Gopher Robot <[email protected]> Reviewed-by: Heschi Kreinick <[email protected]> Run-TryBot: Russ Cox <[email protected]> Reviewed-by: Carlos Amedee <[email protected]>
1 parent 2fcca5d commit 1635205

File tree

6 files changed

+849
-9
lines changed

6 files changed

+849
-9
lines changed

src/cmd/dist/build.go

+39-1
Original file line numberDiff line numberDiff line change
@@ -351,12 +351,40 @@ func chomp(s string) string {
351351
}
352352

353353
// findgoversion determines the Go version to use in the version string.
354+
// It also parses any other metadata found in the version file.
354355
func findgoversion() string {
355356
// The $GOROOT/VERSION file takes priority, for distributions
356357
// without the source repo.
357358
path := pathf("%s/VERSION", goroot)
358359
if isfile(path) {
359360
b := chomp(readfile(path))
361+
362+
// Starting in Go 1.21 the VERSION file starts with the
363+
// version on a line by itself but then can contain other
364+
// metadata about the release, one item per line.
365+
if i := strings.Index(b, "\n"); i >= 0 {
366+
rest := b[i+1:]
367+
b = chomp(b[:i])
368+
for _, line := range strings.Split(rest, "\n") {
369+
f := strings.Fields(line)
370+
if len(f) == 0 {
371+
continue
372+
}
373+
switch f[0] {
374+
default:
375+
fatalf("VERSION: unexpected line: %s", line)
376+
case "time":
377+
if len(f) != 2 {
378+
fatalf("VERSION: unexpected time line: %s", line)
379+
}
380+
_, err := time.Parse(time.RFC3339, f[1])
381+
if err != nil {
382+
fatalf("VERSION: bad time: %s", err)
383+
}
384+
}
385+
}
386+
}
387+
360388
// Commands such as "dist version > VERSION" will cause
361389
// the shell to create an empty VERSION file and set dist's
362390
// stdout to its fd. dist in turn looks at VERSION and uses
@@ -591,6 +619,7 @@ func mustLinkExternal(goos, goarch string, cgoEnabled bool) bool {
591619
// exclude files with that prefix.
592620
// Note that this table applies only to the build of cmd/go,
593621
// after the main compiler bootstrap.
622+
// Files listed here should also be listed in ../distpack/pack.go's srcArch.Remove list.
594623
var deptab = []struct {
595624
prefix string // prefix of target
596625
dep []string // dependency tweaks for targets with that prefix
@@ -1206,6 +1235,9 @@ func clean() {
12061235

12071236
// Remove cached version info.
12081237
xremove(pathf("%s/VERSION.cache", goroot))
1238+
1239+
// Remove distribution packages.
1240+
xremoveall(pathf("%s/pkg/distpack", goroot))
12091241
}
12101242
}
12111243

@@ -1347,9 +1379,10 @@ func cmdbootstrap() {
13471379
timelog("start", "dist bootstrap")
13481380
defer timelog("end", "dist bootstrap")
13491381

1350-
var debug, force, noBanner, noClean bool
1382+
var debug, distpack, force, noBanner, noClean bool
13511383
flag.BoolVar(&rebuildall, "a", rebuildall, "rebuild all")
13521384
flag.BoolVar(&debug, "d", debug, "enable debugging of bootstrap process")
1385+
flag.BoolVar(&distpack, "distpack", distpack, "write distribution files to pkg/distpack")
13531386
flag.BoolVar(&force, "force", force, "build even if the port is marked as broken")
13541387
flag.BoolVar(&noBanner, "no-banner", noBanner, "do not print banner")
13551388
flag.BoolVar(&noClean, "no-clean", noClean, "print deprecation warning")
@@ -1592,6 +1625,11 @@ func cmdbootstrap() {
15921625
os.Setenv("CC", oldcc)
15931626
}
15941627

1628+
if distpack {
1629+
xprintf("Packaging archives for %s/%s.\n", goos, goarch)
1630+
run("", ShowOutput|CheckExit, pathf("%s/distpack", tooldir))
1631+
}
1632+
15951633
// Print trailing banner unless instructed otherwise.
15961634
if !noBanner {
15971635
banner()

src/cmd/distpack/archive.go

+197
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,197 @@
1+
// Copyright 2023 The Go Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style
3+
// license that can be found in the LICENSE file.
4+
5+
package main
6+
7+
import (
8+
"io/fs"
9+
"log"
10+
"os"
11+
"path"
12+
"path/filepath"
13+
"sort"
14+
"strings"
15+
"time"
16+
)
17+
18+
// An Archive describes an archive to write: a collection of files.
19+
// Directories are implied by the files and not explicitly listed.
20+
type Archive struct {
21+
Files []File
22+
}
23+
24+
// A File describes a single file to write to an archive.
25+
type File struct {
26+
Name string // name in archive
27+
Time time.Time // modification time
28+
Mode fs.FileMode
29+
Size int64
30+
Src string // source file in OS file system
31+
}
32+
33+
// Info returns a FileInfo about the file, for use with tar.FileInfoHeader
34+
// and zip.FileInfoHeader.
35+
func (f *File) Info() fs.FileInfo {
36+
return fileInfo{f}
37+
}
38+
39+
// A fileInfo is an implementation of fs.FileInfo describing a File.
40+
type fileInfo struct {
41+
f *File
42+
}
43+
44+
func (i fileInfo) Name() string { return path.Base(i.f.Name) }
45+
func (i fileInfo) ModTime() time.Time { return i.f.Time }
46+
func (i fileInfo) Mode() fs.FileMode { return i.f.Mode }
47+
func (i fileInfo) IsDir() bool { return false }
48+
func (i fileInfo) Size() int64 { return i.f.Size }
49+
func (i fileInfo) Sys() any { return nil }
50+
51+
// NewArchive returns a new Archive containing all the files in the directory dir.
52+
// The archive can be amended afterward using methods like Add and Filter.
53+
func NewArchive(dir string) (*Archive, error) {
54+
a := new(Archive)
55+
err := fs.WalkDir(os.DirFS(dir), ".", func(name string, d fs.DirEntry, err error) error {
56+
if err != nil {
57+
return err
58+
}
59+
if d.IsDir() {
60+
return nil
61+
}
62+
info, err := d.Info()
63+
if err != nil {
64+
return err
65+
}
66+
a.Add(name, filepath.Join(dir, name), info)
67+
return nil
68+
})
69+
if err != nil {
70+
return nil, err
71+
}
72+
a.Sort()
73+
return a, nil
74+
}
75+
76+
// Add adds a file with the given name and info to the archive.
77+
// The content of the file comes from the operating system file src.
78+
// After a sequence of one or more calls to Add,
79+
// the caller should invoke Sort to re-sort the archive's files.
80+
func (a *Archive) Add(name, src string, info fs.FileInfo) {
81+
a.Files = append(a.Files, File{
82+
Name: name,
83+
Time: info.ModTime(),
84+
Mode: info.Mode(),
85+
Size: info.Size(),
86+
Src: src,
87+
})
88+
}
89+
90+
// Sort sorts the files in the archive.
91+
// It is only necessary to call Sort after calling Add.
92+
// ArchiveDir returns a sorted archive, and the other methods
93+
// preserve the sorting of the archive.
94+
func (a *Archive) Sort() {
95+
sort.Slice(a.Files, func(i, j int) bool {
96+
return a.Files[i].Name < a.Files[j].Name
97+
})
98+
}
99+
100+
// Clone returns a copy of the Archive.
101+
// Method calls like Add and Filter invoked on the copy do not affect the original,
102+
// nor do calls on the original affect the copy.
103+
func (a *Archive) Clone() *Archive {
104+
b := &Archive{
105+
Files: make([]File, len(a.Files)),
106+
}
107+
copy(b.Files, a.Files)
108+
return b
109+
}
110+
111+
// AddPrefix adds a prefix to all file names in the archive.
112+
func (a *Archive) AddPrefix(prefix string) {
113+
for i := range a.Files {
114+
a.Files[i].Name = path.Join(prefix, a.Files[i].Name)
115+
}
116+
}
117+
118+
// Filter removes files from the archive for which keep(name) returns false.
119+
func (a *Archive) Filter(keep func(name string) bool) {
120+
files := a.Files[:0]
121+
for _, f := range a.Files {
122+
if keep(f.Name) {
123+
files = append(files, f)
124+
}
125+
}
126+
a.Files = files
127+
}
128+
129+
// SetMode changes the mode of every file in the archive
130+
// to be mode(name, m), where m is the file's current mode.
131+
func (a *Archive) SetMode(mode func(name string, m fs.FileMode) fs.FileMode) {
132+
for i := range a.Files {
133+
a.Files[i].Mode = mode(a.Files[i].Name, a.Files[i].Mode)
134+
}
135+
}
136+
137+
// Remove removes files matching any of the patterns from the archive.
138+
// The patterns use the syntax of path.Match, with an extension of allowing
139+
// a leading **/ or trailing /**, which match any number of path elements
140+
// (including no path elements) before or after the main match.
141+
func (a *Archive) Remove(patterns ...string) {
142+
a.Filter(func(name string) bool {
143+
for _, pattern := range patterns {
144+
match, err := amatch(pattern, name)
145+
if err != nil {
146+
log.Fatalf("archive remove: %v", err)
147+
}
148+
if match {
149+
return false
150+
}
151+
}
152+
return true
153+
})
154+
}
155+
156+
// SetTime sets the modification time of all files in the archive to t.
157+
func (a *Archive) SetTime(t time.Time) {
158+
for i := range a.Files {
159+
a.Files[i].Time = t
160+
}
161+
}
162+
163+
func amatch(pattern, name string) (bool, error) {
164+
// firstN returns the prefix of name corresponding to the first n path elements.
165+
// If n <= 0, firstN returns the entire name.
166+
firstN := func(name string, n int) string {
167+
for i := 0; i < len(name); i++ {
168+
if name[i] == '/' {
169+
if n--; n == 0 {
170+
return name[:i]
171+
}
172+
}
173+
}
174+
return name
175+
}
176+
177+
// lastN returns the suffix of name corresponding to the last n path elements.
178+
// If n <= 0, lastN returns the entire name.
179+
lastN := func(name string, n int) string {
180+
for i := len(name) - 1; i >= 0; i-- {
181+
if name[i] == '/' {
182+
if n--; n == 0 {
183+
return name[i+1:]
184+
}
185+
}
186+
}
187+
return name
188+
}
189+
190+
if p, ok := strings.CutPrefix(pattern, "**/"); ok {
191+
return path.Match(p, lastN(name, 1+strings.Count(p, "/")))
192+
}
193+
if p, ok := strings.CutSuffix(pattern, "/**"); ok {
194+
return path.Match(p, firstN(name, 1+strings.Count(p, "/")))
195+
}
196+
return path.Match(pattern, name)
197+
}

src/cmd/distpack/archive_test.go

+39
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
// Copyright 2023 The Go Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style
3+
// license that can be found in the LICENSE file.
4+
5+
package main
6+
7+
import "testing"
8+
9+
var amatchTests = []struct {
10+
pattern string
11+
name string
12+
ok bool
13+
}{
14+
{"a", "a", true},
15+
{"a", "b", false},
16+
{"a/**", "a", true},
17+
{"a/**", "b", false},
18+
{"a/**", "a/b", true},
19+
{"a/**", "b/b", false},
20+
{"a/**", "a/b/c/d/e/f", true},
21+
{"a/**", "z/a/b/c/d/e/f", false},
22+
{"**/a", "a", true},
23+
{"**/a", "b", false},
24+
{"**/a", "x/a", true},
25+
{"**/a", "x/a/b", false},
26+
{"**/a", "x/y/z/a", true},
27+
{"**/a", "x/y/z/a/b", false},
28+
29+
{"go/pkg/tool/*/compile", "go/pkg/tool/darwin_amd64/compile", true},
30+
}
31+
32+
func TestAmatch(t *testing.T) {
33+
for _, tt := range amatchTests {
34+
ok, err := amatch(tt.pattern, tt.name)
35+
if ok != tt.ok || err != nil {
36+
t.Errorf("amatch(%q, %q) = %v, %v, want %v, nil", tt.pattern, tt.name, ok, err, tt.ok)
37+
}
38+
}
39+
}

0 commit comments

Comments
 (0)