_content: add rebuild page with reproducible build information
We now have a command to reproduce Go builds posted on go.dev/dl.
Add a dashboard that people can check to see its results.
We should be able to link to this page from https://reproducible-builds.org/citests/.
For golang/go#57120.
For golang/go#58884.
Change-Id: I0bd1f9c26a9a003aa1f301125083195fdeb048b4
Reviewed-on: https://go-review.googlesource.com/c/website/+/513700
Reviewed-by: Heschi Kreinick <heschi@google.com>
Reviewed-by: Dmitri Shuralyov <dmitshur@golang.org>
Reviewed-by: Dmitri Shuralyov <dmitshur@google.com>
Run-TryBot: Russ Cox <rsc@golang.org>
TryBot-Result: Gopher Robot <gobot@golang.org>
diff --git a/_content/rebuild.html b/_content/rebuild.html
new file mode 100644
index 0000000..8931be0
--- /dev/null
+++ b/_content/rebuild.html
@@ -0,0 +1,94 @@
+<!--{
+ "Title": "Go Reproducible Build Report",
+ "layout": "article",
+ "template": true
+}-->
+
+<style>
+details { margin-left: 2em; }
+pre { margin-left: 2em; }
+.time { color: #777; }
+</style>
+
+<p>
+As of Go 1.21, Go's binary toolchain downloads served by <a href="/dl/">go.dev/dl/</a> can be
+reproduced and verified by anyone, on any platform,
+using <a href="https://pkg.go.dev/golang.org/x/tools/cmd/gorebuild">golang.org/x/tools/cmd/gorebuild</a>.
+</p>
+
+<p>
+This page is updated daily with the results of running <code>gorebuild</code> in an Ubuntu VM,
+using this script:
+</p>
+
+<pre>
+apt-get update &&
+apt-get -y install software-properties-common &&
+add-apt-repository universe &&
+apt-get -y install golang-go msitools &&
+go run golang.org/x/build/cmd/gorebuild@latest -p=4
+</pre>
+
+<p>
+The installation of <code>msitools</code> lets <code>gorebuild</code>
+check the contents of the Windows MSI installation file.
+The <code>-p=4</code> means to run up to four builds in parallel.
+</p>
+
+{{define "marker"}}<span style="marker">{{template "markersymbol" .}}</span>{{end}}
+{{define "markersymbol"}}
+{{- if eq . "PASS" -}} ✅
+{{- else if eq . "SKIP" -}} —
+{{- else -}} ❌
+{{- end -}}
+{{end}}
+
+{{define "log"}}
+<pre>
+{{range .Messages}}<span class="time">{{(rfc3339 .Time).UTC.Format "15:04:05"}}</span> {{.Text}}
+{{end}}
+</pre>
+{{end}}
+
+{{define "autoopen"}} {{if not (eq . "PASS")}} open {{end}} {{end}}
+
+{{$Report := json gorebuild}}
+{{with $Report}}
+
+Using gorebuild from {{.Version}}<br><br>
+
+Rebuild started at {{(rfc3339 .Start).UTC.Format "2006-01-02 15:04:05"}} UTC.<br>
+Rebuild finished at {{(rfc3339 .End).UTC.Format "2006-01-02 15:04:05"}} UTC.<br>
+Elapsed time: {{((rfc3339 .End).Sub (rfc3339 .Start)).Round 1e9}}.
+
+<h2 id="releases">Releases</h2>
+
+{{range .Releases}}
+<details {{template "autoopen" .Log.Status}} >
+<summary><b id="{{.Version}}">{{template "marker" .Log.Status}} {{.Version}}</b></summary>
+
+<details>
+<summary>Log</summary>
+{{template "log" .Log}}
+</details>
+
+{{range .Files}}
+<details {{template "autoopen" .Log.Status}}>
+<summary><b id="{{.Name}}">{{template "marker" .Log.Status}} {{.Name}}</b></summary>
+{{template "log" .Log}}
+</details>
+{{end}}
+
+</details>
+{{end}}
+
+<h2 id="bootstraps">Bootstraps</h2>
+
+{{range .Bootstraps}}
+<details>
+<summary><b>Bootstrap {{.Version}}</b></summary>
+{{template "log" .Log}}
+</details>
+{{end}}
+
+{{end}}
diff --git a/cmd/golangorg/server.go b/cmd/golangorg/server.go
index 1fb1a7c..2a7f66b 100644
--- a/cmd/golangorg/server.go
+++ b/cmd/golangorg/server.go
@@ -16,6 +16,7 @@
"fmt"
"go/format"
"html/template"
+ "io"
"io/fs"
"io/ioutil"
"log"
@@ -27,6 +28,7 @@
"runtime"
"runtime/debug"
"strings"
+ "sync"
"sync/atomic"
"time"
@@ -242,6 +244,8 @@
return h
}
+var gorebuild = NewCachedURL("https://gorebuild.storage.googleapis.com/gorebuild.json", 5*time.Minute)
+
// newSite creates a new site for a given content and goroot file system pair
// and registers it in mux to handle requests for host.
// If host is the empty string, the registrations are for the wildcard host.
@@ -251,11 +255,14 @@
site.Funcs(template.FuncMap{
"googleAnalytics": func() string { return googleAnalytics },
"googleCN": func() bool { return host == "golang.google.cn" },
+ "gorebuild": gorebuild.Get,
+ "json": jsonUnmarshal,
"newest": newest,
+ "now": func() time.Time { return time.Now() },
"releases": func() []*history.Major { return history.Majors },
+ "rfc3339": parseRFC3339,
"section": section,
"version": func() string { return runtime.Version() },
- "now": func() time.Time { return time.Now() },
})
docs, err := pkgdoc.NewServer(fsys, site, googleCN)
if err != nil {
@@ -269,6 +276,10 @@
return site, nil
}
+func parseRFC3339(s string) (time.Time, error) {
+ return time.Parse(time.RFC3339, s)
+}
+
// watchTip is a background goroutine that watches the main Go repo for updates.
// When a new commit is available, watchTip downloads the new tree and calls
// tipGoroot.Set to install the new file system.
@@ -803,3 +814,66 @@
http.Redirect(w, r, url, http.StatusMovedPermanently)
})
}
+
+type CachedURL struct {
+ url string
+ timeout time.Duration
+
+ mu sync.Mutex
+ data []byte
+ err error
+ etag string
+ updated time.Time
+}
+
+func NewCachedURL(url string, timeout time.Duration) *CachedURL {
+ return &CachedURL{url: url, timeout: timeout}
+}
+
+func (c *CachedURL) Get() (data []byte, err error) {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+
+ if time.Since(c.updated) < c.timeout {
+ return c.data, c.err
+ }
+ defer func() {
+ c.updated = time.Now()
+ c.data, c.err = data, err
+ }()
+
+ cli := &http.Client{Timeout: 60 * time.Second}
+ req, err := http.NewRequest("GET", c.url, nil)
+ if err != nil {
+ return nil, err
+ }
+ if c.etag != "" {
+ req.Header.Set("If-None-Match", c.etag)
+ }
+ resp, err := cli.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("loading rebuild report JSON: %v", err)
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode == 206 {
+ // Unmodified.
+ log.Printf("checked %s - unmodified", c.url)
+ return c.data, c.err
+ }
+ log.Printf("reloading %s", c.url)
+ if resp.StatusCode != 200 {
+ return nil, fmt.Errorf("loading rebuild report JSON: %v", resp.Status)
+ }
+ c.etag = resp.Header.Get("Etag")
+ data, err = io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("loading rebuild report JSON: %v", err)
+ }
+ return data, nil
+}
+
+func jsonUnmarshal(data []byte) (any, error) {
+ var x any
+ err := json.Unmarshal(data, &x)
+ return x, err
+}
