tip: fix, update tip.golang.org
Primarily for golang/go#29251 but also update the base Docker layer
from Go 1.9 to Go 1.11, update other deps, and fix some Kubernetes
config cleanups that happened prior without testing apparently.
Fixes golang/go#29251
Change-Id: I0aafccdfedfc0d9ebb75d0c1a3b0819245ea5f19
Reviewed-on: https://go-review.googlesource.com/c/154181
Reviewed-by: Bryan C. Mills <bcmills@google.com>
diff --git a/cmd/tip/Dockerfile b/cmd/tip/Dockerfile
index bc0374f..5fe8b39 100644
--- a/cmd/tip/Dockerfile
+++ b/cmd/tip/Dockerfile
@@ -1,19 +1,23 @@
-FROM golang:1.9
+FROM golang:1.11
RUN apt-get update && apt-get install --no-install-recommends -y -q build-essential git
+# For implicit GOCACHE (Issues 29243 and 29251), set HOME:
+RUN mkdir -p /home/gopher
+ENV HOME /home/gopher
+
# golang puts its go install here (weird but true)
ENV GOROOT_BOOTSTRAP /usr/local/go
# BEGIN deps (run `make update-deps` to update)
-# Repo cloud.google.com/go at 1d0c2da (2018-01-30)
-ENV REV=1d0c2da40456a9b47f5376165f275424acc15c09
-RUN go get -d cloud.google.com/go/compute/metadata `#and 6 other pkgs` &&\
+# Repo cloud.google.com/go at b5eca92 (2018-10-23)
+ENV REV=b5eca92245a08e245bc29c4880c9779ea4aeaa9a
+RUN go get -d cloud.google.com/go/compute/metadata `#and 7 other pkgs` &&\
(cd /go/src/cloud.google.com/go && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
-# Repo github.com/golang/protobuf at 9255415 (2018-01-25)
-ENV REV=925541529c1fa6821df4e44ce2723319eb2be768
+# Repo github.com/golang/protobuf at b4deda0 (2018-04-30)
+ENV REV=b4deda0973fb4c70b50d226b1af49f3da59f5265
RUN go get -d github.com/golang/protobuf/proto `#and 6 other pkgs` &&\
(cd /go/src/github.com/golang/protobuf && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
@@ -22,44 +26,54 @@
RUN go get -d github.com/googleapis/gax-go &&\
(cd /go/src/github.com/googleapis/gax-go && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
-# Repo golang.org/x/build at e879390 (2018-02-01)
-ENV REV=e8793909ba350594eea4c7c6bdb0f0d9a0d0f77a
+# Repo go.opencensus.io at ebd8d31 (2018-05-16)
+ENV REV=ebd8d31470fedf6c27d0e3056653ddff642509b8
+RUN go get -d go.opencensus.io/exporter/stackdriver/propagation `#and 12 other pkgs` &&\
+ (cd /go/src/go.opencensus.io && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
+
+# Repo golang.org/x/build at 7b78c20 (2018-12-13)
+ENV REV=7b78c2042368d5c56ee9dbd92ab5fa988c763944
RUN go get -d golang.org/x/build/autocertcache &&\
(cd /go/src/golang.org/x/build && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
-# Repo golang.org/x/crypto at 1875d0a (2018-01-27)
-ENV REV=1875d0a70c90e57f11972aefd42276df65e895b9
+# Repo golang.org/x/crypto at e4dc69e (2018-11-06)
+ENV REV=e4dc69e5b2fd71dcaf8bd5d054eb936deb78d1fa
RUN go get -d golang.org/x/crypto/acme `#and 2 other pkgs` &&\
(cd /go/src/golang.org/x/crypto && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
-# Repo golang.org/x/net at 6d90978 (2018-02-01)
-ENV REV=6d90978dc4889d44e8cfbd04c05d17b5417823c7
+# Repo golang.org/x/net at 891ebc4 (2018-12-13)
+ENV REV=891ebc4b82d6e74f468c533b06f983c7be918a96
RUN go get -d golang.org/x/net/context `#and 8 other pkgs` &&\
(cd /go/src/golang.org/x/net && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
-# Repo golang.org/x/oauth2 at 30785a2 (2018-01-04)
-ENV REV=30785a2c434e431ef7c507b54617d6a951d5f2b4
+# Repo golang.org/x/oauth2 at f42d051 (2018-11-06)
+ENV REV=f42d05182288abf10faef86d16c0d07b8d40ea2d
RUN go get -d golang.org/x/oauth2 `#and 5 other pkgs` &&\
(cd /go/src/golang.org/x/oauth2 && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
-# Repo golang.org/x/text at e19ae14 (2017-12-27)
-ENV REV=e19ae1496984b1c655b8044a65c0300a3c878dd3
+# Repo golang.org/x/sys at 4d1cda0 (2018-12-13)
+ENV REV=4d1cda033e0619309c606fc686de3adcf599539e
+RUN go get -d golang.org/x/sys/unix &&\
+ (cd /go/src/golang.org/x/sys && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
+
+# Repo golang.org/x/text at 6f44c5a (2018-10-30)
+ENV REV=6f44c5a2ea40ee3593d98cdcc905cc1fdaa660e2
RUN go get -d golang.org/x/text/secure/bidirule `#and 4 other pkgs` &&\
(cd /go/src/golang.org/x/text && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
-# Repo google.golang.org/api at 7d0e2d3 (2018-01-30)
-ENV REV=7d0e2d350555821bef5a5b8aecf0d12cc1def633
+# Repo google.golang.org/api at 20530fd (2018-05-06)
+ENV REV=20530fd5d65ad2caee87891f9896d7547cb400c9
RUN go get -d google.golang.org/api/gensupport `#and 9 other pkgs` &&\
(cd /go/src/google.golang.org/api && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
-# Repo google.golang.org/genproto at 4eb30f4 (2018-01-25)
-ENV REV=4eb30f4778eed4c258ba66527a0d4f9ec8a36c45
-RUN go get -d google.golang.org/genproto/googleapis/api/annotations `#and 3 other pkgs` &&\
+# Repo google.golang.org/genproto at 86e600f (2018-04-27)
+ENV REV=86e600f69ee4704c6efbf6a2a40a5c10700e76c2
+RUN go get -d google.golang.org/genproto/googleapis/api/annotations `#and 4 other pkgs` &&\
(cd /go/src/google.golang.org/genproto && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
-# Repo google.golang.org/grpc at 0bd008f (2018-01-25)
-ENV REV=0bd008f5fadb62d228f12b18d016709e8139a7af
-RUN go get -d google.golang.org/grpc `#and 23 other pkgs` &&\
+# Repo google.golang.org/grpc at 07ef407 (2018-08-06)
+ENV REV=07ef407d991f1004e6c3367c8f452ed9a02f17ff
+RUN go get -d google.golang.org/grpc `#and 26 other pkgs` &&\
(cd /go/src/google.golang.org/grpc && (git cat-file -t $REV 2>/dev/null || git fetch -q origin $REV) && git reset --hard $REV)
# Optimization to speed up iterative development, not necessary for correctness:
@@ -67,6 +81,7 @@
cloud.google.com/go/iam \
cloud.google.com/go/internal \
cloud.google.com/go/internal/optional \
+ cloud.google.com/go/internal/trace \
cloud.google.com/go/internal/version \
cloud.google.com/go/storage \
github.com/golang/protobuf/proto \
@@ -76,22 +91,35 @@
github.com/golang/protobuf/ptypes/duration \
github.com/golang/protobuf/ptypes/timestamp \
github.com/googleapis/gax-go \
+ go.opencensus.io/exporter/stackdriver/propagation \
+ go.opencensus.io/internal \
+ go.opencensus.io/internal/tagencoding \
+ go.opencensus.io/plugin/ochttp \
+ go.opencensus.io/plugin/ochttp/propagation/b3 \
+ go.opencensus.io/stats \
+ go.opencensus.io/stats/internal \
+ go.opencensus.io/stats/view \
+ go.opencensus.io/tag \
+ go.opencensus.io/trace \
+ go.opencensus.io/trace/internal \
+ go.opencensus.io/trace/propagation \
golang.org/x/build/autocertcache \
golang.org/x/crypto/acme \
golang.org/x/crypto/acme/autocert \
golang.org/x/net/context \
golang.org/x/net/context/ctxhttp \
+ golang.org/x/net/http/httpguts \
golang.org/x/net/http2 \
golang.org/x/net/http2/hpack \
golang.org/x/net/idna \
golang.org/x/net/internal/timeseries \
- golang.org/x/net/lex/httplex \
golang.org/x/net/trace \
golang.org/x/oauth2 \
golang.org/x/oauth2/google \
golang.org/x/oauth2/internal \
golang.org/x/oauth2/jws \
golang.org/x/oauth2/jwt \
+ golang.org/x/sys/unix \
golang.org/x/text/secure/bidirule \
golang.org/x/text/transform \
golang.org/x/text/unicode/bidi \
@@ -107,6 +135,7 @@
google.golang.org/api/transport/http \
google.golang.org/genproto/googleapis/api/annotations \
google.golang.org/genproto/googleapis/iam/v1 \
+ google.golang.org/genproto/googleapis/rpc/code \
google.golang.org/genproto/googleapis/rpc/status \
google.golang.org/grpc \
google.golang.org/grpc/balancer \
@@ -117,9 +146,13 @@
google.golang.org/grpc/credentials \
google.golang.org/grpc/encoding \
google.golang.org/grpc/encoding/proto \
- google.golang.org/grpc/grpclb/grpc_lb_v1/messages \
google.golang.org/grpc/grpclog \
google.golang.org/grpc/internal \
+ google.golang.org/grpc/internal/backoff \
+ google.golang.org/grpc/internal/channelz \
+ google.golang.org/grpc/internal/envconfig \
+ google.golang.org/grpc/internal/grpcrand \
+ google.golang.org/grpc/internal/transport \
google.golang.org/grpc/keepalive \
google.golang.org/grpc/metadata \
google.golang.org/grpc/naming \
@@ -129,8 +162,7 @@
google.golang.org/grpc/resolver/passthrough \
google.golang.org/grpc/stats \
google.golang.org/grpc/status \
- google.golang.org/grpc/tap \
- google.golang.org/grpc/transport
+ google.golang.org/grpc/tap
# END deps.
# golang sets GOPATH=/go
diff --git a/cmd/tip/Makefile b/cmd/tip/Makefile
index ac7d937..8054f17 100644
--- a/cmd/tip/Makefile
+++ b/cmd/tip/Makefile
@@ -33,8 +33,8 @@
deploy-prod: push-prod
go install golang.org/x/build/cmd/xb
- xb --prod kubectl set image deployment/tip-deployment tip=$(IMAGE_PROD):$(VERSION)
+ xb --prod kubectl set image deployment/tipgodoc-deployment tipgodoc=$(IMAGE_PROD):$(VERSION)
deploy-staging: push-staging
go install golang.org/x/build/cmd/xb
- xb --staging kubectl set image deployment/tip-deployment tip=$(IMAGE_STAGING):$(VERSION)
+ xb --staging kubectl set image deployment/tipgodoc-deployment tipgodoc=$(IMAGE_STAGING):$(VERSION)
diff --git a/cmd/tip/README b/cmd/tip/README
index b96c107..b5f42a9 100644
--- a/cmd/tip/README
+++ b/cmd/tip/README
@@ -23,10 +23,6 @@
Kubernetes instructions:
- * build & push images (see Makefile for helpers)
- * create/update resources:
- - kubectl create -f tip-rc.yaml
- - kubectl create -f tip-service.yaml
+ * make deploy-prod
-TODO(bradfitz): flesh out these instructions as I gain experience
-with updating this over time. Also: move talks.golang.org to GKE too?
+Also: move talks.golang.org to GKE too?
diff --git a/cmd/tip/godoc.go b/cmd/tip/godoc.go
index 4332c35..e6808f5 100644
--- a/cmd/tip/godoc.go
+++ b/cmd/tip/godoc.go
@@ -8,53 +8,73 @@
"bytes"
"errors"
"fmt"
+ "io"
+ "log"
"os"
"os/exec"
"path/filepath"
)
-type godocBuilder struct {
+type godocBuilder struct{}
+
+func prefix8(s string) string {
+ if len(s) < 8 {
+ return s
+ }
+ return s[:8]
}
func (b godocBuilder) Signature(heads map[string]string) string {
- return fmt.Sprintf("go=%v/tools=%v", heads["go"], heads["tools"])
+ return fmt.Sprintf("go=%v/tools=%v", prefix8(heads["go"]), prefix8(heads["tools"]))
}
-func (b godocBuilder) Init(dir, hostport string, heads map[string]string) (*exec.Cmd, error) {
+func (b godocBuilder) Init(logger *log.Logger, dir, hostport string, heads map[string]string) (*exec.Cmd, error) {
+
goDir := filepath.Join(dir, "go")
toolsDir := filepath.Join(dir, "gopath/src/golang.org/x/tools")
+ logger.Printf("checking out go repo ...")
if err := checkout(repoURL+"go", heads["go"], goDir); err != nil {
- return nil, err
+ return nil, fmt.Errorf("checkout of go: %v", err)
}
+ logger.Printf("checking out tools repo ...")
if err := checkout(repoURL+"tools", heads["tools"], toolsDir); err != nil {
- return nil, err
+ return nil, fmt.Errorf("checkout of tools: %v", err)
}
+ var logWriter io.Writer = toLoggerWriter{logger}
+
make := exec.Command(filepath.Join(goDir, "src/make.bash"))
make.Dir = filepath.Join(goDir, "src")
- if err := runErr(make); err != nil {
- return nil, err
+ make.Stdout = logWriter
+ make.Stderr = logWriter
+ logger.Printf("running make.bash in %s ...", make.Dir)
+ if err := make.Run(); err != nil {
+ return nil, fmt.Errorf("running make.bash: %v", err)
}
+
+ logger.Printf("installing godoc ...")
goBin := filepath.Join(goDir, "bin/go")
goPath := filepath.Join(dir, "gopath")
install := exec.Command(goBin, "install", "golang.org/x/tools/cmd/godoc")
- install.Env = []string{
- "GOROOT=" + goDir,
- "GOPATH=" + goPath,
- "GOROOT_BOOTSTRAP=" + os.Getenv("GOROOT_BOOTSTRAP"),
- }
- if err := runErr(install); err != nil {
- return nil, err
+ install.Stdout = logWriter
+ install.Stderr = logWriter
+ install.Env = append(os.Environ(),
+ "GOROOT="+goDir,
+ "GOPATH="+goPath,
+ "GOROOT_BOOTSTRAP="+os.Getenv("GOROOT_BOOTSTRAP"),
+ )
+ if err := install.Run(); err != nil {
+ return nil, fmt.Errorf("go install golang.org/x/tools/cmd/godoc: %v", err)
}
+ logger.Printf("starting godoc ...")
godocBin := filepath.Join(goPath, "bin/godoc")
godoc := exec.Command(godocBin, "-http="+hostport, "-index", "-index_interval=-1s", "-play")
- godoc.Env = []string{"GOROOT=" + goDir}
- // TODO(adg): log this somewhere useful
- godoc.Stdout = os.Stdout
- godoc.Stderr = os.Stderr
+ godoc.Env = append(os.Environ(), "GOROOT="+goDir)
+ godoc.Stdout = logWriter
+ godoc.Stderr = logWriter
if err := godoc.Start(); err != nil {
- return nil, err
+ return nil, fmt.Errorf("starting godoc: %v", err)
}
return godoc, nil
}
diff --git a/cmd/tip/godoc.yaml b/cmd/tip/godoc.yaml
deleted file mode 100644
index 64dbb7d..0000000
--- a/cmd/tip/godoc.yaml
+++ /dev/null
@@ -1,17 +0,0 @@
-module: tip
-runtime: custom
-vm: true
-
-manual_scaling:
- instances: 4
-
-env_variables:
- TIP_BUILDER: 'godoc'
-
-health_check:
- enable_health_check: True
- check_interval_sec: 5
- timeout_sec: 4
- unhealthy_threshold: 2
- healthy_threshold: 2
- restart_threshold: 240
diff --git a/cmd/tip/talks.go b/cmd/tip/talks.go
index a1a7a95..cc1472e 100644
--- a/cmd/tip/talks.go
+++ b/cmd/tip/talks.go
@@ -8,14 +8,14 @@
"bytes"
"errors"
"fmt"
+ "log"
"os"
"os/exec"
"path/filepath"
"runtime"
)
-type talksBuilder struct {
-}
+type talksBuilder struct{}
func (b talksBuilder) Signature(heads map[string]string) string {
return heads["talks"]
@@ -23,7 +23,8 @@
const talksToolsRev = "8cab8a1319f0be9798e7fe78b15da75e5f94b2e9"
-func (b talksBuilder) Init(dir, hostport string, heads map[string]string) (*exec.Cmd, error) {
+func (b talksBuilder) Init(logger *log.Logger, dir, hostport string, heads map[string]string) (*exec.Cmd, error) {
+ // TODO: use logger
toolsDir := filepath.Join(dir, "gopath/src/golang.org/x/tools")
if err := checkout(repoURL+"tools", talksToolsRev, toolsDir); err != nil {
return nil, err
@@ -41,7 +42,7 @@
goPath := filepath.Join(dir, "gopath")
presentPath := "golang.org/x/tools/cmd/present"
install := exec.Command(goBin, "install", "-tags=appenginevm", presentPath)
- install.Env = []string{"GOROOT=" + goDir, "GOPATH=" + goPath}
+ install.Env = append(os.Environ(), "GOROOT="+goDir, "GOPATH="+goPath)
if err := runErr(install); err != nil {
return nil, err
}
diff --git a/cmd/tip/tip-prod.yaml b/cmd/tip/tip-prod.yaml
index ce4ad07..8b30a97 100644
--- a/cmd/tip/tip-prod.yaml
+++ b/cmd/tip/tip-prod.yaml
@@ -31,8 +31,8 @@
- containerPort: 443
resources:
requests:
- cpu: "1"
- memory: "2Gi"
- limits:
cpu: "2"
memory: "4Gi"
+ limits:
+ cpu: "2"
+ memory: "8Gi"
diff --git a/cmd/tip/tip.go b/cmd/tip/tip.go
index ce9c4b7..6109560 100644
--- a/cmd/tip/tip.go
+++ b/cmd/tip/tip.go
@@ -100,7 +100,7 @@
type Proxy struct {
builder Builder
- mu sync.Mutex // protects the followin'
+ mu sync.Mutex // protects following fields
proxy http.Handler
cur string // signature of gorepo+toolsrepo
cmd *exec.Cmd // live godoc instance, or nil for none
@@ -111,7 +111,7 @@
type Builder interface {
Signature(heads map[string]string) string
- Init(dir, hostport string, heads map[string]string) (*exec.Cmd, error)
+ Init(logger *log.Logger, dir, hostport string, heads map[string]string) (*exec.Cmd, error)
HealthCheck(hostport string) error
}
@@ -177,7 +177,9 @@
// run runs in its own goroutine.
func (p *Proxy) run() {
+ p.mu.Lock()
p.side = "a"
+ p.mu.Unlock()
for {
p.poll()
time.Sleep(30 * time.Second)
@@ -225,14 +227,20 @@
if newSide == "b" {
hostport = "localhost:8082"
}
- cmd, err := p.builder.Init(dir, hostport, heads)
+ logger := log.New(os.Stderr, sig+": ", log.LstdFlags)
+
+ cmd, err := p.builder.Init(logger, dir, hostport, heads)
if err != nil {
+ logger.Printf("Init failed: %v", err)
err = fmt.Errorf("builder.Init: %v", err)
} else {
go func() {
// TODO(adg,bradfitz): be smarter about dead processes
if err := cmd.Wait(); err != nil {
- log.Printf("process in %v exited: %v", dir, err)
+ logger.Printf("process in %v exited: %v (%T)", dir, err, err)
+ if ee, ok := err.(*exec.ExitError); ok {
+ logger.Printf("ProcessState.Sys() = %v", ee.ProcessState.Sys())
+ }
}
}()
err = waitReady(p.builder, hostport)
@@ -302,7 +310,7 @@
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
return fmt.Errorf("mkdir: %v", err)
}
- if err := runErr(exec.Command("git", "clone", repo, path)); err != nil {
+ if err := runErr(exec.Command("git", "clone", "--depth", "1", repo, path)); err != nil {
return fmt.Errorf("clone: %v", err)
}
} else if err != nil {
@@ -424,3 +432,10 @@
}
h.h.ServeHTTP(w, r)
}
+
+type toLoggerWriter struct{ logger *log.Logger }
+
+func (w toLoggerWriter) Write(p []byte) (int, error) {
+ w.logger.Printf("%s", p)
+ return len(p), nil
+}
