env: convert linux-x86-clang to be VM-based
Change-Id: I583d751652dec9176169b4d4705f3813506017e3
Reviewed-on: https://go-review.googlesource.com/4790
Reviewed-by: Minux Ma <minux@golang.org>
diff --git a/cmd/gomote/gomote_test.go b/cmd/gomote/gomote_test.go
new file mode 100644
index 0000000..97b5e00
--- /dev/null
+++ b/cmd/gomote/gomote_test.go
@@ -0,0 +1,35 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+ "reflect"
+ "testing"
+)
+
+func TestDedupEnv(t *testing.T) {
+ tests := []struct {
+ noCase bool
+ in []string
+ want []string
+ }{
+ {
+ noCase: true,
+ in: []string{"k1=v1", "k2=v2", "K1=v3"},
+ want: []string{"K1=v3", "k2=v2"},
+ },
+ {
+ noCase: false,
+ in: []string{"k1=v1", "K1=V2", "k1=v3"},
+ want: []string{"k1=v3", "K1=V2"},
+ },
+ }
+ for _, tt := range tests {
+ got := dedupEnv(tt.noCase, tt.in)
+ if !reflect.DeepEqual(got, tt.want) {
+ t.Errorf("dedupEnv(%v, %q) = %q; want %q", tt.noCase, tt.in, got, tt.want)
+ }
+ }
+}
diff --git a/cmd/gomote/run.go b/cmd/gomote/run.go
index 7bdab15..ee23ca9 100644
--- a/cmd/gomote/run.go
+++ b/cmd/gomote/run.go
@@ -11,6 +11,7 @@
"strings"
"golang.org/x/build/buildlet"
+ "golang.org/x/build/dashboard"
)
func run(args []string) error {
@@ -32,6 +33,12 @@
fs.Usage()
}
name, cmd := fs.Arg(0), fs.Arg(1)
+
+ conf, ok := dashboard.Builders[name]
+ if !ok {
+ return fmt.Errorf("unknown builder type %q", name)
+ }
+
bc, err := namedClient(name)
if err != nil {
return err
@@ -41,7 +48,7 @@
SystemLevel: sys || strings.HasPrefix(cmd, "/"),
Output: os.Stdout,
Args: fs.Args()[2:],
- ExtraEnv: []string(env),
+ ExtraEnv: dedupEnv(conf.GOOS() == "windows", append(conf.Env(), []string(env)...)),
Debug: debug,
})
if execErr != nil {
@@ -50,6 +57,35 @@
return remoteErr
}
+// dedupEnv a copy of env with any duplicates removed, in favor of
+// later values. Items are expected to be on the normal environment
+// "key=value" form. If caseInsensitive is true, the case of keys is
+// ignored.
+//
+// TODO(bradfitz): move this somewhere else.
+func dedupEnv(caseInsensitive bool, env []string) []string {
+ out := make([]string, 0, len(env))
+ saw := map[string]int{} // to index in the array
+ for _, kv := range env {
+ eq := strings.Index(kv, "=")
+ if eq < 1 {
+ out = append(out, kv)
+ continue
+ }
+ k := kv[:eq]
+ if caseInsensitive {
+ k = strings.ToLower(k)
+ }
+ if dupIdx, isDup := saw[k]; isDup {
+ out[dupIdx] = kv
+ } else {
+ saw[k] = len(out)
+ out = append(out, kv)
+ }
+ }
+ return out
+}
+
// stringSlice implements flag.Value, specifically for storing environment
// variable key=value pairs.
type stringSlice []string
diff --git a/dashboard/builders.go b/dashboard/builders.go
index 076d564..e0381d9 100644
--- a/dashboard/builders.go
+++ b/dashboard/builders.go
@@ -175,11 +175,9 @@
tool: "gccgo",
})
- // TODO(bradfitz,adg,jbd): convert these (sid, clang, nacl) to VMs too:
+ // TODO(bradfitz,adg,jbd): convert these (sid, nacl) to VMs too:
addBuilder(BuildConfig{Name: "linux-386-sid", Image: "gobuilders/linux-x86-sid"})
addBuilder(BuildConfig{Name: "linux-amd64-sid", Image: "gobuilders/linux-x86-sid"})
- addBuilder(BuildConfig{Name: "linux-386-clang", Image: "gobuilders/linux-x86-clang"})
- addBuilder(BuildConfig{Name: "linux-amd64-clang", Image: "gobuilders/linux-x86-clang"})
addBuilder(BuildConfig{Name: "nacl-386"})
addBuilder(BuildConfig{Name: "nacl-amd64p32"})
@@ -259,6 +257,17 @@
env: []string{"GOROOT_BOOTSTRAP=/go1.4"},
})
addBuilder(BuildConfig{
+ Name: "linux-386-clang",
+ VMImage: "linux-buildlet-clang",
+ buildletURL: "http://storage.googleapis.com/go-builder-data/buildlet.linux-amd64",
+ env: []string{"GOROOT_BOOTSTRAP=/go1.4", "CC=/usr/bin/clang"},
+ })
+ addBuilder(BuildConfig{
+ Name: "linux-amd64-clang",
+ VMImage: "linux-buildlet-clang",
+ env: []string{"GOROOT_BOOTSTRAP=/go1.4", "CC=/usr/bin/clang"},
+ })
+ addBuilder(BuildConfig{
Name: "openbsd-amd64-gce56",
VMImage: "openbsd-amd64-56",
machineType: "n1-highcpu-2",
diff --git a/env/linux-x86-clang/Dockerfile b/env/linux-x86-clang/Dockerfile
index cf94bd4..f0e1e1a 100644
--- a/env/linux-x86-clang/Dockerfile
+++ b/env/linux-x86-clang/Dockerfile
@@ -2,7 +2,8 @@
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.
-# gobuilders/linux-x86-clang for building with clang instead of gcc.
+# Linux builder VM with clang instead of gccc.
+# Docker tag gobuilders/linux-x86-clang
FROM debian:wheezy
MAINTAINER golang-dev <golang-dev@googlegroups.com>
@@ -11,13 +12,43 @@
ADD /sources/clang-deps.list /etc/apt/sources.list.d/
-ADD /scripts/install-apt-deps.sh /scripts/
-RUN /scripts/install-apt-deps.sh
+RUN apt-get update
+RUN apt-get install -y --no-install-recommends ca-certificates
+# Optionally used by some net/http tests:
+RUN apt-get install -y --no-install-recommends strace
+# For building Go's bootstrap 'dist' prog
+RUN apt-get install -y --no-install-recommends curl
+RUN curl http://llvm.org/apt/llvm-snapshot.gpg.key | apt-key add -
-ADD /scripts/build-go-builder.sh /scripts/
-RUN GO_REV=go1.4 TOOLS_REV=d79e0375a BUILDER_REV=fa8373a /scripts/build-go-builder.sh && test -f /usr/local/bin/builder
+RUN apt-get update
+RUN apt-get install -y --no-install-recommends clang-3.5
+# TODO(cmang): move these into a 386 image that derives from this one.
+RUN apt-get install -y --no-install-recommends libc6-dev-i386 gcc-multilib
+# Remove gcc binary so it doesn't interfere with clang
+RUN rm -f /usr/bin/gcc
-ENV CC /usr/bin/clang
+# Required for networking in a VM on GCE:
+RUN apt-get install -y --no-install-recommends net-tools ifupdown isc-dhcp-client dhcp3-client
+# And misc basic tools:
+RUN apt-get install -y --no-install-recommends procps lsof psmisc
-RUN mkdir -p /go1.4-386 && (curl --silent https://storage.googleapis.com/golang/go1.4.linux-386.tar.gz | tar -C /go1.4-386 -zxv)
+RUN apt-get clean
+
RUN mkdir -p /go1.4-amd64 && (curl --silent https://storage.googleapis.com/golang/go1.4.linux-amd64.tar.gz | tar -C /go1.4-amd64 -zxv)
+RUN mv /go1.4-amd64/go /go1.4 && rmdir /go1.4-amd64
+
+RUN curl -o /usr/local/bin/stage0 https://storage.googleapis.com/go-builder-data/stage0.linux-amd64.92ebb4ec
+RUN chmod +x /usr/local/bin/stage0
+
+ADD scripts/rc.local /etc/init.d/rc.local
+RUN chmod +x /etc/init.d/rc.local
+
+RUN rm -rf /var/lib/apt/lists /usr/share/doc
+RUN rm -rf /go1.4/pkg/linux_amd64_race /go1.4/api /go1.4/blog /go1.4/doc /go1.4/misc /go1.4/test
+RUN find /go1.4 -type d -name testdata | xargs rm -rf
+RUN rm -rf /go1.4/pkg/linux_amd64_race /go1.4/test /go1.4/api
+RUN (cd /usr/share/locale/ && ls -1 | grep -v en | xargs rm -rf)
+RUN rm -rf /var/cache/debconf/*
+RUN rm -rf /usr/share/man
+
+RUN (echo "auto lo"; echo "iface lo inet loopback"; echo "auto eth0"; echo "iface eth0 inet dhcp") > /etc/network/interfaces
diff --git a/env/linux-x86-clang/Makefile b/env/linux-x86-clang/Makefile
index 2c945bb..b037ed1 100644
--- a/env/linux-x86-clang/Makefile
+++ b/env/linux-x86-clang/Makefile
@@ -5,11 +5,11 @@
docker: Dockerfile
docker build -t gobuilders/linux-x86-clang .
-docker-linux.clang.tar.gz: docker
- docker save gobuilders/linux-x86-clang | gzip | (cd ../../cmd/upload && go run upload.go --public go-builder-data/docker-linux.clang.tar.gz)
+# TODO(bradfitz): test targets, using cmd/gomote and the buildlet in the container.
-check: docker
- docker run -e GOROOT_BOOTSTRAP=/go1.4-amd64/go gobuilders/linux-x86-clang /usr/local/bin/builder -rev=20a10e7ddd1b -buildroot=/ -v -report=false linux-amd64-temp
-
-check32: docker
- docker run -e GOROOT_BOOTSTRAP=/go1.4-386/go gobuilders/linux-x86-clang /usr/local/bin/builder -rev=20a10e7ddd1b -buildroot=/ -v -report=false linux-386-temp
+# TODO(bradfitz): docs on how to build a new image. In a nutshell,
+#
+# $ go install golang.org/x/build/cmd/docker2boot
+# $ sudo docker2boot --image=gobuilders/linux-x86-clang --out=linux-buildlet-clang.tar.gz
+# $ gsutil cp -a public-read linux-buildlet-clang.tar.gz gs://go-builder-data/linux-buildlet-clang.tar.gz
+# $ gcloud compute --project symbolic-datum-552 images create linux-buildlet-clang --source-uri gs://go-builder-data/linux-buildlet-clang.tar.gz
diff --git a/env/linux-x86-clang/scripts/build-go-builder.sh b/env/linux-x86-clang/scripts/build-go-builder.sh
deleted file mode 100755
index 4f06f1b..0000000
--- a/env/linux-x86-clang/scripts/build-go-builder.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-set -ex
-
-export GOPATH=/gopath
-export GOROOT=/goroot
-PREFIX=/usr/local
-: ${GO_REV:?"need to be set to the golang repo revision used to build the builder."}
-: ${TOOLS_REV:?"need to be set to the tools repo revision used to build the builder."}
-: ${BUILDER_REV:?"need to be set to the build repo revision for the builder."}
-
-mkdir -p $GOROOT
-git clone https://go.googlesource.com/go $GOROOT
-(cd $GOROOT/src && git checkout $GO_REV && find && ./make.bash)
-
-GO_TOOLS=$GOPATH/src/golang.org/x/tools
-mkdir -p $GO_TOOLS
-git clone https://go.googlesource.com/tools $GO_TOOLS
-(cd $GO_TOOLS && git reset --hard $TOOLS_REV)
-
-GO_BUILD=$GOPATH/src/golang.org/x/build
-mkdir -p $GO_BUILD
-git clone https://go.googlesource.com/build $GO_BUILD
-
-mkdir -p $PREFIX/bin
-(cd $GO_BUILD && git reset --hard $BUILDER_REV && GOBIN=$PREFIX/bin /goroot/bin/go install golang.org/x/build/cmd/builder)
-
-rm -fR $GOROOT/bin $GOROOT/pkg $GOPATH
-
-cd $GOROOT
-git clean -f -d -x
-git checkout master
diff --git a/env/linux-x86-clang/scripts/install-apt-deps.sh b/env/linux-x86-clang/scripts/install-apt-deps.sh
deleted file mode 100755
index f5d3fda..0000000
--- a/env/linux-x86-clang/scripts/install-apt-deps.sh
+++ /dev/null
@@ -1,22 +0,0 @@
-set -ex
-
-apt-get update
-apt-get install -y --no-install-recommends ca-certificates
-# Optionally used by some net/http tests:
-apt-get install -y --no-install-recommends strace
-# For building Go's bootstrap 'dist' prog
-apt-get install -y --no-install-recommends wget
-wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key | apt-key add -
-apt-get update
-apt-get install -y --no-install-recommends clang-3.5
-# TODO(cmang): move these into a 386 image that derives from this one.
-apt-get install -y --no-install-recommends libc6-dev-i386 gcc-multilib
-# Remove gcc binary so it doesn't interfere with clang
-rm -f /usr/bin/gcc
-# For interacting with the Go source & subrepos:
-apt-get install -y --no-install-recommends git-core
-# For installing Go 1.4:
-apt-get install -y --no-install-recommends curl
-
-apt-get clean
-rm -fr /var/lib/apt/lists
diff --git a/env/linux-x86-clang/scripts/rc.local b/env/linux-x86-clang/scripts/rc.local
new file mode 100644
index 0000000..2fe28bf
--- /dev/null
+++ b/env/linux-x86-clang/scripts/rc.local
@@ -0,0 +1,4 @@
+#!/bin/sh -e
+
+start-stop-daemon --start --exec=/usr/local/bin/stage0 --background /usr/local/bin/stage0 -- --network-wait=15s
+exit 0
diff --git a/env/linux-x86-std/Dockerfile b/env/linux-x86-std/Dockerfile
index efd7038..f46328f 100644
--- a/env/linux-x86-std/Dockerfile
+++ b/env/linux-x86-std/Dockerfile
@@ -2,7 +2,8 @@
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.
-# Base builder image: gobuilders/linux-x86-base
+# Normal Linux builder VM
+# Docker tag gobuilders/linux-x86-std
FROM debian:wheezy
MAINTAINER golang-dev <golang-dev@googlegroups.com>
@@ -11,10 +12,10 @@
RUN apt-get update
RUN apt-get install -y --no-install-recommends ca-certificates
-# Optionally used by some net/http tests:
-RUN apt-get install -y --no-install-recommends strace
# Optionally used by runtime tests for gdb:
RUN apt-get install -y --no-install-recommends gdb
+# Optionally used by some net/http tests:
+RUN apt-get install -y --no-install-recommends strace
# For building Go's bootstrap 'dist' prog
RUN apt-get install -y --no-install-recommends gcc libc6-dev
# For 32-bit builds:
@@ -22,8 +23,12 @@
RUN apt-get install -y --no-install-recommends libc6-dev-i386 gcc-multilib
# For downloading Go 1.4:
RUN apt-get install -y --no-install-recommends curl
+
+# Required for networking in a VM on GCE:
RUN apt-get install -y --no-install-recommends net-tools ifupdown isc-dhcp-client dhcp3-client
+# And misc basic tools:
RUN apt-get install -y --no-install-recommends procps lsof psmisc
+
RUN apt-get clean
RUN mkdir -p /go1.4-amd64 && (curl --silent https://storage.googleapis.com/golang/go1.4.linux-amd64.tar.gz | tar -C /go1.4-amd64 -zxv)
@@ -44,5 +49,3 @@
RUN rm -rf /usr/share/man
RUN (echo "auto lo"; echo "iface lo inet loopback"; echo "auto eth0"; echo "iface eth0 inet dhcp") > /etc/network/interfaces
-
-
