| // Copyright 2017 The Go Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style |
| // license that can be found in the LICENSE file. |
| |
| // The debugnewvm command creates and destroys a VM-based buildlet |
| // with lots of logging for debugging. Nothing depends on this. |
| package main |
| |
| import ( |
| "context" |
| "flag" |
| "fmt" |
| "log" |
| "net/http" |
| "os" |
| "path" |
| "regexp" |
| "strings" |
| "time" |
| |
| "cloud.google.com/go/compute/metadata" |
| "golang.org/x/build/buildenv" |
| "golang.org/x/build/buildlet" |
| "golang.org/x/build/dashboard" |
| "golang.org/x/build/internal/buildgo" |
| "golang.org/x/build/internal/cloud" |
| "golang.org/x/build/internal/secret" |
| "golang.org/x/oauth2" |
| "golang.org/x/oauth2/google" |
| compute "google.golang.org/api/compute/v1" |
| ) |
| |
| var ( |
| hostType = flag.String("host", "", "host type to create") |
| zone = flag.String("zone", "", "if non-empty, force a certain GCP zone") |
| overrideImage = flag.String("override-image", "", "if non-empty, an alternate GCE VM image or container image to use, depending on the host type") |
| serial = flag.Bool("serial", true, "watch serial. Supported for GCE VMs") |
| pauseAfterUp = flag.Duration("pause-after-up", 0, "pause for this duration before buildlet is destroyed") |
| sleepSec = flag.Int("sleep-test-secs", 0, "number of seconds to sleep when buildlet comes up, to test time source; OpenBSD only for now") |
| |
| runBuild = flag.String("run-build", "", "optional builder name to run all.bash or make.bash for") |
| makeOnly = flag.Bool("make-only", false, "if a --run-build builder name is given, this controls whether make.bash or all.bash is run") |
| buildRev = flag.String("rev", "master", "if --run-build is specified, the git hash or branch name to build") |
| |
| useIAPTunnel = flag.Bool("use-iap-tunnel", true, "use an IAP tunnel to connect to GCE builders") |
| |
| awsKeyID = flag.String("aws-key-id", "", "if the builder runs on aws then key id is required. If executed on GCE, it will be retrieved from secrets.") |
| awsAccessKey = flag.String("aws-access-key", "", "if the builder runs on aws then the access key is required. If executed on GCE, it will be retrieved from secrets.") |
| awsRegion = flag.String("aws-region", "", "if non-empty and the requested builder is an EC2 instance, force an EC2 region.") |
| ) |
| |
| var ( |
| computeSvc *compute.Service |
| env *buildenv.Environment |
| ) |
| |
| func main() { |
| buildenv.RegisterFlags() |
| flag.Parse() |
| |
| var bconf *dashboard.BuildConfig |
| if *runBuild != "" { |
| var ok bool |
| bconf, ok = dashboard.Builders[*runBuild] |
| if !ok { |
| log.Fatalf("unknown builder %q", *runBuild) |
| } |
| if *hostType == "" { |
| *hostType = bconf.HostType |
| } |
| } |
| |
| if *hostType == "" { |
| log.Fatalf("missing --host (or --run-build)") |
| } |
| if *sleepSec != 0 && !strings.Contains(*hostType, "openbsd") { |
| log.Fatalf("The --sleep-test-secs is currently only supported for openbsd hosts.") |
| } |
| |
| hconf, ok := dashboard.Hosts[*hostType] |
| if !ok { |
| log.Fatalf("unknown host type %q", *hostType) |
| } |
| if !hconf.IsVM() && !hconf.IsContainer() { |
| log.Fatalf("host type %q is type %q; want a VM or container host type", *hostType, hconf.PoolName()) |
| } |
| if hconf.IsEC2 && (*awsKeyID == "" || *awsAccessKey == "") { |
| if !metadata.OnGCE() { |
| log.Fatal("missing -aws-key-id and -aws-access-key params are required for builders on AWS") |
| } |
| var err error |
| *awsKeyID, *awsAccessKey, err = awsCredentialsFromSecrets() |
| if err != nil { |
| log.Fatalf("unable to retrieve AWS credentials: %s", err) |
| } |
| } |
| if img := *overrideImage; img != "" { |
| if hconf.IsContainer() { |
| hconf.ContainerImage = img |
| } else { |
| hconf.VMImage = img |
| } |
| } |
| vmImageSummary := fmt.Sprintf("%q", hconf.VMImage) |
| if hconf.IsContainer() { |
| containerHost := hconf.ContainerVMImage() |
| if containerHost == "" { |
| containerHost = "default container host" |
| } |
| vmImageSummary = fmt.Sprintf("%s, running container %q", containerHost, hconf.ContainerImage) |
| } |
| |
| env = buildenv.FromFlags() |
| ctx := context.Background() |
| name := fmt.Sprintf("debug-temp-%d-%s", time.Now().Unix(), os.Getenv("USER")) |
| |
| log.Printf("Creating %s (with VM image %s)", name, vmImageSummary) |
| var bc buildlet.Client |
| if hconf.IsEC2 { |
| region := env.AWSRegion |
| if *awsRegion != "" { |
| region = *awsRegion |
| } |
| awsC, err := cloud.NewAWSClient(region, *awsKeyID, *awsAccessKey) |
| if err != nil { |
| log.Fatalf("unable to create aws cloud client: %s", err) |
| } |
| ec2C := buildlet.NewEC2Client(awsC) |
| if err != nil { |
| log.Fatalf("unable to create ec2 client: %v", err) |
| } |
| bc, err = ec2Buildlet(context.Background(), ec2C, hconf, env, name, *hostType, *zone) |
| if err != nil { |
| log.Fatalf("Start EC2 VM: %v", err) |
| } |
| } else { |
| buildenv.CheckUserCredentials() |
| creds, err := env.Credentials(ctx) |
| if err != nil { |
| log.Fatal(err) |
| } |
| computeSvc, _ = compute.New(oauth2.NewClient(ctx, creds.TokenSource)) |
| bc, err = gceBuildlet(creds, env, name, *hostType, *zone) |
| if err != nil { |
| log.Fatalf("Start GCE VM: %v", err) |
| } |
| } |
| dir, err := bc.WorkDir(ctx) |
| log.Printf("WorkDir: %v, %v", dir, err) |
| |
| if *sleepSec > 0 { |
| bc.Exec(ctx, "sysctl", buildlet.ExecOpts{ |
| Output: os.Stdout, |
| SystemLevel: true, |
| Args: []string{"kern.timecounter.hardware"}, |
| }) |
| bc.Exec(ctx, "bash", buildlet.ExecOpts{ |
| Output: os.Stdout, |
| SystemLevel: true, |
| Args: []string{"-c", "rdate -p -v time.nist.gov; sleep " + fmt.Sprint(*sleepSec) + "; rdate -p -v time.nist.gov"}, |
| }) |
| } |
| |
| var buildFailed bool |
| if *runBuild != "" { |
| // Push GOROOT_BOOTSTRAP, if needed. |
| if u := bconf.GoBootstrapURL(env); u != "" { |
| log.Printf("Pushing 'go1.4' Go bootstrap dir from %s...", u) |
| const bootstrapDir = "go1.4" // might be newer; name is the default |
| if err := bc.PutTarFromURL(ctx, u, bootstrapDir); err != nil { |
| bc.Close() |
| log.Fatalf("Putting Go bootstrap: %v", err) |
| } |
| } |
| |
| // Push Go code |
| log.Printf("Pushing 'go' dir...") |
| goTarGz := "https://go.googlesource.com/go/+archive/" + *buildRev + ".tar.gz" |
| if err := bc.PutTarFromURL(ctx, goTarGz, "go"); err != nil { |
| bc.Close() |
| log.Fatalf("Putting go code: %v", err) |
| } |
| |
| // Push a synthetic VERSION file to prevent git usage: |
| if err := bc.PutTar(ctx, buildgo.VersionTgz(*buildRev), "go"); err != nil { |
| bc.Close() |
| log.Fatalf("Putting VERSION file: %v", err) |
| } |
| |
| script := bconf.AllScript() |
| if *makeOnly { |
| script = bconf.MakeScript() |
| } |
| t0 := time.Now() |
| log.Printf("Running %s ...", script) |
| remoteErr, err := bc.Exec(ctx, path.Join("go", script), buildlet.ExecOpts{ |
| Output: os.Stdout, |
| ExtraEnv: bconf.Env(), |
| Debug: true, |
| Args: bconf.AllScriptArgs(), |
| }) |
| if err != nil { |
| log.Fatalf("error trying to run %s: %v", script, err) |
| } |
| if remoteErr != nil { |
| log.Printf("remote failure running %s: %v", script, remoteErr) |
| buildFailed = true |
| } else { |
| log.Printf("ran %s in %v", script, time.Since(t0).Round(time.Second)) |
| } |
| } |
| |
| if *pauseAfterUp != 0 { |
| log.Printf("Sleeping for %v before shutting down...", *pauseAfterUp) |
| time.Sleep(*pauseAfterUp) |
| } |
| if err := bc.Close(); err != nil { |
| log.Fatalf("Close: %v", err) |
| } |
| log.Printf("done.") |
| time.Sleep(2 * time.Second) // wait for serial logging to catch up |
| |
| if buildFailed { |
| os.Exit(1) |
| } |
| } |
| |
| // watchSerial streams the named VM's serial port to log.Printf. It's roughly: |
| // |
| // gcloud compute connect-to-serial-port --zone=xxx $NAME |
| // |
| // but in Go and works. For some reason, gcloud doesn't work as a |
| // child process and has weird errors. |
| // TODO(golang.org/issue/39485) - investigate if this is possible for EC2 instances |
| func watchSerial(zone, name string) { |
| start := int64(0) |
| indent := strings.Repeat(" ", len("2017/07/25 06:37:14 SERIAL: ")) |
| for { |
| sout, err := computeSvc.Instances.GetSerialPortOutput(env.ProjectName, zone, name).Start(start).Do() |
| if err != nil { |
| log.Printf("serial output error: %v", err) |
| return |
| } |
| moved := sout.Next != start |
| start = sout.Next |
| contents := strings.Replace(strings.TrimSpace(sout.Contents), "\r\n", "\r\n"+indent, -1) |
| if contents != "" { |
| log.Printf("SERIAL: %s", contents) |
| } |
| if !moved { |
| time.Sleep(1 * time.Second) |
| } |
| } |
| } |
| |
| // awsCredentialsFromSecrets retrieves AWS credentials from the secret management service. |
| // This function returns the key ID and the access key. |
| func awsCredentialsFromSecrets() (string, string, error) { |
| c, err := secret.NewClient() |
| if err != nil { |
| return "", "", fmt.Errorf("unable to create secret client: %w", err) |
| } |
| ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
| defer cancel() |
| keyID, err := c.Retrieve(ctx, secret.NameAWSKeyID) |
| if err != nil { |
| return "", "", fmt.Errorf("unable to retrieve key ID: %w", err) |
| } |
| accessKey, err := c.Retrieve(ctx, secret.NameAWSAccessKey) |
| if err != nil { |
| return "", "", fmt.Errorf("unable to retrieve access key: %w", err) |
| } |
| return keyID, accessKey, nil |
| } |
| |
| func gceBuildlet(creds *google.Credentials, env *buildenv.Environment, name, hostType, zone string) (buildlet.Client, error) { |
| return buildlet.StartNewVM(creds, env, name, hostType, buildlet.VMOpts{ |
| Zone: zone, |
| OnInstanceRequested: func() { log.Printf("instance requested") }, |
| OnInstanceCreated: func() { |
| log.Printf("instance created") |
| }, |
| OnGotInstanceInfo: func(inst *compute.Instance) { |
| zone := inst.Zone |
| m := regexp.MustCompile(`/projects/([^/]+)/zones/([^/]+)`).FindStringSubmatch(inst.Zone) |
| if m != nil { |
| zone = m[2] |
| } |
| log.Printf("got instance info; running in %v (%v)", inst.Zone, zone) |
| if *serial { |
| go watchSerial(zone, name) |
| } |
| }, |
| OnBeginBuildletProbe: func(buildletURL string) { |
| log.Printf("About to hit %s to see if buildlet is up yet...", buildletURL) |
| }, |
| OnEndBuildletProbe: func(res *http.Response, err error) { |
| if err != nil { |
| log.Printf("client buildlet probe error: %v", err) |
| return |
| } |
| log.Printf("buildlet probe: %s", res.Status) |
| }, |
| UseIAPTunnel: *useIAPTunnel, |
| }) |
| } |
| |
| func ec2Buildlet(ctx context.Context, ec2Client *buildlet.EC2Client, hconf *dashboard.HostConfig, env *buildenv.Environment, name, hostType, zone string) (buildlet.Client, error) { |
| kp, err := buildlet.NewKeyPair() |
| if err != nil { |
| log.Fatalf("key pair failed: %v", err) |
| } |
| return ec2Client.StartNewVM(ctx, env, hconf, name, hostType, &buildlet.VMOpts{ |
| TLS: kp, |
| Zone: zone, |
| OnInstanceRequested: func() { log.Printf("instance requested") }, |
| OnInstanceCreated: func() { |
| log.Printf("instance created") |
| }, |
| OnGotEC2InstanceInfo: func(inst *cloud.Instance) { |
| log.Printf("got instance info: running in %v", inst.Zone) |
| }, |
| OnBeginBuildletProbe: func(buildletURL string) { |
| log.Printf("About to hit %s to see if buildlet is up yet...", buildletURL) |
| }, |
| OnEndBuildletProbe: func(res *http.Response, err error) { |
| if err != nil { |
| log.Printf("client buildlet probe error: %v", err) |
| return |
| } |
| log.Printf("buildlet probe: %s", res.Status) |
| }, |
| }) |
| } |
