cmd/fetchlogs/fetchlogs.go - build - Git at Google

 // Copyright 2015 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

 // Fetchlogs downloads build failure logs from the Go dashboard so
 // they can be accessed and searched from the local file system.
 //
 // It organizes these logs into two directories created in the
 // directory specified by the -dir flag (which typically defaults to
 // ~/.cache/fetchlogs). The log/ directory contains all log files
 // named the same way they are named by the dashboard (which happens
 // to be the SHA-1 of their contents). The rev/ directory contains
 // symlinks back to these logs named
 //
 //    rev/<ISO 8601 commit date>-<git revision>/<builder>
 //
 // Fetchlogs will reuse existing log files and revision symlinks, so
 // it only has to download logs that are new since the last time it
 // was run.
 //
 // This makes failures easily searchable with standard tools. For
 // example, to list the revisions and builders with a particular
 // failure, use:
 //
 //    grep -lR <regexp> rev | sort
 package main

 import (
 	"bytes"
 	"encoding/json"
 	"flag"
 	"fmt"
 	"io"
 	"log"
 	"net/http"
 	"net/url"
 	"os"
 	"path/filepath"
 	"sync"
 	"time"

 	"golang.org/x/build/repos"
 	"golang.org/x/build/types"
 )

 var defaultDir = filepath.Join(xdgCacheDir(), "fetchlogs")

 var (
 	flagN         = flag.Int("n", 300, "limit to most recent `N` commits")
 	flagPar       = flag.Int("j", 5, "number of concurrent download `jobs`")
 	flagDir       = flag.String("dir", defaultDir, "`directory` to save logs to")
 	flagRepo      = flag.String("repo", "go", `repo to fetch logs for`)
 	flagDashboard = flag.String("dashboard", "https://build.golang.org", `the dashboard root url`)
 )

 func main() {
 	log.SetPrefix("fetchlogs: ")
 	log.SetFlags(0)

 	flag.Parse()
 	if flag.NArg() != 0 {
 		flag.Usage()
 		os.Exit(2)
 	}

 	// If the top-level directory is the default XDG cache
 	// directory, make sure it exists.
 	if *flagDir == defaultDir {
 		if err := xdgCreateDir(*flagDir); err != nil {
 			log.Fatal(err)
 		}
 	}

 	// Create directory structure.
 	if err := os.Chdir(*flagDir); err != nil {
 		log.Fatal(err)
 	}
 	ensureDir("log")
 	ensureDir("rev")

 	// Set up fetchers.
 	fetcher := newFetcher(*flagPar)
 	wg := sync.WaitGroup{}

 	// Fetch dashboard pages.
 	haveCommits := 0
 	for page := 0; haveCommits < *flagN; page++ {
 		dashURL := fmt.Sprintf("%s/?mode=json&page=%d", *flagDashboard, page)
 		if *flagRepo != "go" {
 			repo := repos.ByGerritProject[*flagRepo]
 			if repo == nil {
 				log.Fatalf("unknown repo %s", *flagRepo)
 			}
 			dashURL += "&repo=" + url.QueryEscape(repo.ImportPath)
 		}
 		index, err := fetcher.get(dashURL)
 		if err != nil {
 			log.Fatal(err)
 		}

 		var status types.BuildStatus
 		if err = json.NewDecoder(index).Decode(&status); err != nil {
 			log.Fatal("error unmarshalling result: ", err)
 		}
 		index.Close()

 		for _, rev := range status.Revisions {
 			haveCommits++
 			if haveCommits > *flagN {
 				break
 			}
 			if rev.Repo != *flagRepo {
 				continue
 			}

 			// Create a revision directory. This way we
 			// have a record of commits with no failures.
 			date, err := parseRevDate(rev.Date)
 			if err != nil {
 				log.Fatal("malformed revision date: ", err)
 			}
 			revDir := revToDir(rev.Revision, date)
 			ensureDir(revDir)

 			// Save revision metadata.
 			buf := bytes.Buffer{}
 			enc := json.NewEncoder(&buf)
 			if err = enc.Encode(rev); err != nil {
 				log.Fatal(err)
 			}
 			if err = writeFileAtomic(filepath.Join(revDir, ".rev.json"), &buf); err != nil {
 				log.Fatal("error saving revision metadata: ", err)
 			}

 			// Save builders list so Results list can be
 			// interpreted.
 			if err = enc.Encode(status.Builders); err != nil {
 				log.Fatal(err)
 			}
 			if err = writeFileAtomic(filepath.Join(revDir, ".builders.json"), &buf); err != nil {
 				log.Fatal("error saving builders metadata: ", err)
 			}

 			// Fetch revision logs.
 			for i, res := range rev.Results {
 				if res == "" || res == "ok" {
 					continue
 				}

 				wg.Add(1)
 				go func(rev, builder, logURL string) {
 					defer wg.Done()
 					logPath := filepath.Join("log", filepath.Base(logURL))
 					err := fetcher.getFile(logURL, logPath)
 					if err != nil {
 						log.Fatal("error fetching log: ", err)
 					}
 					if err := linkLog(revDir, builder, logPath); err != nil {
 						log.Fatal("error linking log: ", err)
 					}
 				}(revDir, status.Builders[i], res)
 			}
 		}
 	}

 	wg.Wait()
 }

 // A fetcher downloads files over HTTP concurrently. It allows
 // limiting the number of concurrent downloads and correctly handles
 // multiple (possibly concurrent) fetches from the same URL to the
 // same file.
 type fetcher struct {
 	tokens chan struct{}

 	pending struct {
 		sync.Mutex
 		m map[string]*pendingFetch
 	}
 }

 type pendingFetch struct {
 	wchan chan struct{} // closed when fetch completes

 	// err is the error, if any, that occurred during this fetch.
 	// It will be set before wchan is closed.
 	err error
 }

 func newFetcher(jobs int) *fetcher {
 	f := new(fetcher)

 	f.tokens = make(chan struct{}, *flagPar)
 	for i := 0; i < jobs; i++ {
 		f.tokens <- struct{}{}
 	}

 	f.pending.m = make(map[string]*pendingFetch)

 	return f
 }

 // get performs an HTTP GET for URL and returns the body, while
 // obeying the job limit on fetcher.
 func (f *fetcher) get(url string) (io.ReadCloser, error) {
 	<-f.tokens
 	fmt.Println("fetching", url)
 	resp, err := http.Get(url)
 	f.tokens <- struct{}{}
 	if err != nil {
 		return nil, err
 	}

 	return resp.Body, nil
 }

 // getFile performs an HTTP GET for URL and writes it to filename. If
 // the destination file already exists, this returns immediately. If
 // another goroutine is currently fetching filename, this blocks until
 // the fetch is done and then returns.
 func (f *fetcher) getFile(url string, filename string) error {
 	// Do we already have it?
 	if _, err := os.Stat(filename); err == nil {
 		return nil
 	} else if !os.IsNotExist(err) {
 		return err
 	}

 	// Check if another fetcher is working on it.
 	f.pending.Lock()
 	if p, ok := f.pending.m[filename]; ok {
 		f.pending.Unlock()
 		<-p.wchan
 		return p.err
 	}

 	p := &pendingFetch{wchan: make(chan struct{})}
 	f.pending.m[filename] = p
 	f.pending.Unlock()

 	r, err := f.get(url)
 	if err == nil {
 		err = writeFileAtomic(filename, r)
 		r.Close()
 	}
 	p.err = err

 	close(p.wchan)
 	return p.err
 }

 // ensureDir creates directory name if it does not exist.
 func ensureDir(name string) {
 	err := os.Mkdir(name, 0777)
 	if err != nil && !os.IsExist(err) {
 		log.Fatal("error creating directory ", name, ": ", err)
 	}
 }

 // writeFileAtomic atomically creates a file called filename and
 // copies the data from r to the file.
 func writeFileAtomic(filename string, r io.Reader) error {
 	tmpPath := filename + ".tmp"
 	if f, err := os.Create(tmpPath); err != nil {
 		return err
 	} else {
 		_, err := io.Copy(f, r)
 		if err == nil {
 			err = f.Sync()
 		}
 		err2 := f.Close()
 		if err == nil {
 			err = err2
 		}
 		if err != nil {
 			os.Remove(tmpPath)
 			return err
 		}
 	}
 	if err := os.Rename(tmpPath, filename); err != nil {
 		os.Remove(tmpPath)
 		return err
 	}
 	return nil
 }

 // linkLog creates a symlink for finding logPath based on its git
 // revision and builder.
 func linkLog(revDir, builder, logPath string) error {
 	// Create symlink.
 	err := os.Symlink("../../"+logPath, filepath.Join(revDir, builder))
 	if err != nil && !os.IsExist(err) {
 		return err
 	}

 	return nil
 }

 // parseRevDate parses a revision date in RFC3339.
 func parseRevDate(date string) (time.Time, error) {
 	return time.Parse(time.RFC3339, date)
 }

 // revToDir returns the path of the revision directory for revision.
 func revToDir(revision string, date time.Time) string {
 	return filepath.Join("rev", date.Format("2006-01-02T15:04:05")+"-"+revision[:7])
 }
	// Copyright 2015 The Go Authors. All rights reserved.
	// Use of this source code is governed by a BSD-style
	// license that can be found in the LICENSE file.

	// Fetchlogs downloads build failure logs from the Go dashboard so
	// they can be accessed and searched from the local file system.
	//
	// It organizes these logs into two directories created in the
	// directory specified by the -dir flag (which typically defaults to
	// ~/.cache/fetchlogs). The log/ directory contains all log files
	// named the same way they are named by the dashboard (which happens
	// to be the SHA-1 of their contents). The rev/ directory contains
	// symlinks back to these logs named
	//
	// rev/<ISO 8601 commit date>-<git revision>/<builder>
	//
	// Fetchlogs will reuse existing log files and revision symlinks, so
	// it only has to download logs that are new since the last time it
	// was run.
	//
	// This makes failures easily searchable with standard tools. For
	// example, to list the revisions and builders with a particular
	// failure, use:
	//
	// grep -lR <regexp> rev \| sort
	package main

	import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"sync"
	"time"

	"golang.org/x/build/repos"
	"golang.org/x/build/types"
	)

	var defaultDir = filepath.Join(xdgCacheDir(), "fetchlogs")

	var (
	flagN = flag.Int("n", 300, "limit to most recent `N` commits")
	flagPar = flag.Int("j", 5, "number of concurrent download `jobs`")
	flagDir = flag.String("dir", defaultDir, "`directory` to save logs to")
	flagRepo = flag.String("repo", "go", `repo to fetch logs for`)
	flagDashboard = flag.String("dashboard", "https://build.golang.org", `the dashboard root url`)
	)

	func main() {
	log.SetPrefix("fetchlogs: ")
	log.SetFlags(0)

	flag.Parse()
	if flag.NArg() != 0 {
	flag.Usage()
	os.Exit(2)
	}

	// If the top-level directory is the default XDG cache
	// directory, make sure it exists.
	if *flagDir == defaultDir {
	if err := xdgCreateDir(*flagDir); err != nil {
	log.Fatal(err)
	}
	}

	// Create directory structure.
	if err := os.Chdir(*flagDir); err != nil {
	log.Fatal(err)
	}
	ensureDir("log")
	ensureDir("rev")

	// Set up fetchers.
	fetcher := newFetcher(*flagPar)
	wg := sync.WaitGroup{}

	// Fetch dashboard pages.
	haveCommits := 0
	for page := 0; haveCommits < *flagN; page++ {
	dashURL := fmt.Sprintf("%s/?mode=json&page=%d", *flagDashboard, page)
	if *flagRepo != "go" {
	repo := repos.ByGerritProject[*flagRepo]
	if repo == nil {
	log.Fatalf("unknown repo %s", *flagRepo)
	}
	dashURL += "&repo=" + url.QueryEscape(repo.ImportPath)
	}
	index, err := fetcher.get(dashURL)
	if err != nil {
	log.Fatal(err)
	}

	var status types.BuildStatus
	if err = json.NewDecoder(index).Decode(&status); err != nil {
	log.Fatal("error unmarshalling result: ", err)
	}
	index.Close()

	for _, rev := range status.Revisions {
	haveCommits++
	if haveCommits > *flagN {
	break
	}
	if rev.Repo != *flagRepo {
	continue
	}

	// Create a revision directory. This way we
	// have a record of commits with no failures.
	date, err := parseRevDate(rev.Date)
	if err != nil {
	log.Fatal("malformed revision date: ", err)
	}
	revDir := revToDir(rev.Revision, date)
	ensureDir(revDir)

	// Save revision metadata.
	buf := bytes.Buffer{}
	enc := json.NewEncoder(&buf)
	if err = enc.Encode(rev); err != nil {
	log.Fatal(err)
	}
	if err = writeFileAtomic(filepath.Join(revDir, ".rev.json"), &buf); err != nil {
	log.Fatal("error saving revision metadata: ", err)
	}

	// Save builders list so Results list can be
	// interpreted.
	if err = enc.Encode(status.Builders); err != nil {
	log.Fatal(err)
	}
	if err = writeFileAtomic(filepath.Join(revDir, ".builders.json"), &buf); err != nil {
	log.Fatal("error saving builders metadata: ", err)
	}

	// Fetch revision logs.
	for i, res := range rev.Results {
	if res == "" \|\| res == "ok" {
	continue
	}

	wg.Add(1)
	go func(rev, builder, logURL string) {
	defer wg.Done()
	logPath := filepath.Join("log", filepath.Base(logURL))
	err := fetcher.getFile(logURL, logPath)
	if err != nil {
	log.Fatal("error fetching log: ", err)
	}
	if err := linkLog(revDir, builder, logPath); err != nil {
	log.Fatal("error linking log: ", err)
	}
	}(revDir, status.Builders[i], res)
	}
	}
	}

	wg.Wait()
	}

	// A fetcher downloads files over HTTP concurrently. It allows
	// limiting the number of concurrent downloads and correctly handles
	// multiple (possibly concurrent) fetches from the same URL to the
	// same file.
	type fetcher struct {
	tokens chan struct{}

	pending struct {
	sync.Mutex
	m map[string]*pendingFetch
	}
	}

	type pendingFetch struct {
	wchan chan struct{} // closed when fetch completes

	// err is the error, if any, that occurred during this fetch.
	// It will be set before wchan is closed.
	err error
	}

	func newFetcher(jobs int) *fetcher {
	f := new(fetcher)

	f.tokens = make(chan struct{}, *flagPar)
	for i := 0; i < jobs; i++ {
	f.tokens <- struct{}{}
	}

	f.pending.m = make(map[string]*pendingFetch)

	return f
	}

	// get performs an HTTP GET for URL and returns the body, while
	// obeying the job limit on fetcher.
	func (f *fetcher) get(url string) (io.ReadCloser, error) {
	<-f.tokens
	fmt.Println("fetching", url)
	resp, err := http.Get(url)
	f.tokens <- struct{}{}
	if err != nil {
	return nil, err
	}

	return resp.Body, nil
	}

	// getFile performs an HTTP GET for URL and writes it to filename. If
	// the destination file already exists, this returns immediately. If
	// another goroutine is currently fetching filename, this blocks until
	// the fetch is done and then returns.
	func (f *fetcher) getFile(url string, filename string) error {
	// Do we already have it?
	if _, err := os.Stat(filename); err == nil {
	return nil
	} else if !os.IsNotExist(err) {
	return err
	}

	// Check if another fetcher is working on it.
	f.pending.Lock()
	if p, ok := f.pending.m[filename]; ok {
	f.pending.Unlock()
	<-p.wchan
	return p.err
	}

	p := &pendingFetch{wchan: make(chan struct{})}
	f.pending.m[filename] = p
	f.pending.Unlock()

	r, err := f.get(url)
	if err == nil {
	err = writeFileAtomic(filename, r)
	r.Close()
	}
	p.err = err

	close(p.wchan)
	return p.err
	}

	// ensureDir creates directory name if it does not exist.
	func ensureDir(name string) {
	err := os.Mkdir(name, 0777)
	if err != nil && !os.IsExist(err) {
	log.Fatal("error creating directory ", name, ": ", err)
	}
	}

	// writeFileAtomic atomically creates a file called filename and
	// copies the data from r to the file.
	func writeFileAtomic(filename string, r io.Reader) error {
	tmpPath := filename + ".tmp"
	if f, err := os.Create(tmpPath); err != nil {
	return err
	} else {
	_, err := io.Copy(f, r)
	if err == nil {
	err = f.Sync()
	}
	err2 := f.Close()
	if err == nil {
	err = err2
	}
	if err != nil {
	os.Remove(tmpPath)
	return err
	}
	}
	if err := os.Rename(tmpPath, filename); err != nil {
	os.Remove(tmpPath)
	return err
	}
	return nil
	}

	// linkLog creates a symlink for finding logPath based on its git
	// revision and builder.
	func linkLog(revDir, builder, logPath string) error {
	// Create symlink.
	err := os.Symlink("../../"+logPath, filepath.Join(revDir, builder))
	if err != nil && !os.IsExist(err) {
	return err
	}

	return nil
	}

	// parseRevDate parses a revision date in RFC3339.
	func parseRevDate(date string) (time.Time, error) {
	return time.Parse(time.RFC3339, date)
	}

	// revToDir returns the path of the revision directory for revision.
	func revToDir(revision string, date time.Time) string {
	return filepath.Join("rev", date.Format("2006-01-02T15:04:05")+"-"+revision[:7])
	}