cmd/greplogs/main.go - build - Git at Google

 // Copyright 2015 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

 // Command greplogs searches Go builder logs.
 //
 //	greplogs [flags] (-e regexp|-E regexp) paths...
 //	greplogs [flags] (-e regexp|-E regexp) -dashboard
 //
 // greplogs finds builder logs matching a given set of regular
 // expressions in Go syntax (godoc.org/regexp/syntax) and extracts
 // failures from them.
 //
 // greplogs can search an arbitrary set of files just like grep.
 // Alternatively, the -dashboard flag causes it to search the logs
 // saved locally by fetchlogs (golang.org/x/build/cmd/fetchlogs).
 package main

 import (
 	"flag"
 	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"regexp"
 	"runtime/debug"
 	"sort"
 	"strings"
 	"time"

 	"github.com/kballard/go-shellquote"
 	"golang.org/x/build/cmd/greplogs/internal/logparse"
 )

 // TODO: If searching dashboard logs, optionally print to builder URLs
 // instead of local file names.

 // TODO: Optionally extract failures and show only those.

 // TODO: Optionally classify matched logs by failure (and show either
 // file name or extracted failure).

 // TODO: Option to print failure summary versus full failure message.

 var (
 	fileRegexps regexpList
 	failRegexps regexpList
 	omit        regexpList
 	knownIssues regexpMap

 	flagDashboard = flag.Bool("dashboard", true, "search dashboard logs from fetchlogs")
 	flagMD        = flag.Bool("md", true, "output in Markdown")
 	flagTriage    = flag.Bool("triage", false, "adjust Markdown output for failure triage")
 	flagDetails   = flag.Bool("details", false, "surround Markdown results in a <details> tag")
 	flagFilesOnly = flag.Bool("l", false, "print only names of matching files")
 	flagColor     = flag.String("color", "auto", "highlight output in color: `mode` is never, always, or auto")

 	color         *colorizer
 	since, before timeFlag
 )

 const (
 	colorPath      = colorFgMagenta
 	colorPathColon = colorFgCyan
 	colorMatch     = colorBold | colorFgRed
 )

 var brokenBuilders []string

 func main() {
 	// XXX What I want right now is just to point it at a bunch of
 	// logs and have it extract the failures.
 	flag.Var(&knownIssues, "known-issue", "add an issue=regexp mapping; if a log matches regexp it will be categorized under issue. One mapping per flag.")
 	flag.Var(&fileRegexps, "e", "show files matching `regexp`; if provided multiple times, files must match all regexps")
 	flag.Var(&failRegexps, "E", "show only errors matching `regexp`; if provided multiple times, an error must match all regexps")
 	flag.Var(&omit, "omit", "omit results for builder names and/or revisions matching `regexp`; if provided multiple times, logs matching any regexp are omitted")
 	flag.Var(&since, "since", "list only failures on revisions since this date, as an RFC-3339 date or date-time")
 	flag.Var(&before, "before", "list only failures on revisions before this date, in the same format as -since")
 	flag.Parse()

 	// Validate flags.
 	if *flagDashboard && flag.NArg() > 0 {
 		fmt.Fprintf(os.Stderr, "-dashboard and paths are incompatible\n")
 		os.Exit(2)
 	}
 	switch *flagColor {
 	case "never":
 		color = newColorizer(false)
 	case "always":
 		color = newColorizer(true)
 	case "auto":
 		color = newColorizer(canColor())
 	default:
 		fmt.Fprintf(os.Stderr, "-color must be one of never, always, or auto")
 		os.Exit(2)
 	}

 	if *flagTriage {
 		*flagFilesOnly = true
 		if len(failRegexps) == 0 && len(fileRegexps) == 0 {
 			failRegexps.Set(".")
 		}

 		if before.Time.IsZero() {
 			year, month, day := time.Now().UTC().Date()
 			before = timeFlag{Time: time.Date(year, month, day, 0, 0, 0, 0, time.UTC)}
 		}

 		var err error
 		brokenBuilders, err = listBrokenBuilders()
 		if err != nil {
 			fmt.Fprintln(os.Stderr, err)
 			os.Exit(1)
 		}
 		if len(brokenBuilders) > 0 {
 			fmt.Fprintf(os.Stderr, "omitting builders with known issues:\n\t%s\n\n", strings.Join(brokenBuilders, "\n\t"))
 		}
 	}

 	status := 1
 	defer func() { os.Exit(status) }()

 	numMatching := 0
 	if *flagMD {
 		args := append([]string{filepath.Base(os.Args[0])}, os.Args[1:]...)
 		fmt.Printf("`%s`\n", shellquote.Join(args...))

 		defer func() {
 			if numMatching == 0 || *flagTriage || *flagDetails {
 				fmt.Printf("\n(%d matching logs)\n", numMatching)
 			}
 		}()
 		if *flagDetails {
 			os.Stdout.WriteString("<details>\n\n")
 			defer os.Stdout.WriteString("\n</details>\n")
 		}
 	}

 	// Gather paths.
 	var paths []string
 	var stripDir string
 	if *flagDashboard {
 		revDir := filepath.Join(xdgCacheDir(), "fetchlogs", "rev")
 		fis, err := ioutil.ReadDir(revDir)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "%s: %s\n", revDir, err)
 			os.Exit(1)
 		}
 		for _, fi := range fis {
 			if !fi.IsDir() {
 				continue
 			}
 			paths = append(paths, filepath.Join(revDir, fi.Name()))
 		}
 		sort.Sort(sort.Reverse(sort.StringSlice(paths)))
 		stripDir = revDir + "/"
 	} else {
 		paths = flag.Args()
 	}

 	// Process files
 	for _, path := range paths {
 		filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
 			if err != nil {
 				status = 2
 				fmt.Fprintf(os.Stderr, "%s: %v\n", path, err)
 				return nil
 			}
 			if info.IsDir() || strings.HasPrefix(filepath.Base(path), ".") {
 				return nil
 			}

 			nicePath := path
 			if stripDir != "" && strings.HasPrefix(path, stripDir) {
 				nicePath = path[len(stripDir):]
 			}

 			found, err := process(path, nicePath)
 			if err != nil {
 				status = 2
 				fmt.Fprintf(os.Stderr, "%s: %v\n", path, err)
 			} else if found {
 				numMatching++
 				if status == 1 {
 					status = 0
 				}
 			}
 			return nil
 		})
 	}
 }

 var pathDateRE = regexp.MustCompile(`^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})-([0-9a-f]+(?:-[0-9a-f]+)?)$`)

 func process(path, nicePath string) (found bool, err error) {
 	// If this is from the dashboard, filter by builder and date and get the builder URL.
 	builder := filepath.Base(nicePath)
 	for _, b := range brokenBuilders {
 		if builder == b {
 			return false, nil
 		}
 	}
 	if omit.AnyMatchString(builder) {
 		return false, nil
 	}

 	if !since.Time.IsZero() || !before.Time.IsZero() {
 		revDir := filepath.Dir(nicePath)
 		revDirBase := filepath.Base(revDir)
 		match := pathDateRE.FindStringSubmatch(revDirBase)
 		if len(match) != 3 {
 			// Without a valid log date we can't filter by it.
 			return false, fmt.Errorf("timestamp not found in rev dir name: %q", revDirBase)
 		}
 		if omit.AnyMatchString(match[2]) {
 			return false, nil
 		}
 		revTime, err := time.Parse("2006-01-02T15:04:05", match[1])
 		if err != nil {
 			return false, err
 		}
 		if !since.Time.IsZero() && revTime.Before(since.Time) {
 			return false, nil
 		}
 		if !before.Time.IsZero() && !revTime.Before(before.Time) {
 			return false, nil
 		}
 	}

 	// TODO: Get the URL from the rev.json metadata
 	var logURL string
 	if link, err := os.Readlink(path); err == nil {
 		hash := filepath.Base(link)
 		logURL = "https://build.golang.org/log/" + hash
 	}

 	// TODO: Use streaming if possible.
 	data, err := ioutil.ReadFile(path)
 	if err != nil {
 		return false, err
 	}

 	// Check regexp match.
 	if !fileRegexps.AllMatch(data) || !failRegexps.AllMatch(data) {
 		return false, nil
 	}

 	printPath := nicePath
 	kiMatches := 0
 	if *flagMD && logURL != "" {
 		prefix := ""
 		if *flagTriage {
 			matches := knownIssues.Matches(data)
 			if len(matches) == 0 {
 				prefix = "- [ ] "
 			} else {
 				kiMatches++
 				prefix = fmt.Sprintf("- [x] (%v) ", strings.Join(matches, ", "))
 			}
 		}
 		printPath = fmt.Sprintf("%s[%s](%s)", prefix, nicePath, logURL)
 	}

 	if *flagFilesOnly {
 		fmt.Printf("%s\n", color.color(printPath, colorPath))
 		return true, nil
 	}

 	timer := time.AfterFunc(30*time.Second, func() {
 		debug.SetTraceback("all")
 		panic("stuck in extracting " + path)
 	})

 	// Extract failures.
 	failures, err := logparse.Extract(string(data), "", "")
 	if err != nil {
 		return false, err
 	}

 	timer.Stop()

 	// Print failures.
 	for _, failure := range failures {
 		var msg []byte
 		if failure.FullMessage != "" {
 			msg = []byte(failure.FullMessage)
 		} else {
 			msg = []byte(failure.Message)
 		}

 		if len(failRegexps) > 0 && !failRegexps.AllMatch(msg) {
 			continue
 		}

 		fmt.Printf("%s%s\n", color.color(printPath, colorPath), color.color(":", colorPathColon))
 		if *flagMD {
 			fmt.Printf("```\n")
 		}
 		if !color.enabled {
 			fmt.Printf("%s", msg)
 		} else {
 			// Find specific matches and highlight them.
 			matches := mergeMatches(append(fileRegexps.Matches(msg),
 				failRegexps.Matches(msg)...))
 			printed := 0
 			for _, m := range matches {
 				fmt.Printf("%s%s", msg[printed:m[0]], color.color(string(msg[m[0]:m[1]]), colorMatch))
 				printed = m[1]
 			}
 			fmt.Printf("%s", msg[printed:])
 		}
 		if *flagMD {
 			fmt.Printf("\n```")
 		}
 		fmt.Printf("\n\n")
 	}
 	return true, nil
 }

 func mergeMatches(matches [][]int) [][]int {
 	sort.Slice(matches, func(i, j int) bool { return matches[i][0] < matches[j][0] })
 	for i := 0; i < len(matches); {
 		m := matches[i]

 		// Combine with later matches.
 		j := i + 1
 		for ; j < len(matches); j++ {
 			m2 := matches[j]
 			if m[1] <= m2[0] {
 				// Overlapping or exactly adjacent.
 				if m2[1] > m[1] {
 					m[1] = m2[1]
 				}
 				m2[0], m2[1] = 0, 0
 			} else {
 				break
 			}
 		}
 		i = j
 	}

 	// Clear out combined matches.
 	j := 0
 	for _, m := range matches {
 		if m[0] == 0 && m[1] == 0 {
 			continue
 		}
 		matches[j] = m
 		j++
 	}
 	return matches[:j]
 }
	// Copyright 2015 The Go Authors. All rights reserved.
	// Use of this source code is governed by a BSD-style
	// license that can be found in the LICENSE file.

	// Command greplogs searches Go builder logs.
	//
	// greplogs [flags] (-e regexp\|-E regexp) paths...
	// greplogs [flags] (-e regexp\|-E regexp) -dashboard
	//
	// greplogs finds builder logs matching a given set of regular
	// expressions in Go syntax (godoc.org/regexp/syntax) and extracts
	// failures from them.
	//
	// greplogs can search an arbitrary set of files just like grep.
	// Alternatively, the -dashboard flag causes it to search the logs
	// saved locally by fetchlogs (golang.org/x/build/cmd/fetchlogs).
	package main

	import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"runtime/debug"
	"sort"
	"strings"
	"time"

	"github.com/kballard/go-shellquote"
	"golang.org/x/build/cmd/greplogs/internal/logparse"
	)

	// TODO: If searching dashboard logs, optionally print to builder URLs
	// instead of local file names.

	// TODO: Optionally extract failures and show only those.

	// TODO: Optionally classify matched logs by failure (and show either
	// file name or extracted failure).

	// TODO: Option to print failure summary versus full failure message.

	var (
	fileRegexps regexpList
	failRegexps regexpList
	omit regexpList
	knownIssues regexpMap

	flagDashboard = flag.Bool("dashboard", true, "search dashboard logs from fetchlogs")
	flagMD = flag.Bool("md", true, "output in Markdown")
	flagTriage = flag.Bool("triage", false, "adjust Markdown output for failure triage")
	flagDetails = flag.Bool("details", false, "surround Markdown results in a <details> tag")
	flagFilesOnly = flag.Bool("l", false, "print only names of matching files")
	flagColor = flag.String("color", "auto", "highlight output in color: `mode` is never, always, or auto")

	color *colorizer
	since, before timeFlag
	)

	const (
	colorPath = colorFgMagenta
	colorPathColon = colorFgCyan
	colorMatch = colorBold \| colorFgRed
	)

	var brokenBuilders []string

	func main() {
	// XXX What I want right now is just to point it at a bunch of
	// logs and have it extract the failures.
	flag.Var(&knownIssues, "known-issue", "add an issue=regexp mapping; if a log matches regexp it will be categorized under issue. One mapping per flag.")
	flag.Var(&fileRegexps, "e", "show files matching `regexp`; if provided multiple times, files must match all regexps")
	flag.Var(&failRegexps, "E", "show only errors matching `regexp`; if provided multiple times, an error must match all regexps")
	flag.Var(&omit, "omit", "omit results for builder names and/or revisions matching `regexp`; if provided multiple times, logs matching any regexp are omitted")
	flag.Var(&since, "since", "list only failures on revisions since this date, as an RFC-3339 date or date-time")
	flag.Var(&before, "before", "list only failures on revisions before this date, in the same format as -since")
	flag.Parse()

	// Validate flags.
	if *flagDashboard && flag.NArg() > 0 {
	fmt.Fprintf(os.Stderr, "-dashboard and paths are incompatible\n")
	os.Exit(2)
	}
	switch *flagColor {
	case "never":
	color = newColorizer(false)
	case "always":
	color = newColorizer(true)
	case "auto":
	color = newColorizer(canColor())
	default:
	fmt.Fprintf(os.Stderr, "-color must be one of never, always, or auto")
	os.Exit(2)
	}

	if *flagTriage {
	*flagFilesOnly = true
	if len(failRegexps) == 0 && len(fileRegexps) == 0 {
	failRegexps.Set(".")
	}

	if before.Time.IsZero() {
	year, month, day := time.Now().UTC().Date()
	before = timeFlag{Time: time.Date(year, month, day, 0, 0, 0, 0, time.UTC)}
	}

	var err error
	brokenBuilders, err = listBrokenBuilders()
	if err != nil {
	fmt.Fprintln(os.Stderr, err)
	os.Exit(1)
	}
	if len(brokenBuilders) > 0 {
	fmt.Fprintf(os.Stderr, "omitting builders with known issues:\n\t%s\n\n", strings.Join(brokenBuilders, "\n\t"))
	}
	}

	status := 1
	defer func() { os.Exit(status) }()

	numMatching := 0
	if *flagMD {
	args := append([]string{filepath.Base(os.Args[0])}, os.Args[1:]...)
	fmt.Printf("`%s`\n", shellquote.Join(args...))

	defer func() {
	if numMatching == 0 \|\| flagTriage \|\| flagDetails {
	fmt.Printf("\n(%d matching logs)\n", numMatching)
	}
	}()
	if *flagDetails {
	os.Stdout.WriteString("<details>\n\n")
	defer os.Stdout.WriteString("\n</details>\n")
	}
	}

	// Gather paths.
	var paths []string
	var stripDir string
	if *flagDashboard {
	revDir := filepath.Join(xdgCacheDir(), "fetchlogs", "rev")
	fis, err := ioutil.ReadDir(revDir)
	if err != nil {
	fmt.Fprintf(os.Stderr, "%s: %s\n", revDir, err)
	os.Exit(1)
	}
	for _, fi := range fis {
	if !fi.IsDir() {
	continue
	}
	paths = append(paths, filepath.Join(revDir, fi.Name()))
	}
	sort.Sort(sort.Reverse(sort.StringSlice(paths)))
	stripDir = revDir + "/"
	} else {
	paths = flag.Args()
	}

	// Process files
	for _, path := range paths {
	filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
	if err != nil {
	status = 2
	fmt.Fprintf(os.Stderr, "%s: %v\n", path, err)
	return nil
	}
	if info.IsDir() \|\| strings.HasPrefix(filepath.Base(path), ".") {
	return nil
	}

	nicePath := path
	if stripDir != "" && strings.HasPrefix(path, stripDir) {
	nicePath = path[len(stripDir):]
	}

	found, err := process(path, nicePath)
	if err != nil {
	status = 2
	fmt.Fprintf(os.Stderr, "%s: %v\n", path, err)
	} else if found {
	numMatching++
	if status == 1 {
	status = 0
	}
	}
	return nil
	})
	}
	}

	var pathDateRE = regexp.MustCompile(`^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})-([0-9a-f]+(?:-[0-9a-f]+)?)$`)

	func process(path, nicePath string) (found bool, err error) {
	// If this is from the dashboard, filter by builder and date and get the builder URL.
	builder := filepath.Base(nicePath)
	for _, b := range brokenBuilders {
	if builder == b {
	return false, nil
	}
	}
	if omit.AnyMatchString(builder) {
	return false, nil
	}

	if !since.Time.IsZero() \|\| !before.Time.IsZero() {
	revDir := filepath.Dir(nicePath)
	revDirBase := filepath.Base(revDir)
	match := pathDateRE.FindStringSubmatch(revDirBase)
	if len(match) != 3 {
	// Without a valid log date we can't filter by it.
	return false, fmt.Errorf("timestamp not found in rev dir name: %q", revDirBase)
	}
	if omit.AnyMatchString(match[2]) {
	return false, nil
	}
	revTime, err := time.Parse("2006-01-02T15:04:05", match[1])
	if err != nil {
	return false, err
	}
	if !since.Time.IsZero() && revTime.Before(since.Time) {
	return false, nil
	}
	if !before.Time.IsZero() && !revTime.Before(before.Time) {
	return false, nil
	}
	}

	// TODO: Get the URL from the rev.json metadata
	var logURL string
	if link, err := os.Readlink(path); err == nil {
	hash := filepath.Base(link)
	logURL = "https://build.golang.org/log/" + hash
	}

	// TODO: Use streaming if possible.
	data, err := ioutil.ReadFile(path)
	if err != nil {
	return false, err
	}

	// Check regexp match.
	if !fileRegexps.AllMatch(data) \|\| !failRegexps.AllMatch(data) {
	return false, nil
	}

	printPath := nicePath
	kiMatches := 0
	if *flagMD && logURL != "" {
	prefix := ""
	if *flagTriage {
	matches := knownIssues.Matches(data)
	if len(matches) == 0 {
	prefix = "- [ ] "
	} else {
	kiMatches++
	prefix = fmt.Sprintf("- [x] (%v) ", strings.Join(matches, ", "))
	}
	}
	printPath = fmt.Sprintf("%s[%s](%s)", prefix, nicePath, logURL)
	}

	if *flagFilesOnly {
	fmt.Printf("%s\n", color.color(printPath, colorPath))
	return true, nil
	}

	timer := time.AfterFunc(30*time.Second, func() {
	debug.SetTraceback("all")
	panic("stuck in extracting " + path)
	})

	// Extract failures.
	failures, err := logparse.Extract(string(data), "", "")
	if err != nil {
	return false, err
	}

	timer.Stop()

	// Print failures.
	for _, failure := range failures {
	var msg []byte
	if failure.FullMessage != "" {
	msg = []byte(failure.FullMessage)
	} else {
	msg = []byte(failure.Message)
	}

	if len(failRegexps) > 0 && !failRegexps.AllMatch(msg) {
	continue
	}

	fmt.Printf("%s%s\n", color.color(printPath, colorPath), color.color(":", colorPathColon))
	if *flagMD {
	fmt.Printf("```\n")
	}
	if !color.enabled {
	fmt.Printf("%s", msg)
	} else {
	// Find specific matches and highlight them.
	matches := mergeMatches(append(fileRegexps.Matches(msg),
	failRegexps.Matches(msg)...))
	printed := 0
	for _, m := range matches {
	fmt.Printf("%s%s", msg[printed:m[0]], color.color(string(msg[m[0]:m[1]]), colorMatch))
	printed = m[1]
	}
	fmt.Printf("%s", msg[printed:])
	}
	if *flagMD {
	fmt.Printf("\n```")
	}
	fmt.Printf("\n\n")
	}
	return true, nil
	}

	func mergeMatches(matches [][]int) [][]int {
	sort.Slice(matches, func(i, j int) bool { return matches[i][0] < matches[j][0] })
	for i := 0; i < len(matches); {
	m := matches[i]

	// Combine with later matches.
	j := i + 1
	for ; j < len(matches); j++ {
	m2 := matches[j]
	if m[1] <= m2[0] {
	// Overlapping or exactly adjacent.
	if m2[1] > m[1] {
	m[1] = m2[1]
	}
	m2[0], m2[1] = 0, 0
	} else {
	break
	}
	}
	i = j
	}

	// Clear out combined matches.
	j := 0
	for _, m := range matches {
	if m[0] == 0 && m[1] == 0 {
	continue
	}
	matches[j] = m
	j++
	}
	return matches[:j]
	}