blob: 5167a670f0626f37a8820f7ed7094f166e19c755 [file] [log] [blame]
// Copyright 2017 The Go Authors. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd.
package main
import (
"context"
"log"
"time"
"cloud.google.com/go/trace"
"github.com/golang/gddo/gosrc"
)
func (s *server) doCrawl(ctx context.Context) error {
span := s.traceClient.NewSpan("Crawl")
defer span.Finish()
ctx = trace.NewContext(ctx, span)
// Look for new package to crawl.
importPath, hasSubdirs, err := s.db.PopNewCrawl()
if err != nil {
log.Printf("db.PopNewCrawl() returned error %v", err)
return nil
}
if importPath != "" {
if pdoc, err := s.crawlDoc(ctx, "new", importPath, nil, hasSubdirs, time.Time{}); pdoc == nil && err == nil {
if err := s.db.AddBadCrawl(importPath); err != nil {
log.Printf("ERROR db.AddBadCrawl(%q): %v", importPath, err)
}
}
return nil
}
// Crawl existing doc.
pdoc, pkgs, nextCrawl, err := s.db.Get(ctx, "-")
if err != nil {
log.Printf("db.Get(\"-\") returned error %v", err)
return nil
}
if pdoc == nil || nextCrawl.After(time.Now()) {
return nil
}
if _, err = s.crawlDoc(ctx, "crawl", pdoc.ImportPath, pdoc, len(pkgs) > 0, nextCrawl); err != nil {
// Touch package so that crawl advances to next package.
if err := s.db.SetNextCrawl(pdoc.ImportPath, time.Now().Add(s.v.GetDuration(ConfigMaxAge)/3)); err != nil {
log.Printf("ERROR db.SetNextCrawl(%q): %v", pdoc.ImportPath, err)
}
}
return nil
}
func (s *server) readGitHubUpdates(ctx context.Context) error {
span := s.traceClient.NewSpan("GitHubUpdates")
defer span.Finish()
ctx = trace.NewContext(ctx, span)
const key = "gitHubUpdates"
var last string
if err := s.db.GetGob(key, &last); err != nil {
return err
}
last, names, err := gosrc.GetGitHubUpdates(ctx, s.httpClient, last)
if err != nil {
return err
}
for _, name := range names {
log.Printf("bump crawl github.com/%s", name)
if err := s.db.BumpCrawl("github.com/" + name); err != nil {
log.Println("ERROR force crawl:", err)
}
}
if err := s.db.PutGob(key, last); err != nil {
return err
}
return nil
}