blob: 5e2748797b3309adba7b6f836d90fa6cf9cd9bf5 [file] [log] [blame]
// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package worker
import (
"context"
"errors"
"fmt"
"net/http"
"sort"
"strings"
"sync"
"time"
"unicode/utf8"
"go.opencensus.io/trace"
"golang.org/x/mod/semver"
"golang.org/x/pkgsite/internal"
"golang.org/x/pkgsite/internal/cache"
"golang.org/x/pkgsite/internal/derrors"
"golang.org/x/pkgsite/internal/experiment"
"golang.org/x/pkgsite/internal/fetch"
"golang.org/x/pkgsite/internal/log"
"golang.org/x/pkgsite/internal/postgres"
"golang.org/x/pkgsite/internal/proxy"
"golang.org/x/pkgsite/internal/source"
"golang.org/x/pkgsite/internal/stdlib"
)
// ProxyRemoved is a set of module@version that have been removed from the proxy,
// even though they are still in the index.
var ProxyRemoved = map[string]bool{}
// fetchTask represents the result of a fetch task that was processed.
type fetchTask struct {
fetch.FetchResult
timings map[string]time.Duration
}
// A Fetcher holds state for fetching modules.
type Fetcher struct {
ProxyClient *proxy.Client
SourceClient *source.Client
DB *postgres.DB
Cache *cache.Cache
}
// FetchAndUpdateState fetches and processes a module version, and then updates
// the module_version_states table according to the result. It returns an HTTP
// status code representing the result of the fetch operation, and a non-nil
// error if this status code is not 200.
func (f *Fetcher) FetchAndUpdateState(ctx context.Context, modulePath, requestedVersion, appVersionLabel string) (_ int, resolvedVersion string, err error) {
defer derrors.Wrap(&err, "FetchAndUpdateState(%q, %q, %q)", modulePath, requestedVersion, appVersionLabel)
tctx, span := trace.StartSpan(ctx, "FetchAndUpdateState")
ctx = experiment.NewContext(tctx, experiment.FromContext(ctx).Active()...)
ctx = log.NewContextWithLabel(ctx, "fetch", modulePath+"@"+requestedVersion)
if !utf8.ValidString(modulePath) {
log.Errorf(ctx, "module path %q is not valid UTF-8", modulePath)
}
if modulePath == internal.UnknownModulePath {
return http.StatusInternalServerError, "", errors.New("called with internal.UnknownModulePath")
}
if !utf8.ValidString(requestedVersion) {
log.Errorf(ctx, "requested version %q is not valid UTF-8", requestedVersion)
}
span.AddAttributes(
trace.StringAttribute("modulePath", modulePath),
trace.StringAttribute("version", requestedVersion))
defer span.End()
// Get the latest-version information first, and update the DB. We'll need
// it to determine if the current module version is the latest good one for
// its path.
lmv, err := f.FetchAndUpdateLatest(ctx, modulePath)
// The only errors possible here should be DB failures.
if err != nil {
return derrors.ToStatus(err), "", err
}
ft := f.fetchAndInsertModule(ctx, modulePath, requestedVersion, lmv)
span.AddAttributes(trace.Int64Attribute("numPackages", int64(len(ft.PackageVersionStates))))
// If there were any errors processing the module then we didn't insert it.
// Delete it in case we are reprocessing an existing module.
// However, don't delete if the error was internal, or we are shedding load.
if ft.Status >= 400 && ft.Status < 500 {
if err := deleteModule(ctx, f.DB, ft); err != nil {
log.Error(ctx, err)
ft.Error = err
ft.Status = http.StatusInternalServerError
}
// Do not return an error here, because we want to insert into
// module_version_states below.
}
// Regardless of what the status code is, insert the result into
// version_map, so that a response can be returned for frontend_fetch.
if err := updateVersionMap(ctx, f.DB, ft); err != nil {
log.Error(ctx, err)
if ft.Status != http.StatusInternalServerError {
ft.Error = err
ft.Status = http.StatusInternalServerError
}
// Do not return an error here, because we want to insert into
// module_version_states below.
}
if !semver.IsValid(ft.ResolvedVersion) {
// If the requestedVersion was not successfully resolved to a semantic
// version, then at this point it will be the same as the
// resolvedVersion. This fetch request does not need to be recorded in
// module_version_states, since that table is only used to track
// modules that have been published to index.golang.org.
return ft.Status, ft.ResolvedVersion, ft.Error
}
// Check if the latest good version of the module is not the one in search_documents,
// and insert it there and in imports_unique if so.
// Do not bother if this is an alternative module path.
if ft.Status != derrors.ToStatus(derrors.AlternativeModule) || lmv == nil || lmv.CookedVersion != ft.ResolvedVersion {
if err := f.DB.ReInsertLatestVersion(ctx, modulePath); err != nil {
log.Error(ctx, err)
if ft.Status != http.StatusInternalServerError {
ft.Error = err
ft.Status = http.StatusInternalServerError
}
// Do not return an error here, because we want to insert into
// module_version_states below.
}
}
// Update the module_version_states table with the new status of
// module@version. This must happen last, because if it succeeds with a
// code < 500 but a later action fails, we will never retry the later
// action.
// TODO(golang/go#39628): Split UpsertModuleVersionState into
// InsertModuleVersionState and UpdateModuleVersionState.
start := time.Now()
mvs := &postgres.ModuleVersionStateForUpsert{
ModulePath: ft.ModulePath,
Version: ft.ResolvedVersion,
AppVersion: appVersionLabel,
Status: ft.Status,
HasGoMod: ft.HasGoMod,
GoModPath: ft.GoModPath,
FetchErr: ft.Error,
PackageVersionStates: ft.PackageVersionStates,
}
err = f.DB.UpsertModuleVersionState(ctx, mvs)
ft.timings["db.UpsertModuleVersionState"] = time.Since(start)
if err != nil {
log.Error(ctx, err)
if ft.Error != nil {
ft.Status = http.StatusInternalServerError
ft.Error = fmt.Errorf("db.UpsertModuleVersionState: %v, original error: %v", err, ft.Error)
}
logTaskResult(ctx, ft, "Failed to update module version state")
return http.StatusInternalServerError, ft.ResolvedVersion, ft.Error
}
logTaskResult(ctx, ft, "Updated module version state")
return ft.Status, ft.ResolvedVersion, ft.Error
}
// fetchAndInsertModule fetches the given module version from the module proxy
// or (in the case of the standard library) from the Go repo and writes the
// resulting data to the database.
func (f *Fetcher) fetchAndInsertModule(ctx context.Context, modulePath, requestedVersion string, lmv *internal.LatestModuleVersions) *fetchTask {
ft := &fetchTask{
FetchResult: fetch.FetchResult{
ModulePath: modulePath,
RequestedVersion: requestedVersion,
},
timings: map[string]time.Duration{},
}
defer func() {
derrors.Wrap(&ft.Error, "fetchAndInsertModule(%q, %q)", modulePath, requestedVersion)
if ft.Error != nil {
ft.Status = derrors.ToStatus(ft.Error)
ft.ResolvedVersion = requestedVersion
}
}()
if ProxyRemoved[modulePath+"@"+requestedVersion] {
log.Infof(ctx, "not fetching %s@%s because it is on the ProxyRemoved list", modulePath, requestedVersion)
ft.Error = derrors.Excluded
return ft
}
exc, err := f.DB.IsExcluded(ctx, modulePath)
if err != nil {
ft.Error = err
return ft
}
if exc {
ft.Error = derrors.Excluded
return ft
}
// Fetch the module, and the current @main and @master version of this module.
// The @main and @master version will be used to update the version_map
// target if applicable.
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
start := time.Now()
fr := fetch.FetchModule(ctx, modulePath, requestedVersion, f.ProxyClient, f.SourceClient)
if fr == nil {
panic("fetch.FetchModule should never return a nil FetchResult")
}
defer fr.Defer()
ft.FetchResult = *fr
ft.timings["fetch.FetchModule"] = time.Since(start)
}()
// Do not resolve the @main and @master version if proxy fetch is disabled.
var main string
wg.Add(1)
go func() {
defer wg.Done()
if !f.ProxyClient.FetchDisabled() {
main = resolvedVersion(ctx, modulePath, internal.MainVersion, f.ProxyClient)
}
}()
var master string
wg.Add(1)
go func() {
defer wg.Done()
if !f.ProxyClient.FetchDisabled() {
master = resolvedVersion(ctx, modulePath, internal.MasterVersion, f.ProxyClient)
}
}()
wg.Wait()
ft.MainVersion = main
ft.MasterVersion = master
// There was an error fetching this module.
if ft.Error != nil {
logf := log.Infof
if ft.Status >= 500 && ft.Status != derrors.ToStatus(derrors.ProxyTimedOut) {
logf = log.Warningf
}
logf(ctx, "Error executing fetch: %v (code %d)", ft.Error, ft.Status)
return ft
}
// The module was successfully fetched.
log.Infof(ctx, "fetch.FetchModule succeeded for %s@%s", ft.ModulePath, ft.RequestedVersion)
// Determine the current latest-version information for this module.
start := time.Now()
isLatest, err := f.DB.InsertModule(ctx, ft.Module, lmv)
ft.timings["db.InsertModule"] = time.Since(start)
if err != nil {
log.Error(ctx, err)
ft.Status = derrors.ToStatus(err)
ft.Error = err
return ft
}
log.Infof(ctx, "db.InsertModule succeeded for %s@%s", ft.ModulePath, ft.RequestedVersion)
// Invalidate the cache if we just processed the latest version of a module.
if isLatest {
if err := f.invalidateCache(ctx, ft.ModulePath); err != nil {
// Failure to invalidate the cache is not that serious; at worst it means some pages will be stale.
// (Cache TTLs for details pages configured in internal/frontend/server.go must not be too long,
// to account for this possibility.)
log.Errorf(ctx, "failed to invalidate cache for %s: %v", ft.ModulePath, err)
} else {
log.Infof(ctx, "invalidated cache for %s", ft.ModulePath)
}
}
return ft
}
// invalidateCache deletes the series path for modulePath, as well as any
// possible URL path of which it is a componentwise prefix. That is, it deletes
// example.com/mod, example.com/mod@v1.2.3 and example.com/mod/pkg, but not the
// unrelated example.com/module.
//
// We delete the series path, not the module path, because adding a v2 module
// can affect v1 pages. For example, the first v2 module will add a "higher
// major version" banner to all v1 pages. While adding a v1 version won't
// currently affect v2 pages, that could change some day (for instance, if we
// decide to provide history). So it's better to be safe and delete all paths in
// the series.
func (f *Fetcher) invalidateCache(ctx context.Context, modulePath string) error {
if f.Cache == nil {
return nil
}
var errs []error
seriesPath := internal.SeriesPathForModule(modulePath)
// All cache keys are request URLs, so they begin with "/".
if err := f.Cache.Delete(ctx, "/"+seriesPath); err != nil {
errs = append(errs, err)
}
// Delete all suffixes of the series path followed by a character that marks its end.
for _, end := range "/@?#" {
if err := f.Cache.DeletePrefix(ctx, fmt.Sprintf("/%s%c", seriesPath, end)); err != nil {
errs = append(errs, err)
}
}
if len(errs) > 0 {
return fmt.Errorf("%d errors, first is %w", len(errs), errs[0])
}
return nil
}
func resolvedVersion(ctx context.Context, modulePath, requestedVersion string, proxyClient *proxy.Client) string {
if modulePath == stdlib.ModulePath && requestedVersion == internal.MainVersion {
return ""
}
info, err := fetch.GetInfo(ctx, modulePath, requestedVersion, proxyClient)
if err != nil {
if !errors.Is(err, derrors.NotFound) {
// If an error occurs, log it as a warning and insert the module as normal.
log.Warningf(ctx, "fetch.GetInfo(ctx, %q, %q, f.ProxyClient, false): %v", modulePath, requestedVersion, err)
}
return ""
}
return info.Version
}
func updateVersionMap(ctx context.Context, db *postgres.DB, ft *fetchTask) (err error) {
start := time.Now()
defer func() {
ft.timings["worker.updatedVersionMap"] = time.Since(start)
derrors.Wrap(&err, "updateVersionMap(%q, %q, %q, %d, %v)",
ft.ModulePath, ft.RequestedVersion, ft.ResolvedVersion, ft.Status, ft.Error)
}()
ctx, span := trace.StartSpan(ctx, "worker.updateVersionMap")
defer span.End()
var errMsg string
if ft.Error != nil {
errMsg = ft.Error.Error()
}
// If the resolved version for the this module version is also the resolved
// version for @main or @master, update version_map to match.
requestedVersions := []string{ft.RequestedVersion}
if ft.MainVersion == ft.ResolvedVersion {
requestedVersions = append(requestedVersions, internal.MainVersion)
}
if ft.MasterVersion == ft.ResolvedVersion {
requestedVersions = append(requestedVersions, internal.MasterVersion)
}
for _, v := range requestedVersions {
v := v
vm := &internal.VersionMap{
ModulePath: ft.ModulePath,
RequestedVersion: v,
ResolvedVersion: ft.ResolvedVersion,
Status: ft.Status,
GoModPath: ft.GoModPath,
Error: errMsg,
}
if err := db.UpsertVersionMap(ctx, vm); err != nil {
return err
}
}
return nil
}
func deleteModule(ctx context.Context, db *postgres.DB, ft *fetchTask) (err error) {
start := time.Now()
defer func() {
ft.timings["worker.deleteModule"] = time.Since(start)
derrors.Wrap(&err, "deleteModule(%q, %q, %q, %d, %v)",
ft.ModulePath, ft.RequestedVersion, ft.ResolvedVersion, ft.Status, ft.Error)
}()
ctx, span := trace.StartSpan(ctx, "worker.deleteModule")
defer span.End()
log.Infof(ctx, "%s@%s: code=%d, deleting", ft.ModulePath, ft.ResolvedVersion, ft.Status)
if err := db.DeleteModule(ctx, ft.ModulePath, ft.ResolvedVersion); err != nil {
return err
}
// If this was an alternative path (ft.Status == 491) and there is an older
// version in search_documents, delete it. This is the case where a module's
// canonical path was changed by the addition of a go.mod file. For example,
// versions of logrus before it acquired a go.mod file could have the path
// github.com/Sirupsen/logrus, but once the go.mod file specifies that the
// path is all lower-case, the old versions should not show up in search. We
// still leave their pages in the database so users of those old versions
// can still view documentation.
if ft.Status == derrors.ToStatus(derrors.AlternativeModule) {
log.Infof(ctx, "%s@%s: code=491, deleting older version from search", ft.ModulePath, ft.ResolvedVersion)
if err := db.DeleteOlderVersionFromSearchDocuments(ctx, ft.ModulePath, ft.ResolvedVersion); err != nil {
return err
}
}
return nil
}
func logTaskResult(ctx context.Context, ft *fetchTask, prefix string) {
var times []string
for k, v := range ft.timings {
times = append(times, fmt.Sprintf("%s=%.3fs", k, v.Seconds()))
}
sort.Strings(times)
msg := strings.Join(times, ", ")
logf := log.Infof
if ft.Status == http.StatusInternalServerError {
logf = log.Errorf
}
logf(ctx, "%s for %s@%s: code=%d, num_packages=%d, err=%v; timings: %s",
prefix, ft.ModulePath, ft.ResolvedVersion, ft.Status, len(ft.PackageVersionStates), ft.Error, msg)
}
// FetchAndUpdateLatest fetches information about the latest versions from the proxy,
// and updates the database if the version has changed.
// It returns the most recent good information, which may be what it just fetched or
// may be what is already in the DB.
// It does not update the latest good version; that happens inside InsertModule, because
// it must be protected by the module-path advisory lock.
func (f *Fetcher) FetchAndUpdateLatest(ctx context.Context, modulePath string) (_ *internal.LatestModuleVersions, err error) {
defer derrors.Wrap(&err, "FetchAndUpdateLatest(%q)", modulePath)
lmv, err := fetch.LatestModuleVersions(ctx, modulePath, f.ProxyClient, func(v string) (bool, error) {
return f.DB.HasGoMod(ctx, modulePath, v)
})
var status int
switch {
case lmv != nil:
status = 200
case err == nil:
// There may be no version information for the module, even if it exists.
// In that case, we insert a 404 into the DB.
status = 404
default:
status = derrors.ToStatus(err)
}
if status != 200 {
return nil, f.DB.UpdateLatestModuleVersionsStatus(ctx, modulePath, status)
}
return f.DB.UpdateLatestModuleVersions(ctx, lmv)
}