internal/frontend: enqueue fetch request for all requests to path@master
Once we turn on the feature flag for supporting requests to master, we
need a way to ensure that master is not stale for a given path.
As a result, all requests to path@master will be enqueued, once we
determine that it is a valid path.
Updates golang/go#36811
Change-Id: I498179586ee46acbd7347eb70a90104fe9e0b8a8
Reviewed-on: https://team-review.git.corp.google.com/c/golang/discovery/+/769967
Reviewed-by: Jonathan Amsterdam <jba@google.com>
diff --git a/internal/frontend/details.go b/internal/frontend/details.go
index 138de81..868f210 100644
--- a/internal/frontend/details.go
+++ b/internal/frontend/details.go
@@ -16,6 +16,7 @@
"golang.org/x/pkgsite/internal"
"golang.org/x/pkgsite/internal/derrors"
"golang.org/x/pkgsite/internal/experiment"
+ "golang.org/x/pkgsite/internal/log"
"golang.org/x/pkgsite/internal/postgres"
"golang.org/x/pkgsite/internal/stdlib"
)
@@ -88,6 +89,21 @@
if err := checkPathAndVersion(ctx, s.ds, fullPath, requestedVersion); err != nil {
return err
}
+ if isActivePathAtMaster(ctx) && requestedVersion == internal.MasterVersion {
+ // Since path@master is a moving target, we don't want it to be stale.
+ // As a result, we enqueue every request of path@master to the frontend
+ // task queue, which will initiate a fetch request depending on the
+ // last time we tried to fetch this module version.
+ go func() {
+ status, responseText := s.fetchAndPoll(r.Context(), modulePath, fullPath, requestedVersion)
+ logf := log.Infof
+ if status == http.StatusInternalServerError {
+ logf = log.Errorf
+ }
+ logf(ctx, "fetchAndPoll(%q, %q, %q) result from serveDetails(%q): %d %q",
+ modulePath, fullPath, requestedVersion, r.URL.Path, status, responseText)
+ }()
+ }
// Depending on what the request was for, return the module or package page.
if isModule || fullPath == stdlib.ModulePath {
return s.serveModulePage(w, r, fullPath, requestedVersion)
diff --git a/internal/frontend/fetch.go b/internal/frontend/fetch.go
index 99de031..6686bb7 100644
--- a/internal/frontend/fetch.go
+++ b/internal/frontend/fetch.go
@@ -40,7 +40,9 @@
fetchTimeout = 30 * time.Second
pollEvery = 500 * time.Millisecond
- // keyFrontendFetchStatus is a census tag for frontend fetch query types.
+ // keyFrontendFetchVersion is a census tag for frontend fetch version types.
+ keyFrontendFetchVersion = tag.MustNewKey("frontend-fetch.version")
+ // keyFrontendFetchStatus is a census tag for frontend fetch status types.
keyFrontendFetchStatus = tag.MustNewKey("frontend-fetch.status")
// keyFrontendFetchLatency holds observed latency in individual
// frontend fetch queries.
@@ -82,7 +84,8 @@
http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden)
return
}
- if !isActiveFrontendFetch(r.Context()) {
+ ctx := r.Context()
+ if !isActiveFrontendFetch(ctx) {
// If the experiment flag is not on, treat this as a request for the
// "fetch" package, which does not exist.
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
@@ -95,9 +98,11 @@
http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
return
}
- start := time.Now()
+ if !isActivePathAtMaster(ctx) && requestedVersion != internal.MasterVersion {
+ http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
+ return
+ }
status, responseText := s.fetchAndPoll(r.Context(), modulePath, fullPath, requestedVersion)
- recordFrontendFetchMetric(status, time.Since(start))
if status != http.StatusOK {
http.Error(w, responseText, status)
return
@@ -118,9 +123,11 @@
}
func (s *Server) fetchAndPoll(parentCtx context.Context, modulePath, fullPath, requestedVersion string) (status int, responseText string) {
+ start := time.Now()
defer func() {
log.Infof(parentCtx, "fetchAndPoll(ctx, ds, q, %q, %q, %q): status=%d, responseText=%q",
modulePath, fullPath, requestedVersion, status, responseText)
+ recordFrontendFetchMetric(status, requestedVersion, time.Since(start))
}()
if !semver.IsValid(requestedVersion) &&
@@ -274,6 +281,8 @@
// Check the version_map table to see if a row exists for modulePath and
// requestedVersion.
+ // TODO(golang/go#37002): update db.GetVersionMap to return updated_at,
+ // so that we can determine if a module version is stale.
vm, err := db.GetVersionMap(ctx, modulePath, requestedVersion)
if err != nil {
// If an error is returned, there are two possibilities:
@@ -460,9 +469,16 @@
experiment.IsActive(ctx, internal.ExperimentInsertDirectories)
}
-func recordFrontendFetchMetric(status int, latency time.Duration) {
+func recordFrontendFetchMetric(status int, version string, latency time.Duration) {
l := float64(latency) / float64(time.Millisecond)
+
+ // Tag versions based on latest, master and semver.
+ v := version
+ if semver.IsValid(v) {
+ v = "semver"
+ }
stats.RecordWithTags(context.Background(), []tag.Mutator{
tag.Upsert(keyFrontendFetchStatus, strconv.Itoa(status)),
+ tag.Upsert(keyFrontendFetchVersion, v),
}, keyFrontendFetchLatency.M(l))
}
