internal/frontend: return 400 on @master requests for stdlib paths

At the moment, we do not support requests to @master for packages in the
standard library. Until that support is added, return a 400 for those
requests, instead of showing a 404 page where the user could attempt to
fetch the path.

fetchAndPoll is also updated to return a 400 on requests to @master and
@latest for packages in the standard library.

Updates golang/go#36811
Updates golang/go#39973

Change-Id: I372ad2686547278754f2c04e69ddea7fc092df34
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/240682
Reviewed-by: Jonathan Amsterdam <jba@google.com>
diff --git a/internal/frontend/details.go b/internal/frontend/details.go
index 9170239..20ea1fb 100644
--- a/internal/frontend/details.go
+++ b/internal/frontend/details.go
@@ -216,7 +216,7 @@
 // validatePathAndVersion verifies that the requested path and version are
 // acceptable. The given path may be a module or package path.
 func validatePathAndVersion(ctx context.Context, ds internal.DataSource, fullPath, requestedVersion string) error {
-	if !isSupportedVersion(ctx, requestedVersion) {
+	if !isSupportedVersion(ctx, fullPath, requestedVersion) {
 		return &serverError{
 			status: http.StatusBadRequest,
 			epage: &errorPage{
@@ -245,7 +245,10 @@
 }
 
 // isSupportedVersion reports whether the version is supported by the frontend.
-func isSupportedVersion(ctx context.Context, version string) bool {
+func isSupportedVersion(ctx context.Context, fullPath, version string) bool {
+	if stdlib.Contains(fullPath) && version == internal.MasterVersion {
+		return false
+	}
 	if version == internal.LatestVersion || semver.IsValid(version) {
 		return true
 	}
diff --git a/internal/frontend/fetch.go b/internal/frontend/fetch.go
index 826c16f..4cae89d 100644
--- a/internal/frontend/fetch.go
+++ b/internal/frontend/fetch.go
@@ -27,6 +27,7 @@
 	"golang.org/x/pkgsite/internal/postgres"
 	"golang.org/x/pkgsite/internal/proxy"
 	"golang.org/x/pkgsite/internal/source"
+	"golang.org/x/pkgsite/internal/stdlib"
 )
 
 var (
@@ -76,8 +77,9 @@
 // Meanwhile, the request will poll the database until a row is found, or a
 // timeout occurs. A status and responseText will be returned based on the
 // result of the request.
-// TODO(golang/go#37002): This should be a POST request, since it is causing a change in state.
-// update middleware.AcceptMethods so that this can be a POST instead of a GET.
+// TODO(https://golang.org/issue/39979): This should be a POST request, since it is causing a change
+// in state. Update middleware.AcceptMethods so that this can be a POST instead of a
+// GET.
 func (s *Server) fetchHandler(w http.ResponseWriter, r *http.Request) {
 	if _, ok := s.ds.(*postgres.DB); !ok {
 		// There's no reason for the proxydatasource to need this codepath.
@@ -130,9 +132,10 @@
 		recordFrontendFetchMetric(status, requestedVersion, time.Since(start))
 	}()
 
-	if !semver.IsValid(requestedVersion) &&
-		requestedVersion != internal.MasterVersion &&
-		requestedVersion != internal.LatestVersion {
+	if !isSupportedVersion(parentCtx, fullPath, requestedVersion) ||
+		// TODO(https://golang.org/issue/39973): add support for fetching the
+		// latest and master versions of the standard library
+		(stdlib.Contains(fullPath) && requestedVersion == internal.LatestVersion) {
 		return http.StatusBadRequest, http.StatusText(http.StatusBadRequest)
 	}
 
diff --git a/internal/frontend/fetch_test.go b/internal/frontend/fetch_test.go
index 7b56bc1..d18346e 100644
--- a/internal/frontend/fetch_test.go
+++ b/internal/frontend/fetch_test.go
@@ -74,7 +74,10 @@
 			ctx, cancel := context.WithTimeout(context.Background(), testFetchTimeout)
 			defer cancel()
 			ctx = experiment.NewContext(ctx, experiment.NewSet(map[string]bool{
-				internal.ExperimentInsertDirectories: true,
+				internal.ExperimentFrontendFetch:               true,
+				internal.ExperimentFrontendPackageAtMaster:     true,
+				internal.ExperimentInsertDirectories:           true,
+				internal.ExperimentUsePathInfoToCheckExistence: true,
 			}))
 
 			status, responseText := s.fetchAndPoll(ctx, testModulePath, test.fullPath, test.version)