internal,static: prevent crawlers from indexing versioned pages
Some search queries are surfacing results for versioned
pkg.go.dev URLs. This change updates the robots meta tag
to skip pages that are not the latest minor version.
Fixes golang/go#54958.
Change-Id: Ib5b98f9e41a594f2a8ad8980338ac2b5d7f6fca0
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/429675
TryBot-Result: kokoro <noreply+kokoro@google.com>
Reviewed-by: Suzy Mueller <suzmue@golang.org>
Run-TryBot: Jamal Carvalho <jamal@golang.org>
diff --git a/internal/frontend/unit.go b/internal/frontend/unit.go
index 6eddc07..cf3ea7b 100644
--- a/internal/frontend/unit.go
+++ b/internal/frontend/unit.go
@@ -56,6 +56,10 @@
// LatestURL is a url pointing to the latest version of a unit.
LatestURL string
+ // IsLatestMinor is true if the version displayed is the latest minor of the unit.
+ // Used to determine the canonical URL for search engines and robots meta directives.
+ IsLatestMinor bool
+
// LatestMinorClass is the CSS class that describes the current unit's minor
// version in relationship to the latest version of the unit.
LatestMinorClass string
@@ -218,6 +222,7 @@
RedirectedFromPath: redirectPath,
DepsDevURL: makeDepsDevURL(),
IsGoProject: isGoProject(um.ModulePath),
+ IsLatestMinor: lv == latestInfo.MinorVersion,
}
// Show the banner if there was no error getting the latest major version,
diff --git a/static/frontend/unit/main/main.tmpl b/static/frontend/unit/main/main.tmpl
index e310b07..6493905 100644
--- a/static/frontend/unit/main/main.tmpl
+++ b/static/frontend/unit/main/main.tmpl
@@ -5,7 +5,11 @@
-->
{{define "canonical"}}
- <link rel="canonical" href="https://pkg.go.dev/{{.Unit.Path}}">
+ {{if .IsLatestMinor}}
+ <link rel="canonical" href="https://pkg.go.dev/{{.Unit.Path}}">
+ {{else}}
+ <meta name="robots" content="noindex">
+ {{end}}
{{end}}
{{define "main-styles"}}
