internal/fetch: encode and remember source
Add a new experiment for saving source to the DB.
In this CL, the internal/fetch package encodes the source and stores
it in the goPackage struct. Later CLs will save it to the DB.
Change-Id: I4e0ddcec792512974f3f9a4dc1e158c2b409409f
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/257758
Trust: Jonathan Amsterdam <jba@google.com>
Run-TryBot: Jonathan Amsterdam <jba@google.com>
TryBot-Result: kokoro <noreply+kokoro@google.com>
Reviewed-by: Julie Qiu <julie@golang.org>
diff --git a/internal/experiment.go b/internal/experiment.go
index 64fca5a..e57021b 100644
--- a/internal/experiment.go
+++ b/internal/experiment.go
@@ -6,23 +6,25 @@
package internal
const (
- ExperimentAltRequeue = "alt-requeue"
- ExperimentAutocomplete = "autocomplete"
- ExperimentRemoveUnusedAST = "remove-unused-ast"
- ExperimentSidenav = "sidenav"
- ExperimentUnitPage = "unit-page"
- ExperimentUseUnits = "use-units"
+ ExperimentAltRequeue = "alt-requeue"
+ ExperimentAutocomplete = "autocomplete"
+ ExperimentInsertPackageSource = "insert-package-source"
+ ExperimentRemoveUnusedAST = "remove-unused-ast"
+ ExperimentSidenav = "sidenav"
+ ExperimentUnitPage = "unit-page"
+ ExperimentUseUnits = "use-units"
)
// Experiments represents all of the active experiments in the codebase and
// a description of each experiment.
var Experiments = map[string]string{
- ExperimentAltRequeue: "Requeue modules for reprocessing in a different order.",
- ExperimentAutocomplete: "Enable autocomplete with search.",
- ExperimentRemoveUnusedAST: "Prune AST prior to rendering documentation HTML.",
- ExperimentSidenav: "Display documentation index on the left sidenav.",
- ExperimentUnitPage: "Enable the redesigned details page.",
- ExperimentUseUnits: "Read from paths, documentation, readmes, and package_imports tables.",
+ ExperimentAltRequeue: "Requeue modules for reprocessing in a different order.",
+ ExperimentAutocomplete: "Enable autocomplete with search.",
+ ExperimentInsertPackageSource: "Insert the source code of a package in the database.",
+ ExperimentRemoveUnusedAST: "Prune AST prior to rendering documentation HTML.",
+ ExperimentSidenav: "Display documentation index on the left sidenav.",
+ ExperimentUnitPage: "Enable the redesigned details page.",
+ ExperimentUseUnits: "Read from paths, documentation, readmes, and package_imports tables.",
}
// Experiment holds data associated with an experimental feature for frontend
diff --git a/internal/fetch/load.go b/internal/fetch/load.go
index b2d8aac..86203bd 100644
--- a/internal/fetch/load.go
+++ b/internal/fetch/load.go
@@ -130,6 +130,13 @@
if err != nil && !errors.Is(err, dochtml.ErrTooLarge) {
return nil, err
}
+ var src []byte
+ if experiment.IsActive(ctx, internal.ExperimentInsertPackageSource) {
+ src, err = EncodeASTFiles(fset, allGoFiles)
+ if err != nil {
+ return nil, err
+ }
+ }
importPath := path.Join(modulePath, innerPath)
if modulePath == stdlib.ModulePath {
importPath = innerPath
@@ -144,6 +151,7 @@
documentationHTML: docHTML,
goos: goos,
goarch: goarch,
+ source: src,
}, err
}
diff --git a/internal/fetch/package.go b/internal/fetch/package.go
index d803a9b..cefd898 100644
--- a/internal/fetch/package.go
+++ b/internal/fetch/package.go
@@ -44,6 +44,7 @@
// v1path is the package path of a package with major version 1 in a given
// series.
v1path string
+ source []byte // the source files of the package, for generating doc at serving time
}
// extractPackagesFromZip returns a slice of packages from the module zip r.
