internal/postgres: read and write source files
Write the encoded source files to the documentation.source column
when a module is inserted, and read them back from getDocumentation.
Change-Id: I9fc27c570c701522c9e00c6bf9270dbe51f8602d
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/258237
Trust: Jonathan Amsterdam <jba@google.com>
Run-TryBot: Jonathan Amsterdam <jba@google.com>
TryBot-Result: kokoro <noreply+kokoro@google.com>
Reviewed-by: Julie Qiu <julie@golang.org>
diff --git a/internal/fetch/unit.go b/internal/fetch/unit.go
index d9a916d..bfbe19b 100644
--- a/internal/fetch/unit.go
+++ b/internal/fetch/unit.go
@@ -66,6 +66,7 @@
GOARCH: pkg.goarch,
Synopsis: pkg.synopsis,
HTML: pkg.documentationHTML,
+ Source: pkg.source,
}
}
units = append(units, dir)
diff --git a/internal/postgres/insert_module.go b/internal/postgres/insert_module.go
index e7ed5b1..3191d08 100644
--- a/internal/postgres/insert_module.go
+++ b/internal/postgres/insert_module.go
@@ -24,6 +24,7 @@
"golang.org/x/pkgsite/internal"
"golang.org/x/pkgsite/internal/database"
"golang.org/x/pkgsite/internal/derrors"
+ "golang.org/x/pkgsite/internal/experiment"
"golang.org/x/pkgsite/internal/log"
"golang.org/x/pkgsite/internal/stdlib"
"golang.org/x/pkgsite/internal/version"
@@ -426,7 +427,12 @@
pathToReadme[d.Path] = d.Readme
}
if d.Documentation != nil && d.Documentation.HTML.String() == internal.StringFieldMissing {
- return errors.New("saveModule: package missing Documentation.HTML")
+ return errors.New("insertUnits: package missing Documentation.HTML")
+ }
+ if experiment.IsActive(ctx, internal.ExperimentInsertPackageSource) {
+ if d.Documentation.Source == nil {
+ return errors.New("insertUnits: package missing source files")
+ }
}
pathToDoc[d.Path] = d.Documentation
if len(d.Imports) > 0 {
@@ -503,9 +509,15 @@
}
id := pathToID[path]
docValues = append(docValues, id, doc.GOOS, doc.GOARCH, doc.Synopsis, makeValidUnicode(doc.HTML.String()))
+ if experiment.IsActive(ctx, internal.ExperimentInsertPackageSource) {
+ docValues = append(docValues, doc.Source)
+ }
}
uniqueCols := []string{"path_id", "goos", "goarch"}
docCols := append(uniqueCols, "synopsis", "html")
+ if experiment.IsActive(ctx, internal.ExperimentInsertPackageSource) {
+ docCols = append(docCols, "source")
+ }
if err := db.BulkUpsert(ctx, "documentation", docCols, docValues, uniqueCols); err != nil {
return err
}
diff --git a/internal/postgres/unit.go b/internal/postgres/unit.go
index 2e0d2e4..84a588c 100644
--- a/internal/postgres/unit.go
+++ b/internal/postgres/unit.go
@@ -108,7 +108,8 @@
d.goos,
d.goarch,
d.synopsis,
- d.html
+ d.html,
+ d.source
FROM documentation d
WHERE
d.path_id=$1;`, pathID).Scan(
@@ -116,6 +117,7 @@
database.NullIsEmpty(&doc.GOARCH),
database.NullIsEmpty(&doc.Synopsis),
database.NullIsEmpty(&docHTML),
+ &doc.Source,
)
switch err {
case sql.ErrNoRows:
diff --git a/internal/unit.go b/internal/unit.go
index 8b2e13b..c9b5f7f 100644
--- a/internal/unit.go
+++ b/internal/unit.go
@@ -66,6 +66,7 @@
GOARCH string
Synopsis string
HTML safehtml.HTML
+ Source []byte // encoded ast.Files; see fetch.EncodeASTFiles
}
// Readme is a README at the specified filepath.
