internal/postgres: ensure doc HTML is valid UTF-8 We've seen documentation that is not valid UTF-8 (github.com/xuender/oil/math). That results in a DB error when written to a TEXT column. Make the documentation valid before writing to the DB. Change-Id: Ib2ccdd1dfb3b66af9d444b81216f561a8d906d8a Reviewed-on: https://team-review.git.corp.google.com/c/golang/discovery/+/757995 Reviewed-by: Julie Qiu <julieqiu@google.com> CI-Result: Cloud Build <devtools-proctor-result-processor@system.gserviceaccount.com>
diff --git a/internal/postgres/insert_module.go b/internal/postgres/insert_module.go index 2fe2c5b..4c3e33f 100644 --- a/internal/postgres/insert_module.go +++ b/internal/postgres/insert_module.go
@@ -274,7 +274,7 @@ m.ModulePath, p.V1Path, p.IsRedistributable, - p.DocumentationHTML, + makeValidUnicode(p.DocumentationHTML), pq.Array(licenseTypes), pq.Array(licensePaths), p.GOOS, @@ -465,7 +465,7 @@ continue } id := pathToID[path] - docValues = append(docValues, id, doc.GOOS, doc.GOARCH, doc.Synopsis, doc.HTML) + docValues = append(docValues, id, doc.GOOS, doc.GOARCH, doc.Synopsis, makeValidUnicode(doc.HTML)) } docCols := []string{ "path_id",