blob: 26992300738002c73d37e151a410d32ef2003d90 [file] [log] [blame]
// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package worker
import (
"context"
"sort"
"strings"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/google/safehtml/testconversions"
"golang.org/x/pkgsite/internal"
"golang.org/x/pkgsite/internal/licenses"
"golang.org/x/pkgsite/internal/postgres"
"golang.org/x/pkgsite/internal/proxy"
"golang.org/x/pkgsite/internal/source"
"golang.org/x/pkgsite/internal/stdlib"
"golang.org/x/pkgsite/internal/testing/sample"
"golang.org/x/pkgsite/internal/testing/testhelper"
)
const (
// Indicates that although we have a valid module, some packages could not be processed.
hasIncompletePackagesCode = 290
hasIncompletePackagesDesc = "incomplete packages"
testAppVersion = "appVersionLabel"
)
var (
sourceTimeout = 1 * time.Second
testProxyCommitTime = time.Date(2019, 1, 30, 0, 0, 0, 0, time.UTC)
)
var buildConstraintsMod = &proxy.Module{
ModulePath: "build.constraints/module",
Version: sample.VersionString,
Files: map[string]string{
"LICENSE": testhelper.BSD0License,
"cpu/cpu.go": `
// Package cpu implements processor feature detection
// used by the Go standard library.
package cpu`,
"cpu/cpu_arm.go": "package cpu\n\nconst CacheLinePadSize = 1",
"cpu/cpu_arm64.go": "package cpu\n\nconst CacheLinePadSize = 2",
"cpu/cpu_x86.go": "// +build 386 amd64 amd64p32\n\npackage cpu\n\nconst CacheLinePadSize = 3",
"ignore/ignore.go": "// +build ignore\n\npackage ignore",
},
}
var html = testconversions.MakeHTMLForTest
func TestFetchAndUpdateState(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
defer cancel()
const goRepositoryURLPrefix = "https://github.com/golang"
stdlib.UseTestData = true
defer func() { stdlib.UseTestData = false }()
proxyClient, teardownProxy := proxy.SetupTestClient(t, []*proxy.Module{
buildConstraintsMod,
{
ModulePath: "github.com/my/module",
Files: map[string]string{
"go.mod": "module github.com/my/module\n\ngo 1.12",
"LICENSE": testhelper.BSD0License,
"README.md": "README FILE FOR TESTING.",
"bar/LICENSE": testhelper.MITLicense,
"bar/bar.go": `
// package bar
package bar
// Bar returns the string "bar".
func Bar() string {
return "bar"
}`,
"foo/LICENSE.md": testhelper.MITLicense,
"foo/foo.go": `
// package foo
package foo
import (
"fmt"
"github.com/my/module/bar"
)
// FooBar returns the string "foo bar".
func FooBar() string {
return fmt.Sprintf("foo %s", bar.Bar())
}`,
},
},
{
ModulePath: "nonredistributable.mod/module",
Files: map[string]string{
"go.mod": "module nonredistributable.mod/module\n\ngo 1.13",
"LICENSE": testhelper.BSD0License,
"README.md": "README FILE FOR TESTING.",
"bar/baz/COPYING": testhelper.MITLicense,
"bar/baz/baz.go": `
// package baz
package baz
// Baz returns the string "baz".
func Baz() string {
return "baz"
}
`,
"bar/LICENSE": testhelper.MITLicense,
"bar/bar.go": `
// package bar
package bar
// Bar returns the string "bar".
func Bar() string {
return "bar"
}`,
"foo/LICENSE.md": testhelper.UnknownLicense,
"foo/foo.go": `
// package foo
package foo
import (
"fmt"
"github.com/my/module/bar"
)
// FooBar returns the string "foo bar".
func FooBar() string {
return fmt.Sprintf("foo %s", bar.Bar())
}`,
},
},
})
defer teardownProxy()
myModuleV100 := &internal.Unit{
UnitMeta: internal.UnitMeta{
ModulePath: "github.com/my/module",
Version: sample.VersionString,
CommitTime: testProxyCommitTime,
SourceInfo: source.NewGitHubInfo("https://github.com/my/module", "", sample.VersionString),
IsRedistributable: true,
Path: "github.com/my/module/bar",
Name: "bar",
Licenses: []*licenses.Metadata{
{Types: []string{"BSD-0-Clause"}, FilePath: "LICENSE"},
{Types: []string{"MIT"}, FilePath: "bar/LICENSE"},
},
},
Documentation: &internal.Documentation{
Synopsis: "package bar",
HTML: html("Bar returns the string "bar"."),
GOOS: "linux",
GOARCH: "amd64",
},
Readme: &internal.Readme{
Filepath: "README.md",
Contents: "README FILE FOR TESTING.",
},
}
testCases := []struct {
modulePath string
version string
pkg string
want *internal.Unit
moreWantDoc []string // Additional substrings we expect to see in DocumentationHTML.
dontWantDoc []string // Substrings we expect not to see in DocumentationHTML.
}{
{
modulePath: "github.com/my/module",
version: sample.VersionString,
pkg: "github.com/my/module/bar",
want: myModuleV100,
},
{
modulePath: "github.com/my/module",
version: internal.LatestVersion,
pkg: "github.com/my/module/bar",
want: myModuleV100,
},
{
// nonredistributable.mod/module is redistributable, as are its
// packages bar and bar/baz. But package foo is not.
modulePath: "nonredistributable.mod/module",
version: sample.VersionString,
pkg: "nonredistributable.mod/module/bar/baz",
want: &internal.Unit{
UnitMeta: internal.UnitMeta{
ModulePath: "nonredistributable.mod/module",
Version: "v1.0.0",
CommitTime: testProxyCommitTime,
SourceInfo: nil,
IsRedistributable: true,
Path: "nonredistributable.mod/module/bar/baz",
Name: "baz",
Licenses: []*licenses.Metadata{
{Types: []string{"BSD-0-Clause"}, FilePath: "LICENSE"},
{Types: []string{"MIT"}, FilePath: "bar/LICENSE"},
{Types: []string{"MIT"}, FilePath: "bar/baz/COPYING"},
},
},
Documentation: &internal.Documentation{
Synopsis: "package baz",
HTML: html("Baz returns the string "baz"."),
GOOS: "linux",
GOARCH: "amd64",
},
Readme: &internal.Readme{
Filepath: "README.md",
Contents: "README FILE FOR TESTING.",
},
},
}, {
modulePath: "nonredistributable.mod/module",
version: sample.VersionString,
pkg: "nonredistributable.mod/module/foo",
want: &internal.Unit{
UnitMeta: internal.UnitMeta{
ModulePath: "nonredistributable.mod/module",
Version: sample.VersionString,
CommitTime: testProxyCommitTime,
SourceInfo: nil,
IsRedistributable: false,
Path: "nonredistributable.mod/module/foo",
Name: "foo",
Licenses: []*licenses.Metadata{
{Types: []string{"BSD-0-Clause"}, FilePath: "LICENSE"},
{Types: []string{"UNKNOWN"}, FilePath: "foo/LICENSE.md"},
},
},
},
}, {
modulePath: "std",
version: "v1.12.5",
pkg: "context",
want: &internal.Unit{
UnitMeta: internal.UnitMeta{
ModulePath: "std",
Version: "v1.12.5",
CommitTime: stdlib.TestCommitTime,
SourceInfo: source.NewGitHubInfo(goRepositoryURLPrefix+"/go", "src", "go1.12.5"),
IsRedistributable: true,
Path: "context",
Name: "context",
Licenses: []*licenses.Metadata{
{
Types: []string{"BSD-3-Clause"},
FilePath: "LICENSE",
},
},
},
Documentation: &internal.Documentation{
Synopsis: "Package context defines the Context type, which carries deadlines, cancelation signals, and other request-scoped values across API boundaries and between processes.",
HTML: html("This example demonstrates the use of a cancelable context to prevent a\ngoroutine leak."),
GOOS: "linux",
GOARCH: "amd64",
},
Readme: &internal.Readme{
Filepath: "README.md",
Contents: "# The Go Programming Language\n",
},
},
}, {
modulePath: "std",
version: "v1.12.5",
pkg: "builtin",
want: &internal.Unit{
UnitMeta: internal.UnitMeta{
ModulePath: "std",
Version: "v1.12.5",
CommitTime: stdlib.TestCommitTime,
SourceInfo: source.NewGitHubInfo(goRepositoryURLPrefix+"/go", "src", "go1.12.5"),
IsRedistributable: true,
Path: "builtin",
Name: "builtin",
Licenses: []*licenses.Metadata{
{
Types: []string{"BSD-3-Clause"},
FilePath: "LICENSE",
},
},
},
Documentation: &internal.Documentation{
Synopsis: "Package builtin provides documentation for Go's predeclared identifiers.",
HTML: html("int64 is the set of all signed 64-bit integers."),
GOOS: "linux",
GOARCH: "amd64",
},
Readme: &internal.Readme{
Filepath: "README.md",
Contents: "# The Go Programming Language\n",
},
},
}, {
modulePath: "std",
version: "v1.12.5",
pkg: "encoding/json",
want: &internal.Unit{
UnitMeta: internal.UnitMeta{
ModulePath: "std",
Version: "v1.12.5",
CommitTime: stdlib.TestCommitTime,
SourceInfo: source.NewGitHubInfo(goRepositoryURLPrefix+"/go", "src", "go1.12.5"),
IsRedistributable: true,
Path: "encoding/json",
Name: "json",
Licenses: []*licenses.Metadata{
{
Types: []string{"BSD-3-Clause"},
FilePath: "LICENSE",
},
},
},
Documentation: &internal.Documentation{
HTML: html("The mapping between JSON and Go values is described\nin the documentation for the Marshal and Unmarshal functions."),
Synopsis: "Package json implements encoding and decoding of JSON as defined in RFC 7159.",
GOOS: "linux",
GOARCH: "amd64",
},
Readme: &internal.Readme{
Filepath: "README.md",
Contents: "# The Go Programming Language\n",
},
},
moreWantDoc: []string{
"Example (CustomMarshalJSON)",
`<summary class="Documentation-exampleDetailsHeader">Example (CustomMarshalJSON) <a href="#example-package-CustomMarshalJSON">ΒΆ</a></summary>`,
"Package (CustomMarshalJSON)",
`<li><a href="#example-package-CustomMarshalJSON" class="js-exampleHref">Package (CustomMarshalJSON)</a></li>`,
"Decoder.Decode (Stream)",
`<li><a href="#example-Decoder.Decode-Stream" class="js-exampleHref">Decoder.Decode (Stream)</a></li>`,
},
dontWantDoc: []string{
"Example (customMarshalJSON)",
"Package (customMarshalJSON)",
"Decoder.Decode (stream)",
},
}, {
modulePath: buildConstraintsMod.ModulePath,
version: buildConstraintsMod.Version,
pkg: buildConstraintsMod.ModulePath + "/cpu",
want: &internal.Unit{
UnitMeta: internal.UnitMeta{
ModulePath: buildConstraintsMod.ModulePath,
Version: buildConstraintsMod.Version,
CommitTime: testProxyCommitTime,
IsRedistributable: true,
Path: buildConstraintsMod.ModulePath + "/cpu",
Name: "cpu",
Licenses: []*licenses.Metadata{
{Types: []string{"BSD-0-Clause"}, FilePath: "LICENSE"},
},
},
Documentation: &internal.Documentation{
Synopsis: "Package cpu implements processor feature detection used by the Go standard library.",
HTML: html("const CacheLinePadSize = 3"),
GOOS: "linux",
GOARCH: "amd64",
},
},
dontWantDoc: []string{
"const CacheLinePadSize = 1",
"const CacheLinePadSize = 2",
},
},
}
for _, test := range testCases {
t.Run(test.pkg, func(t *testing.T) {
defer postgres.ResetTestDB(testDB, t)
sourceClient := source.NewClient(sourceTimeout)
if _, err := FetchAndUpdateState(ctx, test.modulePath, test.version, proxyClient, sourceClient, testDB, testAppVersion); err != nil {
t.Fatalf("FetchAndUpdateState(%q, %q, %v, %v, %v): %v", test.modulePath, test.version, proxyClient, sourceClient, testDB, err)
}
got, err := testDB.GetUnitMeta(ctx, test.pkg, test.modulePath, test.want.Version)
if err != nil {
t.Fatal(err)
}
sort.Slice(got.Licenses, func(i, j int) bool {
return got.Licenses[i].FilePath < got.Licenses[j].FilePath
})
if diff := cmp.Diff(test.want.UnitMeta, *got, cmp.AllowUnexported(source.Info{})); diff != "" {
t.Fatalf("testDB.GetUnitMeta(ctx, %q, %q) mismatch (-want +got):\n%s", test.modulePath, test.version, diff)
}
gotPkg, err := testDB.GetUnit(ctx, got, internal.WithReadme|internal.WithDocumentation)
if err != nil {
t.Fatal(err)
}
if diff := cmp.Diff(test.want, gotPkg,
cmp.AllowUnexported(source.Info{}),
cmpopts.IgnoreFields(internal.Unit{}, "Documentation")); diff != "" {
t.Errorf("mismatch on readme (-want +got):\n%s", diff)
}
if got, want := gotPkg.Documentation, test.want.Documentation; got == nil || want == nil {
if got != want {
t.Fatalf("mismatch on documentation: got: %v\nwant: %v", got, want)
}
return
}
if !strings.Contains(gotPkg.Documentation.HTML.String(), test.want.Documentation.HTML.String()) {
t.Errorf("got documentation doesn't contain wanted documentation substring:\ngot: %q\nwant (substring): %q",
gotPkg.Documentation.HTML.String(), test.want.Documentation.HTML.String())
}
for _, want := range test.moreWantDoc {
if got := gotPkg.Documentation.HTML.String(); !strings.Contains(got, want) {
t.Errorf("got documentation doesn't contain wanted documentation substring:\ngot: %q\nwant (substring): %q", got, want)
}
}
for _, dontWant := range test.dontWantDoc {
if got := gotPkg.Documentation.HTML.String(); strings.Contains(got, dontWant) {
t.Errorf("got documentation contains unwanted documentation substring:\ngot: %q\ndontWant (substring): %q", got, dontWant)
}
}
})
}
}