| // Copyright 2019 The Go Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style |
| // license that can be found in the LICENSE file. |
| |
| package worker |
| |
| import ( |
| "context" |
| "sort" |
| "strings" |
| "testing" |
| "time" |
| |
| "github.com/google/go-cmp/cmp" |
| "github.com/google/go-cmp/cmp/cmpopts" |
| "golang.org/x/pkgsite/internal" |
| "golang.org/x/pkgsite/internal/godoc" |
| "golang.org/x/pkgsite/internal/licenses" |
| "golang.org/x/pkgsite/internal/postgres" |
| "golang.org/x/pkgsite/internal/proxy/proxytest" |
| "golang.org/x/pkgsite/internal/source" |
| "golang.org/x/pkgsite/internal/stdlib" |
| "golang.org/x/pkgsite/internal/testing/sample" |
| "golang.org/x/pkgsite/internal/version" |
| ) |
| |
| const ( |
| // Indicates that although we have a valid module, some packages could not be processed. |
| hasIncompletePackagesCode = 290 |
| hasIncompletePackagesDesc = "incomplete packages" |
| |
| testAppVersion = "appVersionLabel" |
| buildConstraintsModulePath = "example.com/build-constraints" |
| buildConstraintsVersion = "v1.0.0" |
| ) |
| |
| var ( |
| sourceTimeout = 1 * time.Second |
| testProxyCommitTime = time.Date(2019, 1, 30, 0, 0, 0, 0, time.UTC) |
| ) |
| |
| func TestFetchAndUpdateState(t *testing.T) { |
| ctx, cancel := context.WithTimeout(context.Background(), testTimeout) |
| defer cancel() |
| |
| defer stdlib.WithTestData()() |
| |
| proxyClient, teardownProxy := proxytest.SetupTestClient(t, testModules) |
| defer teardownProxy() |
| |
| myModuleV100 := &internal.Unit{ |
| UnitMeta: internal.UnitMeta{ |
| ModuleInfo: internal.ModuleInfo{ |
| ModulePath: "example.com/multi", |
| HasGoMod: true, |
| Version: sample.VersionString, |
| CommitTime: testProxyCommitTime, |
| SourceInfo: source.NewGitHubInfo("https://example.com/multi", "", sample.VersionString), |
| IsRedistributable: true, |
| }, |
| IsRedistributable: true, |
| Path: "example.com/multi/bar", |
| Name: "bar", |
| Licenses: []*licenses.Metadata{ |
| {Types: []string{"0BSD"}, FilePath: "LICENSE"}, |
| {Types: []string{"MIT"}, FilePath: "bar/LICENSE"}, |
| }, |
| }, |
| Documentation: []*internal.Documentation{{ |
| Synopsis: "package bar", |
| GOOS: "linux", |
| GOARCH: "amd64", |
| }}, |
| Readme: &internal.Readme{ |
| Filepath: "bar/README", |
| Contents: "Another README file for testing.", |
| }, |
| } |
| |
| testCases := []struct { |
| modulePath string |
| version string |
| pkg string |
| want *internal.Unit |
| wantDoc []string // Substrings we expect to see in DocumentationHTML. |
| dontWantDoc []string // Substrings we expect not to see in DocumentationHTML. |
| }{ |
| { |
| modulePath: "example.com/multi", |
| version: sample.VersionString, |
| pkg: "example.com/multi/bar", |
| want: myModuleV100, |
| wantDoc: []string{"Bar returns the string "bar"."}, |
| }, |
| { |
| modulePath: "example.com/multi", |
| version: version.Latest, |
| pkg: "example.com/multi/bar", |
| want: myModuleV100, |
| }, |
| { |
| // example.com/nonredist is redistributable, as are its |
| // packages bar and bar/baz. But package unk is not. |
| modulePath: "example.com/nonredist", |
| version: sample.VersionString, |
| pkg: "example.com/nonredist/bar/baz", |
| want: &internal.Unit{ |
| UnitMeta: internal.UnitMeta{ |
| ModuleInfo: internal.ModuleInfo{ |
| ModulePath: "example.com/nonredist", |
| Version: sample.VersionString, |
| HasGoMod: true, |
| CommitTime: testProxyCommitTime, |
| SourceInfo: source.NewGitHubInfo("https://example.com/nonredist", "", sample.VersionString), |
| IsRedistributable: true, |
| }, |
| IsRedistributable: true, |
| Path: "example.com/nonredist/bar/baz", |
| Name: "baz", |
| Licenses: []*licenses.Metadata{ |
| {Types: []string{"0BSD"}, FilePath: "LICENSE"}, |
| {Types: []string{"MIT"}, FilePath: "bar/LICENSE"}, |
| {Types: []string{"MIT"}, FilePath: "bar/baz/COPYING"}, |
| }, |
| }, |
| Documentation: []*internal.Documentation{{ |
| Synopsis: "package baz", |
| GOOS: "linux", |
| GOARCH: "amd64", |
| }}, |
| }, |
| wantDoc: []string{"Baz returns the string "baz"."}, |
| }, { |
| modulePath: "example.com/nonredist", |
| version: sample.VersionString, |
| pkg: "example.com/nonredist/unk", |
| want: &internal.Unit{ |
| UnitMeta: internal.UnitMeta{ |
| ModuleInfo: internal.ModuleInfo{ |
| ModulePath: "example.com/nonredist", |
| Version: sample.VersionString, |
| HasGoMod: true, |
| CommitTime: testProxyCommitTime, |
| SourceInfo: source.NewGitHubInfo("https://example.com/nonredist", "", sample.VersionString), |
| IsRedistributable: true, |
| }, |
| IsRedistributable: false, |
| Path: "example.com/nonredist/unk", |
| Name: "unk", |
| Licenses: []*licenses.Metadata{ |
| {Types: []string{"0BSD"}, FilePath: "LICENSE"}, |
| {Types: []string{"UNKNOWN"}, FilePath: "unk/LICENSE.md"}, |
| }, |
| }, |
| NumImports: 2, |
| }, |
| }, { |
| modulePath: "std", |
| version: "v1.12.5", |
| pkg: "context", |
| want: &internal.Unit{ |
| UnitMeta: internal.UnitMeta{ |
| ModuleInfo: internal.ModuleInfo{ |
| ModulePath: "std", |
| Version: "v1.12.5", |
| HasGoMod: true, |
| CommitTime: stdlib.TestCommitTime, |
| SourceInfo: source.NewStdlibInfo("v1.12.5"), |
| IsRedistributable: true, |
| }, |
| IsRedistributable: true, |
| Path: "context", |
| Name: "context", |
| Licenses: []*licenses.Metadata{ |
| { |
| Types: []string{"BSD-3-Clause"}, |
| FilePath: "LICENSE", |
| }, |
| }, |
| }, |
| NumImports: 5, |
| Documentation: []*internal.Documentation{{ |
| Synopsis: "Package context defines the Context type, which carries deadlines, cancelation signals, and other request-scoped values across API boundaries and between processes.", |
| GOOS: "linux", |
| GOARCH: "amd64", |
| }}, |
| }, |
| wantDoc: []string{"This example demonstrates the use of a cancelable context to prevent a\ngoroutine leak."}, |
| }, { |
| modulePath: "std", |
| version: "v1.12.5", |
| pkg: "builtin", |
| want: &internal.Unit{ |
| UnitMeta: internal.UnitMeta{ |
| ModuleInfo: internal.ModuleInfo{ |
| ModulePath: "std", |
| Version: "v1.12.5", |
| HasGoMod: true, |
| CommitTime: stdlib.TestCommitTime, |
| SourceInfo: source.NewStdlibInfo("v1.12.5"), |
| IsRedistributable: true, |
| }, |
| IsRedistributable: true, |
| Path: "builtin", |
| Name: "builtin", |
| Licenses: []*licenses.Metadata{ |
| { |
| Types: []string{"BSD-3-Clause"}, |
| FilePath: "LICENSE", |
| }, |
| }, |
| }, |
| Documentation: []*internal.Documentation{{ |
| Synopsis: "Package builtin provides documentation for Go's predeclared identifiers.", |
| GOOS: "linux", |
| GOARCH: "amd64", |
| }}, |
| }, |
| wantDoc: []string{"int64 is the set of all signed 64-bit integers."}, |
| }, { |
| modulePath: "std", |
| version: "v1.12.5", |
| pkg: "encoding/json", |
| want: &internal.Unit{ |
| UnitMeta: internal.UnitMeta{ |
| ModuleInfo: internal.ModuleInfo{ |
| ModulePath: "std", |
| Version: "v1.12.5", |
| HasGoMod: true, |
| CommitTime: stdlib.TestCommitTime, |
| SourceInfo: source.NewStdlibInfo("v1.12.5"), |
| IsRedistributable: true, |
| }, |
| IsRedistributable: true, |
| Path: "encoding/json", |
| Name: "json", |
| Licenses: []*licenses.Metadata{ |
| { |
| Types: []string{"BSD-3-Clause"}, |
| FilePath: "LICENSE", |
| }, |
| }, |
| }, |
| NumImports: 15, |
| Documentation: []*internal.Documentation{{ |
| Synopsis: "Package json implements encoding and decoding of JSON as defined in RFC 7159.", |
| GOOS: "linux", |
| GOARCH: "amd64", |
| }}, |
| }, |
| wantDoc: []string{ |
| "The mapping between JSON and Go values is described\nin the documentation for the Marshal and Unmarshal functions.", |
| "Example (CustomMarshalJSON)", |
| `<summary class="Documentation-exampleDetailsHeader">Example (CustomMarshalJSON) <a href="#example-package-CustomMarshalJSON">ΒΆ</a></summary>`, |
| "Package (CustomMarshalJSON)", |
| `<li><a href="#example-package-CustomMarshalJSON" class="js-exampleHref">Package (CustomMarshalJSON)</a></li>`, |
| "Decoder.Decode (Stream)", |
| `<li><a href="#example-Decoder.Decode-Stream" class="js-exampleHref">Decoder.Decode (Stream)</a></li>`, |
| }, |
| dontWantDoc: []string{ |
| "Example (customMarshalJSON)", |
| "Package (customMarshalJSON)", |
| "Decoder.Decode (stream)", |
| }, |
| }, { |
| modulePath: buildConstraintsModulePath, |
| version: sample.VersionString, |
| pkg: buildConstraintsModulePath + "/cpu", |
| want: &internal.Unit{ |
| UnitMeta: internal.UnitMeta{ |
| ModuleInfo: internal.ModuleInfo{ |
| ModulePath: buildConstraintsModulePath, |
| Version: "v1.0.0", |
| HasGoMod: true, |
| CommitTime: testProxyCommitTime, |
| SourceInfo: source.NewGitHubInfo("https://"+buildConstraintsModulePath, "", sample.VersionString), |
| IsRedistributable: true, |
| }, |
| IsRedistributable: true, |
| Path: buildConstraintsModulePath + "/cpu", |
| Name: "cpu", |
| Licenses: []*licenses.Metadata{ |
| {Types: []string{"0BSD"}, FilePath: "LICENSE"}, |
| }, |
| }, |
| Documentation: []*internal.Documentation{{ |
| Synopsis: "Package cpu implements processor feature detection used by the Go standard library.", |
| GOOS: "linux", |
| GOARCH: "amd64", |
| }}, |
| }, |
| wantDoc: []string{"const CacheLinePadSize = 3"}, |
| dontWantDoc: []string{ |
| "const CacheLinePadSize = 1", |
| "const CacheLinePadSize = 2", |
| }, |
| }, |
| } |
| |
| sourceClient := source.NewClient(sourceTimeout) |
| for _, test := range testCases { |
| t.Run(strings.ReplaceAll(test.pkg+"@"+test.version, "/", " "), func(t *testing.T) { |
| defer postgres.ResetTestDB(testDB, t) |
| f := &Fetcher{ |
| ProxyClient: proxyClient.WithCache(), |
| SourceClient: sourceClient, |
| DB: testDB, |
| Cache: nil, |
| loadShedder: &loadShedder{maxSizeInFlight: 100 * mib}, |
| } |
| if _, _, err := f.FetchAndUpdateState(ctx, test.modulePath, test.version, testAppVersion); err != nil { |
| t.Fatalf("FetchAndUpdateState(%q, %q, %v, %v, %v): %v", test.modulePath, test.version, proxyClient, sourceClient, testDB, err) |
| } |
| |
| got, err := testDB.GetUnitMeta(ctx, test.pkg, test.modulePath, test.want.Version) |
| if err != nil { |
| t.Fatal(err) |
| } |
| sort.Slice(got.Licenses, func(i, j int) bool { |
| return got.Licenses[i].FilePath < got.Licenses[j].FilePath |
| }) |
| if diff := cmp.Diff(test.want.UnitMeta, *got, cmpopts.EquateEmpty(), cmp.AllowUnexported(source.Info{})); diff != "" { |
| t.Fatalf("testDB.GetUnitMeta(ctx, %q, %q) mismatch (-want +got):\n%s", test.modulePath, test.version, diff) |
| } |
| |
| gotPkg, err := testDB.GetUnit(ctx, got, internal.WithMain, internal.BuildContext{}) |
| if err != nil { |
| t.Fatal(err) |
| } |
| |
| if diff := cmp.Diff(test.want, gotPkg, |
| cmpopts.EquateEmpty(), |
| cmp.AllowUnexported(source.Info{}), |
| cmpopts.IgnoreFields(internal.Unit{}, "Documentation", "BuildContexts"), |
| cmpopts.IgnoreFields(internal.Unit{}, "SymbolHistory"), |
| cmpopts.IgnoreFields(internal.Unit{}, "Subdirectories")); diff != "" { |
| t.Errorf("mismatch on readme (-want +got):\n%s", diff) |
| } |
| if got, want := gotPkg.Documentation, test.want.Documentation; got == nil || want == nil { |
| if !cmp.Equal(got, want) { |
| t.Fatalf("mismatch on documentation: got: %v\nwant: %v", got, want) |
| } |
| return |
| } |
| if gotPkg.Documentation != nil { |
| parts, err := godoc.RenderFromUnit(ctx, gotPkg, internal.BuildContext{}) |
| if err != nil { |
| t.Fatal(err) |
| } |
| gotDoc := parts.Body.String() |
| for _, want := range test.wantDoc { |
| if !strings.Contains(gotDoc, want) { |
| t.Errorf("got documentation doesn't contain wanted documentation substring:\ngot: %q\nwant (substring): %q", gotDoc, want) |
| } |
| } |
| for _, dontWant := range test.dontWantDoc { |
| if strings.Contains(gotDoc, dontWant) { |
| t.Errorf("got documentation contains unwanted documentation substring:\ngot: %q\ndontWant (substring): %q", gotDoc, dontWant) |
| } |
| } |
| } |
| // TODO(https://golang.org/issue/43890): fix 500 error for |
| // fetching std@master and update test. |
| if test.modulePath != stdlib.ModulePath { |
| for _, v := range []string{internal.MainVersion, internal.MasterVersion, test.version} { |
| if _, err := testDB.GetVersionMap(ctx, test.modulePath, v); err != nil { |
| t.Error(err) |
| } |
| } |
| } |
| }) |
| } |
| } |
| |
| func TestFetchAndUpdateStateCacheZip(t *testing.T) { |
| // We can try to download a zip from the proxy twice when we are processing |
| // a new module at the latest compatible version, and there is an |
| // incompatible version. In that case, fetch.LatestModuleVersions needs to |
| // download the zip to see if there is a go.mod file, and then the zip is |
| // downloaded again in fetch.FetchModule. To avoid the double download, the |
| // proxy can be set up with a small cache for the last downloaded zip. This |
| // test confirms that that feature works. |
| ctx := context.Background() |
| defer postgres.ResetTestDB(testDB, t) |
| proxyServer := proxytest.NewServer([]*proxytest.Module{ |
| { |
| ModulePath: "m.com", |
| Version: "v2.0.0+incompatible", |
| Files: map[string]string{"a.go": "package a"}, |
| }, |
| { |
| ModulePath: "m.com", |
| Version: "v1.0.0", |
| Files: map[string]string{"a.go": "package a"}, |
| }, |
| }) |
| proxyClient, teardownProxy, err := proxytest.NewClientForServer(proxyServer) |
| if err != nil { |
| t.Fatal(err) |
| } |
| defer teardownProxy() |
| |
| // With a plain proxy, we download the zip twice. |
| f := &Fetcher{proxyClient, source.NewClient(sourceTimeout), testDB, nil, nil, ""} |
| if _, _, err := f.FetchAndUpdateState(ctx, "m.com", "v1.0.0", testAppVersion); err != nil { |
| t.Fatal(err) |
| } |
| if got, want := proxyServer.ZipRequests(), 2; got != want { |
| t.Errorf("got %d downloads, want %d", got, want) |
| } |
| |
| // With the cache, we download it only once. |
| postgres.ResetTestDB(testDB, t) // to avoid finding has_go_mod in the DB |
| f.ProxyClient = proxyClient.WithCache() |
| if _, _, err := f.FetchAndUpdateState(ctx, "m.com", "v1.0.0", testAppVersion); err != nil { |
| t.Fatal(err) |
| } |
| // We want three total zip requests: 2 before, 1 now. |
| if got, want := proxyServer.ZipRequests(), 3; got != want { |
| t.Errorf("got %d downloads, want %d", got, want) |
| } |
| |
| } |
| |
| func TestFetchAndUpdateLatest(t *testing.T) { |
| ctx := context.Background() |
| prox, teardown := proxytest.SetupTestClient(t, testModules) |
| defer teardown() |
| |
| const modulePath = "example.com/retractions" |
| f := &Fetcher{ |
| ProxyClient: prox, |
| SourceClient: source.NewClient(sourceTimeout), |
| DB: testDB, |
| } |
| got, err := f.FetchAndUpdateLatest(ctx, modulePath) |
| if err != nil { |
| t.Fatal(err) |
| } |
| const ( |
| wantRaw = "v1.2.0" |
| wantCooked = "v1.0.0" |
| ) |
| if got.ModulePath != modulePath || got.RawVersion != wantRaw || got.CookedVersion != wantCooked { |
| t.Errorf("got (%q, %q, %q), want (%q, %q, %q)", |
| got.ModulePath, got.RawVersion, got.CookedVersion, |
| modulePath, wantRaw, wantCooked) |
| } |
| } |
