internal/tests/api/api_test.go

// Copyright 2026 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package api

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/http/httptest"
	"net/url"
	"os"
	"path"
	"slices"
	"strings"
	"testing"

	"github.com/google/go-cmp/cmp"
	"github.com/google/go-cmp/cmp/cmpopts"
	"golang.org/x/pkgsite/internal"
	"golang.org/x/pkgsite/internal/api"
	"golang.org/x/pkgsite/internal/derrors"
	"golang.org/x/pkgsite/internal/log"
	"golang.org/x/pkgsite/internal/postgres"
	"golang.org/x/pkgsite/internal/testing/fakedatasource"
	"golang.org/x/pkgsite/internal/testing/sample"
)

type fallbackDataSource struct {
	internal.DataSource
	fallbackMap map[string]string // requested module -> resolved module
}

func (ds fallbackDataSource) GetUnitMeta(ctx context.Context, path, requestedModulePath, requestedVersion string) (*internal.UnitMeta, error) {
	if resolved, ok := ds.fallbackMap[requestedModulePath]; ok {
		um, err := ds.DataSource.GetUnitMeta(ctx, path, resolved, requestedVersion)
		if err != nil {
			return nil, err
		}
		return um, nil
	}
	return ds.DataSource.GetUnitMeta(ctx, path, requestedModulePath, requestedVersion)
}

// setupTestDB sets up a DB for testing.
// It also removes the voluminous DB log output,
// which makes it hard to see test information.
// Note: This function modifies global log state and should not
// be used in tests running with t.Parallel().
func setupTestDB(t *testing.T) internal.TestingDataSource {
	orig := log.GetLevel()
	t.Cleanup(func() { log.SetLevel(orig.String()) })
	log.SetLevel("Info")
	const testDB = "pkgsite_api"
	db, err := postgres.SetupTestDB(testDB)
	if err != nil {
		if errors.Is(err, derrors.NotFound) && os.Getenv("GO_DISCOVERY_TESTDB") != "true" {
			t.Skipf("could not connect to DB (see doc/postgres.md to set up): %v", err)
		}
		t.Fatalf("setting up DB: %v", err)
	}
	t.Cleanup(func() {
		postgres.ResetTestDB(db, t)
		db.Close()
	})
	return db
}

// modinfo creates a ModuleInfo with the given module path
// and version. It sets LatestVersion to version.
func modinfo(path, version string) internal.ModuleInfo {
	return internal.ModuleInfo{
		ModulePath:        path,
		Version:           version,
		LatestVersion:     version,
		IsRedistributable: true,
	}
}

// Module creates a module with the given ModuleInfo and units.
// The units must not have been previously used.
func module(t *testing.T, mi internal.ModuleInfo, units ...*internal.Unit) *internal.Module {
	for _, u := range units {
		// If Name is already set, then this unit was used to
		// build another module, and that's bad.
		if u.Name != "" {
			t.Fatal("unit used in two modules")
		}
		u.ModuleInfo = mi
		// Change relative to absolute path.
		u.Path = path.Join(mi.ModulePath, u.Path)
		// Name is last component of path.
		u.Name = u.Path[strings.LastIndexByte(u.Path, '/')+1:]
	}
	return &internal.Module{
		ModuleInfo: mi,
		Licenses:   sample.Licenses(),
		Units:      units,
	}
}

// unit constructs a Unit with the given relative path and documentation.
// The path is relative to a module path; the full import path
// will be constructed when the Unit is added to a module in
// the [module] function.
func unit(relativePath string, doc ...*internal.Documentation) *internal.Unit {
	return &internal.Unit{
		UnitMeta: internal.UnitMeta{
			Path: relativePath, // expanded in the module function
			// ModuleInfo and Name set in the module function.
		},
		Licenses:          sample.LicenseMetadata(),
		IsRedistributable: true,
		Documentation:     doc,
	}
}

var diffOptions = []cmp.Option{
	cmpopts.IgnoreUnexported(api.Error{}),
	cmpopts.IgnoreFields(api.Error{}, "Fixes"),
	cmpopts.IgnoreFields(api.Module{}, "CommitTime"),
	cmpopts.IgnoreFields(api.ModuleVersion{}, "CommitTime"),
}

func TestAPI(t *testing.T) {
	// TODO(jba): test filters for invalid regex, case sensisitivty and multi-field matching
	t.Setenv("K_SERVICE", "test")
	t.Run("fake", func(t *testing.T) {
		testAPI(t, func(t *testing.T) internal.TestingDataSource {
			return fakedatasource.New()
		})
	})
	t.Run("db", func(t *testing.T) {
		testAPI(t, setupTestDB)
	})
}

func testAPI(t *testing.T, newTestingDataSource func(t *testing.T) internal.TestingDataSource) {
	t.Run("package", func(t *testing.T) {
		testServePackage(t, newTestingDataSource(t))
	})
	t.Run("module", func(t *testing.T) {
		testServeModule(t, newTestingDataSource(t))
	})
	t.Run("module versions", func(t *testing.T) {
		testServeModuleVersions(t, newTestingDataSource(t))
	})
	t.Run("module packages", func(t *testing.T) {
		testServeModulePackages(t, newTestingDataSource(t))
	})
	t.Run("search", func(t *testing.T) {
		testServeSearch(t, newTestingDataSource(t))
	})
	t.Run("search pagination", func(t *testing.T) {
		testServeSearchPagination(t, newTestingDataSource(t))
	})
	t.Run("package symbols", func(t *testing.T) {
		testServePackageSymbols(t, newTestingDataSource(t))
	})
	t.Run("package imported by", func(t *testing.T) {
		testServePackageImportedBy(t, newTestingDataSource(t))
	})
}

func testServePackage(t *testing.T, ds internal.TestingDataSource) {
	const (
		version       = "v1.2.3"
		latestVersion = "v1.2.4"
	)
	u := unit("pkg", sample.Documentation("linux", "amd64", sample.DocContents))
	u.Imports = []string{"example.com/a/b"}
	mi := modinfo("example.com", version)
	mi.LatestVersion = latestVersion
	ds.MustInsertModule(t, module(t, mi, u))

	ub1 := unit("b", sample.Documentation("linux", "amd64", sample.DocContents))
	ub1.Imports = []string{"example.com/pkg"}
	uc1 := unit("c")
	uc1.Imports = []string{"example.com/pkg"}
	ds.MustInsertModule(t, module(t, modinfo("example.com/a", version), ub1, uc1))
	ub2 := unit("")
	ub2.Imports = []string{"example.com/pkg"}
	ud2 := unit("d")
	ud2.Imports = []string{"example.com/pkg"}
	ds.MustInsertModule(t, module(t, modinfo("example.com/a/b", version), ub2, ud2))
	ds.MustInsertModule(t, module(t, modinfo("example.com", latestVersion),
		unit("pkg", sample.DocumentationWithTest("linux", "amd64", `
		// Package p is a package.
		package p
		var V int
		`, `
		package p
	    func Example() {
			fmt.Println("hello")
            // Output: hello
        }`),
		)))

	// Deprecation.
	// The fake data source uses ModuleInfo.Deprecated, but the DB
	// requires a go.mod file.
	modInfo := modinfo("example.com/d/e", version)
	modInfo.Deprecated = true
	ds.MustInsertModuleGoMod(context.TODO(), t, module(t, modInfo, unit("e")), `module example.com/d/e // Deprecated: bad`)
	// The DB needs the above go.mod contents to know that the module
	// is deprecated. It doesn't look at ModuleInfo.Deprecated.
	ds.MustInsertModule(t, module(t, modinfo("example.com/d", version), unit("e")))

	for _, test := range []struct {
		name       string
		url        string
		wantStatus int
		want       any // Can be *Package or *Error
		overrideDS internal.DataSource
	}{
		{
			name:       "missing package path",
			url:        "/v1/package/",
			wantStatus: http.StatusBadRequest,
			want: &api.Error{
				Code:    http.StatusBadRequest,
				Message: "missing package path",
			},
		},
		{
			name:       "basic metadata",
			url:        "/v1/package/example.com/pkg?version=v1.2.3",
			wantStatus: http.StatusOK,
			want: &api.Package{
				PackageInfo: api.PackageInfo{
					Path:              "example.com/pkg",
					Name:              "pkg",
					IsRedistributable: true,
					Synopsis:          "This is a package synopsis for GOOS=linux, GOARCH=amd64",
				},
				ModulePath: "example.com",
				Version:    "v1.2.3",
				IsLatest:   false,
				GOOS:       "linux",
				GOARCH:     "amd64",
			},
		},
		{
			name:       "ambiguous path",
			url:        "/v1/package/example.com/a/b?version=v1.2.3",
			wantStatus: http.StatusBadRequest,
			want: &api.Error{
				Code:    http.StatusBadRequest,
				Message: "ambiguous package path",
				Candidates: []api.Candidate{
					{ModulePath: "example.com/a/b", PackagePath: "example.com/a/b"},
					{ModulePath: "example.com/a", PackagePath: "example.com/a/b"},
				},
			},
		},
		{
			name:       "disambiguated path",
			url:        "/v1/package/example.com/a/b?version=v1.2.3&module=example.com/a",
			wantStatus: http.StatusOK,
			want: &api.Package{
				PackageInfo: api.PackageInfo{
					Path:              "example.com/a/b",
					Name:              "b",
					IsRedistributable: true,
					Synopsis:          "This is a package synopsis for GOOS=linux, GOARCH=amd64",
				},
				ModulePath: "example.com/a",
				Version:    "v1.2.3",
				IsLatest:   true,
				GOOS:       "linux",
				GOARCH:     "amd64",
			},
		},
		{
			name:       "default build context",
			url:        "/v1/package/example.com/pkg?version=v1.2.3",
			wantStatus: http.StatusOK,
			want: &api.Package{
				PackageInfo: api.PackageInfo{
					Path:              "example.com/pkg",
					Name:              "pkg",
					IsRedistributable: true,
					Synopsis:          "This is a package synopsis for GOOS=linux, GOARCH=amd64",
				},
				ModulePath: "example.com",
				Version:    "v1.2.3",
				IsLatest:   false,
				GOOS:       "linux",
				GOARCH:     "amd64",
			},
		},
		{
			name:       "latest version",
			url:        "/v1/package/example.com/pkg?version=v1.2.4",
			wantStatus: http.StatusOK,
			want: &api.Package{
				PackageInfo: api.PackageInfo{
					Name:              "pkg",
					IsRedistributable: true,
					Synopsis:          "This is a package synopsis for GOOS=linux, GOARCH=amd64",
					Path:              "example.com/pkg",
				},
				ModulePath: "example.com",
				Version:    "v1.2.4",
				IsLatest:   true,
				GOOS:       "linux",
				GOARCH:     "amd64",
			},
		},
		{
			name:       "doc",
			url:        "/v1/package/example.com/pkg?version=v1.2.3&doc=text",
			wantStatus: http.StatusOK,
			want: &api.Package{
				PackageInfo: api.PackageInfo{
					Name:              "pkg",
					IsRedistributable: true,
					Synopsis:          "This is a package synopsis for GOOS=linux, GOARCH=amd64",
					Path:              "example.com/pkg",
				},
				ModulePath: "example.com",
				Version:    "v1.2.3",
				GOOS:       "linux",
				GOARCH:     "amd64",
				Docs:       "package p\n\nPackage p is a package.\n\n# Links\n\n- pkg.go.dev, https://pkg.go.dev\n\nVARIABLES\n\nvar V int\n\n",
			},
		},
		{
			name:       "doc with examples",
			url:        "/v1/package/example.com/pkg?version=v1.2.4&doc=text&examples=true",
			wantStatus: http.StatusOK,
			want: &api.Package{
				PackageInfo: api.PackageInfo{
					Name:              "pkg",
					IsRedistributable: true,
					Synopsis:          "This is a package synopsis for GOOS=linux, GOARCH=amd64",
					Path:              "example.com/pkg",
				},
				ModulePath: "example.com",
				Version:    "v1.2.4",
				IsLatest:   true,
				GOOS:       "linux",
				GOARCH:     "amd64",
				Docs:       "package p\n\nPackage p is a package.\n\nExample:\n\t{\n\t\tfmt.Println(\"hello\")\n\t}\n\n\tOutput:\n\thello\n\nVARIABLES\n\nvar V int\n\n",
			},
		},
		{
			name:       "examples without doc",
			url:        "/v1/package/example.com/pkg?version=v1.2.3&examples=true",
			wantStatus: http.StatusBadRequest,
			want: &api.Error{
				Code:    http.StatusBadRequest,
				Message: "examples require doc format to be specified",
			},
		},
		{
			name:       "examples without doc and nonexistent package",
			url:        "/v1/package/nonexistent.com/pkg?version=v1.2.3&examples=true",
			wantStatus: http.StatusBadRequest,
			want: &api.Error{
				Code:    http.StatusBadRequest,
				Message: "examples require doc format to be specified",
			},
		},
		{
			name:       "package not found",
			url:        "/v1/package/nonexistent.com/pkg",
			wantStatus: http.StatusNotFound,
			want:       &api.Error{Code: 404, Message: "not found"},
		},
		{
			name:       "doc without examples",
			url:        "/v1/package/example.com/pkg?version=v1.2.4&doc=text&examples=false",
			wantStatus: http.StatusOK,
			want: &api.Package{
				PackageInfo: api.PackageInfo{
					Path:              "example.com/pkg",
					Name:              "pkg",
					IsRedistributable: true,
					Synopsis:          "This is a package synopsis for GOOS=linux, GOARCH=amd64",
				},
				ModulePath: "example.com",
				Version:    "v1.2.4",
				IsLatest:   true,
				GOOS:       "linux",
				GOARCH:     "amd64",
				Docs:       "package p\n\nPackage p is a package.\n\nVARIABLES\n\nvar V int\n\n",
			},
		},
		{
			name:       "invalid doc format",
			url:        "/v1/package/example.com/pkg?version=v1.2.3&doc=invalid",
			wantStatus: http.StatusBadRequest,
			want: &api.Error{
				Code:    http.StatusBadRequest,
				Message: "bad doc format: need one of 'text', 'md', 'markdown' or 'html'",
			},
		},
		{
			name:       "invalid doc format and nonexistent package",
			url:        "/v1/package/nonexistent.com/pkg?version=v1.2.3&doc=invalid",
			wantStatus: http.StatusBadRequest,
			want: &api.Error{
				Code:    http.StatusBadRequest,
				Message: "bad doc format: need one of 'text', 'md', 'markdown' or 'html'",
			},
		},
		{
			name:       "empty doc format",
			url:        "/v1/package/example.com/pkg?version=v1.2.3&doc=",
			wantStatus: http.StatusOK,
			want: &api.Package{
				PackageInfo: api.PackageInfo{
					Path:              "example.com/pkg",
					Name:              "pkg",
					IsRedistributable: true,
					Synopsis:          "This is a package synopsis for GOOS=linux, GOARCH=amd64",
				},
				ModulePath: "example.com",
				Version:    "v1.2.3",
				GOOS:       "linux",
				GOARCH:     "amd64",
				Docs:       "",
			},
		},
		{
			name:       "licenses",
			url:        "/v1/package/example.com/pkg?version=v1.2.3&licenses=true",
			wantStatus: http.StatusOK,
			want: &api.Package{
				PackageInfo: api.PackageInfo{
					Path:              "example.com/pkg",
					Name:              "pkg",
					IsRedistributable: true,
					Synopsis:          "This is a package synopsis for GOOS=linux, GOARCH=amd64",
				},
				ModulePath: "example.com",
				Version:    "v1.2.3",
				IsLatest:   false,
				GOOS:       "linux",
				GOARCH:     "amd64",
				Licenses: []api.License{
					{
						Types:    []string{sample.LicenseType},
						FilePath: sample.LicenseFilePath,
						Contents: "Lorem Ipsum",
					},
				},
			},
		},
		{
			name:       "imports",
			url:        "/v1/package/example.com/pkg?version=v1.2.3&imports=true",
			wantStatus: http.StatusOK,
			want: &api.Package{
				PackageInfo: api.PackageInfo{
					Path:              "example.com/pkg",
					Name:              "pkg",
					IsRedistributable: true,
					Synopsis:          "This is a package synopsis for GOOS=linux, GOARCH=amd64",
				},
				ModulePath: "example.com",
				Version:    "v1.2.3",
				IsLatest:   false,
				GOOS:       "linux",
				GOARCH:     "amd64",
				Imports:    []string{"example.com/a/b"},
			},
		},
		{
			name:       "fallback prevention (false positive candidate)",
			url:        "/v1/package/example.com/a/b?version=v1.2.3",
			wantStatus: http.StatusBadRequest,
			want: &api.Error{
				Code:    http.StatusBadRequest,
				Message: "ambiguous package path",
				Candidates: []api.Candidate{
					{ModulePath: "example.com/a/b", PackagePath: "example.com/a/b"},
					{ModulePath: "example.com/a", PackagePath: "example.com/a/b"},
				},
			},
			overrideDS: &fallbackDataSource{
				DataSource: ds,
				fallbackMap: map[string]string{
					"example.com": "example.com/a/b", // simulate fallback
				},
			},
		},
		{
			name:       "deprecation filtering",
			url:        "/v1/package/example.com/d/e?version=v1.2.3",
			wantStatus: http.StatusOK,
			want: &api.Package{
				PackageInfo: api.PackageInfo{
					Path:              "example.com/d/e",
					Name:              "e",
					IsRedistributable: true,
				},
				ModulePath: "example.com/d", // picked because example.com/d/e is deprecated
				Version:    "v1.2.3",
				IsLatest:   true,
			},
		},
		{
			name:       "imported-by-counts",
			url:        "/v1/package/example.com/pkg?version=v1.2.3&imported-by-counts=true",
			wantStatus: http.StatusOK,
			want: &api.Package{
				PackageInfo: api.PackageInfo{
					Path:              "example.com/pkg",
					Name:              "pkg",
					IsRedistributable: true,
					Synopsis:          "This is a package synopsis for GOOS=linux, GOARCH=amd64",
				},
				ModulePath: "example.com",
				Version:    "v1.2.3",
				IsLatest:   false,
				GOOS:       "linux",
				GOARCH:     "amd64",
				ImportedByCounts: &api.ImportedByCounts{
					Packages: 3,
					Modules:  2,
				},
			},
		},
		{
			name:       "imported-by-counts unknown",
			url:        "/v1/package/example.com/pkg?version=v1.2.3&imported-by-counts=true",
			overrideDS: unknownCountDataSource{ds},
			wantStatus: http.StatusOK,
			want: &api.Package{
				PackageInfo: api.PackageInfo{
					Path:              "example.com/pkg",
					Name:              "pkg",
					IsRedistributable: true,
					Synopsis:          "This is a package synopsis for GOOS=linux, GOARCH=amd64",
				},
				ModulePath: "example.com",
				Version:    "v1.2.3",
				IsLatest:   false,
				GOOS:       "linux",
				GOARCH:     "amd64",
				ImportedByCounts: &api.ImportedByCounts{
					Packages: -1,
					Modules:  -1,
				},
			},
		},
	} {
		t.Run(test.name, func(t *testing.T) {
			r := httptest.NewRequest("GET", test.url, nil)
			w := httptest.NewRecorder()

			var currentDS internal.DataSource = ds
			if test.overrideDS != nil {
				currentDS = test.overrideDS
			}
			if err := api.ServePackage(w, r, currentDS); err != nil {
				api.ServeError(w, r, err)
			}

			if w.Code != test.wantStatus {
				t.Errorf("status = %d, want %d. Body: %s", w.Code, test.wantStatus, w.Body.String())
			}

			if test.want != nil {
				got, err := unmarshalResponse[api.Package](w.Body.Bytes())
				if err != nil {
					t.Fatal(err)
				}
				if diff := cmp.Diff(test.want, got, diffOptions...); diff != "" {
					t.Errorf("mismatch (-want +got):\n%s", diff)
				}
			}
		})
	}
}

func testServeModule(t *testing.T, ds internal.TestingDataSource) {

	const (
		modulePath = "example.com"
		version    = "v1.2.3"
	)

	mi1 := modinfo(modulePath, version)
	mi1.LatestVersion = "v1.2.4"
	mi1.HasGoMod = true

	u := unit("")
	u.Readme = &internal.Readme{Filepath: "README.md", Contents: "Hello world"}
	ds.MustInsertModule(t, module(t, mi1, u, unit("pkg")))

	mi2 := modinfo(modulePath, "v1.2.4")
	mi2.HasGoMod = true

	ds.MustInsertModule(t, module(t, mi2, unit("")))

	for _, test := range []struct {
		name       string
		url        string
		wantStatus int
		want       any
	}{
		{
			name:       "invalid query parameter",
			url:        "/v1/module/example.com?licenses=invalid",
			wantStatus: http.StatusBadRequest,
			want: &api.Error{
				Code:    http.StatusBadRequest,
				Message: `invalid boolean value "invalid" for licenses`,
			},
		},
		{
			name:       "basic module metadata",
			url:        "/v1/module/example.com?version=v1.2.3",
			wantStatus: http.StatusOK,
			want: &api.Module{
				Path:              modulePath,
				Version:           "v1.2.3",
				IsRedistributable: true,
				HasGoMod:          true,
			},
		},
		{
			name:       "latest module metadata",
			url:        "/v1/module/example.com?version=v1.2.4",
			wantStatus: http.StatusOK,
			want: &api.Module{
				Path:              modulePath,
				Version:           "v1.2.4",
				IsLatest:          true,
				IsRedistributable: true,
				HasGoMod:          true,
			},
		},
		{
			name:       "bad version",
			url:        "/v1/module/example.com?version=nope",
			wantStatus: http.StatusNotFound,
			want:       &api.Error{Code: 404, Message: "not found"},
		},
		{
			name:       "module not found",
			url:        "/v1/module/nonexistent.com",
			wantStatus: http.StatusNotFound,
			want:       &api.Error{Code: 404, Message: "not found"},
		},
		{
			name:       "package path in module endpoint",
			url:        "/v1/module/example.com/pkg?version=v1.2.3",
			wantStatus: http.StatusBadRequest,
			want: &api.Error{
				Code:    http.StatusBadRequest,
				Message: "example.com/pkg is a package, not a module",
			},
		},
		{
			name:       "missing module path",
			url:        "/v1/module/",
			wantStatus: http.StatusBadRequest,
			want:       &api.Error{Code: 400, Message: "missing module path"},
		},
		{
			name:       "module with readme",
			url:        "/v1/module/example.com?version=v1.2.3&readme=true",
			wantStatus: http.StatusOK,
			want: &api.Module{
				Path:              modulePath,
				Version:           "v1.2.3",
				IsRedistributable: true,
				HasGoMod:          true,
				Readme: &api.Readme{
					Filepath: "README.md",
					Contents: "Hello world",
				},
			},
		},
		{
			name:       "module with licenses",
			url:        "/v1/module/example.com?version=v1.2.3&licenses=true",
			wantStatus: http.StatusOK,
			want: &api.Module{
				Path:              modulePath,
				Version:           "v1.2.3",
				IsRedistributable: true,
				HasGoMod:          true,
				Licenses: []api.License{
					{
						Types:    []string{"MIT"},
						FilePath: "LICENSE",
						Contents: "Lorem Ipsum",
					},
				},
			},
		},
	} {
		t.Run(test.name, func(t *testing.T) {
			r := httptest.NewRequest("GET", test.url, nil)
			w := httptest.NewRecorder()

			if err := api.ServeModule(w, r, ds); err != nil {
				api.ServeError(w, r, err)
			}

			if w.Code != test.wantStatus {
				t.Errorf("status = %d, want %d", w.Code, test.wantStatus)
			}

			if test.want != nil {
				got, err := unmarshalResponse[api.Module](w.Body.Bytes())
				if err != nil {
					t.Fatalf("unmarshaling: %v", err)
				}
				if diff := cmp.Diff(test.want, got, diffOptions...); diff != "" {
					t.Errorf("mismatch (-want +got):\n%s", diff)
				}
			}
		})
	}
}

func testServeModuleVersions(t *testing.T, ds internal.TestingDataSource) {
	newMod := func(path, version, latest string) *internal.Module {
		mi := modinfo(path, version)
		mi.LatestVersion = latest
		return module(t, mi, unit(""), unit("pkg"))
	}
	ds.MustInsertModule(t, newMod("example.com", "v1.0.0", "v1.1.0"))
	ds.MustInsertModule(t, newMod("example.com", "v1.1.0", "v1.1.0"))
	ds.MustInsertModule(t, newMod("example.com/v2", "v2.0.0", "v2.0.0"))
	ds.MustInsertModule(t, newMod("example.com", "v0.0.0-20140414041502-3c2ca4d52544", "v1.1.0"))

	for _, test := range []struct {
		name string
		url  string
		want any
	}{
		{
			name: "all versions (cross-major)",
			url:  "/v1/versions/example.com",
			want: &api.PaginatedResponse[api.ModuleVersion]{
				Total: 3,
				Items: []api.ModuleVersion{
					{
						ModulePath:        "example.com/v2",
						Version:           "v2.0.0",
						LatestVersion:     "v2.0.0",
						IsRedistributable: true,
					},
					{
						ModulePath:        "example.com",
						Version:           "v1.1.0",
						LatestVersion:     "v1.1.0",
						IsRedistributable: true,
					},
					{
						ModulePath:        "example.com",
						Version:           "v1.0.0",
						LatestVersion:     "v1.1.0",
						IsRedistributable: true,
					},
				},
			},
		},
		{
			name: "pseudo=true",
			url:  "/v1/versions/example.com?pseudo=true",
			want: &api.PaginatedResponse[api.ModuleVersion]{
				Total: 4,
				Items: []api.ModuleVersion{
					{
						ModulePath:        "example.com/v2",
						Version:           "v2.0.0",
						LatestVersion:     "v2.0.0",
						IsRedistributable: true,
					},
					{
						ModulePath:        "example.com",
						Version:           "v1.1.0",
						LatestVersion:     "v1.1.0",
						IsRedistributable: true,
					},
					{
						ModulePath:        "example.com",
						Version:           "v1.0.0",
						LatestVersion:     "v1.1.0",
						IsRedistributable: true,
					},
					{
						ModulePath:        "example.com",
						Version:           "v0.0.0-20140414041502-3c2ca4d52544",
						LatestVersion:     "v1.1.0",
						IsRedistributable: true,
					},
				},
			},
		},
		{
			name: "module not found",
			url:  "/v1/versions/nonexistent.com",
			want: &api.Error{Code: 404, Message: "not found"},
		},
		{
			name: "package path in versions endpoint",
			url:  "/v1/versions/example.com/pkg",
			want: &api.Error{
				Code:    http.StatusBadRequest,
				Message: "example.com/pkg is a package, not a module",
			},
		},
		{
			name: "missing module path",
			url:  "/v1/versions/",
			want: &api.Error{Code: 400, Message: "missing module path"},
		},
		{
			name: "filter",
			url: "/v1/versions/example.com?filter=" +
				url.QueryEscape(`contains(version, "2")`),
			want: &api.PaginatedResponse[api.ModuleVersion]{
				Total: 1,
				Items: []api.ModuleVersion{
					{
						ModulePath:        "example.com/v2",
						Version:           "v2.0.0",
						LatestVersion:     "v2.0.0",
						IsRedistributable: true,
					},
				},
			},
		},
		{
			name: "invalid filter",
			url:  "/v1/versions/example.com?filter=" + url.QueryEscape(`[`),
			want: &api.Error{
				Code:    400,
				Message: `parsing filter "[": 1:2: expected operand, found 'EOF'`,
			},
		},
		{
			name: "case-sensitive filter",
			url: "/v1/versions/example.com?filter=" +
				url.QueryEscape(`hasPrefix(version, "V")`),
			want: &api.PaginatedResponse[api.ModuleVersion]{
				Total: 0,
				Items: nil,
			},
		},
		{
			name: "case-insensitive filter",
			url: "/v1/versions/example.com?filter=" +
				url.QueryEscape(`matches(version, "[vV]1")`),
			want: &api.PaginatedResponse[api.ModuleVersion]{
				Total: 2,
				Items: []api.ModuleVersion{
					{
						ModulePath:        "example.com",
						Version:           "v1.1.0",
						LatestVersion:     "v1.1.0",
						IsRedistributable: true,
					},
					{
						ModulePath:        "example.com",
						Version:           "v1.0.0",
						LatestVersion:     "v1.1.0",
						IsRedistributable: true,
					},
				},
			},
		},
	} {
		t.Run(test.name, func(t *testing.T) {
			r := httptest.NewRequest("GET", test.url, nil)
			w := httptest.NewRecorder()

			if err := api.ServeModuleVersions(w, r, ds); err != nil {
				api.ServeError(w, r, err)
			}

			var wantStatus int
			switch w := test.want.(type) {
			case *api.Error:
				wantStatus = w.Code
			default:
				wantStatus = http.StatusOK
			}

			if w.Code != wantStatus {
				t.Errorf("status = %d, want %d. Body: %s", w.Code, wantStatus, w.Body.String())
			}

			if test.want != nil {
				got, err := unmarshalResponse[api.PaginatedResponse[api.ModuleVersion]](w.Body.Bytes())
				if err != nil {
					t.Fatal(err)
				}
				if diff := cmp.Diff(test.want, got, diffOptions...); diff != "" {
					t.Errorf("mismatch (-want +got):\n%s", diff)
				}
			}
		})
	}

	testPagination(t, ds, "/v1/versions/example.com?limit=1",
		api.ServeModuleVersions,
		func(r *api.PaginatedResponse[api.ModuleVersion]) (int, int, string) {
			return len(r.Items), r.Total, r.NextPageToken
		},
		[]wantPage{
			{wantCount: 1, wantTotal: -1},
			{wantCount: 1, wantTotal: -1},
			{wantCount: 1, wantTotal: -1},
		})
}

func testServeModulePackages(t *testing.T, ds internal.TestingDataSource) {
	d := sample.Documentation("linux", "amd64", sample.DocContents)
	d.Synopsis = "api.Synopsis for sub"
	ds.MustInsertModule(t,
		module(t, modinfo("example.com", "v1.2.3"),
			unit("", sample.Documentation("linux", "amd64", sample.DocContents)),
			unit("sub", d)))

	info1 := api.PackageInfo{
		Path:              "example.com",
		Name:              "example.com",
		IsRedistributable: true,
		Synopsis:          "This is a package synopsis for GOOS=linux, GOARCH=amd64",
	}
	info2 := api.PackageInfo{
		Path:              "example.com/sub",
		Name:              "sub",
		IsRedistributable: true,
		Synopsis:          "api.Synopsis for sub",
	}

	response := func(infos ...api.PackageInfo) *api.PackagesResponse {
		return &api.PackagesResponse{
			ModulePath: "example.com",
			Version:    "v1.2.3",
			Packages: api.PaginatedResponse[api.PackageInfo]{
				Total: len(infos),
				Items: infos,
			},
		}

	}

	for _, test := range []struct {
		name string
		url  string
		want any
	}{
		{
			name: "all packages",
			url:  "/v1/packages/example.com?version=v1.2.3",
			want: response(info1, info2),
		},
		{
			name: "latest",
			url:  "/v1/packages/example.com",
			want: response(info1, info2),
		},
		{
			name: "module not found",
			url:  "/v1/packages/nonexistent.com?version=v1.2.3",
			want: &api.Error{Code: 404, Message: "not found"},
		},
		{
			name: "package path in packages endpoint",
			url:  "/v1/packages/example.com/sub?version=v1.2.3",
			want: &api.Error{
				Code:    http.StatusBadRequest,
				Message: "example.com/sub is a package, not a module",
			},
		},
		{
			name: "missing module path",
			url:  "/v1/packages/",
			want: &api.Error{Code: 400, Message: "missing module path"},
		},
		{
			name: "filter on path",
			url: "/v1/packages/example.com?version=v1.2.3&filter=" +
				url.QueryEscape(`matches(path, "s[ux].")`),
			want: response(info2),
		},
		{
			name: "filter on synopsis",
			url: "/v1/packages/example.com?version=v1.2.3&filter=" +
				url.QueryEscape(`matches(synopsis, "GO+S")`),
			want: response(info1),
		},
	} {
		t.Run(test.name, func(t *testing.T) {
			r := httptest.NewRequest("GET", test.url, nil)
			w := httptest.NewRecorder()

			if err := api.ServeModulePackages(w, r, ds); err != nil {
				api.ServeError(w, r, err)
			}
			got, err := unmarshalResponse[api.PackagesResponse](w.Body.Bytes())
			if err != nil {
				t.Fatal(err)
			}
			if diff := cmp.Diff(test.want, got, diffOptions...); diff != "" {
				t.Errorf("mismatch (-want +got):\n%s", diff)
			}
		})
	}

	testPagination(t, ds, "/v1/packages/example.com?limit=1",
		api.ServeModulePackages,
		func(r *api.PackagesResponse) (int, int, string) {
			return len(r.Packages.Items), r.Packages.Total, r.Packages.NextPageToken
		},
		[]wantPage{
			{wantCount: 1, wantTotal: 2},
			{wantCount: 1, wantTotal: 2},
		},
	)
}

func testServeSearch(t *testing.T, ds internal.TestingDataSource) {
	ds.MustInsertModule(t, module(t, modinfo("example.com", "v1.0.0"),
		unit("pkg", sample.Documentation("linux", "amd64", sample.DocContents))))
	uz1 := unit("pkg", sample.Documentation("linux", "amd64", sample.DocContents))
	uz1.Imports = []string{"example.com/pkg"}
	uz2 := unit("pkg2", sample.Documentation("linux", "amd64", sample.DocContents))
	uz2.Imports = []string{"example.com/pkg"}
	ds.MustInsertModule(t, module(t, modinfo("example.com/z", "v1.0.0"), uz1, uz2))

	wantCounts := map[string]api.ImportedByCounts{
		"example.com/pkg":    {Packages: 2, Modules: 1},
		"example.com/z/pkg":  {Packages: 0, Modules: 0},
		"example.com/z/pkg2": {Packages: 0, Modules: 0},
	}

	for _, test := range []struct {
		name       string
		url        string
		wantStatus int
		wantCount  int
	}{
		{
			name:       "basic search",
			url:        "/v1/search?q=synopsis",
			wantStatus: http.StatusOK,
			// All packages match, because sample.Documentation creates a synopsis beginning
			// "This is a package synopsis...".
			// There is no grouping, so all packages are part of the list.
			// See https://go.dev/issues/79697 for more on grouping.
			wantCount: 3,
		},
		{
			name:       "no results",
			url:        "/v1/search?q=nonexistent",
			wantStatus: http.StatusOK,
			wantCount:  0,
		},
		{
			name:       "missing query",
			url:        "/v1/search",
			wantStatus: http.StatusBadRequest,
		},
		{
			name: "search with filter",
			url: "/v1/search?q=synopsis&filter=" +
				url.QueryEscape(`contains(modulePath, "z")`),
			wantStatus: http.StatusOK,
			wantCount:  2,
		},
		{
			name: "search with importedByCounts filter",
			url: "/v1/search?q=synopsis&filter=" +
				url.QueryEscape(`importedByCounts.packages > 0`),
			wantStatus: http.StatusOK,
			wantCount:  1,
		},
		{
			name:       "search with non-matching filter",
			url:        "/v1/search?q=great&filter=" + url.QueryEscape(`hasSuffix(packagePath, "moo")`),
			wantStatus: http.StatusOK,
			wantCount:  0,
		},
	} {
		t.Run(test.name, func(t *testing.T) {
			r := httptest.NewRequest("GET", test.url, nil)
			w := httptest.NewRecorder()

			err := api.ServeSearch(w, r, ds)
			if err != nil {
				api.ServeError(w, r, err)
			}

			if w.Code != test.wantStatus {
				t.Errorf("%s: status = %d, want %d", test.name, w.Code, test.wantStatus)
			}

			if test.wantStatus == http.StatusOK {
				var got api.PaginatedResponse[api.SearchResult]
				unmarshalJSON(t, w.Body.Bytes(), &got)
				if len(got.Items) != test.wantCount {
					t.Errorf("%s: count = %d, want %d", test.name, len(got.Items), test.wantCount)
				}
				for _, item := range got.Items {
					if want, ok := wantCounts[item.PackagePath]; ok && item.ImportedByCounts != want {
						t.Errorf("%s: %s ImportedByCounts = %+v, want %+v", test.name, item.PackagePath, item.ImportedByCounts, want)
					}
				}
			}
		})
	}
}

type wantPage struct {
	wantCount int
	wantTotal int
}

// testPagination is a generic helper for testing paginated API endpoints.
// It performs a sequential crawl of pages starting from baseURL.
// For each page request, it asserts that the response has the expected number
// of items and total results. It verifies the presence or absence of NextPageToken,
// and automatically uses the returned token to fetch the subsequent page.
func testPagination[T any](
	t *testing.T,
	ds internal.TestingDataSource,
	baseURL string,
	// serves request
	serve func(http.ResponseWriter, *http.Request, internal.DataSource) error,
	// extracts pagination info from response
	extract func(*T) (count, total int, next string),
	pages []wantPage) {
	t.Helper()
	token := ""

	for i, page := range pages {
		url := baseURL
		if token != "" {
			if strings.Contains(url, "?") {
				url += "&token=" + token
			} else {
				url += "?token=" + token
			}
		}

		r := httptest.NewRequest("GET", url, nil)
		w := httptest.NewRecorder()

		if err := serve(w, r, ds); err != nil {
			api.ServeError(w, r, err)
		}

		if w.Code != http.StatusOK {
			t.Fatalf("page %d: status = %d, want 200", i+1, w.Code)
		}

		var got T
		unmarshalJSON(t, w.Body.Bytes(), &got)
		count, total, nextToken := extract(&got)
		if count != page.wantCount {
			t.Errorf("page %d: count = %d, want %d", i+1, count, page.wantCount)
		}
		if total != page.wantTotal {
			t.Errorf("page %d: total = %d, want %d", i+1, total, page.wantTotal)
		}

		if i == len(pages)-1 {
			if nextToken != "" {
				t.Errorf("page %d: expected empty next page token, got %q", i+1, nextToken)
			}
		} else {
			if nextToken == "" {
				t.Errorf("page %d: expected next page token, got empty", i+1)
			}
		}

		token = nextToken
	}
}

func testServeSearchPagination(t *testing.T, ds internal.TestingDataSource) {
	doc := sample.Documentation("linux", "amd64", sample.DocContents)
	for i := range 10 {
		modPath := fmt.Sprintf("example.com/m%d", i)
		ds.MustInsertModule(t, module(t, modinfo(modPath, "v1.0.0"), unit("pkg", doc)))
	}

	testPagination[api.PaginatedResponse[api.SearchResult]](t, ds, "/v1/search?q=synopsis&limit=3",
		api.ServeSearch,
		func(r *api.PaginatedResponse[api.SearchResult]) (int, int, string) {
			return len(r.Items), r.Total, r.NextPageToken
		},
		[]wantPage{
			{wantCount: 3, wantTotal: 10},
			{wantCount: 3, wantTotal: 10},
			{wantCount: 3, wantTotal: 10},
			{wantCount: 1, wantTotal: 10},
		})
}

func testServePackageSymbols(t *testing.T, ds internal.TestingDataSource) {

	sym := func(doc *internal.Documentation, name string, kind internal.SymbolKind) *internal.Symbol {
		return &internal.Symbol{
			SymbolMeta: internal.SymbolMeta{
				Name:    name,
				Kind:    kind,
				Section: internal.SymbolSectionFunctions,
			},
			GOOS:   doc.GOOS,
			GOARCH: doc.GOARCH,
		}
	}

	linuxDoc := sample.Documentation("linux", "amd64", sample.DocContents)
	linuxDoc.API = []*internal.Symbol{
		sym(linuxDoc, "LinuxSym", internal.SymbolKindFunction),
		sym(linuxDoc, "F", internal.SymbolKindFunction),
		sym(linuxDoc, "LinuxType", internal.SymbolKindType),
	}
	winDoc := sample.Documentation("windows", "amd64", sample.DocContents)
	winDoc.API = []*internal.Symbol{sym(winDoc, "WindowsSym", internal.SymbolKindFunction)}
	wasmDoc := sample.Documentation("js", "wasm", sample.DocContents)
	wasmDoc.API = []*internal.Symbol{sym(wasmDoc, "WasmSym", internal.SymbolKindFunction)}
	ds.MustInsertModule(t,
		module(t, modinfo("example.com", "v1.0.0"),
			unit("pkg", linuxDoc, winDoc, wasmDoc)))

	for _, test := range []struct {
		name       string
		url        string
		wantStatus int
		wantNames  []string // sorted
		want       any
	}{
		{
			name:       "default best match (linux)",
			url:        "/v1/symbols/example.com/pkg?version=v1.0.0",
			wantStatus: http.StatusOK,
			wantNames:  []string{"F", "LinuxSym", "LinuxType"},
		},
		{
			name:       "package not found",
			url:        "/v1/symbols/nonexistent.com/pkg?version=v1.0.0",
			wantStatus: http.StatusNotFound,
			want:       &api.Error{Code: 404, Message: "not found"},
		},
		{
			name:       "missing package path",
			url:        "/v1/symbols/",
			wantStatus: http.StatusBadRequest,
			want:       &api.Error{Code: 400, Message: "missing package path"},
		},
		{
			name:       "explicit linux",
			url:        "/v1/symbols/example.com/pkg?version=v1.0.0&goos=linux&goarch=amd64",
			wantStatus: http.StatusOK,
			wantNames:  []string{"F", "LinuxSym", "LinuxType"},
		},
		{
			name:       "version latest",
			url:        "/v1/symbols/example.com/pkg?version=latest",
			wantStatus: http.StatusOK,
			wantNames:  []string{"F", "LinuxSym", "LinuxType"},
		},
		{
			name:       "explicit windows",
			url:        "/v1/symbols/example.com/pkg?version=v1.0.0&goos=windows&goarch=amd64",
			wantStatus: http.StatusOK,
			wantNames:  []string{"WindowsSym"},
		},
		{
			name:       "explicit wasm",
			url:        "/v1/symbols/example.com/pkg?version=v1.0.0&goos=js&goarch=wasm",
			wantStatus: http.StatusOK,
			wantNames:  []string{"WasmSym"},
		},
		{
			name:       "not found build context",
			url:        "/v1/symbols/example.com/pkg?version=v1.0.0&goos=darwin&goarch=amd64",
			wantStatus: http.StatusNotFound,
			want: &api.Error{
				Code:    http.StatusNotFound,
				Message: "not found",
			},
		},
	} {
		t.Run(test.name, func(t *testing.T) {
			r := httptest.NewRequest("GET", test.url, nil)
			w := httptest.NewRecorder()

			if err := api.ServePackageSymbols(w, r, ds); err != nil {
				api.ServeError(w, r, err)
			}

			if w.Code != test.wantStatus {
				t.Errorf("status = %d, want %d. Body: %s", w.Code, test.wantStatus, w.Body.String())
			}

			if test.want != nil {
				if want, ok := test.want.(*api.Error); ok {
					var got api.Error
					unmarshalJSON(t, w.Body.Bytes(), &got)
					if diff := cmp.Diff(want, &got, diffOptions...); diff != "" {
						t.Errorf("mismatch (-want +got):\n%s", diff)
					}
				}
			}

			if test.wantStatus == http.StatusOK {
				var got api.PackageSymbols
				unmarshalJSON(t, w.Body.Bytes(), &got)
				var gotNames []string
				for _, it := range got.Symbols.Items {
					gotNames = append(gotNames, it.Name)
				}
				slices.Sort(gotNames)
				if !slices.Equal(gotNames, test.wantNames) {
					t.Errorf("got names %q, want %q", gotNames, test.wantNames)
				}
			}
		})
	}

	testPagination(t, ds, "/v1/symbols/example.com/pkg?version=v1.0.0&limit=1",
		api.ServePackageSymbols,
		func(ps *api.PackageSymbols) (int, int, string) {
			return len(ps.Symbols.Items), ps.Symbols.Total, ps.Symbols.NextPageToken
		},
		[]wantPage{
			{wantCount: 1, wantTotal: 3},
			{wantCount: 1, wantTotal: 3},
			{wantCount: 1, wantTotal: 3},
		})

}

type unknownCountDataSource struct {
	internal.TestingDataSource
}

func (unknownCountDataSource) GetImportedByCounts(ctx context.Context, pkgPath, modulePath string) (internal.ImportedByCounts, error) {
	return internal.ImportedByCounts{}, nil
}

func testServePackageImportedBy(t *testing.T, ds internal.TestingDataSource) {

	ds.MustInsertModule(t, module(t, modinfo("example.com", "v1.2.3"), unit("pkg")))

	u := unit("pkg")
	u.Imports = []string{"example.com/pkg"}
	ds.MustInsertModule(t, module(t, modinfo("example.com/mod", "v1.2.3"), u))

	u2 := unit("pkg")
	u2.Imports = []string{"example.com/pkg"}
	ds.MustInsertModule(t, module(t, modinfo("example.com/mod2", "v1.2.3"), u2))

	unknownDS := unknownCountDataSource{ds}

	for _, test := range []struct {
		name       string
		url        string
		overrideDS internal.DataSource
		wantStatus int
		wantCount  int
		wantTotal  int
		want       any
	}{
		{
			name:       "missing package path",
			url:        "/v1/imported-by/",
			wantStatus: http.StatusBadRequest,
			want: &api.Error{
				Code:    http.StatusBadRequest,
				Message: "missing package path",
			},
		},
		{
			name:       "all imported by",
			url:        "/v1/imported-by/example.com/pkg?version=v1.2.3",
			wantStatus: http.StatusOK,
			wantCount:  2,
			wantTotal:  2,
		},
		{
			name: "filtered single page",
			url: "/v1/imported-by/example.com/pkg?version=v1.2.3&filter=" +
				url.QueryEscape(`contains(path, "mod2")`),
			wantStatus: http.StatusOK,
			wantCount:  1,
			wantTotal:  1,
		},
		{
			name:       "unknown count single page",
			url:        "/v1/imported-by/example.com/pkg?version=v1.2.3",
			overrideDS: unknownDS,
			wantStatus: http.StatusOK,
			wantCount:  2,
			wantTotal:  2,
		},
	} {
		t.Run(test.name, func(t *testing.T) {
			r := httptest.NewRequest("GET", test.url, nil)
			w := httptest.NewRecorder()

			useDS := internal.DataSource(ds)
			if test.overrideDS != nil {
				useDS = test.overrideDS
			}
			if err := api.ServePackageImportedBy(w, r, useDS); err != nil {
				api.ServeError(w, r, err)
			}

			if w.Code != test.wantStatus {
				t.Errorf("status = %d, want %d", w.Code, test.wantStatus)
			}

			if test.wantStatus == http.StatusOK {
				var got api.PackageImportedBy
				unmarshalJSON(t, w.Body.Bytes(), &got)
				if len(got.ImportedBy.Items) != test.wantCount {
					t.Errorf("count = %d, want %d", len(got.ImportedBy.Items), test.wantCount)
				}
				if got.ImportedBy.Total != test.wantTotal {
					t.Errorf("total = %d, want %d", got.ImportedBy.Total, test.wantTotal)
				}
			}
		})
	}
	extract := func(pib *api.PackageImportedBy) (int, int, string) {
		return len(pib.ImportedBy.Items), pib.ImportedBy.Total, pib.ImportedBy.NextPageToken
	}
	testPagination[api.PackageImportedBy](t, ds, "/v1/imported-by/example.com/pkg?version=v1.2.3&limit=1",
		api.ServePackageImportedBy,
		extract,
		[]wantPage{
			{wantCount: 1, wantTotal: 2},
			{wantCount: 1, wantTotal: 2},
		})
	testPagination[api.PackageImportedBy](t, ds, "/v1/imported-by/example.com/pkg?version=v1.2.3&limit=1&filter="+
		url.QueryEscape(`contains(path, "mod2")`),
		api.ServePackageImportedBy,
		extract,
		[]wantPage{
			{wantCount: 0, wantTotal: 2},
			{wantCount: 1, wantTotal: 2},
		})
	testPagination[api.PackageImportedBy](t, unknownDS, "/v1/imported-by/example.com/pkg?version=v1.2.3&limit=1",
		api.ServePackageImportedBy,
		extract,
		[]wantPage{
			{wantCount: 1, wantTotal: -1},
			{wantCount: 1, wantTotal: -1},
		})
}

// unmarshalJSON is like json.Unmarshal, but checks for unknown
// fields.
func unmarshalJSON(t *testing.T, data []byte, ptr any) {
	t.Helper()
	d := json.NewDecoder(bytes.NewReader(data))
	d.DisallowUnknownFields()
	if err := d.Decode(ptr); err != nil {
		t.Fatalf("unmarshalling JSON into %T: %v", ptr, err)
	}
}

// unmarshalResponse unmarshals an API response into either
// a *T or an *Error.
func unmarshalResponse[T any](data []byte) (any, error) {
	d := json.NewDecoder(bytes.NewReader(data))
	d.DisallowUnknownFields()
	var t T
	err1 := d.Decode(&t)
	if err1 == nil {
		return &t, nil
	}
	d = json.NewDecoder(bytes.NewReader(data))
	d.DisallowUnknownFields()
	var e api.Error
	err2 := d.Decode(&e)
	if err2 == nil {
		return &e, nil
	}
	return nil, errors.Join(err1, err2)
}