internal/frontend/search.go - pkgsite - Git at Google

 // Copyright 2019 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

 package frontend

 import (
 	"context"
 	"errors"
 	"fmt"
 	"net/http"
 	"path"
 	"regexp"
 	"sort"
 	"strings"
 	"sync"
 	"time"
 	"unicode"
 	"unicode/utf8"

 	"github.com/google/safehtml/template"
 	"golang.org/x/mod/semver"
 	"golang.org/x/pkgsite/internal"
 	"golang.org/x/pkgsite/internal/derrors"
 	"golang.org/x/pkgsite/internal/log"
 	"golang.org/x/pkgsite/internal/middleware"
 	"golang.org/x/pkgsite/internal/postgres"
 	"golang.org/x/pkgsite/internal/stdlib"
 	"golang.org/x/pkgsite/internal/version"
 	"golang.org/x/pkgsite/internal/vulns"
 	"golang.org/x/text/message"
 	vulnc "golang.org/x/vuln/client"
 )

 // serveSearch applies database data to the search template. Handles endpoint
 // /search?q=<query>. If <query> is an exact match for a package path, the user
 // will be redirected to the details page.
 func (s *Server) serveSearch(w http.ResponseWriter, r *http.Request, ds internal.DataSource) error {
 	action, err := determineSearchAction(r, ds, s.vulnClient)
 	if err != nil {
 		return err
 	}
 	if action.redirectURL != "" {
 		http.Redirect(w, r, action.redirectURL, http.StatusFound)
 		return nil
 	}
 	action.page.setBasePage(s.newBasePage(r, action.title))
 	if s.shouldServeJSON(r) {
 		return s.serveJSONPage(w, r, action.page)
 	}
 	s.servePage(r.Context(), w, action.template, action.page)
 	return nil
 }

 type searchAction struct {
 	redirectURL string
 	title       string
 	template    string
 	page        interface{ setBasePage(basePage) }
 }

 func determineSearchAction(r *http.Request, ds internal.DataSource, vulnClient vulnc.Client) (*searchAction, error) {
 	if r.Method != http.MethodGet && r.Method != http.MethodHead {
 		return nil, &serverError{status: http.StatusMethodNotAllowed}
 	}
 	db, ok := ds.(*postgres.DB)
 	if !ok {
 		// The proxydatasource does not support the imported by page.
 		return nil, datasourceNotSupportedErr()
 	}

 	ctx := r.Context()
 	cq, filters := searchQueryAndFilters(r)
 	if !utf8.ValidString(cq) {
 		return nil, &serverError{status: http.StatusBadRequest}
 	}
 	if len(filters) > 1 {
 		return nil, &serverError{
 			status: http.StatusBadRequest,
 			epage: &errorPage{
 				messageTemplate: template.MakeTrustedTemplate(
 					`<h3 class="Error-message">Search query contains more than one symbol.</h3>`),
 			},
 		}
 	}
 	if len(cq) > maxSearchQueryLength {
 		return nil, &serverError{
 			status: http.StatusBadRequest,
 			epage: &errorPage{
 				messageTemplate: template.MakeTrustedTemplate(
 					`<h3 class="Error-message">Search query too long.</h3>`),
 			},
 		}
 	}
 	if cq == "" {
 		return &searchAction{redirectURL: "/"}, nil
 	}
 	pageParams := newPaginationParams(r, defaultSearchLimit)
 	if pageParams.offset() > maxSearchOffset {
 		return nil, &serverError{
 			status: http.StatusBadRequest,
 			epage: &errorPage{
 				messageTemplate: template.MakeTrustedTemplate(
 					`<h3 class="Error-message">Search page number too large.</h3>`),
 			},
 		}
 	}
 	if pageParams.limit > maxSearchPageSize {
 		return nil, &serverError{
 			status: http.StatusBadRequest,
 			epage: &errorPage{
 				messageTemplate: template.MakeTrustedTemplate(
 					`<h3 class="Error-message">Search page size too large.</h3>`),
 			},
 		}
 	}
 	mode := searchMode(r)
 	if path := searchRequestRedirectPath(ctx, ds, cq, mode); path != "" {
 		return &searchAction{redirectURL: path}, nil
 	}
 	action, err := searchVulnAlias(ctx, mode, cq, vulnClient)
 	if action != nil || err != nil {
 		return action, err
 	}
 	action, err = searchVulnModule(ctx, mode, cq, vulnClient)
 	if action != nil || err != nil {
 		return action, err
 	}
 	var symbol string
 	if len(filters) > 0 {
 		symbol = filters[0]
 	}
 	var getVulnEntries vulns.VulnEntriesFunc
 	if vulnClient != nil {
 		getVulnEntries = vulnClient.GetByModule
 	}
 	page, err := fetchSearchPage(ctx, db, cq, symbol, pageParams, mode == searchModeSymbol, getVulnEntries)
 	if err != nil {
 		// Instead of returning a 500, return a 408, since symbol searches may
 		// timeout for very popular symbols.
 		if mode == searchModeSymbol && strings.Contains(err.Error(), "i/o timeout") {
 			return nil, &serverError{
 				status: http.StatusRequestTimeout,
 				epage: &errorPage{
 					messageTemplate: template.MakeTrustedTemplate(
 						`<h3 class="Error-message">Request timed out. Please try again!</h3>`),
 				},
 			}
 		}
 		return nil, fmt.Errorf("fetchSearchPage(ctx, db, %q): %v", cq, err)
 	}
 	page.SearchMode = mode
 	return &searchAction{
 		title:    fmt.Sprintf("%s - Search Results", cq),
 		template: "search",
 		page:     page,
 	}, nil
 }

 const (
 	// defaultSearchLimit is the default number of items that appears on the
 	// search results page if limit is not specified.
 	defaultSearchLimit = 25

 	// maxSearchQueryLength represents the max number of characters that a search
 	// query can be. For PostgreSQL 11, there is a max length of 2K bytes:
 	// https://www.postgresql.org/docs/11/textsearch-limitations.html. No valid
 	// searches on pkg.go.dev will need more than the maxSearchQueryLength.
 	maxSearchQueryLength = 500

 	// maxSearchOffset is the maximum allowed offset into the search results.
 	// This prevents some very CPU-intensive queries from running.
 	maxSearchOffset = 100

 	// maxSearchPageSize is the maximum allowed limit for search results.
 	maxSearchPageSize = 100

 	// searchModePackage is the keyword prefix and query param for searching
 	// by packages.
 	searchModePackage = "package"

 	// searchModeSymbol is the keyword prefix and query param for searching
 	// by symbols.
 	searchModeSymbol = "symbol"

 	// searchModeVuln is the query param for searching by vuln id.
 	searchModeVuln = "vuln"

 	// symbolSearchFilter is a filter that can be used to indicate that the query
 	// contains a symbol. For example, searching for "#unmarshal json" indicates
 	// that unmarshal is a symbol.
 	symbolSearchFilter = "#"
 )

 // SearchPage contains all of the data that the search template needs to
 // populate.
 type SearchPage struct {
 	basePage

 	// PackageTabQuery is the search query, stripped of any filters.
 	// This is used if the user clicks on the package tab.
 	PackageTabQuery string

 	Pagination pagination
 	Results    []*SearchResult
 }

 // SearchResult contains data needed to display a single search result.
 type SearchResult struct {
 	Name           string
 	PackagePath    string
 	ModulePath     string
 	Version        string
 	ChipText       string
 	Synopsis       string
 	DisplayVersion string
 	Licenses       []string
 	CommitTime     string
 	NumImportedBy  string
 	Symbols        *subResult
 	SameModule     *subResult // package paths in the same module
 	OtherMajor     *subResult // package paths in lower major versions
 	SymbolName     string
 	SymbolKind     string
 	SymbolSynopsis string
 	SymbolGOOS     string
 	SymbolGOARCH   string
 	SymbolLink     string
 	Vulns          []vulns.Vuln
 }

 type subResult struct {
 	Heading string
 	Links   []link
 }

 // fetchSearchPage fetches data matching the search query from the database and
 // returns a SearchPage.
 func fetchSearchPage(ctx context.Context, db *postgres.DB, cq, symbol string,
 	pageParams paginationParams, searchSymbols bool, getVulnEntries vulns.VulnEntriesFunc) (*SearchPage, error) {
 	maxResultCount := maxSearchOffset + pageParams.limit

 	// Pageless search: always start from the beginning.
 	offset := 0
 	dbresults, err := db.Search(ctx, cq, postgres.SearchOptions{
 		MaxResults:     pageParams.limit,
 		Offset:         offset,
 		MaxResultCount: maxResultCount,
 		SearchSymbols:  searchSymbols,
 		SymbolFilter:   symbol,
 	})
 	if err != nil {
 		return nil, err
 	}

 	var results []*SearchResult
 	for _, r := range dbresults {
 		sr := newSearchResult(r, searchSymbols, message.NewPrinter(middleware.LanguageTag(ctx)))
 		results = append(results, sr)
 	}

 	if getVulnEntries != nil {
 		addVulns(ctx, results, getVulnEntries)
 	}

 	var numResults int
 	if len(dbresults) > 0 {
 		numResults = int(dbresults[0].NumResults)
 	}

 	numPageResults := 0
 	for _, r := range dbresults {
 		// Grouping will put some results inside others. Each result counts one
 		// for itself plus one for each sub-result in the SameModule list,
 		// because each of those is removed from the top-level slice. Results in
 		// the LowerMajor list are not removed from the top-level slice,
 		// so we don't add them up.
 		numPageResults += 1 + len(r.SameModule)
 	}

 	pgs := newPagination(pageParams, numPageResults, numResults)
 	sp := &SearchPage{
 		PackageTabQuery: cq,
 		Results:         results,
 		Pagination:      pgs,
 	}
 	return sp, nil
 }

 func newSearchResult(r *postgres.SearchResult, searchSymbols bool, pr *message.Printer) *SearchResult {
 	// For commands, change the name from "main" to the last component of the import path.
 	chipText := ""
 	name := r.Name
 	if name == "main" {
 		chipText = "command"
 		name = effectiveName(r.PackagePath, r.Name)
 	}
 	moduleDesc := "Other packages in module " + r.ModulePath
 	if r.ModulePath == stdlib.ModulePath {
 		moduleDesc = "Related packages in the standard library"
 		chipText = "standard library"
 	}
 	sr := &SearchResult{
 		Name:           name,
 		PackagePath:    r.PackagePath,
 		ModulePath:     r.ModulePath,
 		Version:        r.Version,
 		ChipText:       chipText,
 		Synopsis:       r.Synopsis,
 		DisplayVersion: displayVersion(r.ModulePath, r.Version, r.Version),
 		Licenses:       r.Licenses,
 		CommitTime:     elapsedTime(r.CommitTime),
 		NumImportedBy:  pr.Sprint(r.NumImportedBy),
 		SameModule:     packagePaths(moduleDesc+":", r.SameModule),
 		// Say "other" instead of "lower" because at some point we may
 		// prefer to show a tagged, lower major version over an untagged
 		// higher major version.
 		OtherMajor: modulePaths("Other major versions:", r.OtherMajor),
 	}
 	if searchSymbols {
 		sr.SymbolName = r.SymbolName
 		sr.SymbolKind = strings.ToLower(string(r.SymbolKind))
 		sr.SymbolSynopsis = symbolSynopsis(r)
 		sr.SymbolGOOS = r.SymbolGOOS
 		sr.SymbolGOARCH = r.SymbolGOARCH
 		// If the GOOS is "all" or "linux", it doesn't need to be
 		// specified as a query param. "linux" is the default GOOS when a
 		// package has multiple build contexts, since it is first item
 		// listed in internal.BuildContexts.
 		if r.SymbolGOOS == internal.All || r.SymbolGOOS == "linux" {
 			sr.SymbolLink = fmt.Sprintf("/%s#%s", r.PackagePath, r.SymbolName)
 		} else {
 			sr.SymbolLink = fmt.Sprintf("/%s?GOOS=%s#%s", r.PackagePath, r.SymbolGOOS, r.SymbolName)
 		}
 	}
 	return sr
 }

 // A regexp that matches Go vuln IDs.
 var goVulnIDRegexp = regexp.MustCompile("^GO-[0-9]{4}-[0-9]{4}$")

 // searchRequestRedirectPath returns the path that a search request should be
 // redirected to, or the empty string if there is no such path.
 //
 // If the user types an existing package path into the search bar, we will
 // redirect the user to the details page. Standard library packages that only
 // contain one element (such as fmt, errors, etc.) will not redirect, to allow
 // users to search by those terms.
 //
 // If the user types a name that is in the form of a Go vulnerability ID, we will
 // redirect to the page for that ID (whether or not it exists).
 func searchRequestRedirectPath(ctx context.Context, ds internal.DataSource, query, mode string) string {
 	urlSchemeIdx := strings.Index(query, "://")
 	if urlSchemeIdx > -1 {
 		query = query[urlSchemeIdx+3:]
 	}
 	if goVulnIDRegexp.MatchString(query) {
 		return fmt.Sprintf("/vuln/%s?q", query)
 	}
 	requestedPath := path.Clean(query)
 	if !strings.Contains(requestedPath, "/") || mode == searchModeVuln {
 		return ""
 	}
 	_, err := ds.GetUnitMeta(ctx, requestedPath, internal.UnknownModulePath, version.Latest)
 	if err != nil {
 		if !errors.Is(err, derrors.NotFound) {
 			log.Errorf(ctx, "searchRequestRedirectPath(%q): %v", requestedPath, err)
 		}
 		return ""
 	}
 	return fmt.Sprintf("/%s", requestedPath)
 }

 func searchVulnModule(ctx context.Context, mode, cq string, client vulnc.Client) (_ *searchAction, err error) {
 	if mode != searchModeVuln {
 		return nil, nil
 	}
 	allEntries, err := vulnList(ctx, client)
 	if err != nil {
 		return nil, err
 	}
 	prefix := cq + "/"
 	var entries []OSVEntry
 EntryLoop:
 	for _, entry := range allEntries {
 		for _, aff := range entry.Affected {
 			for _, imp := range aff.EcosystemSpecific.Imports {
 				if imp.Path == cq || strings.HasPrefix(imp.Path, prefix) {
 					entries = append(entries, entry)
 					continue EntryLoop
 				}
 			}
 		}
 	}
 	// Sort from most to least recent.
 	sort.Slice(entries, func(i, j int) bool { return entries[i].ID > entries[j].ID })
 	return &searchAction{
 		title:    fmt.Sprintf("%s - Vulnerability Reports", cq),
 		template: "vuln/list",
 		page:     &VulnListPage{Entries: entries},
 	}, nil
 }

 func searchVulnAlias(ctx context.Context, mode, cq string, vulnClient vulnc.Client) (_ *searchAction, err error) {
 	defer derrors.Wrap(&err, "searchVulnAlias(%q, %q)", mode, cq)

 	if mode != searchModeVuln || !isVulnAlias(cq) {
 		return nil, nil
 	}
 	aliasEntries, err := vulnClient.GetByAlias(ctx, cq)
 	if err != nil {
 		return nil, err
 	}
 	switch len(aliasEntries) {
 	case 0:
 		return nil, &serverError{status: http.StatusNotFound}
 	case 1:
 		return &searchAction{redirectURL: "/vuln/" + aliasEntries[0].ID}, nil
 	default:
 		var entries []OSVEntry
 		for _, e := range aliasEntries {
 			entries = append(entries, OSVEntry{e})
 		}
 		return &searchAction{
 			title:    fmt.Sprintf("%s - Vulnerability Reports", cq),
 			template: "vuln/list",
 			page:     &VulnListPage{Entries: entries},
 		}, nil
 	}
 }

 // Regexps that match aliases for Go vulns.
 var (
 	cveRegexp  = regexp.MustCompile("^CVE-[0-9]{4}-[0-9]+$")
 	ghsaRegexp = regexp.MustCompile("^GHSA-.{4}-.{4}-.{4}$")
 )

 func isVulnAlias(s string) bool {
 	return cveRegexp.MatchString(s) || ghsaRegexp.MatchString(s)
 }

 // searchMode reports whether the search performed should be in package or
 // symbol search mode.
 func searchMode(r *http.Request) string {
 	q, filters := searchQueryAndFilters(r)
 	if len(filters) > 0 {
 		return searchModeSymbol
 	}
 	switch rawSearchMode(r) {
 	case searchModePackage:
 		return searchModePackage
 	case searchModeSymbol:
 		return searchModeSymbol
 	case searchModeVuln:
 		return searchModeVuln
 	default:
 		if isVulnAlias(q) {
 			return searchModeVuln
 		}
 		if shouldDefaultToSymbolSearch(q) {
 			return searchModeSymbol
 		}
 		return searchModePackage
 	}
 }

 // searchQueryAndFilters returns the search query, trimmed of any filters, and
 // the array of words that had a filter prefix.
 func searchQueryAndFilters(r *http.Request) (string, []string) {
 	words := strings.Fields(rawSearchQuery(r))
 	var filters []string
 	for i := range words {
 		if strings.HasPrefix(words[i], symbolSearchFilter) {
 			words[i] = strings.TrimLeft(words[i], symbolSearchFilter)
 			filters = append(filters, words[i])
 		}
 	}
 	return strings.Join(words, " "), filters
 }

 // rawSearchQuery returns the exact search query by the user.
 func rawSearchQuery(r *http.Request) string {
 	return strings.TrimSpace(r.FormValue("q"))
 }

 // rawSearchMode returns the exact search mode from the URL request.
 func rawSearchMode(r *http.Request) string {
 	return strings.TrimSpace(r.FormValue("m"))
 }

 // shouldDefaultToSymbolSearch reports whether the search mode should
 // default to symbol based on the input.
 func shouldDefaultToSymbolSearch(q string) bool {
 	if len(strings.Fields(q)) != 1 {
 		return false
 	}
 	if internal.IsGoPkgInPathElement(q) {
 		return false
 	}
 	parts := strings.Split(q, ".")
 	if len(parts) > 1 {
 		if len(parts) == 2 && semver.IsValid(parts[1]) {
 			// The q has the format <text>.<semver> which is likely a
 			// gopkg.in host, such as yaml.v2. Default to package search.
 			return false
 		}
 		return !internal.TopLevelDomains[parts[len(parts)-1]]
 	}
 	// If a user searches for "Unmarshal", assume that they are searching for
 	// the symbol name "Unmarshal", not the package unmarshal.
 	return isCapitalized(q)
 }

 // symbolSynopsis returns the string to be displayed in the code snippet
 // section for a symbol search result.
 func symbolSynopsis(r *postgres.SearchResult) string {
 	switch r.SymbolKind {
 	case internal.SymbolKindField:
 		return fmt.Sprintf(`
 type %s struct {
 	%s
 }
 `, strings.Split(r.SymbolName, ".")[0], r.SymbolSynopsis)
 	case internal.SymbolKindMethod:
 		if !strings.HasPrefix(r.SymbolSynopsis, "func (") {
 			return fmt.Sprintf(`
 type %s interface {
 	%s
 }
 `, strings.Split(r.SymbolName, ".")[0], r.SymbolSynopsis)
 		}
 	}
 	return r.SymbolSynopsis
 }

 func packagePaths(heading string, rs []*postgres.SearchResult) *subResult {
 	if len(rs) == 0 {
 		return nil
 	}
 	var links []link
 	for _, r := range rs {
 		links = append(links, link{Href: r.PackagePath, Body: internal.Suffix(r.PackagePath, r.ModulePath)})
 	}
 	return &subResult{
 		Heading: heading,
 		Links:   links,
 	}
 }

 func modulePaths(heading string, modulePathToMajor map[string]int) *subResult {
 	if len(modulePathToMajor) == 0 {
 		return nil
 	}

 	type mm struct {
 		Path  string
 		Major int
 	}

 	var mms []mm
 	for m, v := range modulePathToMajor {
 		mms = append(mms, mm{m, v})
 	}
 	sort.Slice(mms, func(i, j int) bool { return mms[i].Major > mms[j].Major })
 	links := make([]link, len(mms))
 	for i, m := range mms {
 		links[i] = link{Href: m.Path, Body: fmt.Sprintf("v%d", m.Major)}
 	}
 	return &subResult{
 		Heading: heading,
 		Links:   links,
 	}
 }

 // isCapitalized reports whether the first letter of s is capitalized.
 func isCapitalized(s string) bool {
 	if len(s) == 0 {
 		return false
 	}
 	return unicode.IsUpper(rune(s[0]))
 }

 // elapsedTime takes a date and returns returns human-readable,
 // relative timestamps based on the following rules:
 // (1) 'X hours ago' when X < 6
 // (2) 'today' between 6 hours and 1 day ago
 // (3) 'Y days ago' when Y < 6
 // (4) A date formatted like "Jan 2, 2006" for anything further back
 func elapsedTime(date time.Time) string {
 	elapsedHours := int(time.Since(date).Hours())
 	if elapsedHours == 1 {
 		return "1 hour ago"
 	} else if elapsedHours < 6 {
 		return fmt.Sprintf("%d hours ago", elapsedHours)
 	}

 	elapsedDays := elapsedHours / 24
 	if elapsedDays < 1 {
 		return "today"
 	} else if elapsedDays == 1 {
 		return "1 day ago"
 	} else if elapsedDays < 6 {
 		return fmt.Sprintf("%d days ago", elapsedDays)
 	}

 	return absoluteTime(date)
 }

 // addVulns adds vulnerability information to search results by consulting the
 // vulnerability database.
 func addVulns(ctx context.Context, rs []*SearchResult, getVulnEntries vulns.VulnEntriesFunc) {
 	// Get all vulns concurrently.
 	var wg sync.WaitGroup
 	// TODO(golang/go#48223): throttle concurrency?
 	for _, r := range rs {
 		r := r
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
 			r.Vulns = vulns.VulnsForPackage(ctx, r.ModulePath, r.Version, r.PackagePath, getVulnEntries)
 		}()
 	}
 	wg.Wait()

 }
	// Copyright 2019 The Go Authors. All rights reserved.
	// Use of this source code is governed by a BSD-style
	// license that can be found in the LICENSE file.

	package frontend

	import (
	"context"
	"errors"
	"fmt"
	"net/http"
	"path"
	"regexp"
	"sort"
	"strings"
	"sync"
	"time"
	"unicode"
	"unicode/utf8"

	"github.com/google/safehtml/template"
	"golang.org/x/mod/semver"
	"golang.org/x/pkgsite/internal"
	"golang.org/x/pkgsite/internal/derrors"
	"golang.org/x/pkgsite/internal/log"
	"golang.org/x/pkgsite/internal/middleware"
	"golang.org/x/pkgsite/internal/postgres"
	"golang.org/x/pkgsite/internal/stdlib"
	"golang.org/x/pkgsite/internal/version"
	"golang.org/x/pkgsite/internal/vulns"
	"golang.org/x/text/message"
	vulnc "golang.org/x/vuln/client"
	)

	// serveSearch applies database data to the search template. Handles endpoint
	// /search?q=<query>. If <query> is an exact match for a package path, the user
	// will be redirected to the details page.
	func (s Server) serveSearch(w http.ResponseWriter, r http.Request, ds internal.DataSource) error {
	action, err := determineSearchAction(r, ds, s.vulnClient)
	if err != nil {
	return err
	}
	if action.redirectURL != "" {
	http.Redirect(w, r, action.redirectURL, http.StatusFound)
	return nil
	}
	action.page.setBasePage(s.newBasePage(r, action.title))
	if s.shouldServeJSON(r) {
	return s.serveJSONPage(w, r, action.page)
	}
	s.servePage(r.Context(), w, action.template, action.page)
	return nil
	}

	type searchAction struct {
	redirectURL string
	title string
	template string
	page interface{ setBasePage(basePage) }
	}

	func determineSearchAction(r http.Request, ds internal.DataSource, vulnClient vulnc.Client) (searchAction, error) {
	if r.Method != http.MethodGet && r.Method != http.MethodHead {
	return nil, &serverError{status: http.StatusMethodNotAllowed}
	}
	db, ok := ds.(*postgres.DB)
	if !ok {
	// The proxydatasource does not support the imported by page.
	return nil, datasourceNotSupportedErr()
	}

	ctx := r.Context()
	cq, filters := searchQueryAndFilters(r)
	if !utf8.ValidString(cq) {
	return nil, &serverError{status: http.StatusBadRequest}
	}
	if len(filters) > 1 {
	return nil, &serverError{
	status: http.StatusBadRequest,
	epage: &errorPage{
	messageTemplate: template.MakeTrustedTemplate(
	`<h3 class="Error-message">Search query contains more than one symbol.</h3>`),
	},
	}
	}
	if len(cq) > maxSearchQueryLength {
	return nil, &serverError{
	status: http.StatusBadRequest,
	epage: &errorPage{
	messageTemplate: template.MakeTrustedTemplate(
	`<h3 class="Error-message">Search query too long.</h3>`),
	},
	}
	}
	if cq == "" {
	return &searchAction{redirectURL: "/"}, nil
	}
	pageParams := newPaginationParams(r, defaultSearchLimit)
	if pageParams.offset() > maxSearchOffset {
	return nil, &serverError{
	status: http.StatusBadRequest,
	epage: &errorPage{
	messageTemplate: template.MakeTrustedTemplate(
	`<h3 class="Error-message">Search page number too large.</h3>`),
	},
	}
	}
	if pageParams.limit > maxSearchPageSize {
	return nil, &serverError{
	status: http.StatusBadRequest,
	epage: &errorPage{
	messageTemplate: template.MakeTrustedTemplate(
	`<h3 class="Error-message">Search page size too large.</h3>`),
	},
	}
	}
	mode := searchMode(r)
	if path := searchRequestRedirectPath(ctx, ds, cq, mode); path != "" {
	return &searchAction{redirectURL: path}, nil
	}
	action, err := searchVulnAlias(ctx, mode, cq, vulnClient)
	if action != nil \|\| err != nil {
	return action, err
	}
	action, err = searchVulnModule(ctx, mode, cq, vulnClient)
	if action != nil \|\| err != nil {
	return action, err
	}
	var symbol string
	if len(filters) > 0 {
	symbol = filters[0]
	}
	var getVulnEntries vulns.VulnEntriesFunc
	if vulnClient != nil {
	getVulnEntries = vulnClient.GetByModule
	}
	page, err := fetchSearchPage(ctx, db, cq, symbol, pageParams, mode == searchModeSymbol, getVulnEntries)
	if err != nil {
	// Instead of returning a 500, return a 408, since symbol searches may
	// timeout for very popular symbols.
	if mode == searchModeSymbol && strings.Contains(err.Error(), "i/o timeout") {
	return nil, &serverError{
	status: http.StatusRequestTimeout,
	epage: &errorPage{
	messageTemplate: template.MakeTrustedTemplate(
	`<h3 class="Error-message">Request timed out. Please try again!</h3>`),
	},
	}
	}
	return nil, fmt.Errorf("fetchSearchPage(ctx, db, %q): %v", cq, err)
	}
	page.SearchMode = mode
	return &searchAction{
	title: fmt.Sprintf("%s - Search Results", cq),
	template: "search",
	page: page,
	}, nil
	}

	const (
	// defaultSearchLimit is the default number of items that appears on the
	// search results page if limit is not specified.
	defaultSearchLimit = 25

	// maxSearchQueryLength represents the max number of characters that a search
	// query can be. For PostgreSQL 11, there is a max length of 2K bytes:
	// https://www.postgresql.org/docs/11/textsearch-limitations.html. No valid
	// searches on pkg.go.dev will need more than the maxSearchQueryLength.
	maxSearchQueryLength = 500

	// maxSearchOffset is the maximum allowed offset into the search results.
	// This prevents some very CPU-intensive queries from running.
	maxSearchOffset = 100

	// maxSearchPageSize is the maximum allowed limit for search results.
	maxSearchPageSize = 100

	// searchModePackage is the keyword prefix and query param for searching
	// by packages.
	searchModePackage = "package"

	// searchModeSymbol is the keyword prefix and query param for searching
	// by symbols.
	searchModeSymbol = "symbol"

	// searchModeVuln is the query param for searching by vuln id.
	searchModeVuln = "vuln"

	// symbolSearchFilter is a filter that can be used to indicate that the query
	// contains a symbol. For example, searching for "#unmarshal json" indicates
	// that unmarshal is a symbol.
	symbolSearchFilter = "#"
	)

	// SearchPage contains all of the data that the search template needs to
	// populate.
	type SearchPage struct {
	basePage

	// PackageTabQuery is the search query, stripped of any filters.
	// This is used if the user clicks on the package tab.
	PackageTabQuery string

	Pagination pagination
	Results []*SearchResult
	}

	// SearchResult contains data needed to display a single search result.
	type SearchResult struct {
	Name string
	PackagePath string
	ModulePath string
	Version string
	ChipText string
	Synopsis string
	DisplayVersion string
	Licenses []string
	CommitTime string
	NumImportedBy string
	Symbols *subResult
	SameModule *subResult // package paths in the same module
	OtherMajor *subResult // package paths in lower major versions
	SymbolName string
	SymbolKind string
	SymbolSynopsis string
	SymbolGOOS string
	SymbolGOARCH string
	SymbolLink string
	Vulns []vulns.Vuln
	}

	type subResult struct {
	Heading string
	Links []link
	}

	// fetchSearchPage fetches data matching the search query from the database and
	// returns a SearchPage.
	func fetchSearchPage(ctx context.Context, db *postgres.DB, cq, symbol string,
	pageParams paginationParams, searchSymbols bool, getVulnEntries vulns.VulnEntriesFunc) (*SearchPage, error) {
	maxResultCount := maxSearchOffset + pageParams.limit

	// Pageless search: always start from the beginning.
	offset := 0
	dbresults, err := db.Search(ctx, cq, postgres.SearchOptions{
	MaxResults: pageParams.limit,
	Offset: offset,
	MaxResultCount: maxResultCount,
	SearchSymbols: searchSymbols,
	SymbolFilter: symbol,
	})
	if err != nil {
	return nil, err
	}

	var results []*SearchResult
	for _, r := range dbresults {
	sr := newSearchResult(r, searchSymbols, message.NewPrinter(middleware.LanguageTag(ctx)))
	results = append(results, sr)
	}

	if getVulnEntries != nil {
	addVulns(ctx, results, getVulnEntries)
	}

	var numResults int
	if len(dbresults) > 0 {
	numResults = int(dbresults[0].NumResults)
	}

	numPageResults := 0
	for _, r := range dbresults {
	// Grouping will put some results inside others. Each result counts one
	// for itself plus one for each sub-result in the SameModule list,
	// because each of those is removed from the top-level slice. Results in
	// the LowerMajor list are not removed from the top-level slice,
	// so we don't add them up.
	numPageResults += 1 + len(r.SameModule)
	}

	pgs := newPagination(pageParams, numPageResults, numResults)
	sp := &SearchPage{
	PackageTabQuery: cq,
	Results: results,
	Pagination: pgs,
	}
	return sp, nil
	}

	func newSearchResult(r postgres.SearchResult, searchSymbols bool, pr message.Printer) *SearchResult {
	// For commands, change the name from "main" to the last component of the import path.
	chipText := ""
	name := r.Name
	if name == "main" {
	chipText = "command"
	name = effectiveName(r.PackagePath, r.Name)
	}
	moduleDesc := "Other packages in module " + r.ModulePath
	if r.ModulePath == stdlib.ModulePath {
	moduleDesc = "Related packages in the standard library"
	chipText = "standard library"
	}
	sr := &SearchResult{
	Name: name,
	PackagePath: r.PackagePath,
	ModulePath: r.ModulePath,
	Version: r.Version,
	ChipText: chipText,
	Synopsis: r.Synopsis,
	DisplayVersion: displayVersion(r.ModulePath, r.Version, r.Version),
	Licenses: r.Licenses,
	CommitTime: elapsedTime(r.CommitTime),
	NumImportedBy: pr.Sprint(r.NumImportedBy),
	SameModule: packagePaths(moduleDesc+":", r.SameModule),
	// Say "other" instead of "lower" because at some point we may
	// prefer to show a tagged, lower major version over an untagged
	// higher major version.
	OtherMajor: modulePaths("Other major versions:", r.OtherMajor),
	}
	if searchSymbols {
	sr.SymbolName = r.SymbolName
	sr.SymbolKind = strings.ToLower(string(r.SymbolKind))
	sr.SymbolSynopsis = symbolSynopsis(r)
	sr.SymbolGOOS = r.SymbolGOOS
	sr.SymbolGOARCH = r.SymbolGOARCH
	// If the GOOS is "all" or "linux", it doesn't need to be
	// specified as a query param. "linux" is the default GOOS when a
	// package has multiple build contexts, since it is first item
	// listed in internal.BuildContexts.
	if r.SymbolGOOS == internal.All \|\| r.SymbolGOOS == "linux" {
	sr.SymbolLink = fmt.Sprintf("/%s#%s", r.PackagePath, r.SymbolName)
	} else {
	sr.SymbolLink = fmt.Sprintf("/%s?GOOS=%s#%s", r.PackagePath, r.SymbolGOOS, r.SymbolName)
	}
	}
	return sr
	}

	// A regexp that matches Go vuln IDs.
	var goVulnIDRegexp = regexp.MustCompile("^GO-[0-9]{4}-[0-9]{4}$")

	// searchRequestRedirectPath returns the path that a search request should be
	// redirected to, or the empty string if there is no such path.
	//
	// If the user types an existing package path into the search bar, we will
	// redirect the user to the details page. Standard library packages that only
	// contain one element (such as fmt, errors, etc.) will not redirect, to allow
	// users to search by those terms.
	//
	// If the user types a name that is in the form of a Go vulnerability ID, we will
	// redirect to the page for that ID (whether or not it exists).
	func searchRequestRedirectPath(ctx context.Context, ds internal.DataSource, query, mode string) string {
	urlSchemeIdx := strings.Index(query, "://")
	if urlSchemeIdx > -1 {
	query = query[urlSchemeIdx+3:]
	}
	if goVulnIDRegexp.MatchString(query) {
	return fmt.Sprintf("/vuln/%s?q", query)
	}
	requestedPath := path.Clean(query)
	if !strings.Contains(requestedPath, "/") \|\| mode == searchModeVuln {
	return ""
	}
	_, err := ds.GetUnitMeta(ctx, requestedPath, internal.UnknownModulePath, version.Latest)
	if err != nil {
	if !errors.Is(err, derrors.NotFound) {
	log.Errorf(ctx, "searchRequestRedirectPath(%q): %v", requestedPath, err)
	}
	return ""
	}
	return fmt.Sprintf("/%s", requestedPath)
	}

	func searchVulnModule(ctx context.Context, mode, cq string, client vulnc.Client) (_ *searchAction, err error) {
	if mode != searchModeVuln {
	return nil, nil
	}
	allEntries, err := vulnList(ctx, client)
	if err != nil {
	return nil, err
	}
	prefix := cq + "/"
	var entries []OSVEntry
	EntryLoop:
	for _, entry := range allEntries {
	for _, aff := range entry.Affected {
	for _, imp := range aff.EcosystemSpecific.Imports {
	if imp.Path == cq \|\| strings.HasPrefix(imp.Path, prefix) {
	entries = append(entries, entry)
	continue EntryLoop
	}
	}
	}
	}
	// Sort from most to least recent.
	sort.Slice(entries, func(i, j int) bool { return entries[i].ID > entries[j].ID })
	return &searchAction{
	title: fmt.Sprintf("%s - Vulnerability Reports", cq),
	template: "vuln/list",
	page: &VulnListPage{Entries: entries},
	}, nil
	}

	func searchVulnAlias(ctx context.Context, mode, cq string, vulnClient vulnc.Client) (_ *searchAction, err error) {
	defer derrors.Wrap(&err, "searchVulnAlias(%q, %q)", mode, cq)

	if mode != searchModeVuln \|\| !isVulnAlias(cq) {
	return nil, nil
	}
	aliasEntries, err := vulnClient.GetByAlias(ctx, cq)
	if err != nil {
	return nil, err
	}
	switch len(aliasEntries) {
	case 0:
	return nil, &serverError{status: http.StatusNotFound}
	case 1:
	return &searchAction{redirectURL: "/vuln/" + aliasEntries[0].ID}, nil
	default:
	var entries []OSVEntry
	for _, e := range aliasEntries {
	entries = append(entries, OSVEntry{e})
	}
	return &searchAction{
	title: fmt.Sprintf("%s - Vulnerability Reports", cq),
	template: "vuln/list",
	page: &VulnListPage{Entries: entries},
	}, nil
	}
	}

	// Regexps that match aliases for Go vulns.
	var (
	cveRegexp = regexp.MustCompile("^CVE-[0-9]{4}-[0-9]+$")
	ghsaRegexp = regexp.MustCompile("^GHSA-.{4}-.{4}-.{4}$")
	)

	func isVulnAlias(s string) bool {
	return cveRegexp.MatchString(s) \|\| ghsaRegexp.MatchString(s)
	}

	// searchMode reports whether the search performed should be in package or
	// symbol search mode.
	func searchMode(r *http.Request) string {
	q, filters := searchQueryAndFilters(r)
	if len(filters) > 0 {
	return searchModeSymbol
	}
	switch rawSearchMode(r) {
	case searchModePackage:
	return searchModePackage
	case searchModeSymbol:
	return searchModeSymbol
	case searchModeVuln:
	return searchModeVuln
	default:
	if isVulnAlias(q) {
	return searchModeVuln
	}
	if shouldDefaultToSymbolSearch(q) {
	return searchModeSymbol
	}
	return searchModePackage
	}
	}

	// searchQueryAndFilters returns the search query, trimmed of any filters, and
	// the array of words that had a filter prefix.
	func searchQueryAndFilters(r *http.Request) (string, []string) {
	words := strings.Fields(rawSearchQuery(r))
	var filters []string
	for i := range words {
	if strings.HasPrefix(words[i], symbolSearchFilter) {
	words[i] = strings.TrimLeft(words[i], symbolSearchFilter)
	filters = append(filters, words[i])
	}
	}
	return strings.Join(words, " "), filters
	}

	// rawSearchQuery returns the exact search query by the user.
	func rawSearchQuery(r *http.Request) string {
	return strings.TrimSpace(r.FormValue("q"))
	}

	// rawSearchMode returns the exact search mode from the URL request.
	func rawSearchMode(r *http.Request) string {
	return strings.TrimSpace(r.FormValue("m"))
	}

	// shouldDefaultToSymbolSearch reports whether the search mode should
	// default to symbol based on the input.
	func shouldDefaultToSymbolSearch(q string) bool {
	if len(strings.Fields(q)) != 1 {
	return false
	}
	if internal.IsGoPkgInPathElement(q) {
	return false
	}
	parts := strings.Split(q, ".")
	if len(parts) > 1 {
	if len(parts) == 2 && semver.IsValid(parts[1]) {
	// The q has the format <text>.<semver> which is likely a
	// gopkg.in host, such as yaml.v2. Default to package search.
	return false
	}
	return !internal.TopLevelDomains[parts[len(parts)-1]]
	}
	// If a user searches for "Unmarshal", assume that they are searching for
	// the symbol name "Unmarshal", not the package unmarshal.
	return isCapitalized(q)
	}

	// symbolSynopsis returns the string to be displayed in the code snippet
	// section for a symbol search result.
	func symbolSynopsis(r *postgres.SearchResult) string {
	switch r.SymbolKind {
	case internal.SymbolKindField:
	return fmt.Sprintf(`
	type %s struct {
	%s
	}
	`, strings.Split(r.SymbolName, ".")[0], r.SymbolSynopsis)
	case internal.SymbolKindMethod:
	if !strings.HasPrefix(r.SymbolSynopsis, "func (") {
	return fmt.Sprintf(`
	type %s interface {
	%s
	}
	`, strings.Split(r.SymbolName, ".")[0], r.SymbolSynopsis)
	}
	}
	return r.SymbolSynopsis
	}

	func packagePaths(heading string, rs []postgres.SearchResult) subResult {
	if len(rs) == 0 {
	return nil
	}
	var links []link
	for _, r := range rs {
	links = append(links, link{Href: r.PackagePath, Body: internal.Suffix(r.PackagePath, r.ModulePath)})
	}
	return &subResult{
	Heading: heading,
	Links: links,
	}
	}

	func modulePaths(heading string, modulePathToMajor map[string]int) *subResult {
	if len(modulePathToMajor) == 0 {
	return nil
	}

	type mm struct {
	Path string
	Major int
	}

	var mms []mm
	for m, v := range modulePathToMajor {
	mms = append(mms, mm{m, v})
	}
	sort.Slice(mms, func(i, j int) bool { return mms[i].Major > mms[j].Major })
	links := make([]link, len(mms))
	for i, m := range mms {
	links[i] = link{Href: m.Path, Body: fmt.Sprintf("v%d", m.Major)}
	}
	return &subResult{
	Heading: heading,
	Links: links,
	}
	}

	// isCapitalized reports whether the first letter of s is capitalized.
	func isCapitalized(s string) bool {
	if len(s) == 0 {
	return false
	}
	return unicode.IsUpper(rune(s[0]))
	}

	// elapsedTime takes a date and returns returns human-readable,
	// relative timestamps based on the following rules:
	// (1) 'X hours ago' when X < 6
	// (2) 'today' between 6 hours and 1 day ago
	// (3) 'Y days ago' when Y < 6
	// (4) A date formatted like "Jan 2, 2006" for anything further back
	func elapsedTime(date time.Time) string {
	elapsedHours := int(time.Since(date).Hours())
	if elapsedHours == 1 {
	return "1 hour ago"
	} else if elapsedHours < 6 {
	return fmt.Sprintf("%d hours ago", elapsedHours)
	}

	elapsedDays := elapsedHours / 24
	if elapsedDays < 1 {
	return "today"
	} else if elapsedDays == 1 {
	return "1 day ago"
	} else if elapsedDays < 6 {
	return fmt.Sprintf("%d days ago", elapsedDays)
	}

	return absoluteTime(date)
	}

	// addVulns adds vulnerability information to search results by consulting the
	// vulnerability database.
	func addVulns(ctx context.Context, rs []*SearchResult, getVulnEntries vulns.VulnEntriesFunc) {
	// Get all vulns concurrently.
	var wg sync.WaitGroup
	// TODO(golang/go#48223): throttle concurrency?
	for _, r := range rs {
	r := r
	wg.Add(1)
	go func() {
	defer wg.Done()
	r.Vulns = vulns.VulnsForPackage(ctx, r.ModulePath, r.Version, r.PackagePath, getVulnEntries)
	}()
	}
	wg.Wait()

	}