internal/godoc: delete indexing code
We no longer serve search from the golang.org site.
Change-Id: I8b855cf664f175c0dc46a91e3017df1bebb28281
Reviewed-on: https://go-review.googlesource.com/c/website/+/293427
Trust: Russ Cox <rsc@golang.org>
Reviewed-by: Dmitri Shuralyov <dmitshur@golang.org>
diff --git a/_content/lib/godoc/godoc.html b/_content/lib/godoc/godoc.html
index e22af76..c50b8b3 100644
--- a/_content/lib/godoc/godoc.html
+++ b/_content/lib/godoc/godoc.html
@@ -12,9 +12,6 @@
<link href="https://fonts.googleapis.com/css?family=Work+Sans:600|Roboto:400,700" rel="stylesheet">
<link href="https://fonts.googleapis.com/css?family=Product+Sans&text=Supported%20by%20Google&display=swap" rel="stylesheet">
<link type="text/css" rel="stylesheet" href="/lib/godoc/style.css">
-{{if .SearchBox}}
-<link rel="search" type="application/opensearchdescription+xml" title="godoc" href="/opensearch.xml" />
-{{end}}
<script>window.initFuncs = [];</script>
{{with .GoogleAnalytics}}
<script>
diff --git a/_content/lib/godoc/opensearch.xml b/_content/lib/godoc/opensearch.xml
deleted file mode 100644
index 1b652db..0000000
--- a/_content/lib/godoc/opensearch.xml
+++ /dev/null
@@ -1,11 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.1/">
- <ShortName>godoc</ShortName>
- <Description>The Go Programming Language</Description>
- <Tags>go golang</Tags>
- <Contact />
- <Url type="text/html" template="{{.BaseURL}}/search?q={searchTerms}" />
- <Image height="15" width="16" type="image/x-icon">/favicon.ico</Image>
- <OutputEncoding>UTF-8</OutputEncoding>
- <InputEncoding>UTF-8</InputEncoding>
-</OpenSearchDescription>
diff --git a/_content/lib/godoc/search.html b/_content/lib/godoc/search.html
deleted file mode 100644
index 3714e1d..0000000
--- a/_content/lib/godoc/search.html
+++ /dev/null
@@ -1,66 +0,0 @@
-<!--
- Copyright 2009 The Go Authors. All rights reserved.
- Use of this source code is governed by a BSD-style
- license that can be found in the LICENSE file.
--->
-
-{{ $colCount := tocColCount .}}
-{{/* Generate the TOC */}}
-<nav class="search-nav" style="column-count: {{$colCount}}" role="navigation">
-{{range $key, $val := .Idents}}
- {{if $val}}
- <a href="#{{$key.Name}}">{{$key.Name}}</a>
- <br />
- {{end}}
-{{end}}
-
-{{if not .Idents}}
- {{with .Pak}}
- <a href="#Packages">Package {{html $.Query}}</a>
- <br />
- {{end}}
-{{end}}
-
-{{with .Hit}}
- {{with .Decls}}
- <a href="#Global">Package-level declarations</a>
- <br />
- {{range .}}
- {{$pkg_html := pkgLink .Pak.Path | html}}
- <a href="#Global_{{$pkg_html}}" class="indent">package {{html .Pak.Name}}</a>
- <br />
- {{end}}
- {{end}}
- {{with .Others}}
- <a href="#Local">Local declarations and uses</a>
- <br />
- {{range .}}
- {{$pkg_html := pkgLink .Pak.Path | html}}
- <a href="#Local_{{$pkg_html}}" class="indent">package {{html .Pak.Name}}</a>
- <br />
- {{end}}
- {{end}}
-{{end}}
-
-{{with .Textual}}
- {{if $.Complete}}
- <a href="#Textual">{{html $.Found}} textual occurrences</a>
- {{else}}
- <a href="#Textual">More than {{html $.Found}} textual occurrences</a>
- {{end}}
-{{end}}
-</nav>
-
-{{with .Alert}}
- <p>
- <span class="alert" style="font-size:120%">{{html .}}</span>
- </p>
-{{end}}
-{{with .Alt}}
- <p>
- <span class="alert" style="font-size:120%">Did you mean: </span>
- {{range .Alts}}
- <a href="search?q={{urlquery .}}" style="font-size:120%">{{html .}}</a>
- {{end}}
- </p>
-{{end}}
diff --git a/_content/lib/godoc/searchcode.html b/_content/lib/godoc/searchcode.html
deleted file mode 100644
index a032e64..0000000
--- a/_content/lib/godoc/searchcode.html
+++ /dev/null
@@ -1,64 +0,0 @@
-<!--
- Copyright 2009 The Go Authors. All rights reserved.
- Use of this source code is governed by a BSD-style
- license that can be found in the LICENSE file.
--->
-{{$query_url := urlquery .Query}}
-{{if not .Idents}}
- {{with .Pak}}
- <h2 id="Packages">Package {{html $.Query}}</h2>
- <p>
- <table class="layout">
- {{range .}}
- {{$pkg_html := pkgLink .Pak.Path | html}}
- <tr><td><a href="/{{$pkg_html}}">{{$pkg_html}}</a></td></tr>
- {{end}}
- </table>
- </p>
- {{end}}
-{{end}}
-{{with .Hit}}
- {{with .Decls}}
- <h2 id="Global">Package-level declarations</h2>
- {{range .}}
- {{$pkg_html := pkgLink .Pak.Path | html}}
- <h3 id="Global_{{$pkg_html}}">package <a href="/{{$pkg_html}}">{{html .Pak.Name}}</a></h3>
- {{range .Files}}
- {{$file := .File.Path}}
- {{range .Groups}}
- {{range .}}
- {{$line := infoLine .}}
- <a href="{{queryLink $file $query_url $line | html}}">{{$file}}:{{$line}}</a>
- {{infoSnippet_html .}}
- {{end}}
- {{end}}
- {{end}}
- {{end}}
- {{end}}
- {{with .Others}}
- <h2 id="Local">Local declarations and uses</h2>
- {{range .}}
- {{$pkg_html := pkgLink .Pak.Path | html}}
- <h3 id="Local_{{$pkg_html}}">package <a href="/{{$pkg_html}}">{{html .Pak.Name}}</a></h3>
- {{range .Files}}
- {{$file := .File.Path}}
- <a href="{{queryLink $file $query_url 0 | html}}">{{$file}}</a>
- <table class="layout">
- {{range .Groups}}
- <tr>
- <td width="25"></td>
- <th align="left" valign="top">{{index . 0 | infoKind_html}}</th>
- <td align="left" width="4"></td>
- <td>
- {{range .}}
- {{$line := infoLine .}}
- <a href="{{queryLink $file $query_url $line | html}}">{{$line}}</a>
- {{end}}
- </td>
- </tr>
- {{end}}
- </table>
- {{end}}
- {{end}}
- {{end}}
-{{end}}
diff --git a/_content/lib/godoc/searchdoc.html b/_content/lib/godoc/searchdoc.html
deleted file mode 100644
index 679c02c..0000000
--- a/_content/lib/godoc/searchdoc.html
+++ /dev/null
@@ -1,24 +0,0 @@
-<!--
- Copyright 2009 The Go Authors. All rights reserved.
- Use of this source code is governed by a BSD-style
- license that can be found in the LICENSE file.
--->
-{{range $key, $val := .Idents}}
- {{if $val}}
- <h2 id="{{$key.Name}}">{{$key.Name}}</h2>
- {{range $val}}
- {{$pkg_html := pkgLink .Path | html}}
- {{if eq "Packages" $key.Name}}
- <a href="/{{$pkg_html}}">{{html .Path}}</a>
- {{else}}
- {{$doc_html := docLink .Path .Name| html}}
- <a href="/{{$pkg_html}}">{{html .Package}}</a>.<a href="{{$doc_html}}">{{.Name}}</a>
- {{end}}
- {{if .Doc}}
- <p>{{comment_html .Doc}}</p>
- {{else}}
- <p><em>No documentation available</em></p>
- {{end}}
- {{end}}
- {{end}}
-{{end}}
diff --git a/_content/lib/godoc/searchtxt.html b/_content/lib/godoc/searchtxt.html
deleted file mode 100644
index 7e4a978..0000000
--- a/_content/lib/godoc/searchtxt.html
+++ /dev/null
@@ -1,42 +0,0 @@
-<!--
- Copyright 2009 The Go Authors. All rights reserved.
- Use of this source code is governed by a BSD-style
- license that can be found in the LICENSE file.
--->
-{{$query_url := urlquery .Query}}
-{{with .Textual}}
- {{if $.Complete}}
- <h2 id="Textual">{{html $.Found}} textual occurrences</h2>
- {{else}}
- <h2 id="Textual">More than {{html $.Found}} textual occurrences</h2>
- <p>
- <span class="alert" style="font-size:120%">Not all files or lines containing "{{html $.Query}}" are shown.</span>
- </p>
- {{end}}
- <p>
- <table class="layout">
- {{range .}}
- {{$file := .Filename}}
- <tr>
- <td align="left" valign="top">
- <a href="{{queryLink $file $query_url 0}}">{{$file}}</a>:
- </td>
- <td align="left" width="4"></td>
- <th align="left" valign="top">{{len .Lines}}</th>
- <td align="left" width="4"></td>
- <td align="left">
- {{range .Lines}}
- <a href="{{queryLink $file $query_url .}}">{{html .}}</a>
- {{end}}
- {{if not $.Complete}}
- ...
- {{end}}
- </td>
- </tr>
- {{end}}
- {{if not $.Complete}}
- <tr><td align="left">...</td></tr>
- {{end}}
- </table>
- </p>
-{{end}}
diff --git a/cmd/golangorg/handlers.go b/cmd/golangorg/handlers.go
index faccde7..c1919c9 100644
--- a/cmd/golangorg/handlers.go
+++ b/cmd/golangorg/handlers.go
@@ -121,11 +121,6 @@
p.GodocHTML = readTemplate("godoc.html")
p.PackageHTML = readTemplate("package.html")
p.PackageRootHTML = readTemplate("packageroot.html")
- p.SearchHTML = readTemplate("search.html")
- p.SearchDocHTML = readTemplate("searchdoc.html")
- p.SearchCodeHTML = readTemplate("searchcode.html")
- p.SearchTxtHTML = readTemplate("searchtxt.html")
- p.SearchDescXML = readTemplate("opensearch.xml")
}
type fmtResponse struct {
diff --git a/cmd/golangorg/main.go b/cmd/golangorg/main.go
index 6384472..60f0294 100644
--- a/cmd/golangorg/main.go
+++ b/cmd/golangorg/main.go
@@ -93,7 +93,6 @@
corpus := godoc.NewCorpus(fs)
corpus.Verbose = *verbose
- corpus.IndexEnabled = false
if err := corpus.Init(); err != nil {
log.Fatal(err)
}
diff --git a/internal/godoc/corpus.go b/internal/godoc/corpus.go
index 153e132..74a2488 100644
--- a/internal/godoc/corpus.go
+++ b/internal/godoc/corpus.go
@@ -24,48 +24,6 @@
// Verbose logging.
Verbose bool
- // IndexEnabled controls whether indexing is enabled.
- IndexEnabled bool
-
- // IndexFiles specifies a glob pattern specifying index files.
- // If not empty, the index is read from these files in sorted
- // order.
- IndexFiles string
-
- // IndexThrottle specifies the indexing throttle value
- // between 0.0 and 1.0. At 0.0, the indexer always sleeps.
- // At 1.0, the indexer never sleeps. Because 0.0 is useless
- // and redundant with setting IndexEnabled to false, the
- // zero value for IndexThrottle means 0.9.
- IndexThrottle float64
-
- // IndexInterval specifies the time to sleep between reindexing
- // all the sources.
- // If zero, a default is used. If negative, the index is only
- // built once.
- IndexInterval time.Duration
-
- // IndexDocs enables indexing of Go documentation.
- // This will produce search results for exported types, functions,
- // methods, variables, and constants, and will link to the godoc
- // documentation for those identifiers.
- IndexDocs bool
-
- // IndexGoCode enables indexing of Go source code.
- // This will produce search results for internal and external identifiers
- // and will link to both declarations and uses of those identifiers in
- // source code.
- IndexGoCode bool
-
- // IndexFullText enables full-text indexing.
- // This will provide search results for any matching text in any file that
- // is indexed, including non-Go files (see whitelisted in index.go).
- // Regexp searching is supported via full-text indexing.
- IndexFullText bool
-
- // MaxResults optionally specifies the maximum results for indexing.
- MaxResults int
-
// SummarizePackage optionally specifies a function to
// summarize a package. It exists as an optimization to
// avoid reading files to parse package comments.
@@ -78,13 +36,6 @@
// package listing.
SummarizePackage func(pkg string) (summary string, showList, ok bool)
- // IndexDirectory optionally specifies a function to determine
- // whether the provided directory should be indexed. The dir
- // will be of the form "/src/cmd/6a", "/doc/play",
- // "/src/io", etc.
- // If nil, all directories are indexed if indexing is enabled.
- IndexDirectory func(dir string) bool
-
// Send a value on this channel to trigger a metadata refresh.
// It is buffered so that if a signal is not lost if sent
// during a refresh.
@@ -95,9 +46,6 @@
fsModified util.RWValue // timestamp of last call to invalidateIndex
docMetadata util.RWValue // mapping from paths to *Metadata
- // SearchIndex is the search index in use.
- searchIndex util.RWValue
-
// flag to check whether a corpus is initialized or not
initMu sync.RWMutex
initDone bool
@@ -114,22 +62,10 @@
c := &Corpus{
fs: fs,
refreshMetadataSignal: make(chan bool, 1),
-
- MaxResults: 1000,
- IndexEnabled: true,
- IndexDocs: true,
- IndexGoCode: true,
- IndexFullText: true,
}
return c
}
-func (c *Corpus) CurrentIndex() (*Index, time.Time) {
- v, t := c.searchIndex.Get()
- idx, _ := v.(*Index)
- return idx, t
-}
-
func (c *Corpus) FSModifiedTime() time.Time {
_, ts := c.fsModified.Get()
return ts
@@ -156,6 +92,5 @@
return errors.New("godoc: corpus fstree is nil")
}
c.fsTree.Set(dir)
- c.invalidateIndex()
return nil
}
diff --git a/internal/godoc/godoc.go b/internal/godoc/godoc.go
index 2a0c05c..faec584 100644
--- a/internal/godoc/godoc.go
+++ b/internal/godoc/godoc.go
@@ -69,11 +69,6 @@
"fileInfoName": fileInfoNameFunc,
"fileInfoTime": fileInfoTimeFunc,
- // access to search result information
- "infoKind_html": infoKind_htmlFunc,
- "infoLine": p.infoLineFunc,
- "infoSnippet_html": p.infoSnippet_htmlFunc,
-
// formatting of AST nodes
"node": p.nodeFunc,
"node_html": p.node_htmlFunc,
@@ -105,9 +100,6 @@
// check whether to display third party section or not
"hasThirdParty": hasThirdParty,
-
- // get the no. of columns to split the toc in search page
- "tocColCount": tocColCount,
}
if p.URLForSrc != nil {
p.funcMap["srcLink"] = p.URLForSrc
@@ -142,48 +134,6 @@
return "" // don't return epoch if time is obviously not set
}
-// The strings in infoKinds must be properly html-escaped.
-var infoKinds = [nKinds]string{
- PackageClause: "package clause",
- ImportDecl: "import decl",
- ConstDecl: "const decl",
- TypeDecl: "type decl",
- VarDecl: "var decl",
- FuncDecl: "func decl",
- MethodDecl: "method decl",
- Use: "use",
-}
-
-func infoKind_htmlFunc(info SpotInfo) string {
- return infoKinds[info.Kind()] // infoKind entries are html-escaped
-}
-
-func (p *Presentation) infoLineFunc(info SpotInfo) int {
- line := info.Lori()
- if info.IsIndex() {
- index, _ := p.Corpus.searchIndex.Get()
- if index != nil {
- line = index.(*Index).Snippet(line).Line
- } else {
- // no line information available because
- // we don't have an index - this should
- // never happen; be conservative and don't
- // crash
- line = 0
- }
- }
- return line
-}
-
-func (p *Presentation) infoSnippet_htmlFunc(info SpotInfo) string {
- if info.IsIndex() {
- index, _ := p.Corpus.searchIndex.Get()
- // Snippet.Text was HTML-escaped when it was generated
- return index.(*Index).Snippet(info.Lori()).Text
- }
- return `<span class="alert">no snippet text available</span>`
-}
-
func (p *Presentation) nodeFunc(info *PageInfo, node interface{}) string {
var buf bytes.Buffer
p.writeNode(&buf, info, info.FSet, node)
diff --git a/internal/godoc/index.go b/internal/godoc/index.go
deleted file mode 100644
index a9c0859..0000000
--- a/internal/godoc/index.go
+++ /dev/null
@@ -1,1580 +0,0 @@
-// Copyright 2009 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// This file contains the infrastructure to create an
-// identifier and full-text index for a set of Go files.
-//
-// Algorithm for identifier index:
-// - traverse all .go files of the file tree specified by root
-// - for each identifier (word) encountered, collect all occurrences (spots)
-// into a list; this produces a list of spots for each word
-// - reduce the lists: from a list of spots to a list of FileRuns,
-// and from a list of FileRuns into a list of PakRuns
-// - make a HitList from the PakRuns
-//
-// Details:
-// - keep two lists per word: one containing package-level declarations
-// that have snippets, and one containing all other spots
-// - keep the snippets in a separate table indexed by snippet index
-// and store the snippet index in place of the line number in a SpotInfo
-// (the line number for spots with snippets is stored in the snippet)
-// - at the end, create lists of alternative spellings for a given
-// word
-//
-// Algorithm for full text index:
-// - concatenate all source code in a byte buffer (in memory)
-// - add the files to a file set in lockstep as they are added to the byte
-// buffer such that a byte buffer offset corresponds to the Pos value for
-// that file location
-// - create a suffix array from the concatenated sources
-//
-// String lookup in full text index:
-// - use the suffix array to lookup a string's offsets - the offsets
-// correspond to the Pos values relative to the file set
-// - translate the Pos values back into file and line information and
-// sort the result
-
-package godoc
-
-import (
- "bufio"
- "bytes"
- "encoding/gob"
- "errors"
- "fmt"
- "go/ast"
- "go/doc"
- "go/parser"
- "go/token"
- "index/suffixarray"
- "io"
- "log"
- "os"
- pathpkg "path"
- "path/filepath"
- "regexp"
- "runtime"
- "sort"
- "strconv"
- "strings"
- "sync"
- "time"
- "unicode"
-
- "golang.org/x/website/internal/godoc/util"
- "golang.org/x/website/internal/godoc/vfs"
-)
-
-// ----------------------------------------------------------------------------
-// InterfaceSlice is a helper type for sorting interface
-// slices according to some slice-specific sort criteria.
-
-type comparer func(x, y interface{}) bool
-
-type interfaceSlice struct {
- slice []interface{}
- less comparer
-}
-
-// ----------------------------------------------------------------------------
-// RunList
-
-// A RunList is a list of entries that can be sorted according to some
-// criteria. A RunList may be compressed by grouping "runs" of entries
-// which are equal (according to the sort criteria) into a new RunList of
-// runs. For instance, a RunList containing pairs (x, y) may be compressed
-// into a RunList containing pair runs (x, {y}) where each run consists of
-// a list of y's with the same x.
-type RunList []interface{}
-
-func (h RunList) sort(less comparer) {
- sort.Sort(&interfaceSlice{h, less})
-}
-
-func (p *interfaceSlice) Len() int { return len(p.slice) }
-func (p *interfaceSlice) Less(i, j int) bool { return p.less(p.slice[i], p.slice[j]) }
-func (p *interfaceSlice) Swap(i, j int) { p.slice[i], p.slice[j] = p.slice[j], p.slice[i] }
-
-// Compress entries which are the same according to a sort criteria
-// (specified by less) into "runs".
-func (h RunList) reduce(less comparer, newRun func(h RunList) interface{}) RunList {
- if len(h) == 0 {
- return nil
- }
- // len(h) > 0
-
- // create runs of entries with equal values
- h.sort(less)
-
- // for each run, make a new run object and collect them in a new RunList
- var hh RunList
- i, x := 0, h[0]
- for j, y := range h {
- if less(x, y) {
- hh = append(hh, newRun(h[i:j]))
- i, x = j, h[j] // start a new run
- }
- }
- // add final run, if any
- if i < len(h) {
- hh = append(hh, newRun(h[i:]))
- }
-
- return hh
-}
-
-// ----------------------------------------------------------------------------
-// KindRun
-
-// Debugging support. Disable to see multiple entries per line.
-const removeDuplicates = true
-
-// A KindRun is a run of SpotInfos of the same kind in a given file.
-// The kind (3 bits) is stored in each SpotInfo element; to find the
-// kind of a KindRun, look at any of its elements.
-type KindRun []SpotInfo
-
-// KindRuns are sorted by line number or index. Since the isIndex bit
-// is always the same for all infos in one list we can compare lori's.
-func (k KindRun) Len() int { return len(k) }
-func (k KindRun) Less(i, j int) bool { return k[i].Lori() < k[j].Lori() }
-func (k KindRun) Swap(i, j int) { k[i], k[j] = k[j], k[i] }
-
-// FileRun contents are sorted by Kind for the reduction into KindRuns.
-func lessKind(x, y interface{}) bool { return x.(SpotInfo).Kind() < y.(SpotInfo).Kind() }
-
-// newKindRun allocates a new KindRun from the SpotInfo run h.
-func newKindRun(h RunList) interface{} {
- run := make(KindRun, len(h))
- for i, x := range h {
- run[i] = x.(SpotInfo)
- }
-
- // Spots were sorted by file and kind to create this run.
- // Within this run, sort them by line number or index.
- sort.Sort(run)
-
- if removeDuplicates {
- // Since both the lori and kind field must be
- // same for duplicates, and since the isIndex
- // bit is always the same for all infos in one
- // list we can simply compare the entire info.
- k := 0
- prev := SpotInfo(1<<32 - 1) // an unlikely value
- for _, x := range run {
- if x != prev {
- run[k] = x
- k++
- prev = x
- }
- }
- run = run[0:k]
- }
-
- return run
-}
-
-// ----------------------------------------------------------------------------
-// FileRun
-
-// A Pak describes a Go package.
-type Pak struct {
- Path string // path of directory containing the package
- Name string // package name as declared by package clause
-}
-
-// Paks are sorted by name (primary key) and by import path (secondary key).
-func (p *Pak) less(q *Pak) bool {
- return p.Name < q.Name || p.Name == q.Name && p.Path < q.Path
-}
-
-// A File describes a Go file.
-type File struct {
- Name string // directory-local file name
- Pak *Pak // the package to which the file belongs
-}
-
-// Path returns the file path of f.
-func (f *File) Path() string {
- return pathpkg.Join(f.Pak.Path, f.Name)
-}
-
-// A Spot describes a single occurrence of a word.
-type Spot struct {
- File *File
- Info SpotInfo
-}
-
-// A FileRun is a list of KindRuns belonging to the same file.
-type FileRun struct {
- File *File
- Groups []KindRun
-}
-
-// Spots are sorted by file path for the reduction into FileRuns.
-func lessSpot(x, y interface{}) bool {
- fx := x.(Spot).File
- fy := y.(Spot).File
- // same as "return fx.Path() < fy.Path()" but w/o computing the file path first
- px := fx.Pak.Path
- py := fy.Pak.Path
- return px < py || px == py && fx.Name < fy.Name
-}
-
-// newFileRun allocates a new FileRun from the Spot run h.
-func newFileRun(h RunList) interface{} {
- file := h[0].(Spot).File
-
- // reduce the list of Spots into a list of KindRuns
- h1 := make(RunList, len(h))
- for i, x := range h {
- h1[i] = x.(Spot).Info
- }
- h2 := h1.reduce(lessKind, newKindRun)
-
- // create the FileRun
- groups := make([]KindRun, len(h2))
- for i, x := range h2 {
- groups[i] = x.(KindRun)
- }
- return &FileRun{file, groups}
-}
-
-// ----------------------------------------------------------------------------
-// PakRun
-
-// A PakRun describes a run of *FileRuns of a package.
-type PakRun struct {
- Pak *Pak
- Files []*FileRun
-}
-
-// Sorting support for files within a PakRun.
-func (p *PakRun) Len() int { return len(p.Files) }
-func (p *PakRun) Less(i, j int) bool { return p.Files[i].File.Name < p.Files[j].File.Name }
-func (p *PakRun) Swap(i, j int) { p.Files[i], p.Files[j] = p.Files[j], p.Files[i] }
-
-// FileRuns are sorted by package for the reduction into PakRuns.
-func lessFileRun(x, y interface{}) bool {
- return x.(*FileRun).File.Pak.less(y.(*FileRun).File.Pak)
-}
-
-// newPakRun allocates a new PakRun from the *FileRun run h.
-func newPakRun(h RunList) interface{} {
- pak := h[0].(*FileRun).File.Pak
- files := make([]*FileRun, len(h))
- for i, x := range h {
- files[i] = x.(*FileRun)
- }
- run := &PakRun{pak, files}
- sort.Sort(run) // files were sorted by package; sort them by file now
- return run
-}
-
-// ----------------------------------------------------------------------------
-// HitList
-
-// A HitList describes a list of PakRuns.
-type HitList []*PakRun
-
-// PakRuns are sorted by package.
-func lessPakRun(x, y interface{}) bool { return x.(*PakRun).Pak.less(y.(*PakRun).Pak) }
-
-func reduce(h0 RunList) HitList {
- // reduce a list of Spots into a list of FileRuns
- h1 := h0.reduce(lessSpot, newFileRun)
- // reduce a list of FileRuns into a list of PakRuns
- h2 := h1.reduce(lessFileRun, newPakRun)
- // sort the list of PakRuns by package
- h2.sort(lessPakRun)
- // create a HitList
- h := make(HitList, len(h2))
- for i, p := range h2 {
- h[i] = p.(*PakRun)
- }
- return h
-}
-
-// filter returns a new HitList created by filtering
-// all PakRuns from h that have a matching pakname.
-func (h HitList) filter(pakname string) HitList {
- var hh HitList
- for _, p := range h {
- if p.Pak.Name == pakname {
- hh = append(hh, p)
- }
- }
- return hh
-}
-
-// ----------------------------------------------------------------------------
-// AltWords
-
-type wordPair struct {
- canon string // canonical word spelling (all lowercase)
- alt string // alternative spelling
-}
-
-// An AltWords describes a list of alternative spellings for a
-// canonical (all lowercase) spelling of a word.
-type AltWords struct {
- Canon string // canonical word spelling (all lowercase)
- Alts []string // alternative spelling for the same word
-}
-
-// wordPairs are sorted by their canonical spelling.
-func lessWordPair(x, y interface{}) bool { return x.(*wordPair).canon < y.(*wordPair).canon }
-
-// newAltWords allocates a new AltWords from the *wordPair run h.
-func newAltWords(h RunList) interface{} {
- canon := h[0].(*wordPair).canon
- alts := make([]string, len(h))
- for i, x := range h {
- alts[i] = x.(*wordPair).alt
- }
- return &AltWords{canon, alts}
-}
-
-func (a *AltWords) filter(s string) *AltWords {
- var alts []string
- for _, w := range a.Alts {
- if w != s {
- alts = append(alts, w)
- }
- }
- if len(alts) > 0 {
- return &AltWords{a.Canon, alts}
- }
- return nil
-}
-
-// Ident stores information about external identifiers in order to create
-// links to package documentation.
-type Ident struct {
- Path string // e.g. "net/http"
- Package string // e.g. "http"
- Name string // e.g. "NewRequest"
- Doc string // e.g. "NewRequest returns a new Request..."
-}
-
-// byImportCount sorts the given slice of Idents by the import
-// counts of the packages to which they belong.
-type byImportCount struct {
- Idents []Ident
- ImportCount map[string]int
-}
-
-func (ic byImportCount) Len() int {
- return len(ic.Idents)
-}
-
-func (ic byImportCount) Less(i, j int) bool {
- ri := ic.ImportCount[ic.Idents[i].Path]
- rj := ic.ImportCount[ic.Idents[j].Path]
- if ri == rj {
- return ic.Idents[i].Path < ic.Idents[j].Path
- }
- return ri > rj
-}
-
-func (ic byImportCount) Swap(i, j int) {
- ic.Idents[i], ic.Idents[j] = ic.Idents[j], ic.Idents[i]
-}
-
-func (ic byImportCount) String() string {
- buf := bytes.NewBuffer([]byte("["))
- for _, v := range ic.Idents {
- buf.WriteString(fmt.Sprintf("\n\t%s, %s (%d)", v.Path, v.Name, ic.ImportCount[v.Path]))
- }
- buf.WriteString("\n]")
- return buf.String()
-}
-
-// filter creates a new Ident list where the results match the given
-// package name.
-func (ic byImportCount) filter(pakname string) []Ident {
- if ic.Idents == nil {
- return nil
- }
- var res []Ident
- for _, i := range ic.Idents {
- if i.Package == pakname {
- res = append(res, i)
- }
- }
- return res
-}
-
-// top returns the top n identifiers.
-func (ic byImportCount) top(n int) []Ident {
- if len(ic.Idents) > n {
- return ic.Idents[:n]
- }
- return ic.Idents
-}
-
-// ----------------------------------------------------------------------------
-// Indexer
-
-type IndexResult struct {
- Decls RunList // package-level declarations (with snippets)
- Others RunList // all other occurrences
-}
-
-// Statistics provides statistics information for an index.
-type Statistics struct {
- Bytes int // total size of indexed source files
- Files int // number of indexed source files
- Lines int // number of lines (all files)
- Words int // number of different identifiers
- Spots int // number of identifier occurrences
-}
-
-// An Indexer maintains the data structures and provides the machinery
-// for indexing .go files under a file tree. It implements the path.Visitor
-// interface for walking file trees, and the ast.Visitor interface for
-// walking Go ASTs.
-type Indexer struct {
- c *Corpus
- fset *token.FileSet // file set for all indexed files
- fsOpenGate chan bool // send pre fs.Open; receive on close
-
- mu sync.Mutex // guards all the following
- sources bytes.Buffer // concatenated sources
- strings map[string]string // interned string
- packages map[Pak]*Pak // interned *Paks
- words map[string]*IndexResult // RunLists of Spots
- snippets []*Snippet // indices are stored in SpotInfos
- current *token.File // last file added to file set
- file *File // AST for current file
- decl ast.Decl // AST for current decl
- stats Statistics
- throttle *util.Throttle
- importCount map[string]int // package path ("net/http") => count
- packagePath map[string]map[string]bool // "template" => "text/template" => true
- exports map[string]map[string]SpotKind // "net/http" => "ListenAndServe" => FuncDecl
- curPkgExports map[string]SpotKind
- idents map[SpotKind]map[string][]Ident // kind => name => list of Idents
-}
-
-func (x *Indexer) intern(s string) string {
- if s, ok := x.strings[s]; ok {
- return s
- }
- x.strings[s] = s
- return s
-}
-
-func (x *Indexer) lookupPackage(path, name string) *Pak {
- // In the source directory tree, more than one package may
- // live in the same directory. For the packages map, construct
- // a key that includes both the directory path and the package
- // name.
- key := Pak{Path: x.intern(path), Name: x.intern(name)}
- pak := x.packages[key]
- if pak == nil {
- pak = &key
- x.packages[key] = pak
- }
- return pak
-}
-
-func (x *Indexer) addSnippet(s *Snippet) int {
- index := len(x.snippets)
- x.snippets = append(x.snippets, s)
- return index
-}
-
-func (x *Indexer) visitIdent(kind SpotKind, id *ast.Ident) {
- if id == nil {
- return
- }
- name := x.intern(id.Name)
-
- switch kind {
- case TypeDecl, FuncDecl, ConstDecl, VarDecl:
- x.curPkgExports[name] = kind
- }
-
- lists, found := x.words[name]
- if !found {
- lists = new(IndexResult)
- x.words[name] = lists
- }
-
- if kind == Use || x.decl == nil {
- if x.c.IndexGoCode {
- // not a declaration or no snippet required
- info := makeSpotInfo(kind, x.current.Line(id.Pos()), false)
- lists.Others = append(lists.Others, Spot{x.file, info})
- }
- } else {
- // a declaration with snippet
- index := x.addSnippet(NewSnippet(x.fset, x.decl, id))
- info := makeSpotInfo(kind, index, true)
- lists.Decls = append(lists.Decls, Spot{x.file, info})
- }
-
- x.stats.Spots++
-}
-
-func (x *Indexer) visitFieldList(kind SpotKind, flist *ast.FieldList) {
- for _, f := range flist.List {
- x.decl = nil // no snippets for fields
- for _, name := range f.Names {
- x.visitIdent(kind, name)
- }
- ast.Walk(x, f.Type)
- // ignore tag - not indexed at the moment
- }
-}
-
-func (x *Indexer) visitSpec(kind SpotKind, spec ast.Spec) {
- switch n := spec.(type) {
- case *ast.ImportSpec:
- x.visitIdent(ImportDecl, n.Name)
- if n.Path != nil {
- if imp, err := strconv.Unquote(n.Path.Value); err == nil {
- x.importCount[x.intern(imp)]++
- }
- }
-
- case *ast.ValueSpec:
- for _, n := range n.Names {
- x.visitIdent(kind, n)
- }
- ast.Walk(x, n.Type)
- for _, v := range n.Values {
- ast.Walk(x, v)
- }
-
- case *ast.TypeSpec:
- x.visitIdent(TypeDecl, n.Name)
- ast.Walk(x, n.Type)
- }
-}
-
-func (x *Indexer) visitGenDecl(decl *ast.GenDecl) {
- kind := VarDecl
- if decl.Tok == token.CONST {
- kind = ConstDecl
- }
- x.decl = decl
- for _, s := range decl.Specs {
- x.visitSpec(kind, s)
- }
-}
-
-func (x *Indexer) Visit(node ast.Node) ast.Visitor {
- switch n := node.(type) {
- case nil:
- // nothing to do
-
- case *ast.Ident:
- x.visitIdent(Use, n)
-
- case *ast.FieldList:
- x.visitFieldList(VarDecl, n)
-
- case *ast.InterfaceType:
- x.visitFieldList(MethodDecl, n.Methods)
-
- case *ast.DeclStmt:
- // local declarations should only be *ast.GenDecls;
- // ignore incorrect ASTs
- if decl, ok := n.Decl.(*ast.GenDecl); ok {
- x.decl = nil // no snippets for local declarations
- x.visitGenDecl(decl)
- }
-
- case *ast.GenDecl:
- x.decl = n
- x.visitGenDecl(n)
-
- case *ast.FuncDecl:
- kind := FuncDecl
- if n.Recv != nil {
- kind = MethodDecl
- ast.Walk(x, n.Recv)
- }
- x.decl = n
- x.visitIdent(kind, n.Name)
- ast.Walk(x, n.Type)
- if n.Body != nil {
- ast.Walk(x, n.Body)
- }
-
- case *ast.File:
- x.decl = nil
- x.visitIdent(PackageClause, n.Name)
- for _, d := range n.Decls {
- ast.Walk(x, d)
- }
-
- default:
- return x
- }
-
- return nil
-}
-
-// addFile adds a file to the index if possible and returns the file set file
-// and the file's AST if it was successfully parsed as a Go file. If addFile
-// failed (that is, if the file was not added), it returns file == nil.
-func (x *Indexer) addFile(f vfs.ReadSeekCloser, filename string, goFile bool) (file *token.File, ast *ast.File) {
- defer f.Close()
-
- // The file set's base offset and x.sources size must be in lock-step;
- // this permits the direct mapping of suffix array lookup results to
- // to corresponding Pos values.
- //
- // When a file is added to the file set, its offset base increases by
- // the size of the file + 1; and the initial base offset is 1. Add an
- // extra byte to the sources here.
- x.sources.WriteByte(0)
-
- // If the sources length doesn't match the file set base at this point
- // the file set implementation changed or we have another error.
- base := x.fset.Base()
- if x.sources.Len() != base {
- panic("internal error: file base incorrect")
- }
-
- // append file contents (src) to x.sources
- if _, err := x.sources.ReadFrom(f); err == nil {
- src := x.sources.Bytes()[base:]
-
- if goFile {
- // parse the file and in the process add it to the file set
- if ast, err = parser.ParseFile(x.fset, filename, src, parser.ParseComments); err == nil {
- file = x.fset.File(ast.Pos()) // ast.Pos() is inside the file
- return
- }
- // file has parse errors, and the AST may be incorrect -
- // set lines information explicitly and index as ordinary
- // text file (cannot fall through to the text case below
- // because the file has already been added to the file set
- // by the parser)
- file = x.fset.File(token.Pos(base)) // token.Pos(base) is inside the file
- file.SetLinesForContent(src)
- ast = nil
- return
- }
-
- if util.IsText(src) {
- // only add the file to the file set (for the full text index)
- file = x.fset.AddFile(filename, x.fset.Base(), len(src))
- file.SetLinesForContent(src)
- return
- }
- }
-
- // discard possibly added data
- x.sources.Truncate(base - 1) // -1 to remove added byte 0 since no file was added
- return
-}
-
-// Design note: Using an explicit white list of permitted files for indexing
-// makes sure that the important files are included and massively reduces the
-// number of files to index. The advantage over a blacklist is that unexpected
-// (non-blacklisted) files won't suddenly explode the index.
-
-// Files are whitelisted if they have a file name or extension
-// present as key in whitelisted.
-var whitelisted = map[string]bool{
- ".bash": true,
- ".c": true,
- ".cc": true,
- ".cpp": true,
- ".cxx": true,
- ".css": true,
- ".go": true,
- ".goc": true,
- ".h": true,
- ".hh": true,
- ".hpp": true,
- ".hxx": true,
- ".html": true,
- ".js": true,
- ".out": true,
- ".py": true,
- ".s": true,
- ".sh": true,
- ".txt": true,
- ".xml": true,
- "AUTHORS": true,
- "CONTRIBUTORS": true,
- "LICENSE": true,
- "Makefile": true,
- "PATENTS": true,
- "README": true,
-}
-
-// isWhitelisted returns true if a file is on the list
-// of "permitted" files for indexing. The filename must
-// be the directory-local name of the file.
-func isWhitelisted(filename string) bool {
- key := pathpkg.Ext(filename)
- if key == "" {
- // file has no extension - use entire filename
- key = filename
- }
- return whitelisted[key]
-}
-
-func (x *Indexer) indexDocs(dirname string, filename string, astFile *ast.File) {
- pkgName := x.intern(astFile.Name.Name)
- if pkgName == "main" {
- return
- }
- pkgPath := x.intern(strings.TrimPrefix(strings.TrimPrefix(dirname, "/src/"), "pkg/"))
- astPkg := ast.Package{
- Name: pkgName,
- Files: map[string]*ast.File{
- filename: astFile,
- },
- }
- var m doc.Mode
- docPkg := doc.New(&astPkg, dirname, m)
- addIdent := func(sk SpotKind, name string, docstr string) {
- if x.idents[sk] == nil {
- x.idents[sk] = make(map[string][]Ident)
- }
- name = x.intern(name)
- x.idents[sk][name] = append(x.idents[sk][name], Ident{
- Path: pkgPath,
- Package: pkgName,
- Name: name,
- Doc: doc.Synopsis(docstr),
- })
- }
-
- if x.idents[PackageClause] == nil {
- x.idents[PackageClause] = make(map[string][]Ident)
- }
- // List of words under which the package identifier will be stored.
- // This includes the package name and the components of the directory
- // in which it resides.
- words := strings.Split(pathpkg.Dir(pkgPath), "/")
- if words[0] == "." {
- words = []string{}
- }
- name := x.intern(docPkg.Name)
- synopsis := doc.Synopsis(docPkg.Doc)
- words = append(words, name)
- pkgIdent := Ident{
- Path: pkgPath,
- Package: pkgName,
- Name: name,
- Doc: synopsis,
- }
- for _, word := range words {
- word = x.intern(word)
- found := false
- pkgs := x.idents[PackageClause][word]
- for i, p := range pkgs {
- if p.Path == pkgPath {
- if docPkg.Doc != "" {
- p.Doc = synopsis
- pkgs[i] = p
- }
- found = true
- break
- }
- }
- if !found {
- x.idents[PackageClause][word] = append(x.idents[PackageClause][word], pkgIdent)
- }
- }
-
- for _, c := range docPkg.Consts {
- for _, name := range c.Names {
- addIdent(ConstDecl, name, c.Doc)
- }
- }
- for _, t := range docPkg.Types {
- addIdent(TypeDecl, t.Name, t.Doc)
- for _, c := range t.Consts {
- for _, name := range c.Names {
- addIdent(ConstDecl, name, c.Doc)
- }
- }
- for _, v := range t.Vars {
- for _, name := range v.Names {
- addIdent(VarDecl, name, v.Doc)
- }
- }
- for _, f := range t.Funcs {
- addIdent(FuncDecl, f.Name, f.Doc)
- }
- for _, f := range t.Methods {
- addIdent(MethodDecl, f.Name, f.Doc)
- // Change the name of methods to be "<typename>.<methodname>".
- // They will still be indexed as <methodname>.
- idents := x.idents[MethodDecl][f.Name]
- idents[len(idents)-1].Name = x.intern(t.Name + "." + f.Name)
- }
- }
- for _, v := range docPkg.Vars {
- for _, name := range v.Names {
- addIdent(VarDecl, name, v.Doc)
- }
- }
- for _, f := range docPkg.Funcs {
- addIdent(FuncDecl, f.Name, f.Doc)
- }
-}
-
-func (x *Indexer) indexGoFile(dirname string, filename string, file *token.File, astFile *ast.File) {
- pkgName := astFile.Name.Name
-
- if x.c.IndexGoCode {
- x.current = file
- pak := x.lookupPackage(dirname, pkgName)
- x.file = &File{filename, pak}
- ast.Walk(x, astFile)
- }
-
- if x.c.IndexDocs {
- // Test files are already filtered out in visitFile if IndexGoCode and
- // IndexFullText are false. Otherwise, check here.
- isTestFile := (x.c.IndexGoCode || x.c.IndexFullText) &&
- (strings.HasSuffix(filename, "_test.go") || strings.HasPrefix(dirname, "/test/"))
- if !isTestFile {
- x.indexDocs(dirname, filename, astFile)
- }
- }
-
- ppKey := x.intern(pkgName)
- if _, ok := x.packagePath[ppKey]; !ok {
- x.packagePath[ppKey] = make(map[string]bool)
- }
- pkgPath := x.intern(strings.TrimPrefix(strings.TrimPrefix(dirname, "/src/"), "pkg/"))
- x.packagePath[ppKey][pkgPath] = true
-
- // Merge in exported symbols found walking this file into
- // the map for that package.
- if len(x.curPkgExports) > 0 {
- dest, ok := x.exports[pkgPath]
- if !ok {
- dest = make(map[string]SpotKind)
- x.exports[pkgPath] = dest
- }
- for k, v := range x.curPkgExports {
- dest[k] = v
- }
- }
-}
-
-func (x *Indexer) visitFile(dirname string, fi os.FileInfo) {
- if fi.IsDir() || !x.c.IndexEnabled {
- return
- }
-
- filename := pathpkg.Join(dirname, fi.Name())
- goFile := isGoFile(fi)
-
- switch {
- case x.c.IndexFullText:
- if !isWhitelisted(fi.Name()) {
- return
- }
- case x.c.IndexGoCode:
- if !goFile {
- return
- }
- case x.c.IndexDocs:
- if !goFile ||
- strings.HasSuffix(fi.Name(), "_test.go") ||
- strings.HasPrefix(dirname, "/test/") {
- return
- }
- default:
- // No indexing turned on.
- return
- }
-
- x.fsOpenGate <- true
- defer func() { <-x.fsOpenGate }()
-
- // open file
- f, err := x.c.fs.Open(filename)
- if err != nil {
- return
- }
-
- x.mu.Lock()
- defer x.mu.Unlock()
-
- x.throttle.Throttle()
-
- x.curPkgExports = make(map[string]SpotKind)
- file, fast := x.addFile(f, filename, goFile)
- if file == nil {
- return // addFile failed
- }
-
- if fast != nil {
- x.indexGoFile(dirname, fi.Name(), file, fast)
- }
-
- // update statistics
- x.stats.Bytes += file.Size()
- x.stats.Files++
- x.stats.Lines += file.LineCount()
-}
-
-// indexOptions contains information that affects the contents of an index.
-type indexOptions struct {
- // Docs provides documentation search results.
- // It is only consulted if IndexEnabled is true.
- // The default values is true.
- Docs bool
-
- // GoCode provides Go source code search results.
- // It is only consulted if IndexEnabled is true.
- // The default values is true.
- GoCode bool
-
- // FullText provides search results from all files.
- // It is only consulted if IndexEnabled is true.
- // The default values is true.
- FullText bool
-
- // MaxResults optionally specifies the maximum results for indexing.
- // The default is 1000.
- MaxResults int
-}
-
-// ----------------------------------------------------------------------------
-// Index
-
-type LookupResult struct {
- Decls HitList // package-level declarations (with snippets)
- Others HitList // all other occurrences
-}
-
-type Index struct {
- fset *token.FileSet // file set used during indexing; nil if no textindex
- suffixes *suffixarray.Index // suffixes for concatenated sources; nil if no textindex
- words map[string]*LookupResult // maps words to hit lists
- alts map[string]*AltWords // maps canonical(words) to lists of alternative spellings
- snippets []*Snippet // all snippets, indexed by snippet index
- stats Statistics
- importCount map[string]int // package path ("net/http") => count
- packagePath map[string]map[string]bool // "template" => "text/template" => true
- exports map[string]map[string]SpotKind // "net/http" => "ListenAndServe" => FuncDecl
- idents map[SpotKind]map[string][]Ident
- opts indexOptions
-}
-
-func canonical(w string) string { return strings.ToLower(w) }
-
-// Somewhat arbitrary, but I figure low enough to not hurt disk-based filesystems
-// consuming file descriptors, where some systems have low 256 or 512 limits.
-// Go should have a built-in way to cap fd usage under the ulimit.
-const (
- maxOpenFiles = 200
- maxOpenDirs = 50
-)
-
-func (c *Corpus) throttle() float64 {
- if c.IndexThrottle <= 0 {
- return 0.9
- }
- if c.IndexThrottle > 1.0 {
- return 1.0
- }
- return c.IndexThrottle
-}
-
-// NewIndex creates a new index for the .go files provided by the corpus.
-func (c *Corpus) NewIndex() *Index {
- // initialize Indexer
- // (use some reasonably sized maps to start)
- x := &Indexer{
- c: c,
- fset: token.NewFileSet(),
- fsOpenGate: make(chan bool, maxOpenFiles),
- strings: make(map[string]string),
- packages: make(map[Pak]*Pak, 256),
- words: make(map[string]*IndexResult, 8192),
- throttle: util.NewThrottle(c.throttle(), 100*time.Millisecond), // run at least 0.1s at a time
- importCount: make(map[string]int),
- packagePath: make(map[string]map[string]bool),
- exports: make(map[string]map[string]SpotKind),
- idents: make(map[SpotKind]map[string][]Ident, 4),
- }
-
- // index all files in the directories given by dirnames
- var wg sync.WaitGroup // outstanding ReadDir + visitFile
- dirGate := make(chan bool, maxOpenDirs)
- for dirname := range c.fsDirnames() {
- if c.IndexDirectory != nil && !c.IndexDirectory(dirname) {
- continue
- }
- dirGate <- true
- wg.Add(1)
- go func(dirname string) {
- defer func() { <-dirGate }()
- defer wg.Done()
-
- list, err := c.fs.ReadDir(dirname)
- if err != nil {
- log.Printf("ReadDir(%q): %v; skipping directory", dirname, err)
- return // ignore this directory
- }
- for _, fi := range list {
- wg.Add(1)
- go func(fi os.FileInfo) {
- defer wg.Done()
- x.visitFile(dirname, fi)
- }(fi)
- }
- }(dirname)
- }
- wg.Wait()
-
- if !c.IndexFullText {
- // the file set, the current file, and the sources are
- // not needed after indexing if no text index is built -
- // help GC and clear them
- x.fset = nil
- x.sources.Reset()
- x.current = nil // contains reference to fset!
- }
-
- // for each word, reduce the RunLists into a LookupResult;
- // also collect the word with its canonical spelling in a
- // word list for later computation of alternative spellings
- words := make(map[string]*LookupResult)
- var wlist RunList
- for w, h := range x.words {
- decls := reduce(h.Decls)
- others := reduce(h.Others)
- words[w] = &LookupResult{
- Decls: decls,
- Others: others,
- }
- wlist = append(wlist, &wordPair{canonical(w), w})
- x.throttle.Throttle()
- }
- x.stats.Words = len(words)
-
- // reduce the word list {canonical(w), w} into
- // a list of AltWords runs {canonical(w), {w}}
- alist := wlist.reduce(lessWordPair, newAltWords)
-
- // convert alist into a map of alternative spellings
- alts := make(map[string]*AltWords)
- for i := 0; i < len(alist); i++ {
- a := alist[i].(*AltWords)
- alts[a.Canon] = a
- }
-
- // create text index
- var suffixes *suffixarray.Index
- if c.IndexFullText {
- suffixes = suffixarray.New(x.sources.Bytes())
- }
-
- // sort idents by the number of imports of their respective packages
- for _, idMap := range x.idents {
- for _, ir := range idMap {
- sort.Sort(byImportCount{ir, x.importCount})
- }
- }
-
- return &Index{
- fset: x.fset,
- suffixes: suffixes,
- words: words,
- alts: alts,
- snippets: x.snippets,
- stats: x.stats,
- importCount: x.importCount,
- packagePath: x.packagePath,
- exports: x.exports,
- idents: x.idents,
- opts: indexOptions{
- Docs: x.c.IndexDocs,
- GoCode: x.c.IndexGoCode,
- FullText: x.c.IndexFullText,
- MaxResults: x.c.MaxResults,
- },
- }
-}
-
-var ErrFileIndexVersion = errors.New("file index version out of date")
-
-const fileIndexVersion = 3
-
-// fileIndex is the subset of Index that's gob-encoded for use by
-// Index.Write and Index.Read.
-type fileIndex struct {
- Version int
- Words map[string]*LookupResult
- Alts map[string]*AltWords
- Snippets []*Snippet
- Fulltext bool
- Stats Statistics
- ImportCount map[string]int
- PackagePath map[string]map[string]bool
- Exports map[string]map[string]SpotKind
- Idents map[SpotKind]map[string][]Ident
- Opts indexOptions
-}
-
-func (x *fileIndex) Write(w io.Writer) error {
- return gob.NewEncoder(w).Encode(x)
-}
-
-func (x *fileIndex) Read(r io.Reader) error {
- return gob.NewDecoder(r).Decode(x)
-}
-
-// WriteTo writes the index x to w.
-func (x *Index) WriteTo(w io.Writer) (n int64, err error) {
- w = countingWriter{&n, w}
- fulltext := false
- if x.suffixes != nil {
- fulltext = true
- }
- fx := fileIndex{
- Version: fileIndexVersion,
- Words: x.words,
- Alts: x.alts,
- Snippets: x.snippets,
- Fulltext: fulltext,
- Stats: x.stats,
- ImportCount: x.importCount,
- PackagePath: x.packagePath,
- Exports: x.exports,
- Idents: x.idents,
- Opts: x.opts,
- }
- if err := fx.Write(w); err != nil {
- return 0, err
- }
- if fulltext {
- encode := func(x interface{}) error {
- return gob.NewEncoder(w).Encode(x)
- }
- if err := x.fset.Write(encode); err != nil {
- return 0, err
- }
- if err := x.suffixes.Write(w); err != nil {
- return 0, err
- }
- }
- return n, nil
-}
-
-// ReadFrom reads the index from r into x; x must not be nil.
-// If r does not also implement io.ByteReader, it will be wrapped in a bufio.Reader.
-// If the index is from an old version, the error is ErrFileIndexVersion.
-func (x *Index) ReadFrom(r io.Reader) (n int64, err error) {
- // We use the ability to read bytes as a plausible surrogate for buffering.
- if _, ok := r.(io.ByteReader); !ok {
- r = bufio.NewReader(r)
- }
- r = countingReader{&n, r.(byteReader)}
- var fx fileIndex
- if err := fx.Read(r); err != nil {
- return n, err
- }
- if fx.Version != fileIndexVersion {
- return 0, ErrFileIndexVersion
- }
- x.words = fx.Words
- x.alts = fx.Alts
- x.snippets = fx.Snippets
- x.stats = fx.Stats
- x.importCount = fx.ImportCount
- x.packagePath = fx.PackagePath
- x.exports = fx.Exports
- x.idents = fx.Idents
- x.opts = fx.Opts
- if fx.Fulltext {
- x.fset = token.NewFileSet()
- decode := func(x interface{}) error {
- return gob.NewDecoder(r).Decode(x)
- }
- if err := x.fset.Read(decode); err != nil {
- return n, err
- }
- x.suffixes = new(suffixarray.Index)
- if err := x.suffixes.Read(r); err != nil {
- return n, err
- }
- }
- return n, nil
-}
-
-// Stats returns index statistics.
-func (x *Index) Stats() Statistics {
- return x.stats
-}
-
-// ImportCount returns a map from import paths to how many times they were seen.
-func (x *Index) ImportCount() map[string]int {
- return x.importCount
-}
-
-// PackagePath returns a map from short package name to a set
-// of full package path names that use that short package name.
-func (x *Index) PackagePath() map[string]map[string]bool {
- return x.packagePath
-}
-
-// Exports returns a map from full package path to exported
-// symbol name to its type.
-func (x *Index) Exports() map[string]map[string]SpotKind {
- return x.exports
-}
-
-// Idents returns a map from identifier type to exported
-// symbol name to the list of identifiers matching that name.
-func (x *Index) Idents() map[SpotKind]map[string][]Ident {
- return x.idents
-}
-
-func (x *Index) lookupWord(w string) (match *LookupResult, alt *AltWords) {
- match = x.words[w]
- alt = x.alts[canonical(w)]
- // remove current spelling from alternatives
- // (if there is no match, the alternatives do
- // not contain the current spelling)
- if match != nil && alt != nil {
- alt = alt.filter(w)
- }
- return
-}
-
-// isIdentifier reports whether s is a Go identifier.
-func isIdentifier(s string) bool {
- for i, ch := range s {
- if unicode.IsLetter(ch) || ch == '_' || i > 0 && unicode.IsDigit(ch) {
- continue
- }
- return false
- }
- return len(s) > 0
-}
-
-// For a given query, which is either a single identifier or a qualified
-// identifier, Lookup returns a SearchResult containing packages, a LookupResult, a
-// list of alternative spellings, and identifiers, if any. Any and all results
-// may be nil. If the query syntax is wrong, an error is reported.
-func (x *Index) Lookup(query string) (*SearchResult, error) {
- ss := strings.Split(query, ".")
-
- // check query syntax
- for _, s := range ss {
- if !isIdentifier(s) {
- return nil, errors.New("all query parts must be identifiers")
- }
- }
- rslt := &SearchResult{
- Query: query,
- Idents: make(map[SpotKind][]Ident, 5),
- }
- // handle simple and qualified identifiers
- switch len(ss) {
- case 1:
- ident := ss[0]
- rslt.Hit, rslt.Alt = x.lookupWord(ident)
- if rslt.Hit != nil {
- // found a match - filter packages with same name
- // for the list of packages called ident, if any
- rslt.Pak = rslt.Hit.Others.filter(ident)
- }
- for k, v := range x.idents {
- const rsltLimit = 50
- ids := byImportCount{v[ident], x.importCount}
- rslt.Idents[k] = ids.top(rsltLimit)
- }
-
- case 2:
- pakname, ident := ss[0], ss[1]
- rslt.Hit, rslt.Alt = x.lookupWord(ident)
- if rslt.Hit != nil {
- // found a match - filter by package name
- // (no paks - package names are not qualified)
- decls := rslt.Hit.Decls.filter(pakname)
- others := rslt.Hit.Others.filter(pakname)
- rslt.Hit = &LookupResult{decls, others}
- }
- for k, v := range x.idents {
- ids := byImportCount{v[ident], x.importCount}
- rslt.Idents[k] = ids.filter(pakname)
- }
-
- default:
- return nil, errors.New("query is not a (qualified) identifier")
- }
-
- return rslt, nil
-}
-
-func (x *Index) Snippet(i int) *Snippet {
- // handle illegal snippet indices gracefully
- if 0 <= i && i < len(x.snippets) {
- return x.snippets[i]
- }
- return nil
-}
-
-type positionList []struct {
- filename string
- line int
-}
-
-func (list positionList) Len() int { return len(list) }
-func (list positionList) Less(i, j int) bool { return list[i].filename < list[j].filename }
-func (list positionList) Swap(i, j int) { list[i], list[j] = list[j], list[i] }
-
-// unique returns the list sorted and with duplicate entries removed
-func unique(list []int) []int {
- sort.Ints(list)
- var last int
- i := 0
- for _, x := range list {
- if i == 0 || x != last {
- last = x
- list[i] = x
- i++
- }
- }
- return list[0:i]
-}
-
-// A FileLines value specifies a file and line numbers within that file.
-type FileLines struct {
- Filename string
- Lines []int
-}
-
-// LookupRegexp returns the number of matches and the matches where a regular
-// expression r is found in the full text index. At most n matches are
-// returned (thus found <= n).
-//
-func (x *Index) LookupRegexp(r *regexp.Regexp, n int) (found int, result []FileLines) {
- if x.suffixes == nil || n <= 0 {
- return
- }
- // n > 0
-
- var list positionList
- // FindAllIndex may returns matches that span across file boundaries.
- // Such matches are unlikely, buf after eliminating them we may end up
- // with fewer than n matches. If we don't have enough at the end, redo
- // the search with an increased value n1, but only if FindAllIndex
- // returned all the requested matches in the first place (if it
- // returned fewer than that there cannot be more).
- for n1 := n; found < n; n1 += n - found {
- found = 0
- matches := x.suffixes.FindAllIndex(r, n1)
- // compute files, exclude matches that span file boundaries,
- // and map offsets to file-local offsets
- list = make(positionList, len(matches))
- for _, m := range matches {
- // by construction, an offset corresponds to the Pos value
- // for the file set - use it to get the file and line
- p := token.Pos(m[0])
- if file := x.fset.File(p); file != nil {
- if base := file.Base(); base <= m[1] && m[1] <= base+file.Size() {
- // match [m[0], m[1]) is within the file boundaries
- list[found].filename = file.Name()
- list[found].line = file.Line(p)
- found++
- }
- }
- }
- if found == n || len(matches) < n1 {
- // found all matches or there's no chance to find more
- break
- }
- }
- list = list[0:found]
- sort.Sort(list) // sort by filename
-
- // collect matches belonging to the same file
- var last string
- var lines []int
- addLines := func() {
- if len(lines) > 0 {
- // remove duplicate lines
- result = append(result, FileLines{last, unique(lines)})
- lines = nil
- }
- }
- for _, m := range list {
- if m.filename != last {
- addLines()
- last = m.filename
- }
- lines = append(lines, m.line)
- }
- addLines()
-
- return
-}
-
-// InvalidateIndex should be called whenever any of the file systems
-// under godoc's observation change so that the indexer is kicked on.
-func (c *Corpus) invalidateIndex() {
- c.fsModified.Set(nil)
- c.refreshMetadata()
-}
-
-// feedDirnames feeds the directory names of all directories
-// under the file system given by root to channel c.
-//
-func (c *Corpus) feedDirnames(ch chan<- string) {
- if dir, _ := c.fsTree.Get(); dir != nil {
- for d := range dir.(*Directory).iter(false) {
- ch <- d.Path
- }
- }
-}
-
-// fsDirnames() returns a channel sending all directory names
-// of all the file systems under godoc's observation.
-//
-func (c *Corpus) fsDirnames() <-chan string {
- ch := make(chan string, 256) // buffered for fewer context switches
- go func() {
- c.feedDirnames(ch)
- close(ch)
- }()
- return ch
-}
-
-// CompatibleWith reports whether the Index x is compatible with the corpus
-// indexing options set in c.
-func (x *Index) CompatibleWith(c *Corpus) bool {
- return x.opts.Docs == c.IndexDocs &&
- x.opts.GoCode == c.IndexGoCode &&
- x.opts.FullText == c.IndexFullText &&
- x.opts.MaxResults == c.MaxResults
-}
-
-func (c *Corpus) readIndex(filenames string) error {
- matches, err := filepath.Glob(filenames)
- if err != nil {
- return err
- } else if matches == nil {
- return fmt.Errorf("no index files match %q", filenames)
- }
- sort.Strings(matches) // make sure files are in the right order
- files := make([]io.Reader, 0, len(matches))
- for _, filename := range matches {
- f, err := os.Open(filename)
- if err != nil {
- return err
- }
- defer f.Close()
- files = append(files, f)
- }
- return c.ReadIndexFrom(io.MultiReader(files...))
-}
-
-// ReadIndexFrom sets the current index from the serialized version found in r.
-func (c *Corpus) ReadIndexFrom(r io.Reader) error {
- x := new(Index)
- if _, err := x.ReadFrom(r); err != nil {
- return err
- }
- if !x.CompatibleWith(c) {
- return fmt.Errorf("index file options are incompatible: %v", x.opts)
- }
- c.searchIndex.Set(x)
- return nil
-}
-
-func (c *Corpus) UpdateIndex() {
- if c.Verbose {
- log.Printf("updating index...")
- }
- start := time.Now()
- index := c.NewIndex()
- stop := time.Now()
- c.searchIndex.Set(index)
- if c.Verbose {
- secs := stop.Sub(start).Seconds()
- stats := index.Stats()
- log.Printf("index updated (%gs, %d bytes of source, %d files, %d lines, %d unique words, %d spots)",
- secs, stats.Bytes, stats.Files, stats.Lines, stats.Words, stats.Spots)
- }
- memstats := new(runtime.MemStats)
- runtime.ReadMemStats(memstats)
- if c.Verbose {
- log.Printf("before GC: bytes = %d footprint = %d", memstats.HeapAlloc, memstats.Sys)
- }
- runtime.GC()
- runtime.ReadMemStats(memstats)
- if c.Verbose {
- log.Printf("after GC: bytes = %d footprint = %d", memstats.HeapAlloc, memstats.Sys)
- }
-}
-
-// RunIndexer runs forever, indexing.
-func (c *Corpus) RunIndexer() {
- // initialize the index from disk if possible
- if c.IndexFiles != "" {
- c.initFSTree()
- if err := c.readIndex(c.IndexFiles); err != nil {
- log.Printf("error reading index from file %s: %v", c.IndexFiles, err)
- }
- return
- }
-
- // Repeatedly update the package directory tree and index.
- for {
- c.initFSTree()
- c.UpdateIndex()
- if c.IndexInterval < 0 {
- return
- }
- delay := 5 * time.Minute // by default, reindex every 5 minutes
- if c.IndexInterval > 0 {
- delay = c.IndexInterval
- }
- time.Sleep(delay)
- }
-}
-
-type countingWriter struct {
- n *int64
- w io.Writer
-}
-
-func (c countingWriter) Write(p []byte) (n int, err error) {
- n, err = c.w.Write(p)
- *c.n += int64(n)
- return
-}
-
-type byteReader interface {
- io.Reader
- io.ByteReader
-}
-
-type countingReader struct {
- n *int64
- r byteReader
-}
-
-func (c countingReader) Read(p []byte) (n int, err error) {
- n, err = c.r.Read(p)
- *c.n += int64(n)
- return
-}
-
-func (c countingReader) ReadByte() (b byte, err error) {
- b, err = c.r.ReadByte()
- *c.n += 1
- return
-}
diff --git a/internal/godoc/index_test.go b/internal/godoc/index_test.go
deleted file mode 100644
index 3718699..0000000
--- a/internal/godoc/index_test.go
+++ /dev/null
@@ -1,323 +0,0 @@
-// Copyright 2013 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package godoc
-
-import (
- "bytes"
- "reflect"
- "sort"
- "strings"
- "testing"
-
- "golang.org/x/website/internal/godoc/vfs/mapfs"
-)
-
-func newCorpus(t *testing.T) *Corpus {
- c := NewCorpus(mapfs.New(map[string]string{
- "src/foo/foo.go": `// Package foo is an example.
-package foo
-
-import "bar"
-
-const Pi = 3.1415
-
-var Foos []Foo
-
-// Foo is stuff.
-type Foo struct{}
-
-func New() *Foo {
- return new(Foo)
-}
-`,
- "src/bar/bar.go": `// Package bar is another example to test races.
-package bar
-`,
- "src/other/bar/bar.go": `// Package bar is another bar package.
-package bar
-func X() {}
-`,
- "src/skip/skip.go": `// Package skip should be skipped.
-package skip
-func Skip() {}
-`,
- "src/bar/readme.txt": `Whitelisted text file.
-`,
- "src/bar/baz.zzz": `Text file not whitelisted.
-`,
- }))
- c.IndexEnabled = true
- c.IndexDirectory = func(dir string) bool {
- return !strings.Contains(dir, "skip")
- }
-
- if err := c.Init(); err != nil {
- t.Fatal(err)
- }
- return c
-}
-
-func TestIndex(t *testing.T) {
- for _, docs := range []bool{true, false} {
- for _, goCode := range []bool{true, false} {
- for _, fullText := range []bool{true, false} {
- c := newCorpus(t)
- c.IndexDocs = docs
- c.IndexGoCode = goCode
- c.IndexFullText = fullText
- c.UpdateIndex()
- ix, _ := c.CurrentIndex()
- if ix == nil {
- t.Fatal("no index")
- }
- t.Logf("docs, goCode, fullText = %v,%v,%v", docs, goCode, fullText)
- testIndex(t, c, ix)
- }
- }
- }
-}
-
-func TestIndexWriteRead(t *testing.T) {
- type key struct {
- docs, goCode, fullText bool
- }
- type val struct {
- buf *bytes.Buffer
- c *Corpus
- }
- m := map[key]val{}
-
- for _, docs := range []bool{true, false} {
- for _, goCode := range []bool{true, false} {
- for _, fullText := range []bool{true, false} {
- k := key{docs, goCode, fullText}
- c := newCorpus(t)
- c.IndexDocs = docs
- c.IndexGoCode = goCode
- c.IndexFullText = fullText
- c.UpdateIndex()
- ix, _ := c.CurrentIndex()
- if ix == nil {
- t.Fatal("no index")
- }
- var buf bytes.Buffer
- nw, err := ix.WriteTo(&buf)
- if err != nil {
- t.Fatalf("Index.WriteTo: %v", err)
- }
- m[k] = val{bytes.NewBuffer(buf.Bytes()), c}
- ix2 := new(Index)
- nr, err := ix2.ReadFrom(&buf)
- if err != nil {
- t.Fatalf("Index.ReadFrom: %v", err)
- }
- if nr != nw {
- t.Errorf("Wrote %d bytes to index but read %d", nw, nr)
- }
- testIndex(t, c, ix)
- }
- }
- }
- // Test CompatibleWith
- for k1, v1 := range m {
- ix := new(Index)
- if _, err := ix.ReadFrom(v1.buf); err != nil {
- t.Fatalf("Index.ReadFrom: %v", err)
- }
- for k2, v2 := range m {
- if got, want := ix.CompatibleWith(v2.c), k1 == k2; got != want {
- t.Errorf("CompatibleWith = %v; want %v for %v, %v", got, want, k1, k2)
- }
- }
- }
-}
-
-func testIndex(t *testing.T, c *Corpus, ix *Index) {
- if _, ok := ix.words["Skip"]; ok {
- t.Errorf("the word Skip was found; expected it to be skipped")
- }
- checkStats(t, c, ix)
- checkImportCount(t, c, ix)
- checkPackagePath(t, c, ix)
- checkExports(t, c, ix)
- checkIdents(t, c, ix)
-}
-
-// checkStats checks the Index's statistics.
-// Some statistics are only set when we're indexing Go code.
-func checkStats(t *testing.T, c *Corpus, ix *Index) {
- want := Statistics{}
- if c.IndexFullText {
- want.Bytes = 314
- want.Files = 4
- want.Lines = 21
- } else if c.IndexDocs || c.IndexGoCode {
- want.Bytes = 291
- want.Files = 3
- want.Lines = 20
- }
- if c.IndexGoCode {
- want.Words = 8
- want.Spots = 12
- }
- if got := ix.Stats(); !reflect.DeepEqual(got, want) {
- t.Errorf("Stats = %#v; want %#v", got, want)
- }
-}
-
-// checkImportCount checks the Index's import count map.
-// It is only set when we're indexing Go code.
-func checkImportCount(t *testing.T, c *Corpus, ix *Index) {
- want := map[string]int{}
- if c.IndexGoCode {
- want = map[string]int{
- "bar": 1,
- }
- }
- if got := ix.ImportCount(); !reflect.DeepEqual(got, want) {
- t.Errorf("ImportCount = %v; want %v", got, want)
- }
-}
-
-// checkPackagePath checks the Index's package path map.
-// It is set if at least one of the indexing options is enabled.
-func checkPackagePath(t *testing.T, c *Corpus, ix *Index) {
- want := map[string]map[string]bool{}
- if c.IndexDocs || c.IndexGoCode || c.IndexFullText {
- want = map[string]map[string]bool{
- "foo": {
- "foo": true,
- },
- "bar": {
- "bar": true,
- "other/bar": true,
- },
- }
- }
- if got := ix.PackagePath(); !reflect.DeepEqual(got, want) {
- t.Errorf("PackagePath = %v; want %v", got, want)
- }
-}
-
-// checkExports checks the Index's exports map.
-// It is only set when we're indexing Go code.
-func checkExports(t *testing.T, c *Corpus, ix *Index) {
- want := map[string]map[string]SpotKind{}
- if c.IndexGoCode {
- want = map[string]map[string]SpotKind{
- "foo": {
- "Pi": ConstDecl,
- "Foos": VarDecl,
- "Foo": TypeDecl,
- "New": FuncDecl,
- },
- "other/bar": {
- "X": FuncDecl,
- },
- }
- }
- if got := ix.Exports(); !reflect.DeepEqual(got, want) {
- t.Errorf("Exports = %v; want %v", got, want)
- }
-}
-
-// checkIdents checks the Index's indents map.
-// It is only set when we're indexing documentation.
-func checkIdents(t *testing.T, c *Corpus, ix *Index) {
- want := map[SpotKind]map[string][]Ident{}
- if c.IndexDocs {
- want = map[SpotKind]map[string][]Ident{
- PackageClause: {
- "bar": {
- {"bar", "bar", "bar", "Package bar is another example to test races."},
- {"other/bar", "bar", "bar", "Package bar is another bar package."},
- },
- "foo": {{"foo", "foo", "foo", "Package foo is an example."}},
- "other": {{"other/bar", "bar", "bar", "Package bar is another bar package."}},
- },
- ConstDecl: {
- "Pi": {{"foo", "foo", "Pi", ""}},
- },
- VarDecl: {
- "Foos": {{"foo", "foo", "Foos", ""}},
- },
- TypeDecl: {
- "Foo": {{"foo", "foo", "Foo", "Foo is stuff."}},
- },
- FuncDecl: {
- "New": {{"foo", "foo", "New", ""}},
- "X": {{"other/bar", "bar", "X", ""}},
- },
- }
- }
- if got := ix.Idents(); !reflect.DeepEqual(got, want) {
- t.Errorf("Idents = %v; want %v", got, want)
- }
-}
-
-func TestIdentResultSort(t *testing.T) {
- ic := map[string]int{
- "/a/b/pkg1": 10,
- "/a/b/pkg2": 2,
- "/b/d/pkg3": 20,
- }
- for _, tc := range []struct {
- ir []Ident
- exp []Ident
- }{
- {
- ir: []Ident{
- {"/a/b/pkg2", "pkg2", "MyFunc2", ""},
- {"/b/d/pkg3", "pkg3", "MyFunc3", ""},
- {"/a/b/pkg1", "pkg1", "MyFunc1", ""},
- },
- exp: []Ident{
- {"/b/d/pkg3", "pkg3", "MyFunc3", ""},
- {"/a/b/pkg1", "pkg1", "MyFunc1", ""},
- {"/a/b/pkg2", "pkg2", "MyFunc2", ""},
- },
- },
- {
- ir: []Ident{
- {"/a/a/pkg1", "pkg1", "MyFunc1", ""},
- {"/a/b/pkg1", "pkg1", "MyFunc1", ""},
- },
- exp: []Ident{
- {"/a/b/pkg1", "pkg1", "MyFunc1", ""},
- {"/a/a/pkg1", "pkg1", "MyFunc1", ""},
- },
- },
- } {
- if sort.Sort(byImportCount{tc.ir, ic}); !reflect.DeepEqual(tc.ir, tc.exp) {
- t.Errorf("got: %v, want %v", tc.ir, tc.exp)
- }
- }
-}
-
-func TestIdentFilter(t *testing.T) {
- ic := map[string]int{}
- for _, tc := range []struct {
- ir []Ident
- pak string
- exp []Ident
- }{
- {
- ir: []Ident{
- {"/a/b/pkg2", "pkg2", "MyFunc2", ""},
- {"/b/d/pkg3", "pkg3", "MyFunc3", ""},
- {"/a/b/pkg1", "pkg1", "MyFunc1", ""},
- },
- pak: "pkg2",
- exp: []Ident{
- {"/a/b/pkg2", "pkg2", "MyFunc2", ""},
- },
- },
- } {
- res := byImportCount{tc.ir, ic}.filter(tc.pak)
- if !reflect.DeepEqual(res, tc.exp) {
- t.Errorf("got: %v, want %v", res, tc.exp)
- }
- }
-}
diff --git a/internal/godoc/page.go b/internal/godoc/page.go
index a892a5a..279d803 100644
--- a/internal/godoc/page.go
+++ b/internal/godoc/page.go
@@ -25,7 +25,6 @@
GoogleCN bool // page is being served from golang.google.cn
// filled in by ServePage
- SearchBox bool
Playground bool
Version string
GoogleAnalytics string
@@ -35,7 +34,6 @@
if page.Tabtitle == "" {
page.Tabtitle = page.Title
}
- page.SearchBox = p.Corpus.IndexEnabled
page.Playground = p.ShowPlayground
page.Version = runtime.Version()
page.GoogleAnalytics = p.GoogleAnalytics
diff --git a/internal/godoc/pres.go b/internal/godoc/pres.go
index bb0057b..cc01204 100644
--- a/internal/godoc/pres.go
+++ b/internal/godoc/pres.go
@@ -13,9 +13,6 @@
"golang.org/x/website/internal/godoc/vfs/httpfs"
)
-// SearchResultFunc functions return an HTML body for displaying search results.
-type SearchResultFunc func(p *Presentation, result SearchResult) []byte
-
// Presentation generates output from a corpus.
type Presentation struct {
Corpus *Corpus
@@ -30,12 +27,7 @@
ExampleHTML,
GodocHTML,
PackageHTML,
- PackageRootHTML,
- SearchHTML,
- SearchDocHTML,
- SearchCodeHTML,
- SearchTxtHTML,
- SearchDescXML *template.Template // If not nil, register a /opensearch.xml handler with this template.
+ PackageRootHTML *template.Template
// TabWidth optionally specifies the tab width.
TabWidth int
@@ -75,10 +67,6 @@
// the query string highlighted.
URLForSrcQuery func(src, query string, line int) string
- // SearchResults optionally specifies a list of functions returning an HTML
- // body for displaying search results.
- SearchResults []SearchResultFunc
-
// GoogleAnalytics optionally adds Google Analytics via the provided
// tracking ID to each page.
GoogleAnalytics string
@@ -89,8 +77,6 @@
}
// NewPresentation returns a new Presentation from a corpus.
-// It sets SearchResults to:
-// [SearchResultDoc SearchResultCode SearchResultTxt].
func NewPresentation(c *Corpus) *Presentation {
if c == nil {
panic("nil Corpus")
@@ -102,11 +88,6 @@
TabWidth: 4,
DeclLinks: true,
- SearchResults: []SearchResultFunc{
- (*Presentation).SearchResultDoc,
- (*Presentation).SearchResultCode,
- (*Presentation).SearchResultTxt,
- },
}
p.cmdHandler = handlerServer{
p: p,
@@ -125,10 +106,6 @@
p.cmdHandler.registerWithMux(p.mux)
p.pkgHandler.registerWithMux(p.mux)
p.mux.HandleFunc("/", p.ServeFile)
- p.mux.HandleFunc("/search", p.HandleSearch)
- if p.SearchDescXML != nil {
- p.mux.HandleFunc("/opensearch.xml", p.serveSearchDesc)
- }
return p
}
diff --git a/internal/godoc/search.go b/internal/godoc/search.go
deleted file mode 100644
index 0e7509a..0000000
--- a/internal/godoc/search.go
+++ /dev/null
@@ -1,186 +0,0 @@
-// Copyright 2009 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package godoc
-
-import (
- "bytes"
- "fmt"
- "net/http"
- "regexp"
- "strings"
-)
-
-type SearchResult struct {
- Query string
- Alert string // error or warning message
-
- // identifier matches
- Pak HitList // packages matching Query
- Hit *LookupResult // identifier matches of Query
- Alt *AltWords // alternative identifiers to look for
-
- // textual matches
- Found int // number of textual occurrences found
- Textual []FileLines // textual matches of Query
- Complete bool // true if all textual occurrences of Query are reported
- Idents map[SpotKind][]Ident
-}
-
-func (c *Corpus) Lookup(query string) SearchResult {
- result := &SearchResult{Query: query}
-
- index, timestamp := c.CurrentIndex()
- if index != nil {
- // identifier search
- if r, err := index.Lookup(query); err == nil {
- result = r
- } else if err != nil && !c.IndexFullText {
- // ignore the error if full text search is enabled
- // since the query may be a valid regular expression
- result.Alert = "Error in query string: " + err.Error()
- return *result
- }
-
- // full text search
- if c.IndexFullText && query != "" {
- rx, err := regexp.Compile(query)
- if err != nil {
- result.Alert = "Error in query regular expression: " + err.Error()
- return *result
- }
- // If we get maxResults+1 results we know that there are more than
- // maxResults results and thus the result may be incomplete (to be
- // precise, we should remove one result from the result set, but
- // nobody is going to count the results on the result page).
- result.Found, result.Textual = index.LookupRegexp(rx, c.MaxResults+1)
- result.Complete = result.Found <= c.MaxResults
- if !result.Complete {
- result.Found-- // since we looked for maxResults+1
- }
- }
- }
-
- // is the result accurate?
- if c.IndexEnabled {
- if ts := c.FSModifiedTime(); timestamp.Before(ts) {
- // The index is older than the latest file system change under godoc's observation.
- result.Alert = "Indexing in progress: result may be inaccurate"
- }
- } else {
- result.Alert = "Search index disabled: no results available"
- }
-
- return *result
-}
-
-// SearchResultDoc optionally specifies a function returning an HTML body
-// displaying search results matching godoc documentation.
-func (p *Presentation) SearchResultDoc(result SearchResult) []byte {
- return applyTemplate(p.SearchDocHTML, "searchDocHTML", result)
-}
-
-// SearchResultCode optionally specifies a function returning an HTML body
-// displaying search results matching source code.
-func (p *Presentation) SearchResultCode(result SearchResult) []byte {
- return applyTemplate(p.SearchCodeHTML, "searchCodeHTML", result)
-}
-
-// SearchResultTxt optionally specifies a function returning an HTML body
-// displaying search results of textual matches.
-func (p *Presentation) SearchResultTxt(result SearchResult) []byte {
- return applyTemplate(p.SearchTxtHTML, "searchTxtHTML", result)
-}
-
-// HandleSearch obtains results for the requested search and returns a page
-// to display them.
-func (p *Presentation) HandleSearch(w http.ResponseWriter, r *http.Request) {
- query := strings.TrimSpace(r.FormValue("q"))
- result := p.Corpus.Lookup(query)
-
- var contents bytes.Buffer
- for _, f := range p.SearchResults {
- contents.Write(f(p, result))
- }
-
- var title string
- if haveResults := contents.Len() > 0; haveResults {
- title = fmt.Sprintf(`Results for query: %v`, query)
- if !p.Corpus.IndexEnabled {
- result.Alert = ""
- }
- } else {
- title = fmt.Sprintf(`No results found for query %q`, query)
- }
-
- body := bytes.NewBuffer(applyTemplate(p.SearchHTML, "searchHTML", result))
- body.Write(contents.Bytes())
-
- p.ServePage(w, Page{
- Title: title,
- Tabtitle: query,
- Query: query,
- Body: body.Bytes(),
- GoogleCN: googleCN(r),
- })
-}
-
-func (p *Presentation) serveSearchDesc(w http.ResponseWriter, r *http.Request) {
- w.Header().Set("Content-Type", "application/opensearchdescription+xml")
- data := map[string]interface{}{
- "BaseURL": fmt.Sprintf("http://%s", r.Host),
- }
- applyTemplateToResponseWriter(w, p.SearchDescXML, &data)
-}
-
-// tocColCount returns the no. of columns
-// to split the toc table to.
-func tocColCount(result SearchResult) int {
- tocLen := tocLen(result)
- colCount := 0
- // Simple heuristic based on visual aesthetic in manual testing.
- switch {
- case tocLen <= 10:
- colCount = 1
- case tocLen <= 20:
- colCount = 2
- case tocLen <= 80:
- colCount = 3
- default:
- colCount = 4
- }
- return colCount
-}
-
-// tocLen calculates the no. of items in the toc table
-// by going through various fields in the SearchResult
-// that is rendered in the UI.
-func tocLen(result SearchResult) int {
- tocLen := 0
- for _, val := range result.Idents {
- if len(val) != 0 {
- tocLen++
- }
- }
- // If no identifiers, then just one item for the header text "Package <result.Query>".
- // See searchcode.html for further details.
- if len(result.Idents) == 0 {
- tocLen++
- }
- if result.Hit != nil {
- if len(result.Hit.Decls) > 0 {
- tocLen += len(result.Hit.Decls)
- // We need one extra item for the header text "Package-level declarations".
- tocLen++
- }
- if len(result.Hit.Others) > 0 {
- tocLen += len(result.Hit.Others)
- // We need one extra item for the header text "Local declarations and uses".
- tocLen++
- }
- }
- // For "textual occurrences".
- tocLen++
- return tocLen
-}
diff --git a/internal/godoc/snippet.go b/internal/godoc/snippet.go
deleted file mode 100644
index 1750478..0000000
--- a/internal/godoc/snippet.go
+++ /dev/null
@@ -1,123 +0,0 @@
-// Copyright 2009 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// This file contains the infrastructure to create a code
-// snippet for search results.
-//
-// Note: At the moment, this only creates HTML snippets.
-
-package godoc
-
-import (
- "bytes"
- "fmt"
- "go/ast"
- "go/token"
-)
-
-type Snippet struct {
- Line int
- Text string // HTML-escaped
-}
-
-func (p *Presentation) newSnippet(fset *token.FileSet, decl ast.Decl, id *ast.Ident) *Snippet {
- // TODO instead of pretty-printing the node, should use the original source instead
- var buf1 bytes.Buffer
- p.writeNode(&buf1, nil, fset, decl)
- // wrap text with <pre> tag
- var buf2 bytes.Buffer
- buf2.WriteString("<pre>")
- FormatText(&buf2, buf1.Bytes(), -1, true, id.Name, nil)
- buf2.WriteString("</pre>")
- return &Snippet{fset.Position(id.Pos()).Line, buf2.String()}
-}
-
-func findSpec(list []ast.Spec, id *ast.Ident) ast.Spec {
- for _, spec := range list {
- switch s := spec.(type) {
- case *ast.ImportSpec:
- if s.Name == id {
- return s
- }
- case *ast.ValueSpec:
- for _, n := range s.Names {
- if n == id {
- return s
- }
- }
- case *ast.TypeSpec:
- if s.Name == id {
- return s
- }
- }
- }
- return nil
-}
-
-func (p *Presentation) genSnippet(fset *token.FileSet, d *ast.GenDecl, id *ast.Ident) *Snippet {
- s := findSpec(d.Specs, id)
- if s == nil {
- return nil // declaration doesn't contain id - exit gracefully
- }
-
- // only use the spec containing the id for the snippet
- dd := &ast.GenDecl{
- Doc: d.Doc,
- TokPos: d.Pos(),
- Tok: d.Tok,
- Lparen: d.Lparen,
- Specs: []ast.Spec{s},
- Rparen: d.Rparen,
- }
-
- return p.newSnippet(fset, dd, id)
-}
-
-func (p *Presentation) funcSnippet(fset *token.FileSet, d *ast.FuncDecl, id *ast.Ident) *Snippet {
- if d.Name != id {
- return nil // declaration doesn't contain id - exit gracefully
- }
-
- // only use the function signature for the snippet
- dd := &ast.FuncDecl{
- Doc: d.Doc,
- Recv: d.Recv,
- Name: d.Name,
- Type: d.Type,
- }
-
- return p.newSnippet(fset, dd, id)
-}
-
-// NewSnippet creates a text snippet from a declaration decl containing an
-// identifier id. Parts of the declaration not containing the identifier
-// may be removed for a more compact snippet.
-func NewSnippet(fset *token.FileSet, decl ast.Decl, id *ast.Ident) *Snippet {
- // TODO(bradfitz, adg): remove this function. But it's used by indexer, which
- // doesn't have a *Presentation, and NewSnippet needs a TabWidth.
- var p Presentation
- p.TabWidth = 4
- return p.NewSnippet(fset, decl, id)
-}
-
-// NewSnippet creates a text snippet from a declaration decl containing an
-// identifier id. Parts of the declaration not containing the identifier
-// may be removed for a more compact snippet.
-func (p *Presentation) NewSnippet(fset *token.FileSet, decl ast.Decl, id *ast.Ident) *Snippet {
- var s *Snippet
- switch d := decl.(type) {
- case *ast.GenDecl:
- s = p.genSnippet(fset, d, id)
- case *ast.FuncDecl:
- s = p.funcSnippet(fset, d, id)
- }
-
- // handle failure gracefully
- if s == nil {
- var buf bytes.Buffer
- fmt.Fprintf(&buf, `<span class="alert">could not generate a snippet for <span class="highlight">%s</span></span>`, id.Name)
- s = &Snippet{fset.Position(id.Pos()).Line, buf.String()}
- }
- return s
-}
diff --git a/internal/godoc/spot.go b/internal/godoc/spot.go
deleted file mode 100644
index 95ffa4b..0000000
--- a/internal/godoc/spot.go
+++ /dev/null
@@ -1,83 +0,0 @@
-// Copyright 2013 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package godoc
-
-// ----------------------------------------------------------------------------
-// SpotInfo
-
-// A SpotInfo value describes a particular identifier spot in a given file;
-// It encodes three values: the SpotKind (declaration or use), a line or
-// snippet index "lori", and whether it's a line or index.
-//
-// The following encoding is used:
-//
-// bits 32 4 1 0
-// value [lori|kind|isIndex]
-//
-type SpotInfo uint32
-
-// SpotKind describes whether an identifier is declared (and what kind of
-// declaration) or used.
-type SpotKind uint32
-
-const (
- PackageClause SpotKind = iota
- ImportDecl
- ConstDecl
- TypeDecl
- VarDecl
- FuncDecl
- MethodDecl
- Use
- nKinds
-)
-
-var (
- // These must match the SpotKind values above.
- name = []string{
- "Packages",
- "Imports",
- "Constants",
- "Types",
- "Variables",
- "Functions",
- "Methods",
- "Uses",
- "Unknown",
- }
-)
-
-func (x SpotKind) Name() string { return name[x] }
-
-func init() {
- // sanity check: if nKinds is too large, the SpotInfo
- // accessor functions may need to be updated
- if nKinds > 8 {
- panic("internal error: nKinds > 8")
- }
-}
-
-// makeSpotInfo makes a SpotInfo.
-func makeSpotInfo(kind SpotKind, lori int, isIndex bool) SpotInfo {
- // encode lori: bits [4..32)
- x := SpotInfo(lori) << 4
- if int(x>>4) != lori {
- // lori value doesn't fit - since snippet indices are
- // most certainly always smaller then 1<<28, this can
- // only happen for line numbers; give it no line number (= 0)
- x = 0
- }
- // encode kind: bits [1..4)
- x |= SpotInfo(kind) << 1
- // encode isIndex: bit 0
- if isIndex {
- x |= 1
- }
- return x
-}
-
-func (x SpotInfo) Kind() SpotKind { return SpotKind(x >> 1 & 7) }
-func (x SpotInfo) Lori() int { return int(x >> 4) }
-func (x SpotInfo) IsIndex() bool { return x&1 != 0 }
diff --git a/internal/godoc/util/throttle.go b/internal/godoc/util/throttle.go
deleted file mode 100644
index 53d9ba6..0000000
--- a/internal/godoc/util/throttle.go
+++ /dev/null
@@ -1,88 +0,0 @@
-// Copyright 2011 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package util
-
-import "time"
-
-// A Throttle permits throttling of a goroutine by
-// calling the Throttle method repeatedly.
-//
-type Throttle struct {
- f float64 // f = (1-r)/r for 0 < r < 1
- dt time.Duration // minimum run time slice; >= 0
- tr time.Duration // accumulated time running
- ts time.Duration // accumulated time stopped
- tt time.Time // earliest throttle time (= time Throttle returned + tm)
-}
-
-// NewThrottle creates a new Throttle with a throttle value r and
-// a minimum allocated run time slice of dt:
-//
-// r == 0: "empty" throttle; the goroutine is always sleeping
-// r == 1: full throttle; the goroutine is never sleeping
-//
-// A value of r == 0.6 throttles a goroutine such that it runs
-// approx. 60% of the time, and sleeps approx. 40% of the time.
-// Values of r < 0 or r > 1 are clamped down to values between 0 and 1.
-// Values of dt < 0 are set to 0.
-//
-func NewThrottle(r float64, dt time.Duration) *Throttle {
- var f float64
- switch {
- case r <= 0:
- f = -1 // indicates always sleep
- case r >= 1:
- f = 0 // assume r == 1 (never sleep)
- default:
- // 0 < r < 1
- f = (1 - r) / r
- }
- if dt < 0 {
- dt = 0
- }
- return &Throttle{f: f, dt: dt, tt: time.Now().Add(dt)}
-}
-
-// Throttle calls time.Sleep such that over time the ratio tr/ts between
-// accumulated run (tr) and sleep times (ts) approximates the value 1/(1-r)
-// where r is the throttle value. Throttle returns immediately (w/o sleeping)
-// if less than tm ns have passed since the last call to Throttle.
-//
-func (p *Throttle) Throttle() {
- if p.f < 0 {
- select {} // always sleep
- }
-
- t0 := time.Now()
- if t0.Before(p.tt) {
- return // keep running (minimum time slice not exhausted yet)
- }
-
- // accumulate running time
- p.tr += t0.Sub(p.tt) + p.dt
-
- // compute sleep time
- // Over time we want:
- //
- // tr/ts = r/(1-r)
- //
- // Thus:
- //
- // ts = tr*f with f = (1-r)/r
- //
- // After some incremental run time δr added to the total run time
- // tr, the incremental sleep-time δs to get to the same ratio again
- // after waking up from time.Sleep is:
- if δs := time.Duration(float64(p.tr)*p.f) - p.ts; δs > 0 {
- time.Sleep(δs)
- }
-
- // accumulate (actual) sleep time
- t1 := time.Now()
- p.ts += t1.Sub(t0)
-
- // set earliest next throttle time
- p.tt = t1.Add(p.dt)
-}
