blob: fa8ecf10f4281542d40a5b5bca6fdb010b64208c [file] [log] [blame]
// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// This file exercises the import parser but also checks that
// some low-level packages do not have new dependencies added.
package build
import (
"bytes"
"fmt"
"internal/testenv"
"io/ioutil"
"os"
"path/filepath"
"runtime"
"sort"
"strconv"
"strings"
"testing"
)
// depsRules defines the expected dependencies between packages in
// the Go source tree. It is a statement of policy.
//
// DO NOT CHANGE THIS DATA TO FIX BUILDS.
// Existing packages should not have their constraints relaxed
// without prior discussion.
// Negative assertions should almost never be removed.
//
// The general syntax of a rule is:
//
// a, b < c, d;
//
// which means c and d come after a and b in the partial order
// (that is, c and d can import a and b),
// but doesn't provide a relative order between a vs b or c vs d.
//
// The rules can chain together, as in:
//
// e < f, g < h;
//
// which is equivalent to
//
// e < f, g;
// f, g < h;
//
// Except for the special bottom element "NONE", each name
// must appear exactly once on the right-hand side of a rule.
// That rule serves as the definition of the allowed dependencies
// for that name. The definition must appear before any uses
// of the name on the left-hand side of a rule. (That is, the
// rules themselves must be ordered according to the partial
// order, for easier reading by people.)
//
// Negative assertions double-check the partial order:
//
// i !< j
//
// means that it must NOT be the case that i < j.
// Negative assertions may appear anywhere in the rules,
// even before i and j have been defined.
//
// Comments begin with #.
//
// All-caps names are pseudo-names for specific points
// in the dependency lattice.
//
var depsRules = `
# No dependencies allowed for any of these packages.
NONE
< container/list, container/ring,
internal/cfg, internal/cpu,
internal/goversion, internal/nettrace,
unicode/utf8, unicode/utf16, unicode,
unsafe;
# RUNTIME is the core runtime group of packages, all of them very light-weight.
internal/cpu, unsafe
< internal/bytealg
< internal/unsafeheader
< runtime/internal/sys
< runtime/internal/atomic
< runtime/internal/math
< runtime
< sync/atomic
< internal/race
< sync
< internal/reflectlite
< errors
< internal/oserror, math/bits
< RUNTIME;
RUNTIME
< sort
< container/heap;
RUNTIME
< io;
reflect !< sort;
# SYSCALL is RUNTIME plus the packages necessary for basic system calls.
RUNTIME, unicode/utf8, unicode/utf16, io
< internal/syscall/windows/sysdll, syscall/js
< syscall
< internal/syscall/unix, internal/syscall/windows, internal/syscall/windows/registry
< internal/syscall/execenv
< SYSCALL;
# TIME is SYSCALL plus the core packages about time, including context.
SYSCALL
< time/tzdata
< time
< context
< TIME;
# MATH is RUNTIME plus the basic math packages.
RUNTIME
< math
< MATH;
unicode !< math;
MATH
< math/cmplx;
MATH
< math/rand;
MATH, unicode/utf8
< strconv;
unicode !< strconv;
# STR is basic string and buffer manipulation.
RUNTIME, io, unicode/utf8, unicode/utf16, unicode
< bytes, strings
< bufio, path;
bufio, path, strconv
< STR;
# OS is basic OS access, including helpers (path/filepath, os/exec, etc).
# OS includes string routines, but those must be layered above package os.
# OS does not include reflection.
TIME, io, sort
< internal/testlog
< internal/poll
< os
< os/signal;
unicode, fmt !< os, os/signal;
os/signal, STR
< path/filepath
< io/ioutil, os/exec
< OS;
reflect !< OS;
OS
< golang.org/x/sys/cpu, internal/goroot;
# FMT is OS (which includes string routines) plus reflect and fmt.
# It does not include package log, which should be avoided in core packages.
strconv, unicode
< reflect;
os, reflect
< internal/fmtsort
< fmt;
OS, fmt
< FMT;
log !< FMT;
# Misc packages needing only FMT.
FMT
< flag,
html,
mime/quotedprintable,
net/internal/socktest,
net/url,
runtime/debug,
runtime/trace,
text/scanner,
text/tabwriter;
# encodings
# core ones do not use fmt.
io, strconv
< encoding;
encoding, reflect
< encoding/binary
< encoding/base32, encoding/base64;
fmt !< encoding/base32, encoding/base64;
FMT, encoding/base32, encoding/base64
< encoding/ascii85, encoding/csv, encoding/gob, encoding/hex,
encoding/json, encoding/pem, encoding/xml, mime;
# hashes
io
< hash
< hash/adler32, hash/crc32, hash/crc64, hash/fnv, hash/maphash;
# math/big
FMT, encoding/binary, math/rand
< math/big;
# compression
FMT, encoding/binary, hash/adler32, hash/crc32
< compress/bzip2, compress/flate, compress/lzw
< archive/zip, compress/gzip, compress/zlib;
# templates
FMT
< text/template/parse;
net/url, text/template/parse
< text/template
< internal/lazytemplate;
encoding/json, html, text/template
< html/template;
# regexp
FMT
< regexp/syntax
< regexp
< internal/lazyregexp;
# suffix array
encoding/binary, regexp
< index/suffixarray;
# executable parsing
FMT, encoding/binary, compress/zlib
< debug/dwarf
< debug/elf, debug/gosym, debug/macho, debug/pe, debug/plan9obj, internal/xcoff
< DEBUG;
# go parser and friends.
FMT
< go/token
< go/scanner
< go/ast
< go/parser;
go/parser, text/tabwriter
< go/printer
< go/format;
go/parser, internal/lazyregexp, text/template
< go/doc;
math/big, go/token
< go/constant;
container/heap, go/constant, go/parser
< go/types;
go/doc, go/parser, internal/goroot, internal/goversion
< go/build;
DEBUG, go/build, go/types, text/scanner
< go/internal/gcimporter, go/internal/gccgoimporter, go/internal/srcimporter
< go/importer;
# databases
FMT
< database/sql/internal
< database/sql/driver
< database/sql;
# images
FMT, compress/lzw, compress/zlib
< image/color
< image, image/color/palette
< image/internal/imageutil
< image/draw
< image/gif, image/jpeg, image/png;
# cgo, delayed as long as possible.
# If you add a dependency on CGO, you must add the package
# to cgoPackages in cmd/dist/test.go as well.
RUNTIME
< C
< runtime/cgo
< CGO
< runtime/race, runtime/msan;
# Bulk of the standard library must not use cgo.
# The prohibition stops at net and os/user.
C !< fmt, go/types, CRYPTO-MATH;
CGO, OS
< plugin;
CGO, FMT
< os/user
< archive/tar;
sync
< internal/singleflight;
os
< golang.org/x/net/dns/dnsmessage,
golang.org/x/net/lif,
golang.org/x/net/route;
# net is unavoidable when doing any networking,
# so large dependencies must be kept out.
# This is a long-looking list but most of these
# are small with few dependencies.
# math/rand should probably be removed at some point.
CGO,
golang.org/x/net/dns/dnsmessage,
golang.org/x/net/lif,
golang.org/x/net/route,
internal/nettrace,
internal/poll,
internal/singleflight,
internal/race,
math/rand,
os
< net;
fmt, unicode !< net;
# NET is net plus net-helper packages.
FMT, net
< net/textproto;
mime, net/textproto, net/url
< NET;
# logging - most packages should not import; http and up is allowed
FMT
< log;
log !< crypto/tls, database/sql, go/importer, testing;
FMT, log, net
< log/syslog;
NET, log
< net/mail;
# CRYPTO is core crypto algorithms - no cgo, fmt, net.
# Unfortunately, stuck with reflect via encoding/binary.
encoding/binary, golang.org/x/sys/cpu, hash
< crypto
< crypto/subtle
< crypto/internal/subtle
< crypto/cipher
< crypto/aes, crypto/des, crypto/hmac, crypto/md5, crypto/rc4,
crypto/sha1, crypto/sha256, crypto/sha512
< CRYPTO;
CGO, fmt, net !< CRYPTO;
# CRYPTO-MATH is core bignum-based crypto - no cgo, net; fmt now ok.
CRYPTO, FMT, math/big
< crypto/rand
< crypto/internal/randutil
< crypto/ed25519/internal/edwards25519
< crypto/ed25519
< encoding/asn1
< golang.org/x/crypto/cryptobyte/asn1
< golang.org/x/crypto/cryptobyte
< golang.org/x/crypto/curve25519
< crypto/dsa, crypto/elliptic, crypto/rsa
< crypto/ecdsa
< CRYPTO-MATH;
CGO, net !< CRYPTO-MATH;
# TLS, Prince of Dependencies.
CGO, CRYPTO-MATH, NET, container/list, encoding/hex, encoding/pem
< golang.org/x/crypto/internal/subtle
< golang.org/x/crypto/chacha20
< golang.org/x/crypto/poly1305
< golang.org/x/crypto/chacha20poly1305
< golang.org/x/crypto/hkdf
< crypto/x509/internal/macos
< crypto/x509/pkix
< crypto/x509
< crypto/tls;
# crypto-aware packages
NET, crypto/rand, mime/quotedprintable
< mime/multipart;
crypto/tls
< net/smtp;
# HTTP, King of Dependencies.
FMT
< golang.org/x/net/http2/hpack, net/http/internal;
FMT, NET, container/list, encoding/binary, log
< golang.org/x/text/transform
< golang.org/x/text/unicode/norm
< golang.org/x/text/unicode/bidi
< golang.org/x/text/secure/bidirule
< golang.org/x/net/idna
< golang.org/x/net/http/httpguts, golang.org/x/net/http/httpproxy;
NET, crypto/tls
< net/http/httptrace;
compress/gzip,
golang.org/x/net/http/httpguts,
golang.org/x/net/http/httpproxy,
golang.org/x/net/http2/hpack,
net/http/internal,
net/http/httptrace,
mime/multipart,
log
< net/http;
# HTTP-aware packages
encoding/json, net/http
< expvar;
net/http
< net/http/cookiejar, net/http/httputil;
net/http, flag
< net/http/httptest;
net/http, regexp
< net/http/cgi
< net/http/fcgi;
# Profiling
FMT, compress/gzip, encoding/binary, text/tabwriter
< runtime/pprof;
OS, compress/gzip, regexp
< internal/profile;
html/template, internal/profile, net/http, runtime/pprof, runtime/trace
< net/http/pprof;
# RPC
encoding/gob, encoding/json, go/token, html/template, net/http
< net/rpc
< net/rpc/jsonrpc;
# Test-only
log
< testing/iotest;
FMT, flag, math/rand
< testing/quick;
FMT, flag, runtime/debug, runtime/trace
< testing;
internal/testlog, runtime/pprof, regexp
< testing/internal/testdeps;
OS, flag, testing, internal/cfg
< internal/testenv;
OS, encoding/base64
< internal/obscuretestdata;
CGO, OS, fmt
< os/signal/internal/pty;
NET, testing
< golang.org/x/net/nettest;
FMT, container/heap, math/rand
< internal/trace;
`
// listStdPkgs returns the same list of packages as "go list std".
func listStdPkgs(goroot string) ([]string, error) {
// Based on cmd/go's matchPackages function.
var pkgs []string
src := filepath.Join(goroot, "src") + string(filepath.Separator)
walkFn := func(path string, fi os.FileInfo, err error) error {
if err != nil || !fi.IsDir() || path == src {
return nil
}
base := filepath.Base(path)
if strings.HasPrefix(base, ".") || strings.HasPrefix(base, "_") || base == "testdata" {
return filepath.SkipDir
}
name := filepath.ToSlash(path[len(src):])
if name == "builtin" || name == "cmd" {
return filepath.SkipDir
}
pkgs = append(pkgs, strings.TrimPrefix(name, "vendor/"))
return nil
}
if err := filepath.Walk(src, walkFn); err != nil {
return nil, err
}
return pkgs, nil
}
func TestDependencies(t *testing.T) {
if !testenv.HasSrc() {
// Tests run in a limited file system and we do not
// provide access to every source file.
t.Skipf("skipping on %s/%s, missing full GOROOT", runtime.GOOS, runtime.GOARCH)
}
ctxt := Default
all, err := listStdPkgs(ctxt.GOROOT)
if err != nil {
t.Fatal(err)
}
sort.Strings(all)
sawImport := map[string]map[string]bool{} // from package => to package => true
policy := depsPolicy(t)
for _, pkg := range all {
imports, err := findImports(pkg)
if err != nil {
t.Error(err)
continue
}
if sawImport[pkg] == nil {
sawImport[pkg] = map[string]bool{}
}
ok := policy[pkg]
var bad []string
for _, imp := range imports {
sawImport[pkg][imp] = true
if !ok[imp] {
bad = append(bad, imp)
}
}
if bad != nil {
t.Errorf("unexpected dependency: %s imports %v", pkg, bad)
}
}
// depPath returns the path between the given from and to packages.
// It returns the empty string if there's no dependency path.
var depPath func(string, string) string
depPath = func(from, to string) string {
if sawImport[from][to] {
return from + " => " + to
}
for pkg := range sawImport[from] {
if p := depPath(pkg, to); p != "" {
return from + " => " + p
}
}
return ""
}
}
var buildIgnore = []byte("\n// +build ignore")
func findImports(pkg string) ([]string, error) {
vpkg := pkg
if strings.HasPrefix(pkg, "golang.org") {
vpkg = "vendor/" + pkg
}
dir := filepath.Join(Default.GOROOT, "src", vpkg)
files, err := ioutil.ReadDir(dir)
if err != nil {
return nil, err
}
var imports []string
var haveImport = map[string]bool{}
for _, file := range files {
name := file.Name()
if name == "slice_go14.go" || name == "slice_go18.go" {
// These files are for compiler bootstrap with older versions of Go and not built in the standard build.
continue
}
if !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") {
continue
}
f, err := os.Open(filepath.Join(dir, name))
if err != nil {
return nil, err
}
var imp []string
data, err := readImports(f, false, &imp)
f.Close()
if err != nil {
return nil, fmt.Errorf("reading %v: %v", name, err)
}
if bytes.Contains(data, buildIgnore) {
continue
}
for _, quoted := range imp {
path, err := strconv.Unquote(quoted)
if err != nil {
continue
}
if !haveImport[path] {
haveImport[path] = true
imports = append(imports, path)
}
}
}
sort.Strings(imports)
return imports, nil
}
// depsPolicy returns a map m such that m[p][d] == true when p can import d.
func depsPolicy(t *testing.T) map[string]map[string]bool {
allowed := map[string]map[string]bool{"NONE": {}}
disallowed := [][2][]string{}
parseDepsRules(t, func(deps []string, op string, users []string) {
if op == "!<" {
disallowed = append(disallowed, [2][]string{deps, users})
return
}
for _, u := range users {
if allowed[u] != nil {
t.Errorf("multiple deps lists for %s", u)
}
allowed[u] = make(map[string]bool)
for _, d := range deps {
if allowed[d] == nil {
t.Errorf("use of %s before its deps list", d)
}
allowed[u][d] = true
}
}
})
// Check for missing deps info.
for _, deps := range allowed {
for d := range deps {
if allowed[d] == nil {
t.Errorf("missing deps list for %s", d)
}
}
}
// Complete transitive allowed deps.
for k := range allowed {
for i := range allowed {
for j := range allowed {
if i != k && k != j && allowed[i][k] && allowed[k][j] {
if i == j {
// Can only happen along with a "use of X before deps" error above,
// but this error is more specific - it makes clear that reordering the
// rules will not be enough to fix the problem.
t.Errorf("deps policy cycle: %s < %s < %s", j, k, i)
}
allowed[i][j] = true
}
}
}
}
// Check negative assertions against completed allowed deps.
for _, bad := range disallowed {
deps, users := bad[0], bad[1]
for _, d := range deps {
for _, u := range users {
if allowed[u][d] {
t.Errorf("deps policy incorrect: assertion failed: %s !< %s", d, u)
}
}
}
}
if t.Failed() {
t.FailNow()
}
return allowed
}
// parseDepsRules parses depsRules, calling save(deps, op, users)
// for each deps < users or deps !< users rule
// (op is "<" or "!<").
func parseDepsRules(t *testing.T, save func(deps []string, op string, users []string)) {
p := &depsParser{t: t, lineno: 1, text: depsRules}
var prev []string
var op string
for {
list, tok := p.nextList()
if tok == "" {
if prev == nil {
break
}
p.syntaxError("unexpected EOF")
}
if prev != nil {
save(prev, op, list)
}
prev = list
if tok == ";" {
prev = nil
op = ""
continue
}
if tok != "<" && tok != "!<" {
p.syntaxError("missing <")
}
op = tok
}
}
// A depsParser parses the depsRules syntax described above.
type depsParser struct {
t *testing.T
lineno int
lastWord string
text string
}
// syntaxError reports a parsing error.
func (p *depsParser) syntaxError(msg string) {
p.t.Fatalf("deps:%d: syntax error: %s near %s", p.lineno, msg, p.lastWord)
}
// nextList parses and returns a comma-separated list of names.
func (p *depsParser) nextList() (list []string, token string) {
for {
tok := p.nextToken()
switch tok {
case "":
if len(list) == 0 {
return nil, ""
}
fallthrough
case ",", "<", "!<", ";":
p.syntaxError("bad list syntax")
}
list = append(list, tok)
tok = p.nextToken()
if tok != "," {
return list, tok
}
}
}
// nextToken returns the next token in the deps rules,
// one of ";" "," "<" "!<" or a name.
func (p *depsParser) nextToken() string {
for {
if p.text == "" {
return ""
}
switch p.text[0] {
case ';', ',', '<':
t := p.text[:1]
p.text = p.text[1:]
return t
case '!':
if len(p.text) < 2 || p.text[1] != '<' {
p.syntaxError("unexpected token !")
}
p.text = p.text[2:]
return "!<"
case '#':
i := strings.Index(p.text, "\n")
if i < 0 {
i = len(p.text)
}
p.text = p.text[i:]
continue
case '\n':
p.lineno++
fallthrough
case ' ', '\t':
p.text = p.text[1:]
continue
default:
i := strings.IndexAny(p.text, "!;,<#\n \t")
if i < 0 {
i = len(p.text)
}
t := p.text[:i]
p.text = p.text[i:]
p.lastWord = t
return t
}
}
}
// TestStdlibLowercase tests that all standard library package names are
// lowercase. See Issue 40065.
func TestStdlibLowercase(t *testing.T) {
if !testenv.HasSrc() {
t.Skipf("skipping on %s/%s, missing full GOROOT", runtime.GOOS, runtime.GOARCH)
}
ctxt := Default
all, err := listStdPkgs(ctxt.GOROOT)
if err != nil {
t.Fatal(err)
}
for _, pkgname := range all {
if strings.ToLower(pkgname) != pkgname {
t.Errorf("package %q should not use upper-case path", pkgname)
}
}
}