blob: aecbba73003f139c9933cc6b336d68dac81b5e25 [file] [log] [blame]
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package licenses detects licenses and determines whether they are redistributable.
// The functions in this package do not return errors; instead, they log any problems
// they encounter and fail closed by reporting that the module or package is not
// redistributable.
//
// Example (modproxy):
//
// d := licenses.NewDetector(modulePath, version, zipReader, log.Infof)
// modRedist := d.ModuleIsRedistributable()
//
// Example (discovery):
//
// d := licenses.NewDetector(modulePath, version, zipReader, log.Infof)
// modRedist := d.ModuleIsRedistributable()
// lics := d.AllLicenses()
// pkgRedist, pkgMetas := d.PackageInfo(pkgSubdir)
package licenses
import (
"archive/zip"
"context"
"fmt"
"io"
"io/fs"
"path"
"path/filepath"
"sort"
"strings"
"sync"
"github.com/google/licensecheck"
"golang.org/x/mod/module"
modzip "golang.org/x/mod/zip"
"golang.org/x/pkgsite/internal/log"
)
//go:generate rm -f exceptions.gen.go
//go:generate go run gen_exceptions.go
const (
// coverageThreshold is the minimum percentage of the file that must contain
// license text.
coverageThreshold = 75
// unknownLicenseType is for text in a license file that's not recognized.
unknownLicenseType = "UNKNOWN"
)
// maxLicenseSize is the maximum allowable size (in bytes) for a license file.
// There are some license files larger than 1 million bytes: https://github.com/vmware/vic/LICENSE
// and github.com/goharbor/harbor/LICENSE, for example.
// var for testing
var maxLicenseSize int64 = modzip.MaxLICENSE
// Metadata holds information extracted from a license file.
type Metadata struct {
// Types is the set of license types, as determined by the licensecheck package.
Types []string
// FilePath is the '/'-separated path to the license file in the module zip,
// relative to the contents directory.
FilePath string
Coverage licensecheck.Coverage
}
// A License is a classified license file path and its contents.
type License struct {
*Metadata
Contents []byte
}
// RemoveNonRedistributableData methods removes the license contents
// if the license is non-redistributable.
func (l *License) RemoveNonRedistributableData() {
if !Redistributable(l.Types) {
l.Contents = nil
}
}
var (
FileNames = []string{
"COPYING",
"COPYING.md",
"COPYING.markdown",
"COPYING.txt",
"LICENCE",
"LICENCE.md",
"LICENCE.markdown",
"LICENCE.txt",
"LICENSE",
"LICENSE.md",
"LICENSE.markdown",
"LICENSE.txt",
"LICENSE-2.0.txt",
"LICENCE-2.0.txt",
"LICENSE-APACHE",
"LICENCE-APACHE",
"LICENSE-APACHE-2.0.txt",
"LICENCE-APACHE-2.0.txt",
"LICENSE-MIT",
"LICENCE-MIT",
"LICENSE.MIT",
"LICENCE.MIT",
"LICENSE.code",
"LICENCE.code",
"LICENSE.docs",
"LICENCE.docs",
"LICENSE.rst",
"LICENCE.rst",
"MIT-LICENSE",
"MIT-LICENCE",
"MIT-LICENSE.md",
"MIT-LICENCE.md",
"MIT-LICENSE.markdown",
"MIT-LICENCE.markdown",
"MIT-LICENSE.txt",
"MIT-LICENCE.txt",
"MIT_LICENSE",
"MIT_LICENCE",
"UNLICENSE",
"UNLICENCE",
}
// standardRedistributableLicenseTypes is the list of license types, as reported by
// licensecheck, that allow redistribution, and also have a name that is an OSI or SPDX
// identifier.
standardRedistributableLicenseTypes = []string{
// Licenses acceptable by OSI.
"AFL-3.0",
"AGPL-3.0",
"AGPL-3.0-only",
"AGPL-3.0-or-later",
"Apache-1.1",
"Apache-2.0",
"Artistic-2.0",
"BlueOak-1.0.0",
"0BSD",
"BSD-1-Clause",
"BSD-2-Clause",
"BSD-2-Clause-Patent",
"BSD-2-Clause-Views",
"BSD-3-Clause",
"BSD-3-Clause-Clear",
"BSD-3-Clause-LBNL",
"BSD-3-Clause-Open-MPI",
"BSD-4-Clause",
"BSD-4-Clause-UC",
"BSL-1.0",
"CC-BY-3.0",
"CC-BY-4.0",
"CC-BY-SA-3.0",
"CC-BY-SA-4.0",
"CECILL-2.1",
"CC0-1.0",
"EPL-1.0",
"EPL-2.0",
"EUPL-1.2",
"GPL-2.0",
"GPL-2.0-only",
"GPL-2.0-or-later",
"GPL-3.0",
"GPL-3.0-only",
"GPL-3.0-or-later",
"HPND",
"ISC",
"JSON",
"LGPL-2.1",
"LGPL-2.1-or-later",
"LGPL-3.0",
"LGPL-3.0-or-later",
"MIT",
"MIT-0",
"MPL-2.0",
"MPL-2.0-no-copyleft-exception",
"MulanPSL-2.0",
"NIST-PD",
"NIST-PD-fallback",
"NCSA",
"OpenSSL",
"OSL-3.0",
"PostgreSQL", // TODO: ask legal
"Python-2.0",
"Unlicense",
"UPL-1.0",
"Zlib",
}
// These aren't technically licenses, but they are recognized by
// licensecheck and safe to ignore.
ignorableLicenseTypes = map[string]bool{
"CC-Notice": true,
"GooglePatentClause": true,
"GooglePatentsFile": true,
"blessing": true,
"OFL-1.1": true, // concerns fonts only
}
// redistributableLicenseTypes is the set of license types, as reported by
// licensecheck, that allow redistribution. It consists of the standard
// types along with some exception types.
redistributableLicenseTypes = map[string]bool{}
)
func init() {
for _, t := range standardRedistributableLicenseTypes {
redistributableLicenseTypes[t] = true
}
// Add here all other types defined in the exceptions.
redistributableLicenseTypes["Freetype"] = true
// exceptionTypes is a map from License IDs from LREs in the exception
// directory to license types. Any type mentioned in an exception should
// be redistributable. If not, there's a problem.
for _, types := range exceptionTypes {
for _, t := range types {
if !redistributableLicenseTypes[t] {
log.Fatalf(context.Background(), "%s is an exception type that is not redistributable.", t)
}
}
}
}
// nonOSILicenses lists licenses that are not approved by OSI.
var nonOSILicenses = map[string]bool{
"AGPL-3.0-only": true,
"AGPL-3.0-or-later": true,
"BlueOak-1.0.0": true,
"BSD-2-Clause-Views": true,
"BSD-3-Clause-Clear": true,
"BSD-3-Clause-LBNL": true,
"BSD-3-Clause-Open-MPI": true,
"BSD-4-Clause": true,
"BSD-4-Clause-UC": true,
"CC-BY-3.0": true,
"CC-BY-4.0": true,
"CC-BY-SA-3.0": true,
"CC-BY-SA-4.0": true,
"CC0-1.0": true,
"GPL-2.0-only": true,
"GPL-2.0-or-later": true,
"GPL-3.0-only": true, // it is referred in https://opensource.org/licenses/GPL-3.0
"GPL-3.0-or-later": true,
"LGPL-2.1-or-later": true,
"LGPL-3.0-or-later": true,
"MPL-2.0-no-copyleft-exception": true,
"NIST-PD": true,
"NIST-PD-fallback": true,
"JSON": true,
"OpenSSL": true,
"UPL-1.0": true,
}
// fileNamesLowercase has all the entries of FileNames, downcased and made a set
// for fast case-insensitive matching.
var fileNamesLowercase = map[string]bool{}
func init() {
for _, f := range FileNames {
fileNamesLowercase[strings.ToLower(f)] = true
}
}
// AcceptedLicenseInfo describes a license that is accepted by the discovery site.
type AcceptedLicenseInfo struct {
Name string
URL string
}
// AcceptedLicenses returns a sorted slice of license types that are accepted as
// redistributable. Its result is intended to be displayed to users.
func AcceptedLicenses() []AcceptedLicenseInfo {
var lics []AcceptedLicenseInfo
for _, identifier := range standardRedistributableLicenseTypes {
var link string
if nonOSILicenses[identifier] {
link = fmt.Sprintf("https://spdx.org/licenses/%s.html", identifier)
} else {
link = fmt.Sprintf("https://opensource.org/licenses/%s", identifier)
}
lics = append(lics, AcceptedLicenseInfo{identifier, link})
}
sort.Slice(lics, func(i, j int) bool { return lics[i].Name < lics[j].Name })
return lics
}
var (
// OmitExceptions causes the list of exceptions to be omitted from license detection.
// It is intended only to speed up testing, and must be set before the first use
// of this package.
OmitExceptions bool
_scanner *licensecheck.Scanner
scannerOnce sync.Once
)
func scanner() *licensecheck.Scanner {
scannerOnce.Do(func() {
if OmitExceptions {
exceptionLicenses = nil
}
var err error
_scanner, err = licensecheck.NewScanner(append(exceptionLicenses, licensecheck.BuiltinLicenses()...))
if err != nil {
log.Fatalf(context.Background(), "licensecheck.NewScanner: %v", err)
}
})
return _scanner
}
// A Detector detects licenses in a module and its packages.
type Detector struct {
modulePath string
version string
fsys fs.FS
logf func(string, ...any)
moduleRedist bool
moduleLicenses []*License // licenses at module root directory, or list from exceptions
allLicenses []*License
licsByDir map[string][]*License // from directory to list of licenses
}
// NewDetector returns a Detector for the given module and version.
// zr should be the zip file for that module and version.
// logf is for logging; if nil, no logging is done.
// Deprecated: use NewDetectorFS.
func NewDetector(modulePath, version string, zr *zip.Reader, logf func(string, ...any)) *Detector {
sub, err := fs.Sub(zr, modulePath+"@"+version)
// This should only fail if the prefix is not a valid path, which shouldn't be possible.
if err != nil && logf != nil {
logf("fs.Sub: %v", err)
}
return NewDetectorFS(modulePath, version, sub, logf)
}
// NewDetectorFS returns a Detector for the given module and version.
// fsys should represent the content directory of the module (not the zip root).
// logf is for logging; if nil, no logging is done.
func NewDetectorFS(modulePath, version string, fsys fs.FS, logf func(string, ...any)) *Detector {
if logf == nil {
logf = func(string, ...any) {}
}
d := &Detector{
modulePath: modulePath,
version: version,
fsys: fsys,
logf: logf,
}
d.computeModuleInfo()
return d
}
// ModuleIsRedistributable reports whether the given module is redistributable.
func (d *Detector) ModuleIsRedistributable() bool {
return d.moduleRedist
}
// ModuleLicenses returns the licenses that apply to the module.
func (d *Detector) ModuleLicenses() []*License {
return d.moduleLicenses
}
// AllLicenses returns all the licenses detected in the entire module, including
// package licenses.
func (d *Detector) AllLicenses() []*License {
if d.allLicenses == nil {
d.computeAllLicenseInfo()
}
return d.allLicenses
}
// PackageInfo reports whether the package at dir, a directory relative to the
// module root, is redistributable. It also returns all the licenses that apply
// to the package.
func (d *Detector) PackageInfo(dir string) (isRedistributable bool, lics []*License) {
cleanDir := filepath.ToSlash(filepath.Clean(dir))
if path.IsAbs(cleanDir) || strings.HasPrefix(cleanDir, "..") {
return false, nil
}
if d.allLicenses == nil {
d.computeAllLicenseInfo()
}
// Collect all the license metadata for directories dir and above, excluding the root.
for prefix, plics := range d.licsByDir {
// append a slash so that prefix a/b does not match a/bc/d
if strings.HasPrefix(cleanDir+"/", prefix+"/") {
lics = append(lics, plics...)
}
}
// A package is redistributable if its module is, and if other licenses on
// the path to the root are redistributable. Note that this is not the same
// as asking if the module licenses plus the package licenses are
// redistributable. A module that is granted an exception (see DetectFiles)
// may have licenses that are non-redistributable.
ltypes := types(lics)
isRedistributable = d.ModuleIsRedistributable() && (len(ltypes) == 0 || Redistributable(ltypes))
// A package's licenses include the ones we've already computed, as well
// as the module licenses.
return isRedistributable, append(lics, d.moduleLicenses...)
}
// computeModuleInfo determines values for the moduleRedist and moduleLicenses fields of d.
func (d *Detector) computeModuleInfo() {
// Check that all licenses in the contents directory are redistributable.
d.moduleLicenses = d.detectFiles(d.paths(RootFiles))
d.moduleRedist = Redistributable(types(d.moduleLicenses))
}
// computeAllLicenseInfo collects all the detected licenses in the zip and
// stores them in the allLicenses field of d. It also maps detected licenses to
// their directories, to optimize Detector.PackageInfo.
func (d *Detector) computeAllLicenseInfo() {
d.allLicenses = []*License{}
d.allLicenses = append(d.allLicenses, d.moduleLicenses...)
nonRootLicenses := d.detectFiles(d.paths(NonRootFiles))
d.allLicenses = append(d.allLicenses, nonRootLicenses...)
d.licsByDir = map[string][]*License{}
for _, l := range nonRootLicenses {
prefix := path.Dir(l.FilePath)
d.licsByDir[prefix] = append(d.licsByDir[prefix], l)
}
}
// WhichFiles describes which files from the zip should be returned by Detector.Files.
type WhichFiles int
const (
// Only files from the root (contents) directory.
RootFiles WhichFiles = iota
// Only files that are not in the root directory.
NonRootFiles
// All files; the union of root and non-root.
AllFiles
)
// paths returns a list of license file paths from the Detector's filesystem.
// The which argument determines the location of the files considered.
// If paths encounters an error, it logs it and returns nil.
func (d *Detector) paths(which WhichFiles) []string {
if d.fsys == nil {
return nil
}
var paths []string
err := fs.WalkDir(d.fsys, ".", func(pathname string, de fs.DirEntry, err error) error {
if err != nil {
return err
}
if de.IsDir() {
return nil
}
if !fileNamesLowercase[strings.ToLower(de.Name())] {
return nil
}
// Skip files we should ignore.
if ignoreFiles[d.modulePath+" "+pathname] {
return nil
}
if which == RootFiles && path.Dir(pathname) != "." {
// Skip f since it's not at root.
return nil
}
if which == NonRootFiles && path.Dir(pathname) == "." {
// Skip f since it is at root.
return nil
}
if isVendoredFile(pathname) {
// Skip if f is in the vendor directory.
return nil
}
if err := module.CheckFilePath(pathname); err != nil {
// Skip if the file path is bad.
d.logf("module.CheckFilePath(%q): %v", pathname, err)
return nil
}
paths = append(paths, pathname)
return nil
})
if err != nil {
d.logf("licenses.Detector.paths: %v", err)
return nil
}
return paths
}
// isVendoredFile reports if the given file is in a proper subdirectory nested
// under a 'vendor' directory, to allow for Go packages named 'vendor'.
// For example:
// - isVendoredFile("vendor/LICENSE") == false, and
// - isVendoredFile("vendor/foo/LICENSE") == true.
func isVendoredFile(name string) bool {
var vendorOffset int
if strings.HasPrefix(name, "vendor/") {
vendorOffset = len("vendor/")
} else if i := strings.Index(name, "/vendor/"); i >= 0 {
vendorOffset = i + len("/vendor/")
} else {
// no vendor directory
return false
}
// check if the file is in a proper subdirectory of vendor
return strings.Contains(name[vendorOffset:], "/")
}
// detectFiles runs DetectFile on each of the given files.
// If a file cannot be read, the error is logged and a license
// of type unknown is added.
func (d *Detector) detectFiles(pathnames []string) []*License {
var licenses []*License
for _, p := range pathnames {
bytes, err := d.readFile(p)
if err != nil {
d.logf("reading file %s: %v", p, err)
licenses = append(licenses, &License{
Metadata: &Metadata{
Types: []string{unknownLicenseType},
FilePath: p,
},
})
continue
}
types, cov := DetectFile(bytes, p, d.logf)
licenses = append(licenses, &License{
Metadata: &Metadata{
Types: types,
FilePath: p,
Coverage: cov,
},
Contents: bytes,
})
}
return licenses
}
func (d *Detector) readFile(pathname string) ([]byte, error) {
f, err := d.fsys.Open(pathname)
if err != nil {
return nil, err
}
defer f.Close()
info, err := f.Stat()
if err != nil {
return nil, err
}
if info.Size() > maxLicenseSize {
return nil, fmt.Errorf("file size %d exceeds max license size %d", info.Size(), maxLicenseSize)
}
return io.ReadAll(io.LimitReader(f, int64(maxLicenseSize)))
}
// DetectFile return the set of license types for the given file contents. It
// also returns the licensecheck coverage information. The filename is used
// solely for logging.
func DetectFile(contents []byte, filename string, logf func(string, ...any)) ([]string, licensecheck.Coverage) {
if logf == nil {
logf = func(string, ...any) {}
}
cov := scanner().Scan(contents)
if cov.Percent < float64(coverageThreshold) {
logf("%s license coverage too low (%+v), skipping", filename, cov)
return []string{unknownLicenseType}, cov
}
types := make(map[string]bool)
for _, m := range cov.Match {
ts := exceptionTypes[m.ID]
if ts == nil {
ts = []string{m.ID}
}
for _, t := range ts {
types[t] = true
}
}
if len(types) == 0 {
logf("%s failed to classify license (%+v), skipping", filename, cov)
return []string{unknownLicenseType}, cov
}
return setToSortedSlice(types), cov
}
// Redistributable reports whether the set of license types establishes that a
// module or package is redistributable.
// All the licenses we see that are relevant must be redistributable, and
// we must see at least one such license.
func Redistributable(licenseTypes []string) bool {
sawRedist := false
for _, t := range licenseTypes {
if ignorableLicenseTypes[t] {
continue
}
if !redistributableLicenseTypes[t] {
return false
}
sawRedist = true
}
return sawRedist
}
func types(lics []*License) []string {
var types []string
for _, l := range lics {
types = append(types, l.Types...)
}
return types
}
func setToSortedSlice(m map[string]bool) []string {
var s []string
for e := range m {
s = append(s, e)
}
sort.Strings(s)
return s
}