internal/postgres/directory.go

// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package postgres

import (
	"context"
	"database/sql"
	"fmt"
	"sort"

	"github.com/lib/pq"
	"golang.org/x/pkgsite/internal"
	"golang.org/x/pkgsite/internal/database"
	"golang.org/x/pkgsite/internal/derrors"
	"golang.org/x/pkgsite/internal/stdlib"
)

// GetDirectoryNew returns a directory from the database, along with all of the
// data associated with that directory, including the package, imports, readme,
// documentation, and licenses.
func (db *DB) GetDirectoryNew(ctx context.Context, path, modulePath, version string) (_ *internal.VersionedDirectory, err error) {
	query := `
		SELECT
			m.module_path,
			m.version,
			m.commit_time,
			m.version_type,
			m.redistributable,
			m.has_go_mod,
			m.source_info,
			p.id,
			p.path,
			p.name,
			p.v1_path,
			p.redistributable,
			p.license_types,
			p.license_paths,
			d.goos,
			d.goarch,
			d.synopsis,
			d.html
		FROM modules m
		INNER JOIN paths p
		ON p.module_id = m.id
		LEFT JOIN documentation d
		ON d.path_id = p.id
		WHERE
			p.path = $1
			AND m.module_path = $2
			AND m.version = $3;`
	var (
		mi                         internal.ModuleInfo
		dir                        internal.DirectoryNew
		doc                        internal.Documentation
		pkg                        internal.PackageNew
		licenseTypes, licensePaths []string
		pathID                     int
	)
	row := db.db.QueryRow(ctx, query, path, modulePath, version)
	if err := row.Scan(
		&mi.ModulePath,
		&mi.Version,
		&mi.CommitTime,
		&mi.VersionType,
		&mi.IsRedistributable,
		&mi.HasGoMod,
		jsonbScanner{&mi.SourceInfo},
		&pathID,
		&dir.Path,
		database.NullIsEmpty(&pkg.Name),
		&dir.V1Path,
		&dir.IsRedistributable,
		pq.Array(&licenseTypes),
		pq.Array(&licensePaths),
		database.NullIsEmpty(&doc.GOOS),
		database.NullIsEmpty(&doc.GOARCH),
		database.NullIsEmpty(&doc.Synopsis),
		database.NullIsEmpty(&doc.HTML),
	); err != nil {
		if err == sql.ErrNoRows {
			return nil, fmt.Errorf("directory %s@%s: %w", path, version, derrors.NotFound)
		}
		return nil, fmt.Errorf("row.Scan(): %v", err)
	}

	lics, err := zipLicenseMetadata(licenseTypes, licensePaths)
	if err != nil {
		return nil, err
	}
	dir.Licenses = lics
	if pkg.Name != "" {
		dir.Package = &pkg
		pkg.Path = dir.Path
		pkg.Documentation = &doc
		collect := func(rows *sql.Rows) error {
			var path string
			if err := rows.Scan(&path); err != nil {
				return fmt.Errorf("row.Scan(): %v", err)
			}
			pkg.Imports = append(pkg.Imports, path)
			return nil
		}
		if err := db.db.RunQuery(ctx, `
		SELECT to_path
		FROM package_imports
		WHERE path_id = $1`, collect, pathID); err != nil {
			return nil, err
		}
	}

	// TODO(golang/go#38513): remove and query the readmes table directly once
	// we start displaying READMEs for directories instead of the top-level
	// module.
	var readme internal.Readme
	row = db.db.QueryRow(ctx, `
		SELECT file_path, contents
		FROM modules m
		INNER JOIN paths p
		ON p.module_id = m.id
		INNER JOIN readmes r
		ON p.id = r.path_id
		WHERE
		    module_path=$1
			AND m.version=$2
			AND m.module_path=p.path`, modulePath, version)
	if err := row.Scan(&readme.Filepath, &readme.Contents); err != nil && err != sql.ErrNoRows {
		return nil, err
	}
	if readme.Filepath != "" {
		dir.Readme = &readme
	}
	return &internal.VersionedDirectory{
		ModuleInfo:   mi,
		DirectoryNew: dir,
	}, nil
}

// LegacyGetDirectory returns the directory corresponding to the provided dirPath,
// modulePath, and version. The directory will contain all packages for that
// version, in sorted order by package path.
//
// If version = internal.LatestVersion, the directory corresponding to the
// latest matching module version will be fetched.
//
// fields is a set of fields to read (see internal.FieldSet and related
// definitions). If a field is not in fields, it will not be read from the DB
// and its value will be one of the XXXFieldMissing constants. Only certain
// large fields are treated specially in this way.
//
// If more than one module ties for a given dirPath and version pair, and
// modulePath = internal.UnknownModulePath, the directory for the module with
// the  longest module path will be fetched.
// For example, if there are
// two rows in the packages table:
// (1) path = "github.com/hashicorp/vault/api"
//     module_path = "github.com/hashicorp/vault"
// AND
// (2) path = "github.com/hashicorp/vault/api"
//     module_path = "github.com/hashicorp/vault/api"
// Only directories in the latter module will be returned.
//
// Packages will be returned for a given dirPath if: (1) the package path has a
// prefix of dirPath (2) the dirPath has a prefix matching the package's
// module_path
//
// For example, if the package "golang.org/x/tools/go/packages" in module
// "golang.org/x/tools" is in the database, it will match on:
// golang.org/x/tools
// golang.org/x/tools/go
// golang.org/x/tools/go/packages
//
// It will not match on:
// golang.org/x/tools/g
func (db *DB) LegacyGetDirectory(ctx context.Context, dirPath, modulePath, version string, fields internal.FieldSet) (_ *internal.LegacyDirectory, err error) {
	defer derrors.Wrap(&err, "DB.LegacyGetDirectory(ctx, %q, %q, %q)", dirPath, modulePath, version)

	if dirPath == "" || modulePath == "" || version == "" {
		return nil, fmt.Errorf("none of pkgPath, modulePath, or version can be empty: %w", derrors.InvalidArgument)
	}

	var (
		query string
		args  []interface{}
	)
	if modulePath == internal.UnknownModulePath || modulePath == stdlib.ModulePath {
		query, args = directoryQueryWithoutModulePath(dirPath, version, fields)
	} else {
		query, args = directoryQueryWithModulePath(dirPath, modulePath, version, fields)
	}

	var (
		packages []*internal.LegacyPackage
		mi       = internal.LegacyModuleInfo{LegacyReadmeContents: internal.StringFieldMissing}
	)
	collect := func(rows *sql.Rows) error {
		var (
			pkg          = internal.LegacyPackage{DocumentationHTML: internal.StringFieldMissing}
			licenseTypes []string
			licensePaths []string
		)
		scanArgs := []interface{}{
			&pkg.Path,
			&pkg.Name,
			&pkg.Synopsis,
			&pkg.V1Path,
		}
		if fields&internal.WithDocumentationHTML != 0 {
			scanArgs = append(scanArgs, database.NullIsEmpty(&pkg.DocumentationHTML))
		}
		scanArgs = append(scanArgs,
			pq.Array(&licenseTypes),
			pq.Array(&licensePaths),
			&pkg.IsRedistributable,
			&pkg.GOOS,
			&pkg.GOARCH,
			&mi.Version,
			&mi.ModulePath,
			database.NullIsEmpty(&mi.LegacyReadmeFilePath))
		if fields&internal.WithReadmeContents != 0 {
			scanArgs = append(scanArgs, database.NullIsEmpty(&mi.LegacyReadmeContents))
		}
		var hasGoMod sql.NullBool
		scanArgs = append(scanArgs,
			&mi.CommitTime,
			&mi.VersionType,
			jsonbScanner{&mi.SourceInfo},
			&mi.IsRedistributable,
			&hasGoMod)
		if err := rows.Scan(scanArgs...); err != nil {
			return fmt.Errorf("row.Scan(): %v", err)
		}
		setHasGoMod(&mi.ModuleInfo, hasGoMod)
		lics, err := zipLicenseMetadata(licenseTypes, licensePaths)
		if err != nil {
			return err
		}
		pkg.Licenses = lics
		packages = append(packages, &pkg)
		return nil
	}
	if err := db.db.RunQuery(ctx, query, collect, args...); err != nil {
		return nil, err
	}
	if len(packages) == 0 {
		return nil, fmt.Errorf("packages in directory not found: %w", derrors.NotFound)
	}
	sort.Slice(packages, func(i, j int) bool {
		return packages[i].Path < packages[j].Path
	})
	return &internal.LegacyDirectory{
		Path:             dirPath,
		LegacyModuleInfo: mi,
		Packages:         packages,
	}, nil
}

func directoryColumns(fields internal.FieldSet) string {
	var doc, readme string
	if fields&internal.WithDocumentationHTML != 0 {
		doc = "p.documentation,"
	}
	if fields&internal.WithReadmeContents != 0 {
		readme = "m.readme_contents,"
	}
	return `
			p.path,
			p.name,
			p.synopsis,
			p.v1_path,
			` + doc + `
			p.license_types,
			p.license_paths,
			p.redistributable,
			p.goos,
			p.goarch,
			p.version,
			p.module_path,
			m.readme_file_path,
			` + readme + `
			m.commit_time,
			m.version_type,
			m.source_info,
			m.redistributable,
			m.has_go_mod`
}

const orderByLatest = `
			ORDER BY
				-- Order the versions by release then prerelease.
				-- The default version should be the first release
				-- version available, if one exists.
				version_type = 'release' DESC,
				sort_version DESC,
				module_path DESC`

// directoryQueryWithoutModulePath returns the query and args needed to fetch a
// directory when no module path is provided.
func directoryQueryWithoutModulePath(dirPath, version string, fields internal.FieldSet) (string, []interface{}) {
	if version == internal.LatestVersion {
		// internal packages are filtered out from the search_documents table.
		// However, for other packages, fetching from search_documents is
		// significantly faster than fetching from packages.
		var table string
		if !isInternalPackage(dirPath) {
			table = "search_documents"
		} else {
			table = "packages"
		}

		// Only dirPath is specified, so get the latest version of the
		// package found in any module that contains that directory.
		//
		// This might not necessarily be the latest module version that
		// matches the directory path. For example,
		// github.com/hashicorp/vault@v1.2.3 does not contain
		// github.com/hashicorp/vault/api, but
		// github.com/hashicorp/vault/api@v1.1.5 does.
		return fmt.Sprintf(`
			SELECT %s
			FROM
				packages p
			INNER JOIN (
				SELECT *
				FROM
					modules
				WHERE
					(module_path, version) IN (
						SELECT module_path, version
						FROM %s
						WHERE tsv_parent_directories @@ $1::tsquery
						GROUP BY 1, 2
					)
				%s
				LIMIT 1
			) m
			ON
				p.module_path = m.module_path
				AND p.version = m.version
			WHERE tsv_parent_directories @@ $1::tsquery;`,
			directoryColumns(fields), table, orderByLatest), []interface{}{dirPath}
	}

	// dirPath and version are specified, so get that directory version
	// from any module.  If it exists in multiple modules, return the one
	// with the longest path.
	return fmt.Sprintf(`
		WITH potential_packages AS (
			SELECT *
			FROM packages
			WHERE tsv_parent_directories @@ $1::tsquery
		),
		module_version AS (
			SELECT m.*
			FROM modules m
			INNER JOIN potential_packages p
			ON
				p.module_path = m.module_path
				AND p.version = m.version
			WHERE
				p.version = $2
			ORDER BY
				module_path DESC
			LIMIT 1
		)
		SELECT %s
		FROM potential_packages p
		INNER JOIN module_version m
		ON
			p.module_path = m.module_path
			AND p.version = m.version;`, directoryColumns(fields)), []interface{}{dirPath, version}
}

// directoryQueryWithoutModulePath returns the query and args needed to fetch a
// directory when a module path is provided.
func directoryQueryWithModulePath(dirPath, modulePath, version string, fields internal.FieldSet) (string, []interface{}) {
	if version == internal.LatestVersion {
		// dirPath and modulePath are specified, so get the latest version of
		// the package in the specified module.
		return fmt.Sprintf(`
			SELECT %s
			FROM packages p
			INNER JOIN (
				SELECT *
				FROM modules
				WHERE
					module_path = $2
					AND version IN (
						SELECT version
						FROM packages
						WHERE
							tsv_parent_directories @@ $1::tsquery
							AND module_path=$2
					)
				%s
				LIMIT 1
			) m
			ON
				p.module_path = m.module_path
				AND p.version = m.version
			WHERE
				p.module_path = $2
				AND tsv_parent_directories @@ $1::tsquery;`,
			directoryColumns(fields), orderByLatest), []interface{}{dirPath, modulePath}
	}

	// dirPath, modulePath and version were all specified. Only one
	// directory should ever match this query.
	return fmt.Sprintf(`
			SELECT %s
			FROM
				packages p
			INNER JOIN
				modules m
			ON
				p.module_path = m.module_path
				AND p.version = m.version
			WHERE
				tsv_parent_directories @@ $1::tsquery
				AND p.module_path = $2
				AND p.version = $3;`, directoryColumns(fields)), []interface{}{dirPath, modulePath, version}
}