internal/config: separate config initialization into serverconfig
This change creates a new package that does config initialization and
other GCP-specific operations that were previously done in package
config, so that config can have no cloud dependencies.
For golang/go#61399
Change-Id: I8d78294834e325b47d838892a1cef87003a4b90a
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/522516
Run-TryBot: Michael Matloob <matloob@golang.org>
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Robert Findley <rfindley@google.com>
kokoro-CI: kokoro <noreply+kokoro@google.com>
diff --git a/cmd/frontend/main.go b/cmd/frontend/main.go
index f931d85..2a05e14 100644
--- a/cmd/frontend/main.go
+++ b/cmd/frontend/main.go
@@ -18,6 +18,7 @@
"golang.org/x/pkgsite/cmd/internal/cmdconfig"
"golang.org/x/pkgsite/internal"
"golang.org/x/pkgsite/internal/config"
+ "golang.org/x/pkgsite/internal/config/serverconfig"
"golang.org/x/pkgsite/internal/dcensus"
"golang.org/x/pkgsite/internal/fetch"
"golang.org/x/pkgsite/internal/fetchdatasource"
@@ -35,7 +36,7 @@
)
var (
- queueName = config.GetEnv("GO_DISCOVERY_FRONTEND_TASK_QUEUE", "")
+ queueName = serverconfig.GetEnv("GO_DISCOVERY_FRONTEND_TASK_QUEUE", "")
workers = flag.Int("workers", 10, "number of concurrent requests to the fetch service, when running locally")
staticFlag = flag.String("static", "static", "path to folder containing static files served")
thirdPartyPath = flag.String("third_party", "third_party", "path to folder containing third-party libraries")
@@ -53,7 +54,7 @@
func main() {
flag.Parse()
ctx := context.Background()
- cfg, err := config.Init(ctx)
+ cfg, err := serverconfig.Init(ctx)
if err != nil {
log.Fatal(ctx, err)
}
@@ -169,7 +170,7 @@
}
// We are not currently forwarding any ports on AppEngine, so serving debug
// information is broken.
- if !cfg.OnAppEngine() {
+ if !serverconfig.OnAppEngine() {
dcensusServer, err := dcensus.NewServer()
if err != nil {
log.Fatal(ctx, err)
diff --git a/cmd/internal/cmdconfig/cmdconfig.go b/cmd/internal/cmdconfig/cmdconfig.go
index 3b5d4d5..6d46e09 100644
--- a/cmd/internal/cmdconfig/cmdconfig.go
+++ b/cmd/internal/cmdconfig/cmdconfig.go
@@ -19,6 +19,7 @@
"golang.org/x/pkgsite/internal"
"golang.org/x/pkgsite/internal/config"
"golang.org/x/pkgsite/internal/config/dynconfig"
+ "golang.org/x/pkgsite/internal/config/serverconfig"
"golang.org/x/pkgsite/internal/database"
"golang.org/x/pkgsite/internal/derrors"
"golang.org/x/pkgsite/internal/log"
@@ -30,12 +31,12 @@
// Logger configures a middleware.Logger.
func Logger(ctx context.Context, cfg *config.Config, logName string) middleware.Logger {
- if cfg.OnGCP() {
+ if serverconfig.OnGCP() {
opts := []logging.LoggerOption{logging.CommonResource(&mrpb.MonitoredResource{
Type: cfg.MonitoredResource.Type,
Labels: cfg.MonitoredResource.Labels,
})}
- if cfg.OnGKE() {
+ if serverconfig.OnGKE() {
opts = append(opts, logging.CommonLabels(map[string]string{
"k8s-pod/env": cfg.DeploymentEnvironment(),
"k8s-pod/app": cfg.Application(),
@@ -53,7 +54,7 @@
// Reporter configures an Error Reporting client.
func Reporter(ctx context.Context, cfg *config.Config) derrors.Reporter {
- if !cfg.OnGCP() || cfg.DisableErrorReporting {
+ if !serverconfig.OnGCP() || cfg.DisableErrorReporting {
return nil
}
reportingClient, err := errorreporting.NewClient(ctx, cfg.ProjectID, errorreporting.Config{
diff --git a/cmd/worker/main.go b/cmd/worker/main.go
index d500159..495d1b3 100644
--- a/cmd/worker/main.go
+++ b/cmd/worker/main.go
@@ -19,6 +19,7 @@
_ "github.com/jackc/pgx/v4/stdlib" // for pgx driver
"golang.org/x/pkgsite/cmd/internal/cmdconfig"
"golang.org/x/pkgsite/internal/config"
+ "golang.org/x/pkgsite/internal/config/serverconfig"
"golang.org/x/pkgsite/internal/dcensus"
"golang.org/x/pkgsite/internal/index"
"golang.org/x/pkgsite/internal/log"
@@ -31,8 +32,8 @@
)
var (
- timeout = config.GetEnvInt(context.Background(), "GO_DISCOVERY_WORKER_TIMEOUT_MINUTES", 10)
- queueName = config.GetEnv("GO_DISCOVERY_WORKER_TASK_QUEUE", "")
+ timeout = serverconfig.GetEnvInt(context.Background(), "GO_DISCOVERY_WORKER_TIMEOUT_MINUTES", 10)
+ queueName = serverconfig.GetEnv("GO_DISCOVERY_WORKER_TASK_QUEUE", "")
workers = flag.Int("workers", 10, "number of concurrent requests to the fetch service, when running locally")
// flag used in call to safehtml/template.TrustedSourceFromFlag
_ = flag.String("static", "static", "path to folder containing static files served")
@@ -44,7 +45,7 @@
ctx := context.Background()
- cfg, err := config.Init(ctx)
+ cfg, err := serverconfig.Init(ctx)
if err != nil {
log.Fatal(ctx, err)
}
@@ -128,7 +129,7 @@
}
// We are not currently forwarding any ports on AppEngine, so serving debug
// information is broken.
- if !cfg.OnAppEngine() {
+ if !serverconfig.OnAppEngine() {
dcensusServer, err := dcensus.NewServer()
if err != nil {
log.Fatal(ctx, err)
diff --git a/devtools/cmd/db/main.go b/devtools/cmd/db/main.go
index 848a491..d2e5a03 100644
--- a/devtools/cmd/db/main.go
+++ b/devtools/cmd/db/main.go
@@ -14,7 +14,7 @@
"strings"
_ "github.com/jackc/pgx/v4/stdlib" // for pgx driver
- "golang.org/x/pkgsite/internal/config"
+ "golang.org/x/pkgsite/internal/config/serverconfig"
"golang.org/x/pkgsite/internal/database"
"golang.org/x/pkgsite/internal/log"
)
@@ -39,12 +39,12 @@
}
ctx := context.Background()
- cfg, err := config.Init(ctx)
+ cfg, err := serverconfig.Init(ctx)
if err != nil {
log.Fatal(ctx, err)
}
- dbName := config.GetEnv("GO_DISCOVERY_DATABASE_NAME", "discovery-db")
+ dbName := serverconfig.GetEnv("GO_DISCOVERY_DATABASE_NAME", "discovery-db")
if err := run(ctx, flag.Args()[0], dbName, cfg.DBConnInfo()); err != nil {
log.Fatal(ctx, err)
}
diff --git a/devtools/cmd/seeddb/main.go b/devtools/cmd/seeddb/main.go
index d729ca6..5fd7b06 100644
--- a/devtools/cmd/seeddb/main.go
+++ b/devtools/cmd/seeddb/main.go
@@ -21,6 +21,7 @@
"golang.org/x/pkgsite/internal"
"golang.org/x/pkgsite/internal/config"
"golang.org/x/pkgsite/internal/config/dynconfig"
+ "golang.org/x/pkgsite/internal/config/serverconfig"
"golang.org/x/pkgsite/internal/database"
"golang.org/x/pkgsite/internal/derrors"
"golang.org/x/pkgsite/internal/experiment"
@@ -45,7 +46,7 @@
flag.Parse()
ctx := context.Background()
- cfg, err := config.Init(ctx)
+ cfg, err := serverconfig.Init(ctx)
if err != nil {
log.Fatal(ctx, err)
}
diff --git a/internal/config/config.go b/internal/config/config.go
index 5c07106..2b8f045 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -2,96 +2,21 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
-// Package config resolves shared configuration for Go Discovery services, and
-// provides functions to access this configuration.
-//
-// The Init function should be called before using any of the configuration accessors.
+// Package config provides the definition of the configuration for the
+// frontend.
package config
import (
- "context"
- "encoding/hex"
"encoding/json"
- "errors"
"fmt"
"io"
- "math/rand"
- "net/http"
- "os"
- "path"
- "strconv"
"strings"
"time"
-
- "cloud.google.com/go/storage"
- "golang.org/x/net/context/ctxhttp"
- "golang.org/x/pkgsite/internal/derrors"
- "golang.org/x/pkgsite/internal/log"
- "golang.org/x/pkgsite/internal/secrets"
- "gopkg.in/yaml.v3"
)
-// GetEnv looks up the given key from the environment, returning its value if
-// it exists, and otherwise returning the given fallback value.
-func GetEnv(key, fallback string) string {
- if value, ok := os.LookupEnv(key); ok {
- return value
- }
- return fallback
-}
-
-// GetEnvInt looks up the given key from the environment and expects an integer,
-// returning the integer value if it exists, and otherwise returning the given
-// fallback value.
-// If the environment variable has a value but it can't be parsed as an integer,
-// GetEnvInt terminates the program.
-func GetEnvInt(ctx context.Context, key string, fallback int) int {
- if s, ok := os.LookupEnv(key); ok {
- v, err := strconv.Atoi(s)
- if err != nil {
- log.Fatalf(ctx, "bad value %q for %s: %v", s, key, err)
- }
- return v
- }
- return fallback
-}
-
-// GetEnvFloat64 looks up the given key from the environment and expects a
-// float64, returning the float64 value if it exists, and otherwise returning
-// the given fallback value.
-func GetEnvFloat64(key string, fallback float64) float64 {
- if valueStr, ok := os.LookupEnv(key); ok {
- if value, err := strconv.ParseFloat(valueStr, 64); err == nil {
- return value
- }
- }
- return fallback
-}
-
// AppVersionFormat is the expected format of the app version timestamp.
const AppVersionFormat = "20060102t150405"
-// ValidateAppVersion validates that appVersion follows the expected format
-// defined by AppVersionFormat.
-func ValidateAppVersion(appVersion string) error {
- // Accept GKE versions, which start with the docker image name.
- if strings.HasPrefix(appVersion, "gcr.io/") {
- return nil
- }
- if _, err := time.Parse(AppVersionFormat, appVersion); err != nil {
- // Accept alternative version, used by our AppEngine deployment script.
- const altDateFormat = "2006-01-02t15-04"
- if len(appVersion) > len(altDateFormat) {
- appVersion = appVersion[:len(altDateFormat)]
- }
- if _, err := time.Parse(altDateFormat, appVersion); err != nil {
- return fmt.Errorf("app version %q does not match time formats %q or %q: %v",
- appVersion, AppVersionFormat, altDateFormat, err)
- }
- }
- return nil
-}
-
const (
// BypassQuotaAuthHeader is the header key used by the frontend server to know
// that a request can bypass the quota server.
@@ -215,35 +140,6 @@
return c.FallbackVersionLabel
}
-// OnAppEngine reports if the current process is running in an AppEngine
-// environment.
-func (c *Config) OnAppEngine() bool {
- return os.Getenv("GAE_ENV") == "standard"
-}
-
-// OnGKE reports whether the current process is running on GKE.
-func (c *Config) OnGKE() bool {
- return os.Getenv("GO_DISCOVERY_ON_GKE") == "true"
-}
-
-// OnCloudRun reports whether the current process is running on Cloud Run.
-func (c *Config) OnCloudRun() bool {
- // Use the presence of the environment variables provided by Cloud Run.
- // See https://cloud.google.com/run/docs/reference/container-contract.
- for _, ev := range []string{"K_SERVICE", "K_REVISION", "K_CONFIGURATION"} {
- if os.Getenv(ev) == "" {
- return false
- }
- }
- return true
-}
-
-// OnGCP reports whether the current process is running on Google Cloud
-// Platform.
-func (c *Config) OnGCP() bool {
- return c.OnAppEngine() || c.OnGKE() || c.OnCloudRun()
-}
-
// StatementTimeout is the value of the Postgres statement_timeout parameter.
// Statements that run longer than this are terminated.
// 10 minutes is the App Engine standard request timeout,
@@ -341,14 +237,6 @@
}
}
-// configOverride holds selected config settings that can be dynamically overridden.
-type configOverride struct {
- DBHost string `yaml:"DBHost"`
- DBSecondaryHost string `yaml:"DBSecondaryHost"`
- DBName string `yaml:"DBName"`
- Quota QuotaSettings `yaml:"Quota"`
-}
-
// QuotaSettings is config for internal/middleware/quota.go
type QuotaSettings struct {
Enable bool `yaml:"Enable"`
@@ -364,229 +252,6 @@
HMACKey []byte `json:"-" yaml:"-"` // key for obfuscating IPs
}
-// Init resolves all configuration values provided by the config package. It
-// must be called before any configuration values are used.
-func Init(ctx context.Context) (_ *Config, err error) {
- defer derrors.Add(&err, "config.Init(ctx)")
- // Build a Config from the execution environment, loading some values
- // from envvars and others from remote services.
- cfg := &Config{
- AuthValues: parseCommaList(os.Getenv("GO_DISCOVERY_AUTH_VALUES")),
- IndexURL: GetEnv("GO_MODULE_INDEX_URL", "https://index.golang.org/index"),
- ProxyURL: GetEnv("GO_MODULE_PROXY_URL", "https://proxy.golang.org"),
- Port: os.Getenv("PORT"),
- DebugPort: os.Getenv("DEBUG_PORT"),
- // Resolve AppEngine identifiers
- ProjectID: os.Getenv("GOOGLE_CLOUD_PROJECT"),
- ServiceID: GetEnv("GAE_SERVICE", os.Getenv("GO_DISCOVERY_SERVICE")),
- // Version ID from either AppEngine, Cloud Run (see
- // https://cloud.google.com/run/docs/reference/container-contract) or
- // GKE (set by our own config).
- VersionID: GetEnv("GAE_VERSION", GetEnv("K_REVISION", os.Getenv("DOCKER_IMAGE"))),
- InstanceID: GetEnv("GAE_INSTANCE", os.Getenv("GO_DISCOVERY_INSTANCE")),
- GoogleTagManagerID: os.Getenv("GO_DISCOVERY_GOOGLE_TAG_MANAGER_ID"),
- QueueURL: os.Getenv("GO_DISCOVERY_QUEUE_URL"),
- QueueAudience: os.Getenv("GO_DISCOVERY_QUEUE_AUDIENCE"),
-
- // LocationID is essentially hard-coded until we figure out a good way to
- // determine it programmatically, but we check an environment variable in
- // case it needs to be overridden.
- LocationID: GetEnv("GO_DISCOVERY_GAE_LOCATION_ID", "us-central1"),
- // This fallback should only be used when developing locally.
- FallbackVersionLabel: time.Now().Format(AppVersionFormat),
- DBHost: chooseOne(GetEnv("GO_DISCOVERY_DATABASE_HOST", "localhost")),
- DBUser: GetEnv("GO_DISCOVERY_DATABASE_USER", "postgres"),
- DBPassword: os.Getenv("GO_DISCOVERY_DATABASE_PASSWORD"),
- DBSecondaryHost: chooseOne(os.Getenv("GO_DISCOVERY_DATABASE_SECONDARY_HOST")),
- DBPort: GetEnv("GO_DISCOVERY_DATABASE_PORT", "5432"),
- DBName: GetEnv("GO_DISCOVERY_DATABASE_NAME", "discovery-db"),
- DBSecret: os.Getenv("GO_DISCOVERY_DATABASE_SECRET"),
- DBSSL: GetEnv("GO_DISCOVERY_DATABASE_SSL", "disable"),
- RedisCacheHost: os.Getenv("GO_DISCOVERY_REDIS_HOST"),
- RedisBetaCacheHost: os.Getenv("GO_DISCOVERY_REDIS_BETA_HOST"),
- RedisCachePort: GetEnv("GO_DISCOVERY_REDIS_PORT", "6379"),
- Quota: QuotaSettings{
- Enable: os.Getenv("GO_DISCOVERY_ENABLE_QUOTA") == "true",
- QPS: GetEnvInt(ctx, "GO_DISCOVERY_QUOTA_QPS", 10),
- Burst: 20, // ignored in redis-based quota implementation
- MaxEntries: 1000, // ignored in redis-based quota implementation
- RecordOnly: func() *bool {
- t := (os.Getenv("GO_DISCOVERY_QUOTA_RECORD_ONLY") != "false")
- return &t
- }(),
- AuthValues: parseCommaList(os.Getenv("GO_DISCOVERY_AUTH_VALUES")),
- },
- UseProfiler: os.Getenv("GO_DISCOVERY_USE_PROFILER") == "true",
- LogLevel: os.Getenv("GO_DISCOVERY_LOG_LEVEL"),
- ServeStats: os.Getenv("GO_DISCOVERY_SERVE_STATS") == "true",
- DisableErrorReporting: os.Getenv("GO_DISCOVERY_DISABLE_ERROR_REPORTING") == "true",
- VulnDB: GetEnv("GO_DISCOVERY_VULN_DB", "https://storage.googleapis.com/go-vulndb"),
- }
- log.SetLevel(cfg.LogLevel)
-
- bucket := os.Getenv("GO_DISCOVERY_CONFIG_BUCKET")
- config := os.Getenv("GO_DISCOVERY_CONFIG_DYNAMIC")
- exclude := os.Getenv("GO_DISCOVERY_EXCLUDED_FILENAME")
- if bucket != "" {
- if config == "" {
- return nil, errors.New("GO_DISCOVERY_CONFIG_DYNAMIC must be set if GO_DISCOVERY_CONFIG_BUCKET is")
- }
- cfg.DynamicConfigLocation = fmt.Sprintf("gs://%s/%s", bucket, config)
- if exclude != "" {
- cfg.DynamicExcludeLocation = fmt.Sprintf("gs://%s/%s", bucket, exclude)
- }
- } else {
- cfg.DynamicConfigLocation = config
- cfg.DynamicExcludeLocation = exclude
- }
- if cfg.OnGCP() {
- // Zone is not available in the environment but can be queried via the metadata API.
- zone, err := gceMetadata(ctx, "instance/zone")
- if err != nil {
- return nil, err
- }
- cfg.ZoneID = zone
- sa, err := gceMetadata(ctx, "instance/service-accounts/default/email")
- if err != nil {
- return nil, err
- }
- cfg.ServiceAccount = sa
- switch {
- case cfg.OnAppEngine():
- // Use the gae_app monitored resource. It would be better to use the
- // gae_instance monitored resource, but that's not currently supported:
- // https://cloud.google.com/logging/docs/api/v2/resource-list#resource-types
- cfg.MonitoredResource = &MonitoredResource{
- Type: "gae_app",
- Labels: map[string]string{
- "project_id": cfg.ProjectID,
- "module_id": cfg.ServiceID,
- "version_id": cfg.VersionID,
- "zone": cfg.ZoneID,
- },
- }
- case cfg.OnCloudRun():
- cfg.MonitoredResource = &MonitoredResource{
- Type: "cloud_run_revision",
- Labels: map[string]string{
- "project_id": cfg.ProjectID,
- "service_name": cfg.ServiceID,
- "revision_name": cfg.VersionID,
- "configuration_name": os.Getenv("K_CONFIGURATION"),
- },
- }
- case cfg.OnGKE():
- cfg.MonitoredResource = &MonitoredResource{
- Type: "k8s_container",
- Labels: map[string]string{
- "project_id": cfg.ProjectID,
- "location": path.Base(cfg.ZoneID),
- "cluster_name": cfg.DeploymentEnvironment() + "-pkgsite",
- "namespace_name": "default",
- "pod_name": os.Getenv("HOSTNAME"),
- "container_name": cfg.Application(),
- },
- }
- default:
- return nil, errors.New("on GCP but using an unknown product")
- }
- if cfg.InstanceID == "" {
- id, err := gceMetadata(ctx, "instance/id")
- if err != nil {
- return nil, fmt.Errorf("getting instance ID: %v", err)
- }
- cfg.InstanceID = id
- }
- } else { // running locally, perhaps
- cfg.MonitoredResource = &MonitoredResource{
- Type: "global",
- Labels: map[string]string{"project_id": cfg.ProjectID},
- }
- }
- if cfg.DBHost == "" {
- panic("DBHost is empty; impossible")
- }
- if cfg.DBSecret != "" {
- var err error
- cfg.DBPassword, err = secrets.Get(ctx, cfg.DBSecret)
- if err != nil {
- return nil, fmt.Errorf("could not get database password secret: %v", err)
- }
- }
- if cfg.Quota.Enable {
- s, err := secrets.Get(ctx, "quota-hmac-key")
- if err != nil {
- return nil, err
- }
- hmacKey, err := hex.DecodeString(s)
- if err != nil {
- return nil, err
- }
- if len(hmacKey) < 16 {
- return nil, errors.New("HMAC secret must be at least 16 bytes")
- }
- cfg.Quota.HMACKey = hmacKey
- log.Debugf(ctx, "quota enforcement enabled: qps=%d burst=%d maxentry=%d", cfg.Quota.QPS, cfg.Quota.Burst, cfg.Quota.MaxEntries)
- } else {
- log.Debugf(ctx, "quota enforcement disabled")
- }
-
- // If the <env>-override.yaml file exists in the configured bucket, it
- // should provide overrides for selected configuration.
- // Use this when you want to fix something in prod quickly, without waiting
- // to re-deploy. (Otherwise, do not use it.)
- if cfg.DeploymentEnvironment() != "local" {
- overrideObj := fmt.Sprintf("%s-override.yaml", cfg.DeploymentEnvironment())
- overrideBytes, err := readOverrideFile(ctx, bucket, overrideObj)
- if err != nil {
- log.Error(ctx, err)
- } else {
- log.Infof(ctx, "processing overrides from gs://%s/%s", bucket, overrideObj)
- processOverrides(ctx, cfg, overrideBytes)
- }
- }
- return cfg, nil
-}
-
-func readOverrideFile(ctx context.Context, bucketName, objName string) (_ []byte, err error) {
- defer derrors.Wrap(&err, "readOverrideFile(ctx, %q)", objName)
-
- client, err := storage.NewClient(ctx)
- if err != nil {
- return nil, err
- }
- defer client.Close()
- r, err := client.Bucket(bucketName).Object(objName).NewReader(ctx)
- if err != nil {
- return nil, err
- }
- defer r.Close()
- return io.ReadAll(r)
-}
-
-func processOverrides(ctx context.Context, cfg *Config, bytes []byte) {
- var ov configOverride
- if err := yaml.Unmarshal(bytes, &ov); err != nil {
- log.Errorf(ctx, "processOverrides: yaml.Unmarshal: %v", err)
- return
- }
- override(ctx, "DBHost", &cfg.DBHost, ov.DBHost)
- override(ctx, "DBSecondaryHost", &cfg.DBSecondaryHost, ov.DBSecondaryHost)
- override(ctx, "DBName", &cfg.DBName, ov.DBName)
- override(ctx, "Quota.QPS", &cfg.Quota.QPS, ov.Quota.QPS)
- override(ctx, "Quota.Burst", &cfg.Quota.Burst, ov.Quota.Burst)
- override(ctx, "Quota.MaxEntries", &cfg.Quota.MaxEntries, ov.Quota.MaxEntries)
- override(ctx, "Quota.RecordOnly", &cfg.Quota.RecordOnly, ov.Quota.RecordOnly)
-}
-
-func override[T comparable](ctx context.Context, name string, field *T, val T) {
- var zero T
- if val != zero {
- *field = val
- log.Infof(ctx, "overriding %s with %v", name, val)
- }
-}
-
// Dump outputs the current config information to the given Writer.
func (c *Config) Dump(w io.Writer) error {
fmt.Fprint(w, "config: ")
@@ -594,55 +259,3 @@
enc.SetIndent("", " ")
return enc.Encode(c)
}
-
-// chooseOne selects one entry at random from a whitespace-separated
-// string. It returns the empty string if there are no elements.
-func chooseOne(configVar string) string {
- fields := strings.Fields(configVar)
- if len(fields) == 0 {
- return ""
- }
- src := rand.NewSource(time.Now().UnixNano())
- rng := rand.New(src)
- return fields[rng.Intn(len(fields))]
-}
-
-// gceMetadata reads a metadata value from GCE.
-// For the possible values of name, see
-// https://cloud.google.com/appengine/docs/standard/java/accessing-instance-metadata.
-func gceMetadata(ctx context.Context, name string) (_ string, err error) {
- // See https://cloud.google.com/appengine/docs/standard/java/accessing-instance-metadata.
- // (This documentation doesn't exist for Golang, but it seems to work).
- defer derrors.Wrap(&err, "gceMetadata(ctx, %q)", name)
-
- const metadataURL = "http://metadata.google.internal/computeMetadata/v1/"
- req, err := http.NewRequest("GET", metadataURL+name, nil)
- if err != nil {
- return "", fmt.Errorf("http.NewRequest: %v", err)
- }
- req.Header.Set("Metadata-Flavor", "Google")
- resp, err := ctxhttp.Do(ctx, nil, req)
- if err != nil {
- return "", fmt.Errorf("ctxhttp.Do: %v", err)
- }
- defer resp.Body.Close()
- if resp.StatusCode != http.StatusOK {
- return "", fmt.Errorf("bad status: %s", resp.Status)
- }
- bytes, err := io.ReadAll(resp.Body)
- if err != nil {
- return "", fmt.Errorf("io.ReadAll: %v", err)
- }
- return string(bytes), nil
-}
-
-func parseCommaList(s string) []string {
- var a []string
- for _, p := range strings.Split(s, ",") {
- p = strings.TrimSpace(p)
- if p != "" {
- a = append(a, p)
- }
- }
- return a
-}
diff --git a/internal/config/serverconfig/config.go b/internal/config/serverconfig/config.go
new file mode 100644
index 0000000..7b37a70
--- /dev/null
+++ b/internal/config/serverconfig/config.go
@@ -0,0 +1,387 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package serverconfig resolves shared configuration for Go Discovery services.
+package serverconfig
+
+import (
+ "context"
+ "encoding/hex"
+ "errors"
+ "fmt"
+ "io"
+ "math/rand"
+ "net/http"
+ "os"
+ "path"
+ "strconv"
+ "strings"
+ "time"
+
+ "cloud.google.com/go/storage"
+ "golang.org/x/net/context/ctxhttp"
+ "golang.org/x/pkgsite/internal/config"
+ "golang.org/x/pkgsite/internal/derrors"
+ "golang.org/x/pkgsite/internal/log"
+ "golang.org/x/pkgsite/internal/secrets"
+ "gopkg.in/yaml.v3"
+)
+
+// GetEnv looks up the given key from the environment, returning its value if
+// it exists, and otherwise returning the given fallback value.
+func GetEnv(key, fallback string) string {
+ if value, ok := os.LookupEnv(key); ok {
+ return value
+ }
+ return fallback
+}
+
+// GetEnvInt looks up the given key from the environment and expects an integer,
+// returning the integer value if it exists, and otherwise returning the given
+// fallback value.
+// If the environment variable has a value but it can't be parsed as an integer,
+// GetEnvInt terminates the program.
+func GetEnvInt(ctx context.Context, key string, fallback int) int {
+ if s, ok := os.LookupEnv(key); ok {
+ v, err := strconv.Atoi(s)
+ if err != nil {
+ log.Fatalf(ctx, "bad value %q for %s: %v", s, key, err)
+ }
+ return v
+ }
+ return fallback
+}
+
+// ValidateAppVersion validates that appVersion follows the expected format
+// defined by AppVersionFormat.
+func ValidateAppVersion(appVersion string) error {
+ // Accept GKE versions, which start with the docker image name.
+ if strings.HasPrefix(appVersion, "gcr.io/") {
+ return nil
+ }
+ if _, err := time.Parse(config.AppVersionFormat, appVersion); err != nil {
+ // Accept alternative version, used by our AppEngine deployment script.
+ const altDateFormat = "2006-01-02t15-04"
+ if len(appVersion) > len(altDateFormat) {
+ appVersion = appVersion[:len(altDateFormat)]
+ }
+ if _, err := time.Parse(altDateFormat, appVersion); err != nil {
+ return fmt.Errorf("app version %q does not match time formats %q or %q: %v",
+ appVersion, config.AppVersionFormat, altDateFormat, err)
+ }
+ }
+ return nil
+}
+
+// OnAppEngine reports if the current process is running in an AppEngine
+// environment.
+func OnAppEngine() bool {
+ return os.Getenv("GAE_ENV") == "standard"
+}
+
+// OnGKE reports whether the current process is running on GKE.
+func OnGKE() bool {
+ return os.Getenv("GO_DISCOVERY_ON_GKE") == "true"
+}
+
+// onCloudRun reports whether the current process is running on Cloud Run.
+func onCloudRun() bool {
+ // Use the presence of the environment variables provided by Cloud Run.
+ // See https://cloud.google.com/run/docs/reference/container-contract.
+ for _, ev := range []string{"K_SERVICE", "K_REVISION", "K_CONFIGURATION"} {
+ if os.Getenv(ev) == "" {
+ return false
+ }
+ }
+ return true
+}
+
+// OnGCP reports whether the current process is running on Google Cloud
+// Platform.
+func OnGCP() bool {
+ return OnAppEngine() || OnGKE() || onCloudRun()
+}
+
+// configOverride holds selected config settings that can be dynamically overridden.
+type configOverride struct {
+ DBHost string `yaml:"DBHost"`
+ DBSecondaryHost string `yaml:"DBSecondaryHost"`
+ DBName string `yaml:"DBName"`
+ Quota config.QuotaSettings `yaml:"Quota"`
+}
+
+// Init resolves all configuration values provided by the config package. It
+// must be called before any configuration values are used.
+func Init(ctx context.Context) (_ *config.Config, err error) {
+ defer derrors.Add(&err, "config.Init(ctx)")
+ // Build a Config from the execution environment, loading some values
+ // from envvars and others from remote services.
+ cfg := &config.Config{
+ AuthValues: parseCommaList(os.Getenv("GO_DISCOVERY_AUTH_VALUES")),
+ IndexURL: GetEnv("GO_MODULE_INDEX_URL", "https://index.golang.org/index"),
+ ProxyURL: GetEnv("GO_MODULE_PROXY_URL", "https://proxy.golang.org"),
+ Port: os.Getenv("PORT"),
+ DebugPort: os.Getenv("DEBUG_PORT"),
+ // Resolve AppEngine identifiers
+ ProjectID: os.Getenv("GOOGLE_CLOUD_PROJECT"),
+ ServiceID: GetEnv("GAE_SERVICE", os.Getenv("GO_DISCOVERY_SERVICE")),
+ // Version ID from either AppEngine, Cloud Run (see
+ // https://cloud.google.com/run/docs/reference/container-contract) or
+ // GKE (set by our own config).
+ VersionID: GetEnv("GAE_VERSION", GetEnv("K_REVISION", os.Getenv("DOCKER_IMAGE"))),
+ InstanceID: GetEnv("GAE_INSTANCE", os.Getenv("GO_DISCOVERY_INSTANCE")),
+ GoogleTagManagerID: os.Getenv("GO_DISCOVERY_GOOGLE_TAG_MANAGER_ID"),
+ QueueURL: os.Getenv("GO_DISCOVERY_QUEUE_URL"),
+ QueueAudience: os.Getenv("GO_DISCOVERY_QUEUE_AUDIENCE"),
+
+ // LocationID is essentially hard-coded until we figure out a good way to
+ // determine it programmatically, but we check an environment variable in
+ // case it needs to be overridden.
+ LocationID: GetEnv("GO_DISCOVERY_GAE_LOCATION_ID", "us-central1"),
+ // This fallback should only be used when developing locally.
+ FallbackVersionLabel: time.Now().Format(config.AppVersionFormat),
+ DBHost: chooseOne(GetEnv("GO_DISCOVERY_DATABASE_HOST", "localhost")),
+ DBUser: GetEnv("GO_DISCOVERY_DATABASE_USER", "postgres"),
+ DBPassword: os.Getenv("GO_DISCOVERY_DATABASE_PASSWORD"),
+ DBSecondaryHost: chooseOne(os.Getenv("GO_DISCOVERY_DATABASE_SECONDARY_HOST")),
+ DBPort: GetEnv("GO_DISCOVERY_DATABASE_PORT", "5432"),
+ DBName: GetEnv("GO_DISCOVERY_DATABASE_NAME", "discovery-db"),
+ DBSecret: os.Getenv("GO_DISCOVERY_DATABASE_SECRET"),
+ DBSSL: GetEnv("GO_DISCOVERY_DATABASE_SSL", "disable"),
+ RedisCacheHost: os.Getenv("GO_DISCOVERY_REDIS_HOST"),
+ RedisBetaCacheHost: os.Getenv("GO_DISCOVERY_REDIS_BETA_HOST"),
+ RedisCachePort: GetEnv("GO_DISCOVERY_REDIS_PORT", "6379"),
+ Quota: config.QuotaSettings{
+ Enable: os.Getenv("GO_DISCOVERY_ENABLE_QUOTA") == "true",
+ QPS: GetEnvInt(ctx, "GO_DISCOVERY_QUOTA_QPS", 10),
+ Burst: 20, // ignored in redis-based quota implementation
+ MaxEntries: 1000, // ignored in redis-based quota implementation
+ RecordOnly: func() *bool {
+ t := (os.Getenv("GO_DISCOVERY_QUOTA_RECORD_ONLY") != "false")
+ return &t
+ }(),
+ AuthValues: parseCommaList(os.Getenv("GO_DISCOVERY_AUTH_VALUES")),
+ },
+ UseProfiler: os.Getenv("GO_DISCOVERY_USE_PROFILER") == "true",
+ LogLevel: os.Getenv("GO_DISCOVERY_LOG_LEVEL"),
+ ServeStats: os.Getenv("GO_DISCOVERY_SERVE_STATS") == "true",
+ DisableErrorReporting: os.Getenv("GO_DISCOVERY_DISABLE_ERROR_REPORTING") == "true",
+ VulnDB: GetEnv("GO_DISCOVERY_VULN_DB", "https://storage.googleapis.com/go-vulndb"),
+ }
+ log.SetLevel(cfg.LogLevel)
+
+ bucket := os.Getenv("GO_DISCOVERY_CONFIG_BUCKET")
+ configDynamic := os.Getenv("GO_DISCOVERY_CONFIG_DYNAMIC")
+ exclude := os.Getenv("GO_DISCOVERY_EXCLUDED_FILENAME")
+ if bucket != "" {
+ if configDynamic == "" {
+ return nil, errors.New("GO_DISCOVERY_CONFIG_DYNAMIC must be set if GO_DISCOVERY_CONFIG_BUCKET is")
+ }
+ cfg.DynamicConfigLocation = fmt.Sprintf("gs://%s/%s", bucket, configDynamic)
+ if exclude != "" {
+ cfg.DynamicExcludeLocation = fmt.Sprintf("gs://%s/%s", bucket, exclude)
+ }
+ } else {
+ cfg.DynamicConfigLocation = configDynamic
+ cfg.DynamicExcludeLocation = exclude
+ }
+ if OnGCP() {
+ // Zone is not available in the environment but can be queried via the metadata API.
+ zone, err := gceMetadata(ctx, "instance/zone")
+ if err != nil {
+ return nil, err
+ }
+ cfg.ZoneID = zone
+ sa, err := gceMetadata(ctx, "instance/service-accounts/default/email")
+ if err != nil {
+ return nil, err
+ }
+ cfg.ServiceAccount = sa
+ switch {
+ case OnAppEngine():
+ // Use the gae_app monitored resource. It would be better to use the
+ // gae_instance monitored resource, but that's not currently supported:
+ // https://cloud.google.com/logging/docs/api/v2/resource-list#resource-types
+ cfg.MonitoredResource = &config.MonitoredResource{
+ Type: "gae_app",
+ Labels: map[string]string{
+ "project_id": cfg.ProjectID,
+ "module_id": cfg.ServiceID,
+ "version_id": cfg.VersionID,
+ "zone": cfg.ZoneID,
+ },
+ }
+ case onCloudRun():
+ cfg.MonitoredResource = &config.MonitoredResource{
+ Type: "cloud_run_revision",
+ Labels: map[string]string{
+ "project_id": cfg.ProjectID,
+ "service_name": cfg.ServiceID,
+ "revision_name": cfg.VersionID,
+ "configuration_name": os.Getenv("K_CONFIGURATION"),
+ },
+ }
+ case OnGKE():
+ cfg.MonitoredResource = &config.MonitoredResource{
+ Type: "k8s_container",
+ Labels: map[string]string{
+ "project_id": cfg.ProjectID,
+ "location": path.Base(cfg.ZoneID),
+ "cluster_name": cfg.DeploymentEnvironment() + "-pkgsite",
+ "namespace_name": "default",
+ "pod_name": os.Getenv("HOSTNAME"),
+ "container_name": cfg.Application(),
+ },
+ }
+ default:
+ return nil, errors.New("on GCP but using an unknown product")
+ }
+ if cfg.InstanceID == "" {
+ id, err := gceMetadata(ctx, "instance/id")
+ if err != nil {
+ return nil, fmt.Errorf("getting instance ID: %v", err)
+ }
+ cfg.InstanceID = id
+ }
+ } else { // running locally, perhaps
+ cfg.MonitoredResource = &config.MonitoredResource{
+ Type: "global",
+ Labels: map[string]string{"project_id": cfg.ProjectID},
+ }
+ }
+ if cfg.DBHost == "" {
+ panic("DBHost is empty; impossible")
+ }
+ if cfg.DBSecret != "" {
+ var err error
+ cfg.DBPassword, err = secrets.Get(ctx, cfg.DBSecret)
+ if err != nil {
+ return nil, fmt.Errorf("could not get database password secret: %v", err)
+ }
+ }
+ if cfg.Quota.Enable {
+ s, err := secrets.Get(ctx, "quota-hmac-key")
+ if err != nil {
+ return nil, err
+ }
+ hmacKey, err := hex.DecodeString(s)
+ if err != nil {
+ return nil, err
+ }
+ if len(hmacKey) < 16 {
+ return nil, errors.New("HMAC secret must be at least 16 bytes")
+ }
+ cfg.Quota.HMACKey = hmacKey
+ log.Debugf(ctx, "quota enforcement enabled: qps=%d burst=%d maxentry=%d", cfg.Quota.QPS, cfg.Quota.Burst, cfg.Quota.MaxEntries)
+ } else {
+ log.Debugf(ctx, "quota enforcement disabled")
+ }
+
+ // If the <env>-override.yaml file exists in the configured bucket, it
+ // should provide overrides for selected configuration.
+ // Use this when you want to fix something in prod quickly, without waiting
+ // to re-deploy. (Otherwise, do not use it.)
+ if cfg.DeploymentEnvironment() != "local" {
+ overrideObj := fmt.Sprintf("%s-override.yaml", cfg.DeploymentEnvironment())
+ overrideBytes, err := readOverrideFile(ctx, bucket, overrideObj)
+ if err != nil {
+ log.Error(ctx, err)
+ } else {
+ log.Infof(ctx, "processing overrides from gs://%s/%s", bucket, overrideObj)
+ processOverrides(ctx, cfg, overrideBytes)
+ }
+ }
+ return cfg, nil
+}
+
+func readOverrideFile(ctx context.Context, bucketName, objName string) (_ []byte, err error) {
+ defer derrors.Wrap(&err, "readOverrideFile(ctx, %q)", objName)
+
+ client, err := storage.NewClient(ctx)
+ if err != nil {
+ return nil, err
+ }
+ defer client.Close()
+ r, err := client.Bucket(bucketName).Object(objName).NewReader(ctx)
+ if err != nil {
+ return nil, err
+ }
+ defer r.Close()
+ return io.ReadAll(r)
+}
+
+func processOverrides(ctx context.Context, cfg *config.Config, bytes []byte) {
+ var ov configOverride
+ if err := yaml.Unmarshal(bytes, &ov); err != nil {
+ log.Errorf(ctx, "processOverrides: yaml.Unmarshal: %v", err)
+ return
+ }
+ override(ctx, "DBHost", &cfg.DBHost, ov.DBHost)
+ override(ctx, "DBSecondaryHost", &cfg.DBSecondaryHost, ov.DBSecondaryHost)
+ override(ctx, "DBName", &cfg.DBName, ov.DBName)
+ override(ctx, "Quota.QPS", &cfg.Quota.QPS, ov.Quota.QPS)
+ override(ctx, "Quota.Burst", &cfg.Quota.Burst, ov.Quota.Burst)
+ override(ctx, "Quota.MaxEntries", &cfg.Quota.MaxEntries, ov.Quota.MaxEntries)
+ override(ctx, "Quota.RecordOnly", &cfg.Quota.RecordOnly, ov.Quota.RecordOnly)
+}
+
+func override[T comparable](ctx context.Context, name string, field *T, val T) {
+ var zero T
+ if val != zero {
+ *field = val
+ log.Infof(ctx, "overriding %s with %v", name, val)
+ }
+}
+
+// chooseOne selects one entry at random from a whitespace-separated
+// string. It returns the empty string if there are no elements.
+func chooseOne(configVar string) string {
+ fields := strings.Fields(configVar)
+ if len(fields) == 0 {
+ return ""
+ }
+ src := rand.NewSource(time.Now().UnixNano())
+ rng := rand.New(src)
+ return fields[rng.Intn(len(fields))]
+}
+
+// gceMetadata reads a metadata value from GCE.
+// For the possible values of name, see
+// https://cloud.google.com/appengine/docs/standard/java/accessing-instance-metadata.
+func gceMetadata(ctx context.Context, name string) (_ string, err error) {
+ // See https://cloud.google.com/appengine/docs/standard/java/accessing-instance-metadata.
+ // (This documentation doesn't exist for Golang, but it seems to work).
+ defer derrors.Wrap(&err, "gceMetadata(ctx, %q)", name)
+
+ const metadataURL = "http://metadata.google.internal/computeMetadata/v1/"
+ req, err := http.NewRequest("GET", metadataURL+name, nil)
+ if err != nil {
+ return "", fmt.Errorf("http.NewRequest: %v", err)
+ }
+ req.Header.Set("Metadata-Flavor", "Google")
+ resp, err := ctxhttp.Do(ctx, nil, req)
+ if err != nil {
+ return "", fmt.Errorf("ctxhttp.Do: %v", err)
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != http.StatusOK {
+ return "", fmt.Errorf("bad status: %s", resp.Status)
+ }
+ bytes, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return "", fmt.Errorf("io.ReadAll: %v", err)
+ }
+ return string(bytes), nil
+}
+
+func parseCommaList(s string) []string {
+ var a []string
+ for _, p := range strings.Split(s, ",") {
+ p = strings.TrimSpace(p)
+ if p != "" {
+ a = append(a, p)
+ }
+ }
+ return a
+}
diff --git a/internal/config/config_test.go b/internal/config/serverconfig/config_test.go
similarity index 86%
rename from internal/config/config_test.go
rename to internal/config/serverconfig/config_test.go
index dd141fd..c45f6a0 100644
--- a/internal/config/config_test.go
+++ b/internal/config/serverconfig/config_test.go
@@ -2,7 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
-package config
+package serverconfig
import (
"context"
@@ -10,6 +10,7 @@
"testing"
"github.com/google/go-cmp/cmp"
+ "golang.org/x/pkgsite/internal/config"
)
func TestValidateAppVersion(t *testing.T) {
@@ -56,10 +57,10 @@
func TestProcessOverrides(t *testing.T) {
tr := true
f := false
- cfg := Config{
+ cfg := config.Config{
DBHost: "origHost",
DBName: "origName",
- Quota: QuotaSettings{QPS: 1, Burst: 2, MaxEntries: 3, RecordOnly: &tr},
+ Quota: config.QuotaSettings{QPS: 1, Burst: 2, MaxEntries: 3, RecordOnly: &tr},
}
ov := `
DBHost: newHost
@@ -69,12 +70,12 @@
`
processOverrides(context.Background(), &cfg, []byte(ov))
got := cfg
- want := Config{
+ want := config.Config{
DBHost: "newHost",
DBName: "origName",
- Quota: QuotaSettings{QPS: 1, Burst: 2, MaxEntries: 17, RecordOnly: &f},
+ Quota: config.QuotaSettings{QPS: 1, Burst: 2, MaxEntries: 17, RecordOnly: &f},
}
- if diff := cmp.Diff(want, got, cmp.AllowUnexported(Config{})); diff != "" {
+ if diff := cmp.Diff(want, got, cmp.AllowUnexported(config.Config{})); diff != "" {
t.Errorf("mismatch (-want, +got):\n%s", diff)
}
}
@@ -108,7 +109,7 @@
{"-foo-bar", "unknownEnv", "foo-bar"},
{"", "local", "unknownApp"},
} {
- cfg := &Config{ServiceID: test.serviceID}
+ cfg := &config.Config{ServiceID: test.serviceID}
gotEnv := cfg.DeploymentEnvironment()
if gotEnv != test.wantEnv {
t.Errorf("%q: got %q, want %q", test.serviceID, gotEnv, test.wantEnv)
diff --git a/internal/database/dbutil.go b/internal/database/dbutil.go
index d750fba..db6eab6 100644
--- a/internal/database/dbutil.go
+++ b/internal/database/dbutil.go
@@ -13,7 +13,7 @@
"path/filepath"
"strings"
- "golang.org/x/pkgsite/internal/config"
+ "golang.org/x/pkgsite/internal/config/serverconfig"
"golang.org/x/pkgsite/internal/derrors"
"golang.org/x/pkgsite/internal/testing/testhelper"
@@ -31,10 +31,10 @@
// necessary as migrate expects a URI.
func DBConnURI(dbName string) string {
var (
- user = config.GetEnv("GO_DISCOVERY_DATABASE_USER", "postgres")
- password = config.GetEnv("GO_DISCOVERY_DATABASE_PASSWORD", "")
- host = config.GetEnv("GO_DISCOVERY_DATABASE_HOST", "localhost")
- port = config.GetEnv("GO_DISCOVERY_DATABASE_PORT", "5432")
+ user = serverconfig.GetEnv("GO_DISCOVERY_DATABASE_USER", "postgres")
+ password = serverconfig.GetEnv("GO_DISCOVERY_DATABASE_PASSWORD", "")
+ host = serverconfig.GetEnv("GO_DISCOVERY_DATABASE_HOST", "localhost")
+ port = serverconfig.GetEnv("GO_DISCOVERY_DATABASE_PORT", "5432")
)
cs := fmt.Sprintf("postgres://%s/%s?sslmode=disable&user=%s&password=%s&port=%s&timezone=UTC",
host, dbName, url.QueryEscape(user), url.QueryEscape(password), url.QueryEscape(port))
diff --git a/internal/dcensus/dcensus.go b/internal/dcensus/dcensus.go
index 3b2f76e..f67de36 100644
--- a/internal/dcensus/dcensus.go
+++ b/internal/dcensus/dcensus.go
@@ -21,6 +21,7 @@
"go.opencensus.io/trace"
"go.opencensus.io/zpages"
"golang.org/x/pkgsite/internal/config"
+ "golang.org/x/pkgsite/internal/config/serverconfig"
"golang.org/x/pkgsite/internal/derrors"
"golang.org/x/pkgsite/internal/log"
)
@@ -169,7 +170,7 @@
"task_id": cfg.InstanceID,
},
}
- if cfg.OnGKE() {
+ if serverconfig.OnGKE() {
mr = (*monitoredResource)(cfg.MonitoredResource)
}
log.Debugf(context.Background(), "monitored resource for monitoring: Type %q, Labels %v",
diff --git a/internal/postgres/benchmarks_test.go b/internal/postgres/benchmarks_test.go
index 57f2cf2..766e9b0 100644
--- a/internal/postgres/benchmarks_test.go
+++ b/internal/postgres/benchmarks_test.go
@@ -8,7 +8,7 @@
"context"
"testing"
- "golang.org/x/pkgsite/internal/config"
+ "golang.org/x/pkgsite/internal/config/serverconfig"
"golang.org/x/pkgsite/internal/database"
)
@@ -33,7 +33,7 @@
func BenchmarkSearch(b *testing.B) {
ctx := context.Background()
- cfg, err := config.Init(ctx)
+ cfg, err := serverconfig.Init(ctx)
if err != nil {
b.Fatal(err)
}
diff --git a/internal/postgres/requeue.go b/internal/postgres/requeue.go
index 618ad0e..fec8002 100644
--- a/internal/postgres/requeue.go
+++ b/internal/postgres/requeue.go
@@ -12,7 +12,7 @@
"strconv"
"golang.org/x/pkgsite/internal"
- "golang.org/x/pkgsite/internal/config"
+ "golang.org/x/pkgsite/internal/config/serverconfig"
"golang.org/x/pkgsite/internal/derrors"
"golang.org/x/pkgsite/internal/log"
)
@@ -156,7 +156,7 @@
// largeModulesLimit represents the number of large modules that we are
// willing to enqueue at a given time.
// var for testing.
-var largeModulesLimit = config.GetEnvInt(context.Background(), "GO_DISCOVERY_LARGE_MODULES_LIMIT", 100)
+var largeModulesLimit = serverconfig.GetEnvInt(context.Background(), "GO_DISCOVERY_LARGE_MODULES_LIMIT", 100)
// GetNextModulesToFetch returns the next batch of modules that need to be
// processed. We prioritize modules based on (1) whether it has status zero
diff --git a/internal/queue/gcpqueue/queue.go b/internal/queue/gcpqueue/queue.go
index 23e4ec6..bd56852 100644
--- a/internal/queue/gcpqueue/queue.go
+++ b/internal/queue/gcpqueue/queue.go
@@ -17,6 +17,7 @@
"time"
"cloud.google.com/go/cloudtasks/apiv2"
+ "golang.org/x/pkgsite/internal/config/serverconfig"
taskspb "google.golang.org/genproto/googleapis/cloud/tasks/v2"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
@@ -33,7 +34,7 @@
// New creates a new Queue with name queueName based on the configuration
// in cfg. When running locally, Queue uses numWorkers concurrent workers.
func New(ctx context.Context, cfg *config.Config, queueName string, numWorkers int, expGetter middleware.ExperimentGetter, processFunc queue.InMemoryProcessFunc) (queue.Queue, error) {
- if !cfg.OnGCP() {
+ if !serverconfig.OnGCP() {
experiments, err := expGetter(ctx)
if err != nil {
return nil, err
diff --git a/internal/worker/pages.go b/internal/worker/pages.go
index 8e6d4b7..f35a326 100644
--- a/internal/worker/pages.go
+++ b/internal/worker/pages.go
@@ -19,6 +19,7 @@
"github.com/google/safehtml/template"
"golang.org/x/pkgsite/internal"
"golang.org/x/pkgsite/internal/config"
+ "golang.org/x/pkgsite/internal/config/serverconfig"
"golang.org/x/pkgsite/internal/derrors"
"golang.org/x/pkgsite/internal/log"
"golang.org/x/pkgsite/internal/memory"
@@ -75,7 +76,7 @@
log.Warningf(ctx, "could not get cgroup stats: %v", err)
}
var logsURL string
- if s.cfg.OnGKE() {
+ if serverconfig.OnGKE() {
env := s.cfg.DeploymentEnvironment()
cluster := env + "-" + "pkgsite"
logsURL = `https://pantheon.corp.google.com/logs/query;query=resource.type%3D%22k8s_container%22%20resource.labels.cluster_name%3D%22` +
diff --git a/internal/worker/server.go b/internal/worker/server.go
index be0ac6a..a7283b6 100644
--- a/internal/worker/server.go
+++ b/internal/worker/server.go
@@ -27,6 +27,7 @@
"golang.org/x/pkgsite/internal"
"golang.org/x/pkgsite/internal/cache"
"golang.org/x/pkgsite/internal/config"
+ "golang.org/x/pkgsite/internal/config/serverconfig"
"golang.org/x/pkgsite/internal/derrors"
"golang.org/x/pkgsite/internal/godoc/dochtml"
"golang.org/x/pkgsite/internal/index"
@@ -605,7 +606,7 @@
if appVersion == "" {
return &serverError{http.StatusBadRequest, errors.New("app_version was not specified")}
}
- if err := config.ValidateAppVersion(appVersion); err != nil {
+ if err := serverconfig.ValidateAppVersion(appVersion); err != nil {
return &serverError{http.StatusBadRequest, fmt.Errorf("config.ValidateAppVersion(%q): %v", appVersion, err)}
}
@@ -874,14 +875,14 @@
var maxModuleZipSize int64 = math.MaxInt64
func init() {
- v := config.GetEnvInt(context.Background(), "GO_DISCOVERY_MAX_MODULE_ZIP_MI", -1)
+ v := serverconfig.GetEnvInt(context.Background(), "GO_DISCOVERY_MAX_MODULE_ZIP_MI", -1)
if v > 0 {
maxModuleZipSize = int64(v) * mib
}
}
func (s *Server) setLoadShedder(ctx context.Context) {
- mebis := config.GetEnvInt(ctx, "GO_DISCOVERY_MAX_IN_FLIGHT_ZIP_MI", -1)
+ mebis := serverconfig.GetEnvInt(ctx, "GO_DISCOVERY_MAX_IN_FLIGHT_ZIP_MI", -1)
if mebis > 0 {
log.Infof(ctx, "shedding load over %dMi", mebis)
s.loadShedder = &loadShedder{
diff --git a/tests/search/main.go b/tests/search/main.go
index 508f4a3..9e950af 100755
--- a/tests/search/main.go
+++ b/tests/search/main.go
@@ -18,7 +18,7 @@
"strings"
_ "github.com/jackc/pgx/v4/stdlib" // for pgx driver
- "golang.org/x/pkgsite/internal/config"
+ "golang.org/x/pkgsite/internal/config/serverconfig"
"golang.org/x/pkgsite/internal/database"
"golang.org/x/pkgsite/internal/derrors"
"golang.org/x/pkgsite/internal/frontend"
@@ -33,7 +33,7 @@
flag.Parse()
ctx := context.Background()
- cfg, err := config.Init(ctx)
+ cfg, err := serverconfig.Init(ctx)
if err != nil {
log.Fatal(ctx, err)
}
