internal/config: support dynamic config from a file
If the environment doesn't specify a bucket, read the dynamic
config from file.
If neither is provided, don't crash.
Fixes golang/go#41608
Change-Id: I5de91f2e394e9d02d238bcfed3f5e13f173321f9
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/257098
Trust: Jonathan Amsterdam <jba@google.com>
Run-TryBot: Jonathan Amsterdam <jba@google.com>
Reviewed-by: Julie Qiu <julie@golang.org>
diff --git a/cmd/internal/cmdconfig/cmdconfig.go b/cmd/internal/cmdconfig/cmdconfig.go
index f6f3acf..240add4 100644
--- a/cmd/internal/cmdconfig/cmdconfig.go
+++ b/cmd/internal/cmdconfig/cmdconfig.go
@@ -51,13 +51,18 @@
func Experimenter(ctx context.Context, cfg *config.Config, getter middleware.ExperimentGetter, reportingClient *errorreporting.Client) *middleware.Experimenter {
if os.Getenv("GO_DISCOVERY_EXPERIMENTS_FROM_CONFIG") == "true" {
// Ignore getter, use dynamic config.
- log.Infof(ctx, "using dynamic config for experiments")
- getter = func(ctx context.Context) ([]*internal.Experiment, error) {
- dc, err := dynconfig.Read(ctx, cfg.Bucket, cfg.DynamicObject)
- if err != nil {
- return nil, err
+ if cfg.DynamicConfigLocation == "" {
+ log.Warningf(ctx, "experiments are not configured")
+ getter = func(context.Context) ([]*internal.Experiment, error) { return nil, nil }
+ } else {
+ log.Infof(ctx, "using dynamic config from %s for experiments", cfg.DynamicConfigLocation)
+ getter = func(ctx context.Context) ([]*internal.Experiment, error) {
+ dc, err := dynconfig.Read(ctx, cfg.DynamicConfigLocation)
+ if err != nil {
+ return nil, err
+ }
+ return dc.Experiments, nil
}
- return dc.Experiments, nil
}
}
e, err := middleware.NewExperimenter(ctx, 1*time.Minute, getter, reportingClient)
diff --git a/internal/config/config.go b/internal/config/config.go
index 8d88576..061d153 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -173,10 +173,9 @@
// In case of invalid/empty value, all logs will be printed.
LogLevel string
- // GCS Bucket for overrides and dynamic config.
- Bucket string
- // GCS object name for dynamic config.
- DynamicObject string
+ // DynamicConfigLocation is the location (either a file or gs://bucket/object) for
+ // dynamic configuration.
+ DynamicConfigLocation string
}
// AppVersionLabel returns the version label for the current instance. This is
@@ -403,17 +402,17 @@
MaxTimeout: time.Duration(GetEnvInt("GO_DISCOVERY_TEEPROXY_MAX_TIMEOUT_SECONDS", 240)) * time.Second,
SuccsToGreen: GetEnvInt("GO_DISCOVERY_TEEPROXY_SUCCS_TO_GREEN", 20),
},
- LogLevel: os.Getenv("GO_DISCOVERY_LOG_LEVEL"),
- Bucket: os.Getenv("GO_DISCOVERY_CONFIG_BUCKET"),
- DynamicObject: os.Getenv("GO_DISCOVERY_CONFIG_DYNAMIC"),
+ LogLevel: os.Getenv("GO_DISCOVERY_LOG_LEVEL"),
}
-
- // Check that the dynamic config is readable, but don't do anything with it.
- if cfg.Bucket == "" {
- return nil, errors.New("GO_DISCOVERY_CONFIG_BUCKET must be set")
- }
- if cfg.DynamicObject == "" {
- return nil, errors.New("GO_DISCOVERY_CONFIG_DYNAMIC must be set")
+ bucket := os.Getenv("GO_DISCOVERY_CONFIG_BUCKET")
+ object := os.Getenv("GO_DISCOVERY_CONFIG_DYNAMIC")
+ if bucket != "" {
+ if object == "" {
+ return nil, errors.New("GO_DISCOVERY_CONFIG_DYNAMIC must be set if GO_DISCOVERY_CONFIG_BUCKET is")
+ }
+ cfg.DynamicConfigLocation = fmt.Sprintf("gs://%s/%s", bucket, object)
+ } else {
+ cfg.DynamicConfigLocation = object
}
if cfg.OnGCP() {
// Zone is not available in the environment but can be queried via the metadata API.
@@ -479,11 +478,11 @@
// to re-deploy. (Otherwise, do not use it.)
overrideObj := os.Getenv("GO_DISCOVERY_CONFIG_OVERRIDE")
if overrideObj != "" {
- overrideBytes, err := readOverrideFile(ctx, cfg.Bucket, overrideObj)
+ overrideBytes, err := readOverrideFile(ctx, bucket, overrideObj)
if err != nil {
log.Print(err)
} else {
- log.Printf("processing overrides from gs://%s/%s", cfg.Bucket, overrideObj)
+ log.Printf("processing overrides from gs://%s/%s", bucket, overrideObj)
processOverrides(cfg, overrideBytes)
}
}
diff --git a/internal/config/dynconfig/dynconfig.go b/internal/config/dynconfig/dynconfig.go
index 7a4815a..bfbe2af 100644
--- a/internal/config/dynconfig/dynconfig.go
+++ b/internal/config/dynconfig/dynconfig.go
@@ -9,7 +9,11 @@
import (
"context"
+ "errors"
+ "io"
"io/ioutil"
+ "os"
+ "strings"
"cloud.google.com/go/storage"
"github.com/ghodss/yaml"
@@ -27,19 +31,38 @@
Experiments []*internal.Experiment
}
-// Read reads dynamic configuration from the given GCS bucket and object.
-func Read(ctx context.Context, bucket, object string) (_ *DynamicConfig, err error) {
- defer derrors.Wrap(&err, "dynconfig.Read(%q, %q)", bucket, object)
+// Read reads dynamic configuration from the given location.
+// Location may be of the form gs://bucket/object, denoting a GCS bucket.
+// Otherwise it is interpreted as a filename.
+func Read(ctx context.Context, location string) (_ *DynamicConfig, err error) {
+ defer derrors.Wrap(&err, "dynconfig.Read(%q)", location)
- log.Infof(ctx, "reading dynamic config from gs://%s/%s", bucket, object)
- client, err := storage.NewClient(ctx)
- if err != nil {
- return nil, err
- }
- defer client.Close()
- r, err := client.Bucket(bucket).Object(object).NewReader(ctx)
- if err != nil {
- return nil, err
+ log.Infof(ctx, "reading dynamic config from %s", location)
+ var r io.ReadCloser
+ if strings.HasPrefix(location, "gs://") {
+ parts := strings.SplitN(location[5:], "/", 2)
+ if len(parts) != 2 {
+ return nil, errors.New("bad GCS URL")
+ }
+ bucket := parts[0]
+ object := parts[1]
+ if err != nil {
+ return nil, err
+ }
+ client, err := storage.NewClient(ctx)
+ if err != nil {
+ return nil, err
+ }
+ defer client.Close()
+ r, err = client.Bucket(bucket).Object(object).NewReader(ctx)
+ if err != nil {
+ return nil, err
+ }
+ } else {
+ r, err = os.Open(location)
+ if err != nil {
+ return nil, err
+ }
}
defer r.Close()
data, err := ioutil.ReadAll(r)
