cmd: remove DB experiments from worker and frontend
All code paths get experiments from a config file.
Change-Id: I53e3c22e443573546d8ebdf65303012f845b980c
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/259840
Trust: Jonathan Amsterdam <jba@google.com>
Run-TryBot: Jonathan Amsterdam <jba@google.com>
TryBot-Result: kokoro <noreply+kokoro@google.com>
Reviewed-by: Julie Qiu <julie@golang.org>
diff --git a/cmd/frontend/main.go b/cmd/frontend/main.go
index 8e716dd..89468e4 100644
--- a/cmd/frontend/main.go
+++ b/cmd/frontend/main.go
@@ -6,13 +6,10 @@
package main
import (
- "bufio"
"context"
"flag"
"net/http"
"os"
- "strconv"
- "strings"
"time"
"cloud.google.com/go/profiler"
@@ -67,7 +64,6 @@
var (
dsg func(context.Context) internal.DataSource
- expg middleware.ExperimentGetter
fetchQueue queue.Queue
)
proxyClient, err := proxy.New(*proxyURL)
@@ -77,6 +73,7 @@
if *bypassLicenseCheck {
log.Info(ctx, "BYPASSING LICENSE CHECKING: DISPLAYING NON-REDISTRIBUTABLE INFORMATION")
}
+ expg := cmdconfig.ExperimentGetter(ctx, cfg)
if *directProxy {
var pds *proxydatasource.DataSource
if *bypassLicenseCheck {
@@ -85,8 +82,6 @@
pds = proxydatasource.New(proxyClient)
}
dsg = func(context.Context) internal.DataSource { return pds }
- exps := readLocalExperiments(ctx)
- expg = func(context.Context) ([]*internal.Experiment, error) { return exps, nil }
} else {
// Wrap the postgres driver with OpenCensus instrumentation.
ocDriver, err := ocsql.Register("postgres", ocsql.WithAllTraceOptions())
@@ -105,7 +100,6 @@
}
defer db.Close()
dsg = func(context.Context) internal.DataSource { return db }
- expg = db.GetExperiments
sourceClient := source.NewClient(config.SourceTimeout)
// The closure passed to queue.New is only used for testing and local
// execution, not in production. So it's okay that it doesn't use a
@@ -221,51 +215,3 @@
cfg.DBHost, err, cfg.DBSecondaryHost)
return database.Open(driver, ci, cfg.InstanceID)
}
-
-// Read a file of experiments used to initialize the local experiment source
-// for use in direct proxy mode.
-// Format of the file: each line is
-// name,rollout
-// For each experiment.
-func readLocalExperiments(ctx context.Context) []*internal.Experiment {
- filename := config.GetEnv("GO_DISCOVERY_LOCAL_EXPERIMENTS", "")
- if filename == "" {
- return nil
- }
- f, err := os.Open(filename)
- if err != nil {
- log.Fatal(ctx, err)
- }
- defer f.Close()
- scan := bufio.NewScanner(f)
- var experiments []*internal.Experiment
- log.Infof(ctx, "reading experiments from %q for local development", filename)
- for scan.Scan() {
- line := strings.TrimSpace(scan.Text())
- parts := strings.SplitN(line, ",", 3)
- if len(parts) != 2 {
- log.Fatalf(ctx, "invalid experiment in file: %q", line)
- }
- name := parts[0]
- if name == "" {
- log.Fatalf(ctx, "invalid experiment in file (name cannot be empty): %q", line)
- }
- rollout, err := strconv.ParseUint(parts[1], 10, 0)
- if err != nil {
- log.Fatalf(ctx, "invalid experiment in file (invalid rollout): %v", err)
- }
- if rollout > 100 {
- log.Fatalf(ctx, "invalid experiment in file (rollout must be between 0 - 100): %q", line)
- }
- experiments = append(experiments, &internal.Experiment{
- Name: name,
- Rollout: uint(rollout),
- })
- log.Infof(ctx, "experiment %q: rollout = %d", name, rollout)
- }
- if err := scan.Err(); err != nil {
- log.Fatalf(ctx, "scanning %s: %v", filename, err)
- }
- log.Infof(ctx, "found %d experiment(s)", len(experiments))
- return experiments
-}
diff --git a/cmd/internal/cmdconfig/cmdconfig.go b/cmd/internal/cmdconfig/cmdconfig.go
index 240add4..983e7b4 100644
--- a/cmd/internal/cmdconfig/cmdconfig.go
+++ b/cmd/internal/cmdconfig/cmdconfig.go
@@ -7,7 +7,8 @@
import (
"context"
- "os"
+ "fmt"
+ "strings"
"time"
"cloud.google.com/go/errorreporting"
@@ -49,25 +50,32 @@
// Experimenter configures a middleware.Experimenter.
func Experimenter(ctx context.Context, cfg *config.Config, getter middleware.ExperimentGetter, reportingClient *errorreporting.Client) *middleware.Experimenter {
- if os.Getenv("GO_DISCOVERY_EXPERIMENTS_FROM_CONFIG") == "true" {
- // Ignore getter, use dynamic config.
- if cfg.DynamicConfigLocation == "" {
- log.Warningf(ctx, "experiments are not configured")
- getter = func(context.Context) ([]*internal.Experiment, error) { return nil, nil }
- } else {
- log.Infof(ctx, "using dynamic config from %s for experiments", cfg.DynamicConfigLocation)
- getter = func(ctx context.Context) ([]*internal.Experiment, error) {
- dc, err := dynconfig.Read(ctx, cfg.DynamicConfigLocation)
- if err != nil {
- return nil, err
- }
- return dc.Experiments, nil
- }
- }
- }
e, err := middleware.NewExperimenter(ctx, 1*time.Minute, getter, reportingClient)
if err != nil {
log.Fatal(ctx, err)
}
return e
}
+
+// ExperimentGetter returns an ExperimentGetter using the config.
+func ExperimentGetter(ctx context.Context, cfg *config.Config) middleware.ExperimentGetter {
+ if cfg.DynamicConfigLocation == "" {
+ log.Warningf(ctx, "experiments are not configured")
+ return func(context.Context) ([]*internal.Experiment, error) { return nil, nil }
+ }
+ log.Infof(ctx, "using dynamic config from %s for experiments", cfg.DynamicConfigLocation)
+ return func(ctx context.Context) ([]*internal.Experiment, error) {
+ dc, err := dynconfig.Read(ctx, cfg.DynamicConfigLocation)
+ if err != nil {
+ return nil, err
+ }
+
+ var s []string
+ for _, e := range dc.Experiments {
+ s = append(s, fmt.Sprintf("%s:%d", e.Name, e.Rollout))
+ }
+ log.Infof(ctx, "read experiments %s", strings.Join(s, ", "))
+
+ return dc.Experiments, nil
+ }
+}
diff --git a/cmd/worker/main.go b/cmd/worker/main.go
index 41ecfba..f914f79 100644
--- a/cmd/worker/main.go
+++ b/cmd/worker/main.go
@@ -94,7 +94,8 @@
log.Fatal(ctx, err)
}
sourceClient := source.NewClient(config.SourceTimeout)
- fetchQueue, err := queue.New(ctx, cfg, queueName, *workers, db.GetExperiments,
+ expg := cmdconfig.ExperimentGetter(ctx, cfg)
+ fetchQueue, err := queue.New(ctx, cfg, queueName, *workers, expg,
func(ctx context.Context, modulePath, version string) (int, error) {
return worker.FetchAndUpdateState(ctx, modulePath, version, proxyClient, sourceClient, db, cfg.AppVersionLabel())
})
@@ -105,7 +106,7 @@
reportingClient := cmdconfig.ReportingClient(ctx, cfg)
redisHAClient := getHARedis(ctx, cfg)
redisCacheClient := getCacheRedis(ctx, cfg)
- experimenter := cmdconfig.Experimenter(ctx, cfg, db.GetExperiments, reportingClient)
+ experimenter := cmdconfig.Experimenter(ctx, cfg, expg, reportingClient)
server, err := worker.NewServer(cfg, worker.ServerConfig{
DB: db,
IndexClient: indexClient,
