all: use Workload Identity for gitmirror, move to "prod" namespace
Rather than using a service account key, enable Workload Identity for
gitmirror. This involved moving into a non-default namespace, so the
coordinator's monitoring logic needs updating too.
Change-Id: I196fae0825f806248571aae6fb125d8c0917934d
Reviewed-on: https://go-review.googlesource.com/c/build/+/347410
Trust: Heschi Kreinick <heschi@google.com>
Run-TryBot: Heschi Kreinick <heschi@google.com>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Alexander Rakoczy <alex@golang.org>
Reviewed-by: Carlos Amedee <carlos@golang.org>
diff --git a/buildenv/envs.go b/buildenv/envs.go
index 7e586ad..f737feb 100644
--- a/buildenv/envs.go
+++ b/buildenv/envs.go
@@ -44,6 +44,9 @@
// Name is the name of the Kubernetes cluster that will be created.
Name string
+
+ // Namespace is the Kubernetes namespace to use within the cluster.
+ Namespace string
}
// Environment describes the configuration of the infrastructure for a
@@ -261,6 +264,7 @@
MaxNodes: 3,
Name: "go",
MachineType: "n1-standard-4",
+ Namespace: "default",
},
DashURL: "https://build-staging.golang.org/",
PerfDataURL: "https://perfdata.golang.org",
@@ -296,6 +300,7 @@
MaxNodes: 4,
Name: "go",
MachineType: "n1-standard-4",
+ Namespace: "prod",
},
DashURL: "https://build.golang.org/",
PerfDataURL: "https://perfdata.golang.org",
diff --git a/cmd/gitmirror/deployment-mirroring.yaml b/cmd/gitmirror/deployment-mirroring.yaml
index 192e8da..131092c 100644
--- a/cmd/gitmirror/deployment-mirroring.yaml
+++ b/cmd/gitmirror/deployment-mirroring.yaml
@@ -1,6 +1,7 @@
apiVersion: apps/v1
kind: Deployment
metadata:
+ namespace: prod
name: gitmirror-mirroring-deployment
labels:
app: gitmirror-mirroring
@@ -14,6 +15,9 @@
labels:
app: gitmirror-mirroring
spec:
+ serviceAccountName: gitmirror
+ nodeSelector:
+ cloud.google.com/gke-nodepool: workload-identity-pool
volumes:
- name: cache-volume
emptyDir:
diff --git a/cmd/gitmirror/deployment-serving.yaml b/cmd/gitmirror/deployment-serving.yaml
index 0fe6512..27831b9 100644
--- a/cmd/gitmirror/deployment-serving.yaml
+++ b/cmd/gitmirror/deployment-serving.yaml
@@ -1,6 +1,7 @@
apiVersion: apps/v1
kind: Deployment
metadata:
+ namespace: prod
name: gitmirror-serving-deployment
labels:
app: gitmirror-serving
@@ -17,6 +18,9 @@
labels:
app: gitmirror-serving
spec:
+ serviceAccountName: gitmirror
+ nodeSelector:
+ cloud.google.com/gke-nodepool: workload-identity-pool
volumes:
- name: cache-volume
emptyDir:
diff --git a/cmd/gitmirror/gitmirror.go b/cmd/gitmirror/gitmirror.go
index f664aab..84ea914 100644
--- a/cmd/gitmirror/gitmirror.go
+++ b/cmd/gitmirror/gitmirror.go
@@ -143,21 +143,8 @@
fmt.Fprintf(sshConfig, "Host github.com\n IdentityFile %v\n", privKeyPath)
}
- // gitmirror service key, used to gcloud auth for CSR writes.
+ // The gitmirror service account should already be available via GKE workload identity.
if *flagMirrorCSR {
- serviceKey, err := retrieveSecret(ctx, secret.NameGitMirrorServiceKey)
- if err != nil {
- return fmt.Errorf("reading service key from secret manager: %v", err)
- }
- serviceKeyPath := filepath.Join(home, secret.NameGitMirrorServiceKey)
- if err := ioutil.WriteFile(serviceKeyPath, []byte(serviceKey), 0600); err != nil {
- return err
- }
- gcloud := exec.CommandContext(ctx, "gcloud", "auth", "activate-service-account", "--key-file", serviceKeyPath)
- gcloud.Env = append(os.Environ(), "HOME="+home)
- if out, err := gcloud.CombinedOutput(); err != nil {
- return fmt.Errorf("gcloud auth failed: %v\n%v", err, out)
- }
fmt.Fprintf(gitConfig, "[credential \"https://source.developers.google.com\"]\n helper=gcloud.sh\n")
}
diff --git a/cmd/gitmirror/service.yaml b/cmd/gitmirror/service.yaml
index dce13f4..2501098 100644
--- a/cmd/gitmirror/service.yaml
+++ b/cmd/gitmirror/service.yaml
@@ -2,6 +2,7 @@
kind: Service
metadata:
name: gitmirror
+ namespace: prod
spec:
ports:
- port: 8585
diff --git a/internal/coordinator/pool/kube.go b/internal/coordinator/pool/kube.go
index f73201f..32aaf5c 100644
--- a/internal/coordinator/pool/kube.go
+++ b/internal/coordinator/pool/kube.go
@@ -83,6 +83,7 @@
goKubeClient, err = gke.NewClient(ctx,
gceBuildEnv.KubeTools.Name,
+ gke.OptNamespace(gceBuildEnv.KubeTools.Namespace),
gke.OptZone(gceBuildEnv.ControlZone),
gke.OptProject(gceBuildEnv.ProjectName),
gke.OptTokenSource(gce.GCPCredentials().TokenSource))
diff --git a/internal/secret/gcp_secret_manager.go b/internal/secret/gcp_secret_manager.go
index 4b6dc57..f2f2e55 100644
--- a/internal/secret/gcp_secret_manager.go
+++ b/internal/secret/gcp_secret_manager.go
@@ -34,9 +34,6 @@
// NameGitHubSSHKey is the secret name for the GitHub SSH private key.
NameGitHubSSHKey = "github-ssh-private-key"
- // NameGitMirrorServiceKey is the secret name for the gitmirror service account key.
- NameGitMirrorServiceKey = "gitmirror-service-key"
-
// NameGobotPassword is the secret name for the Gobot password.
NameGobotPassword = "gobot-password"
diff --git a/kubernetes/client.go b/kubernetes/client.go
index d6a6185..d52e4c6 100644
--- a/kubernetes/client.go
+++ b/kubernetes/client.go
@@ -39,7 +39,10 @@
// The provided host is an url (scheme://hostname[:port]) of a
// Kubernetes master without any path.
// The provided client is an authorized http.Client used to perform requests to the Kubernetes API master.
-func NewClient(baseURL string, client *http.Client) (*Client, error) {
+func NewClient(baseURL, namespace string, client *http.Client) (*Client, error) {
+ if namespace == "" {
+ return nil, fmt.Errorf("must specify Kubernetes namespace")
+ }
validURL, err := url.Parse(baseURL)
if err != nil {
return nil, fmt.Errorf("failed to parse URL %q: %v", baseURL, err)
@@ -47,7 +50,7 @@
return &Client{
endpointURL: strings.TrimSuffix(validURL.String(), "/") + "/api/v1",
httpClient: client,
- namespace: "default",
+ namespace: namespace,
}, nil
}
diff --git a/kubernetes/client_test.go b/kubernetes/client_test.go
index debf7f5..3deabdc 100644
--- a/kubernetes/client_test.go
+++ b/kubernetes/client_test.go
@@ -74,7 +74,7 @@
s := httptest.NewServer(&hs)
defer s.Close()
- c, err := kubernetes.NewClient(s.URL, http.DefaultClient)
+ c, err := kubernetes.NewClient(s.URL, "default", http.DefaultClient)
if err != nil {
t.Fatalf("NewClient: %v", err)
}
diff --git a/kubernetes/gke/gke.go b/kubernetes/gke/gke.go
index 0811b53..a7ab1ac 100644
--- a/kubernetes/gke/gke.go
+++ b/kubernetes/gke/gke.go
@@ -35,6 +35,7 @@
Project string
TokenSource oauth2.TokenSource
Zone string
+ Namespace string
}
type clientOptFunc func(*clientOpt)
@@ -69,9 +70,16 @@
})
}
+// OptNamespace sets the Kubernetes namespace to look in.
+func OptNamespace(namespace string) ClientOpt {
+ return clientOptFunc(func(o *clientOpt) {
+ o.Namespace = namespace
+ })
+}
+
// NewClient returns an Kubernetes client to a GKE cluster.
func NewClient(ctx context.Context, clusterName string, opts ...ClientOpt) (*kubernetes.Client, error) {
- var opt clientOpt
+ opt := clientOpt{Namespace: "default"}
for _, o := range opts {
o.modify(&opt)
}
@@ -166,7 +174,7 @@
},
}
- kubeClient, err := kubernetes.NewClient("https://"+cluster.Endpoint, kubeHTTPClient)
+ kubeClient, err := kubernetes.NewClient("https://"+cluster.Endpoint, opt.Namespace, kubeHTTPClient)
if err != nil {
return nil, fmt.Errorf("kubernetes HTTP client could not be created: %v", err)
}
