deploy,devtools: add files needed to deploy worker
Change-Id: Ife8b67606d3fcab56d9c220807314e6f88abf705
Reviewed-on: https://go-review.googlesource.com/c/pkgsite-metrics/+/466197
Run-TryBot: Jonathan Amsterdam <jba@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Julie Qiu <julieqiu@google.com>
diff --git a/deploy/worker.yaml b/deploy/worker.yaml
new file mode 100644
index 0000000..76ebc12
--- /dev/null
+++ b/deploy/worker.yaml
@@ -0,0 +1,108 @@
+# Copyright 2022 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+# This is a Cloud Build config file for the go-ecosystem worker.
+# Invoke locally from the command line using devtools/deploy.sh.
+# It can also be configured to run from a trigger, by supplying the _ENV
+# substitution.
+
+substitutions:
+ _ENV: ''
+ _BQ_DATASET: ''
+
+steps:
+ - id: Lock
+ name: golang:1.19.4
+ entrypoint: bash
+ args:
+ - -ec
+ - |
+ if [[ "$COMMIT_SHA" = '' ]]; then
+ echo "no COMMIT_SHA, not locking"
+ exit 0
+ fi
+ go run golang.org/x/website/cmd/locktrigger@latest \
+ -project $PROJECT_ID -build $BUILD_ID -repo https://go.googlesource.com/pkgsite-metrics
+
+ - id: Test
+ # Run tests. Do this early, to avoid wasting time if they fail.
+ name: golang:1.19.4
+ entrypoint: bash
+ args:
+ - -ec
+ - go test ./...
+
+ - id: Prepare
+ name: gcr.io/cloud-builders/gcloud
+ entrypoint: bash
+ args:
+ - -ec
+ - |
+ # Determine the image name and save for later steps.
+ if [[ "$SHORT_SHA" = '' ]]; then
+ echo >&2 "missing SHORT_SHA; use --substitutions on command line"
+ exit 1
+ fi
+ if [[ "$_ENV" = '' ]]; then
+ echo >&2 "missing _ENV; use --substitutions on command line"
+ exit 1
+ fi
+ if [[ "$_BQ_DATASET" = '' ]]; then
+ echo >&2 "missing _BQ_DATASET; use --substitutions on command line"
+ exit 1
+ fi
+
+ tag=$(date +%Y%m%dt%H%M%S)-$SHORT_SHA
+ image=gcr.io/$PROJECT_ID/${_ENV}-ecosystem-worker:$tag
+ echo "image is $image"
+ echo $image > /workspace/image.txt
+
+ # Convert the commented config.json file to valid json.
+ sed '/^[ \t]*#/d' config.json.commented > /workspace/config.json
+
+ # Download the vuln DB from its bucket to a local directory, and remember
+ # its last-modified time in a file.
+ gsutil -m -q cp -r gs://go-vulndb /workspace
+ gsutil stat gs://go-vulndb/index.json | \
+ awk '$$1 == "Update" { for (i = 4; i <= NF; i++) printf("%s ", $$i); printf("\n"); }' \
+ > /workspace/go-vulndb/LAST_MODIFIED
+ # Download a tarball of a docker Go image.
+ gsutil cp gs://go-ecosystem/go-image.tar.gz /workspace
+
+ - id: Build
+ # Build the docker image.
+ #
+ # The files we put in /workspace in the previous step need to be
+ # in the repo root so they get uploaded to the Docker daemon.
+ # However it turns out that /workspace is in fact the same directory,
+ # so no copying is necessary.
+ name: gcr.io/cloud-builders/docker
+ entrypoint: bash
+ args:
+ - -ec
+ - |
+ image=$(cat /workspace/image.txt)
+ docker build -t $image -f cmd/worker/Dockerfile . \
+ --build-arg DOCKER_IMAGE=$image \
+ --build-arg BQ_DATASET=${_BQ_DATASET}
+ docker push $image
+
+ - id: Deploy
+ name: gcr.io/cloud-builders/gcloud
+ entrypoint: bash
+ args:
+ - -ec
+ - |
+ image=$(cat /workspace/image.txt)
+ service=${_ENV}-ecosystem-worker
+ args="--project $PROJECT_ID --region us-central1"
+ gcloud beta run deploy $args $service --image $image --execution-environment=gen2
+ # If there was a rollback, `gcloud run deploy` will create a revision but
+ # not point traffic to it. The following command ensures that the new revision
+ # will get traffic.
+ latestTraffic=$(gcloud run services $args describe $service \
+ --format='value(status.traffic.latestRevision)')
+ if [[ $latestTraffic != True ]]; then
+ gcloud run services $args update-traffic $service --to-latest
+ fi
diff --git a/devtools/deploy.sh b/devtools/deploy.sh
new file mode 100755
index 0000000..8b95673
--- /dev/null
+++ b/devtools/deploy.sh
@@ -0,0 +1,69 @@
+#!/bin/bash
+
+# Copyright 2022 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+# Deploy the go-ecosystem worker to Cloud Run, using Cloud Build.
+
+set -e
+
+source devtools/lib.sh || { echo "Are you at repo root?"; exit 1; }
+
+usage() {
+ die "usage: $0 [-n] (dev | prod) BIGQUERY_DATASET"
+}
+
+# Report whether the current repo's workspace has no uncommitted files.
+clean_workspace() {
+ [[ $(git status --porcelain) == '' ]]
+}
+
+main() {
+ local prefix=
+ if [[ $1 = '-n' ]]; then
+ prefix='echo dryrun: '
+ shift
+ fi
+
+ local env=$1
+
+ case $env in
+ dev|prod);;
+ *) usage;;
+ esac
+
+ local dataset=$2
+ if [[ $dataset = '' ]]; then
+ usage
+ fi
+
+ if which grants > /dev/null; then
+ local allowed=false
+ while read g _ ok _; do
+ if [[ $ok = OK ]]; then
+ allowed=true
+ fi
+ done < <(grants check $GO_ECOSYSTEM_DEPLOY_GROUPS)
+ if ! $allowed; then
+ die "You need a grant for one of: $GO_ECOSYSTEM_DEPLOY_GROUPS"
+ fi
+ fi
+
+ local project=$(tfvar ${env}_project)
+ if [[ $project = '' ]]; then
+ die "no ${env}_project in terraform.tfvars"
+ fi
+ local commit=$(git rev-parse --short HEAD)
+ local unclean
+ if ! clean_workspace; then
+ unclean="-unclean"
+ fi
+
+ $prefix gcloud builds submit \
+ --project $project \
+ --config deploy/worker.yaml \
+ --substitutions SHORT_SHA=${commit}${unclean},_ENV=$env,_BQ_DATASET=$dataset
+}
+
+main $@
diff --git a/devtools/lib.sh b/devtools/lib.sh
index 09e0e64..ff2f3ac 100644
--- a/devtools/lib.sh
+++ b/devtools/lib.sh
@@ -1,4 +1,4 @@
-# Copyright 2023 The Go Authors. All rights reserved.
+# Copyright 2022 The Go Authors. All rights reserved.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.
@@ -47,25 +47,21 @@
$@ || err "command failed"
}
-# tfvar NAME returns the value of NAME in the terraform.tfvars file.
+# tfvar NAME returns the value of the terraform variable NAME.
tfvar() {
- local name=$1
- awk '$1 == "'$name'" { print substr($3, 2, length($3)-2) }' terraform/terraform.tfvars
+ local v=TF_VAR_$1
+ echo ${!v}
}
worker_url() {
local env=$1
- case $env in
- prod) echo https://prod-metrics-worker-7x5g2mdvca-uc.a.run.app;;
- dev) echo https://dev-metrics-worker-7x5g2mdvca-uc.a.run.app;;
- *) die "usage: $0 (dev | prod)";;
- esac
+ echo https://${env}-${GO_ECOSYSTEM_WORKER_URL_SUFFIX}
}
impersonation_service_account() {
local env=$1
case $env in
- prod|dev) echo impersonate@go-ecosystem.iam.gserviceaccount.com;;
+ prod|dev) echo impersonate@$(tfvar ${env}_project).iam.gserviceaccount.com;;
*) die "usage: $0 (dev | prod)";;
esac
}
