devapp: initial support for App Engine Flex
devapp now supports 3 modes of operation:
- App Engine Standard. This supports Go 1.6 and requires a modified
local go-github to continue working. This app is built with the
"appengine" build tag.
- App Engine Flex. This builds Go 1.8 in a Docker container, using the
build tag "appenginevm", and works with the Google Cloud platform.
- Normal. This uses an in-memory datastore, and does not
support user login/logout.
To get this working I copied appengine.go to noappengine.go and
modified the calls until they were working. You can view a running app
at https://devapp-161505.appspot.com.
Change-Id: I8ea018e63baf2dafb44150f7eee419e09065ba2c
Reviewed-on: https://go-review.googlesource.com/38161
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>
diff --git a/cmd/devapp/main.go b/cmd/devapp/main.go
deleted file mode 100644
index 0f41a48..0000000
--- a/cmd/devapp/main.go
+++ /dev/null
@@ -1,48 +0,0 @@
-// Copyright 2017 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Devapp generates the dashboard that powers dev.golang.org.
-//
-// Usage:
-//
-// devapp --port=8081
-//
-// By default devapp listens on port 8081.
-//
-// Github issues and Gerrit CL's are stored in memory in the running process.
-// To trigger an initial download, visit http://localhost:8081/update or
-// http://localhost:8081/update/stats in your browser.
-package main
-
-import (
- "flag"
- "fmt"
- "log"
- "net"
- "net/http"
- "os"
-
- _ "golang.org/x/build/devapp"
-)
-
-func init() {
- flag.Usage = func() {
- os.Stderr.WriteString(`usage: devapp [-port=port]
-
-Devapp generates the dashboard that powers dev.golang.org.
-`)
- os.Exit(2)
- }
-}
-
-func main() {
- port := flag.Uint("port", 8081, "Port to listen on")
- flag.Parse()
- ln, err := net.Listen("tcp", fmt.Sprintf(":%d", *port))
- if err != nil {
- log.Fatal(err)
- }
- fmt.Fprintf(os.Stderr, "Listening on port %d\n", *port)
- log.Fatal(http.Serve(ln, nil))
-}
diff --git a/devapp/appengine.go b/devapp/appengine.go
index 3d56980..e0b5578 100644
--- a/devapp/appengine.go
+++ b/devapp/appengine.go
@@ -18,23 +18,22 @@
)
func init() {
- onAppengine = !appengine.IsDevAppServer()
- log = &appengineLogger{}
+ logger = appengineLogger{}
http.HandleFunc("/setToken", setTokenHandler)
}
type appengineLogger struct{}
-func (a *appengineLogger) Infof(ctx context.Context, format string, args ...interface{}) {
+func (a appengineLogger) Infof(ctx context.Context, format string, args ...interface{}) {
applog.Infof(ctx, format, args...)
}
-func (a *appengineLogger) Errorf(ctx context.Context, format string, args ...interface{}) {
+func (a appengineLogger) Errorf(ctx context.Context, format string, args ...interface{}) {
applog.Errorf(ctx, format, args...)
}
-func (a *appengineLogger) Criticalf(ctx context.Context, format string, args ...interface{}) {
+func (a appengineLogger) Criticalf(ctx context.Context, format string, args ...interface{}) {
applog.Criticalf(ctx, format, args...)
}
@@ -108,19 +107,6 @@
return string(cache.Value), nil
}
-// Store a token in the database
-func setTokenHandler(w http.ResponseWriter, r *http.Request) {
- ctx := appengine.NewContext(r)
- r.ParseForm()
- if value := r.Form.Get("value"); value != "" {
- var token Cache
- token.Value = []byte(value)
- if err := putCache(ctx, "github-token", &token); err != nil {
- http.Error(w, err.Error(), 500)
- }
- }
-}
-
func getContext(r *http.Request) context.Context {
return appengine.NewContext(r)
}
diff --git a/devapp/appenginevm.go b/devapp/appenginevm.go
new file mode 100644
index 0000000..275f0bf
--- /dev/null
+++ b/devapp/appenginevm.go
@@ -0,0 +1,149 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file gets built on App Engine Flex.
+
+// +build appenginevm
+
+package devapp
+
+import (
+ "errors"
+ "fmt"
+ "net/http"
+
+ "cloud.google.com/go/compute/metadata"
+ "cloud.google.com/go/datastore"
+ "cloud.google.com/go/logging"
+ "golang.org/x/net/context"
+)
+
+var lg *logging.Logger
+var dsClient *datastore.Client
+
+func init() {
+ logger = appengineLogger{}
+ id, err := metadata.ProjectID()
+ if err != nil {
+ id = "devapp"
+ }
+ ctx := context.Background()
+ client, err := logging.NewClient(ctx, id)
+ if err == nil {
+ lg = client.Logger("log")
+ }
+ dsClient, _ = datastore.NewClient(ctx, id)
+
+ http.Handle("/static/", http.FileServer(http.Dir(".")))
+ http.HandleFunc("/favicon.ico", faviconHandler)
+ http.HandleFunc("/setToken", setTokenHandler)
+}
+
+type appengineLogger struct{}
+
+func (a appengineLogger) Infof(_ context.Context, format string, args ...interface{}) {
+ if lg == nil {
+ return
+ }
+ lg.Log(logging.Entry{
+ Severity: logging.Info,
+ Payload: fmt.Sprintf(format, args...),
+ })
+}
+
+func (a appengineLogger) Errorf(_ context.Context, format string, args ...interface{}) {
+ if lg == nil {
+ return
+ }
+ lg.Log(logging.Entry{
+ Severity: logging.Error,
+ Payload: fmt.Sprintf(format, args...),
+ })
+}
+
+func (a appengineLogger) Criticalf(ctx context.Context, format string, args ...interface{}) {
+ if lg == nil {
+ return
+ }
+ lg.LogSync(ctx, logging.Entry{
+ Severity: logging.Critical,
+ Payload: fmt.Sprintf(format, args...),
+ })
+}
+
+func newTransport(ctx context.Context) http.RoundTripper {
+ // This doesn't have a context, but we should be setting it on the request
+ // when it comes through.
+ return &http.Transport{}
+}
+
+func currentUserEmail(ctx context.Context) string {
+ return ""
+}
+
+// loginURL returns a URL that, when visited, prompts the user to sign in,
+// then redirects the user to the URL specified by dest.
+func loginURL(ctx context.Context, path string) (string, error) {
+ return "", errors.New("unimplemented")
+}
+
+func logoutURL(ctx context.Context, path string) (string, error) {
+ return "", errors.New("unimplemented")
+}
+
+func getCache(ctx context.Context, name string) (*Cache, error) {
+ cache := new(Cache)
+ key := datastore.NameKey(entityPrefix+"Cache", name, nil)
+ if err := dsClient.Get(ctx, key, cache); err != nil {
+ return cache, err
+ }
+ return cache, nil
+}
+
+func getCaches(ctx context.Context, names ...string) map[string]*Cache {
+ out := make(map[string]*Cache)
+ var keys []*datastore.Key
+ var ptrs []*Cache
+ for _, name := range names {
+ keys = append(keys, datastore.NameKey(entityPrefix+"Cache", name, nil))
+ out[name] = new(Cache)
+ ptrs = append(ptrs, out[name])
+ }
+ dsClient.GetMulti(ctx, keys, ptrs) // Ignore errors since they might not exist.
+ return out
+}
+
+func getPage(ctx context.Context, page string) (*Page, error) {
+ entity := new(Page)
+ key := datastore.NameKey(entityPrefix+"Page", page, nil)
+ err := dsClient.Get(ctx, key, entity)
+ return entity, err
+}
+
+func writePage(ctx context.Context, page string, content []byte) error {
+ entity := &Page{
+ Content: content,
+ }
+ key := datastore.NameKey(entityPrefix+"Page", page, nil)
+ _, err := dsClient.Put(ctx, key, entity)
+ return err
+}
+
+func putCache(ctx context.Context, name string, c *Cache) error {
+ key := datastore.NameKey(entityPrefix+"Cache", name, nil)
+ _, err := dsClient.Put(ctx, key, c)
+ return err
+}
+
+func getToken(ctx context.Context) (string, error) {
+ cache, err := getCache(ctx, "github-token")
+ if err != nil {
+ return "", err
+ }
+ return string(cache.Value), nil
+}
+
+func getContext(r *http.Request) context.Context {
+ return r.Context()
+}
diff --git a/devapp/appenginevm_test.go b/devapp/appenginevm_test.go
new file mode 100644
index 0000000..61ddce7
--- /dev/null
+++ b/devapp/appenginevm_test.go
@@ -0,0 +1,22 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build appenginevm
+
+package devapp
+
+import (
+ "net/http"
+ "net/http/httptest"
+ "testing"
+)
+
+func TestSetTokenMethodNotAllowed(t *testing.T) {
+ w := httptest.NewRecorder()
+ req := httptest.NewRequest("GET", "/setToken", nil)
+ http.DefaultServeMux.ServeHTTP(w, req)
+ if w.Code != 405 {
+ t.Errorf("GET /setToken: got %d, want 405", w.Code)
+ }
+}
diff --git a/devapp/cache.go b/devapp/cache.go
index 2bfd49a..22b1956 100644
--- a/devapp/cache.go
+++ b/devapp/cache.go
@@ -17,7 +17,7 @@
// Value contains a gzipped gob'd serialization of the object
// to be cached. It must be []byte to avail ourselves of the
// datastore's 1 MB size limit.
- Value []byte
+ Value []byte `datastore:"value,noindex"`
}
func unpackCache(cache *Cache, data interface{}) error {
@@ -54,6 +54,6 @@
return err
}
cache.Value = cacheout.Bytes()
- log.Infof(ctx, "Cache %q update finished; writing %d bytes", name, cacheout.Len())
+ logger.Infof(ctx, "Cache %q update finished; writing %d bytes", name, cacheout.Len())
return putCache(ctx, name, &cache)
}
diff --git a/devapp/dash.go b/devapp/dash.go
index 6e51c91..c26c104 100644
--- a/devapp/dash.go
+++ b/devapp/dash.go
@@ -18,15 +18,15 @@
"golang.org/x/net/context"
)
-var onAppengine = false
-
-type logger interface {
+type pkgLogger interface {
+ // This needs to be x/net/context because App Engine Standard still runs on
+ // Go 1.6.
Infof(context.Context, string, ...interface{})
Errorf(context.Context, string, ...interface{})
Criticalf(context.Context, string, ...interface{})
}
-var log logger
+var logger pkgLogger
func findEmail(ctx context.Context, data *godash.Data) string {
email := currentUserEmail(ctx)
@@ -118,7 +118,7 @@
tmpl, err := ioutil.ReadFile("template/dash.html")
if err != nil {
- log.Errorf(ctx, "reading template: %v", err)
+ logger.Errorf(ctx, "reading template: %v", err)
return
}
t, err := template.New("main").Funcs(template.FuncMap{
@@ -135,7 +135,7 @@
"release": d.release,
}).Parse(string(tmpl))
if err != nil {
- log.Errorf(ctx, "parsing template: %v", err)
+ logger.Errorf(ctx, "parsing template: %v", err)
return
}
@@ -174,7 +174,7 @@
}
if err := t.Execute(w, tData); err != nil {
- log.Errorf(ctx, "execute: %v", err)
+ logger.Errorf(ctx, "execute: %v", err)
http.Error(w, "error executing template", 500)
return
}
diff --git a/devapp/devapp.go b/devapp/devapp.go
index 0ed9ef9..88e247d 100644
--- a/devapp/devapp.go
+++ b/devapp/devapp.go
@@ -11,7 +11,7 @@
"bytes"
"fmt"
"io"
- stdlog "log"
+ "log"
"net/http"
"strings"
"sync/atomic"
@@ -53,23 +53,23 @@
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ctx := getContext(r)
if err := fn(ctx, w, r); err != nil {
- log.Criticalf(ctx, "handler failed: %v", err)
+ logger.Criticalf(ctx, "handler failed: %v", err)
http.Error(w, err.Error(), 500)
}
})
}
func logFn(ctx context.Context, w io.Writer) func(string, ...interface{}) {
- logger := stdlog.New(w, "", stdlog.Lmicroseconds)
+ stdLogger := log.New(w, "", log.Lmicroseconds)
return func(format string, args ...interface{}) {
- logger.Printf(format, args...)
- log.Infof(ctx, format, args...)
+ stdLogger.Printf(format, args...)
+ logger.Infof(ctx, format, args...)
}
}
type Page struct {
// Content is the complete HTML of the page.
- Content []byte
+ Content []byte `datastore:"content,noindex"`
}
func servePage(w http.ResponseWriter, r *http.Request, page string) {
@@ -106,7 +106,7 @@
ct := &countTransport{newTransport(ctx), 0}
gh := godash.NewGitHubClient("golang/go", token, ct)
defer func() {
- log.Infof(ctx, "Sent %d requests to GitHub", ct.Count())
+ logger.Infof(ctx, "Sent %d requests to GitHub", ct.Count())
}()
ger := gerrit.NewClient("https://go-review.googlesource.com", gerrit.NoAuth)
// Without a deadline, urlfetch will use a 5s timeout which is too slow for Gerrit.
@@ -120,12 +120,12 @@
}
if err := data.Reviewers.LoadGithub(ctx, gh); err != nil {
- log.Criticalf(ctx, "failed to load reviewers: %v", err)
+ logger.Criticalf(ctx, "failed to load reviewers: %v", err)
return err
}
l := logFn(ctx, w)
if err := data.FetchData(gerctx, gh, ger, l, 7, false, false); err != nil {
- log.Criticalf(ctx, "failed to fetch data: %v", err)
+ logger.Criticalf(ctx, "failed to fetch data: %v", err)
return err
}
@@ -153,3 +153,30 @@
}
return writeCache(ctx, "gzdata", &data)
}
+
+// POST /setToken
+//
+// Store a github token in the database, so we can use it for API calls.
+// Necessary because the only available configuration method is the app.yaml
+// file, which we want to check in, and can't store secrets.
+func setTokenHandler(w http.ResponseWriter, r *http.Request) {
+ if r.Method != "POST" {
+ w.Header().Set("Allow", "POST")
+ http.Error(w, "Method not allowed", 405)
+ return
+ }
+ ctx := r.Context()
+ r.ParseForm()
+ if value := r.Form.Get("value"); value != "" {
+ var token Cache
+ token.Value = []byte(value)
+ if err := putCache(ctx, "github-token", &token); err != nil {
+ http.Error(w, err.Error(), 500)
+ }
+ }
+}
+
+// GET /favicon.ico
+func faviconHandler(w http.ResponseWriter, r *http.Request) {
+ http.ServeFile(w, r, "./static/favicon.ico")
+}
diff --git a/devapp/devapp_test.go b/devapp/devapp_test.go
index 43755f8..2244d6a 100644
--- a/devapp/devapp_test.go
+++ b/devapp/devapp_test.go
@@ -2,6 +2,8 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
+// +build !appenginevm
+
package devapp
import (
diff --git a/devapp/devappserver/app.yaml b/devapp/devappserver/app.yaml
new file mode 100644
index 0000000..09d1ac3
--- /dev/null
+++ b/devapp/devappserver/app.yaml
@@ -0,0 +1,13 @@
+# YAML file necessary to deploy to App Engine Flex.
+#
+# To deploy, run "gcloud app deploy". You will need to sign up for an account
+# and configure your local gcloud settings.
+#
+# As of March 2017 deployments take 4-8 minutes (to upload all files, build
+# a Docker container, and provision that container onto multiple servers). Note
+# that libraries will be deployed as they exist on your local filesystem, so
+# ensure third party dependencies are up to date.
+#
+# More info: https://cloud.google.com/appengine/docs/flexible/go/quickstart
+runtime: go
+env: flex
diff --git a/devapp/devappserver/main.go b/devapp/devappserver/main.go
new file mode 100644
index 0000000..e623bcf
--- /dev/null
+++ b/devapp/devappserver/main.go
@@ -0,0 +1,66 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Devapp generates the dashboard that powers dev.golang.org. This binary is
+// designed to be run outside of an App Engine context.
+//
+// Usage:
+//
+// devappserver --port=8081
+//
+// By default devapp listens on port 8081. You can also configure the port by
+// setting the PORT environment variable (but not both).
+//
+// For the moment, Github issues and Gerrit CL's are stored in memory
+// in the running process. To trigger an initial download, visit
+// http://localhost:8081/update and/or http://localhost:8081/update/stats in
+// your browser.
+
+package main
+
+import (
+ "flag"
+ "fmt"
+ "log"
+ "net"
+ "net/http"
+ "os"
+
+ _ "golang.org/x/build/devapp" // registers HTTP handlers
+)
+
+func init() {
+ flag.Usage = func() {
+ os.Stderr.WriteString(`usage: devapp [-port=port]
+
+Devapp generates the dashboard that powers dev.golang.org.
+ `)
+ }
+}
+
+const defaultPort = "8081"
+
+func main() {
+ portFlag := flag.String("port", "", "Port to listen on")
+ flag.Parse()
+ if *portFlag != "" && os.Getenv("PORT") != "" {
+ os.Stderr.WriteString("cannot set both $PORT and --port flags\n")
+ os.Exit(2)
+ }
+ var port string
+ if p := os.Getenv("PORT"); p != "" {
+ port = p
+ } else if *portFlag != "" {
+ port = *portFlag
+ } else {
+ port = defaultPort
+ }
+ ln, err := net.Listen("tcp", ":"+port)
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Error listening on %s: %v\n", port, err)
+ os.Exit(2)
+ }
+ fmt.Fprintf(os.Stderr, "Listening on port %s\n", port)
+ log.Fatal(http.Serve(ln, nil))
+}
diff --git a/devapp/noappengine.go b/devapp/noappengine.go
index 292a84f..759b12d 100644
--- a/devapp/noappengine.go
+++ b/devapp/noappengine.go
@@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
+// +build !appenginevm
// +build !appengine
package devapp
@@ -25,7 +26,7 @@
var tokenFile = flag.String("token", "", "read GitHub token personal access token from `file` (default $HOME/.github-issue-token)")
func init() {
- log = &stderrLogger{}
+ logger = &stderrLogger{}
// TODO don't bind to the working directory.
http.Handle("/static/", http.FileServer(http.Dir(".")))
http.HandleFunc("/favicon.ico", faviconHandler)
@@ -171,8 +172,3 @@
func getContext(r *http.Request) context.Context {
return r.Context()
}
-
-func faviconHandler(w http.ResponseWriter, r *http.Request) {
- w.Header().Set("Content-Type", "image/x-icon")
- http.ServeFile(w, r, "./static/favicon.ico")
-}
