message/pipeline: copy over files from cmd/gotext Most of the code will be turned into a package. Change-Id: I47609e5afb176e42d3b0b29bfa04cca23ab6867f Reviewed-on: https://go-review.googlesource.com/82235 Run-TryBot: Marcel van Lohuizen <mpvl@golang.org> TryBot-Result: Gobot Gobot <gobot@golang.org> Reviewed-by: Nigel Tao <nigeltao@golang.org>
diff --git a/message/pipeline/common.go b/message/pipeline/common.go new file mode 100644 index 0000000..ae87e44 --- /dev/null +++ b/message/pipeline/common.go
@@ -0,0 +1,48 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "fmt" + "go/build" + "go/parser" + + "golang.org/x/tools/go/loader" +) + +const ( + extractFile = "extracted.gotext.json" + outFile = "out.gotext.json" + gotextSuffix = ".gotext.json" +) + +// NOTE: The command line tool already prefixes with "gotext:". +var ( + wrap = func(err error, msg string) error { + return fmt.Errorf("%s: %v", msg, err) + } + wrapf = func(err error, msg string, args ...interface{}) error { + return wrap(err, fmt.Sprintf(msg, args...)) + } + errorf = fmt.Errorf +) + +func loadPackages(conf *loader.Config, args []string) (*loader.Program, error) { + if len(args) == 0 { + args = []string{"."} + } + + conf.Build = &build.Default + conf.ParserMode = parser.ParseComments + + // Use the initial packages from the command line. + args, err := conf.FromArgs(args, false) + if err != nil { + return nil, wrap(err, "loading packages failed") + } + + // Load, parse and type-check the whole program. + return conf.Load() +}
diff --git a/message/pipeline/doc.go b/message/pipeline/doc.go new file mode 100644 index 0000000..2a274f7 --- /dev/null +++ b/message/pipeline/doc.go
@@ -0,0 +1,53 @@ +// Code generated by go generate. DO NOT EDIT. + +// gotext is a tool for managing text in Go source code. +// +// Usage: +// +// gotext command [arguments] +// +// The commands are: +// +// extract extracts strings to be translated from code +// rewrite rewrites fmt functions to use a message Printer +// generate generates code to insert translated messages +// +// Use "go help [command]" for more information about a command. +// +// Additional help topics: +// +// +// Use "gotext help [topic]" for more information about that topic. +// +// +// Extracts strings to be translated from code +// +// Usage: +// +// go extract <package>* +// +// +// +// +// Rewrites fmt functions to use a message Printer +// +// Usage: +// +// go rewrite <package> +// +// rewrite is typically done once for a project. It rewrites all usages of +// fmt to use x/text's message package whenever a message.Printer is in scope. +// It rewrites Print and Println calls with constant strings to the equivalent +// using Printf to allow translators to reorder arguments. +// +// +// Generates code to insert translated messages +// +// Usage: +// +// go generate <package> +// +// +// +// +package main
diff --git a/message/pipeline/extract.go b/message/pipeline/extract.go new file mode 100644 index 0000000..561f050 --- /dev/null +++ b/message/pipeline/extract.go
@@ -0,0 +1,358 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "go/ast" + "go/constant" + "go/format" + "go/token" + "go/types" + "io/ioutil" + "os" + "path" + "path/filepath" + "strings" + "unicode" + "unicode/utf8" + + "golang.org/x/text/internal" + fmtparser "golang.org/x/text/internal/format" + "golang.org/x/text/language" + "golang.org/x/tools/go/loader" +) + +// TODO: +// - merge information into existing files +// - handle different file formats (PO, XLIFF) +// - handle features (gender, plural) +// - message rewriting + +var ( + srcLang *string + lang *string +) + +func init() { + srcLang = cmdExtract.Flag.String("srclang", "en-US", "the source-code language") + lang = cmdExtract.Flag.String("lang", "en-US", "comma-separated list of languages to process") +} + +var cmdExtract = &Command{ + Run: runExtract, + UsageLine: "extract <package>*", + Short: "extracts strings to be translated from code", +} + +func runExtract(cmd *Command, args []string) error { + conf := loader.Config{} + prog, err := loadPackages(&conf, args) + if err != nil { + return wrap(err, "") + } + + // print returns Go syntax for the specified node. + print := func(n ast.Node) string { + var buf bytes.Buffer + format.Node(&buf, conf.Fset, n) + return buf.String() + } + + var messages []Message + + for _, info := range prog.AllPackages { + for _, f := range info.Files { + // Associate comments with nodes. + cmap := ast.NewCommentMap(prog.Fset, f, f.Comments) + getComment := func(n ast.Node) string { + cs := cmap.Filter(n).Comments() + if len(cs) > 0 { + return strings.TrimSpace(cs[0].Text()) + } + return "" + } + + // Find function calls. + ast.Inspect(f, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok { + return true + } + + // Skip calls of functions other than + // (*message.Printer).{Sp,Fp,P}rintf. + sel, ok := call.Fun.(*ast.SelectorExpr) + if !ok { + return true + } + meth := info.Selections[sel] + if meth == nil || meth.Kind() != types.MethodVal { + return true + } + // TODO: remove cheap hack and check if the type either + // implements some interface or is specifically of type + // "golang.org/x/text/message".Printer. + m, ok := extractFuncs[path.Base(meth.Recv().String())] + if !ok { + return true + } + + fmtType, ok := m[meth.Obj().Name()] + if !ok { + return true + } + // argn is the index of the format string. + argn := fmtType.arg + if argn >= len(call.Args) { + return true + } + + args := call.Args[fmtType.arg:] + + fmtMsg, ok := msgStr(info, args[0]) + if !ok { + // TODO: identify the type of the format argument. If it + // is not a string, multiple keys may be defined. + return true + } + comment := "" + key := []string{} + if ident, ok := args[0].(*ast.Ident); ok { + key = append(key, ident.Name) + if v, ok := ident.Obj.Decl.(*ast.ValueSpec); ok && v.Comment != nil { + // TODO: get comment above ValueSpec as well + comment = v.Comment.Text() + } + } + + arguments := []argument{} + args = args[1:] + simArgs := make([]interface{}, len(args)) + for i, arg := range args { + expr := print(arg) + val := "" + if v := info.Types[arg].Value; v != nil { + val = v.ExactString() + simArgs[i] = val + switch arg.(type) { + case *ast.BinaryExpr, *ast.UnaryExpr: + expr = val + } + } + arguments = append(arguments, argument{ + ArgNum: i + 1, + Type: info.Types[arg].Type.String(), + UnderlyingType: info.Types[arg].Type.Underlying().String(), + Expr: expr, + Value: val, + Comment: getComment(arg), + Position: posString(conf, info, arg.Pos()), + // TODO report whether it implements + // interfaces plural.Interface, + // gender.Interface. + }) + } + msg := "" + + ph := placeholders{index: map[string]string{}} + + p := fmtparser.Parser{} + p.Reset(simArgs) + for p.SetFormat(fmtMsg); p.Scan(); { + switch p.Status { + case fmtparser.StatusText: + msg += p.Text() + case fmtparser.StatusSubstitution, + fmtparser.StatusBadWidthSubstitution, + fmtparser.StatusBadPrecSubstitution: + arguments[p.ArgNum-1].used = true + arg := arguments[p.ArgNum-1] + sub := p.Text() + if !p.HasIndex { + r, sz := utf8.DecodeLastRuneInString(sub) + sub = fmt.Sprintf("%s[%d]%c", sub[:len(sub)-sz], p.ArgNum, r) + } + msg += fmt.Sprintf("{%s}", ph.addArg(&arg, sub)) + } + } + key = append(key, msg) + + // Add additional Placeholders that can be used in translations + // that are not present in the string. + for _, arg := range arguments { + if arg.used { + continue + } + ph.addArg(&arg, fmt.Sprintf("%%[%d]v", arg.ArgNum)) + } + + if c := getComment(call.Args[0]); c != "" { + comment = c + } + + messages = append(messages, Message{ + ID: key, + Key: fmtMsg, + Message: Text{Msg: msg}, + // TODO(fix): this doesn't get the before comment. + Comment: comment, + Placeholders: ph.slice, + Position: posString(conf, info, call.Lparen), + }) + return true + }) + } + } + + tag, err := language.Parse(*srcLang) + if err != nil { + return wrap(err, "") + } + out := Locale{ + Language: tag, + Messages: messages, + } + data, err := json.MarshalIndent(out, "", " ") + if err != nil { + return wrap(err, "") + } + os.MkdirAll(*dir, 0755) + // TODO: this file can probably go if we replace the extract + generate + // cycle with a init once and update cycle. + file := filepath.Join(*dir, extractFile) + if err := ioutil.WriteFile(file, data, 0644); err != nil { + return wrapf(err, "could not create file") + } + + langs := append(getLangs(), tag) + langs = internal.UniqueTags(langs) + for _, tag := range langs { + // TODO: inject translations from existing files to avoid retranslation. + out.Language = tag + data, err := json.MarshalIndent(out, "", " ") + if err != nil { + return wrap(err, "JSON marshal failed") + } + file := filepath.Join(*dir, tag.String(), outFile) + if err := os.MkdirAll(filepath.Dir(file), 0750); err != nil { + return wrap(err, "dir create failed") + } + if err := ioutil.WriteFile(file, data, 0740); err != nil { + return wrap(err, "write failed") + } + } + return nil +} + +func posString(conf loader.Config, info *loader.PackageInfo, pos token.Pos) string { + p := conf.Fset.Position(pos) + file := fmt.Sprintf("%s:%d:%d", filepath.Base(p.Filename), p.Line, p.Column) + return filepath.Join(info.Pkg.Path(), file) +} + +// extractFuncs indicates the types and methods for which to extract strings, +// and which argument to extract. +// TODO: use the types in conf.Import("golang.org/x/text/message") to extract +// the correct instances. +var extractFuncs = map[string]map[string]extractType{ + // TODO: Printer -> *golang.org/x/text/message.Printer + "message.Printer": { + "Printf": extractType{arg: 0, format: true}, + "Sprintf": extractType{arg: 0, format: true}, + "Fprintf": extractType{arg: 1, format: true}, + + "Lookup": extractType{arg: 0}, + }, +} + +type extractType struct { + // format indicates if the next arg is a formatted string or whether to + // concatenate all arguments + format bool + // arg indicates the position of the argument to extract. + arg int +} + +func getID(arg *argument) string { + s := getLastComponent(arg.Expr) + s = strip(s) + s = strings.Replace(s, " ", "", -1) + // For small variable names, use user-defined types for more info. + if len(s) <= 2 && arg.UnderlyingType != arg.Type { + s = getLastComponent(arg.Type) + } + return strings.Title(s) +} + +// strip is a dirty hack to convert function calls to placeholder IDs. +func strip(s string) string { + s = strings.Map(func(r rune) rune { + if unicode.IsSpace(r) || r == '-' { + return '_' + } + if !unicode.In(r, unicode.Letter, unicode.Mark) { + return -1 + } + return r + }, s) + // Strip "Get" from getter functions. + if strings.HasPrefix(s, "Get") || strings.HasPrefix(s, "get") { + if len(s) > len("get") { + r, _ := utf8.DecodeRuneInString(s) + if !unicode.In(r, unicode.Ll, unicode.M) { // not lower or mark + s = s[len("get"):] + } + } + } + return s +} + +type placeholders struct { + index map[string]string + slice []Placeholder +} + +func (p *placeholders) addArg(arg *argument, sub string) (id string) { + id = getID(arg) + id1 := id + alt, ok := p.index[id1] + for i := 1; ok && alt != sub; i++ { + id1 = fmt.Sprintf("%s_%d", id, i) + alt, ok = p.index[id1] + } + p.index[id1] = sub + p.slice = append(p.slice, Placeholder{ + ID: id1, + String: sub, + Type: arg.Type, + UnderlyingType: arg.UnderlyingType, + ArgNum: arg.ArgNum, + Expr: arg.Expr, + Comment: arg.Comment, + }) + return id1 +} + +func getLastComponent(s string) string { + return s[1+strings.LastIndexByte(s, '.'):] +} + +func msgStr(info *loader.PackageInfo, e ast.Expr) (s string, ok bool) { + v := info.Types[e].Value + if v == nil || v.Kind() != constant.String { + return "", false + } + s = constant.StringVal(v) + // Only record strings with letters. + for _, r := range s { + if unicode.In(r, unicode.L) { + return s, true + } + } + return "", false +}
diff --git a/message/pipeline/generate.go b/message/pipeline/generate.go new file mode 100644 index 0000000..0293273 --- /dev/null +++ b/message/pipeline/generate.go
@@ -0,0 +1,322 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "text/template" + + "golang.org/x/text/collate" + "golang.org/x/text/feature/plural" + "golang.org/x/text/internal" + "golang.org/x/text/internal/catmsg" + "golang.org/x/text/internal/gen" + "golang.org/x/text/language" + "golang.org/x/tools/go/loader" +) + +func init() { + out = cmdGenerate.Flag.String("out", "", "output file to write to") +} + +var ( + out *string +) + +var cmdGenerate = &Command{ + Run: runGenerate, + UsageLine: "generate <package>", + Short: "generates code to insert translated messages", +} + +var transRe = regexp.MustCompile(`messages\.(.*)\.json`) + +func runGenerate(cmd *Command, args []string) error { + prog, err := loadPackages(&loader.Config{}, args) + if err != nil { + return wrap(err, "could not load package") + } + + pkgs := prog.InitialPackages() + if len(pkgs) != 1 { + return fmt.Errorf("more than one package selected: %v", pkgs) + } + + // TODO: add in external input. Right now we assume that all files are + // manually created and stored in the textdata directory. + + // Build up index of translations and original messages. + extracted := Locale{} + translations := map[language.Tag]map[string]Message{} + languages := []language.Tag{} + langVars := []string{} + usedKeys := map[string]int{} + + err = filepath.Walk(*dir, func(path string, f os.FileInfo, err error) error { + if err != nil { + return wrap(err, "loading data") + } + if f.IsDir() { + return nil + } + if f.Name() == extractFile { + b, err := ioutil.ReadFile(path) + if err != nil { + return wrap(err, "read file failed") + } + if err := json.Unmarshal(b, &extracted); err != nil { + return wrap(err, "unmarshal source failed") + } + return nil + } + if f.Name() == outFile { + return nil + } + if !strings.HasSuffix(path, gotextSuffix) { + return nil + } + b, err := ioutil.ReadFile(path) + if err != nil { + return wrap(err, "read file failed") + } + var locale Locale + if err := json.Unmarshal(b, &locale); err != nil { + return wrap(err, "parsing translation file failed") + } + tag := locale.Language + if _, ok := translations[tag]; !ok { + translations[tag] = map[string]Message{} + languages = append(languages, tag) + } + for _, m := range locale.Messages { + if !m.Translation.IsEmpty() { + for _, id := range m.ID { + if _, ok := translations[tag][id]; ok { + logf("Duplicate translation in locale %q for message %q", tag, id) + } + translations[tag][id] = m + } + } + } + return nil + }) + if err != nil { + return err + } + + // Verify completeness and register keys. + internal.SortTags(languages) + + for _, tag := range languages { + langVars = append(langVars, strings.Replace(tag.String(), "-", "_", -1)) + dict := translations[tag] + for _, msg := range extracted.Messages { + for _, id := range msg.ID { + if trans, ok := dict[id]; ok && !trans.Translation.IsEmpty() { + if _, ok := usedKeys[msg.Key]; !ok { + usedKeys[msg.Key] = len(usedKeys) + } + break + } + // TODO: log missing entry. + logf("%s: Missing entry for %q.", tag, id) + } + } + } + + w := gen.NewCodeWriter() + + x := &struct { + Fallback language.Tag + Languages []string + }{ + Fallback: extracted.Language, + Languages: langVars, + } + + if err := lookup.Execute(w, x); err != nil { + return wrap(err, "error") + } + + keyToIndex := []string{} + for k := range usedKeys { + keyToIndex = append(keyToIndex, k) + } + sort.Strings(keyToIndex) + fmt.Fprint(w, "var messageKeyToIndex = map[string]int{\n") + for _, k := range keyToIndex { + fmt.Fprintf(w, "%q: %d,\n", k, usedKeys[k]) + } + fmt.Fprint(w, "}\n\n") + + for i, tag := range languages { + dict := translations[tag] + a := make([]string, len(usedKeys)) + for _, msg := range extracted.Messages { + for _, id := range msg.ID { + if trans, ok := dict[id]; ok && !trans.Translation.IsEmpty() { + m, err := assemble(&msg, &trans.Translation) + if err != nil { + return wrap(err, "error") + } + // TODO: support macros. + data, err := catmsg.Compile(tag, nil, m) + if err != nil { + return wrap(err, "error") + } + key := usedKeys[msg.Key] + if d := a[key]; d != "" && d != data { + logf("Duplicate non-consistent translation for key %q, picking the one for message %q", msg.Key, id) + } + a[key] = string(data) + break + } + } + } + index := []uint32{0} + p := 0 + for _, s := range a { + p += len(s) + index = append(index, uint32(p)) + } + + w.WriteVar(langVars[i]+"Index", index) + w.WriteConst(langVars[i]+"Data", strings.Join(a, "")) + } + + pkg := pkgs[0].Pkg.Name() + if *out == "" { + if _, err = w.WriteGo(os.Stdout, pkg); err != nil { + return wrap(err, "could not write go file") + } + } else { + w.WriteGoFile(*out, pkg) + } + + return nil +} + +func assemble(m *Message, t *Text) (msg catmsg.Message, err error) { + keys := []string{} + for k := range t.Var { + keys = append(keys, k) + } + sort.Strings(keys) + var a []catmsg.Message + for _, k := range keys { + t := t.Var[k] + m, err := assemble(m, &t) + if err != nil { + return nil, err + } + a = append(a, &catmsg.Var{Name: k, Message: m}) + } + if t.Select != nil { + s, err := assembleSelect(m, t.Select) + if err != nil { + return nil, err + } + a = append(a, s) + } + if t.Msg != "" { + sub, err := m.Substitute(t.Msg) + if err != nil { + return nil, err + } + a = append(a, catmsg.String(sub)) + } + switch len(a) { + case 0: + return nil, errorf("generate: empty message") + case 1: + return a[0], nil + default: + return catmsg.FirstOf(a), nil + + } +} + +func assembleSelect(m *Message, s *Select) (msg catmsg.Message, err error) { + cases := []string{} + for c := range s.Cases { + cases = append(cases, c) + } + sortCases(cases) + + caseMsg := []interface{}{} + for _, c := range cases { + cm := s.Cases[c] + m, err := assemble(m, &cm) + if err != nil { + return nil, err + } + caseMsg = append(caseMsg, c, m) + } + + ph := m.Placeholder(s.Arg) + + switch s.Feature { + case "plural": + // TODO: only printf-style selects are supported as of yet. + return plural.Selectf(ph.ArgNum, ph.String, caseMsg...), nil + } + return nil, errorf("unknown feature type %q", s.Feature) +} + +func sortCases(cases []string) { + // TODO: implement full interface. + sort.Slice(cases, func(i, j int) bool { + if cases[j] == "other" && cases[i] != "other" { + return true + } + // the following code relies on '<' < '=' < any letter. + return cmpNumeric(cases[i], cases[j]) == -1 + }) +} + +var cmpNumeric = collate.New(language.Und, collate.Numeric).CompareString + +var lookup = template.Must(template.New("gen").Parse(` +import ( + "golang.org/x/text/language" + "golang.org/x/text/message" + "golang.org/x/text/message/catalog" +) + +type dictionary struct { + index []uint32 + data string +} + +func (d *dictionary) Lookup(key string) (data string, ok bool) { + p := messageKeyToIndex[key] + start, end := d.index[p], d.index[p+1] + if start == end { + return "", false + } + return d.data[start:end], true +} + +func init() { + dict := map[string]catalog.Dictionary{ + {{range .Languages}}"{{.}}": &dictionary{index: {{.}}Index, data: {{.}}Data }, + {{end}} + } + fallback := language.MustParse("{{.Fallback}}") + cat, err := catalog.NewFromMap(dict, catalog.Fallback(fallback)) + if err != nil { + panic(err) + } + message.DefaultCatalog = cat +} + +`))
diff --git a/message/pipeline/main.go b/message/pipeline/main.go new file mode 100644 index 0000000..f3f50d7 --- /dev/null +++ b/message/pipeline/main.go
@@ -0,0 +1,352 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:generate go build -o gotext.latest +//go:generate ./gotext.latest help gendocumentation +//go:generate rm gotext.latest + +package main + +import ( + "bufio" + "bytes" + "flag" + "fmt" + "go/build" + "go/format" + "io" + "io/ioutil" + "log" + "os" + "strings" + "sync" + "text/template" + "unicode" + "unicode/utf8" + + "golang.org/x/text/language" + "golang.org/x/tools/go/buildutil" +) + +func init() { + flag.Var((*buildutil.TagsFlag)(&build.Default.BuildTags), "tags", buildutil.TagsFlagDoc) +} + +var dir = flag.String("dir", "locales", "default subdirectory to store translation files") + +// NOTE: the Command struct is copied from the go tool in core. + +// A Command is an implementation of a go command +// like go build or go fix. +type Command struct { + // Run runs the command. + // The args are the arguments after the command name. + Run func(cmd *Command, args []string) error + + // UsageLine is the one-line usage message. + // The first word in the line is taken to be the command name. + UsageLine string + + // Short is the short description shown in the 'go help' output. + Short string + + // Long is the long message shown in the 'go help <this-command>' output. + Long string + + // Flag is a set of flags specific to this command. + Flag flag.FlagSet +} + +// Name returns the command's name: the first word in the usage line. +func (c *Command) Name() string { + name := c.UsageLine + i := strings.Index(name, " ") + if i >= 0 { + name = name[:i] + } + return name +} + +func (c *Command) Usage() { + fmt.Fprintf(os.Stderr, "usage: %s\n\n", c.UsageLine) + fmt.Fprintf(os.Stderr, "%s\n", strings.TrimSpace(c.Long)) + os.Exit(2) +} + +// Runnable reports whether the command can be run; otherwise +// it is a documentation pseudo-command such as importpath. +func (c *Command) Runnable() bool { + return c.Run != nil +} + +// Commands lists the available commands and help topics. +// The order here is the order in which they are printed by 'go help'. +var commands = []*Command{ + cmdExtract, + cmdRewrite, + cmdGenerate, + // TODO: + // - update: full-cycle update of extraction, sending, and integration + // - report: report of freshness of translations +} + +var exitStatus = 0 +var exitMu sync.Mutex + +func setExitStatus(n int) { + exitMu.Lock() + if exitStatus < n { + exitStatus = n + } + exitMu.Unlock() +} + +var origEnv []string + +func main() { + flag.Usage = usage + flag.Parse() + log.SetFlags(0) + + args := flag.Args() + if len(args) < 1 { + usage() + } + + if args[0] == "help" { + help(args[1:]) + return + } + + for _, cmd := range commands { + if cmd.Name() == args[0] && cmd.Runnable() { + cmd.Flag.Usage = func() { cmd.Usage() } + cmd.Flag.Parse(args[1:]) + args = cmd.Flag.Args() + if err := cmd.Run(cmd, args); err != nil { + fatalf("gotext: %+v", err) + } + exit() + return + } + } + + fmt.Fprintf(os.Stderr, "gotext: unknown subcommand %q\nRun 'go help' for usage.\n", args[0]) + setExitStatus(2) + exit() +} + +var usageTemplate = `gotext is a tool for managing text in Go source code. + +Usage: + + gotext command [arguments] + +The commands are: +{{range .}}{{if .Runnable}} + {{.Name | printf "%-11s"}} {{.Short}}{{end}}{{end}} + +Use "go help [command]" for more information about a command. + +Additional help topics: +{{range .}}{{if not .Runnable}} + {{.Name | printf "%-11s"}} {{.Short}}{{end}}{{end}} + +Use "gotext help [topic]" for more information about that topic. + +` + +var helpTemplate = `{{if .Runnable}}usage: go {{.UsageLine}} + +{{end}}{{.Long | trim}} +` + +var documentationTemplate = `{{range .}}{{if .Short}}{{.Short | capitalize}} + +{{end}}{{if .Runnable}}Usage: + + go {{.UsageLine}} + +{{end}}{{.Long | trim}} + + +{{end}}` + +// commentWriter writes a Go comment to the underlying io.Writer, +// using line comment form (//). +type commentWriter struct { + W io.Writer + wroteSlashes bool // Wrote "//" at the beginning of the current line. +} + +func (c *commentWriter) Write(p []byte) (int, error) { + var n int + for i, b := range p { + if !c.wroteSlashes { + s := "//" + if b != '\n' { + s = "// " + } + if _, err := io.WriteString(c.W, s); err != nil { + return n, err + } + c.wroteSlashes = true + } + n0, err := c.W.Write(p[i : i+1]) + n += n0 + if err != nil { + return n, err + } + if b == '\n' { + c.wroteSlashes = false + } + } + return len(p), nil +} + +// An errWriter wraps a writer, recording whether a write error occurred. +type errWriter struct { + w io.Writer + err error +} + +func (w *errWriter) Write(b []byte) (int, error) { + n, err := w.w.Write(b) + if err != nil { + w.err = err + } + return n, err +} + +// tmpl executes the given template text on data, writing the result to w. +func tmpl(w io.Writer, text string, data interface{}) { + t := template.New("top") + t.Funcs(template.FuncMap{"trim": strings.TrimSpace, "capitalize": capitalize}) + template.Must(t.Parse(text)) + ew := &errWriter{w: w} + err := t.Execute(ew, data) + if ew.err != nil { + // I/O error writing. Ignore write on closed pipe. + if strings.Contains(ew.err.Error(), "pipe") { + os.Exit(1) + } + fatalf("writing output: %v", ew.err) + } + if err != nil { + panic(err) + } +} + +func capitalize(s string) string { + if s == "" { + return s + } + r, n := utf8.DecodeRuneInString(s) + return string(unicode.ToTitle(r)) + s[n:] +} + +func printUsage(w io.Writer) { + bw := bufio.NewWriter(w) + tmpl(bw, usageTemplate, commands) + bw.Flush() +} + +func usage() { + printUsage(os.Stderr) + os.Exit(2) +} + +// help implements the 'help' command. +func help(args []string) { + if len(args) == 0 { + printUsage(os.Stdout) + // not exit 2: succeeded at 'go help'. + return + } + if len(args) != 1 { + fmt.Fprintf(os.Stderr, "usage: go help command\n\nToo many arguments given.\n") + os.Exit(2) // failed at 'go help' + } + + arg := args[0] + + // 'go help documentation' generates doc.go. + if strings.HasSuffix(arg, "documentation") { + w := &bytes.Buffer{} + + fmt.Fprintln(w, "// Code generated by go generate. DO NOT EDIT.") + fmt.Fprintln(w) + buf := new(bytes.Buffer) + printUsage(buf) + usage := &Command{Long: buf.String()} + tmpl(&commentWriter{W: w}, documentationTemplate, append([]*Command{usage}, commands...)) + fmt.Fprintln(w, "package main") + if arg == "gendocumentation" { + b, err := format.Source(w.Bytes()) + if err != nil { + logf("Could not format generated docs: %v\n", err) + } + if err := ioutil.WriteFile("doc.go", b, 0666); err != nil { + logf("Could not create file alldocs.go: %v\n", err) + } + } else { + fmt.Println(w.String()) + } + return + } + + for _, cmd := range commands { + if cmd.Name() == arg { + tmpl(os.Stdout, helpTemplate, cmd) + // not exit 2: succeeded at 'go help cmd'. + return + } + } + + fmt.Fprintf(os.Stderr, "Unknown help topic %#q. Run 'go help'.\n", arg) + os.Exit(2) // failed at 'go help cmd' +} + +func getLangs() (tags []language.Tag) { + for _, t := range strings.Split(*lang, ",") { + if t == "" { + continue + } + tag, err := language.Parse(t) + if err != nil { + fatalf("gotext: could not parse language %q: %v", t, err) + } + tags = append(tags, tag) + } + return tags +} + +var atexitFuncs []func() + +func atexit(f func()) { + atexitFuncs = append(atexitFuncs, f) +} + +func exit() { + for _, f := range atexitFuncs { + f() + } + os.Exit(exitStatus) +} + +func fatalf(format string, args ...interface{}) { + logf(format, args...) + exit() +} + +func logf(format string, args ...interface{}) { + log.Printf(format, args...) + setExitStatus(1) +} + +func exitIfErrors() { + if exitStatus != 0 { + exit() + } +}
diff --git a/message/pipeline/message.go b/message/pipeline/message.go new file mode 100644 index 0000000..990ba52 --- /dev/null +++ b/message/pipeline/message.go
@@ -0,0 +1,222 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "encoding/json" + "strings" + + "golang.org/x/text/language" +) + +// TODO: these definitions should be moved to a package so that the can be used +// by other tools. + +// The file contains the structures used to define translations of a certain +// messages. +// +// A translation may have multiple translations strings, or messages, depending +// on the feature values of the various arguments. For instance, consider +// a hypothetical translation from English to English, where the source defines +// the format string "%d file(s) remaining". +// See the examples directory for examples of extracted messages. + +// A Locale is used to store all information for a single locale. This type is +// used both for extraction and injection. +type Locale struct { + Language language.Tag `json:"language"` + Messages []Message `json:"messages"` + Macros map[string]Text `json:"macros,omitempty"` +} + +// A Message describes a message to be translated. +type Message struct { + // ID contains a list of identifiers for the message. + ID IDList `json:"id"` + // Key is the string that is used to look up the message at runtime. + Key string `json:"key"` + Meaning string `json:"meaning,omitempty"` + Message Text `json:"message"` + Translation Text `json:"translation"` + + Comment string `json:"comment,omitempty"` + TranslatorComment string `json:"translatorComment,omitempty"` + + Placeholders []Placeholder `json:"placeholders,omitempty"` + + // TODO: default placeholder syntax is {foo}. Allow alternative escaping + // like `foo`. + + // Extraction information. + Position string `json:"position,omitempty"` // filePosition:line +} + +// Placeholder reports the placeholder for the given ID if it is defined or nil +// otherwise. +func (m *Message) Placeholder(id string) *Placeholder { + for _, p := range m.Placeholders { + if p.ID == id { + return &p + } + } + return nil +} + +// Substitute replaces placeholders in msg with their original value. +func (m *Message) Substitute(msg string) (sub string, err error) { + last := 0 + for i := 0; i < len(msg); { + pLeft := strings.IndexByte(msg[i:], '{') + if pLeft == -1 { + break + } + pLeft += i + pRight := strings.IndexByte(msg[pLeft:], '}') + if pRight == -1 { + return "", errorf("unmatched '}'") + } + pRight += pLeft + id := strings.TrimSpace(msg[pLeft+1 : pRight]) + i = pRight + 1 + if id != "" && id[0] == '$' { + continue + } + sub += msg[last:pLeft] + last = i + ph := m.Placeholder(id) + if ph == nil { + return "", errorf("unknown placeholder %q in message %q", id, msg) + } + sub += ph.String + } + sub += msg[last:] + return sub, err +} + +// A Placeholder is a part of the message that should not be changed by a +// translator. It can be used to hide or prettify format strings (e.g. %d or +// {{.Count}}), hide HTML, or mark common names that should not be translated. +type Placeholder struct { + // ID is the placeholder identifier without the curly braces. + ID string `json:"id"` + + // String is the string with which to replace the placeholder. This may be a + // formatting string (for instance "%d" or "{{.Count}}") or a literal string + // (<div>). + String string `json:"string"` + + Type string `json:"type"` + UnderlyingType string `json:"underlyingType"` + // ArgNum and Expr are set if the placeholder is a substitution of an + // argument. + ArgNum int `json:"argNum,omitempty"` + Expr string `json:"expr,omitempty"` + + Comment string `json:"comment,omitempty"` + Example string `json:"example,omitempty"` + + // Features contains the features that are available for the implementation + // of this argument. + Features []Feature `json:"features,omitempty"` +} + +// An argument contains information about the arguments passed to a message. +type argument struct { + // ArgNum corresponds to the number that should be used for explicit argument indexes (e.g. + // "%[1]d"). + ArgNum int `json:"argNum,omitempty"` + + used bool // Used by Placeholder + Type string `json:"type"` + UnderlyingType string `json:"underlyingType"` + Expr string `json:"expr"` + Value string `json:"value,omitempty"` + Comment string `json:"comment,omitempty"` + Position string `json:"position,omitempty"` +} + +// Feature holds information about a feature that can be implemented by +// an Argument. +type Feature struct { + Type string `json:"type"` // Right now this is only gender and plural. + + // TODO: possible values and examples for the language under consideration. + +} + +// Text defines a message to be displayed. +type Text struct { + // Msg and Select contains the message to be displayed. Msg may be used as + // a fallback value if none of the select cases match. + Msg string `json:"msg,omitempty"` + Select *Select `json:"select,omitempty"` + + // Var defines a map of variables that may be substituted in the selected + // message. + Var map[string]Text `json:"var,omitempty"` + + // Example contains an example message formatted with default values. + Example string `json:"example,omitempty"` +} + +// IsEmpty reports whether this Text can generate anything. +func (t *Text) IsEmpty() bool { + return t.Msg == "" && t.Select == nil && t.Var == nil +} + +// rawText erases the UnmarshalJSON method. +type rawText Text + +// UnmarshalJSON implements json.Unmarshaler. +func (t *Text) UnmarshalJSON(b []byte) error { + if b[0] == '"' { + return json.Unmarshal(b, &t.Msg) + } + return json.Unmarshal(b, (*rawText)(t)) +} + +// MarshalJSON implements json.Marshaler. +func (t *Text) MarshalJSON() ([]byte, error) { + if t.Select == nil && t.Var == nil && t.Example == "" { + return json.Marshal(t.Msg) + } + return json.Marshal((*rawText)(t)) +} + +// IDList is a set identifiers that each may refer to possibly different +// versions of the same message. When looking up a messages, the first +// identifier in the list takes precedence. +type IDList []string + +// UnmarshalJSON implements json.Unmarshaler. +func (id *IDList) UnmarshalJSON(b []byte) error { + if b[0] == '"' { + *id = []string{""} + return json.Unmarshal(b, &((*id)[0])) + } + return json.Unmarshal(b, (*[]string)(id)) +} + +// MarshalJSON implements json.Marshaler. +func (id *IDList) MarshalJSON() ([]byte, error) { + if len(*id) == 1 { + return json.Marshal((*id)[0]) + } + return json.Marshal((*[]string)(id)) +} + +// Select selects a Text based on the feature value associated with a feature of +// a certain argument. +type Select struct { + Feature string `json:"feature"` // Name of Feature type (e.g plural) + Arg string `json:"arg"` // The placeholder ID + Cases map[string]Text `json:"cases"` +} + +// TODO: order matters, but can we derive the ordering from the case keys? +// type Case struct { +// Key string `json:"key"` +// Value Text `json:"value"` +// }
diff --git a/message/pipeline/rewrite.go b/message/pipeline/rewrite.go new file mode 100644 index 0000000..f4641c8 --- /dev/null +++ b/message/pipeline/rewrite.go
@@ -0,0 +1,287 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "bytes" + "fmt" + "go/ast" + "go/constant" + "go/format" + "go/token" + "os" + "strings" + + "golang.org/x/tools/go/loader" +) + +const printerType = "golang.org/x/text/message.Printer" + +// TODO: +// - merge information into existing files +// - handle different file formats (PO, XLIFF) +// - handle features (gender, plural) +// - message rewriting + +func init() { + overwrite = cmdRewrite.Flag.Bool("w", false, "write files in place") +} + +var ( + overwrite *bool +) + +var cmdRewrite = &Command{ + Run: runRewrite, + UsageLine: "rewrite <package>", + Short: "rewrites fmt functions to use a message Printer", + Long: ` +rewrite is typically done once for a project. It rewrites all usages of +fmt to use x/text's message package whenever a message.Printer is in scope. +It rewrites Print and Println calls with constant strings to the equivalent +using Printf to allow translators to reorder arguments. +`, +} + +func runRewrite(cmd *Command, args []string) error { + conf := &loader.Config{ + AllowErrors: true, // Allow unused instances of message.Printer. + } + prog, err := loadPackages(conf, args) + if err != nil { + return wrap(err, "") + } + + for _, info := range prog.InitialPackages() { + for _, f := range info.Files { + // Associate comments with nodes. + + // Pick up initialized Printers at the package level. + r := rewriter{info: info, conf: conf} + for _, n := range info.InitOrder { + if t := r.info.Types[n.Rhs].Type.String(); strings.HasSuffix(t, printerType) { + r.printerVar = n.Lhs[0].Name() + } + } + + ast.Walk(&r, f) + + w := os.Stdout + if *overwrite { + var err error + if w, err = os.Create(conf.Fset.File(f.Pos()).Name()); err != nil { + return wrap(err, "open failed") + } + } + + if err := format.Node(w, conf.Fset, f); err != nil { + return wrap(err, "go format failed") + } + } + } + + return nil +} + +type rewriter struct { + info *loader.PackageInfo + conf *loader.Config + printerVar string +} + +// print returns Go syntax for the specified node. +func (r *rewriter) print(n ast.Node) string { + var buf bytes.Buffer + format.Node(&buf, r.conf.Fset, n) + return buf.String() +} + +func (r *rewriter) Visit(n ast.Node) ast.Visitor { + // Save the state by scope. + if _, ok := n.(*ast.BlockStmt); ok { + r := *r + return &r + } + // Find Printers created by assignment. + stmt, ok := n.(*ast.AssignStmt) + if ok { + for _, v := range stmt.Lhs { + if r.printerVar == r.print(v) { + r.printerVar = "" + } + } + for i, v := range stmt.Rhs { + if t := r.info.Types[v].Type.String(); strings.HasSuffix(t, printerType) { + r.printerVar = r.print(stmt.Lhs[i]) + return r + } + } + } + // Find Printers created by variable declaration. + spec, ok := n.(*ast.ValueSpec) + if ok { + for _, v := range spec.Names { + if r.printerVar == r.print(v) { + r.printerVar = "" + } + } + for i, v := range spec.Values { + if t := r.info.Types[v].Type.String(); strings.HasSuffix(t, printerType) { + r.printerVar = r.print(spec.Names[i]) + return r + } + } + } + if r.printerVar == "" { + return r + } + call, ok := n.(*ast.CallExpr) + if !ok { + return r + } + + // TODO: Handle literal values? + sel, ok := call.Fun.(*ast.SelectorExpr) + if !ok { + return r + } + meth := r.info.Selections[sel] + + source := r.print(sel.X) + fun := r.print(sel.Sel) + if meth != nil { + source = meth.Recv().String() + fun = meth.Obj().Name() + } + + // TODO: remove cheap hack and check if the type either + // implements some interface or is specifically of type + // "golang.org/x/text/message".Printer. + m, ok := rewriteFuncs[source] + if !ok { + return r + } + + rewriteType, ok := m[fun] + if !ok { + return r + } + ident := ast.NewIdent(r.printerVar) + ident.NamePos = sel.X.Pos() + sel.X = ident + if rewriteType.method != "" { + sel.Sel.Name = rewriteType.method + } + + // Analyze arguments. + argn := rewriteType.arg + if rewriteType.format || argn >= len(call.Args) { + return r + } + hasConst := false + for _, a := range call.Args[argn:] { + if v := r.info.Types[a].Value; v != nil && v.Kind() == constant.String { + hasConst = true + break + } + } + if !hasConst { + return r + } + sel.Sel.Name = rewriteType.methodf + + // We are done if there is only a single string that does not need to be + // escaped. + if len(call.Args) == 1 { + s, ok := constStr(r.info, call.Args[0]) + if ok && !strings.Contains(s, "%") && !rewriteType.newLine { + return r + } + } + + // Rewrite arguments as format string. + expr := &ast.BasicLit{ + ValuePos: call.Lparen, + Kind: token.STRING, + } + newArgs := append(call.Args[:argn:argn], expr) + newStr := []string{} + for i, a := range call.Args[argn:] { + if s, ok := constStr(r.info, a); ok { + newStr = append(newStr, strings.Replace(s, "%", "%%", -1)) + } else { + newStr = append(newStr, "%v") + newArgs = append(newArgs, call.Args[argn+i]) + } + } + s := strings.Join(newStr, rewriteType.sep) + if rewriteType.newLine { + s += "\n" + } + expr.Value = fmt.Sprintf("%q", s) + + call.Args = newArgs + + // TODO: consider creating an expression instead of a constant string and + // then wrapping it in an escape function or so: + // call.Args[argn+i] = &ast.CallExpr{ + // Fun: &ast.SelectorExpr{ + // X: ast.NewIdent("message"), + // Sel: ast.NewIdent("Lookup"), + // }, + // Args: []ast.Expr{a}, + // } + // } + + return r +} + +type rewriteType struct { + // method is the name of the equivalent method on a printer, or "" if it is + // the same. + method string + + // methodf is the method to use if the arguments can be rewritten as a + // arguments to a printf-style call. + methodf string + + // format is true if the method takes a formatting string followed by + // substitution arguments. + format bool + + // arg indicates the position of the argument to extract. If all is + // positive, all arguments from this argument onwards needs to be extracted. + arg int + + sep string + newLine bool +} + +// rewriteFuncs list functions that can be directly mapped to the printer +// functions of the message package. +var rewriteFuncs = map[string]map[string]rewriteType{ + // TODO: Printer -> *golang.org/x/text/message.Printer + "fmt": { + "Print": rewriteType{methodf: "Printf"}, + "Sprint": rewriteType{methodf: "Sprintf"}, + "Fprint": rewriteType{methodf: "Fprintf"}, + + "Println": rewriteType{methodf: "Printf", sep: " ", newLine: true}, + "Sprintln": rewriteType{methodf: "Sprintf", sep: " ", newLine: true}, + "Fprintln": rewriteType{methodf: "Fprintf", sep: " ", newLine: true}, + + "Printf": rewriteType{method: "Printf", format: true}, + "Sprintf": rewriteType{method: "Sprintf", format: true}, + "Fprintf": rewriteType{method: "Fprintf", format: true}, + }, +} + +func constStr(info *loader.PackageInfo, e ast.Expr) (s string, ok bool) { + v := info.Types[e].Value + if v == nil || v.Kind() != constant.String { + return "", false + } + return constant.StringVal(v), true +}