message/pipeline: converted to API
and changed cmd/gotext to use it.
Change-Id: I418957cfcbcad3acb2ebcd2f65c88a43e5e7f254
Reviewed-on: https://go-review.googlesource.com/82236
Run-TryBot: Marcel van Lohuizen <mpvl@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Nigel Tao <nigeltao@golang.org>
diff --git a/cmd/gotext/common.go b/cmd/gotext/common.go
index ae87e44..1036592 100644
--- a/cmd/gotext/common.go
+++ b/cmd/gotext/common.go
@@ -23,12 +23,10 @@
wrap = func(err error, msg string) error {
return fmt.Errorf("%s: %v", msg, err)
}
- wrapf = func(err error, msg string, args ...interface{}) error {
- return wrap(err, fmt.Sprintf(msg, args...))
- }
errorf = fmt.Errorf
)
+// TODO: still used. Remove when possible.
func loadPackages(conf *loader.Config, args []string) (*loader.Program, error) {
if len(args) == 0 {
args = []string{"."}
diff --git a/cmd/gotext/extract.go b/cmd/gotext/extract.go
index 561f050..f1f700b 100644
--- a/cmd/gotext/extract.go
+++ b/cmd/gotext/extract.go
@@ -5,26 +5,14 @@
package main
import (
- "bytes"
"encoding/json"
- "fmt"
- "go/ast"
- "go/constant"
- "go/format"
- "go/token"
- "go/types"
"io/ioutil"
"os"
- "path"
"path/filepath"
- "strings"
- "unicode"
- "unicode/utf8"
"golang.org/x/text/internal"
- fmtparser "golang.org/x/text/internal/format"
"golang.org/x/text/language"
- "golang.org/x/tools/go/loader"
+ "golang.org/x/text/message/pipeline"
)
// TODO:
@@ -50,173 +38,16 @@
}
func runExtract(cmd *Command, args []string) error {
- conf := loader.Config{}
- prog, err := loadPackages(&conf, args)
- if err != nil {
- return wrap(err, "")
- }
-
- // print returns Go syntax for the specified node.
- print := func(n ast.Node) string {
- var buf bytes.Buffer
- format.Node(&buf, conf.Fset, n)
- return buf.String()
- }
-
- var messages []Message
-
- for _, info := range prog.AllPackages {
- for _, f := range info.Files {
- // Associate comments with nodes.
- cmap := ast.NewCommentMap(prog.Fset, f, f.Comments)
- getComment := func(n ast.Node) string {
- cs := cmap.Filter(n).Comments()
- if len(cs) > 0 {
- return strings.TrimSpace(cs[0].Text())
- }
- return ""
- }
-
- // Find function calls.
- ast.Inspect(f, func(n ast.Node) bool {
- call, ok := n.(*ast.CallExpr)
- if !ok {
- return true
- }
-
- // Skip calls of functions other than
- // (*message.Printer).{Sp,Fp,P}rintf.
- sel, ok := call.Fun.(*ast.SelectorExpr)
- if !ok {
- return true
- }
- meth := info.Selections[sel]
- if meth == nil || meth.Kind() != types.MethodVal {
- return true
- }
- // TODO: remove cheap hack and check if the type either
- // implements some interface or is specifically of type
- // "golang.org/x/text/message".Printer.
- m, ok := extractFuncs[path.Base(meth.Recv().String())]
- if !ok {
- return true
- }
-
- fmtType, ok := m[meth.Obj().Name()]
- if !ok {
- return true
- }
- // argn is the index of the format string.
- argn := fmtType.arg
- if argn >= len(call.Args) {
- return true
- }
-
- args := call.Args[fmtType.arg:]
-
- fmtMsg, ok := msgStr(info, args[0])
- if !ok {
- // TODO: identify the type of the format argument. If it
- // is not a string, multiple keys may be defined.
- return true
- }
- comment := ""
- key := []string{}
- if ident, ok := args[0].(*ast.Ident); ok {
- key = append(key, ident.Name)
- if v, ok := ident.Obj.Decl.(*ast.ValueSpec); ok && v.Comment != nil {
- // TODO: get comment above ValueSpec as well
- comment = v.Comment.Text()
- }
- }
-
- arguments := []argument{}
- args = args[1:]
- simArgs := make([]interface{}, len(args))
- for i, arg := range args {
- expr := print(arg)
- val := ""
- if v := info.Types[arg].Value; v != nil {
- val = v.ExactString()
- simArgs[i] = val
- switch arg.(type) {
- case *ast.BinaryExpr, *ast.UnaryExpr:
- expr = val
- }
- }
- arguments = append(arguments, argument{
- ArgNum: i + 1,
- Type: info.Types[arg].Type.String(),
- UnderlyingType: info.Types[arg].Type.Underlying().String(),
- Expr: expr,
- Value: val,
- Comment: getComment(arg),
- Position: posString(conf, info, arg.Pos()),
- // TODO report whether it implements
- // interfaces plural.Interface,
- // gender.Interface.
- })
- }
- msg := ""
-
- ph := placeholders{index: map[string]string{}}
-
- p := fmtparser.Parser{}
- p.Reset(simArgs)
- for p.SetFormat(fmtMsg); p.Scan(); {
- switch p.Status {
- case fmtparser.StatusText:
- msg += p.Text()
- case fmtparser.StatusSubstitution,
- fmtparser.StatusBadWidthSubstitution,
- fmtparser.StatusBadPrecSubstitution:
- arguments[p.ArgNum-1].used = true
- arg := arguments[p.ArgNum-1]
- sub := p.Text()
- if !p.HasIndex {
- r, sz := utf8.DecodeLastRuneInString(sub)
- sub = fmt.Sprintf("%s[%d]%c", sub[:len(sub)-sz], p.ArgNum, r)
- }
- msg += fmt.Sprintf("{%s}", ph.addArg(&arg, sub))
- }
- }
- key = append(key, msg)
-
- // Add additional Placeholders that can be used in translations
- // that are not present in the string.
- for _, arg := range arguments {
- if arg.used {
- continue
- }
- ph.addArg(&arg, fmt.Sprintf("%%[%d]v", arg.ArgNum))
- }
-
- if c := getComment(call.Args[0]); c != "" {
- comment = c
- }
-
- messages = append(messages, Message{
- ID: key,
- Key: fmtMsg,
- Message: Text{Msg: msg},
- // TODO(fix): this doesn't get the before comment.
- Comment: comment,
- Placeholders: ph.slice,
- Position: posString(conf, info, call.Lparen),
- })
- return true
- })
- }
- }
-
tag, err := language.Parse(*srcLang)
if err != nil {
return wrap(err, "")
}
- out := Locale{
- Language: tag,
- Messages: messages,
+ config := &pipeline.Config{
+ SourceLanguage: tag,
+ Packages: args,
}
+ out, err := pipeline.Extract(config)
+
data, err := json.MarshalIndent(out, "", " ")
if err != nil {
return wrap(err, "")
@@ -226,7 +57,7 @@
// cycle with a init once and update cycle.
file := filepath.Join(*dir, extractFile)
if err := ioutil.WriteFile(file, data, 0644); err != nil {
- return wrapf(err, "could not create file")
+ return wrap(err, "could not create file")
}
langs := append(getLangs(), tag)
@@ -248,111 +79,3 @@
}
return nil
}
-
-func posString(conf loader.Config, info *loader.PackageInfo, pos token.Pos) string {
- p := conf.Fset.Position(pos)
- file := fmt.Sprintf("%s:%d:%d", filepath.Base(p.Filename), p.Line, p.Column)
- return filepath.Join(info.Pkg.Path(), file)
-}
-
-// extractFuncs indicates the types and methods for which to extract strings,
-// and which argument to extract.
-// TODO: use the types in conf.Import("golang.org/x/text/message") to extract
-// the correct instances.
-var extractFuncs = map[string]map[string]extractType{
- // TODO: Printer -> *golang.org/x/text/message.Printer
- "message.Printer": {
- "Printf": extractType{arg: 0, format: true},
- "Sprintf": extractType{arg: 0, format: true},
- "Fprintf": extractType{arg: 1, format: true},
-
- "Lookup": extractType{arg: 0},
- },
-}
-
-type extractType struct {
- // format indicates if the next arg is a formatted string or whether to
- // concatenate all arguments
- format bool
- // arg indicates the position of the argument to extract.
- arg int
-}
-
-func getID(arg *argument) string {
- s := getLastComponent(arg.Expr)
- s = strip(s)
- s = strings.Replace(s, " ", "", -1)
- // For small variable names, use user-defined types for more info.
- if len(s) <= 2 && arg.UnderlyingType != arg.Type {
- s = getLastComponent(arg.Type)
- }
- return strings.Title(s)
-}
-
-// strip is a dirty hack to convert function calls to placeholder IDs.
-func strip(s string) string {
- s = strings.Map(func(r rune) rune {
- if unicode.IsSpace(r) || r == '-' {
- return '_'
- }
- if !unicode.In(r, unicode.Letter, unicode.Mark) {
- return -1
- }
- return r
- }, s)
- // Strip "Get" from getter functions.
- if strings.HasPrefix(s, "Get") || strings.HasPrefix(s, "get") {
- if len(s) > len("get") {
- r, _ := utf8.DecodeRuneInString(s)
- if !unicode.In(r, unicode.Ll, unicode.M) { // not lower or mark
- s = s[len("get"):]
- }
- }
- }
- return s
-}
-
-type placeholders struct {
- index map[string]string
- slice []Placeholder
-}
-
-func (p *placeholders) addArg(arg *argument, sub string) (id string) {
- id = getID(arg)
- id1 := id
- alt, ok := p.index[id1]
- for i := 1; ok && alt != sub; i++ {
- id1 = fmt.Sprintf("%s_%d", id, i)
- alt, ok = p.index[id1]
- }
- p.index[id1] = sub
- p.slice = append(p.slice, Placeholder{
- ID: id1,
- String: sub,
- Type: arg.Type,
- UnderlyingType: arg.UnderlyingType,
- ArgNum: arg.ArgNum,
- Expr: arg.Expr,
- Comment: arg.Comment,
- })
- return id1
-}
-
-func getLastComponent(s string) string {
- return s[1+strings.LastIndexByte(s, '.'):]
-}
-
-func msgStr(info *loader.PackageInfo, e ast.Expr) (s string, ok bool) {
- v := info.Types[e].Value
- if v == nil || v.Kind() != constant.String {
- return "", false
- }
- s = constant.StringVal(v)
- // Only record strings with letters.
- for _, r := range s {
- if unicode.In(r, unicode.L) {
- return s, true
- }
- }
- return "", false
-}
diff --git a/cmd/gotext/generate.go b/cmd/gotext/generate.go
index 0293273..2d34465 100644
--- a/cmd/gotext/generate.go
+++ b/cmd/gotext/generate.go
@@ -11,16 +11,9 @@
"os"
"path/filepath"
"regexp"
- "sort"
"strings"
- "text/template"
- "golang.org/x/text/collate"
- "golang.org/x/text/feature/plural"
- "golang.org/x/text/internal"
- "golang.org/x/text/internal/catmsg"
- "golang.org/x/text/internal/gen"
- "golang.org/x/text/language"
+ "golang.org/x/text/message/pipeline"
"golang.org/x/tools/go/loader"
)
@@ -41,6 +34,7 @@
var transRe = regexp.MustCompile(`messages\.(.*)\.json`)
func runGenerate(cmd *Command, args []string) error {
+
prog, err := loadPackages(&loader.Config{}, args)
if err != nil {
return wrap(err, "could not load package")
@@ -50,16 +44,14 @@
if len(pkgs) != 1 {
return fmt.Errorf("more than one package selected: %v", pkgs)
}
+ pkg := pkgs[0].Pkg.Name()
// TODO: add in external input. Right now we assume that all files are
// manually created and stored in the textdata directory.
// Build up index of translations and original messages.
- extracted := Locale{}
- translations := map[language.Tag]map[string]Message{}
- languages := []language.Tag{}
- langVars := []string{}
- usedKeys := map[string]int{}
+ extracted := pipeline.Locale{}
+ translations := []*pipeline.Locale{}
err = filepath.Walk(*dir, func(path string, f os.FileInfo, err error) error {
if err != nil {
@@ -88,235 +80,25 @@
if err != nil {
return wrap(err, "read file failed")
}
- var locale Locale
+ var locale pipeline.Locale
if err := json.Unmarshal(b, &locale); err != nil {
return wrap(err, "parsing translation file failed")
}
- tag := locale.Language
- if _, ok := translations[tag]; !ok {
- translations[tag] = map[string]Message{}
- languages = append(languages, tag)
- }
- for _, m := range locale.Messages {
- if !m.Translation.IsEmpty() {
- for _, id := range m.ID {
- if _, ok := translations[tag][id]; ok {
- logf("Duplicate translation in locale %q for message %q", tag, id)
- }
- translations[tag][id] = m
- }
- }
- }
+ translations = append(translations, &locale)
return nil
})
if err != nil {
return err
}
- // Verify completeness and register keys.
- internal.SortTags(languages)
-
- for _, tag := range languages {
- langVars = append(langVars, strings.Replace(tag.String(), "-", "_", -1))
- dict := translations[tag]
- for _, msg := range extracted.Messages {
- for _, id := range msg.ID {
- if trans, ok := dict[id]; ok && !trans.Translation.IsEmpty() {
- if _, ok := usedKeys[msg.Key]; !ok {
- usedKeys[msg.Key] = len(usedKeys)
- }
- break
- }
- // TODO: log missing entry.
- logf("%s: Missing entry for %q.", tag, id)
- }
- }
- }
-
- w := gen.NewCodeWriter()
-
- x := &struct {
- Fallback language.Tag
- Languages []string
- }{
- Fallback: extracted.Language,
- Languages: langVars,
- }
-
- if err := lookup.Execute(w, x); err != nil {
- return wrap(err, "error")
- }
-
- keyToIndex := []string{}
- for k := range usedKeys {
- keyToIndex = append(keyToIndex, k)
- }
- sort.Strings(keyToIndex)
- fmt.Fprint(w, "var messageKeyToIndex = map[string]int{\n")
- for _, k := range keyToIndex {
- fmt.Fprintf(w, "%q: %d,\n", k, usedKeys[k])
- }
- fmt.Fprint(w, "}\n\n")
-
- for i, tag := range languages {
- dict := translations[tag]
- a := make([]string, len(usedKeys))
- for _, msg := range extracted.Messages {
- for _, id := range msg.ID {
- if trans, ok := dict[id]; ok && !trans.Translation.IsEmpty() {
- m, err := assemble(&msg, &trans.Translation)
- if err != nil {
- return wrap(err, "error")
- }
- // TODO: support macros.
- data, err := catmsg.Compile(tag, nil, m)
- if err != nil {
- return wrap(err, "error")
- }
- key := usedKeys[msg.Key]
- if d := a[key]; d != "" && d != data {
- logf("Duplicate non-consistent translation for key %q, picking the one for message %q", msg.Key, id)
- }
- a[key] = string(data)
- break
- }
- }
- }
- index := []uint32{0}
- p := 0
- for _, s := range a {
- p += len(s)
- index = append(index, uint32(p))
- }
-
- w.WriteVar(langVars[i]+"Index", index)
- w.WriteConst(langVars[i]+"Data", strings.Join(a, ""))
- }
-
- pkg := pkgs[0].Pkg.Name()
- if *out == "" {
- if _, err = w.WriteGo(os.Stdout, pkg); err != nil {
- return wrap(err, "could not write go file")
- }
- } else {
- w.WriteGoFile(*out, pkg)
- }
-
- return nil
-}
-
-func assemble(m *Message, t *Text) (msg catmsg.Message, err error) {
- keys := []string{}
- for k := range t.Var {
- keys = append(keys, k)
- }
- sort.Strings(keys)
- var a []catmsg.Message
- for _, k := range keys {
- t := t.Var[k]
- m, err := assemble(m, &t)
+ w := os.Stdout
+ if *out != "" {
+ w, err = os.Create(*out)
if err != nil {
- return nil, err
+ return wrap(err, "create file failed")
}
- a = append(a, &catmsg.Var{Name: k, Message: m})
}
- if t.Select != nil {
- s, err := assembleSelect(m, t.Select)
- if err != nil {
- return nil, err
- }
- a = append(a, s)
- }
- if t.Msg != "" {
- sub, err := m.Substitute(t.Msg)
- if err != nil {
- return nil, err
- }
- a = append(a, catmsg.String(sub))
- }
- switch len(a) {
- case 0:
- return nil, errorf("generate: empty message")
- case 1:
- return a[0], nil
- default:
- return catmsg.FirstOf(a), nil
- }
+ _, err = pipeline.Generate(w, pkg, &extracted, translations...)
+ return err
}
-
-func assembleSelect(m *Message, s *Select) (msg catmsg.Message, err error) {
- cases := []string{}
- for c := range s.Cases {
- cases = append(cases, c)
- }
- sortCases(cases)
-
- caseMsg := []interface{}{}
- for _, c := range cases {
- cm := s.Cases[c]
- m, err := assemble(m, &cm)
- if err != nil {
- return nil, err
- }
- caseMsg = append(caseMsg, c, m)
- }
-
- ph := m.Placeholder(s.Arg)
-
- switch s.Feature {
- case "plural":
- // TODO: only printf-style selects are supported as of yet.
- return plural.Selectf(ph.ArgNum, ph.String, caseMsg...), nil
- }
- return nil, errorf("unknown feature type %q", s.Feature)
-}
-
-func sortCases(cases []string) {
- // TODO: implement full interface.
- sort.Slice(cases, func(i, j int) bool {
- if cases[j] == "other" && cases[i] != "other" {
- return true
- }
- // the following code relies on '<' < '=' < any letter.
- return cmpNumeric(cases[i], cases[j]) == -1
- })
-}
-
-var cmpNumeric = collate.New(language.Und, collate.Numeric).CompareString
-
-var lookup = template.Must(template.New("gen").Parse(`
-import (
- "golang.org/x/text/language"
- "golang.org/x/text/message"
- "golang.org/x/text/message/catalog"
-)
-
-type dictionary struct {
- index []uint32
- data string
-}
-
-func (d *dictionary) Lookup(key string) (data string, ok bool) {
- p := messageKeyToIndex[key]
- start, end := d.index[p], d.index[p+1]
- if start == end {
- return "", false
- }
- return d.data[start:end], true
-}
-
-func init() {
- dict := map[string]catalog.Dictionary{
- {{range .Languages}}"{{.}}": &dictionary{index: {{.}}Index, data: {{.}}Data },
- {{end}}
- }
- fallback := language.MustParse("{{.Fallback}}")
- cat, err := catalog.NewFromMap(dict, catalog.Fallback(fallback))
- if err != nil {
- panic(err)
- }
- message.DefaultCatalog = cat
-}
-
-`))
diff --git a/cmd/gotext/message.go b/cmd/gotext/message.go
deleted file mode 100644
index 990ba52..0000000
--- a/cmd/gotext/message.go
+++ /dev/null
@@ -1,222 +0,0 @@
-// Copyright 2016 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package main
-
-import (
- "encoding/json"
- "strings"
-
- "golang.org/x/text/language"
-)
-
-// TODO: these definitions should be moved to a package so that the can be used
-// by other tools.
-
-// The file contains the structures used to define translations of a certain
-// messages.
-//
-// A translation may have multiple translations strings, or messages, depending
-// on the feature values of the various arguments. For instance, consider
-// a hypothetical translation from English to English, where the source defines
-// the format string "%d file(s) remaining".
-// See the examples directory for examples of extracted messages.
-
-// A Locale is used to store all information for a single locale. This type is
-// used both for extraction and injection.
-type Locale struct {
- Language language.Tag `json:"language"`
- Messages []Message `json:"messages"`
- Macros map[string]Text `json:"macros,omitempty"`
-}
-
-// A Message describes a message to be translated.
-type Message struct {
- // ID contains a list of identifiers for the message.
- ID IDList `json:"id"`
- // Key is the string that is used to look up the message at runtime.
- Key string `json:"key"`
- Meaning string `json:"meaning,omitempty"`
- Message Text `json:"message"`
- Translation Text `json:"translation"`
-
- Comment string `json:"comment,omitempty"`
- TranslatorComment string `json:"translatorComment,omitempty"`
-
- Placeholders []Placeholder `json:"placeholders,omitempty"`
-
- // TODO: default placeholder syntax is {foo}. Allow alternative escaping
- // like `foo`.
-
- // Extraction information.
- Position string `json:"position,omitempty"` // filePosition:line
-}
-
-// Placeholder reports the placeholder for the given ID if it is defined or nil
-// otherwise.
-func (m *Message) Placeholder(id string) *Placeholder {
- for _, p := range m.Placeholders {
- if p.ID == id {
- return &p
- }
- }
- return nil
-}
-
-// Substitute replaces placeholders in msg with their original value.
-func (m *Message) Substitute(msg string) (sub string, err error) {
- last := 0
- for i := 0; i < len(msg); {
- pLeft := strings.IndexByte(msg[i:], '{')
- if pLeft == -1 {
- break
- }
- pLeft += i
- pRight := strings.IndexByte(msg[pLeft:], '}')
- if pRight == -1 {
- return "", errorf("unmatched '}'")
- }
- pRight += pLeft
- id := strings.TrimSpace(msg[pLeft+1 : pRight])
- i = pRight + 1
- if id != "" && id[0] == '$' {
- continue
- }
- sub += msg[last:pLeft]
- last = i
- ph := m.Placeholder(id)
- if ph == nil {
- return "", errorf("unknown placeholder %q in message %q", id, msg)
- }
- sub += ph.String
- }
- sub += msg[last:]
- return sub, err
-}
-
-// A Placeholder is a part of the message that should not be changed by a
-// translator. It can be used to hide or prettify format strings (e.g. %d or
-// {{.Count}}), hide HTML, or mark common names that should not be translated.
-type Placeholder struct {
- // ID is the placeholder identifier without the curly braces.
- ID string `json:"id"`
-
- // String is the string with which to replace the placeholder. This may be a
- // formatting string (for instance "%d" or "{{.Count}}") or a literal string
- // (<div>).
- String string `json:"string"`
-
- Type string `json:"type"`
- UnderlyingType string `json:"underlyingType"`
- // ArgNum and Expr are set if the placeholder is a substitution of an
- // argument.
- ArgNum int `json:"argNum,omitempty"`
- Expr string `json:"expr,omitempty"`
-
- Comment string `json:"comment,omitempty"`
- Example string `json:"example,omitempty"`
-
- // Features contains the features that are available for the implementation
- // of this argument.
- Features []Feature `json:"features,omitempty"`
-}
-
-// An argument contains information about the arguments passed to a message.
-type argument struct {
- // ArgNum corresponds to the number that should be used for explicit argument indexes (e.g.
- // "%[1]d").
- ArgNum int `json:"argNum,omitempty"`
-
- used bool // Used by Placeholder
- Type string `json:"type"`
- UnderlyingType string `json:"underlyingType"`
- Expr string `json:"expr"`
- Value string `json:"value,omitempty"`
- Comment string `json:"comment,omitempty"`
- Position string `json:"position,omitempty"`
-}
-
-// Feature holds information about a feature that can be implemented by
-// an Argument.
-type Feature struct {
- Type string `json:"type"` // Right now this is only gender and plural.
-
- // TODO: possible values and examples for the language under consideration.
-
-}
-
-// Text defines a message to be displayed.
-type Text struct {
- // Msg and Select contains the message to be displayed. Msg may be used as
- // a fallback value if none of the select cases match.
- Msg string `json:"msg,omitempty"`
- Select *Select `json:"select,omitempty"`
-
- // Var defines a map of variables that may be substituted in the selected
- // message.
- Var map[string]Text `json:"var,omitempty"`
-
- // Example contains an example message formatted with default values.
- Example string `json:"example,omitempty"`
-}
-
-// IsEmpty reports whether this Text can generate anything.
-func (t *Text) IsEmpty() bool {
- return t.Msg == "" && t.Select == nil && t.Var == nil
-}
-
-// rawText erases the UnmarshalJSON method.
-type rawText Text
-
-// UnmarshalJSON implements json.Unmarshaler.
-func (t *Text) UnmarshalJSON(b []byte) error {
- if b[0] == '"' {
- return json.Unmarshal(b, &t.Msg)
- }
- return json.Unmarshal(b, (*rawText)(t))
-}
-
-// MarshalJSON implements json.Marshaler.
-func (t *Text) MarshalJSON() ([]byte, error) {
- if t.Select == nil && t.Var == nil && t.Example == "" {
- return json.Marshal(t.Msg)
- }
- return json.Marshal((*rawText)(t))
-}
-
-// IDList is a set identifiers that each may refer to possibly different
-// versions of the same message. When looking up a messages, the first
-// identifier in the list takes precedence.
-type IDList []string
-
-// UnmarshalJSON implements json.Unmarshaler.
-func (id *IDList) UnmarshalJSON(b []byte) error {
- if b[0] == '"' {
- *id = []string{""}
- return json.Unmarshal(b, &((*id)[0]))
- }
- return json.Unmarshal(b, (*[]string)(id))
-}
-
-// MarshalJSON implements json.Marshaler.
-func (id *IDList) MarshalJSON() ([]byte, error) {
- if len(*id) == 1 {
- return json.Marshal((*id)[0])
- }
- return json.Marshal((*[]string)(id))
-}
-
-// Select selects a Text based on the feature value associated with a feature of
-// a certain argument.
-type Select struct {
- Feature string `json:"feature"` // Name of Feature type (e.g plural)
- Arg string `json:"arg"` // The placeholder ID
- Cases map[string]Text `json:"cases"`
-}
-
-// TODO: order matters, but can we derive the ordering from the case keys?
-// type Case struct {
-// Key string `json:"key"`
-// Value Text `json:"value"`
-// }
diff --git a/cmd/gotext/rewrite.go b/cmd/gotext/rewrite.go
index f4641c8..a35b727 100644
--- a/cmd/gotext/rewrite.go
+++ b/cmd/gotext/rewrite.go
@@ -5,16 +5,9 @@
package main
import (
- "bytes"
- "fmt"
- "go/ast"
- "go/constant"
- "go/format"
- "go/token"
"os"
- "strings"
- "golang.org/x/tools/go/loader"
+ "golang.org/x/text/message/pipeline"
)
const printerType = "golang.org/x/text/message.Printer"
@@ -46,242 +39,17 @@
}
func runRewrite(cmd *Command, args []string) error {
- conf := &loader.Config{
- AllowErrors: true, // Allow unused instances of message.Printer.
+ w := os.Stdout
+ if *overwrite {
+ w = nil
}
- prog, err := loadPackages(conf, args)
- if err != nil {
- return wrap(err, "")
+ pkg := "."
+ switch len(args) {
+ case 0:
+ case 1:
+ pkg = args[0]
+ default:
+ return errorf("can only specify at most one package")
}
-
- for _, info := range prog.InitialPackages() {
- for _, f := range info.Files {
- // Associate comments with nodes.
-
- // Pick up initialized Printers at the package level.
- r := rewriter{info: info, conf: conf}
- for _, n := range info.InitOrder {
- if t := r.info.Types[n.Rhs].Type.String(); strings.HasSuffix(t, printerType) {
- r.printerVar = n.Lhs[0].Name()
- }
- }
-
- ast.Walk(&r, f)
-
- w := os.Stdout
- if *overwrite {
- var err error
- if w, err = os.Create(conf.Fset.File(f.Pos()).Name()); err != nil {
- return wrap(err, "open failed")
- }
- }
-
- if err := format.Node(w, conf.Fset, f); err != nil {
- return wrap(err, "go format failed")
- }
- }
- }
-
- return nil
-}
-
-type rewriter struct {
- info *loader.PackageInfo
- conf *loader.Config
- printerVar string
-}
-
-// print returns Go syntax for the specified node.
-func (r *rewriter) print(n ast.Node) string {
- var buf bytes.Buffer
- format.Node(&buf, r.conf.Fset, n)
- return buf.String()
-}
-
-func (r *rewriter) Visit(n ast.Node) ast.Visitor {
- // Save the state by scope.
- if _, ok := n.(*ast.BlockStmt); ok {
- r := *r
- return &r
- }
- // Find Printers created by assignment.
- stmt, ok := n.(*ast.AssignStmt)
- if ok {
- for _, v := range stmt.Lhs {
- if r.printerVar == r.print(v) {
- r.printerVar = ""
- }
- }
- for i, v := range stmt.Rhs {
- if t := r.info.Types[v].Type.String(); strings.HasSuffix(t, printerType) {
- r.printerVar = r.print(stmt.Lhs[i])
- return r
- }
- }
- }
- // Find Printers created by variable declaration.
- spec, ok := n.(*ast.ValueSpec)
- if ok {
- for _, v := range spec.Names {
- if r.printerVar == r.print(v) {
- r.printerVar = ""
- }
- }
- for i, v := range spec.Values {
- if t := r.info.Types[v].Type.String(); strings.HasSuffix(t, printerType) {
- r.printerVar = r.print(spec.Names[i])
- return r
- }
- }
- }
- if r.printerVar == "" {
- return r
- }
- call, ok := n.(*ast.CallExpr)
- if !ok {
- return r
- }
-
- // TODO: Handle literal values?
- sel, ok := call.Fun.(*ast.SelectorExpr)
- if !ok {
- return r
- }
- meth := r.info.Selections[sel]
-
- source := r.print(sel.X)
- fun := r.print(sel.Sel)
- if meth != nil {
- source = meth.Recv().String()
- fun = meth.Obj().Name()
- }
-
- // TODO: remove cheap hack and check if the type either
- // implements some interface or is specifically of type
- // "golang.org/x/text/message".Printer.
- m, ok := rewriteFuncs[source]
- if !ok {
- return r
- }
-
- rewriteType, ok := m[fun]
- if !ok {
- return r
- }
- ident := ast.NewIdent(r.printerVar)
- ident.NamePos = sel.X.Pos()
- sel.X = ident
- if rewriteType.method != "" {
- sel.Sel.Name = rewriteType.method
- }
-
- // Analyze arguments.
- argn := rewriteType.arg
- if rewriteType.format || argn >= len(call.Args) {
- return r
- }
- hasConst := false
- for _, a := range call.Args[argn:] {
- if v := r.info.Types[a].Value; v != nil && v.Kind() == constant.String {
- hasConst = true
- break
- }
- }
- if !hasConst {
- return r
- }
- sel.Sel.Name = rewriteType.methodf
-
- // We are done if there is only a single string that does not need to be
- // escaped.
- if len(call.Args) == 1 {
- s, ok := constStr(r.info, call.Args[0])
- if ok && !strings.Contains(s, "%") && !rewriteType.newLine {
- return r
- }
- }
-
- // Rewrite arguments as format string.
- expr := &ast.BasicLit{
- ValuePos: call.Lparen,
- Kind: token.STRING,
- }
- newArgs := append(call.Args[:argn:argn], expr)
- newStr := []string{}
- for i, a := range call.Args[argn:] {
- if s, ok := constStr(r.info, a); ok {
- newStr = append(newStr, strings.Replace(s, "%", "%%", -1))
- } else {
- newStr = append(newStr, "%v")
- newArgs = append(newArgs, call.Args[argn+i])
- }
- }
- s := strings.Join(newStr, rewriteType.sep)
- if rewriteType.newLine {
- s += "\n"
- }
- expr.Value = fmt.Sprintf("%q", s)
-
- call.Args = newArgs
-
- // TODO: consider creating an expression instead of a constant string and
- // then wrapping it in an escape function or so:
- // call.Args[argn+i] = &ast.CallExpr{
- // Fun: &ast.SelectorExpr{
- // X: ast.NewIdent("message"),
- // Sel: ast.NewIdent("Lookup"),
- // },
- // Args: []ast.Expr{a},
- // }
- // }
-
- return r
-}
-
-type rewriteType struct {
- // method is the name of the equivalent method on a printer, or "" if it is
- // the same.
- method string
-
- // methodf is the method to use if the arguments can be rewritten as a
- // arguments to a printf-style call.
- methodf string
-
- // format is true if the method takes a formatting string followed by
- // substitution arguments.
- format bool
-
- // arg indicates the position of the argument to extract. If all is
- // positive, all arguments from this argument onwards needs to be extracted.
- arg int
-
- sep string
- newLine bool
-}
-
-// rewriteFuncs list functions that can be directly mapped to the printer
-// functions of the message package.
-var rewriteFuncs = map[string]map[string]rewriteType{
- // TODO: Printer -> *golang.org/x/text/message.Printer
- "fmt": {
- "Print": rewriteType{methodf: "Printf"},
- "Sprint": rewriteType{methodf: "Sprintf"},
- "Fprint": rewriteType{methodf: "Fprintf"},
-
- "Println": rewriteType{methodf: "Printf", sep: " ", newLine: true},
- "Sprintln": rewriteType{methodf: "Sprintf", sep: " ", newLine: true},
- "Fprintln": rewriteType{methodf: "Fprintf", sep: " ", newLine: true},
-
- "Printf": rewriteType{method: "Printf", format: true},
- "Sprintf": rewriteType{method: "Sprintf", format: true},
- "Fprintf": rewriteType{method: "Fprintf", format: true},
- },
-}
-
-func constStr(info *loader.PackageInfo, e ast.Expr) (s string, ok bool) {
- v := info.Types[e].Value
- if v == nil || v.Kind() != constant.String {
- return "", false
- }
- return constant.StringVal(v), true
+ return pipeline.Rewrite(w, pkg)
}
diff --git a/message/pipeline/doc.go b/message/pipeline/doc.go
deleted file mode 100644
index 2a274f7..0000000
--- a/message/pipeline/doc.go
+++ /dev/null
@@ -1,53 +0,0 @@
-// Code generated by go generate. DO NOT EDIT.
-
-// gotext is a tool for managing text in Go source code.
-//
-// Usage:
-//
-// gotext command [arguments]
-//
-// The commands are:
-//
-// extract extracts strings to be translated from code
-// rewrite rewrites fmt functions to use a message Printer
-// generate generates code to insert translated messages
-//
-// Use "go help [command]" for more information about a command.
-//
-// Additional help topics:
-//
-//
-// Use "gotext help [topic]" for more information about that topic.
-//
-//
-// Extracts strings to be translated from code
-//
-// Usage:
-//
-// go extract <package>*
-//
-//
-//
-//
-// Rewrites fmt functions to use a message Printer
-//
-// Usage:
-//
-// go rewrite <package>
-//
-// rewrite is typically done once for a project. It rewrites all usages of
-// fmt to use x/text's message package whenever a message.Printer is in scope.
-// It rewrites Print and Println calls with constant strings to the equivalent
-// using Printf to allow translators to reorder arguments.
-//
-//
-// Generates code to insert translated messages
-//
-// Usage:
-//
-// go generate <package>
-//
-//
-//
-//
-package main
diff --git a/message/pipeline/extract.go b/message/pipeline/extract.go
index 561f050..20ad914 100644
--- a/message/pipeline/extract.go
+++ b/message/pipeline/extract.go
@@ -2,28 +2,23 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
-package main
+package pipeline
import (
"bytes"
- "encoding/json"
"fmt"
"go/ast"
"go/constant"
"go/format"
"go/token"
"go/types"
- "io/ioutil"
- "os"
"path"
"path/filepath"
"strings"
"unicode"
"unicode/utf8"
- "golang.org/x/text/internal"
fmtparser "golang.org/x/text/internal/format"
- "golang.org/x/text/language"
"golang.org/x/tools/go/loader"
)
@@ -33,27 +28,12 @@
// - handle features (gender, plural)
// - message rewriting
-var (
- srcLang *string
- lang *string
-)
-
-func init() {
- srcLang = cmdExtract.Flag.String("srclang", "en-US", "the source-code language")
- lang = cmdExtract.Flag.String("lang", "en-US", "comma-separated list of languages to process")
-}
-
-var cmdExtract = &Command{
- Run: runExtract,
- UsageLine: "extract <package>*",
- Short: "extracts strings to be translated from code",
-}
-
-func runExtract(cmd *Command, args []string) error {
+// Extract extracts all strings form the package defined in Config.
+func Extract(c *Config) (*Locale, error) {
conf := loader.Config{}
- prog, err := loadPackages(&conf, args)
+ prog, err := loadPackages(&conf, c.Packages)
if err != nil {
- return wrap(err, "")
+ return nil, wrap(err, "")
}
// print returns Go syntax for the specified node.
@@ -209,44 +189,11 @@
}
}
- tag, err := language.Parse(*srcLang)
- if err != nil {
- return wrap(err, "")
- }
- out := Locale{
- Language: tag,
+ out := &Locale{
+ Language: c.SourceLanguage,
Messages: messages,
}
- data, err := json.MarshalIndent(out, "", " ")
- if err != nil {
- return wrap(err, "")
- }
- os.MkdirAll(*dir, 0755)
- // TODO: this file can probably go if we replace the extract + generate
- // cycle with a init once and update cycle.
- file := filepath.Join(*dir, extractFile)
- if err := ioutil.WriteFile(file, data, 0644); err != nil {
- return wrapf(err, "could not create file")
- }
-
- langs := append(getLangs(), tag)
- langs = internal.UniqueTags(langs)
- for _, tag := range langs {
- // TODO: inject translations from existing files to avoid retranslation.
- out.Language = tag
- data, err := json.MarshalIndent(out, "", " ")
- if err != nil {
- return wrap(err, "JSON marshal failed")
- }
- file := filepath.Join(*dir, tag.String(), outFile)
- if err := os.MkdirAll(filepath.Dir(file), 0750); err != nil {
- return wrap(err, "dir create failed")
- }
- if err := ioutil.WriteFile(file, data, 0740); err != nil {
- return wrap(err, "write failed")
- }
- }
- return nil
+ return out, nil
}
func posString(conf loader.Config, info *loader.PackageInfo, pos token.Pos) string {
diff --git a/message/pipeline/generate.go b/message/pipeline/generate.go
index 0293273..b03d9a2 100644
--- a/message/pipeline/generate.go
+++ b/message/pipeline/generate.go
@@ -2,14 +2,11 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
-package main
+package pipeline
import (
- "encoding/json"
"fmt"
- "io/ioutil"
- "os"
- "path/filepath"
+ "io"
"regexp"
"sort"
"strings"
@@ -21,83 +18,29 @@
"golang.org/x/text/internal/catmsg"
"golang.org/x/text/internal/gen"
"golang.org/x/text/language"
- "golang.org/x/tools/go/loader"
)
-func init() {
- out = cmdGenerate.Flag.String("out", "", "output file to write to")
-}
-
-var (
- out *string
-)
-
-var cmdGenerate = &Command{
- Run: runGenerate,
- UsageLine: "generate <package>",
- Short: "generates code to insert translated messages",
-}
-
var transRe = regexp.MustCompile(`messages\.(.*)\.json`)
-func runGenerate(cmd *Command, args []string) error {
- prog, err := loadPackages(&loader.Config{}, args)
- if err != nil {
- return wrap(err, "could not load package")
- }
-
- pkgs := prog.InitialPackages()
- if len(pkgs) != 1 {
- return fmt.Errorf("more than one package selected: %v", pkgs)
- }
-
+// Generate writes a Go file with the given package name to w, which defines a
+// Catalog with translated messages.
+func Generate(w io.Writer, pkg string, extracted *Locale, trans ...*Locale) (n int, err error) {
// TODO: add in external input. Right now we assume that all files are
// manually created and stored in the textdata directory.
// Build up index of translations and original messages.
- extracted := Locale{}
translations := map[language.Tag]map[string]Message{}
languages := []language.Tag{}
langVars := []string{}
usedKeys := map[string]int{}
- err = filepath.Walk(*dir, func(path string, f os.FileInfo, err error) error {
- if err != nil {
- return wrap(err, "loading data")
- }
- if f.IsDir() {
- return nil
- }
- if f.Name() == extractFile {
- b, err := ioutil.ReadFile(path)
- if err != nil {
- return wrap(err, "read file failed")
- }
- if err := json.Unmarshal(b, &extracted); err != nil {
- return wrap(err, "unmarshal source failed")
- }
- return nil
- }
- if f.Name() == outFile {
- return nil
- }
- if !strings.HasSuffix(path, gotextSuffix) {
- return nil
- }
- b, err := ioutil.ReadFile(path)
- if err != nil {
- return wrap(err, "read file failed")
- }
- var locale Locale
- if err := json.Unmarshal(b, &locale); err != nil {
- return wrap(err, "parsing translation file failed")
- }
- tag := locale.Language
+ for _, loc := range trans {
+ tag := loc.Language
if _, ok := translations[tag]; !ok {
translations[tag] = map[string]Message{}
languages = append(languages, tag)
}
- for _, m := range locale.Messages {
+ for _, m := range loc.Messages {
if !m.Translation.IsEmpty() {
for _, id := range m.ID {
if _, ok := translations[tag][id]; ok {
@@ -107,10 +50,6 @@
}
}
}
- return nil
- })
- if err != nil {
- return err
}
// Verify completeness and register keys.
@@ -133,7 +72,7 @@
}
}
- w := gen.NewCodeWriter()
+ cw := gen.NewCodeWriter()
x := &struct {
Fallback language.Tag
@@ -143,8 +82,8 @@
Languages: langVars,
}
- if err := lookup.Execute(w, x); err != nil {
- return wrap(err, "error")
+ if err := lookup.Execute(cw, x); err != nil {
+ return 0, wrap(err, "error")
}
keyToIndex := []string{}
@@ -152,11 +91,11 @@
keyToIndex = append(keyToIndex, k)
}
sort.Strings(keyToIndex)
- fmt.Fprint(w, "var messageKeyToIndex = map[string]int{\n")
+ fmt.Fprint(cw, "var messageKeyToIndex = map[string]int{\n")
for _, k := range keyToIndex {
- fmt.Fprintf(w, "%q: %d,\n", k, usedKeys[k])
+ fmt.Fprintf(cw, "%q: %d,\n", k, usedKeys[k])
}
- fmt.Fprint(w, "}\n\n")
+ fmt.Fprint(cw, "}\n\n")
for i, tag := range languages {
dict := translations[tag]
@@ -166,12 +105,12 @@
if trans, ok := dict[id]; ok && !trans.Translation.IsEmpty() {
m, err := assemble(&msg, &trans.Translation)
if err != nil {
- return wrap(err, "error")
+ return 0, wrap(err, "error")
}
// TODO: support macros.
data, err := catmsg.Compile(tag, nil, m)
if err != nil {
- return wrap(err, "error")
+ return 0, wrap(err, "error")
}
key := usedKeys[msg.Key]
if d := a[key]; d != "" && d != data {
@@ -189,20 +128,10 @@
index = append(index, uint32(p))
}
- w.WriteVar(langVars[i]+"Index", index)
- w.WriteConst(langVars[i]+"Data", strings.Join(a, ""))
+ cw.WriteVar(langVars[i]+"Index", index)
+ cw.WriteConst(langVars[i]+"Data", strings.Join(a, ""))
}
-
- pkg := pkgs[0].Pkg.Name()
- if *out == "" {
- if _, err = w.WriteGo(os.Stdout, pkg); err != nil {
- return wrap(err, "could not write go file")
- }
- } else {
- w.WriteGoFile(*out, pkg)
- }
-
- return nil
+ return cw.WriteGo(w, pkg)
}
func assemble(m *Message, t *Text) (msg catmsg.Message, err error) {
diff --git a/message/pipeline/main.go b/message/pipeline/main.go
deleted file mode 100644
index f3f50d7..0000000
--- a/message/pipeline/main.go
+++ /dev/null
@@ -1,352 +0,0 @@
-// Copyright 2016 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-//go:generate go build -o gotext.latest
-//go:generate ./gotext.latest help gendocumentation
-//go:generate rm gotext.latest
-
-package main
-
-import (
- "bufio"
- "bytes"
- "flag"
- "fmt"
- "go/build"
- "go/format"
- "io"
- "io/ioutil"
- "log"
- "os"
- "strings"
- "sync"
- "text/template"
- "unicode"
- "unicode/utf8"
-
- "golang.org/x/text/language"
- "golang.org/x/tools/go/buildutil"
-)
-
-func init() {
- flag.Var((*buildutil.TagsFlag)(&build.Default.BuildTags), "tags", buildutil.TagsFlagDoc)
-}
-
-var dir = flag.String("dir", "locales", "default subdirectory to store translation files")
-
-// NOTE: the Command struct is copied from the go tool in core.
-
-// A Command is an implementation of a go command
-// like go build or go fix.
-type Command struct {
- // Run runs the command.
- // The args are the arguments after the command name.
- Run func(cmd *Command, args []string) error
-
- // UsageLine is the one-line usage message.
- // The first word in the line is taken to be the command name.
- UsageLine string
-
- // Short is the short description shown in the 'go help' output.
- Short string
-
- // Long is the long message shown in the 'go help <this-command>' output.
- Long string
-
- // Flag is a set of flags specific to this command.
- Flag flag.FlagSet
-}
-
-// Name returns the command's name: the first word in the usage line.
-func (c *Command) Name() string {
- name := c.UsageLine
- i := strings.Index(name, " ")
- if i >= 0 {
- name = name[:i]
- }
- return name
-}
-
-func (c *Command) Usage() {
- fmt.Fprintf(os.Stderr, "usage: %s\n\n", c.UsageLine)
- fmt.Fprintf(os.Stderr, "%s\n", strings.TrimSpace(c.Long))
- os.Exit(2)
-}
-
-// Runnable reports whether the command can be run; otherwise
-// it is a documentation pseudo-command such as importpath.
-func (c *Command) Runnable() bool {
- return c.Run != nil
-}
-
-// Commands lists the available commands and help topics.
-// The order here is the order in which they are printed by 'go help'.
-var commands = []*Command{
- cmdExtract,
- cmdRewrite,
- cmdGenerate,
- // TODO:
- // - update: full-cycle update of extraction, sending, and integration
- // - report: report of freshness of translations
-}
-
-var exitStatus = 0
-var exitMu sync.Mutex
-
-func setExitStatus(n int) {
- exitMu.Lock()
- if exitStatus < n {
- exitStatus = n
- }
- exitMu.Unlock()
-}
-
-var origEnv []string
-
-func main() {
- flag.Usage = usage
- flag.Parse()
- log.SetFlags(0)
-
- args := flag.Args()
- if len(args) < 1 {
- usage()
- }
-
- if args[0] == "help" {
- help(args[1:])
- return
- }
-
- for _, cmd := range commands {
- if cmd.Name() == args[0] && cmd.Runnable() {
- cmd.Flag.Usage = func() { cmd.Usage() }
- cmd.Flag.Parse(args[1:])
- args = cmd.Flag.Args()
- if err := cmd.Run(cmd, args); err != nil {
- fatalf("gotext: %+v", err)
- }
- exit()
- return
- }
- }
-
- fmt.Fprintf(os.Stderr, "gotext: unknown subcommand %q\nRun 'go help' for usage.\n", args[0])
- setExitStatus(2)
- exit()
-}
-
-var usageTemplate = `gotext is a tool for managing text in Go source code.
-
-Usage:
-
- gotext command [arguments]
-
-The commands are:
-{{range .}}{{if .Runnable}}
- {{.Name | printf "%-11s"}} {{.Short}}{{end}}{{end}}
-
-Use "go help [command]" for more information about a command.
-
-Additional help topics:
-{{range .}}{{if not .Runnable}}
- {{.Name | printf "%-11s"}} {{.Short}}{{end}}{{end}}
-
-Use "gotext help [topic]" for more information about that topic.
-
-`
-
-var helpTemplate = `{{if .Runnable}}usage: go {{.UsageLine}}
-
-{{end}}{{.Long | trim}}
-`
-
-var documentationTemplate = `{{range .}}{{if .Short}}{{.Short | capitalize}}
-
-{{end}}{{if .Runnable}}Usage:
-
- go {{.UsageLine}}
-
-{{end}}{{.Long | trim}}
-
-
-{{end}}`
-
-// commentWriter writes a Go comment to the underlying io.Writer,
-// using line comment form (//).
-type commentWriter struct {
- W io.Writer
- wroteSlashes bool // Wrote "//" at the beginning of the current line.
-}
-
-func (c *commentWriter) Write(p []byte) (int, error) {
- var n int
- for i, b := range p {
- if !c.wroteSlashes {
- s := "//"
- if b != '\n' {
- s = "// "
- }
- if _, err := io.WriteString(c.W, s); err != nil {
- return n, err
- }
- c.wroteSlashes = true
- }
- n0, err := c.W.Write(p[i : i+1])
- n += n0
- if err != nil {
- return n, err
- }
- if b == '\n' {
- c.wroteSlashes = false
- }
- }
- return len(p), nil
-}
-
-// An errWriter wraps a writer, recording whether a write error occurred.
-type errWriter struct {
- w io.Writer
- err error
-}
-
-func (w *errWriter) Write(b []byte) (int, error) {
- n, err := w.w.Write(b)
- if err != nil {
- w.err = err
- }
- return n, err
-}
-
-// tmpl executes the given template text on data, writing the result to w.
-func tmpl(w io.Writer, text string, data interface{}) {
- t := template.New("top")
- t.Funcs(template.FuncMap{"trim": strings.TrimSpace, "capitalize": capitalize})
- template.Must(t.Parse(text))
- ew := &errWriter{w: w}
- err := t.Execute(ew, data)
- if ew.err != nil {
- // I/O error writing. Ignore write on closed pipe.
- if strings.Contains(ew.err.Error(), "pipe") {
- os.Exit(1)
- }
- fatalf("writing output: %v", ew.err)
- }
- if err != nil {
- panic(err)
- }
-}
-
-func capitalize(s string) string {
- if s == "" {
- return s
- }
- r, n := utf8.DecodeRuneInString(s)
- return string(unicode.ToTitle(r)) + s[n:]
-}
-
-func printUsage(w io.Writer) {
- bw := bufio.NewWriter(w)
- tmpl(bw, usageTemplate, commands)
- bw.Flush()
-}
-
-func usage() {
- printUsage(os.Stderr)
- os.Exit(2)
-}
-
-// help implements the 'help' command.
-func help(args []string) {
- if len(args) == 0 {
- printUsage(os.Stdout)
- // not exit 2: succeeded at 'go help'.
- return
- }
- if len(args) != 1 {
- fmt.Fprintf(os.Stderr, "usage: go help command\n\nToo many arguments given.\n")
- os.Exit(2) // failed at 'go help'
- }
-
- arg := args[0]
-
- // 'go help documentation' generates doc.go.
- if strings.HasSuffix(arg, "documentation") {
- w := &bytes.Buffer{}
-
- fmt.Fprintln(w, "// Code generated by go generate. DO NOT EDIT.")
- fmt.Fprintln(w)
- buf := new(bytes.Buffer)
- printUsage(buf)
- usage := &Command{Long: buf.String()}
- tmpl(&commentWriter{W: w}, documentationTemplate, append([]*Command{usage}, commands...))
- fmt.Fprintln(w, "package main")
- if arg == "gendocumentation" {
- b, err := format.Source(w.Bytes())
- if err != nil {
- logf("Could not format generated docs: %v\n", err)
- }
- if err := ioutil.WriteFile("doc.go", b, 0666); err != nil {
- logf("Could not create file alldocs.go: %v\n", err)
- }
- } else {
- fmt.Println(w.String())
- }
- return
- }
-
- for _, cmd := range commands {
- if cmd.Name() == arg {
- tmpl(os.Stdout, helpTemplate, cmd)
- // not exit 2: succeeded at 'go help cmd'.
- return
- }
- }
-
- fmt.Fprintf(os.Stderr, "Unknown help topic %#q. Run 'go help'.\n", arg)
- os.Exit(2) // failed at 'go help cmd'
-}
-
-func getLangs() (tags []language.Tag) {
- for _, t := range strings.Split(*lang, ",") {
- if t == "" {
- continue
- }
- tag, err := language.Parse(t)
- if err != nil {
- fatalf("gotext: could not parse language %q: %v", t, err)
- }
- tags = append(tags, tag)
- }
- return tags
-}
-
-var atexitFuncs []func()
-
-func atexit(f func()) {
- atexitFuncs = append(atexitFuncs, f)
-}
-
-func exit() {
- for _, f := range atexitFuncs {
- f()
- }
- os.Exit(exitStatus)
-}
-
-func fatalf(format string, args ...interface{}) {
- logf(format, args...)
- exit()
-}
-
-func logf(format string, args ...interface{}) {
- log.Printf(format, args...)
- setExitStatus(1)
-}
-
-func exitIfErrors() {
- if exitStatus != 0 {
- exit()
- }
-}
diff --git a/message/pipeline/message.go b/message/pipeline/message.go
index 990ba52..8e54700 100644
--- a/message/pipeline/message.go
+++ b/message/pipeline/message.go
@@ -1,8 +1,8 @@
-// Copyright 2016 The Go Authors. All rights reserved.
+// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
-package main
+package pipeline
import (
"encoding/json"
@@ -23,6 +23,25 @@
// the format string "%d file(s) remaining".
// See the examples directory for examples of extracted messages.
+// Config contains configuration for the translation pipeline.
+type Config struct {
+ SourceLanguage language.Tag
+
+ // Supported indicates the languages for which data should be generated.
+ // If unspecified, it will attempt to derive the set of supported languages
+ // from the context.
+ Supported []language.Tag
+
+ Packages []string
+
+ // TODO:
+ // - Printf-style configuration
+ // - Template-style configuration
+ // - Extraction options
+ // - Rewrite options
+ // - Generation options
+}
+
// A Locale is used to store all information for a single locale. This type is
// used both for extraction and injection.
type Locale struct {
diff --git a/message/pipeline/common.go b/message/pipeline/pipeline.go
similarity index 81%
rename from message/pipeline/common.go
rename to message/pipeline/pipeline.go
index ae87e44..733a50c 100644
--- a/message/pipeline/common.go
+++ b/message/pipeline/pipeline.go
@@ -2,12 +2,16 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
-package main
+// Package pipeline provides tools for creating translation pipelines.
+//
+// NOTE: UNDER DEVELOPMENT. API MAY CHANGE.
+package pipeline
import (
"fmt"
"go/build"
"go/parser"
+ "log"
"golang.org/x/tools/go/loader"
)
@@ -29,6 +33,11 @@
errorf = fmt.Errorf
)
+// TODO: don't log.
+func logf(format string, args ...interface{}) {
+ log.Printf(format, args...)
+}
+
func loadPackages(conf *loader.Config, args []string) (*loader.Program, error) {
if len(args) == 0 {
args = []string{"."}
diff --git a/message/pipeline/rewrite.go b/message/pipeline/rewrite.go
index f4641c8..fa78324 100644
--- a/message/pipeline/rewrite.go
+++ b/message/pipeline/rewrite.go
@@ -2,7 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
-package main
+package pipeline
import (
"bytes"
@@ -11,6 +11,7 @@
"go/constant"
"go/format"
"go/token"
+ "io"
"os"
"strings"
@@ -19,37 +20,15 @@
const printerType = "golang.org/x/text/message.Printer"
-// TODO:
-// - merge information into existing files
-// - handle different file formats (PO, XLIFF)
-// - handle features (gender, plural)
-// - message rewriting
-
-func init() {
- overwrite = cmdRewrite.Flag.Bool("w", false, "write files in place")
-}
-
-var (
- overwrite *bool
-)
-
-var cmdRewrite = &Command{
- Run: runRewrite,
- UsageLine: "rewrite <package>",
- Short: "rewrites fmt functions to use a message Printer",
- Long: `
-rewrite is typically done once for a project. It rewrites all usages of
-fmt to use x/text's message package whenever a message.Printer is in scope.
-It rewrites Print and Println calls with constant strings to the equivalent
-using Printf to allow translators to reorder arguments.
-`,
-}
-
-func runRewrite(cmd *Command, args []string) error {
+// Rewrite rewrites the Go files in a single package to use the localization
+// machinery and rewrites strings to adopt best practices when possible.
+// If w is not nil the generated files are written to it, each files with a
+// "--- <filename>" header. Otherwise the files are overwritten.
+func Rewrite(w io.Writer, goPackage string) error {
conf := &loader.Config{
AllowErrors: true, // Allow unused instances of message.Printer.
}
- prog, err := loadPackages(conf, args)
+ prog, err := loadPackages(conf, []string{goPackage})
if err != nil {
return wrap(err, "")
}
@@ -68,12 +47,14 @@
ast.Walk(&r, f)
- w := os.Stdout
- if *overwrite {
+ w := w
+ if w == nil {
var err error
if w, err = os.Create(conf.Fset.File(f.Pos()).Name()); err != nil {
return wrap(err, "open failed")
}
+ } else {
+ fmt.Fprintln(w, "---", conf.Fset.File(f.Pos()).Name())
}
if err := format.Node(w, conf.Fset, f); err != nil {
