message/pipeline: hoist importing from cmd/gotext
- cmd/gotext generate now extracts, so no need to
create intermediate extraction file.
(next step is to just have an update command)
- shared config creation code in cmd/text
- added tests in pipeline package
Change-Id: Ifdb69710554712a79bda79502e5e03a3f3a2c9d3
Reviewed-on: https://go-review.googlesource.com/83657
Run-TryBot: Marcel van Lohuizen <mpvl@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Nigel Tao <nigeltao@golang.org>
diff --git a/cmd/gotext/examples/extract_http/locales/extracted.gotext.json b/cmd/gotext/examples/extract_http/locales/extracted.gotext.json
deleted file mode 100755
index d0c4684..0000000
--- a/cmd/gotext/examples/extract_http/locales/extracted.gotext.json
+++ /dev/null
@@ -1,39 +0,0 @@
-{
- "language": "en-US",
- "messages": [
- {
- "id": "Hello {From}!\n",
- "key": "Hello %s!\n",
- "message": "Hello {From}!\n",
- "translation": "",
- "placeholders": [
- {
- "id": "From",
- "string": "%[1]s",
- "type": "string",
- "underlyingType": "string",
- "argNum": 1,
- "expr": "r.Header.Get(\"From\")"
- }
- ],
- "position": "golang.org/x/text/cmd/gotext/examples/extract_http/pkg/pkg.go:22:11"
- },
- {
- "id": "Do you like your browser ({User_Agent})?\n",
- "key": "Do you like your browser (%s)?\n",
- "message": "Do you like your browser ({User_Agent})?\n",
- "translation": "",
- "placeholders": [
- {
- "id": "User_Agent",
- "string": "%[1]s",
- "type": "string",
- "underlyingType": "string",
- "argNum": 1,
- "expr": "r.Header.Get(\"User-Agent\")"
- }
- ],
- "position": "golang.org/x/text/cmd/gotext/examples/extract_http/pkg/pkg.go:24:11"
- }
- ]
-}
\ No newline at end of file
diff --git a/cmd/gotext/extract.go b/cmd/gotext/extract.go
index 20b94d3..221e776 100644
--- a/cmd/gotext/extract.go
+++ b/cmd/gotext/extract.go
@@ -11,7 +11,6 @@
"path/filepath"
"golang.org/x/text/internal"
- "golang.org/x/text/language"
"golang.org/x/text/message/pipeline"
)
@@ -22,12 +21,10 @@
// - message rewriting
var (
- srcLang *string
- lang *string
+ lang *string
)
func init() {
- srcLang = cmdExtract.Flag.String("srclang", "en-US", "the source-code language")
lang = cmdExtract.Flag.String("lang", "en-US", "comma-separated list of languages to process")
}
@@ -37,34 +34,15 @@
Short: "extracts strings to be translated from code",
}
-func runExtract(cmd *Command, args []string) error {
- tag, err := language.Parse(*srcLang)
- if err != nil {
- return wrap(err, "")
- }
- config := &pipeline.Config{
- SourceLanguage: tag,
- Packages: args,
- }
+func runExtract(cmd *Command, config *pipeline.Config, args []string) error {
+ config.Packages = args
state, err := pipeline.Extract(config)
if err != nil {
return wrap(err, "extract failed")
}
out := state.Extracted
- data, err := json.MarshalIndent(out, "", " ")
- if err != nil {
- return wrap(err, "")
- }
- os.MkdirAll(*dir, 0755)
- // TODO: this file can probably go if we replace the extract + generate
- // cycle with a init once and update cycle.
- file := filepath.Join(*dir, extractFile)
- if err := ioutil.WriteFile(file, data, 0644); err != nil {
- return wrap(err, "could not create file")
- }
-
- langs := append(getLangs(), tag)
+ langs := append(getLangs(), config.SourceLanguage)
langs = internal.UniqueTags(langs)
for _, tag := range langs {
// TODO: inject translations from existing files to avoid retranslation.
diff --git a/cmd/gotext/generate.go b/cmd/gotext/generate.go
index a108dee..c8d5d79 100644
--- a/cmd/gotext/generate.go
+++ b/cmd/gotext/generate.go
@@ -5,16 +5,7 @@
package main
import (
- "encoding/json"
- "fmt"
- "io/ioutil"
- "os"
- "path/filepath"
- "regexp"
- "strings"
-
"golang.org/x/text/message/pipeline"
- "golang.org/x/tools/go/loader"
)
func init() {
@@ -31,74 +22,14 @@
Short: "generates code to insert translated messages",
}
-var transRe = regexp.MustCompile(`messages\.(.*)\.json`)
-
-func runGenerate(cmd *Command, args []string) error {
-
- prog, err := loadPackages(&loader.Config{}, args)
+func runGenerate(cmd *Command, config *pipeline.Config, args []string) error {
+ config.Packages = args
+ s, err := pipeline.Extract(config)
if err != nil {
- return wrap(err, "could not load package")
+ return wrap(err, "extraction failed")
}
-
- pkgs := prog.InitialPackages()
- if len(pkgs) != 1 {
- return fmt.Errorf("more than one package selected: %v", pkgs)
+ if err := s.Import(); err != nil {
+ return wrap(err, "import failed")
}
- pkg := pkgs[0].Pkg.Name()
-
- // TODO: add in external input. Right now we assume that all files are
- // manually created and stored in the textdata directory.
-
- // Build up index of translations and original messages.
- extracted := pipeline.Messages{}
- translations := []pipeline.Messages{}
-
- err = filepath.Walk(*dir, func(path string, f os.FileInfo, err error) error {
- if err != nil {
- return wrap(err, "loading data")
- }
- if f.IsDir() {
- return nil
- }
- if f.Name() == extractFile {
- b, err := ioutil.ReadFile(path)
- if err != nil {
- return wrap(err, "read file failed")
- }
- if err := json.Unmarshal(b, &extracted); err != nil {
- return wrap(err, "unmarshal source failed")
- }
- return nil
- }
- if f.Name() == outFile {
- return nil
- }
- if !strings.HasSuffix(path, gotextSuffix) {
- return nil
- }
- b, err := ioutil.ReadFile(path)
- if err != nil {
- return wrap(err, "read file failed")
- }
- var locale pipeline.Messages
- if err := json.Unmarshal(b, &locale); err != nil {
- return wrap(err, "parsing translation file failed")
- }
- translations = append(translations, locale)
- return nil
- })
- if err != nil {
- return err
- }
-
- w := os.Stdout
- if *out != "" {
- w, err = os.Create(*out)
- if err != nil {
- return wrap(err, "create file failed")
- }
- }
-
- _, err = pipeline.Generate(w, pkg, &extracted, translations...)
- return err
+ return wrap(s.Generate(), "generation failed")
}
diff --git a/cmd/gotext/main.go b/cmd/gotext/main.go
index f3f50d7..1e6c3aa 100644
--- a/cmd/gotext/main.go
+++ b/cmd/gotext/main.go
@@ -25,6 +25,8 @@
"unicode"
"unicode/utf8"
+ "golang.org/x/text/message/pipeline"
+
"golang.org/x/text/language"
"golang.org/x/tools/go/buildutil"
)
@@ -33,7 +35,22 @@
flag.Var((*buildutil.TagsFlag)(&build.Default.BuildTags), "tags", buildutil.TagsFlagDoc)
}
-var dir = flag.String("dir", "locales", "default subdirectory to store translation files")
+var (
+ srcLang = flag.String("srclang", "en-US", "the source-code language")
+ dir = flag.String("dir", "locales", "default subdirectory to store translation files")
+)
+
+func config() (*pipeline.Config, error) {
+ tag, err := language.Parse(*srcLang)
+ if err != nil {
+ return nil, wrap(err, "invalid srclang")
+ }
+ return &pipeline.Config{
+ SourceLanguage: tag,
+ TranslationsPattern: `messages\.(.*)\.json`,
+ GenFile: *out,
+ }, nil
+}
// NOTE: the Command struct is copied from the go tool in core.
@@ -42,7 +59,7 @@
type Command struct {
// Run runs the command.
// The args are the arguments after the command name.
- Run func(cmd *Command, args []string) error
+ Run func(cmd *Command, c *pipeline.Config, args []string) error
// UsageLine is the one-line usage message.
// The first word in the line is taken to be the command name.
@@ -124,7 +141,11 @@
cmd.Flag.Usage = func() { cmd.Usage() }
cmd.Flag.Parse(args[1:])
args = cmd.Flag.Args()
- if err := cmd.Run(cmd, args); err != nil {
+ config, err := config()
+ if err != nil {
+ fatalf("gotext: %+v", err)
+ }
+ if err := cmd.Run(cmd, config, args); err != nil {
fatalf("gotext: %+v", err)
}
exit()
diff --git a/cmd/gotext/rewrite.go b/cmd/gotext/rewrite.go
index a35b727..3ee9555 100644
--- a/cmd/gotext/rewrite.go
+++ b/cmd/gotext/rewrite.go
@@ -38,7 +38,7 @@
`,
}
-func runRewrite(cmd *Command, args []string) error {
+func runRewrite(cmd *Command, _ *pipeline.Config, args []string) error {
w := os.Stdout
if *overwrite {
w = nil
diff --git a/message/pipeline/generate.go b/message/pipeline/generate.go
index 2b1c875..f21568d 100644
--- a/message/pipeline/generate.go
+++ b/message/pipeline/generate.go
@@ -6,7 +6,9 @@
import (
"fmt"
+ "go/build"
"io"
+ "path/filepath"
"regexp"
"sort"
"strings"
@@ -25,8 +27,12 @@
// Generate writes a Go file that defines a Catalog with translated messages.
func (s *State) Generate() error {
- filename := s.Config.CatalogFile
- prog, err := loadPackages(&loader.Config{}, []string{filename})
+ path := s.Config.GenPackage
+ if path == "" {
+ path = "."
+ }
+ isDir := path[0] == '.'
+ prog, err := loadPackages(&loader.Config{}, []string{path})
if err != nil {
return wrap(err, "could not load package")
}
@@ -40,7 +46,12 @@
if err != nil {
return err
}
- cw.WriteGoFile(filename, pkg) // TODO: WriteGoFile should return error.
+ if !isDir {
+ gopath := build.Default.GOPATH
+ path = filepath.Join(gopath, filepath.FromSlash(pkgs[0].Pkg.Path()))
+ }
+ path = filepath.Join(path, s.Config.GenFile)
+ cw.WriteGoFile(path, pkg) // TODO: WriteGoFile should return error.
return err
}
diff --git a/message/pipeline/pipeline.go b/message/pipeline/pipeline.go
index 848a47a..876e470 100644
--- a/message/pipeline/pipeline.go
+++ b/message/pipeline/pipeline.go
@@ -8,10 +8,17 @@
package pipeline
import (
+ "bytes"
+ "encoding/json"
"fmt"
"go/build"
"go/parser"
+ "io/ioutil"
"log"
+ "path/filepath"
+ "regexp"
+ "strings"
+ "text/template"
"golang.org/x/text/language"
"golang.org/x/tools/go/loader"
@@ -20,7 +27,7 @@
const (
extractFile = "extracted.gotext.json"
outFile = "out.gotext.json"
- gotextSuffix = ".gotext.json"
+ gotextSuffix = "gotext.json"
)
// Config contains configuration for the translation pipeline.
@@ -41,8 +48,14 @@
// Dir is the root dir for all operations.
Dir string
- // TranslationsPattern is a regular expression for input translation files
- // that match anywhere in the directory structure rooted at Dir.
+ // TranslationsPattern is a regular expression to match incoming translation
+ // files. These files may appear in any directory rooted at Dir.
+ // language for the translation files is determined as follows:
+ // 1. From the Language field in the file.
+ // 2. If not present, from a valid language tag in the filename, separated
+ // by dots (e.g. "en-US.json" or "incoming.pt_PT.xmb").
+ // 3. If not present, from a the closest subdirectory in which the file
+ // is contained that parses as a valid language tag.
TranslationsPattern string
// OutPattern defines the location for translation files for a certain
@@ -65,9 +78,13 @@
// --- Generation
- // CatalogFile may be in a different package. It is not defined, it will
+ // GenFile may be in a different package. It is not defined, it will
// be written to stdout.
- CatalogFile string
+ GenFile string
+
+ // GenPackage is the package or relative path into which to generate the
+ // file. If not specified it is relative to the current directory.
+ GenPackage string
// DeclareVar defines a variable to which to assing the generated Catalog.
DeclareVar string
@@ -117,30 +134,106 @@
Translations []Messages
}
-// A full-cycle pipeline example:
-//
-// func updateAll(c *Config) error {
-// s := Extract(c)
-// s.Import()
-// s.Merge()
-// for range s.Config.Actions {
-// if s.Err != nil {
-// return s.Err
-// }
-// // TODO: do the actions.
-// }
-// if err := s.Export(); err != nil {
-// return err
-// }
-// if err := s.Generate(); err != nil {
-// return err
-// }
-// return nil
-// }
+func (s *State) dir() string {
+ if d := s.Config.Dir; d != "" {
+ return d
+ }
+ return "./locales"
+}
+
+func outPattern(s *State) (string, error) {
+ c := s.Config
+ pat := c.OutPattern
+ if pat == "" {
+ pat = "{{.Dir}}/{{.Language}}/out.{{.Ext}}"
+ }
+
+ ext := c.Ext
+ if ext == "" {
+ ext = c.Format
+ }
+ if ext == "" {
+ ext = gotextSuffix
+ }
+ t, err := template.New("").Parse(pat)
+ if err != nil {
+ return "", wrap(err, "error parsing template")
+ }
+ buf := bytes.Buffer{}
+ err = t.Execute(&buf, map[string]string{
+ "Dir": s.dir(),
+ "Language": "%s",
+ "Ext": ext,
+ })
+ return filepath.FromSlash(buf.String()), wrap(err, "incorrect OutPattern")
+}
+
+var transRE = regexp.MustCompile(`.*\.` + gotextSuffix)
// Import loads existing translation files.
func (s *State) Import() error {
- panic("unimplemented")
+ outPattern, err := outPattern(s)
+ if err != nil {
+ return err
+ }
+ re := transRE
+ if pat := s.Config.TranslationsPattern; pat != "" {
+ if re, err = regexp.Compile(pat); err != nil {
+ return wrapf(err, "error parsing regexp %q", s.Config.TranslationsPattern)
+ }
+ }
+ x := importer{s, outPattern, re}
+ return x.walkImport(s.dir(), s.Config.SourceLanguage)
+}
+
+type importer struct {
+ state *State
+ outPattern string
+ transFile *regexp.Regexp
+}
+
+func (i *importer) walkImport(path string, tag language.Tag) error {
+ files, err := ioutil.ReadDir(path)
+ if err != nil {
+ return nil
+ }
+ for _, f := range files {
+ name := f.Name()
+ tag := tag
+ if f.IsDir() {
+ if t, err := language.Parse(name); err == nil {
+ tag = t
+ }
+ // We ignore errors
+ if err := i.walkImport(filepath.Join(path, name), tag); err != nil {
+ return err
+ }
+ continue
+ }
+ for _, l := range strings.Split(name, ".") {
+ if t, err := language.Parse(l); err == nil {
+ tag = t
+ }
+ }
+ file := filepath.Join(path, name)
+ // TODO: Should we skip files that match output files?
+ if fmt.Sprintf(i.outPattern, tag) == file {
+ continue
+ }
+ // TODO: handle different file formats.
+ if !i.transFile.MatchString(name) {
+ continue
+ }
+ b, err := ioutil.ReadFile(file)
+ if err != nil {
+ return wrap(err, "read file failed")
+ }
+ var translations Messages
+ if err := json.Unmarshal(b, &translations); err != nil {
+ return wrap(err, "parsing translation file failed")
+ }
+ i.state.Translations = append(i.state.Translations, translations)
+ }
return nil
}
@@ -160,9 +253,15 @@
// NOTE: The command line tool already prefixes with "gotext:".
var (
wrap = func(err error, msg string) error {
+ if err == nil {
+ return nil
+ }
return fmt.Errorf("%s: %v", msg, err)
}
wrapf = func(err error, msg string, args ...interface{}) error {
+ if err == nil {
+ return nil
+ }
return wrap(err, fmt.Sprintf(msg, args...))
}
errorf = fmt.Errorf
diff --git a/message/pipeline/pipeline_test.go b/message/pipeline/pipeline_test.go
new file mode 100644
index 0000000..cb1d3b5
--- /dev/null
+++ b/message/pipeline/pipeline_test.go
@@ -0,0 +1,113 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package pipeline
+
+import (
+ "bufio"
+ "bytes"
+ "encoding/json"
+ "flag"
+ "fmt"
+ "io/ioutil"
+ "os"
+ "path"
+ "path/filepath"
+ "testing"
+)
+
+var genFiles = flag.Bool("gen", false, "generate output files instead of comparing")
+
+func TestFullCycle(t *testing.T) {
+ const path = "./testdata"
+ dirs, err := ioutil.ReadDir(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, f := range dirs {
+ t.Run(f.Name(), func(t *testing.T) {
+ chk := func(t *testing.T, err error) {
+ t.Helper()
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+ dir := filepath.Join(path, f.Name())
+ pkgPath := fmt.Sprintf("%s/%s", path, f.Name())
+ config := Config{
+ Packages: []string{pkgPath},
+ Dir: filepath.Join(dir, "locales"),
+ GenFile: "catalog_gen.go",
+ GenPackage: pkgPath,
+ }
+ // TODO: load config if available.
+ s, err := Extract(&config)
+ chk(t, err)
+ chk(t, s.Import())
+ // chk(t, s.Merge()) // TODO
+ // TODO:
+ // for range s.Config.Actions {
+ // // TODO: do the actions.
+ // }
+ // chk(t, s.Export()) // TODO
+ chk(t, s.Generate())
+
+ writeJSON(t, filepath.Join(dir, "extracted.gotext.json"), s.Extracted)
+ checkOutput(t, dir)
+ })
+ }
+}
+
+func checkOutput(t *testing.T, p string) {
+ filepath.Walk(p, func(p string, f os.FileInfo, err error) error {
+ if f.IsDir() {
+ return nil
+ }
+ if filepath.Ext(p) != ".want" {
+ return nil
+ }
+ gotFile := p[:len(p)-len(".want")]
+ got, err := ioutil.ReadFile(gotFile)
+ if err != nil {
+ t.Errorf("failed to read %q", p)
+ return nil
+ }
+ if *genFiles {
+ if err := ioutil.WriteFile(p, got, 0644); err != nil {
+ t.Fatal(err)
+ }
+ }
+ want, err := ioutil.ReadFile(p)
+ if err != nil {
+ t.Errorf("failed to read %q", p)
+ } else {
+ scanGot := bufio.NewScanner(bytes.NewReader(got))
+ scanWant := bufio.NewScanner(bytes.NewReader(want))
+ line := 0
+ for scanGot.Scan() && scanWant.Scan() {
+ got := path.Clean(filepath.ToSlash(scanGot.Text()))
+ want := path.Clean(filepath.ToSlash(scanWant.Text()))
+ if got != want {
+ t.Errorf("file %q differs from .want file at line %d:\n\t%s\n\t%s", gotFile, line, got, want)
+ break
+ }
+ line++
+ }
+ if scanGot.Scan() || scanWant.Scan() {
+ t.Errorf("file %q differs from .want file at line %d.", gotFile, line)
+ }
+ }
+ return nil
+ })
+}
+
+func writeJSON(t *testing.T, path string, x interface{}) {
+ data, err := json.MarshalIndent(x, "", " ")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := ioutil.WriteFile(path, data, 0644); err != nil {
+ t.Fatal(err)
+ }
+}
diff --git a/message/pipeline/testdata/test1/catalog_gen.go b/message/pipeline/testdata/test1/catalog_gen.go
new file mode 100644
index 0000000..76b0bf0
--- /dev/null
+++ b/message/pipeline/testdata/test1/catalog_gen.go
@@ -0,0 +1,81 @@
+// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
+
+package main
+
+import (
+ "golang.org/x/text/language"
+ "golang.org/x/text/message"
+ "golang.org/x/text/message/catalog"
+)
+
+type dictionary struct {
+ index []uint32
+ data string
+}
+
+func (d *dictionary) Lookup(key string) (data string, ok bool) {
+ p := messageKeyToIndex[key]
+ start, end := d.index[p], d.index[p+1]
+ if start == end {
+ return "", false
+ }
+ return d.data[start:end], true
+}
+
+func init() {
+ dict := map[string]catalog.Dictionary{
+ "de": &dictionary{index: deIndex, data: deData},
+ "en_US": &dictionary{index: en_USIndex, data: en_USData},
+ "zh": &dictionary{index: zhIndex, data: zhData},
+ }
+ fallback := language.MustParse("und")
+ cat, err := catalog.NewFromMap(dict, catalog.Fallback(fallback))
+ if err != nil {
+ panic(err)
+ }
+ message.DefaultCatalog = cat
+}
+
+var messageKeyToIndex = map[string]int{
+ "%.2[1]f miles traveled (%[1]f)": 7,
+ "%[1]s is visiting %[3]s!\n": 3,
+ "%d files remaining!": 4,
+ "%d more files remaining!": 5,
+ "%s is out of order!": 6,
+ "%s is visiting %s!\n": 2,
+ "Hello %s!\n": 1,
+ "Hello world!\n": 0,
+}
+
+var deIndex = []uint32{ // 9 elements
+ 0x00000000, 0x0000000d, 0x0000001b, 0x00000031,
+ 0x00000047, 0x00000065, 0x00000084, 0x00000084,
+ 0x00000084,
+} // Size: 60 bytes
+
+const deData string = "" + // Size: 132 bytes
+ "\x02Hallo Welt!\x0a\x02Hallo %[1]s!\x0a\x02%[1]s besucht %[2]s!\x0a\x02%" +
+ "[1]s besucht %[3]s!\x0a\x02Noch zwei Bestände zu gehen!\x02Noch %[1]d Be" +
+ "stände zu gehen!"
+
+var en_USIndex = []uint32{ // 9 elements
+ 0x00000000, 0x0000000e, 0x0000001c, 0x00000036,
+ 0x00000050, 0x00000050, 0x00000093, 0x000000aa,
+ 0x000000c9,
+} // Size: 60 bytes
+
+const en_USData string = "" + // Size: 201 bytes
+ "\x02Hello world!\x0a\x02Hello %[1]s!\x0a\x02%[1]s is visiting %[2]s!\x0a" +
+ "\x02%[1]s is visiting %[3]s!\x0a\x04\x01\x81\x01\x00\x02\x14\x02One file" +
+ " remaining!\x00&\x02There are %[1]d more files remaining!\x02%[1]s is ou" +
+ "t of order!\x02%.2[1]f miles traveled (%[1]f)"
+
+var zhIndex = []uint32{ // 9 elements
+ 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+ 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+ 0x00000000,
+} // Size: 60 bytes
+
+const zhData string = ""
+
+// Total table size 513 bytes (0KiB); checksum: B21180E3
diff --git a/cmd/gotext/examples/extract/locales/extracted.gotext.json b/message/pipeline/testdata/test1/extracted.gotext.json
old mode 100755
new mode 100644
similarity index 77%
rename from cmd/gotext/examples/extract/locales/extracted.gotext.json
rename to message/pipeline/testdata/test1/extracted.gotext.json
index 27d8b56..dfb9153
--- a/cmd/gotext/examples/extract/locales/extracted.gotext.json
+++ b/message/pipeline/testdata/test1/extracted.gotext.json
@@ -1,12 +1,12 @@
{
- "language": "en-US",
+ "language": "und",
"messages": [
{
"id": "Hello world!\n",
"key": "Hello world!\n",
"message": "Hello world!\n",
"translation": "",
- "position": "golang.org/x/text/cmd/gotext/examples/extract/main.go:28:10"
+ "position": "testdata/test1/test1.go:19:10"
},
{
"id": "Hello {City}!\n",
@@ -23,25 +23,7 @@
"expr": "city"
}
],
- "position": "golang.org/x/text/cmd/gotext/examples/extract/main.go:32:10"
- },
- {
- "id": "Hello {Town}!\n",
- "key": "Hello %s!\n",
- "message": "Hello {Town}!\n",
- "translation": "",
- "placeholders": [
- {
- "id": "Town",
- "string": "%[1]s",
- "type": "string",
- "underlyingType": "string",
- "argNum": 1,
- "expr": "town",
- "comment": "Town"
- }
- ],
- "position": "golang.org/x/text/cmd/gotext/examples/extract/main.go:36:10"
+ "position": "testdata/test1/test1.go:24:10"
},
{
"id": "{Person} is visiting {Place}!\n",
@@ -68,14 +50,14 @@
"comment": "Place the person is visiting."
}
],
- "position": "golang.org/x/text/cmd/gotext/examples/extract/main.go:41:10"
+ "position": "testdata/test1/test1.go:30:10"
},
{
"id": "{Person} is visiting {Place}!\n",
"key": "%[1]s is visiting %[3]s!\n",
"message": "{Person} is visiting {Place}!\n",
"translation": "",
- "comment": "Person visiting a place.",
+ "comment": "Field names are placeholders.",
"placeholders": [
{
"id": "Person",
@@ -103,7 +85,7 @@
"expr": "pp.extra"
}
],
- "position": "golang.org/x/text/cmd/gotext/examples/extract/main.go:56:10"
+ "position": "testdata/test1/test1.go:44:10"
},
{
"id": "{2} files remaining!",
@@ -120,7 +102,7 @@
"expr": "2"
}
],
- "position": "golang.org/x/text/cmd/gotext/examples/extract/main.go:63:10"
+ "position": "testdata/test1/test1.go:51:10"
},
{
"id": "{N} more files remaining!",
@@ -137,7 +119,7 @@
"expr": "n"
}
],
- "position": "golang.org/x/text/cmd/gotext/examples/extract/main.go:68:10"
+ "position": "testdata/test1/test1.go:56:10"
},
{
"id": "Use the following code for your discount: {ReferralCode}\n",
@@ -148,13 +130,13 @@
{
"id": "ReferralCode",
"string": "%[1]d",
- "type": "golang.org/x/text/cmd/gotext/examples/extract.referralCode",
+ "type": "./testdata/test1.referralCode",
"underlyingType": "int",
"argNum": 1,
"expr": "c"
}
],
- "position": "golang.org/x/text/cmd/gotext/examples/extract/main.go:74:10"
+ "position": "testdata/test1/test1.go:64:10"
},
{
"id": [
@@ -164,7 +146,7 @@
"key": "%s is out of order!",
"message": "{Device} is out of order!",
"translation": "",
- "comment": "FOO\n",
+ "comment": "This comment wins.\n",
"placeholders": [
{
"id": "Device",
@@ -175,7 +157,7 @@
"expr": "device"
}
],
- "position": "golang.org/x/text/cmd/gotext/examples/extract/main.go:82:10"
+ "position": "testdata/test1/test1.go:70:10"
},
{
"id": "{Miles} miles traveled ({Miles_1})",
@@ -200,7 +182,7 @@
"expr": "miles"
}
],
- "position": "golang.org/x/text/cmd/gotext/examples/extract/main.go:86:10"
+ "position": "testdata/test1/test1.go:74:10"
}
]
}
\ No newline at end of file
diff --git a/cmd/gotext/examples/extract/locales/extracted.gotext.json b/message/pipeline/testdata/test1/extracted.gotext.json.want
old mode 100755
new mode 100644
similarity index 77%
copy from cmd/gotext/examples/extract/locales/extracted.gotext.json
copy to message/pipeline/testdata/test1/extracted.gotext.json.want
index 27d8b56..dfb9153
--- a/cmd/gotext/examples/extract/locales/extracted.gotext.json
+++ b/message/pipeline/testdata/test1/extracted.gotext.json.want
@@ -1,12 +1,12 @@
{
- "language": "en-US",
+ "language": "und",
"messages": [
{
"id": "Hello world!\n",
"key": "Hello world!\n",
"message": "Hello world!\n",
"translation": "",
- "position": "golang.org/x/text/cmd/gotext/examples/extract/main.go:28:10"
+ "position": "testdata/test1/test1.go:19:10"
},
{
"id": "Hello {City}!\n",
@@ -23,25 +23,7 @@
"expr": "city"
}
],
- "position": "golang.org/x/text/cmd/gotext/examples/extract/main.go:32:10"
- },
- {
- "id": "Hello {Town}!\n",
- "key": "Hello %s!\n",
- "message": "Hello {Town}!\n",
- "translation": "",
- "placeholders": [
- {
- "id": "Town",
- "string": "%[1]s",
- "type": "string",
- "underlyingType": "string",
- "argNum": 1,
- "expr": "town",
- "comment": "Town"
- }
- ],
- "position": "golang.org/x/text/cmd/gotext/examples/extract/main.go:36:10"
+ "position": "testdata/test1/test1.go:24:10"
},
{
"id": "{Person} is visiting {Place}!\n",
@@ -68,14 +50,14 @@
"comment": "Place the person is visiting."
}
],
- "position": "golang.org/x/text/cmd/gotext/examples/extract/main.go:41:10"
+ "position": "testdata/test1/test1.go:30:10"
},
{
"id": "{Person} is visiting {Place}!\n",
"key": "%[1]s is visiting %[3]s!\n",
"message": "{Person} is visiting {Place}!\n",
"translation": "",
- "comment": "Person visiting a place.",
+ "comment": "Field names are placeholders.",
"placeholders": [
{
"id": "Person",
@@ -103,7 +85,7 @@
"expr": "pp.extra"
}
],
- "position": "golang.org/x/text/cmd/gotext/examples/extract/main.go:56:10"
+ "position": "testdata/test1/test1.go:44:10"
},
{
"id": "{2} files remaining!",
@@ -120,7 +102,7 @@
"expr": "2"
}
],
- "position": "golang.org/x/text/cmd/gotext/examples/extract/main.go:63:10"
+ "position": "testdata/test1/test1.go:51:10"
},
{
"id": "{N} more files remaining!",
@@ -137,7 +119,7 @@
"expr": "n"
}
],
- "position": "golang.org/x/text/cmd/gotext/examples/extract/main.go:68:10"
+ "position": "testdata/test1/test1.go:56:10"
},
{
"id": "Use the following code for your discount: {ReferralCode}\n",
@@ -148,13 +130,13 @@
{
"id": "ReferralCode",
"string": "%[1]d",
- "type": "golang.org/x/text/cmd/gotext/examples/extract.referralCode",
+ "type": "./testdata/test1.referralCode",
"underlyingType": "int",
"argNum": 1,
"expr": "c"
}
],
- "position": "golang.org/x/text/cmd/gotext/examples/extract/main.go:74:10"
+ "position": "testdata/test1/test1.go:64:10"
},
{
"id": [
@@ -164,7 +146,7 @@
"key": "%s is out of order!",
"message": "{Device} is out of order!",
"translation": "",
- "comment": "FOO\n",
+ "comment": "This comment wins.\n",
"placeholders": [
{
"id": "Device",
@@ -175,7 +157,7 @@
"expr": "device"
}
],
- "position": "golang.org/x/text/cmd/gotext/examples/extract/main.go:82:10"
+ "position": "testdata/test1/test1.go:70:10"
},
{
"id": "{Miles} miles traveled ({Miles_1})",
@@ -200,7 +182,7 @@
"expr": "miles"
}
],
- "position": "golang.org/x/text/cmd/gotext/examples/extract/main.go:86:10"
+ "position": "testdata/test1/test1.go:74:10"
}
]
}
\ No newline at end of file
diff --git a/message/pipeline/testdata/test1/locales/de/messages.gotext.json b/message/pipeline/testdata/test1/locales/de/messages.gotext.json
new file mode 100755
index 0000000..a4a46ed
--- /dev/null
+++ b/message/pipeline/testdata/test1/locales/de/messages.gotext.json
@@ -0,0 +1,123 @@
+{
+ "language": "de",
+ "messages": [
+ {
+ "id": "Hello world!\n",
+ "key": "Hello world!\n",
+ "message": "Hello world!\n",
+ "translation": "Hallo Welt!\n"
+ },
+ {
+ "id": "Hello {City}!\n",
+ "key": "Hello %s!\n",
+ "message": "Hello {City}!\n",
+ "translation": "Hallo {City}!\n",
+ "placeholders": [
+ {
+ "id": "City",
+ "string": "%[1]s"
+ }
+ ]
+ },
+ {
+ "id": "{Person} is visiting {Place}!\n",
+ "key": "%s is visiting %s!\n",
+ "message": "{Person} is visiting {Place}!\n",
+ "translation": "{Person} besucht {Place}!\n",
+ "placeholders": [
+ {
+ "id": "Person",
+ "string": "%[1]s"
+ },
+ {
+ "id": "Place",
+ "string": "%[2]s"
+ }
+ ]
+ },
+ {
+ "id": "{Person} is visiting {Place}!\n",
+ "key": "%[1]s is visiting %[3]s!\n",
+ "message": "{Person} is visiting {Place}!\n",
+ "translation": "{Person} besucht {Place}!\n",
+ "placeholders": [
+ {
+ "id": "Person",
+ "string": "%[1]s"
+ },
+ {
+ "id": "Place",
+ "string": "%[3]s"
+ },
+ {
+ "id": "Extra",
+ "string": "%[2]v"
+ }
+ ]
+ },
+ {
+ "id": "{2} files remaining!",
+ "key": "%d files remaining!",
+ "message": "{N} files remaining!",
+ "translation": "Noch zwei Bestände zu gehen!",
+ "placeholders": [
+ {
+ "id": "2",
+ "string": "%[1]d"
+ }
+ ]
+ },
+ {
+ "id": "{N} more files remaining!",
+ "key": "%d more files remaining!",
+ "message": "{N} more files remaining!",
+ "translation": "Noch {N} Bestände zu gehen!",
+ "placeholders": [
+ {
+ "id": "N",
+ "string": "%[1]d"
+ }
+ ]
+ },
+ {
+ "id": "Use the following code for your discount: {ReferralCode}\n",
+ "key": "Use the following code for your discount: %d\n",
+ "message": "Use the following code for your discount: {ReferralCode}\n",
+ "translation": "",
+ "placeholders": [
+ {
+ "id": "ReferralCode",
+ "string": "%[1]d"
+ }
+ ]
+ },
+ {
+ "id": [ "msgOutOfOrder", "{Device} is out of order!" ],
+ "key": "%s is out of order!",
+ "message": "{Device} is out of order!",
+ "translation": "",
+ "placeholders": [
+ {
+ "id": "Device",
+ "string": "%[1]s"
+ }
+ ]
+ },
+ {
+ "id": "{Miles} miles traveled ({Miles_1})",
+ "key": "%.2[1]f miles traveled (%[1]f)",
+ "message": "{Miles} miles traveled ({Miles_1})",
+ "translation": "",
+ "placeholders": [
+ {
+ "id": "Miles",
+ "string": "%.2[1]f"
+ },
+ {
+ "id": "Miles_1",
+ "string": "%[1]f"
+ }
+ ]
+ }
+ ]
+}
diff --git a/message/pipeline/testdata/test1/locales/en-US/messages.gotext.json b/message/pipeline/testdata/test1/locales/en-US/messages.gotext.json
new file mode 100755
index 0000000..bf97577
--- /dev/null
+++ b/message/pipeline/testdata/test1/locales/en-US/messages.gotext.json
@@ -0,0 +1,91 @@
+{
+ "language": "en-US",
+ "messages": [
+ {
+"id": "Hello world!\n",
+ "key": "Hello world!\n",
+ "message": "Hello world!\n",
+ "translation": "Hello world!\n"
+ },
+ {
+ "id": "Hello {City}!\n",
+ "key": "Hello %s!\n",
+ "message": "Hello {City}!\n",
+ "translation": "Hello {City}!\n"
+ },
+ {
+ "id": "Hello {Town}!\n",
+ "key": "Hello %s!\n",
+ "message": "Hello {Town}!\n",
+ "translation": "Hello {Town}!\n",
+ "placeholders": [
+ {
+ "id": "Town",
+ "string": "%[1]s",
+ "type": "string",
+ "underlyingType": "string",
+ "argNum": 1,
+ "expr": "town",
+ "comment": "Town"
+ }
+ ]
+ },
+ {
+ "id": "{Person} is visiting {Place}!\n",
+ "key": "%s is visiting %s!\n",
+ "message": "{Person} is visiting {Place}!\n",
+ "translation": "{Person} is visiting {Place}!\n"
+ },
+ {
+ "id": "{Person} is visiting {Place}!\n",
+ "key": "%[1]s is visiting %[3]s!\n",
+ "message": "{Person} is visiting {Place}!\n",
+ "translation": "{Person} is visiting {Place}!\n"
+ },
+ {
+ "id": "{2} files remaining!",
+ "key": "%d files remaining!",
+ "message": "{N} files remaining!",
+ "translation": "",
+ "placeholders": [
+ {
+ "id": "2",
+ "string": "%[1]d"
+ }
+ ]
+ },
+ {
+ "id": "{N} more files remaining!",
+ "key": "%d more files remaining!",
+ "message": "{N} more files remaining!",
+ "translation": {
+ "select": {
+ "feature": "plural",
+ "arg": "N",
+ "cases": {
+ "one": "One file remaining!",
+ "other": "There are {N} more files remaining!"
+ }
+ }
+ }
+ },
+ {
+ "id": "Use the following code for your discount: {ReferralCode}\n",
+ "key": "Use the following code for your discount: %d\n",
+ "message": "Use the following code for your discount: {ReferralCode}\n",
+ "translation": ""
+ },
+ {
+ "id": [ "msgOutOfOrder", "{Device} is out of order!" ],
+ "key": "%s is out of order!",
+ "message": "{Device} is out of order!",
+ "translation": "{Device} is out of order!"
+ },
+ {
+ "id": "{Miles} miles traveled ({Miles_1})",
+ "key": "%.2[1]f miles traveled (%[1]f)",
+ "message": "{Miles} miles traveled ({Miles_1})",
+ "translation": "{Miles} miles traveled ({Miles_1})"
+ }
+ ]
+}
diff --git a/message/pipeline/testdata/test1/locales/zh/messages.gotext.json b/message/pipeline/testdata/test1/locales/zh/messages.gotext.json
new file mode 100755
index 0000000..557a3d2
--- /dev/null
+++ b/message/pipeline/testdata/test1/locales/zh/messages.gotext.json
@@ -0,0 +1,135 @@
+{
+ "language": "zh",
+ "messages": [
+ {
+ "id": "Hello world!\n",
+ "key": "Hello world!\n",
+ "message": "Hello world!\n",
+ "translation": ""
+ },
+ {
+ "id": "Hello {City}!\n",
+ "key": "Hello %s!\n",
+ "message": "Hello {City}!\n",
+ "translation": "",
+ "placeholders": [
+ {
+ "id": "City",
+ "string": "%[1]s"
+ }
+ ]
+ },
+ {
+ "id": "Hello {Town}!\n",
+ "key": "Hello %s!\n",
+ "message": "Hello {Town}!\n",
+ "translation": "",
+ "placeholders": [
+ {
+ "id": "Town",
+ "string": "%[1]s"
+ }
+ ]
+ },
+ {
+ "id": "{Person} is visiting {Place}!\n",
+ "key": "%s is visiting %s!\n",
+ "message": "{Person} is visiting {Place}!\n",
+ "translation": "",
+ "placeholders": [
+ {
+ "id": "Person",
+ "string": "%[1]s"
+ },
+ {
+ "id": "Place",
+ "string": "%[2]s"
+ }
+ ]
+ },
+ {
+ "id": "{Person} is visiting {Place}!\n",
+ "key": "%[1]s is visiting %[3]s!\n",
+ "message": "{Person} is visiting {Place}!\n",
+ "translation": "",
+ "placeholders": [
+ {
+ "id": "Person",
+ "string": "%[1]s"
+ },
+ {
+ "id": "Place",
+ "string": "%[3]s"
+ },
+ {
+ "id": "Extra",
+ "string": "%[2]v"
+ }
+ ]
+ },
+ {
+ "id": "{2} files remaining!",
+ "key": "%d files remaining!",
+ "message": "{2} files remaining!",
+ "translation": "",
+ "placeholders": [
+ {
+ "id": "",
+ "string": "%[1]d"
+ }
+ ]
+ },
+ {
+ "id": "{N} more files remaining!",
+ "key": "%d more files remaining!",
+ "message": "{N} more files remaining!",
+ "translation": "",
+ "placeholders": [
+ {
+ "id": "N",
+ "string": "%[1]d"
+ }
+ ]
+ },
+ {
+ "id": "Use the following code for your discount: {ReferralCode}\n",
+ "key": "Use the following code for your discount: %d\n",
+ "message": "Use the following code for your discount: {ReferralCode}\n",
+ "translation": "",
+ "placeholders": [
+ {
+ "id": "ReferralCode",
+ "string": "%[1]d"
+ }
+ ]
+ },
+ {
+ "id": [ "{Device} is out of order!", "msgOutOfOrder" ],
+ "key": "%s is out of order!",
+ "message": "{Device} is out of order!",
+ "translation": "",
+ "placeholders": [
+ {
+ "id": "Device",
+ "string": "%[1]s"
+ }
+ ]
+ },
+ {
+ "id": "{Miles} miles traveled ({Miles_1})",
+ "key": "%.2[1]f miles traveled (%[1]f)",
+ "message": "{Miles} miles traveled ({Miles_1})",
+ "translation": "",
+ "placeholders": [
+ {
+ "id": "Miles",
+ "string": "%.2[1]f"
+ },
+ {
+ "id": "Miles_1",
+ "string": "%[1]f"
+ }
+ ]
+ }
+ ]
+}
\ No newline at end of file
diff --git a/message/pipeline/testdata/test1/test1.go b/message/pipeline/testdata/test1/test1.go
new file mode 100644
index 0000000..88051f9
--- /dev/null
+++ b/message/pipeline/testdata/test1/test1.go
@@ -0,0 +1,75 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import "golang.org/x/text/message"
+
+func main() {
+ p := message.NewPrinter(message.MatchLanguage("en"))
+
+ // NOT EXTRACTED: strings passed to Println are not extracted.
+ p.Println("Hello world!")
+
+ // NOT EXTRACTED: strings passed to Print are not extracted.
+ p.Print("Hello world!\n")
+
+ // Extract and trim whitespace (TODO).
+ p.Printf("Hello world!\n")
+
+ // NOT EXTRACTED: city is not used as a pattern or passed to %m.
+ city := "Amsterdam"
+ // This comment is extracted.
+ p.Printf("Hello %s!\n", city)
+
+ person := "Sheila"
+ place := "Zürich"
+
+ // Substitutions replaced by variable names.
+ p.Printf("%s is visiting %s!\n",
+ person, // The person of matter.
+ place, // Place the person is visiting.
+ )
+
+ pp := struct {
+ Person string // The person of matter. // TODO: get this comment.
+ Place string
+ extra int
+ }{
+ person, place, 4,
+ }
+
+ // extract will drop this comment in favor of the one below.
+ p.Printf("%[1]s is visiting %[3]s!\n", // Field names are placeholders.
+ pp.Person,
+ pp.extra,
+ pp.Place, // Place the person is visiting.
+ )
+
+ // Numeric literal becomes placeholder.
+ p.Printf("%d files remaining!", 2)
+
+ const n = 2
+
+ // Constant identifier becomes placeholder.
+ p.Printf("%d more files remaining!", n)
+
+ // Infer better names from type names.
+ type referralCode int
+
+ const c = referralCode(5)
+
+ // Use type name as placeholder.
+ p.Printf("Use the following code for your discount: %d\n", c)
+
+ // Use constant name as message ID.
+ const msgOutOfOrder = "%s is out of order!" // This comment wins.
+ const device = "Soda machine"
+ // This message has two IDs.
+ p.Printf(msgOutOfOrder, device)
+
+ // Multiple substitutions for same argument.
+ miles := 1.2345
+ p.Printf("%.2[1]f miles traveled (%[1]f)", miles)
+}
