cmd/gotext: add update command This is the recommended approach and will likely replace export and generate. Change-Id: Ie7dc0523c580715b8576de074f312b490a8727b6 Reviewed-on: https://go-review.googlesource.com/83776 Run-TryBot: Marcel van Lohuizen <mpvl@golang.org> TryBot-Result: Gobot Gobot <gobot@golang.org> Reviewed-by: Nigel Tao <nigeltao@golang.org>
diff --git a/cmd/gotext/examples/extract/catalog.go b/cmd/gotext/examples/extract/catalog.go index 1a0d6d0..bc6130a 100644 --- a/cmd/gotext/examples/extract/catalog.go +++ b/cmd/gotext/examples/extract/catalog.go
@@ -37,42 +37,48 @@ } var messageKeyToIndex = map[string]int{ - "%.2[1]f miles traveled (%[1]f)": 6, - "%[1]s is visiting %[3]s!\n": 3, - "%d more files remaining!": 4, - "%s is out of order!": 5, - "%s is visiting %s!\n": 2, - "Hello %s!\n": 1, - "Hello world!\n": 0, + "%.2[1]f miles traveled (%[1]f)": 8, + "%[1]s is visiting %[3]s!\n": 3, + "%d files remaining!": 5, + "%d more files remaining!": 4, + "%s is out of order!": 7, + "%s is visiting %s!\n": 2, + "Hello %s!\n": 1, + "Hello world!\n": 0, + "Use the following code for your discount: %d\n": 6, } -var deIndex = []uint32{ // 8 elements +var deIndex = []uint32{ // 10 elements 0x00000000, 0x00000011, 0x00000023, 0x0000003d, 0x00000057, 0x00000076, 0x00000076, 0x00000076, -} // Size: 56 bytes + 0x00000076, 0x00000076, +} // Size: 64 bytes const deData string = "" + // Size: 118 bytes "\x04\x00\x01\x0a\x0c\x02Hallo Welt!\x04\x00\x01\x0a\x0d\x02Hallo %[1]s!" + "\x04\x00\x01\x0a\x15\x02%[1]s besucht %[2]s!\x04\x00\x01\x0a\x15\x02%[1]" + "s besucht %[3]s!\x02Noch %[1]d Bestände zu gehen!" -var en_USIndex = []uint32{ // 8 elements +var en_USIndex = []uint32{ // 10 elements 0x00000000, 0x00000012, 0x00000024, 0x00000042, - 0x00000060, 0x000000a3, 0x000000ba, 0x000000d9, -} // Size: 56 bytes + 0x00000060, 0x000000a3, 0x000000ba, 0x000000ef, + 0x00000106, 0x00000125, +} // Size: 64 bytes -const en_USData string = "" + // Size: 217 bytes - "\x04\x00\x01\x0a\x0d\x02Hello world!\x04\x00\x01\x0a\x0d\x02Hello %[1]s!" + +const en_USData string = "" + // Size: 293 bytes + "\x04\x00\x01\x0a\x0d\x02Hello world!\x04\x00\x01\x0a\x0d\x02Hello %[1]sn" + "\x04\x00\x01\x0a\x19\x02%[1]s is visiting %[2]s!\x04\x00\x01\x0a\x19\x02" + "%[1]s is visiting %[3]s!\x14\x01\x81\x01\x00\x02\x14\x02One file remaini" + - "ng!\x00&\x02There are %[1]d more files remaining!\x02%[1]s is out of ord" + - "er!\x02%.2[1]f miles traveled (%[1]f)" + "ng!\x00&\x02There are %[1]d more files remaining!\x02%[1]d files remaini" + + "ng!\x04\x00\x01\x0a0\x02Use the following code for your discount: %[1]d" + + "\x02%[1]s is out of order!\x02%.2[1]f miles traveled (%[1]f)" -var zhIndex = []uint32{ // 8 elements +var zhIndex = []uint32{ // 10 elements 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, -} // Size: 56 bytes + 0x00000000, 0x00000000, +} // Size: 64 bytes const zhData string = "" -// Total table size 503 bytes (0KiB); checksum: A968BD6 +// Total table size 603 bytes (0KiB); checksum: 1D2754EE
diff --git a/cmd/gotext/examples/extract/main.go b/cmd/gotext/examples/extract/main.go index cbbc370..414b453 100644 --- a/cmd/gotext/examples/extract/main.go +++ b/cmd/gotext/examples/extract/main.go
@@ -4,8 +4,7 @@ package main -//go:generate gotext extract --lang=de,zh -//go:generate gotext generate -out catalog.go +//go:generate gotext update -out catalog.go import ( "golang.org/x/text/language"
diff --git a/cmd/gotext/examples/extract_http/catalog_gen.go b/cmd/gotext/examples/extract_http/catalog_gen.go new file mode 100644 index 0000000..2c410dc --- /dev/null +++ b/cmd/gotext/examples/extract_http/catalog_gen.go
@@ -0,0 +1,57 @@ +// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. + +package main + +import ( + "golang.org/x/text/language" + "golang.org/x/text/message" + "golang.org/x/text/message/catalog" +) + +type dictionary struct { + index []uint32 + data string +} + +func (d *dictionary) Lookup(key string) (data string, ok bool) { + p := messageKeyToIndex[key] + start, end := d.index[p], d.index[p+1] + if start == end { + return "", false + } + return d.data[start:end], true +} + +func init() { + dict := map[string]catalog.Dictionary{ + "en": &dictionary{index: enIndex, data: enData}, + "zh": &dictionary{index: zhIndex, data: zhData}, + } + fallback := language.MustParse("en") + cat, err := catalog.NewFromMap(dict, catalog.Fallback(fallback)) + if err != nil { + panic(err) + } + message.DefaultCatalog = cat +} + +var messageKeyToIndex = map[string]int{ + "Do you like your browser (%s)?\n": 1, + "Hello %s!\n": 0, +} + +var enIndex = []uint32{ // 3 elements + 0x00000000, 0x00000012, 0x00000039, +} // Size: 36 bytes + +const enData string = "" + // Size: 57 bytes + "\x04\x00\x01\x0a\x0d\x02Hello %[1]s!\x04\x00\x01\x0a\x22\x02Do you like " + + "your browser (%[1]s)?" + +var zhIndex = []uint32{ // 3 elements + 0x00000000, 0x00000000, 0x00000000, +} // Size: 36 bytes + +const zhData string = "" + +// Total table size 129 bytes (0KiB); checksum: 9C146C82
diff --git a/cmd/gotext/examples/extract_http/locales/en/out.gotext.json b/cmd/gotext/examples/extract_http/locales/en/out.gotext.json new file mode 100644 index 0000000..1391e58 --- /dev/null +++ b/cmd/gotext/examples/extract_http/locales/en/out.gotext.json
@@ -0,0 +1,39 @@ +{ + "language": "en", + "messages": [ + { + "id": "Hello {From}!", + "message": "Hello {From}!", + "translation": "Hello {From}!", + "translatorComment": "Copied from source.", + "placeholders": [ + { + "id": "From", + "string": "%[1]s", + "type": "string", + "underlyingType": "string", + "argNum": 1, + "expr": "r.Header.Get(\"From\")" + } + ], + "fuzzy": true + }, + { + "id": "Do you like your browser ({User_Agent})?", + "message": "Do you like your browser ({User_Agent})?", + "translation": "Do you like your browser ({User_Agent})?", + "translatorComment": "Copied from source.", + "placeholders": [ + { + "id": "User_Agent", + "string": "%[1]s", + "type": "string", + "underlyingType": "string", + "argNum": 1, + "expr": "r.Header.Get(\"User-Agent\")" + } + ], + "fuzzy": true + } + ] +} \ No newline at end of file
diff --git a/cmd/gotext/examples/extract_http/locales/zh/out.gotext.json b/cmd/gotext/examples/extract_http/locales/zh/out.gotext.json index 0bab4a7..7b26974 100755 --- a/cmd/gotext/examples/extract_http/locales/zh/out.gotext.json +++ b/cmd/gotext/examples/extract_http/locales/zh/out.gotext.json
@@ -3,7 +3,6 @@ "messages": [ { "id": "Hello {From}!", - "key": "Hello %s!\n", "message": "Hello {From}!", "translation": "", "placeholders": [ @@ -15,12 +14,10 @@ "argNum": 1, "expr": "r.Header.Get(\"From\")" } - ], - "position": "golang.org/x/text/cmd/gotext/examples/extract_http/pkg/pkg.go:22:11" + ] }, { "id": "Do you like your browser ({User_Agent})?", - "key": "Do you like your browser (%s)?\n", "message": "Do you like your browser ({User_Agent})?", "translation": "", "placeholders": [ @@ -32,8 +29,7 @@ "argNum": 1, "expr": "r.Header.Get(\"User-Agent\")" } - ], - "position": "golang.org/x/text/cmd/gotext/examples/extract_http/pkg/pkg.go:24:11" + ] } ] } \ No newline at end of file
diff --git a/cmd/gotext/examples/extract_http/main.go b/cmd/gotext/examples/extract_http/main.go index c1ac449..b5eb3b33 100644 --- a/cmd/gotext/examples/extract_http/main.go +++ b/cmd/gotext/examples/extract_http/main.go
@@ -4,7 +4,7 @@ package main -//go:generate gotext extract --lang=de,zh +//go:generate gotext -srclang=en update -out=catalog_gen.go -lang=en,zh import ( "net/http"
diff --git a/cmd/gotext/extract.go b/cmd/gotext/extract.go index b59097b..103d7e6 100644 --- a/cmd/gotext/extract.go +++ b/cmd/gotext/extract.go
@@ -14,10 +14,6 @@ // - handle features (gender, plural) // - message rewriting -var ( - lang *string -) - func init() { lang = cmdExtract.Flag.String("lang", "en-US", "comma-separated list of languages to process") }
diff --git a/cmd/gotext/generate.go b/cmd/gotext/generate.go index c8d5d79..36820df 100644 --- a/cmd/gotext/generate.go +++ b/cmd/gotext/generate.go
@@ -12,10 +12,6 @@ out = cmdGenerate.Flag.String("out", "", "output file to write to") } -var ( - out *string -) - var cmdGenerate = &Command{ Run: runGenerate, UsageLine: "generate <package>",
diff --git a/cmd/gotext/main.go b/cmd/gotext/main.go index 1e6c3aa..73f6d91 100644 --- a/cmd/gotext/main.go +++ b/cmd/gotext/main.go
@@ -47,6 +47,7 @@ } return &pipeline.Config{ SourceLanguage: tag, + Supported: getLangs(), TranslationsPattern: `messages\.(.*)\.json`, GenFile: *out, }, nil @@ -100,6 +101,7 @@ // Commands lists the available commands and help topics. // The order here is the order in which they are printed by 'go help'. var commands = []*Command{ + cmdUpdate, cmdExtract, cmdRewrite, cmdGenerate,
diff --git a/cmd/gotext/update.go b/cmd/gotext/update.go new file mode 100644 index 0000000..1260750 --- /dev/null +++ b/cmd/gotext/update.go
@@ -0,0 +1,52 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "golang.org/x/text/message/pipeline" +) + +// TODO: +// - merge information into existing files +// - handle different file formats (PO, XLIFF) +// - handle features (gender, plural) +// - message rewriting + +var ( + lang *string + out *string +) + +func init() { + lang = cmdUpdate.Flag.String("lang", "en-US", "comma-separated list of languages to process") + out = cmdUpdate.Flag.String("out", "", "output file to write to") +} + +var cmdUpdate = &Command{ + Run: runUpdate, + UsageLine: "update <package>* [-out <gofile>]", + Short: "merge translations and generate catalog", +} + +func runUpdate(cmd *Command, config *pipeline.Config, args []string) error { + config.Packages = args + state, err := pipeline.Extract(config) + if err != nil { + return wrap(err, "extract failed") + } + if err := state.Import(); err != nil { + return wrap(err, "import failed") + } + if err := state.Merge(); err != nil { + return wrap(err, "merge failed") + } + if err := state.Export(); err != nil { + return wrap(err, "export failed") + } + if *out != "" { + return wrap(state.Generate(), "generation failed") + } + return nil +}
diff --git a/message/pipeline/generate.go b/message/pipeline/generate.go index cb62c25..5d329b2 100644 --- a/message/pipeline/generate.go +++ b/message/pipeline/generate.go
@@ -26,6 +26,8 @@ var transRe = regexp.MustCompile(`messages\.(.*)\.json`) // Generate writes a Go file that defines a Catalog with translated messages. +// Translations are retrieved from s.Messages, not s.Translations, so it +// is assumed Merge has been called. func (s *State) Generate() error { path := s.Config.GenPackage if path == "" { @@ -56,7 +58,8 @@ } // WriteGen writes a Go file with the given package name to w that defines a -// Catalog with translated messages. +// Catalog with translated messages. Translations are retrieved from s.Messages, +// not s.Translations, so it is assumed Merge has been called. func (s *State) WriteGen(w io.Writer, pkg string) error { cw, err := s.generate() if err != nil { @@ -80,15 +83,12 @@ } func (s *State) generate() (*gen.CodeWriter, error) { - // TODO: add in external input. Right now we assume that all files are - // manually created and stored in the textdata directory. - // Build up index of translations and original messages. translations := map[language.Tag]map[string]Message{} languages := []language.Tag{} usedKeys := map[string]int{} - for _, loc := range s.Translations { + for _, loc := range s.Messages { tag := loc.Language if _, ok := translations[tag]; !ok { translations[tag] = map[string]Message{}
diff --git a/message/pipeline/pipeline.go b/message/pipeline/pipeline.go index 93d2602..cafd6f2 100644 --- a/message/pipeline/pipeline.go +++ b/message/pipeline/pipeline.go
@@ -289,7 +289,7 @@ // Build index of translations. translations := map[language.Tag]map[string]Message{} - languages := []language.Tag{} + languages := append([]language.Tag{}, s.Config.Supported...) for _, t := range s.Translations { tag := t.Language @@ -308,7 +308,7 @@ } } } - internal.SortTags(languages) + languages = internal.UniqueTags(languages) for _, tag := range languages { ms := Messages{Language: tag}