cmd/gotext: add update command
This is the recommended approach and will
likely replace export and generate.
Change-Id: Ie7dc0523c580715b8576de074f312b490a8727b6
Reviewed-on: https://go-review.googlesource.com/83776
Run-TryBot: Marcel van Lohuizen <mpvl@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Nigel Tao <nigeltao@golang.org>
diff --git a/cmd/gotext/examples/extract/catalog.go b/cmd/gotext/examples/extract/catalog.go
index 1a0d6d0..bc6130a 100644
--- a/cmd/gotext/examples/extract/catalog.go
+++ b/cmd/gotext/examples/extract/catalog.go
@@ -37,42 +37,48 @@
}
var messageKeyToIndex = map[string]int{
- "%.2[1]f miles traveled (%[1]f)": 6,
- "%[1]s is visiting %[3]s!\n": 3,
- "%d more files remaining!": 4,
- "%s is out of order!": 5,
- "%s is visiting %s!\n": 2,
- "Hello %s!\n": 1,
- "Hello world!\n": 0,
+ "%.2[1]f miles traveled (%[1]f)": 8,
+ "%[1]s is visiting %[3]s!\n": 3,
+ "%d files remaining!": 5,
+ "%d more files remaining!": 4,
+ "%s is out of order!": 7,
+ "%s is visiting %s!\n": 2,
+ "Hello %s!\n": 1,
+ "Hello world!\n": 0,
+ "Use the following code for your discount: %d\n": 6,
}
-var deIndex = []uint32{ // 8 elements
+var deIndex = []uint32{ // 10 elements
0x00000000, 0x00000011, 0x00000023, 0x0000003d,
0x00000057, 0x00000076, 0x00000076, 0x00000076,
-} // Size: 56 bytes
+ 0x00000076, 0x00000076,
+} // Size: 64 bytes
const deData string = "" + // Size: 118 bytes
"\x04\x00\x01\x0a\x0c\x02Hallo Welt!\x04\x00\x01\x0a\x0d\x02Hallo %[1]s!" +
"\x04\x00\x01\x0a\x15\x02%[1]s besucht %[2]s!\x04\x00\x01\x0a\x15\x02%[1]" +
"s besucht %[3]s!\x02Noch %[1]d Bestände zu gehen!"
-var en_USIndex = []uint32{ // 8 elements
+var en_USIndex = []uint32{ // 10 elements
0x00000000, 0x00000012, 0x00000024, 0x00000042,
- 0x00000060, 0x000000a3, 0x000000ba, 0x000000d9,
-} // Size: 56 bytes
+ 0x00000060, 0x000000a3, 0x000000ba, 0x000000ef,
+ 0x00000106, 0x00000125,
+} // Size: 64 bytes
-const en_USData string = "" + // Size: 217 bytes
- "\x04\x00\x01\x0a\x0d\x02Hello world!\x04\x00\x01\x0a\x0d\x02Hello %[1]s!" +
+const en_USData string = "" + // Size: 293 bytes
+ "\x04\x00\x01\x0a\x0d\x02Hello world!\x04\x00\x01\x0a\x0d\x02Hello %[1]sn" +
"\x04\x00\x01\x0a\x19\x02%[1]s is visiting %[2]s!\x04\x00\x01\x0a\x19\x02" +
"%[1]s is visiting %[3]s!\x14\x01\x81\x01\x00\x02\x14\x02One file remaini" +
- "ng!\x00&\x02There are %[1]d more files remaining!\x02%[1]s is out of ord" +
- "er!\x02%.2[1]f miles traveled (%[1]f)"
+ "ng!\x00&\x02There are %[1]d more files remaining!\x02%[1]d files remaini" +
+ "ng!\x04\x00\x01\x0a0\x02Use the following code for your discount: %[1]d" +
+ "\x02%[1]s is out of order!\x02%.2[1]f miles traveled (%[1]f)"
-var zhIndex = []uint32{ // 8 elements
+var zhIndex = []uint32{ // 10 elements
0x00000000, 0x00000000, 0x00000000, 0x00000000,
0x00000000, 0x00000000, 0x00000000, 0x00000000,
-} // Size: 56 bytes
+ 0x00000000, 0x00000000,
+} // Size: 64 bytes
const zhData string = ""
-// Total table size 503 bytes (0KiB); checksum: A968BD6
+// Total table size 603 bytes (0KiB); checksum: 1D2754EE
diff --git a/cmd/gotext/examples/extract/main.go b/cmd/gotext/examples/extract/main.go
index cbbc370..414b453 100644
--- a/cmd/gotext/examples/extract/main.go
+++ b/cmd/gotext/examples/extract/main.go
@@ -4,8 +4,7 @@
package main
-//go:generate gotext extract --lang=de,zh
-//go:generate gotext generate -out catalog.go
+//go:generate gotext update -out catalog.go
import (
"golang.org/x/text/language"
diff --git a/cmd/gotext/examples/extract_http/catalog_gen.go b/cmd/gotext/examples/extract_http/catalog_gen.go
new file mode 100644
index 0000000..2c410dc
--- /dev/null
+++ b/cmd/gotext/examples/extract_http/catalog_gen.go
@@ -0,0 +1,57 @@
+// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
+
+package main
+
+import (
+ "golang.org/x/text/language"
+ "golang.org/x/text/message"
+ "golang.org/x/text/message/catalog"
+)
+
+type dictionary struct {
+ index []uint32
+ data string
+}
+
+func (d *dictionary) Lookup(key string) (data string, ok bool) {
+ p := messageKeyToIndex[key]
+ start, end := d.index[p], d.index[p+1]
+ if start == end {
+ return "", false
+ }
+ return d.data[start:end], true
+}
+
+func init() {
+ dict := map[string]catalog.Dictionary{
+ "en": &dictionary{index: enIndex, data: enData},
+ "zh": &dictionary{index: zhIndex, data: zhData},
+ }
+ fallback := language.MustParse("en")
+ cat, err := catalog.NewFromMap(dict, catalog.Fallback(fallback))
+ if err != nil {
+ panic(err)
+ }
+ message.DefaultCatalog = cat
+}
+
+var messageKeyToIndex = map[string]int{
+ "Do you like your browser (%s)?\n": 1,
+ "Hello %s!\n": 0,
+}
+
+var enIndex = []uint32{ // 3 elements
+ 0x00000000, 0x00000012, 0x00000039,
+} // Size: 36 bytes
+
+const enData string = "" + // Size: 57 bytes
+ "\x04\x00\x01\x0a\x0d\x02Hello %[1]s!\x04\x00\x01\x0a\x22\x02Do you like " +
+ "your browser (%[1]s)?"
+
+var zhIndex = []uint32{ // 3 elements
+ 0x00000000, 0x00000000, 0x00000000,
+} // Size: 36 bytes
+
+const zhData string = ""
+
+// Total table size 129 bytes (0KiB); checksum: 9C146C82
diff --git a/cmd/gotext/examples/extract_http/locales/en/out.gotext.json b/cmd/gotext/examples/extract_http/locales/en/out.gotext.json
new file mode 100644
index 0000000..1391e58
--- /dev/null
+++ b/cmd/gotext/examples/extract_http/locales/en/out.gotext.json
@@ -0,0 +1,39 @@
+{
+ "language": "en",
+ "messages": [
+ {
+ "id": "Hello {From}!",
+ "message": "Hello {From}!",
+ "translation": "Hello {From}!",
+ "translatorComment": "Copied from source.",
+ "placeholders": [
+ {
+ "id": "From",
+ "string": "%[1]s",
+ "type": "string",
+ "underlyingType": "string",
+ "argNum": 1,
+ "expr": "r.Header.Get(\"From\")"
+ }
+ ],
+ "fuzzy": true
+ },
+ {
+ "id": "Do you like your browser ({User_Agent})?",
+ "message": "Do you like your browser ({User_Agent})?",
+ "translation": "Do you like your browser ({User_Agent})?",
+ "translatorComment": "Copied from source.",
+ "placeholders": [
+ {
+ "id": "User_Agent",
+ "string": "%[1]s",
+ "type": "string",
+ "underlyingType": "string",
+ "argNum": 1,
+ "expr": "r.Header.Get(\"User-Agent\")"
+ }
+ ],
+ "fuzzy": true
+ }
+ ]
+}
\ No newline at end of file
diff --git a/cmd/gotext/examples/extract_http/locales/zh/out.gotext.json b/cmd/gotext/examples/extract_http/locales/zh/out.gotext.json
index 0bab4a7..7b26974 100755
--- a/cmd/gotext/examples/extract_http/locales/zh/out.gotext.json
+++ b/cmd/gotext/examples/extract_http/locales/zh/out.gotext.json
@@ -3,7 +3,6 @@
"messages": [
{
"id": "Hello {From}!",
- "key": "Hello %s!\n",
"message": "Hello {From}!",
"translation": "",
"placeholders": [
@@ -15,12 +14,10 @@
"argNum": 1,
"expr": "r.Header.Get(\"From\")"
}
- ],
- "position": "golang.org/x/text/cmd/gotext/examples/extract_http/pkg/pkg.go:22:11"
+ ]
},
{
"id": "Do you like your browser ({User_Agent})?",
- "key": "Do you like your browser (%s)?\n",
"message": "Do you like your browser ({User_Agent})?",
"translation": "",
"placeholders": [
@@ -32,8 +29,7 @@
"argNum": 1,
"expr": "r.Header.Get(\"User-Agent\")"
}
- ],
- "position": "golang.org/x/text/cmd/gotext/examples/extract_http/pkg/pkg.go:24:11"
+ ]
}
]
}
\ No newline at end of file
diff --git a/cmd/gotext/examples/extract_http/main.go b/cmd/gotext/examples/extract_http/main.go
index c1ac449..b5eb3b33 100644
--- a/cmd/gotext/examples/extract_http/main.go
+++ b/cmd/gotext/examples/extract_http/main.go
@@ -4,7 +4,7 @@
package main
-//go:generate gotext extract --lang=de,zh
+//go:generate gotext -srclang=en update -out=catalog_gen.go -lang=en,zh
import (
"net/http"
diff --git a/cmd/gotext/extract.go b/cmd/gotext/extract.go
index b59097b..103d7e6 100644
--- a/cmd/gotext/extract.go
+++ b/cmd/gotext/extract.go
@@ -14,10 +14,6 @@
// - handle features (gender, plural)
// - message rewriting
-var (
- lang *string
-)
-
func init() {
lang = cmdExtract.Flag.String("lang", "en-US", "comma-separated list of languages to process")
}
diff --git a/cmd/gotext/generate.go b/cmd/gotext/generate.go
index c8d5d79..36820df 100644
--- a/cmd/gotext/generate.go
+++ b/cmd/gotext/generate.go
@@ -12,10 +12,6 @@
out = cmdGenerate.Flag.String("out", "", "output file to write to")
}
-var (
- out *string
-)
-
var cmdGenerate = &Command{
Run: runGenerate,
UsageLine: "generate <package>",
diff --git a/cmd/gotext/main.go b/cmd/gotext/main.go
index 1e6c3aa..73f6d91 100644
--- a/cmd/gotext/main.go
+++ b/cmd/gotext/main.go
@@ -47,6 +47,7 @@
}
return &pipeline.Config{
SourceLanguage: tag,
+ Supported: getLangs(),
TranslationsPattern: `messages\.(.*)\.json`,
GenFile: *out,
}, nil
@@ -100,6 +101,7 @@
// Commands lists the available commands and help topics.
// The order here is the order in which they are printed by 'go help'.
var commands = []*Command{
+ cmdUpdate,
cmdExtract,
cmdRewrite,
cmdGenerate,
diff --git a/cmd/gotext/update.go b/cmd/gotext/update.go
new file mode 100644
index 0000000..1260750
--- /dev/null
+++ b/cmd/gotext/update.go
@@ -0,0 +1,52 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+ "golang.org/x/text/message/pipeline"
+)
+
+// TODO:
+// - merge information into existing files
+// - handle different file formats (PO, XLIFF)
+// - handle features (gender, plural)
+// - message rewriting
+
+var (
+ lang *string
+ out *string
+)
+
+func init() {
+ lang = cmdUpdate.Flag.String("lang", "en-US", "comma-separated list of languages to process")
+ out = cmdUpdate.Flag.String("out", "", "output file to write to")
+}
+
+var cmdUpdate = &Command{
+ Run: runUpdate,
+ UsageLine: "update <package>* [-out <gofile>]",
+ Short: "merge translations and generate catalog",
+}
+
+func runUpdate(cmd *Command, config *pipeline.Config, args []string) error {
+ config.Packages = args
+ state, err := pipeline.Extract(config)
+ if err != nil {
+ return wrap(err, "extract failed")
+ }
+ if err := state.Import(); err != nil {
+ return wrap(err, "import failed")
+ }
+ if err := state.Merge(); err != nil {
+ return wrap(err, "merge failed")
+ }
+ if err := state.Export(); err != nil {
+ return wrap(err, "export failed")
+ }
+ if *out != "" {
+ return wrap(state.Generate(), "generation failed")
+ }
+ return nil
+}
diff --git a/message/pipeline/generate.go b/message/pipeline/generate.go
index cb62c25..5d329b2 100644
--- a/message/pipeline/generate.go
+++ b/message/pipeline/generate.go
@@ -26,6 +26,8 @@
var transRe = regexp.MustCompile(`messages\.(.*)\.json`)
// Generate writes a Go file that defines a Catalog with translated messages.
+// Translations are retrieved from s.Messages, not s.Translations, so it
+// is assumed Merge has been called.
func (s *State) Generate() error {
path := s.Config.GenPackage
if path == "" {
@@ -56,7 +58,8 @@
}
// WriteGen writes a Go file with the given package name to w that defines a
-// Catalog with translated messages.
+// Catalog with translated messages. Translations are retrieved from s.Messages,
+// not s.Translations, so it is assumed Merge has been called.
func (s *State) WriteGen(w io.Writer, pkg string) error {
cw, err := s.generate()
if err != nil {
@@ -80,15 +83,12 @@
}
func (s *State) generate() (*gen.CodeWriter, error) {
- // TODO: add in external input. Right now we assume that all files are
- // manually created and stored in the textdata directory.
-
// Build up index of translations and original messages.
translations := map[language.Tag]map[string]Message{}
languages := []language.Tag{}
usedKeys := map[string]int{}
- for _, loc := range s.Translations {
+ for _, loc := range s.Messages {
tag := loc.Language
if _, ok := translations[tag]; !ok {
translations[tag] = map[string]Message{}
diff --git a/message/pipeline/pipeline.go b/message/pipeline/pipeline.go
index 93d2602..cafd6f2 100644
--- a/message/pipeline/pipeline.go
+++ b/message/pipeline/pipeline.go
@@ -289,7 +289,7 @@
// Build index of translations.
translations := map[language.Tag]map[string]Message{}
- languages := []language.Tag{}
+ languages := append([]language.Tag{}, s.Config.Supported...)
for _, t := range s.Translations {
tag := t.Language
@@ -308,7 +308,7 @@
}
}
}
- internal.SortTags(languages)
+ languages = internal.UniqueTags(languages)
for _, tag := range languages {
ms := Messages{Language: tag}
