cmd/gotext: move to Placeholder model
This more general model fits better with template-
style substitution, while still fitting well with
printf-style. It also allows hiding HTML and the like.
Modifies printf-substitution to be position-independent.
Change-Id: Ie8bd64c4fec9b8833bf8952bd02a8f3f56139e59
Reviewed-on: https://go-review.googlesource.com/79916
Run-TryBot: Marcel van Lohuizen <mpvl@golang.org>
Reviewed-by: Nigel Tao <nigeltao@golang.org>
diff --git a/cmd/gotext/examples/main.go b/cmd/gotext/examples/main.go
index e1a84e0..08ac143 100644
--- a/cmd/gotext/examples/main.go
+++ b/cmd/gotext/examples/main.go
@@ -45,13 +45,16 @@
pp := struct {
Person string // The person of matter. // TODO: get this comment.
Place string
+ extra int
}{
- person, place,
+ person, place, 4,
}
// extract will drop this comment in favor of the one below.
- p.Printf("%s is visiting %s!\n", // Person visiting a place.
+ // argument is added as a placeholder.
+ p.Printf("%[1]s is visiting %[3]s!\n", // Person visiting a place.
pp.Person,
+ pp.extra,
pp.Place, // Place the person is visiting.
)
@@ -76,4 +79,8 @@
const msgOutOfOrder = "%s is out of order!" // FOO
const device = "Soda machine"
p.Printf(msgOutOfOrder, device)
+
+ // Double arguments.
+ miles := 1.2345
+ p.Printf("%.2[1]f miles traveled (%[1]f)", miles)
}
diff --git a/cmd/gotext/examples/textdata/gotext_en.out.json b/cmd/gotext/examples/textdata/gotext_en.out.json
index e6ebe5d..3dd943a 100755
--- a/cmd/gotext/examples/textdata/gotext_en.out.json
+++ b/cmd/gotext/examples/textdata/gotext_en.out.json
@@ -15,17 +15,14 @@
"message": {
"msg": "Hello {City}!\n"
},
- "args": [
+ "placeholders": [
{
"id": "City",
- "argNum": 1,
- "format": [
- "%s"
- ],
+ "string": "%[1]s",
"type": "string",
"underlyingType": "string",
- "expr": "city",
- "position": "golang.org/x/text/cmd/gotext/examples/main.go:31:26"
+ "argNum": 1,
+ "expr": "city"
}
],
"position": "golang.org/x/text/cmd/gotext/examples/main.go:31:10"
@@ -37,18 +34,15 @@
"message": {
"msg": "Hello {Town}!\n"
},
- "args": [
+ "placeholders": [
{
"id": "Town",
- "argNum": 1,
- "format": [
- "%s"
- ],
+ "string": "%[1]s",
"type": "string",
"underlyingType": "string",
+ "argNum": 1,
"expr": "town",
- "comment": "Town",
- "position": "golang.org/x/text/cmd/gotext/examples/main.go:36:3"
+ "comment": "Town"
}
],
"position": "golang.org/x/text/cmd/gotext/examples/main.go:35:10"
@@ -60,68 +54,64 @@
"message": {
"msg": "{Person} is visiting {Place}!\n"
},
- "args": [
+ "placeholders": [
{
"id": "Person",
- "argNum": 1,
- "format": [
- "%s"
- ],
+ "string": "%[1]s",
"type": "string",
"underlyingType": "string",
+ "argNum": 1,
"expr": "person",
- "comment": "The person of matter.",
- "position": "golang.org/x/text/cmd/gotext/examples/main.go:41:3"
+ "comment": "The person of matter."
},
{
"id": "Place",
- "argNum": 2,
- "format": [
- "%s"
- ],
+ "string": "%[2]s",
"type": "string",
"underlyingType": "string",
+ "argNum": 2,
"expr": "place",
- "comment": "Place the person is visiting.",
- "position": "golang.org/x/text/cmd/gotext/examples/main.go:42:3"
+ "comment": "Place the person is visiting."
}
],
"position": "golang.org/x/text/cmd/gotext/examples/main.go:40:10"
},
{
"key": [
- "%s is visiting %s!\n"
+ "%[1]s is visiting %[3]s!\n"
],
"message": {
"msg": "{Person} is visiting {Place}!\n"
},
"comment": "Person visiting a place.",
- "args": [
+ "placeholders": [
{
"id": "Person",
- "argNum": 1,
- "format": [
- "%s"
- ],
+ "string": "%[1]s",
"type": "string",
"underlyingType": "string",
- "expr": "pp.Person",
- "position": "golang.org/x/text/cmd/gotext/examples/main.go:54:3"
+ "argNum": 1,
+ "expr": "pp.Person"
},
{
"id": "Place",
- "argNum": 2,
- "format": [
- "%s"
- ],
+ "string": "%[3]s",
"type": "string",
"underlyingType": "string",
+ "argNum": 3,
"expr": "pp.Place",
- "comment": "Place the person is visiting.",
- "position": "golang.org/x/text/cmd/gotext/examples/main.go:55:3"
+ "comment": "Place the person is visiting."
+ },
+ {
+ "id": "Extra",
+ "string": "%[2]v",
+ "type": "int",
+ "underlyingType": "int",
+ "argNum": 2,
+ "expr": "pp.extra"
}
],
- "position": "golang.org/x/text/cmd/gotext/examples/main.go:53:10"
+ "position": "golang.org/x/text/cmd/gotext/examples/main.go:55:10"
},
{
"key": [
@@ -130,21 +120,17 @@
"message": {
"msg": "{2} files remaining!"
},
- "args": [
+ "placeholders": [
{
"id": "2",
- "argNum": 1,
- "format": [
- "%d"
- ],
+ "string": "%[1]d",
"type": "int",
"underlyingType": "int",
- "expr": "2",
- "value": "2",
- "position": "golang.org/x/text/cmd/gotext/examples/main.go:59:34"
+ "argNum": 1,
+ "expr": "2"
}
],
- "position": "golang.org/x/text/cmd/gotext/examples/main.go:59:10"
+ "position": "golang.org/x/text/cmd/gotext/examples/main.go:62:10"
},
{
"key": [
@@ -153,21 +139,17 @@
"message": {
"msg": "{N} more files remaining!"
},
- "args": [
+ "placeholders": [
{
"id": "N",
- "argNum": 1,
- "format": [
- "%d"
- ],
+ "string": "%[1]d",
"type": "int",
"underlyingType": "int",
- "expr": "n",
- "value": "2",
- "position": "golang.org/x/text/cmd/gotext/examples/main.go:64:39"
+ "argNum": 1,
+ "expr": "n"
}
],
- "position": "golang.org/x/text/cmd/gotext/examples/main.go:64:10"
+ "position": "golang.org/x/text/cmd/gotext/examples/main.go:67:10"
},
{
"key": [
@@ -176,21 +158,17 @@
"message": {
"msg": "Use the following code for your discount: {ReferralCode}\n"
},
- "args": [
+ "placeholders": [
{
"id": "ReferralCode",
- "argNum": 1,
- "format": [
- "%d"
- ],
+ "string": "%[1]d",
"type": "golang.org/x/text/cmd/gotext/examples.referralCode",
"underlyingType": "int",
- "expr": "c",
- "value": "5",
- "position": "golang.org/x/text/cmd/gotext/examples/main.go:70:61"
+ "argNum": 1,
+ "expr": "c"
}
],
- "position": "golang.org/x/text/cmd/gotext/examples/main.go:70:10"
+ "position": "golang.org/x/text/cmd/gotext/examples/main.go:73:10"
},
{
"key": [
@@ -201,20 +179,43 @@
"msg": "{Device} is out of order!"
},
"comment": "FOO\n",
- "args": [
+ "placeholders": [
{
"id": "Device",
- "argNum": 1,
- "format": [
- "%s"
- ],
+ "string": "%[1]s",
"type": "string",
"underlyingType": "string",
- "expr": "device",
- "value": "\"Soda machine\"",
- "position": "golang.org/x/text/cmd/gotext/examples/main.go:78:26"
+ "argNum": 1,
+ "expr": "device"
}
],
- "position": "golang.org/x/text/cmd/gotext/examples/main.go:78:10"
+ "position": "golang.org/x/text/cmd/gotext/examples/main.go:81:10"
+ },
+ {
+ "key": [
+ "%.2[1]f miles traveled (%[1]f)"
+ ],
+ "message": {
+ "msg": "{Miles} miles traveled ({Miles_1})"
+ },
+ "placeholders": [
+ {
+ "id": "Miles",
+ "string": "%.2[1]f",
+ "type": "float64",
+ "underlyingType": "float64",
+ "argNum": 1,
+ "expr": "miles"
+ },
+ {
+ "id": "Miles_1",
+ "string": "%[1]f",
+ "type": "float64",
+ "underlyingType": "float64",
+ "argNum": 1,
+ "expr": "miles"
+ }
+ ],
+ "position": "golang.org/x/text/cmd/gotext/examples/main.go:85:10"
}
]
\ No newline at end of file
diff --git a/cmd/gotext/extract.go b/cmd/gotext/extract.go
index ac318d1..3b6120d 100644
--- a/cmd/gotext/extract.go
+++ b/cmd/gotext/extract.go
@@ -21,6 +21,7 @@
"path/filepath"
"strings"
"unicode"
+ "unicode/utf8"
fmtparser "golang.org/x/text/internal/format"
"golang.org/x/tools/go/loader"
@@ -135,7 +136,7 @@
}
key = append(key, fmtMsg)
- arguments := []Argument{}
+ arguments := []argument{}
args = args[1:]
simArgs := make([]interface{}, len(args))
for i, arg := range args {
@@ -149,7 +150,7 @@
expr = val
}
}
- arguments = append(arguments, Argument{
+ arguments = append(arguments, argument{
ArgNum: i + 1,
Type: info.Types[arg].Type.String(),
UnderlyingType: info.Types[arg].Type.Underlying().String(),
@@ -164,6 +165,8 @@
}
msg := ""
+ ph := placeholders{index: map[string]string{}}
+
p := fmtparser.Parser{}
p.Reset(simArgs)
for p.SetFormat(fmtMsg); p.Scan(); {
@@ -173,28 +176,37 @@
case fmtparser.StatusSubstitution,
fmtparser.StatusBadWidthSubstitution,
fmtparser.StatusBadPrecSubstitution:
+ arguments[p.ArgNum-1].used = true
arg := arguments[p.ArgNum-1]
- id := getID(&arg)
- arguments[p.ArgNum-1].ID = id
- // TODO: do we allow the same entry to be formatted
- // differently within the same string, do we give
- // a warning, or is this an error?
- arguments[p.ArgNum-1].Format = append(arguments[p.ArgNum-1].Format, p.Text())
- msg += fmt.Sprintf("{%s}", id)
+ sub := p.Text()
+ if !p.HasIndex {
+ r, sz := utf8.DecodeLastRuneInString(sub)
+ sub = fmt.Sprintf("%s[%d]%c", sub[:len(sub)-sz], p.ArgNum, r)
+ }
+ msg += fmt.Sprintf("{%s}", ph.addArg(&arg, sub))
}
}
+ // Add additional Placeholders that can be used in translations
+ // that are not present in the string.
+ for _, arg := range arguments {
+ if arg.used {
+ continue
+ }
+ ph.addArg(&arg, fmt.Sprintf("%%[%d]v", arg.ArgNum))
+ }
+
if c := getComment(call.Args[0]); c != "" {
comment = c
}
messages = append(messages, Message{
- Key: key,
- Position: posString(conf, info, call.Lparen),
- Message: Text{Msg: msg},
+ Key: key,
+ Message: Text{Msg: msg},
// TODO(fix): this doesn't get the before comment.
- Comment: comment,
- Args: arguments,
+ Comment: comment,
+ Placeholders: ph.slice,
+ Position: posString(conf, info, call.Lparen),
})
return true
})
@@ -245,7 +257,7 @@
arg int
}
-func getID(arg *Argument) string {
+func getID(arg *argument) string {
s := getLastComponent(arg.Expr)
s = strings.Replace(s, " ", "", -1)
// For small variable names, use user-defined types for more info.
@@ -255,6 +267,32 @@
return strings.Title(s)
}
+type placeholders struct {
+ index map[string]string
+ slice []Placeholder
+}
+
+func (p *placeholders) addArg(arg *argument, sub string) (id string) {
+ id = getID(arg)
+ id1 := id
+ alt, ok := p.index[id1]
+ for i := 1; ok && alt != sub; i++ {
+ id1 = fmt.Sprintf("%s_%d", id, i)
+ alt, ok = p.index[id1]
+ }
+ p.index[id1] = sub
+ p.slice = append(p.slice, Placeholder{
+ ID: id1,
+ String: sub,
+ Type: arg.Type,
+ UnderlyingType: arg.UnderlyingType,
+ ArgNum: arg.ArgNum,
+ Expr: arg.Expr,
+ Comment: arg.Comment,
+ })
+ return id1
+}
+
func getLastComponent(s string) string {
return s[1+strings.LastIndexByte(s, '.'):]
}
diff --git a/cmd/gotext/message.go b/cmd/gotext/message.go
index 7344f8d..567cf5d 100644
--- a/cmd/gotext/message.go
+++ b/cmd/gotext/message.go
@@ -28,36 +28,55 @@
Comment string `json:"comment,omitempty"`
TranslatorComment string `json:"translatorComment,omitempty"`
- // TODO: have a separate placeholder list, mapping placeholders
- // to arguments or constant strings.
- // TODO: default placeholder syntax is {foo}. Allow alternatives
- // like `foo`.
+ Placeholders []Placeholder `json:"placeholders,omitempty"`
- Args []Argument `json:"args,omitempty"`
+ // TODO: default placeholder syntax is {foo}. Allow alternative escaping
+ // like `foo`.
// Extraction information.
Position string `json:"position,omitempty"` // filePosition:line
}
-// An Argument contains information about the arguments passed to a message.
-type Argument struct {
- ID string `json:"id"` // An int for printf-style calls, but could be a string.
- // Argument position for printf-style format strings. ArgNum corresponds to
- // the number that should be used for explicit argument indexes (e.g.
- // "%[1]d").
- ArgNum int `json:"argNum,omitempty"`
- Format []string `json:"format,omitempty"`
+// A Placeholder is a part of the message that should not be changed by a
+// translator. It can be used to hide or prettify format strings (e.g. %d or
+// {{.Count}}), hide HTML, or mark common names that should not be translated.
+type Placeholder struct {
+ // ID is the placeholder identifier without the curly braces.
+ ID string `json:"id"`
+ // String is the string with which to replace the placeholder. This may be a
+ // formatting string (for instance "%d" or "{{.Count}}") or a literal string
+ // (<div>).
+ String string `json:"string"`
+
+ Type string `json:"type"`
+ UnderlyingType string `json:"underlyingType"`
+ // ArgNum and Expr are set if the placeholder is a substitution of an
+ // argument.
+ ArgNum int `json:"argNum,omitempty"`
+ Expr string `json:"expr,omitempty"`
+
+ Comment string `json:"comment,omitempty"`
+ Example string `json:"example,omitempty"`
+
+ // Features contains the features that are available for the implementation
+ // of this argument.
+ Features []Feature `json:"features,omitempty"`
+}
+
+// An argument contains information about the arguments passed to a message.
+type argument struct {
+ // ArgNum corresponds to the number that should be used for explicit argument indexes (e.g.
+ // "%[1]d").
+ ArgNum int `json:"argNum,omitempty"`
+
+ used bool // Used by Placeholder
Type string `json:"type"`
UnderlyingType string `json:"underlyingType"`
Expr string `json:"expr"`
Value string `json:"value,omitempty"`
Comment string `json:"comment,omitempty"`
Position string `json:"position,omitempty"`
-
- // Features contains the features that are available for the implementation
- // of this argument.
- Features []Feature `json:"features,omitempty"`
}
// Feature holds information about a feature that can be implemented by
@@ -71,13 +90,15 @@
// Text defines a message to be displayed.
type Text struct {
- // Msg and Select contains the message to be displayed. Within a Text value
- // either Msg or Select is defined.
+ // Msg and Select contains the message to be displayed. Msg may be used as
+ // a fallback value if none of the select cases match.
Msg string `json:"msg,omitempty"`
Select *Select `json:"select,omitempty"`
+
// Var defines a map of variables that may be substituted in the selected
// message.
Var map[string]Text `json:"var,omitempty"`
+
// Example contains an example message formatted with default values.
Example string `json:"example,omitempty"`
}
@@ -86,6 +107,12 @@
// a certain argument.
type Select struct {
Feature string `json:"feature"` // Name of variable or Feature type
- Arg interface{} `json:"arg"` // The argument ID.
+ Arg string `json:"arg"` // The placeholder ID
Cases map[string]Text `json:"cases"`
}
+
+// TODO: order matters, but can we derive the ordering from the case keys?
+// type Case struct {
+// Key string `json:"key"`
+// Value Text `json:"value"`
+// }
diff --git a/internal/format/parser.go b/internal/format/parser.go
index 68a8e8e..f4f37f4 100644
--- a/internal/format/parser.go
+++ b/internal/format/parser.go
@@ -28,6 +28,8 @@
PlusV bool
SharpV bool
+ HasIndex bool
+
Width int
Prec int // precision
@@ -94,6 +96,8 @@
p.PlusV = false
p.SharpV = false
+
+ p.HasIndex = false
}
// Scan scans the next part of the format string and sets the status to
@@ -220,6 +224,7 @@
if !afterIndex {
i, afterIndex = p.updateArgNumber(format, i)
}
+ p.HasIndex = afterIndex
if i >= end {
p.endPos = i
