blob: a15a7f9d37ddf7f2021af455f76913af9e5e4735 [file] [log] [blame] [edit]
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package pipeline
import (
"bytes"
"errors"
"fmt"
"go/ast"
"go/constant"
"go/format"
"go/token"
"go/types"
"path/filepath"
"sort"
"strings"
"unicode"
"unicode/utf8"
fmtparser "golang.org/x/text/internal/format"
"golang.org/x/tools/go/callgraph"
"golang.org/x/tools/go/callgraph/cha"
"golang.org/x/tools/go/loader"
"golang.org/x/tools/go/ssa"
"golang.org/x/tools/go/ssa/ssautil"
)
const debug = false
// TODO:
// - merge information into existing files
// - handle different file formats (PO, XLIFF)
// - handle features (gender, plural)
// - message rewriting
// - `msg:"etc"` tags
// Extract extracts all strings form the package defined in Config.
func Extract(c *Config) (*State, error) {
x, err := newExtracter(c)
if err != nil {
return nil, wrap(err, "")
}
if err := x.seedEndpoints(); err != nil {
return nil, err
}
x.extractMessages()
return &State{
Config: *c,
program: x.iprog,
Extracted: Messages{
Language: c.SourceLanguage,
Messages: x.messages,
},
}, nil
}
type extracter struct {
conf loader.Config
iprog *loader.Program
prog *ssa.Program
callGraph *callgraph.Graph
// Calls and other expressions to collect.
globals map[token.Pos]*constData
funcs map[token.Pos]*callData
messages []Message
}
func newExtracter(c *Config) (x *extracter, err error) {
x = &extracter{
conf: loader.Config{},
globals: map[token.Pos]*constData{},
funcs: map[token.Pos]*callData{},
}
x.iprog, err = loadPackages(&x.conf, c.Packages)
if err != nil {
return nil, wrap(err, "")
}
x.prog = ssautil.CreateProgram(x.iprog, ssa.GlobalDebug|ssa.BareInits)
x.prog.Build()
x.callGraph = cha.CallGraph(x.prog)
return x, nil
}
func (x *extracter) globalData(pos token.Pos) *constData {
cd := x.globals[pos]
if cd == nil {
cd = &constData{}
x.globals[pos] = cd
}
return cd
}
func (x *extracter) seedEndpoints() error {
pkgInfo := x.iprog.Package("golang.org/x/text/message")
if pkgInfo == nil {
return errors.New("pipeline: golang.org/x/text/message is not imported")
}
pkg := x.prog.Package(pkgInfo.Pkg)
typ := types.NewPointer(pkg.Type("Printer").Type())
x.processGlobalVars()
x.handleFunc(x.prog.LookupMethod(typ, pkg.Pkg, "Printf"), &callData{
formatPos: 1,
argPos: 2,
isMethod: true,
})
x.handleFunc(x.prog.LookupMethod(typ, pkg.Pkg, "Sprintf"), &callData{
formatPos: 1,
argPos: 2,
isMethod: true,
})
x.handleFunc(x.prog.LookupMethod(typ, pkg.Pkg, "Fprintf"), &callData{
formatPos: 2,
argPos: 3,
isMethod: true,
})
return nil
}
// processGlobalVars finds string constants that are assigned to global
// variables.
func (x *extracter) processGlobalVars() {
for _, p := range x.prog.AllPackages() {
m, ok := p.Members["init"]
if !ok {
continue
}
for _, b := range m.(*ssa.Function).Blocks {
for _, i := range b.Instrs {
s, ok := i.(*ssa.Store)
if !ok {
continue
}
a, ok := s.Addr.(*ssa.Global)
if !ok {
continue
}
t := a.Type()
for {
p, ok := t.(*types.Pointer)
if !ok {
break
}
t = p.Elem()
}
if b, ok := t.(*types.Basic); !ok || b.Kind() != types.String {
continue
}
x.visitInit(a, s.Val)
}
}
}
}
type constData struct {
call *callData // to provide a signature for the constants
values []constVal
others []token.Pos // Assigned to other global data.
}
func (d *constData) visit(x *extracter, f func(c constant.Value)) {
for _, v := range d.values {
f(v.value)
}
for _, p := range d.others {
if od, ok := x.globals[p]; ok {
od.visit(x, f)
}
}
}
type constVal struct {
value constant.Value
pos token.Pos
}
type callData struct {
call ssa.CallInstruction
expr *ast.CallExpr
formats []constant.Value
callee *callData
isMethod bool
formatPos int
argPos int // varargs at this position in the call
argTypes []int // arguments extractable from this position
}
func (c *callData) callFormatPos() int {
c = c.callee
if c.isMethod {
return c.formatPos - 1
}
return c.formatPos
}
func (c *callData) callArgsStart() int {
c = c.callee
if c.isMethod {
return c.argPos - 1
}
return c.argPos
}
func (c *callData) Pos() token.Pos { return c.call.Pos() }
func (c *callData) Pkg() *types.Package { return c.call.Parent().Pkg.Pkg }
func (x *extracter) handleFunc(f *ssa.Function, fd *callData) {
for _, e := range x.callGraph.Nodes[f].In {
if e.Pos() == 0 {
continue
}
call := e.Site
caller := x.funcs[call.Pos()]
if caller != nil {
// TODO: theoretically a format string could be passed to multiple
// arguments of a function. Support this eventually.
continue
}
x.debug(call, "CALL", f.String())
caller = &callData{
call: call,
callee: fd,
formatPos: -1,
argPos: -1,
}
// Offset by one if we are invoking an interface method.
offset := 0
if call.Common().IsInvoke() {
offset = -1
}
x.funcs[call.Pos()] = caller
if fd.argPos >= 0 {
x.visitArgs(caller, call.Common().Args[fd.argPos+offset])
}
x.visitFormats(caller, call.Common().Args[fd.formatPos+offset])
}
}
type posser interface {
Pos() token.Pos
Parent() *ssa.Function
}
func (x *extracter) debug(v posser, header string, args ...interface{}) {
if debug {
pos := ""
if p := v.Parent(); p != nil {
pos = posString(&x.conf, p.Package().Pkg, v.Pos())
}
if header != "CALL" && header != "INSERT" {
header = " " + header
}
fmt.Printf("%-32s%-10s%-15T ", pos+fmt.Sprintf("@%d", v.Pos()), header, v)
for _, a := range args {
fmt.Printf(" %v", a)
}
fmt.Println()
}
}
// visitInit evaluates and collects values assigned to global variables in an
// init function.
func (x *extracter) visitInit(global *ssa.Global, v ssa.Value) {
if v == nil {
return
}
x.debug(v, "GLOBAL", v)
switch v := v.(type) {
case *ssa.Phi:
for _, e := range v.Edges {
x.visitInit(global, e)
}
case *ssa.Const:
// Only record strings with letters.
if str := constant.StringVal(v.Value); isMsg(str) {
cd := x.globalData(global.Pos())
cd.values = append(cd.values, constVal{v.Value, v.Pos()})
}
// TODO: handle %m-directive.
case *ssa.Global:
cd := x.globalData(global.Pos())
cd.others = append(cd.others, v.Pos())
case *ssa.FieldAddr, *ssa.Field:
// TODO: mark field index v.Field of v.X.Type() for extraction. extract
// an example args as to give parameters for the translator.
case *ssa.Slice:
if v.Low == nil && v.High == nil && v.Max == nil {
x.visitInit(global, v.X)
}
case *ssa.Alloc:
if ref := v.Referrers(); ref == nil {
for _, r := range *ref {
values := []ssa.Value{}
for _, o := range r.Operands(nil) {
if o == nil || *o == v {
continue
}
values = append(values, *o)
}
// TODO: return something different if we care about multiple
// values as well.
if len(values) == 1 {
x.visitInit(global, values[0])
}
}
}
case ssa.Instruction:
rands := v.Operands(nil)
if len(rands) == 1 && rands[0] != nil {
x.visitInit(global, *rands[0])
}
}
return
}
// visitFormats finds the original source of the value. The returned index is
// position of the argument if originated from a function argument or -1
// otherwise.
func (x *extracter) visitFormats(call *callData, v ssa.Value) {
if v == nil {
return
}
x.debug(v, "VALUE", v)
switch v := v.(type) {
case *ssa.Phi:
for _, e := range v.Edges {
x.visitFormats(call, e)
}
case *ssa.Const:
// Only record strings with letters.
if isMsg(constant.StringVal(v.Value)) {
x.debug(call.call, "FORMAT", v.Value.ExactString())
call.formats = append(call.formats, v.Value)
}
// TODO: handle %m-directive.
case *ssa.Global:
x.globalData(v.Pos()).call = call
case *ssa.FieldAddr, *ssa.Field:
// TODO: mark field index v.Field of v.X.Type() for extraction. extract
// an example args as to give parameters for the translator.
case *ssa.Slice:
if v.Low == nil && v.High == nil && v.Max == nil {
x.visitFormats(call, v.X)
}
case *ssa.Parameter:
// TODO: handle the function for the index parameter.
f := v.Parent()
for i, p := range f.Params {
if p == v {
if call.formatPos < 0 {
call.formatPos = i
// TODO: is there a better way to detect this is calling
// a method rather than a function?
call.isMethod = len(f.Params) > f.Signature.Params().Len()
x.handleFunc(v.Parent(), call)
} else if debug && i != call.formatPos {
// TODO: support this.
fmt.Printf("WARNING:%s: format string passed to arg %d and %d\n",
posString(&x.conf, call.Pkg(), call.Pos()),
call.formatPos, i)
}
}
}
case *ssa.Alloc:
if ref := v.Referrers(); ref == nil {
for _, r := range *ref {
values := []ssa.Value{}
for _, o := range r.Operands(nil) {
if o == nil || *o == v {
continue
}
values = append(values, *o)
}
// TODO: return something different if we care about multiple
// values as well.
if len(values) == 1 {
x.visitFormats(call, values[0])
}
}
}
// TODO:
// case *ssa.Index:
// // Get all values in the array if applicable
// case *ssa.IndexAddr:
// // Get all values in the slice or *array if applicable.
// case *ssa.Lookup:
// // Get all values in the map if applicable.
case *ssa.FreeVar:
// TODO: find the link between free variables and parameters:
//
// func freeVar(p *message.Printer, str string) {
// fn := func(p *message.Printer) {
// p.Printf(str)
// }
// fn(p)
// }
case *ssa.Call:
case ssa.Instruction:
rands := v.Operands(nil)
if len(rands) == 1 && rands[0] != nil {
x.visitFormats(call, *rands[0])
}
}
}
// Note: a function may have an argument marked as both format and passthrough.
// visitArgs collects information on arguments. For wrapped functions it will
// just determine the position of the variable args slice.
func (x *extracter) visitArgs(fd *callData, v ssa.Value) {
if v == nil {
return
}
x.debug(v, "ARGV", v)
switch v := v.(type) {
case *ssa.Slice:
if v.Low == nil && v.High == nil && v.Max == nil {
x.visitArgs(fd, v.X)
}
case *ssa.Parameter:
// TODO: handle the function for the index parameter.
f := v.Parent()
for i, p := range f.Params {
if p == v {
fd.argPos = i
}
}
case *ssa.Alloc:
if ref := v.Referrers(); ref == nil {
for _, r := range *ref {
values := []ssa.Value{}
for _, o := range r.Operands(nil) {
if o == nil || *o == v {
continue
}
values = append(values, *o)
}
// TODO: return something different if we care about
// multiple values as well.
if len(values) == 1 {
x.visitArgs(fd, values[0])
}
}
}
case ssa.Instruction:
rands := v.Operands(nil)
if len(rands) == 1 && rands[0] != nil {
x.visitArgs(fd, *rands[0])
}
}
}
// print returns Go syntax for the specified node.
func (x *extracter) print(n ast.Node) string {
var buf bytes.Buffer
format.Node(&buf, x.conf.Fset, n)
return buf.String()
}
type packageExtracter struct {
f *ast.File
x *extracter
info *loader.PackageInfo
cmap ast.CommentMap
}
func (px packageExtracter) getComment(n ast.Node) string {
cs := px.cmap.Filter(n).Comments()
if len(cs) > 0 {
return strings.TrimSpace(cs[0].Text())
}
return ""
}
func (x *extracter) extractMessages() {
prog := x.iprog
keys := make([]*types.Package, 0, len(x.iprog.AllPackages))
for k := range x.iprog.AllPackages {
keys = append(keys, k)
}
sort.Slice(keys, func(i, j int) bool { return keys[i].Path() < keys[j].Path() })
files := []packageExtracter{}
for _, k := range keys {
info := x.iprog.AllPackages[k]
for _, f := range info.Files {
// Associate comments with nodes.
px := packageExtracter{
f, x, info,
ast.NewCommentMap(prog.Fset, f, f.Comments),
}
files = append(files, px)
}
}
for _, px := range files {
ast.Inspect(px.f, func(n ast.Node) bool {
switch v := n.(type) {
case *ast.CallExpr:
if d := x.funcs[v.Lparen]; d != nil {
d.expr = v
}
}
return true
})
}
for _, px := range files {
ast.Inspect(px.f, func(n ast.Node) bool {
switch v := n.(type) {
case *ast.CallExpr:
return px.handleCall(v)
case *ast.ValueSpec:
return px.handleGlobal(v)
}
return true
})
}
}
func (px packageExtracter) handleGlobal(spec *ast.ValueSpec) bool {
comment := px.getComment(spec)
for _, ident := range spec.Names {
data, ok := px.x.globals[ident.Pos()]
if !ok {
continue
}
name := ident.Name
var arguments []argument
if data.call != nil {
arguments = px.getArguments(data.call)
} else if !strings.HasPrefix(name, "msg") && !strings.HasPrefix(name, "Msg") {
continue
}
data.visit(px.x, func(c constant.Value) {
px.addMessage(spec.Pos(), []string{name}, c, comment, arguments)
})
}
return true
}
func (px packageExtracter) handleCall(call *ast.CallExpr) bool {
x := px.x
data := x.funcs[call.Lparen]
if data == nil || len(data.formats) == 0 {
return true
}
if data.expr != call {
panic("invariant `data.call != call` failed")
}
x.debug(data.call, "INSERT", data.formats)
argn := data.callFormatPos()
if argn >= len(call.Args) {
return true
}
format := call.Args[argn]
arguments := px.getArguments(data)
comment := ""
key := []string{}
if ident, ok := format.(*ast.Ident); ok {
key = append(key, ident.Name)
if v, ok := ident.Obj.Decl.(*ast.ValueSpec); ok && v.Comment != nil {
// TODO: get comment above ValueSpec as well
comment = v.Comment.Text()
}
}
if c := px.getComment(call.Args[0]); c != "" {
comment = c
}
formats := data.formats
for _, c := range formats {
px.addMessage(call.Lparen, key, c, comment, arguments)
}
return true
}
func (px packageExtracter) getArguments(data *callData) []argument {
arguments := []argument{}
x := px.x
info := px.info
if data.callArgsStart() >= 0 {
args := data.expr.Args[data.callArgsStart():]
for i, arg := range args {
expr := x.print(arg)
val := ""
if v := info.Types[arg].Value; v != nil {
val = v.ExactString()
switch arg.(type) {
case *ast.BinaryExpr, *ast.UnaryExpr:
expr = val
}
}
arguments = append(arguments, argument{
ArgNum: i + 1,
Type: info.Types[arg].Type.String(),
UnderlyingType: info.Types[arg].Type.Underlying().String(),
Expr: expr,
Value: val,
Comment: px.getComment(arg),
Position: posString(&x.conf, info.Pkg, arg.Pos()),
// TODO report whether it implements
// interfaces plural.Interface,
// gender.Interface.
})
}
}
return arguments
}
func (px packageExtracter) addMessage(
pos token.Pos,
key []string,
c constant.Value,
comment string,
arguments []argument) {
x := px.x
fmtMsg := constant.StringVal(c)
ph := placeholders{index: map[string]string{}}
trimmed, _, _ := trimWS(fmtMsg)
p := fmtparser.Parser{}
simArgs := make([]interface{}, len(arguments))
for i, v := range arguments {
simArgs[i] = v
}
msg := ""
p.Reset(simArgs)
for p.SetFormat(trimmed); p.Scan(); {
name := ""
var arg *argument
switch p.Status {
case fmtparser.StatusText:
msg += p.Text()
continue
case fmtparser.StatusSubstitution,
fmtparser.StatusBadWidthSubstitution,
fmtparser.StatusBadPrecSubstitution:
arguments[p.ArgNum-1].used = true
arg = &arguments[p.ArgNum-1]
name = getID(arg)
case fmtparser.StatusBadArgNum, fmtparser.StatusMissingArg:
arg = &argument{
ArgNum: p.ArgNum,
Position: posString(&x.conf, px.info.Pkg, pos),
}
name, arg.UnderlyingType = verbToPlaceholder(p.Text(), p.ArgNum)
}
sub := p.Text()
if !p.HasIndex {
r, sz := utf8.DecodeLastRuneInString(sub)
sub = fmt.Sprintf("%s[%d]%c", sub[:len(sub)-sz], p.ArgNum, r)
}
msg += fmt.Sprintf("{%s}", ph.addArg(arg, name, sub))
}
key = append(key, msg)
// Add additional Placeholders that can be used in translations
// that are not present in the string.
for _, arg := range arguments {
if arg.used {
continue
}
ph.addArg(&arg, getID(&arg), fmt.Sprintf("%%[%d]v", arg.ArgNum))
}
x.messages = append(x.messages, Message{
ID: key,
Key: fmtMsg,
Message: Text{Msg: msg},
// TODO(fix): this doesn't get the before comment.
Comment: comment,
Placeholders: ph.slice,
Position: posString(&x.conf, px.info.Pkg, pos),
})
}
func posString(conf *loader.Config, pkg *types.Package, pos token.Pos) string {
p := conf.Fset.Position(pos)
file := fmt.Sprintf("%s:%d:%d", filepath.Base(p.Filename), p.Line, p.Column)
return filepath.Join(pkg.Path(), file)
}
func getID(arg *argument) string {
s := getLastComponent(arg.Expr)
s = strip(s)
s = strings.Replace(s, " ", "", -1)
// For small variable names, use user-defined types for more info.
if len(s) <= 2 && arg.UnderlyingType != arg.Type {
s = getLastComponent(arg.Type)
}
return strings.Title(s)
}
// strip is a dirty hack to convert function calls to placeholder IDs.
func strip(s string) string {
s = strings.Map(func(r rune) rune {
if unicode.IsSpace(r) || r == '-' {
return '_'
}
if !unicode.In(r, unicode.Letter, unicode.Mark, unicode.Number) {
return -1
}
return r
}, s)
// Strip "Get" from getter functions.
if strings.HasPrefix(s, "Get") || strings.HasPrefix(s, "get") {
if len(s) > len("get") {
r, _ := utf8.DecodeRuneInString(s)
if !unicode.In(r, unicode.Ll, unicode.M) { // not lower or mark
s = s[len("get"):]
}
}
}
return s
}
// verbToPlaceholder gives a name for a placeholder based on the substitution
// verb. This is only to be used if there is otherwise no other type information
// available.
func verbToPlaceholder(sub string, pos int) (name, underlying string) {
r, _ := utf8.DecodeLastRuneInString(sub)
name = fmt.Sprintf("Arg_%d", pos)
switch r {
case 's', 'q':
underlying = "string"
case 'd':
name = "Integer"
underlying = "int"
case 'e', 'f', 'g':
name = "Number"
underlying = "float64"
case 'm':
name = "Message"
underlying = "string"
default:
underlying = "interface{}"
}
return name, underlying
}
type placeholders struct {
index map[string]string
slice []Placeholder
}
func (p *placeholders) addArg(arg *argument, name, sub string) (id string) {
id = name
alt, ok := p.index[id]
for i := 1; ok && alt != sub; i++ {
id = fmt.Sprintf("%s_%d", name, i)
alt, ok = p.index[id]
}
p.index[id] = sub
p.slice = append(p.slice, Placeholder{
ID: id,
String: sub,
Type: arg.Type,
UnderlyingType: arg.UnderlyingType,
ArgNum: arg.ArgNum,
Expr: arg.Expr,
Comment: arg.Comment,
})
return id
}
func getLastComponent(s string) string {
return s[1+strings.LastIndexByte(s, '.'):]
}
// isMsg returns whether s should be translated.
func isMsg(s string) bool {
// TODO: parse as format string and omit strings that contain letters
// coming from format verbs.
for _, r := range s {
if unicode.In(r, unicode.L) {
return true
}
}
return false
}