blob: 99e1a8b327302cb5ba76ae00b619ab866dfbf387 [file] [log] [blame]
// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package protojson
import (
"encoding/base64"
"fmt"
"math"
"strconv"
"strings"
"google.golang.org/protobuf/internal/encoding/json"
"google.golang.org/protobuf/internal/errors"
"google.golang.org/protobuf/internal/pragma"
"google.golang.org/protobuf/internal/set"
"google.golang.org/protobuf/proto"
pref "google.golang.org/protobuf/reflect/protoreflect"
"google.golang.org/protobuf/reflect/protoregistry"
)
// Unmarshal reads the given []byte into the given proto.Message.
func Unmarshal(b []byte, m proto.Message) error {
return UnmarshalOptions{}.Unmarshal(b, m)
}
// UnmarshalOptions is a configurable JSON format parser.
type UnmarshalOptions struct {
pragma.NoUnkeyedLiterals
// If AllowPartial is set, input for messages that will result in missing
// required fields will not return an error.
AllowPartial bool
// If DiscardUnknown is set, unknown fields are ignored.
DiscardUnknown bool
// Resolver is the registry used for type lookups when unmarshaling extensions
// and processing Any. If Resolver is not set, unmarshaling will default to
// using protoregistry.GlobalTypes.
Resolver *protoregistry.Types
decoder *json.Decoder
}
// Unmarshal reads the given []byte and populates the given proto.Message using
// options in UnmarshalOptions object. It will clear the message first before
// setting the fields. If it returns an error, the given message may be
// partially set.
func (o UnmarshalOptions) Unmarshal(b []byte, m proto.Message) error {
mr := m.ProtoReflect()
// TODO: Determine if we would like to have an option for merging or only
// have merging behavior. We should at least be consistent with textproto
// marshaling.
resetMessage(mr)
if o.Resolver == nil {
o.Resolver = protoregistry.GlobalTypes
}
o.decoder = json.NewDecoder(b)
var nerr errors.NonFatal
if err := o.unmarshalMessage(mr, false); !nerr.Merge(err) {
return err
}
// Check for EOF.
val, err := o.decoder.Read()
if err != nil {
return err
}
if val.Type() != json.EOF {
return unexpectedJSONError{val}
}
if !o.AllowPartial {
nerr.Merge(proto.IsInitialized(m))
}
return nerr.E
}
// resetMessage clears all fields of given protoreflect.Message.
func resetMessage(m pref.Message) {
knownFields := m.KnownFields()
knownFields.Range(func(num pref.FieldNumber, _ pref.Value) bool {
knownFields.Clear(num)
return true
})
unknownFields := m.UnknownFields()
unknownFields.Range(func(num pref.FieldNumber, _ pref.RawFields) bool {
unknownFields.Set(num, nil)
return true
})
extTypes := knownFields.ExtensionTypes()
extTypes.Range(func(xt pref.ExtensionType) bool {
extTypes.Remove(xt)
return true
})
}
// unexpectedJSONError is an error that contains the unexpected json.Value. This
// is returned by methods to provide callers the read json.Value that it did not
// expect.
// TODO: Consider moving this to internal/encoding/json for consistency with
// errors that package returns.
type unexpectedJSONError struct {
value json.Value
}
func (e unexpectedJSONError) Error() string {
return newError("unexpected value %s", e.value).Error()
}
// newError returns an error object. If one of the values passed in is of
// json.Value type, it produces an error with position info.
func newError(f string, x ...interface{}) error {
var hasValue bool
var line, column int
for i := 0; i < len(x); i++ {
if val, ok := x[i].(json.Value); ok {
line, column = val.Position()
hasValue = true
break
}
}
e := errors.New(f, x...)
if hasValue {
return errors.New("(line %d:%d): %v", line, column, e)
}
return e
}
// unmarshalMessage unmarshals a message into the given protoreflect.Message.
func (o UnmarshalOptions) unmarshalMessage(m pref.Message, skipTypeURL bool) error {
var nerr errors.NonFatal
if isCustomType(m.Descriptor().FullName()) {
return o.unmarshalCustomType(m)
}
jval, err := o.decoder.Read()
if !nerr.Merge(err) {
return err
}
if jval.Type() != json.StartObject {
return unexpectedJSONError{jval}
}
if err := o.unmarshalFields(m, skipTypeURL); !nerr.Merge(err) {
return err
}
return nerr.E
}
// unmarshalFields unmarshals the fields into the given protoreflect.Message.
func (o UnmarshalOptions) unmarshalFields(m pref.Message, skipTypeURL bool) error {
var nerr errors.NonFatal
var seenNums set.Ints
var seenOneofs set.Ints
messageDesc := m.Descriptor()
knownFields := m.KnownFields()
fieldDescs := messageDesc.Fields()
xtTypes := knownFields.ExtensionTypes()
Loop:
for {
// Read field name.
jval, err := o.decoder.Read()
if !nerr.Merge(err) {
return err
}
switch jval.Type() {
default:
return unexpectedJSONError{jval}
case json.EndObject:
break Loop
case json.Name:
// Continue below.
}
name, err := jval.Name()
if !nerr.Merge(err) {
return err
}
// Unmarshaling a non-custom embedded message in Any will contain the
// JSON field "@type" which should be skipped because it is not a field
// of the embedded message, but simply an artifact of the Any format.
if skipTypeURL && name == "@type" {
o.decoder.Read()
continue
}
// Get the FieldDescriptor.
var fd pref.FieldDescriptor
if strings.HasPrefix(name, "[") && strings.HasSuffix(name, "]") {
// Only extension names are in [name] format.
xtName := pref.FullName(name[1 : len(name)-1])
xt := xtTypes.ByName(xtName)
if xt == nil {
xt, err = o.findExtension(xtName)
if err != nil && err != protoregistry.NotFound {
return errors.New("unable to resolve [%v]: %v", xtName, err)
}
if xt != nil {
xtTypes.Register(xt)
}
}
if xt != nil {
fd = xt.Descriptor()
}
} else {
// The name can either be the JSON name or the proto field name.
fd = fieldDescs.ByJSONName(name)
if fd == nil {
fd = fieldDescs.ByName(pref.Name(name))
}
}
if fd == nil {
// Field is unknown.
if o.DiscardUnknown {
if err := skipJSONValue(o.decoder); !nerr.Merge(err) {
return err
}
continue
}
return newError("%v contains unknown field %s", messageDesc.FullName(), jval)
}
// Do not allow duplicate fields.
num := uint64(fd.Number())
if seenNums.Has(num) {
return newError("%v contains repeated field %s", messageDesc.FullName(), jval)
}
seenNums.Set(num)
// No need to set values for JSON null unless the field type is
// google.protobuf.Value or google.protobuf.NullValue.
if o.decoder.Peek() == json.Null && !isKnownValue(fd) && !isNullValue(fd) {
o.decoder.Read()
continue
}
switch {
case fd.IsList():
list := knownFields.Get(fd.Number()).List()
if err := o.unmarshalList(list, fd); !nerr.Merge(err) {
return errors.New("%v|%q: %v", fd.FullName(), name, err)
}
case fd.IsMap():
mmap := knownFields.Get(fd.Number()).Map()
if err := o.unmarshalMap(mmap, fd); !nerr.Merge(err) {
return errors.New("%v|%q: %v", fd.FullName(), name, err)
}
default:
// If field is a oneof, check if it has already been set.
if od := fd.ContainingOneof(); od != nil {
idx := uint64(od.Index())
if seenOneofs.Has(idx) {
return errors.New("%v: oneof is already set", od.FullName())
}
seenOneofs.Set(idx)
}
// Required or optional fields.
if err := o.unmarshalSingular(knownFields, fd); !nerr.Merge(err) {
return errors.New("%v|%q: %v", fd.FullName(), name, err)
}
}
}
return nerr.E
}
// findExtension returns protoreflect.ExtensionType from the resolver if found.
func (o UnmarshalOptions) findExtension(xtName pref.FullName) (pref.ExtensionType, error) {
xt, err := o.Resolver.FindExtensionByName(xtName)
if err == nil {
return xt, nil
}
// Check if this is a MessageSet extension field.
xt, err = o.Resolver.FindExtensionByName(xtName + ".message_set_extension")
if err == nil && isMessageSetExtension(xt) {
return xt, nil
}
return nil, protoregistry.NotFound
}
func isKnownValue(fd pref.FieldDescriptor) bool {
md := fd.Message()
return md != nil && md.FullName() == "google.protobuf.Value"
}
func isNullValue(fd pref.FieldDescriptor) bool {
ed := fd.Enum()
return ed != nil && ed.FullName() == "google.protobuf.NullValue"
}
// unmarshalSingular unmarshals to the non-repeated field specified by the given
// FieldDescriptor.
func (o UnmarshalOptions) unmarshalSingular(knownFields pref.KnownFields, fd pref.FieldDescriptor) error {
var val pref.Value
var err error
num := fd.Number()
switch fd.Kind() {
case pref.MessageKind, pref.GroupKind:
m := knownFields.NewMessage(num)
err = o.unmarshalMessage(m, false)
val = pref.ValueOf(m)
default:
val, err = o.unmarshalScalar(fd)
}
var nerr errors.NonFatal
if !nerr.Merge(err) {
return err
}
knownFields.Set(num, val)
return nerr.E
}
// unmarshalScalar unmarshals to a scalar/enum protoreflect.Value specified by
// the given FieldDescriptor.
func (o UnmarshalOptions) unmarshalScalar(fd pref.FieldDescriptor) (pref.Value, error) {
const b32 int = 32
const b64 int = 64
var nerr errors.NonFatal
jval, err := o.decoder.Read()
if !nerr.Merge(err) {
return pref.Value{}, err
}
kind := fd.Kind()
switch kind {
case pref.BoolKind:
return unmarshalBool(jval)
case pref.Int32Kind, pref.Sint32Kind, pref.Sfixed32Kind:
return unmarshalInt(jval, b32)
case pref.Int64Kind, pref.Sint64Kind, pref.Sfixed64Kind:
return unmarshalInt(jval, b64)
case pref.Uint32Kind, pref.Fixed32Kind:
return unmarshalUint(jval, b32)
case pref.Uint64Kind, pref.Fixed64Kind:
return unmarshalUint(jval, b64)
case pref.FloatKind:
return unmarshalFloat(jval, b32)
case pref.DoubleKind:
return unmarshalFloat(jval, b64)
case pref.StringKind:
pval, err := unmarshalString(jval)
if !nerr.Merge(err) {
return pval, err
}
return pval, nerr.E
case pref.BytesKind:
return unmarshalBytes(jval)
case pref.EnumKind:
return unmarshalEnum(jval, fd)
}
panic(fmt.Sprintf("invalid scalar kind %v", kind))
}
func unmarshalBool(jval json.Value) (pref.Value, error) {
if jval.Type() != json.Bool {
return pref.Value{}, unexpectedJSONError{jval}
}
b, err := jval.Bool()
return pref.ValueOf(b), err
}
func unmarshalInt(jval json.Value, bitSize int) (pref.Value, error) {
switch jval.Type() {
case json.Number:
return getInt(jval, bitSize)
case json.String:
// Decode number from string.
s := strings.TrimSpace(jval.String())
if len(s) != len(jval.String()) {
return pref.Value{}, errors.New("invalid number %v", jval.Raw())
}
dec := json.NewDecoder([]byte(s))
var nerr errors.NonFatal
jval, err := dec.Read()
if !nerr.Merge(err) {
return pref.Value{}, err
}
return getInt(jval, bitSize)
}
return pref.Value{}, unexpectedJSONError{jval}
}
func getInt(jval json.Value, bitSize int) (pref.Value, error) {
n, err := jval.Int(bitSize)
if err != nil {
return pref.Value{}, err
}
if bitSize == 32 {
return pref.ValueOf(int32(n)), nil
}
return pref.ValueOf(n), nil
}
func unmarshalUint(jval json.Value, bitSize int) (pref.Value, error) {
switch jval.Type() {
case json.Number:
return getUint(jval, bitSize)
case json.String:
// Decode number from string.
s := strings.TrimSpace(jval.String())
if len(s) != len(jval.String()) {
return pref.Value{}, errors.New("invalid number %v", jval.Raw())
}
dec := json.NewDecoder([]byte(s))
var nerr errors.NonFatal
jval, err := dec.Read()
if !nerr.Merge(err) {
return pref.Value{}, err
}
return getUint(jval, bitSize)
}
return pref.Value{}, unexpectedJSONError{jval}
}
func getUint(jval json.Value, bitSize int) (pref.Value, error) {
n, err := jval.Uint(bitSize)
if err != nil {
return pref.Value{}, err
}
if bitSize == 32 {
return pref.ValueOf(uint32(n)), nil
}
return pref.ValueOf(n), nil
}
func unmarshalFloat(jval json.Value, bitSize int) (pref.Value, error) {
switch jval.Type() {
case json.Number:
return getFloat(jval, bitSize)
case json.String:
s := jval.String()
switch s {
case "NaN":
if bitSize == 32 {
return pref.ValueOf(float32(math.NaN())), nil
}
return pref.ValueOf(math.NaN()), nil
case "Infinity":
if bitSize == 32 {
return pref.ValueOf(float32(math.Inf(+1))), nil
}
return pref.ValueOf(math.Inf(+1)), nil
case "-Infinity":
if bitSize == 32 {
return pref.ValueOf(float32(math.Inf(-1))), nil
}
return pref.ValueOf(math.Inf(-1)), nil
}
// Decode number from string.
if len(s) != len(strings.TrimSpace(s)) {
return pref.Value{}, errors.New("invalid number %v", jval.Raw())
}
dec := json.NewDecoder([]byte(s))
var nerr errors.NonFatal
jval, err := dec.Read()
if !nerr.Merge(err) {
return pref.Value{}, err
}
return getFloat(jval, bitSize)
}
return pref.Value{}, unexpectedJSONError{jval}
}
func getFloat(jval json.Value, bitSize int) (pref.Value, error) {
n, err := jval.Float(bitSize)
if err != nil {
return pref.Value{}, err
}
if bitSize == 32 {
return pref.ValueOf(float32(n)), nil
}
return pref.ValueOf(n), nil
}
func unmarshalString(jval json.Value) (pref.Value, error) {
if jval.Type() != json.String {
return pref.Value{}, unexpectedJSONError{jval}
}
return pref.ValueOf(jval.String()), nil
}
func unmarshalBytes(jval json.Value) (pref.Value, error) {
if jval.Type() != json.String {
return pref.Value{}, unexpectedJSONError{jval}
}
s := jval.String()
enc := base64.StdEncoding
if strings.ContainsAny(s, "-_") {
enc = base64.URLEncoding
}
if len(s)%4 != 0 {
enc = enc.WithPadding(base64.NoPadding)
}
b, err := enc.DecodeString(s)
if err != nil {
return pref.Value{}, err
}
return pref.ValueOf(b), nil
}
func unmarshalEnum(jval json.Value, fd pref.FieldDescriptor) (pref.Value, error) {
switch jval.Type() {
case json.String:
// Lookup EnumNumber based on name.
s := jval.String()
if enumVal := fd.Enum().Values().ByName(pref.Name(s)); enumVal != nil {
return pref.ValueOf(enumVal.Number()), nil
}
return pref.Value{}, newError("invalid enum value %q", jval)
case json.Number:
n, err := jval.Int(32)
if err != nil {
return pref.Value{}, err
}
return pref.ValueOf(pref.EnumNumber(n)), nil
case json.Null:
// This is only valid for google.protobuf.NullValue.
if isNullValue(fd) {
return pref.ValueOf(pref.EnumNumber(0)), nil
}
}
return pref.Value{}, unexpectedJSONError{jval}
}
func (o UnmarshalOptions) unmarshalList(list pref.List, fd pref.FieldDescriptor) error {
var nerr errors.NonFatal
jval, err := o.decoder.Read()
if !nerr.Merge(err) {
return err
}
if jval.Type() != json.StartArray {
return unexpectedJSONError{jval}
}
switch fd.Kind() {
case pref.MessageKind, pref.GroupKind:
for {
m := list.NewMessage()
err := o.unmarshalMessage(m, false)
if !nerr.Merge(err) {
if e, ok := err.(unexpectedJSONError); ok {
if e.value.Type() == json.EndArray {
// Done with list.
return nerr.E
}
}
return err
}
list.Append(pref.ValueOf(m))
}
default:
for {
val, err := o.unmarshalScalar(fd)
if !nerr.Merge(err) {
if e, ok := err.(unexpectedJSONError); ok {
if e.value.Type() == json.EndArray {
// Done with list.
return nerr.E
}
}
return err
}
list.Append(val)
}
}
return nerr.E
}
func (o UnmarshalOptions) unmarshalMap(mmap pref.Map, fd pref.FieldDescriptor) error {
var nerr errors.NonFatal
jval, err := o.decoder.Read()
if !nerr.Merge(err) {
return err
}
if jval.Type() != json.StartObject {
return unexpectedJSONError{jval}
}
// Determine ahead whether map entry is a scalar type or a message type in
// order to call the appropriate unmarshalMapValue func inside the for loop
// below.
var unmarshalMapValue func() (pref.Value, error)
switch fd.MapValue().Kind() {
case pref.MessageKind, pref.GroupKind:
unmarshalMapValue = func() (pref.Value, error) {
var nerr errors.NonFatal
m := mmap.NewMessage()
if err := o.unmarshalMessage(m, false); !nerr.Merge(err) {
return pref.Value{}, err
}
return pref.ValueOf(m), nerr.E
}
default:
unmarshalMapValue = func() (pref.Value, error) {
return o.unmarshalScalar(fd.MapValue())
}
}
Loop:
for {
// Read field name.
jval, err := o.decoder.Read()
if !nerr.Merge(err) {
return err
}
switch jval.Type() {
default:
return unexpectedJSONError{jval}
case json.EndObject:
break Loop
case json.Name:
// Continue.
}
name, err := jval.Name()
if !nerr.Merge(err) {
return err
}
// Unmarshal field name.
pkey, err := unmarshalMapKey(name, fd.MapKey())
if !nerr.Merge(err) {
return err
}
// Check for duplicate field name.
if mmap.Has(pkey) {
return newError("duplicate map key %q", jval)
}
// Read and unmarshal field value.
pval, err := unmarshalMapValue()
if !nerr.Merge(err) {
return err
}
mmap.Set(pkey, pval)
}
return nerr.E
}
// unmarshalMapKey converts given string into a protoreflect.MapKey. A map key type is any
// integral or string type.
func unmarshalMapKey(name string, fd pref.FieldDescriptor) (pref.MapKey, error) {
const b32 = 32
const b64 = 64
const base10 = 10
kind := fd.Kind()
switch kind {
case pref.StringKind:
return pref.ValueOf(name).MapKey(), nil
case pref.BoolKind:
switch name {
case "true":
return pref.ValueOf(true).MapKey(), nil
case "false":
return pref.ValueOf(false).MapKey(), nil
}
return pref.MapKey{}, errors.New("invalid value for boolean key %q", name)
case pref.Int32Kind, pref.Sint32Kind, pref.Sfixed32Kind:
n, err := strconv.ParseInt(name, base10, b32)
if err != nil {
return pref.MapKey{}, err
}
return pref.ValueOf(int32(n)).MapKey(), nil
case pref.Int64Kind, pref.Sint64Kind, pref.Sfixed64Kind:
n, err := strconv.ParseInt(name, base10, b64)
if err != nil {
return pref.MapKey{}, err
}
return pref.ValueOf(int64(n)).MapKey(), nil
case pref.Uint32Kind, pref.Fixed32Kind:
n, err := strconv.ParseUint(name, base10, b32)
if err != nil {
return pref.MapKey{}, err
}
return pref.ValueOf(uint32(n)).MapKey(), nil
case pref.Uint64Kind, pref.Fixed64Kind:
n, err := strconv.ParseUint(name, base10, b64)
if err != nil {
return pref.MapKey{}, err
}
return pref.ValueOf(uint64(n)).MapKey(), nil
}
panic(fmt.Sprintf("%s: invalid kind %s for map key", fd.FullName(), kind))
}