encoding/protojson: refactor to follow prototext pattern
All unmarshaling error messages now contain line number and column
information, except for the following errors:
- `unexpected EOF`
- `no support for proto1 MessageSets`
- `required fields X not set`
Changes to internal/encoding/json:
- Moved encoding funcs in string.go and number.go into encode.go.
- Separated out encoding kind constants from decoding ones.
- Renamed file string.go to decode_string.go.
- Renamed file number.go to decode_number.go.
- Renamed Type struct to Kind.
- Renamed Value struct to Token.
- Token accessor methods no longer return error.
Name, Bool, ParsedString will panic if called on the wrong kind.
Float, Int, Uint has ok bool result to check against.
- Changed Peek to return Token and error.
Changes to encoding/protojson:
- Updated internal/encoding/json API calls.
- Added line info on most unmarshaling error messages and kept
description simple and consistent.
Change-Id: Ie50456694f2214c5c4fafd2c9b9239680da0deec
Reviewed-on: https://go-review.googlesource.com/c/protobuf/+/218978
Reviewed-by: Joe Tsai <thebrokentoaster@gmail.com>
diff --git a/encoding/protojson/decode.go b/encoding/protojson/decode.go
index db8a3df..4d13d77 100644
--- a/encoding/protojson/decode.go
+++ b/encoding/protojson/decode.go
@@ -45,8 +45,6 @@
protoregistry.MessageTypeResolver
protoregistry.ExtensionTypeResolver
}
-
- decoder *json.Decoder
}
// Unmarshal reads the given []byte and populates the given proto.Message using
@@ -59,19 +57,19 @@
if o.Resolver == nil {
o.Resolver = protoregistry.GlobalTypes
}
- o.decoder = json.NewDecoder(b)
- if err := o.unmarshalMessage(m.ProtoReflect(), false); err != nil {
+ dec := decoder{json.NewDecoder(b), o}
+ if err := dec.unmarshalMessage(m.ProtoReflect(), false); err != nil {
return err
}
// Check for EOF.
- val, err := o.decoder.Read()
+ tok, err := dec.Read()
if err != nil {
return err
}
- if val.Type() != json.EOF {
- return unexpectedJSONError{val}
+ if tok.Kind() != json.EOF {
+ return dec.unexpectedTokenError(tok)
}
if o.AllowPartial {
@@ -80,53 +78,45 @@
return proto.IsInitialized(m)
}
-// unexpectedJSONError is an error that contains the unexpected json.Value. This
-// is returned by methods to provide callers the read json.Value that it did not
-// expect.
-// TODO: Consider moving this to internal/encoding/json for consistency with
-// errors that package returns.
-type unexpectedJSONError struct {
- value json.Value
+type decoder struct {
+ *json.Decoder
+ opts UnmarshalOptions
}
-func (e unexpectedJSONError) Error() string {
- return newError("unexpected value %s", e.value).Error()
+// newError returns an error object with position info.
+func (d decoder) newError(pos int, f string, x ...interface{}) error {
+ line, column := d.Position(pos)
+ head := fmt.Sprintf("(line %d:%d): ", line, column)
+ return errors.New(head+f, x...)
}
-// newError returns an error object. If one of the values passed in is of
-// json.Value type, it produces an error with position info.
-func newError(f string, x ...interface{}) error {
- var hasValue bool
- var line, column int
- for i := 0; i < len(x); i++ {
- if val, ok := x[i].(json.Value); ok {
- line, column = val.Position()
- hasValue = true
- break
- }
- }
- e := errors.New(f, x...)
- if hasValue {
- return errors.New("(line %d:%d): %v", line, column, e)
- }
- return e
+// unexpectedTokenError returns a syntax error for the given unexpected token.
+func (d decoder) unexpectedTokenError(tok json.Token) error {
+ return d.syntaxError(tok.Pos(), "unexpected token %s", tok.RawString())
+}
+
+// syntaxError returns a syntax error for given position.
+func (d decoder) syntaxError(pos int, f string, x ...interface{}) error {
+ line, column := d.Position(pos)
+ head := fmt.Sprintf("syntax error (line %d:%d): ", line, column)
+ return errors.New(head+f, x...)
}
// unmarshalMessage unmarshals a message into the given protoreflect.Message.
-func (o UnmarshalOptions) unmarshalMessage(m pref.Message, skipTypeURL bool) error {
+func (d decoder) unmarshalMessage(m pref.Message, skipTypeURL bool) error {
if isCustomType(m.Descriptor().FullName()) {
- return o.unmarshalCustomType(m)
+ return d.unmarshalCustomType(m)
}
- jval, err := o.decoder.Read()
+ tok, err := d.Read()
if err != nil {
return err
}
- if jval.Type() != json.StartObject {
- return unexpectedJSONError{jval}
+ if tok.Kind() != json.ObjectOpen {
+ return d.unexpectedTokenError(tok)
}
- if err := o.unmarshalFields(m, skipTypeURL); err != nil {
+ if err := d.unmarshalFields(m, skipTypeURL); err != nil {
return err
}
@@ -134,7 +124,7 @@
}
// unmarshalFields unmarshals the fields into the given protoreflect.Message.
-func (o UnmarshalOptions) unmarshalFields(m pref.Message, skipTypeURL bool) error {
+func (d decoder) unmarshalFields(m pref.Message, skipTypeURL bool) error {
messageDesc := m.Descriptor()
if !flags.ProtoLegacy && messageset.IsMessageSet(messageDesc) {
return errors.New("no support for proto1 MessageSets")
@@ -145,28 +135,25 @@
fieldDescs := messageDesc.Fields()
for {
// Read field name.
- jval, err := o.decoder.Read()
+ tok, err := d.Read()
if err != nil {
return err
}
- switch jval.Type() {
+ switch tok.Kind() {
default:
- return unexpectedJSONError{jval}
- case json.EndObject:
+ return d.unexpectedTokenError(tok)
+ case json.ObjectClose:
return nil
case json.Name:
// Continue below.
}
- name, err := jval.Name()
- if err != nil {
- return err
- }
+ name := tok.Name()
// Unmarshaling a non-custom embedded message in Any will contain the
// JSON field "@type" which should be skipped because it is not a field
// of the embedded message, but simply an artifact of the Any format.
if skipTypeURL && name == "@type" {
- o.decoder.Read()
+ d.Read()
continue
}
@@ -175,14 +162,14 @@
if strings.HasPrefix(name, "[") && strings.HasSuffix(name, "]") {
// Only extension names are in [name] format.
extName := pref.FullName(name[1 : len(name)-1])
- extType, err := o.findExtension(extName)
+ extType, err := d.findExtension(extName)
if err != nil && err != protoregistry.NotFound {
- return errors.New("unable to resolve [%v]: %v", extName, err)
+ return d.newError(tok.Pos(), "unable to resolve %s: %v", tok.RawString(), err)
}
if extType != nil {
fd = extType.TypeDescriptor()
if !messageDesc.ExtensionRanges().Has(fd.Number()) || fd.ContainingMessage().FullName() != messageDesc.FullName() {
- return errors.New("message %v cannot be extended by %v", messageDesc.FullName(), fd.FullName())
+ return d.newError(tok.Pos(), "message %v cannot be extended by %v", messageDesc.FullName(), fd.FullName())
}
}
} else {
@@ -210,65 +197,65 @@
if fd == nil {
// Field is unknown.
- if o.DiscardUnknown {
- if err := skipJSONValue(o.decoder); err != nil {
+ if d.opts.DiscardUnknown {
+ if err := d.skipJSONValue(); err != nil {
return err
}
continue
}
- return newError("%v contains unknown field %s", messageDesc.FullName(), jval)
+ return d.newError(tok.Pos(), "unknown field %v", tok.RawString())
}
// Do not allow duplicate fields.
num := uint64(fd.Number())
if seenNums.Has(num) {
- return newError("%v contains repeated field %s", messageDesc.FullName(), jval)
+ return d.newError(tok.Pos(), "duplicate field %v", tok.RawString())
}
seenNums.Set(num)
// No need to set values for JSON null unless the field type is
// google.protobuf.Value or google.protobuf.NullValue.
- if o.decoder.Peek() == json.Null && !isKnownValue(fd) && !isNullValue(fd) {
- o.decoder.Read()
+ if tok, _ := d.Peek(); tok.Kind() == json.Null && !isKnownValue(fd) && !isNullValue(fd) {
+ d.Read()
continue
}
switch {
case fd.IsList():
list := m.Mutable(fd).List()
- if err := o.unmarshalList(list, fd); err != nil {
- return errors.New("%v|%q: %v", fd.FullName(), name, err)
+ if err := d.unmarshalList(list, fd); err != nil {
+ return err
}
case fd.IsMap():
mmap := m.Mutable(fd).Map()
- if err := o.unmarshalMap(mmap, fd); err != nil {
- return errors.New("%v|%q: %v", fd.FullName(), name, err)
+ if err := d.unmarshalMap(mmap, fd); err != nil {
+ return err
}
default:
// If field is a oneof, check if it has already been set.
if od := fd.ContainingOneof(); od != nil {
idx := uint64(od.Index())
if seenOneofs.Has(idx) {
- return errors.New("%v: oneof is already set", od.FullName())
+ return d.newError(tok.Pos(), "error parsing %s, oneof %v is already set", tok.RawString(), od.FullName())
}
seenOneofs.Set(idx)
}
// Required or optional fields.
- if err := o.unmarshalSingular(m, fd); err != nil {
- return errors.New("%v|%q: %v", fd.FullName(), name, err)
+ if err := d.unmarshalSingular(m, fd); err != nil {
+ return err
}
}
}
}
// findExtension returns protoreflect.ExtensionType from the resolver if found.
-func (o UnmarshalOptions) findExtension(xtName pref.FullName) (pref.ExtensionType, error) {
- xt, err := o.Resolver.FindExtensionByName(xtName)
+func (d decoder) findExtension(xtName pref.FullName) (pref.ExtensionType, error) {
+ xt, err := d.opts.Resolver.FindExtensionByName(xtName)
if err == nil {
return xt, nil
}
- return messageset.FindMessageSetExtension(o.Resolver, xtName)
+ return messageset.FindMessageSetExtension(d.opts.Resolver, xtName)
}
func isKnownValue(fd pref.FieldDescriptor) bool {
@@ -281,17 +268,17 @@
return ed != nil && ed.FullName() == "google.protobuf.NullValue"
}
-// unmarshalSingular unmarshals to the non-repeated field specified by the given
-// FieldDescriptor.
-func (o UnmarshalOptions) unmarshalSingular(m pref.Message, fd pref.FieldDescriptor) error {
+// unmarshalSingular unmarshals to the non-repeated field specified
+// by the given FieldDescriptor.
+func (d decoder) unmarshalSingular(m pref.Message, fd pref.FieldDescriptor) error {
var val pref.Value
var err error
switch fd.Kind() {
case pref.MessageKind, pref.GroupKind:
val = m.NewField(fd)
- err = o.unmarshalMessage(val.Message(), false)
+ err = d.unmarshalMessage(val.Message(), false)
default:
- val, err = o.unmarshalScalar(fd)
+ val, err = d.unmarshalScalar(fd)
}
if err != nil {
@@ -303,11 +290,11 @@
// unmarshalScalar unmarshals to a scalar/enum protoreflect.Value specified by
// the given FieldDescriptor.
-func (o UnmarshalOptions) unmarshalScalar(fd pref.FieldDescriptor) (pref.Value, error) {
+func (d decoder) unmarshalScalar(fd pref.FieldDescriptor) (pref.Value, error) {
const b32 int = 32
const b64 int = 64
- jval, err := o.decoder.Read()
+ tok, err := d.Read()
if err != nil {
return pref.Value{}, err
}
@@ -315,177 +302,182 @@
kind := fd.Kind()
switch kind {
case pref.BoolKind:
- return unmarshalBool(jval)
+ if tok.Kind() == json.Bool {
+ return pref.ValueOfBool(tok.Bool()), nil
+ }
case pref.Int32Kind, pref.Sint32Kind, pref.Sfixed32Kind:
- return unmarshalInt(jval, b32)
+ if v, ok := unmarshalInt(tok, b32); ok {
+ return v, nil
+ }
case pref.Int64Kind, pref.Sint64Kind, pref.Sfixed64Kind:
- return unmarshalInt(jval, b64)
+ if v, ok := unmarshalInt(tok, b64); ok {
+ return v, nil
+ }
case pref.Uint32Kind, pref.Fixed32Kind:
- return unmarshalUint(jval, b32)
+ if v, ok := unmarshalUint(tok, b32); ok {
+ return v, nil
+ }
case pref.Uint64Kind, pref.Fixed64Kind:
- return unmarshalUint(jval, b64)
+ if v, ok := unmarshalUint(tok, b64); ok {
+ return v, nil
+ }
case pref.FloatKind:
- return unmarshalFloat(jval, b32)
+ if v, ok := unmarshalFloat(tok, b32); ok {
+ return v, nil
+ }
case pref.DoubleKind:
- return unmarshalFloat(jval, b64)
+ if v, ok := unmarshalFloat(tok, b64); ok {
+ return v, nil
+ }
case pref.StringKind:
- pval, err := unmarshalString(jval)
- if err != nil {
- return pval, err
+ if tok.Kind() == json.String {
+ return pref.ValueOfString(tok.ParsedString()), nil
}
- return pval, nil
case pref.BytesKind:
- return unmarshalBytes(jval)
+ if v, ok := unmarshalBytes(tok); ok {
+ return v, nil
+ }
case pref.EnumKind:
- return unmarshalEnum(jval, fd)
+ if v, ok := unmarshalEnum(tok, fd); ok {
+ return v, nil
+ }
+
+ default:
+ panic(fmt.Sprintf("unmarshalScalar: invalid scalar kind %v", kind))
}
- panic(fmt.Sprintf("invalid scalar kind %v", kind))
+ return pref.Value{}, d.newError(tok.Pos(), "invalid value for %v type: %v", kind, tok.RawString())
}
-func unmarshalBool(jval json.Value) (pref.Value, error) {
- if jval.Type() != json.Bool {
- return pref.Value{}, unexpectedJSONError{jval}
- }
- b, err := jval.Bool()
- return pref.ValueOfBool(b), err
-}
-
-func unmarshalInt(jval json.Value, bitSize int) (pref.Value, error) {
- switch jval.Type() {
+func unmarshalInt(tok json.Token, bitSize int) (pref.Value, bool) {
+ switch tok.Kind() {
case json.Number:
- return getInt(jval, bitSize)
+ return getInt(tok, bitSize)
case json.String:
// Decode number from string.
- s := strings.TrimSpace(jval.String())
- if len(s) != len(jval.String()) {
- return pref.Value{}, errors.New("invalid number %v", jval.Raw())
+ s := strings.TrimSpace(tok.ParsedString())
+ if len(s) != len(tok.ParsedString()) {
+ return pref.Value{}, false
}
dec := json.NewDecoder([]byte(s))
- jval, err := dec.Read()
+ tok, err := dec.Read()
if err != nil {
- return pref.Value{}, err
+ return pref.Value{}, false
}
- return getInt(jval, bitSize)
+ return getInt(tok, bitSize)
}
- return pref.Value{}, unexpectedJSONError{jval}
+ return pref.Value{}, false
}
-func getInt(jval json.Value, bitSize int) (pref.Value, error) {
- n, err := jval.Int(bitSize)
- if err != nil {
- return pref.Value{}, err
+func getInt(tok json.Token, bitSize int) (pref.Value, bool) {
+ n, ok := tok.Int(bitSize)
+ if !ok {
+ return pref.Value{}, false
}
if bitSize == 32 {
- return pref.ValueOfInt32(int32(n)), nil
+ return pref.ValueOfInt32(int32(n)), true
}
- return pref.ValueOfInt64(n), nil
+ return pref.ValueOfInt64(n), true
}
-func unmarshalUint(jval json.Value, bitSize int) (pref.Value, error) {
- switch jval.Type() {
+func unmarshalUint(tok json.Token, bitSize int) (pref.Value, bool) {
+ switch tok.Kind() {
case json.Number:
- return getUint(jval, bitSize)
+ return getUint(tok, bitSize)
case json.String:
// Decode number from string.
- s := strings.TrimSpace(jval.String())
- if len(s) != len(jval.String()) {
- return pref.Value{}, errors.New("invalid number %v", jval.Raw())
+ s := strings.TrimSpace(tok.ParsedString())
+ if len(s) != len(tok.ParsedString()) {
+ return pref.Value{}, false
}
dec := json.NewDecoder([]byte(s))
- jval, err := dec.Read()
+ tok, err := dec.Read()
if err != nil {
- return pref.Value{}, err
+ return pref.Value{}, false
}
- return getUint(jval, bitSize)
+ return getUint(tok, bitSize)
}
- return pref.Value{}, unexpectedJSONError{jval}
+ return pref.Value{}, false
}
-func getUint(jval json.Value, bitSize int) (pref.Value, error) {
- n, err := jval.Uint(bitSize)
- if err != nil {
- return pref.Value{}, err
+func getUint(tok json.Token, bitSize int) (pref.Value, bool) {
+ n, ok := tok.Uint(bitSize)
+ if !ok {
+ return pref.Value{}, false
}
if bitSize == 32 {
- return pref.ValueOfUint32(uint32(n)), nil
+ return pref.ValueOfUint32(uint32(n)), true
}
- return pref.ValueOfUint64(n), nil
+ return pref.ValueOfUint64(n), true
}
-func unmarshalFloat(jval json.Value, bitSize int) (pref.Value, error) {
- switch jval.Type() {
+func unmarshalFloat(tok json.Token, bitSize int) (pref.Value, bool) {
+ switch tok.Kind() {
case json.Number:
- return getFloat(jval, bitSize)
+ return getFloat(tok, bitSize)
case json.String:
- s := jval.String()
+ s := tok.ParsedString()
switch s {
case "NaN":
if bitSize == 32 {
- return pref.ValueOfFloat32(float32(math.NaN())), nil
+ return pref.ValueOfFloat32(float32(math.NaN())), true
}
- return pref.ValueOfFloat64(math.NaN()), nil
+ return pref.ValueOfFloat64(math.NaN()), true
case "Infinity":
if bitSize == 32 {
- return pref.ValueOfFloat32(float32(math.Inf(+1))), nil
+ return pref.ValueOfFloat32(float32(math.Inf(+1))), true
}
- return pref.ValueOfFloat64(math.Inf(+1)), nil
+ return pref.ValueOfFloat64(math.Inf(+1)), true
case "-Infinity":
if bitSize == 32 {
- return pref.ValueOfFloat32(float32(math.Inf(-1))), nil
+ return pref.ValueOfFloat32(float32(math.Inf(-1))), true
}
- return pref.ValueOfFloat64(math.Inf(-1)), nil
+ return pref.ValueOfFloat64(math.Inf(-1)), true
}
+
// Decode number from string.
if len(s) != len(strings.TrimSpace(s)) {
- return pref.Value{}, errors.New("invalid number %v", jval.Raw())
+ return pref.Value{}, false
}
dec := json.NewDecoder([]byte(s))
- jval, err := dec.Read()
+ tok, err := dec.Read()
if err != nil {
- return pref.Value{}, err
+ return pref.Value{}, false
}
- return getFloat(jval, bitSize)
+ return getFloat(tok, bitSize)
}
- return pref.Value{}, unexpectedJSONError{jval}
+ return pref.Value{}, false
}
-func getFloat(jval json.Value, bitSize int) (pref.Value, error) {
- n, err := jval.Float(bitSize)
- if err != nil {
- return pref.Value{}, err
+func getFloat(tok json.Token, bitSize int) (pref.Value, bool) {
+ n, ok := tok.Float(bitSize)
+ if !ok {
+ return pref.Value{}, false
}
if bitSize == 32 {
- return pref.ValueOfFloat32(float32(n)), nil
+ return pref.ValueOfFloat32(float32(n)), true
}
- return pref.ValueOfFloat64(n), nil
+ return pref.ValueOfFloat64(n), true
}
-func unmarshalString(jval json.Value) (pref.Value, error) {
- if jval.Type() != json.String {
- return pref.Value{}, unexpectedJSONError{jval}
- }
- return pref.ValueOfString(jval.String()), nil
-}
-
-func unmarshalBytes(jval json.Value) (pref.Value, error) {
- if jval.Type() != json.String {
- return pref.Value{}, unexpectedJSONError{jval}
+func unmarshalBytes(tok json.Token) (pref.Value, bool) {
+ if tok.Kind() != json.String {
+ return pref.Value{}, false
}
- s := jval.String()
+ s := tok.ParsedString()
enc := base64.StdEncoding
if strings.ContainsAny(s, "-_") {
enc = base64.URLEncoding
@@ -495,88 +487,93 @@
}
b, err := enc.DecodeString(s)
if err != nil {
- return pref.Value{}, err
+ return pref.Value{}, false
}
- return pref.ValueOfBytes(b), nil
+ return pref.ValueOfBytes(b), true
}
-func unmarshalEnum(jval json.Value, fd pref.FieldDescriptor) (pref.Value, error) {
- switch jval.Type() {
+func unmarshalEnum(tok json.Token, fd pref.FieldDescriptor) (pref.Value, bool) {
+ switch tok.Kind() {
case json.String:
// Lookup EnumNumber based on name.
- s := jval.String()
+ s := tok.ParsedString()
if enumVal := fd.Enum().Values().ByName(pref.Name(s)); enumVal != nil {
- return pref.ValueOfEnum(enumVal.Number()), nil
+ return pref.ValueOfEnum(enumVal.Number()), true
}
- return pref.Value{}, newError("invalid enum value %q", jval)
case json.Number:
- n, err := jval.Int(32)
- if err != nil {
- return pref.Value{}, err
+ if n, ok := tok.Int(32); ok {
+ return pref.ValueOfEnum(pref.EnumNumber(n)), true
}
- return pref.ValueOfEnum(pref.EnumNumber(n)), nil
case json.Null:
// This is only valid for google.protobuf.NullValue.
if isNullValue(fd) {
- return pref.ValueOfEnum(0), nil
+ return pref.ValueOfEnum(0), true
}
}
- return pref.Value{}, unexpectedJSONError{jval}
+ return pref.Value{}, false
}
-func (o UnmarshalOptions) unmarshalList(list pref.List, fd pref.FieldDescriptor) error {
- jval, err := o.decoder.Read()
+func (d decoder) unmarshalList(list pref.List, fd pref.FieldDescriptor) error {
+ tok, err := d.Read()
if err != nil {
return err
}
- if jval.Type() != json.StartArray {
- return unexpectedJSONError{jval}
+ if tok.Kind() != json.ArrayOpen {
+ return d.unexpectedTokenError(tok)
}
switch fd.Kind() {
case pref.MessageKind, pref.GroupKind:
for {
- val := list.NewElement()
- err := o.unmarshalMessage(val.Message(), false)
+ tok, err := d.Peek()
if err != nil {
- if e, ok := err.(unexpectedJSONError); ok {
- if e.value.Type() == json.EndArray {
- // Done with list.
- return nil
- }
- }
+ return err
+ }
+
+ if tok.Kind() == json.ArrayClose {
+ d.Read()
+ return nil
+ }
+
+ val := list.NewElement()
+ if err := d.unmarshalMessage(val.Message(), false); err != nil {
return err
}
list.Append(val)
}
default:
for {
- val, err := o.unmarshalScalar(fd)
+ tok, err := d.Peek()
if err != nil {
- if e, ok := err.(unexpectedJSONError); ok {
- if e.value.Type() == json.EndArray {
- // Done with list.
- return nil
- }
- }
+ return err
+ }
+
+ if tok.Kind() == json.ArrayClose {
+ d.Read()
+ return nil
+ }
+
+ val, err := d.unmarshalScalar(fd)
+ if err != nil {
return err
}
list.Append(val)
}
}
+
return nil
}
-func (o UnmarshalOptions) unmarshalMap(mmap pref.Map, fd pref.FieldDescriptor) error {
- jval, err := o.decoder.Read()
+func (d decoder) unmarshalMap(mmap pref.Map, fd pref.FieldDescriptor) error {
+ tok, err := d.Read()
if err != nil {
return err
}
- if jval.Type() != json.StartObject {
- return unexpectedJSONError{jval}
+ if tok.Kind() != json.ObjectOpen {
+ return d.unexpectedTokenError(tok)
}
// Determine ahead whether map entry is a scalar type or a message type in
@@ -587,47 +584,42 @@
case pref.MessageKind, pref.GroupKind:
unmarshalMapValue = func() (pref.Value, error) {
val := mmap.NewValue()
- if err := o.unmarshalMessage(val.Message(), false); err != nil {
+ if err := d.unmarshalMessage(val.Message(), false); err != nil {
return pref.Value{}, err
}
return val, nil
}
default:
unmarshalMapValue = func() (pref.Value, error) {
- return o.unmarshalScalar(fd.MapValue())
+ return d.unmarshalScalar(fd.MapValue())
}
}
Loop:
for {
// Read field name.
- jval, err := o.decoder.Read()
+ tok, err := d.Read()
if err != nil {
return err
}
- switch jval.Type() {
+ switch tok.Kind() {
default:
- return unexpectedJSONError{jval}
- case json.EndObject:
+ return d.unexpectedTokenError(tok)
+ case json.ObjectClose:
break Loop
case json.Name:
// Continue.
}
- name, err := jval.Name()
- if err != nil {
- return err
- }
-
// Unmarshal field name.
- pkey, err := unmarshalMapKey(name, fd.MapKey())
+ pkey, err := d.unmarshalMapKey(tok, fd.MapKey())
if err != nil {
return err
}
// Check for duplicate field name.
if mmap.Has(pkey) {
- return newError("duplicate map key %q", jval)
+ return d.newError(tok.Pos(), "duplicate map key %v", tok.RawString())
}
// Read and unmarshal field value.
@@ -642,13 +634,14 @@
return nil
}
-// unmarshalMapKey converts given string into a protoreflect.MapKey. A map key type is any
-// integral or string type.
-func unmarshalMapKey(name string, fd pref.FieldDescriptor) (pref.MapKey, error) {
+// unmarshalMapKey converts given token of Name kind into a protoreflect.MapKey.
+// A map key type is any integral or string type.
+func (d decoder) unmarshalMapKey(tok json.Token, fd pref.FieldDescriptor) (pref.MapKey, error) {
const b32 = 32
const b64 = 64
const base10 = 10
+ name := tok.Name()
kind := fd.Kind()
switch kind {
case pref.StringKind:
@@ -661,36 +654,30 @@
case "false":
return pref.ValueOfBool(false).MapKey(), nil
}
- return pref.MapKey{}, errors.New("invalid value for boolean key %q", name)
case pref.Int32Kind, pref.Sint32Kind, pref.Sfixed32Kind:
- n, err := strconv.ParseInt(name, base10, b32)
- if err != nil {
- return pref.MapKey{}, err
+ if n, err := strconv.ParseInt(name, base10, b32); err == nil {
+ return pref.ValueOfInt32(int32(n)).MapKey(), nil
}
- return pref.ValueOfInt32(int32(n)).MapKey(), nil
case pref.Int64Kind, pref.Sint64Kind, pref.Sfixed64Kind:
- n, err := strconv.ParseInt(name, base10, b64)
- if err != nil {
- return pref.MapKey{}, err
+ if n, err := strconv.ParseInt(name, base10, b64); err == nil {
+ return pref.ValueOfInt64(int64(n)).MapKey(), nil
}
- return pref.ValueOfInt64(int64(n)).MapKey(), nil
case pref.Uint32Kind, pref.Fixed32Kind:
- n, err := strconv.ParseUint(name, base10, b32)
- if err != nil {
- return pref.MapKey{}, err
+ if n, err := strconv.ParseUint(name, base10, b32); err == nil {
+ return pref.ValueOfUint32(uint32(n)).MapKey(), nil
}
- return pref.ValueOfUint32(uint32(n)).MapKey(), nil
case pref.Uint64Kind, pref.Fixed64Kind:
- n, err := strconv.ParseUint(name, base10, b64)
- if err != nil {
- return pref.MapKey{}, err
+ if n, err := strconv.ParseUint(name, base10, b64); err == nil {
+ return pref.ValueOfUint64(uint64(n)).MapKey(), nil
}
- return pref.ValueOfUint64(uint64(n)).MapKey(), nil
+
+ default:
+ panic(fmt.Sprintf("invalid kind for map key: %v", kind))
}
- panic(fmt.Sprintf("%s: invalid kind %s for map key", fd.FullName(), kind))
+ return pref.MapKey{}, d.newError(tok.Pos(), "invalid value for %v key: %s", kind, tok.RawString())
}
diff --git a/encoding/protojson/decode_test.go b/encoding/protojson/decode_test.go
index e37a9a3..5ef4594 100644
--- a/encoding/protojson/decode_test.go
+++ b/encoding/protojson/decode_test.go
@@ -6,9 +6,11 @@
import (
"math"
+ "strings"
"testing"
"google.golang.org/protobuf/encoding/protojson"
+ "google.golang.org/protobuf/internal/errors"
"google.golang.org/protobuf/internal/flags"
"google.golang.org/protobuf/proto"
preg "google.golang.org/protobuf/reflect/protoregistry"
@@ -33,9 +35,8 @@
inputMessage proto.Message
inputText string
wantMessage proto.Message
- // TODO: verify expected error message substring.
- wantErr bool
- skip bool
+ wantErr string // Expected error substring.
+ skip bool
}{{
desc: "proto2 empty message",
inputMessage: &pb2.Scalars{},
@@ -45,7 +46,7 @@
desc: "unexpected value instead of EOF",
inputMessage: &pb2.Scalars{},
inputText: "{} {}",
- wantErr: true,
+ wantErr: `(line 1:4): unexpected token {`,
}, {
desc: "proto2 optional scalars set to zero values",
inputMessage: &pb2.Scalars{},
@@ -157,7 +158,7 @@
desc: "not boolean",
inputMessage: &pb3.Scalars{},
inputText: `{"sBool": "true"}`,
- wantErr: true,
+ wantErr: `invalid value for bool type: "true"`,
}, {
desc: "float and double",
inputMessage: &pb3.Scalars{},
@@ -206,22 +207,22 @@
desc: "float exceeds limit",
inputMessage: &pb3.Scalars{},
inputText: `{"sFloat": 3.4e39}`,
- wantErr: true,
+ wantErr: `invalid value for float type: 3.4e39`,
}, {
desc: "float in string exceeds limit",
inputMessage: &pb3.Scalars{},
inputText: `{"sFloat": "-3.4e39"}`,
- wantErr: true,
+ wantErr: `invalid value for float type: "-3.4e39"`,
}, {
desc: "double exceeds limit",
inputMessage: &pb3.Scalars{},
- inputText: `{"sFloat": -1.79e+309}`,
- wantErr: true,
+ inputText: `{"sDouble": -1.79e+309}`,
+ wantErr: `invalid value for double type: -1.79e+309`,
}, {
desc: "double in string exceeds limit",
inputMessage: &pb3.Scalars{},
- inputText: `{"sFloat": "1.79e+309"}`,
- wantErr: true,
+ inputText: `{"sDouble": "1.79e+309"}`,
+ wantErr: `invalid value for double type: "1.79e+309"`,
}, {
desc: "infinites",
inputMessage: &pb3.Scalars{},
@@ -234,22 +235,22 @@
desc: "float string with leading space",
inputMessage: &pb3.Scalars{},
inputText: `{"sFloat": " 1.234"}`,
- wantErr: true,
+ wantErr: `invalid value for float type: " 1.234"`,
}, {
desc: "double string with trailing space",
inputMessage: &pb3.Scalars{},
inputText: `{"sDouble": "5.678 "}`,
- wantErr: true,
+ wantErr: `invalid value for double type: "5.678 "`,
}, {
desc: "not float",
inputMessage: &pb3.Scalars{},
inputText: `{"sFloat": true}`,
- wantErr: true,
+ wantErr: `invalid value for float type: true`,
}, {
desc: "not double",
inputMessage: &pb3.Scalars{},
inputText: `{"sDouble": "not a number"}`,
- wantErr: true,
+ wantErr: `invalid value for double type: "not a number"`,
}, {
desc: "integers",
inputMessage: &pb3.Scalars{},
@@ -315,42 +316,42 @@
desc: "integer string with leading space",
inputMessage: &pb3.Scalars{},
inputText: `{"sInt32": " 1234"}`,
- wantErr: true,
+ wantErr: `invalid value for int32 type: " 1234"`,
}, {
desc: "integer string with trailing space",
inputMessage: &pb3.Scalars{},
inputText: `{"sUint32": "1e2 "}`,
- wantErr: true,
+ wantErr: `invalid value for uint32 type: "1e2 "`,
}, {
desc: "number is not an integer",
inputMessage: &pb3.Scalars{},
inputText: `{"sInt32": 1.001}`,
- wantErr: true,
+ wantErr: `invalid value for int32 type: 1.001`,
}, {
desc: "32-bit int exceeds limit",
inputMessage: &pb3.Scalars{},
inputText: `{"sInt32": 2e10}`,
- wantErr: true,
+ wantErr: `invalid value for int32 type: 2e10`,
}, {
desc: "64-bit int exceeds limit",
inputMessage: &pb3.Scalars{},
inputText: `{"sSfixed64": -9e19}`,
- wantErr: true,
+ wantErr: `invalid value for sfixed64 type: -9e19`,
}, {
desc: "not integer",
inputMessage: &pb3.Scalars{},
inputText: `{"sInt32": "not a number"}`,
- wantErr: true,
+ wantErr: `invalid value for int32 type: "not a number"`,
}, {
desc: "not unsigned integer",
inputMessage: &pb3.Scalars{},
inputText: `{"sUint32": "not a number"}`,
- wantErr: true,
+ wantErr: `invalid value for uint32 type: "not a number"`,
}, {
desc: "number is not an unsigned integer",
inputMessage: &pb3.Scalars{},
inputText: `{"sUint32": -1}`,
- wantErr: true,
+ wantErr: `invalid value for uint32 type: -1`,
}, {
desc: "string",
inputMessage: &pb2.Scalars{},
@@ -362,12 +363,12 @@
desc: "string with invalid UTF-8",
inputMessage: &pb3.Scalars{},
inputText: "{\"sString\": \"\xff\"}",
- wantErr: true,
+ wantErr: `(line 1:13): invalid UTF-8 in string`,
}, {
desc: "not string",
inputMessage: &pb2.Scalars{},
inputText: `{"optString": 42}`,
- wantErr: true,
+ wantErr: `invalid value for string type: 42`,
}, {
desc: "bytes",
inputMessage: &pb3.Scalars{},
@@ -386,7 +387,7 @@
desc: "not bytes",
inputMessage: &pb3.Scalars{},
inputText: `{"sBytes": true}`,
- wantErr: true,
+ wantErr: `invalid value for bytes type: true`,
}, {
desc: "proto2 enum",
inputMessage: &pb2.Enums{},
@@ -437,21 +438,21 @@
inputText: `{
"sEnum": "1"
}`,
- wantErr: true,
+ wantErr: `invalid value for enum type: "1"`,
}, {
desc: "enum set to invalid named",
inputMessage: &pb3.Enums{},
inputText: `{
"sEnum": "UNNAMED"
}`,
- wantErr: true,
+ wantErr: `invalid value for enum type: "UNNAMED"`,
}, {
desc: "enum set to not enum",
inputMessage: &pb3.Enums{},
inputText: `{
"sEnum": true
}`,
- wantErr: true,
+ wantErr: `invalid value for enum type: true`,
}, {
desc: "enum set to JSON null",
inputMessage: &pb3.Enums{},
@@ -494,7 +495,7 @@
inputText: `{
"sString": "camelcase used"
}`,
- wantErr: true,
+ wantErr: `unknown field "sString"`,
}, {
desc: "proto name and json_name",
inputMessage: &pb3.JSONNames{},
@@ -502,7 +503,7 @@
"foo_bar": "json_name used",
"s_string": "proto name used"
}`,
- wantErr: true,
+ wantErr: `(line 3:3): duplicate field "s_string"`,
}, {
desc: "duplicate field names",
inputMessage: &pb3.JSONNames{},
@@ -510,12 +511,12 @@
"foo_bar": "one",
"foo_bar": "two",
}`,
- wantErr: true,
+ wantErr: `(line 3:3): duplicate field "foo_bar"`,
}, {
desc: "null message",
inputMessage: &pb2.Nests{},
inputText: "null",
- wantErr: true,
+ wantErr: `unexpected token null`,
}, {
desc: "proto2 nested message not set",
inputMessage: &pb2.Nests{},
@@ -624,12 +625,12 @@
desc: "message set to non-message",
inputMessage: &pb3.Nests{},
inputText: `"not valid"`,
- wantErr: true,
+ wantErr: `unexpected token "not valid"`,
}, {
desc: "nested message set to non-message",
inputMessage: &pb3.Nests{},
inputText: `{"sNested": true}`,
- wantErr: true,
+ wantErr: `(line 1:13): unexpected token true`,
}, {
desc: "oneof not set",
inputMessage: &pb3.Oneofs{},
@@ -691,7 +692,7 @@
"oneofEnum": "ZERO",
"oneofString": "hello"
}`,
- wantErr: true,
+ wantErr: `(line 3:3): error parsing "oneofString", oneof pb3.Oneofs.union is already set`,
}, {
desc: "oneof set to null and value",
inputMessage: &pb3.Oneofs{},
@@ -793,22 +794,22 @@
desc: "repeated string contains invalid UTF8",
inputMessage: &pb2.Repeats{},
inputText: `{"rptString": ["` + "abc\xff" + `"]}`,
- wantErr: true,
+ wantErr: `invalid UTF-8`,
}, {
desc: "repeated messages contain invalid UTF8",
inputMessage: &pb2.Nests{},
inputText: `{"rptNested": [{"optString": "` + "abc\xff" + `"}]}`,
- wantErr: true,
+ wantErr: `invalid UTF-8`,
}, {
desc: "repeated scalars contain invalid type",
inputMessage: &pb2.Repeats{},
inputText: `{"rptString": ["hello", null, "world"]}`,
- wantErr: true,
+ wantErr: `invalid value for string type: null`,
}, {
desc: "repeated messages contain invalid type",
inputMessage: &pb2.Nests{},
inputText: `{"rptNested": [{}, null]}`,
- wantErr: true,
+ wantErr: `unexpected token null`,
}, {
desc: "map fields 1",
inputMessage: &pb3.Maps{},
@@ -907,11 +908,11 @@
inputText: `{
"int32ToStr": {
"0": "cero",
- "0": "zero"
+ "0": "zero"
}
}
`,
- wantErr: true,
+ wantErr: `(line 4:5): duplicate map key "0"`,
}, {
desc: "map key empty string",
inputMessage: &pb3.Maps{},
@@ -931,24 +932,27 @@
inputText: `{
"int32ToStr": {
"invalid": "cero"
+ }
}`,
- wantErr: true,
+ wantErr: `invalid value for int32 key: "invalid"`,
}, {
desc: "map contains invalid key 2",
inputMessage: &pb3.Maps{},
inputText: `{
"int32ToStr": {
"1.02": "float"
+ }
}`,
- wantErr: true,
+ wantErr: `invalid value for int32 key: "1.02"`,
}, {
desc: "map contains invalid key 3",
inputMessage: &pb3.Maps{},
inputText: `{
"int32ToStr": {
"2147483648": "exceeds 32-bit integer max limit"
+ }
}`,
- wantErr: true,
+ wantErr: `invalid value for int32 key: "2147483648"`,
}, {
desc: "map contains invalid key 4",
inputMessage: &pb3.Maps{},
@@ -957,7 +961,7 @@
"-1": 0
}
}`,
- wantErr: true,
+ wantErr: `invalid value for uint64 key: "-1"`,
}, {
desc: "map contains invalid value",
inputMessage: &pb3.Maps{},
@@ -965,7 +969,7 @@
"int32ToStr": {
"101": true
}`,
- wantErr: true,
+ wantErr: `invalid value for string type: true`,
}, {
desc: "map contains null for scalar value",
inputMessage: &pb3.Maps{},
@@ -973,7 +977,7 @@
"int32ToStr": {
"101": null
}`,
- wantErr: true,
+ wantErr: `invalid value for string type: null`,
}, {
desc: "map contains null for message value",
inputMessage: &pb3.Maps{},
@@ -982,7 +986,7 @@
"hello": null
}
}`,
- wantErr: true,
+ wantErr: `unexpected token null`,
}, {
desc: "map contains contains message value with invalid UTF8",
inputMessage: &pb3.Maps{},
@@ -993,7 +997,7 @@
}
}
}`,
- wantErr: true,
+ wantErr: `invalid UTF-8`,
}, {
desc: "map key contains invalid UTF8",
inputMessage: &pb3.Maps{},
@@ -1002,11 +1006,12 @@
"` + "abc\xff" + `": {}
}
}`,
- wantErr: true,
+ wantErr: `invalid UTF-8`,
}, {
desc: "required fields not set",
inputMessage: &pb2.Requireds{},
- wantErr: true,
+ inputText: `{}`,
+ wantErr: errors.RequiredNotSet("pb2.Requireds.req_bool").Error(),
}, {
desc: "required field set",
inputMessage: &pb2.PartialRequired{},
@@ -1031,7 +1036,7 @@
ReqString: proto.String("hello"),
ReqEnum: pb2.Enum_ONE.Enum(),
},
- wantErr: true,
+ wantErr: errors.RequiredNotSet("pb2.Requireds.req_double").Error(),
}, {
desc: "required fields partially set with AllowPartial",
umo: protojson.UnmarshalOptions{AllowPartial: true},
@@ -1076,7 +1081,7 @@
wantMessage: &pb2.IndirectRequired{
OptNested: &pb2.NestedWithRequired{},
},
- wantErr: true,
+ wantErr: errors.RequiredNotSet("pb2.NestedWithRequired.req_string").Error(),
}, {
desc: "indirect required field with AllowPartial",
umo: protojson.UnmarshalOptions{AllowPartial: true},
@@ -1104,7 +1109,7 @@
{},
},
},
- wantErr: true,
+ wantErr: errors.RequiredNotSet("pb2.NestedWithRequired.req_string").Error(),
}, {
desc: "indirect required field in repeated with AllowPartial",
umo: protojson.UnmarshalOptions{AllowPartial: true},
@@ -1142,7 +1147,7 @@
},
},
},
- wantErr: true,
+ wantErr: errors.RequiredNotSet("pb2.NestedWithRequired.req_string").Error(),
}, {
desc: "indirect required field in map with AllowPartial",
umo: protojson.UnmarshalOptions{AllowPartial: true},
@@ -1174,7 +1179,7 @@
OneofNested: &pb2.NestedWithRequired{},
},
},
- wantErr: true,
+ wantErr: errors.RequiredNotSet("pb2.NestedWithRequired.req_string").Error(),
}, {
desc: "indirect required field in oneof with AllowPartial",
umo: protojson.UnmarshalOptions{AllowPartial: true},
@@ -1305,18 +1310,18 @@
desc: "invalid extension field name",
inputMessage: &pb2.Extensions{},
inputText: `{ "[pb2.invalid_message_field]": true }`,
- wantErr: true,
+ wantErr: `(line 1:3): unknown field "[pb2.invalid_message_field]"`,
}, {
desc: "extensions of repeated field contains null",
inputMessage: &pb2.Extensions{},
inputText: `{
"[pb2.ExtensionsContainer.rpt_ext_nested]": [
{"optString": "one"},
- null,
+ null,
{"optString": "three"}
],
}`,
- wantErr: true,
+ wantErr: `(line 4:5): unexpected token null`,
}, {
desc: "MessageSet",
inputMessage: &pb2.MessageSet{},
@@ -1369,7 +1374,7 @@
"optString": "not a messageset extension"
}
}`,
- wantErr: true,
+ wantErr: `unknown field "[pb2.FakeMessageSetExtension]"`,
skip: !flags.ProtoLegacy,
}, {
desc: "not real MessageSet 3",
@@ -1396,7 +1401,7 @@
desc: "Empty contains unknown",
inputMessage: &emptypb.Empty{},
inputText: `{"unknown": null}`,
- wantErr: true,
+ wantErr: `unknown field "unknown"`,
}, {
desc: "BoolValue false",
inputMessage: &wrapperspb.BoolValue{},
@@ -1411,7 +1416,7 @@
desc: "BoolValue invalid value",
inputMessage: &wrapperspb.BoolValue{},
inputText: `{}`,
- wantErr: true,
+ wantErr: `invalid value for bool type: {`,
}, {
desc: "Int32Value",
inputMessage: &wrapperspb.Int32Value{},
@@ -1445,8 +1450,8 @@
}, {
desc: "FloatValue exceeds max limit",
inputMessage: &wrapperspb.FloatValue{},
- inputText: `1.23+40`,
- wantErr: true,
+ inputText: `1.23e+40`,
+ wantErr: `invalid value for float type: 1.23e+40`,
}, {
desc: "FloatValue Infinity",
inputMessage: &wrapperspb.FloatValue{},
@@ -1476,12 +1481,12 @@
desc: "StringValue with invalid UTF8 error",
inputMessage: &wrapperspb.StringValue{},
inputText: "\"abc\xff\"",
- wantErr: true,
+ wantErr: `invalid UTF-8`,
}, {
desc: "StringValue field with invalid UTF8 error",
inputMessage: &pb2.KnownTypes{},
inputText: "{\n \"optString\": \"abc\xff\"\n}",
- wantErr: true,
+ wantErr: `invalid UTF-8`,
}, {
desc: "NullValue field with JSON null",
inputMessage: &pb2.KnownTypes{},
@@ -1552,7 +1557,7 @@
desc: "Value string with invalid UTF8",
inputMessage: &structpb.Value{},
inputText: "\"\xff\"",
- wantErr: true,
+ wantErr: `invalid UTF-8`,
}, {
desc: "Value field string",
inputMessage: &pb2.KnownTypes{},
@@ -1568,7 +1573,7 @@
inputText: `{
"optValue": "` + "\xff" + `"
}`,
- wantErr: true,
+ wantErr: `invalid UTF-8`,
}, {
desc: "Value empty struct",
inputMessage: &structpb.Value{},
@@ -1619,7 +1624,7 @@
desc: "Value struct with invalid UTF8 string",
inputMessage: &structpb.Value{},
inputText: "{\"string\": \"abc\xff\"}",
- wantErr: true,
+ wantErr: `invalid UTF-8`,
}, {
desc: "Value field struct",
inputMessage: &pb2.KnownTypes{},
@@ -1693,19 +1698,19 @@
desc: "Value list with invalid UTF8 string",
inputMessage: &structpb.Value{},
inputText: "[\"abc\xff\"]",
- wantErr: true,
+ wantErr: `invalid UTF-8`,
}, {
desc: "Value field list with invalid UTF8 string",
inputMessage: &pb2.KnownTypes{},
inputText: `{
"optValue": [ "` + "abc\xff" + `"]
}`,
- wantErr: true,
+ wantErr: `(line 2:17): invalid UTF-8`,
}, {
desc: "Duration empty string",
inputMessage: &durationpb.Duration{},
inputText: `""`,
- wantErr: true,
+ wantErr: `invalid google.protobuf.Duration value ""`,
}, {
desc: "Duration with secs",
inputMessage: &durationpb.Duration{},
@@ -1780,37 +1785,37 @@
desc: "Duration with +secs out of range",
inputMessage: &durationpb.Duration{},
inputText: `"315576000001s"`,
- wantErr: true,
+ wantErr: `google.protobuf.Duration value out of range: "315576000001s"`,
}, {
desc: "Duration with -secs out of range",
inputMessage: &durationpb.Duration{},
inputText: `"-315576000001s"`,
- wantErr: true,
+ wantErr: `google.protobuf.Duration value out of range: "-315576000001s"`,
}, {
desc: "Duration with nanos beyond 9 digits",
inputMessage: &durationpb.Duration{},
inputText: `"0.1000000000s"`,
- wantErr: true,
+ wantErr: `invalid google.protobuf.Duration value "0.1000000000s"`,
}, {
desc: "Duration without suffix s",
inputMessage: &durationpb.Duration{},
inputText: `"123"`,
- wantErr: true,
+ wantErr: `invalid google.protobuf.Duration value "123"`,
}, {
desc: "Duration invalid signed fraction",
inputMessage: &durationpb.Duration{},
inputText: `"123.+123s"`,
- wantErr: true,
+ wantErr: `invalid google.protobuf.Duration value "123.+123s"`,
}, {
desc: "Duration invalid multiple .",
inputMessage: &durationpb.Duration{},
inputText: `"123.123.s"`,
- wantErr: true,
+ wantErr: `invalid google.protobuf.Duration value "123.123.s"`,
}, {
desc: "Duration invalid integer",
inputMessage: &durationpb.Duration{},
inputText: `"01s"`,
- wantErr: true,
+ wantErr: `invalid google.protobuf.Duration value "01s"`,
}, {
desc: "Timestamp zero",
inputMessage: ×tamppb.Timestamp{},
@@ -1845,7 +1850,7 @@
desc: "Timestamp above max value",
inputMessage: ×tamppb.Timestamp{},
inputText: `"9999-12-31T23:59:59-01:00"`,
- wantErr: true,
+ wantErr: `google.protobuf.Timestamp value out of range: "9999-12-31T23:59:59-01:00"`,
}, {
desc: "Timestamp min value",
inputMessage: ×tamppb.Timestamp{},
@@ -1855,12 +1860,12 @@
desc: "Timestamp below min value",
inputMessage: ×tamppb.Timestamp{},
inputText: `"0001-01-01T00:00:00+01:00"`,
- wantErr: true,
+ wantErr: `google.protobuf.Timestamp value out of range: "0001-01-01T00:00:00+01:00"`,
}, {
desc: "Timestamp with nanos beyond 9 digits",
inputMessage: ×tamppb.Timestamp{},
inputText: `"1970-01-01T00:00:00.0000000001Z"`,
- wantErr: true,
+ wantErr: `invalid google.protobuf.Timestamp value`,
}, {
desc: "FieldMask empty",
inputMessage: &fieldmaskpb.FieldMask{},
@@ -1933,7 +1938,7 @@
umo: protojson.UnmarshalOptions{Resolver: new(preg.Types)},
inputMessage: &anypb.Any{},
inputText: `{"@type": "foo/pb2.Nested"}`,
- wantErr: true,
+ wantErr: `(line 1:11): unable to resolve "foo/pb2.Nested":`,
}, {
desc: "Any with missing required",
inputMessage: &anypb.Any{},
@@ -1990,7 +1995,7 @@
"optString": "` + "abc\xff" + `",
"@type": "foo/pb2.Nested"
}`,
- wantErr: true,
+ wantErr: `(line 2:16): invalid UTF-8`,
}, {
desc: "Any with BoolValue",
inputMessage: &anypb.Any{},
@@ -2025,7 +2030,7 @@
inputText: `{
"@type": "type.googleapis.com/google.protobuf.Empty"
}`,
- wantErr: true,
+ wantErr: `(line 3:1): missing "value" field`,
}, {
desc: "Any with StringValue containing invalid UTF8",
inputMessage: &anypb.Any{},
@@ -2033,7 +2038,7 @@
"@type": "google.protobuf.StringValue",
"value": "` + "abc\xff" + `"
}`,
- wantErr: true,
+ wantErr: `(line 3:12): invalid UTF-8`,
}, {
desc: "Any with Int64Value",
inputMessage: &anypb.Any{},
@@ -2059,7 +2064,7 @@
"@type": "google.protobuf.Int64Value",
"value": "forty-two"
}`,
- wantErr: true,
+ wantErr: `(line 3:12): invalid value for int64 type: "forty-two"`,
}, {
desc: "Any with invalid UInt64Value",
inputMessage: &anypb.Any{},
@@ -2067,7 +2072,7 @@
"@type": "google.protobuf.UInt64Value",
"value": -42
}`,
- wantErr: true,
+ wantErr: `(line 3:12): invalid value for uint64 type: -42`,
}, {
desc: "Any with Duration",
inputMessage: &anypb.Any{},
@@ -2093,7 +2098,7 @@
"@type": "google.protobuf.Value",
"value": "` + "abc\xff" + `"
}`,
- wantErr: true,
+ wantErr: `(line 3:12): invalid UTF-8`,
}, {
desc: "Any with Value of NullValue",
inputMessage: &anypb.Any{},
@@ -2159,14 +2164,14 @@
inputText: `{
"value": {}
}`,
- wantErr: true,
+ wantErr: `(line 1:1): missing "@type" field`,
}, {
desc: "Any with empty @type",
inputMessage: &anypb.Any{},
inputText: `{
"@type": ""
}`,
- wantErr: true,
+ wantErr: `(line 2:12): @type field contains empty value`,
}, {
desc: "Any with duplicate @type",
inputMessage: &anypb.Any{},
@@ -2175,7 +2180,7 @@
"value": "hello",
"@type": "pb2.Nested"
}`,
- wantErr: true,
+ wantErr: `(line 4:3): duplicate "@type" field`,
}, {
desc: "Any with duplicate value",
inputMessage: &anypb.Any{},
@@ -2184,7 +2189,7 @@
"value": "hello",
"value": "world"
}`,
- wantErr: true,
+ wantErr: `(line 4:3): duplicate "value" field`,
}, {
desc: "Any with unknown field",
inputMessage: &anypb.Any{},
@@ -2193,7 +2198,7 @@
"optString": "hello",
"unknown": "world"
}`,
- wantErr: true,
+ wantErr: `(line 4:3): unknown field "unknown"`,
}, {
desc: "Any with embedded type containing Any",
inputMessage: &anypb.Any{},
@@ -2201,10 +2206,10 @@
"@type": "pb2.KnownTypes",
"optAny": {
"@type": "google.protobuf.StringValue",
- "value": "` + "abc\xff" + `"
+ "value": "` + "abc\xff" + `"
}
}`,
- wantErr: true,
+ wantErr: `(line 5:14): invalid UTF-8`,
}, {
desc: "well known types as field values",
inputMessage: &pb2.KnownTypes{},
@@ -2396,7 +2401,7 @@
desc: "weak fields; unknown field",
inputMessage: &testpb.TestWeak{},
inputText: `{"weak_message1":{"a":1}, "weak_message2":{"a":1}}`,
- wantErr: true, // weak_message2 is unknown since the package containing it is not imported
+ wantErr: `unknown field "weak_message2"`, // weak_message2 is unknown since the package containing it is not imported
skip: !flags.ProtoLegacy,
}}
@@ -2407,11 +2412,16 @@
}
t.Run(tt.desc, func(t *testing.T) {
err := tt.umo.Unmarshal([]byte(tt.inputText), tt.inputMessage)
- if err != nil && !tt.wantErr {
- t.Errorf("Unmarshal() returned error: %v\n\n", err)
+ if err != nil {
+ if tt.wantErr == "" {
+ t.Errorf("Unmarshal() got unexpected error: %v", err)
+ } else if !strings.Contains(err.Error(), tt.wantErr) {
+ t.Errorf("Unmarshal() error got %q, want %q", err, tt.wantErr)
+ }
+ return
}
- if err == nil && tt.wantErr {
- t.Error("Unmarshal() got nil error, want error\n\n")
+ if tt.wantErr != "" {
+ t.Errorf("Unmarshal() got nil error, want error %q", tt.wantErr)
}
if tt.wantMessage != nil && !proto.Equal(tt.inputMessage, tt.wantMessage) {
t.Errorf("Unmarshal()\n<got>\n%v\n<want>\n%v\n", tt.inputMessage, tt.wantMessage)
diff --git a/encoding/protojson/encode.go b/encoding/protojson/encode.go
index 02723c0..ad29455 100644
--- a/encoding/protojson/encode.go
+++ b/encoding/protojson/encode.go
@@ -39,7 +39,6 @@
// MarshalOptions is a configurable JSON format marshaler.
type MarshalOptions struct {
pragma.NoUnkeyedLiterals
- encoder *json.Encoder
// AllowPartial allows messages that have missing required fields to marshal
// without returning an error. If AllowPartial is false (the default),
@@ -112,31 +111,35 @@
o.Resolver = protoregistry.GlobalTypes
}
- var err error
- o.encoder, err = json.NewEncoder(o.Indent)
+ internalEnc, err := json.NewEncoder(o.Indent)
if err != nil {
return nil, err
}
- err = o.marshalMessage(m.ProtoReflect())
- if err != nil {
+ enc := encoder{internalEnc, o}
+ if err := enc.marshalMessage(m.ProtoReflect()); err != nil {
return nil, err
}
if o.AllowPartial {
- return o.encoder.Bytes(), nil
+ return enc.Bytes(), nil
}
- return o.encoder.Bytes(), proto.IsInitialized(m)
+ return enc.Bytes(), proto.IsInitialized(m)
+}
+
+type encoder struct {
+ *json.Encoder
+ opts MarshalOptions
}
// marshalMessage marshals the given protoreflect.Message.
-func (o MarshalOptions) marshalMessage(m pref.Message) error {
+func (e encoder) marshalMessage(m pref.Message) error {
if isCustomType(m.Descriptor().FullName()) {
- return o.marshalCustomType(m)
+ return e.marshalCustomType(m)
}
- o.encoder.StartObject()
- defer o.encoder.EndObject()
- if err := o.marshalFields(m); err != nil {
+ e.StartObject()
+ defer e.EndObject()
+ if err := e.marshalFields(m); err != nil {
return err
}
@@ -144,7 +147,7 @@
}
// marshalFields marshals the fields in the given protoreflect.Message.
-func (o MarshalOptions) marshalFields(m pref.Message) error {
+func (e encoder) marshalFields(m pref.Message) error {
messageDesc := m.Descriptor()
if !flags.ProtoLegacy && messageset.IsMessageSet(messageDesc) {
return errors.New("no support for proto1 MessageSets")
@@ -166,7 +169,7 @@
val := m.Get(fd)
if !m.Has(fd) {
- if !o.EmitUnpopulated {
+ if !e.opts.EmitUnpopulated {
continue
}
isProto2Scalar := fd.Syntax() == pref.Proto2 && fd.Default().IsValid()
@@ -178,99 +181,93 @@
}
name := fd.JSONName()
- if o.UseProtoNames {
+ if e.opts.UseProtoNames {
name = string(fd.Name())
// Use type name for group field name.
if fd.Kind() == pref.GroupKind {
name = string(fd.Message().Name())
}
}
- if err := o.encoder.WriteName(name); err != nil {
+ if err := e.WriteName(name); err != nil {
return err
}
- if err := o.marshalValue(val, fd); err != nil {
+ if err := e.marshalValue(val, fd); err != nil {
return err
}
}
// Marshal out extensions.
- if err := o.marshalExtensions(m); err != nil {
+ if err := e.marshalExtensions(m); err != nil {
return err
}
return nil
}
// marshalValue marshals the given protoreflect.Value.
-func (o MarshalOptions) marshalValue(val pref.Value, fd pref.FieldDescriptor) error {
+func (e encoder) marshalValue(val pref.Value, fd pref.FieldDescriptor) error {
switch {
case fd.IsList():
- return o.marshalList(val.List(), fd)
+ return e.marshalList(val.List(), fd)
case fd.IsMap():
- return o.marshalMap(val.Map(), fd)
+ return e.marshalMap(val.Map(), fd)
default:
- return o.marshalSingular(val, fd)
+ return e.marshalSingular(val, fd)
}
}
// marshalSingular marshals the given non-repeated field value. This includes
// all scalar types, enums, messages, and groups.
-func (o MarshalOptions) marshalSingular(val pref.Value, fd pref.FieldDescriptor) error {
+func (e encoder) marshalSingular(val pref.Value, fd pref.FieldDescriptor) error {
if !val.IsValid() {
- o.encoder.WriteNull()
+ e.WriteNull()
return nil
}
switch kind := fd.Kind(); kind {
case pref.BoolKind:
- o.encoder.WriteBool(val.Bool())
+ e.WriteBool(val.Bool())
case pref.StringKind:
- if err := o.encoder.WriteString(val.String()); err != nil {
- return err
+ if e.WriteString(val.String()) != nil {
+ return errors.InvalidUTF8(string(fd.FullName()))
}
case pref.Int32Kind, pref.Sint32Kind, pref.Sfixed32Kind:
- o.encoder.WriteInt(val.Int())
+ e.WriteInt(val.Int())
case pref.Uint32Kind, pref.Fixed32Kind:
- o.encoder.WriteUint(val.Uint())
+ e.WriteUint(val.Uint())
case pref.Int64Kind, pref.Sint64Kind, pref.Uint64Kind,
pref.Sfixed64Kind, pref.Fixed64Kind:
// 64-bit integers are written out as JSON string.
- o.encoder.WriteString(val.String())
+ e.WriteString(val.String())
case pref.FloatKind:
// Encoder.WriteFloat handles the special numbers NaN and infinites.
- o.encoder.WriteFloat(val.Float(), 32)
+ e.WriteFloat(val.Float(), 32)
case pref.DoubleKind:
// Encoder.WriteFloat handles the special numbers NaN and infinites.
- o.encoder.WriteFloat(val.Float(), 64)
+ e.WriteFloat(val.Float(), 64)
case pref.BytesKind:
- err := o.encoder.WriteString(base64.StdEncoding.EncodeToString(val.Bytes()))
- if err != nil {
- return err
- }
+ e.WriteString(base64.StdEncoding.EncodeToString(val.Bytes()))
case pref.EnumKind:
if fd.Enum().FullName() == "google.protobuf.NullValue" {
- o.encoder.WriteNull()
+ e.WriteNull()
} else {
desc := fd.Enum().Values().ByNumber(val.Enum())
- if o.UseEnumNumbers || desc == nil {
- o.encoder.WriteInt(int64(val.Enum()))
+ if e.opts.UseEnumNumbers || desc == nil {
+ e.WriteInt(int64(val.Enum()))
} else {
- err := o.encoder.WriteString(string(desc.Name()))
- if err != nil {
- return err
- }
+ e.WriteString(string(desc.Name()))
}
}
case pref.MessageKind, pref.GroupKind:
- if err := o.marshalMessage(val.Message()); err != nil {
+ if err := e.marshalMessage(val.Message()); err != nil {
return err
}
@@ -281,13 +278,13 @@
}
// marshalList marshals the given protoreflect.List.
-func (o MarshalOptions) marshalList(list pref.List, fd pref.FieldDescriptor) error {
- o.encoder.StartArray()
- defer o.encoder.EndArray()
+func (e encoder) marshalList(list pref.List, fd pref.FieldDescriptor) error {
+ e.StartArray()
+ defer e.EndArray()
for i := 0; i < list.Len(); i++ {
item := list.Get(i)
- if err := o.marshalSingular(item, fd); err != nil {
+ if err := e.marshalSingular(item, fd); err != nil {
return err
}
}
@@ -300,9 +297,9 @@
}
// marshalMap marshals given protoreflect.Map.
-func (o MarshalOptions) marshalMap(mmap pref.Map, fd pref.FieldDescriptor) error {
- o.encoder.StartObject()
- defer o.encoder.EndObject()
+func (e encoder) marshalMap(mmap pref.Map, fd pref.FieldDescriptor) error {
+ e.StartObject()
+ defer e.EndObject()
// Get a sorted list based on keyType first.
entries := make([]mapEntry, 0, mmap.Len())
@@ -314,10 +311,10 @@
// Write out sorted list.
for _, entry := range entries {
- if err := o.encoder.WriteName(entry.key.String()); err != nil {
+ if err := e.WriteName(entry.key.String()); err != nil {
return err
}
- if err := o.marshalSingular(entry.value, fd.MapValue()); err != nil {
+ if err := e.marshalSingular(entry.value, fd.MapValue()); err != nil {
return err
}
}
@@ -341,7 +338,7 @@
}
// marshalExtensions marshals extension fields.
-func (o MarshalOptions) marshalExtensions(m pref.Message) error {
+func (e encoder) marshalExtensions(m pref.Message) error {
type entry struct {
key string
value pref.Value
@@ -380,10 +377,10 @@
// JSON field name is the proto field name enclosed in [], similar to
// textproto. This is consistent with Go v1 lib. C++ lib v3.7.0 does not
// marshal out extension fields.
- if err := o.encoder.WriteName("[" + entry.key + "]"); err != nil {
+ if err := e.WriteName("[" + entry.key + "]"); err != nil {
return err
}
- if err := o.marshalValue(entry.value, entry.desc); err != nil {
+ if err := e.marshalValue(entry.value, entry.desc); err != nil {
return err
}
}
diff --git a/encoding/protojson/well_known_types.go b/encoding/protojson/well_known_types.go
index a30b590..e5e9b2e 100644
--- a/encoding/protojson/well_known_types.go
+++ b/encoding/protojson/well_known_types.go
@@ -49,11 +49,11 @@
// marshalCustomType marshals given well-known type message that have special
// JSON conversion rules. It needs to be a message type where isCustomType
// returns true, else it will panic.
-func (o MarshalOptions) marshalCustomType(m pref.Message) error {
+func (e encoder) marshalCustomType(m pref.Message) error {
name := m.Descriptor().FullName()
switch name {
case "google.protobuf.Any":
- return o.marshalAny(m)
+ return e.marshalAny(m)
case "google.protobuf.BoolValue",
"google.protobuf.DoubleValue",
@@ -64,41 +64,41 @@
"google.protobuf.UInt64Value",
"google.protobuf.StringValue",
"google.protobuf.BytesValue":
- return o.marshalWrapperType(m)
+ return e.marshalWrapperType(m)
case "google.protobuf.Empty":
- return o.marshalEmpty(m)
+ return e.marshalEmpty(m)
case "google.protobuf.Struct":
- return o.marshalStruct(m)
+ return e.marshalStruct(m)
case "google.protobuf.ListValue":
- return o.marshalListValue(m)
+ return e.marshalListValue(m)
case "google.protobuf.Value":
- return o.marshalKnownValue(m)
+ return e.marshalKnownValue(m)
case "google.protobuf.Duration":
- return o.marshalDuration(m)
+ return e.marshalDuration(m)
case "google.protobuf.Timestamp":
- return o.marshalTimestamp(m)
+ return e.marshalTimestamp(m)
case "google.protobuf.FieldMask":
- return o.marshalFieldMask(m)
+ return e.marshalFieldMask(m)
}
- panic(fmt.Sprintf("%q does not have a custom marshaler", name))
+ panic(fmt.Sprintf("%s does not have a custom marshaler", name))
}
// unmarshalCustomType unmarshals given well-known type message that have
// special JSON conversion rules. It needs to be a message type where
// isCustomType returns true, else it will panic.
-func (o UnmarshalOptions) unmarshalCustomType(m pref.Message) error {
+func (d decoder) unmarshalCustomType(m pref.Message) error {
name := m.Descriptor().FullName()
switch name {
case "google.protobuf.Any":
- return o.unmarshalAny(m)
+ return d.unmarshalAny(m)
case "google.protobuf.BoolValue",
"google.protobuf.DoubleValue",
@@ -109,31 +109,31 @@
"google.protobuf.UInt64Value",
"google.protobuf.StringValue",
"google.protobuf.BytesValue":
- return o.unmarshalWrapperType(m)
+ return d.unmarshalWrapperType(m)
case "google.protobuf.Empty":
- return o.unmarshalEmpty(m)
+ return d.unmarshalEmpty(m)
case "google.protobuf.Struct":
- return o.unmarshalStruct(m)
+ return d.unmarshalStruct(m)
case "google.protobuf.ListValue":
- return o.unmarshalListValue(m)
+ return d.unmarshalListValue(m)
case "google.protobuf.Value":
- return o.unmarshalKnownValue(m)
+ return d.unmarshalKnownValue(m)
case "google.protobuf.Duration":
- return o.unmarshalDuration(m)
+ return d.unmarshalDuration(m)
case "google.protobuf.Timestamp":
- return o.unmarshalTimestamp(m)
+ return d.unmarshalTimestamp(m)
case "google.protobuf.FieldMask":
- return o.unmarshalFieldMask(m)
+ return d.unmarshalFieldMask(m)
}
- panic(fmt.Sprintf("%q does not have a custom unmarshaler", name))
+ panic(fmt.Sprintf("%s does not have a custom unmarshaler", name))
}
// The JSON representation of an Any message uses the regular representation of
@@ -142,14 +142,14 @@
// custom JSON representation, that representation will be embedded adding a
// field `value` which holds the custom JSON in addition to the `@type` field.
-func (o MarshalOptions) marshalAny(m pref.Message) error {
+func (e encoder) marshalAny(m pref.Message) error {
fds := m.Descriptor().Fields()
fdType := fds.ByNumber(fieldnum.Any_TypeUrl)
fdValue := fds.ByNumber(fieldnum.Any_Value)
// Start writing the JSON object.
- o.encoder.StartObject()
- defer o.encoder.EndObject()
+ e.StartObject()
+ defer e.EndObject()
if !m.Has(fdType) {
if !m.Has(fdValue) {
@@ -166,13 +166,13 @@
// Marshal out @type field.
typeURL := typeVal.String()
- o.encoder.WriteName("@type")
- if err := o.encoder.WriteString(typeURL); err != nil {
+ e.WriteName("@type")
+ if err := e.WriteString(typeURL); err != nil {
return err
}
// Resolve the type in order to unmarshal value field.
- emt, err := o.Resolver.FindMessageByURL(typeURL)
+ emt, err := e.opts.Resolver.FindMessageByURL(typeURL)
if err != nil {
return errors.New("%s: unable to resolve %q: %v", m.Descriptor().FullName(), typeURL, err)
}
@@ -180,7 +180,7 @@
em := emt.New()
err = proto.UnmarshalOptions{
AllowPartial: true, // never check required fields inside an Any
- Resolver: o.Resolver,
+ Resolver: e.opts.Resolver,
}.Unmarshal(valueVal.Bytes(), em.Interface())
if err != nil {
return errors.New("%s: unable to unmarshal %q: %v", m.Descriptor().FullName(), typeURL, err)
@@ -190,71 +190,82 @@
// with corresponding custom JSON encoding of the embedded message as a
// field.
if isCustomType(emt.Descriptor().FullName()) {
- o.encoder.WriteName("value")
- return o.marshalCustomType(em)
+ e.WriteName("value")
+ return e.marshalCustomType(em)
}
// Else, marshal out the embedded message's fields in this Any object.
- if err := o.marshalFields(em); err != nil {
+ if err := e.marshalFields(em); err != nil {
return err
}
return nil
}
-func (o UnmarshalOptions) unmarshalAny(m pref.Message) error {
- // Use Peek to check for json.StartObject to avoid advancing a read.
- if o.decoder.Peek() != json.StartObject {
- jval, _ := o.decoder.Read()
- return unexpectedJSONError{jval}
+func (d decoder) unmarshalAny(m pref.Message) error {
+ // Peek to check for json.ObjectOpen to avoid advancing a read.
+ start, err := d.Peek()
+ if err != nil {
+ return err
+ }
+ if start.Kind() != json.ObjectOpen {
+ return d.unexpectedTokenError(start)
}
- // Use another json.Decoder to parse the unread bytes from o.decoder for
- // @type field. This avoids advancing a read from o.decoder because the
- // current JSON object may contain the fields of the embedded type.
- dec := o.decoder.Clone()
- typeURL, err := findTypeURL(dec)
- if err == errEmptyObject {
+ // Use another decoder to parse the unread bytes for @type field. This
+ // avoids advancing a read from current decoder because the current JSON
+ // object may contain the fields of the embedded type.
+ dec := decoder{d.Clone(), UnmarshalOptions{}}
+ tok, err := findTypeURL(dec)
+ switch err {
+ case errEmptyObject:
// An empty JSON object translates to an empty Any message.
- o.decoder.Read() // Read json.StartObject.
- o.decoder.Read() // Read json.EndObject.
+ d.Read() // Read json.ObjectOpen.
+ d.Read() // Read json.ObjectClose.
return nil
- }
- if o.DiscardUnknown && err == errMissingType {
- // Treat all fields as unknowns, similar to an empty object.
- return skipJSONValue(o.decoder)
- }
- if err != nil {
- return errors.New("google.protobuf.Any: %v", err)
+
+ case errMissingType:
+ if d.opts.DiscardUnknown {
+ // Treat all fields as unknowns, similar to an empty object.
+ return d.skipJSONValue()
+ }
+ // Use start.Pos() for line position.
+ return d.newError(start.Pos(), err.Error())
+
+ default:
+ if err != nil {
+ return err
+ }
}
- emt, err := o.Resolver.FindMessageByURL(typeURL)
+ typeURL := tok.ParsedString()
+ emt, err := d.opts.Resolver.FindMessageByURL(typeURL)
if err != nil {
- return errors.New("google.protobuf.Any: unable to resolve type %q: %v", typeURL, err)
+ return d.newError(tok.Pos(), "unable to resolve %v: %q", tok.RawString(), err)
}
// Create new message for the embedded message type and unmarshal into it.
em := emt.New()
if isCustomType(emt.Descriptor().FullName()) {
- // If embedded message is a custom type, unmarshal the JSON "value" field
- // into it.
- if err := o.unmarshalAnyValue(em); err != nil {
- return errors.New("google.protobuf.Any: %v", err)
+ // If embedded message is a custom type,
+ // unmarshal the JSON "value" field into it.
+ if err := d.unmarshalAnyValue(em); err != nil {
+ return err
}
} else {
// Else unmarshal the current JSON object into it.
- if err := o.unmarshalMessage(em, true); err != nil {
- return errors.New("google.protobuf.Any: %v", err)
+ if err := d.unmarshalMessage(em, true); err != nil {
+ return err
}
}
// Serialize the embedded message and assign the resulting bytes to the
// proto value field.
b, err := proto.MarshalOptions{
- AllowPartial: true, // never check required fields inside an Any
+ AllowPartial: true, // No need to check required fields inside an Any.
Deterministic: true,
}.Marshal(em.Interface())
if err != nil {
- return errors.New("google.protobuf.Any: %v", err)
+ return d.newError(start.Pos(), "error in marshaling Any.value field: %v", err)
}
fds := m.Descriptor().Fields()
@@ -266,113 +277,112 @@
return nil
}
-var errEmptyObject = errors.New(`empty object`)
-var errMissingType = errors.New(`missing "@type" field`)
+var errEmptyObject = fmt.Errorf(`empty object`)
+var errMissingType = fmt.Errorf(`missing "@type" field`)
-// findTypeURL returns the "@type" field value from the given JSON bytes. It is
-// expected that the given bytes start with json.StartObject. It returns
-// errEmptyObject if the JSON object is empty. It returns error if the object
-// does not contain the field or other decoding problems.
-func findTypeURL(dec *json.Decoder) (string, error) {
+// findTypeURL returns the token for the "@type" field value from the given
+// JSON bytes. It is expected that the given bytes start with json.ObjectOpen.
+// It returns errEmptyObject if the JSON object is empty or errMissingType if
+// @type field does not exist. It returns other error if the @type field is not
+// valid or other decoding issues.
+func findTypeURL(d decoder) (json.Token, error) {
var typeURL string
+ var typeTok json.Token
numFields := 0
// Skip start object.
- dec.Read()
+ d.Read()
Loop:
for {
- jval, err := dec.Read()
+ tok, err := d.Read()
if err != nil {
- return "", err
+ return json.Token{}, err
}
- switch jval.Type() {
- case json.EndObject:
+ switch tok.Kind() {
+ case json.ObjectClose:
if typeURL == "" {
// Did not find @type field.
if numFields > 0 {
- return "", errMissingType
+ return json.Token{}, errMissingType
}
- return "", errEmptyObject
+ return json.Token{}, errEmptyObject
}
break Loop
case json.Name:
numFields++
- name, err := jval.Name()
- if err != nil {
- return "", err
- }
- if name != "@type" {
+ if tok.Name() != "@type" {
// Skip value.
- if err := skipJSONValue(dec); err != nil {
- return "", err
+ if err := d.skipJSONValue(); err != nil {
+ return json.Token{}, err
}
continue
}
// Return error if this was previously set already.
if typeURL != "" {
- return "", errors.New(`duplicate "@type" field`)
+ return json.Token{}, d.newError(tok.Pos(), `duplicate "@type" field`)
}
// Read field value.
- jval, err := dec.Read()
+ tok, err := d.Read()
if err != nil {
- return "", err
+ return json.Token{}, err
}
- if jval.Type() != json.String {
- return "", unexpectedJSONError{jval}
+ if tok.Kind() != json.String {
+ return json.Token{}, d.newError(tok.Pos(), `@type field value is not a string: %v`, tok.RawString())
}
- typeURL = jval.String()
+ typeURL = tok.ParsedString()
if typeURL == "" {
- return "", errors.New(`"@type" field contains empty value`)
+ return json.Token{}, d.newError(tok.Pos(), `@type field contains empty value`)
}
+ typeTok = tok
}
}
- return typeURL, nil
+ return typeTok, nil
}
-// skipJSONValue makes the given decoder parse a JSON value (null, boolean,
-// string, number, object and array) in order to advance the read to the next
-// JSON value. It relies on Decoder.Read returning an error if the types are
-// not in valid sequence.
-func skipJSONValue(dec *json.Decoder) error {
- jval, err := dec.Read()
+// skipJSONValue parses a JSON value (null, boolean, string, number, object and
+// array) in order to advance the read to the next JSON value. It relies on
+// the decoder returning an error if the types are not in valid sequence.
+func (d decoder) skipJSONValue() error {
+ tok, err := d.Read()
if err != nil {
return err
}
// Only need to continue reading for objects and arrays.
- switch jval.Type() {
- case json.StartObject:
+ switch tok.Kind() {
+ case json.ObjectOpen:
for {
- jval, err := dec.Read()
+ tok, err := d.Read()
if err != nil {
return err
}
- switch jval.Type() {
- case json.EndObject:
+ switch tok.Kind() {
+ case json.ObjectClose:
return nil
case json.Name:
// Skip object field value.
- if err := skipJSONValue(dec); err != nil {
+ if err := d.skipJSONValue(); err != nil {
return err
}
}
}
- case json.StartArray:
+ case json.ArrayOpen:
for {
- switch dec.Peek() {
- case json.EndArray:
- dec.Read()
- return nil
- case json.Invalid:
- _, err := dec.Read()
+ tok, err := d.Peek()
+ if err != nil {
return err
+ }
+ switch tok.Kind() {
+ case json.ArrayClose:
+ d.Read()
+ return nil
default:
// Skip array item.
- if err := skipJSONValue(dec); err != nil {
+ if err := d.skipJSONValue(); err != nil {
return err
}
}
@@ -383,51 +393,47 @@
// unmarshalAnyValue unmarshals the given custom-type message from the JSON
// object's "value" field.
-func (o UnmarshalOptions) unmarshalAnyValue(m pref.Message) error {
- // Skip StartObject, and start reading the fields.
- o.decoder.Read()
+func (d decoder) unmarshalAnyValue(m pref.Message) error {
+ // Skip ObjectOpen, and start reading the fields.
+ d.Read()
var found bool // Used for detecting duplicate "value".
for {
- jval, err := o.decoder.Read()
+ tok, err := d.Read()
if err != nil {
return err
}
- switch jval.Type() {
- case json.EndObject:
+ switch tok.Kind() {
+ case json.ObjectClose:
if !found {
- return errors.New(`missing "value" field`)
+ return d.newError(tok.Pos(), `missing "value" field`)
}
return nil
case json.Name:
- name, err := jval.Name()
- if err != nil {
- return err
- }
- switch name {
+ switch tok.Name() {
+ case "@type":
+ // Skip the value as this was previously parsed already.
+ d.Read()
+
+ case "value":
+ if found {
+ return d.newError(tok.Pos(), `duplicate "value" field`)
+ }
+ // Unmarshal the field value into the given message.
+ if err := d.unmarshalCustomType(m); err != nil {
+ return err
+ }
+ found = true
+
default:
- if o.DiscardUnknown {
- if err := skipJSONValue(o.decoder); err != nil {
+ if d.opts.DiscardUnknown {
+ if err := d.skipJSONValue(); err != nil {
return err
}
continue
}
- return errors.New("unknown field %q", name)
-
- case "@type":
- // Skip the value as this was previously parsed already.
- o.decoder.Read()
-
- case "value":
- if found {
- return errors.New(`duplicate "value" field`)
- }
- // Unmarshal the field value into the given message.
- if err := o.unmarshalCustomType(m); err != nil {
- return err
- }
- found = true
+ return d.newError(tok.Pos(), "unknown field %v", tok.RawString())
}
}
}
@@ -438,15 +444,15 @@
// The "value" field has the same field number for all wrapper types.
const wrapperFieldNumber = fieldnum.BoolValue_Value
-func (o MarshalOptions) marshalWrapperType(m pref.Message) error {
+func (e encoder) marshalWrapperType(m pref.Message) error {
fd := m.Descriptor().Fields().ByNumber(wrapperFieldNumber)
val := m.Get(fd)
- return o.marshalSingular(val, fd)
+ return e.marshalSingular(val, fd)
}
-func (o UnmarshalOptions) unmarshalWrapperType(m pref.Message) error {
+func (d decoder) unmarshalWrapperType(m pref.Message) error {
fd := m.Descriptor().Fields().ByNumber(wrapperFieldNumber)
- val, err := o.unmarshalScalar(fd)
+ val, err := d.unmarshalScalar(fd)
if err != nil {
return err
}
@@ -456,42 +462,41 @@
// The JSON representation for Empty is an empty JSON object.
-func (o MarshalOptions) marshalEmpty(pref.Message) error {
- o.encoder.StartObject()
- o.encoder.EndObject()
+func (e encoder) marshalEmpty(pref.Message) error {
+ e.StartObject()
+ e.EndObject()
return nil
}
-func (o UnmarshalOptions) unmarshalEmpty(pref.Message) error {
- jval, err := o.decoder.Read()
+func (d decoder) unmarshalEmpty(pref.Message) error {
+ tok, err := d.Read()
if err != nil {
return err
}
- if jval.Type() != json.StartObject {
- return unexpectedJSONError{jval}
+ if tok.Kind() != json.ObjectOpen {
+ return d.unexpectedTokenError(tok)
}
for {
- jval, err := o.decoder.Read()
+ tok, err := d.Read()
if err != nil {
return err
}
- switch jval.Type() {
- case json.EndObject:
+ switch tok.Kind() {
+ case json.ObjectClose:
return nil
case json.Name:
- if o.DiscardUnknown {
- if err := skipJSONValue(o.decoder); err != nil {
+ if d.opts.DiscardUnknown {
+ if err := d.skipJSONValue(); err != nil {
return err
}
continue
}
- name, _ := jval.Name()
- return errors.New("unknown field %q", name)
+ return d.newError(tok.Pos(), "unknown field %v", tok.RawString())
default:
- return unexpectedJSONError{jval}
+ return d.unexpectedTokenError(tok)
}
}
}
@@ -499,73 +504,76 @@
// The JSON representation for Struct is a JSON object that contains the encoded
// Struct.fields map and follows the serialization rules for a map.
-func (o MarshalOptions) marshalStruct(m pref.Message) error {
+func (e encoder) marshalStruct(m pref.Message) error {
fd := m.Descriptor().Fields().ByNumber(fieldnum.Struct_Fields)
- return o.marshalMap(m.Get(fd).Map(), fd)
+ return e.marshalMap(m.Get(fd).Map(), fd)
}
-func (o UnmarshalOptions) unmarshalStruct(m pref.Message) error {
+func (d decoder) unmarshalStruct(m pref.Message) error {
fd := m.Descriptor().Fields().ByNumber(fieldnum.Struct_Fields)
- return o.unmarshalMap(m.Mutable(fd).Map(), fd)
+ return d.unmarshalMap(m.Mutable(fd).Map(), fd)
}
// The JSON representation for ListValue is JSON array that contains the encoded
// ListValue.values repeated field and follows the serialization rules for a
// repeated field.
-func (o MarshalOptions) marshalListValue(m pref.Message) error {
+func (e encoder) marshalListValue(m pref.Message) error {
fd := m.Descriptor().Fields().ByNumber(fieldnum.ListValue_Values)
- return o.marshalList(m.Get(fd).List(), fd)
+ return e.marshalList(m.Get(fd).List(), fd)
}
-func (o UnmarshalOptions) unmarshalListValue(m pref.Message) error {
+func (d decoder) unmarshalListValue(m pref.Message) error {
fd := m.Descriptor().Fields().ByNumber(fieldnum.ListValue_Values)
- return o.unmarshalList(m.Mutable(fd).List(), fd)
+ return d.unmarshalList(m.Mutable(fd).List(), fd)
}
// The JSON representation for a Value is dependent on the oneof field that is
// set. Each of the field in the oneof has its own custom serialization rule. A
// Value message needs to be a oneof field set, else it is an error.
-func (o MarshalOptions) marshalKnownValue(m pref.Message) error {
+func (e encoder) marshalKnownValue(m pref.Message) error {
od := m.Descriptor().Oneofs().ByName("kind")
fd := m.WhichOneof(od)
if fd == nil {
return errors.New("%s: none of the oneof fields is set", m.Descriptor().FullName())
}
- return o.marshalSingular(m.Get(fd), fd)
+ return e.marshalSingular(m.Get(fd), fd)
}
-func (o UnmarshalOptions) unmarshalKnownValue(m pref.Message) error {
- switch o.decoder.Peek() {
+func (d decoder) unmarshalKnownValue(m pref.Message) error {
+ tok, err := d.Peek()
+ if err != nil {
+ return err
+ }
+
+ var fd pref.FieldDescriptor
+ var val pref.Value
+ switch tok.Kind() {
case json.Null:
- o.decoder.Read()
- fd := m.Descriptor().Fields().ByNumber(fieldnum.Value_NullValue)
- m.Set(fd, pref.ValueOfEnum(0))
+ d.Read()
+ fd = m.Descriptor().Fields().ByNumber(fieldnum.Value_NullValue)
+ val = pref.ValueOfEnum(0)
case json.Bool:
- jval, err := o.decoder.Read()
+ tok, err := d.Read()
if err != nil {
return err
}
- val, err := unmarshalBool(jval)
- if err != nil {
- return err
- }
- fd := m.Descriptor().Fields().ByNumber(fieldnum.Value_BoolValue)
- m.Set(fd, val)
+ fd = m.Descriptor().Fields().ByNumber(fieldnum.Value_BoolValue)
+ val = pref.ValueOfBool(tok.Bool())
case json.Number:
- jval, err := o.decoder.Read()
+ tok, err := d.Read()
if err != nil {
return err
}
- val, err := unmarshalFloat(jval, 64)
- if err != nil {
- return err
+ fd = m.Descriptor().Fields().ByNumber(fieldnum.Value_NumberValue)
+ var ok bool
+ val, ok = unmarshalFloat(tok, 64)
+ if !ok {
+ return d.newError(tok.Pos(), "invalid google.protobuf.Value: %v", tok.RawString())
}
- fd := m.Descriptor().Fields().ByNumber(fieldnum.Value_NumberValue)
- m.Set(fd, val)
case json.String:
// A JSON string may have been encoded from the number_value field,
@@ -574,40 +582,32 @@
// however, there is no way to identify that and hence a JSON string is
// always assigned to the string_value field, which means that certain
// encoding cannot be parsed back to the same field.
- jval, err := o.decoder.Read()
+ tok, err := d.Read()
if err != nil {
return err
}
- val, err := unmarshalString(jval)
- if err != nil {
- return err
- }
- fd := m.Descriptor().Fields().ByNumber(fieldnum.Value_StringValue)
- m.Set(fd, val)
+ fd = m.Descriptor().Fields().ByNumber(fieldnum.Value_StringValue)
+ val = pref.ValueOfString(tok.ParsedString())
- case json.StartObject:
- fd := m.Descriptor().Fields().ByNumber(fieldnum.Value_StructValue)
- val := m.NewField(fd)
- if err := o.unmarshalStruct(val.Message()); err != nil {
+ case json.ObjectOpen:
+ fd = m.Descriptor().Fields().ByNumber(fieldnum.Value_StructValue)
+ val = m.NewField(fd)
+ if err := d.unmarshalStruct(val.Message()); err != nil {
return err
}
- m.Set(fd, val)
- case json.StartArray:
- fd := m.Descriptor().Fields().ByNumber(fieldnum.Value_ListValue)
- val := m.NewField(fd)
- if err := o.unmarshalListValue(val.Message()); err != nil {
+ case json.ArrayOpen:
+ fd = m.Descriptor().Fields().ByNumber(fieldnum.Value_ListValue)
+ val = m.NewField(fd)
+ if err := d.unmarshalListValue(val.Message()); err != nil {
return err
}
- m.Set(fd, val)
default:
- jval, err := o.decoder.Read()
- if err != nil {
- return err
- }
- return unexpectedJSONError{jval}
+ return d.newError(tok.Pos(), "invalid google.protobuf.Value: %v", tok.RawString())
}
+
+ m.Set(fd, val)
return nil
}
@@ -628,7 +628,7 @@
maxSecondsInDuration = 315576000000
)
-func (o MarshalOptions) marshalDuration(m pref.Message) error {
+func (e encoder) marshalDuration(m pref.Message) error {
fds := m.Descriptor().Fields()
fdSeconds := fds.ByNumber(fieldnum.Duration_Seconds)
fdNanos := fds.ByNumber(fieldnum.Duration_Nanos)
@@ -659,28 +659,27 @@
x = strings.TrimSuffix(x, "000")
x = strings.TrimSuffix(x, "000")
x = strings.TrimSuffix(x, ".000")
- o.encoder.WriteString(x + "s")
+ e.WriteString(x + "s")
return nil
}
-func (o UnmarshalOptions) unmarshalDuration(m pref.Message) error {
- jval, err := o.decoder.Read()
+func (d decoder) unmarshalDuration(m pref.Message) error {
+ tok, err := d.Read()
if err != nil {
return err
}
- if jval.Type() != json.String {
- return unexpectedJSONError{jval}
+ if tok.Kind() != json.String {
+ return d.unexpectedTokenError(tok)
}
- input := jval.String()
- secs, nanos, ok := parseDuration(input)
+ secs, nanos, ok := parseDuration(tok.ParsedString())
if !ok {
- return errors.New("%s: invalid duration value %q", m.Descriptor().FullName(), input)
+ return d.newError(tok.Pos(), "invalid google.protobuf.Duration value %v", tok.RawString())
}
// Validate seconds. No need to validate nanos because parseDuration would
// have covered that already.
if secs < -maxSecondsInDuration || secs > maxSecondsInDuration {
- return errors.New("%s: out of range %q", m.Descriptor().FullName(), input)
+ return d.newError(tok.Pos(), "google.protobuf.Duration value out of range: %v", tok.RawString())
}
fds := m.Descriptor().Fields()
@@ -820,7 +819,7 @@
minTimestampSeconds = -62135596800
)
-func (o MarshalOptions) marshalTimestamp(m pref.Message) error {
+func (e encoder) marshalTimestamp(m pref.Message) error {
fds := m.Descriptor().Fields()
fdSeconds := fds.ByNumber(fieldnum.Timestamp_Seconds)
fdNanos := fds.ByNumber(fieldnum.Timestamp_Nanos)
@@ -842,29 +841,28 @@
x = strings.TrimSuffix(x, "000")
x = strings.TrimSuffix(x, "000")
x = strings.TrimSuffix(x, ".000")
- o.encoder.WriteString(x + "Z")
+ e.WriteString(x + "Z")
return nil
}
-func (o UnmarshalOptions) unmarshalTimestamp(m pref.Message) error {
- jval, err := o.decoder.Read()
+func (d decoder) unmarshalTimestamp(m pref.Message) error {
+ tok, err := d.Read()
if err != nil {
return err
}
- if jval.Type() != json.String {
- return unexpectedJSONError{jval}
+ if tok.Kind() != json.String {
+ return d.unexpectedTokenError(tok)
}
- input := jval.String()
- t, err := time.Parse(time.RFC3339Nano, input)
+ t, err := time.Parse(time.RFC3339Nano, tok.ParsedString())
if err != nil {
- return errors.New("%s: invalid timestamp value %q", m.Descriptor().FullName(), input)
+ return d.newError(tok.Pos(), "invalid google.protobuf.Timestamp value %v", tok.RawString())
}
// Validate seconds. No need to validate nanos because time.Parse would have
// covered that already.
secs := t.Unix()
if secs < minTimestampSeconds || secs > maxTimestampSeconds {
- return errors.New("%s: out of range %q", m.Descriptor().FullName(), input)
+ return d.newError(tok.Pos(), "google.protobuf.Timestamp value out of range: %v", tok.RawString())
}
fds := m.Descriptor().Fields()
@@ -881,7 +879,7 @@
// lower-camel naming conventions. Encoding should fail if the path name would
// end up differently after a round-trip.
-func (o MarshalOptions) marshalFieldMask(m pref.Message) error {
+func (e encoder) marshalFieldMask(m pref.Message) error {
fd := m.Descriptor().Fields().ByNumber(fieldnum.FieldMask_Paths)
list := m.Get(fd).List()
paths := make([]string, 0, list.Len())
@@ -896,19 +894,19 @@
paths = append(paths, cc)
}
- o.encoder.WriteString(strings.Join(paths, ","))
+ e.WriteString(strings.Join(paths, ","))
return nil
}
-func (o UnmarshalOptions) unmarshalFieldMask(m pref.Message) error {
- jval, err := o.decoder.Read()
+func (d decoder) unmarshalFieldMask(m pref.Message) error {
+ tok, err := d.Read()
if err != nil {
return err
}
- if jval.Type() != json.String {
- return unexpectedJSONError{jval}
+ if tok.Kind() != json.String {
+ return d.unexpectedTokenError(tok)
}
- str := strings.TrimSpace(jval.String())
+ str := strings.TrimSpace(tok.ParsedString())
if str == "" {
return nil
}
diff --git a/internal/encoding/json/bench_test.go b/internal/encoding/json/bench_test.go
index db7d761..bee8545 100644
--- a/internal/encoding/json/bench_test.go
+++ b/internal/encoding/json/bench_test.go
@@ -18,9 +18,8 @@
if err != nil {
b.Fatal(err)
}
- _, err = val.Float(64)
- if err != nil {
- b.Fatal(err)
+ if _, ok := val.Float(64); !ok {
+ b.Fatal("not a flaot")
}
}
}
@@ -33,9 +32,8 @@
if err != nil {
b.Fatal(err)
}
- _, err = val.Int(64)
- if err != nil {
- b.Fatal(err)
+ if _, ok := val.Int(64); !ok {
+ b.Fatal("not an int64")
}
}
}
@@ -48,7 +46,7 @@
if err != nil {
b.Fatal(err)
}
- _ = val.String()
+ _ = val.ParsedString()
}
}
@@ -60,9 +58,6 @@
if err != nil {
b.Fatal(err)
}
- _, err = val.Bool()
- if err != nil {
- b.Fatal(err)
- }
+ _ = val.Bool()
}
}
diff --git a/internal/encoding/json/decode.go b/internal/encoding/json/decode.go
index 3e58d06..b13fd29 100644
--- a/internal/encoding/json/decode.go
+++ b/internal/encoding/json/decode.go
@@ -9,7 +9,6 @@
"fmt"
"io"
"regexp"
- "strconv"
"unicode/utf8"
"google.golang.org/protobuf/internal/errors"
@@ -23,22 +22,27 @@
peekCall
)
+const unexpectedFmt = "unexpected token %s"
+
+// ErrUnexpectedEOF means that EOF was encountered in the middle of the input.
+var ErrUnexpectedEOF = errors.New("%v", io.ErrUnexpectedEOF)
+
// Decoder is a token-based JSON decoder.
type Decoder struct {
// lastCall is last method called, either readCall or peekCall.
// Initial value is readCall.
lastCall call
- // value contains the last read value.
- value Value
+ // lastToken contains the last read token.
+ lastToken Token
- // err contains the last read error.
- err error
+ // lastErr contains the last read error.
+ lastErr error
- // startStack is a stack containing StartObject and StartArray types. The
+ // openStack is a stack containing ObjectOpen and ArrayOpen values. The
// top of stack represents the object or the array the current value is
// directly located in.
- startStack []Type
+ openStack []Kind
// orig is used in reporting line and column.
orig []byte
@@ -51,193 +55,188 @@
return &Decoder{orig: b, in: b}
}
-// Peek looks ahead and returns the next JSON type without advancing a read.
-func (d *Decoder) Peek() Type {
+// Peek looks ahead and returns the next token kind without advancing a read.
+func (d *Decoder) Peek() (Token, error) {
defer func() { d.lastCall = peekCall }()
if d.lastCall == readCall {
- d.value, d.err = d.Read()
+ d.lastToken, d.lastErr = d.Read()
}
- return d.value.typ
+ return d.lastToken, d.lastErr
}
-// Read returns the next JSON value. It will return an error if there is no
-// valid value. For String types containing invalid UTF8 characters, a non-fatal
-// error is returned and caller can call Read for the next value.
-func (d *Decoder) Read() (Value, error) {
+// Read returns the next JSON token.
+// It will return an error if there is no valid token.
+func (d *Decoder) Read() (Token, error) {
+ const scalar = Null | Bool | Number | String
+
defer func() { d.lastCall = readCall }()
if d.lastCall == peekCall {
- return d.value, d.err
+ return d.lastToken, d.lastErr
}
- value, err := d.parseNext()
+ tok, err := d.parseNext()
if err != nil {
- return Value{}, err
+ return Token{}, err
}
- n := value.size
- switch value.typ {
+ switch tok.kind {
case EOF:
- if len(d.startStack) != 0 ||
- d.value.typ&Null|Bool|Number|String|EndObject|EndArray == 0 {
- return Value{}, io.ErrUnexpectedEOF
+ if len(d.openStack) != 0 ||
+ d.lastToken.kind&scalar|ObjectClose|ArrayClose == 0 {
+ return Token{}, ErrUnexpectedEOF
}
case Null:
if !d.isValueNext() {
- return Value{}, d.newSyntaxError("unexpected value null")
+ return Token{}, d.newSyntaxError(tok.pos, unexpectedFmt, tok.RawString())
}
case Bool, Number:
if !d.isValueNext() {
- return Value{}, d.newSyntaxError("unexpected value %v", value.Raw())
+ return Token{}, d.newSyntaxError(tok.pos, unexpectedFmt, tok.RawString())
}
case String:
if d.isValueNext() {
break
}
- // Check if this is for an object name.
- if d.value.typ&(StartObject|comma) == 0 {
- return Value{}, d.newSyntaxError("unexpected value %v", value.Raw())
+ // This string token should only be for a field name.
+ if d.lastToken.kind&(ObjectOpen|comma) == 0 {
+ return Token{}, d.newSyntaxError(tok.pos, unexpectedFmt, tok.RawString())
}
- d.in = d.in[n:]
- d.consume(0)
if len(d.in) == 0 {
- return Value{}, d.newSyntaxError(`unexpected EOF, missing ":" after object name`)
+ return Token{}, ErrUnexpectedEOF
}
if c := d.in[0]; c != ':' {
- return Value{}, d.newSyntaxError(`unexpected character %v, missing ":" after object name`, string(c))
+ return Token{}, d.newSyntaxError(d.currPos(), `unexpected character %s, missing ":" after field name`, string(c))
}
- n = 1
- value.typ = Name
+ tok.kind = Name
+ d.consume(1)
- case StartObject, StartArray:
+ case ObjectOpen, ArrayOpen:
if !d.isValueNext() {
- return Value{}, d.newSyntaxError("unexpected character %v", value.Raw())
+ return Token{}, d.newSyntaxError(tok.pos, unexpectedFmt, tok.RawString())
}
- d.startStack = append(d.startStack, value.typ)
+ d.openStack = append(d.openStack, tok.kind)
- case EndObject:
- if len(d.startStack) == 0 ||
- d.value.typ == comma ||
- d.startStack[len(d.startStack)-1] != StartObject {
- return Value{}, d.newSyntaxError("unexpected character }")
+ case ObjectClose:
+ if len(d.openStack) == 0 ||
+ d.lastToken.kind == comma ||
+ d.openStack[len(d.openStack)-1] != ObjectOpen {
+ return Token{}, d.newSyntaxError(tok.pos, unexpectedFmt, tok.RawString())
}
- d.startStack = d.startStack[:len(d.startStack)-1]
+ d.openStack = d.openStack[:len(d.openStack)-1]
- case EndArray:
- if len(d.startStack) == 0 ||
- d.value.typ == comma ||
- d.startStack[len(d.startStack)-1] != StartArray {
- return Value{}, d.newSyntaxError("unexpected character ]")
+ case ArrayClose:
+ if len(d.openStack) == 0 ||
+ d.lastToken.kind == comma ||
+ d.openStack[len(d.openStack)-1] != ArrayOpen {
+ return Token{}, d.newSyntaxError(tok.pos, unexpectedFmt, tok.RawString())
}
- d.startStack = d.startStack[:len(d.startStack)-1]
+ d.openStack = d.openStack[:len(d.openStack)-1]
case comma:
- if len(d.startStack) == 0 ||
- d.value.typ&(Null|Bool|Number|String|EndObject|EndArray) == 0 {
- return Value{}, d.newSyntaxError("unexpected character ,")
+ if len(d.openStack) == 0 ||
+ d.lastToken.kind&(scalar|ObjectClose|ArrayClose) == 0 {
+ return Token{}, d.newSyntaxError(tok.pos, unexpectedFmt, tok.RawString())
}
}
- // Update d.value only after validating value to be in the right sequence.
- d.value = value
- d.in = d.in[n:]
+ // Update d.lastToken only after validating token to be in the right sequence.
+ d.lastToken = tok
- if d.value.typ == comma {
+ if d.lastToken.kind == comma {
return d.Read()
}
- return value, nil
+ return tok, nil
}
// Any sequence that looks like a non-delimiter (for error reporting).
var errRegexp = regexp.MustCompile(`^([-+._a-zA-Z0-9]{1,32}|.)`)
-// parseNext parses for the next JSON value. It returns a Value object for
-// different types, except for Name. It does not handle whether the next value
+// parseNext parses for the next JSON token. It returns a Token object for
+// different types, except for Name. It does not handle whether the next token
// is in a valid sequence or not.
-func (d *Decoder) parseNext() (value Value, err error) {
+func (d *Decoder) parseNext() (Token, error) {
// Trim leading spaces.
d.consume(0)
in := d.in
if len(in) == 0 {
- return d.newValue(EOF, nil, 0), nil
+ return d.consumeToken(EOF, 0), nil
}
switch in[0] {
case 'n':
- n := matchWithDelim("null", in)
- if n == 0 {
- return Value{}, d.newSyntaxError("invalid value %s", errRegexp.Find(in))
+ if n := matchWithDelim("null", in); n != 0 {
+ return d.consumeToken(Null, n), nil
}
- return d.newValue(Null, in, n), nil
case 't':
- n := matchWithDelim("true", in)
- if n == 0 {
- return Value{}, d.newSyntaxError("invalid value %s", errRegexp.Find(in))
+ if n := matchWithDelim("true", in); n != 0 {
+ return d.consumeBoolToken(true, n), nil
}
- return d.newBoolValue(in, n, true), nil
case 'f':
- n := matchWithDelim("false", in)
- if n == 0 {
- return Value{}, d.newSyntaxError("invalid value %s", errRegexp.Find(in))
+ if n := matchWithDelim("false", in); n != 0 {
+ return d.consumeBoolToken(false, n), nil
}
- return d.newBoolValue(in, n, false), nil
case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
- n, ok := consumeNumber(in)
- if !ok {
- return Value{}, d.newSyntaxError("invalid number %s", errRegexp.Find(in))
+ if n, ok := parseNumber(in); ok {
+ return d.consumeToken(Number, n), nil
}
- return d.newValue(Number, in, n), nil
case '"':
s, n, err := d.parseString(in)
if err != nil {
- return Value{}, err
+ return Token{}, err
}
- return d.newStringValue(in, n, s), nil
+ return d.consumeStringToken(s, n), nil
case '{':
- return d.newValue(StartObject, in, 1), nil
+ return d.consumeToken(ObjectOpen, 1), nil
case '}':
- return d.newValue(EndObject, in, 1), nil
+ return d.consumeToken(ObjectClose, 1), nil
case '[':
- return d.newValue(StartArray, in, 1), nil
+ return d.consumeToken(ArrayOpen, 1), nil
case ']':
- return d.newValue(EndArray, in, 1), nil
+ return d.consumeToken(ArrayClose, 1), nil
case ',':
- return d.newValue(comma, in, 1), nil
+ return d.consumeToken(comma, 1), nil
}
- return Value{}, d.newSyntaxError("invalid value %s", errRegexp.Find(in))
-}
-
-// position returns line and column number of index in given orig slice.
-func position(orig []byte, idx int) (int, int) {
- b := orig[:idx]
- line := bytes.Count(b, []byte("\n")) + 1
- if i := bytes.LastIndexByte(b, '\n'); i >= 0 {
- b = b[i+1:]
- }
- column := utf8.RuneCount(b) + 1 // ignore multi-rune characters
- return line, column
+ return Token{}, d.newSyntaxError(d.currPos(), "invalid value %s", errRegexp.Find(in))
}
// newSyntaxError returns an error with line and column information useful for
// syntax errors.
-func (d *Decoder) newSyntaxError(f string, x ...interface{}) error {
+func (d *Decoder) newSyntaxError(pos int, f string, x ...interface{}) error {
e := errors.New(f, x...)
- line, column := position(d.orig, len(d.orig)-len(d.in))
+ line, column := d.Position(pos)
return errors.New("syntax error (line %d:%d): %v", line, column, e)
}
+// Position returns line and column number of given index of the original input.
+// It will panic if index is out of range.
+func (d *Decoder) Position(idx int) (line int, column int) {
+ b := d.orig[:idx]
+ line = bytes.Count(b, []byte("\n")) + 1
+ if i := bytes.LastIndexByte(b, '\n'); i >= 0 {
+ b = b[i+1:]
+ }
+ column = utf8.RuneCount(b) + 1 // ignore multi-rune characters
+ return line, column
+}
+
+// currPos returns the current index position of d.in from d.orig.
+func (d *Decoder) currPos() int {
+ return len(d.orig) - len(d.in)
+}
+
// matchWithDelim matches s with the input b and verifies that the match
// terminates with a delimiter of some form (e.g., r"[^-+_.a-zA-Z0-9]").
// As a special case, EOF is considered a delimiter. It returns the length of s
@@ -278,193 +277,64 @@
// isValueNext returns true if next type should be a JSON value: Null,
// Number, String or Bool.
func (d *Decoder) isValueNext() bool {
- if len(d.startStack) == 0 {
- return d.value.typ == 0
+ if len(d.openStack) == 0 {
+ return d.lastToken.kind == 0
}
- start := d.startStack[len(d.startStack)-1]
+ start := d.openStack[len(d.openStack)-1]
switch start {
- case StartObject:
- return d.value.typ&Name != 0
- case StartArray:
- return d.value.typ&(StartArray|comma) != 0
+ case ObjectOpen:
+ return d.lastToken.kind&Name != 0
+ case ArrayOpen:
+ return d.lastToken.kind&(ArrayOpen|comma) != 0
}
panic(fmt.Sprintf(
- "unreachable logic in Decoder.isValueNext, lastType: %v, startStack: %v",
- d.value.typ, start))
+ "unreachable logic in Decoder.isValueNext, lastToken.kind: %v, openStack: %v",
+ d.lastToken.kind, start))
}
-// newValue constructs a Value for given Type.
-func (d *Decoder) newValue(typ Type, input []byte, size int) Value {
- return Value{
- typ: typ,
- input: d.orig,
- start: len(d.orig) - len(input),
- size: size,
+// consumeToken constructs a Token for given Kind with raw value derived from
+// current d.in and given size, and consumes the given size-lenght of it.
+func (d *Decoder) consumeToken(kind Kind, size int) Token {
+ tok := Token{
+ kind: kind,
+ raw: d.in[:size],
+ pos: len(d.orig) - len(d.in),
}
+ d.consume(size)
+ return tok
}
-// newBoolValue constructs a Value for a JSON boolean.
-func (d *Decoder) newBoolValue(input []byte, size int, b bool) Value {
- return Value{
- typ: Bool,
- input: d.orig,
- start: len(d.orig) - len(input),
- size: size,
- boo: b,
+// consumeBoolToken constructs a Token for a Bool kind with raw value derived from
+// current d.in and given size.
+func (d *Decoder) consumeBoolToken(b bool, size int) Token {
+ tok := Token{
+ kind: Bool,
+ raw: d.in[:size],
+ pos: len(d.orig) - len(d.in),
+ boo: b,
}
+ d.consume(size)
+ return tok
}
-// newStringValue constructs a Value for a JSON string.
-func (d *Decoder) newStringValue(input []byte, size int, s string) Value {
- return Value{
- typ: String,
- input: d.orig,
- start: len(d.orig) - len(input),
- size: size,
- str: s,
+// consumeStringToken constructs a Token for a String kind with raw value derived
+// from current d.in and given size.
+func (d *Decoder) consumeStringToken(s string, size int) Token {
+ tok := Token{
+ kind: String,
+ raw: d.in[:size],
+ pos: len(d.orig) - len(d.in),
+ str: s,
}
+ d.consume(size)
+ return tok
}
// Clone returns a copy of the Decoder for use in reading ahead the next JSON
// object, array or other values without affecting current Decoder.
func (d *Decoder) Clone() *Decoder {
ret := *d
- ret.startStack = append([]Type(nil), ret.startStack...)
+ ret.openStack = append([]Kind(nil), ret.openStack...)
return &ret
}
-
-// Value provides a parsed JSON type and value.
-//
-// The original input slice is stored in this struct in order to compute for
-// position as needed. The raw JSON value is derived from the original input
-// slice given start and size.
-//
-// For JSON boolean and string, it holds the converted value in boo and str
-// fields respectively. For JSON number, the raw JSON value holds a valid number
-// which is converted only in Int or Float. Other JSON types do not require any
-// additional data.
-type Value struct {
- typ Type
- input []byte
- start int
- size int
- boo bool
- str string
-}
-
-func (v Value) newError(f string, x ...interface{}) error {
- e := errors.New(f, x...)
- line, col := v.Position()
- return errors.New("error (line %d:%d): %v", line, col, e)
-}
-
-// Type returns the JSON type.
-func (v Value) Type() Type {
- return v.typ
-}
-
-// Position returns the line and column of the value.
-func (v Value) Position() (int, int) {
- return position(v.input, v.start)
-}
-
-// Bool returns the bool value if token is Bool, else it will return an error.
-func (v Value) Bool() (bool, error) {
- if v.typ != Bool {
- return false, v.newError("%s is not a bool", v.Raw())
- }
- return v.boo, nil
-}
-
-// String returns the string value for a JSON string token or the read value in
-// string if token is not a string.
-func (v Value) String() string {
- if v.typ != String {
- return v.Raw()
- }
- return v.str
-}
-
-// Name returns the object name if token is Name, else it will return an error.
-func (v Value) Name() (string, error) {
- if v.typ != Name {
- return "", v.newError("%s is not an object name", v.Raw())
- }
- return v.str, nil
-}
-
-// Raw returns the read value in string.
-func (v Value) Raw() string {
- return string(v.input[v.start : v.start+v.size])
-}
-
-// Float returns the floating-point number if token is Number, else it will
-// return an error.
-//
-// The floating-point precision is specified by the bitSize parameter: 32 for
-// float32 or 64 for float64. If bitSize=32, the result still has type float64,
-// but it will be convertible to float32 without changing its value. It will
-// return an error if the number exceeds the floating point limits for given
-// bitSize.
-func (v Value) Float(bitSize int) (float64, error) {
- if v.typ != Number {
- return 0, v.newError("%s is not a number", v.Raw())
- }
- f, err := strconv.ParseFloat(v.Raw(), bitSize)
- if err != nil {
- return 0, v.newError("%v", err)
- }
- return f, nil
-}
-
-// Int returns the signed integer number if token is Number, else it will
-// return an error.
-//
-// The given bitSize specifies the integer type that the result must fit into.
-// It returns an error if the number is not an integer value or if the result
-// exceeds the limits for given bitSize.
-func (v Value) Int(bitSize int) (int64, error) {
- s, err := v.getIntStr()
- if err != nil {
- return 0, err
- }
- n, err := strconv.ParseInt(s, 10, bitSize)
- if err != nil {
- return 0, v.newError("%v", err)
- }
- return n, nil
-}
-
-// Uint returns the signed integer number if token is Number, else it will
-// return an error.
-//
-// The given bitSize specifies the unsigned integer type that the result must
-// fit into. It returns an error if the number is not an unsigned integer value
-// or if the result exceeds the limits for given bitSize.
-func (v Value) Uint(bitSize int) (uint64, error) {
- s, err := v.getIntStr()
- if err != nil {
- return 0, err
- }
- n, err := strconv.ParseUint(s, 10, bitSize)
- if err != nil {
- return 0, v.newError("%v", err)
- }
- return n, nil
-}
-
-func (v Value) getIntStr() (string, error) {
- if v.typ != Number {
- return "", v.newError("%s is not a number", v.input)
- }
- parts, ok := parseNumber(v.input[v.start : v.start+v.size])
- if !ok {
- return "", v.newError("%s is not a number", v.input)
- }
- num, ok := normalizeToIntString(parts)
- if !ok {
- return "", v.newError("cannot convert %s to integer", v.input)
- }
- return num, nil
-}
diff --git a/internal/encoding/json/number.go b/internal/encoding/json/decode_number.go
similarity index 83%
rename from internal/encoding/json/number.go
rename to internal/encoding/json/decode_number.go
index 529331f..a695689 100644
--- a/internal/encoding/json/number.go
+++ b/internal/encoding/json/decode_number.go
@@ -6,46 +6,14 @@
import (
"bytes"
- "math"
"strconv"
)
-// appendFloat formats given float in bitSize, and appends to the given []byte.
-func appendFloat(out []byte, n float64, bitSize int) []byte {
- switch {
- case math.IsNaN(n):
- return append(out, `"NaN"`...)
- case math.IsInf(n, +1):
- return append(out, `"Infinity"`...)
- case math.IsInf(n, -1):
- return append(out, `"-Infinity"`...)
- }
-
- // JSON number formatting logic based on encoding/json.
- // See floatEncoder.encode for reference.
- fmt := byte('f')
- if abs := math.Abs(n); abs != 0 {
- if bitSize == 64 && (abs < 1e-6 || abs >= 1e21) ||
- bitSize == 32 && (float32(abs) < 1e-6 || float32(abs) >= 1e21) {
- fmt = 'e'
- }
- }
- out = strconv.AppendFloat(out, n, fmt, -1, bitSize)
- if fmt == 'e' {
- n := len(out)
- if n >= 4 && out[n-4] == 'e' && out[n-3] == '-' && out[n-2] == '0' {
- out[n-2] = out[n-1]
- out = out[:n-1]
- }
- }
- return out
-}
-
-// consumeNumber reads the given []byte for a valid JSON number. If it is valid,
+// parseNumber reads the given []byte for a valid JSON number. If it is valid,
// it returns the number of bytes. Parsing logic follows the definition in
// https://tools.ietf.org/html/rfc7159#section-6, and is based off
// encoding/json.isValidNumber function.
-func consumeNumber(input []byte) (int, bool) {
+func parseNumber(input []byte) (int, bool) {
var n int
s := input
@@ -128,7 +96,7 @@
// parseNumber constructs numberParts from given []byte. The logic here is
// similar to consumeNumber above with the difference of having to construct
// numberParts. The slice fields in numberParts are subslices of the input.
-func parseNumber(input []byte) (numberParts, bool) {
+func parseNumberParts(input []byte) (numberParts, bool) {
var neg bool
var intp []byte
var frac []byte
diff --git a/internal/encoding/json/decode_string.go b/internal/encoding/json/decode_string.go
new file mode 100644
index 0000000..f7fea7d
--- /dev/null
+++ b/internal/encoding/json/decode_string.go
@@ -0,0 +1,91 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package json
+
+import (
+ "strconv"
+ "unicode"
+ "unicode/utf16"
+ "unicode/utf8"
+
+ "google.golang.org/protobuf/internal/strs"
+)
+
+func (d *Decoder) parseString(in []byte) (string, int, error) {
+ in0 := in
+ if len(in) == 0 {
+ return "", 0, ErrUnexpectedEOF
+ }
+ if in[0] != '"' {
+ return "", 0, d.newSyntaxError(d.currPos(), "invalid character %q at start of string", in[0])
+ }
+ in = in[1:]
+ i := indexNeedEscapeInBytes(in)
+ in, out := in[i:], in[:i:i] // set cap to prevent mutations
+ for len(in) > 0 {
+ switch r, n := utf8.DecodeRune(in); {
+ case r == utf8.RuneError && n == 1:
+ return "", 0, d.newSyntaxError(d.currPos(), "invalid UTF-8 in string")
+ case r < ' ':
+ return "", 0, d.newSyntaxError(d.currPos(), "invalid character %q in string", r)
+ case r == '"':
+ in = in[1:]
+ n := len(in0) - len(in)
+ return string(out), n, nil
+ case r == '\\':
+ if len(in) < 2 {
+ return "", 0, ErrUnexpectedEOF
+ }
+ switch r := in[1]; r {
+ case '"', '\\', '/':
+ in, out = in[2:], append(out, r)
+ case 'b':
+ in, out = in[2:], append(out, '\b')
+ case 'f':
+ in, out = in[2:], append(out, '\f')
+ case 'n':
+ in, out = in[2:], append(out, '\n')
+ case 'r':
+ in, out = in[2:], append(out, '\r')
+ case 't':
+ in, out = in[2:], append(out, '\t')
+ case 'u':
+ if len(in) < 6 {
+ return "", 0, ErrUnexpectedEOF
+ }
+ v, err := strconv.ParseUint(string(in[2:6]), 16, 16)
+ if err != nil {
+ return "", 0, d.newSyntaxError(d.currPos(), "invalid escape code %q in string", in[:6])
+ }
+ in = in[6:]
+
+ r := rune(v)
+ if utf16.IsSurrogate(r) {
+ if len(in) < 6 {
+ return "", 0, ErrUnexpectedEOF
+ }
+ v, err := strconv.ParseUint(string(in[2:6]), 16, 16)
+ r = utf16.DecodeRune(r, rune(v))
+ if in[0] != '\\' || in[1] != 'u' ||
+ r == unicode.ReplacementChar || err != nil {
+ return "", 0, d.newSyntaxError(d.currPos(), "invalid escape code %q in string", in[:6])
+ }
+ in = in[6:]
+ }
+ out = append(out, string(r)...)
+ default:
+ return "", 0, d.newSyntaxError(d.currPos(), "invalid escape code %q in string", in[:2])
+ }
+ default:
+ i := indexNeedEscapeInBytes(in[n:])
+ in, out = in[n+i:], append(out, in[:n+i]...)
+ }
+ }
+ return "", 0, ErrUnexpectedEOF
+}
+
+// indexNeedEscapeInBytes returns the index of the character that needs
+// escaping. If no characters need escaping, this returns the input length.
+func indexNeedEscapeInBytes(b []byte) int { return indexNeedEscapeInString(strs.UnsafeString(b)) }
diff --git a/internal/encoding/json/decode_test.go b/internal/encoding/json/decode_test.go
index f4905d0..b788c39 100644
--- a/internal/encoding/json/decode_test.go
+++ b/internal/encoding/json/decode_test.go
@@ -5,84 +5,374 @@
package json_test
import (
+ "fmt"
"math"
"strings"
"testing"
"unicode/utf8"
+ "github.com/google/go-cmp/cmp"
"google.golang.org/protobuf/internal/encoding/json"
)
type R struct {
- // T is expected Type returned from calling Decoder.Read.
- T json.Type
// E is expected error substring from calling Decoder.Read if set.
E string
- // V is expected value from calling
- // Value.{Bool()|Float()|Int()|Uint()|String()} depending on type.
- V interface{}
- // VE is expected error substring from calling
- // Value.{Bool()|Float()|Int()|Uint()|String()} depending on type if set.
- VE string
+ // V is one of the checker implementations that validates the token value.
+ V checker
+ // P is expected Token.Pos() if set > 0.
+ P int
+ // RS is expected result from Token.RawString() if not empty.
+ RS string
}
+// checker defines API for Token validation.
+type checker interface {
+ // check checks and expects for token API call to return and compare
+ // against implementation-stored value. Returns empty string if success,
+ // else returns error message describing the error.
+ check(json.Token) string
+}
+
+// checkers that checks the token kind only.
+var (
+ EOF = kindOnly{json.EOF}
+ Null = kindOnly{json.Null}
+ ObjectOpen = kindOnly{json.ObjectOpen}
+ ObjectClose = kindOnly{json.ObjectClose}
+ ArrayOpen = kindOnly{json.ArrayOpen}
+ ArrayClose = kindOnly{json.ArrayClose}
+)
+
+type kindOnly struct {
+ want json.Kind
+}
+
+func (x kindOnly) check(tok json.Token) string {
+ if got := tok.Kind(); got != x.want {
+ return fmt.Sprintf("Token.Kind(): got %v, want %v", got, x.want)
+ }
+ return ""
+}
+
+type Name struct {
+ val string
+}
+
+func (x Name) check(tok json.Token) string {
+ if got := tok.Kind(); got != json.Name {
+ return fmt.Sprintf("Token.Kind(): got %v, want %v", got, json.Name)
+ }
+
+ if got := tok.Name(); got != x.val {
+ return fmt.Sprintf("Token.Name(): got %v, want %v", got, x.val)
+ }
+ return ""
+}
+
+type Bool struct {
+ val bool
+}
+
+func (x Bool) check(tok json.Token) string {
+ if got := tok.Kind(); got != json.Bool {
+ return fmt.Sprintf("Token.Kind(): got %v, want %v", got, json.Bool)
+ }
+
+ if got := tok.Bool(); got != x.val {
+ return fmt.Sprintf("Token.Bool(): got %v, want %v", got, x.val)
+ }
+ return ""
+}
+
+type Str struct {
+ val string
+}
+
+func (x Str) check(tok json.Token) string {
+ if got := tok.Kind(); got != json.String {
+ return fmt.Sprintf("Token.Kind(): got %v, want %v", got, json.String)
+ }
+
+ if got := tok.ParsedString(); got != x.val {
+ return fmt.Sprintf("Token.ParsedString(): got %v, want %v", got, x.val)
+ }
+ return ""
+}
+
+type F64 struct {
+ val float64
+}
+
+func (x F64) check(tok json.Token) string {
+ if got := tok.Kind(); got != json.Number {
+ return fmt.Sprintf("Token.Kind(): got %v, want %v", got, json.Number)
+ }
+
+ got, ok := tok.Float(64)
+ if !ok {
+ return fmt.Sprintf("Token.Float(64): returned not ok")
+ }
+ if got != x.val {
+ return fmt.Sprintf("Token.Float(64): got %v, want %v", got, x.val)
+ }
+ return ""
+}
+
+type F32 struct {
+ val float32
+}
+
+func (x F32) check(tok json.Token) string {
+ if got := tok.Kind(); got != json.Number {
+ return fmt.Sprintf("Token.Kind(): got %v, want %v", got, json.Number)
+ }
+
+ got, ok := tok.Float(32)
+ if !ok {
+ return fmt.Sprintf("Token.Float(32): returned not ok")
+ }
+ if float32(got) != x.val {
+ return fmt.Sprintf("Token.Float(32): got %v, want %v", got, x.val)
+ }
+ return ""
+}
+
+// NotF64 is a checker to validate a Number token where Token.Float(64) returns not ok.
+var NotF64 = xf64{}
+
+type xf64 struct{}
+
+func (x xf64) check(tok json.Token) string {
+ if got := tok.Kind(); got != json.Number {
+ return fmt.Sprintf("Token.Kind(): got %v, want %v", got, json.Number)
+ }
+
+ _, ok := tok.Float(64)
+ if ok {
+ return fmt.Sprintf("Token.Float(64): returned ok")
+ }
+ return ""
+}
+
+// NotF32 is a checker to validate a Number token where Token.Float(32) returns not ok.
+var NotF32 = xf32{}
+
+type xf32 struct{}
+
+func (x xf32) check(tok json.Token) string {
+ if got := tok.Kind(); got != json.Number {
+ return fmt.Sprintf("Token.Kind(): got %v, want %v", got, json.Number)
+ }
+
+ _, ok := tok.Float(32)
+ if ok {
+ return fmt.Sprintf("Token.Float(32): returned ok")
+ }
+ return ""
+}
+
+type I64 struct {
+ val int64
+}
+
+func (x I64) check(tok json.Token) string {
+ if got := tok.Kind(); got != json.Number {
+ return fmt.Sprintf("Token.Kind(): got %v, want %v", got, json.Number)
+ }
+
+ got, ok := tok.Int(64)
+ if !ok {
+ return fmt.Sprintf("Token.Int(64): returned not ok")
+ }
+ if got != x.val {
+ return fmt.Sprintf("Token.Int(64): got %v, want %v", got, x.val)
+ }
+ return ""
+}
+
+type I32 struct {
+ val int32
+}
+
+func (x I32) check(tok json.Token) string {
+ if got := tok.Kind(); got != json.Number {
+ return fmt.Sprintf("Token.Kind(): got %v, want %v", got, json.Number)
+ }
+
+ got, ok := tok.Int(32)
+ if !ok {
+ return fmt.Sprintf("Token.Int(32): returned not ok")
+ }
+ if int32(got) != x.val {
+ return fmt.Sprintf("Token.Int(32): got %v, want %v", got, x.val)
+ }
+ return ""
+}
+
+// NotI64 is a checker to validate a Number token where Token.Int(64) returns not ok.
+var NotI64 = xi64{}
+
+type xi64 struct{}
+
+func (x xi64) check(tok json.Token) string {
+ if got := tok.Kind(); got != json.Number {
+ return fmt.Sprintf("Token.Kind(): got %v, want %v", got, json.Number)
+ }
+
+ _, ok := tok.Int(64)
+ if ok {
+ return fmt.Sprintf("Token.Int(64): returned ok")
+ }
+ return ""
+}
+
+// NotI32 is a checker to validate a Number token where Token.Int(32) returns not ok.
+var NotI32 = xi32{}
+
+type xi32 struct{}
+
+func (x xi32) check(tok json.Token) string {
+ if got := tok.Kind(); got != json.Number {
+ return fmt.Sprintf("Token.Kind(): got %v, want %v", got, json.Number)
+ }
+
+ _, ok := tok.Int(32)
+ if ok {
+ return fmt.Sprintf("Token.Int(32): returned ok")
+ }
+ return ""
+}
+
+type Ui64 struct {
+ val uint64
+}
+
+func (x Ui64) check(tok json.Token) string {
+ if got := tok.Kind(); got != json.Number {
+ return fmt.Sprintf("Token.Kind(): got %v, want %v", got, json.Number)
+ }
+
+ got, ok := tok.Uint(64)
+ if !ok {
+ return fmt.Sprintf("Token.Uint(64): returned not ok")
+ }
+ if got != x.val {
+ return fmt.Sprintf("Token.Uint(64): got %v, want %v", got, x.val)
+ }
+ return ""
+}
+
+type Ui32 struct {
+ val uint32
+}
+
+func (x Ui32) check(tok json.Token) string {
+ if got := tok.Kind(); got != json.Number {
+ return fmt.Sprintf("Token.Kind(): got %v, want %v", got, json.Number)
+ }
+
+ got, ok := tok.Uint(32)
+ if !ok {
+ return fmt.Sprintf("Token.Uint(32): returned not ok")
+ }
+ if uint32(got) != x.val {
+ return fmt.Sprintf("Token.Uint(32): got %v, want %v", got, x.val)
+ }
+ return ""
+}
+
+// NotUi64 is a checker to validate a Number token where Token.Uint(64) returns not ok.
+var NotUi64 = xui64{}
+
+type xui64 struct{}
+
+func (x xui64) check(tok json.Token) string {
+ if got := tok.Kind(); got != json.Number {
+ return fmt.Sprintf("Token.Kind(): got %v, want %v", got, json.Number)
+ }
+
+ _, ok := tok.Uint(64)
+ if ok {
+ return fmt.Sprintf("Token.Uint(64): returned ok")
+ }
+ return ""
+}
+
+// NotI32 is a checker to validate a Number token where Token.Uint(32) returns not ok.
+var NotUi32 = xui32{}
+
+type xui32 struct{}
+
+func (x xui32) check(tok json.Token) string {
+ if got := tok.Kind(); got != json.Number {
+ return fmt.Sprintf("Token.Kind(): got %v, want %v", got, json.Number)
+ }
+
+ _, ok := tok.Uint(32)
+ if ok {
+ return fmt.Sprintf("Token.Uint(32): returned ok")
+ }
+ return ""
+}
+
+var errEOF = json.ErrUnexpectedEOF.Error()
+
func TestDecoder(t *testing.T) {
const space = " \n\r\t"
tests := []struct {
- input string
+ in string
// want is a list of expected values returned from calling
// Decoder.Read. An item makes the test code invoke
- // Decoder.Read and compare against R.T and R.E. For Bool,
- // Number and String tokens, it invokes the corresponding getter method
- // and compares the returned value against R.V or R.VE if it returned an
- // error.
+ // Decoder.Read and compare against R.E for error returned or use R.V to
+ // validate the returned Token object.
want []R
}{
{
- input: ``,
- want: []R{{T: json.EOF}},
+ in: ``,
+ want: []R{{V: EOF}},
},
{
- input: space,
- want: []R{{T: json.EOF}},
+ in: space,
+ want: []R{{V: EOF}},
},
{
// Calling Read after EOF will keep returning EOF for
// succeeding Read calls.
- input: space,
+ in: space,
want: []R{
- {T: json.EOF},
- {T: json.EOF},
- {T: json.EOF},
+ {V: EOF},
+ {V: EOF},
+ {V: EOF},
},
},
// JSON literals.
{
- input: space + `null` + space,
+ in: space + `null` + space,
want: []R{
- {T: json.Null},
- {T: json.EOF},
+ {V: Null, P: len(space), RS: `null`},
+ {V: EOF},
},
},
{
- input: space + `true` + space,
+ in: space + `true` + space,
want: []R{
- {T: json.Bool, V: true},
- {T: json.EOF},
+ {V: Bool{true}},
+ {V: EOF},
},
},
{
- input: space + `false` + space,
+ in: space + `false` + space,
want: []R{
- {T: json.Bool, V: false},
- {T: json.EOF},
+ {V: Bool{false}},
+ {V: EOF},
},
},
{
// Error returned will produce the same error again.
- input: space + `foo` + space,
+ in: space + `foo` + space,
want: []R{
{E: `invalid value foo`},
{E: `invalid value foo`},
@@ -91,1027 +381,1001 @@
// JSON strings.
{
- input: space + `""` + space,
+ in: space + `""` + space,
want: []R{
- {T: json.String, V: ""},
- {T: json.EOF},
+ {V: Str{}},
+ {V: EOF},
},
},
{
- input: space + `"hello"` + space,
+ in: space + `"hello"` + space,
want: []R{
- {T: json.String, V: "hello"},
- {T: json.EOF},
+ {V: Str{"hello"}, RS: `"hello"`},
+ {V: EOF},
},
},
{
- input: `"hello`,
- want: []R{{E: `unexpected EOF`}},
+ in: `"hello`,
+ want: []R{{E: errEOF}},
},
{
- input: "\"\x00\"",
- want: []R{{E: `invalid character '\x00' in string`}},
+ in: "\"\x00\"",
+ want: []R{{E: `invalid character '\x00' in string`}},
},
{
- input: "\"\u0031\u0032\"",
+ in: "\"\u0031\u0032\"",
want: []R{
- {T: json.String, V: "12"},
- {T: json.EOF},
+ {V: Str{"12"}, RS: "\"\u0031\u0032\""},
+ {V: EOF},
},
},
{
// Invalid UTF-8 error is returned in ReadString instead of Read.
- input: "\"\xff\"",
- want: []R{{E: `syntax error (line 1:1): invalid UTF-8 in string`}},
+ in: "\"\xff\"",
+ want: []R{{E: `syntax error (line 1:1): invalid UTF-8 in string`}},
},
{
- input: `"` + string(utf8.RuneError) + `"`,
+ in: `"` + string(utf8.RuneError) + `"`,
want: []R{
- {T: json.String, V: string(utf8.RuneError)},
- {T: json.EOF},
+ {V: Str{string(utf8.RuneError)}},
+ {V: EOF},
},
},
{
- input: `"\uFFFD"`,
+ in: `"\uFFFD"`,
want: []R{
- {T: json.String, V: string(utf8.RuneError)},
- {T: json.EOF},
+ {V: Str{string(utf8.RuneError)}},
+ {V: EOF},
},
},
{
- input: `"\x"`,
- want: []R{{E: `invalid escape code "\\x" in string`}},
+ in: `"\x"`,
+ want: []R{{E: `invalid escape code "\\x" in string`}},
},
{
- input: `"\uXXXX"`,
- want: []R{{E: `invalid escape code "\\uXXXX" in string`}},
+ in: `"\uXXXX"`,
+ want: []R{{E: `invalid escape code "\\uXXXX" in string`}},
},
{
- input: `"\uDEAD"`, // unmatched surrogate pair
- want: []R{{E: `unexpected EOF`}},
+ in: `"\uDEAD"`, // unmatched surrogate pair
+ want: []R{{E: errEOF}},
},
{
- input: `"\uDEAD\uBEEF"`, // invalid surrogate half
- want: []R{{E: `invalid escape code "\\uBEEF" in string`}},
+ in: `"\uDEAD\uBEEF"`, // invalid surrogate half
+ want: []R{{E: `invalid escape code "\\uBEEF" in string`}},
},
{
- input: `"\uD800\udead"`, // valid surrogate pair
+ in: `"\uD800\udead"`, // valid surrogate pair
want: []R{
- {T: json.String, V: `𐊭`},
- {T: json.EOF},
+ {V: Str{`𐊭`}},
+ {V: EOF},
},
},
{
- input: `"\u0000\"\\\/\b\f\n\r\t"`,
+ in: `"\u0000\"\\\/\b\f\n\r\t"`,
want: []R{
- {T: json.String, V: "\u0000\"\\/\b\f\n\r\t"},
- {T: json.EOF},
+ {V: Str{"\u0000\"\\/\b\f\n\r\t"}},
+ {V: EOF},
},
},
// Invalid JSON numbers.
{
- input: `-`,
- want: []R{{E: `invalid number -`}},
+ in: `-`,
+ want: []R{{E: `invalid value -`}},
},
{
- input: `+0`,
- want: []R{{E: `invalid value +0`}},
+ in: `+0`,
+ want: []R{{E: `invalid value +0`}},
},
{
- input: `-+`,
- want: []R{{E: `invalid number -+`}},
+ in: `-+`,
+ want: []R{{E: `invalid value -+`}},
},
{
- input: `0.`,
- want: []R{{E: `invalid number 0.`}},
+ in: `0.`,
+ want: []R{{E: `invalid value 0.`}},
},
{
- input: `.1`,
- want: []R{{E: `invalid value .1`}},
+ in: `.1`,
+ want: []R{{E: `invalid value .1`}},
},
{
- input: `1.0.1`,
- want: []R{{E: `invalid number 1.0.1`}},
+ in: `1.0.1`,
+ want: []R{{E: `invalid value 1.0.1`}},
},
{
- input: `1..1`,
- want: []R{{E: `invalid number 1..1`}},
+ in: `1..1`,
+ want: []R{{E: `invalid value 1..1`}},
},
{
- input: `-1-2`,
- want: []R{{E: `invalid number -1-2`}},
+ in: `-1-2`,
+ want: []R{{E: `invalid value -1-2`}},
},
{
- input: `01`,
- want: []R{{E: `invalid number 01`}},
+ in: `01`,
+ want: []R{{E: `invalid value 01`}},
},
{
- input: `1e`,
- want: []R{{E: `invalid number 1e`}},
+ in: `1e`,
+ want: []R{{E: `invalid value 1e`}},
},
{
- input: `1e1.2`,
- want: []R{{E: `invalid number 1e1.2`}},
+ in: `1e1.2`,
+ want: []R{{E: `invalid value 1e1.2`}},
},
{
- input: `1Ee`,
- want: []R{{E: `invalid number 1Ee`}},
+ in: `1Ee`,
+ want: []R{{E: `invalid value 1Ee`}},
},
{
- input: `1.e1`,
- want: []R{{E: `invalid number 1.e1`}},
+ in: `1.e1`,
+ want: []R{{E: `invalid value 1.e1`}},
},
{
- input: `1.e+`,
- want: []R{{E: `invalid number 1.e+`}},
+ in: `1.e+`,
+ want: []R{{E: `invalid value 1.e+`}},
},
{
- input: `1e+-2`,
- want: []R{{E: `invalid number 1e+-2`}},
+ in: `1e+-2`,
+ want: []R{{E: `invalid value 1e+-2`}},
},
{
- input: `1e--2`,
- want: []R{{E: `invalid number 1e--2`}},
+ in: `1e--2`,
+ want: []R{{E: `invalid value 1e--2`}},
},
{
- input: `1.0true`,
- want: []R{{E: `invalid number 1.0true`}},
+ in: `1.0true`,
+ want: []R{{E: `invalid value 1.0true`}},
},
// JSON numbers as floating point.
{
- input: space + `0.0` + space,
+ in: space + `0.0` + space,
want: []R{
- {T: json.Number, V: float32(0)},
- {T: json.EOF},
+ {V: F32{0}, P: len(space), RS: `0.0`},
+ {V: EOF},
},
},
{
- input: space + `0` + space,
+ in: space + `0` + space,
want: []R{
- {T: json.Number, V: float32(0)},
- {T: json.EOF},
+ {V: F32{0}},
+ {V: EOF},
},
},
{
- input: space + `-0` + space,
+ in: space + `-0` + space,
want: []R{
- {T: json.Number, V: float32(math.Copysign(0, -1))},
- {T: json.EOF},
+ {V: F32{float32(math.Copysign(0, -1))}},
+ {V: EOF},
},
},
{
- input: `-0`,
+ in: `-0`,
want: []R{
- {T: json.Number, V: math.Copysign(0, -1)},
- {T: json.EOF},
+ {V: F64{math.Copysign(0, -1)}},
+ {V: EOF},
},
},
{
- input: `-0.0`,
+ in: `-0.0`,
want: []R{
- {T: json.Number, V: float32(math.Copysign(0, -1))},
- {T: json.EOF},
+ {V: F32{float32(math.Copysign(0, -1))}},
+ {V: EOF},
},
},
{
- input: `-0.0`,
+ in: `-0.0`,
want: []R{
- {T: json.Number, V: math.Copysign(0, -1)},
- {T: json.EOF},
+ {V: F64{math.Copysign(0, -1)}},
+ {V: EOF},
},
},
{
- input: `-1.02`,
+ in: `-1.02`,
want: []R{
- {T: json.Number, V: float32(-1.02)},
- {T: json.EOF},
+ {V: F32{-1.02}},
+ {V: EOF},
},
},
{
- input: `1.020000`,
+ in: `1.020000`,
want: []R{
- {T: json.Number, V: float32(1.02)},
- {T: json.EOF},
+ {V: F32{1.02}},
+ {V: EOF},
},
},
{
- input: `-1.0e0`,
+ in: `-1.0e0`,
want: []R{
- {T: json.Number, V: float32(-1)},
- {T: json.EOF},
+ {V: F32{-1}},
+ {V: EOF},
},
},
{
- input: `1.0e-000`,
+ in: `1.0e-000`,
want: []R{
- {T: json.Number, V: float32(1)},
- {T: json.EOF},
+ {V: F32{1}},
+ {V: EOF},
},
},
{
- input: `1e+00`,
+ in: `1e+00`,
want: []R{
- {T: json.Number, V: float32(1)},
- {T: json.EOF},
+ {V: F32{1}},
+ {V: EOF},
},
},
{
- input: `1.02e3`,
+ in: `1.02e3`,
want: []R{
- {T: json.Number, V: float32(1.02e3)},
- {T: json.EOF},
+ {V: F32{1.02e3}},
+ {V: EOF},
},
},
{
- input: `-1.02E03`,
+ in: `-1.02E03`,
want: []R{
- {T: json.Number, V: float32(-1.02e3)},
- {T: json.EOF},
+ {V: F32{-1.02e3}},
+ {V: EOF},
},
},
{
- input: `1.0200e+3`,
+ in: `1.0200e+3`,
want: []R{
- {T: json.Number, V: float32(1.02e3)},
- {T: json.EOF},
+ {V: F32{1.02e3}},
+ {V: EOF},
},
},
{
- input: `-1.0200E+03`,
+ in: `-1.0200E+03`,
want: []R{
- {T: json.Number, V: float32(-1.02e3)},
- {T: json.EOF},
+ {V: F32{-1.02e3}},
+ {V: EOF},
},
},
{
- input: `1.0200e-3`,
+ in: `1.0200e-3`,
want: []R{
- {T: json.Number, V: float32(1.02e-3)},
- {T: json.EOF},
+ {V: F32{1.02e-3}},
+ {V: EOF},
},
},
{
- input: `-1.0200E-03`,
+ in: `-1.0200E-03`,
want: []R{
- {T: json.Number, V: float32(-1.02e-3)},
- {T: json.EOF},
+ {V: F32{-1.02e-3}},
+ {V: EOF},
},
},
{
// Exceeds max float32 limit, but should be ok for float64.
- input: `3.4e39`,
+ in: `3.4e39`,
want: []R{
- {T: json.Number, V: float64(3.4e39)},
- {T: json.EOF},
+ {V: F64{3.4e39}},
+ {V: EOF},
},
},
+
{
// Exceeds max float32 limit.
- input: `3.4e39`,
+ in: `3.4e39`,
want: []R{
- {T: json.Number, V: float32(0), VE: `value out of range`},
- {T: json.EOF},
+ {V: NotF32},
+ {V: EOF},
},
},
{
// Less than negative max float32 limit.
- input: `-3.4e39`,
+ in: `-3.4e39`,
want: []R{
- {T: json.Number, V: float32(0), VE: `value out of range`},
- {T: json.EOF},
+ {V: NotF32},
+ {V: EOF},
},
},
{
// Exceeds max float64 limit.
- input: `1.79e+309`,
+ in: `1.79e+309`,
want: []R{
- {T: json.Number, V: float64(0), VE: `value out of range`},
- {T: json.EOF},
+ {V: NotF64},
+ {V: EOF},
},
},
{
// Less than negative max float64 limit.
- input: `-1.79e+309`,
+ in: `-1.79e+309`,
want: []R{
- {T: json.Number, V: float64(0), VE: `value out of range`},
- {T: json.EOF},
+ {V: NotF64},
+ {V: EOF},
},
},
// JSON numbers as signed integers.
{
- input: space + `0` + space,
+ in: space + `0` + space,
want: []R{
- {T: json.Number, V: int32(0)},
- {T: json.EOF},
+ {V: I32{0}},
+ {V: EOF},
},
},
{
- input: space + `-0` + space,
+ in: space + `-0` + space,
want: []R{
- {T: json.Number, V: int32(0)},
- {T: json.EOF},
+ {V: I32{0}},
+ {V: EOF},
},
},
{
// Fractional part equals 0 is ok.
- input: `1.00000`,
+ in: `1.00000`,
want: []R{
- {T: json.Number, V: int32(1)},
- {T: json.EOF},
+ {V: I32{1}},
+ {V: EOF},
},
},
{
// Fractional part not equals 0 returns error.
- input: `1.0000000001`,
+ in: `1.0000000001`,
want: []R{
- {T: json.Number, V: int32(0), VE: `cannot convert 1.0000000001 to integer`},
- {T: json.EOF},
+ {V: NotI32},
+ {V: EOF},
},
},
{
- input: `0e0`,
+ in: `0e0`,
want: []R{
- {T: json.Number, V: int32(0)},
- {T: json.EOF},
+ {V: I32{0}},
+ {V: EOF},
},
},
{
- input: `0.0E0`,
+ in: `0.0E0`,
want: []R{
- {T: json.Number, V: int32(0)},
- {T: json.EOF},
+ {V: I32{0}},
+ {V: EOF},
},
},
{
- input: `0.0E10`,
+ in: `0.0E10`,
want: []R{
- {T: json.Number, V: int32(0)},
- {T: json.EOF},
+ {V: I32{0}},
+ {V: EOF},
},
},
{
- input: `-1`,
+ in: `-1`,
want: []R{
- {T: json.Number, V: int32(-1)},
- {T: json.EOF},
+ {V: I32{-1}},
+ {V: EOF},
},
},
{
- input: `1.0e+0`,
+ in: `1.0e+0`,
want: []R{
- {T: json.Number, V: int32(1)},
- {T: json.EOF},
+ {V: I32{1}},
+ {V: EOF},
},
},
{
- input: `-1E-0`,
+ in: `-1E-0`,
want: []R{
- {T: json.Number, V: int32(-1)},
- {T: json.EOF},
+ {V: I32{-1}},
+ {V: EOF},
},
},
{
- input: `1E1`,
+ in: `1E1`,
want: []R{
- {T: json.Number, V: int32(10)},
- {T: json.EOF},
+ {V: I32{10}},
+ {V: EOF},
},
},
{
- input: `-100.00e-02`,
+ in: `-100.00e-02`,
want: []R{
- {T: json.Number, V: int32(-1)},
- {T: json.EOF},
+ {V: I32{-1}},
+ {V: EOF},
},
},
{
- input: `0.1200E+02`,
+ in: `0.1200E+02`,
want: []R{
- {T: json.Number, V: int64(12)},
- {T: json.EOF},
+ {V: I64{12}},
+ {V: EOF},
},
},
{
- input: `0.012e2`,
+ in: `0.012e2`,
want: []R{
- {T: json.Number, V: int32(0), VE: `cannot convert 0.012e2 to integer`},
- {T: json.EOF},
+ {V: NotI32},
+ {V: EOF},
},
},
{
- input: `12e-2`,
+ in: `12e-2`,
want: []R{
- {T: json.Number, V: int32(0), VE: `cannot convert 12e-2 to integer`},
- {T: json.EOF},
+ {V: NotI32},
+ {V: EOF},
},
},
{
// Exceeds math.MaxInt32.
- input: `2147483648`,
+ in: `2147483648`,
want: []R{
- {T: json.Number, V: int32(0), VE: `value out of range`},
- {T: json.EOF},
+ {V: NotI32},
+ {V: EOF},
},
},
{
// Exceeds math.MinInt32.
- input: `-2147483649`,
+ in: `-2147483649`,
want: []R{
- {T: json.Number, V: int32(0), VE: `value out of range`},
- {T: json.EOF},
+ {V: NotI32},
+ {V: EOF},
},
},
{
// Exceeds math.MaxInt32, but ok for int64.
- input: `2147483648`,
+ in: `2147483648`,
want: []R{
- {T: json.Number, V: int64(2147483648)},
- {T: json.EOF},
+ {V: I64{2147483648}},
+ {V: EOF},
},
},
{
// Exceeds math.MinInt32, but ok for int64.
- input: `-2147483649`,
+ in: `-2147483649`,
want: []R{
- {T: json.Number, V: int64(-2147483649)},
- {T: json.EOF},
+ {V: I64{-2147483649}},
+ {V: EOF},
},
},
{
// Exceeds math.MaxInt64.
- input: `9223372036854775808`,
+ in: `9223372036854775808`,
want: []R{
- {T: json.Number, V: int64(0), VE: `value out of range`},
- {T: json.EOF},
+ {V: NotI64},
+ {V: EOF},
},
},
{
// Exceeds math.MinInt64.
- input: `-9223372036854775809`,
+ in: `-9223372036854775809`,
want: []R{
- {T: json.Number, V: int64(0), VE: `value out of range`},
- {T: json.EOF},
+ {V: NotI64},
+ {V: EOF},
},
},
// JSON numbers as unsigned integers.
{
- input: space + `0` + space,
+ in: space + `0` + space,
want: []R{
- {T: json.Number, V: uint32(0)},
- {T: json.EOF},
+ {V: Ui32{0}},
+ {V: EOF},
},
},
{
- input: space + `-0` + space,
+ in: space + `-0` + space,
want: []R{
- {T: json.Number, V: uint32(0)},
- {T: json.EOF},
+ {V: Ui32{0}},
+ {V: EOF},
},
},
{
- input: `-1`,
+ in: `-1`,
want: []R{
- {T: json.Number, V: uint32(0), VE: `invalid syntax`},
- {T: json.EOF},
+ {V: NotUi32},
+ {V: EOF},
},
},
{
// Exceeds math.MaxUint32.
- input: `4294967296`,
+ in: `4294967296`,
want: []R{
- {T: json.Number, V: uint32(0), VE: `value out of range`},
- {T: json.EOF},
+ {V: NotUi32},
+ {V: EOF},
},
},
{
// Exceeds math.MaxUint64.
- input: `18446744073709551616`,
+ in: `18446744073709551616`,
want: []R{
- {T: json.Number, V: uint64(0), VE: `value out of range`},
- {T: json.EOF},
+ {V: NotUi64},
+ {V: EOF},
},
},
// JSON sequence of values.
{
- input: `true null`,
+ in: `true null`,
want: []R{
- {T: json.Bool, V: true},
- {E: `unexpected value null`},
+ {V: Bool{true}},
+ {E: `(line 1:6): unexpected token null`},
},
},
{
- input: "null false",
+ in: "null false",
want: []R{
- {T: json.Null},
- {E: `unexpected value false`},
+ {V: Null},
+ {E: `unexpected token false`},
},
},
{
- input: `true,false`,
+ in: `true,false`,
want: []R{
- {T: json.Bool, V: true},
- {E: `unexpected character ,`},
+ {V: Bool{true}},
+ {E: `unexpected token ,`},
},
},
{
- input: `47"hello"`,
+ in: `47"hello"`,
want: []R{
- {T: json.Number, V: int32(47)},
- {E: `unexpected value "hello"`},
+ {V: I32{47}},
+ {E: `unexpected token "hello"`},
},
},
{
- input: `47 "hello"`,
+ in: `47 "hello"`,
want: []R{
- {T: json.Number, V: int32(47)},
- {E: `unexpected value "hello"`},
+ {V: I32{47}},
+ {E: `unexpected token "hello"`},
},
},
{
- input: `true 42`,
+ in: `true 42`,
want: []R{
- {T: json.Bool, V: true},
- {E: `unexpected value 42`},
+ {V: Bool{true}},
+ {E: `unexpected token 42`},
},
},
// JSON arrays.
{
- input: space + `[]` + space,
+ in: space + `[]` + space,
want: []R{
- {T: json.StartArray},
- {T: json.EndArray},
- {T: json.EOF},
+ {V: ArrayOpen},
+ {V: ArrayClose},
+ {V: EOF},
},
},
{
- input: space + `[` + space + `]` + space,
+ in: space + `[` + space + `]` + space,
want: []R{
- {T: json.StartArray},
- {T: json.EndArray},
- {T: json.EOF},
+ {V: ArrayOpen, P: len(space), RS: `[`},
+ {V: ArrayClose},
+ {V: EOF},
},
},
{
- input: space + `[` + space,
+ in: space + `[` + space,
want: []R{
- {T: json.StartArray},
- {E: `unexpected EOF`},
+ {V: ArrayOpen},
+ {E: errEOF},
},
},
{
- input: space + `]` + space,
- want: []R{{E: `unexpected character ]`}},
+ in: space + `]` + space,
+ want: []R{{E: `unexpected token ]`}},
},
{
- input: `[null,true,false, 1e1, "hello" ]`,
+ in: `[null,true,false, 1e1, "hello" ]`,
want: []R{
- {T: json.StartArray},
- {T: json.Null},
- {T: json.Bool, V: true},
- {T: json.Bool, V: false},
- {T: json.Number, V: int32(10)},
- {T: json.String, V: "hello"},
- {T: json.EndArray},
- {T: json.EOF},
+ {V: ArrayOpen},
+ {V: Null},
+ {V: Bool{true}},
+ {V: Bool{false}},
+ {V: I32{10}},
+ {V: Str{"hello"}},
+ {V: ArrayClose},
+ {V: EOF},
},
},
{
- input: `[` + space + `true` + space + `,` + space + `"hello"` + space + `]`,
+ in: `[` + space + `true` + space + `,` + space + `"hello"` + space + `]`,
want: []R{
- {T: json.StartArray},
- {T: json.Bool, V: true},
- {T: json.String, V: "hello"},
- {T: json.EndArray},
- {T: json.EOF},
+ {V: ArrayOpen},
+ {V: Bool{true}},
+ {V: Str{"hello"}},
+ {V: ArrayClose},
+ {V: EOF},
},
},
{
- input: `[` + space + `true` + space + `,` + space + `]`,
+ in: `[` + space + `true` + space + `,` + space + `]`,
want: []R{
- {T: json.StartArray},
- {T: json.Bool, V: true},
- {E: `unexpected character ]`},
+ {V: ArrayOpen},
+ {V: Bool{true}},
+ {E: `unexpected token ]`},
},
},
{
- input: `[` + space + `false` + space + `]`,
+ in: `[` + space + `false` + space + `]`,
want: []R{
- {T: json.StartArray},
- {T: json.Bool, V: false},
- {T: json.EndArray},
- {T: json.EOF},
+ {V: ArrayOpen},
+ {V: Bool{false}},
+ {V: ArrayClose},
+ {V: EOF},
},
},
{
- input: `[` + space + `1` + space + `0` + space + `]`,
+ in: `[` + space + `1` + space + `0` + space + `]`,
want: []R{
- {T: json.StartArray},
- {T: json.Number, V: int64(1)},
- {E: `unexpected value 0`},
+ {V: ArrayOpen},
+ {V: I64{1}},
+ {E: `unexpected token 0`},
},
},
{
- input: `[null`,
+ in: `[null`,
want: []R{
- {T: json.StartArray},
- {T: json.Null},
- {E: `unexpected EOF`},
+ {V: ArrayOpen},
+ {V: Null},
+ {E: errEOF},
},
},
{
- input: `[foo]`,
+ in: `[foo]`,
want: []R{
- {T: json.StartArray},
+ {V: ArrayOpen},
{E: `invalid value foo`},
},
},
{
- input: `[{}, "hello", [true, false], null]`,
+ in: `[{}, "hello", [true, false], null]`,
want: []R{
- {T: json.StartArray},
- {T: json.StartObject},
- {T: json.EndObject},
- {T: json.String, V: "hello"},
- {T: json.StartArray},
- {T: json.Bool, V: true},
- {T: json.Bool, V: false},
- {T: json.EndArray},
- {T: json.Null},
- {T: json.EndArray},
- {T: json.EOF},
+ {V: ArrayOpen},
+ {V: ObjectOpen},
+ {V: ObjectClose},
+ {V: Str{"hello"}},
+ {V: ArrayOpen},
+ {V: Bool{true}},
+ {V: Bool{false}},
+ {V: ArrayClose},
+ {V: Null},
+ {V: ArrayClose},
+ {V: EOF},
},
},
{
- input: `[{ ]`,
+ in: `[{ ]`,
want: []R{
- {T: json.StartArray},
- {T: json.StartObject},
- {E: `unexpected character ]`},
+ {V: ArrayOpen},
+ {V: ObjectOpen},
+ {E: `unexpected token ]`},
},
},
{
- input: `[[ ]`,
+ in: `[[ ]`,
want: []R{
- {T: json.StartArray},
- {T: json.StartArray},
- {T: json.EndArray},
- {E: `unexpected EOF`},
+ {V: ArrayOpen},
+ {V: ArrayOpen},
+ {V: ArrayClose},
+ {E: errEOF},
},
},
{
- input: `[,]`,
+ in: `[,]`,
want: []R{
- {T: json.StartArray},
- {E: `unexpected character ,`},
+ {V: ArrayOpen},
+ {E: `unexpected token ,`},
},
},
{
- input: `[true "hello"]`,
+ in: `[true "hello"]`,
want: []R{
- {T: json.StartArray},
- {T: json.Bool, V: true},
- {E: `unexpected value "hello"`},
+ {V: ArrayOpen},
+ {V: Bool{true}},
+ {E: `unexpected token "hello"`},
},
},
{
- input: `[] null`,
+ in: `[] null`,
want: []R{
- {T: json.StartArray},
- {T: json.EndArray},
- {E: `unexpected value null`},
+ {V: ArrayOpen},
+ {V: ArrayClose},
+ {E: `unexpected token null`},
},
},
{
- input: `true []`,
+ in: `true []`,
want: []R{
- {T: json.Bool, V: true},
- {E: `unexpected character [`},
+ {V: Bool{true}},
+ {E: `unexpected token [`},
},
},
// JSON objects.
{
- input: space + `{}` + space,
+ in: space + `{}` + space,
want: []R{
- {T: json.StartObject},
- {T: json.EndObject},
- {T: json.EOF},
+ {V: ObjectOpen},
+ {V: ObjectClose},
+ {V: EOF},
},
},
{
- input: space + `{` + space + `}` + space,
+ in: space + `{` + space + `}` + space,
want: []R{
- {T: json.StartObject},
- {T: json.EndObject},
- {T: json.EOF},
+ {V: ObjectOpen},
+ {V: ObjectClose},
+ {V: EOF},
},
},
{
- input: space + `{` + space,
+ in: space + `{` + space,
want: []R{
- {T: json.StartObject},
- {E: `unexpected EOF`},
+ {V: ObjectOpen},
+ {E: errEOF},
},
},
{
- input: space + `}` + space,
- want: []R{{E: `unexpected character }`}},
+ in: space + `}` + space,
+ want: []R{{E: `unexpected token }`}},
},
{
- input: `{` + space + `null` + space + `}`,
+ in: `{` + space + `null` + space + `}`,
want: []R{
- {T: json.StartObject},
- {E: `unexpected value null`},
+ {V: ObjectOpen},
+ {E: `unexpected token null`},
},
},
{
- input: `{[]}`,
+ in: `{[]}`,
want: []R{
- {T: json.StartObject},
- {E: `unexpected character [`},
+ {V: ObjectOpen},
+ {E: `(line 1:2): unexpected token [`},
},
},
{
- input: `{,}`,
+ in: `{,}`,
want: []R{
- {T: json.StartObject},
- {E: `unexpected character ,`},
+ {V: ObjectOpen},
+ {E: `unexpected token ,`},
},
},
{
- input: `{"345678"}`,
+ in: `{"345678"}`,
want: []R{
- {T: json.StartObject},
- {E: `unexpected character }, missing ":" after object name`},
+ {V: ObjectOpen},
+ {E: `(line 1:10): unexpected character }, missing ":" after field name`},
},
},
{
- input: `{` + space + `"hello"` + space + `:` + space + `"world"` + space + `}`,
+ in: `{` + space + `"hello"` + space + `:` + space + `"world"` + space + `}`,
want: []R{
- {T: json.StartObject},
- {T: json.Name, V: "hello"},
- {T: json.String, V: "world"},
- {T: json.EndObject},
- {T: json.EOF},
+ {V: ObjectOpen},
+ {V: Name{"hello"}, P: len(space) + 1, RS: `"hello"`},
+ {V: Str{"world"}, RS: `"world"`},
+ {V: ObjectClose},
+ {V: EOF},
},
},
{
- input: `{"hello" "world"}`,
+ in: `{"hello" "world"}`,
want: []R{
- {T: json.StartObject},
- {E: `unexpected character ", missing ":" after object name`},
+ {V: ObjectOpen},
+ {E: `(line 1:10): unexpected character ", missing ":" after field name`},
},
},
{
- input: `{"hello":`,
+ in: `{"hello":`,
want: []R{
- {T: json.StartObject},
- {T: json.Name, V: "hello"},
- {E: `unexpected EOF`},
+ {V: ObjectOpen},
+ {V: Name{"hello"}},
+ {E: errEOF},
},
},
{
- input: `{"hello":"world"`,
+ in: `{"hello":"world"`,
want: []R{
- {T: json.StartObject},
- {T: json.Name, V: "hello"},
- {T: json.String, V: "world"},
- {E: `unexpected EOF`},
+ {V: ObjectOpen},
+ {V: Name{"hello"}},
+ {V: Str{"world"}},
+ {E: errEOF},
},
},
{
- input: `{"hello":"world",`,
+ in: `{"hello":"world",`,
want: []R{
- {T: json.StartObject},
- {T: json.Name, V: "hello"},
- {T: json.String, V: "world"},
- {E: `unexpected EOF`},
+ {V: ObjectOpen},
+ {V: Name{"hello"}},
+ {V: Str{"world"}},
+ {E: errEOF},
},
},
{
- input: `{""`,
+ in: `{""`,
want: []R{
- {T: json.StartObject},
- {E: `syntax error (line 1:4): unexpected EOF`},
+ {V: ObjectOpen},
+ {E: errEOF},
},
},
{
- input: `{"34":"89",}`,
+ in: `{"34":"89",}`,
want: []R{
- {T: json.StartObject},
- {T: json.Name, V: "34"},
- {T: json.String, V: "89"},
- {E: `syntax error (line 1:12): unexpected character }`},
+ {V: ObjectOpen},
+ {V: Name{"34"}, RS: `"34"`},
+ {V: Str{"89"}},
+ {E: `syntax error (line 1:12): unexpected token }`},
},
},
{
- input: `{
- "number": 123e2,
- "bool" : false,
- "object": {"string": "world"},
- "null" : null,
- "array" : [1.01, "hello", true],
- "string": "hello"
-}`,
+ in: `{
+ "number": 123e2,
+ "bool" : false,
+ "object": {"string": "world"},
+ "null" : null,
+ "array" : [1.01, "hello", true],
+ "string": "hello"
+ }`,
want: []R{
- {T: json.StartObject},
+ {V: ObjectOpen},
- {T: json.Name, V: "number"},
- {T: json.Number, V: int32(12300)},
+ {V: Name{"number"}},
+ {V: I32{12300}},
- {T: json.Name, V: "bool"},
- {T: json.Bool, V: false},
+ {V: Name{"bool"}},
+ {V: Bool{false}},
- {T: json.Name, V: "object"},
- {T: json.StartObject},
- {T: json.Name, V: "string"},
- {T: json.String, V: "world"},
- {T: json.EndObject},
+ {V: Name{"object"}},
+ {V: ObjectOpen},
+ {V: Name{"string"}},
+ {V: Str{"world"}},
+ {V: ObjectClose},
- {T: json.Name, V: "null"},
- {T: json.Null},
+ {V: Name{"null"}},
+ {V: Null},
- {T: json.Name, V: "array"},
- {T: json.StartArray},
- {T: json.Number, V: float32(1.01)},
- {T: json.String, V: "hello"},
- {T: json.Bool, V: true},
- {T: json.EndArray},
+ {V: Name{"array"}},
+ {V: ArrayOpen},
+ {V: F32{1.01}},
+ {V: Str{"hello"}},
+ {V: Bool{true}},
+ {V: ArrayClose},
- {T: json.Name, V: "string"},
- {T: json.String, V: "hello"},
+ {V: Name{"string"}},
+ {V: Str{"hello"}},
- {T: json.EndObject},
- {T: json.EOF},
+ {V: ObjectClose},
+ {V: EOF},
},
},
{
- input: `[
- {"object": {"number": 47}},
- ["list"],
- null
-]`,
+ in: `[
+ {"object": {"number": 47}},
+ ["list"],
+ null
+ ]`,
want: []R{
- {T: json.StartArray},
+ {V: ArrayOpen},
- {T: json.StartObject},
- {T: json.Name, V: "object"},
- {T: json.StartObject},
- {T: json.Name, V: "number"},
- {T: json.Number, V: uint32(47)},
- {T: json.EndObject},
- {T: json.EndObject},
+ {V: ObjectOpen},
+ {V: Name{"object"}},
+ {V: ObjectOpen},
+ {V: Name{"number"}},
+ {V: I32{47}},
+ {V: ObjectClose},
+ {V: ObjectClose},
- {T: json.StartArray},
- {T: json.String, V: "list"},
- {T: json.EndArray},
+ {V: ArrayOpen},
+ {V: Str{"list"}},
+ {V: ArrayClose},
- {T: json.Null},
+ {V: Null},
- {T: json.EndArray},
- {T: json.EOF},
+ {V: ArrayClose},
+ {V: EOF},
},
},
// Tests for line and column info.
{
- input: `12345678 x`,
+ in: `12345678 x`,
want: []R{
- {T: json.Number, V: int64(12345678)},
+ {V: I64{12345678}},
{E: `syntax error (line 1:10): invalid value x`},
},
},
{
- input: "\ntrue\n x",
+ in: "\ntrue\n x",
want: []R{
- {T: json.Bool, V: true},
+ {V: Bool{true}},
{E: `syntax error (line 3:4): invalid value x`},
},
},
{
- input: `"💩"x`,
+ in: `"💩"x`,
want: []R{
- {T: json.String, V: "💩"},
+ {V: Str{"💩"}},
{E: `syntax error (line 1:4): invalid value x`},
},
},
{
- input: "\n\n[\"🔥🔥🔥\"x",
+ in: "\n\n[\"🔥🔥🔥\"x",
want: []R{
- {T: json.StartArray},
- {T: json.String, V: "🔥🔥🔥"},
+ {V: ArrayOpen},
+ {V: Str{"🔥🔥🔥"}},
{E: `syntax error (line 3:7): invalid value x`},
},
},
{
// Multi-rune emojis.
- input: `["👍🏻👍🏿"x`,
+ in: `["👍🏻👍🏿"x`,
want: []R{
- {T: json.StartArray},
- {T: json.String, V: "👍🏻👍🏿"},
+ {V: ArrayOpen},
+ {V: Str{"👍🏻👍🏿"}},
{E: `syntax error (line 1:8): invalid value x`},
},
},
- {
- input: `{
- "45678":-1
-}`,
- want: []R{
- {T: json.StartObject},
- {T: json.Name, V: "45678"},
- {T: json.Number, V: uint64(1), VE: "error (line 2:11)"},
- },
- },
}
for _, tc := range tests {
tc := tc
t.Run("", func(t *testing.T) {
- dec := json.NewDecoder([]byte(tc.input))
+ dec := json.NewDecoder([]byte(tc.in))
for i, want := range tc.want {
- typ := dec.Peek()
- if typ != want.T {
- t.Errorf("input: %v\nPeek() got %v want %v", tc.input, typ, want.T)
- }
- value, err := dec.Read()
+ peekTok, peekErr := dec.Peek()
+ tok, err := dec.Read()
if err != nil {
if want.E == "" {
- t.Errorf("input: %v\nRead() got unexpected error: %v", tc.input, err)
-
+ errorf(t, tc.in, "want#%d: Read() got unexpected error: %v", i, err)
} else if !strings.Contains(err.Error(), want.E) {
- t.Errorf("input: %v\nRead() got %q, want %q", tc.input, err, want.E)
+ errorf(t, tc.in, "want#%d: Read() got %q, want %q", i, err, want.E)
}
- } else {
- if want.E != "" {
- t.Errorf("input: %v\nRead() got nil error, want %q", tc.input, want.E)
- }
+ return
}
- token := value.Type()
- if token != want.T {
- t.Errorf("input: %v\nRead() got %v, want %v", tc.input, token, want.T)
- break
+ if want.E != "" {
+ errorf(t, tc.in, "want#%d: Read() got nil error, want %q", i, want.E)
+ return
}
- checkValue(t, value, i, want)
+ checkToken(t, tok, i, want, tc.in)
+ if !cmp.Equal(tok, peekTok, cmp.Comparer(json.TokenEquals)) {
+ errorf(t, tc.in, "want#%d: Peek() %+v != Read() token %+v", i, peekTok, tok)
+ }
+ if err != peekErr {
+ errorf(t, tc.in, "want#%d: Peek() error %v != Read() error %v", i, err, peekErr)
+ }
}
})
}
}
-func checkValue(t *testing.T, value json.Value, wantIdx int, want R) {
- var got interface{}
- var err error
- switch value.Type() {
- case json.Bool:
- got, err = value.Bool()
- case json.String:
- got = value.String()
- case json.Name:
- got, err = value.Name()
- case json.Number:
- switch want.V.(type) {
- case float32:
- got, err = value.Float(32)
- got = float32(got.(float64))
- case float64:
- got, err = value.Float(64)
- case int32:
- got, err = value.Int(32)
- got = int32(got.(int64))
- case int64:
- got, err = value.Int(64)
- case uint32:
- got, err = value.Uint(32)
- got = uint32(got.(uint64))
- case uint64:
- got, err = value.Uint(64)
+func checkToken(t *testing.T, tok json.Token, idx int, r R, in string) {
+ // Validate Token.Pos() if R.P is set.
+ if r.P > 0 {
+ got := tok.Pos()
+ if got != r.P {
+ errorf(t, in, "want#%d: Token.Pos() got %v want %v", idx, got, r.P)
}
- default:
- return
}
-
- if err != nil {
- if want.VE == "" {
- t.Errorf("want%d: %v got unexpected error: %v", wantIdx, value, err)
- } else if !strings.Contains(err.Error(), want.VE) {
- t.Errorf("want#%d: %v got %q, want %q", wantIdx, value, err, want.VE)
- }
- return
- } else {
- if want.VE != "" {
- t.Errorf("want#%d: %v got nil error, want %q", wantIdx, value, want.VE)
- return
+ // Validate Token.RawString if R.RS is set.
+ if len(r.RS) > 0 {
+ got := tok.RawString()
+ if got != r.RS {
+ errorf(t, in, "want#%d: Token.RawString() got %v want %v", idx, got, r.P)
}
}
- if got != want.V {
- t.Errorf("want#%d: %v got %v, want %v", wantIdx, value, got, want.V)
+ // Skip checking for Token details if r.V is not set.
+ if r.V == nil {
+ return
}
+
+ if err := r.V.check(tok); err != "" {
+ errorf(t, in, "want#%d: %s", idx, err)
+ }
+ return
+}
+
+func errorf(t *testing.T, in string, fmtStr string, args ...interface{}) {
+ t.Helper()
+ vargs := []interface{}{in}
+ for _, arg := range args {
+ vargs = append(vargs, arg)
+ }
+ t.Errorf("input:\n%s\n~end~\n"+fmtStr, vargs...)
}
func TestClone(t *testing.T) {
@@ -1123,7 +1387,7 @@
compareDecoders(t, dec, clone)
// Advance to inner object, clone and compare again.
- dec.Read() // Read StartObject.
+ dec.Read() // Read ObjectOpen.
dec.Read() // Read Name.
clone = dec.Clone()
compareDecoders(t, dec, clone)
@@ -1131,18 +1395,18 @@
func compareDecoders(t *testing.T, d1 *json.Decoder, d2 *json.Decoder) {
for {
- v1, err1 := d1.Read()
- v2, err2 := d2.Read()
- if v1.Type() != v2.Type() {
- t.Errorf("cloned decoder: got Type %v, want %v", v2.Type(), v1.Type())
+ tok1, err1 := d1.Read()
+ tok2, err2 := d2.Read()
+ if tok1.Kind() != tok2.Kind() {
+ t.Errorf("cloned decoder: got Kind %v, want %v", tok2.Kind(), tok1.Kind())
}
- if v1.Raw() != v2.Raw() {
- t.Errorf("cloned decoder: got Raw %v, want %v", v2.Raw(), v1.Raw())
+ if tok1.RawString() != tok2.RawString() {
+ t.Errorf("cloned decoder: got RawString %v, want %v", tok2.RawString(), tok1.RawString())
}
if err1 != err2 {
t.Errorf("cloned decoder: got error %v, want %v", err2, err1)
}
- if v1.Type() == json.EOF {
+ if tok1.Kind() == json.EOF {
break
}
}
diff --git a/internal/encoding/json/decode_token.go b/internal/encoding/json/decode_token.go
new file mode 100644
index 0000000..2eb7023
--- /dev/null
+++ b/internal/encoding/json/decode_token.go
@@ -0,0 +1,193 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package json
+
+import (
+ "bytes"
+ "fmt"
+ "strconv"
+)
+
+// Kind represents a token kind expressible in the JSON format.
+type Kind uint16
+
+const (
+ Invalid Kind = (1 << iota) / 2
+ EOF
+ Null
+ Bool
+ Number
+ String
+ Name
+ ObjectOpen
+ ObjectClose
+ ArrayOpen
+ ArrayClose
+
+ // comma is only for parsing in between tokens and
+ // does not need to be exported.
+ comma
+)
+
+func (k Kind) String() string {
+ switch k {
+ case EOF:
+ return "eof"
+ case Null:
+ return "null"
+ case Bool:
+ return "bool"
+ case Number:
+ return "number"
+ case String:
+ return "string"
+ case ObjectOpen:
+ return "{"
+ case ObjectClose:
+ return "}"
+ case Name:
+ return "name"
+ case ArrayOpen:
+ return "["
+ case ArrayClose:
+ return "]"
+ case comma:
+ return ","
+ }
+ return "<invalid>"
+}
+
+// Token provides a parsed token kind and value.
+//
+// Values are provided by the difference accessor methods. The accessor methods
+// Name, Bool, and ParsedString will panic if called on the wrong kind. There
+// are different accessor methods for the Number kind for converting to the
+// appropriate Go numeric type and those methods have the ok return value.
+type Token struct {
+ // Token kind.
+ kind Kind
+ // pos provides the position of the token in the original input.
+ pos int
+ // raw bytes of the serialized token.
+ // This is a subslice into the original input.
+ raw []byte
+ // boo is parsed boolean value.
+ boo bool
+ // str is parsed string value.
+ str string
+}
+
+// Kind returns the token kind.
+func (t Token) Kind() Kind {
+ return t.kind
+}
+
+// RawString returns the read value in string.
+func (t Token) RawString() string {
+ return string(t.raw)
+}
+
+// Pos returns the token position from the input.
+func (t Token) Pos() int {
+ return t.pos
+}
+
+// Name returns the object name if token is Name, else it will return an error.
+func (t Token) Name() string {
+ if t.kind == Name {
+ return t.str
+ }
+ panic(fmt.Sprintf("Token is not a Name: %v", t.RawString()))
+}
+
+// Bool returns the bool value if token kind is Bool, else it panics.
+func (t Token) Bool() bool {
+ if t.kind == Bool {
+ return t.boo
+ }
+ panic(fmt.Sprintf("Token is not a Bool: %v", t.RawString()))
+}
+
+// ParsedString returns the string value for a JSON string token or the read
+// value in string if token is not a string.
+func (t Token) ParsedString() string {
+ if t.kind == String {
+ return t.str
+ }
+ panic(fmt.Sprintf("Token is not a String: %v", t.RawString()))
+}
+
+// Float returns the floating-point number if token kind is Number.
+//
+// The floating-point precision is specified by the bitSize parameter: 32 for
+// float32 or 64 for float64. If bitSize=32, the result still has type float64,
+// but it will be convertible to float32 without changing its value. It will
+// return false if the number exceeds the floating point limits for given
+// bitSize.
+func (t Token) Float(bitSize int) (float64, bool) {
+ if t.kind != Number {
+ return 0, false
+ }
+ f, err := strconv.ParseFloat(t.RawString(), bitSize)
+ if err != nil {
+ return 0, false
+ }
+ return f, true
+}
+
+// Int returns the signed integer number if token is Number.
+//
+// The given bitSize specifies the integer type that the result must fit into.
+// It returns false if the number is not an integer value or if the result
+// exceeds the limits for given bitSize.
+func (t Token) Int(bitSize int) (int64, bool) {
+ s, ok := t.getIntStr()
+ if !ok {
+ return 0, false
+ }
+ n, err := strconv.ParseInt(s, 10, bitSize)
+ if err != nil {
+ return 0, false
+ }
+ return n, true
+}
+
+// Uint returns the signed integer number if token is Number, else it will
+// return an error.
+//
+// The given bitSize specifies the unsigned integer type that the result must
+// fit into. It returns false if the number is not an unsigned integer value
+// or if the result exceeds the limits for given bitSize.
+func (t Token) Uint(bitSize int) (uint64, bool) {
+ s, ok := t.getIntStr()
+ if !ok {
+ return 0, false
+ }
+ n, err := strconv.ParseUint(s, 10, bitSize)
+ if err != nil {
+ return 0, false
+ }
+ return n, true
+}
+
+func (t Token) getIntStr() (string, bool) {
+ if t.kind != Number {
+ return "", false
+ }
+ parts, ok := parseNumberParts(t.raw)
+ if !ok {
+ return "", false
+ }
+ return normalizeToIntString(parts)
+}
+
+// TokenEquals returns true if given Tokens are equal, else false.
+func TokenEquals(x, y Token) bool {
+ return x.kind == y.kind &&
+ x.pos == y.pos &&
+ bytes.Equal(x.raw, y.raw) &&
+ x.boo == y.boo &&
+ x.str == y.str
+}
diff --git a/internal/encoding/json/encode.go b/internal/encoding/json/encode.go
index 741f34f..17dec31 100644
--- a/internal/encoding/json/encode.go
+++ b/internal/encoding/json/encode.go
@@ -5,18 +5,34 @@
package json
import (
+ "math"
+ "math/bits"
"strconv"
"strings"
+ "unicode/utf8"
"google.golang.org/protobuf/internal/detrand"
"google.golang.org/protobuf/internal/errors"
)
+// kind represents an encoding type.
+type kind uint8
+
+const (
+ _ kind = (1 << iota) / 2
+ name
+ scalar
+ objectOpen
+ objectClose
+ arrayOpen
+ arrayClose
+)
+
// Encoder provides methods to write out JSON constructs and values. The user is
// responsible for producing valid sequences of JSON constructs and values.
type Encoder struct {
indent string
- lastType Type
+ lastKind kind
indents []byte
out []byte
}
@@ -43,13 +59,13 @@
// WriteNull writes out the null value.
func (e *Encoder) WriteNull() {
- e.prepareNext(Null)
+ e.prepareNext(scalar)
e.out = append(e.out, "null"...)
}
// WriteBool writes out the given boolean value.
func (e *Encoder) WriteBool(b bool) {
- e.prepareNext(Bool)
+ e.prepareNext(scalar)
if b {
e.out = append(e.out, "true"...)
} else {
@@ -57,9 +73,10 @@
}
}
-// WriteString writes out the given string in JSON string value.
+// WriteString writes out the given string in JSON string value. Returns error
+// if input string contains invalid UTF-8.
func (e *Encoder) WriteString(s string) error {
- e.prepareNext(String)
+ e.prepareNext(scalar)
var err error
if e.out, err = appendString(e.out, s); err != nil {
return err
@@ -67,42 +84,126 @@
return nil
}
+// Sentinel error used for indicating invalid UTF-8.
+var invalidUTF8Err = errors.New("invalid UTF-8")
+
+func appendString(out []byte, in string) ([]byte, error) {
+ out = append(out, '"')
+ i := indexNeedEscapeInString(in)
+ in, out = in[i:], append(out, in[:i]...)
+ for len(in) > 0 {
+ switch r, n := utf8.DecodeRuneInString(in); {
+ case r == utf8.RuneError && n == 1:
+ return out, invalidUTF8Err
+ case r < ' ' || r == '"' || r == '\\':
+ out = append(out, '\\')
+ switch r {
+ case '"', '\\':
+ out = append(out, byte(r))
+ case '\b':
+ out = append(out, 'b')
+ case '\f':
+ out = append(out, 'f')
+ case '\n':
+ out = append(out, 'n')
+ case '\r':
+ out = append(out, 'r')
+ case '\t':
+ out = append(out, 't')
+ default:
+ out = append(out, 'u')
+ out = append(out, "0000"[1+(bits.Len32(uint32(r))-1)/4:]...)
+ out = strconv.AppendUint(out, uint64(r), 16)
+ }
+ in = in[n:]
+ default:
+ i := indexNeedEscapeInString(in[n:])
+ in, out = in[n+i:], append(out, in[:n+i]...)
+ }
+ }
+ out = append(out, '"')
+ return out, nil
+}
+
+// indexNeedEscapeInString returns the index of the character that needs
+// escaping. If no characters need escaping, this returns the input length.
+func indexNeedEscapeInString(s string) int {
+ for i, r := range s {
+ if r < ' ' || r == '\\' || r == '"' || r == utf8.RuneError {
+ return i
+ }
+ }
+ return len(s)
+}
+
// WriteFloat writes out the given float and bitSize in JSON number value.
func (e *Encoder) WriteFloat(n float64, bitSize int) {
- e.prepareNext(Number)
+ e.prepareNext(scalar)
e.out = appendFloat(e.out, n, bitSize)
}
+// appendFloat formats given float in bitSize, and appends to the given []byte.
+func appendFloat(out []byte, n float64, bitSize int) []byte {
+ switch {
+ case math.IsNaN(n):
+ return append(out, `"NaN"`...)
+ case math.IsInf(n, +1):
+ return append(out, `"Infinity"`...)
+ case math.IsInf(n, -1):
+ return append(out, `"-Infinity"`...)
+ }
+
+ // JSON number formatting logic based on encoding/json.
+ // See floatEncoder.encode for reference.
+ fmt := byte('f')
+ if abs := math.Abs(n); abs != 0 {
+ if bitSize == 64 && (abs < 1e-6 || abs >= 1e21) ||
+ bitSize == 32 && (float32(abs) < 1e-6 || float32(abs) >= 1e21) {
+ fmt = 'e'
+ }
+ }
+ out = strconv.AppendFloat(out, n, fmt, -1, bitSize)
+ if fmt == 'e' {
+ n := len(out)
+ if n >= 4 && out[n-4] == 'e' && out[n-3] == '-' && out[n-2] == '0' {
+ out[n-2] = out[n-1]
+ out = out[:n-1]
+ }
+ }
+ return out
+}
+
// WriteInt writes out the given signed integer in JSON number value.
func (e *Encoder) WriteInt(n int64) {
- e.prepareNext(Number)
+ e.prepareNext(scalar)
e.out = append(e.out, strconv.FormatInt(n, 10)...)
}
// WriteUint writes out the given unsigned integer in JSON number value.
func (e *Encoder) WriteUint(n uint64) {
- e.prepareNext(Number)
+ e.prepareNext(scalar)
e.out = append(e.out, strconv.FormatUint(n, 10)...)
}
// StartObject writes out the '{' symbol.
func (e *Encoder) StartObject() {
- e.prepareNext(StartObject)
+ e.prepareNext(objectOpen)
e.out = append(e.out, '{')
}
// EndObject writes out the '}' symbol.
func (e *Encoder) EndObject() {
- e.prepareNext(EndObject)
+ e.prepareNext(objectClose)
e.out = append(e.out, '}')
}
// WriteName writes out the given string in JSON string value and the name
-// separator ':'.
+// separator ':'. Returns error if input string contains invalid UTF-8, which
+// should not be likely as protobuf field names should be valid.
func (e *Encoder) WriteName(s string) error {
- e.prepareNext(Name)
- // Errors returned by appendString() are non-fatal.
+ e.prepareNext(name)
var err error
+ // Append to output regardless of error.
e.out, err = appendString(e.out, s)
e.out = append(e.out, ':')
return err
@@ -110,28 +211,28 @@
// StartArray writes out the '[' symbol.
func (e *Encoder) StartArray() {
- e.prepareNext(StartArray)
+ e.prepareNext(arrayOpen)
e.out = append(e.out, '[')
}
// EndArray writes out the ']' symbol.
func (e *Encoder) EndArray() {
- e.prepareNext(EndArray)
+ e.prepareNext(arrayClose)
e.out = append(e.out, ']')
}
// prepareNext adds possible comma and indentation for the next value based
-// on last type and indent option. It also updates lastType to next.
-func (e *Encoder) prepareNext(next Type) {
+// on last type and indent option. It also updates lastKind to next.
+func (e *Encoder) prepareNext(next kind) {
defer func() {
- // Set lastType to next.
- e.lastType = next
+ // Set lastKind to next.
+ e.lastKind = next
}()
if len(e.indent) == 0 {
// Need to add comma on the following condition.
- if e.lastType&(Null|Bool|Number|String|EndObject|EndArray) != 0 &&
- next&(Name|Null|Bool|Number|String|StartObject|StartArray) != 0 {
+ if e.lastKind&(scalar|objectClose|arrayClose) != 0 &&
+ next&(name|scalar|objectOpen|arrayOpen) != 0 {
e.out = append(e.out, ',')
// For single-line output, add a random extra space after each
// comma to make output unstable.
@@ -143,28 +244,28 @@
}
switch {
- case e.lastType&(StartObject|StartArray) != 0:
+ case e.lastKind&(objectOpen|arrayOpen) != 0:
// If next type is NOT closing, add indent and newline.
- if next&(EndObject|EndArray) == 0 {
+ if next&(objectClose|arrayClose) == 0 {
e.indents = append(e.indents, e.indent...)
e.out = append(e.out, '\n')
e.out = append(e.out, e.indents...)
}
- case e.lastType&(Null|Bool|Number|String|EndObject|EndArray) != 0:
+ case e.lastKind&(scalar|objectClose|arrayClose) != 0:
switch {
// If next type is either a value or name, add comma and newline.
- case next&(Name|Null|Bool|Number|String|StartObject|StartArray) != 0:
+ case next&(name|scalar|objectOpen|arrayOpen) != 0:
e.out = append(e.out, ',', '\n')
// If next type is a closing object or array, adjust indentation.
- case next&(EndObject|EndArray) != 0:
+ case next&(objectClose|arrayClose) != 0:
e.indents = e.indents[:len(e.indents)-len(e.indent)]
e.out = append(e.out, '\n')
}
e.out = append(e.out, e.indents...)
- case e.lastType&Name != 0:
+ case e.lastKind&name != 0:
e.out = append(e.out, ' ')
// For multi-line output, add a random extra space after key: to make
// output unstable.
diff --git a/internal/encoding/json/string.go b/internal/encoding/json/string.go
deleted file mode 100644
index 0730ffa..0000000
--- a/internal/encoding/json/string.go
+++ /dev/null
@@ -1,140 +0,0 @@
-// Copyright 2018 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package json
-
-import (
- "io"
- "math/bits"
- "strconv"
- "unicode"
- "unicode/utf16"
- "unicode/utf8"
-
- "google.golang.org/protobuf/internal/errors"
- "google.golang.org/protobuf/internal/strs"
-)
-
-func appendString(out []byte, in string) ([]byte, error) {
- out = append(out, '"')
- i := indexNeedEscapeInString(in)
- in, out = in[i:], append(out, in[:i]...)
- for len(in) > 0 {
- switch r, n := utf8.DecodeRuneInString(in); {
- case r == utf8.RuneError && n == 1:
- return out, errors.InvalidUTF8("")
- case r < ' ' || r == '"' || r == '\\':
- out = append(out, '\\')
- switch r {
- case '"', '\\':
- out = append(out, byte(r))
- case '\b':
- out = append(out, 'b')
- case '\f':
- out = append(out, 'f')
- case '\n':
- out = append(out, 'n')
- case '\r':
- out = append(out, 'r')
- case '\t':
- out = append(out, 't')
- default:
- out = append(out, 'u')
- out = append(out, "0000"[1+(bits.Len32(uint32(r))-1)/4:]...)
- out = strconv.AppendUint(out, uint64(r), 16)
- }
- in = in[n:]
- default:
- i := indexNeedEscapeInString(in[n:])
- in, out = in[n+i:], append(out, in[:n+i]...)
- }
- }
- out = append(out, '"')
- return out, nil
-}
-
-func (d *Decoder) parseString(in []byte) (string, int, error) {
- in0 := in
- if len(in) == 0 {
- return "", 0, io.ErrUnexpectedEOF
- }
- if in[0] != '"' {
- return "", 0, d.newSyntaxError("invalid character %q at start of string", in[0])
- }
- in = in[1:]
- i := indexNeedEscapeInBytes(in)
- in, out := in[i:], in[:i:i] // set cap to prevent mutations
- for len(in) > 0 {
- switch r, n := utf8.DecodeRune(in); {
- case r == utf8.RuneError && n == 1:
- return "", 0, d.newSyntaxError("invalid UTF-8 in string")
- case r < ' ':
- return "", 0, d.newSyntaxError("invalid character %q in string", r)
- case r == '"':
- in = in[1:]
- n := len(in0) - len(in)
- return string(out), n, nil
- case r == '\\':
- if len(in) < 2 {
- return "", 0, io.ErrUnexpectedEOF
- }
- switch r := in[1]; r {
- case '"', '\\', '/':
- in, out = in[2:], append(out, r)
- case 'b':
- in, out = in[2:], append(out, '\b')
- case 'f':
- in, out = in[2:], append(out, '\f')
- case 'n':
- in, out = in[2:], append(out, '\n')
- case 'r':
- in, out = in[2:], append(out, '\r')
- case 't':
- in, out = in[2:], append(out, '\t')
- case 'u':
- if len(in) < 6 {
- return "", 0, io.ErrUnexpectedEOF
- }
- v, err := strconv.ParseUint(string(in[2:6]), 16, 16)
- if err != nil {
- return "", 0, d.newSyntaxError("invalid escape code %q in string", in[:6])
- }
- in = in[6:]
-
- r := rune(v)
- if utf16.IsSurrogate(r) {
- if len(in) < 6 {
- return "", 0, io.ErrUnexpectedEOF
- }
- v, err := strconv.ParseUint(string(in[2:6]), 16, 16)
- r = utf16.DecodeRune(r, rune(v))
- if in[0] != '\\' || in[1] != 'u' ||
- r == unicode.ReplacementChar || err != nil {
- return "", 0, d.newSyntaxError("invalid escape code %q in string", in[:6])
- }
- in = in[6:]
- }
- out = append(out, string(r)...)
- default:
- return "", 0, d.newSyntaxError("invalid escape code %q in string", in[:2])
- }
- default:
- i := indexNeedEscapeInBytes(in[n:])
- in, out = in[n+i:], append(out, in[:n+i]...)
- }
- }
- return "", 0, io.ErrUnexpectedEOF
-}
-
-// indexNeedEscapeInString returns the index of the character that needs
-// escaping. If no characters need escaping, this returns the input length.
-func indexNeedEscapeInString(s string) int {
- for i, r := range s {
- if r < ' ' || r == '\\' || r == '"' || r == utf8.RuneError {
- return i
- }
- }
- return len(s)
-}
-func indexNeedEscapeInBytes(b []byte) int { return indexNeedEscapeInString(strs.UnsafeString(b)) }
diff --git a/internal/encoding/json/types.go b/internal/encoding/json/types.go
deleted file mode 100644
index 35feeb7..0000000
--- a/internal/encoding/json/types.go
+++ /dev/null
@@ -1,51 +0,0 @@
-// Copyright 2019 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package json
-
-// Type represents a type expressible in the JSON format.
-type Type uint
-
-const (
- Invalid Type = (1 << iota) / 2
- EOF
- Null
- Bool
- Number
- String
- StartObject
- EndObject
- Name
- StartArray
- EndArray
-
- // comma is only for parsing in between values and should not be exposed.
- comma
-)
-
-func (t Type) String() string {
- switch t {
- case EOF:
- return "eof"
- case Null:
- return "null"
- case Bool:
- return "bool"
- case Number:
- return "number"
- case String:
- return "string"
- case StartObject:
- return "{"
- case EndObject:
- return "}"
- case Name:
- return "name"
- case StartArray:
- return "["
- case EndArray:
- return "]"
- }
- return "<invalid>"
-}
