encoding/jsonpb: add support for unmarshaling Any
Also added json.Decoder.Clone API for unmarshaling Any to look
ahead remaining bytes for @type field.
Change-Id: I2f803743534dfb64f9092d716805b115faa5975a
Reviewed-on: https://go-review.googlesource.com/c/protobuf/+/170102
Reviewed-by: Joe Tsai <thebrokentoaster@gmail.com>
diff --git a/encoding/jsonpb/decode.go b/encoding/jsonpb/decode.go
index 816ba0a..8511a8d 100644
--- a/encoding/jsonpb/decode.go
+++ b/encoding/jsonpb/decode.go
@@ -59,7 +59,7 @@
o.decoder = json.NewDecoder(b)
var nerr errors.NonFatal
- if err := o.unmarshalMessage(mr); !nerr.Merge(err) {
+ if err := o.unmarshalMessage(mr, false); !nerr.Merge(err) {
return err
}
@@ -126,7 +126,7 @@
}
// unmarshalMessage unmarshals a message into the given protoreflect.Message.
-func (o UnmarshalOptions) unmarshalMessage(m pref.Message) error {
+func (o UnmarshalOptions) unmarshalMessage(m pref.Message, skipTypeURL bool) error {
var nerr errors.NonFatal
if isCustomType(m.Type().FullName()) {
@@ -141,7 +141,7 @@
return unexpectedJSONError{jval}
}
- if err := o.unmarshalFields(m); !nerr.Merge(err) {
+ if err := o.unmarshalFields(m, skipTypeURL); !nerr.Merge(err) {
return err
}
@@ -149,7 +149,7 @@
}
// unmarshalFields unmarshals the fields into the given protoreflect.Message.
-func (o UnmarshalOptions) unmarshalFields(m pref.Message) error {
+func (o UnmarshalOptions) unmarshalFields(m pref.Message, skipTypeURL bool) error {
var nerr errors.NonFatal
var reqNums set.Ints
var seenNums set.Ints
@@ -179,6 +179,13 @@
if !nerr.Merge(err) {
return err
}
+ // Unmarshaling a non-custom embedded message in Any will contain the
+ // JSON field "@type" which should be skipped because it is not a field
+ // of the embedded message, but simply an artifact of the Any format.
+ if skipTypeURL && name == "@type" {
+ o.decoder.Read()
+ continue
+ }
// Get the FieldDescriptor.
var fd pref.FieldDescriptor
@@ -280,7 +287,7 @@
switch fd.Kind() {
case pref.MessageKind, pref.GroupKind:
m := knownFields.NewMessage(num)
- err = o.unmarshalMessage(m)
+ err = o.unmarshalMessage(m, false)
val = pref.ValueOf(m)
default:
val, err = o.unmarshalScalar(fd)
@@ -539,7 +546,7 @@
case pref.MessageKind, pref.GroupKind:
for {
m := list.NewMessage()
- err := o.unmarshalMessage(m)
+ err := o.unmarshalMessage(m, false)
if !nerr.Merge(err) {
if e, ok := err.(unexpectedJSONError); ok {
if e.value.Type() == json.EndArray {
@@ -596,7 +603,7 @@
unmarshalMapValue = func() (pref.Value, error) {
var nerr errors.NonFatal
m := mmap.NewMessage()
- if err := o.unmarshalMessage(m); !nerr.Merge(err) {
+ if err := o.unmarshalMessage(m, false); !nerr.Merge(err) {
return pref.Value{}, err
}
return pref.ValueOf(m), nerr.E
diff --git a/encoding/jsonpb/decode_test.go b/encoding/jsonpb/decode_test.go
index a1ad766..1e9bd98 100644
--- a/encoding/jsonpb/decode_test.go
+++ b/encoding/jsonpb/decode_test.go
@@ -1866,6 +1866,535 @@
},
},
},
+ }, {
+ desc: "Any empty",
+ inputMessage: &knownpb.Any{},
+ inputText: `{}`,
+ wantMessage: &knownpb.Any{},
+ }, {
+ desc: "Any with non-custom message",
+ umo: jsonpb.UnmarshalOptions{
+ Resolver: preg.NewTypes((&pb2.Nested{}).ProtoReflect().Type()),
+ },
+ inputMessage: &knownpb.Any{},
+ inputText: `{
+ "@type": "foo/pb2.Nested",
+ "optString": "embedded inside Any",
+ "optNested": {
+ "optString": "inception"
+ }
+}`,
+ wantMessage: func() proto.Message {
+ m := &pb2.Nested{
+ OptString: scalar.String("embedded inside Any"),
+ OptNested: &pb2.Nested{
+ OptString: scalar.String("inception"),
+ },
+ }
+ b, err := proto.MarshalOptions{Deterministic: true}.Marshal(m)
+ if err != nil {
+ t.Fatalf("error in binary marshaling message for Any.value: %v", err)
+ }
+ return &knownpb.Any{
+ TypeUrl: "foo/pb2.Nested",
+ Value: b,
+ }
+ }(),
+ }, {
+ desc: "Any with empty embedded message",
+ umo: jsonpb.UnmarshalOptions{
+ Resolver: preg.NewTypes((&pb2.Nested{}).ProtoReflect().Type()),
+ },
+ inputMessage: &knownpb.Any{},
+ inputText: `{
+ "@type": "foo/pb2.Nested"
+}`,
+ wantMessage: &knownpb.Any{TypeUrl: "foo/pb2.Nested"},
+ }, {
+ desc: "Any without registered type",
+ umo: jsonpb.UnmarshalOptions{Resolver: preg.NewTypes()},
+ inputMessage: &knownpb.Any{},
+ inputText: `{
+ "@type": "foo/pb2.Nested"
+}`,
+ wantErr: true,
+ }, {
+ desc: "Any with missing required error",
+ umo: jsonpb.UnmarshalOptions{
+ Resolver: preg.NewTypes((&pb2.PartialRequired{}).ProtoReflect().Type()),
+ },
+ inputMessage: &knownpb.Any{},
+ inputText: `{
+ "@type": "pb2.PartialRequired",
+ "optString": "embedded inside Any"
+}`,
+ wantMessage: func() proto.Message {
+ m := &pb2.PartialRequired{
+ OptString: scalar.String("embedded inside Any"),
+ }
+ b, err := proto.MarshalOptions{Deterministic: true}.Marshal(m)
+ // TODO: Marshal may fail due to required field not set at some
+ // point. Need to ignore required not set error here.
+ if err != nil {
+ t.Fatalf("error in binary marshaling message for Any.value: %v", err)
+ }
+ return &knownpb.Any{
+ TypeUrl: string(m.ProtoReflect().Type().FullName()),
+ Value: b,
+ }
+ }(),
+ wantErr: true,
+ }, {
+ desc: "Any with partial required and AllowPartial",
+ umo: jsonpb.UnmarshalOptions{
+ AllowPartial: true,
+ Resolver: preg.NewTypes((&pb2.PartialRequired{}).ProtoReflect().Type()),
+ },
+ inputMessage: &knownpb.Any{},
+ inputText: `{
+ "@type": "pb2.PartialRequired",
+ "optString": "embedded inside Any"
+}`,
+ wantMessage: func() proto.Message {
+ m := &pb2.PartialRequired{
+ OptString: scalar.String("embedded inside Any"),
+ }
+ b, err := proto.MarshalOptions{Deterministic: true}.Marshal(m)
+ // TODO: Marshal may fail due to required field not set at some
+ // point. Need to ignore required not set error here.
+ if err != nil {
+ t.Fatalf("error in binary marshaling message for Any.value: %v", err)
+ }
+ return &knownpb.Any{
+ TypeUrl: string(m.ProtoReflect().Type().FullName()),
+ Value: b,
+ }
+ }(),
+ }, {
+ desc: "Any with invalid UTF8",
+ umo: jsonpb.UnmarshalOptions{
+ Resolver: preg.NewTypes((&pb2.Nested{}).ProtoReflect().Type()),
+ },
+ inputMessage: &knownpb.Any{},
+ inputText: `{
+ "optString": "` + "abc\xff" + `",
+ "@type": "foo/pb2.Nested"
+}`,
+ wantMessage: func() proto.Message {
+ m := &pb2.Nested{
+ OptString: scalar.String("abc\xff"),
+ }
+ b, err := proto.MarshalOptions{Deterministic: true}.Marshal(m)
+ if err != nil {
+ t.Fatalf("error in binary marshaling message for Any.value: %v", err)
+ }
+ return &knownpb.Any{
+ TypeUrl: "foo/pb2.Nested",
+ Value: b,
+ }
+ }(),
+ wantErr: true,
+ }, {
+ desc: "Any with BoolValue",
+ umo: jsonpb.UnmarshalOptions{
+ Resolver: preg.NewTypes((&knownpb.BoolValue{}).ProtoReflect().Type()),
+ },
+ inputMessage: &knownpb.Any{},
+ inputText: `{
+ "@type": "type.googleapis.com/google.protobuf.BoolValue",
+ "value": true
+}`,
+ wantMessage: func() proto.Message {
+ m := &knownpb.BoolValue{Value: true}
+ b, err := proto.MarshalOptions{Deterministic: true}.Marshal(m)
+ if err != nil {
+ t.Fatalf("error in binary marshaling message for Any.value: %v", err)
+ }
+ return &knownpb.Any{
+ TypeUrl: "type.googleapis.com/google.protobuf.BoolValue",
+ Value: b,
+ }
+ }(),
+ }, {
+ desc: "Any with Empty",
+ umo: jsonpb.UnmarshalOptions{
+ Resolver: preg.NewTypes((&knownpb.Empty{}).ProtoReflect().Type()),
+ },
+ inputMessage: &knownpb.Any{},
+ inputText: `{
+ "value": {},
+ "@type": "type.googleapis.com/google.protobuf.Empty"
+}`,
+ wantMessage: func() proto.Message {
+ m := &knownpb.Empty{}
+ b, err := proto.MarshalOptions{Deterministic: true}.Marshal(m)
+ if err != nil {
+ t.Fatalf("error in binary marshaling message for Any.value: %v", err)
+ }
+ return &knownpb.Any{
+ TypeUrl: "type.googleapis.com/google.protobuf.Empty",
+ Value: b,
+ }
+ }(),
+ }, {
+ desc: "Any with missing Empty",
+ umo: jsonpb.UnmarshalOptions{
+ Resolver: preg.NewTypes((&knownpb.Empty{}).ProtoReflect().Type()),
+ },
+ inputMessage: &knownpb.Any{},
+ inputText: `{
+ "@type": "type.googleapis.com/google.protobuf.Empty"
+}`,
+ wantErr: true,
+ }, {
+ desc: "Any with StringValue containing invalid UTF8",
+ umo: jsonpb.UnmarshalOptions{
+ Resolver: preg.NewTypes((&knownpb.StringValue{}).ProtoReflect().Type()),
+ },
+ inputMessage: &knownpb.Any{},
+ inputText: `{
+ "@type": "google.protobuf.StringValue",
+ "value": "` + "abc\xff" + `"
+}`,
+ wantMessage: func() proto.Message {
+ m := &knownpb.StringValue{Value: "abc\xff"}
+ b, err := proto.MarshalOptions{Deterministic: true}.Marshal(m)
+ if err != nil {
+ t.Fatalf("error in binary marshaling message for Any.value: %v", err)
+ }
+ return &knownpb.Any{
+ TypeUrl: "google.protobuf.StringValue",
+ Value: b,
+ }
+ }(),
+ wantErr: true,
+ }, {
+ desc: "Any with Int64Value",
+ umo: jsonpb.UnmarshalOptions{
+ Resolver: preg.NewTypes((&knownpb.Int64Value{}).ProtoReflect().Type()),
+ },
+ inputMessage: &knownpb.Any{},
+ inputText: `{
+ "@type": "google.protobuf.Int64Value",
+ "value": "42"
+}`,
+ wantMessage: func() proto.Message {
+ m := &knownpb.Int64Value{Value: 42}
+ b, err := proto.MarshalOptions{Deterministic: true}.Marshal(m)
+ if err != nil {
+ t.Fatalf("error in binary marshaling message for Any.value: %v", err)
+ }
+ return &knownpb.Any{
+ TypeUrl: "google.protobuf.Int64Value",
+ Value: b,
+ }
+ }(),
+ }, {
+ desc: "Any with invalid Int64Value",
+ umo: jsonpb.UnmarshalOptions{
+ Resolver: preg.NewTypes((&knownpb.Int64Value{}).ProtoReflect().Type()),
+ },
+ inputMessage: &knownpb.Any{},
+ inputText: `{
+ "@type": "google.protobuf.Int64Value",
+ "value": "forty-two"
+}`,
+ wantErr: true,
+ }, {
+ desc: "Any with invalid UInt64Value",
+ umo: jsonpb.UnmarshalOptions{
+ Resolver: preg.NewTypes((&knownpb.UInt64Value{}).ProtoReflect().Type()),
+ },
+ inputMessage: &knownpb.Any{},
+ inputText: `{
+ "@type": "google.protobuf.UInt64Value",
+ "value": -42
+}`,
+ wantErr: true,
+ }, {
+ desc: "Any with Duration",
+ umo: jsonpb.UnmarshalOptions{
+ Resolver: preg.NewTypes((&knownpb.Duration{}).ProtoReflect().Type()),
+ },
+ inputMessage: &knownpb.Any{},
+ inputText: `{
+ "@type": "type.googleapis.com/google.protobuf.Duration",
+ "value": "0s"
+}`,
+ wantMessage: func() proto.Message {
+ m := &knownpb.Duration{}
+ b, err := proto.MarshalOptions{Deterministic: true}.Marshal(m)
+ if err != nil {
+ t.Fatalf("error in binary marshaling message for Any.value: %v", err)
+ }
+ return &knownpb.Any{
+ TypeUrl: "type.googleapis.com/google.protobuf.Duration",
+ Value: b,
+ }
+ }(),
+ }, {
+ desc: "Any with Value of StringValue",
+ umo: jsonpb.UnmarshalOptions{
+ Resolver: preg.NewTypes((&knownpb.Value{}).ProtoReflect().Type()),
+ },
+ inputMessage: &knownpb.Any{},
+ inputText: `{
+ "@type": "google.protobuf.Value",
+ "value": "` + "abc\xff" + `"
+}`,
+ wantMessage: func() proto.Message {
+ m := &knownpb.Value{Kind: &knownpb.Value_StringValue{"abc\xff"}}
+ b, err := proto.MarshalOptions{Deterministic: true}.Marshal(m)
+ if err != nil {
+ t.Fatalf("error in binary marshaling message for Any.value: %v", err)
+ }
+ return &knownpb.Any{
+ TypeUrl: "google.protobuf.Value",
+ Value: b,
+ }
+ }(),
+ wantErr: true,
+ }, {
+ desc: "Any with Value of NullValue",
+ umo: jsonpb.UnmarshalOptions{
+ Resolver: preg.NewTypes((&knownpb.Value{}).ProtoReflect().Type()),
+ },
+ inputMessage: &knownpb.Any{},
+ inputText: `{
+ "@type": "google.protobuf.Value",
+ "value": null
+}`,
+ wantMessage: func() proto.Message {
+ m := &knownpb.Value{Kind: &knownpb.Value_NullValue{}}
+ b, err := proto.MarshalOptions{Deterministic: true}.Marshal(m)
+ if err != nil {
+ t.Fatalf("error in binary marshaling message for Any.value: %v", err)
+ }
+ return &knownpb.Any{
+ TypeUrl: "google.protobuf.Value",
+ Value: b,
+ }
+ }(),
+ }, {
+ desc: "Any with Struct",
+ umo: jsonpb.UnmarshalOptions{
+ Resolver: preg.NewTypes(
+ (&knownpb.Struct{}).ProtoReflect().Type(),
+ (&knownpb.Value{}).ProtoReflect().Type(),
+ (&knownpb.BoolValue{}).ProtoReflect().Type(),
+ knownpb.NullValue_NULL_VALUE.Type(),
+ (&knownpb.StringValue{}).ProtoReflect().Type(),
+ ),
+ },
+ inputMessage: &knownpb.Any{},
+ inputText: `{
+ "@type": "google.protobuf.Struct",
+ "value": {
+ "bool": true,
+ "null": null,
+ "string": "hello",
+ "struct": {
+ "string": "world"
+ }
+ }
+}`,
+ wantMessage: func() proto.Message {
+ m := &knownpb.Struct{
+ Fields: map[string]*knownpb.Value{
+ "bool": {Kind: &knownpb.Value_BoolValue{true}},
+ "null": {Kind: &knownpb.Value_NullValue{}},
+ "string": {Kind: &knownpb.Value_StringValue{"hello"}},
+ "struct": {
+ Kind: &knownpb.Value_StructValue{
+ &knownpb.Struct{
+ Fields: map[string]*knownpb.Value{
+ "string": {Kind: &knownpb.Value_StringValue{"world"}},
+ },
+ },
+ },
+ },
+ },
+ }
+ b, err := proto.MarshalOptions{Deterministic: true}.Marshal(m)
+ if err != nil {
+ t.Fatalf("error in binary marshaling message for Any.value: %v", err)
+ }
+ return &knownpb.Any{
+ TypeUrl: "google.protobuf.Struct",
+ Value: b,
+ }
+ }(),
+ }, {
+ desc: "Any with missing @type",
+ umo: jsonpb.UnmarshalOptions{},
+ inputMessage: &knownpb.Any{},
+ inputText: `{
+ "value": {}
+}`,
+ wantErr: true,
+ }, {
+ desc: "Any with empty @type",
+ inputMessage: &knownpb.Any{},
+ inputText: `{
+ "@type": ""
+}`,
+ wantErr: true,
+ }, {
+ desc: "Any with duplicate @type",
+ umo: jsonpb.UnmarshalOptions{
+ Resolver: preg.NewTypes(
+ (&pb2.Nested{}).ProtoReflect().Type(),
+ (&knownpb.StringValue{}).ProtoReflect().Type(),
+ ),
+ },
+ inputMessage: &knownpb.Any{},
+ inputText: `{
+ "@type": "google.protobuf.StringValue",
+ "value": "hello",
+ "@type": "pb2.Nested"
+}`,
+ wantErr: true,
+ }, {
+ desc: "Any with duplicate value",
+ umo: jsonpb.UnmarshalOptions{
+ Resolver: preg.NewTypes((&knownpb.StringValue{}).ProtoReflect().Type()),
+ },
+ inputMessage: &knownpb.Any{},
+ inputText: `{
+ "@type": "google.protobuf.StringValue",
+ "value": "hello",
+ "value": "world"
+}`,
+ wantErr: true,
+ }, {
+ desc: "Any with unknown field",
+ umo: jsonpb.UnmarshalOptions{
+ Resolver: preg.NewTypes((&pb2.Nested{}).ProtoReflect().Type()),
+ },
+ inputMessage: &knownpb.Any{},
+ inputText: `{
+ "@type": "pb2.Nested",
+ "optString": "hello",
+ "unknown": "world"
+}`,
+ wantErr: true,
+ }, {
+ desc: "Any with embedded type containing Any",
+ umo: jsonpb.UnmarshalOptions{
+ Resolver: preg.NewTypes(
+ (&pb2.KnownTypes{}).ProtoReflect().Type(),
+ (&knownpb.Any{}).ProtoReflect().Type(),
+ (&knownpb.StringValue{}).ProtoReflect().Type(),
+ ),
+ },
+ inputMessage: &knownpb.Any{},
+ inputText: `{
+ "@type": "pb2.KnownTypes",
+ "optAny": {
+ "@type": "google.protobuf.StringValue",
+ "value": "` + "abc\xff" + `"
+ }
+}`,
+ wantMessage: func() proto.Message {
+ m1 := &knownpb.StringValue{Value: "abc\xff"}
+ b, err := proto.MarshalOptions{Deterministic: true}.Marshal(m1)
+ if err != nil {
+ t.Fatalf("error in binary marshaling message for Any.value: %v", err)
+ }
+ m2 := &knownpb.Any{
+ TypeUrl: "google.protobuf.StringValue",
+ Value: b,
+ }
+ m3 := &pb2.KnownTypes{OptAny: m2}
+ b, err = proto.MarshalOptions{Deterministic: true}.Marshal(m3)
+ if err != nil {
+ t.Fatalf("error in binary marshaling message for Any.value: %v", err)
+ }
+ return &knownpb.Any{
+ TypeUrl: "pb2.KnownTypes",
+ Value: b,
+ }
+ }(),
+ wantErr: true,
+ }, {
+ desc: "well known types as field values",
+ umo: jsonpb.UnmarshalOptions{
+ Resolver: preg.NewTypes((&knownpb.Empty{}).ProtoReflect().Type()),
+ },
+ inputMessage: &pb2.KnownTypes{},
+ inputText: `{
+ "optBool": false,
+ "optInt32": 42,
+ "optInt64": "42",
+ "optUint32": 42,
+ "optUint64": "42",
+ "optFloat": 1.23,
+ "optDouble": 3.1415,
+ "optString": "hello",
+ "optBytes": "aGVsbG8=",
+ "optDuration": "123s",
+ "optTimestamp": "2019-03-19T23:03:21Z",
+ "optStruct": {
+ "string": "hello"
+ },
+ "optList": [
+ null,
+ "",
+ {},
+ []
+ ],
+ "optValue": "world",
+ "optEmpty": {},
+ "optAny": {
+ "@type": "google.protobuf.Empty",
+ "value": {}
+ },
+ "optFieldmask": "fooBar,barFoo"
+}`,
+ wantMessage: &pb2.KnownTypes{
+ OptBool: &knownpb.BoolValue{Value: false},
+ OptInt32: &knownpb.Int32Value{Value: 42},
+ OptInt64: &knownpb.Int64Value{Value: 42},
+ OptUint32: &knownpb.UInt32Value{Value: 42},
+ OptUint64: &knownpb.UInt64Value{Value: 42},
+ OptFloat: &knownpb.FloatValue{Value: 1.23},
+ OptDouble: &knownpb.DoubleValue{Value: 3.1415},
+ OptString: &knownpb.StringValue{Value: "hello"},
+ OptBytes: &knownpb.BytesValue{Value: []byte("hello")},
+ OptDuration: &knownpb.Duration{Seconds: 123},
+ OptTimestamp: &knownpb.Timestamp{Seconds: 1553036601},
+ OptStruct: &knownpb.Struct{
+ Fields: map[string]*knownpb.Value{
+ "string": {Kind: &knownpb.Value_StringValue{"hello"}},
+ },
+ },
+ OptList: &knownpb.ListValue{
+ Values: []*knownpb.Value{
+ {Kind: &knownpb.Value_NullValue{}},
+ {Kind: &knownpb.Value_StringValue{}},
+ {
+ Kind: &knownpb.Value_StructValue{
+ &knownpb.Struct{Fields: map[string]*knownpb.Value{}},
+ },
+ },
+ {
+ Kind: &knownpb.Value_ListValue{
+ &knownpb.ListValue{Values: []*knownpb.Value{}},
+ },
+ },
+ },
+ },
+ OptValue: &knownpb.Value{
+ Kind: &knownpb.Value_StringValue{"world"},
+ },
+ OptEmpty: &knownpb.Empty{},
+ OptAny: &knownpb.Any{
+ TypeUrl: "google.protobuf.Empty",
+ },
+ OptFieldmask: &knownpb.FieldMask{
+ Paths: []string{"foo_bar", "bar_foo"},
+ },
+ },
}}
for _, tt := range tests {
diff --git a/encoding/jsonpb/encode_test.go b/encoding/jsonpb/encode_test.go
index e9bf571..005de06 100644
--- a/encoding/jsonpb/encode_test.go
+++ b/encoding/jsonpb/encode_test.go
@@ -1509,7 +1509,7 @@
input: &knownpb.Any{},
want: `{}`,
}, {
- desc: "Any",
+ desc: "Any with non-custom message",
mo: jsonpb.MarshalOptions{
Resolver: preg.NewTypes((&pb2.Nested{}).ProtoReflect().Type()),
},
@@ -1537,7 +1537,7 @@
}
}`,
}, {
- desc: "Any without value",
+ desc: "Any with empty embedded message",
mo: jsonpb.MarshalOptions{
Resolver: preg.NewTypes((&pb2.Nested{}).ProtoReflect().Type()),
},
@@ -1546,11 +1546,9 @@
"@type": "foo/pb2.Nested"
}`,
}, {
- desc: "Any without registered type",
- mo: jsonpb.MarshalOptions{Resolver: preg.NewTypes()},
- input: func() proto.Message {
- return &knownpb.Any{TypeUrl: "foo/pb2.Nested"}
- }(),
+ desc: "Any without registered type",
+ mo: jsonpb.MarshalOptions{Resolver: preg.NewTypes()},
+ input: &knownpb.Any{TypeUrl: "foo/pb2.Nested"},
wantErr: true,
}, {
desc: "Any with missing required error",
@@ -1578,6 +1576,31 @@
}`,
wantErr: true,
}, {
+ desc: "Any with partial required and AllowPartial",
+ mo: jsonpb.MarshalOptions{
+ AllowPartial: true,
+ Resolver: preg.NewTypes((&pb2.PartialRequired{}).ProtoReflect().Type()),
+ },
+ input: func() proto.Message {
+ m := &pb2.PartialRequired{
+ OptString: scalar.String("embedded inside Any"),
+ }
+ b, err := proto.MarshalOptions{Deterministic: true}.Marshal(m)
+ // TODO: Marshal may fail due to required field not set at some
+ // point. Need to ignore required not set error here.
+ if err != nil {
+ t.Fatalf("error in binary marshaling message for Any.value: %v", err)
+ }
+ return &knownpb.Any{
+ TypeUrl: string(m.ProtoReflect().Type().FullName()),
+ Value: b,
+ }
+ }(),
+ want: `{
+ "@type": "pb2.PartialRequired",
+ "optString": "embedded inside Any"
+}`,
+ }, {
desc: "Any with invalid UTF8",
mo: jsonpb.MarshalOptions{
Resolver: preg.NewTypes((&pb2.Nested{}).ProtoReflect().Type()),
@@ -1672,6 +1695,63 @@
}`,
wantErr: true,
}, {
+ desc: "Any with Int64Value",
+ mo: jsonpb.MarshalOptions{
+ Resolver: preg.NewTypes((&knownpb.Int64Value{}).ProtoReflect().Type()),
+ },
+ input: func() proto.Message {
+ m := &knownpb.Int64Value{Value: 42}
+ b, err := proto.MarshalOptions{Deterministic: true}.Marshal(m)
+ if err != nil {
+ t.Fatalf("error in binary marshaling message for Any.value: %v", err)
+ }
+ return &knownpb.Any{
+ TypeUrl: "google.protobuf.Int64Value",
+ Value: b,
+ }
+ }(),
+ want: `{
+ "@type": "google.protobuf.Int64Value",
+ "value": "42"
+}`,
+ }, {
+ desc: "Any with Duration",
+ mo: jsonpb.MarshalOptions{
+ Resolver: preg.NewTypes((&knownpb.Duration{}).ProtoReflect().Type()),
+ },
+ input: func() proto.Message {
+ m := &knownpb.Duration{}
+ b, err := proto.MarshalOptions{Deterministic: true}.Marshal(m)
+ if err != nil {
+ t.Fatalf("error in binary marshaling message for Any.value: %v", err)
+ }
+ return &knownpb.Any{
+ TypeUrl: "type.googleapis.com/google.protobuf.Duration",
+ Value: b,
+ }
+ }(),
+ want: `{
+ "@type": "type.googleapis.com/google.protobuf.Duration",
+ "value": "0s"
+}`,
+ }, {
+ desc: "Any with empty Value",
+ mo: jsonpb.MarshalOptions{
+ Resolver: preg.NewTypes((&knownpb.Value{}).ProtoReflect().Type()),
+ },
+ input: func() proto.Message {
+ m := &knownpb.Value{}
+ b, err := proto.Marshal(m)
+ if err != nil {
+ t.Fatalf("error in binary marshaling message for Any.value: %v", err)
+ }
+ return &knownpb.Any{
+ TypeUrl: "type.googleapis.com/google.protobuf.Value",
+ Value: b,
+ }
+ }(),
+ wantErr: true,
+ }, {
desc: "Any with Value of StringValue",
mo: jsonpb.MarshalOptions{
Resolver: preg.NewTypes((&knownpb.Value{}).ProtoReflect().Type()),
@@ -1693,13 +1773,13 @@
}`,
wantErr: true,
}, {
- desc: "Any with empty Value",
+ desc: "Any with Value of NullValue",
mo: jsonpb.MarshalOptions{
Resolver: preg.NewTypes((&knownpb.Value{}).ProtoReflect().Type()),
},
input: func() proto.Message {
- m := &knownpb.Value{}
- b, err := proto.Marshal(m)
+ m := &knownpb.Value{Kind: &knownpb.Value_NullValue{}}
+ b, err := proto.MarshalOptions{Deterministic: true}.Marshal(m)
if err != nil {
t.Fatalf("error in binary marshaling message for Any.value: %v", err)
}
@@ -1708,26 +1788,9 @@
Value: b,
}
}(),
- wantErr: true,
- }, {
- desc: "Any with Duration",
- mo: jsonpb.MarshalOptions{
- Resolver: preg.NewTypes((&knownpb.Duration{}).ProtoReflect().Type()),
- },
- input: func() proto.Message {
- m := &knownpb.Duration{}
- b, err := proto.MarshalOptions{Deterministic: true}.Marshal(m)
- if err != nil {
- t.Fatalf("error in binary marshaling message for Any.value: %v", err)
- }
- return &knownpb.Any{
- TypeUrl: "type.googleapis.com/google.protobuf.Duration",
- Value: b,
- }
- }(),
want: `{
- "@type": "type.googleapis.com/google.protobuf.Duration",
- "value": "0s"
+ "@type": "type.googleapis.com/google.protobuf.Value",
+ "value": null
}`,
}, {
desc: "Any with Struct",
@@ -1778,6 +1841,22 @@
}
}`,
}, {
+ desc: "Any with missing type_url",
+ mo: jsonpb.MarshalOptions{
+ Resolver: preg.NewTypes((&knownpb.BoolValue{}).ProtoReflect().Type()),
+ },
+ input: func() proto.Message {
+ m := &knownpb.BoolValue{Value: true}
+ b, err := proto.MarshalOptions{Deterministic: true}.Marshal(m)
+ if err != nil {
+ t.Fatalf("error in binary marshaling message for Any.value: %v", err)
+ }
+ return &knownpb.Any{
+ Value: b,
+ }
+ }(),
+ wantErr: true,
+ }, {
desc: "well known types as field values",
mo: jsonpb.MarshalOptions{
Resolver: preg.NewTypes((&knownpb.Empty{}).ProtoReflect().Type()),
diff --git a/encoding/jsonpb/well_known_types.go b/encoding/jsonpb/well_known_types.go
index 091998d..db1e365 100644
--- a/encoding/jsonpb/well_known_types.go
+++ b/encoding/jsonpb/well_known_types.go
@@ -98,7 +98,7 @@
name := m.Type().FullName()
switch name {
case "google.protobuf.Any":
- panic("unmarshaling of google.protobuf.Any is not implemented yet")
+ return o.unmarshalAny(m)
case "google.protobuf.BoolValue",
"google.protobuf.DoubleValue",
@@ -138,12 +138,11 @@
// The JSON representation of an Any message uses the regular representation of
// the deserialized, embedded message, with an additional field `@type` which
-// contains the type URL. If the embedded message type is well-known and has a
+// contains the type URL. If the embedded message type is well-known and has a
// custom JSON representation, that representation will be embedded adding a
// field `value` which holds the custom JSON in addition to the `@type` field.
func (o MarshalOptions) marshalAny(m pref.Message) error {
- var nerr errors.NonFatal
msgType := m.Type()
knownFields := m.KnownFields()
@@ -167,6 +166,7 @@
// Marshal out @type field.
typeURL := typeVal.String()
o.encoder.WriteName("@type")
+ var nerr errors.NonFatal
if err := o.encoder.WriteString(typeURL); !nerr.Merge(err) {
return err
}
@@ -200,6 +200,225 @@
return nerr.E
}
+func (o UnmarshalOptions) unmarshalAny(m pref.Message) error {
+ // Use Peek to check for json.StartObject to avoid advancing a read.
+ if o.decoder.Peek() != json.StartObject {
+ jval, _ := o.decoder.Read()
+ return unexpectedJSONError{jval}
+ }
+
+ // Use another json.Decoder to parse the unread bytes from o.decoder for
+ // @type field. This avoids advancing a read from o.decoder because the
+ // current JSON object may contain the fields of the embedded type.
+ dec := o.decoder.Clone()
+ typeURL, err := findTypeURL(dec)
+ if err == errEmptyObject {
+ // An empty JSON object translates to an empty Any message.
+ o.decoder.Read() // Read json.StartObject.
+ o.decoder.Read() // Read json.EndObject.
+ return nil
+ }
+ var nerr errors.NonFatal
+ if !nerr.Merge(err) {
+ return errors.New("google.protobuf.Any: %v", err)
+ }
+
+ emt, err := o.Resolver.FindMessageByURL(typeURL)
+ if err != nil {
+ return errors.New("google.protobuf.Any: unable to resolve type %q: %v", typeURL, err)
+ }
+
+ // Create new message for the embedded message type and unmarshal into it.
+ em := emt.New()
+ if isCustomType(emt.FullName()) {
+ // If embedded message is a custom type, unmarshal the JSON "value" field
+ // into it.
+ if err := o.unmarshalAnyValue(em); !nerr.Merge(err) {
+ return errors.New("google.protobuf.Any: %v", err)
+ }
+ } else {
+ // Else unmarshal the current JSON object into it.
+ if err := o.unmarshalMessage(em, true); !nerr.Merge(err) {
+ return errors.New("google.protobuf.Any: %v", err)
+ }
+ }
+ // Serialize the embedded message and assign the resulting bytes to the
+ // proto value field.
+ b, err := proto.MarshalOptions{Deterministic: true}.Marshal(em.Interface())
+ if !nerr.Merge(err) {
+ return errors.New("google.protobuf.Any: %v", err)
+ }
+
+ knownFields := m.KnownFields()
+ knownFields.Set(fieldnum.Any_TypeUrl, pref.ValueOf(typeURL))
+ knownFields.Set(fieldnum.Any_Value, pref.ValueOf(b))
+ return nerr.E
+}
+
+var errEmptyObject = errors.New(`empty object`)
+
+// findTypeURL returns the "@type" field value from the given JSON bytes. It is
+// expected that the given bytes start with json.StartObject. It returns
+// errEmptyObject if the JSON object is empty. It returns error if the object
+// does not contain the field or other decoding problems.
+func findTypeURL(dec *json.Decoder) (string, error) {
+ var typeURL string
+ var nerr errors.NonFatal
+ numFields := 0
+ // Skip start object.
+ dec.Read()
+
+Loop:
+ for {
+ jval, err := dec.Read()
+ if !nerr.Merge(err) {
+ return "", err
+ }
+
+ switch jval.Type() {
+ case json.EndObject:
+ if typeURL == "" {
+ // Did not find @type field.
+ if numFields > 0 {
+ return "", errors.New(`missing "@type" field`)
+ }
+ return "", errEmptyObject
+ }
+ break Loop
+
+ case json.Name:
+ numFields++
+ name, err := jval.Name()
+ if !nerr.Merge(err) {
+ return "", err
+ }
+ if name != "@type" {
+ // Skip value.
+ if err := skipJSONValue(dec); err != nil {
+ return "", err
+ }
+ continue
+ }
+
+ // Return error if this was previously set already.
+ if typeURL != "" {
+ return "", errors.New(`duplicate "@type" field`)
+ }
+ // Read field value.
+ jval, err := dec.Read()
+ if !nerr.Merge(err) {
+ return "", err
+ }
+ if jval.Type() != json.String {
+ return "", unexpectedJSONError{jval}
+ }
+ typeURL = jval.String()
+ if typeURL == "" {
+ return "", errors.New(`"@type" field contains empty value`)
+ }
+ }
+ }
+
+ return typeURL, nerr.E
+}
+
+// skipJSONValue makes the given decoder parse a JSON value (null, boolean,
+// string, number, object and array) in order to advance the read to the next
+// JSON value. It relies on Decoder.Read returning an error if the types are
+// not in valid sequence.
+func skipJSONValue(dec *json.Decoder) error {
+ // Ignore non-fatal errors, do not return nerr.E.
+ var nerr errors.NonFatal
+ jval, err := dec.Read()
+ if !nerr.Merge(err) {
+ return err
+ }
+ // Only need to continue reading for objects and arrays.
+ switch jval.Type() {
+ case json.StartObject:
+ for {
+ jval, err := dec.Read()
+ if !nerr.Merge(err) {
+ return err
+ }
+ switch jval.Type() {
+ case json.EndObject:
+ return nil
+ case json.Name:
+ // Skip object field value.
+ if err := skipJSONValue(dec); err != nil {
+ return err
+ }
+ }
+ }
+
+ case json.StartArray:
+ for {
+ switch dec.Peek() {
+ case json.EndArray:
+ dec.Read()
+ return nil
+ case json.Invalid:
+ _, err := dec.Read()
+ return err
+ default:
+ // Skip array item.
+ if err := skipJSONValue(dec); err != nil {
+ return err
+ }
+ }
+ }
+ }
+ return nil
+}
+
+// unmarshalAnyValue unmarshals the given custom-type message from the JSON
+// object's "value" field.
+func (o UnmarshalOptions) unmarshalAnyValue(m pref.Message) error {
+ var nerr errors.NonFatal
+ // Skip StartObject, and start reading the fields.
+ o.decoder.Read()
+
+ var found bool // Used for detecting duplicate "value".
+ for {
+ jval, err := o.decoder.Read()
+ if !nerr.Merge(err) {
+ return err
+ }
+ switch jval.Type() {
+ case json.EndObject:
+ if !found {
+ return errors.New(`missing "value" field`)
+ }
+ return nerr.E
+
+ case json.Name:
+ name, err := jval.Name()
+ if !nerr.Merge(err) {
+ return err
+ }
+ switch name {
+ default:
+ return errors.New("unknown field %q", name)
+
+ case "@type":
+ // Skip the value as this was previously parsed already.
+ o.decoder.Read()
+
+ case "value":
+ if found {
+ return errors.New(`duplicate "value" field`)
+ }
+ // Unmarshal the field value into the given message.
+ if err := o.unmarshalCustomType(m); !nerr.Merge(err) {
+ return err
+ }
+ found = true
+ }
+ }
+ }
+}
+
// Wrapper types are encoded as JSON primitives like string, number or boolean.
func (o MarshalOptions) marshalWrapperType(m pref.Message) error {
diff --git a/internal/encoding/json/decode.go b/internal/encoding/json/decode.go
index a9262b8..0ee6c85 100644
--- a/internal/encoding/json/decode.go
+++ b/internal/encoding/json/decode.go
@@ -25,7 +25,8 @@
// Decoder is a token-based JSON decoder.
type Decoder struct {
- // lastCall is last method called, eiterh readCall or peekCall.
+ // lastCall is last method called, either readCall or peekCall.
+ // Initial value is readCall.
lastCall call
// value contains the last read value.
@@ -88,7 +89,7 @@
case Bool, Number:
if !d.isValueNext() {
- return Value{}, d.newSyntaxError("unexpected value %v", value)
+ return Value{}, d.newSyntaxError("unexpected value %v", value.Raw())
}
case String:
@@ -97,7 +98,7 @@
}
// Check if this is for an object name.
if d.value.typ&(StartObject|comma) == 0 {
- return Value{}, d.newSyntaxError("unexpected value %q", value)
+ return Value{}, d.newSyntaxError("unexpected value %v", value.Raw())
}
d.in = d.in[n:]
d.consume(0)
@@ -109,7 +110,7 @@
case StartObject, StartArray:
if !d.isValueNext() {
- return Value{}, d.newSyntaxError("unexpected character %v", value)
+ return Value{}, d.newSyntaxError("unexpected character %v", value.Raw())
}
d.startStack = append(d.startStack, value.typ)
@@ -323,6 +324,14 @@
}
}
+// Clone returns a copy of the Decoder for use in reading ahead the next JSON
+// object, array or other values without affecting current Decoder.
+func (d *Decoder) Clone() *Decoder {
+ ret := *d
+ ret.startStack = append([]Type(nil), ret.startStack...)
+ return &ret
+}
+
// Value contains a JSON type and value parsed from calling Decoder.Read.
// For JSON boolean and string, it holds the converted value in boo and str
// fields respectively. For JSON number, input field holds a valid number which
@@ -377,6 +386,11 @@
return v.str, nil
}
+// Raw returns the read value in string.
+func (v Value) Raw() string {
+ return string(v.input)
+}
+
// Float returns the floating-point number if token is Number, else it will
// return an error.
//
diff --git a/internal/encoding/json/decode_test.go b/internal/encoding/json/decode_test.go
index dc7f25f..b402b0a 100644
--- a/internal/encoding/json/decode_test.go
+++ b/internal/encoding/json/decode_test.go
@@ -1087,3 +1087,37 @@
t.Errorf("want#%d: %v got %v, want %v", wantIdx, value, got, want.V)
}
}
+
+func TestClone(t *testing.T) {
+ input := `{"outer":{"str":"hello", "number": 123}}`
+ dec := json.NewDecoder([]byte(input))
+
+ // Clone at the start should produce the same reads as the original.
+ clone := dec.Clone()
+ compareDecoders(t, dec, clone)
+
+ // Advance to inner object, clone and compare again.
+ dec.Read() // Read StartObject.
+ dec.Read() // Read Name.
+ clone = dec.Clone()
+ compareDecoders(t, dec, clone)
+}
+
+func compareDecoders(t *testing.T, d1 *json.Decoder, d2 *json.Decoder) {
+ for {
+ v1, err1 := d1.Read()
+ v2, err2 := d2.Read()
+ if v1.Type() != v2.Type() {
+ t.Errorf("cloned decoder: got Type %v, want %v", v2.Type(), v1.Type())
+ }
+ if v1.Raw() != v2.Raw() {
+ t.Errorf("cloned decoder: got Raw %v, want %v", v2.Raw(), v1.Raw())
+ }
+ if err1 != err2 {
+ t.Errorf("cloned decoder: got error %v, want %v", err2, err1)
+ }
+ if v1.Type() == json.EOF {
+ break
+ }
+ }
+}
