encoding/simplifiedchinese: make GB18030 replace with FFFD on error
Updates golang/go#18898
Change-Id: I0636d4ff317bbca015cf2a2a2ce9f452bdc601fd
Reviewed-on: https://go-review.googlesource.com/37322
Run-TryBot: Marcel van Lohuizen <mpvl@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Nigel Tao <nigeltao@golang.org>
diff --git a/encoding/simplifiedchinese/all_test.go b/encoding/simplifiedchinese/all_test.go
index 2d44f1b..96a0585 100644
--- a/encoding/simplifiedchinese/all_test.go
+++ b/encoding/simplifiedchinese/all_test.go
@@ -5,6 +5,7 @@
package simplifiedchinese
import (
+ "strings"
"testing"
"golang.org/x/text/encoding"
@@ -21,6 +22,8 @@
}
func TestNonRepertoire(t *testing.T) {
+ // Pick n large enough to overflow the destination buffer of transform.String.
+ const n = 10000
testCases := []struct {
init func(e encoding.Encoding) (string, transform.Transformer, error)
e encoding.Encoding
@@ -36,6 +39,18 @@
{enc, HZGB2312, "갂", ""},
{enc, HZGB2312, "a갂", "a"},
{enc, HZGB2312, "\u6cf5갂", "~{1C~}"},
+
+ {dec, GB18030, "\x80", "€"},
+ {dec, GB18030, "\x81", "\ufffd"},
+ {dec, GB18030, "\x81\x20", "\ufffd "},
+ {dec, GB18030, "\xfe\xfe", "\ufffd"},
+ {dec, GB18030, "\xfe\xff", "\ufffd\ufffd"},
+ {dec, GB18030, "\xfe\x30", "\ufffd0"},
+ {dec, GB18030, "\xfe\x30\x30 ", "\ufffd00 "},
+ {dec, GB18030, "\xfe\x30\xff ", "\ufffd0\ufffd "},
+ {dec, GB18030, "\xfe\x30\x81\x21", "\ufffd0\ufffd!"},
+
+ {dec, GB18030, strings.Repeat("\xfe\x30", n), strings.Repeat("\ufffd0", n)},
}
for _, tc := range testCases {
dir, tr, wantErr := tc.init(tc.e)
diff --git a/encoding/simplifiedchinese/gbk.go b/encoding/simplifiedchinese/gbk.go
index e0b15bb..b89c45b 100644
--- a/encoding/simplifiedchinese/gbk.go
+++ b/encoding/simplifiedchinese/gbk.go
@@ -5,7 +5,6 @@
package simplifiedchinese
import (
- "errors"
"unicode/utf8"
"golang.org/x/text/encoding"
@@ -40,11 +39,6 @@
identifier.GB18030,
}
-var (
- errInvalidGB18030 = errors.New("simplifiedchinese: invalid GB18030 encoding")
- errInvalidGBK = errors.New("simplifiedchinese: invalid GBK encoding")
-)
-
type gbkDecoder struct {
transform.NopResetter
gb18030 bool
@@ -66,8 +60,12 @@
case c0 < 0xff:
if nSrc+1 >= len(src) {
- err = transform.ErrShortSrc
- break loop
+ if !atEOF {
+ err = transform.ErrShortSrc
+ break loop
+ }
+ r, size = utf8.RuneError, 1
+ goto write
}
c1 := src[nSrc+1]
switch {
@@ -77,18 +75,24 @@
c1 -= 0x41
case d.gb18030 && 0x30 <= c1 && c1 < 0x40:
if nSrc+3 >= len(src) {
- err = transform.ErrShortSrc
- break loop
+ if !atEOF {
+ err = transform.ErrShortSrc
+ break loop
+ }
+ // The second byte here is always ASCII, so we can set size
+ // to 1 in all cases.
+ r, size = utf8.RuneError, 1
+ goto write
}
c2 := src[nSrc+2]
if c2 < 0x81 || 0xff <= c2 {
- err = errInvalidGB18030
- break loop
+ r, size = utf8.RuneError, 1
+ goto write
}
c3 := src[nSrc+3]
if c3 < 0x30 || 0x3a <= c3 {
- err = errInvalidGB18030
- break loop
+ r, size = utf8.RuneError, 1
+ goto write
}
size = 4
r = ((rune(c0-0x81)*10+rune(c1-0x30))*126+rune(c2-0x81))*10 + rune(c3-0x30)
@@ -109,17 +113,13 @@
r -= 189000
if 0 <= r && r < 0x100000 {
r += 0x10000
- goto write
- }
- err = errInvalidGB18030
- break loop
- default:
- if d.gb18030 {
- err = errInvalidGB18030
} else {
- err = errInvalidGBK
+ r, size = utf8.RuneError, 1
}
- break loop
+ goto write
+ default:
+ r, size = utf8.RuneError, 1
+ goto write
}
r, size = '\ufffd', 2
if i := int(c0-0x81)*190 + int(c1); i < len(decode) {
@@ -130,12 +130,7 @@
}
default:
- if d.gb18030 {
- err = errInvalidGB18030
- } else {
- err = errInvalidGBK
- }
- break loop
+ r, size = utf8.RuneError, 1
}
write:
@@ -145,13 +140,6 @@
}
nDst += utf8.EncodeRune(dst[nDst:], r)
}
- if atEOF && err == transform.ErrShortSrc {
- if d.gb18030 {
- err = errInvalidGB18030
- } else {
- err = errInvalidGBK
- }
- }
return nDst, nSrc, err
}
