encoding/japanese: make EUCJP replace with FFFD on error
Updates golang/go#18898
Change-Id: Ic5fb77af67656889a387fa75a3e6efc9b9975817
Reviewed-on: https://go-review.googlesource.com/37316
Run-TryBot: Marcel van Lohuizen <mpvl@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Nigel Tao <nigeltao@golang.org>
diff --git a/encoding/japanese/all_test.go b/encoding/japanese/all_test.go
index 39fb27d..cc7e8dd 100644
--- a/encoding/japanese/all_test.go
+++ b/encoding/japanese/all_test.go
@@ -5,6 +5,7 @@
package japanese_test
import (
+ "strings"
"testing"
"golang.org/x/text/encoding"
@@ -22,6 +23,8 @@
}
func TestNonRepertoire(t *testing.T) {
+ // Pick n to cause the destination buffer in transform.String to overflow.
+ const n = 10000
testCases := []struct {
init func(e encoding.Encoding) (string, transform.Transformer, error)
e encoding.Encoding
@@ -42,6 +45,26 @@
{enc, japanese.ShiftJIS, "갂", ""},
{enc, japanese.ShiftJIS, "a갂", "a"},
{enc, japanese.ShiftJIS, "\u2190갂", "\x81\xa9"},
+
+ // Continue correctly after errors
+ {dec, japanese.EUCJP, "\x8e\xa0", "\ufffd\ufffd"},
+ {dec, japanese.EUCJP, "\x8e\xe0", "\ufffd"},
+ {dec, japanese.EUCJP, "\x8e\xff", "\ufffd\ufffd"},
+ {dec, japanese.EUCJP, "\x8ea", "\ufffda"},
+ {dec, japanese.EUCJP, "\x8f\xa0", "\ufffd\ufffd"},
+ {dec, japanese.EUCJP, "\x8f\xa1a", "\ufffda"},
+ {dec, japanese.EUCJP, "\x8f\xa1a", "\ufffda"},
+ {dec, japanese.EUCJP, "\x8f\xa1\xa0", "\ufffd\ufffd"},
+ {dec, japanese.EUCJP, "\x8f\xa1a", "\ufffda"},
+ {dec, japanese.EUCJP, "\x8f\xa2\xa2", "\ufffd"},
+ {dec, japanese.EUCJP, "\xfe", "\ufffd"},
+ {dec, japanese.EUCJP, "\xfe\xff", "\ufffd\ufffd"},
+ // Correct handling of end of source
+ {dec, japanese.EUCJP, strings.Repeat("\x8e", n), strings.Repeat("\ufffd", n)},
+ {dec, japanese.EUCJP, strings.Repeat("\x8f", n), strings.Repeat("\ufffd", n)},
+ {dec, japanese.EUCJP, strings.Repeat("\x8f\xa0", n), strings.Repeat("\ufffd", 2*n)},
+ {dec, japanese.EUCJP, "a" + strings.Repeat("\x8f\xa1", n), "a" + strings.Repeat("\ufffd", n)},
+ {dec, japanese.EUCJP, "a" + strings.Repeat("\x8f\xa1\xff", n), "a" + strings.Repeat("\ufffd", 2*n)},
}
for _, tc := range testCases {
dir, tr, wantErr := tc.init(tc.e)
diff --git a/encoding/japanese/eucjp.go b/encoding/japanese/eucjp.go
index 40f9b05..79313fa 100644
--- a/encoding/japanese/eucjp.go
+++ b/encoding/japanese/eucjp.go
@@ -5,7 +5,6 @@
package japanese
import (
- "errors"
"unicode/utf8"
"golang.org/x/text/encoding"
@@ -23,10 +22,9 @@
identifier.EUCPkdFmtJapanese,
}
-var errInvalidEUCJP = errors.New("japanese: invalid EUC-JP encoding")
-
type eucJPDecoder struct{ transform.NopResetter }
+// See https://encoding.spec.whatwg.org/#euc-jp-decoder.
func (eucJPDecoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
r, size := rune(0), 0
loop:
@@ -37,60 +35,79 @@
case c0 == 0x8e:
if nSrc+1 >= len(src) {
- err = transform.ErrShortSrc
- break loop
+ if !atEOF {
+ err = transform.ErrShortSrc
+ break loop
+ }
+ r, size = utf8.RuneError, 1
+ break
}
c1 := src[nSrc+1]
- if c1 < 0xa1 || 0xdf < c1 {
- err = errInvalidEUCJP
- break loop
+ switch {
+ case c1 < 0xa1:
+ r, size = utf8.RuneError, 1
+ case c1 > 0xdf:
+ r, size = utf8.RuneError, 2
+ if c1 == 0xff {
+ size = 1
+ }
+ default:
+ r, size = rune(c1)+(0xff61-0xa1), 2
}
- r, size = rune(c1)+(0xff61-0xa1), 2
-
case c0 == 0x8f:
if nSrc+2 >= len(src) {
- err = transform.ErrShortSrc
- break loop
+ if !atEOF {
+ err = transform.ErrShortSrc
+ break loop
+ }
+ r, size = utf8.RuneError, 1
+ if p := nSrc + 1; p < len(src) && 0xa1 <= src[p] && src[p] < 0xfe {
+ size = 2
+ }
+ break
}
c1 := src[nSrc+1]
if c1 < 0xa1 || 0xfe < c1 {
- err = errInvalidEUCJP
- break loop
+ r, size = utf8.RuneError, 1
+ break
}
c2 := src[nSrc+2]
if c2 < 0xa1 || 0xfe < c2 {
- err = errInvalidEUCJP
- break loop
+ r, size = utf8.RuneError, 2
+ break
}
- r, size = '\ufffd', 3
+ r, size = utf8.RuneError, 3
if i := int(c1-0xa1)*94 + int(c2-0xa1); i < len(jis0212Decode) {
r = rune(jis0212Decode[i])
if r == 0 {
- r = '\ufffd'
+ r = utf8.RuneError
}
}
case 0xa1 <= c0 && c0 <= 0xfe:
if nSrc+1 >= len(src) {
- err = transform.ErrShortSrc
- break loop
+ if !atEOF {
+ err = transform.ErrShortSrc
+ break loop
+ }
+ r, size = utf8.RuneError, 1
+ break
}
c1 := src[nSrc+1]
if c1 < 0xa1 || 0xfe < c1 {
- err = errInvalidEUCJP
- break loop
+ r, size = utf8.RuneError, 1
+ break
}
- r, size = '\ufffd', 2
+ r, size = utf8.RuneError, 2
if i := int(c0-0xa1)*94 + int(c1-0xa1); i < len(jis0208Decode) {
r = rune(jis0208Decode[i])
if r == 0 {
- r = '\ufffd'
+ r = utf8.RuneError
}
}
default:
- err = errInvalidEUCJP
- break loop
+ r, size = utf8.RuneError, 1
}
if nDst+utf8.RuneLen(r) > len(dst) {
@@ -99,9 +116,6 @@
}
nDst += utf8.EncodeRune(dst[nDst:], r)
}
- if atEOF && err == transform.ErrShortSrc {
- err = errInvalidEUCJP
- }
return nDst, nSrc, err
}
