encoding/japanese: make iso2022jp replace with FFFD on error This CL also fixes a bug in the JIS 212 mode. Updates golang/go#18898 Change-Id: Idb10e375bbc5d278db4e8550f26986a5d2bb6caa Reviewed-on: https://go-review.googlesource.com/37595 Run-TryBot: Marcel van Lohuizen <mpvl@golang.org> TryBot-Result: Gobot Gobot <gobot@golang.org> Reviewed-by: Nigel Tao <nigeltao@golang.org>
diff --git a/encoding/japanese/all_test.go b/encoding/japanese/all_test.go index a3def34..9cffe10 100644 --- a/encoding/japanese/all_test.go +++ b/encoding/japanese/all_test.go
@@ -5,6 +5,7 @@ package japanese import ( + "fmt" "strings" "testing" @@ -23,16 +24,13 @@ func TestNonRepertoire(t *testing.T) { // Pick n to cause the destination buffer in transform.String to overflow. - const n = 10000 + const n = 100 + long := strings.Repeat(".", n) testCases := []struct { init func(e encoding.Encoding) (string, transform.Transformer, error) e encoding.Encoding src, want string }{ - {dec, EUCJP, "\xfe\xfc", "\ufffd"}, - {dec, ISO2022JP, "\x1b$B\x7e\x7e", "\ufffd"}, - {dec, ShiftJIS, "\xef\xfc", "\ufffd"}, - {enc, EUCJP, "갂", ""}, {enc, EUCJP, "a갂", "a"}, {enc, EUCJP, "丌갂", "\x8f\xb0\xa4"}, @@ -51,12 +49,13 @@ {dec, EUCJP, "\x8e\xff", "\ufffd\ufffd"}, {dec, EUCJP, "\x8ea", "\ufffda"}, {dec, EUCJP, "\x8f\xa0", "\ufffd\ufffd"}, - {dec, EUCJP, "\x8f\xa1a", "\ufffda"}, - {dec, EUCJP, "\x8f\xa1a", "\ufffda"}, {dec, EUCJP, "\x8f\xa1\xa0", "\ufffd\ufffd"}, {dec, EUCJP, "\x8f\xa1a", "\ufffda"}, + {dec, EUCJP, "\x8f\xa1a", "\ufffda"}, + {dec, EUCJP, "\x8f\xa1a", "\ufffda"}, {dec, EUCJP, "\x8f\xa2\xa2", "\ufffd"}, {dec, EUCJP, "\xfe", "\ufffd"}, + {dec, EUCJP, "\xfe\xfc", "\ufffd"}, {dec, EUCJP, "\xfe\xff", "\ufffd\ufffd"}, // Correct handling of end of source {dec, EUCJP, strings.Repeat("\x8e", n), strings.Repeat("\ufffd", n)}, @@ -68,11 +67,12 @@ // Continue correctly after errors {dec, ShiftJIS, "\x80", "\u0080"}, // It's what the spec says. {dec, ShiftJIS, "\x81", "\ufffd"}, + {dec, ShiftJIS, "\x81\x7f", "\ufffd\u007f"}, {dec, ShiftJIS, "\xe0", "\ufffd"}, {dec, ShiftJIS, "\xe0\x39", "\ufffd\u0039"}, - {dec, ShiftJIS, "\x81\x7f", "\ufffd\u007f"}, - {dec, ShiftJIS, "\xe0\xfd", "\ufffd"}, {dec, ShiftJIS, "\xe0\x9f", "燹"}, + {dec, ShiftJIS, "\xe0\xfd", "\ufffd"}, + {dec, ShiftJIS, "\xef\xfc", "\ufffd"}, {dec, ShiftJIS, "\xfc\xfc", "\ufffd"}, {dec, ShiftJIS, "\xfc\xfd", "\ufffd"}, {dec, ShiftJIS, "\xfdaa", "\ufffdaa"}, @@ -80,17 +80,70 @@ {dec, ShiftJIS, strings.Repeat("\x81\x81", n), strings.Repeat("＝", n)}, {dec, ShiftJIS, strings.Repeat("\xe0\xfd", n), strings.Repeat("\ufffd", n)}, {dec, ShiftJIS, "a" + strings.Repeat("\xe0\xfd", n), "a" + strings.Repeat("\ufffd", n)}, + + {dec, ISO2022JP, "\x1b$", "\ufffd$"}, + {dec, ISO2022JP, "\x1b(", "\ufffd("}, + {dec, ISO2022JP, "\x1b@", "\ufffd@"}, + {dec, ISO2022JP, "\x1bZ", "\ufffdZ"}, + // incomplete escapes + {dec, ISO2022JP, "\x1b$", "\ufffd$"}, + {dec, ISO2022JP, "\x1b$J.", "\ufffd$J."}, // illegal + {dec, ISO2022JP, "\x1b$B.", "\ufffd"}, // JIS208 + {dec, ISO2022JP, "\x1b$(", "\ufffd$("}, // JIS212 + {dec, ISO2022JP, "\x1b$(..", "\ufffd$(.."}, // JIS212 + {dec, ISO2022JP, "\x1b$(" + long, "\ufffd$(" + long}, // JIS212 + {dec, ISO2022JP, "\x1b$(D.", "\ufffd"}, // JIS212 + {dec, ISO2022JP, "\x1b$(D..", "\ufffd"}, // JIS212 + {dec, ISO2022JP, "\x1b$(D...", "\ufffd\ufffd"}, // JIS212 + {dec, ISO2022JP, "\x1b(B.", "."}, // ascii + {dec, ISO2022JP, "\x1b(B..", ".."}, // ascii + {dec, ISO2022JP, "\x1b(J.", "."}, // roman + {dec, ISO2022JP, "\x1b(J..", ".."}, // roman + {dec, ISO2022JP, "\x1b(I\x20", "\ufffd"}, // katakana + {dec, ISO2022JP, "\x1b(I\x20\x20", "\ufffd\ufffd"}, // katakana + // recover to same state + {dec, ISO2022JP, "\x1b(B\x1b.", "\ufffd."}, + {dec, ISO2022JP, "\x1b(I\x1b.", "\ufffdｮ"}, + {dec, ISO2022JP, "\x1b(I\x1b$.", "\ufffd､ｮ"}, + {dec, ISO2022JP, "\x1b(I\x1b(.", "\ufffdｨｮ"}, + {dec, ISO2022JP, "\x1b$B\x7e\x7e", "\ufffd"}, + {dec, ISO2022JP, "\x1b$@\x0a.", "\x0a."}, + {dec, ISO2022JP, "\x1b$B\x0a.", "\x0a."}, + {dec, ISO2022JP, "\x1b$(D\x0a.", "\x0a."}, + {dec, ISO2022JP, "\x1b$(D\x7e\x7e", "\ufffd"}, + {dec, ISO2022JP, "\x80", "\ufffd"}, + + // TODO: according to https://encoding.spec.whatwg.org/#iso-2022-jp, + // these should all be correct. + // {dec, ISO2022JP, "\x1b(B\x0E", "\ufffd"}, + // {dec, ISO2022JP, "\x1b(B\x0F", "\ufffd"}, + {dec, ISO2022JP, "\x1b(B\x5C", "\u005C"}, + {dec, ISO2022JP, "\x1b(B\x7E", "\u007E"}, + // {dec, ISO2022JP, "\x1b(J\x0E", "\ufffd"}, + // {dec, ISO2022JP, "\x1b(J\x0F", "\ufffd"}, + // {dec, ISO2022JP, "\x1b(J\x5C", "\u00A5"}, + // {dec, ISO2022JP, "\x1b(J\x7E", "\u203E"}, } for _, tc := range testCases { dir, tr, wantErr := tc.init(tc.e) - - dst, _, err := transform.String(tr, tc.src) - if err != wantErr { - t.Errorf("%s %v(%q): got %v; want %v", dir, tc.e, tc.src, err, wantErr) - } - if got := string(dst); got != tc.want { - t.Errorf("%s %v(%q):\ngot %q\nwant %q", dir, tc.e, tc.src, got, tc.want) - } + t.Run(fmt.Sprintf("%s/%v/%q", dir, tc.e, tc.src), func(t *testing.T) { + dst := make([]byte, 100000) + src := []byte(tc.src) + for i := 0; i <= len(tc.src); i++ { + nDst, nSrc, err := tr.Transform(dst, src[:i], false) + if err != nil && err != transform.ErrShortSrc && err != wantErr { + t.Fatalf("error on first call to Transform: %v", err) + } + n, _, err := tr.Transform(dst[nDst:], src[nSrc:], true) + nDst += n + if err != wantErr { + t.Fatalf("(%q|%q): got %v; want %v", tc.src[:i], tc.src[i:], err, wantErr) + } + if got := string(dst[:nDst]); got != tc.want { + t.Errorf("(%q|%q):\ngot %q\nwant %q", tc.src[:i], tc.src[i:], got, tc.want) + } + } + }) } }
diff --git a/encoding/japanese/iso2022jp.go b/encoding/japanese/iso2022jp.go index b63e7d5..613226d 100644 --- a/encoding/japanese/iso2022jp.go +++ b/encoding/japanese/iso2022jp.go
@@ -5,7 +5,6 @@ package japanese import ( - "errors" "unicode/utf8" "golang.org/x/text/encoding" @@ -31,8 +30,6 @@ return new(iso2022JPEncoder) } -var errInvalidISO2022JP = errors.New("japanese: invalid ISO-2022-JP encoding") - const ( asciiState = iota katakanaState @@ -50,45 +47,51 @@ func (d *iso2022JPDecoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { r, size := rune(0), 0 -loop: for ; nSrc < len(src); nSrc += size { c0 := src[nSrc] if c0 >= utf8.RuneSelf { - err = errInvalidISO2022JP - break loop + r, size = '\ufffd', 1 + goto write } if c0 == asciiEsc { if nSrc+2 >= len(src) { - err = transform.ErrShortSrc - break loop + if !atEOF { + return nDst, nSrc, transform.ErrShortSrc + } + // TODO: is it correct to only skip 1?? + r, size = '\ufffd', 1 + goto write } size = 3 c1 := src[nSrc+1] c2 := src[nSrc+2] switch { - case c1 == '$' && (c2 == '@' || c2 == 'B'): + case c1 == '$' && (c2 == '@' || c2 == 'B'): // 0x24 {0x40, 0x42} *d = jis0208State continue - case c1 == '$' && c2 == '(': + case c1 == '$' && c2 == '(': // 0x24 0x28 if nSrc+3 >= len(src) { - err = transform.ErrShortSrc - break loop + if !atEOF { + return nDst, nSrc, transform.ErrShortSrc + } + r, size = '\ufffd', 1 + goto write } size = 4 - if src[nSrc]+3 == 'D' { + if src[nSrc+3] == 'D' { *d = jis0212State continue } - case c1 == '(' && (c2 == 'B' || c2 == 'J'): + case c1 == '(' && (c2 == 'B' || c2 == 'J'): // 0x28 {0x42, 0x4A} *d = asciiState continue - case c1 == '(' && c2 == 'I': + case c1 == '(' && c2 == 'I': // 0x28 0x49 *d = katakanaState continue } - err = errInvalidISO2022JP - break loop + r, size = '\ufffd', 1 + goto write } switch *d { @@ -97,8 +100,8 @@ case katakanaState: if c0 < 0x21 || 0x60 <= c0 { - err = errInvalidISO2022JP - break loop + r, size = '\ufffd', 1 + goto write } r, size = rune(c0)+(0xff61-0x21), 1 @@ -106,11 +109,14 @@ if c0 == 0x0a { *d = asciiState r, size = rune(c0), 1 - break + goto write } if nSrc+1 >= len(src) { - err = transform.ErrShortSrc - break loop + if !atEOF { + return nDst, nSrc, transform.ErrShortSrc + } + r, size = '\ufffd', 1 + goto write } size = 2 c1 := src[nSrc+1] @@ -121,22 +127,19 @@ r = rune(jis0212Decode[i]) } else { r = '\ufffd' - break + goto write } if r == 0 { r = '\ufffd' } } + write: if nDst+utf8.RuneLen(r) > len(dst) { - err = transform.ErrShortDst - break loop + return nDst, nSrc, transform.ErrShortDst } nDst += utf8.EncodeRune(dst[nDst:], r) } - if atEOF && err == transform.ErrShortSrc { - err = errInvalidISO2022JP - } return nDst, nSrc, err }
diff --git a/encoding/japanese/shiftjis.go b/encoding/japanese/shiftjis.go index 9667b1f..16fd8a6 100644 --- a/encoding/japanese/shiftjis.go +++ b/encoding/japanese/shiftjis.go
@@ -45,8 +45,8 @@ c0 = 2*c0 - 0x21 if nSrc+1 >= len(src) { - err = transform.ErrShortSrc if !atEOF { + err = transform.ErrShortSrc break loop } r, size = '\ufffd', 1