encoding/japanese: make iso2022jp replace with FFFD on error

This CL also fixes a bug in the JIS 212 mode.

Updates golang/go#18898

Change-Id: Idb10e375bbc5d278db4e8550f26986a5d2bb6caa
Reviewed-on: https://go-review.googlesource.com/37595
Run-TryBot: Marcel van Lohuizen <mpvl@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Nigel Tao <nigeltao@golang.org>
diff --git a/encoding/japanese/all_test.go b/encoding/japanese/all_test.go
index a3def34..9cffe10 100644
--- a/encoding/japanese/all_test.go
+++ b/encoding/japanese/all_test.go
@@ -5,6 +5,7 @@
 package japanese
 
 import (
+	"fmt"
 	"strings"
 	"testing"
 
@@ -23,16 +24,13 @@
 
 func TestNonRepertoire(t *testing.T) {
 	// Pick n to cause the destination buffer in transform.String to overflow.
-	const n = 10000
+	const n = 100
+	long := strings.Repeat(".", n)
 	testCases := []struct {
 		init      func(e encoding.Encoding) (string, transform.Transformer, error)
 		e         encoding.Encoding
 		src, want string
 	}{
-		{dec, EUCJP, "\xfe\xfc", "\ufffd"},
-		{dec, ISO2022JP, "\x1b$B\x7e\x7e", "\ufffd"},
-		{dec, ShiftJIS, "\xef\xfc", "\ufffd"},
-
 		{enc, EUCJP, "갂", ""},
 		{enc, EUCJP, "a갂", "a"},
 		{enc, EUCJP, "丌갂", "\x8f\xb0\xa4"},
@@ -51,12 +49,13 @@
 		{dec, EUCJP, "\x8e\xff", "\ufffd\ufffd"},
 		{dec, EUCJP, "\x8ea", "\ufffda"},
 		{dec, EUCJP, "\x8f\xa0", "\ufffd\ufffd"},
-		{dec, EUCJP, "\x8f\xa1a", "\ufffda"},
-		{dec, EUCJP, "\x8f\xa1a", "\ufffda"},
 		{dec, EUCJP, "\x8f\xa1\xa0", "\ufffd\ufffd"},
 		{dec, EUCJP, "\x8f\xa1a", "\ufffda"},
+		{dec, EUCJP, "\x8f\xa1a", "\ufffda"},
+		{dec, EUCJP, "\x8f\xa1a", "\ufffda"},
 		{dec, EUCJP, "\x8f\xa2\xa2", "\ufffd"},
 		{dec, EUCJP, "\xfe", "\ufffd"},
+		{dec, EUCJP, "\xfe\xfc", "\ufffd"},
 		{dec, EUCJP, "\xfe\xff", "\ufffd\ufffd"},
 		// Correct handling of end of source
 		{dec, EUCJP, strings.Repeat("\x8e", n), strings.Repeat("\ufffd", n)},
@@ -68,11 +67,12 @@
 		// Continue correctly after errors
 		{dec, ShiftJIS, "\x80", "\u0080"}, // It's what the spec says.
 		{dec, ShiftJIS, "\x81", "\ufffd"},
+		{dec, ShiftJIS, "\x81\x7f", "\ufffd\u007f"},
 		{dec, ShiftJIS, "\xe0", "\ufffd"},
 		{dec, ShiftJIS, "\xe0\x39", "\ufffd\u0039"},
-		{dec, ShiftJIS, "\x81\x7f", "\ufffd\u007f"},
-		{dec, ShiftJIS, "\xe0\xfd", "\ufffd"},
 		{dec, ShiftJIS, "\xe0\x9f", "燹"},
+		{dec, ShiftJIS, "\xe0\xfd", "\ufffd"},
+		{dec, ShiftJIS, "\xef\xfc", "\ufffd"},
 		{dec, ShiftJIS, "\xfc\xfc", "\ufffd"},
 		{dec, ShiftJIS, "\xfc\xfd", "\ufffd"},
 		{dec, ShiftJIS, "\xfdaa", "\ufffdaa"},
@@ -80,17 +80,70 @@
 		{dec, ShiftJIS, strings.Repeat("\x81\x81", n), strings.Repeat("=", n)},
 		{dec, ShiftJIS, strings.Repeat("\xe0\xfd", n), strings.Repeat("\ufffd", n)},
 		{dec, ShiftJIS, "a" + strings.Repeat("\xe0\xfd", n), "a" + strings.Repeat("\ufffd", n)},
+
+		{dec, ISO2022JP, "\x1b$", "\ufffd$"},
+		{dec, ISO2022JP, "\x1b(", "\ufffd("},
+		{dec, ISO2022JP, "\x1b@", "\ufffd@"},
+		{dec, ISO2022JP, "\x1bZ", "\ufffdZ"},
+		// incomplete escapes
+		{dec, ISO2022JP, "\x1b$", "\ufffd$"},
+		{dec, ISO2022JP, "\x1b$J.", "\ufffd$J."},             // illegal
+		{dec, ISO2022JP, "\x1b$B.", "\ufffd"},                // JIS208
+		{dec, ISO2022JP, "\x1b$(", "\ufffd$("},               // JIS212
+		{dec, ISO2022JP, "\x1b$(..", "\ufffd$(.."},           // JIS212
+		{dec, ISO2022JP, "\x1b$(" + long, "\ufffd$(" + long}, // JIS212
+		{dec, ISO2022JP, "\x1b$(D.", "\ufffd"},               // JIS212
+		{dec, ISO2022JP, "\x1b$(D..", "\ufffd"},              // JIS212
+		{dec, ISO2022JP, "\x1b$(D...", "\ufffd\ufffd"},       // JIS212
+		{dec, ISO2022JP, "\x1b(B.", "."},                     // ascii
+		{dec, ISO2022JP, "\x1b(B..", ".."},                   // ascii
+		{dec, ISO2022JP, "\x1b(J.", "."},                     // roman
+		{dec, ISO2022JP, "\x1b(J..", ".."},                   // roman
+		{dec, ISO2022JP, "\x1b(I\x20", "\ufffd"},             // katakana
+		{dec, ISO2022JP, "\x1b(I\x20\x20", "\ufffd\ufffd"},   // katakana
+		// recover to same state
+		{dec, ISO2022JP, "\x1b(B\x1b.", "\ufffd."},
+		{dec, ISO2022JP, "\x1b(I\x1b.", "\ufffdョ"},
+		{dec, ISO2022JP, "\x1b(I\x1b$.", "\ufffd、ョ"},
+		{dec, ISO2022JP, "\x1b(I\x1b(.", "\ufffdィョ"},
+		{dec, ISO2022JP, "\x1b$B\x7e\x7e", "\ufffd"},
+		{dec, ISO2022JP, "\x1b$@\x0a.", "\x0a."},
+		{dec, ISO2022JP, "\x1b$B\x0a.", "\x0a."},
+		{dec, ISO2022JP, "\x1b$(D\x0a.", "\x0a."},
+		{dec, ISO2022JP, "\x1b$(D\x7e\x7e", "\ufffd"},
+		{dec, ISO2022JP, "\x80", "\ufffd"},
+
+		// TODO: according to https://encoding.spec.whatwg.org/#iso-2022-jp,
+		// these should all be correct.
+		// {dec, ISO2022JP, "\x1b(B\x0E", "\ufffd"},
+		// {dec, ISO2022JP, "\x1b(B\x0F", "\ufffd"},
+		{dec, ISO2022JP, "\x1b(B\x5C", "\u005C"},
+		{dec, ISO2022JP, "\x1b(B\x7E", "\u007E"},
+		// {dec, ISO2022JP, "\x1b(J\x0E", "\ufffd"},
+		// {dec, ISO2022JP, "\x1b(J\x0F", "\ufffd"},
+		// {dec, ISO2022JP, "\x1b(J\x5C", "\u00A5"},
+		// {dec, ISO2022JP, "\x1b(J\x7E", "\u203E"},
 	}
 	for _, tc := range testCases {
 		dir, tr, wantErr := tc.init(tc.e)
-
-		dst, _, err := transform.String(tr, tc.src)
-		if err != wantErr {
-			t.Errorf("%s %v(%q): got %v; want %v", dir, tc.e, tc.src, err, wantErr)
-		}
-		if got := string(dst); got != tc.want {
-			t.Errorf("%s %v(%q):\ngot  %q\nwant %q", dir, tc.e, tc.src, got, tc.want)
-		}
+		t.Run(fmt.Sprintf("%s/%v/%q", dir, tc.e, tc.src), func(t *testing.T) {
+			dst := make([]byte, 100000)
+			src := []byte(tc.src)
+			for i := 0; i <= len(tc.src); i++ {
+				nDst, nSrc, err := tr.Transform(dst, src[:i], false)
+				if err != nil && err != transform.ErrShortSrc && err != wantErr {
+					t.Fatalf("error on first call to Transform: %v", err)
+				}
+				n, _, err := tr.Transform(dst[nDst:], src[nSrc:], true)
+				nDst += n
+				if err != wantErr {
+					t.Fatalf("(%q|%q): got %v; want %v", tc.src[:i], tc.src[i:], err, wantErr)
+				}
+				if got := string(dst[:nDst]); got != tc.want {
+					t.Errorf("(%q|%q):\ngot  %q\nwant %q", tc.src[:i], tc.src[i:], got, tc.want)
+				}
+			}
+		})
 	}
 }
 
diff --git a/encoding/japanese/iso2022jp.go b/encoding/japanese/iso2022jp.go
index b63e7d5..613226d 100644
--- a/encoding/japanese/iso2022jp.go
+++ b/encoding/japanese/iso2022jp.go
@@ -5,7 +5,6 @@
 package japanese
 
 import (
-	"errors"
 	"unicode/utf8"
 
 	"golang.org/x/text/encoding"
@@ -31,8 +30,6 @@
 	return new(iso2022JPEncoder)
 }
 
-var errInvalidISO2022JP = errors.New("japanese: invalid ISO-2022-JP encoding")
-
 const (
 	asciiState = iota
 	katakanaState
@@ -50,45 +47,51 @@
 
 func (d *iso2022JPDecoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
 	r, size := rune(0), 0
-loop:
 	for ; nSrc < len(src); nSrc += size {
 		c0 := src[nSrc]
 		if c0 >= utf8.RuneSelf {
-			err = errInvalidISO2022JP
-			break loop
+			r, size = '\ufffd', 1
+			goto write
 		}
 
 		if c0 == asciiEsc {
 			if nSrc+2 >= len(src) {
-				err = transform.ErrShortSrc
-				break loop
+				if !atEOF {
+					return nDst, nSrc, transform.ErrShortSrc
+				}
+				// TODO: is it correct to only skip 1??
+				r, size = '\ufffd', 1
+				goto write
 			}
 			size = 3
 			c1 := src[nSrc+1]
 			c2 := src[nSrc+2]
 			switch {
-			case c1 == '$' && (c2 == '@' || c2 == 'B'):
+			case c1 == '$' && (c2 == '@' || c2 == 'B'): // 0x24 {0x40, 0x42}
 				*d = jis0208State
 				continue
-			case c1 == '$' && c2 == '(':
+			case c1 == '$' && c2 == '(': // 0x24 0x28
 				if nSrc+3 >= len(src) {
-					err = transform.ErrShortSrc
-					break loop
+					if !atEOF {
+						return nDst, nSrc, transform.ErrShortSrc
+					}
+					r, size = '\ufffd', 1
+					goto write
 				}
 				size = 4
-				if src[nSrc]+3 == 'D' {
+				if src[nSrc+3] == 'D' {
 					*d = jis0212State
 					continue
 				}
-			case c1 == '(' && (c2 == 'B' || c2 == 'J'):
+			case c1 == '(' && (c2 == 'B' || c2 == 'J'): // 0x28 {0x42, 0x4A}
 				*d = asciiState
 				continue
-			case c1 == '(' && c2 == 'I':
+			case c1 == '(' && c2 == 'I': // 0x28 0x49
 				*d = katakanaState
 				continue
 			}
-			err = errInvalidISO2022JP
-			break loop
+			r, size = '\ufffd', 1
+			goto write
 		}
 
 		switch *d {
@@ -97,8 +100,8 @@
 
 		case katakanaState:
 			if c0 < 0x21 || 0x60 <= c0 {
-				err = errInvalidISO2022JP
-				break loop
+				r, size = '\ufffd', 1
+				goto write
 			}
 			r, size = rune(c0)+(0xff61-0x21), 1
 
@@ -106,11 +109,14 @@
 			if c0 == 0x0a {
 				*d = asciiState
 				r, size = rune(c0), 1
-				break
+				goto write
 			}
 			if nSrc+1 >= len(src) {
-				err = transform.ErrShortSrc
-				break loop
+				if !atEOF {
+					return nDst, nSrc, transform.ErrShortSrc
+				}
+				r, size = '\ufffd', 1
+				goto write
 			}
 			size = 2
 			c1 := src[nSrc+1]
@@ -121,22 +127,19 @@
 				r = rune(jis0212Decode[i])
 			} else {
 				r = '\ufffd'
-				break
+				goto write
 			}
 			if r == 0 {
 				r = '\ufffd'
 			}
 		}
 
+	write:
 		if nDst+utf8.RuneLen(r) > len(dst) {
-			err = transform.ErrShortDst
-			break loop
+			return nDst, nSrc, transform.ErrShortDst
 		}
 		nDst += utf8.EncodeRune(dst[nDst:], r)
 	}
-	if atEOF && err == transform.ErrShortSrc {
-		err = errInvalidISO2022JP
-	}
 	return nDst, nSrc, err
 }
 
diff --git a/encoding/japanese/shiftjis.go b/encoding/japanese/shiftjis.go
index 9667b1f..16fd8a6 100644
--- a/encoding/japanese/shiftjis.go
+++ b/encoding/japanese/shiftjis.go
@@ -45,8 +45,8 @@
 			c0 = 2*c0 - 0x21
 
 			if nSrc+1 >= len(src) {
-				err = transform.ErrShortSrc
 				if !atEOF {
+					err = transform.ErrShortSrc
 					break loop
 				}
 				r, size = '\ufffd', 1