language: compact index bug fix: match zh-HK to zh-Hant-HK

Also add some other tests.

Add GetCoreKey to language/internal in preperation of
generation.

Change-Id: I675e6236e468231b53bd295251d43f7db0c4b49b
Reviewed-on: https://go-review.googlesource.com/95826
Run-TryBot: Marcel van Lohuizen <mpvl@golang.org>
Reviewed-by: Nigel Tao <nigeltao@golang.org>

language/compact.go

diff --git a/language/compact.go b/language/compact.go
new file mode 100644
index 0000000..125cc56
--- /dev/null
+++ b/language/compact.go
@@ -0,0 +1,14 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package language
+
+import "golang.org/x/text/language/internal"
+
+type compactID uint16
+
+func getCoreIndex(t language.Tag) (id compactID, ok bool) {
+	x, ok := coreTags[language.GetCoreKey(t)]
+	return compactID(x), ok
+}

language/internal/compact.go

diff --git a/language/internal/compact.go b/language/internal/compact.go
new file mode 100644
index 0000000..9e5dc8f
--- /dev/null
+++ b/language/internal/compact.go
@@ -0,0 +1,17 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package language
+
+// GetCoreKey generates a uint32 value that is guaranteed to be unique for
+// different language, region, and script values.
+func GetCoreKey(t Tag) (key uint32) {
+	if t.LangID > langNoIndexOffset {
+		return 0xfff00000
+	}
+	key |= uint32(t.LangID) << (8 + 12)
+	key |= uint32(t.ScriptID) << 12
+	key |= uint32(t.RegionID)
+	return key
+}

language/language.go

diff --git a/language/language.go b/language/language.go
index 3829d6d..b800fd9 100644
--- a/language/language.go
+++ b/language/language.go
@@ -438,17 +438,24 @@
 			exact = false
 		}
 	}
-	for ; t != Und; t = t.Parent() {
+	if x, ok := getCoreIndex(t.tag); ok {
+		return int(x), exact
+	}
+	exact = false
+	if r.regionID != 0 && s.scriptID == 0 {
+		// Deal with cases where an extra script is inserted for the region.
+		t, _ := t.tag.Maximize()
+		if x, ok := getCoreIndex(t); ok {
+			return int(x), exact
+		}
+	}
+	for t = t.Parent(); t != Und; t = t.Parent() {
 		// No variants specified: just compare core components.
 		// The key has the form lllssrrr, where l, s, and r are nibbles for
 		// respectively the langID, scriptID, and regionID.
-		key := uint32(b.langID) << (8 + 12)
-		key |= uint32(s.scriptID) << 12
-		key |= uint32(r.regionID)
-		if x, ok := coreTags[key]; ok {
+		if x, ok := getCoreIndex(t.tag); ok {
 			return int(x), exact
 		}
-		exact = false
 	}
 	return int(0), exact
 }

language/language_test.go

diff --git a/language/language_test.go b/language/language_test.go
index 084a86f..1a605c9 100644
--- a/language/language_test.go
+++ b/language/language_test.go
@@ -65,6 +65,14 @@
 		{"en", 136, true},
 		{"en-u-co-phonebk", 136, false},
 		{"en-001", 137, true},
+		{"zh-Hant-HK", 763, true},
+		{"zh-HK", 763, false}, // maximized to zh-Hant-HK
+		{"nl-Beng", 0, false}, // parent skips script
+		{"nl-NO", 524, false}, // region is ignored
+		{"nl-Latn-NO", 524, false},
+		{"nl-Latn-NO-u-co-phonebk", 524, false},
+		{"nl-Latn-NO-valencia", 524, false},
+		{"nl-Latn-NO-oxendict", 524, false},
 		{"sh", 0, false}, // We don't normalize.
 	}
 	for _, tt := range tests {