mime: use sync.Map instead of RWMutex for type lookups
This provides a significant speedup for TypeByExtension and
ExtensionsByType when using many CPU cores.
updates #17973
updates #18177
name old time/op new time/op delta
QEncodeWord 526ns ± 3% 525ns ± 3% ~ (p=0.990 n=15+28)
QEncodeWord-6 945ns ± 7% 913ns ±20% ~ (p=0.220 n=14+28)
QEncodeWord-48 1.02µs ± 2% 1.00µs ± 6% -2.22% (p=0.036 n=13+27)
QDecodeWord 311ns ±18% 323ns ±20% ~ (p=0.107 n=16+28)
QDecodeWord-6 595ns ±12% 612ns ±11% ~ (p=0.093 n=15+27)
QDecodeWord-48 592ns ± 6% 606ns ± 8% +2.39% (p=0.045 n=16+26)
QDecodeHeader 389ns ± 4% 394ns ± 8% ~ (p=0.161 n=12+26)
QDecodeHeader-6 685ns ±12% 674ns ±20% ~ (p=0.773 n=14+27)
QDecodeHeader-48 658ns ±13% 669ns ±14% ~ (p=0.457 n=16+28)
TypeByExtension/.html 77.4ns ±15% 55.5ns ±13% -28.35% (p=0.000 n=8+8)
TypeByExtension/.html-6 263ns ± 9% 10ns ±21% -96.29% (p=0.000 n=8+8)
TypeByExtension/.html-48 175ns ± 5% 2ns ±16% -98.88% (p=0.000 n=8+8)
TypeByExtension/.HTML 113ns ± 6% 97ns ± 6% -14.37% (p=0.000 n=8+8)
TypeByExtension/.HTML-6 273ns ± 7% 17ns ± 4% -93.93% (p=0.000 n=7+8)
TypeByExtension/.HTML-48 175ns ± 4% 4ns ± 4% -97.73% (p=0.000 n=8+8)
TypeByExtension/.unused 116ns ± 4% 90ns ± 4% -22.89% (p=0.001 n=7+7)
TypeByExtension/.unused-6 262ns ± 5% 15ns ± 4% -94.17% (p=0.000 n=8+8)
TypeByExtension/.unused-48 176ns ± 4% 3ns ±10% -98.10% (p=0.000 n=8+8)
ExtensionsByType/text/html 630ns ± 5% 522ns ± 5% -17.19% (p=0.000 n=8+7)
ExtensionsByType/text/html-6 314ns ±20% 136ns ± 6% -56.80% (p=0.000 n=8+8)
ExtensionsByType/text/html-48 298ns ± 4% 104ns ± 6% -65.06% (p=0.000 n=8+8)
ExtensionsByType/text/html;_charset=utf-8 1.12µs ± 3% 1.05µs ± 7% -6.19% (p=0.004 n=8+7)
ExtensionsByType/text/html;_charset=utf-8-6 402ns ±11% 307ns ± 4% -23.77% (p=0.000 n=8+8)
ExtensionsByType/text/html;_charset=utf-8-48 422ns ± 3% 309ns ± 4% -26.86% (p=0.000 n=8+8)
ExtensionsByType/application/octet-stream 810ns ± 2% 747ns ± 5% -7.74% (p=0.000 n=8+8)
ExtensionsByType/application/octet-stream-6 289ns ± 9% 185ns ± 8% -36.15% (p=0.000 n=7+8)
ExtensionsByType/application/octet-stream-48 267ns ± 6% 94ns ± 2% -64.91% (p=0.000 n=8+7)
name old alloc/op new alloc/op delta
QEncodeWord 48.0B ± 0% 48.0B ± 0% ~ (all equal)
QEncodeWord-6 48.0B ± 0% 48.0B ± 0% ~ (all equal)
QEncodeWord-48 48.0B ± 0% 48.0B ± 0% ~ (all equal)
QDecodeWord 48.0B ± 0% 48.0B ± 0% ~ (all equal)
QDecodeWord-6 48.0B ± 0% 48.0B ± 0% ~ (all equal)
QDecodeWord-48 48.0B ± 0% 48.0B ± 0% ~ (all equal)
QDecodeHeader 48.0B ± 0% 48.0B ± 0% ~ (all equal)
QDecodeHeader-6 48.0B ± 0% 48.0B ± 0% ~ (all equal)
QDecodeHeader-48 48.0B ± 0% 48.0B ± 0% ~ (all equal)
TypeByExtension/.html 0.00B 0.00B ~ (all equal)
TypeByExtension/.html-6 0.00B 0.00B ~ (all equal)
TypeByExtension/.html-48 0.00B 0.00B ~ (all equal)
TypeByExtension/.HTML 0.00B 0.00B ~ (all equal)
TypeByExtension/.HTML-6 0.00B 0.00B ~ (all equal)
TypeByExtension/.HTML-48 0.00B 0.00B ~ (all equal)
TypeByExtension/.unused 0.00B 0.00B ~ (all equal)
TypeByExtension/.unused-6 0.00B 0.00B ~ (all equal)
TypeByExtension/.unused-48 0.00B 0.00B ~ (all equal)
ExtensionsByType/text/html 192B ± 0% 176B ± 0% -8.33% (p=0.000 n=8+8)
ExtensionsByType/text/html-6 192B ± 0% 176B ± 0% -8.33% (p=0.000 n=8+8)
ExtensionsByType/text/html-48 192B ± 0% 176B ± 0% -8.33% (p=0.000 n=8+8)
ExtensionsByType/text/html;_charset=utf-8 480B ± 0% 464B ± 0% -3.33% (p=0.000 n=8+8)
ExtensionsByType/text/html;_charset=utf-8-6 480B ± 0% 464B ± 0% -3.33% (p=0.000 n=8+8)
ExtensionsByType/text/html;_charset=utf-8-48 480B ± 0% 464B ± 0% -3.33% (p=0.000 n=8+8)
ExtensionsByType/application/octet-stream 160B ± 0% 160B ± 0% ~ (all equal)
ExtensionsByType/application/octet-stream-6 160B ± 0% 160B ± 0% ~ (all equal)
ExtensionsByType/application/octet-stream-48 160B ± 0% 160B ± 0% ~ (all equal)
name old allocs/op new allocs/op delta
QEncodeWord 1.00 ± 0% 1.00 ± 0% ~ (all equal)
QEncodeWord-6 1.00 ± 0% 1.00 ± 0% ~ (all equal)
QEncodeWord-48 1.00 ± 0% 1.00 ± 0% ~ (all equal)
QDecodeWord 2.00 ± 0% 2.00 ± 0% ~ (all equal)
QDecodeWord-6 2.00 ± 0% 2.00 ± 0% ~ (all equal)
QDecodeWord-48 2.00 ± 0% 2.00 ± 0% ~ (all equal)
QDecodeHeader 2.00 ± 0% 2.00 ± 0% ~ (all equal)
QDecodeHeader-6 2.00 ± 0% 2.00 ± 0% ~ (all equal)
QDecodeHeader-48 2.00 ± 0% 2.00 ± 0% ~ (all equal)
TypeByExtension/.html 0.00 0.00 ~ (all equal)
TypeByExtension/.html-6 0.00 0.00 ~ (all equal)
TypeByExtension/.html-48 0.00 0.00 ~ (all equal)
TypeByExtension/.HTML 0.00 0.00 ~ (all equal)
TypeByExtension/.HTML-6 0.00 0.00 ~ (all equal)
TypeByExtension/.HTML-48 0.00 0.00 ~ (all equal)
TypeByExtension/.unused 0.00 0.00 ~ (all equal)
TypeByExtension/.unused-6 0.00 0.00 ~ (all equal)
TypeByExtension/.unused-48 0.00 0.00 ~ (all equal)
ExtensionsByType/text/html 3.00 ± 0% 3.00 ± 0% ~ (all equal)
ExtensionsByType/text/html-6 3.00 ± 0% 3.00 ± 0% ~ (all equal)
ExtensionsByType/text/html-48 3.00 ± 0% 3.00 ± 0% ~ (all equal)
ExtensionsByType/text/html;_charset=utf-8 4.00 ± 0% 4.00 ± 0% ~ (all equal)
ExtensionsByType/text/html;_charset=utf-8-6 4.00 ± 0% 4.00 ± 0% ~ (all equal)
ExtensionsByType/text/html;_charset=utf-8-48 4.00 ± 0% 4.00 ± 0% ~ (all equal)
ExtensionsByType/application/octet-stream 2.00 ± 0% 2.00 ± 0% ~ (all equal)
ExtensionsByType/application/octet-stream-6 2.00 ± 0% 2.00 ± 0% ~ (all equal)
ExtensionsByType/application/octet-stream-48 2.00 ± 0% 2.00 ± 0% ~ (all equal)
https://perf.golang.org/search?q=upload:20170427.4
Change-Id: I35438be087ad6eb3d5da9119b395723ea5babaf6
Reviewed-on: https://go-review.googlesource.com/41990
Run-TryBot: Bryan Mills <bcmills@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Ian Lance Taylor <iant@golang.org>
diff --git a/src/mime/type.go b/src/mime/type.go
index d369259..78fc6b6 100644
--- a/src/mime/type.go
+++ b/src/mime/type.go
@@ -12,24 +12,48 @@
)
var (
- mimeLock sync.RWMutex // guards following 3 maps
- mimeTypes map[string]string // ".Z" => "application/x-compress"
- mimeTypesLower map[string]string // ".z" => "application/x-compress"
+ mimeTypes sync.Map // map[string]string; ".Z" => "application/x-compress"
+ mimeTypesLower sync.Map // map[string]string; ".z" => "application/x-compress"
// extensions maps from MIME type to list of lowercase file
// extensions: "image/jpeg" => [".jpg", ".jpeg"]
- extensions map[string][]string
+ extensionsMu sync.Mutex // Guards stores (but not loads) on extensions.
+ extensions sync.Map // map[string][]string; slice values are append-only.
)
+func clearSyncMap(m *sync.Map) {
+ m.Range(func(k, _ interface{}) bool {
+ m.Delete(k)
+ return true
+ })
+}
+
// setMimeTypes is used by initMime's non-test path, and by tests.
-// The two maps must not be the same, or nil.
func setMimeTypes(lowerExt, mixExt map[string]string) {
- if lowerExt == nil || mixExt == nil {
- panic("nil map")
+ clearSyncMap(&mimeTypes)
+ clearSyncMap(&mimeTypesLower)
+ clearSyncMap(&extensions)
+
+ for k, v := range lowerExt {
+ mimeTypesLower.Store(k, v)
}
- mimeTypesLower = lowerExt
- mimeTypes = mixExt
- extensions = invert(lowerExt)
+ for k, v := range mixExt {
+ mimeTypes.Store(k, v)
+ }
+
+ extensionsMu.Lock()
+ defer extensionsMu.Unlock()
+ for k, v := range lowerExt {
+ justType, _, err := ParseMediaType(v)
+ if err != nil {
+ panic(err)
+ }
+ var exts []string
+ if ei, ok := extensions.Load(k); ok {
+ exts = ei.([]string)
+ }
+ extensions.Store(justType, append(exts, k))
+ }
}
var builtinTypesLower = map[string]string{
@@ -45,29 +69,6 @@
".xml": "text/xml; charset=utf-8",
}
-func clone(m map[string]string) map[string]string {
- m2 := make(map[string]string, len(m))
- for k, v := range m {
- m2[k] = v
- if strings.ToLower(k) != k {
- panic("keys in builtinTypesLower must be lowercase")
- }
- }
- return m2
-}
-
-func invert(m map[string]string) map[string][]string {
- m2 := make(map[string][]string, len(m))
- for k, v := range m {
- justType, _, err := ParseMediaType(v)
- if err != nil {
- panic(err)
- }
- m2[justType] = append(m2[justType], k)
- }
- return m2
-}
-
var once sync.Once // guards initMime
var testInitMime, osInitMime func()
@@ -76,7 +77,7 @@
if fn := testInitMime; fn != nil {
fn()
} else {
- setMimeTypes(builtinTypesLower, clone(builtinTypesLower))
+ setMimeTypes(builtinTypesLower, builtinTypesLower)
osInitMime()
}
}
@@ -100,12 +101,10 @@
// Text types have the charset parameter set to "utf-8" by default.
func TypeByExtension(ext string) string {
once.Do(initMime)
- mimeLock.RLock()
- defer mimeLock.RUnlock()
// Case-sensitive lookup.
- if v := mimeTypes[ext]; v != "" {
- return v
+ if v, ok := mimeTypes.Load(ext); ok {
+ return v.(string)
}
// Case-insensitive lookup.
@@ -118,7 +117,9 @@
c := ext[i]
if c >= utf8RuneSelf {
// Slow path.
- return mimeTypesLower[strings.ToLower(ext)]
+ si, _ := mimeTypesLower.Load(strings.ToLower(ext))
+ s, _ := si.(string)
+ return s
}
if 'A' <= c && c <= 'Z' {
lower = append(lower, c+('a'-'A'))
@@ -126,9 +127,9 @@
lower = append(lower, c)
}
}
- // The conversion from []byte to string doesn't allocate in
- // a map lookup.
- return mimeTypesLower[string(lower)]
+ si, _ := mimeTypesLower.Load(string(lower))
+ s, _ := si.(string)
+ return s
}
// ExtensionsByType returns the extensions known to be associated with the MIME
@@ -142,13 +143,11 @@
}
once.Do(initMime)
- mimeLock.RLock()
- defer mimeLock.RUnlock()
- s, ok := extensions[justType]
+ s, ok := extensions.Load(justType)
if !ok {
return nil, nil
}
- return append([]string{}, s...), nil
+ return append([]string{}, s.([]string)...), nil
}
// AddExtensionType sets the MIME type associated with
@@ -173,15 +172,20 @@
}
extLower := strings.ToLower(extension)
- mimeLock.Lock()
- defer mimeLock.Unlock()
- mimeTypes[extension] = mimeType
- mimeTypesLower[extLower] = mimeType
- for _, v := range extensions[justType] {
+ mimeTypes.Store(extension, mimeType)
+ mimeTypesLower.Store(extLower, mimeType)
+
+ extensionsMu.Lock()
+ defer extensionsMu.Unlock()
+ var exts []string
+ if ei, ok := extensions.Load(justType); ok {
+ exts = ei.([]string)
+ }
+ for _, v := range exts {
if v == extLower {
return nil
}
}
- extensions[justType] = append(extensions[justType], extLower)
+ extensions.Store(justType, append(exts, extLower))
return nil
}
