Skip to content

Commit e8d7e5d

Browse files
author
Bryan C. Mills
committed
mime: use sync.Map instead of RWMutex for type lookups
This provides a significant speedup for TypeByExtension and ExtensionsByType when using many CPU cores. updates #17973 updates #18177 name old time/op new time/op delta QEncodeWord 526ns ± 3% 525ns ± 3% ~ (p=0.990 n=15+28) QEncodeWord-6 945ns ± 7% 913ns ±20% ~ (p=0.220 n=14+28) QEncodeWord-48 1.02µs ± 2% 1.00µs ± 6% -2.22% (p=0.036 n=13+27) QDecodeWord 311ns ±18% 323ns ±20% ~ (p=0.107 n=16+28) QDecodeWord-6 595ns ±12% 612ns ±11% ~ (p=0.093 n=15+27) QDecodeWord-48 592ns ± 6% 606ns ± 8% +2.39% (p=0.045 n=16+26) QDecodeHeader 389ns ± 4% 394ns ± 8% ~ (p=0.161 n=12+26) QDecodeHeader-6 685ns ±12% 674ns ±20% ~ (p=0.773 n=14+27) QDecodeHeader-48 658ns ±13% 669ns ±14% ~ (p=0.457 n=16+28) TypeByExtension/.html 77.4ns ±15% 55.5ns ±13% -28.35% (p=0.000 n=8+8) TypeByExtension/.html-6 263ns ± 9% 10ns ±21% -96.29% (p=0.000 n=8+8) TypeByExtension/.html-48 175ns ± 5% 2ns ±16% -98.88% (p=0.000 n=8+8) TypeByExtension/.HTML 113ns ± 6% 97ns ± 6% -14.37% (p=0.000 n=8+8) TypeByExtension/.HTML-6 273ns ± 7% 17ns ± 4% -93.93% (p=0.000 n=7+8) TypeByExtension/.HTML-48 175ns ± 4% 4ns ± 4% -97.73% (p=0.000 n=8+8) TypeByExtension/.unused 116ns ± 4% 90ns ± 4% -22.89% (p=0.001 n=7+7) TypeByExtension/.unused-6 262ns ± 5% 15ns ± 4% -94.17% (p=0.000 n=8+8) TypeByExtension/.unused-48 176ns ± 4% 3ns ±10% -98.10% (p=0.000 n=8+8) ExtensionsByType/text/html 630ns ± 5% 522ns ± 5% -17.19% (p=0.000 n=8+7) ExtensionsByType/text/html-6 314ns ±20% 136ns ± 6% -56.80% (p=0.000 n=8+8) ExtensionsByType/text/html-48 298ns ± 4% 104ns ± 6% -65.06% (p=0.000 n=8+8) ExtensionsByType/text/html;_charset=utf-8 1.12µs ± 3% 1.05µs ± 7% -6.19% (p=0.004 n=8+7) ExtensionsByType/text/html;_charset=utf-8-6 402ns ±11% 307ns ± 4% -23.77% (p=0.000 n=8+8) ExtensionsByType/text/html;_charset=utf-8-48 422ns ± 3% 309ns ± 4% -26.86% (p=0.000 n=8+8) ExtensionsByType/application/octet-stream 810ns ± 2% 747ns ± 5% -7.74% (p=0.000 n=8+8) ExtensionsByType/application/octet-stream-6 289ns ± 9% 185ns ± 8% -36.15% (p=0.000 n=7+8) ExtensionsByType/application/octet-stream-48 267ns ± 6% 94ns ± 2% -64.91% (p=0.000 n=8+7) name old alloc/op new alloc/op delta QEncodeWord 48.0B ± 0% 48.0B ± 0% ~ (all equal) QEncodeWord-6 48.0B ± 0% 48.0B ± 0% ~ (all equal) QEncodeWord-48 48.0B ± 0% 48.0B ± 0% ~ (all equal) QDecodeWord 48.0B ± 0% 48.0B ± 0% ~ (all equal) QDecodeWord-6 48.0B ± 0% 48.0B ± 0% ~ (all equal) QDecodeWord-48 48.0B ± 0% 48.0B ± 0% ~ (all equal) QDecodeHeader 48.0B ± 0% 48.0B ± 0% ~ (all equal) QDecodeHeader-6 48.0B ± 0% 48.0B ± 0% ~ (all equal) QDecodeHeader-48 48.0B ± 0% 48.0B ± 0% ~ (all equal) TypeByExtension/.html 0.00B 0.00B ~ (all equal) TypeByExtension/.html-6 0.00B 0.00B ~ (all equal) TypeByExtension/.html-48 0.00B 0.00B ~ (all equal) TypeByExtension/.HTML 0.00B 0.00B ~ (all equal) TypeByExtension/.HTML-6 0.00B 0.00B ~ (all equal) TypeByExtension/.HTML-48 0.00B 0.00B ~ (all equal) TypeByExtension/.unused 0.00B 0.00B ~ (all equal) TypeByExtension/.unused-6 0.00B 0.00B ~ (all equal) TypeByExtension/.unused-48 0.00B 0.00B ~ (all equal) ExtensionsByType/text/html 192B ± 0% 176B ± 0% -8.33% (p=0.000 n=8+8) ExtensionsByType/text/html-6 192B ± 0% 176B ± 0% -8.33% (p=0.000 n=8+8) ExtensionsByType/text/html-48 192B ± 0% 176B ± 0% -8.33% (p=0.000 n=8+8) ExtensionsByType/text/html;_charset=utf-8 480B ± 0% 464B ± 0% -3.33% (p=0.000 n=8+8) ExtensionsByType/text/html;_charset=utf-8-6 480B ± 0% 464B ± 0% -3.33% (p=0.000 n=8+8) ExtensionsByType/text/html;_charset=utf-8-48 480B ± 0% 464B ± 0% -3.33% (p=0.000 n=8+8) ExtensionsByType/application/octet-stream 160B ± 0% 160B ± 0% ~ (all equal) ExtensionsByType/application/octet-stream-6 160B ± 0% 160B ± 0% ~ (all equal) ExtensionsByType/application/octet-stream-48 160B ± 0% 160B ± 0% ~ (all equal) name old allocs/op new allocs/op delta QEncodeWord 1.00 ± 0% 1.00 ± 0% ~ (all equal) QEncodeWord-6 1.00 ± 0% 1.00 ± 0% ~ (all equal) QEncodeWord-48 1.00 ± 0% 1.00 ± 0% ~ (all equal) QDecodeWord 2.00 ± 0% 2.00 ± 0% ~ (all equal) QDecodeWord-6 2.00 ± 0% 2.00 ± 0% ~ (all equal) QDecodeWord-48 2.00 ± 0% 2.00 ± 0% ~ (all equal) QDecodeHeader 2.00 ± 0% 2.00 ± 0% ~ (all equal) QDecodeHeader-6 2.00 ± 0% 2.00 ± 0% ~ (all equal) QDecodeHeader-48 2.00 ± 0% 2.00 ± 0% ~ (all equal) TypeByExtension/.html 0.00 0.00 ~ (all equal) TypeByExtension/.html-6 0.00 0.00 ~ (all equal) TypeByExtension/.html-48 0.00 0.00 ~ (all equal) TypeByExtension/.HTML 0.00 0.00 ~ (all equal) TypeByExtension/.HTML-6 0.00 0.00 ~ (all equal) TypeByExtension/.HTML-48 0.00 0.00 ~ (all equal) TypeByExtension/.unused 0.00 0.00 ~ (all equal) TypeByExtension/.unused-6 0.00 0.00 ~ (all equal) TypeByExtension/.unused-48 0.00 0.00 ~ (all equal) ExtensionsByType/text/html 3.00 ± 0% 3.00 ± 0% ~ (all equal) ExtensionsByType/text/html-6 3.00 ± 0% 3.00 ± 0% ~ (all equal) ExtensionsByType/text/html-48 3.00 ± 0% 3.00 ± 0% ~ (all equal) ExtensionsByType/text/html;_charset=utf-8 4.00 ± 0% 4.00 ± 0% ~ (all equal) ExtensionsByType/text/html;_charset=utf-8-6 4.00 ± 0% 4.00 ± 0% ~ (all equal) ExtensionsByType/text/html;_charset=utf-8-48 4.00 ± 0% 4.00 ± 0% ~ (all equal) ExtensionsByType/application/octet-stream 2.00 ± 0% 2.00 ± 0% ~ (all equal) ExtensionsByType/application/octet-stream-6 2.00 ± 0% 2.00 ± 0% ~ (all equal) ExtensionsByType/application/octet-stream-48 2.00 ± 0% 2.00 ± 0% ~ (all equal) https://perf.golang.org/search?q=upload:20170427.4 Change-Id: I35438be087ad6eb3d5da9119b395723ea5babaf6 Reviewed-on: https://go-review.googlesource.com/41990 Run-TryBot: Bryan Mills <[email protected]> TryBot-Result: Gobot Gobot <[email protected]> Reviewed-by: Ian Lance Taylor <[email protected]>
1 parent ce5263f commit e8d7e5d

File tree

1 file changed

+56
-52
lines changed

1 file changed

+56
-52
lines changed

src/mime/type.go

+56-52
Original file line numberDiff line numberDiff line change
@@ -12,24 +12,48 @@ import (
1212
)
1313

1414
var (
15-
mimeLock sync.RWMutex // guards following 3 maps
16-
mimeTypes map[string]string // ".Z" => "application/x-compress"
17-
mimeTypesLower map[string]string // ".z" => "application/x-compress"
15+
mimeTypes sync.Map // map[string]string; ".Z" => "application/x-compress"
16+
mimeTypesLower sync.Map // map[string]string; ".z" => "application/x-compress"
1817

1918
// extensions maps from MIME type to list of lowercase file
2019
// extensions: "image/jpeg" => [".jpg", ".jpeg"]
21-
extensions map[string][]string
20+
extensionsMu sync.Mutex // Guards stores (but not loads) on extensions.
21+
extensions sync.Map // map[string][]string; slice values are append-only.
2222
)
2323

24+
func clearSyncMap(m *sync.Map) {
25+
m.Range(func(k, _ interface{}) bool {
26+
m.Delete(k)
27+
return true
28+
})
29+
}
30+
2431
// setMimeTypes is used by initMime's non-test path, and by tests.
25-
// The two maps must not be the same, or nil.
2632
func setMimeTypes(lowerExt, mixExt map[string]string) {
27-
if lowerExt == nil || mixExt == nil {
28-
panic("nil map")
33+
clearSyncMap(&mimeTypes)
34+
clearSyncMap(&mimeTypesLower)
35+
clearSyncMap(&extensions)
36+
37+
for k, v := range lowerExt {
38+
mimeTypesLower.Store(k, v)
39+
}
40+
for k, v := range mixExt {
41+
mimeTypes.Store(k, v)
42+
}
43+
44+
extensionsMu.Lock()
45+
defer extensionsMu.Unlock()
46+
for k, v := range lowerExt {
47+
justType, _, err := ParseMediaType(v)
48+
if err != nil {
49+
panic(err)
50+
}
51+
var exts []string
52+
if ei, ok := extensions.Load(k); ok {
53+
exts = ei.([]string)
54+
}
55+
extensions.Store(justType, append(exts, k))
2956
}
30-
mimeTypesLower = lowerExt
31-
mimeTypes = mixExt
32-
extensions = invert(lowerExt)
3357
}
3458

3559
var builtinTypesLower = map[string]string{
@@ -45,29 +69,6 @@ var builtinTypesLower = map[string]string{
4569
".xml": "text/xml; charset=utf-8",
4670
}
4771

48-
func clone(m map[string]string) map[string]string {
49-
m2 := make(map[string]string, len(m))
50-
for k, v := range m {
51-
m2[k] = v
52-
if strings.ToLower(k) != k {
53-
panic("keys in builtinTypesLower must be lowercase")
54-
}
55-
}
56-
return m2
57-
}
58-
59-
func invert(m map[string]string) map[string][]string {
60-
m2 := make(map[string][]string, len(m))
61-
for k, v := range m {
62-
justType, _, err := ParseMediaType(v)
63-
if err != nil {
64-
panic(err)
65-
}
66-
m2[justType] = append(m2[justType], k)
67-
}
68-
return m2
69-
}
70-
7172
var once sync.Once // guards initMime
7273

7374
var testInitMime, osInitMime func()
@@ -76,7 +77,7 @@ func initMime() {
7677
if fn := testInitMime; fn != nil {
7778
fn()
7879
} else {
79-
setMimeTypes(builtinTypesLower, clone(builtinTypesLower))
80+
setMimeTypes(builtinTypesLower, builtinTypesLower)
8081
osInitMime()
8182
}
8283
}
@@ -100,12 +101,10 @@ func initMime() {
100101
// Text types have the charset parameter set to "utf-8" by default.
101102
func TypeByExtension(ext string) string {
102103
once.Do(initMime)
103-
mimeLock.RLock()
104-
defer mimeLock.RUnlock()
105104

106105
// Case-sensitive lookup.
107-
if v := mimeTypes[ext]; v != "" {
108-
return v
106+
if v, ok := mimeTypes.Load(ext); ok {
107+
return v.(string)
109108
}
110109

111110
// Case-insensitive lookup.
@@ -118,17 +117,19 @@ func TypeByExtension(ext string) string {
118117
c := ext[i]
119118
if c >= utf8RuneSelf {
120119
// Slow path.
121-
return mimeTypesLower[strings.ToLower(ext)]
120+
si, _ := mimeTypesLower.Load(strings.ToLower(ext))
121+
s, _ := si.(string)
122+
return s
122123
}
123124
if 'A' <= c && c <= 'Z' {
124125
lower = append(lower, c+('a'-'A'))
125126
} else {
126127
lower = append(lower, c)
127128
}
128129
}
129-
// The conversion from []byte to string doesn't allocate in
130-
// a map lookup.
131-
return mimeTypesLower[string(lower)]
130+
si, _ := mimeTypesLower.Load(string(lower))
131+
s, _ := si.(string)
132+
return s
132133
}
133134

134135
// ExtensionsByType returns the extensions known to be associated with the MIME
@@ -142,13 +143,11 @@ func ExtensionsByType(typ string) ([]string, error) {
142143
}
143144

144145
once.Do(initMime)
145-
mimeLock.RLock()
146-
defer mimeLock.RUnlock()
147-
s, ok := extensions[justType]
146+
s, ok := extensions.Load(justType)
148147
if !ok {
149148
return nil, nil
150149
}
151-
return append([]string{}, s...), nil
150+
return append([]string{}, s.([]string)...), nil
152151
}
153152

154153
// AddExtensionType sets the MIME type associated with
@@ -173,15 +172,20 @@ func setExtensionType(extension, mimeType string) error {
173172
}
174173
extLower := strings.ToLower(extension)
175174

176-
mimeLock.Lock()
177-
defer mimeLock.Unlock()
178-
mimeTypes[extension] = mimeType
179-
mimeTypesLower[extLower] = mimeType
180-
for _, v := range extensions[justType] {
175+
mimeTypes.Store(extension, mimeType)
176+
mimeTypesLower.Store(extLower, mimeType)
177+
178+
extensionsMu.Lock()
179+
defer extensionsMu.Unlock()
180+
var exts []string
181+
if ei, ok := extensions.Load(justType); ok {
182+
exts = ei.([]string)
183+
}
184+
for _, v := range exts {
181185
if v == extLower {
182186
return nil
183187
}
184188
}
185-
extensions[justType] = append(extensions[justType], extLower)
189+
extensions.Store(justType, append(exts, extLower))
186190
return nil
187191
}

0 commit comments

Comments
 (0)