local.go 7.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337
  1. // Copyright (c) 2017 Uber Technologies, Inc.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package metrics
  15. import (
  16. "sort"
  17. "sync"
  18. "sync/atomic"
  19. "time"
  20. "github.com/codahale/hdrhistogram"
  21. )
  22. // This is intentionally very similar to github.com/codahale/metrics, the
  23. // main difference being that counters/gauges are scoped to the provider
  24. // rather than being global (to facilitate testing).
  25. // A LocalBackend is a metrics provider which aggregates data in-vm, and
  26. // allows exporting snapshots to shove the data into a remote collector
  27. type LocalBackend struct {
  28. cm sync.Mutex
  29. gm sync.Mutex
  30. tm sync.Mutex
  31. counters map[string]*int64
  32. gauges map[string]*int64
  33. timers map[string]*localBackendTimer
  34. stop chan struct{}
  35. wg sync.WaitGroup
  36. TagsSep string
  37. TagKVSep string
  38. }
  39. // NewLocalBackend returns a new LocalBackend. The collectionInterval is the histogram
  40. // time window for each timer.
  41. func NewLocalBackend(collectionInterval time.Duration) *LocalBackend {
  42. b := &LocalBackend{
  43. counters: make(map[string]*int64),
  44. gauges: make(map[string]*int64),
  45. timers: make(map[string]*localBackendTimer),
  46. stop: make(chan struct{}),
  47. TagsSep: "|",
  48. TagKVSep: "=",
  49. }
  50. if collectionInterval == 0 {
  51. // Use one histogram time window for all timers
  52. return b
  53. }
  54. b.wg.Add(1)
  55. go b.runLoop(collectionInterval)
  56. return b
  57. }
  58. // Clear discards accumulated stats
  59. func (b *LocalBackend) Clear() {
  60. b.cm.Lock()
  61. defer b.cm.Unlock()
  62. b.gm.Lock()
  63. defer b.gm.Unlock()
  64. b.tm.Lock()
  65. defer b.tm.Unlock()
  66. b.counters = make(map[string]*int64)
  67. b.gauges = make(map[string]*int64)
  68. b.timers = make(map[string]*localBackendTimer)
  69. }
  70. func (b *LocalBackend) runLoop(collectionInterval time.Duration) {
  71. defer b.wg.Done()
  72. ticker := time.NewTicker(collectionInterval)
  73. for {
  74. select {
  75. case <-ticker.C:
  76. b.tm.Lock()
  77. timers := make(map[string]*localBackendTimer, len(b.timers))
  78. for timerName, timer := range b.timers {
  79. timers[timerName] = timer
  80. }
  81. b.tm.Unlock()
  82. for _, t := range timers {
  83. t.Lock()
  84. t.hist.Rotate()
  85. t.Unlock()
  86. }
  87. case <-b.stop:
  88. ticker.Stop()
  89. return
  90. }
  91. }
  92. }
  93. // IncCounter increments a counter value
  94. func (b *LocalBackend) IncCounter(name string, tags map[string]string, delta int64) {
  95. name = GetKey(name, tags, b.TagsSep, b.TagKVSep)
  96. b.cm.Lock()
  97. defer b.cm.Unlock()
  98. counter := b.counters[name]
  99. if counter == nil {
  100. b.counters[name] = new(int64)
  101. *b.counters[name] = delta
  102. return
  103. }
  104. atomic.AddInt64(counter, delta)
  105. }
  106. // UpdateGauge updates the value of a gauge
  107. func (b *LocalBackend) UpdateGauge(name string, tags map[string]string, value int64) {
  108. name = GetKey(name, tags, b.TagsSep, b.TagKVSep)
  109. b.gm.Lock()
  110. defer b.gm.Unlock()
  111. gauge := b.gauges[name]
  112. if gauge == nil {
  113. b.gauges[name] = new(int64)
  114. *b.gauges[name] = value
  115. return
  116. }
  117. atomic.StoreInt64(gauge, value)
  118. }
  119. // RecordTimer records a timing duration
  120. func (b *LocalBackend) RecordTimer(name string, tags map[string]string, d time.Duration) {
  121. name = GetKey(name, tags, b.TagsSep, b.TagKVSep)
  122. timer := b.findOrCreateTimer(name)
  123. timer.Lock()
  124. timer.hist.Current.RecordValue(int64(d / time.Millisecond))
  125. timer.Unlock()
  126. }
  127. func (b *LocalBackend) findOrCreateTimer(name string) *localBackendTimer {
  128. b.tm.Lock()
  129. defer b.tm.Unlock()
  130. if t, ok := b.timers[name]; ok {
  131. return t
  132. }
  133. t := &localBackendTimer{
  134. hist: hdrhistogram.NewWindowed(5, 0, int64((5*time.Minute)/time.Millisecond), 1),
  135. }
  136. b.timers[name] = t
  137. return t
  138. }
  139. type localBackendTimer struct {
  140. sync.Mutex
  141. hist *hdrhistogram.WindowedHistogram
  142. }
  143. var (
  144. percentiles = map[string]float64{
  145. "P50": 50,
  146. "P75": 75,
  147. "P90": 90,
  148. "P95": 95,
  149. "P99": 99,
  150. "P999": 99.9,
  151. }
  152. )
  153. // Snapshot captures a snapshot of the current counter and gauge values
  154. func (b *LocalBackend) Snapshot() (counters, gauges map[string]int64) {
  155. b.cm.Lock()
  156. defer b.cm.Unlock()
  157. counters = make(map[string]int64, len(b.counters))
  158. for name, value := range b.counters {
  159. counters[name] = atomic.LoadInt64(value)
  160. }
  161. b.gm.Lock()
  162. defer b.gm.Unlock()
  163. gauges = make(map[string]int64, len(b.gauges))
  164. for name, value := range b.gauges {
  165. gauges[name] = atomic.LoadInt64(value)
  166. }
  167. b.tm.Lock()
  168. timers := make(map[string]*localBackendTimer)
  169. for timerName, timer := range b.timers {
  170. timers[timerName] = timer
  171. }
  172. b.tm.Unlock()
  173. for timerName, timer := range timers {
  174. timer.Lock()
  175. hist := timer.hist.Merge()
  176. timer.Unlock()
  177. for name, q := range percentiles {
  178. gauges[timerName+"."+name] = hist.ValueAtQuantile(q)
  179. }
  180. }
  181. return
  182. }
  183. // Stop cleanly closes the background goroutine spawned by NewLocalBackend.
  184. func (b *LocalBackend) Stop() {
  185. close(b.stop)
  186. b.wg.Wait()
  187. }
  188. // GetKey converts name+tags into a single string of the form
  189. // "name|tag1=value1|...|tagN=valueN", where tag names are
  190. // sorted alphabetically.
  191. func GetKey(name string, tags map[string]string, tagsSep string, tagKVSep string) string {
  192. keys := make([]string, 0, len(tags))
  193. for k := range tags {
  194. keys = append(keys, k)
  195. }
  196. sort.Strings(keys)
  197. key := name
  198. for _, k := range keys {
  199. key = key + tagsSep + k + tagKVSep + tags[k]
  200. }
  201. return key
  202. }
  203. type stats struct {
  204. name string
  205. tags map[string]string
  206. localBackend *LocalBackend
  207. }
  208. type localTimer struct {
  209. stats
  210. }
  211. func (l *localTimer) Record(d time.Duration) {
  212. l.localBackend.RecordTimer(l.name, l.tags, d)
  213. }
  214. type localCounter struct {
  215. stats
  216. }
  217. func (l *localCounter) Inc(delta int64) {
  218. l.localBackend.IncCounter(l.name, l.tags, delta)
  219. }
  220. type localGauge struct {
  221. stats
  222. }
  223. func (l *localGauge) Update(value int64) {
  224. l.localBackend.UpdateGauge(l.name, l.tags, value)
  225. }
  226. // LocalFactory stats factory that creates metrics that are stored locally
  227. type LocalFactory struct {
  228. *LocalBackend
  229. namespace string
  230. tags map[string]string
  231. }
  232. // NewLocalFactory returns a new LocalMetricsFactory
  233. func NewLocalFactory(collectionInterval time.Duration) *LocalFactory {
  234. return &LocalFactory{
  235. LocalBackend: NewLocalBackend(collectionInterval),
  236. }
  237. }
  238. // appendTags adds the tags to the namespace tags and returns a combined map.
  239. func (l *LocalFactory) appendTags(tags map[string]string) map[string]string {
  240. newTags := make(map[string]string)
  241. for k, v := range l.tags {
  242. newTags[k] = v
  243. }
  244. for k, v := range tags {
  245. newTags[k] = v
  246. }
  247. return newTags
  248. }
  249. func (l *LocalFactory) newNamespace(name string) string {
  250. if l.namespace == "" {
  251. return name
  252. }
  253. if name == "" {
  254. return l.namespace
  255. }
  256. return l.namespace + "." + name
  257. }
  258. // Counter returns a local stats counter
  259. func (l *LocalFactory) Counter(name string, tags map[string]string) Counter {
  260. return &localCounter{
  261. stats{
  262. name: l.newNamespace(name),
  263. tags: l.appendTags(tags),
  264. localBackend: l.LocalBackend,
  265. },
  266. }
  267. }
  268. // Timer returns a local stats timer.
  269. func (l *LocalFactory) Timer(name string, tags map[string]string) Timer {
  270. return &localTimer{
  271. stats{
  272. name: l.newNamespace(name),
  273. tags: l.appendTags(tags),
  274. localBackend: l.LocalBackend,
  275. },
  276. }
  277. }
  278. // Gauge returns a local stats gauge.
  279. func (l *LocalFactory) Gauge(name string, tags map[string]string) Gauge {
  280. return &localGauge{
  281. stats{
  282. name: l.newNamespace(name),
  283. tags: l.appendTags(tags),
  284. localBackend: l.LocalBackend,
  285. },
  286. }
  287. }
  288. // Namespace returns a new namespace.
  289. func (l *LocalFactory) Namespace(name string, tags map[string]string) Factory {
  290. return &LocalFactory{
  291. namespace: l.newNamespace(name),
  292. tags: l.appendTags(tags),
  293. LocalBackend: l.LocalBackend,
  294. }
  295. }