stack.go 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832
  1. // Copyright 2015 Marc-Antoine Ruel. All rights reserved.
  2. // Use of this source code is governed under the Apache License, Version 2.0
  3. // that can be found in the LICENSE file.
  4. // Package stack analyzes stack dump of Go processes and simplifies it.
  5. //
  6. // It is mostly useful on servers will large number of identical goroutines,
  7. // making the crash dump harder to read than strictly necesary.
  8. package stack
  9. import (
  10. "bufio"
  11. "bytes"
  12. "errors"
  13. "fmt"
  14. "io"
  15. "math"
  16. "net/url"
  17. "os"
  18. "path/filepath"
  19. "regexp"
  20. "runtime"
  21. "sort"
  22. "strconv"
  23. "strings"
  24. "unicode"
  25. "unicode/utf8"
  26. )
  27. const lockedToThread = "locked to thread"
  28. var (
  29. // TODO(maruel): Handle corrupted stack cases:
  30. // - missed stack barrier
  31. // - found next stack barrier at 0x123; expected
  32. // - runtime: unexpected return pc for FUNC_NAME called from 0x123
  33. reRoutineHeader = regexp.MustCompile("^goroutine (\\d+) \\[([^\\]]+)\\]\\:\n$")
  34. reMinutes = regexp.MustCompile("^(\\d+) minutes$")
  35. reUnavail = regexp.MustCompile("^(?:\t| +)goroutine running on other thread; stack unavailable")
  36. // See gentraceback() in src/runtime/traceback.go for more information.
  37. // - Sometimes the source file comes up as "<autogenerated>". It is the
  38. // compiler than generated these, not the runtime.
  39. // - The tab may be replaced with spaces when a user copy-paste it, handle
  40. // this transparently.
  41. // - "runtime.gopanic" is explicitly replaced with "panic" by gentraceback().
  42. // - The +0x123 byte offset is printed when frame.pc > _func.entry. _func is
  43. // generated by the linker.
  44. // - The +0x123 byte offset is not included with generated code, e.g. unnamed
  45. // functions "func·006()" which is generally go func() { ... }()
  46. // statements. Since the _func is generated at runtime, it's probably why
  47. // _func.entry is not set.
  48. // - C calls may have fp=0x123 sp=0x123 appended. I think it normally happens
  49. // when a signal is not correctly handled. It is printed with m.throwing>0.
  50. // These are discarded.
  51. // - For cgo, the source file may be "??".
  52. reFile = regexp.MustCompile("^(?:\t| +)(\\?\\?|\\<autogenerated\\>|.+\\.(?:c|go|s))\\:(\\d+)(?:| \\+0x[0-9a-f]+)(?:| fp=0x[0-9a-f]+ sp=0x[0-9a-f]+)\n$")
  53. // Sadly, it doesn't note the goroutine number so we could cascade them per
  54. // parenthood.
  55. reCreated = regexp.MustCompile("^created by (.+)\n$")
  56. reFunc = regexp.MustCompile("^(.+)\\((.*)\\)\n$")
  57. reElided = regexp.MustCompile("^\\.\\.\\.additional frames elided\\.\\.\\.\n$")
  58. // Include frequent GOROOT value on Windows, distro provided and user
  59. // installed path. This simplifies the user's life when processing a trace
  60. // generated on another VM.
  61. // TODO(maruel): Guess the path automatically via traces containing the
  62. // 'runtime' package, which is very frequent. This would be "less bad" than
  63. // throwing up random values at the parser.
  64. goroots = []string{runtime.GOROOT(), "c:/go", "/usr/lib/go", "/usr/local/go"}
  65. )
  66. // Similarity is the level at which two call lines arguments must match to be
  67. // considered similar enough to coalesce them.
  68. type Similarity int
  69. const (
  70. // ExactFlags requires same bits (e.g. Locked).
  71. ExactFlags Similarity = iota
  72. // ExactLines requests the exact same arguments on the call line.
  73. ExactLines
  74. // AnyPointer considers different pointers a similar call line.
  75. AnyPointer
  76. // AnyValue accepts any value as similar call line.
  77. AnyValue
  78. )
  79. // Function is a function call.
  80. //
  81. // Go stack traces print a mangled function call, this wrapper unmangle the
  82. // string before printing and adds other filtering methods.
  83. type Function struct {
  84. Raw string
  85. }
  86. // String is the fully qualified function name.
  87. //
  88. // Sadly Go is a bit confused when the package name doesn't match the directory
  89. // containing the source file and will use the directory name instead of the
  90. // real package name.
  91. func (f Function) String() string {
  92. s, _ := url.QueryUnescape(f.Raw)
  93. return s
  94. }
  95. // Name is the naked function name.
  96. func (f Function) Name() string {
  97. parts := strings.SplitN(filepath.Base(f.Raw), ".", 2)
  98. if len(parts) == 1 {
  99. return parts[0]
  100. }
  101. return parts[1]
  102. }
  103. // PkgName is the package name for this function reference.
  104. func (f Function) PkgName() string {
  105. parts := strings.SplitN(filepath.Base(f.Raw), ".", 2)
  106. if len(parts) == 1 {
  107. return ""
  108. }
  109. s, _ := url.QueryUnescape(parts[0])
  110. return s
  111. }
  112. // PkgDotName returns "<package>.<func>" format.
  113. func (f Function) PkgDotName() string {
  114. parts := strings.SplitN(filepath.Base(f.Raw), ".", 2)
  115. s, _ := url.QueryUnescape(parts[0])
  116. if len(parts) == 1 {
  117. return parts[0]
  118. }
  119. if s != "" || parts[1] != "" {
  120. return s + "." + parts[1]
  121. }
  122. return ""
  123. }
  124. // IsExported returns true if the function is exported.
  125. func (f Function) IsExported() bool {
  126. name := f.Name()
  127. parts := strings.Split(name, ".")
  128. r, _ := utf8.DecodeRuneInString(parts[len(parts)-1])
  129. if unicode.ToUpper(r) == r {
  130. return true
  131. }
  132. return f.PkgName() == "main" && name == "main"
  133. }
  134. // Arg is an argument on a Call.
  135. type Arg struct {
  136. Value uint64 // Value is the raw value as found in the stack trace
  137. Name string // Name is a pseudo name given to the argument
  138. }
  139. // IsPtr returns true if we guess it's a pointer. It's only a guess, it can be
  140. // easily be confused by a bitmask.
  141. func (a *Arg) IsPtr() bool {
  142. // Assumes all pointers are above 16Mb and positive.
  143. return a.Value > 16*1024*1024 && a.Value < math.MaxInt64
  144. }
  145. func (a Arg) String() string {
  146. if a.Name != "" {
  147. return a.Name
  148. }
  149. if a.Value == 0 {
  150. return "0"
  151. }
  152. return fmt.Sprintf("0x%x", a.Value)
  153. }
  154. // Args is a series of function call arguments.
  155. type Args struct {
  156. Values []Arg // Values is the arguments as shown on the stack trace. They are mangled via simplification.
  157. Processed []string // Processed is the arguments generated from processing the source files. It can have a length lower than Values.
  158. Elided bool // If set, it means there was a trailing ", ..."
  159. }
  160. func (a Args) String() string {
  161. var v []string
  162. if len(a.Processed) != 0 {
  163. v = make([]string, 0, len(a.Processed))
  164. for _, item := range a.Processed {
  165. v = append(v, item)
  166. }
  167. } else {
  168. v = make([]string, 0, len(a.Values))
  169. for _, item := range a.Values {
  170. v = append(v, item.String())
  171. }
  172. }
  173. if a.Elided {
  174. v = append(v, "...")
  175. }
  176. return strings.Join(v, ", ")
  177. }
  178. // Equal returns true only if both arguments are exactly equal.
  179. func (a *Args) Equal(r *Args) bool {
  180. if a.Elided != r.Elided || len(a.Values) != len(r.Values) {
  181. return false
  182. }
  183. for i, l := range a.Values {
  184. if l != r.Values[i] {
  185. return false
  186. }
  187. }
  188. return true
  189. }
  190. // Similar returns true if the two Args are equal or almost but not quite
  191. // equal.
  192. func (a *Args) Similar(r *Args, similar Similarity) bool {
  193. if a.Elided != r.Elided || len(a.Values) != len(r.Values) {
  194. return false
  195. }
  196. if similar == AnyValue {
  197. return true
  198. }
  199. for i, l := range a.Values {
  200. switch similar {
  201. case ExactFlags, ExactLines:
  202. if l != r.Values[i] {
  203. return false
  204. }
  205. default:
  206. if l.IsPtr() != r.Values[i].IsPtr() || (!l.IsPtr() && l != r.Values[i]) {
  207. return false
  208. }
  209. }
  210. }
  211. return true
  212. }
  213. // Merge merges two similar Args, zapping out differences.
  214. func (a *Args) Merge(r *Args) Args {
  215. out := Args{
  216. Values: make([]Arg, len(a.Values)),
  217. Elided: a.Elided,
  218. }
  219. for i, l := range a.Values {
  220. if l != r.Values[i] {
  221. out.Values[i].Name = "*"
  222. out.Values[i].Value = l.Value
  223. } else {
  224. out.Values[i] = l
  225. }
  226. }
  227. return out
  228. }
  229. // Call is an item in the stack trace.
  230. type Call struct {
  231. SourcePath string // Full path name of the source file
  232. Line int // Line number
  233. Func Function // Fully qualified function name (encoded).
  234. Args Args // Call arguments
  235. }
  236. // Equal returns true only if both calls are exactly equal.
  237. func (c *Call) Equal(r *Call) bool {
  238. return c.SourcePath == r.SourcePath && c.Line == r.Line && c.Func == r.Func && c.Args.Equal(&r.Args)
  239. }
  240. // Similar returns true if the two Call are equal or almost but not quite
  241. // equal.
  242. func (c *Call) Similar(r *Call, similar Similarity) bool {
  243. return c.SourcePath == r.SourcePath && c.Line == r.Line && c.Func == r.Func && c.Args.Similar(&r.Args, similar)
  244. }
  245. // Merge merges two similar Call, zapping out differences.
  246. func (c *Call) Merge(r *Call) Call {
  247. return Call{
  248. SourcePath: c.SourcePath,
  249. Line: c.Line,
  250. Func: c.Func,
  251. Args: c.Args.Merge(&r.Args),
  252. }
  253. }
  254. // SourceName returns the base file name of the source file.
  255. func (c *Call) SourceName() string {
  256. return filepath.Base(c.SourcePath)
  257. }
  258. // SourceLine returns "source.go:line", including only the base file name.
  259. func (c *Call) SourceLine() string {
  260. return fmt.Sprintf("%s:%d", c.SourceName(), c.Line)
  261. }
  262. // FullSourceLine returns "/path/to/source.go:line".
  263. func (c *Call) FullSourceLine() string {
  264. return fmt.Sprintf("%s:%d", c.SourcePath, c.Line)
  265. }
  266. // PkgSource is one directory plus the file name of the source file.
  267. func (c *Call) PkgSource() string {
  268. return filepath.Join(filepath.Base(filepath.Dir(c.SourcePath)), c.SourceName())
  269. }
  270. const testMainSource = "_test" + string(os.PathSeparator) + "_testmain.go"
  271. // IsStdlib returns true if it is a Go standard library function. This includes
  272. // the 'go test' generated main executable.
  273. func (c *Call) IsStdlib() bool {
  274. for _, goroot := range goroots {
  275. if strings.HasPrefix(c.SourcePath, goroot) {
  276. return true
  277. }
  278. }
  279. // Consider _test/_testmain.go as stdlib since it's injected by "go test".
  280. return c.PkgSource() == testMainSource
  281. }
  282. // IsPkgMain returns true if it is in the main package.
  283. func (c *Call) IsPkgMain() bool {
  284. return c.Func.PkgName() == "main"
  285. }
  286. // Stack is a call stack.
  287. type Stack struct {
  288. Calls []Call // Call stack. First is original function, last is leaf function.
  289. Elided bool // Happens when there's >100 items in Stack, currently hardcoded in package runtime.
  290. }
  291. // Equal returns true on if both call stacks are exactly equal.
  292. func (s *Stack) Equal(r *Stack) bool {
  293. if len(s.Calls) != len(r.Calls) || s.Elided != r.Elided {
  294. return false
  295. }
  296. for i := range s.Calls {
  297. if !s.Calls[i].Equal(&r.Calls[i]) {
  298. return false
  299. }
  300. }
  301. return true
  302. }
  303. // Similar returns true if the two Stack are equal or almost but not quite
  304. // equal.
  305. func (s *Stack) Similar(r *Stack, similar Similarity) bool {
  306. if len(s.Calls) != len(r.Calls) || s.Elided != r.Elided {
  307. return false
  308. }
  309. for i := range s.Calls {
  310. if !s.Calls[i].Similar(&r.Calls[i], similar) {
  311. return false
  312. }
  313. }
  314. return true
  315. }
  316. // Merge merges two similar Stack, zapping out differences.
  317. func (s *Stack) Merge(r *Stack) *Stack {
  318. // Assumes similar stacks have the same length.
  319. out := &Stack{
  320. Calls: make([]Call, len(s.Calls)),
  321. Elided: s.Elided,
  322. }
  323. for i := range s.Calls {
  324. out.Calls[i] = s.Calls[i].Merge(&r.Calls[i])
  325. }
  326. return out
  327. }
  328. // Less compares two Stack, where the ones that are less are more
  329. // important, so they come up front. A Stack with more private functions is
  330. // 'less' so it is at the top. Inversely, a Stack with only public
  331. // functions is 'more' so it is at the bottom.
  332. func (s *Stack) Less(r *Stack) bool {
  333. lStdlib := 0
  334. lPrivate := 0
  335. for _, c := range s.Calls {
  336. if c.IsStdlib() {
  337. lStdlib++
  338. } else {
  339. lPrivate++
  340. }
  341. }
  342. rStdlib := 0
  343. rPrivate := 0
  344. for _, s := range r.Calls {
  345. if s.IsStdlib() {
  346. rStdlib++
  347. } else {
  348. rPrivate++
  349. }
  350. }
  351. if lPrivate > rPrivate {
  352. return true
  353. }
  354. if lPrivate < rPrivate {
  355. return false
  356. }
  357. if lStdlib > rStdlib {
  358. return false
  359. }
  360. if lStdlib < rStdlib {
  361. return true
  362. }
  363. // Stack lengths are the same.
  364. for x := range s.Calls {
  365. if s.Calls[x].Func.Raw < r.Calls[x].Func.Raw {
  366. return true
  367. }
  368. if s.Calls[x].Func.Raw > r.Calls[x].Func.Raw {
  369. return true
  370. }
  371. if s.Calls[x].PkgSource() < r.Calls[x].PkgSource() {
  372. return true
  373. }
  374. if s.Calls[x].PkgSource() > r.Calls[x].PkgSource() {
  375. return true
  376. }
  377. if s.Calls[x].Line < r.Calls[x].Line {
  378. return true
  379. }
  380. if s.Calls[x].Line > r.Calls[x].Line {
  381. return true
  382. }
  383. }
  384. return false
  385. }
  386. // Signature represents the signature of one or multiple goroutines.
  387. //
  388. // It is effectively the stack trace plus the goroutine internal bits, like
  389. // it's state, if it is thread locked, which call site created this goroutine,
  390. // etc.
  391. type Signature struct {
  392. // Use git grep 'gopark(|unlock)\(' to find them all plus everything listed
  393. // in runtime/traceback.go. Valid values includes:
  394. // - chan send, chan receive, select
  395. // - finalizer wait, mark wait (idle),
  396. // - Concurrent GC wait, GC sweep wait, force gc (idle)
  397. // - IO wait, panicwait
  398. // - semacquire, semarelease
  399. // - sleep, timer goroutine (idle)
  400. // - trace reader (blocked)
  401. // Stuck cases:
  402. // - chan send (nil chan), chan receive (nil chan), select (no cases)
  403. // Runnable states:
  404. // - idle, runnable, running, syscall, waiting, dead, enqueue, copystack,
  405. // Scan states:
  406. // - scan, scanrunnable, scanrunning, scansyscall, scanwaiting, scandead,
  407. // scanenqueue
  408. State string
  409. CreatedBy Call // Which other goroutine which created this one.
  410. SleepMin int // Wait time in minutes, if applicable.
  411. SleepMax int // Wait time in minutes, if applicable.
  412. Stack Stack
  413. Locked bool // Locked to an OS thread.
  414. }
  415. // Equal returns true only if both signatures are exactly equal.
  416. func (s *Signature) Equal(r *Signature) bool {
  417. if s.State != r.State || !s.CreatedBy.Equal(&r.CreatedBy) || s.Locked != r.Locked || s.SleepMin != r.SleepMin || s.SleepMax != r.SleepMax {
  418. return false
  419. }
  420. return s.Stack.Equal(&r.Stack)
  421. }
  422. // Similar returns true if the two Signature are equal or almost but not quite
  423. // equal.
  424. func (s *Signature) Similar(r *Signature, similar Similarity) bool {
  425. if s.State != r.State || !s.CreatedBy.Similar(&r.CreatedBy, similar) {
  426. return false
  427. }
  428. if similar == ExactFlags && s.Locked != r.Locked {
  429. return false
  430. }
  431. return s.Stack.Similar(&r.Stack, similar)
  432. }
  433. // Merge merges two similar Signature, zapping out differences.
  434. func (s *Signature) Merge(r *Signature) *Signature {
  435. min := s.SleepMin
  436. if r.SleepMin < min {
  437. min = r.SleepMin
  438. }
  439. max := s.SleepMax
  440. if r.SleepMax > max {
  441. max = r.SleepMax
  442. }
  443. return &Signature{
  444. State: s.State, // Drop right side.
  445. CreatedBy: s.CreatedBy, // Drop right side.
  446. SleepMin: min,
  447. SleepMax: max,
  448. Stack: *s.Stack.Merge(&r.Stack),
  449. Locked: s.Locked || r.Locked, // TODO(maruel): This is weirdo.
  450. }
  451. }
  452. // Less compares two Signature, where the ones that are less are more
  453. // important, so they come up front. A Signature with more private functions is
  454. // 'less' so it is at the top. Inversely, a Signature with only public
  455. // functions is 'more' so it is at the bottom.
  456. func (s *Signature) Less(r *Signature) bool {
  457. if s.Stack.Less(&r.Stack) {
  458. return true
  459. }
  460. if r.Stack.Less(&s.Stack) {
  461. return false
  462. }
  463. if s.Locked && !r.Locked {
  464. return true
  465. }
  466. if r.Locked && !s.Locked {
  467. return false
  468. }
  469. if s.State < r.State {
  470. return true
  471. }
  472. if s.State > r.State {
  473. return false
  474. }
  475. return false
  476. }
  477. // Goroutine represents the state of one goroutine, including the stack trace.
  478. type Goroutine struct {
  479. Signature // It's stack trace, internal bits, state, which call site created it, etc.
  480. ID int // Goroutine ID.
  481. First bool // First is the goroutine first printed, normally the one that crashed.
  482. }
  483. // Bucketize returns the number of similar goroutines.
  484. func Bucketize(goroutines []Goroutine, similar Similarity) map[*Signature][]Goroutine {
  485. out := map[*Signature][]Goroutine{}
  486. // O(n²). Fix eventually.
  487. for _, routine := range goroutines {
  488. found := false
  489. for key := range out {
  490. // When a match is found, this effectively drops the other goroutine ID.
  491. if key.Similar(&routine.Signature, similar) {
  492. found = true
  493. if !key.Equal(&routine.Signature) {
  494. // Almost but not quite equal. There's different pointers passed
  495. // around but the same values. Zap out the different values.
  496. newKey := key.Merge(&routine.Signature)
  497. out[newKey] = append(out[key], routine)
  498. delete(out, key)
  499. } else {
  500. out[key] = append(out[key], routine)
  501. }
  502. break
  503. }
  504. }
  505. if !found {
  506. key := &Signature{}
  507. *key = routine.Signature
  508. out[key] = []Goroutine{routine}
  509. }
  510. }
  511. return out
  512. }
  513. // Bucket is a stack trace signature and the list of goroutines that fits this
  514. // signature.
  515. type Bucket struct {
  516. Signature
  517. Routines []Goroutine
  518. }
  519. // First returns true if it contains the first goroutine, e.g. the ones that
  520. // likely generated the panic() call, if any.
  521. func (b *Bucket) First() bool {
  522. for _, r := range b.Routines {
  523. if r.First {
  524. return true
  525. }
  526. }
  527. return false
  528. }
  529. // Less does reverse sort.
  530. func (b *Bucket) Less(r *Bucket) bool {
  531. if b.First() {
  532. return true
  533. }
  534. if r.First() {
  535. return false
  536. }
  537. return b.Signature.Less(&r.Signature)
  538. }
  539. // Buckets is a list of Bucket sorted by repeation count.
  540. type Buckets []Bucket
  541. func (b Buckets) Len() int {
  542. return len(b)
  543. }
  544. func (b Buckets) Less(i, j int) bool {
  545. return b[i].Less(&b[j])
  546. }
  547. func (b Buckets) Swap(i, j int) {
  548. b[j], b[i] = b[i], b[j]
  549. }
  550. // SortBuckets creates a list of Bucket from each goroutine stack trace count.
  551. func SortBuckets(buckets map[*Signature][]Goroutine) Buckets {
  552. out := make(Buckets, 0, len(buckets))
  553. for signature, count := range buckets {
  554. out = append(out, Bucket{*signature, count})
  555. }
  556. sort.Sort(out)
  557. return out
  558. }
  559. // scanLines is similar to bufio.ScanLines except that it:
  560. // - doesn't drop '\n'
  561. // - doesn't strip '\r'
  562. // - returns when the data is bufio.MaxScanTokenSize bytes
  563. func scanLines(data []byte, atEOF bool) (advance int, token []byte, err error) {
  564. if atEOF && len(data) == 0 {
  565. return 0, nil, nil
  566. }
  567. if i := bytes.IndexByte(data, '\n'); i >= 0 {
  568. return i + 1, data[0 : i+1], nil
  569. }
  570. if atEOF {
  571. return len(data), data, nil
  572. }
  573. if len(data) >= bufio.MaxScanTokenSize {
  574. // Returns the line even if it is not at EOF nor has a '\n', otherwise the
  575. // scanner will return bufio.ErrTooLong which is definitely not what we
  576. // want.
  577. return len(data), data, nil
  578. }
  579. return 0, nil, nil
  580. }
  581. // ParseDump processes the output from runtime.Stack().
  582. //
  583. // It supports piping from another command and assumes there is junk before the
  584. // actual stack trace. The junk is streamed to out.
  585. func ParseDump(r io.Reader, out io.Writer) ([]Goroutine, error) {
  586. goroutines := make([]Goroutine, 0, 16)
  587. var goroutine *Goroutine
  588. scanner := bufio.NewScanner(r)
  589. scanner.Split(scanLines)
  590. // TODO(maruel): Use a formal state machine. Patterns follows:
  591. // - reRoutineHeader
  592. // Either:
  593. // - reUnavail
  594. // - reFunc + reFile in a loop
  595. // - reElided
  596. // Optionally ends with:
  597. // - reCreated + reFile
  598. // Between each goroutine stack dump: an empty line
  599. created := false
  600. // firstLine is the first line after the reRoutineHeader header line.
  601. firstLine := false
  602. for scanner.Scan() {
  603. line := scanner.Text()
  604. if line == "\n" {
  605. if goroutine != nil {
  606. goroutine = nil
  607. continue
  608. }
  609. } else if line[len(line)-1] == '\n' {
  610. if goroutine == nil {
  611. if match := reRoutineHeader.FindStringSubmatch(line); match != nil {
  612. if id, err := strconv.Atoi(match[1]); err == nil {
  613. // See runtime/traceback.go.
  614. // "<state>, \d+ minutes, locked to thread"
  615. items := strings.Split(match[2], ", ")
  616. sleep := 0
  617. locked := false
  618. for i := 1; i < len(items); i++ {
  619. if items[i] == lockedToThread {
  620. locked = true
  621. continue
  622. }
  623. // Look for duration, if any.
  624. if match2 := reMinutes.FindStringSubmatch(items[i]); match2 != nil {
  625. sleep, _ = strconv.Atoi(match2[1])
  626. }
  627. }
  628. goroutines = append(goroutines, Goroutine{
  629. Signature: Signature{
  630. State: items[0],
  631. SleepMin: sleep,
  632. SleepMax: sleep,
  633. Locked: locked,
  634. },
  635. ID: id,
  636. First: len(goroutines) == 0,
  637. })
  638. goroutine = &goroutines[len(goroutines)-1]
  639. firstLine = true
  640. continue
  641. }
  642. }
  643. } else {
  644. if firstLine {
  645. firstLine = false
  646. if match := reUnavail.FindStringSubmatch(line); match != nil {
  647. // Generate a fake stack entry.
  648. goroutine.Stack.Calls = []Call{{SourcePath: "<unavailable>"}}
  649. continue
  650. }
  651. }
  652. if match := reFile.FindStringSubmatch(line); match != nil {
  653. // Triggers after a reFunc or a reCreated.
  654. num, err := strconv.Atoi(match[2])
  655. if err != nil {
  656. return goroutines, fmt.Errorf("failed to parse int on line: \"%s\"", line)
  657. }
  658. if created {
  659. created = false
  660. goroutine.CreatedBy.SourcePath = match[1]
  661. goroutine.CreatedBy.Line = num
  662. } else {
  663. i := len(goroutine.Stack.Calls) - 1
  664. if i < 0 {
  665. return goroutines, errors.New("unexpected order")
  666. }
  667. goroutine.Stack.Calls[i].SourcePath = match[1]
  668. goroutine.Stack.Calls[i].Line = num
  669. }
  670. continue
  671. }
  672. if match := reCreated.FindStringSubmatch(line); match != nil {
  673. created = true
  674. goroutine.CreatedBy.Func.Raw = match[1]
  675. continue
  676. }
  677. if match := reFunc.FindStringSubmatch(line); match != nil {
  678. args := Args{}
  679. for _, a := range strings.Split(match[2], ", ") {
  680. if a == "..." {
  681. args.Elided = true
  682. continue
  683. }
  684. if a == "" {
  685. // Remaining values were dropped.
  686. break
  687. }
  688. v, err := strconv.ParseUint(a, 0, 64)
  689. if err != nil {
  690. return goroutines, fmt.Errorf("failed to parse int on line: \"%s\"", line)
  691. }
  692. args.Values = append(args.Values, Arg{Value: v})
  693. }
  694. goroutine.Stack.Calls = append(goroutine.Stack.Calls, Call{Func: Function{match[1]}, Args: args})
  695. continue
  696. }
  697. if match := reElided.FindStringSubmatch(line); match != nil {
  698. goroutine.Stack.Elided = true
  699. continue
  700. }
  701. }
  702. }
  703. _, _ = io.WriteString(out, line)
  704. goroutine = nil
  705. }
  706. nameArguments(goroutines)
  707. return goroutines, scanner.Err()
  708. }
  709. // Private stuff.
  710. func nameArguments(goroutines []Goroutine) {
  711. // Set a name for any pointer occuring more than once.
  712. type object struct {
  713. args []*Arg
  714. inPrimary bool
  715. id int
  716. }
  717. objects := map[uint64]object{}
  718. // Enumerate all the arguments.
  719. for i := range goroutines {
  720. for j := range goroutines[i].Stack.Calls {
  721. for k := range goroutines[i].Stack.Calls[j].Args.Values {
  722. arg := goroutines[i].Stack.Calls[j].Args.Values[k]
  723. if arg.IsPtr() {
  724. objects[arg.Value] = object{
  725. args: append(objects[arg.Value].args, &goroutines[i].Stack.Calls[j].Args.Values[k]),
  726. inPrimary: objects[arg.Value].inPrimary || i == 0,
  727. }
  728. }
  729. }
  730. }
  731. // CreatedBy.Args is never set.
  732. }
  733. order := uint64Slice{}
  734. for k, obj := range objects {
  735. if len(obj.args) > 1 && obj.inPrimary {
  736. order = append(order, k)
  737. }
  738. }
  739. sort.Sort(order)
  740. nextID := 1
  741. for _, k := range order {
  742. for _, arg := range objects[k].args {
  743. arg.Name = fmt.Sprintf("#%d", nextID)
  744. }
  745. nextID++
  746. }
  747. // Now do the rest. This is done so the output is deterministic.
  748. order = uint64Slice{}
  749. for k := range objects {
  750. order = append(order, k)
  751. }
  752. sort.Sort(order)
  753. for _, k := range order {
  754. // Process the remaining pointers, they were not referenced by primary
  755. // thread so will have higher IDs.
  756. if objects[k].inPrimary {
  757. continue
  758. }
  759. for _, arg := range objects[k].args {
  760. arg.Name = fmt.Sprintf("#%d", nextID)
  761. }
  762. nextID++
  763. }
  764. }
  765. type uint64Slice []uint64
  766. func (a uint64Slice) Len() int { return len(a) }
  767. func (a uint64Slice) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
  768. func (a uint64Slice) Less(i, j int) bool { return a[i] < a[j] }