iterator_test.go 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723
  1. // Copyright 2019 The go-ethereum Authors
  2. // This file is part of the go-ethereum library.
  3. //
  4. // The go-ethereum library is free software: you can redistribute it and/or modify
  5. // it under the terms of the GNU Lesser General Public License as published by
  6. // the Free Software Foundation, either version 3 of the License, or
  7. // (at your option) any later version.
  8. //
  9. // The go-ethereum library is distributed in the hope that it will be useful,
  10. // but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12. // GNU Lesser General Public License for more details.
  13. //
  14. // You should have received a copy of the GNU Lesser General Public License
  15. // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
  16. package snapshot
  17. import (
  18. "bytes"
  19. "encoding/binary"
  20. "fmt"
  21. "math/rand"
  22. "testing"
  23. "github.com/VictoriaMetrics/fastcache"
  24. "github.com/ethereum/go-ethereum/common"
  25. "github.com/ethereum/go-ethereum/core/rawdb"
  26. )
  27. // TestAccountIteratorBasics tests some simple single-layer iteration
  28. func TestAccountIteratorBasics(t *testing.T) {
  29. var (
  30. destructs = make(map[common.Hash]struct{})
  31. accounts = make(map[common.Hash][]byte)
  32. storage = make(map[common.Hash]map[common.Hash][]byte)
  33. )
  34. // Fill up a parent
  35. for i := 0; i < 100; i++ {
  36. h := randomHash()
  37. data := randomAccount()
  38. accounts[h] = data
  39. if rand.Intn(4) == 0 {
  40. destructs[h] = struct{}{}
  41. }
  42. if rand.Intn(2) == 0 {
  43. accStorage := make(map[common.Hash][]byte)
  44. value := make([]byte, 32)
  45. rand.Read(value)
  46. accStorage[randomHash()] = value
  47. storage[h] = accStorage
  48. }
  49. }
  50. // Add some (identical) layers on top
  51. parent := newDiffLayer(emptyLayer(), common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage))
  52. it := parent.AccountIterator(common.Hash{})
  53. verifyIterator(t, 100, it)
  54. }
  55. type testIterator struct {
  56. values []byte
  57. }
  58. func newTestIterator(values ...byte) *testIterator {
  59. return &testIterator{values}
  60. }
  61. func (ti *testIterator) Seek(common.Hash) {
  62. panic("implement me")
  63. }
  64. func (ti *testIterator) Next() bool {
  65. ti.values = ti.values[1:]
  66. return len(ti.values) > 0
  67. }
  68. func (ti *testIterator) Error() error {
  69. return nil
  70. }
  71. func (ti *testIterator) Hash() common.Hash {
  72. return common.BytesToHash([]byte{ti.values[0]})
  73. }
  74. func (ti *testIterator) Account() []byte {
  75. return nil
  76. }
  77. func (ti *testIterator) Release() {}
  78. func TestFastIteratorBasics(t *testing.T) {
  79. type testCase struct {
  80. lists [][]byte
  81. expKeys []byte
  82. }
  83. for i, tc := range []testCase{
  84. {lists: [][]byte{{0, 1, 8}, {1, 2, 8}, {2, 9}, {4},
  85. {7, 14, 15}, {9, 13, 15, 16}},
  86. expKeys: []byte{0, 1, 2, 4, 7, 8, 9, 13, 14, 15, 16}},
  87. {lists: [][]byte{{0, 8}, {1, 2, 8}, {7, 14, 15}, {8, 9},
  88. {9, 10}, {10, 13, 15, 16}},
  89. expKeys: []byte{0, 1, 2, 7, 8, 9, 10, 13, 14, 15, 16}},
  90. } {
  91. var iterators []*weightedAccountIterator
  92. for i, data := range tc.lists {
  93. it := newTestIterator(data...)
  94. iterators = append(iterators, &weightedAccountIterator{it, i})
  95. }
  96. fi := &fastAccountIterator{
  97. iterators: iterators,
  98. initiated: false,
  99. }
  100. count := 0
  101. for fi.Next() {
  102. if got, exp := fi.Hash()[31], tc.expKeys[count]; exp != got {
  103. t.Errorf("tc %d, [%d]: got %d exp %d", i, count, got, exp)
  104. }
  105. count++
  106. }
  107. }
  108. }
  109. func verifyIterator(t *testing.T, expCount int, it AccountIterator) {
  110. t.Helper()
  111. var (
  112. count = 0
  113. last = common.Hash{}
  114. )
  115. for it.Next() {
  116. hash := it.Hash()
  117. if bytes.Compare(last[:], hash[:]) >= 0 {
  118. t.Errorf("wrong order: %x >= %x", last, hash)
  119. }
  120. if it.Account() == nil {
  121. t.Errorf("iterator returned nil-value for hash %x", hash)
  122. }
  123. count++
  124. }
  125. if count != expCount {
  126. t.Errorf("iterator count mismatch: have %d, want %d", count, expCount)
  127. }
  128. if err := it.Error(); err != nil {
  129. t.Errorf("iterator failed: %v", err)
  130. }
  131. }
  132. // TestAccountIteratorTraversal tests some simple multi-layer iteration.
  133. func TestAccountIteratorTraversal(t *testing.T) {
  134. // Create an empty base layer and a snapshot tree out of it
  135. base := &diskLayer{
  136. diskdb: rawdb.NewMemoryDatabase(),
  137. root: common.HexToHash("0x01"),
  138. cache: fastcache.New(1024 * 500),
  139. }
  140. snaps := &Tree{
  141. layers: map[common.Hash]snapshot{
  142. base.root: base,
  143. },
  144. }
  145. // Stack three diff layers on top with various overlaps
  146. snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil,
  147. randomAccountSet("0xaa", "0xee", "0xff", "0xf0"), nil)
  148. snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil,
  149. randomAccountSet("0xbb", "0xdd", "0xf0"), nil)
  150. snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil,
  151. randomAccountSet("0xcc", "0xf0", "0xff"), nil)
  152. // Verify the single and multi-layer iterators
  153. head := snaps.Snapshot(common.HexToHash("0x04"))
  154. verifyIterator(t, 3, head.(snapshot).AccountIterator(common.Hash{}))
  155. verifyIterator(t, 7, head.(*diffLayer).newBinaryAccountIterator())
  156. it, _ := snaps.AccountIterator(common.HexToHash("0x04"), common.Hash{})
  157. verifyIterator(t, 7, it)
  158. it.Release()
  159. // Test after persist some bottom-most layers into the disk,
  160. // the functionalities still work.
  161. limit := aggregatorMemoryLimit
  162. defer func() {
  163. aggregatorMemoryLimit = limit
  164. }()
  165. aggregatorMemoryLimit = 0 // Force pushing the bottom-most layer into disk
  166. snaps.Cap(common.HexToHash("0x04"), 2)
  167. verifyIterator(t, 7, head.(*diffLayer).newBinaryAccountIterator())
  168. it, _ = snaps.AccountIterator(common.HexToHash("0x04"), common.Hash{})
  169. verifyIterator(t, 7, it)
  170. it.Release()
  171. }
  172. // TestAccountIteratorTraversalValues tests some multi-layer iteration, where we
  173. // also expect the correct values to show up.
  174. func TestAccountIteratorTraversalValues(t *testing.T) {
  175. // Create an empty base layer and a snapshot tree out of it
  176. base := &diskLayer{
  177. diskdb: rawdb.NewMemoryDatabase(),
  178. root: common.HexToHash("0x01"),
  179. cache: fastcache.New(1024 * 500),
  180. }
  181. snaps := &Tree{
  182. layers: map[common.Hash]snapshot{
  183. base.root: base,
  184. },
  185. }
  186. // Create a batch of account sets to seed subsequent layers with
  187. var (
  188. a = make(map[common.Hash][]byte)
  189. b = make(map[common.Hash][]byte)
  190. c = make(map[common.Hash][]byte)
  191. d = make(map[common.Hash][]byte)
  192. e = make(map[common.Hash][]byte)
  193. f = make(map[common.Hash][]byte)
  194. g = make(map[common.Hash][]byte)
  195. h = make(map[common.Hash][]byte)
  196. )
  197. for i := byte(2); i < 0xff; i++ {
  198. a[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 0, i))
  199. if i > 20 && i%2 == 0 {
  200. b[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 1, i))
  201. }
  202. if i%4 == 0 {
  203. c[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 2, i))
  204. }
  205. if i%7 == 0 {
  206. d[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 3, i))
  207. }
  208. if i%8 == 0 {
  209. e[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 4, i))
  210. }
  211. if i > 50 || i < 85 {
  212. f[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 5, i))
  213. }
  214. if i%64 == 0 {
  215. g[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 6, i))
  216. }
  217. if i%128 == 0 {
  218. h[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 7, i))
  219. }
  220. }
  221. // Assemble a stack of snapshots from the account layers
  222. snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, a, nil)
  223. snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, b, nil)
  224. snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, c, nil)
  225. snaps.Update(common.HexToHash("0x05"), common.HexToHash("0x04"), nil, d, nil)
  226. snaps.Update(common.HexToHash("0x06"), common.HexToHash("0x05"), nil, e, nil)
  227. snaps.Update(common.HexToHash("0x07"), common.HexToHash("0x06"), nil, f, nil)
  228. snaps.Update(common.HexToHash("0x08"), common.HexToHash("0x07"), nil, g, nil)
  229. snaps.Update(common.HexToHash("0x09"), common.HexToHash("0x08"), nil, h, nil)
  230. it, _ := snaps.AccountIterator(common.HexToHash("0x09"), common.Hash{})
  231. head := snaps.Snapshot(common.HexToHash("0x09"))
  232. for it.Next() {
  233. hash := it.Hash()
  234. want, err := head.AccountRLP(hash)
  235. if err != nil {
  236. t.Fatalf("failed to retrieve expected account: %v", err)
  237. }
  238. if have := it.Account(); !bytes.Equal(want, have) {
  239. t.Fatalf("hash %x: account mismatch: have %x, want %x", hash, have, want)
  240. }
  241. }
  242. it.Release()
  243. // Test after persist some bottom-most layers into the disk,
  244. // the functionalities still work.
  245. limit := aggregatorMemoryLimit
  246. defer func() {
  247. aggregatorMemoryLimit = limit
  248. }()
  249. aggregatorMemoryLimit = 0 // Force pushing the bottom-most layer into disk
  250. snaps.Cap(common.HexToHash("0x09"), 2)
  251. it, _ = snaps.AccountIterator(common.HexToHash("0x09"), common.Hash{})
  252. for it.Next() {
  253. hash := it.Hash()
  254. want, err := head.AccountRLP(hash)
  255. if err != nil {
  256. t.Fatalf("failed to retrieve expected account: %v", err)
  257. }
  258. if have := it.Account(); !bytes.Equal(want, have) {
  259. t.Fatalf("hash %x: account mismatch: have %x, want %x", hash, have, want)
  260. }
  261. }
  262. it.Release()
  263. }
  264. // This testcase is notorious, all layers contain the exact same 200 accounts.
  265. func TestAccountIteratorLargeTraversal(t *testing.T) {
  266. // Create a custom account factory to recreate the same addresses
  267. makeAccounts := func(num int) map[common.Hash][]byte {
  268. accounts := make(map[common.Hash][]byte)
  269. for i := 0; i < num; i++ {
  270. h := common.Hash{}
  271. binary.BigEndian.PutUint64(h[:], uint64(i+1))
  272. accounts[h] = randomAccount()
  273. }
  274. return accounts
  275. }
  276. // Build up a large stack of snapshots
  277. base := &diskLayer{
  278. diskdb: rawdb.NewMemoryDatabase(),
  279. root: common.HexToHash("0x01"),
  280. cache: fastcache.New(1024 * 500),
  281. }
  282. snaps := &Tree{
  283. layers: map[common.Hash]snapshot{
  284. base.root: base,
  285. },
  286. }
  287. for i := 1; i < 128; i++ {
  288. snaps.Update(common.HexToHash(fmt.Sprintf("0x%02x", i+1)), common.HexToHash(fmt.Sprintf("0x%02x", i)), nil, makeAccounts(200), nil)
  289. }
  290. // Iterate the entire stack and ensure everything is hit only once
  291. head := snaps.Snapshot(common.HexToHash("0x80"))
  292. verifyIterator(t, 200, head.(snapshot).AccountIterator(common.Hash{}))
  293. verifyIterator(t, 200, head.(*diffLayer).newBinaryAccountIterator())
  294. it, _ := snaps.AccountIterator(common.HexToHash("0x80"), common.Hash{})
  295. verifyIterator(t, 200, it)
  296. it.Release()
  297. // Test after persist some bottom-most layers into the disk,
  298. // the functionalities still work.
  299. limit := aggregatorMemoryLimit
  300. defer func() {
  301. aggregatorMemoryLimit = limit
  302. }()
  303. aggregatorMemoryLimit = 0 // Force pushing the bottom-most layer into disk
  304. snaps.Cap(common.HexToHash("0x80"), 2)
  305. verifyIterator(t, 200, head.(*diffLayer).newBinaryAccountIterator())
  306. it, _ = snaps.AccountIterator(common.HexToHash("0x80"), common.Hash{})
  307. verifyIterator(t, 200, it)
  308. it.Release()
  309. }
  310. // TestAccountIteratorFlattening tests what happens when we
  311. // - have a live iterator on child C (parent C1 -> C2 .. CN)
  312. // - flattens C2 all the way into CN
  313. // - continues iterating
  314. func TestAccountIteratorFlattening(t *testing.T) {
  315. // Create an empty base layer and a snapshot tree out of it
  316. base := &diskLayer{
  317. diskdb: rawdb.NewMemoryDatabase(),
  318. root: common.HexToHash("0x01"),
  319. cache: fastcache.New(1024 * 500),
  320. }
  321. snaps := &Tree{
  322. layers: map[common.Hash]snapshot{
  323. base.root: base,
  324. },
  325. }
  326. // Create a stack of diffs on top
  327. snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil,
  328. randomAccountSet("0xaa", "0xee", "0xff", "0xf0"), nil)
  329. snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil,
  330. randomAccountSet("0xbb", "0xdd", "0xf0"), nil)
  331. snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil,
  332. randomAccountSet("0xcc", "0xf0", "0xff"), nil)
  333. // Create an iterator and flatten the data from underneath it
  334. it, _ := snaps.AccountIterator(common.HexToHash("0x04"), common.Hash{})
  335. defer it.Release()
  336. if err := snaps.Cap(common.HexToHash("0x04"), 1); err != nil {
  337. t.Fatalf("failed to flatten snapshot stack: %v", err)
  338. }
  339. //verifyIterator(t, 7, it)
  340. }
  341. func TestAccountIteratorSeek(t *testing.T) {
  342. // Create a snapshot stack with some initial data
  343. base := &diskLayer{
  344. diskdb: rawdb.NewMemoryDatabase(),
  345. root: common.HexToHash("0x01"),
  346. cache: fastcache.New(1024 * 500),
  347. }
  348. snaps := &Tree{
  349. layers: map[common.Hash]snapshot{
  350. base.root: base,
  351. },
  352. }
  353. snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil,
  354. randomAccountSet("0xaa", "0xee", "0xff", "0xf0"), nil)
  355. snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil,
  356. randomAccountSet("0xbb", "0xdd", "0xf0"), nil)
  357. snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil,
  358. randomAccountSet("0xcc", "0xf0", "0xff"), nil)
  359. // Account set is now
  360. // 02: aa, ee, f0, ff
  361. // 03: aa, bb, dd, ee, f0 (, f0), ff
  362. // 04: aa, bb, cc, dd, ee, f0 (, f0), ff (, ff)
  363. // Construct various iterators and ensure their traversal is correct
  364. it, _ := snaps.AccountIterator(common.HexToHash("0x02"), common.HexToHash("0xdd"))
  365. defer it.Release()
  366. verifyIterator(t, 3, it) // expected: ee, f0, ff
  367. it, _ = snaps.AccountIterator(common.HexToHash("0x02"), common.HexToHash("0xaa"))
  368. defer it.Release()
  369. verifyIterator(t, 4, it) // expected: aa, ee, f0, ff
  370. it, _ = snaps.AccountIterator(common.HexToHash("0x02"), common.HexToHash("0xff"))
  371. defer it.Release()
  372. verifyIterator(t, 1, it) // expected: ff
  373. it, _ = snaps.AccountIterator(common.HexToHash("0x02"), common.HexToHash("0xff1"))
  374. defer it.Release()
  375. verifyIterator(t, 0, it) // expected: nothing
  376. it, _ = snaps.AccountIterator(common.HexToHash("0x04"), common.HexToHash("0xbb"))
  377. defer it.Release()
  378. verifyIterator(t, 6, it) // expected: bb, cc, dd, ee, f0, ff
  379. it, _ = snaps.AccountIterator(common.HexToHash("0x04"), common.HexToHash("0xef"))
  380. defer it.Release()
  381. verifyIterator(t, 2, it) // expected: f0, ff
  382. it, _ = snaps.AccountIterator(common.HexToHash("0x04"), common.HexToHash("0xf0"))
  383. defer it.Release()
  384. verifyIterator(t, 2, it) // expected: f0, ff
  385. it, _ = snaps.AccountIterator(common.HexToHash("0x04"), common.HexToHash("0xff"))
  386. defer it.Release()
  387. verifyIterator(t, 1, it) // expected: ff
  388. it, _ = snaps.AccountIterator(common.HexToHash("0x04"), common.HexToHash("0xff1"))
  389. defer it.Release()
  390. verifyIterator(t, 0, it) // expected: nothing
  391. }
  392. // TestIteratorDeletions tests that the iterator behaves correct when there are
  393. // deleted accounts (where the Account() value is nil). The iterator
  394. // should not output any accounts or nil-values for those cases.
  395. func TestIteratorDeletions(t *testing.T) {
  396. // Create an empty base layer and a snapshot tree out of it
  397. base := &diskLayer{
  398. diskdb: rawdb.NewMemoryDatabase(),
  399. root: common.HexToHash("0x01"),
  400. cache: fastcache.New(1024 * 500),
  401. }
  402. snaps := &Tree{
  403. layers: map[common.Hash]snapshot{
  404. base.root: base,
  405. },
  406. }
  407. // Stack three diff layers on top with various overlaps
  408. snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"),
  409. nil, randomAccountSet("0x11", "0x22", "0x33"), nil)
  410. deleted := common.HexToHash("0x22")
  411. destructed := map[common.Hash]struct{}{
  412. deleted: {},
  413. }
  414. snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"),
  415. destructed, randomAccountSet("0x11", "0x33"), nil)
  416. snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"),
  417. nil, randomAccountSet("0x33", "0x44", "0x55"), nil)
  418. // The output should be 11,33,44,55
  419. it, _ := snaps.AccountIterator(common.HexToHash("0x04"), common.Hash{})
  420. // Do a quick check
  421. verifyIterator(t, 4, it)
  422. it.Release()
  423. // And a more detailed verification that we indeed do not see '0x22'
  424. it, _ = snaps.AccountIterator(common.HexToHash("0x04"), common.Hash{})
  425. defer it.Release()
  426. for it.Next() {
  427. hash := it.Hash()
  428. if it.Account() == nil {
  429. t.Errorf("iterator returned nil-value for hash %x", hash)
  430. }
  431. if hash == deleted {
  432. t.Errorf("expected deleted elem %x to not be returned by iterator", deleted)
  433. }
  434. }
  435. }
  436. // BenchmarkAccountIteratorTraversal is a bit a bit notorious -- all layers contain the
  437. // exact same 200 accounts. That means that we need to process 2000 items, but
  438. // only spit out 200 values eventually.
  439. //
  440. // The value-fetching benchmark is easy on the binary iterator, since it never has to reach
  441. // down at any depth for retrieving the values -- all are on the toppmost layer
  442. //
  443. // BenchmarkAccountIteratorTraversal/binary_iterator_keys-6 2239 483674 ns/op
  444. // BenchmarkAccountIteratorTraversal/binary_iterator_values-6 2403 501810 ns/op
  445. // BenchmarkAccountIteratorTraversal/fast_iterator_keys-6 1923 677966 ns/op
  446. // BenchmarkAccountIteratorTraversal/fast_iterator_values-6 1741 649967 ns/op
  447. func BenchmarkAccountIteratorTraversal(b *testing.B) {
  448. // Create a custom account factory to recreate the same addresses
  449. makeAccounts := func(num int) map[common.Hash][]byte {
  450. accounts := make(map[common.Hash][]byte)
  451. for i := 0; i < num; i++ {
  452. h := common.Hash{}
  453. binary.BigEndian.PutUint64(h[:], uint64(i+1))
  454. accounts[h] = randomAccount()
  455. }
  456. return accounts
  457. }
  458. // Build up a large stack of snapshots
  459. base := &diskLayer{
  460. diskdb: rawdb.NewMemoryDatabase(),
  461. root: common.HexToHash("0x01"),
  462. cache: fastcache.New(1024 * 500),
  463. }
  464. snaps := &Tree{
  465. layers: map[common.Hash]snapshot{
  466. base.root: base,
  467. },
  468. }
  469. for i := 1; i <= 100; i++ {
  470. snaps.Update(common.HexToHash(fmt.Sprintf("0x%02x", i+1)), common.HexToHash(fmt.Sprintf("0x%02x", i)), nil, makeAccounts(200), nil)
  471. }
  472. // We call this once before the benchmark, so the creation of
  473. // sorted accountlists are not included in the results.
  474. head := snaps.Snapshot(common.HexToHash("0x65"))
  475. head.(*diffLayer).newBinaryAccountIterator()
  476. b.Run("binary iterator keys", func(b *testing.B) {
  477. for i := 0; i < b.N; i++ {
  478. got := 0
  479. it := head.(*diffLayer).newBinaryAccountIterator()
  480. for it.Next() {
  481. got++
  482. }
  483. if exp := 200; got != exp {
  484. b.Errorf("iterator len wrong, expected %d, got %d", exp, got)
  485. }
  486. }
  487. })
  488. b.Run("binary iterator values", func(b *testing.B) {
  489. for i := 0; i < b.N; i++ {
  490. got := 0
  491. it := head.(*diffLayer).newBinaryAccountIterator()
  492. for it.Next() {
  493. got++
  494. head.(*diffLayer).accountRLP(it.Hash(), 0)
  495. }
  496. if exp := 200; got != exp {
  497. b.Errorf("iterator len wrong, expected %d, got %d", exp, got)
  498. }
  499. }
  500. })
  501. b.Run("fast iterator keys", func(b *testing.B) {
  502. for i := 0; i < b.N; i++ {
  503. it, _ := snaps.AccountIterator(common.HexToHash("0x65"), common.Hash{})
  504. defer it.Release()
  505. got := 0
  506. for it.Next() {
  507. got++
  508. }
  509. if exp := 200; got != exp {
  510. b.Errorf("iterator len wrong, expected %d, got %d", exp, got)
  511. }
  512. }
  513. })
  514. b.Run("fast iterator values", func(b *testing.B) {
  515. for i := 0; i < b.N; i++ {
  516. it, _ := snaps.AccountIterator(common.HexToHash("0x65"), common.Hash{})
  517. defer it.Release()
  518. got := 0
  519. for it.Next() {
  520. got++
  521. it.Account()
  522. }
  523. if exp := 200; got != exp {
  524. b.Errorf("iterator len wrong, expected %d, got %d", exp, got)
  525. }
  526. }
  527. })
  528. }
  529. // BenchmarkAccountIteratorLargeBaselayer is a pretty realistic benchmark, where
  530. // the baselayer is a lot larger than the upper layer.
  531. //
  532. // This is heavy on the binary iterator, which in most cases will have to
  533. // call recursively 100 times for the majority of the values
  534. //
  535. // BenchmarkAccountIteratorLargeBaselayer/binary_iterator_(keys)-6 514 1971999 ns/op
  536. // BenchmarkAccountIteratorLargeBaselayer/binary_iterator_(values)-6 61 18997492 ns/op
  537. // BenchmarkAccountIteratorLargeBaselayer/fast_iterator_(keys)-6 10000 114385 ns/op
  538. // BenchmarkAccountIteratorLargeBaselayer/fast_iterator_(values)-6 4047 296823 ns/op
  539. func BenchmarkAccountIteratorLargeBaselayer(b *testing.B) {
  540. // Create a custom account factory to recreate the same addresses
  541. makeAccounts := func(num int) map[common.Hash][]byte {
  542. accounts := make(map[common.Hash][]byte)
  543. for i := 0; i < num; i++ {
  544. h := common.Hash{}
  545. binary.BigEndian.PutUint64(h[:], uint64(i+1))
  546. accounts[h] = randomAccount()
  547. }
  548. return accounts
  549. }
  550. // Build up a large stack of snapshots
  551. base := &diskLayer{
  552. diskdb: rawdb.NewMemoryDatabase(),
  553. root: common.HexToHash("0x01"),
  554. cache: fastcache.New(1024 * 500),
  555. }
  556. snaps := &Tree{
  557. layers: map[common.Hash]snapshot{
  558. base.root: base,
  559. },
  560. }
  561. snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, makeAccounts(2000), nil)
  562. for i := 2; i <= 100; i++ {
  563. snaps.Update(common.HexToHash(fmt.Sprintf("0x%02x", i+1)), common.HexToHash(fmt.Sprintf("0x%02x", i)), nil, makeAccounts(20), nil)
  564. }
  565. // We call this once before the benchmark, so the creation of
  566. // sorted accountlists are not included in the results.
  567. head := snaps.Snapshot(common.HexToHash("0x65"))
  568. head.(*diffLayer).newBinaryAccountIterator()
  569. b.Run("binary iterator (keys)", func(b *testing.B) {
  570. for i := 0; i < b.N; i++ {
  571. got := 0
  572. it := head.(*diffLayer).newBinaryAccountIterator()
  573. for it.Next() {
  574. got++
  575. }
  576. if exp := 2000; got != exp {
  577. b.Errorf("iterator len wrong, expected %d, got %d", exp, got)
  578. }
  579. }
  580. })
  581. b.Run("binary iterator (values)", func(b *testing.B) {
  582. for i := 0; i < b.N; i++ {
  583. got := 0
  584. it := head.(*diffLayer).newBinaryAccountIterator()
  585. for it.Next() {
  586. got++
  587. v := it.Hash()
  588. head.(*diffLayer).accountRLP(v, 0)
  589. }
  590. if exp := 2000; got != exp {
  591. b.Errorf("iterator len wrong, expected %d, got %d", exp, got)
  592. }
  593. }
  594. })
  595. b.Run("fast iterator (keys)", func(b *testing.B) {
  596. for i := 0; i < b.N; i++ {
  597. it, _ := snaps.AccountIterator(common.HexToHash("0x65"), common.Hash{})
  598. defer it.Release()
  599. got := 0
  600. for it.Next() {
  601. got++
  602. }
  603. if exp := 2000; got != exp {
  604. b.Errorf("iterator len wrong, expected %d, got %d", exp, got)
  605. }
  606. }
  607. })
  608. b.Run("fast iterator (values)", func(b *testing.B) {
  609. for i := 0; i < b.N; i++ {
  610. it, _ := snaps.AccountIterator(common.HexToHash("0x65"), common.Hash{})
  611. defer it.Release()
  612. got := 0
  613. for it.Next() {
  614. it.Account()
  615. got++
  616. }
  617. if exp := 2000; got != exp {
  618. b.Errorf("iterator len wrong, expected %d, got %d", exp, got)
  619. }
  620. }
  621. })
  622. }
  623. /*
  624. func BenchmarkBinaryAccountIteration(b *testing.B) {
  625. benchmarkAccountIteration(b, func(snap snapshot) AccountIterator {
  626. return snap.(*diffLayer).newBinaryAccountIterator()
  627. })
  628. }
  629. func BenchmarkFastAccountIteration(b *testing.B) {
  630. benchmarkAccountIteration(b, newFastAccountIterator)
  631. }
  632. func benchmarkAccountIteration(b *testing.B, iterator func(snap snapshot) AccountIterator) {
  633. // Create a diff stack and randomize the accounts across them
  634. layers := make([]map[common.Hash][]byte, 128)
  635. for i := 0; i < len(layers); i++ {
  636. layers[i] = make(map[common.Hash][]byte)
  637. }
  638. for i := 0; i < b.N; i++ {
  639. depth := rand.Intn(len(layers))
  640. layers[depth][randomHash()] = randomAccount()
  641. }
  642. stack := snapshot(emptyLayer())
  643. for _, layer := range layers {
  644. stack = stack.Update(common.Hash{}, layer, nil, nil)
  645. }
  646. // Reset the timers and report all the stats
  647. it := iterator(stack)
  648. b.ResetTimer()
  649. b.ReportAllocs()
  650. for it.Next() {
  651. }
  652. }
  653. */