eth: add debug_accountRange API (#19645)

This new API allows reading accounts and their content by address range.

Co-authored-by: Martin Holst Swende <martin@swende.se>
Co-authored-by: Felix Lange <fjl@twurst.com>

core/state/dump.go

@@ -27,7 +27,7 @@ import (
 	"github.com/ethereum/go-ethereum/trie"
 )
 
-// DumpAccount represents an account in the state
+// DumpAccount represents an account in the state.
 type DumpAccount struct {
 	Balance   string                 `json:"balance"`
 	Nonce     uint64                 `json:"nonce"`
@@ -40,17 +40,24 @@ type DumpAccount struct {
 
 }
 
-// Dump represents the full dump in a collected format, as one large map
+// Dump represents the full dump in a collected format, as one large map.
 type Dump struct {
 	Root     string                         `json:"root"`
 	Accounts map[common.Address]DumpAccount `json:"accounts"`
 }
 
-// iterativeDump is a 'collector'-implementation which dump output line-by-line iteratively
+// iterativeDump is a 'collector'-implementation which dump output line-by-line iteratively.
 type iterativeDump struct {
 	*json.Encoder
 }
 
+// IteratorDump is an implementation for iterating over data.
+type IteratorDump struct {
+	Root     string                         `json:"root"`
+	Accounts map[common.Address]DumpAccount `json:"accounts"`
+	Next     []byte                         `json:"next,omitempty"` // nil if no more accounts
+}
+
 // Collector interface which the state trie calls during iteration
 type collector interface {
 	onRoot(common.Hash)
@@ -64,6 +71,13 @@ func (d *Dump) onRoot(root common.Hash) {
 func (d *Dump) onAccount(addr common.Address, account DumpAccount) {
 	d.Accounts[addr] = account
 }
+func (d *IteratorDump) onRoot(root common.Hash) {
+	d.Root = fmt.Sprintf("%x", root)
+}
+
+func (d *IteratorDump) onAccount(addr common.Address, account DumpAccount) {
+	d.Accounts[addr] = account
+}
 
 func (d iterativeDump) onAccount(addr common.Address, account DumpAccount) {
 	dumpAccount := &DumpAccount{
@@ -88,11 +102,13 @@ func (d iterativeDump) onRoot(root common.Hash) {
 	}{root})
 }
 
-func (s *StateDB) dump(c collector, excludeCode, excludeStorage, excludeMissingPreimages bool) {
+func (s *StateDB) dump(c collector, excludeCode, excludeStorage, excludeMissingPreimages bool, start []byte, maxResults int) (nextKey []byte) {
 	emptyAddress := (common.Address{})
 	missingPreimages := 0
 	c.onRoot(s.trie.Hash())
-	it := trie.NewIterator(s.trie.NodeIterator(nil))
+
+	var count int
+	it := trie.NewIterator(s.trie.NodeIterator(start))
 	for it.Next() {
 		var data Account
 		if err := rlp.DecodeBytes(it.Value, &data); err != nil {
@@ -130,10 +146,19 @@ func (s *StateDB) dump(c collector, excludeCode, excludeStorage, excludeMissingP
 			}
 		}
 		c.onAccount(addr, account)
+		count++
+		if maxResults > 0 && count >= maxResults {
+			if it.Next() {
+				nextKey = it.Key
+			}
+			break
+		}
 	}
 	if missingPreimages > 0 {
 		log.Warn("Dump incomplete due to missing preimages", "missing", missingPreimages)
 	}
+
+	return nextKey
 }
 
 // RawDump returns the entire state an a single large object
@@ -141,7 +166,7 @@ func (s *StateDB) RawDump(excludeCode, excludeStorage, excludeMissingPreimages b
 	dump := &Dump{
 		Accounts: make(map[common.Address]DumpAccount),
 	}
-	s.dump(dump, excludeCode, excludeStorage, excludeMissingPreimages)
+	s.dump(dump, excludeCode, excludeStorage, excludeMissingPreimages, nil, 0)
 	return *dump
 }
 
@@ -157,5 +182,14 @@ func (s *StateDB) Dump(excludeCode, excludeStorage, excludeMissingPreimages bool
 
 // IterativeDump dumps out accounts as json-objects, delimited by linebreaks on stdout
 func (s *StateDB) IterativeDump(excludeCode, excludeStorage, excludeMissingPreimages bool, output *json.Encoder) {
-	s.dump(iterativeDump{output}, excludeCode, excludeStorage, excludeMissingPreimages)
+	s.dump(iterativeDump{output}, excludeCode, excludeStorage, excludeMissingPreimages, nil, 0)
+}
+
+// IteratorDump dumps out a batch of accounts starts with the given start key
+func (s *StateDB) IteratorDump(excludeCode, excludeStorage, excludeMissingPreimages bool, start []byte, maxResults int) IteratorDump {
+	iterator := &IteratorDump{
+		Accounts: make(map[common.Address]DumpAccount),
+	}
+	iterator.Next = s.dump(iterator, excludeCode, excludeStorage, excludeMissingPreimages, start, maxResults)
+	return *iterator
 }

eth/api.go

@@ -351,70 +351,50 @@ func (api *PrivateDebugAPI) GetBadBlocks(ctx context.Context) ([]*BadBlockArgs,
 	return results, nil
 }
 
-// AccountRangeResult returns a mapping from the hash of an account addresses
-// to its preimage. It will return the JSON null if no preimage is found.
-// Since a query can return a limited amount of results, a "next" field is
-// also present for paging.
-type AccountRangeResult struct {
-	Accounts map[common.Hash]*common.Address `json:"accounts"`
-	Next     common.Hash                     `json:"next"`
-}
-
-func accountRange(st state.Trie, start *common.Hash, maxResults int) (AccountRangeResult, error) {
-	if start == nil {
-		start = &common.Hash{0}
-	}
-	it := trie.NewIterator(st.NodeIterator(start.Bytes()))
-	result := AccountRangeResult{Accounts: make(map[common.Hash]*common.Address), Next: common.Hash{}}
-
-	if maxResults > AccountRangeMaxResults {
-		maxResults = AccountRangeMaxResults
-	}
-
-	for i := 0; i < maxResults && it.Next(); i++ {
-		if preimage := st.GetKey(it.Key); preimage != nil {
-			addr := &common.Address{}
-			addr.SetBytes(preimage)
-			result.Accounts[common.BytesToHash(it.Key)] = addr
-		} else {
-			result.Accounts[common.BytesToHash(it.Key)] = nil
-		}
-	}
-
-	if it.Next() {
-		result.Next = common.BytesToHash(it.Key)
-	}
-
-	return result, nil
-}
-
 // AccountRangeMaxResults is the maximum number of results to be returned per call
 const AccountRangeMaxResults = 256
 
-// AccountRange enumerates all accounts in the latest state
-func (api *PrivateDebugAPI) AccountRange(ctx context.Context, start *common.Hash, maxResults int) (AccountRangeResult, error) {
-	var statedb *state.StateDB
+// AccountRangeAt enumerates all accounts in the given block and start point in paging request
+func (api *PublicDebugAPI) AccountRange(blockNrOrHash rpc.BlockNumberOrHash, start []byte, maxResults int, nocode, nostorage, incompletes bool) (state.IteratorDump, error) {
+	var stateDb *state.StateDB
 	var err error
-	block := api.eth.blockchain.CurrentBlock()
 
-	if len(block.Transactions()) == 0 {
-		statedb, err = api.computeStateDB(block, defaultTraceReexec)
-		if err != nil {
-			return AccountRangeResult{}, err
+	if number, ok := blockNrOrHash.Number(); ok {
+		if number == rpc.PendingBlockNumber {
+			// If we're dumping the pending state, we need to request
+			// both the pending block as well as the pending state from
+			// the miner and operate on those
+			_, stateDb = api.eth.miner.Pending()
+		} else {
+			var block *types.Block
+			if number == rpc.LatestBlockNumber {
+				block = api.eth.blockchain.CurrentBlock()
+			} else {
+				block = api.eth.blockchain.GetBlockByNumber(uint64(number))
+			}
+			if block == nil {
+				return state.IteratorDump{}, fmt.Errorf("block #%d not found", number)
+			}
+			stateDb, err = api.eth.BlockChain().StateAt(block.Root())
+			if err != nil {
+				return state.IteratorDump{}, err
+			}
 		}
-	} else {
-		_, _, statedb, err = api.computeTxEnv(block.Hash(), len(block.Transactions())-1, 0)
+	} else if hash, ok := blockNrOrHash.Hash(); ok {
+		block := api.eth.blockchain.GetBlockByHash(hash)
+		if block == nil {
+			return state.IteratorDump{}, fmt.Errorf("block %s not found", hash.Hex())
+		}
+		stateDb, err = api.eth.BlockChain().StateAt(block.Root())
 		if err != nil {
-			return AccountRangeResult{}, err
+			return state.IteratorDump{}, err
 		}
 	}
 
-	trie, err := statedb.Database().OpenTrie(block.Header().Root)
-	if err != nil {
-		return AccountRangeResult{}, err
+	if maxResults > AccountRangeMaxResults || maxResults <= 0 {
+		maxResults = AccountRangeMaxResults
 	}
-
-	return accountRange(trie, start, maxResults)
+	return stateDb.IteratorDump(nocode, nostorage, incompletes, start, maxResults), nil
 }
 
 // StorageRangeResult is the result of a debug_storageRangeAt API call.
@@ -431,7 +411,7 @@ type storageEntry struct {
 }
 
 // StorageRangeAt returns the storage at the given block height and transaction index.
-func (api *PrivateDebugAPI) StorageRangeAt(ctx context.Context, blockHash common.Hash, txIndex int, contractAddress common.Address, keyStart hexutil.Bytes, maxResult int) (StorageRangeResult, error) {
+func (api *PrivateDebugAPI) StorageRangeAt(blockHash common.Hash, txIndex int, contractAddress common.Address, keyStart hexutil.Bytes, maxResult int) (StorageRangeResult, error) {
 	_, _, statedb, err := api.computeTxEnv(blockHash, txIndex, 0)
 	if err != nil {
 		return StorageRangeResult{}, err

eth/api_test.go

@@ -33,29 +33,24 @@ import (
 
 var dumper = spew.ConfigState{Indent: "    "}
 
-func accountRangeTest(t *testing.T, trie *state.Trie, statedb *state.StateDB, start *common.Hash, requestedNum int, expectedNum int) AccountRangeResult {
-	result, err := accountRange(*trie, start, requestedNum)
-	if err != nil {
-		t.Fatal(err)
-	}
+func accountRangeTest(t *testing.T, trie *state.Trie, statedb *state.StateDB, start common.Hash, requestedNum int, expectedNum int) state.IteratorDump {
+	result := statedb.IteratorDump(true, true, false, start.Bytes(), requestedNum)
 
 	if len(result.Accounts) != expectedNum {
-		t.Fatalf("expected %d results.  Got %d", expectedNum, len(result.Accounts))
+		t.Fatalf("expected %d results, got %d", expectedNum, len(result.Accounts))
 	}
-
-	for _, address := range result.Accounts {
-		if address == nil {
-			t.Fatalf("null address returned")
+	for address := range result.Accounts {
+		if address == (common.Address{}) {
+			t.Fatalf("empty address returned")
 		}
-		if !statedb.Exist(*address) {
+		if !statedb.Exist(address) {
 			t.Fatalf("account not found in state %s", address.Hex())
 		}
 	}
-
 	return result
 }
 
-type resultHash []*common.Hash
+type resultHash []common.Hash
 
 func (h resultHash) Len() int           { return len(h) }
 func (h resultHash) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }
@@ -80,7 +75,6 @@ func TestAccountRange(t *testing.T) {
 			m[addr] = true
 		}
 	}
-
 	state.Commit(true)
 	root := state.IntermediateRoot(true)
 
@@ -88,68 +82,40 @@ func TestAccountRange(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-
-	t.Logf("test getting number of results less than max")
-	accountRangeTest(t, &trie, state, &common.Hash{0x0}, AccountRangeMaxResults/2, AccountRangeMaxResults/2)
-
-	t.Logf("test getting number of results greater than max %d", AccountRangeMaxResults)
-	accountRangeTest(t, &trie, state, &common.Hash{0x0}, AccountRangeMaxResults*2, AccountRangeMaxResults)
-
-	t.Logf("test with empty 'start' hash")
-	accountRangeTest(t, &trie, state, nil, AccountRangeMaxResults, AccountRangeMaxResults)
-
-	t.Logf("test pagination")
-
+	accountRangeTest(t, &trie, state, common.Hash{}, AccountRangeMaxResults/2, AccountRangeMaxResults/2)
 	// test pagination
-	firstResult := accountRangeTest(t, &trie, state, &common.Hash{0x0}, AccountRangeMaxResults, AccountRangeMaxResults)
-
-	t.Logf("test pagination 2")
-	secondResult := accountRangeTest(t, &trie, state, &firstResult.Next, AccountRangeMaxResults, AccountRangeMaxResults)
+	firstResult := accountRangeTest(t, &trie, state, common.Hash{}, AccountRangeMaxResults, AccountRangeMaxResults)
+	secondResult := accountRangeTest(t, &trie, state, common.BytesToHash(firstResult.Next), AccountRangeMaxResults, AccountRangeMaxResults)
 
 	hList := make(resultHash, 0)
-	for h1, addr1 := range firstResult.Accounts {
-		h := &common.Hash{}
-		h.SetBytes(h1.Bytes())
-		hList = append(hList, h)
-		for h2, addr2 := range secondResult.Accounts {
-			// Make sure that the hashes aren't the same
-			if bytes.Equal(h1.Bytes(), h2.Bytes()) {
-				t.Fatalf("pagination test failed:  results should not overlap")
-			}
-
-			// If either address is nil, then it makes no sense to compare
-			// them as they might be two different accounts.
-			if addr1 == nil || addr2 == nil {
-				continue
-			}
-
-			// Since the two hashes are different, they should not have
-			// the same preimage, but let's check anyway in case there
-			// is a bug in the (hash, addr) map generation code.
-			if bytes.Equal(addr1.Bytes(), addr2.Bytes()) {
-				t.Fatalf("pagination test failed: addresses should not repeat")
-			}
+	for addr1 := range firstResult.Accounts {
+		// If address is empty, then it makes no sense to compare
+		// them as they might be two different accounts.
+		if addr1 == (common.Address{}) {
+			continue
+		}
+		if _, duplicate := secondResult.Accounts[addr1]; duplicate {
+			t.Fatalf("pagination test failed:  results should not overlap")
 		}
+		hList = append(hList, crypto.Keccak256Hash(addr1.Bytes()))
 	}
-
 	// Test to see if it's possible to recover from the middle of the previous
 	// set and get an even split between the first and second sets.
-	t.Logf("test random access pagination")
 	sort.Sort(hList)
 	middleH := hList[AccountRangeMaxResults/2]
 	middleResult := accountRangeTest(t, &trie, state, middleH, AccountRangeMaxResults, AccountRangeMaxResults)
-	innone, infirst, insecond := 0, 0, 0
+	missing, infirst, insecond := 0, 0, 0
 	for h := range middleResult.Accounts {
 		if _, ok := firstResult.Accounts[h]; ok {
 			infirst++
 		} else if _, ok := secondResult.Accounts[h]; ok {
 			insecond++
 		} else {
-			innone++
+			missing++
 		}
 	}
-	if innone != 0 {
-		t.Fatalf("%d hashes in the 'middle' set were neither in the first not the second set", innone)
+	if missing != 0 {
+		t.Fatalf("%d hashes in the 'middle' set were neither in the first not the second set", missing)
 	}
 	if infirst != AccountRangeMaxResults/2 {
 		t.Fatalf("Imbalance in the number of first-test results: %d != %d", infirst, AccountRangeMaxResults/2)
@@ -164,20 +130,10 @@ func TestEmptyAccountRange(t *testing.T) {
 		statedb  = state.NewDatabase(rawdb.NewMemoryDatabase())
 		state, _ = state.New(common.Hash{}, statedb, nil)
 	)
-
 	state.Commit(true)
-	root := state.IntermediateRoot(true)
-
-	trie, err := statedb.OpenTrie(root)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	results, err := accountRange(trie, &common.Hash{0x0}, AccountRangeMaxResults)
-	if err != nil {
-		t.Fatalf("Empty results should not trigger an error: %v", err)
-	}
-	if results.Next != common.HexToHash("0") {
+	state.IntermediateRoot(true)
+	results := state.IteratorDump(true, true, true, (common.Hash{}).Bytes(), AccountRangeMaxResults)
+	if bytes.Equal(results.Next, (common.Hash{}).Bytes()) {
 		t.Fatalf("Empty results should not return a second page")
 	}
 	if len(results.Accounts) != 0 {