tests: update to latest tests (#22290)

This updates the consensus tests to commit 31d6630 and
adds support for access list transactions in the test runner.

Co-authored-by: Martin Holst Swende <martin@swende.se>

tests/block_test.go

@@ -25,7 +25,11 @@ func TestBlockchain(t *testing.T) {
 
 	bt := new(testMatcher)
 	// General state tests are 'exported' as blockchain tests, but we can run them natively.
-	bt.skipLoad(`^GeneralStateTests/`)
+	// For speedier CI-runs, the line below can be uncommented, so those are skipped.
+	// For now, in hardfork-times (Berlin), we run the tests both as StateTests and
+	// as blockchain tests, since the latter also covers things like receipt root
+	//bt.skipLoad(`^GeneralStateTests/`)
+
 	// Skip random failures due to selfish mining test
 	bt.skipLoad(`.*bcForgedTest/bcForkUncle\.json`)
 
@@ -43,15 +47,7 @@ func TestBlockchain(t *testing.T) {
 	// test takes a lot for time and goes easily OOM because of sha3 calculation on a huge range,
 	// using 4.6 TGas
 	bt.skipLoad(`.*randomStatetest94.json.*`)
-
 	bt.walk(t, blockTestDir, func(t *testing.T, name string, test *BlockTest) {
-		if test.json.Network == "Berlin" {
-			// Our current berlin-tests were generated using YOLOv2 rules, hence a lot of them
-			// fail when berlin is defined as YOLOv3. We skip those, until they've been
-			// regenerated and re-imported
-			// TODO (@holiman)
-			return
-		}
 		if err := bt.checkFailure(t, name+"/trie", test.Run(false)); err != nil {
 			t.Errorf("test without snapshotter failed: %v", err)
 		}

tests/gen_stenv.go

@@ -13,6 +13,7 @@ import (
 
 var _ = (*stEnvMarshaling)(nil)
 
+// MarshalJSON marshals as JSON.
 func (s stEnv) MarshalJSON() ([]byte, error) {
 	type stEnv struct {
 		Coinbase   common.UnprefixedAddress `json:"currentCoinbase"   gencodec:"required"`
@@ -30,6 +31,7 @@ func (s stEnv) MarshalJSON() ([]byte, error) {
 	return json.Marshal(&enc)
 }
 
+// UnmarshalJSON unmarshals from JSON.
 func (s *stEnv) UnmarshalJSON(input []byte) error {
 	type stEnv struct {
 		Coinbase   *common.UnprefixedAddress `json:"currentCoinbase"   gencodec:"required"`

tests/gen_sttransaction.go

@@ -8,25 +8,29 @@ import (
 
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/common/math"
+	"github.com/ethereum/go-ethereum/core/types"
 )
 
 var _ = (*stTransactionMarshaling)(nil)
 
+// MarshalJSON marshals as JSON.
 func (s stTransaction) MarshalJSON() ([]byte, error) {
 	type stTransaction struct {
-		GasPrice   *math.HexOrDecimal256 `json:"gasPrice"`
-		Nonce      math.HexOrDecimal64   `json:"nonce"`
-		To         string                `json:"to"`
-		Data       []string              `json:"data"`
-		GasLimit   []math.HexOrDecimal64 `json:"gasLimit"`
-		Value      []string              `json:"value"`
-		PrivateKey hexutil.Bytes         `json:"secretKey"`
+		GasPrice    *math.HexOrDecimal256 `json:"gasPrice"`
+		Nonce       math.HexOrDecimal64   `json:"nonce"`
+		To          string                `json:"to"`
+		Data        []string              `json:"data"`
+		AccessLists []*types.AccessList   `json:"accessLists,omitempty"`
+		GasLimit    []math.HexOrDecimal64 `json:"gasLimit"`
+		Value       []string              `json:"value"`
+		PrivateKey  hexutil.Bytes         `json:"secretKey"`
 	}
 	var enc stTransaction
 	enc.GasPrice = (*math.HexOrDecimal256)(s.GasPrice)
 	enc.Nonce = math.HexOrDecimal64(s.Nonce)
 	enc.To = s.To
 	enc.Data = s.Data
+	enc.AccessLists = s.AccessLists
 	if s.GasLimit != nil {
 		enc.GasLimit = make([]math.HexOrDecimal64, len(s.GasLimit))
 		for k, v := range s.GasLimit {
@@ -38,15 +42,17 @@ func (s stTransaction) MarshalJSON() ([]byte, error) {
 	return json.Marshal(&enc)
 }
 
+// UnmarshalJSON unmarshals from JSON.
 func (s *stTransaction) UnmarshalJSON(input []byte) error {
 	type stTransaction struct {
-		GasPrice   *math.HexOrDecimal256 `json:"gasPrice"`
-		Nonce      *math.HexOrDecimal64  `json:"nonce"`
-		To         *string               `json:"to"`
-		Data       []string              `json:"data"`
-		GasLimit   []math.HexOrDecimal64 `json:"gasLimit"`
-		Value      []string              `json:"value"`
-		PrivateKey *hexutil.Bytes        `json:"secretKey"`
+		GasPrice    *math.HexOrDecimal256 `json:"gasPrice"`
+		Nonce       *math.HexOrDecimal64  `json:"nonce"`
+		To          *string               `json:"to"`
+		Data        []string              `json:"data"`
+		AccessLists []*types.AccessList   `json:"accessLists,omitempty"`
+		GasLimit    []math.HexOrDecimal64 `json:"gasLimit"`
+		Value       []string              `json:"value"`
+		PrivateKey  *hexutil.Bytes        `json:"secretKey"`
 	}
 	var dec stTransaction
 	if err := json.Unmarshal(input, &dec); err != nil {
@@ -64,6 +70,9 @@ func (s *stTransaction) UnmarshalJSON(input []byte) error {
 	if dec.Data != nil {
 		s.Data = dec.Data
 	}
+	if dec.AccessLists != nil {
+		s.AccessLists = dec.AccessLists
+	}
 	if dec.GasLimit != nil {
 		s.GasLimit = make([]uint64, len(dec.GasLimit))
 		for k, v := range dec.GasLimit {

tests/gen_vmexec.go

@@ -14,6 +14,7 @@ import (
 
 var _ = (*vmExecMarshaling)(nil)
 
+// MarshalJSON marshals as JSON.
 func (v vmExec) MarshalJSON() ([]byte, error) {
 	type vmExec struct {
 		Address  common.UnprefixedAddress `json:"address"  gencodec:"required"`
@@ -37,6 +38,7 @@ func (v vmExec) MarshalJSON() ([]byte, error) {
 	return json.Marshal(&enc)
 }
 
+// UnmarshalJSON unmarshals from JSON.
 func (v *vmExec) UnmarshalJSON(input []byte) error {
 	type vmExec struct {
 		Address  *common.UnprefixedAddress `json:"address"  gencodec:"required"`

tests/state_test.go

@@ -62,14 +62,6 @@ func TestState(t *testing.T) {
 	} {
 		st.walk(t, dir, func(t *testing.T, name string, test *StateTest) {
 			for _, subtest := range test.Subtests() {
-				if subtest.Fork == "Berlin" {
-					// Our current berlin-tests were generated using YOLOv2 rules, hence a lot of them
-					// fail when berlin is defined as YOLOv3. We skip those, until they've been
-					// regenerated and re-imported
-					// TODO (@holiman)
-					continue
-				}
-
 				subtest := subtest
 				key := fmt.Sprintf("%s/%d", subtest.Fork, subtest.Index)
 				name := name + "/" + key

tests/state_test_util.go

@@ -95,13 +95,14 @@ type stEnvMarshaling struct {
 //go:generate gencodec -type stTransaction -field-override stTransactionMarshaling -out gen_sttransaction.go
 
 type stTransaction struct {
-	GasPrice   *big.Int `json:"gasPrice"`
-	Nonce      uint64   `json:"nonce"`
-	To         string   `json:"to"`
-	Data       []string `json:"data"`
-	GasLimit   []uint64 `json:"gasLimit"`
-	Value      []string `json:"value"`
-	PrivateKey []byte   `json:"secretKey"`
+	GasPrice    *big.Int            `json:"gasPrice"`
+	Nonce       uint64              `json:"nonce"`
+	To          string              `json:"to"`
+	Data        []string            `json:"data"`
+	AccessLists []*types.AccessList `json:"accessLists,omitempty"`
+	GasLimit    []uint64            `json:"gasLimit"`
+	Value       []string            `json:"value"`
+	PrivateKey  []byte              `json:"secretKey"`
 }
 
 type stTransactionMarshaling struct {
@@ -292,8 +293,11 @@ func (tx *stTransaction) toMessage(ps stPostState) (core.Message, error) {
 	if err != nil {
 		return nil, fmt.Errorf("invalid tx data %q", dataHex)
 	}
-
-	msg := types.NewMessage(from, to, tx.Nonce, value, gasLimit, tx.GasPrice, data, nil, true)
+	var accessList types.AccessList
+	if tx.AccessLists != nil && tx.AccessLists[ps.Indexes.Data] != nil {
+		accessList = *tx.AccessLists[ps.Indexes.Data]
+	}
+	msg := types.NewMessage(from, to, tx.Nonce, value, gasLimit, tx.GasPrice, data, accessList, true)
 	return msg, nil
 }
 

tests/testdata

@@ -1 +1 @@
-Subproject commit 6c863f03bee8d7a66bb7a028a9f880a86a5f4975
+Subproject commit 31d663076b6678df18983d6da912d7cad4ad3416