Return error up stack instead of passing testing var down

tests/block_test.go

@@ -10,40 +10,38 @@ var blockTestDir = filepath.Join(baseDir, "BlockTests")
 
 // TODO: refactor test setup & execution to better align with vm and tx tests
 func TestBcValidBlockTests(t *testing.T) {
-	// SimpleTx3 genesis block does not validate against calculated state root
-	// as of 2015-06-09. unskip once working /Gustav
-	runBlockTestsInFile(filepath.Join(blockTestDir, "bcValidBlockTest.json"), []string{"SimpleTx3"}, t)
+	runBlockTestsInFile(filepath.Join(blockTestDir, "bcValidBlockTest.json"), []string{"SimpleTx3"})
 }
 
 func TestBcUncleTests(t *testing.T) {
-	runBlockTestsInFile(filepath.Join(blockTestDir, "bcUncleTest.json"), []string{}, t)
-	runBlockTestsInFile(filepath.Join(blockTestDir, "bcBruncleTest.json"), []string{}, t)
+	runBlockTestsInFile(filepath.Join(blockTestDir, "bcUncleTest.json"), []string{})
+	runBlockTestsInFile(filepath.Join(blockTestDir, "bcBruncleTest.json"), []string{})
 }
 
 func TestBcUncleHeaderValidityTests(t *testing.T) {
-	runBlockTestsInFile(filepath.Join(blockTestDir, "bcUncleHeaderValiditiy.json"), []string{}, t)
+	runBlockTestsInFile(filepath.Join(blockTestDir, "bcUncleHeaderValiditiy.json"), []string{})
 }
 
 func TestBcInvalidHeaderTests(t *testing.T) {
-	runBlockTestsInFile(filepath.Join(blockTestDir, "bcInvalidHeaderTest.json"), []string{}, t)
+	runBlockTestsInFile(filepath.Join(blockTestDir, "bcInvalidHeaderTest.json"), []string{})
 }
 
 func TestBcInvalidRLPTests(t *testing.T) {
-	runBlockTestsInFile(filepath.Join(blockTestDir, "bcInvalidRLPTest.json"), []string{}, t)
+	runBlockTestsInFile(filepath.Join(blockTestDir, "bcInvalidRLPTest.json"), []string{})
 }
 
 func TestBcRPCAPITests(t *testing.T) {
-	runBlockTestsInFile(filepath.Join(blockTestDir, "bcRPC_API_Test.json"), []string{}, t)
+	runBlockTestsInFile(filepath.Join(blockTestDir, "bcRPC_API_Test.json"), []string{})
 }
 
 func TestBcForkBlockTests(t *testing.T) {
-	runBlockTestsInFile(filepath.Join(blockTestDir, "bcForkBlockTest.json"), []string{}, t)
+	runBlockTestsInFile(filepath.Join(blockTestDir, "bcForkBlockTest.json"), []string{})
 }
 
 func TestBcTotalDifficulty(t *testing.T) {
-	runBlockTestsInFile(filepath.Join(blockTestDir, "bcTotalDifficultyTest.json"), []string{}, t)
+	runBlockTestsInFile(filepath.Join(blockTestDir, "bcTotalDifficultyTest.json"), []string{})
 }
 
 func TestBcWallet(t *testing.T) {
-	runBlockTestsInFile(filepath.Join(blockTestDir, "bcWalletTest.json"), []string{}, t)
+	runBlockTestsInFile(filepath.Join(blockTestDir, "bcWalletTest.json"), []string{})
 }

tests/block_test_util.go

@@ -11,7 +11,6 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
-	"testing"
 	"time"
 
 	"github.com/ethereum/go-ethereum/accounts"
@@ -87,10 +86,10 @@ type btTransaction struct {
 	Value    string
 }
 
-func runBlockTestsInFile(filepath string, snafus []string, t *testing.T) {
+func runBlockTestsInFile(filepath string, snafus []string) error {
 	bt, err := LoadBlockTests(filepath)
 	if err != nil {
-		t.Fatal(err)
+		return nil
 	}
 
 	notWorking := make(map[string]bool, 100)
@@ -100,21 +99,24 @@ func runBlockTestsInFile(filepath string, snafus []string, t *testing.T) {
 
 	for name, test := range bt {
 		if !notWorking[name] {
-			runBlockTest(name, test, t)
+			if err := runBlockTest(name, test); err != nil {
+				return err
+			}
 		}
 	}
+	return nil
 }
 
-func runBlockTest(name string, test *BlockTest, t *testing.T) {
+func runBlockTest(name string, test *BlockTest) error {
 	cfg := testEthConfig()
 	ethereum, err := eth.New(cfg)
 	if err != nil {
-		t.Fatalf("%v", err)
+		return err
 	}
 
 	err = ethereum.Start()
 	if err != nil {
-		t.Fatalf("%v", err)
+		return err
 	}
 
 	// import the genesis block
@@ -123,19 +125,20 @@ func runBlockTest(name string, test *BlockTest, t *testing.T) {
 	// import pre accounts
 	statedb, err := test.InsertPreState(ethereum)
 	if err != nil {
-		t.Fatalf("InsertPreState: %v", err)
+		return fmt.Errorf("InsertPreState: %v", err)
 	}
 
 	err = test.TryBlocksInsert(ethereum.ChainManager())
 	if err != nil {
-		t.Fatal(err)
+		return err
 	}
 
 	if err = test.ValidatePostState(statedb); err != nil {
-		t.Fatal("post state validation failed: %v", err)
+		return fmt.Errorf("post state validation failed: %v", err)
 	}
 	fmt.Println("Block test passed: ", name)
 	// t.Log("Block test passed: ", name)
+	return nil
 }
 
 func testEthConfig() *eth.Config {

tests/init.go

@@ -1,13 +1,14 @@
 package tests
 
 import (
+	"bytes"
 	"encoding/json"
+	"fmt"
 	"io"
 	"io/ioutil"
 	// "log"
 	"net/http"
 	"os"
-	"testing"
 
 	// logpkg "github.com/ethereum/go-ethereum/logger"
 )
@@ -20,34 +21,40 @@ import (
 // 	logpkg.AddLogSystem(Logger)
 // }
 
-func readJSON(t *testing.T, reader io.Reader, value interface{}) {
+func readJSON(reader io.Reader, value interface{}) error {
 	data, err := ioutil.ReadAll(reader)
 	err = json.Unmarshal(data, &value)
 	if err != nil {
-		t.Error(err)
+		return err
 	}
+	return nil
 }
 
-func CreateHttpTests(t *testing.T, uri string, value interface{}) {
+func CreateHttpTests(uri string, value interface{}) error {
 	resp, err := http.Get(uri)
 	if err != nil {
-		t.Error(err)
-
-		return
+		return err
 	}
 	defer resp.Body.Close()
 
-	readJSON(t, resp.Body, value)
+	err = readJSON(resp.Body, value)
+	if err != nil {
+		return err
+	}
+	return nil
 }
 
-func CreateFileTests(t *testing.T, fn string, value interface{}) {
+func CreateFileTests(fn string, value interface{}) error {
 	file, err := os.Open(fn)
 	if err != nil {
-		t.Error(err)
-
-		return
+		return err
 	}
 	defer file.Close()
 
-	readJSON(t, file, value)
+	err = readJSON(file, value)
+	if err != nil {
+		return err
+	}
+	return nil
+}
 }

tests/state_test.go

@@ -10,62 +10,86 @@ var stateTestDir = filepath.Join(baseDir, "StateTests")
 
 func TestStateSystemOperations(t *testing.T) {
 	fn := filepath.Join(stateTestDir, "stSystemOperationsTest.json")
-	RunStateTest(fn, t)
+	if err := RunStateTest(fn); err != nil {
+		t.Error(err)
+	}
 }
 
 func TestStateExample(t *testing.T) {
 	fn := filepath.Join(stateTestDir, "stExample.json")
-	RunStateTest(fn, t)
+	if err := RunStateTest(fn); err != nil {
+		t.Error(err)
+	}
 }
 
 func TestStatePreCompiledContracts(t *testing.T) {
 	fn := filepath.Join(stateTestDir, "stPreCompiledContracts.json")
-	RunStateTest(fn, t)
+	if err := RunStateTest(fn); err != nil {
+		t.Error(err)
+	}
 }
 
 func TestStateRecursiveCreate(t *testing.T) {
 	fn := filepath.Join(stateTestDir, "stRecursiveCreate.json")
-	RunStateTest(fn, t)
+	if err := RunStateTest(fn); err != nil {
+		t.Error(err)
+	}
 }
 
 func TestStateSpecial(t *testing.T) {
 	fn := filepath.Join(stateTestDir, "stSpecialTest.json")
-	RunStateTest(fn, t)
+	if err := RunStateTest(fn); err != nil {
+		t.Error(err)
+	}
 }
 
 func TestStateRefund(t *testing.T) {
 	fn := filepath.Join(stateTestDir, "stRefundTest.json")
-	RunStateTest(fn, t)
+	if err := RunStateTest(fn); err != nil {
+		t.Error(err)
+	}
 }
 
 func TestStateBlockHash(t *testing.T) {
 	fn := filepath.Join(stateTestDir, "stBlockHashTest.json")
-	RunStateTest(fn, t)
+	if err := RunStateTest(fn); err != nil {
+		t.Error(err)
+	}
 }
 
 func TestStateInitCode(t *testing.T) {
 	fn := filepath.Join(stateTestDir, "stInitCodeTest.json")
-	RunStateTest(fn, t)
+	if err := RunStateTest(fn); err != nil {
+		t.Error(err)
+	}
 }
 
 func TestStateLog(t *testing.T) {
 	fn := filepath.Join(stateTestDir, "stLogTests.json")
-	RunStateTest(fn, t)
+	if err := RunStateTest(fn); err != nil {
+		t.Error(err)
+	}
 }
 
 func TestStateTransaction(t *testing.T) {
 	fn := filepath.Join(stateTestDir, "stTransactionTest.json")
-	RunStateTest(fn, t)
+	if err := RunStateTest(fn); err != nil {
+		t.Error(err)
+	}
 }
 
 func TestCallCreateCallCode(t *testing.T) {
 	fn := filepath.Join(stateTestDir, "stCallCreateCallCodeTest.json")
-	RunStateTest(fn, t)
+	if err := RunStateTest(fn); err != nil {
+		t.Error(err)
+	}
 }
 
 func TestMemory(t *testing.T) {
 	fn := filepath.Join(stateTestDir, "stMemoryTest.json")
-	RunStateTest(fn, t)
+	if err := RunStateTest(fn); err != nil {
+		t.Error(err)
+	}
 }
 
 func TestMemoryStress(t *testing.T) {
@@ -73,7 +97,9 @@ func TestMemoryStress(t *testing.T) {
 		t.Skip()
 	}
 	fn := filepath.Join(stateTestDir, "stMemoryStressTest.json")
-	RunStateTest(fn, t)
+	if err := RunStateTest(fn); err != nil {
+		t.Error(err)
+	}
 }
 
 func TestQuadraticComplexity(t *testing.T) {
@@ -81,22 +107,30 @@ func TestQuadraticComplexity(t *testing.T) {
 		t.Skip()
 	}
 	fn := filepath.Join(stateTestDir, "stQuadraticComplexityTest.json")
-	RunStateTest(fn, t)
+	if err := RunStateTest(fn); err != nil {
+		t.Error(err)
+	}
 }
 
 func TestSolidity(t *testing.T) {
 	fn := filepath.Join(stateTestDir, "stSolidityTest.json")
-	RunStateTest(fn, t)
+	if err := RunStateTest(fn); err != nil {
+		t.Error(err)
+	}
 }
 
 func TestWallet(t *testing.T) {
 	fn := filepath.Join(stateTestDir, "stWalletTest.json")
-	RunStateTest(fn, t)
+	if err := RunStateTest(fn); err != nil {
+		t.Error(err)
+	}
 }
 
 func TestStateTestsRandom(t *testing.T) {
 	fns, _ := filepath.Glob("./files/StateTests/RandomTests/*")
 	for _, fn := range fns {
-		RunStateTest(fn, t)
+		if err := RunStateTest(fn); err != nil {
+			t.Error(err)
+		}
 	}
 }

tests/state_test_util.go

@@ -6,22 +6,22 @@ import (
 	"fmt"
 	"math/big"
 	"strconv"
-	"testing"
+	// "testing"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/state"
-	"github.com/ethereum/go-ethereum/core/types"
+	// "github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
 	// "github.com/ethereum/go-ethereum/logger"
 )
 
-func RunStateTest(p string, t *testing.T) {
+func RunStateTest(p string) error {
 
 	tests := make(map[string]VmTest)
-	CreateFileTests(t, p, &tests)
+	CreateFileTests(p, &tests)
 
 	for name, test := range tests {
 		/*
@@ -38,7 +38,7 @@ func RunStateTest(p string, t *testing.T) {
 			obj := StateObjectFromAccount(db, addr, account)
 			statedb.SetStateObject(obj)
 			for a, v := range account.Storage {
-				obj.SetState(common.HexToHash(a), common.NewValue(common.FromHex(v)))
+				obj.SetState(common.HexToHash(a), common.HexToHash(s))
 			}
 		}
 
@@ -64,6 +64,7 @@ func RunStateTest(p string, t *testing.T) {
 
 		ret, logs, _, _ = RunState(statedb, env, test.Transaction)
 
+		// Compare expected  and actual return
 		switch name {
 		// the memory required for these tests (4294967297 bytes) would take too much time.
 		// on 19 May 2015 decided to skip these tests their output.
@@ -71,7 +72,7 @@ func RunStateTest(p string, t *testing.T) {
 		default:
 			rexp := common.FromHex(test.Out)
 			if bytes.Compare(rexp, ret) != 0 {
-				t.Errorf("%s's return failed. Expected %x, got %x\n", name, rexp, ret)
+				return fmt.Errorf("%s's return failed. Expected %x, got %x\n", name, rexp, ret)
 			}
 		}
 
@@ -83,11 +84,11 @@ func RunStateTest(p string, t *testing.T) {
 			}
 
 			if obj.Balance().Cmp(common.Big(account.Balance)) != 0 {
-				t.Errorf("%s's : (%x) balance failed. Expected %v, got %v => %v\n", name, obj.Address().Bytes()[:4], account.Balance, obj.Balance(), new(big.Int).Sub(common.Big(account.Balance), obj.Balance()))
+				return fmt.Errorf("%s's : (%x) balance failed. Expected %v, got %v => %v\n", name, obj.Address().Bytes()[:4], account.Balance, obj.Balance(), new(big.Int).Sub(common.Big(account.Balance), obj.Balance()))
 			}
 
 			if obj.Nonce() != common.String2Big(account.Nonce).Uint64() {
-				t.Errorf("%s's : (%x) nonce failed. Expected %v, got %v\n", name, obj.Address().Bytes()[:4], account.Nonce, obj.Nonce())
+				return fmt.Errorf("%s's : (%x) nonce failed. Expected %v, got %v\n", name, obj.Address().Bytes()[:4], account.Nonce, obj.Nonce())
 			}
 
 			for addr, value := range account.Storage {
@@ -95,7 +96,7 @@ func RunStateTest(p string, t *testing.T) {
 				vexp := common.FromHex(value)
 
 				if bytes.Compare(v, vexp) != 0 {
-					t.Errorf("%s's : (%x: %s) storage failed. Expected %x, got %x (%v %v)\n", name, obj.Address().Bytes()[0:4], addr, vexp, v, common.BigD(vexp), common.BigD(v))
+					return fmt.Errorf("%s's : (%x: %s) storage failed. Expected %x, got %x (%v %v)\n", name, obj.Address().Bytes()[0:4], addr, vexp, v, common.BigD(vexp), common.BigD(v))
 				}
 			}
 		}
@@ -103,14 +104,14 @@ func RunStateTest(p string, t *testing.T) {
 		statedb.Sync()
 		//if !bytes.Equal(common.Hex2Bytes(test.PostStateRoot), statedb.Root()) {
 		if common.HexToHash(test.PostStateRoot) != statedb.Root() {
-			t.Errorf("%s's : Post state root error. Expected %s, got %x", name, test.PostStateRoot, statedb.Root())
+			return fmt.Errorf("%s's : Post state root error. Expected %s, got %x", name, test.PostStateRoot, statedb.Root())
 		}
 
 		// check logs
 		if len(test.Logs) > 0 {
 			lerr := CheckLogs(test.Logs, logs, t)
 			if lerr != nil {
-				t.Errorf("'%s' ", name, lerr.Error())
+				return fmt.Errorf("'%s' ", name, lerr.Error())
 			}
 		}
 

tests/vm_test.go

@@ -10,72 +10,100 @@ var vmTestDir = filepath.Join(baseDir, "VMTests")
 // I've created a new function for each tests so it's easier to identify where the problem lies if any of them fail.
 func TestVMArithmetic(t *testing.T) {
 	fn := filepath.Join(vmTestDir, "vmArithmeticTest.json")
-	RunVmTest(fn, t)
+	if err := RunVmTest(fn); err != nil {
+		t.Error(err)
+	}
 }
 
 func TestBitwiseLogicOperation(t *testing.T) {
 	fn := filepath.Join(vmTestDir, "vmBitwiseLogicOperationTest.json")
-	RunVmTest(fn, t)
+	if err := RunVmTest(fn); err != nil {
+		t.Error(err)
+	}
 }
 
 func TestBlockInfo(t *testing.T) {
 	fn := filepath.Join(vmTestDir, "vmBlockInfoTest.json")
-	RunVmTest(fn, t)
+	if err := RunVmTest(fn); err != nil {
+		t.Error(err)
+	}
 }
 
 func TestEnvironmentalInfo(t *testing.T) {
 	fn := filepath.Join(vmTestDir, "vmEnvironmentalInfoTest.json")
-	RunVmTest(fn, t)
+	if err := RunVmTest(fn); err != nil {
+		t.Error(err)
+	}
 }
 
 func TestFlowOperation(t *testing.T) {
 	fn := filepath.Join(vmTestDir, "vmIOandFlowOperationsTest.json")
-	RunVmTest(fn, t)
+	if err := RunVmTest(fn); err != nil {
+		t.Error(err)
+	}
 }
 
 func TestLogTest(t *testing.T) {
 	fn := filepath.Join(vmTestDir, "vmLogTest.json")
-	RunVmTest(fn, t)
+	if err := RunVmTest(fn); err != nil {
+		t.Error(err)
+	}
 }
 
 func TestPerformance(t *testing.T) {
 	fn := filepath.Join(vmTestDir, "vmPerformanceTest.json")
-	RunVmTest(fn, t)
+	if err := RunVmTest(fn); err != nil {
+		t.Error(err)
+	}
 }
 
 func TestPushDupSwap(t *testing.T) {
 	fn := filepath.Join(vmTestDir, "vmPushDupSwapTest.json")
-	RunVmTest(fn, t)
+	if err := RunVmTest(fn); err != nil {
+		t.Error(err)
+	}
 }
 
 func TestVMSha3(t *testing.T) {
 	fn := filepath.Join(vmTestDir, "vmSha3Test.json")
-	RunVmTest(fn, t)
+	if err := RunVmTest(fn); err != nil {
+		t.Error(err)
+	}
 }
 
 func TestVm(t *testing.T) {
 	fn := filepath.Join(vmTestDir, "vmtests.json")
-	RunVmTest(fn, t)
+	if err := RunVmTest(fn); err != nil {
+		t.Error(err)
+	}
 }
 
 func TestVmLog(t *testing.T) {
 	fn := filepath.Join(vmTestDir, "vmLogTest.json")
-	RunVmTest(fn, t)
+	if err := RunVmTest(fn); err != nil {
+		t.Error(err)
+	}
 }
 
 func TestInputLimits(t *testing.T) {
 	fn := filepath.Join(vmTestDir, "vmInputLimits.json")
-	RunVmTest(fn, t)
+	if err := RunVmTest(fn); err != nil {
+		t.Error(err)
+	}
 }
 
 func TestInputLimitsLight(t *testing.T) {
 	fn := filepath.Join(vmTestDir, "vmInputLimitsLight.json")
-	RunVmTest(fn, t)
+	if err := RunVmTest(fn); err != nil {
+		t.Error(err)
+	}
 }
 
 func TestVMRandom(t *testing.T) {
 	fns, _ := filepath.Glob(filepath.Join(baseDir, "RandomTests", "*"))
 	for _, fn := range fns {
-		RunVmTest(fn, t)
+		if err := RunVmTest(fn); err != nil {
+			t.Error(err)
+		}
 	}
 }

tests/vm_test_util.go

@@ -6,7 +6,7 @@ import (
 	"fmt"
 	"math/big"
 	"strconv"
-	"testing"
+	// "testing"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core"
@@ -79,10 +79,13 @@ type VmTest struct {
 	PostStateRoot string
 }
 
-func RunVmTest(p string, t *testing.T) {
+func RunVmTest(p string) error {
 
 	tests := make(map[string]VmTest)
-	CreateFileTests(t, p, &tests)
+	err := CreateFileTests(p, &tests)
+	if err != nil {
+		return err
+	}
 
 	for name, test := range tests {
 		/*
@@ -128,16 +131,16 @@ func RunVmTest(p string, t *testing.T) {
 		// Compare expectedand actual return
 		rexp := common.FromHex(test.Out)
 		if bytes.Compare(rexp, ret) != 0 {
-			t.Errorf("%s's return failed. Expected %x, got %x\n", name, rexp, ret)
+			return fmt.Errorf("%s's return failed. Expected %x, got %x\n", name, rexp, ret)
 		}
 
 		// Check gas usage
 		if len(test.Gas) == 0 && err == nil {
-			t.Errorf("%s's gas unspecified, indicating an error. VM returned (incorrectly) successfull", name)
+			return fmt.Errorf("%s's gas unspecified, indicating an error. VM returned (incorrectly) successfull", name)
 		} else {
 			gexp := common.Big(test.Gas)
 			if gexp.Cmp(gas) != 0 {
-				t.Errorf("%s's gas failed. Expected %v, got %v\n", name, gexp, gas)
+				return fmt.Errorf("%s's gas failed. Expected %v, got %v\n", name, gexp, gas)
 			}
 		}
 
@@ -153,7 +156,7 @@ func RunVmTest(p string, t *testing.T) {
 				vexp := common.HexToHash(value)
 
 				if v != vexp {
-					t.Errorf("%s's : (%x: %s) storage failed. Expected %x, got %x (%v %v)\n", name, obj.Address().Bytes()[0:4], addr, vexp, v, vexp.Big(), v.Big())
+					return t.Errorf("%s's : (%x: %s) storage failed. Expected %x, got %x (%v %v)\n", name, obj.Address().Bytes()[0:4], addr, vexp, v, vexp.Big(), v.Big())
 				}
 			}
 		}
@@ -168,6 +171,7 @@ func RunVmTest(p string, t *testing.T) {
 		//fmt.Println(string(statedb.Dump()))
 	}
 	// logger.Flush()
+	return nil
 }
 
 type Env struct {