Merge pull request #5 from MariusVanDerWijden/evaluation

Evaluation

benchmark/bench.go

@@ -18,44 +18,46 @@ func RunFullBench(N int) {
 	time, err := testGeneration(N)
 	// Basic building blocks
 	printResult("BenchmarkTestGeneration", time, err)
-	time, err = execution(N)
-	printResult("BenchmarkExecution", time, err)
-	time, err = verify(N)
-	printResult("BenchmarkVerification", time, err)
-	// single thread execution
-	time, err = single(N)
-	printResult("BenchmarkSingle", time, err)
-	time, err = singleBatch(N)
-	printResult("BenchmarkSingleBatch", time, err)
-	time, err = singleDocker(N)
-	printResult("BenchmarkSingleDocker", time, err)
-	time, err = singleBatchDocker(N)
-	printResult("BenchmarkSingleBatchDocker", time, err)
-
+	/*
+		time, err = execution(N)
+		printResult("BenchmarkExecution", time, err)
+		time, err = verify(N)
+		printResult("BenchmarkVerification", time, err)
+		// single thread execution
+		time, err = single(N)
+		printResult("BenchmarkSingle", time, err)
+		time, err = singleBatch(N)
+		printResult("BenchmarkSingleBatch", time, err)
+		time, err = singleDocker(N)
+		printResult("BenchmarkSingleDocker", time, err)
+		time, err = singleBatchDocker(N)
+		printResult("BenchmarkSingleBatchDocker", time, err)
+	*/
 	// parallel execution linear evms (structure 3.1)
-	time, err = linear(N)
+	//time, err = linear(N)
 	printResult("BenchmarkLinear", time, err)
-	time, err = linearBatch(N)
+	//time, err = linearBatch(N)
 	printResult("BenchmarkLinearBatch", time, err)
-	time, err = linearDocker(N)
-	printResult("BenchmarkLinearDocker", time, err)
-	time, err = linearBatchDocker(N)
-	printResult("BenchmarkLinearBatchDocker", time, err)
-
-	// parallel execution parallel evms (structure 3.2)
-	time, err = parallel(N)
-	printResult("BenchmarkParallel", time, err)
-	time, err = parallelBatch(N)
-	printResult("BenchmarkParallelBatch", time, err)
-	time, err = parallelDocker(N)
-	printResult("BenchmarkParallelDocker", time, err)
+	/*
+		time, err = linearDocker(N)
+		printResult("BenchmarkLinearDocker", time, err)
+		time, err = linearBatchDocker(N)
+		printResult("BenchmarkLinearBatchDocker", time, err)
 
-	time, err = parallelBatchDocker(N)
-	printResult("BenchmarkParallelBatchDocker", time, err)
+		// parallel execution parallel evms (structure 3.2)
+		time, err = parallel(N)
+		printResult("BenchmarkParallel", time, err)
+		time, err = parallelBatch(N)
+		printResult("BenchmarkParallelBatch", time, err)
+		time, err = parallelDocker(N)
+		printResult("BenchmarkParallelDocker", time, err)
 
+		time, err = parallelBatchDocker(N)
+		printResult("BenchmarkParallelBatchDocker", time, err)
+	*/
 	// pipe strategy besu
-	time, err = piping(N)
-	printResult("BenchmarkPipeStrategy", time, err)
+	//time, err = piping(N)
+	//printResult("BenchmarkPipeStrategy", time, err)
 }
 
 func printResult(name string, time time.Duration, err error) {
@@ -106,10 +108,18 @@ func createTempDirs() (string, string, error) {
 	if err != nil {
 		return "", "", err
 	}
+	// set the permissions
+	if err = os.Chmod(outputDir, 0755); err != nil {
+		return "", "", err
+	}
 	crashers, err := ioutil.TempDir("", "")
 	if err != nil {
 		return "", "", err
 	}
+	// set the permissions
+	if err = os.Chmod(crashers, 0755); err != nil {
+		return "", "", err
+	}
 	return outputDir, crashers, nil
 }
 

executor/batch.go

@@ -28,7 +28,7 @@ import (
 
 var (
 	batchSize        = 20
-	concurrencyLimit = 8
+	concurrencyLimit = 1
 )
 
 // ExecuteBatch runs all tests in `dirName` in batches and saves crashers in `outDir`

executor/executor.go

@@ -33,15 +33,15 @@ import (
 
 var (
 	vms = []evms.Evm{
-		evms.NewGethEVM("~/ethereum/FuzzyVM/vms/geth-evm"),
-		evms.NewParityVM("~/ethereum/FuzzyVM/vms/openethereum-evm"),
-		evms.NewNethermindVM("~/ethereum/FuzzyVM/vms/nethtest"),
-		evms.NewBesuVM("~/ethereum/besu/ethereum/evmtool/build/install/evmtool/bin/evm"),
-		evms.NewTurboGethEVM("~/ethereum/FuzzyVM/vms/turbogeth-evm"),
+		evms.NewGethEVM("/home/matematik/ethereum/FuzzyVM/vms/geth-evm"),
+		evms.NewParityVM("/home/matematik/ethereum/FuzzyVM/vms/openethereum-evm"),
+		//evms.NewNethermindVM("/home/matematik/ethereum/FuzzyVM/vms/nethtest"),
+		evms.NewBesuVM("/home/matematik/ethereum/besu/ethereum/evmtool/build/install/evmtool/bin/evm"),
+		evms.NewTurboGethEVM("/home/matematik/ethereum/FuzzyVM/vms/turbogeth-evm"),
 	}
 	PrintTrace   = true
 	ParallelEVMS = false
-	threadlimit  = 8
+	threadlimit  = 10
 )
 
 // Execute runs all tests in `dirName` and saves crashers in `outDir`
@@ -51,7 +51,7 @@ func Execute(dirName, outDir string) error {
 		return err
 	}
 	errChan := make(chan error)
-	if ParallelEVMS {
+	if ParallelEVMS || evms.PipeStrategy {
 		threadlimit = 1
 	}
 	limit := limiter.NewConcurrencyLimiter(threadlimit)

fuzzer/corpus_test.go

@@ -64,12 +64,21 @@ func TestCreateMaxTest(t *testing.T) {
 func TestSample(t *testing.T) {
 	results := SampleLengthCorpus(10000)
 	sort.Ints(results)
-	fmt.Printf("%v", results)
-	groups := make(map[int]int)
+	/*
+		fmt.Printf("%v", results)
+		groups := make(map[int]int)
+		for _, res := range results {
+			groups[res]++
+		}
+		for k, r := range groups {
+			//fmt.Printf("%v : %v\n", k, r)
+		}*/
+	subgroups := make(map[int]int)
 	for _, res := range results {
-		groups[res]++
+		group := res / 500
+		subgroups[group*500]++
 	}
-	for k, r := range groups {
+	for k, r := range subgroups {
 		fmt.Printf("%v : %v\n", k, r)
 	}
 	panic("asdf")

generator/generator.go

@@ -118,8 +118,8 @@ func GenerateProgram(f *filler.Filler) (*fuzzing.GstMaker, []byte) {
 		case 11:
 			// Returns with offset, len
 			var (
-				offset = f.Uint32()
-				len    = f.Uint32()
+				offset = uint32(f.Uint16())
+				len    = uint32(f.Uint16())
 			)
 			p.Return(offset, len)
 		case 12:

generator/jumptable.go

@@ -18,7 +18,6 @@ package generator
 
 import (
 	"encoding/binary"
-	"fmt"
 )
 
 type destination struct {
@@ -83,11 +82,6 @@ func (j *Jumptable) InsertJumps(bytecode []byte) []byte {
 				// if no suitable destination found, set jumpdest to 0
 				bytecode = insertJumpdest(bytecode, pc, 0)
 			}
-		} else {
-			fmt.Printf("%v\n", bytecode)
-			fmt.Printf("%v\n", pc)
-			fmt.Printf("%v\n", bytecode[pc])
-			panic("invalid jumpdest")
 		}
 	}
 	return bytecode

generator/precompiles/ecdsa.go

@@ -38,6 +38,7 @@ func (*ecdsaCaller) call(p *program.Program, f *filler.Filler) error {
 	if err != nil {
 		return err
 	}
+	// Sig is in [R | S | V] we need it in components
 	c := CallObj{
 		Gas:       f.GasInt(),
 		Address:   ecdsaAddr,

generator/precompiles/identity.go

@@ -27,14 +27,14 @@ var identityAddr = common.HexToAddress("0x4")
 type identityCaller struct{}
 
 func (*identityCaller) call(p *program.Program, f *filler.Filler) error {
-	data := f.ByteSlice(int(f.Uint16()))
+	data := f.ByteSlice256()
 	c := CallObj{
 		Gas:       f.GasInt(),
 		Address:   identityAddr,
 		InOffset:  0,
 		InSize:    uint32(len(data)),
 		OutOffset: 0,
-		OutSize:   20,
+		OutSize:   uint32(len(data)),
 		Value:     f.BigInt(),
 	}
 	p.Mstore(data, 0)

generator/precompiles/ripemd.go

@@ -27,7 +27,7 @@ var ripemdAddr = common.HexToAddress("0x3")
 type ripemdCaller struct{}
 
 func (*ripemdCaller) call(p *program.Program, f *filler.Filler) error {
-	data := f.ByteSlice(int(f.Uint16()))
+	data := f.ByteSlice256()
 	c := CallObj{
 		Gas:       f.GasInt(),
 		Address:   ripemdAddr,

go.mod

@@ -3,13 +3,18 @@ module github.com/MariusVanDerWijden/FuzzyVM
 go 1.15
 
 require (
-	github.com/ethereum/go-ethereum v1.9.23
+	github.com/dvyukov/go-fuzz v0.0.0-20210103155950-6a8e9d1f2415 // indirect
+	github.com/elazarl/go-bindata-assetfs v1.0.1 // indirect
+	github.com/ethereum/go-ethereum v1.9.25
 	github.com/google/godepq v0.0.0-20190501212251-2c635fd1e5fe // indirect
 	github.com/hhatto/gocloc v0.3.3 // indirect
 	github.com/holiman/goevmlab v0.0.0-20200925112252-8249743488ae
 	github.com/korovkin/limiter v0.0.0-20190919045942-dac5a6b2a536
 	github.com/pkg/errors v0.9.1
 	github.com/pkg/profile v1.5.0
+	github.com/stephens2424/writerset v1.0.2 // indirect
+	golang.org/x/mod v0.4.1 // indirect
+	golang.org/x/tools v0.0.0-20210114065538-d78b04bdf963 // indirect
 	gopkg.in/urfave/cli.v1 v1.20.0
 )