Merge pull request #10 from MariusVanDerWijden/strats

all: split out strategies, add config files

README.md

@@ -2,9 +2,14 @@
 
 A framework to fuzz Ethereum Virtual Machine implementations.
 
+FuzzyVM uses two different processes. 
+One process generates test cases and the other one executes them on different EVM implementations.
+The traces of the EVM's are collected and compared against each other.
+If a crasher is found, 
+
 ## Environment
-You need to have golang < 1.15 installed as go version 1.15 includes some 
-changes that break go-fuzz.
+You might need to have golang < 1.15 installed as go version 1.15 includes some 
+changes that might break go-fuzz.
 
 ## Install instructions
 
@@ -21,3 +26,14 @@ go build
 ./FuzzyVM
 ```
 
+If the fuzzer breaks (or you shut it down) it might leave test cases laying around.
+You can execute the test cases with `./FuzzyVM --exec`.
+
+It makes sense to create an initial corpus in order to improve the efficiency of the fuzzer.
+You can generate corpus elements with `./FuzzyVM --corpus N`.
+
+## Config
+
+FuzzyVM has to be configured to know which EVMs it should use.
+You can specify the paths to the EVMs in a config file.
+An example for the config file can be found in `config.toml`.

benchmark/linear.go

@@ -6,6 +6,7 @@ import (
 
  "github.com/MariusVanDerWijden/FuzzyVM/executor"
  "github.com/MariusVanDerWijden/FuzzyVM/generator"
+ "github.com/holiman/goevmlab/evms"
 )
 
 // testGeneration generates N programs.
@@ -32,13 +33,14 @@ func verify(N int) (time.Duration, error) {
  return time.Nanosecond, err
  }
  name = fmt.Sprintf("%v/%v.json", outDir, name)
- out, err := executor.ExecuteTest(name)
+ exc := newExecutor()
+ out, err := exc.ExecuteTest(name)
  if err != nil {
  return time.Nanosecond, err
  }
  start := time.Now()
  for i := 0; i < N; i++ {
- if !executor.Verify(name, out) {
+ if !exc.Verify(name, out) {
  return time.Nanosecond, fmt.Errorf("Verification failed: %v", name)
  }
  }
@@ -56,10 +58,10 @@ func execution(N int) (time.Duration, error) {
  return time.Nanosecond, err
  }
  name = fmt.Sprintf("%v.json", name)
- executor.PrintTrace = false
+ exc := newExecutor()
  start := time.Now()
  for i := 0; i < N; i++ {
- executor.ExecuteFullTest(outDir, crashers, name, false)
+ exc.ExecuteFullTest(outDir, crashers, name, false)
  }
  return time.Since(start), nil
 }
@@ -87,10 +89,15 @@ func execMultiple(N int, threadlimit int) (time.Duration, error) {
  }
  names = append(names, fmt.Sprintf("%v.json", name))
  }
- executor.PrintTrace = false
+ exc := newExecutor()
  start := time.Now()
- if err := executor.Execute(outDir, crashers, threadlimit); err != nil {
+ if err := exc.Execute(outDir, crashers, threadlimit); err != nil {
  return time.Nanosecond, err
  }
  return time.Since(start), nil
 }
+
+func newExecutor() *executor.Executor {
+ // TODO add meaningful vms for benchmarks
+ return executor.NewExecutor([]evms.Evm{}, false)
+}

config.go

@@ -0,0 +1,94 @@
+// Copyright 2020 Marius van der Wijden
+// This file is part of the fuzzy-vm library.
+//
+// The fuzzy-vm library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The fuzzy-vm library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the fuzzy-vm library. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+ "bufio"
+ "errors"
+ "fmt"
+ "os"
+ "reflect"
+ "unicode"
+
+ "github.com/holiman/goevmlab/evms"
+ "github.com/naoina/toml"
+)
+
+func getVMsFromConfig(file string) ([]evms.Evm, error) {
+ conf, err := loadConfig(file)
+ if err != nil {
+ return []evms.Evm{}, err
+ }
+ var vms []evms.Evm
+ for _, s := range conf.Geth {
+ vms = append(vms, evms.NewGethEVM(s))
+ }
+ for _, s := range conf.Nethermind {
+ vms = append(vms, evms.NewNethermindVM(s))
+ }
+ for _, s := range conf.Besu {
+ vms = append(vms, evms.NewBesuVM(s))
+ }
+ for _, s := range conf.OpenEthereum {
+ vms = append(vms, evms.NewParityVM(s))
+ }
+ for _, s := range conf.Aleth {
+ vms = append(vms, evms.NewAlethVM(s))
+ }
+ return vms, nil
+}
+
+type config struct {
+ Geth []string
+ Besu []string
+ OpenEthereum []string
+ Nethermind []string
+ Aleth []string
+}
+
+func loadConfig(file string) (*config, error) {
+ f, err := os.Open(file)
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+
+ conf := &config{}
+ err = tomlSettings.NewDecoder(bufio.NewReader(f)).Decode(conf)
+ // Add file name to errors that have a line number.
+ if _, ok := err.(*toml.LineError); ok {
+ err = errors.New(file + ", " + err.Error())
+ }
+ return conf, err
+}
+
+// These settings ensure that TOML keys use the same names as Go struct fields.
+var tomlSettings = toml.Config{
+ NormFieldName: func(rt reflect.Type, key string) string {
+ return key
+ },
+ FieldToKey: func(rt reflect.Type, field string) string {
+ return field
+ },
+ MissingField: func(rt reflect.Type, field string) error {
+ link := ""
+ if unicode.IsUpper(rune(rt.Name()[0])) && rt.PkgPath() != "main" {
+ link = fmt.Sprintf(", see https://godoc.org/%s#%s for available fields", rt.PkgPath(), rt.Name())
+ }
+ return fmt.Errorf("field '%s' is not defined in %s%s", field, rt.String(), link)
+ },
+}

config.toml

@@ -0,0 +1,5 @@
+Geth = ["/home/matematik/ethereum/FuzzyVM/vms/geth-evm", "/home/matematik/ethereum/FuzzyVM/vms/turbogeth-evm"]
+OpenEthereum = ["/home/matematik/ethereum/FuzzyVM/vms/openethereum-evm"]
+Nethermind = ["/home/matematik/ethereum/neth_test/nethermind/src/Nethermind/Nethermind.State.Test.Runner/bin/Release/netcoreapp3.1/nethtest"]
+Besu = ["/home/matematik/ethereum/besu/ethereum/evmtool/build/install/evmtool/bin/evm"]
+Aleth = []

executor/executor.go

@@ -31,19 +31,22 @@ import (
  "github.com/pkg/errors"
 )
 
-var (
- vms = []evms.Evm{
- evms.NewGethEVM("/home/matematik/ethereum/FuzzyVM/vms/geth-evm"),
- evms.NewParityVM("/home/matematik/ethereum/FuzzyVM/vms/openethereum-evm"),
- evms.NewNethermindVM("/home/matematik/ethereum/neth_test/nethermind/src/Nethermind/Nethermind.State.Test.Runner/bin/Release/netcoreapp3.1/nethtest"),
- evms.NewBesuVM("/home/matematik/ethereum/besu/ethereum/evmtool/build/install/evmtool/bin/evm"),
- evms.NewGethEVM("/home/matematik/ethereum/FuzzyVM/vms/turbogeth-evm"),
+var PrintTrace = true
+
+type Executor struct {
+ Vms []evms.Evm
+ PrintTrace bool
+}
+
+func NewExecutor(vms []evms.Evm, printTrace bool) *Executor {
+ return &Executor{
+ Vms: vms,
+ PrintTrace: printTrace,
  }
- PrintTrace = true
-)
+}
 
 // Execute runs all tests in `dirName` and saves crashers in `outDir`
-func Execute(dirName, outDir string, threadlimit int) error {
+func (e *Executor) Execute(dirName, outDir string, threadlimit int) error {
  infos, err := ioutil.ReadDir(dirName)
  if err != nil {
  return err
@@ -59,7 +62,7 @@ func Execute(dirName, outDir string, threadlimit int) error {
  meter.Mark(1)
  name := info.Name()
  job := func() {
- if err := ExecuteFullTest(dirName, outDir, name, true); err != nil {
+ if err := e.ExecuteFullTest(dirName, outDir, name, true); err != nil {
  err := errors.Wrap(err, fmt.Sprintf("in file: %v", name))
  fmt.Println(err)
  //errChan <- err
@@ -81,25 +84,25 @@ func Execute(dirName, outDir string, threadlimit int) error {
 }
 
 // ExecuteFullTest executes a single test.
-func ExecuteFullTest(dirName, outDir, filename string, doPurge bool) error {
+func (e *Executor) ExecuteFullTest(dirName, outDir, filename string, doPurge bool) error {
  var (
  testFile = fmt.Sprintf("%v/%v", dirName, filename)
  testName = strings.TrimRight(filename, ".json")
  traceFile = fmt.Sprintf("%v/%v-trace.jsonl", dirName, testName)
  outputs [][]byte
  err error
  )
- outputs, err = ExecuteTest(testFile)
+ outputs, err = e.ExecuteTest(testFile)
  if err != nil {
  return err
  }
- return verifyAndPurge(traceFile, testName, outDir, testFile, outputs, doPurge)
+ return e.verifyAndPurge(traceFile, testName, outDir, testFile, outputs, doPurge)
 }
 
-func verifyAndPurge(traceFile, testName, outDir, testFile string, outputs [][]byte, doPurge bool) error {
- if !Verify(traceFile, outputs) {
+func (e *Executor) verifyAndPurge(traceFile, testName, outDir, testFile string, outputs [][]byte, doPurge bool) error {
+ if !e.Verify(traceFile, outputs) {
  fmt.Printf("Test %v failed, dumping\n", testName)
- if err := dump(testName, outDir, vms, outputs); err != nil {
+ if err := dump(testName, outDir, e.Vms, outputs); err != nil {
  return err
  }
  } else {
@@ -109,17 +112,17 @@ func verifyAndPurge(traceFile, testName, outDir, testFile string, outputs [][]by
  fmt.Printf("Purging failed: %v\n", err)
  }
  } else if PrintTrace {
- printOutputs(outputs)
+ e.printOutputs(outputs)
  }
  }
  return nil
 }
 
 // ExecuteTest executes a state test.
-func ExecuteTest(testName string) ([][]byte, error) {
+func (e *Executor) ExecuteTest(testName string) ([][]byte, error) {
  var buf [][]byte
  var buffer bytes.Buffer
- for _, vm := range vms {
+ for _, vm := range e.Vms {
  buffer.Reset()
  if _, err := vm.RunStateTest(testName, &buffer, false); err != nil {
  return nil, err
@@ -130,7 +133,7 @@ func ExecuteTest(testName string) ([][]byte, error) {
 }
 
 // Verify checks if the traces match the default trace.
-func Verify(traceName string, outputs [][]byte) bool {
+func (e *Executor) Verify(traceName string, outputs [][]byte) bool {
  var ioReaders []io.Reader
  for _, out := range outputs {
  ioReaders = append(ioReaders, bytes.NewReader(out))
@@ -143,7 +146,7 @@ func Verify(traceName string, outputs [][]byte) bool {
  }
  ioReaders = append(ioReaders, bytes.NewBuffer(ref))
  */
- return evms.CompareFiles(vms, ioReaders)
+ return evms.CompareFiles(e.Vms, ioReaders)
 }
 
 // dump writes outputs to a file in case of a verification problem
@@ -168,11 +171,11 @@ func purge(filename, tracename string) error {
 }
 
 // printOutputs prints out the produced traces
-func printOutputs(outputs [][]byte) {
+func (e *Executor) printOutputs(outputs [][]byte) {
  fmt.Println("TRACES:")
  fmt.Println("--------------")
  for i, out := range outputs {
- fmt.Printf("%v: \n", vms[i].Name())
+ fmt.Printf("%v: \n", e.Vms[i].Name())
  fmt.Print(string(out))
  fmt.Println("--------------")
  }

flags.go

@@ -64,4 +64,9 @@ var (
  Name: "corpus",
  Usage: "Number of corpus elements that should be created",
  }
+ configFileFlag = cli.StringFlag{
+ Name: "config",
+ Usage: "Path to the config file required to run FuzzyVM",
+ Value: "config.toml",
+ }
 )