common,crypto: move fuzzers out of core (#22029)

* common,crypto: move fuzzers out of core

* fuzzers: move vm fuzzer out from core

* fuzzing: rework cover package logic

* fuzzers: lint

Author: holiman

oss-fuzz.sh

@@ -26,38 +26,80 @@
 # $CFLAGS, $CXXFLAGS    C and C++ compiler flags.
 # $LIB_FUZZING_ENGINE   C++ compiler argument to link fuzz target against the prebuilt engine library (e.g. libFuzzer).
 
+# This sets the -coverpgk for the coverage report when the corpus is executed through go test
+coverpkg="github.com/ethereum/go-ethereum/..."
+
+function coverbuild {
+  path=$1
+  function=$2
+  fuzzer=$3
+  tags=""
+
+  if [[ $#  -eq 4 ]]; then
+    tags="-tags $4"
+  fi
+  cd $path
+  fuzzed_package=`pwd | rev | cut -d'/' -f 1 | rev`
+  cp $GOPATH/ossfuzz_coverage_runner.go ./"${function,,}"_test.go
+  sed -i -e 's/FuzzFunction/'$function'/' ./"${function,,}"_test.go
+  sed -i -e 's/mypackagebeingfuzzed/'$fuzzed_package'/' ./"${function,,}"_test.go
+  sed -i -e 's/TestFuzzCorpus/Test'$function'Corpus/' ./"${function,,}"_test.go
+
+cat << DOG > $OUT/$fuzzer
+#/bin/sh
+
+  cd $OUT/$path
+  go test -run Test${function}Corpus -v $tags -coverprofile \$1 -coverpkg $coverpkg
+
+DOG
+
+  chmod +x $OUT/$fuzzer
+  #echo "Built script $OUT/$fuzzer"
+  #cat $OUT/$fuzzer
+  cd -
+}
+
 function compile_fuzzer {
-  path=$SRC/go-ethereum/$1
+  # Inputs:
+  # $1: The package to fuzz, within go-ethereum
+  # $2: The name of the fuzzing function
+  # $3: The name to give to the final fuzzing-binary
+
+  path=$GOPATH/src/github.com/ethereum/go-ethereum/$1
   func=$2
   fuzzer=$3
-  corpusfile="${path}/testdata/${fuzzer}_seed_corpus.zip"
-  echo "Building $fuzzer (expecting corpus at $corpusfile)"
-  (cd $path && \
+
+  echo "Building $fuzzer"
+
+  # Do a coverage-build or a regular build
+  if [[ $SANITIZER = *coverage* ]]; then
+    coverbuild $path $func $fuzzer $coverpkg
+  else
+    (cd $path && \
         go-fuzz -func $func -o $WORK/$fuzzer.a . && \
-        echo "First stage built OK" && \
-        $CXX $CXXFLAGS $LIB_FUZZING_ENGINE $WORK/$fuzzer.a -o $OUT/$fuzzer && \
-        echo "Second stage built ok" )
-
-        ## Check if there exists a seed corpus file
-        if [ -f $corpusfile ]
-        then
-          cp $corpusfile $OUT/
-          echo "Found seed corpus: $corpusfile"
-        fi
+        $CXX $CXXFLAGS $LIB_FUZZING_ENGINE $WORK/$fuzzer.a -o $OUT/$fuzzer)
+  fi
+
+  ## Check if there exists a seed corpus file
+  corpusfile="${path}/testdata/${fuzzer}_seed_corpus.zip"
+  if [ -f $corpusfile ]
+  then
+    cp $corpusfile $OUT/
+    echo "Found seed corpus: $corpusfile"
+  fi
 }
 
-compile_fuzzer common/bitutil  Fuzz      fuzzBitutilCompress
-compile_fuzzer crypto/bn256    FuzzAdd   fuzzBn256Add
-compile_fuzzer crypto/bn256    FuzzMul   fuzzBn256Mul
-compile_fuzzer crypto/bn256    FuzzPair  fuzzBn256Pair
-compile_fuzzer core/vm/runtime Fuzz      fuzzVmRuntime
-compile_fuzzer crypto/blake2b  Fuzz      fuzzBlake2b
+compile_fuzzer tests/fuzzers/bitutil  Fuzz      fuzzBitutilCompress
+compile_fuzzer tests/fuzzers/bn256    FuzzAdd   fuzzBn256Add
+compile_fuzzer tests/fuzzers/bn256    FuzzMul   fuzzBn256Mul
+compile_fuzzer tests/fuzzers/bn256    FuzzPair  fuzzBn256Pair
+compile_fuzzer tests/fuzzers/runtime  Fuzz      fuzzVmRuntime
 compile_fuzzer tests/fuzzers/keystore   Fuzz fuzzKeystore
 compile_fuzzer tests/fuzzers/txfetcher  Fuzz fuzzTxfetcher
 compile_fuzzer tests/fuzzers/rlp        Fuzz fuzzRlp
 compile_fuzzer tests/fuzzers/trie       Fuzz fuzzTrie
 compile_fuzzer tests/fuzzers/stacktrie  Fuzz fuzzStackTrie
-compile_fuzzer tests/fuzzers/difficulty  Fuzz fuzzDifficulty
+compile_fuzzer tests/fuzzers/difficulty Fuzz fuzzDifficulty
 
 compile_fuzzer tests/fuzzers/bls12381  FuzzG1Add fuzz_g1_add
 compile_fuzzer tests/fuzzers/bls12381  FuzzG1Mul fuzz_g1_mul
@@ -69,6 +111,10 @@ compile_fuzzer tests/fuzzers/bls12381  FuzzPairing fuzz_pairing
 compile_fuzzer tests/fuzzers/bls12381  FuzzMapG1 fuzz_map_g1
 compile_fuzzer tests/fuzzers/bls12381  FuzzMapG2 fuzz_map_g2
 
+#TODO: move this to tests/fuzzers, if possible
+compile_fuzzer crypto/blake2b  Fuzz      fuzzBlake2b
+
+
 # This doesn't work very well @TODO
 #compile_fuzzertests/fuzzers/abi Fuzz fuzzAbi
 

common/bitutil/compress_fuzz.go renamed to tests/fuzzers/bitutil/compress_fuzz.go

@@ -14,11 +14,13 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-// +build gofuzz
-
 package bitutil
 
-import "bytes"
+import (
+	"bytes"
+
+	"github.com/ethereum/go-ethereum/common/bitutil"
+)
 
 // Fuzz implements a go-fuzz fuzzer method to test various encoding method
 // invocations.
@@ -35,7 +37,7 @@ func Fuzz(data []byte) int {
 // fuzzEncode implements a go-fuzz fuzzer method to test the bitset encoding and
 // decoding algorithm.
 func fuzzEncode(data []byte) int {
-	proc, _ := bitsetDecodeBytes(bitsetEncodeBytes(data), len(data))
+	proc, _ := bitutil.DecompressBytes(bitutil.CompressBytes(data), len(data))
 	if !bytes.Equal(data, proc) {
 		panic("content mismatch")
 	}
@@ -45,11 +47,11 @@ func fuzzEncode(data []byte) int {
 // fuzzDecode implements a go-fuzz fuzzer method to test the bit decoding and
 // reencoding algorithm.
 func fuzzDecode(data []byte) int {
-	blob, err := bitsetDecodeBytes(data, 1024)
+	blob, err := bitutil.DecompressBytes(data, 1024)
 	if err != nil {
 		return 0
 	}
-	if comp := bitsetEncodeBytes(blob); !bytes.Equal(comp, data) {
+	if comp := bitutil.CompressBytes(blob); !bytes.Equal(comp, data) {
 		panic("content mismatch")
 	}
 	return 1

crypto/bn256/bn256_fuzz.go renamed to tests/fuzzers/bn256/bn256_fuzz.go

@@ -2,8 +2,6 @@
 // Use of this source code is governed by a BSD-style license that can be found
 // in the LICENSE file.
 
-// +build gofuzz
-
 package bn256
 
 import (
@@ -24,7 +22,7 @@ func getG1Points(input io.Reader) (*cloudflare.G1, *google.G1) {
 	}
 	xg := new(google.G1)
 	if _, err := xg.Unmarshal(xc.Marshal()); err != nil {
-		panic(fmt.Sprintf("Could not marshal cloudflare -> google:", err))
+		panic(fmt.Sprintf("Could not marshal cloudflare -> google: %v", err))
 	}
 	return xc, xg
 }
@@ -37,7 +35,7 @@ func getG2Points(input io.Reader) (*cloudflare.G2, *google.G2) {
 	}
 	xg := new(google.G2)
 	if _, err := xg.Unmarshal(xc.Marshal()); err != nil {
-		panic(fmt.Sprintf("Could not marshal cloudflare -> google:", err))
+		panic(fmt.Sprintf("Could not marshal cloudflare -> google: %v", err))
 	}
 	return xc, xg
 }

core/vm/runtime/fuzz.go renamed to tests/fuzzers/runtime/runtime_fuzz.go

@@ -14,23 +14,23 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-// +build gofuzz
-
 package runtime
 
+import (
+	"github.com/ethereum/go-ethereum/core/vm/runtime"
+)
+
 // Fuzz is the basic entry point for the go-fuzz tool
 //
 // This returns 1 for valid parsable/runable code, 0
 // for invalid opcode.
 func Fuzz(input []byte) int {
-	_, _, err := Execute(input, input, &Config{
-		GasLimit: 3000000,
+	_, _, err := runtime.Execute(input, input, &runtime.Config{
+		GasLimit: 12000000,
 	})
-
 	// invalid opcode
-	if err != nil && len(err.Error()) > 6 && string(err.Error()[:7]) == "invalid" {
+	if err != nil && len(err.Error()) > 6 && err.Error()[:7] == "invalid" {
 		return 0
 	}
-
 	return 1
 }