add md5 hash support for favicon (#1799)

Author: dogancanbakir

common/stringz/stringz.go

@@ -2,7 +2,9 @@ package stringz
 
 import (
 	"bytes"
+	"crypto/md5"
 	"encoding/base64"
+	"encoding/hex"
 	"errors"
 	"net/http"
 	"net/url"
@@ -121,12 +123,19 @@ func murmurhash(data []byte) int32 {
 	return int32(hasher.Sum32())
 }
 
-func FaviconHash(data []byte) (int32, error) {
+// md5Hash returns the md5 hash of the data
+func md5Hash(data []byte) string {
+	hasher := md5.New()
+	hasher.Write(data)
+	return hex.EncodeToString(hasher.Sum(nil))
+}
+
+func FaviconHash(data []byte) (int32, string, error) {
 	if isContentTypeImage(data) {
-		return murmurhash(data), nil
+		return murmurhash(data), md5Hash(data), nil
 	}
 
-	return 0, errors.New("content type is not image")
+	return 0, "", errors.New("content type is not image")
 }
 
 func InsertInto(s string, interval int, sep rune) string {

runner/runner.go

@@ -1948,11 +1948,11 @@ retry:
 		builder.WriteRune(']')
 	}
 
-	var faviconMMH3, faviconPath, faviconURL string
+	var faviconMMH3, faviconMD5, faviconPath, faviconURL string
 	var faviconData []byte
 	if scanopts.Favicon {
 		var err error
-		faviconMMH3, faviconPath, faviconData, faviconURL, err = r.HandleFaviconHash(hp, req, resp.Data, true)
+		faviconMMH3, faviconMD5, faviconPath, faviconData, faviconURL, err = r.HandleFaviconHash(hp, req, resp.Data, true)
 		if err == nil {
 			builder.WriteString(" [")
 			if !scanopts.OutputWithNoColor {
@@ -2199,6 +2199,7 @@ retry:
 		Technologies:     technologies,
 		FinalURL:         finalURL,
 		FavIconMMH3:      faviconMMH3,
+		FavIconMD5:       faviconMD5,
 		FaviconPath:      faviconPath,
 		FaviconURL:       faviconURL,
 		Hashes:           hashesMap,
@@ -2257,24 +2258,24 @@ func calculatePerceptionHash(screenshotBytes []byte) (uint64, error) {
 	return pHash.GetHash(), nil
 }
 
-func (r *Runner) HandleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request, currentResp []byte, defaultProbe bool) (string, string, []byte, string, error) {
+func (r *Runner) HandleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request, currentResp []byte, defaultProbe bool) (string, string, string, []byte, string, error) {
 	// Check if current URI is ending with .ico => use current body without additional requests
 	if path.Ext(req.URL.Path) == ".ico" {
-		hash, err := r.calculateFaviconHashWithRaw(currentResp)
-		return hash, req.URL.Path, currentResp, "", err
+		MMH3Hash, MD5Hash, err := r.calculateFaviconHashWithRaw(currentResp)
+		return MMH3Hash, MD5Hash, req.URL.Path, currentResp, "", err
 	}
 
 	// search in the response of the requested path for element and rel shortcut/mask/apple-touch icon
 	// link with .ico extension (which will be prioritized if available)
 	// if not, any of link from other icons can be requested
 	potentialURLs, err := extractPotentialFavIconsURLs(currentResp)
 	if err != nil {
-		return "", "", nil, "", err
+		return "", "", "", nil, "", err
 	}
 
 	clone := req.Clone(context.Background())
 
-	var faviconHash, faviconPath, faviconURL string
+	var faviconMMH3, faviconMD5, faviconPath, faviconURL string
 	var faviconData []byte
 	errCount := 0
 	if len(potentialURLs) == 0 && defaultProbe {
@@ -2309,25 +2310,26 @@ func (r *Runner) HandleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request,
 			errCount++
 			continue
 		}
-		hash, err := r.calculateFaviconHashWithRaw(resp.Data)
+		MMH3Hash, MD5Hash, err := r.calculateFaviconHashWithRaw(resp.Data)
 		if err != nil {
 			continue
 		}
 		faviconURL = clone.URL.String()
 		faviconPath = potentialURL
-		faviconHash = hash
+		faviconMMH3 = MMH3Hash
+		faviconMD5 = MD5Hash
 		faviconData = resp.Data
 		break
 	}
-	return faviconHash, faviconPath, faviconData, faviconURL, nil
+	return faviconMMH3, faviconMD5, faviconPath, faviconData, faviconURL, nil
 }
 
-func (r *Runner) calculateFaviconHashWithRaw(data []byte) (string, error) {
-	hashNum, err := stringz.FaviconHash(data)
+func (r *Runner) calculateFaviconHashWithRaw(data []byte) (string, string, error) {
+	hashNum, md5Hash, err := stringz.FaviconHash(data)
 	if err != nil {
-		return "", errorutil.NewWithTag("favicon", "could not calculate favicon hash").Wrap(err)
+		return "", "", errorutil.NewWithTag("favicon", "could not calculate favicon hash").Wrap(err)
 	}
-	return fmt.Sprintf("%d", hashNum), nil
+	return fmt.Sprintf("%d", hashNum), md5Hash, nil
 }
 
 func extractPotentialFavIconsURLs(resp []byte) ([]string, error) {

runner/types.go

@@ -59,6 +59,7 @@ type Result struct {
 	Host               string                 `json:"host,omitempty" csv:"host"`
 	Path               string                 `json:"path,omitempty" csv:"path"`
 	FavIconMMH3        string                 `json:"favicon,omitempty" csv:"favicon"`
+	FavIconMD5         string                 `json:"favicon_md5,omitempty" csv:"favicon"`
 	FaviconPath        string                 `json:"favicon_path,omitempty" csv:"favicon_path"`
 	FaviconURL         string                 `json:"favicon_url,omitempty" csv:"favicon_url"`
 	FinalURL           string                 `json:"final_url,omitempty" csv:"final_url"`