Skip to content

use mapstructure tag #1898

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Mzack9999 merged 1 commit into from
Aug 26, 2024
Merged

use mapstructure tag #1898

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ require (
github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6
github.com/microcosm-cc/bluemonday v1.0.26
github.com/miekg/dns v1.1.59 // indirect
github.com/mitchellh/mapstructure v1.5.0
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/asnmap v1.1.1
github.com/projectdiscovery/cdncheck v1.1.0
Expand Down Expand Up @@ -51,7 +50,10 @@ require (
golang.org/x/text v0.16.0
)

require github.com/weppos/publicsuffix-go v0.30.2
require (
github.com/go-viper/mapstructure/v2 v2.1.0
github.com/weppos/publicsuffix-go v0.30.2
)

require (
aead.dev/minisign v0.2.0 // indirect
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,8 @@ github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/go-rod/rod v0.114.0 h1:P+zLOqsj+vKf4C86SfjP6ymyPl9VXoYKm+ceCeQms6Y=
github.com/go-rod/rod v0.114.0/go.mod h1:aiedSEFg5DwG/fnNbUOTPMTTWX3MRj6vIs/a684Mthw=
github.com/go-viper/mapstructure/v2 v2.1.0 h1:gHnMa2Y/pIxElCH2GlZZ1lZSsn6XMtufpGyP1XxdC/w=
github.com/go-viper/mapstructure/v2 v2.1.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
Expand Down Expand Up @@ -176,8 +178,6 @@ github.com/miekg/dns v1.1.59 h1:C9EXc/UToRwKLhK5wKU/I4QVsBUc8kE6MkHBkeypWZs=
github.com/miekg/dns v1.1.59/go.mod h1:nZpewl5p6IvctfgrckopVx2OlSEHPRO/U4SYkRklrEk=
github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 h1:yRZGarbxsRytL6EGgbqK2mCY+Lk5MWKQYKJT2gEglhc=
github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
Expand Down
137 changes: 68 additions & 69 deletions runner/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (

"github.com/go-faker/faker/v4"
"github.com/go-faker/faker/v4/pkg/options"
"github.com/mitchellh/mapstructure"
mapstructure "github.com/go-viper/mapstructure/v2"
"github.com/projectdiscovery/dsl"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/tlsx/pkg/tlsx/clients"
Expand All @@ -32,74 +32,74 @@ func (o AsnResponse) String() string {

// Result of a scan
type Result struct {
Timestamp time.Time `json:"timestamp,omitempty" csv:"timestamp"`
ASN *AsnResponse `json:"asn,omitempty" csv:"asn"`
Err error `json:"-" csv:"-"`
CSPData *httpx.CSPData `json:"csp,omitempty" csv:"csp"`
TLSData *clients.Response `json:"tls,omitempty" csv:"tls"`
Hashes map[string]interface{} `json:"hash,omitempty" csv:"hash"`
ExtractRegex []string `json:"extract_regex,omitempty" csv:"extract_regex"`
CDNName string `json:"cdn_name,omitempty" csv:"cdn_name"`
CDNType string `json:"cdn_type,omitempty" csv:"cdn_type"`
SNI string `json:"sni,omitempty" csv:"sni"`
Port string `json:"port,omitempty" csv:"port"`
Raw string `json:"-" csv:"-"`
URL string `json:"url,omitempty" csv:"url"`
Input string `json:"input,omitempty" csv:"input"`
Location string `json:"location,omitempty" csv:"location"`
Title string `json:"title,omitempty" csv:"title"`
str string
Scheme string `json:"scheme,omitempty" csv:"scheme"`
Error string `json:"error,omitempty" csv:"error"`
WebServer string `json:"webserver,omitempty" csv:"webserver"`
ResponseBody string `json:"body,omitempty" csv:"body"`
BodyPreview string `json:"body_preview,omitempty" csv:"body_preview"`
ContentType string `json:"content_type,omitempty" csv:"content_type"`
Method string `json:"method,omitempty" csv:"method"`
Host string `json:"host,omitempty" csv:"host"`
Path string `json:"path,omitempty" csv:"path"`
FavIconMMH3 string `json:"favicon,omitempty" csv:"favicon"`
FavIconMD5 string `json:"favicon_md5,omitempty" csv:"favicon"`
FaviconPath string `json:"favicon_path,omitempty" csv:"favicon_path"`
FaviconURL string `json:"favicon_url,omitempty" csv:"favicon_url"`
FinalURL string `json:"final_url,omitempty" csv:"final_url"`
ResponseHeaders map[string]interface{} `json:"header,omitempty" csv:"header"`
RawHeaders string `json:"raw_header,omitempty" csv:"raw_header"`
Request string `json:"request,omitempty" csv:"request"`
ResponseTime string `json:"time,omitempty" csv:"time"`
JarmHash string `json:"jarm_hash,omitempty" csv:"jarm_hash"`
ChainStatusCodes []int `json:"chain_status_codes,omitempty" csv:"chain_status_codes"`
A []string `json:"a,omitempty" csv:"a"`
AAAA []string `json:"aaaa,omitempty" csv:"aaaa"`
CNAMEs []string `json:"cname,omitempty" csv:"cname"`
Technologies []string `json:"tech,omitempty" csv:"tech"`
Extracts map[string][]string `json:"extracts,omitempty" csv:"extracts"`
Chain []httpx.ChainItem `json:"chain,omitempty" csv:"chain"`
Words int `json:"words" csv:"words"`
Lines int `json:"lines" csv:"lines"`
StatusCode int `json:"status_code" csv:"status_code"`
ContentLength int `json:"content_length" csv:"content_length"`
Failed bool `json:"failed" csv:"failed"`
VHost bool `json:"vhost,omitempty" csv:"vhost"`
WebSocket bool `json:"websocket,omitempty" csv:"websocket"`
CDN bool `json:"cdn,omitempty" csv:"cdn"`
HTTP2 bool `json:"http2,omitempty" csv:"http2"`
Pipeline bool `json:"pipeline,omitempty" csv:"pipeline"`
HeadlessBody string `json:"headless_body,omitempty" csv:"headless_body"`
ScreenshotBytes []byte `json:"screenshot_bytes,omitempty" csv:"screenshot_bytes"`
StoredResponsePath string `json:"stored_response_path,omitempty" csv:"stored_response_path"`
ScreenshotPath string `json:"screenshot_path,omitempty" csv:"screenshot_path"`
ScreenshotPathRel string `json:"screenshot_path_rel,omitempty" csv:"screenshot_path_rel"`
KnowledgeBase map[string]interface{} `json:"knowledgebase,omitempty" csv:"knowledgebase"`
Resolvers []string `json:"resolvers,omitempty" csv:"resolvers"`
Fqdns []string `json:"body_fqdn,omitempty"`
Domains []string `json:"body_domains,omitempty"`
Timestamp time.Time `json:"timestamp,omitempty" csv:"timestamp" mapstructure:"timestamp"`
ASN *AsnResponse `json:"asn,omitempty" csv:"asn" mapstructure:"asn"`
Err error `json:"-" csv:"-" mapstructure:"-"`
CSPData *httpx.CSPData `json:"csp,omitempty" csv:"csp" mapstructure:"csp"`
TLSData *clients.Response `json:"tls,omitempty" csv:"tls" mapstructure:"tls"`
Hashes map[string]interface{} `json:"hash,omitempty" csv:"hash" mapstructure:"hash"`
ExtractRegex []string `json:"extract_regex,omitempty" csv:"extract_regex" mapstructure:"extract_regex"`
CDNName string `json:"cdn_name,omitempty" csv:"cdn_name" mapstructure:"cdn_name"`
CDNType string `json:"cdn_type,omitempty" csv:"cdn_type" mapstructure:"cdn_type"`
SNI string `json:"sni,omitempty" csv:"sni" mapstructure:"sni"`
Port string `json:"port,omitempty" csv:"port" mapstructure:"port"`
Raw string `json:"-" csv:"-" mapstructure:"-"`
URL string `json:"url,omitempty" csv:"url" mapstructure:"url"`
Input string `json:"input,omitempty" csv:"input" mapstructure:"input"`
Location string `json:"location,omitempty" csv:"location" mapstructure:"location"`
Title string `json:"title,omitempty" csv:"title" mapstructure:"title"`
str string `mapstructure:"-"`
Scheme string `json:"scheme,omitempty" csv:"scheme" mapstructure:"scheme"`
Error string `json:"error,omitempty" csv:"error" mapstructure:"error"`
WebServer string `json:"webserver,omitempty" csv:"webserver" mapstructure:"webserver"`
ResponseBody string `json:"body,omitempty" csv:"body" mapstructure:"body"`
BodyPreview string `json:"body_preview,omitempty" csv:"body_preview" mapstructure:"body_preview"`
ContentType string `json:"content_type,omitempty" csv:"content_type" mapstructure:"content_type"`
Method string `json:"method,omitempty" csv:"method" mapstructure:"method"`
Host string `json:"host,omitempty" csv:"host" mapstructure:"host"`
Path string `json:"path,omitempty" csv:"path" mapstructure:"path"`
FavIconMMH3 string `json:"favicon,omitempty" csv:"favicon" mapstructure:"favicon"`
FavIconMD5 string `json:"favicon_md5,omitempty" csv:"favicon" mapstructure:"favicon_md5"`
FaviconPath string `json:"favicon_path,omitempty" csv:"favicon_path" mapstructure:"favicon_path"`
FaviconURL string `json:"favicon_url,omitempty" csv:"favicon_url" mapstructure:"favicon_url"`
FinalURL string `json:"final_url,omitempty" csv:"final_url" mapstructure:"final_url"`
ResponseHeaders map[string]interface{} `json:"header,omitempty" csv:"header" mapstructure:"header"`
RawHeaders string `json:"raw_header,omitempty" csv:"raw_header" mapstructure:"raw_header"`
Request string `json:"request,omitempty" csv:"request" mapstructure:"request"`
ResponseTime string `json:"time,omitempty" csv:"time" mapstructure:"time"`
JarmHash string `json:"jarm_hash,omitempty" csv:"jarm_hash" mapstructure:"jarm_hash"`
ChainStatusCodes []int `json:"chain_status_codes,omitempty" csv:"chain_status_codes" mapstructure:"chain_status_codes"`
A []string `json:"a,omitempty" csv:"a" mapstructure:"a"`
AAAA []string `json:"aaaa,omitempty" csv:"aaaa" mapstructure:"aaaa"`
CNAMEs []string `json:"cname,omitempty" csv:"cname" mapstructure:"cname"`
Technologies []string `json:"tech,omitempty" csv:"tech" mapstructure:"tech"`
Extracts map[string][]string `json:"extracts,omitempty" csv:"extracts" mapstructure:"extracts"`
Chain []httpx.ChainItem `json:"chain,omitempty" csv:"chain" mapstructure:"chain"`
Words int `json:"words" csv:"words" mapstructure:"words"`
Lines int `json:"lines" csv:"lines" mapstructure:"lines"`
StatusCode int `json:"status_code" csv:"status_code" mapstructure:"status_code"`
ContentLength int `json:"content_length" csv:"content_length" mapstructure:"content_length"`
Failed bool `json:"failed" csv:"failed" mapstructure:"failed"`
VHost bool `json:"vhost,omitempty" csv:"vhost" mapstructure:"vhost"`
WebSocket bool `json:"websocket,omitempty" csv:"websocket" mapstructure:"websocket"`
CDN bool `json:"cdn,omitempty" csv:"cdn" mapstructure:"cdn"`
HTTP2 bool `json:"http2,omitempty" csv:"http2" mapstructure:"http2"`
Pipeline bool `json:"pipeline,omitempty" csv:"pipeline" mapstructure:"pipeline"`
HeadlessBody string `json:"headless_body,omitempty" csv:"headless_body" mapstructure:"headless_body"`
ScreenshotBytes []byte `json:"screenshot_bytes,omitempty" csv:"screenshot_bytes" mapstructure:"screenshot_bytes"`
StoredResponsePath string `json:"stored_response_path,omitempty" csv:"stored_response_path" mapstructure:"stored_response_path"`
ScreenshotPath string `json:"screenshot_path,omitempty" csv:"screenshot_path" mapstructure:"screenshot_path"`
ScreenshotPathRel string `json:"screenshot_path_rel,omitempty" csv:"screenshot_path_rel" mapstructure:"screenshot_path_rel"`
KnowledgeBase map[string]interface{} `json:"knowledgebase,omitempty" csv:"knowledgebase" mapstructure:"knowledgebase"`
Resolvers []string `json:"resolvers,omitempty" csv:"resolvers" mapstructure:"resolvers"`
Fqdns []string `json:"body_fqdn,omitempty" mapstructure:"body_fqdn"`
Domains []string `json:"body_domains,omitempty" mapstructure:"body_domains"`

// Internal Fields
TechnologyDetails map[string]wappalyzer.AppInfo `json:"-" csv:"-"`
RequestRaw []byte `json:"-" csv:"-"`
Response *httpx.Response `json:"-" csv:"-"`
FaviconData []byte `json:"-" csv:"-"`
TechnologyDetails map[string]wappalyzer.AppInfo `json:"-" csv:"-" mapstructure:"-"`
RequestRaw []byte `json:"-" csv:"-" mapstructure:"-"`
Response *httpx.Response `json:"-" csv:"-" mapstructure:"-"`
FaviconData []byte `json:"-" csv:"-" mapstructure:"-"`
}

// function to get dsl variables from result struct
Expand Down Expand Up @@ -139,8 +139,7 @@ func evalDslExpr(result Result, dslExpr string) bool {
func resultToMap(resp Result) (map[string]any, error) {
m := make(map[string]any)
config := &mapstructure.DecoderConfig{
TagName: "json",
Result: &m,
Result: &m,
}
decoder, err := mapstructure.NewDecoder(config)
if err != nil {
Expand Down