Skip to content

fix: enable all template types for template list/display #6668

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dwisiswant0 merged 2 commits into from
Dec 8, 2025
Merged

fix: enable all template types for template list/display #6668

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3d652a7
fix: enable all template types for template list/display
dwisiswant0 Dec 7, 2025
2fc2ed6
chore: cleanup messy messy messy
dwisiswant0 Dec 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 0 additions & 8 deletions internal/runner/runner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,14 +125,6 @@ func New(options *types.Options) (*Runner, error) {
}
}

// if template list or template display is enabled, enable all templates
if options.TemplateList || options.TemplateDisplay {
options.EnableCodeTemplates = true
options.EnableFileTemplates = true
options.EnableSelfContainedTemplates = true
options.EnableGlobalMatchersTemplates = true
}

Comment on lines -128 to -135
Copy link
Member Author

@dwisiswant0 dwisiswant0 Dec 7, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Introduced in #6343. Removed, since mutating global opts in the runner could affects the entire engine state (UB).

// check for custom template updates and update if available
ctm, err := customtemplates.NewCustomTemplatesManager(options)
if err != nil {
Expand Down
101 changes: 61 additions & 40 deletions pkg/catalog/loader/loader.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,7 @@ func New(cfg *Config) (*Store, error) {
templatesFinal = append(templatesFinal, v) // something went wrong, treat it as a file
}
}

cfg.TemplateURLs = remoteTemplates
store.finalTemplates = templatesFinal

Expand All @@ -214,6 +215,7 @@ func New(cfg *Config) (*Store, error) {
if err != nil {
return store, err
}

store.finalTemplates = append(store.finalTemplates, remoteTemplates...)
store.finalWorkflows = append(store.finalWorkflows, remoteWorkflows...)
}
Expand Down Expand Up @@ -249,14 +251,18 @@ func handleTemplatesEditorURLs(input string) string {
if err != nil {
return input
}

if !strings.HasSuffix(parsed.Hostname(), "cloud.projectdiscovery.io") {
return input
}

if strings.HasSuffix(parsed.Path, ".yaml") {
return input
}

parsed.Path = fmt.Sprintf("%s.yaml", parsed.Path)
finalURL := parsed.String()

return finalURL
}

Expand All @@ -266,17 +272,21 @@ func handleTemplatesEditorURLs(input string) string {
func (store *Store) ReadTemplateFromURI(uri string, remote bool) ([]byte, error) {
if stringsutil.HasPrefixAny(uri, httpPrefix, httpsPrefix) && remote {
uri = handleTemplatesEditorURLs(uri)

remoteTemplates, _, err := getRemoteTemplatesAndWorkflows([]string{uri}, nil, store.config.RemoteTemplateDomainList)
if err != nil || len(remoteTemplates) == 0 {
return nil, errkit.Wrapf(err, "Could not load template %s: got %v", uri, remoteTemplates)
}

resp, err := retryablehttp.Get(remoteTemplates[0])
if err != nil {
return nil, err
}

defer func() {
_ = resp.Body.Close()
}()

return io.ReadAll(resp.Body)
} else {
return os.ReadFile(uri)
Expand Down Expand Up @@ -396,6 +406,7 @@ func (store *Store) LoadTemplatesOnlyMetadata() error {
if templatesCache != nil {
if template, _, _ := templatesCache.Has(templatePath); template != nil {
var metadata *index.Metadata

if store.metadataIndex != nil {
metadata, _ = store.metadataIndex.SetFromTemplate(templatePath, template)
} else {
Expand All @@ -413,6 +424,7 @@ func (store *Store) LoadTemplatesOnlyMetadata() error {

validPaths[templatePath] = struct{}{}
}

if err != nil {
if strings.Contains(err.Error(), templates.ErrExcluded.Error()) {
stats.Increment(templates.TemplatesExcludedStats)
Expand All @@ -421,6 +433,7 @@ func (store *Store) LoadTemplatesOnlyMetadata() error {
}
continue
}

store.logger.Warning().Msg(err.Error())
}
}
Expand All @@ -431,41 +444,36 @@ func (store *Store) LoadTemplatesOnlyMetadata() error {
}

loadedTemplateIDs := mapsutil.NewSyncLockMap[string, struct{}]()
caps := templates.Capabilities{
Headless: store.config.ExecutorOptions.Options.Headless,
Code: store.config.ExecutorOptions.Options.EnableCodeTemplates,
DAST: store.config.ExecutorOptions.Options.DAST,
SelfContained: store.config.ExecutorOptions.Options.EnableSelfContainedTemplates,
File: store.config.ExecutorOptions.Options.EnableFileTemplates,
}
isListOrDisplay := store.config.ExecutorOptions.Options.TemplateList ||
store.config.ExecutorOptions.Options.TemplateDisplay

for templatePath := range validPaths {
template, _, _ := templatesCache.Has(templatePath)

if len(template.RequestsHeadless) > 0 && !store.config.ExecutorOptions.Options.Headless {
continue
}

if len(template.RequestsCode) > 0 && !store.config.ExecutorOptions.Options.EnableCodeTemplates {
continue
}

if template.IsFuzzing() && !store.config.ExecutorOptions.Options.DAST {
if template == nil {
continue
}

if template.SelfContained && !store.config.ExecutorOptions.Options.EnableSelfContainedTemplates {
if !isListOrDisplay && !template.IsEnabledFor(caps) {
continue
}

if template.HasFileProtocol() && !store.config.ExecutorOptions.Options.EnableFileTemplates {
if loadedTemplateIDs.Has(template.ID) {
store.logger.Debug().Msgf("Skipping duplicate template ID '%s' from path '%s'", template.ID, templatePath)
continue
}

if template != nil {
if loadedTemplateIDs.Has(template.ID) {
store.logger.Debug().Msgf("Skipping duplicate template ID '%s' from path '%s'", template.ID, templatePath)
continue
}

_ = loadedTemplateIDs.Set(template.ID, struct{}{})
template.Path = templatePath
store.templates = append(store.templates, template)
}
_ = loadedTemplateIDs.Set(template.ID, struct{}{})
template.Path = templatePath
store.templates = append(store.templates, template)
}

return nil
}

Expand Down Expand Up @@ -561,17 +569,20 @@ func (store *Store) areWorkflowOrTemplatesValid(filteredTemplatePaths map[string
// areTemplatesValid = false
store.logger.Warning().Msgf("Found duplicate template ID during validation '%s' => '%s': %s\n", templatePath, existingTemplatePath, template.ID)
}
if !isWorkflow && len(template.Workflows) > 0 {

if !isWorkflow && template.HasWorkflows() {
continue
}
}

if isWorkflow {
if !areWorkflowTemplatesValid(store, template.Workflows) {
areTemplatesValid = false
continue
}
}
}

return areTemplatesValid
}

Expand All @@ -580,24 +591,29 @@ func areWorkflowTemplatesValid(store *Store, workflows []*workflows.WorkflowTemp
if !areWorkflowTemplatesValid(store, workflow.Subtemplates) {
return false
}

_, err := store.config.Catalog.GetTemplatePath(workflow.Template)
if err != nil {
if isParsingError(store, "Error occurred loading template %s: %s\n", workflow.Template, err) {
return false
}
}
}

return true
}

func isParsingError(store *Store, message string, template string, err error) bool {
if errors.Is(err, templates.ErrExcluded) {
return false
}

if errors.Is(err, templates.ErrCreateTemplateExecutor) {
return false
}

store.logger.Error().Msgf(message, template, err)

return true
}

Expand All @@ -617,6 +633,7 @@ func (store *Store) LoadWorkflows(workflowsList []string) []*templates.Template
if err != nil {
store.logger.Warning().Msgf("Could not load workflow %s: %s\n", workflowPath, err)
}

if loaded {
parsed, err := templates.Parse(workflowPath, store.preprocessor, store.config.ExecutorOptions)
if err != nil {
Expand All @@ -626,6 +643,7 @@ func (store *Store) LoadWorkflows(workflowsList []string) []*templates.Template
}
}
}

return loadedWorkflows
}

Expand Down Expand Up @@ -663,22 +681,24 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ
}
}

concurrency := store.config.ExecutorOptions.Options.TemplateLoadingConcurrency
typesOpts := store.config.ExecutorOptions.Options
concurrency := typesOpts.TemplateLoadingConcurrency
if concurrency <= 0 {
concurrency = types.DefaultTemplateLoadingConcurrency
}

wgLoadTemplates, errWg := syncutil.New(syncutil.WithSize(concurrency))
if errWg != nil {
panic("could not create wait group")
}

if store.config.ExecutorOptions.Options.ExecutionId == "" {
store.config.ExecutorOptions.Options.ExecutionId = xid.New().String()
if typesOpts.ExecutionId == "" {
typesOpts.ExecutionId = xid.New().String()
}

dialers := protocolstate.GetDialersWithId(store.config.ExecutorOptions.Options.ExecutionId)
dialers := protocolstate.GetDialersWithId(typesOpts.ExecutionId)
if dialers == nil {
panic("dialers with executionId " + store.config.ExecutorOptions.Options.ExecutionId + " not found")
panic("dialers with executionId " + typesOpts.ExecutionId + " not found")
}

for _, templatePath := range includedTemplates {
Expand Down Expand Up @@ -728,18 +748,18 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ
}
store.logger.Warning().Msgf("Could not parse template %s: %s\n", templatePath, err)
} else if parsed != nil {
if !parsed.Verified && store.config.ExecutorOptions.Options.DisableUnsignedTemplates {
if !parsed.Verified && typesOpts.DisableUnsignedTemplates {
// skip unverified templates when prompted to
stats.Increment(templates.SkippedUnsignedStats)
return
}

if parsed.SelfContained && !store.config.ExecutorOptions.Options.EnableSelfContainedTemplates {
if parsed.SelfContained && !typesOpts.EnableSelfContainedTemplates {
stats.Increment(templates.ExcludedSelfContainedStats)
return
}

if parsed.HasFileProtocol() && !store.config.ExecutorOptions.Options.EnableFileTemplates {
if parsed.HasFileRequest() && !typesOpts.EnableFileTemplates {
stats.Increment(templates.ExcludedFileStats)
return
}
Expand All @@ -751,11 +771,11 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ
}
// DAST only templates
// Skip DAST filter when loading auth templates
if store.ID() != AuthStoreId && store.config.ExecutorOptions.Options.DAST {
if store.ID() != AuthStoreId && typesOpts.DAST {
// check if the template is a DAST template
// also allow global matchers template to be loaded
if parsed.IsFuzzing() || parsed.Options.GlobalMatchers != nil && parsed.Options.GlobalMatchers.HasMatchers() {
if len(parsed.RequestsHeadless) > 0 && !store.config.ExecutorOptions.Options.Headless {
if parsed.IsFuzzableRequest() || parsed.IsGlobalMatchersTemplate() {
if parsed.HasHeadlessRequest() && !typesOpts.Headless {
stats.Increment(templates.ExcludedHeadlessTmplStats)
if config.DefaultConfig.LogAllEvents {
store.logger.Print().Msgf("[%v] Headless flag is required for headless template '%s'.\n", aurora.Yellow("WRN").String(), templatePath)
Expand All @@ -764,27 +784,27 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ
loadTemplate(parsed)
}
}
} else if len(parsed.RequestsHeadless) > 0 && !store.config.ExecutorOptions.Options.Headless {
} else if parsed.HasHeadlessRequest() && !typesOpts.Headless {
// donot include headless template in final list if headless flag is not set
stats.Increment(templates.ExcludedHeadlessTmplStats)
if config.DefaultConfig.LogAllEvents {
store.logger.Print().Msgf("[%v] Headless flag is required for headless template '%s'.\n", aurora.Yellow("WRN").String(), templatePath)
}
} else if len(parsed.RequestsCode) > 0 && !store.config.ExecutorOptions.Options.EnableCodeTemplates {
} else if parsed.HasCodeRequest() && !typesOpts.EnableCodeTemplates {
// donot include 'Code' protocol custom template in final list if code flag is not set
stats.Increment(templates.ExcludedCodeTmplStats)
if config.DefaultConfig.LogAllEvents {
store.logger.Print().Msgf("[%v] Code flag is required for code protocol template '%s'.\n", aurora.Yellow("WRN").String(), templatePath)
}
} else if len(parsed.RequestsCode) > 0 && !parsed.Verified && len(parsed.Workflows) == 0 {
} else if parsed.HasCodeRequest() && !parsed.Verified && !parsed.HasWorkflows() {
// donot include unverified 'Code' protocol custom template in final list
stats.Increment(templates.SkippedCodeTmplTamperedStats)
// these will be skipped so increment skip counter
stats.Increment(templates.SkippedUnsignedStats)
if config.DefaultConfig.LogAllEvents {
store.logger.Print().Msgf("[%v] Tampered/Unsigned template at %v.\n", aurora.Yellow("WRN").String(), templatePath)
}
} else if parsed.IsFuzzing() && !store.config.ExecutorOptions.Options.DAST {
} else if parsed.IsFuzzableRequest() && !typesOpts.DAST {
stats.Increment(templates.ExludedDastTmplStats)
if config.DefaultConfig.LogAllEvents {
store.logger.Print().Msgf("[%v] -dast flag is required for DAST template '%s'.\n", aurora.Yellow("WRN").String(), templatePath)
Expand Down Expand Up @@ -822,10 +842,11 @@ func IsHTTPBasedProtocolUsed(store *Store) bool {
templates := append(store.Templates(), store.Workflows()...)

for _, template := range templates {
if len(template.RequestsHTTP) > 0 || len(template.RequestsHeadless) > 0 {
if template.HasHTTPRequest() || template.HasHeadlessRequest() {
return true
}
if len(template.Workflows) > 0 {

if template.HasWorkflows() {
if workflowContainsProtocol(template.Workflows) {
return true
}
Expand Down
5 changes: 5 additions & 0 deletions pkg/protocols/headless/headless.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,3 +175,8 @@ func (request *Request) Requests() int {
func (r *Request) UpdateOptions(opts *protocols.ExecutorOptions) {
r.options.ApplyNewEngineOptions(opts)
}

// HasFuzzing checks if the request has fuzzing rules defined.
func (request *Request) HasFuzzing() bool {
return len(request.Fuzzing) > 0
}
5 changes: 5 additions & 0 deletions pkg/protocols/http/http.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -544,3 +544,8 @@ func init() {
func (r *Request) UpdateOptions(opts *protocols.ExecutorOptions) {
r.options.ApplyNewEngineOptions(opts)
}

// HasFuzzing indicates whether the request has fuzzing rules defined.
func (request *Request) HasFuzzing() bool {
return len(request.Fuzzing) > 0
}
20 changes: 10 additions & 10 deletions pkg/templates/compile.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -320,34 +320,34 @@ func (template *Template) compileProtocolRequests(options *protocols.ExecutorOpt
options.IsMultiProtocol = true
}
} else {
if len(template.RequestsDNS) > 0 {
if template.HasDNSRequest() {
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsDNS)...)
}
if len(template.RequestsFile) > 0 {
if template.HasFileRequest() {
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsFile)...)
}
if len(template.RequestsNetwork) > 0 {
if template.HasNetworkRequest() {
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsNetwork)...)
}
if len(template.RequestsHTTP) > 0 {
if template.HasHTTPRequest() {
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsHTTP)...)
}
if len(template.RequestsHeadless) > 0 && options.Options.Headless {
if template.HasHeadlessRequest() && options.Options.Headless {
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsHeadless)...)
}
if len(template.RequestsSSL) > 0 {
if template.HasSSLRequest() {
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsSSL)...)
}
if len(template.RequestsWebsocket) > 0 {
if template.HasWebsocketRequest() {
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsWebsocket)...)
}
if len(template.RequestsWHOIS) > 0 {
if template.HasWHOISRequest() {
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsWHOIS)...)
}
if len(template.RequestsCode) > 0 && options.Options.EnableCodeTemplates {
if template.HasCodeRequest() && options.Options.EnableCodeTemplates {
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsCode)...)
}
if len(template.RequestsJavascript) > 0 {
if template.HasJavascriptRequest() {
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsJavascript)...)
}
}
Expand Down
Loading
Loading