Skip to content
Open
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions cmd/nuclei/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -435,6 +435,7 @@ on extensive configurability, massive extensibility and ease of use.`)
)

flagSet.CreateGroup("headless", "Headless",
flagSet.BoolVarP(&options.DisableTechStackFiltering, "disable-tech-filter", "dtf", false, "disable automatic template filtering based on Server response header tech detection"),
flagSet.BoolVar(&options.Headless, "headless", false, "enable templates that require headless browser support (root user on Linux will disable sandbox)"),
flagSet.IntVar(&options.PageTimeout, "page-timeout", 20, "seconds to wait for each page in headless mode"),
flagSet.BoolVarP(&options.ShowBrowser, "show-browser", "sb", false, "show the browser on the screen when running templates with headless mode"),
Expand Down
7 changes: 6 additions & 1 deletion pkg/core/engine.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/nuclei/v3/pkg/core/hosttechcache"
)

// Engine is an executer for running Nuclei Templates/Workflows.
Expand All @@ -21,6 +22,7 @@ type Engine struct {
executerOpts *protocols.ExecutorOptions
Callback func(*output.ResultEvent) // Executed on results
Logger *gologger.Logger
HostTechCache *hosttechcache.HostTechCache
}

// New returns a new Engine instance
Expand All @@ -30,6 +32,9 @@ func New(options *types.Options) *Engine {
Logger: options.Logger,
}
engine.workPool = engine.GetWorkPool()
if !options.DisableTechStackFiltering {
engine.HostTechCache = hosttechcache.NewHostTechCache()
}
return engine
}

Expand Down Expand Up @@ -64,4 +69,4 @@ func (e *Engine) WorkPool() *WorkPool {
// resize check point - nop if there are no changes
e.workPool.RefreshWithConfig(e.GetWorkPoolConfig())
return e.workPool
}
}
2 changes: 1 addition & 1 deletion pkg/core/execute_options.go
Original file line number Diff line number Diff line change
Expand Up @@ -175,4 +175,4 @@ func getRequestCount(templates []*templates.Template) int {
count += template.TotalRequests
}
return count
}
}
2 changes: 1 addition & 1 deletion pkg/core/executors.go
Original file line number Diff line number Diff line change
Expand Up @@ -245,4 +245,4 @@ func (e *Engine) executeTemplateOnInput(ctx context.Context, template *templates
}
return template.Executer.Execute(scanCtx)
}
}
}
88 changes: 88 additions & 0 deletions pkg/core/hosttechcache/hosttechcache.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
package hosttechcache

import (
"strings"
"sync"
"github.com/projectdiscovery/gologger"
)

// TechHint represents a detected technology on a host that can be used
// to filter templates before execution.
type TechHint struct {
// Tags is the set of template tags that are REQUIRED for this host.
// A template is skipped unless it contains at least one of these tags,
// or the set is empty (meaning: no filtering).
Tags map[string]struct{}
}

// HostTechCache stores per-host technology hints derived from early HTTP
// responses (e.g. the Server: header). It is safe for concurrent use.
type HostTechCache struct {
mu sync.RWMutex
hints map[string]*TechHint // keyed by normalised host (scheme+host)
}

// NewHostTechCache returns an initialised HostTechCache.
func NewHostTechCache() *HostTechCache {
return &HostTechCache{hints: make(map[string]*TechHint)}
}

// RecordServerHeader inspects a raw Server header value and, if it contains
// a known technology keyword, records a tag requirement for that host.
//
// Currently understood keywords → required tag:
//
// "apache" → "apache"
//
// The mapping is intentionally simple and lowercase-compared so that
// "Apache/2.4.51 (Unix)" and "apache" both resolve to the same hint.
func (c *HostTechCache) RecordServerHeader(host, serverHeader string) {
lower := strings.ToLower(serverHeader)

var requiredTags []string
if strings.Contains(lower, "apache") {
requiredTags = append(requiredTags, "apache")
}

c.mu.Lock()
defer c.mu.Unlock()

if len(requiredTags) == 0 {
if _, exists := c.hints[host]; exists {
gologger.Debug().Msgf("[tech-filter] CLEARED hint for host '%s' (unrecognised Server header: '%s')",
host, serverHeader)
}
delete(c.hints, host)
return
}

gologger.Debug().Msgf("[tech-filter] RECORDED hint for host '%s' — Server: '%s' → required tags: %v",
host, serverHeader, requiredTags)

hint := &TechHint{Tags: make(map[string]struct{}, len(requiredTags))}
for _, t := range requiredTags {
hint.Tags[t] = struct{}{}
}
c.hints[host] = hint
}
Comment on lines +41 to +73
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major

🧩 Analysis chain

🏁 Script executed:

#!/bin/bash
# Verify how MetaInput.Input is populated in different contexts

# Check HTTP request context - what value is passed to RecordServerHeader
rg -n -A5 'RecordServerHeader\(' --type go

# Check template executor context - what value is passed to ShouldSkipTemplate  
rg -n -A5 'ShouldSkipTemplate\(' --type go

# Check how MetaInput.Input is set/modified during execution
rg -n -B3 -A3 'MetaInput\.Input\s*=' --type go

Repository: projectdiscovery/nuclei

Length of output: 22286


🏁 Script executed:

# Check the context around where ShouldSkipTemplate is called in tmplexec/exec.go
rg -n -B15 'ShouldSkipTemplate\(' pkg/tmplexec/exec.go | head -60

Repository: projectdiscovery/nuclei

Length of output: 783


🏁 Script executed:

# Check the context around line 928 in http/request.go where RecordServerHeader is called
rg -n -B20 'RecordServerHeader\(' pkg/protocols/http/request.go | head -80

Repository: projectdiscovery/nuclei

Length of output: 891


🏁 Script executed:

# Check what GetCopyIfHostOutdated does and how MetaInput.Input is modified
rg -n -B5 -A10 'GetCopyIfHostOutdated\(input' pkg/protocols/http/request.go | head -100

Repository: projectdiscovery/nuclei

Length of output: 2050


🏁 Script executed:

# Check how ctx.Input is created/passed in template execution
rg -n -B10 'ExecuteWithResults' pkg/tmplexec/exec.go | grep -A10 "func.*ExecuteWithResults"

Repository: projectdiscovery/nuclei

Length of output: 525


🏁 Script executed:

# Check how ctx (ScanContext) Input is set up initially
rg -n 'type ScanContext' --type go -A20

Repository: projectdiscovery/nuclei

Length of output: 1467


🏁 Script executed:

# Look for how template execution flow receives its input context
rg -n 'ExecuteWithResults' pkg/runner/runner.go -B10 -A5

Repository: projectdiscovery/nuclei

Length of output: 127


🏁 Script executed:

# Search for where ExecuteWithResults is called
rg -n 'ExecuteWithResults' --type go -B5 -A2

Repository: projectdiscovery/nuclei

Length of output: 40798


🏁 Script executed:

# Search for InputHelper and what it transforms
rg -n 'type InputHelper' --type go -A20

Repository: projectdiscovery/nuclei

Length of output: 49


🏁 Script executed:

# Search for InputHelper interface/struct definition
rg -n 'InputHelper' --type go | head -40

Repository: projectdiscovery/nuclei

Length of output: 2668


🏁 Script executed:

# Find the input.Helper implementation and its Transform method
fd -e go 'input' pkg/ | xargs grep -l 'type Helper' | head -5

Repository: projectdiscovery/nuclei

Length of output: 49


🏁 Script executed:

# Search for input package and Helper type
rg -n 'package input' --type go | head -10

Repository: projectdiscovery/nuclei

Length of output: 149


🏁 Script executed:

# Check the input package files
ls -la pkg/input/

Repository: projectdiscovery/nuclei

Length of output: 544


🏁 Script executed:

# Read the transform.go file to see Helper and Transform implementation
cat -n pkg/input/transform.go

Repository: projectdiscovery/nuclei

Length of output: 4978


🏁 Script executed:

# Check the test file to understand how inputs are actually passed
cat -n pkg/core/hosttechcache/hosttestcache_test.go | head -100

Repository: projectdiscovery/nuclei

Length of output: 4607


🏁 Script executed:

# Check how input is initially created and what format it has
rg -n 'NewWithInput\|ctx.Input.MetaInput.Input =' pkg/core/execute_options.go pkg/core/executors.go

Repository: projectdiscovery/nuclei

Length of output: 49


🏁 Script executed:

# Check exactly where in the HTTP request flow RecordServerHeader is called vs transformations
sed -n '925,935p' pkg/protocols/http/request.go

Repository: projectdiscovery/nuclei

Length of output: 498


Normalize host key consistently between cache recording and lookups to prevent cache misses.

The ShouldSkipTemplate check at template execution time uses ctx.Input.MetaInput.Input in its original form, but RecordServerHeader is called after the HTTP protocol executor has processed the input through InputHelper.Transform(). For HTTP protocol requests, this transforms raw inputs (like "example.com") into full URLs (like "https://example.com"), causing a key format mismatch.

Scenario where this fails:

  1. Template execution starts with original input "example.com"
  2. ShouldSkipTemplate() checks cache with key "example.com" → miss (cache empty)
  3. HTTP protocol transforms input to "https://example.com" via InputHelper.Transform()
  4. RecordServerHeader() records hint with key "https://example.com"
  5. Next scan of "example.com" looks up "example.com" again → still a miss, despite hint existing

Apply the suggested normalization approach to extract a consistent key (host+port only) in both RecordServerHeader and ShouldSkipTemplate methods:

🔧 Suggested normalization
+import (
+	"net"
+	"strings"
+	urlutil "github.com/projectdiscovery/utils/url"
+)
+
+// normalizeHost extracts host:port or just host from various input formats
+func normalizeHost(input string) string {
+	if parsed, err := urlutil.Parse(input); err == nil && parsed.Host != "" {
+		// Extract just the host part (removing path, query, fragment)
+		return strings.ToLower(parsed.Host)
+	}
+	return strings.ToLower(input)
+}
+
 func (c *HostTechCache) RecordServerHeader(host, serverHeader string) {
+	host = normalizeHost(host)
 	lower := strings.ToLower(serverHeader)
 	// ... rest of function
 }
 
 func (c *HostTechCache) ShouldSkipTemplate(host string, templateTags []string) bool {
+	host = normalizeHost(host)
 	c.mu.RLock()
 	// ... rest of function
 }
🤖 Prompt for AI Agents
Verify each finding against the current code and only fix it if needed.

In `@pkg/core/hosttechcache/hosttechcache.go` around lines 39 - 67,
RecordServerHeader currently stores hints keyed by the transformed input (e.g.
full URL) which mismatches lookups in ShouldSkipTemplate that use the original
raw input; fix by normalizing the cache key to the same host:port form in both
places: implement a helper (e.g., normalizeHostKey(input string) string) that
tries net/url.Parse(input) and returns url.Host when parse succeeds, otherwise
returns the original input, and use that helper when setting c.hints in
RecordServerHeader (and in ShouldSkipTemplate lookups) so keys are consistently
host[:port] across recording and lookup; preserve existing locking around
c.hints and keep TechHint/Tags behavior unchanged.


// ShouldSkipTemplate returns true when the cache has a hint for the given host
// AND the template's tags contain none of the required tags.
//
// If there is no hint for the host the function always returns false (no skip).
func (c *HostTechCache) ShouldSkipTemplate(host string, templateTags []string) bool {
c.mu.RLock()
hint, ok := c.hints[host]
c.mu.RUnlock()

if !ok || len(hint.Tags) == 0 {
return false // no information → don't skip
}

for _, tag := range templateTags {
if _, required := hint.Tags[strings.ToLower(tag)]; required {
return false // template has at least one matching tag → keep it
}
}
return true // no matching tag found → skip
}
Loading