Merge pull request #951 from projectdiscovery/host-max-errors

Added HostErrorsCache to track errors per-host for skipping failures

v2/cmd/nuclei/main.go

@@ -126,6 +126,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 	createGroup(flagSet, "optimization", "Optimizations",
 		flagSet.IntVar(&options.Timeout, "timeout", 5, "time to wait in seconds before timeout"),
 		flagSet.IntVar(&options.Retries, "retries", 1, "number of times to retry a failed request"),
+		flagSet.IntVar(&options.HostMaxErrors, "host-max-error", 30, "max errors for a host before skipping from scan"),
 
 		flagSet.BoolVar(&options.Project, "project", false, "use a project folder to avoid sending same request multiple times"),
 		flagSet.StringVar(&options.ProjectPath, "project-path", os.TempDir(), "set a specific project path"),

v2/go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/antchfx/htmlquery v1.2.3
 	github.com/apex/log v1.9.0
 	github.com/blang/semver v3.5.1+incompatible
+	github.com/bluele/gcache v0.0.2 // indirect
 	github.com/c4milo/unpackit v0.1.0 // indirect
 	github.com/corpix/uarand v0.1.1
 	github.com/fatih/structs v1.1.0 // indirect
@@ -18,6 +19,7 @@ require (
 	github.com/gosuri/uiprogress v0.0.1 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-retryablehttp v0.6.8 // indirect
+	github.com/json-iterator/go v1.1.10
 	github.com/itchyny/gojq v0.12.4
 	github.com/json-iterator/go v1.1.11
 	github.com/julienschmidt/httprouter v1.3.0

v2/go.sum

@@ -60,6 +60,8 @@ github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN
 github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I=
 github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
+github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw=
+github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0=
 github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaqKnf+7Qs6GbEPfd4iMOitWzXJx8=
 github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8/go.mod h1:spo1JLcs67NmW1aVLEgtA8Yy1elc+X8y5SRW1sFW4Og=
 github.com/c4milo/unpackit v0.1.0 h1:91pWJ6B3svZ4LOE+p3rnyucRK5fZwBdF/yQ/pcZO31I=
@@ -183,6 +185,8 @@ github.com/hashicorp/go-retryablehttp v0.6.8 h1:92lWxgpa+fF3FozM4B3UZtHZMJX8T5XT
 github.com/hashicorp/go-retryablehttp v0.6.8/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
+github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hooklift/assert v0.1.0 h1:UZzFxx5dSb9aBtvMHTtnPuvFnBvcEhHTPb9+0+jpEjs=
 github.com/hooklift/assert v0.1.0/go.mod h1:pfexfvIHnKCdjh6CkkIZv5ic6dQ6aU2jhKghBlXuwwY=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=

v2/internal/runner/processor.go

@@ -14,6 +14,10 @@ func (r *Runner) processTemplateWithList(template *templates.Template) bool {
 	r.hostMap.Scan(func(k, _ []byte) error {
 		URL := string(k)
 
+		// Skip if the host has had errors
+		if r.hostErrors != nil && r.hostErrors.Check(URL) {
+			return nil
+		}
 		wg.Add()
 		go func(URL string) {
 			defer wg.Done()
@@ -37,6 +41,11 @@ func (r *Runner) processWorkflowWithList(template *templates.Template) bool {
 
 	r.hostMap.Scan(func(k, _ []byte) error {
 		URL := string(k)
+
+		// Skip if the host has had errors
+		if r.hostErrors != nil && r.hostErrors.Check(URL) {
+			return nil
+		}
 		wg.Add()
 		go func(URL string) {
 			defer wg.Done()

v2/internal/runner/runner.go

@@ -29,6 +29,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/projectfile"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/clusterer"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine"
@@ -56,6 +57,7 @@ type Runner struct {
 	addColor        func(severity.Severity) string
 	browser         *engine.Browser
 	ratelimiter     ratelimit.Limiter
+	hostErrors      *hosterrorscache.Cache
 }
 
 // New creates a new client for running enumeration process.
@@ -293,16 +295,22 @@ func (r *Runner) RunEnumeration() error {
 	r.options.ExcludeTags = append(r.options.ExcludeTags, ignoreFile.Tags...)
 	r.options.ExcludedTemplates = append(r.options.ExcludedTemplates, ignoreFile.Files...)
 
+	var cache *hosterrorscache.Cache
+	if r.options.HostMaxErrors > 0 {
+		cache = hosterrorscache.New(r.options.HostMaxErrors, hosterrorscache.DefaultMaxHostsCount).SetVerbose(r.options.Verbose)
+	}
+	r.hostErrors = cache
 	executerOpts := protocols.ExecuterOptions{
-		Output:       r.output,
-		Options:      r.options,
-		Progress:     r.progress,
-		Catalog:      r.catalog,
-		IssuesClient: r.issuesClient,
-		RateLimiter:  r.ratelimiter,
-		Interactsh:   r.interactsh,
-		ProjectFile:  r.projectFile,
-		Browser:      r.browser,
+		Output:          r.output,
+		Options:         r.options,
+		Progress:        r.progress,
+		Catalog:         r.catalog,
+		IssuesClient:    r.issuesClient,
+		RateLimiter:     r.ratelimiter,
+		Interactsh:      r.interactsh,
+		ProjectFile:     r.projectFile,
+		Browser:         r.browser,
+		HostErrorsCache: cache,
 	}
 
 	workflowLoader, err := parsers.NewLoader(&executerOpts)
@@ -413,15 +421,16 @@ func (r *Runner) RunEnumeration() error {
 	for _, cluster := range clusters {
 		if len(cluster) > 1 && !r.options.OfflineHTTP {
 			executerOpts := protocols.ExecuterOptions{
-				Output:       r.output,
-				Options:      r.options,
-				Progress:     r.progress,
-				Catalog:      r.catalog,
-				RateLimiter:  r.ratelimiter,
-				IssuesClient: r.issuesClient,
-				Browser:      r.browser,
-				ProjectFile:  r.projectFile,
-				Interactsh:   r.interactsh,
+				Output:          r.output,
+				Options:         r.options,
+				Progress:        r.progress,
+				Catalog:         r.catalog,
+				RateLimiter:     r.ratelimiter,
+				IssuesClient:    r.issuesClient,
+				Browser:         r.browser,
+				ProjectFile:     r.projectFile,
+				Interactsh:      r.interactsh,
+				HostErrorsCache: cache,
 			}
 			clusterID := fmt.Sprintf("cluster-%s", xid.New().String())
 

v2/pkg/protocols/common/clusterer/executer.go

@@ -87,6 +87,9 @@ func (e *Executer) Execute(input string) (bool, error) {
 			}
 		}
 	})
+	if err != nil && e.options.HostErrorsCache != nil && e.options.HostErrorsCache.CheckError(err) {
+		e.options.HostErrorsCache.MarkFailed(input)
+	}
 	return results, err
 }
 
@@ -106,5 +109,8 @@ func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEve
 			}
 		}
 	})
+	if err != nil && e.options.HostErrorsCache != nil && e.options.HostErrorsCache.CheckError(err) {
+		e.options.HostErrorsCache.MarkFailed(input)
+	}
 	return err
 }

v2/pkg/protocols/common/executer/executer.go

@@ -77,6 +77,11 @@ func (e *Executer) Execute(input string) (bool, error) {
 			}
 		})
 		if err != nil {
+			if e.options.HostErrorsCache != nil {
+				if e.options.HostErrorsCache.CheckError(err) {
+					e.options.HostErrorsCache.MarkFailed(input)
+				}
+			}
 			gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input, err)
 		}
 	}
@@ -109,6 +114,11 @@ func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEve
 			callback(event)
 		})
 		if err != nil {
+			if e.options.HostErrorsCache != nil {
+				if e.options.HostErrorsCache.CheckError(err) {
+					e.options.HostErrorsCache.MarkFailed(input)
+				}
+			}
 			gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input, err)
 		}
 	}

v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go

@@ -0,0 +1,126 @@
+package hosterrorscache
+
+import (
+	"net"
+	"net/url"
+	"regexp"
+	"strings"
+
+	"github.com/bluele/gcache"
+	"github.com/projectdiscovery/gologger"
+)
+
+// Cache is a cache for host based errors. It allows skipping
+// certain hosts based on an error threshold.
+//
+// It uses an LRU cache internally for skipping unresponsive hosts
+// that remain so for a duration.
+type Cache struct {
+	hostMaxErrors int
+	verbose       bool
+	failedTargets gcache.Cache
+}
+
+const DefaultMaxHostsCount = 10000
+
+// New returns a new host max errors cache
+func New(hostMaxErrors, maxHostsCount int) *Cache {
+	gc := gcache.New(maxHostsCount).
+		ARC().
+		Build()
+	return &Cache{failedTargets: gc, hostMaxErrors: hostMaxErrors}
+}
+
+// SetVerbose sets the cache to log at verbose level
+func (c *Cache) SetVerbose(verbose bool) *Cache {
+	c.verbose = verbose
+	return c
+}
+
+// Close closes the host errors cache
+func (c *Cache) Close() {
+	c.failedTargets.Purge()
+}
+
+func (c *Cache) normalizeCacheValue(value string) string {
+	finalValue := value
+	if strings.HasPrefix(value, "http") {
+		if parsed, err := url.Parse(value); err == nil {
+
+			hostname := parsed.Host
+			finalPort := parsed.Port()
+			if finalPort == "" {
+				if parsed.Scheme == "https" {
+					finalPort = "443"
+				} else {
+					finalPort = "80"
+				}
+				hostname = net.JoinHostPort(parsed.Host, finalPort)
+			}
+			finalValue = hostname
+		}
+	}
+	return finalValue
+}
+
+// ErrUnresponsiveHost is returned when a host is unresponsive
+//var ErrUnresponsiveHost = errors.New("skipping as host is unresponsive")
+
+// Check returns true if a host should be skipped as it has been
+// unresponsive for a certain number of times.
+//
+// The value can be many formats -
+//   - URL: https?:// type
+//   - Host:port type
+//   - host type
+func (c *Cache) Check(value string) bool {
+	finalValue := c.normalizeCacheValue(value)
+	if !c.failedTargets.Has(finalValue) {
+		return false
+	}
+
+	numberOfErrors, err := c.failedTargets.GetIFPresent(finalValue)
+	if err != nil {
+		return false
+	}
+	numberOfErrorsValue := numberOfErrors.(int)
+
+	if numberOfErrors == -1 {
+		return true
+	}
+	if numberOfErrorsValue >= c.hostMaxErrors {
+		_ = c.failedTargets.Set(finalValue, -1)
+		if c.verbose {
+			gologger.Verbose().Msgf("Skipping %s as previously unresponsive %d times", finalValue, numberOfErrorsValue)
+		}
+		return true
+	}
+	return false
+}
+
+// MarkFailed marks a host as failed previously
+func (c *Cache) MarkFailed(value string) {
+	finalValue := c.normalizeCacheValue(value)
+	if !c.failedTargets.Has(finalValue) {
+		_ = c.failedTargets.Set(finalValue, 1)
+		return
+	}
+
+	numberOfErrors, err := c.failedTargets.GetIFPresent(finalValue)
+	if err != nil || numberOfErrors == nil {
+		_ = c.failedTargets.Set(finalValue, 1)
+		return
+	}
+	numberOfErrorsValue := numberOfErrors.(int)
+
+	_ = c.failedTargets.Set(finalValue, numberOfErrorsValue+1)
+}
+
+var checkErrorRegexp = regexp.MustCompile(`(no address found for host|Client\.Timeout exceeded while awaiting headers|could not resolve host)`)
+
+// CheckError checks if an error represents a type that should be
+// added to the host skipping table.
+func (c *Cache) CheckError(err error) bool {
+	errString := err.Error()
+	return checkErrorRegexp.MatchString(errString)
+}

v2/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go

@@ -0,0 +1,30 @@
+package hosterrorscache
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestCacheCheckMarkFailed(t *testing.T) {
+	cache := New(3, DefaultMaxHostsCount)
+
+	cache.MarkFailed("http://example.com:80")
+	if value, err := cache.failedTargets.Get("http://example.com:80"); err == nil && value != nil {
+		require.Equal(t, 1, value, "could not get correct markfailed")
+	}
+	cache.MarkFailed("example.com:80")
+	if value, err := cache.failedTargets.Get("example.com:80"); err == nil && value != nil {
+		require.Equal(t, 2, value, "could not get correct markfailed")
+	}
+	cache.MarkFailed("example.com")
+	if value, err := cache.failedTargets.Get("example.com"); err == nil && value != nil {
+		require.Equal(t, 1, value, "could not get correct markfailed")
+	}
+	for i := 0; i < 3; i++ {
+		cache.MarkFailed("test")
+	}
+
+	value := cache.Check("test")
+	require.Equal(t, true, value, "could not get checked value")
+}

v2/pkg/protocols/http/request.go

@@ -216,6 +216,10 @@ func (r *Request) ExecuteWithResults(reqURL string, dynamicValues, previous outp
 			return err
 		}
 
+		// Check if hosts just keep erroring
+		if r.options.HostErrorsCache != nil && r.options.HostErrorsCache.Check(reqURL) {
+			break
+		}
 		var gotOutput bool
 		r.options.RateLimiter.Take()
 		err = r.executeRequest(reqURL, request, previous, func(event *output.InternalWrappedEvent) {
@@ -237,7 +241,10 @@ func (r *Request) ExecuteWithResults(reqURL string, dynamicValues, previous outp
 			}
 		}, requestCount)
 		if err != nil {
-			requestErr = multierr.Append(requestErr, err)
+			if r.options.HostErrorsCache != nil && r.options.HostErrorsCache.CheckError(err) {
+				r.options.HostErrorsCache.MarkFailed(reqURL)
+			}
+			requestErr = err
 		}
 		requestCount++
 		r.options.Progress.IncrementRequests()
@@ -304,9 +311,10 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ
 
 	// For race conditions we can't dump the request body at this point as it's already waiting the open-gate event, already handled with a similar code within the race function
 	if !request.original.Race {
-		dumpedRequest, err = dump(request, reqURL)
-		if err != nil {
-			return err
+		var dumpError error
+		dumpedRequest, dumpError = dump(request, reqURL)
+		if dumpError != nil {
+			return dumpError
 		}
 
 		if r.options.Options.Debug || r.options.Options.DebugRequests {
@@ -314,10 +322,6 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ
 			gologger.Print().Msgf("%s", string(dumpedRequest))
 		}
 	}
-
-	if resp == nil {
-		err = errors.New("no response got for request")
-	}
 	if err != nil {
 		// rawhttp doesn't supports draining response bodies.
 		if resp != nil && resp.Body != nil && request.rawRequest == nil {

v2/pkg/protocols/protocols.go

@@ -9,6 +9,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
 	"github.com/projectdiscovery/nuclei/v2/pkg/progress"
 	"github.com/projectdiscovery/nuclei/v2/pkg/projectfile"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine"
 	"github.com/projectdiscovery/nuclei/v2/pkg/reporting"
@@ -54,6 +55,8 @@ type ExecuterOptions struct {
 	Browser *engine.Browser
 	// Interactsh is a client for interactsh oob polling server
 	Interactsh *interactsh.Client
+	// HostErrorsCache is an optional cache for handling host errors
+	HostErrorsCache *hosterrorscache.Cache
 
 	Operators []*operators.Operators // only used by offlinehttp module
 

v2/pkg/templates/workflows.go

@@ -59,15 +59,16 @@ func parseWorkflowTemplate(workflow *workflows.WorkflowTemplate, preprocessor Pr
 	}
 	for _, path := range paths {
 		opts := protocols.ExecuterOptions{
-			Output:       options.Output,
-			Options:      options.Options,
-			Progress:     options.Progress,
-			Catalog:      options.Catalog,
-			Browser:      options.Browser,
-			RateLimiter:  options.RateLimiter,
-			IssuesClient: options.IssuesClient,
-			Interactsh:   options.Interactsh,
-			ProjectFile:  options.ProjectFile,
+			Output:          options.Output,
+			Options:         options.Options,
+			Progress:        options.Progress,
+			Catalog:         options.Catalog,
+			Browser:         options.Browser,
+			RateLimiter:     options.RateLimiter,
+			IssuesClient:    options.IssuesClient,
+			Interactsh:      options.Interactsh,
+			ProjectFile:     options.ProjectFile,
+			HostErrorsCache: options.HostErrorsCache,
 		}
 		template, err := Parse(path, preprocessor, opts)
 		if err != nil {

v2/pkg/types/types.go

@@ -65,6 +65,8 @@ type Options struct {
 	StatsInterval int
 	// MetricsPort is the port to show metrics on
 	MetricsPort int
+	// HostMaxErrors is the maximum number of errors allowed for a host
+	HostMaxErrors int
 	// BulkSize is the of targets analyzed in parallel for each template
 	BulkSize int
 	// TemplateThreads is the number of templates executed in parallel

v2/pkg/workflows/execute.go

@@ -55,6 +55,11 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res
 				}
 			}
 			if err != nil {
+				if w.Options.HostErrorsCache != nil {
+					if w.Options.HostErrorsCache.CheckError(err) {
+						w.Options.HostErrorsCache.MarkFailed(input)
+					}
+				}
 				if len(template.Executers) == 1 {
 					mainErr = err
 				} else {