diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index cec94d7a..0e8d66e5 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -126,6 +126,7 @@ on extensive configurability, massive extensibility and ease of use.`) createGroup(flagSet, "optimization", "Optimizations", flagSet.IntVar(&options.Timeout, "timeout", 5, "time to wait in seconds before timeout"), flagSet.IntVar(&options.Retries, "retries", 1, "number of times to retry a failed request"), + flagSet.IntVar(&options.HostMaxErrors, "host-max-error", 30, "max errors for a host before skipping from scan"), flagSet.BoolVar(&options.Project, "project", false, "use a project folder to avoid sending same request multiple times"), flagSet.StringVar(&options.ProjectPath, "project-path", os.TempDir(), "set a specific project path"), diff --git a/v2/go.mod b/v2/go.mod index 580e6596..2552e575 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -8,6 +8,7 @@ require ( github.com/antchfx/htmlquery v1.2.3 github.com/apex/log v1.9.0 github.com/blang/semver v3.5.1+incompatible + github.com/bluele/gcache v0.0.2 // indirect github.com/c4milo/unpackit v0.1.0 // indirect github.com/corpix/uarand v0.1.1 github.com/fatih/structs v1.1.0 // indirect @@ -18,6 +19,7 @@ require ( github.com/gosuri/uiprogress v0.0.1 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-retryablehttp v0.6.8 // indirect + github.com/json-iterator/go v1.1.10 github.com/itchyny/gojq v0.12.4 github.com/json-iterator/go v1.1.11 github.com/julienschmidt/httprouter v1.3.0 diff --git a/v2/go.sum b/v2/go.sum index 6ec5e06f..9ef2cc0d 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -60,6 +60,8 @@ github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= +github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaqKnf+7Qs6GbEPfd4iMOitWzXJx8= github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8/go.mod h1:spo1JLcs67NmW1aVLEgtA8Yy1elc+X8y5SRW1sFW4Og= github.com/c4milo/unpackit v0.1.0 h1:91pWJ6B3svZ4LOE+p3rnyucRK5fZwBdF/yQ/pcZO31I= @@ -183,6 +185,8 @@ github.com/hashicorp/go-retryablehttp v0.6.8 h1:92lWxgpa+fF3FozM4B3UZtHZMJX8T5XT github.com/hashicorp/go-retryablehttp v0.6.8/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hooklift/assert v0.1.0 h1:UZzFxx5dSb9aBtvMHTtnPuvFnBvcEhHTPb9+0+jpEjs= github.com/hooklift/assert v0.1.0/go.mod h1:pfexfvIHnKCdjh6CkkIZv5ic6dQ6aU2jhKghBlXuwwY= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= diff --git a/v2/internal/runner/processor.go b/v2/internal/runner/processor.go index a00993b8..6006823e 100644 --- a/v2/internal/runner/processor.go +++ b/v2/internal/runner/processor.go @@ -14,6 +14,10 @@ func (r *Runner) processTemplateWithList(template *templates.Template) bool { r.hostMap.Scan(func(k, _ []byte) error { URL := string(k) + // Skip if the host has had errors + if r.hostErrors != nil && r.hostErrors.Check(URL) { + return nil + } wg.Add() go func(URL string) { defer wg.Done() @@ -37,6 +41,11 @@ func (r *Runner) processWorkflowWithList(template *templates.Template) bool { r.hostMap.Scan(func(k, _ []byte) error { URL := string(k) + + // Skip if the host has had errors + if r.hostErrors != nil && r.hostErrors.Check(URL) { + return nil + } wg.Add() go func(URL string) { defer wg.Done() diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index b2a36151..6aaccebc 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -29,6 +29,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/projectfile" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/clusterer" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine" @@ -56,6 +57,7 @@ type Runner struct { addColor func(severity.Severity) string browser *engine.Browser ratelimiter ratelimit.Limiter + hostErrors *hosterrorscache.Cache } // New creates a new client for running enumeration process. @@ -293,16 +295,22 @@ func (r *Runner) RunEnumeration() error { r.options.ExcludeTags = append(r.options.ExcludeTags, ignoreFile.Tags...) r.options.ExcludedTemplates = append(r.options.ExcludedTemplates, ignoreFile.Files...) + var cache *hosterrorscache.Cache + if r.options.HostMaxErrors > 0 { + cache = hosterrorscache.New(r.options.HostMaxErrors, hosterrorscache.DefaultMaxHostsCount).SetVerbose(r.options.Verbose) + } + r.hostErrors = cache executerOpts := protocols.ExecuterOptions{ - Output: r.output, - Options: r.options, - Progress: r.progress, - Catalog: r.catalog, - IssuesClient: r.issuesClient, - RateLimiter: r.ratelimiter, - Interactsh: r.interactsh, - ProjectFile: r.projectFile, - Browser: r.browser, + Output: r.output, + Options: r.options, + Progress: r.progress, + Catalog: r.catalog, + IssuesClient: r.issuesClient, + RateLimiter: r.ratelimiter, + Interactsh: r.interactsh, + ProjectFile: r.projectFile, + Browser: r.browser, + HostErrorsCache: cache, } workflowLoader, err := parsers.NewLoader(&executerOpts) @@ -413,15 +421,16 @@ func (r *Runner) RunEnumeration() error { for _, cluster := range clusters { if len(cluster) > 1 && !r.options.OfflineHTTP { executerOpts := protocols.ExecuterOptions{ - Output: r.output, - Options: r.options, - Progress: r.progress, - Catalog: r.catalog, - RateLimiter: r.ratelimiter, - IssuesClient: r.issuesClient, - Browser: r.browser, - ProjectFile: r.projectFile, - Interactsh: r.interactsh, + Output: r.output, + Options: r.options, + Progress: r.progress, + Catalog: r.catalog, + RateLimiter: r.ratelimiter, + IssuesClient: r.issuesClient, + Browser: r.browser, + ProjectFile: r.projectFile, + Interactsh: r.interactsh, + HostErrorsCache: cache, } clusterID := fmt.Sprintf("cluster-%s", xid.New().String()) diff --git a/v2/pkg/operators/common/dsl/dsl.go b/v2/pkg/operators/common/dsl/dsl.go index 8116ef13..5c6871c6 100644 --- a/v2/pkg/operators/common/dsl/dsl.go +++ b/v2/pkg/operators/common/dsl/dsl.go @@ -52,7 +52,7 @@ var functions = map[string]govaluate.ExpressionFunction{ return compiled.ReplaceAllString(types.ToString(args[0]), types.ToString(args[2])), nil }, "trim": func(args ...interface{}) (interface{}, error) { - return strings.Trim(types.ToString(args[0]), types.ToString(args[2])), nil + return strings.Trim(types.ToString(args[0]), types.ToString(args[1])), nil }, "trimleft": func(args ...interface{}) (interface{}, error) { return strings.TrimLeft(types.ToString(args[0]), types.ToString(args[1])), nil @@ -162,7 +162,7 @@ var functions = map[string]govaluate.ExpressionFunction{ base := letters + numbers if len(args) >= 1 { - l = args[0].(int) + l = int(args[0].(float64)) } if len(args) >= withCutSetArgsSize { bad = types.ToString(args[1]) @@ -179,7 +179,7 @@ var functions = map[string]govaluate.ExpressionFunction{ chars := letters + numbers if len(args) >= 1 { - l = args[0].(int) + l = int(args[0].(float64)) } if len(args) >= withCutSetArgsSize { bad = types.ToString(args[1]) @@ -193,7 +193,7 @@ var functions = map[string]govaluate.ExpressionFunction{ chars := letters if len(args) >= 1 { - l = args[0].(int) + l = int(args[0].(float64)) } if len(args) >= withCutSetArgsSize { bad = types.ToString(args[1]) @@ -207,7 +207,7 @@ var functions = map[string]govaluate.ExpressionFunction{ chars := numbers if len(args) >= 1 { - l = args[0].(int) + l = int(args[0].(float64)) } if len(args) >= withCutSetArgsSize { bad = types.ToString(args[1]) @@ -220,10 +220,10 @@ var functions = map[string]govaluate.ExpressionFunction{ max := math.MaxInt32 if len(args) >= 1 { - min = args[0].(int) + min = int(args[0].(float64)) } if len(args) >= withMaxRandArgsSize { - max = args[1].(int) + max = int(args[1].(float64)) } return rand.Intn(max-min) + min, nil }, diff --git a/v2/pkg/protocols/common/clusterer/executer.go b/v2/pkg/protocols/common/clusterer/executer.go index ebaa7f00..ac523c8d 100644 --- a/v2/pkg/protocols/common/clusterer/executer.go +++ b/v2/pkg/protocols/common/clusterer/executer.go @@ -87,6 +87,9 @@ func (e *Executer) Execute(input string) (bool, error) { } } }) + if err != nil && e.options.HostErrorsCache != nil && e.options.HostErrorsCache.CheckError(err) { + e.options.HostErrorsCache.MarkFailed(input) + } return results, err } @@ -106,5 +109,8 @@ func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEve } } }) + if err != nil && e.options.HostErrorsCache != nil && e.options.HostErrorsCache.CheckError(err) { + e.options.HostErrorsCache.MarkFailed(input) + } return err } diff --git a/v2/pkg/protocols/common/executer/executer.go b/v2/pkg/protocols/common/executer/executer.go index ab1c4eaf..65ea5eb3 100644 --- a/v2/pkg/protocols/common/executer/executer.go +++ b/v2/pkg/protocols/common/executer/executer.go @@ -77,6 +77,11 @@ func (e *Executer) Execute(input string) (bool, error) { } }) if err != nil { + if e.options.HostErrorsCache != nil { + if e.options.HostErrorsCache.CheckError(err) { + e.options.HostErrorsCache.MarkFailed(input) + } + } gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input, err) } } @@ -109,6 +114,11 @@ func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEve callback(event) }) if err != nil { + if e.options.HostErrorsCache != nil { + if e.options.HostErrorsCache.CheckError(err) { + e.options.HostErrorsCache.MarkFailed(input) + } + } gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input, err) } } diff --git a/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go b/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go new file mode 100644 index 00000000..ae6f5bc2 --- /dev/null +++ b/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go @@ -0,0 +1,126 @@ +package hosterrorscache + +import ( + "net" + "net/url" + "regexp" + "strings" + + "github.com/bluele/gcache" + "github.com/projectdiscovery/gologger" +) + +// Cache is a cache for host based errors. It allows skipping +// certain hosts based on an error threshold. +// +// It uses an LRU cache internally for skipping unresponsive hosts +// that remain so for a duration. +type Cache struct { + hostMaxErrors int + verbose bool + failedTargets gcache.Cache +} + +const DefaultMaxHostsCount = 10000 + +// New returns a new host max errors cache +func New(hostMaxErrors, maxHostsCount int) *Cache { + gc := gcache.New(maxHostsCount). + ARC(). + Build() + return &Cache{failedTargets: gc, hostMaxErrors: hostMaxErrors} +} + +// SetVerbose sets the cache to log at verbose level +func (c *Cache) SetVerbose(verbose bool) *Cache { + c.verbose = verbose + return c +} + +// Close closes the host errors cache +func (c *Cache) Close() { + c.failedTargets.Purge() +} + +func (c *Cache) normalizeCacheValue(value string) string { + finalValue := value + if strings.HasPrefix(value, "http") { + if parsed, err := url.Parse(value); err == nil { + + hostname := parsed.Host + finalPort := parsed.Port() + if finalPort == "" { + if parsed.Scheme == "https" { + finalPort = "443" + } else { + finalPort = "80" + } + hostname = net.JoinHostPort(parsed.Host, finalPort) + } + finalValue = hostname + } + } + return finalValue +} + +// ErrUnresponsiveHost is returned when a host is unresponsive +//var ErrUnresponsiveHost = errors.New("skipping as host is unresponsive") + +// Check returns true if a host should be skipped as it has been +// unresponsive for a certain number of times. +// +// The value can be many formats - +// - URL: https?:// type +// - Host:port type +// - host type +func (c *Cache) Check(value string) bool { + finalValue := c.normalizeCacheValue(value) + if !c.failedTargets.Has(finalValue) { + return false + } + + numberOfErrors, err := c.failedTargets.GetIFPresent(finalValue) + if err != nil { + return false + } + numberOfErrorsValue := numberOfErrors.(int) + + if numberOfErrors == -1 { + return true + } + if numberOfErrorsValue >= c.hostMaxErrors { + _ = c.failedTargets.Set(finalValue, -1) + if c.verbose { + gologger.Verbose().Msgf("Skipping %s as previously unresponsive %d times", finalValue, numberOfErrorsValue) + } + return true + } + return false +} + +// MarkFailed marks a host as failed previously +func (c *Cache) MarkFailed(value string) { + finalValue := c.normalizeCacheValue(value) + if !c.failedTargets.Has(finalValue) { + _ = c.failedTargets.Set(finalValue, 1) + return + } + + numberOfErrors, err := c.failedTargets.GetIFPresent(finalValue) + if err != nil || numberOfErrors == nil { + _ = c.failedTargets.Set(finalValue, 1) + return + } + numberOfErrorsValue := numberOfErrors.(int) + + _ = c.failedTargets.Set(finalValue, numberOfErrorsValue+1) +} + +var checkErrorRegexp = regexp.MustCompile(`(no address found for host|Client\.Timeout exceeded while awaiting headers|could not resolve host)`) + +// CheckError checks if an error represents a type that should be +// added to the host skipping table. +func (c *Cache) CheckError(err error) bool { + errString := err.Error() + return checkErrorRegexp.MatchString(errString) +} diff --git a/v2/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go b/v2/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go new file mode 100644 index 00000000..fa13bd82 --- /dev/null +++ b/v2/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go @@ -0,0 +1,30 @@ +package hosterrorscache + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCacheCheckMarkFailed(t *testing.T) { + cache := New(3, DefaultMaxHostsCount) + + cache.MarkFailed("http://example.com:80") + if value, err := cache.failedTargets.Get("http://example.com:80"); err == nil && value != nil { + require.Equal(t, 1, value, "could not get correct markfailed") + } + cache.MarkFailed("example.com:80") + if value, err := cache.failedTargets.Get("example.com:80"); err == nil && value != nil { + require.Equal(t, 2, value, "could not get correct markfailed") + } + cache.MarkFailed("example.com") + if value, err := cache.failedTargets.Get("example.com"); err == nil && value != nil { + require.Equal(t, 1, value, "could not get correct markfailed") + } + for i := 0; i < 3; i++ { + cache.MarkFailed("test") + } + + value := cache.Check("test") + require.Equal(t, true, value, "could not get checked value") +} diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index a3de4f44..ba1784d6 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -216,6 +216,10 @@ func (r *Request) ExecuteWithResults(reqURL string, dynamicValues, previous outp return err } + // Check if hosts just keep erroring + if r.options.HostErrorsCache != nil && r.options.HostErrorsCache.Check(reqURL) { + break + } var gotOutput bool r.options.RateLimiter.Take() err = r.executeRequest(reqURL, request, previous, func(event *output.InternalWrappedEvent) { @@ -237,7 +241,10 @@ func (r *Request) ExecuteWithResults(reqURL string, dynamicValues, previous outp } }, requestCount) if err != nil { - requestErr = multierr.Append(requestErr, err) + if r.options.HostErrorsCache != nil && r.options.HostErrorsCache.CheckError(err) { + r.options.HostErrorsCache.MarkFailed(reqURL) + } + requestErr = err } requestCount++ r.options.Progress.IncrementRequests() @@ -304,9 +311,10 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ // For race conditions we can't dump the request body at this point as it's already waiting the open-gate event, already handled with a similar code within the race function if !request.original.Race { - dumpedRequest, err = dump(request, reqURL) - if err != nil { - return err + var dumpError error + dumpedRequest, dumpError = dump(request, reqURL) + if dumpError != nil { + return dumpError } if r.options.Options.Debug || r.options.Options.DebugRequests { @@ -314,10 +322,6 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ gologger.Print().Msgf("%s", string(dumpedRequest)) } } - - if resp == nil { - err = errors.New("no response got for request") - } if err != nil { // rawhttp doesn't supports draining response bodies. if resp != nil && resp.Body != nil && request.rawRequest == nil { diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 21fea928..4b3984b6 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -9,6 +9,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/progress" "github.com/projectdiscovery/nuclei/v2/pkg/projectfile" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine" "github.com/projectdiscovery/nuclei/v2/pkg/reporting" @@ -54,6 +55,8 @@ type ExecuterOptions struct { Browser *engine.Browser // Interactsh is a client for interactsh oob polling server Interactsh *interactsh.Client + // HostErrorsCache is an optional cache for handling host errors + HostErrorsCache *hosterrorscache.Cache Operators []*operators.Operators // only used by offlinehttp module diff --git a/v2/pkg/templates/workflows.go b/v2/pkg/templates/workflows.go index 04532205..0fcdd9b3 100644 --- a/v2/pkg/templates/workflows.go +++ b/v2/pkg/templates/workflows.go @@ -59,15 +59,16 @@ func parseWorkflowTemplate(workflow *workflows.WorkflowTemplate, preprocessor Pr } for _, path := range paths { opts := protocols.ExecuterOptions{ - Output: options.Output, - Options: options.Options, - Progress: options.Progress, - Catalog: options.Catalog, - Browser: options.Browser, - RateLimiter: options.RateLimiter, - IssuesClient: options.IssuesClient, - Interactsh: options.Interactsh, - ProjectFile: options.ProjectFile, + Output: options.Output, + Options: options.Options, + Progress: options.Progress, + Catalog: options.Catalog, + Browser: options.Browser, + RateLimiter: options.RateLimiter, + IssuesClient: options.IssuesClient, + Interactsh: options.Interactsh, + ProjectFile: options.ProjectFile, + HostErrorsCache: options.HostErrorsCache, } template, err := Parse(path, preprocessor, opts) if err != nil { diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 8545b37f..5fe2a414 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -65,6 +65,8 @@ type Options struct { StatsInterval int // MetricsPort is the port to show metrics on MetricsPort int + // HostMaxErrors is the maximum number of errors allowed for a host + HostMaxErrors int // BulkSize is the of targets analyzed in parallel for each template BulkSize int // TemplateThreads is the number of templates executed in parallel diff --git a/v2/pkg/workflows/execute.go b/v2/pkg/workflows/execute.go index 163cc982..f4b29489 100644 --- a/v2/pkg/workflows/execute.go +++ b/v2/pkg/workflows/execute.go @@ -55,6 +55,11 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res } } if err != nil { + if w.Options.HostErrorsCache != nil { + if w.Options.HostErrorsCache.CheckError(err) { + w.Options.HostErrorsCache.MarkFailed(input) + } + } if len(template.Executers) == 1 { mainErr = err } else {