From f216c6f6b39a3f8681d16ee84de6680e16ddf1e0 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 16 Aug 2021 21:24:37 +0530 Subject: [PATCH 1/8] Added HostErrorsCache for tracking failed hosts --- v2/cmd/nuclei/main.go | 1 + v2/go.mod | 2 + v2/go.sum | 4 + v2/internal/runner/processor.go | 9 ++ v2/internal/runner/runner.go | 45 ++++--- v2/pkg/protocols/common/executer/executer.go | 10 ++ .../common/hosterrorscache/hosterrorscache.go | 115 ++++++++++++++++++ .../hosterrorscache/hosterrorscache_test.go | 30 +++++ v2/pkg/protocols/http/request.go | 11 +- v2/pkg/protocols/protocols.go | 3 + v2/pkg/templates/workflows.go | 19 +-- v2/pkg/types/types.go | 2 + v2/pkg/workflows/execute.go | 5 + 13 files changed, 222 insertions(+), 34 deletions(-) create mode 100644 v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go create mode 100644 v2/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 6d381126..e2fb0c27 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -116,6 +116,7 @@ on extensive configurability, massive extensibility and ease of use.`) createGroup(flagSet, "optimization", "Optimizations", flagSet.IntVar(&options.Timeout, "timeout", 5, "time to wait in seconds before timeout"), flagSet.IntVar(&options.Retries, "retries", 1, "number of times to retry a failed request"), + flagSet.IntVar(&options.HostMaxErrors, "host-max-errors", 30, "max errors allowed for a host before skipping"), flagSet.BoolVar(&options.Project, "project", false, "use a project folder to avoid sending same request multiple times"), flagSet.StringVar(&options.ProjectPath, "project-path", os.TempDir(), "set a specific project path"), diff --git a/v2/go.mod b/v2/go.mod index 4920a1f8..29633925 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -7,6 +7,7 @@ require ( github.com/andygrunwald/go-jira v1.13.0 github.com/apex/log v1.9.0 github.com/blang/semver v3.5.1+incompatible + github.com/bluele/gcache v0.0.2 // indirect github.com/c4milo/unpackit v0.1.0 // indirect github.com/corpix/uarand v0.1.1 github.com/fatih/structs v1.1.0 // indirect @@ -17,6 +18,7 @@ require ( github.com/gosuri/uiprogress v0.0.1 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-retryablehttp v0.6.8 // indirect + github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/json-iterator/go v1.1.10 github.com/julienschmidt/httprouter v1.3.0 github.com/karlseguin/ccache v2.0.3+incompatible diff --git a/v2/go.sum b/v2/go.sum index 97fb14f0..8407b7eb 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -52,6 +52,8 @@ github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= +github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaqKnf+7Qs6GbEPfd4iMOitWzXJx8= github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8/go.mod h1:spo1JLcs67NmW1aVLEgtA8Yy1elc+X8y5SRW1sFW4Og= github.com/c4milo/unpackit v0.1.0 h1:91pWJ6B3svZ4LOE+p3rnyucRK5fZwBdF/yQ/pcZO31I= @@ -173,6 +175,8 @@ github.com/hashicorp/go-retryablehttp v0.6.8 h1:92lWxgpa+fF3FozM4B3UZtHZMJX8T5XT github.com/hashicorp/go-retryablehttp v0.6.8/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hooklift/assert v0.1.0 h1:UZzFxx5dSb9aBtvMHTtnPuvFnBvcEhHTPb9+0+jpEjs= github.com/hooklift/assert v0.1.0/go.mod h1:pfexfvIHnKCdjh6CkkIZv5ic6dQ6aU2jhKghBlXuwwY= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= diff --git a/v2/internal/runner/processor.go b/v2/internal/runner/processor.go index a00993b8..6006823e 100644 --- a/v2/internal/runner/processor.go +++ b/v2/internal/runner/processor.go @@ -14,6 +14,10 @@ func (r *Runner) processTemplateWithList(template *templates.Template) bool { r.hostMap.Scan(func(k, _ []byte) error { URL := string(k) + // Skip if the host has had errors + if r.hostErrors != nil && r.hostErrors.Check(URL) { + return nil + } wg.Add() go func(URL string) { defer wg.Done() @@ -37,6 +41,11 @@ func (r *Runner) processWorkflowWithList(template *templates.Template) bool { r.hostMap.Scan(func(k, _ []byte) error { URL := string(k) + + // Skip if the host has had errors + if r.hostErrors != nil && r.hostErrors.Check(URL) { + return nil + } wg.Add() go func(URL string) { defer wg.Done() diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 017d3d86..d4a121ba 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -21,6 +21,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/projectfile" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/clusterer" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine" @@ -52,6 +53,7 @@ type Runner struct { severityColors *colorizer.Colorizer browser *engine.Browser ratelimiter ratelimit.Limiter + hostErrors *hosterrorscache.Cache } // New creates a new client for running enumeration process. @@ -273,16 +275,22 @@ func (r *Runner) RunEnumeration() error { r.options.ExcludeTags = append(r.options.ExcludeTags, ignoreFile.Tags...) r.options.ExcludedTemplates = append(r.options.ExcludedTemplates, ignoreFile.Files...) + var cache *hosterrorscache.Cache + if r.options.HostMaxErrors > 0 { + cache = hosterrorscache.New(r.options.HostMaxErrors, hosterrorscache.DefaultMaxHostsCount) + } + r.hostErrors = cache executerOpts := protocols.ExecuterOptions{ - Output: r.output, - Options: r.options, - Progress: r.progress, - Catalog: r.catalog, - IssuesClient: r.issuesClient, - RateLimiter: r.ratelimiter, - Interactsh: r.interactsh, - ProjectFile: r.projectFile, - Browser: r.browser, + Output: r.output, + Options: r.options, + Progress: r.progress, + Catalog: r.catalog, + IssuesClient: r.issuesClient, + RateLimiter: r.ratelimiter, + Interactsh: r.interactsh, + ProjectFile: r.projectFile, + Browser: r.browser, + HostErrorsCache: cache, } loaderConfig := loader.Config{ Templates: r.options.Templates, @@ -385,15 +393,16 @@ func (r *Runner) RunEnumeration() error { for _, cluster := range clusters { if len(cluster) > 1 && !r.options.OfflineHTTP { executerOpts := protocols.ExecuterOptions{ - Output: r.output, - Options: r.options, - Progress: r.progress, - Catalog: r.catalog, - RateLimiter: r.ratelimiter, - IssuesClient: r.issuesClient, - Browser: r.browser, - ProjectFile: r.projectFile, - Interactsh: r.interactsh, + Output: r.output, + Options: r.options, + Progress: r.progress, + Catalog: r.catalog, + RateLimiter: r.ratelimiter, + IssuesClient: r.issuesClient, + Browser: r.browser, + ProjectFile: r.projectFile, + Interactsh: r.interactsh, + HostErrorsCache: cache, } clusterID := fmt.Sprintf("cluster-%s", xid.New().String()) diff --git a/v2/pkg/protocols/common/executer/executer.go b/v2/pkg/protocols/common/executer/executer.go index ab1c4eaf..65ea5eb3 100644 --- a/v2/pkg/protocols/common/executer/executer.go +++ b/v2/pkg/protocols/common/executer/executer.go @@ -77,6 +77,11 @@ func (e *Executer) Execute(input string) (bool, error) { } }) if err != nil { + if e.options.HostErrorsCache != nil { + if e.options.HostErrorsCache.CheckError(err) { + e.options.HostErrorsCache.MarkFailed(input) + } + } gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input, err) } } @@ -109,6 +114,11 @@ func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEve callback(event) }) if err != nil { + if e.options.HostErrorsCache != nil { + if e.options.HostErrorsCache.CheckError(err) { + e.options.HostErrorsCache.MarkFailed(input) + } + } gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input, err) } } diff --git a/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go b/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go new file mode 100644 index 00000000..dfe56848 --- /dev/null +++ b/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go @@ -0,0 +1,115 @@ +package hosterrorscache + +import ( + "net" + "net/url" + "regexp" + "strings" + + "github.com/bluele/gcache" +) + +// Cache is a cache for host based errors. It allows skipping +// certain hosts based on an error threshold. +// +// It uses an LRU cache internally for skipping unresponsive hosts +// that remain so for a duration. +type Cache struct { + hostMaxErrors int + failedTargets gcache.Cache +} + +const DefaultMaxHostsCount = 10000 + +// New returns a new host max errors cache +func New(hostMaxErrors, maxHostsCount int) *Cache { + gc := gcache.New(maxHostsCount). + ARC(). + Build() + return &Cache{failedTargets: gc, hostMaxErrors: hostMaxErrors} +} + +// Close closes the host errors cache +func (c *Cache) Close() { + c.failedTargets.Purge() +} + +func (c *Cache) normalizeCacheValue(value string) string { + finalValue := value + if strings.HasPrefix(value, "http") { + if parsed, err := url.Parse(value); err == nil { + + hostname := parsed.Host + finalPort := parsed.Port() + if finalPort == "" { + if parsed.Scheme == "https" { + finalPort = "443" + } else { + finalPort = "80" + } + hostname = net.JoinHostPort(parsed.Host, finalPort) + } + finalValue = hostname + } + } + return finalValue +} + +// ErrUnresponsiveHost is returned when a host is unresponsive +//var ErrUnresponsiveHost = errors.New("skipping as host is unresponsive") + +// Check returns true if a host should be skipped as it has been +// unresponsive for a certain number of times. +// +// The value can be many formats - +// - URL: https?:// type +// - Host:port type +// - host type +func (c *Cache) Check(value string) bool { + finalValue := c.normalizeCacheValue(value) + if !c.failedTargets.Has(finalValue) { + return false + } + + numberOfErrors, err := c.failedTargets.GetIFPresent(finalValue) + if err != nil { + return false + } + numberOfErrorsValue := numberOfErrors.(int) + + if numberOfErrorsValue >= c.hostMaxErrors { + return true + } + return false +} + +// MarkFailed marks a host as failed previously +func (c *Cache) MarkFailed(value string) { + finalValue := c.normalizeCacheValue(value) + if !c.failedTargets.Has(finalValue) { + _ = c.failedTargets.Set(finalValue, 1) + return + } + + numberOfErrors, err := c.failedTargets.GetIFPresent(finalValue) + if err != nil || numberOfErrors == nil { + _ = c.failedTargets.Set(finalValue, 1) + return + } + numberOfErrorsValue := numberOfErrors.(int) + + _ = c.failedTargets.Set(finalValue, numberOfErrorsValue+1) +} + +var checkErrorRegexp = regexp.MustCompile(`(no address found for host|Client\.Timeout exceeded while awaiting headers|could not resolve host)`) + +// CheckError checks if an error represents a type that should be +// added to the host skipping table. +func (c *Cache) CheckError(err error) bool { + errString := err.Error() + + if checkErrorRegexp.MatchString(errString) { + return true + } + return false +} diff --git a/v2/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go b/v2/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go new file mode 100644 index 00000000..fa13bd82 --- /dev/null +++ b/v2/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go @@ -0,0 +1,30 @@ +package hosterrorscache + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCacheCheckMarkFailed(t *testing.T) { + cache := New(3, DefaultMaxHostsCount) + + cache.MarkFailed("http://example.com:80") + if value, err := cache.failedTargets.Get("http://example.com:80"); err == nil && value != nil { + require.Equal(t, 1, value, "could not get correct markfailed") + } + cache.MarkFailed("example.com:80") + if value, err := cache.failedTargets.Get("example.com:80"); err == nil && value != nil { + require.Equal(t, 2, value, "could not get correct markfailed") + } + cache.MarkFailed("example.com") + if value, err := cache.failedTargets.Get("example.com"); err == nil && value != nil { + require.Equal(t, 1, value, "could not get correct markfailed") + } + for i := 0; i < 3; i++ { + cache.MarkFailed("test") + } + + value := cache.Check("test") + require.Equal(t, true, value, "could not get checked value") +} diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 0ece585a..377d752f 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -304,9 +304,10 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ // For race conditions we can't dump the request body at this point as it's already waiting the open-gate event, already handled with a similar code within the race function if !request.original.Race { - dumpedRequest, err = dump(request, reqURL) - if err != nil { - return err + var dumpError error + dumpedRequest, dumpError = dump(request, reqURL) + if dumpError != nil { + return dumpError } if r.options.Options.Debug || r.options.Options.DebugRequests { @@ -314,10 +315,6 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ gologger.Print().Msgf("%s", string(dumpedRequest)) } } - - if resp == nil { - err = errors.New("no response got for request") - } if err != nil { // rawhttp doesn't supports draining response bodies. if resp != nil && resp.Body != nil && request.rawRequest == nil { diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 9225c523..634bccd1 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -8,6 +8,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/progress" "github.com/projectdiscovery/nuclei/v2/pkg/projectfile" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine" "github.com/projectdiscovery/nuclei/v2/pkg/reporting" @@ -53,6 +54,8 @@ type ExecuterOptions struct { Browser *engine.Browser // Interactsh is a client for interactsh oob polling server Interactsh *interactsh.Client + // HostErrorsCache is an optional cache for handling host errors + HostErrorsCache *hosterrorscache.Cache Operators []*operators.Operators // only used by offlinehttp module } diff --git a/v2/pkg/templates/workflows.go b/v2/pkg/templates/workflows.go index 628365d0..f290b75b 100644 --- a/v2/pkg/templates/workflows.go +++ b/v2/pkg/templates/workflows.go @@ -58,15 +58,16 @@ func parseWorkflowTemplate(workflow *workflows.WorkflowTemplate, preprocessor Pr } for _, path := range paths { opts := protocols.ExecuterOptions{ - Output: options.Output, - Options: options.Options, - Progress: options.Progress, - Catalog: options.Catalog, - Browser: options.Browser, - RateLimiter: options.RateLimiter, - IssuesClient: options.IssuesClient, - Interactsh: options.Interactsh, - ProjectFile: options.ProjectFile, + Output: options.Output, + Options: options.Options, + Progress: options.Progress, + Catalog: options.Catalog, + Browser: options.Browser, + RateLimiter: options.RateLimiter, + IssuesClient: options.IssuesClient, + Interactsh: options.Interactsh, + ProjectFile: options.ProjectFile, + HostErrorsCache: options.HostErrorsCache, } template, err := Parse(path, preprocessor, opts) if err != nil { diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 291ec872..b8effe06 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -60,6 +60,8 @@ type Options struct { StatsInterval int // MetricsPort is the port to show metrics on MetricsPort int + // HostMaxErrors is the maximum number of errors allowed for a host + HostMaxErrors int // BulkSize is the of targets analyzed in parallel for each template BulkSize int // TemplateThreads is the number of templates executed in parallel diff --git a/v2/pkg/workflows/execute.go b/v2/pkg/workflows/execute.go index 163cc982..f4b29489 100644 --- a/v2/pkg/workflows/execute.go +++ b/v2/pkg/workflows/execute.go @@ -55,6 +55,11 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res } } if err != nil { + if w.Options.HostErrorsCache != nil { + if w.Options.HostErrorsCache.CheckError(err) { + w.Options.HostErrorsCache.MarkFailed(input) + } + } if len(template.Executers) == 1 { mainErr = err } else { From 8452adfa7307b4c9149830378f2daa4bc9821239 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 16 Aug 2021 21:28:58 +0530 Subject: [PATCH 2/8] Fixing the linter --- .../common/hosterrorscache/hosterrorscache.go | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go b/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go index dfe56848..eb00a1da 100644 --- a/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go +++ b/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go @@ -76,11 +76,7 @@ func (c *Cache) Check(value string) bool { return false } numberOfErrorsValue := numberOfErrors.(int) - - if numberOfErrorsValue >= c.hostMaxErrors { - return true - } - return false + return numberOfErrorsValue >= c.hostMaxErrors } // MarkFailed marks a host as failed previously @@ -107,9 +103,5 @@ var checkErrorRegexp = regexp.MustCompile(`(no address found for host|Client\.Ti // added to the host skipping table. func (c *Cache) CheckError(err error) bool { errString := err.Error() - - if checkErrorRegexp.MatchString(errString) { - return true - } - return false + return checkErrorRegexp.MatchString(errString) } From b800e2cce2f1361ac37c66a9ca63178d95d137fa Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 17 Aug 2021 14:23:42 +0530 Subject: [PATCH 3/8] Cancel http requests if the host keeps erroring --- v2/internal/runner/processor.go | 3 +++ v2/pkg/protocols/http/request.go | 9 ++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/v2/internal/runner/processor.go b/v2/internal/runner/processor.go index 6006823e..55c47fa8 100644 --- a/v2/internal/runner/processor.go +++ b/v2/internal/runner/processor.go @@ -1,6 +1,8 @@ package runner import ( + "fmt" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/remeh/sizedwaitgroup" @@ -22,6 +24,7 @@ func (r *Runner) processTemplateWithList(template *templates.Template) bool { go func(URL string) { defer wg.Done() + fmt.Printf("%v %v\n", URL, template.Info) match, err := template.Executer.Execute(URL) if err != nil { gologger.Warning().Msgf("[%s] Could not execute step: %s\n", r.colorizer.BrightBlue(template.ID), err) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index cfe8e466..ba1784d6 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -216,6 +216,10 @@ func (r *Request) ExecuteWithResults(reqURL string, dynamicValues, previous outp return err } + // Check if hosts just keep erroring + if r.options.HostErrorsCache != nil && r.options.HostErrorsCache.Check(reqURL) { + break + } var gotOutput bool r.options.RateLimiter.Take() err = r.executeRequest(reqURL, request, previous, func(event *output.InternalWrappedEvent) { @@ -237,7 +241,10 @@ func (r *Request) ExecuteWithResults(reqURL string, dynamicValues, previous outp } }, requestCount) if err != nil { - requestErr = multierr.Append(requestErr, err) + if r.options.HostErrorsCache != nil && r.options.HostErrorsCache.CheckError(err) { + r.options.HostErrorsCache.MarkFailed(reqURL) + } + requestErr = err } requestCount++ r.options.Progress.IncrementRequests() From 3177aea6f9c3f7c8591edbf56986db98c9e3663a Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 17 Aug 2021 14:33:50 +0530 Subject: [PATCH 4/8] Misc --- v2/internal/runner/processor.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/v2/internal/runner/processor.go b/v2/internal/runner/processor.go index 55c47fa8..6006823e 100644 --- a/v2/internal/runner/processor.go +++ b/v2/internal/runner/processor.go @@ -1,8 +1,6 @@ package runner import ( - "fmt" - "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/remeh/sizedwaitgroup" @@ -24,7 +22,6 @@ func (r *Runner) processTemplateWithList(template *templates.Template) bool { go func(URL string) { defer wg.Done() - fmt.Printf("%v %v\n", URL, template.Info) match, err := template.Executer.Execute(URL) if err != nil { gologger.Warning().Msgf("[%s] Could not execute step: %s\n", r.colorizer.BrightBlue(template.ID), err) From 586b4c0f8edac46fddb8fad5eb1da6bcc5f173c8 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 17 Aug 2021 14:50:54 +0530 Subject: [PATCH 5/8] Show skipped hosts --- v2/internal/runner/runner.go | 2 +- .../common/hosterrorscache/hosterrorscache.go | 21 ++++++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index d07063f3..c1e95f27 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -293,7 +293,7 @@ func (r *Runner) RunEnumeration() error { var cache *hosterrorscache.Cache if r.options.HostMaxErrors > 0 { - cache = hosterrorscache.New(r.options.HostMaxErrors, hosterrorscache.DefaultMaxHostsCount) + cache = hosterrorscache.New(r.options.HostMaxErrors, hosterrorscache.DefaultMaxHostsCount).SetVerbose(r.options.Verbose) } r.hostErrors = cache executerOpts := protocols.ExecuterOptions{ diff --git a/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go b/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go index eb00a1da..9c046790 100644 --- a/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go +++ b/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/bluele/gcache" + "github.com/projectdiscovery/gologger" ) // Cache is a cache for host based errors. It allows skipping @@ -16,6 +17,7 @@ import ( // that remain so for a duration. type Cache struct { hostMaxErrors int + verbose bool failedTargets gcache.Cache } @@ -29,6 +31,12 @@ func New(hostMaxErrors, maxHostsCount int) *Cache { return &Cache{failedTargets: gc, hostMaxErrors: hostMaxErrors} } +// SetVerbose sets the cache to log at verbose level +func (c *Cache) SetVerbose(verbose bool) *Cache { + c.verbose = verbose + return c +} + // Close closes the host errors cache func (c *Cache) Close() { c.failedTargets.Purge() @@ -76,7 +84,18 @@ func (c *Cache) Check(value string) bool { return false } numberOfErrorsValue := numberOfErrors.(int) - return numberOfErrorsValue >= c.hostMaxErrors + + if numberOfErrors == -1 { + return true + } + if numberOfErrorsValue >= c.hostMaxErrors { + _ = c.failedTargets.Set(finalValue, -1) + if c.verbose { + gologger.Verbose().Msgf("Skipping %s as it has failed %d times", finalValue, numberOfErrorsValue) + } + return true + } + return false } // MarkFailed marks a host as failed previously From a9b4cb076b2f07fd306ea590fd0c7b3a21466105 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 17 Aug 2021 16:05:29 +0530 Subject: [PATCH 6/8] Add support to clusterer for host errors --- v2/pkg/protocols/common/clusterer/executer.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/v2/pkg/protocols/common/clusterer/executer.go b/v2/pkg/protocols/common/clusterer/executer.go index 758e99b8..6085ffba 100644 --- a/v2/pkg/protocols/common/clusterer/executer.go +++ b/v2/pkg/protocols/common/clusterer/executer.go @@ -86,6 +86,9 @@ func (e *Executer) Execute(input string) (bool, error) { } } }) + if err != nil && e.options.HostErrorsCache != nil && e.options.HostErrorsCache.CheckError(err) { + e.options.HostErrorsCache.MarkFailed(input) + } return results, err } @@ -105,5 +108,8 @@ func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEve } } }) + if err != nil && e.options.HostErrorsCache != nil && e.options.HostErrorsCache.CheckError(err) { + e.options.HostErrorsCache.MarkFailed(input) + } return err } From 213b8be0f78ee2112ea88ccaea01d94b584866fb Mon Sep 17 00:00:00 2001 From: sandeep Date: Tue, 17 Aug 2021 16:25:59 +0530 Subject: [PATCH 7/8] misc update --- v2/cmd/nuclei/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 3dd1415b..0ec47f3e 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -120,7 +120,7 @@ on extensive configurability, massive extensibility and ease of use.`) createGroup(flagSet, "optimization", "Optimizations", flagSet.IntVar(&options.Timeout, "timeout", 5, "time to wait in seconds before timeout"), flagSet.IntVar(&options.Retries, "retries", 1, "number of times to retry a failed request"), - flagSet.IntVar(&options.HostMaxErrors, "host-max-errors", 30, "max errors allowed for a host before skipping"), + flagSet.IntVar(&options.HostMaxErrors, "host-max-error", 30, "max errors for a host before skipping from scan"), flagSet.BoolVar(&options.Project, "project", false, "use a project folder to avoid sending same request multiple times"), flagSet.StringVar(&options.ProjectPath, "project-path", os.TempDir(), "set a specific project path"), From 03d2405c3d6796b903d957aed94babae3fb138f1 Mon Sep 17 00:00:00 2001 From: sandeep Date: Tue, 17 Aug 2021 17:56:35 +0530 Subject: [PATCH 8/8] misc update --- v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go b/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go index 9c046790..ae6f5bc2 100644 --- a/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go +++ b/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go @@ -91,7 +91,7 @@ func (c *Cache) Check(value string) bool { if numberOfErrorsValue >= c.hostMaxErrors { _ = c.failedTargets.Set(finalValue, -1) if c.verbose { - gologger.Verbose().Msgf("Skipping %s as it has failed %d times", finalValue, numberOfErrorsValue) + gologger.Verbose().Msgf("Skipping %s as previously unresponsive %d times", finalValue, numberOfErrorsValue) } return true }