From 0d90a555f67289397d091f03da4a2a75ef691f37 Mon Sep 17 00:00:00 2001 From: Austin Traver Date: Tue, 14 Mar 2023 01:29:42 -0700 Subject: [PATCH] adds `-track-error` option to add custom errors to max-host-error watchlist (#3399) * Allow user to specify for "context deadline exceeded" errors to count toward the max host error count * Convert flag to a string slice `--track-error` * Minimize diff * Add documentation for `-track-error` * adds unit test & minor improvements * update flag description --------- Co-authored-by: Austin Traver Co-authored-by: Tarun Koyalwar --- README.md | 1 + README_CN.md | 1 + README_ID.md | 1 + README_KR.md | 1 + v2/cmd/integration-test/code.go | 2 +- v2/cmd/nuclei/main.go | 3 ++- v2/examples/simple.go | 2 +- v2/internal/runner/runner.go | 2 +- .../common/hosterrorscache/hosterrorscache.go | 13 ++++++++-- .../hosterrorscache/hosterrorscache_test.go | 24 ++++++++++++++++--- v2/pkg/types/types.go | 2 ++ 11 files changed, 43 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index f2da76f5..2fe74397 100644 --- a/README.md +++ b/README.md @@ -225,6 +225,7 @@ OPTIMIZATIONS: -retries int number of times to retry a failed request (default 1) -ldp, -leave-default-ports leave default HTTP/HTTPS ports (eg. host:80,host:443) -mhe, -max-host-error int max errors for a host before skipping from scan (default 30) + -te, -track-error string[] adds given error to max-host-error watchlist (standard, file) -nmhe, -no-mhe disable skipping host from scan based on errors -project use a project folder to avoid sending same request multiple times -project-path string set a specific project path (default "/tmp") diff --git a/README_CN.md b/README_CN.md index 0764c88e..7ab8ced0 100644 --- a/README_CN.md +++ b/README_CN.md @@ -189,6 +189,7 @@ Nuclei是一款注重于可配置性、可扩展性和易用性的基于模板 -retries int 重试次数（默认：1） -ldp, -leave-default-ports 指定HTTP/HTTPS默认端口（例如：host:80，host:443） -mhe, -max-host-error int 某主机扫描失败次数，跳过该主机（默认：30） + -te, -track-error string[] 将给定错误添加到最大主机错误监视列表（标准、文件） -nmhe, -no-mhe disable skipping host from scan based on errors -project 使用项目文件夹避免多次发送同一请求 -project-path string 设置特定的项目文件夹 diff --git a/README_ID.md b/README_ID.md index ca4d6a7f..b3022cc9 100644 --- a/README_ID.md +++ b/README_ID.md @@ -188,6 +188,7 @@ OPTIMIZATIONS: -retries int number of times to retry a failed request (default 1) -ldp, -leave-default-ports leave default HTTP/HTTPS ports (eg. host:80,host:443 -mhe, -max-host-error int max errors for a host before skipping from scan (default 30) + -te, -track-error string[] adds given error to max-host-error watchlist (standard, file) -nmhe, -no-mhe disable skipping host from scan based on errors -project use a project folder to avoid sending same request multiple times -project-path string set a specific project path diff --git a/README_KR.md b/README_KR.md index 4daa7099..343d87ac 100644 --- a/README_KR.md +++ b/README_KR.md @@ -178,6 +178,7 @@ OPTIMIZATIONS: -retries int 실패한 요청을 재시도하는 횟수 (기본 1) -ldp, -leave-default-ports leave default HTTP/HTTPS ports (eg. host:80,host:443 -mhe, -max-host-error int 스캔을 건너뛰기 전에 호스트에 대한 최대 오류 수 (기본 30) + -te, -track-error string[] 주어진 오류를 max-host-error 감시 목록(표준, 파일)에 추가 -nmhe, -no-mhe disable skipping host from scan based on errors -project 프로젝트 폴더를 사용하여 동일한 요청을 여러 번 보내지 않음 -project-path string 특정 프로젝트 경로 설정 diff --git a/v2/cmd/integration-test/code.go b/v2/cmd/integration-test/code.go index dc2ce1c3..eb7fb03c 100644 --- a/v2/cmd/integration-test/code.go +++ b/v2/cmd/integration-test/code.go @@ -65,7 +65,7 @@ func (h *goIntegrationTest) Execute(templatePath string) error { // executeNucleiAsCode contains an example func executeNucleiAsCode(templatePath, templateURL string) ([]string, error) { - cache := hosterrorscache.New(30, hosterrorscache.DefaultMaxHostsCount) + cache := hosterrorscache.New(30, hosterrorscache.DefaultMaxHostsCount, nil) defer cache.Close() mockProgress := &testutils.MockProgressClient{} diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 91efbe37..902511d7 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -249,6 +249,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.IntVar(&options.Retries, "retries", 1, "number of times to retry a failed request"), flagSet.BoolVarP(&options.LeaveDefaultPorts, "leave-default-ports", "ldp", false, "leave default HTTP/HTTPS ports (eg. host:80,host:443)"), flagSet.IntVarP(&options.MaxHostError, "max-host-error", "mhe", 30, "max errors for a host before skipping from scan"), + flagSet.StringSliceVarP(&options.TrackError, "track-error", "te", nil, "adds given error to max-host-error watchlist (standard, file)", goflags.FileStringSliceOptions), flagSet.BoolVarP(&options.NoHostErrors, "no-mhe", "nmhe", false, "disable skipping host from scan based on errors"), flagSet.BoolVar(&options.Project, "project", false, "use a project folder to avoid sending same request multiple times"), flagSet.StringVar(&options.ProjectPath, "project-path", os.TempDir(), "set a specific project path"), @@ -334,7 +335,7 @@ on extensive configurability, massive extensibility and ease of use.`) _ = flagSet.Parse() gologger.DefaultLogger.SetTimestamp(options.Timestamp, levels.LevelDebug) - + if options.LeaveDefaultPorts { http.LeaveDefaultPorts = true } diff --git a/v2/examples/simple.go b/v2/examples/simple.go index c23b0d3b..129561fa 100644 --- a/v2/examples/simple.go +++ b/v2/examples/simple.go @@ -31,7 +31,7 @@ import ( ) func main() { - cache := hosterrorscache.New(30, hosterrorscache.DefaultMaxHostsCount) + cache := hosterrorscache.New(30, hosterrorscache.DefaultMaxHostsCount, nil) defer cache.Close() mockProgress := &testutils.MockProgressClient{} diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 22f207b2..db0b97c0 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -411,7 +411,7 @@ func (r *Runner) RunEnumeration() error { } if r.options.ShouldUseHostError() { - cache := hosterrorscache.New(r.options.MaxHostError, hosterrorscache.DefaultMaxHostsCount) + cache := hosterrorscache.New(r.options.MaxHostError, hosterrorscache.DefaultMaxHostsCount, r.options.TrackError) cache.SetVerbose(r.options.Verbose) r.hostErrors = cache executerOpts.HostErrorsCache = cache diff --git a/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go b/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go index d7e6610a..53d77b49 100644 --- a/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go +++ b/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go @@ -30,6 +30,7 @@ type Cache struct { MaxHostError int verbose bool failedTargets gcache.Cache + TrackError []string } type cacheItem struct { @@ -40,11 +41,11 @@ type cacheItem struct { const DefaultMaxHostsCount = 10000 // New returns a new host max errors cache -func New(maxHostError, maxHostsCount int) *Cache { +func New(maxHostError, maxHostsCount int, trackError []string) *Cache { gc := gcache.New(maxHostsCount). ARC(). Build() - return &Cache{failedTargets: gc, MaxHostError: maxHostError} + return &Cache{failedTargets: gc, MaxHostError: maxHostError, TrackError: trackError} } // SetVerbose sets the cache to log at verbose level @@ -128,6 +129,14 @@ var reCheckError = regexp.MustCompile(`(no address found for host|Client\.Timeou // checkError checks if an error represents a type that should be // added to the host skipping table. func (c *Cache) checkError(err error) bool { + if err == nil { + return false + } errString := err.Error() + for _, msg := range c.TrackError { + if strings.Contains(errString, msg) { + return true + } + } return reCheckError.MatchString(errString) } diff --git a/v2/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go b/v2/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go index 15debcdf..a4730366 100644 --- a/v2/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go +++ b/v2/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go @@ -10,7 +10,7 @@ import ( ) func TestCacheCheck(t *testing.T) { - cache := New(3, DefaultMaxHostsCount) + cache := New(3, DefaultMaxHostsCount, nil) for i := 0; i < 100; i++ { cache.MarkFailed("test", fmt.Errorf("could not resolve host")) @@ -28,6 +28,24 @@ func TestCacheCheck(t *testing.T) { require.Equal(t, true, value, "could not get checked value") } +func TestTrackErrors(t *testing.T) { + cache := New(3, DefaultMaxHostsCount, []string{"custom error"}) + + for i := 0; i < 100; i++ { + cache.MarkFailed("custom", fmt.Errorf("got: nested: custom error")) + got := cache.Check("custom") + if i < 2 { + // till 3 the host is not flagged to skip + require.False(t, got) + } else { + // above 3 it must remain flagged to skip + require.True(t, got) + } + } + value := cache.Check("custom") + require.Equal(t, true, value, "could not get checked value") +} + func TestCacheItemDo(t *testing.T) { var ( count int @@ -51,7 +69,7 @@ func TestCacheItemDo(t *testing.T) { } func TestCacheMarkFailed(t *testing.T) { - cache := New(3, DefaultMaxHostsCount) + cache := New(3, DefaultMaxHostsCount, nil) tests := []struct { host string @@ -76,7 +94,7 @@ func TestCacheMarkFailed(t *testing.T) { } func TestCacheMarkFailedConcurrent(t *testing.T) { - cache := New(3, DefaultMaxHostsCount) + cache := New(3, DefaultMaxHostsCount, nil) tests := []struct { host string diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 40f69d4a..3a989d02 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -139,6 +139,8 @@ type Options struct { MetricsPort int // MaxHostError is the maximum number of errors allowed for a host MaxHostError int + // TrackError contains additional error messages that count towards the maximum number of errors allowed for a host + TrackError goflags.StringSlice // NoHostErrors disables host skipping after maximum number of errors NoHostErrors bool // BulkSize is the of targets analyzed in parallel for each template