adds `-track-error` option to add custom errors to max-host-error watchlist (#3399)

* Allow user to specify for "context deadline exceeded" errors to count toward the max host error count

* Convert flag to a string slice `--track-error`

* Minimize diff

* Add documentation for `-track-error`

* adds unit test & minor improvements

* update flag description

---------

Co-authored-by: Austin Traver <austin_traver@intuit.com>
Co-authored-by: Tarun Koyalwar <tarun@projectdiscovery.io>

README.md

@@ -225,6 +225,7 @@ OPTIMIZATIONS:
    -retries int                        number of times to retry a failed request (default 1)
    -ldp, -leave-default-ports          leave default HTTP/HTTPS ports (eg. host:80,host:443)
    -mhe, -max-host-error int           max errors for a host before skipping from scan (default 30)
+   -te, -track-error string[]          adds given error to max-host-error watchlist (standard, file)
    -nmhe, -no-mhe                      disable skipping host from scan based on errors
    -project                            use a project folder to avoid sending same request multiple times
    -project-path string                set a specific project path (default "/tmp")

README_CN.md

@@ -189,6 +189,7 @@ Nuclei是一款注重于可配置性、可扩展性和易用性的基于模板
    -retries int                          重试次数（默认：1）
    -ldp, -leave-default-ports            指定HTTP/HTTPS默认端口（例如：host:80，host:443）
    -mhe, -max-host-error int             某主机扫描失败次数，跳过该主机（默认：30）
+   -te, -track-error string[]            将给定错误添加到最大主机错误监视列表（标准、文件）
    -nmhe, -no-mhe                        disable skipping host from scan based on errors
    -project                              使用项目文件夹避免多次发送同一请求
    -project-path string                  设置特定的项目文件夹

README_ID.md

@@ -188,6 +188,7 @@ OPTIMIZATIONS:
    -retries int                        number of times to retry a failed request (default 1)
    -ldp, -leave-default-ports          leave default HTTP/HTTPS ports (eg. host:80,host:443
    -mhe, -max-host-error int           max errors for a host before skipping from scan (default 30)
+   -te, -track-error string[]          adds given error to max-host-error watchlist (standard, file)
    -nmhe, -no-mhe                      disable skipping host from scan based on errors
    -project                            use a project folder to avoid sending same request multiple times
    -project-path string                set a specific project path

README_KR.md

@@ -178,6 +178,7 @@ OPTIMIZATIONS:
    -retries int                실패한 요청을 재시도하는 횟수 (기본 1)
    -ldp, -leave-default-ports  leave default HTTP/HTTPS ports (eg. host:80,host:443
    -mhe, -max-host-error int   스캔을 건너뛰기 전에 호스트에 대한 최대 오류 수 (기본 30)
+   -te, -track-error string[]  주어진 오류를 max-host-error 감시 목록(표준, 파일)에 추가
    -nmhe, -no-mhe                      disable skipping host from scan based on errors
    -project                    프로젝트 폴더를 사용하여 동일한 요청을 여러 번 보내지 않음
    -project-path string        특정 프로젝트 경로 설정

v2/cmd/integration-test/code.go

@@ -65,7 +65,7 @@ func (h *goIntegrationTest) Execute(templatePath string) error {
 
 // executeNucleiAsCode contains an example
 func executeNucleiAsCode(templatePath, templateURL string) ([]string, error) {
-	cache := hosterrorscache.New(30, hosterrorscache.DefaultMaxHostsCount)
+	cache := hosterrorscache.New(30, hosterrorscache.DefaultMaxHostsCount, nil)
 	defer cache.Close()
 
 	mockProgress := &testutils.MockProgressClient{}

v2/cmd/nuclei/main.go

@@ -249,6 +249,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.IntVar(&options.Retries, "retries", 1, "number of times to retry a failed request"),
 		flagSet.BoolVarP(&options.LeaveDefaultPorts, "leave-default-ports", "ldp", false, "leave default HTTP/HTTPS ports (eg. host:80,host:443)"),
 		flagSet.IntVarP(&options.MaxHostError, "max-host-error", "mhe", 30, "max errors for a host before skipping from scan"),
+		flagSet.StringSliceVarP(&options.TrackError, "track-error", "te", nil, "adds given error to max-host-error watchlist (standard, file)", goflags.FileStringSliceOptions),
 		flagSet.BoolVarP(&options.NoHostErrors, "no-mhe", "nmhe", false, "disable skipping host from scan based on errors"),
 		flagSet.BoolVar(&options.Project, "project", false, "use a project folder to avoid sending same request multiple times"),
 		flagSet.StringVar(&options.ProjectPath, "project-path", os.TempDir(), "set a specific project path"),
@@ -334,7 +335,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 	_ = flagSet.Parse()
 
 	gologger.DefaultLogger.SetTimestamp(options.Timestamp, levels.LevelDebug)
-	
+
 	if options.LeaveDefaultPorts {
 		http.LeaveDefaultPorts = true
 	}

v2/examples/simple.go

@@ -31,7 +31,7 @@ import (
 )
 
 func main() {
-	cache := hosterrorscache.New(30, hosterrorscache.DefaultMaxHostsCount)
+	cache := hosterrorscache.New(30, hosterrorscache.DefaultMaxHostsCount, nil)
 	defer cache.Close()
 
 	mockProgress := &testutils.MockProgressClient{}

v2/internal/runner/runner.go

@@ -411,7 +411,7 @@ func (r *Runner) RunEnumeration() error {
 	}
 
 	if r.options.ShouldUseHostError() {
-		cache := hosterrorscache.New(r.options.MaxHostError, hosterrorscache.DefaultMaxHostsCount)
+		cache := hosterrorscache.New(r.options.MaxHostError, hosterrorscache.DefaultMaxHostsCount, r.options.TrackError)
 		cache.SetVerbose(r.options.Verbose)
 		r.hostErrors = cache
 		executerOpts.HostErrorsCache = cache

v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go

@@ -30,6 +30,7 @@ type Cache struct {
 	MaxHostError  int
 	verbose       bool
 	failedTargets gcache.Cache
+	TrackError    []string
 }
 
 type cacheItem struct {
@@ -40,11 +41,11 @@ type cacheItem struct {
 const DefaultMaxHostsCount = 10000
 
 // New returns a new host max errors cache
-func New(maxHostError, maxHostsCount int) *Cache {
+func New(maxHostError, maxHostsCount int, trackError []string) *Cache {
 	gc := gcache.New(maxHostsCount).
 		ARC().
 		Build()
-	return &Cache{failedTargets: gc, MaxHostError: maxHostError}
+	return &Cache{failedTargets: gc, MaxHostError: maxHostError, TrackError: trackError}
 }
 
 // SetVerbose sets the cache to log at verbose level
@@ -128,6 +129,14 @@ var reCheckError = regexp.MustCompile(`(no address found for host|Client\.Timeou
 // checkError checks if an error represents a type that should be
 // added to the host skipping table.
 func (c *Cache) checkError(err error) bool {
+	if err == nil {
+		return false
+	}
 	errString := err.Error()
+	for _, msg := range c.TrackError {
+		if strings.Contains(errString, msg) {
+			return true
+		}
+	}
 	return reCheckError.MatchString(errString)
 }

v2/pkg/protocols/common/hosterrorscache/hosterrorscache_test.go

@@ -10,7 +10,7 @@ import (
 )
 
 func TestCacheCheck(t *testing.T) {
-	cache := New(3, DefaultMaxHostsCount)
+	cache := New(3, DefaultMaxHostsCount, nil)
 
 	for i := 0; i < 100; i++ {
 		cache.MarkFailed("test", fmt.Errorf("could not resolve host"))
@@ -28,6 +28,24 @@ func TestCacheCheck(t *testing.T) {
 	require.Equal(t, true, value, "could not get checked value")
 }
 
+func TestTrackErrors(t *testing.T) {
+	cache := New(3, DefaultMaxHostsCount, []string{"custom error"})
+
+	for i := 0; i < 100; i++ {
+		cache.MarkFailed("custom", fmt.Errorf("got: nested: custom error"))
+		got := cache.Check("custom")
+		if i < 2 {
+			// till 3 the host is not flagged to skip
+			require.False(t, got)
+		} else {
+			// above 3 it must remain flagged to skip
+			require.True(t, got)
+		}
+	}
+	value := cache.Check("custom")
+	require.Equal(t, true, value, "could not get checked value")
+}
+
 func TestCacheItemDo(t *testing.T) {
 	var (
 		count int
@@ -51,7 +69,7 @@ func TestCacheItemDo(t *testing.T) {
 }
 
 func TestCacheMarkFailed(t *testing.T) {
-	cache := New(3, DefaultMaxHostsCount)
+	cache := New(3, DefaultMaxHostsCount, nil)
 
 	tests := []struct {
 		host     string
@@ -76,7 +94,7 @@ func TestCacheMarkFailed(t *testing.T) {
 }
 
 func TestCacheMarkFailedConcurrent(t *testing.T) {
-	cache := New(3, DefaultMaxHostsCount)
+	cache := New(3, DefaultMaxHostsCount, nil)
 
 	tests := []struct {
 		host     string

v2/pkg/types/types.go

@@ -139,6 +139,8 @@ type Options struct {
 	MetricsPort int
 	// MaxHostError is the maximum number of errors allowed for a host
 	MaxHostError int
+	// TrackError contains additional error messages that count towards the maximum number of errors allowed for a host
+	TrackError goflags.StringSlice
 	// NoHostErrors disables host skipping after maximum number of errors
 	NoHostErrors bool
 	// BulkSize is the of targets analyzed in parallel for each template