introduce exclude target flag (#4214)

* introduce exclude target flag

* change logging

* update docs

* misc option update

* update input count

* separate funcs

* exclude with host-based comparison

* fix test

* fix lint

* remove duplicate

* introduce SetWithExclusions

* init map /shrug

---------

Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com>

README.md

@@ -115,11 +115,12 @@ Usage:
 
 Flags:
 TARGET:
-   -u, -target string[]       target URLs/hosts to scan
-   -l, -list string           path to file containing a list of target URLs/hosts to scan (one per line)
-   -resume string             resume scan using resume.cfg (clustering will be disabled)
-   -sa, -scan-all-ips         scan all the IP's associated with dns record
-   -iv, -ip-version string[]  IP version to scan of hostname (4,6) - (default 4)
+   -u, -target string[]             target URLs/hosts to scan
+   -l, -list string                 path to file containing a list of target URLs/hosts to scan (one per line)
+   -eh, -exclude-hosts string[]     hosts to exclude to scan from the input list (ip, cidr, hostname)
+   -resume string                   resume scan using resume.cfg (clustering will be disabled)
+   -sa, -scan-all-ips               scan all the IP's associated with dns record
+   -iv, -ip-version string[]        IP version to scan of hostname (4,6) - (default 4)
 
 TEMPLATES:
    -nt, -new-templates                    run only new templates added in latest nuclei-templates release

cmd/nuclei/main.go

@@ -178,6 +178,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 	flagSet.CreateGroup("input", "Target",
 		flagSet.StringSliceVarP(&options.Targets, "target", "u", nil, "target URLs/hosts to scan", goflags.StringSliceOptions),
 		flagSet.StringVarP(&options.TargetsFilePath, "list", "l", "", "path to file containing a list of target URLs/hosts to scan (one per line)"),
+		flagSet.StringSliceVarP(&options.ExcludeTargets, "exclude-hosts", "eh", nil, "hosts to exclude to scan from the input list (ip, cidr, hostname)", goflags.FileCommaSeparatedStringSliceOptions),
 		flagSet.StringVar(&options.Resume, "resume", "", "resume scan using resume.cfg (clustering will be disabled)"),
 		flagSet.BoolVarP(&options.ScanAllIPs, "scan-all-ips", "sa", false, "scan all the IP's associated with dns record"),
 		flagSet.StringSliceVarP(&options.IPVersion, "ip-version", "iv", nil, "IP version to scan of hostname (4,6) - (default 4)", goflags.CommaSeparatedStringSliceOptions),

internal/runner/runner.go

@@ -453,7 +453,7 @@ func (r *Runner) RunEnumeration() error {
 		}
 		ret := uncover.GetUncoverTargetsFromMetadata(context.TODO(), store.Templates(), r.options.UncoverField, uncoverOpts)
 		for host := range ret {
-			r.hmapInputProvider.Set(host)
+			r.hmapInputProvider.SetWithExclusions(host)
 		}
 	}
 	// list all templates

pkg/core/inputs/hybrid/hmap.go

@@ -37,8 +37,11 @@ const DefaultMaxDedupeItemsCount = 10000
 type Input struct {
 	ipOptions         *ipOptions
 	inputCount        int64
+	excludedCount     int64
 	dupeCount         int64
+	skippedCount      int64
 	hostMap           *hybrid.HybridMap
+	excludedHosts     map[string]struct{}
 	hostMapStream     *filekv.FileDB
 	hostMapStreamOnce sync.Once
 	sync.Once
@@ -70,6 +73,7 @@ func New(opts *Options) (*Input, error) {
 			IPV4:       sliceutil.Contains(options.IPVersion, "4"),
 			IPV6:       sliceutil.Contains(options.IPVersion, "6"),
 		},
+		excludedHosts: make(map[string]struct{}),
 	}
 	if options.Stream {
 		fkvOptions := filekv.DefaultOptions
@@ -88,9 +92,15 @@ func New(opts *Options) (*Input, error) {
 	if initErr := input.initializeInputSources(opts); initErr != nil {
 		return nil, initErr
 	}
+	if input.excludedCount > 0 {
+		gologger.Info().Msgf("Number of hosts excluded from input: %d", input.excludedCount)
+	}
 	if input.dupeCount > 0 {
 		gologger.Info().Msgf("Supplied input was automatically deduplicated (%d removed).", input.dupeCount)
 	}
+	if input.skippedCount > 0 {
+		gologger.Info().Msgf("Number of hosts skipped from input due to exclusion: %d", input.skippedCount)
+	}
 	return input, nil
 }
 
@@ -110,9 +120,11 @@ func (i *Input) initializeInputSources(opts *Options) error {
 	for _, target := range options.Targets {
 		switch {
 		case iputil.IsCIDR(target):
-			i.expandCIDRInputValue(target)
+			ips := i.expandCIDRInputValue(target)
+			i.addTargets(ips)
 		case asn.IsASN(target):
-			i.expandASNInputValue(target)
+			ips := i.expandASNInputValue(target)
+			i.addTargets(ips)
 		default:
 			i.Set(target)
 		}
@@ -156,6 +168,22 @@ func (i *Input) initializeInputSources(opts *Options) error {
 			i.Set(c)
 		}
 	}
+
+	if len(options.ExcludeTargets) > 0 {
+		for _, target := range options.ExcludeTargets {
+			switch {
+			case iputil.IsCIDR(target):
+				ips := i.expandCIDRInputValue(target)
+				i.removeTargets(ips)
+			case asn.IsASN(target):
+				ips := i.expandASNInputValue(target)
+				i.removeTargets(ips)
+			default:
+				i.Del(target)
+			}
+		}
+	}
+
 	return nil
 }
 
@@ -166,9 +194,11 @@ func (i *Input) scanInputFromReader(reader io.Reader) {
 		item := scanner.Text()
 		switch {
 		case iputil.IsCIDR(item):
-			i.expandCIDRInputValue(item)
+			ips := i.expandCIDRInputValue(item)
+			i.addTargets(ips)
 		case asn.IsASN(item):
-			i.expandASNInputValue(item)
+			ips := i.expandASNInputValue(item)
+			i.addTargets(ips)
 		default:
 			i.Set(item)
 		}
@@ -258,6 +288,114 @@ func (i *Input) Set(value string) {
 	}
 }
 
+// SetWithExclusions normalizes and stores passed input values if not excluded
+func (i *Input) SetWithExclusions(value string) {
+	URL := strings.TrimSpace(value)
+	if URL == "" {
+		return
+	}
+	if i.isExcluded(URL) {
+		i.skippedCount++
+		return
+	}
+	i.Set(URL)
+}
+
+// isExcluded checks if a URL is in the exclusion list
+func (i *Input) isExcluded(URL string) bool {
+	metaInput := &contextargs.MetaInput{Input: URL}
+	key, err := metaInput.MarshalString()
+	if err != nil {
+		gologger.Warning().Msgf("%s\n", err)
+		return false
+	}
+
+	_, exists := i.excludedHosts[key]
+	return exists
+}
+
+func (i *Input) Del(value string) {
+	URL := strings.TrimSpace(value)
+	if URL == "" {
+		return
+	}
+	// parse hostname if url is given
+	urlx, err := urlutil.Parse(URL)
+	if err != nil || (urlx != nil && urlx.Host == "") {
+		gologger.Debug().Label("url").MsgFunc(func() string {
+			if err != nil {
+				return fmt.Sprintf("failed to parse url %v got %v skipping ip selection", URL, err)
+			}
+			return fmt.Sprintf("got empty hostname for %v skipping ip selection", URL)
+		})
+		metaInput := &contextargs.MetaInput{Input: URL}
+		i.delItem(metaInput)
+		return
+	}
+
+	// Check if input is ip or hostname
+	if iputil.IsIP(urlx.Hostname()) {
+		metaInput := &contextargs.MetaInput{Input: URL}
+		i.delItem(metaInput)
+		return
+	}
+
+	if i.ipOptions.ScanAllIPs {
+		// scan all ips
+		dnsData, err := protocolstate.Dialer.GetDNSData(urlx.Hostname())
+		if err == nil {
+			if (len(dnsData.A) + len(dnsData.AAAA)) > 0 {
+				var ips []string
+				if i.ipOptions.IPV4 {
+					ips = append(ips, dnsData.A...)
+				}
+				if i.ipOptions.IPV6 {
+					ips = append(ips, dnsData.AAAA...)
+				}
+				for _, ip := range ips {
+					if ip == "" {
+						continue
+					}
+					metaInput := &contextargs.MetaInput{Input: value, CustomIP: ip}
+					i.delItem(metaInput)
+				}
+				return
+			} else {
+				gologger.Debug().Msgf("scanAllIps: no ip's found reverting to default")
+			}
+		} else {
+			// failed to scanallips falling back to defaults
+			gologger.Debug().Msgf("scanAllIps: dns resolution failed: %v", err)
+		}
+	}
+
+	ips := []string{}
+	// only scan the target but ipv6 if it has one
+	if i.ipOptions.IPV6 {
+		dnsData, err := protocolstate.Dialer.GetDNSData(urlx.Hostname())
+		if err == nil && len(dnsData.AAAA) > 0 {
+			// pick/ prefer 1st
+			ips = append(ips, dnsData.AAAA[0])
+		} else {
+			gologger.Warning().Msgf("target does not have ipv6 address falling back to ipv4 %v\n", err)
+		}
+	}
+	if i.ipOptions.IPV4 {
+		// if IPV4 is enabled do not specify ip let dialer handle it
+		ips = append(ips, "")
+	}
+
+	for _, ip := range ips {
+		if ip != "" {
+			metaInput := &contextargs.MetaInput{Input: URL, CustomIP: ip}
+			i.delItem(metaInput)
+		} else {
+			metaInput := &contextargs.MetaInput{Input: URL}
+			i.delItem(metaInput)
+		}
+	}
+}
+
 // setItem in the kv store
 func (i *Input) setItem(metaInput *contextargs.MetaInput) {
 	key, err := metaInput.MarshalString()
@@ -277,6 +415,38 @@ func (i *Input) setItem(metaInput *contextargs.MetaInput) {
 	}
 }
 
+// setItem in the kv store
+func (i *Input) delItem(metaInput *contextargs.MetaInput) {
+	targetUrl, err := urlutil.ParseURL(metaInput.Input, true)
+	if err != nil {
+		gologger.Warning().Msgf("%s\n", err)
+		return
+	}
+
+	i.hostMap.Scan(func(k, _ []byte) error {
+		var tmpMetaInput contextargs.MetaInput
+		if err := tmpMetaInput.Unmarshal(string(k)); err != nil {
+			return err
+		}
+		tmpKey, err := tmpMetaInput.MarshalString()
+		if err != nil {
+			return err
+		}
+		tmpUrl, err := urlutil.ParseURL(tmpMetaInput.Input, true)
+		if err != nil {
+			return err
+		}
+
+		if tmpUrl.Host == targetUrl.Host {
+			_ = i.hostMap.Del(tmpKey)
+			i.excludedHosts[tmpKey] = struct{}{}
+			i.excludedCount++
+			i.inputCount--
+		}
+		return nil
+	})
+}
+
 // setHostMapStream sets item in stream mode
 func (i *Input) setHostMapStream(data string) {
 	if _, err := i.hostMapStream.Merge([][]byte{[]byte(data)}); err != nil {
@@ -318,31 +488,34 @@ func (i *Input) Scan(callback func(value *contextargs.MetaInput) bool) {
 }
 
 // expandCIDRInputValue expands CIDR and stores expanded IPs
-func (i *Input) expandCIDRInputValue(value string) {
-	ips, _ := mapcidr.IPAddressesAsStream(value)
-	for ip := range ips {
-		metaInput := &contextargs.MetaInput{Input: ip}
-		key, err := metaInput.MarshalString()
-		if err != nil {
-			gologger.Warning().Msgf("%s\n", err)
-			return
-		}
-		if _, ok := i.hostMap.Get(key); ok {
-			i.dupeCount++
-			continue
-		}
-		i.inputCount++
-		_ = i.hostMap.Set(key, nil)
-		if i.hostMapStream != nil {
-			i.setHostMapStream(key)
-		}
+func (i *Input) expandCIDRInputValue(value string) []string {
+	var ips []string
+	ipsCh, _ := mapcidr.IPAddressesAsStream(value)
+	for ip := range ipsCh {
+		ips = append(ips, ip)
 	}
+	return ips
 }
 
 // expandASNInputValue expands CIDRs for given ASN and stores expanded IPs
-func (i *Input) expandASNInputValue(value string) {
+func (i *Input) expandASNInputValue(value string) []string {
+	var ips []string
 	cidrs, _ := asn.GetCIDRsForASNNum(value)
 	for _, cidr := range cidrs {
-		i.expandCIDRInputValue(cidr.String())
+		ips = append(ips, i.expandCIDRInputValue(cidr.String())...)
+	}
+	return ips
+}
+
+func (i *Input) addTargets(targets []string) {
+	for _, target := range targets {
+		i.Set(target)
+	}
+}
+
+func (i *Input) removeTargets(targets []string) {
+	for _, target := range targets {
+		metaInput := &contextargs.MetaInput{Input: target}
+		i.delItem(metaInput)
 	}
 }

pkg/core/inputs/hybrid/hmap_test.go

@@ -33,7 +33,8 @@ func Test_expandCIDRInputValue(t *testing.T) {
 		require.Nil(t, err, "could not create temporary input file")
 		input := &Input{hostMap: hm}
 
-		input.expandCIDRInputValue(tt.cidr)
+		ips := input.expandCIDRInputValue(tt.cidr)
+		input.addTargets(ips)
 		// scan
 		got := []string{}
 		input.hostMap.Scan(func(k, _ []byte) error {
@@ -169,7 +170,8 @@ func Test_expandASNInputValue(t *testing.T) {
 		require.Nil(t, err, "could not create temporary input file")
 		input := &Input{hostMap: hm}
 		// get the IP addresses for ASN number
-		input.expandASNInputValue(tt.asn)
+		ips := input.expandASNInputValue(tt.asn)
+		input.addTargets(ips)
 		// scan the hmap
 		got := []string{}
 		input.hostMap.Scan(func(k, v []byte) error {

pkg/types/types.go

@@ -76,6 +76,8 @@ type Options struct {
 	InteractshToken string
 	// Target URLs/Domains to scan using a template
 	Targets goflags.StringSlice
+	// ExcludeTargets URLs/Domains to exclude from scanning
+	ExcludeTargets goflags.StringSlice
 	// TargetsFilePath specifies the targets from a file to scan using templates.
 	TargetsFilePath string
 	// Resume the scan from the state stored in the resume config file