diff --git a/.github/workflows/publish-docs.yaml b/.github/workflows/publish-docs.yaml
index 0b4e2a76..03fb105b 100644
--- a/.github/workflows/publish-docs.yaml
+++ b/.github/workflows/publish-docs.yaml
@@ -12,6 +12,8 @@ jobs:
steps:
- name: Checkout code
uses: actions/checkout@v3
+ with:
+ fetch-depth: 0
- name: "Set up Go"
uses: actions/setup-go@v3
@@ -36,7 +38,6 @@ jobs:
run: |
git config --local user.email "action@github.com"
git config --local user.name "GitHub Action"
- git pull
git add SYNTAX-REFERENCE.md nuclei-jsonschema.json
git commit -m "Auto Generate Syntax Docs + JSONSchema [$(date)] :robot:" -a
diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md
index 7308db44..b0311441 100755
--- a/SYNTAX-REFERENCE.md
+++ b/SYNTAX-REFERENCE.md
@@ -3519,6 +3519,19 @@ description: |
+stop-at-first-match bool
+
+
+
+
+StopAtFirstMatch stops the execution of the requests and template as soon as a match is found.
+
+
+
+
+
+
diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json
index ef08a307..cf9b5fd6 100644
--- a/nuclei-jsonschema.json
+++ b/nuclei-jsonschema.json
@@ -576,6 +576,11 @@
"title": "custom user agent for the headless request",
"description": "Custom user agent for the headless request"
},
+ "stop-at-first-match": {
+ "type": "boolean",
+ "title": "stop at first match",
+ "description": "Stop the execution after a match is found"
+ },
"matchers": {
"items": {
"$ref": "#/definitions/matchers.Matcher"
diff --git a/v2/pkg/core/inputs/hybrid/hmap.go b/v2/pkg/core/inputs/hybrid/hmap.go
index 3937e283..82d2c94d 100644
--- a/v2/pkg/core/inputs/hybrid/hmap.go
+++ b/v2/pkg/core/inputs/hybrid/hmap.go
@@ -5,7 +5,7 @@ package hybrid
import (
"bufio"
"io"
- "net/url"
+ "net"
"os"
"strings"
"sync"
@@ -22,6 +22,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/uncover"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
+ "github.com/projectdiscovery/nuclei/v2/pkg/utils"
fileutil "github.com/projectdiscovery/utils/file"
iputil "github.com/projectdiscovery/utils/ip"
readerutil "github.com/projectdiscovery/utils/reader"
@@ -169,39 +170,49 @@ func (i *Input) Set(value string) {
if URL == "" {
return
}
- // actual hostname
- var host string
// parse hostname if url is given
- parsedURL, err := url.Parse(value)
- if err == nil && parsedURL.Host != "" {
- host = parsedURL.Host
+ host := utils.ParseHostname(value)
+ if host == "" {
+ // not a valid url hence scanallips is skipped
+ gologger.Debug().Msgf("scanAllIps: failed to parse hostname of %v falling back to default", value)
+ i.setItem(&contextargs.MetaInput{Input: value})
+ return
} else {
- parsedURL = nil
- host = value
+ // case when hostname contains port
+ hostwithoutport, _, erx := net.SplitHostPort(host)
+ if erx == nil && hostwithoutport != "" {
+ // given host contains port
+ host = hostwithoutport
+ }
}
if i.ipOptions.ScanAllIPs {
// scan all ips
dnsData, err := protocolstate.Dialer.GetDNSData(host)
- if err == nil && (len(dnsData.A)+len(dnsData.AAAA)) > 0 {
- var ips []string
- if i.ipOptions.IPV4 {
- ips = append(ips, dnsData.A...)
- }
- if i.ipOptions.IPV6 {
- ips = append(ips, dnsData.AAAA...)
- }
- for _, ip := range ips {
- if ip == "" {
- continue
+ if err == nil {
+ if (len(dnsData.A) + len(dnsData.AAAA)) > 0 {
+ var ips []string
+ if i.ipOptions.IPV4 {
+ ips = append(ips, dnsData.A...)
}
- metaInput := &contextargs.MetaInput{Input: value, CustomIP: ip}
- i.setItem(metaInput)
+ if i.ipOptions.IPV6 {
+ ips = append(ips, dnsData.AAAA...)
+ }
+ for _, ip := range ips {
+ if ip == "" {
+ continue
+ }
+ metaInput := &contextargs.MetaInput{Input: value, CustomIP: ip}
+ i.setItem(metaInput)
+ }
+ return
+ } else {
+ gologger.Debug().Msgf("scanAllIps: no ip's found reverting to default")
}
- return
+ } else {
+ // failed to scanallips falling back to defaults
+ gologger.Debug().Msgf("scanAllIps: dns resolution failed: %v", err)
}
- // failed to scanallips falling back to defaults
- gologger.Error().Msgf("failed to scan all ips reverting to default %v", err)
}
ips := []string{}
@@ -212,7 +223,7 @@ func (i *Input) Set(value string) {
// pick/ prefer 1st
ips = append(ips, dnsData.AAAA[0])
} else {
- gologger.Warning().Msgf("target does not have ipv6 address falling back to ipv4 %s\n", err)
+ gologger.Warning().Msgf("target does not have ipv6 address falling back to ipv4 %v\n", err)
}
}
if i.ipOptions.IPV4 {
diff --git a/v2/pkg/core/inputs/hybrid/hmap_test.go b/v2/pkg/core/inputs/hybrid/hmap_test.go
index c4aa453e..60f12972 100644
--- a/v2/pkg/core/inputs/hybrid/hmap_test.go
+++ b/v2/pkg/core/inputs/hybrid/hmap_test.go
@@ -51,7 +51,7 @@ func Test_expandCIDRInputValue(t *testing.T) {
type mockDnsHandler struct{}
-func (this *mockDnsHandler) ServeDNS(w dns.ResponseWriter, r *dns.Msg) {
+func (m *mockDnsHandler) ServeDNS(w dns.ResponseWriter, r *dns.Msg) {
msg := dns.Msg{}
msg.SetReply(r)
switch r.Question[0].Qtype {
@@ -85,18 +85,14 @@ func Test_scanallips_normalizeStoreInputValue(t *testing.T) {
defaultOpts := types.DefaultOptions()
defaultOpts.InternalResolversList = []string{"127.0.0.1:61234"}
_ = protocolstate.Init(defaultOpts)
- tests := []struct {
+ type testcase struct {
hostname string
ipv4 bool
ipv6 bool
expected []string
- }{
+ }
+ tests := []testcase{
{
- hostname: "scanme.sh",
- ipv4: true,
- ipv6: true,
- expected: []string{"128.199.158.128", "2400:6180:0:d0::91:1001"},
- }, {
hostname: "scanme.sh",
ipv4: true,
expected: []string{"128.199.158.128"},
@@ -104,12 +100,27 @@ func Test_scanallips_normalizeStoreInputValue(t *testing.T) {
hostname: "scanme.sh",
ipv6: true,
expected: []string{"2400:6180:0:d0::91:1001"},
- }, {
- hostname: "http://scanme.sh",
+ },
+ }
+ // add extra edge cases
+ urls := []string{
+ "https://scanme.sh/",
+ "http://scanme.sh",
+ "https://scanme.sh:443/",
+ "https://scanme.sh:443/somepath",
+ "http://scanme.sh:80/?with=param",
+ "scanme.sh/home",
+ "scanme.sh",
+ }
+ resolvedIps := []string{"128.199.158.128", "2400:6180:0:d0::91:1001"}
+
+ for _, v := range urls {
+ tests = append(tests, testcase{
+ hostname: v,
ipv4: true,
ipv6: true,
- expected: []string{"128.199.158.128", "2400:6180:0:d0::91:1001"},
- },
+ expected: resolvedIps,
+ })
}
for _, tt := range tests {
hm, err := hybrid.New(hybrid.DefaultDiskOptions)
@@ -134,7 +145,7 @@ func Test_scanallips_normalizeStoreInputValue(t *testing.T) {
got = append(got, metainput.CustomIP)
return nil
})
- require.ElementsMatch(t, tt.expected, got, "could not get correct ips")
+ require.ElementsMatchf(t, tt.expected, got, "could not get correct ips for hostname %v", tt.hostname)
input.Close()
}
}
diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go
index f85b3567..d96fbe4a 100644
--- a/v2/pkg/templates/templates_doc.go
+++ b/v2/pkg/templates/templates_doc.go
@@ -1542,7 +1542,7 @@ func init() {
Value: "Headless response received from client (default)",
},
}
- HEADLESSRequestDoc.Fields = make([]encoder.Doc, 9)
+ HEADLESSRequestDoc.Fields = make([]encoder.Doc, 10)
HEADLESSRequestDoc.Fields[0].Name = "id"
HEADLESSRequestDoc.Fields[0].Type = "string"
HEADLESSRequestDoc.Fields[0].Note = ""
@@ -1573,22 +1573,27 @@ func init() {
HEADLESSRequestDoc.Fields[5].Note = ""
HEADLESSRequestDoc.Fields[5].Description = "description: |\n If UserAgent is set to custom, customUserAgent is the custom user-agent to use for the request."
HEADLESSRequestDoc.Fields[5].Comments[encoder.LineComment] = " description: |"
- HEADLESSRequestDoc.Fields[6].Name = "matchers"
- HEADLESSRequestDoc.Fields[6].Type = "[]matchers.Matcher"
+ HEADLESSRequestDoc.Fields[6].Name = "stop-at-first-match"
+ HEADLESSRequestDoc.Fields[6].Type = "bool"
HEADLESSRequestDoc.Fields[6].Note = ""
- HEADLESSRequestDoc.Fields[6].Description = "Matchers contains the detection mechanism for the request to identify\nwhether the request was successful by doing pattern matching\non request/responses.\n\nMultiple matchers can be combined with `matcher-condition` flag\nwhich accepts either `and` or `or` as argument."
- HEADLESSRequestDoc.Fields[6].Comments[encoder.LineComment] = "Matchers contains the detection mechanism for the request to identify"
- HEADLESSRequestDoc.Fields[7].Name = "extractors"
- HEADLESSRequestDoc.Fields[7].Type = "[]extractors.Extractor"
+ HEADLESSRequestDoc.Fields[6].Description = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found."
+ HEADLESSRequestDoc.Fields[6].Comments[encoder.LineComment] = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found."
+ HEADLESSRequestDoc.Fields[7].Name = "matchers"
+ HEADLESSRequestDoc.Fields[7].Type = "[]matchers.Matcher"
HEADLESSRequestDoc.Fields[7].Note = ""
- HEADLESSRequestDoc.Fields[7].Description = "Extractors contains the extraction mechanism for the request to identify\nand extract parts of the response."
- HEADLESSRequestDoc.Fields[7].Comments[encoder.LineComment] = "Extractors contains the extraction mechanism for the request to identify"
- HEADLESSRequestDoc.Fields[8].Name = "matchers-condition"
- HEADLESSRequestDoc.Fields[8].Type = "string"
+ HEADLESSRequestDoc.Fields[7].Description = "Matchers contains the detection mechanism for the request to identify\nwhether the request was successful by doing pattern matching\non request/responses.\n\nMultiple matchers can be combined with `matcher-condition` flag\nwhich accepts either `and` or `or` as argument."
+ HEADLESSRequestDoc.Fields[7].Comments[encoder.LineComment] = "Matchers contains the detection mechanism for the request to identify"
+ HEADLESSRequestDoc.Fields[8].Name = "extractors"
+ HEADLESSRequestDoc.Fields[8].Type = "[]extractors.Extractor"
HEADLESSRequestDoc.Fields[8].Note = ""
- HEADLESSRequestDoc.Fields[8].Description = "MatchersCondition is the condition between the matchers. Default is OR."
- HEADLESSRequestDoc.Fields[8].Comments[encoder.LineComment] = "MatchersCondition is the condition between the matchers. Default is OR."
- HEADLESSRequestDoc.Fields[8].Values = []string{
+ HEADLESSRequestDoc.Fields[8].Description = "Extractors contains the extraction mechanism for the request to identify\nand extract parts of the response."
+ HEADLESSRequestDoc.Fields[8].Comments[encoder.LineComment] = "Extractors contains the extraction mechanism for the request to identify"
+ HEADLESSRequestDoc.Fields[9].Name = "matchers-condition"
+ HEADLESSRequestDoc.Fields[9].Type = "string"
+ HEADLESSRequestDoc.Fields[9].Note = ""
+ HEADLESSRequestDoc.Fields[9].Description = "MatchersCondition is the condition between the matchers. Default is OR."
+ HEADLESSRequestDoc.Fields[9].Comments[encoder.LineComment] = "MatchersCondition is the condition between the matchers. Default is OR."
+ HEADLESSRequestDoc.Fields[9].Values = []string{
"and",
"or",
}
diff --git a/v2/pkg/utils/utils.go b/v2/pkg/utils/utils.go
index 0db1e6b3..73efad3d 100644
--- a/v2/pkg/utils/utils.go
+++ b/v2/pkg/utils/utils.go
@@ -77,3 +77,24 @@ func StringSliceContains(slice []string, item string) bool {
}
return false
}
+
+// ParseHostname returns hostname
+func ParseHostname(inputURL string) string {
+ /*
+ currently if URL is scanme.sh/path or scanme.sh:443 i.e without protocol then
+ url.Parse considers this as valid url but fails to parse hostname
+ this can be handled by adding schema
+ */
+ input, err := url.Parse(inputURL)
+ if err != nil {
+ return ""
+ }
+ if input.Host == "" {
+ newinput, err := url.Parse("https://" + inputURL)
+ if err != nil {
+ return ""
+ }
+ return newinput.Host
+ }
+ return input.Host
+}
diff --git a/v2/pkg/utils/utils_test.go b/v2/pkg/utils/utils_test.go
index ede9d530..49597214 100644
--- a/v2/pkg/utils/utils_test.go
+++ b/v2/pkg/utils/utils_test.go
@@ -19,3 +19,24 @@ func TestUnwrapError(t *testing.T) {
errThree := fmt.Errorf("error with error: %w", errTwo)
require.Equal(t, errOne, UnwrapError(errThree))
}
+
+func TestParseURL(t *testing.T) {
+ testcases := []struct {
+ URL string
+ Hostname string
+ }{
+ {"https://scanme.sh:443", "scanme.sh:443"},
+ {"http://scanme.sh/path", "scanme.sh"},
+ {"scanme.sh:443/path", "scanme.sh:443"},
+ {"scanme.sh/path", "scanme.sh"},
+ }
+ for _, v := range testcases {
+ urlx := ParseHostname(v.URL)
+ if urlx == "" {
+ t.Errorf("failed to hostname of url %v", v)
+ }
+ if urlx != v.Hostname {
+ t.Errorf("hostname mismatch expected scanme.sh got %v", urlx)
+ }
+ }
+}