mirror of https://github.com/daffainfo/nuclei.git
handle 1 more edgecase (#4868)
* handle 1 more edgecase * add integration test for this edgecase * fix multi-http-var-sharing with integration test * add -payload-concurrency (-pc) flag * fix missing internal:true login in multiprotocol engine * fix/handle absolute invalid url parsing * support -pc & -jc in go sdk * fix missing variables in code protocol operators * add payload count parallelhttp checkdev
Tarun Koyalwar
GitHub
parent
a66b56fc79
commit
49ef5cbf16
|
|
@ -80,6 +80,55 @@ var httpTestcases = []TestCaseInfo{
|
|||
{Path: "protocols/http/disable-path-automerge.yaml", TestCase: &httpDisablePathAutomerge{}},
|
||||
{Path: "protocols/http/http-preprocessor.yaml", TestCase: &httpPreprocessor{}},
|
||||
{Path: "protocols/http/multi-request.yaml", TestCase: &httpMultiRequest{}},
|
||||
{Path: "protocols/http/http-matcher-extractor-dy-extractor.yaml", TestCase: &httpMatcherExtractorDynamicExtractor{}},
|
||||
{Path: "protocols/http/multi-http-var-sharing.yaml", TestCase: &httpMultiVarSharing{}},
|
||||
}
|
||||
|
||||
type httpMultiVarSharing struct{}
|
||||
|
||||
func (h *httpMultiVarSharing) Execute(filePath string) error {
|
||||
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "https://scanme.sh", debug)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return expectResultsCount(results, 1)
|
||||
}
|
||||
|
||||
type httpMatcherExtractorDynamicExtractor struct{}
|
||||
|
||||
func (h *httpMatcherExtractorDynamicExtractor) Execute(filePath string) error {
|
||||
router := httprouter.New()
|
||||
router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
|
||||
html := `<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<body>
|
||||
<a href="/domains">Domains</a>
|
||||
</body>
|
||||
</html>`
|
||||
fmt.Fprint(w, html)
|
||||
})
|
||||
router.GET("/domains", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
|
||||
html := `<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<title>Dynamic Extractor Test</title>
|
||||
</head>
|
||||
<body>
|
||||
<!-- The content of the title tag matches the regex pattern for both the extractor and matcher for 'title' -->
|
||||
</body>
|
||||
</html>
|
||||
`
|
||||
fmt.Fprint(w, html)
|
||||
})
|
||||
ts := httptest.NewServer(router)
|
||||
defer ts.Close()
|
||||
|
||||
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return expectResultsCount(results, 1)
|
||||
}
|
||||
|
||||
type httpInteractshRequest struct{}
|
||||
|
|
@ -776,8 +825,18 @@ func (h *httpPaths) Execute(filepath string) error {
|
|||
}
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(expected, actual) {
|
||||
return fmt.Errorf("%8v: %v\n%-8v: %v", "expected", expected, "actual", actual)
|
||||
if len(expected) > len(actual) {
|
||||
actualValuesIndex := len(actual) - 1
|
||||
if actualValuesIndex < 0 {
|
||||
actualValuesIndex = 0
|
||||
}
|
||||
return fmt.Errorf("missing values : %v", expected[actualValuesIndex:])
|
||||
} else if len(expected) < len(actual) {
|
||||
return fmt.Errorf("unexpected values : %v", actual[len(expected)-1:])
|
||||
} else {
|
||||
if !reflect.DeepEqual(expected, actual) {
|
||||
return fmt.Errorf("expected: %v\n\nactual: %v", expected, actual)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -322,6 +322,7 @@ on extensive configurability, massive extensibility and ease of use.`)
|
|||
flagSet.IntVarP(&options.HeadlessBulkSize, "headless-bulk-size", "hbs", 10, "maximum number of headless hosts to be analyzed in parallel per template"),
|
||||
flagSet.IntVarP(&options.HeadlessTemplateThreads, "headless-concurrency", "headc", 10, "maximum number of headless templates to be executed in parallel"),
|
||||
flagSet.IntVarP(&options.JsConcurrency, "js-concurrency", "jsc", 120, "maximum number of javascript runtimes to be executed in parallel"),
|
||||
flagSet.IntVarP(&options.PayloadConcurrency, "payload-concurrency", "pc", 25, "max payload concurrency for each template"),
|
||||
)
|
||||
flagSet.CreateGroup("optimization", "Optimizations",
|
||||
flagSet.IntVar(&options.Timeout, "timeout", 10, "time to wait in seconds before timeout"),
|
||||
|
|
|
|||
7
go.mod
7
go.mod
|
|
@ -20,12 +20,12 @@ require (
|
|||
github.com/olekukonko/tablewriter v0.0.5
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/projectdiscovery/clistats v0.0.20
|
||||
github.com/projectdiscovery/fastdialer v0.0.61
|
||||
github.com/projectdiscovery/fastdialer v0.0.62
|
||||
github.com/projectdiscovery/hmap v0.0.41
|
||||
github.com/projectdiscovery/interactsh v1.1.9
|
||||
github.com/projectdiscovery/rawhttp v0.1.40
|
||||
github.com/projectdiscovery/retryabledns v1.0.58
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.50
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.51
|
||||
github.com/projectdiscovery/yamldoc-go v1.0.4
|
||||
github.com/remeh/sizedwaitgroup v1.0.0
|
||||
github.com/rs/xid v1.5.0
|
||||
|
|
@ -91,7 +91,7 @@ require (
|
|||
github.com/projectdiscovery/tlsx v1.1.6
|
||||
github.com/projectdiscovery/uncover v1.0.7
|
||||
github.com/projectdiscovery/useragent v0.0.40
|
||||
github.com/projectdiscovery/utils v0.0.82
|
||||
github.com/projectdiscovery/utils v0.0.84-0.20240312214300-d3ba70dbb9ca
|
||||
github.com/projectdiscovery/wappalyzergo v0.0.112
|
||||
github.com/redis/go-redis/v9 v9.1.0
|
||||
github.com/sashabaranov/go-openai v1.15.3
|
||||
|
|
@ -274,7 +274,6 @@ require (
|
|||
github.com/tklauser/numcpus v0.6.0 // indirect
|
||||
github.com/trivago/tgo v1.0.7
|
||||
github.com/ulikunitz/xz v0.5.11 // indirect
|
||||
github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 // indirect
|
||||
github.com/valyala/bytebufferpool v1.0.0 // indirect
|
||||
github.com/yl2chen/cidranger v1.0.2 // indirect
|
||||
github.com/ysmood/goob v0.4.0 // indirect
|
||||
|
|
|
|||
14
go.sum
14
go.sum
|
|
@ -812,8 +812,8 @@ github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPo
|
|||
github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
|
||||
github.com/projectdiscovery/dsl v0.0.46 h1:zBNNzSBA1aakGY44w6KhnjJQx/zO+oW2Wx7TR8xZm/A=
|
||||
github.com/projectdiscovery/dsl v0.0.46/go.mod h1:eoZEJFCIT5emI00xj8HTQwHz4YwwGAD+grL3G7CDlfs=
|
||||
github.com/projectdiscovery/fastdialer v0.0.61 h1:z5OzP9lRbn6fSIezgReKC3hkzRh+YX41ST9OgkVEm/s=
|
||||
github.com/projectdiscovery/fastdialer v0.0.61/go.mod h1:FyxJ0m1MwB69nLmdXYqK32f3a0Pf+5YpC8wBY73baiE=
|
||||
github.com/projectdiscovery/fastdialer v0.0.62 h1:Wyba2hD6ZF3S04MgCn380mC+1RXJ+dq14Yq8u2yk7ps=
|
||||
github.com/projectdiscovery/fastdialer v0.0.62/go.mod h1:2baj2TRXTw+hHbKTW9IZR4dhpxCGJkq5AKL1ge5gis8=
|
||||
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
|
||||
github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
|
||||
github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
|
||||
|
|
@ -852,8 +852,8 @@ github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gB
|
|||
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
|
||||
github.com/projectdiscovery/retryabledns v1.0.58 h1:ut1FSB9+GZ6zQIlKJFLqIz2RZs81EmkbsHTuIrWfYLE=
|
||||
github.com/projectdiscovery/retryabledns v1.0.58/go.mod h1:RobmKoNBgngAVE4H9REQtaLP1pa4TCyypHy1MWHT1mY=
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.50 h1:QiVqNzpDFoTZm2z1oO68FEtl2VTk2kFscPetxmCK2gc=
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.50/go.mod h1:yY/go76Lx/MgJmR+dSg/Xa8wOiXNjmuEdBDVDODKeDk=
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.51 h1:8XMrNC8JrwvySESe2d+XWF9bq4unWqD4PUPEC4Cai8s=
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.51/go.mod h1:6cdh/acYHpeYWg7+Iblh4xBRb87bC118L4G4mpvCMuA=
|
||||
github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
|
||||
github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ=
|
||||
github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
|
||||
|
|
@ -864,8 +864,8 @@ github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7
|
|||
github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE=
|
||||
github.com/projectdiscovery/useragent v0.0.40 h1:1LUhReSGPkhqsM5n40OOC9dIoNqMGs1dyGFJcOmg2Fo=
|
||||
github.com/projectdiscovery/useragent v0.0.40/go.mod h1:EvK1x3s948Gtqb/XOahXcauyejCL/rSgy5d1IAvsKT4=
|
||||
github.com/projectdiscovery/utils v0.0.82 h1:U//02floCSFxJluN7MP+rJSwI4Px7o454JL7ukERArI=
|
||||
github.com/projectdiscovery/utils v0.0.82/go.mod h1:AbmIvy0TTlsfXxPDEMaNPVrxmqDmYiCnbGqh0TTthE4=
|
||||
github.com/projectdiscovery/utils v0.0.84-0.20240312214300-d3ba70dbb9ca h1:GY9lUYDlENXPSFPJH01Bm1BfhrUF2jpnUBR+K4VPJIs=
|
||||
github.com/projectdiscovery/utils v0.0.84-0.20240312214300-d3ba70dbb9ca/go.mod h1:wzMfHBq2I9oy+DEiMfUYV86g1D7eXKaQsgWnqFpmMtI=
|
||||
github.com/projectdiscovery/wappalyzergo v0.0.112 h1:QPpp5jmj1lqLd5mFdFKQ9VvcYhQNqyU9Mr+IB0US2zA=
|
||||
github.com/projectdiscovery/wappalyzergo v0.0.112/go.mod h1:hc/o+fgM8KtdpFesjfBTmHTwsR+yBd+4kYZW/DGy/x8=
|
||||
github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
|
||||
|
|
@ -1034,8 +1034,6 @@ github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oW
|
|||
github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
|
||||
github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8=
|
||||
github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
|
||||
github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 h1:TtyC78WMafNW8QFfv3TeP3yWNDG+uxNkk9vOrnDu6JA=
|
||||
github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6/go.mod h1:h8272+G2omSmi30fBXiZDMkmHuOgonplfKIKjQWzlfs=
|
||||
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
|
||||
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
|
||||
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
|
||||
|
|
|
|||
|
|
@ -20,4 +20,4 @@ code:
|
|||
- type: word
|
||||
words:
|
||||
- "hello from input baz"
|
||||
# digest: 4a0a00473045022100b290a0c40f27573f0de9a950be13457a9bf59ade1ff2f497bf01a3b526e5db750220761942acffd6d27e2714ddaa1c73c699ccd7de48839f08cff1d6a9456bc8ff1f:4a3eb6b4988d95847d4203be25ed1d46
|
||||
# digest: 4a0a0047304502207e3a5eda5f3207c3c01c820562243281926c1215224a7c80ed7528559b9f52cb022100f6ef99bb45843f481705778630f2cfd8f4d1cc3acb96392ff016f22e06aa91af:4a3eb6b4988d95847d4203be25ed1d46
|
||||
|
|
@ -18,4 +18,4 @@ code:
|
|||
- type: word
|
||||
words:
|
||||
- "hello from input"
|
||||
# digest: 490a004630440220335663a6a4db720ee6276ab7179a87a6be0b4030771ec5ee82ecf6982342113602200a2570db7eb9721f6ceb1a89543fc436ee62b30d1b720c75ea3834ed3d2b64f3:4a3eb6b4988d95847d4203be25ed1d46
|
||||
# digest: 4a0a004730450220069673af9bd6d6677f9529d06f5d8bd46d543089a4731ed18ee806761d75fd60022100913a3e27b0a5809baf710ba9585bf9fe729634c0e19e3e13eef70a6bd100df34:4a3eb6b4988d95847d4203be25ed1d46
|
||||
|
|
@ -26,4 +26,4 @@ code:
|
|||
part: interactsh_protocol
|
||||
words:
|
||||
- "http"
|
||||
# digest: 490a004630440220400892730a62fa1bbb1064e4d88eea760dbf8f01c6b630ff0f5b126fd1952839022025a6d52e730c1f1cfcbd440e6269f93489db3a77cb2a27d0f47522c0819dc8d3:4a3eb6b4988d95847d4203be25ed1d46
|
||||
# digest: 490a00463044022003b8d069e3c84412729c43e33013a52ee04eabcf096d511979691d71d8e905f60220011f4475899abed4f86b4bd5e6c2423750759135206e4729826afe1ed8a44f4d:4a3eb6b4988d95847d4203be25ed1d46
|
||||
|
|
@ -21,4 +21,4 @@ code:
|
|||
- type: word
|
||||
words:
|
||||
- "hello from input"
|
||||
# digest: 490a0046304402206b14abdc0d5fc13466f5c292da9fb2a19d1b2c5e683cc052037fe367b372f82b02202c00b9acbd8106a769eb411794c567d3019433671397bf909e16b286105ed69e:4a3eb6b4988d95847d4203be25ed1d46
|
||||
# digest: 4a0a00473045022100c291615cf2a8005450c17a6554e81a9cdab14743b299f0679c644247929198b502206fdacc8ab173bde2b4015340012637916bf2659f66f320fcc06b97ac639072a1:4a3eb6b4988d95847d4203be25ed1d46
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
id: http-matcher-extractor-dy-extractor
|
||||
info:
|
||||
name: HTTP matcher and extractor & dynamic extractor
|
||||
description: >
|
||||
Edgecase to test for a combination of matchers , extractors and dynamic extractors
|
||||
author: pdteam
|
||||
severity: info
|
||||
|
||||
http:
|
||||
- raw:
|
||||
- |
|
||||
GET {{BaseURL}} HTTP/1.1
|
||||
- |
|
||||
GET {{absolutePath}} HTTP/1.1
|
||||
|
||||
req-condition: true
|
||||
extractors:
|
||||
- type: regex
|
||||
internal: true
|
||||
part: body_1
|
||||
name: absolutePath
|
||||
regex:
|
||||
- '<a href="(/domains)">'
|
||||
group: 1
|
||||
- type: regex
|
||||
internal: false
|
||||
part: body_2
|
||||
name: title
|
||||
regex:
|
||||
- '<title[^>]*>([^<]+)</title>'
|
||||
group: 1
|
||||
matchers:
|
||||
- type: regex
|
||||
part: body_2
|
||||
regex:
|
||||
- '<title[^>]*>([^<]+)</title>'
|
||||
|
|
@ -28,7 +28,7 @@ info:
|
|||
- "//CFIDE/wizards/common/utils.cfc"
|
||||
|
||||
# Test all templates with FullURLs
|
||||
requests:
|
||||
http:
|
||||
- raw:
|
||||
# relative path without leading slash with param
|
||||
# If relative path does not have `/` prefix it is autocorrected
|
||||
|
|
|
|||
|
|
@ -0,0 +1,36 @@
|
|||
id: multi-http-var-sharing
|
||||
|
||||
info:
|
||||
name: Multi HTTP var sharing
|
||||
author: pdteam
|
||||
severity: info
|
||||
description: |
|
||||
A template which has multiple HTTP requests block and variables are shared between them
|
||||
|
||||
http:
|
||||
- method: GET
|
||||
path:
|
||||
- "{{BaseURL}}"
|
||||
|
||||
matchers:
|
||||
- type: word
|
||||
words:
|
||||
- "This is test matcher text"
|
||||
negative: true
|
||||
internal: true
|
||||
|
||||
extractors:
|
||||
- type: dsl
|
||||
name: ffff
|
||||
dsl:
|
||||
- status_code
|
||||
internal: true
|
||||
|
||||
- method: GET
|
||||
path:
|
||||
- "{{BaseURL}}/{{ffff}}"
|
||||
|
||||
matchers:
|
||||
- type: status
|
||||
status:
|
||||
- 200
|
||||
|
|
@ -107,10 +107,12 @@ func WithInteractshOptions(opts InteractshOpts) NucleiSDKOptions {
|
|||
|
||||
// Concurrency options
|
||||
type Concurrency struct {
|
||||
TemplateConcurrency int // number of templates to run concurrently (per host in host-spray mode)
|
||||
HostConcurrency int // number of hosts to scan concurrently (per template in template-spray mode)
|
||||
HeadlessHostConcurrency int // number of hosts to scan concurrently for headless templates (per template in template-spray mode)
|
||||
HeadlessTemplateConcurrency int // number of templates to run concurrently for headless templates (per host in host-spray mode)
|
||||
TemplateConcurrency int // number of templates to run concurrently (per host in host-spray mode)
|
||||
HostConcurrency int // number of hosts to scan concurrently (per template in template-spray mode)
|
||||
HeadlessHostConcurrency int // number of hosts to scan concurrently for headless templates (per template in template-spray mode)
|
||||
HeadlessTemplateConcurrency int // number of templates to run concurrently for headless templates (per host in host-spray mode)
|
||||
JavascriptTemplateConcurrency int // number of templates to run concurrently for javascript templates (per host in host-spray mode)
|
||||
TemplatePayloadConcurrency int // max concurrent payloads to run for a template (a good default is 25)
|
||||
}
|
||||
|
||||
// WithConcurrency sets concurrency options
|
||||
|
|
@ -120,6 +122,8 @@ func WithConcurrency(opts Concurrency) NucleiSDKOptions {
|
|||
e.opts.BulkSize = opts.HostConcurrency
|
||||
e.opts.HeadlessBulkSize = opts.HeadlessHostConcurrency
|
||||
e.opts.HeadlessTemplateThreads = opts.HeadlessTemplateConcurrency
|
||||
e.opts.JsConcurrency = opts.JavascriptTemplateConcurrency
|
||||
e.opts.PayloadConcurrency = opts.TemplatePayloadConcurrency
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -299,7 +299,7 @@ func (operators *Operators) Execute(data map[string]interface{}, match MatchFunc
|
|||
if len(result.DynamicValues) > 0 {
|
||||
return result, true
|
||||
}
|
||||
return nil, false
|
||||
return result, false
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -313,6 +313,10 @@ func (operators *Operators) Execute(data map[string]interface{}, match MatchFunc
|
|||
|
||||
// Don't print if we have matchers, and they have not matched, regardless of extractor
|
||||
if len(operators.Matchers) > 0 && !matches {
|
||||
// if dynamic values are present then it is not a failure
|
||||
if len(result.DynamicValues) > 0 {
|
||||
return result, true
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
// Write a final string of output if matcher type is
|
||||
|
|
@ -320,6 +324,10 @@ func (operators *Operators) Execute(data map[string]interface{}, match MatchFunc
|
|||
if len(result.Extracts) > 0 || len(result.OutputExtracts) > 0 || matches {
|
||||
return result, true
|
||||
}
|
||||
// if dynamic values are present then it is not a failure
|
||||
if len(result.DynamicValues) > 0 {
|
||||
return result, true
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -192,6 +192,10 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
|
|||
dataOutputString := fmtStdout(gOutput.Stdout.String())
|
||||
|
||||
data := make(output.InternalEvent)
|
||||
// also include all request variables in result event
|
||||
for _, value := range metaSrc.Variables {
|
||||
data[value.Name] = value.Value
|
||||
}
|
||||
|
||||
data["type"] = request.Type().String()
|
||||
data["response"] = dataOutputString // response contains filtered output (eg without trailing \n)
|
||||
|
|
|
|||
|
|
@ -97,7 +97,7 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context,
|
|||
}
|
||||
|
||||
// Parse target url
|
||||
parsed, err := urlutil.Parse(input.MetaInput.Input)
|
||||
parsed, err := urlutil.ParseAbsoluteURL(input.MetaInput.Input, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -153,7 +153,7 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context,
|
|||
return r.generateRawRequest(ctx, reqData, parsed, finalVars, payloads)
|
||||
}
|
||||
|
||||
reqURL, err := urlutil.ParseURL(reqData, true)
|
||||
reqURL, err := urlutil.ParseAbsoluteURL(reqData, true)
|
||||
if err != nil {
|
||||
return nil, errorutil.NewWithTag("http", "failed to parse url %v while creating http request", reqData)
|
||||
}
|
||||
|
|
@ -291,8 +291,7 @@ func (r *requestGenerator) generateRawRequest(ctx context.Context, rawRequest st
|
|||
unsafeReq := &generatedRequest{rawRequest: rawRequestData, meta: generatorValues, original: r.request, interactshURLs: r.interactshURLs}
|
||||
return unsafeReq, nil
|
||||
}
|
||||
|
||||
urlx, err := urlutil.ParseURL(rawRequestData.FullURL, true)
|
||||
urlx, err := urlutil.ParseAbsoluteURL(rawRequestData.FullURL, true)
|
||||
if err != nil {
|
||||
return nil, errorutil.NewWithErr(err).Msgf("failed to create request with url %v got %v", rawRequestData.FullURL, err).WithTag("raw")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ import (
|
|||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/projectdiscovery/rawhttp"
|
||||
convUtil "github.com/projectdiscovery/utils/conversion"
|
||||
errorutil "github.com/projectdiscovery/utils/errors"
|
||||
httpUtils "github.com/projectdiscovery/utils/http"
|
||||
"github.com/projectdiscovery/utils/reader"
|
||||
sliceutil "github.com/projectdiscovery/utils/slice"
|
||||
|
|
@ -453,7 +454,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
|
|||
}
|
||||
|
||||
// verify if parallel elaboration was requested
|
||||
if request.Threads > 0 {
|
||||
if request.Threads > 0 && len(request.Payloads) > 0 {
|
||||
return request.executeParallelHTTP(input, dynamicValues, callback)
|
||||
}
|
||||
|
||||
|
|
@ -494,7 +495,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
|
|||
return true, nil
|
||||
}
|
||||
var gotMatches bool
|
||||
err = request.executeRequest(input, generatedHttpRequest, previous, hasInteractMatchers, func(event *output.InternalWrappedEvent) {
|
||||
execReqErr := request.executeRequest(input, generatedHttpRequest, previous, hasInteractMatchers, func(event *output.InternalWrappedEvent) {
|
||||
// a special case where operators has interactsh matchers and multiple request are made
|
||||
// ex: status_code_2 , interactsh_protocol (from 1st request) etc
|
||||
needsRequestEvent := interactsh.HasMatchers(request.CompiledOperators) && request.NeedsRequestCondition()
|
||||
|
|
@ -525,14 +526,14 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
|
|||
}, generator.currentIndex)
|
||||
|
||||
// If a variable is unresolved, skip all further requests
|
||||
if errors.Is(err, errStopExecution) {
|
||||
if errors.Is(execReqErr, errStopExecution) {
|
||||
return true, nil
|
||||
}
|
||||
if err != nil {
|
||||
if execReqErr != nil {
|
||||
if request.options.HostErrorsCache != nil {
|
||||
request.options.HostErrorsCache.MarkFailed(input.MetaInput.ID(), err)
|
||||
}
|
||||
requestErr = err
|
||||
requestErr = errorutil.NewWithErr(execReqErr).Msgf("got err while executing %v", generatedHttpRequest.URL())
|
||||
}
|
||||
request.options.Progress.IncrementRequests()
|
||||
|
||||
|
|
|
|||
|
|
@ -6,13 +6,22 @@ import (
|
|||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
|
||||
"github.com/projectdiscovery/rawhttp"
|
||||
errorutil "github.com/projectdiscovery/utils/errors"
|
||||
)
|
||||
|
||||
// dump creates a dump of the http request in form of a byte slice
|
||||
func dump(req *generatedRequest, reqURL string) ([]byte, error) {
|
||||
if req.request != nil {
|
||||
return req.request.Dump()
|
||||
bin, err := req.request.Dump()
|
||||
if err != nil {
|
||||
return nil, errorutil.NewWithErr(err).WithTag("http").Msgf("could not dump request: %v", req.request.URL.String())
|
||||
}
|
||||
return bin, nil
|
||||
}
|
||||
rawHttpOptions := &rawhttp.Options{CustomHeaders: req.rawRequest.UnsafeHeaders, CustomRawBytes: req.rawRequest.UnsafeRawBytes}
|
||||
return rawhttp.DumpRequestRaw(req.rawRequest.Method, reqURL, req.rawRequest.Path, generators.ExpandMapValues(req.rawRequest.Headers), io.NopCloser(strings.NewReader(req.rawRequest.Data)), rawHttpOptions)
|
||||
bin, err := rawhttp.DumpRequestRaw(req.rawRequest.Method, reqURL, req.rawRequest.Path, generators.ExpandMapValues(req.rawRequest.Headers), io.NopCloser(strings.NewReader(req.rawRequest.Data)), rawHttpOptions)
|
||||
if err != nil {
|
||||
return nil, errorutil.NewWithErr(err).WithTag("http").Msgf("could not dump request: %v", reqURL)
|
||||
}
|
||||
return bin, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -129,7 +129,7 @@ func (e *ExecutorOptions) GetThreadsForNPayloadRequests(totalRequests int, curre
|
|||
if currentThreads > 0 {
|
||||
return currentThreads
|
||||
} else {
|
||||
return e.Options.TemplateThreads
|
||||
return e.Options.PayloadConcurrency
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ package tmplexec
|
|||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"runtime/debug"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
|
||||
|
|
@ -34,16 +35,6 @@ var _ protocols.Executer = &TemplateExecuter{}
|
|||
|
||||
// NewTemplateExecuter creates a new request TemplateExecuter for list of requests
|
||||
func NewTemplateExecuter(requests []protocols.Request, options *protocols.ExecutorOptions) (*TemplateExecuter, error) {
|
||||
isMultiProto := false
|
||||
lastProto := ""
|
||||
for _, request := range requests {
|
||||
if request.Type().String() != lastProto && lastProto != "" {
|
||||
isMultiProto = true
|
||||
break
|
||||
}
|
||||
lastProto = request.Type().String()
|
||||
}
|
||||
|
||||
e := &TemplateExecuter{requests: requests, options: options, results: &atomic.Bool{}}
|
||||
if options.Flow != "" {
|
||||
// we use a dummy input here because goal of flow executor at this point is to just check
|
||||
|
|
@ -55,13 +46,11 @@ func NewTemplateExecuter(requests []protocols.Request, options *protocols.Execut
|
|||
}
|
||||
e.program = p
|
||||
} else {
|
||||
// Review:
|
||||
// multiproto engine is only used if there is more than one protocol in template
|
||||
// else we use generic engine (should we use multiproto engine for single protocol with multiple requests as well ?)
|
||||
if isMultiProto {
|
||||
e.engine = multiproto.NewMultiProtocol(requests, options, e.results)
|
||||
} else {
|
||||
// only use generic if there is only 1 protocol with only 1 section
|
||||
if len(requests) == 1 {
|
||||
e.engine = generic.NewGenericEngine(requests, options, e.results)
|
||||
} else {
|
||||
e.engine = multiproto.NewMultiProtocol(requests, options, e.results)
|
||||
}
|
||||
}
|
||||
return e, nil
|
||||
|
|
@ -113,8 +102,9 @@ func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) {
|
|||
defer func() {
|
||||
// try catching unknown panics
|
||||
if r := recover(); r != nil {
|
||||
ctx.LogError(fmt.Errorf("panic: %v", r))
|
||||
gologger.Verbose().Msgf("panic: %v", r)
|
||||
stacktrace := debug.Stack()
|
||||
ctx.LogError(fmt.Errorf("panic: %v\n%s", r, stacktrace))
|
||||
gologger.Verbose().Msgf("panic: %v\n%s", r, stacktrace)
|
||||
}
|
||||
}()
|
||||
|
||||
|
|
|
|||
|
|
@ -46,12 +46,13 @@ func (m *MultiProtocol) Compile() error {
|
|||
func (m *MultiProtocol) ExecuteWithResults(ctx *scan.ScanContext) error {
|
||||
// put all readonly args into template context
|
||||
m.options.GetTemplateCtx(ctx.Input.MetaInput).Merge(m.readOnlyArgs)
|
||||
var finalProtoEvent *output.InternalWrappedEvent
|
||||
// callback to process results from all protocols
|
||||
multiProtoCallback := func(event *output.InternalWrappedEvent) {
|
||||
if event != nil {
|
||||
finalProtoEvent = event
|
||||
if event == nil {
|
||||
return
|
||||
}
|
||||
// log event and generate result for the event
|
||||
ctx.LogEvent(event)
|
||||
// export dynamic values from operators (i.e internal:true)
|
||||
if event.OperatorsResult != nil && len(event.OperatorsResult.DynamicValues) > 0 {
|
||||
for k, v := range event.OperatorsResult.DynamicValues {
|
||||
|
|
@ -97,12 +98,6 @@ func (m *MultiProtocol) ExecuteWithResults(ctx *scan.ScanContext) error {
|
|||
return err
|
||||
}
|
||||
}
|
||||
// Review: how to handle events of multiple protocols in a single template
|
||||
// currently the outer callback is only executed once (for the last protocol in queue)
|
||||
// due to workflow logic at https://github.com/projectdiscovery/nuclei/blob/main/pkg/protocols/common/executer/executem.go#L150
|
||||
// this causes addition of duplicated / unncessary variables with prefix template_id_all_variables
|
||||
ctx.LogEvent(finalProtoEvent)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -370,6 +370,8 @@ type Options struct {
|
|||
ScanID string
|
||||
// JsConcurrency is the number of concurrent js routines to run
|
||||
JsConcurrency int
|
||||
// PayloadConcurrency is the number of concurrent payloads to run per template
|
||||
PayloadConcurrency int
|
||||
}
|
||||
|
||||
// ShouldLoadResume resume file
|
||||
|
|
|
|||
Loading…
Reference in New Issue