daffainfo
/
nuclei
mirror of https://github.com/daffainfo/nuclei.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' into maint-runner-cache

dev
mzack 2024-03-13 20:16:27 +01:00
parent fd14472181 66b722faaf
commit 3685379960
21 changed files with 207 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
cmd/integration-test/http.go
View File

@ -80,6 +80,55 @@ var httpTestcases = []TestCaseInfo{
{Path: "protocols/http/disable-path-automerge.yaml", TestCase: &httpDisablePathAutomerge{}}, {Path: "protocols/http/disable-path-automerge.yaml", TestCase: &httpDisablePathAutomerge{}},
{Path: "protocols/http/http-preprocessor.yaml", TestCase: &httpPreprocessor{}}, {Path: "protocols/http/http-preprocessor.yaml", TestCase: &httpPreprocessor{}},
{Path: "protocols/http/multi-request.yaml", TestCase: &httpMultiRequest{}}, {Path: "protocols/http/multi-request.yaml", TestCase: &httpMultiRequest{}},
{Path: "protocols/http/http-matcher-extractor-dy-extractor.yaml", TestCase: &httpMatcherExtractorDynamicExtractor{}},
{Path: "protocols/http/multi-http-var-sharing.yaml", TestCase: &httpMultiVarSharing{}},
}
type httpMultiVarSharing struct{}
func (h *httpMultiVarSharing) Execute(filePath string) error {
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "https://scanme.sh", debug)
if err != nil {
return err
}
return expectResultsCount(results, 1)
}
type httpMatcherExtractorDynamicExtractor struct{}
func (h *httpMatcherExtractorDynamicExtractor) Execute(filePath string) error {
router := httprouter.New()
router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
html := `<!DOCTYPE html>
<html lang="en">
<body>
<a href="/domains">Domains</a>
</body>
</html>`
fmt.Fprint(w, html)
})
router.GET("/domains", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
html := `<!DOCTYPE html>
<html lang="en">
<head>
<title>Dynamic Extractor Test</title>
</head>
<body>
<!-- The content of the title tag matches the regex pattern for both the extractor and matcher for 'title' -->
</body>
</html>
`
fmt.Fprint(w, html)
})
ts := httptest.NewServer(router)
defer ts.Close()
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
if err != nil {
return err
}
return expectResultsCount(results, 1)
} }
type httpInteractshRequest struct{} type httpInteractshRequest struct{}
@ -776,8 +825,18 @@ func (h *httpPaths) Execute(filepath string) error {
} }
} }
if !reflect.DeepEqual(expected, actual) { if len(expected) > len(actual) {
return fmt.Errorf("%8v: %v\n%-8v: %v", "expected", expected, "actual", actual) actualValuesIndex := len(actual) - 1
if actualValuesIndex < 0 {
actualValuesIndex = 0
}
return fmt.Errorf("missing values : %v", expected[actualValuesIndex:])
} else if len(expected) < len(actual) {
return fmt.Errorf("unexpected values : %v", actual[len(expected)-1:])
} else {
if !reflect.DeepEqual(expected, actual) {
return fmt.Errorf("expected: %v\n\nactual: %v", expected, actual)
}
} }
return nil return nil
} }

1
cmd/nuclei/main.go
View File

@ -322,6 +322,7 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.IntVarP(&options.HeadlessBulkSize, "headless-bulk-size", "hbs", 10, "maximum number of headless hosts to be analyzed in parallel per template"), flagSet.IntVarP(&options.HeadlessBulkSize, "headless-bulk-size", "hbs", 10, "maximum number of headless hosts to be analyzed in parallel per template"),
flagSet.IntVarP(&options.HeadlessTemplateThreads, "headless-concurrency", "headc", 10, "maximum number of headless templates to be executed in parallel"), flagSet.IntVarP(&options.HeadlessTemplateThreads, "headless-concurrency", "headc", 10, "maximum number of headless templates to be executed in parallel"),
flagSet.IntVarP(&options.JsConcurrency, "js-concurrency", "jsc", 120, "maximum number of javascript runtimes to be executed in parallel"), flagSet.IntVarP(&options.JsConcurrency, "js-concurrency", "jsc", 120, "maximum number of javascript runtimes to be executed in parallel"),
flagSet.IntVarP(&options.PayloadConcurrency, "payload-concurrency", "pc", 25, "max payload concurrency for each template"),
) )
flagSet.CreateGroup("optimization", "Optimizations", flagSet.CreateGroup("optimization", "Optimizations",
flagSet.IntVar(&options.Timeout, "timeout", 10, "time to wait in seconds before timeout"), flagSet.IntVar(&options.Timeout, "timeout", 10, "time to wait in seconds before timeout"),

9
go.mod
View File

@ -20,12 +20,12 @@ require (
github.com/olekukonko/tablewriter v0.0.5 github.com/olekukonko/tablewriter v0.0.5
github.com/pkg/errors v0.9.1 github.com/pkg/errors v0.9.1
github.com/projectdiscovery/clistats v0.0.20 github.com/projectdiscovery/clistats v0.0.20
github.com/projectdiscovery/fastdialer v0.0.61 github.com/projectdiscovery/fastdialer v0.0.62
github.com/projectdiscovery/hmap v0.0.41 github.com/projectdiscovery/hmap v0.0.41
github.com/projectdiscovery/interactsh v1.1.9 github.com/projectdiscovery/interactsh v1.1.9
github.com/projectdiscovery/rawhttp v0.1.40 github.com/projectdiscovery/rawhttp v0.1.40
github.com/projectdiscovery/retryabledns v1.0.58 github.com/projectdiscovery/retryabledns v1.0.58
github.com/projectdiscovery/retryablehttp-go v1.0.50 github.com/projectdiscovery/retryablehttp-go v1.0.51
github.com/projectdiscovery/yamldoc-go v1.0.4 github.com/projectdiscovery/yamldoc-go v1.0.4
github.com/remeh/sizedwaitgroup v1.0.0 github.com/remeh/sizedwaitgroup v1.0.0
github.com/rs/xid v1.5.0 github.com/rs/xid v1.5.0
@ -81,7 +81,7 @@ require (
github.com/projectdiscovery/goflags v0.1.42 github.com/projectdiscovery/goflags v0.1.42
github.com/projectdiscovery/gologger v1.1.12 github.com/projectdiscovery/gologger v1.1.12
github.com/projectdiscovery/gostruct v0.0.2 github.com/projectdiscovery/gostruct v0.0.2
github.com/projectdiscovery/gozero v0.0.2-0.20240305085154-99aa5ddb9f98 github.com/projectdiscovery/gozero v0.0.2
github.com/projectdiscovery/httpx v1.6.0 github.com/projectdiscovery/httpx v1.6.0
github.com/projectdiscovery/mapcidr v1.1.16 github.com/projectdiscovery/mapcidr v1.1.16
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5
@ -91,7 +91,7 @@ require (
github.com/projectdiscovery/tlsx v1.1.6 github.com/projectdiscovery/tlsx v1.1.6
github.com/projectdiscovery/uncover v1.0.7 github.com/projectdiscovery/uncover v1.0.7
github.com/projectdiscovery/useragent v0.0.40 github.com/projectdiscovery/useragent v0.0.40
github.com/projectdiscovery/utils v0.0.82 github.com/projectdiscovery/utils v0.0.84-0.20240312214300-d3ba70dbb9ca
github.com/projectdiscovery/wappalyzergo v0.0.112 github.com/projectdiscovery/wappalyzergo v0.0.112
github.com/redis/go-redis/v9 v9.1.0 github.com/redis/go-redis/v9 v9.1.0
github.com/sashabaranov/go-openai v1.15.3 github.com/sashabaranov/go-openai v1.15.3
@ -274,7 +274,6 @@ require (
github.com/tklauser/numcpus v0.6.0 // indirect github.com/tklauser/numcpus v0.6.0 // indirect
github.com/trivago/tgo v1.0.7 github.com/trivago/tgo v1.0.7
github.com/ulikunitz/xz v0.5.11 // indirect github.com/ulikunitz/xz v0.5.11 // indirect
github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/yl2chen/cidranger v1.0.2 // indirect github.com/yl2chen/cidranger v1.0.2 // indirect
github.com/ysmood/goob v0.4.0 // indirect github.com/ysmood/goob v0.4.0 // indirect

16
go.sum
View File

@ -812,8 +812,8 @@ github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPo
github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4= github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
github.com/projectdiscovery/dsl v0.0.46 h1:zBNNzSBA1aakGY44w6KhnjJQx/zO+oW2Wx7TR8xZm/A= github.com/projectdiscovery/dsl v0.0.46 h1:zBNNzSBA1aakGY44w6KhnjJQx/zO+oW2Wx7TR8xZm/A=
github.com/projectdiscovery/dsl v0.0.46/go.mod h1:eoZEJFCIT5emI00xj8HTQwHz4YwwGAD+grL3G7CDlfs= github.com/projectdiscovery/dsl v0.0.46/go.mod h1:eoZEJFCIT5emI00xj8HTQwHz4YwwGAD+grL3G7CDlfs=
github.com/projectdiscovery/fastdialer v0.0.61 h1:z5OzP9lRbn6fSIezgReKC3hkzRh+YX41ST9OgkVEm/s= github.com/projectdiscovery/fastdialer v0.0.62 h1:Wyba2hD6ZF3S04MgCn380mC+1RXJ+dq14Yq8u2yk7ps=
github.com/projectdiscovery/fastdialer v0.0.61/go.mod h1:FyxJ0m1MwB69nLmdXYqK32f3a0Pf+5YpC8wBY73baiE= github.com/projectdiscovery/fastdialer v0.0.62/go.mod h1:2baj2TRXTw+hHbKTW9IZR4dhpxCGJkq5AKL1ge5gis8=
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw= github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q= github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
@ -828,6 +828,8 @@ github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBD
github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
github.com/projectdiscovery/gozero v0.0.2-0.20240305085154-99aa5ddb9f98 h1:KKS26wFrlcfPxKDmop+2NmI8HbGn8pgotHJBTh+3R4k= github.com/projectdiscovery/gozero v0.0.2-0.20240305085154-99aa5ddb9f98 h1:KKS26wFrlcfPxKDmop+2NmI8HbGn8pgotHJBTh+3R4k=
github.com/projectdiscovery/gozero v0.0.2-0.20240305085154-99aa5ddb9f98/go.mod h1:/dHwbly+1lhOX9UreVure4lEe7K4hIHeu/c/wZGNTDo= github.com/projectdiscovery/gozero v0.0.2-0.20240305085154-99aa5ddb9f98/go.mod h1:/dHwbly+1lhOX9UreVure4lEe7K4hIHeu/c/wZGNTDo=
github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3cjkilRQ+E=
github.com/projectdiscovery/gozero v0.0.2/go.mod h1:d8bZvDWW07LWNYWrwjZ4OO1I0cpkfqaysyDfSs9ibK8=
github.com/projectdiscovery/hmap v0.0.41 h1:8IgTyDce3/2JzcfPVA4H+XpBRFfETULx8td3BMdSYVE= github.com/projectdiscovery/hmap v0.0.41 h1:8IgTyDce3/2JzcfPVA4H+XpBRFfETULx8td3BMdSYVE=
github.com/projectdiscovery/hmap v0.0.41/go.mod h1:bCrai6x5Eijqm2U+jtcH0wZX5ZcaZhcvzoMGTZgLAf0= github.com/projectdiscovery/hmap v0.0.41/go.mod h1:bCrai6x5Eijqm2U+jtcH0wZX5ZcaZhcvzoMGTZgLAf0=
github.com/projectdiscovery/httpx v1.6.0 h1:6g4UoSQpsOyZgaK+SMLLnZIAU0eYyTxBUwVl+jtm0JQ= github.com/projectdiscovery/httpx v1.6.0 h1:6g4UoSQpsOyZgaK+SMLLnZIAU0eYyTxBUwVl+jtm0JQ=
@ -852,8 +854,8 @@ github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gB
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
github.com/projectdiscovery/retryabledns v1.0.58 h1:ut1FSB9+GZ6zQIlKJFLqIz2RZs81EmkbsHTuIrWfYLE= github.com/projectdiscovery/retryabledns v1.0.58 h1:ut1FSB9+GZ6zQIlKJFLqIz2RZs81EmkbsHTuIrWfYLE=
github.com/projectdiscovery/retryabledns v1.0.58/go.mod h1:RobmKoNBgngAVE4H9REQtaLP1pa4TCyypHy1MWHT1mY= github.com/projectdiscovery/retryabledns v1.0.58/go.mod h1:RobmKoNBgngAVE4H9REQtaLP1pa4TCyypHy1MWHT1mY=
github.com/projectdiscovery/retryablehttp-go v1.0.50 h1:QiVqNzpDFoTZm2z1oO68FEtl2VTk2kFscPetxmCK2gc= github.com/projectdiscovery/retryablehttp-go v1.0.51 h1:8XMrNC8JrwvySESe2d+XWF9bq4unWqD4PUPEC4Cai8s=
github.com/projectdiscovery/retryablehttp-go v1.0.50/go.mod h1:yY/go76Lx/MgJmR+dSg/Xa8wOiXNjmuEdBDVDODKeDk= github.com/projectdiscovery/retryablehttp-go v1.0.51/go.mod h1:6cdh/acYHpeYWg7+Iblh4xBRb87bC118L4G4mpvCMuA=
github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us= github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ= github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ=
github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
@ -864,8 +866,8 @@ github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7
github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE= github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE=
github.com/projectdiscovery/useragent v0.0.40 h1:1LUhReSGPkhqsM5n40OOC9dIoNqMGs1dyGFJcOmg2Fo= github.com/projectdiscovery/useragent v0.0.40 h1:1LUhReSGPkhqsM5n40OOC9dIoNqMGs1dyGFJcOmg2Fo=
github.com/projectdiscovery/useragent v0.0.40/go.mod h1:EvK1x3s948Gtqb/XOahXcauyejCL/rSgy5d1IAvsKT4= github.com/projectdiscovery/useragent v0.0.40/go.mod h1:EvK1x3s948Gtqb/XOahXcauyejCL/rSgy5d1IAvsKT4=
github.com/projectdiscovery/utils v0.0.82 h1:U//02floCSFxJluN7MP+rJSwI4Px7o454JL7ukERArI= github.com/projectdiscovery/utils v0.0.84-0.20240312214300-d3ba70dbb9ca h1:GY9lUYDlENXPSFPJH01Bm1BfhrUF2jpnUBR+K4VPJIs=
github.com/projectdiscovery/utils v0.0.82/go.mod h1:AbmIvy0TTlsfXxPDEMaNPVrxmqDmYiCnbGqh0TTthE4= github.com/projectdiscovery/utils v0.0.84-0.20240312214300-d3ba70dbb9ca/go.mod h1:wzMfHBq2I9oy+DEiMfUYV86g1D7eXKaQsgWnqFpmMtI=
github.com/projectdiscovery/wappalyzergo v0.0.112 h1:QPpp5jmj1lqLd5mFdFKQ9VvcYhQNqyU9Mr+IB0US2zA= github.com/projectdiscovery/wappalyzergo v0.0.112 h1:QPpp5jmj1lqLd5mFdFKQ9VvcYhQNqyU9Mr+IB0US2zA=
github.com/projectdiscovery/wappalyzergo v0.0.112/go.mod h1:hc/o+fgM8KtdpFesjfBTmHTwsR+yBd+4kYZW/DGy/x8= github.com/projectdiscovery/wappalyzergo v0.0.112/go.mod h1:hc/o+fgM8KtdpFesjfBTmHTwsR+yBd+4kYZW/DGy/x8=
github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
@ -1034,8 +1036,6 @@ github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oW
github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8=
github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 h1:TtyC78WMafNW8QFfv3TeP3yWNDG+uxNkk9vOrnDu6JA=
github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6/go.mod h1:h8272+G2omSmi30fBXiZDMkmHuOgonplfKIKjQWzlfs=
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=

2
integration_tests/protocols/code/py-env-var.yaml
View File

@ -20,4 +20,4 @@ code:
- type: word - type: word
words: words:
- "hello from input baz" - "hello from input baz"
# digest: 4a0a00473045022100b290a0c40f27573f0de9a950be13457a9bf59ade1ff2f497bf01a3b526e5db750220761942acffd6d27e2714ddaa1c73c699ccd7de48839f08cff1d6a9456bc8ff1f:4a3eb6b4988d95847d4203be25ed1d46 # digest: 4a0a0047304502207e3a5eda5f3207c3c01c820562243281926c1215224a7c80ed7528559b9f52cb022100f6ef99bb45843f481705778630f2cfd8f4d1cc3acb96392ff016f22e06aa91af:4a3eb6b4988d95847d4203be25ed1d46

2
integration_tests/protocols/code/py-file.yaml
View File

@ -18,4 +18,4 @@ code:
- type: word - type: word
words: words:
- "hello from input" - "hello from input"
# digest: 490a004630440220335663a6a4db720ee6276ab7179a87a6be0b4030771ec5ee82ecf6982342113602200a2570db7eb9721f6ceb1a89543fc436ee62b30d1b720c75ea3834ed3d2b64f3:4a3eb6b4988d95847d4203be25ed1d46 # digest: 4a0a004730450220069673af9bd6d6677f9529d06f5d8bd46d543089a4731ed18ee806761d75fd60022100913a3e27b0a5809baf710ba9585bf9fe729634c0e19e3e13eef70a6bd100df34:4a3eb6b4988d95847d4203be25ed1d46

2
integration_tests/protocols/code/py-interactsh.yaml
View File

@ -26,4 +26,4 @@ code:
part: interactsh_protocol part: interactsh_protocol
words: words:
- "http" - "http"
# digest: 490a004630440220400892730a62fa1bbb1064e4d88eea760dbf8f01c6b630ff0f5b126fd1952839022025a6d52e730c1f1cfcbd440e6269f93489db3a77cb2a27d0f47522c0819dc8d3:4a3eb6b4988d95847d4203be25ed1d46 # digest: 490a00463044022003b8d069e3c84412729c43e33013a52ee04eabcf096d511979691d71d8e905f60220011f4475899abed4f86b4bd5e6c2423750759135206e4729826afe1ed8a44f4d:4a3eb6b4988d95847d4203be25ed1d46

2
integration_tests/protocols/code/py-snippet.yaml
View File

@ -21,4 +21,4 @@ code:
- type: word - type: word
words: words:
- "hello from input" - "hello from input"
# digest: 490a0046304402206b14abdc0d5fc13466f5c292da9fb2a19d1b2c5e683cc052037fe367b372f82b02202c00b9acbd8106a769eb411794c567d3019433671397bf909e16b286105ed69e:4a3eb6b4988d95847d4203be25ed1d46 # digest: 4a0a00473045022100c291615cf2a8005450c17a6554e81a9cdab14743b299f0679c644247929198b502206fdacc8ab173bde2b4015340012637916bf2659f66f320fcc06b97ac639072a1:4a3eb6b4988d95847d4203be25ed1d46

36
integration_tests/protocols/http/http-matcher-extractor-dy-extractor.yaml Normal file
View File

@ -0,0 +1,36 @@
id: http-matcher-extractor-dy-extractor
info:
name: HTTP matcher and extractor & dynamic extractor
description: >
Edgecase to test for a combination of matchers , extractors and dynamic extractors
author: pdteam
severity: info
http:
- raw:
- |
GET {{BaseURL}} HTTP/1.1
- |
GET {{absolutePath}} HTTP/1.1
req-condition: true
extractors:
- type: regex
internal: true
part: body_1
name: absolutePath
regex:
- '<a href="(/domains)">'
group: 1
- type: regex
internal: false
part: body_2
name: title
regex:
- '<title[^>]*>([^<]+)</title>'
group: 1
matchers:
- type: regex
part: body_2
regex:
- '<title[^>]*>([^<]+)</title>'

2
integration_tests/protocols/http/http-paths.yaml
View File

@ -28,7 +28,7 @@ info:
- "//CFIDE/wizards/common/utils.cfc" - "//CFIDE/wizards/common/utils.cfc"
# Test all templates with FullURLs # Test all templates with FullURLs
requests: http:
- raw: - raw:
# relative path without leading slash with param # relative path without leading slash with param
# If relative path does not have `/` prefix it is autocorrected # If relative path does not have `/` prefix it is autocorrected

36
integration_tests/protocols/http/multi-http-var-sharing.yaml Normal file
View File

@ -0,0 +1,36 @@
id: multi-http-var-sharing
info:
name: Multi HTTP var sharing
author: pdteam
severity: info
description: |
A template which has multiple HTTP requests block and variables are shared between them
http:
- method: GET
path:
- "{{BaseURL}}"
matchers:
- type: word
words:
- "This is test matcher text"
negative: true
internal: true
extractors:
- type: dsl
name: ffff
dsl:
- status_code
internal: true
- method: GET
path:
- "{{BaseURL}}/{{ffff}}"
matchers:
- type: status
status:
- 200

12
lib/config.go
View File

@ -107,10 +107,12 @@ func WithInteractshOptions(opts InteractshOpts) NucleiSDKOptions {
// Concurrency options // Concurrency options
type Concurrency struct { type Concurrency struct {
TemplateConcurrency int // number of templates to run concurrently (per host in host-spray mode) TemplateConcurrency int // number of templates to run concurrently (per host in host-spray mode)
HostConcurrency int // number of hosts to scan concurrently (per template in template-spray mode) HostConcurrency int // number of hosts to scan concurrently (per template in template-spray mode)
HeadlessHostConcurrency int // number of hosts to scan concurrently for headless templates (per template in template-spray mode) HeadlessHostConcurrency int // number of hosts to scan concurrently for headless templates (per template in template-spray mode)
HeadlessTemplateConcurrency int // number of templates to run concurrently for headless templates (per host in host-spray mode) HeadlessTemplateConcurrency int // number of templates to run concurrently for headless templates (per host in host-spray mode)
JavascriptTemplateConcurrency int // number of templates to run concurrently for javascript templates (per host in host-spray mode)
TemplatePayloadConcurrency int // max concurrent payloads to run for a template (a good default is 25)
} }
// WithConcurrency sets concurrency options // WithConcurrency sets concurrency options
@ -120,6 +122,8 @@ func WithConcurrency(opts Concurrency) NucleiSDKOptions {
e.opts.BulkSize = opts.HostConcurrency e.opts.BulkSize = opts.HostConcurrency
e.opts.HeadlessBulkSize = opts.HeadlessHostConcurrency e.opts.HeadlessBulkSize = opts.HeadlessHostConcurrency
e.opts.HeadlessTemplateThreads = opts.HeadlessTemplateConcurrency e.opts.HeadlessTemplateThreads = opts.HeadlessTemplateConcurrency
e.opts.JsConcurrency = opts.JavascriptTemplateConcurrency
e.opts.PayloadConcurrency = opts.TemplatePayloadConcurrency
return nil return nil
} }
} }

10
pkg/operators/operators.go
View File

@ -299,7 +299,7 @@ func (operators *Operators) Execute(data map[string]interface{}, match MatchFunc
if len(result.DynamicValues) > 0 { if len(result.DynamicValues) > 0 {
return result, true return result, true
} }
return nil, false return result, false
} }
} }
@ -313,6 +313,10 @@ func (operators *Operators) Execute(data map[string]interface{}, match MatchFunc
// Don't print if we have matchers, and they have not matched, regardless of extractor // Don't print if we have matchers, and they have not matched, regardless of extractor
if len(operators.Matchers) > 0 && !matches { if len(operators.Matchers) > 0 && !matches {
// if dynamic values are present then it is not a failure
if len(result.DynamicValues) > 0 {
return result, true
}
return nil, false return nil, false
} }
// Write a final string of output if matcher type is // Write a final string of output if matcher type is
@ -320,6 +324,10 @@ func (operators *Operators) Execute(data map[string]interface{}, match MatchFunc
if len(result.Extracts) > 0 || len(result.OutputExtracts) > 0 || matches { if len(result.Extracts) > 0 || len(result.OutputExtracts) > 0 || matches {
return result, true return result, true
} }
// if dynamic values are present then it is not a failure
if len(result.DynamicValues) > 0 {
return result, true
}
return nil, false return nil, false
} }

4
pkg/protocols/code/code.go
View File

@ -192,6 +192,10 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
dataOutputString := fmtStdout(gOutput.Stdout.String()) dataOutputString := fmtStdout(gOutput.Stdout.String())
data := make(output.InternalEvent) data := make(output.InternalEvent)
// also include all request variables in result event
for _, value := range metaSrc.Variables {
data[value.Name] = value.Value
}
data["type"] = request.Type().String() data["type"] = request.Type().String()
data["response"] = dataOutputString // response contains filtered output (eg without trailing \n) data["response"] = dataOutputString // response contains filtered output (eg without trailing \n)

7
pkg/protocols/http/build_request.go
View File

@ -97,7 +97,7 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context,
} }
// Parse target url // Parse target url
parsed, err := urlutil.Parse(input.MetaInput.Input) parsed, err := urlutil.ParseAbsoluteURL(input.MetaInput.Input, false)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -153,7 +153,7 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context,
return r.generateRawRequest(ctx, reqData, parsed, finalVars, payloads) return r.generateRawRequest(ctx, reqData, parsed, finalVars, payloads)
} }
reqURL, err := urlutil.ParseURL(reqData, true) reqURL, err := urlutil.ParseAbsoluteURL(reqData, true)
if err != nil { if err != nil {
return nil, errorutil.NewWithTag("http", "failed to parse url %v while creating http request", reqData) return nil, errorutil.NewWithTag("http", "failed to parse url %v while creating http request", reqData)
} }
@ -291,8 +291,7 @@ func (r *requestGenerator) generateRawRequest(ctx context.Context, rawRequest st
unsafeReq := &generatedRequest{rawRequest: rawRequestData, meta: generatorValues, original: r.request, interactshURLs: r.interactshURLs} unsafeReq := &generatedRequest{rawRequest: rawRequestData, meta: generatorValues, original: r.request, interactshURLs: r.interactshURLs}
return unsafeReq, nil return unsafeReq, nil
} }
urlx, err := urlutil.ParseAbsoluteURL(rawRequestData.FullURL, true)
urlx, err := urlutil.ParseURL(rawRequestData.FullURL, true)
if err != nil { if err != nil {
return nil, errorutil.NewWithErr(err).Msgf("failed to create request with url %v got %v", rawRequestData.FullURL, err).WithTag("raw") return nil, errorutil.NewWithErr(err).Msgf("failed to create request with url %v got %v", rawRequestData.FullURL, err).WithTag("raw")
} }

11
pkg/protocols/http/request.go
View File

@ -38,6 +38,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/types" "github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/rawhttp" "github.com/projectdiscovery/rawhttp"
convUtil "github.com/projectdiscovery/utils/conversion" convUtil "github.com/projectdiscovery/utils/conversion"
errorutil "github.com/projectdiscovery/utils/errors"
httpUtils "github.com/projectdiscovery/utils/http" httpUtils "github.com/projectdiscovery/utils/http"
"github.com/projectdiscovery/utils/reader" "github.com/projectdiscovery/utils/reader"
sliceutil "github.com/projectdiscovery/utils/slice" sliceutil "github.com/projectdiscovery/utils/slice"
@ -453,7 +454,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
} }
// verify if parallel elaboration was requested // verify if parallel elaboration was requested
if request.Threads > 0 { if request.Threads > 0 && len(request.Payloads) > 0 {
return request.executeParallelHTTP(input, dynamicValues, callback) return request.executeParallelHTTP(input, dynamicValues, callback)
} }
@ -494,7 +495,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
return true, nil return true, nil
} }
var gotMatches bool var gotMatches bool
err = request.executeRequest(input, generatedHttpRequest, previous, hasInteractMatchers, func(event *output.InternalWrappedEvent) { execReqErr := request.executeRequest(input, generatedHttpRequest, previous, hasInteractMatchers, func(event *output.InternalWrappedEvent) {
// a special case where operators has interactsh matchers and multiple request are made // a special case where operators has interactsh matchers and multiple request are made
// ex: status_code_2 , interactsh_protocol (from 1st request) etc // ex: status_code_2 , interactsh_protocol (from 1st request) etc
needsRequestEvent := interactsh.HasMatchers(request.CompiledOperators) && request.NeedsRequestCondition() needsRequestEvent := interactsh.HasMatchers(request.CompiledOperators) && request.NeedsRequestCondition()
@ -525,14 +526,14 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
}, generator.currentIndex) }, generator.currentIndex)
// If a variable is unresolved, skip all further requests // If a variable is unresolved, skip all further requests
if errors.Is(err, errStopExecution) { if errors.Is(execReqErr, errStopExecution) {
return true, nil return true, nil
} }
if err != nil { if execReqErr != nil {
if request.options.HostErrorsCache != nil { if request.options.HostErrorsCache != nil {
request.options.HostErrorsCache.MarkFailed(input.MetaInput.ID(), err) request.options.HostErrorsCache.MarkFailed(input.MetaInput.ID(), err)
} }
requestErr = err requestErr = errorutil.NewWithErr(execReqErr).Msgf("got err while executing %v", generatedHttpRequest.URL())
} }
request.options.Progress.IncrementRequests() request.options.Progress.IncrementRequests()

13
pkg/protocols/http/utils.go
View File

@ -6,13 +6,22 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
"github.com/projectdiscovery/rawhttp" "github.com/projectdiscovery/rawhttp"
errorutil "github.com/projectdiscovery/utils/errors"
) )
// dump creates a dump of the http request in form of a byte slice // dump creates a dump of the http request in form of a byte slice
func dump(req *generatedRequest, reqURL string) ([]byte, error) { func dump(req *generatedRequest, reqURL string) ([]byte, error) {
if req.request != nil { if req.request != nil {
return req.request.Dump() bin, err := req.request.Dump()
if err != nil {
return nil, errorutil.NewWithErr(err).WithTag("http").Msgf("could not dump request: %v", req.request.URL.String())
}
return bin, nil
} }
rawHttpOptions := &rawhttp.Options{CustomHeaders: req.rawRequest.UnsafeHeaders, CustomRawBytes: req.rawRequest.UnsafeRawBytes} rawHttpOptions := &rawhttp.Options{CustomHeaders: req.rawRequest.UnsafeHeaders, CustomRawBytes: req.rawRequest.UnsafeRawBytes}
return rawhttp.DumpRequestRaw(req.rawRequest.Method, reqURL, req.rawRequest.Path, generators.ExpandMapValues(req.rawRequest.Headers), io.NopCloser(strings.NewReader(req.rawRequest.Data)), rawHttpOptions) bin, err := rawhttp.DumpRequestRaw(req.rawRequest.Method, reqURL, req.rawRequest.Path, generators.ExpandMapValues(req.rawRequest.Headers), io.NopCloser(strings.NewReader(req.rawRequest.Data)), rawHttpOptions)
if err != nil {
return nil, errorutil.NewWithErr(err).WithTag("http").Msgf("could not dump request: %v", reqURL)
}
return bin, nil
} }

2
pkg/protocols/protocols.go
View File

@ -131,7 +131,7 @@ func (e *ExecutorOptions) GetThreadsForNPayloadRequests(totalRequests int, curre
if currentThreads > 0 { if currentThreads > 0 {
return currentThreads return currentThreads
} else { } else {
return e.Options.TemplateThreads return e.Options.PayloadConcurrency
} }
} }

26
pkg/tmplexec/exec.go
View File

@ -3,6 +3,7 @@ package tmplexec
import ( import (
"errors" "errors"
"fmt" "fmt"
"runtime/debug"
"strings" "strings"
"sync/atomic" "sync/atomic"
@ -34,16 +35,6 @@ var _ protocols.Executer = &TemplateExecuter{}
// NewTemplateExecuter creates a new request TemplateExecuter for list of requests // NewTemplateExecuter creates a new request TemplateExecuter for list of requests
func NewTemplateExecuter(requests []protocols.Request, options *protocols.ExecutorOptions) (*TemplateExecuter, error) { func NewTemplateExecuter(requests []protocols.Request, options *protocols.ExecutorOptions) (*TemplateExecuter, error) {
isMultiProto := false
lastProto := ""
for _, request := range requests {
if request.Type().String() != lastProto && lastProto != "" {
isMultiProto = true
break
}
lastProto = request.Type().String()
}
e := &TemplateExecuter{requests: requests, options: options, results: &atomic.Bool{}} e := &TemplateExecuter{requests: requests, options: options, results: &atomic.Bool{}}
if options.Flow != "" { if options.Flow != "" {
// we use a dummy input here because goal of flow executor at this point is to just check // we use a dummy input here because goal of flow executor at this point is to just check
@ -55,13 +46,11 @@ func NewTemplateExecuter(requests []protocols.Request, options *protocols.Execut
} }
e.program = p e.program = p
} else { } else {
// Review: // only use generic if there is only 1 protocol with only 1 section
// multiproto engine is only used if there is more than one protocol in template if len(requests) == 1 {
// else we use generic engine (should we use multiproto engine for single protocol with multiple requests as well ?)
if isMultiProto {
e.engine = multiproto.NewMultiProtocol(requests, options, e.results)
} else {
e.engine = generic.NewGenericEngine(requests, options, e.results) e.engine = generic.NewGenericEngine(requests, options, e.results)
} else {
e.engine = multiproto.NewMultiProtocol(requests, options, e.results)
} }
} }
return e, nil return e, nil
@ -113,8 +102,9 @@ func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) {
defer func() { defer func() {
// try catching unknown panics // try catching unknown panics
if r := recover(); r != nil { if r := recover(); r != nil {
ctx.LogError(fmt.Errorf("panic: %v", r)) stacktrace := debug.Stack()
gologger.Verbose().Msgf("panic: %v", r) ctx.LogError(fmt.Errorf("panic: %v\n%s", r, stacktrace))
gologger.Verbose().Msgf("panic: %v\n%s", r, stacktrace)
} }
}() }()

13
pkg/tmplexec/multiproto/multi.go
View File

@ -46,12 +46,13 @@ func (m *MultiProtocol) Compile() error {
func (m *MultiProtocol) ExecuteWithResults(ctx *scan.ScanContext) error { func (m *MultiProtocol) ExecuteWithResults(ctx *scan.ScanContext) error {
// put all readonly args into template context // put all readonly args into template context
m.options.GetTemplateCtx(ctx.Input.MetaInput).Merge(m.readOnlyArgs) m.options.GetTemplateCtx(ctx.Input.MetaInput).Merge(m.readOnlyArgs)
var finalProtoEvent *output.InternalWrappedEvent
// callback to process results from all protocols // callback to process results from all protocols
multiProtoCallback := func(event *output.InternalWrappedEvent) { multiProtoCallback := func(event *output.InternalWrappedEvent) {
if event != nil { if event == nil {
finalProtoEvent = event return
} }
// log event and generate result for the event
ctx.LogEvent(event)
// export dynamic values from operators (i.e internal:true) // export dynamic values from operators (i.e internal:true)
if event.OperatorsResult != nil && len(event.OperatorsResult.DynamicValues) > 0 { if event.OperatorsResult != nil && len(event.OperatorsResult.DynamicValues) > 0 {
for k, v := range event.OperatorsResult.DynamicValues { for k, v := range event.OperatorsResult.DynamicValues {
@ -97,12 +98,6 @@ func (m *MultiProtocol) ExecuteWithResults(ctx *scan.ScanContext) error {
return err return err
} }
} }
// Review: how to handle events of multiple protocols in a single template
// currently the outer callback is only executed once (for the last protocol in queue)
// due to workflow logic at https://github.com/projectdiscovery/nuclei/blob/main/pkg/protocols/common/executer/executem.go#L150
// this causes addition of duplicated / unncessary variables with prefix template_id_all_variables
ctx.LogEvent(finalProtoEvent)
return nil return nil
} }

2
pkg/types/types.go
View File

@ -370,6 +370,8 @@ type Options struct {
ScanID string ScanID string
// JsConcurrency is the number of concurrent js routines to run // JsConcurrency is the number of concurrent js routines to run
JsConcurrency int JsConcurrency int
// PayloadConcurrency is the number of concurrent payloads to run per template
PayloadConcurrency int
} }
// ShouldLoadResume resume file // ShouldLoadResume resume file