Shared Execution Context Prototype (#2576)

* renaming var

* Introducing shared execution context prototype

* fixing field name

* adding shared values propagation

* adding shared context lock

* add slice values normalization

* adding integration tests

* adding metadata support for dns

* adding multi-protocol context sharing test

* removing debug test files

* moving contextargs around

* adding comments

* refactoring code

- getter/setter for complex types
- using pointers to avoid heap allocations
dev
Mzack9999 2022-10-03 12:12:20 +02:00 committed by GitHub
parent 2e66c8ddd6
commit 781e4e6105
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
34 changed files with 523 additions and 125 deletions

View File

@ -0,0 +1,17 @@
id: dns-value-sharing-template1
info:
name: dns-value-sharing-template1
author: pdteam
severity: info
dns:
- name: "{{FQDN}}"
type: A
extractors:
- type: regex
name: extracted
group: 1
regex:
- "IN\tA\t(.+)"

View File

@ -0,0 +1,10 @@
id: dns-value-sharing-template2
info:
name: dns-value-sharing-template2
author: pdteam
severity: info
dns:
- name: "{{extracted}}"
type: PTR

View File

@ -0,0 +1,19 @@
id: value-sharing-template2
info:
name: value-sharing-template2
author: pdteam
severity: info
requests:
- raw:
- |
GET / HTTP/1.1
Host: {{Hostname}}
{{extracted}}
matchers:
- type: word
words:
- "ok"

View File

@ -0,0 +1,11 @@
id: dns-value-sharing-workflow
info:
name: DNS Value Sharing Test
author: pdteam
severity: info
workflows:
- template: workflow/dns-value-share-template-1.yaml
subtemplates:
- template: workflow/dns-value-share-template-2.yaml
- template: workflow/dns-value-share-template-3.yaml

View File

@ -0,0 +1,17 @@
id: value-sharing-template1
info:
name: value-sharing-template1
author: pdteam
severity: info
requests:
- path:
- "{{BaseURL}}/path1"
extractors:
- type: regex
part: body
name: extracted
regex:
- 'href="(.*)"'
group: 1

View File

@ -0,0 +1,19 @@
id: value-sharing-template2
info:
name: value-sharing-template2
author: pdteam
severity: info
requests:
- raw:
- |
GET /path2 HTTP/1.1
Host: {{Hostname}}
{{extracted}}
matchers:
- type: word
words:
- "test-value"

View File

@ -0,0 +1,10 @@
id: http-value-sharing-workflow
info:
name: HTTP Value Sharing Test
author: pdteam
severity: info
workflows:
- template: workflow/http-value-share-template-1.yaml
subtemplates:
- template: workflow/http-value-share-template-2.yaml

View File

@ -2,6 +2,7 @@ package main
import ( import (
"fmt" "fmt"
"io"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
@ -11,10 +12,12 @@ import (
) )
var workflowTestcases = map[string]testutils.TestCase{ var workflowTestcases = map[string]testutils.TestCase{
"workflow/basic.yaml": &workflowBasic{}, "workflow/basic.yaml": &workflowBasic{},
"workflow/condition-matched.yaml": &workflowConditionMatched{}, "workflow/condition-matched.yaml": &workflowConditionMatched{},
"workflow/condition-unmatched.yaml": &workflowConditionUnmatch{}, "workflow/condition-unmatched.yaml": &workflowConditionUnmatch{},
"workflow/matcher-name.yaml": &workflowMatcherName{}, "workflow/matcher-name.yaml": &workflowMatcherName{},
"workflow/http-value-share-workflow.yaml": &workflowHttpKeyValueShare{},
"workflow/dns-value-share-workflow.yaml": &workflowDnsKeyValueShare{},
} }
type workflowBasic struct{} type workflowBasic struct{}
@ -92,3 +95,39 @@ func (h *workflowMatcherName) Execute(filePath string) error {
return expectResultsCount(results, 1) return expectResultsCount(results, 1)
} }
type workflowHttpKeyValueShare struct{}
// Execute executes a test case and returns an error if occurred
func (h *workflowHttpKeyValueShare) Execute(filePath string) error {
router := httprouter.New()
router.GET("/path1", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
fmt.Fprintf(w, "href=\"test-value\"")
})
router.GET("/path2", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
body, _ := io.ReadAll(r.Body)
fmt.Fprintf(w, "%s", body)
})
ts := httptest.NewServer(router)
defer ts.Close()
results, err := testutils.RunNucleiWorkflowAndGetResults(filePath, ts.URL, debug)
if err != nil {
return err
}
return expectResultsCount(results, 1)
}
type workflowDnsKeyValueShare struct{}
// Execute executes a test case and returns an error if occurred
func (h *workflowDnsKeyValueShare) Execute(filePath string) error {
results, err := testutils.RunNucleiWorkflowAndGetResults(filePath, "http://scanme.sh", debug)
if err != nil {
return err
}
// no results - ensure that the variable sharing works
return expectResultsCount(results, 1)
}

View File

@ -28,7 +28,6 @@ require (
github.com/projectdiscovery/cryptoutil v1.0.0 github.com/projectdiscovery/cryptoutil v1.0.0
github.com/projectdiscovery/fastdialer v0.0.16-0.20220908084548-3eab0c2a02d5 github.com/projectdiscovery/fastdialer v0.0.16-0.20220908084548-3eab0c2a02d5
github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08 github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08
github.com/projectdiscovery/goflags v0.0.10-0.20220829035232-bc74fe1cb567
github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/gologger v1.1.4
github.com/projectdiscovery/hmap v0.0.2 github.com/projectdiscovery/hmap v0.0.2
github.com/projectdiscovery/interactsh v1.0.6-0.20220827132222-460cc6270053 github.com/projectdiscovery/interactsh v1.0.6-0.20220827132222-460cc6270053
@ -36,7 +35,7 @@ require (
github.com/projectdiscovery/rawhttp v0.1.1 github.com/projectdiscovery/rawhttp v0.1.1
github.com/projectdiscovery/retryabledns v1.0.15 github.com/projectdiscovery/retryabledns v1.0.15
github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d
github.com/projectdiscovery/stringsutil v0.0.0-20220731064040-4b67f194751e github.com/projectdiscovery/stringsutil v0.0.1
github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6 github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6
github.com/remeh/sizedwaitgroup v1.0.0 github.com/remeh/sizedwaitgroup v1.0.0
github.com/rs/xid v1.4.0 github.com/rs/xid v1.4.0
@ -51,7 +50,7 @@ require (
github.com/xanzy/go-gitlab v0.73.1 github.com/xanzy/go-gitlab v0.73.1
go.uber.org/atomic v1.10.0 go.uber.org/atomic v1.10.0
go.uber.org/multierr v1.8.0 go.uber.org/multierr v1.8.0
golang.org/x/net v0.0.0-20220805013720-a33c5aa5df48 golang.org/x/net v0.0.0-20221002022538-bcab6841153b
golang.org/x/oauth2 v0.0.0-20220722155238-128564f6959c golang.org/x/oauth2 v0.0.0-20220722155238-128564f6959c
golang.org/x/text v0.3.7 golang.org/x/text v0.3.7
gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v2 v2.4.0
@ -73,7 +72,7 @@ require (
github.com/mholt/archiver v3.1.1+incompatible github.com/mholt/archiver v3.1.1+incompatible
github.com/mitchellh/go-homedir v1.1.0 github.com/mitchellh/go-homedir v1.1.0
github.com/openrdap/rdap v0.9.1-0.20191017185644-af93e7ef17b7 github.com/openrdap/rdap v0.9.1-0.20191017185644-af93e7ef17b7
github.com/projectdiscovery/fileutil v0.0.0-20220705195237-01becc2a8963 github.com/projectdiscovery/fileutil v0.0.1
github.com/projectdiscovery/iputil v0.0.0-20220712175312-b9406f31cdd8 github.com/projectdiscovery/iputil v0.0.0-20220712175312-b9406f31cdd8
github.com/projectdiscovery/nvd v1.0.9 github.com/projectdiscovery/nvd v1.0.9
github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4 github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4
@ -148,7 +147,7 @@ require (
github.com/mattn/go-isatty v0.0.16 // indirect github.com/mattn/go-isatty v0.0.16 // indirect
github.com/mattn/go-runewidth v0.0.13 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect
github.com/mholt/acmez v1.0.4 // indirect github.com/mholt/acmez v1.0.4 // indirect
github.com/microcosm-cc/bluemonday v1.0.19 // indirect github.com/microcosm-cc/bluemonday v1.0.20 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/onsi/ginkgo v1.16.4 // indirect github.com/onsi/ginkgo v1.16.4 // indirect
@ -195,5 +194,6 @@ require (
github.com/klauspost/compress v1.15.8 // indirect github.com/klauspost/compress v1.15.8 // indirect
github.com/nwaples/rardecode v1.1.2 // indirect github.com/nwaples/rardecode v1.1.2 // indirect
github.com/pierrec/lz4 v2.6.1+incompatible // indirect github.com/pierrec/lz4 v2.6.1+incompatible // indirect
github.com/projectdiscovery/goflags v0.1.0 // indirect
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
) )

View File

@ -469,6 +469,8 @@ github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/le
github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM=
github.com/microcosm-cc/bluemonday v1.0.19 h1:OI7hoF5FY4pFz2VA//RN8TfM0YJ2dJcl4P4APrCWy6c= github.com/microcosm-cc/bluemonday v1.0.19 h1:OI7hoF5FY4pFz2VA//RN8TfM0YJ2dJcl4P4APrCWy6c=
github.com/microcosm-cc/bluemonday v1.0.19/go.mod h1:QNzV2UbLK2/53oIIwTOyLUSABMkjZ4tqiyC1g/DyqxE= github.com/microcosm-cc/bluemonday v1.0.19/go.mod h1:QNzV2UbLK2/53oIIwTOyLUSABMkjZ4tqiyC1g/DyqxE=
github.com/microcosm-cc/bluemonday v1.0.20 h1:flpzsq4KU3QIYAYGV/szUat7H+GPOXR0B2JU5A1Wp8Y=
github.com/microcosm-cc/bluemonday v1.0.20/go.mod h1:yfBmMi8mxvaZut3Yytv+jTXRY8mxyjJ0/kQBTElld50=
github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
@ -556,12 +558,16 @@ github.com/projectdiscovery/fileutil v0.0.0-20220308101036-16c79af1cf5d/go.mod h
github.com/projectdiscovery/fileutil v0.0.0-20220609150212-453ac591c36c/go.mod h1:g8wsrb0S5NtEN0JgVyyPeb3FQdArx+UMESmFX94bcGY= github.com/projectdiscovery/fileutil v0.0.0-20220609150212-453ac591c36c/go.mod h1:g8wsrb0S5NtEN0JgVyyPeb3FQdArx+UMESmFX94bcGY=
github.com/projectdiscovery/fileutil v0.0.0-20220705195237-01becc2a8963 h1:4o97N9ftX1J3iKlIRVMPVOVZs4qbCczJvoFF2WA40t4= github.com/projectdiscovery/fileutil v0.0.0-20220705195237-01becc2a8963 h1:4o97N9ftX1J3iKlIRVMPVOVZs4qbCczJvoFF2WA40t4=
github.com/projectdiscovery/fileutil v0.0.0-20220705195237-01becc2a8963/go.mod h1:DaY7wmLPMleyHDCD/14YApPCDtrARY4J8Eny2ZGsG/g= github.com/projectdiscovery/fileutil v0.0.0-20220705195237-01becc2a8963/go.mod h1:DaY7wmLPMleyHDCD/14YApPCDtrARY4J8Eny2ZGsG/g=
github.com/projectdiscovery/fileutil v0.0.1 h1:3K3UqCDOan3LsvWhV0nyvVuMWSwCloNPUJIGcXsi1os=
github.com/projectdiscovery/fileutil v0.0.1/go.mod h1:Oo6ZEvXmQz/xPF0YukzmwpdW2LYinWCSEmzZOQsJCLg=
github.com/projectdiscovery/folderutil v0.0.0-20220215113126-add60a1e8e08 h1:m1pgJisawU7zP9lKGktOEk6KNrNAR7e4Q07Kt3ox0NM= github.com/projectdiscovery/folderutil v0.0.0-20220215113126-add60a1e8e08 h1:m1pgJisawU7zP9lKGktOEk6KNrNAR7e4Q07Kt3ox0NM=
github.com/projectdiscovery/folderutil v0.0.0-20220215113126-add60a1e8e08/go.mod h1:BMqXH4jNGByVdE2iLtKvc/6XStaiZRuCIaKv1vw9PnI= github.com/projectdiscovery/folderutil v0.0.0-20220215113126-add60a1e8e08/go.mod h1:BMqXH4jNGByVdE2iLtKvc/6XStaiZRuCIaKv1vw9PnI=
github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY=
github.com/projectdiscovery/goflags v0.0.8/go.mod h1:GDSkWyXa6kfQjpJu10SO64DN8lXuKXVENlBMk8N7H80= github.com/projectdiscovery/goflags v0.0.8/go.mod h1:GDSkWyXa6kfQjpJu10SO64DN8lXuKXVENlBMk8N7H80=
github.com/projectdiscovery/goflags v0.0.10-0.20220829035232-bc74fe1cb567 h1:0CFt+l9ZO7lT1ZPU/NCTAdRzkZZ5OH+b4FvxsJo1GhE= github.com/projectdiscovery/goflags v0.0.10-0.20220829035232-bc74fe1cb567 h1:0CFt+l9ZO7lT1ZPU/NCTAdRzkZZ5OH+b4FvxsJo1GhE=
github.com/projectdiscovery/goflags v0.0.10-0.20220829035232-bc74fe1cb567/go.mod h1:t/dEhv2VDOzayugXZCkbkX8n+pPeVmRD+WgQRSgReeI= github.com/projectdiscovery/goflags v0.0.10-0.20220829035232-bc74fe1cb567/go.mod h1:t/dEhv2VDOzayugXZCkbkX8n+pPeVmRD+WgQRSgReeI=
github.com/projectdiscovery/goflags v0.1.0 h1:Z7sUVK8wgH6aGJWinmGQEtsn+GNf/0RQ+z1wQcpCeeA=
github.com/projectdiscovery/goflags v0.1.0/go.mod h1:/YBPA+1igSkQbwD7a91o0HUIwMDlsmQDRZL2oSYSyEQ=
github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI= github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI=
github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY= github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY=
@ -616,6 +622,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20220422150559-b54fb5dc6833/go.mo
github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3/go.mod h1:mF5sh4jTghoGWwgUb9qWi5waTFklClDbtrqtJU93awc= github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3/go.mod h1:mF5sh4jTghoGWwgUb9qWi5waTFklClDbtrqtJU93awc=
github.com/projectdiscovery/stringsutil v0.0.0-20220731064040-4b67f194751e h1:JBdwX+DJNq5FIJGsZCSKLGV4EhSRiPtNk22Vi+3cTDg= github.com/projectdiscovery/stringsutil v0.0.0-20220731064040-4b67f194751e h1:JBdwX+DJNq5FIJGsZCSKLGV4EhSRiPtNk22Vi+3cTDg=
github.com/projectdiscovery/stringsutil v0.0.0-20220731064040-4b67f194751e/go.mod h1:32NYmKyHkKsmisAOAaWrR15lz2ysz2M8x3KMeeoRHoU= github.com/projectdiscovery/stringsutil v0.0.0-20220731064040-4b67f194751e/go.mod h1:32NYmKyHkKsmisAOAaWrR15lz2ysz2M8x3KMeeoRHoU=
github.com/projectdiscovery/stringsutil v0.0.1 h1:a6TCMT+D1aUsoZxNiYf9O30wiDOoLOHDwj89HBjr5BQ=
github.com/projectdiscovery/stringsutil v0.0.1/go.mod h1:TDi2LEqR3OML0BxGoMbbfAHSk5AdfHX762Oc302sgmM=
github.com/projectdiscovery/tlsx v0.0.7 h1:McoDo4Ju7aetogatU4lVTRcQpxkf0qgQSoWgtkDavCk= github.com/projectdiscovery/tlsx v0.0.7 h1:McoDo4Ju7aetogatU4lVTRcQpxkf0qgQSoWgtkDavCk=
github.com/projectdiscovery/tlsx v0.0.7/go.mod h1:/ZCk/zzyuDdXx2E511yhNIj3LERnGUXghqkASBbdh5M= github.com/projectdiscovery/tlsx v0.0.7/go.mod h1:/ZCk/zzyuDdXx2E511yhNIj3LERnGUXghqkASBbdh5M=
github.com/projectdiscovery/urlutil v0.0.0-20210525140139-b874f06ad921 h1:EgaxpJm7+lKppfAHkFHs+S+II0lodp4Gu3leZCCkWlc= github.com/projectdiscovery/urlutil v0.0.0-20210525140139-b874f06ad921 h1:EgaxpJm7+lKppfAHkFHs+S+II0lodp4Gu3leZCCkWlc=
@ -922,6 +930,9 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
golang.org/x/net v0.0.0-20220728211354-c7608f3a8462/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20220728211354-c7608f3a8462/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/net v0.0.0-20220805013720-a33c5aa5df48 h1:N9Vc/rorQUDes6B9CNdIxAn5jODGj2wzfrei2x4wNj4= golang.org/x/net v0.0.0-20220805013720-a33c5aa5df48 h1:N9Vc/rorQUDes6B9CNdIxAn5jODGj2wzfrei2x4wNj4=
golang.org/x/net v0.0.0-20220805013720-a33c5aa5df48/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20220805013720-a33c5aa5df48/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/net v0.0.0-20221002022538-bcab6841153b h1:6e93nYa3hNqAvLr0pD4PN1fFS+gKzp2zAXqrnTCstqU=
golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=

View File

@ -6,6 +6,7 @@ import (
"github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/templates"
"github.com/projectdiscovery/nuclei/v2/pkg/templates/types" "github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
generalTypes "github.com/projectdiscovery/nuclei/v2/pkg/types" generalTypes "github.com/projectdiscovery/nuclei/v2/pkg/types"
@ -59,7 +60,7 @@ func (e *Engine) ExecuteWithOpts(templatesList []*templates.Template, target Inp
// processSelfContainedTemplates execute a self-contained template. // processSelfContainedTemplates execute a self-contained template.
func (e *Engine) executeSelfContainedTemplateWithInput(template *templates.Template, results *atomic.Bool) { func (e *Engine) executeSelfContainedTemplateWithInput(template *templates.Template, results *atomic.Bool) {
match, err := template.Executer.Execute("") match, err := template.Executer.Execute(contextargs.New())
if err != nil { if err != nil {
gologger.Warning().Msgf("[%s] Could not execute step: %s\n", e.executerOpts.Colorizer.BrightBlue(template.ID), err) gologger.Warning().Msgf("[%s] Could not execute step: %s\n", e.executerOpts.Colorizer.BrightBlue(template.ID), err)
} }
@ -139,7 +140,9 @@ func (e *Engine) executeModelWithInput(templateType types.ProtocolType, template
case types.WorkflowProtocol: case types.WorkflowProtocol:
match = e.executeWorkflow(value, template.CompiledWorkflow) match = e.executeWorkflow(value, template.CompiledWorkflow)
default: default:
match, err = template.Executer.Execute(value) ctxArgs := contextargs.New()
ctxArgs.Input = value
match, err = template.Executer.Execute(ctxArgs)
} }
if err != nil { if err != nil {
gologger.Warning().Msgf("[%s] Could not execute step: %s\n", e.executerOpts.Colorizer.BrightBlue(template.ID), err) gologger.Warning().Msgf("[%s] Could not execute step: %s\n", e.executerOpts.Colorizer.BrightBlue(template.ID), err)
@ -200,7 +203,9 @@ func (e *Engine) executeModelWithInputAndResult(templateType types.ProtocolType,
case types.WorkflowProtocol: case types.WorkflowProtocol:
match = e.executeWorkflow(value, template.CompiledWorkflow) match = e.executeWorkflow(value, template.CompiledWorkflow)
default: default:
err = template.Executer.ExecuteWithResults(value, func(event *output.InternalWrappedEvent) { ctxArgs := contextargs.New()
ctxArgs.Input = value
err = template.Executer.ExecuteWithResults(ctxArgs, func(event *output.InternalWrappedEvent) {
for _, result := range event.Results { for _, result := range event.Results {
callback(result) callback(result)
} }
@ -240,7 +245,9 @@ func (e *ChildExecuter) Execute(template *templates.Template, URL string) {
wg.Add() wg.Add()
go func(tpl *templates.Template) { go func(tpl *templates.Template) {
match, err := template.Executer.Execute(URL) ctxArgs := contextargs.New()
ctxArgs.Input = URL
match, err := template.Executer.Execute(ctxArgs)
if err != nil { if err != nil {
gologger.Warning().Msgf("[%s] Could not execute step: %s\n", e.e.executerOpts.Colorizer.BrightBlue(template.ID), err) gologger.Warning().Msgf("[%s] Could not execute step: %s\n", e.e.executerOpts.Colorizer.BrightBlue(template.ID), err)
} }

View File

@ -1,26 +1,36 @@
package core package core
import ( import (
"fmt"
"net/http/cookiejar"
"github.com/remeh/sizedwaitgroup" "github.com/remeh/sizedwaitgroup"
"go.uber.org/atomic" "go.uber.org/atomic"
"github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/workflows" "github.com/projectdiscovery/nuclei/v2/pkg/workflows"
) )
const workflowExecutionErrorMessageTemplate = "[%s] Could not execute workflow step: %s\n" const workflowStepExecutionError = "[%s] Could not execute workflow step: %s\n"
// executeWorkflow runs a workflow on an input and returns true or false // executeWorkflow runs a workflow on an input and returns true or false
func (e *Engine) executeWorkflow(input string, w *workflows.Workflow) bool { func (e *Engine) executeWorkflow(input string, w *workflows.Workflow) bool {
results := &atomic.Bool{} results := &atomic.Bool{}
// at this point we should be at the start root execution of a workflow tree, hence we create global shared instances
workflowCookieJar, _ := cookiejar.New(nil)
ctxArgs := contextargs.New()
ctxArgs.Input = input
ctxArgs.CookieJar = workflowCookieJar
swg := sizedwaitgroup.New(w.Options.Options.TemplateThreads) swg := sizedwaitgroup.New(w.Options.Options.TemplateThreads)
for _, template := range w.Workflows { for _, template := range w.Workflows {
swg.Add() swg.Add()
func(template *workflows.WorkflowTemplate) { func(template *workflows.WorkflowTemplate) {
if err := e.runWorkflowStep(template, input, results, &swg, w); err != nil { if err := e.runWorkflowStep(template, ctxArgs, results, &swg, w); err != nil {
gologger.Warning().Msgf(workflowExecutionErrorMessageTemplate, template.Template, err) gologger.Warning().Msgf(workflowStepExecutionError, template.Template, err)
} }
swg.Done() swg.Done()
}(template) }(template)
@ -31,7 +41,7 @@ func (e *Engine) executeWorkflow(input string, w *workflows.Workflow) bool {
// runWorkflowStep runs a workflow step for the workflow. It executes the workflow // runWorkflowStep runs a workflow step for the workflow. It executes the workflow
// in a recursive manner running all subtemplates and matchers. // in a recursive manner running all subtemplates and matchers.
func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, input string, results *atomic.Bool, swg *sizedwaitgroup.SizedWaitGroup, w *workflows.Workflow) error { func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, input *contextargs.Context, results *atomic.Bool, swg *sizedwaitgroup.SizedWaitGroup, w *workflows.Workflow) error {
var firstMatched bool var firstMatched bool
var err error var err error
var mainErr error var mainErr error
@ -49,6 +59,25 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, input str
if len(result.Results) > 0 { if len(result.Results) > 0 {
firstMatched = true firstMatched = true
} }
if result.OperatorsResult != nil && result.OperatorsResult.Extracts != nil {
for k, v := range result.OperatorsResult.Extracts {
// normalize items:
switch len(v) {
case 0, 1:
// - key:[item] => key: item
input.Set(k, v[0])
default:
// - key:[item_0, ..., item_n] => key0:item_0, keyn:item_n
for vIdx, vVal := range v {
normalizedKIdx := fmt.Sprintf("%s%d", k, vIdx)
input.Set(normalizedKIdx, vVal)
}
// also add the original name with full slice
input.Set(k, v)
}
}
}
}) })
} else { } else {
var matched bool var matched bool
@ -59,12 +88,12 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, input str
} }
if err != nil { if err != nil {
if w.Options.HostErrorsCache != nil { if w.Options.HostErrorsCache != nil {
w.Options.HostErrorsCache.MarkFailed(input, err) w.Options.HostErrorsCache.MarkFailed(input.Input, err)
} }
if len(template.Executers) == 1 { if len(template.Executers) == 1 {
mainErr = err mainErr = err
} else { } else {
gologger.Warning().Msgf(workflowExecutionErrorMessageTemplate, template.Template, err) gologger.Warning().Msgf(workflowStepExecutionError, template.Template, err)
} }
continue continue
} }
@ -82,6 +111,12 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, input str
return return
} }
if event.OperatorsResult.Extracts != nil {
for k, v := range event.OperatorsResult.Extracts {
input.Set(k, v)
}
}
for _, matcher := range template.Matchers { for _, matcher := range template.Matchers {
if !matcher.Match(event.OperatorsResult) { if !matcher.Match(event.OperatorsResult) {
continue continue
@ -92,7 +127,7 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, input str
go func(subtemplate *workflows.WorkflowTemplate) { go func(subtemplate *workflows.WorkflowTemplate) {
if err := e.runWorkflowStep(subtemplate, input, results, swg, w); err != nil { if err := e.runWorkflowStep(subtemplate, input, results, swg, w); err != nil {
gologger.Warning().Msgf(workflowExecutionErrorMessageTemplate, subtemplate.Template, err) gologger.Warning().Msgf(workflowStepExecutionError, subtemplate.Template, err)
} }
swg.Done() swg.Done()
}(subtemplate) }(subtemplate)
@ -103,7 +138,7 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, input str
if len(template.Executers) == 1 { if len(template.Executers) == 1 {
mainErr = err mainErr = err
} else { } else {
gologger.Warning().Msgf(workflowExecutionErrorMessageTemplate, template.Template, err) gologger.Warning().Msgf(workflowStepExecutionError, template.Template, err)
} }
continue continue
} }
@ -116,7 +151,7 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, input str
go func(template *workflows.WorkflowTemplate) { go func(template *workflows.WorkflowTemplate) {
if err := e.runWorkflowStep(template, input, results, swg, w); err != nil { if err := e.runWorkflowStep(template, input, results, swg, w); err != nil {
gologger.Warning().Msgf(workflowExecutionErrorMessageTemplate, template.Template, err) gologger.Warning().Msgf(workflowStepExecutionError, template.Template, err)
} }
swg.Done() swg.Done()
}(subtemplate) }(subtemplate)

View File

@ -10,6 +10,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/progress" "github.com/projectdiscovery/nuclei/v2/pkg/progress"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/nuclei/v2/pkg/types"
"github.com/projectdiscovery/nuclei/v2/pkg/workflows" "github.com/projectdiscovery/nuclei/v2/pkg/workflows"
) )
@ -178,17 +179,17 @@ func (m *mockExecuter) Requests() int {
} }
// Execute executes the protocol group and returns true or false if results were found. // Execute executes the protocol group and returns true or false if results were found.
func (m *mockExecuter) Execute(input string) (bool, error) { func (m *mockExecuter) Execute(input *contextargs.Context) (bool, error) {
if m.executeHook != nil { if m.executeHook != nil {
m.executeHook(input) m.executeHook(input.Input)
} }
return m.result, nil return m.result, nil
} }
// ExecuteWithResults executes the protocol requests and returns results instead of writing them. // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (m *mockExecuter) ExecuteWithResults(input string, callback protocols.OutputEventCallback) error { func (m *mockExecuter) ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error {
if m.executeHook != nil { if m.executeHook != nil {
m.executeHook(input) m.executeHook(input.Input)
} }
for _, output := range m.outputs { for _, output := range m.outputs {
callback(output) callback(output)

View File

@ -0,0 +1,31 @@
package contextargs
// Args is a generic map with helpers
type Args map[string]interface{}
// Set a key with value
func (args Args) Set(key string, value interface{}) {
args[key] = value
}
// Get the value associated to a key
func (args Args) Get(key string) (interface{}, bool) {
value, ok := args[key]
return value, ok
}
// Has verifies if the map contains the key
func (args Args) Has(key string) bool {
_, ok := args[key]
return ok
}
// IsEmpty verifies if the map is empty
func (Args Args) IsEmpty() bool {
return len(Args) == 0
}
// create a new args map instance
func newArgs() map[string]interface{} {
return make(map[string]interface{})
}

View File

@ -0,0 +1,109 @@
package contextargs
import (
"net/http/cookiejar"
"sync"
"golang.org/x/exp/maps"
)
// Context implements a shared context struct to share information across multiple templates within a workflow
type Context struct {
// Input target for the executor
Input string
// CookieJar shared within workflow's http templates
CookieJar *cookiejar.Jar
// Access to Args must use lock strategies to prevent data races
*sync.RWMutex
// Args is a workflow shared key-value store
args Args
}
// Create a new contextargs instance
func New() *Context {
return &Context{}
}
// Create a new contextargs instance with input string
func NewWithInput(input string) *Context {
return &Context{Input: input}
}
func (ctx *Context) initialize() {
ctx.args = newArgs()
ctx.RWMutex = &sync.RWMutex{}
}
func (ctx *Context) set(key string, value interface{}) {
ctx.Lock()
defer ctx.Unlock()
ctx.args.Set(key, value)
}
// Set the specific key-value pair
func (ctx *Context) Set(key string, value interface{}) {
if !ctx.isInitialized() {
ctx.initialize()
}
ctx.set(key, value)
}
func (ctx *Context) isInitialized() bool {
return ctx.args != nil
}
func (ctx *Context) hasArgs() bool {
return ctx.isInitialized() && !ctx.args.IsEmpty()
}
func (ctx *Context) get(key string) (interface{}, bool) {
ctx.RLock()
defer ctx.RUnlock()
return ctx.args.Get(key)
}
// Get the value with specific key if exists
func (ctx *Context) Get(key string) (interface{}, bool) {
if !ctx.hasArgs() {
return nil, false
}
return ctx.get(key)
}
func (ctx *Context) GetAll() Args {
if !ctx.hasArgs() {
return nil
}
return maps.Clone(ctx.args)
}
func (ctx *Context) ForEach(f func(string, interface{})) {
ctx.RLock()
defer ctx.RUnlock()
for k, v := range ctx.args {
f(k, v)
}
}
func (ctx *Context) has(key string) bool {
ctx.RLock()
defer ctx.RUnlock()
return ctx.args.Has(key)
}
// Has check if the key exists
func (ctx *Context) Has(key string) bool {
return ctx.hasArgs() && ctx.has(key)
}
func (ctx *Context) HasArgs() bool {
return ctx.hasArgs()
}

View File

@ -0,0 +1,4 @@
// Package contextargs implements a generic entity for shared context within workflows
//
// All templates within a workflow shares the same cookiejar and a key-value store with shared items
package contextargs

View File

@ -10,6 +10,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl" "github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl"
"github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/writer" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/writer"
) )
@ -58,10 +59,15 @@ func (e *Executer) Requests() int {
} }
// Execute executes the protocol group and returns true or false if results were found. // Execute executes the protocol group and returns true or false if results were found.
func (e *Executer) Execute(input string) (bool, error) { func (e *Executer) Execute(input *contextargs.Context) (bool, error) {
var results bool var results bool
dynamicValues := make(map[string]interface{}) dynamicValues := make(map[string]interface{})
if input.HasArgs() {
input.ForEach(func(key string, value interface{}) {
dynamicValues[key] = value
})
}
previous := make(map[string]interface{}) previous := make(map[string]interface{})
for _, req := range e.requests { for _, req := range e.requests {
err := req.ExecuteWithResults(input, dynamicValues, previous, func(event *output.InternalWrappedEvent) { err := req.ExecuteWithResults(input, dynamicValues, previous, func(event *output.InternalWrappedEvent) {
@ -95,9 +101,9 @@ func (e *Executer) Execute(input string) (bool, error) {
}) })
if err != nil { if err != nil {
if e.options.HostErrorsCache != nil { if e.options.HostErrorsCache != nil {
e.options.HostErrorsCache.MarkFailed(input, err) e.options.HostErrorsCache.MarkFailed(input.Input, err)
} }
gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input, err) gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input.Input, err)
} }
// If a match was found and stop at first match is set, break out of the loop and return // If a match was found and stop at first match is set, break out of the loop and return
if results && (e.options.StopAtFirstMatch || e.options.Options.StopAtFirstMatch) { if results && (e.options.StopAtFirstMatch || e.options.Options.StopAtFirstMatch) {
@ -108,8 +114,13 @@ func (e *Executer) Execute(input string) (bool, error) {
} }
// ExecuteWithResults executes the protocol requests and returns results instead of writing them. // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEventCallback) error { func (e *Executer) ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error {
dynamicValues := make(map[string]interface{}) dynamicValues := make(map[string]interface{})
if input.HasArgs() {
input.ForEach(func(key string, value interface{}) {
dynamicValues[key] = value
})
}
previous := make(map[string]interface{}) previous := make(map[string]interface{})
var results bool var results bool
@ -136,9 +147,9 @@ func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEve
}) })
if err != nil { if err != nil {
if e.options.HostErrorsCache != nil { if e.options.HostErrorsCache != nil {
e.options.HostErrorsCache.MarkFailed(input, err) e.options.HostErrorsCache.MarkFailed(input.Input, err)
} }
gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input, err) gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input.Input, err)
} }
// If a match was found and stop at first match is set, break out of the loop and return // If a match was found and stop at first match is set, break out of the loop and return
if results && (e.options.StopAtFirstMatch || e.options.Options.StopAtFirstMatch) { if results && (e.options.StopAtFirstMatch || e.options.Options.StopAtFirstMatch) {

View File

@ -12,6 +12,7 @@ import (
"github.com/projectdiscovery/iputil" "github.com/projectdiscovery/iputil"
"github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
@ -30,13 +31,13 @@ func (request *Request) Type() templateTypes.ProtocolType {
} }
// ExecuteWithResults executes the protocol requests and returns results instead of writing them. // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (request *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
// Parse the URL and return domain if URL. // Parse the URL and return domain if URL.
var domain string var domain string
if utils.IsURL(input) { if utils.IsURL(input.Input) {
domain = extractDomain(input) domain = extractDomain(input.Input)
} else { } else {
domain = input domain = input.Input
} }
var err error var err error
@ -45,6 +46,8 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
return errors.Wrap(err, "could not build request") return errors.Wrap(err, "could not build request")
} }
vars := GenerateVariables(domain) vars := GenerateVariables(domain)
// merge with metadata (eg. from workflow context)
vars = generators.MergeMaps(vars, metadata)
variablesMap := request.options.Variables.Evaluate(vars) variablesMap := request.options.Variables.Evaluate(vars)
vars = generators.MergeMaps(variablesMap, vars) vars = generators.MergeMaps(variablesMap, vars)
@ -107,7 +110,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
} }
} }
outputEvent := request.responseToDSLMap(compiledRequest, response, input, input, traceData) outputEvent := request.responseToDSLMap(compiledRequest, response, input.Input, input.Input, traceData)
for k, v := range previous { for k, v := range previous {
outputEvent[k] = v outputEvent[k] = v
} }
@ -115,7 +118,6 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
outputEvent[k] = v outputEvent[k] = v
} }
event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse) event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse)
// TODO: dynamic values are not supported yet
dumpResponse(event, request, request.options, response.String(), domain) dumpResponse(event, request, request.options, response.String(), domain)
if request.Trace { if request.Trace {

View File

@ -11,6 +11,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/testutils"
) )
@ -53,7 +54,9 @@ func TestDNSExecuteWithResults(t *testing.T) {
t.Run("domain-valid", func(t *testing.T) { t.Run("domain-valid", func(t *testing.T) {
metadata := make(output.InternalEvent) metadata := make(output.InternalEvent)
previous := make(output.InternalEvent) previous := make(output.InternalEvent)
err := request.ExecuteWithResults("example.com", metadata, previous, func(event *output.InternalWrappedEvent) { ctxArgs := contextargs.New()
ctxArgs.Input = "example.com"
err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
finalEvent = event finalEvent = event
}) })
require.Nil(t, err, "could not execute dns request") require.Nil(t, err, "could not execute dns request")
@ -68,7 +71,7 @@ func TestDNSExecuteWithResults(t *testing.T) {
t.Run("url-to-domain", func(t *testing.T) { t.Run("url-to-domain", func(t *testing.T) {
metadata := make(output.InternalEvent) metadata := make(output.InternalEvent)
previous := make(output.InternalEvent) previous := make(output.InternalEvent)
err := request.ExecuteWithResults("https://example.com", metadata, previous, func(event *output.InternalWrappedEvent) { err := request.ExecuteWithResults(contextargs.NewWithInput("https://example.com"), metadata, previous, func(event *output.InternalWrappedEvent) {
finalEvent = event finalEvent = event
}) })
require.Nil(t, err, "could not execute dns request") require.Nil(t, err, "could not execute dns request")

View File

@ -18,6 +18,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
@ -44,9 +45,9 @@ type FileMatch struct {
var emptyResultErr = errors.New("Empty result") var emptyResultErr = errors.New("Empty result")
// ExecuteWithResults executes the protocol requests and returns results instead of writing them. // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (request *Request) ExecuteWithResults(input string, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error { func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
wg := sizedwaitgroup.New(request.options.Options.BulkSize) wg := sizedwaitgroup.New(request.options.Options.BulkSize)
err := request.getInputPaths(input, func(filePath string) { err := request.getInputPaths(input.Input, func(filePath string) {
wg.Add() wg.Add()
func(filePath string) { func(filePath string) {
defer wg.Done() defer wg.Done()
@ -62,7 +63,7 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp
// every new file in the compressed multi-file archive counts 1 // every new file in the compressed multi-file archive counts 1
request.options.Progress.AddToTotal(1) request.options.Progress.AddToTotal(1)
archiveFileName := filepath.Join(filePath, file.Name()) archiveFileName := filepath.Join(filePath, file.Name())
event, fileMatches, err := request.processReader(file.ReadCloser, archiveFileName, input, file.Size(), previous) event, fileMatches, err := request.processReader(file.ReadCloser, archiveFileName, input.Input, file.Size(), previous)
if err != nil { if err != nil {
if errors.Is(err, emptyResultErr) { if errors.Is(err, emptyResultErr) {
// no matches but one file elaborated // no matches but one file elaborated
@ -115,7 +116,7 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp
_ = tmpFileOut.Sync() _ = tmpFileOut.Sync()
// rewind the file // rewind the file
_, _ = tmpFileOut.Seek(0, 0) _, _ = tmpFileOut.Seek(0, 0)
event, fileMatches, err := request.processReader(tmpFileOut, filePath, input, fileStat.Size(), previous) event, fileMatches, err := request.processReader(tmpFileOut, filePath, input.Input, fileStat.Size(), previous)
if err != nil { if err != nil {
if errors.Is(err, emptyResultErr) { if errors.Is(err, emptyResultErr) {
// no matches but one file elaborated // no matches but one file elaborated
@ -135,7 +136,7 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp
default: default:
// normal file - increments the counter by 1 // normal file - increments the counter by 1
request.options.Progress.AddToTotal(1) request.options.Progress.AddToTotal(1)
event, fileMatches, err := request.processFile(filePath, input, previous) event, fileMatches, err := request.processFile(filePath, input.Input, previous)
if err != nil { if err != nil {
if errors.Is(err, emptyResultErr) { if errors.Is(err, emptyResultErr) {
// no matches but one file elaborated // no matches but one file elaborated
@ -157,7 +158,7 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp
wg.Wait() wg.Wait()
if err != nil { if err != nil {
request.options.Output.Request(request.options.TemplatePath, input, request.Type().String(), err) request.options.Output.Request(request.options.TemplatePath, input.Input, request.Type().String(), err)
request.options.Progress.IncrementFailedRequestsBy(1) request.options.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, "could not send file request") return errors.Wrap(err, "could not send file request")
} }

View File

@ -13,6 +13,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/testutils"
) )
@ -65,7 +66,9 @@ func TestFileExecuteWithResults(t *testing.T) {
t.Run("valid", func(t *testing.T) { t.Run("valid", func(t *testing.T) {
metadata := make(output.InternalEvent) metadata := make(output.InternalEvent)
previous := make(output.InternalEvent) previous := make(output.InternalEvent)
err := request.ExecuteWithResults(tempDir, metadata, previous, func(event *output.InternalWrappedEvent) { ctxArgs := contextargs.New()
ctxArgs.Input = tempDir
err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
finalEvent = event finalEvent = event
}) })
require.Nil(t, err, "could not execute file request") require.Nil(t, err, "could not execute file request")

View File

@ -10,6 +10,7 @@ import (
"github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
@ -29,7 +30,8 @@ func (request *Request) Type() templateTypes.ProtocolType {
} }
// ExecuteWithResults executes the protocol requests and returns results instead of writing them. // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (request *Request) ExecuteWithResults(inputURL string, metadata, previous output.InternalEvent /*TODO review unused parameter*/, callback protocols.OutputEventCallback) error { func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, previous output.InternalEvent /*TODO review unused parameter*/, callback protocols.OutputEventCallback) error {
inputURL := input.Input
if request.options.Browser.UserAgent() == "" { if request.options.Browser.UserAgent() == "" {
request.options.Browser.SetUserAgent(request.compiledUserAgent) request.options.Browser.SetUserAgent(request.compiledUserAgent)
} }

View File

@ -59,6 +59,7 @@ func Init(options *types.Options) error {
type ConnectionConfiguration struct { type ConnectionConfiguration struct {
// DisableKeepAlive of the connection // DisableKeepAlive of the connection
DisableKeepAlive bool DisableKeepAlive bool
Cookiejar *cookiejar.Jar
} }
// Configuration contains the custom configuration options for a client // Configuration contains the custom configuration options for a client
@ -241,7 +242,9 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl
} }
var jar *cookiejar.Jar var jar *cookiejar.Jar
if configuration.CookieReuse { if configuration.Connection != nil && configuration.Connection.Cookiejar != nil {
jar = configuration.Connection.Cookiejar
} else if configuration.CookieReuse {
if jar, err = cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List}); err != nil { if jar, err = cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List}); err != nil {
return nil, errors.Wrap(err, "could not create cookiejar") return nil, errors.Wrap(err, "could not create cookiejar")
} }

View File

@ -22,6 +22,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
@ -45,7 +46,8 @@ func (request *Request) Type() templateTypes.ProtocolType {
} }
// executeRaceRequest executes race condition request for a URL // executeRaceRequest executes race condition request for a URL
func (request *Request) executeRaceRequest(reqURL string, previous output.InternalEvent, callback protocols.OutputEventCallback) error { func (request *Request) executeRaceRequest(input *contextargs.Context, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
reqURL := input.Input
var generatedRequests []*generatedRequest var generatedRequests []*generatedRequest
// Requests within race condition should be dumped once and the output prefilled to allow DSL language to work // Requests within race condition should be dumped once and the output prefilled to allow DSL language to work
@ -98,7 +100,7 @@ func (request *Request) executeRaceRequest(reqURL string, previous output.Intern
wg.Add(1) wg.Add(1)
go func(httpRequest *generatedRequest) { go func(httpRequest *generatedRequest) {
defer wg.Done() defer wg.Done()
err := request.executeRequest(reqURL, httpRequest, previous, false, callback, 0) err := request.executeRequest(input, httpRequest, previous, false, callback, 0)
mutex.Lock() mutex.Lock()
if err != nil { if err != nil {
requestErr = multierr.Append(requestErr, err) requestErr = multierr.Append(requestErr, err)
@ -113,7 +115,7 @@ func (request *Request) executeRaceRequest(reqURL string, previous output.Intern
} }
// executeRaceRequest executes parallel requests for a template // executeRaceRequest executes parallel requests for a template
func (request *Request) executeParallelHTTP(reqURL string, dynamicValues output.InternalEvent, callback protocols.OutputEventCallback) error { func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicValues output.InternalEvent, callback protocols.OutputEventCallback) error {
generator := request.newGenerator() generator := request.newGenerator()
// Workers that keeps enqueuing new requests // Workers that keeps enqueuing new requests
@ -127,7 +129,7 @@ func (request *Request) executeParallelHTTP(reqURL string, dynamicValues output.
if !ok { if !ok {
break break
} }
generatedHttpRequest, err := generator.Make(context.Background(), reqURL, inputData, payloads, dynamicValues) generatedHttpRequest, err := generator.Make(context.Background(), input.Input, inputData, payloads, dynamicValues)
if err != nil { if err != nil {
if err == io.EOF { if err == io.EOF {
break break
@ -135,8 +137,8 @@ func (request *Request) executeParallelHTTP(reqURL string, dynamicValues output.
request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
return err return err
} }
if reqURL == "" { if input.Input == "" {
reqURL = generatedHttpRequest.URL() input.Input = generatedHttpRequest.URL()
} }
swg.Add() swg.Add()
go func(httpRequest *generatedRequest) { go func(httpRequest *generatedRequest) {
@ -145,7 +147,7 @@ func (request *Request) executeParallelHTTP(reqURL string, dynamicValues output.
request.options.RateLimiter.Take() request.options.RateLimiter.Take()
previous := make(map[string]interface{}) previous := make(map[string]interface{})
err := request.executeRequest(reqURL, httpRequest, previous, false, callback, 0) err := request.executeRequest(input, httpRequest, previous, false, callback, 0)
mutex.Lock() mutex.Lock()
if err != nil { if err != nil {
requestErr = multierr.Append(requestErr, err) requestErr = multierr.Append(requestErr, err)
@ -159,11 +161,11 @@ func (request *Request) executeParallelHTTP(reqURL string, dynamicValues output.
} }
// executeTurboHTTP executes turbo http request for a URL // executeTurboHTTP executes turbo http request for a URL
func (request *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
generator := request.newGenerator() generator := request.newGenerator()
// need to extract the target from the url // need to extract the target from the url
URL, err := url.Parse(reqURL) URL, err := url.Parse(input.Input)
if err != nil { if err != nil {
return err return err
} }
@ -194,20 +196,20 @@ func (request *Request) executeTurboHTTP(reqURL string, dynamicValues, previous
if !ok { if !ok {
break break
} }
generatedHttpRequest, err := generator.Make(context.Background(), reqURL, inputData, payloads, dynamicValues) generatedHttpRequest, err := generator.Make(context.Background(), input.Input, inputData, payloads, dynamicValues)
if err != nil { if err != nil {
request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
return err return err
} }
if reqURL == "" { if input.Input == "" {
reqURL = generatedHttpRequest.URL() input.Input = generatedHttpRequest.URL()
} }
generatedHttpRequest.pipelinedClient = pipeClient generatedHttpRequest.pipelinedClient = pipeClient
swg.Add() swg.Add()
go func(httpRequest *generatedRequest) { go func(httpRequest *generatedRequest) {
defer swg.Done() defer swg.Done()
err := request.executeRequest(reqURL, httpRequest, previous, false, callback, 0) err := request.executeRequest(input, httpRequest, previous, false, callback, 0)
mutex.Lock() mutex.Lock()
if err != nil { if err != nil {
requestErr = multierr.Append(requestErr, err) requestErr = multierr.Append(requestErr, err)
@ -221,24 +223,24 @@ func (request *Request) executeTurboHTTP(reqURL string, dynamicValues, previous
} }
// ExecuteWithResults executes the final request on a URL // ExecuteWithResults executes the final request on a URL
func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
if request.Pipeline || request.Race && request.RaceNumberRequests > 0 || request.Threads > 0 { if request.Pipeline || request.Race && request.RaceNumberRequests > 0 || request.Threads > 0 {
variablesMap := request.options.Variables.Evaluate(generators.MergeMaps(dynamicValues, previous)) variablesMap := request.options.Variables.Evaluate(generators.MergeMaps(dynamicValues, previous))
dynamicValues = generators.MergeMaps(variablesMap, dynamicValues) dynamicValues = generators.MergeMaps(variablesMap, dynamicValues)
} }
// verify if pipeline was requested // verify if pipeline was requested
if request.Pipeline { if request.Pipeline {
return request.executeTurboHTTP(reqURL, dynamicValues, previous, callback) return request.executeTurboHTTP(input, dynamicValues, previous, callback)
} }
// verify if a basic race condition was requested // verify if a basic race condition was requested
if request.Race && request.RaceNumberRequests > 0 { if request.Race && request.RaceNumberRequests > 0 {
return request.executeRaceRequest(reqURL, dynamicValues, callback) return request.executeRaceRequest(input, dynamicValues, callback)
} }
// verify if parallel elaboration was requested // verify if parallel elaboration was requested
if request.Threads > 0 { if request.Threads > 0 {
return request.executeParallelHTTP(reqURL, dynamicValues, callback) return request.executeParallelHTTP(input, dynamicValues, callback)
} }
generator := request.newGenerator() generator := request.newGenerator()
@ -257,7 +259,7 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(request.options.Options.Timeout)*time.Second) ctx, cancel := context.WithTimeout(context.Background(), time.Duration(request.options.Options.Timeout)*time.Second)
defer cancel() defer cancel()
generatedHttpRequest, err := generator.Make(ctx, reqURL, data, payloads, dynamicValue) generatedHttpRequest, err := generator.Make(ctx, input.Input, data, payloads, dynamicValue)
if err != nil { if err != nil {
if err == io.EOF { if err == io.EOF {
return true, nil return true, nil
@ -270,15 +272,15 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou
generatedHttpRequest.interactshURLs = append(generatedHttpRequest.interactshURLs, interactURLs...) generatedHttpRequest.interactshURLs = append(generatedHttpRequest.interactshURLs, interactURLs...)
} }
hasInteractMarkers := interactsh.HasMarkers(data) || len(generatedHttpRequest.interactshURLs) > 0 hasInteractMarkers := interactsh.HasMarkers(data) || len(generatedHttpRequest.interactshURLs) > 0
if reqURL == "" { if input.Input == "" {
reqURL = generatedHttpRequest.URL() input.Input = generatedHttpRequest.URL()
} }
// Check if hosts keep erroring // Check if hosts keep erroring
if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(reqURL) { if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(input.Input) {
return true, nil return true, nil
} }
var gotMatches bool var gotMatches bool
err = request.executeRequest(reqURL, generatedHttpRequest, previous, hasInteractMatchers, func(event *output.InternalWrappedEvent) { err = request.executeRequest(input, generatedHttpRequest, previous, hasInteractMatchers, func(event *output.InternalWrappedEvent) {
// Add the extracts to the dynamic values if any. // Add the extracts to the dynamic values if any.
if event.OperatorsResult != nil { if event.OperatorsResult != nil {
gotMatches = event.OperatorsResult.Matched gotMatches = event.OperatorsResult.Matched
@ -303,7 +305,7 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou
} }
if err != nil { if err != nil {
if request.options.HostErrorsCache != nil { if request.options.HostErrorsCache != nil {
request.options.HostErrorsCache.MarkFailed(reqURL, err) request.options.HostErrorsCache.MarkFailed(input.Input, err)
} }
requestErr = err requestErr = err
} }
@ -347,7 +349,7 @@ const drainReqSize = int64(8 * 1024)
var errStopExecution = errors.New("stop execution due to unresolved variables") var errStopExecution = errors.New("stop execution due to unresolved variables")
// executeRequest executes the actual generated request and returns error if occurred // executeRequest executes the actual generated request and returns error if occurred
func (request *Request) executeRequest(reqURL string, generatedRequest *generatedRequest, previousEvent output.InternalEvent, hasInteractMatchers bool, callback protocols.OutputEventCallback, requestCount int) error { func (request *Request) executeRequest(input *contextargs.Context, generatedRequest *generatedRequest, previousEvent output.InternalEvent, hasInteractMatchers bool, callback protocols.OutputEventCallback, requestCount int) error {
request.setCustomHeaders(generatedRequest) request.setCustomHeaders(generatedRequest)
// Try to evaluate any payloads before replacement // Try to evaluate any payloads before replacement
@ -375,7 +377,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
if !generatedRequest.original.Race { if !generatedRequest.original.Race {
var dumpError error var dumpError error
// TODO: dump is currently not working with post-processors - somehow it alters the signature // TODO: dump is currently not working with post-processors - somehow it alters the signature
dumpedRequest, dumpError = dump(generatedRequest, reqURL) dumpedRequest, dumpError = dump(generatedRequest, input.Input)
if dumpError != nil { if dumpError != nil {
return dumpError return dumpError
} }
@ -383,12 +385,12 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
if ignoreList := GetVariablesNamesSkipList(generatedRequest.original.Signature.Value); ignoreList != nil { if ignoreList := GetVariablesNamesSkipList(generatedRequest.original.Signature.Value); ignoreList != nil {
if varErr := expressions.ContainsVariablesWithIgnoreList(ignoreList, dumpedRequestString); varErr != nil && !request.SkipVariablesCheck { if varErr := expressions.ContainsVariablesWithIgnoreList(ignoreList, dumpedRequestString); varErr != nil && !request.SkipVariablesCheck {
gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, reqURL, varErr) gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, input.Input, varErr)
return errStopExecution return errStopExecution
} }
} else { // Check if are there any unresolved variables. If yes, skip unless overridden by user. } else { // Check if are there any unresolved variables. If yes, skip unless overridden by user.
if varErr := expressions.ContainsUnresolvedVariables(dumpedRequestString); varErr != nil && !request.SkipVariablesCheck { if varErr := expressions.ContainsUnresolvedVariables(dumpedRequestString); varErr != nil && !request.SkipVariablesCheck {
gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, reqURL, varErr) gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, input.Input, varErr)
return errStopExecution return errStopExecution
} }
} }
@ -402,7 +404,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
if parsed, parseErr := url.Parse(formedURL); parseErr == nil { if parsed, parseErr := url.Parse(formedURL); parseErr == nil {
hostname = parsed.Host hostname = parsed.Host
} }
resp, err = generatedRequest.pipelinedClient.DoRaw(generatedRequest.rawRequest.Method, reqURL, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data))) resp, err = generatedRequest.pipelinedClient.DoRaw(generatedRequest.rawRequest.Method, input.Input, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)))
} else if generatedRequest.request != nil { } else if generatedRequest.request != nil {
resp, err = generatedRequest.pipelinedClient.Dor(generatedRequest.request) resp, err = generatedRequest.pipelinedClient.Dor(generatedRequest.request)
} }
@ -410,7 +412,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
formedURL = generatedRequest.rawRequest.FullURL formedURL = generatedRequest.rawRequest.FullURL
// use request url as matched url if empty // use request url as matched url if empty
if formedURL == "" { if formedURL == "" {
formedURL = reqURL formedURL = input.Input
} }
if parsed, parseErr := url.Parse(formedURL); parseErr == nil { if parsed, parseErr := url.Parse(formedURL); parseErr == nil {
hostname = parsed.Host hostname = parsed.Host
@ -420,7 +422,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
options.CustomRawBytes = generatedRequest.rawRequest.UnsafeRawBytes options.CustomRawBytes = generatedRequest.rawRequest.UnsafeRawBytes
options.ForceReadAllBody = request.ForceReadAllBody options.ForceReadAllBody = request.ForceReadAllBody
options.SNI = request.options.Options.SNI options.SNI = request.options.Options.SNI
resp, err = generatedRequest.original.rawhttpClient.DoRawWithOptions(generatedRequest.rawRequest.Method, reqURL, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)), &options) resp, err = generatedRequest.original.rawhttpClient.DoRawWithOptions(generatedRequest.rawRequest.Method, input.Input, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)), &options)
} else { } else {
hostname = generatedRequest.request.URL.Host hostname = generatedRequest.request.URL.Host
formedURL = generatedRequest.request.URL.String() formedURL = generatedRequest.request.URL.String()
@ -437,18 +439,29 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
if errSignature := request.handleSignature(generatedRequest); errSignature != nil { if errSignature := request.handleSignature(generatedRequest); errSignature != nil {
return errSignature return errSignature
} }
resp, err = request.httpClient.Do(generatedRequest.request)
httpclient := request.httpClient
if input.CookieJar != nil {
connConfiguration := request.connConfiguration
connConfiguration.Connection.Cookiejar = input.CookieJar
client, err := httpclientpool.Get(request.options.Options, connConfiguration)
if err != nil {
return errors.Wrap(err, "could not get http client")
}
httpclient = client
}
resp, err = httpclient.Do(generatedRequest.request)
} }
} }
// use request url as matched url if empty // use request url as matched url if empty
if formedURL == "" { if formedURL == "" {
formedURL = reqURL formedURL = input.Input
} }
// Dump the requests containing all headers // Dump the requests containing all headers
if !generatedRequest.original.Race { if !generatedRequest.original.Race {
var dumpError error var dumpError error
dumpedRequest, dumpError = dump(generatedRequest, reqURL) dumpedRequest, dumpError = dump(generatedRequest, input.Input)
if dumpError != nil { if dumpError != nil {
return dumpError return dumpError
} }
@ -461,7 +474,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
gologger.Print().Msgf("%s", dumpedRequestString) gologger.Print().Msgf("%s", dumpedRequestString)
} }
if request.options.Options.StoreResponse { if request.options.Options.StoreResponse {
request.options.Output.WriteStoreDebugData(reqURL, request.options.TemplateID, request.Type().String(), fmt.Sprintf("%s\n%s", msg, dumpedRequestString)) request.options.Output.WriteStoreDebugData(input.Input, request.options.TemplateID, request.Type().String(), fmt.Sprintf("%s\n%s", msg, dumpedRequestString))
} }
} }
} }
@ -477,7 +490,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
// If we have interactsh markers and request times out, still send // If we have interactsh markers and request times out, still send
// a callback event so in case we receive an interaction, correlation is possible. // a callback event so in case we receive an interaction, correlation is possible.
if hasInteractMatchers { if hasInteractMatchers {
outputEvent := request.responseToDSLMap(&http.Response{}, reqURL, formedURL, tostring.UnsafeToString(dumpedRequest), "", "", "", 0, generatedRequest.meta) outputEvent := request.responseToDSLMap(&http.Response{}, input.Input, formedURL, tostring.UnsafeToString(dumpedRequest), "", "", "", 0, generatedRequest.meta)
if i := strings.LastIndex(hostname, ":"); i != -1 { if i := strings.LastIndex(hostname, ":"); i != -1 {
hostname = hostname[:i] hostname = hostname[:i]
} }
@ -559,7 +572,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
if response.resp == nil { if response.resp == nil {
continue // Skip nil responses continue // Skip nil responses
} }
matchedURL := reqURL matchedURL := input.Input
if generatedRequest.rawRequest != nil && generatedRequest.rawRequest.FullURL != "" { if generatedRequest.rawRequest != nil && generatedRequest.rawRequest.FullURL != "" {
matchedURL = generatedRequest.rawRequest.FullURL matchedURL = generatedRequest.rawRequest.FullURL
} }
@ -574,7 +587,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
} }
finalEvent := make(output.InternalEvent) finalEvent := make(output.InternalEvent)
outputEvent := request.responseToDSLMap(response.resp, reqURL, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(response.fullResponse), tostring.UnsafeToString(response.body), tostring.UnsafeToString(response.headers), duration, generatedRequest.meta) outputEvent := request.responseToDSLMap(response.resp, input.Input, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(response.fullResponse), tostring.UnsafeToString(response.body), tostring.UnsafeToString(response.headers), duration, generatedRequest.meta)
if i := strings.LastIndex(hostname, ":"); i != -1 { if i := strings.LastIndex(hostname, ":"); i != -1 {
hostname = hostname[:i] hostname = hostname[:i]
} }
@ -610,7 +623,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate
responseContentType := resp.Header.Get("Content-Type") responseContentType := resp.Header.Get("Content-Type")
isResponseTruncated := request.MaxSize > 0 && len(gotData) >= request.MaxSize isResponseTruncated := request.MaxSize > 0 && len(gotData) >= request.MaxSize
dumpResponse(event, request, response.fullResponse, formedURL, responseContentType, isResponseTruncated, reqURL) dumpResponse(event, request, response.fullResponse, formedURL, responseContentType, isResponseTruncated, input.Input)
callback(event) callback(event)

View File

@ -14,6 +14,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/testutils"
) )
@ -81,7 +82,9 @@ Disallow: /c`))
t.Run("test", func(t *testing.T) { t.Run("test", func(t *testing.T) {
metadata := make(output.InternalEvent) metadata := make(output.InternalEvent)
previous := make(output.InternalEvent) previous := make(output.InternalEvent)
err := request.ExecuteWithResults(ts.URL, metadata, previous, func(event *output.InternalWrappedEvent) { ctxArgs := contextargs.New()
ctxArgs.Input = ts.URL
err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
if event.OperatorsResult != nil && event.OperatorsResult.Matched { if event.OperatorsResult != nil && event.OperatorsResult.Matched {
matchCount++ matchCount++
} }

View File

@ -17,6 +17,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
@ -35,17 +36,17 @@ func (request *Request) Type() templateTypes.ProtocolType {
} }
// ExecuteWithResults executes the protocol requests and returns results instead of writing them. // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (request *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
var address string var address string
var err error var err error
if request.SelfContained { if request.SelfContained {
address = "" address = ""
} else { } else {
address, err = getAddress(input) address, err = getAddress(input.Input)
} }
if err != nil { if err != nil {
request.options.Output.Request(request.options.TemplatePath, input, request.Type().String(), err) request.options.Output.Request(request.options.TemplatePath, input.Input, request.Type().String(), err)
request.options.Progress.IncrementFailedRequestsBy(1) request.options.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, "could not get address from url") return errors.Wrap(err, "could not get address from url")
} }
@ -56,7 +57,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
variables = generators.MergeMaps(variablesMap, variables) variables = generators.MergeMaps(variablesMap, variables)
actualAddress := replacer.Replace(kv.address, variables) actualAddress := replacer.Replace(kv.address, variables)
if err := request.executeAddress(variables, actualAddress, address, input, kv.tls, previous, callback); err != nil { if err := request.executeAddress(variables, actualAddress, address, input.Input, kv.tls, previous, callback); err != nil {
gologger.Warning().Msgf("Could not make network request for %s: %s\n", actualAddress, err) gologger.Warning().Msgf("Could not make network request for %s: %s\n", actualAddress, err)
continue continue
} }

View File

@ -16,6 +16,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/testutils"
) )
@ -64,7 +65,9 @@ func TestNetworkExecuteWithResults(t *testing.T) {
t.Run("domain-valid", func(t *testing.T) { t.Run("domain-valid", func(t *testing.T) {
metadata := make(output.InternalEvent) metadata := make(output.InternalEvent)
previous := make(output.InternalEvent) previous := make(output.InternalEvent)
err := request.ExecuteWithResults(parsed.Host, metadata, previous, func(event *output.InternalWrappedEvent) { ctxArgs := contextargs.New()
ctxArgs.Input = parsed.Host
err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
finalEvent = event finalEvent = event
}) })
require.Nil(t, err, "could not execute network request") require.Nil(t, err, "could not execute network request")
@ -79,7 +82,9 @@ func TestNetworkExecuteWithResults(t *testing.T) {
t.Run("invalid-port-override", func(t *testing.T) { t.Run("invalid-port-override", func(t *testing.T) {
metadata := make(output.InternalEvent) metadata := make(output.InternalEvent)
previous := make(output.InternalEvent) previous := make(output.InternalEvent)
err := request.ExecuteWithResults("127.0.0.1:11211", metadata, previous, func(event *output.InternalWrappedEvent) { ctxArgs := contextargs.New()
ctxArgs.Input = "127.0.0.1:11211"
err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
finalEvent = event finalEvent = event
}) })
require.Nil(t, err, "could not execute network request") require.Nil(t, err, "could not execute network request")
@ -92,7 +97,9 @@ func TestNetworkExecuteWithResults(t *testing.T) {
t.Run("hex-to-string", func(t *testing.T) { t.Run("hex-to-string", func(t *testing.T) {
metadata := make(output.InternalEvent) metadata := make(output.InternalEvent)
previous := make(output.InternalEvent) previous := make(output.InternalEvent)
err := request.ExecuteWithResults(parsed.Host, metadata, previous, func(event *output.InternalWrappedEvent) { ctxArgs := contextargs.New()
ctxArgs.Input = parsed.Host
err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
finalEvent = event finalEvent = event
}) })
require.Nil(t, err, "could not execute network request") require.Nil(t, err, "could not execute network request")

View File

@ -13,6 +13,7 @@ import (
"github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring"
templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
@ -28,10 +29,10 @@ func (request *Request) Type() templateTypes.ProtocolType {
} }
// ExecuteWithResults executes the protocol requests and returns results instead of writing them. // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (request *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
wg := sizedwaitgroup.New(request.options.Options.BulkSize) wg := sizedwaitgroup.New(request.options.Options.BulkSize)
err := request.getInputPaths(input, func(data string) { err := request.getInputPaths(input.Input, func(data string) {
wg.Add() wg.Add()
go func(data string) { go func(data string) {
@ -97,7 +98,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review
}) })
wg.Wait() wg.Wait()
if err != nil { if err != nil {
request.options.Output.Request(request.options.TemplatePath, input, "file", err) request.options.Output.Request(request.options.TemplatePath, input.Input, "file", err)
request.options.Progress.IncrementFailedRequestsBy(1) request.options.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, "could not send file request") return errors.Wrap(err, "could not send file request")
} }

View File

@ -13,6 +13,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/progress" "github.com/projectdiscovery/nuclei/v2/pkg/progress"
"github.com/projectdiscovery/nuclei/v2/pkg/projectfile" "github.com/projectdiscovery/nuclei/v2/pkg/projectfile"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/excludematchers" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/excludematchers"
@ -30,9 +31,9 @@ type Executer interface {
// Requests returns the total number of requests the rule will perform // Requests returns the total number of requests the rule will perform
Requests() int Requests() int
// Execute executes the protocol group and returns true or false if results were found. // Execute executes the protocol group and returns true or false if results were found.
Execute(input string) (bool, error) Execute(input *contextargs.Context) (bool, error)
// ExecuteWithResults executes the protocol requests and returns results instead of writing them. // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
ExecuteWithResults(input string, callback OutputEventCallback) error ExecuteWithResults(input *contextargs.Context, callback OutputEventCallback) error
} }
// ExecuterOptions contains the configuration options for executer clients // ExecuterOptions contains the configuration options for executer clients
@ -100,7 +101,7 @@ type Request interface {
// Extract performs extracting operation for an extractor on model and returns true or false. // Extract performs extracting operation for an extractor on model and returns true or false.
Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{}
// ExecuteWithResults executes the protocol requests and returns results instead of writing them. // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback OutputEventCallback) error ExecuteWithResults(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback OutputEventCallback) error
// MakeResultEventItem creates a result event from internal wrapped event. Intended to be used by MakeResultEventItem internally // MakeResultEventItem creates a result event from internal wrapped event. Intended to be used by MakeResultEventItem internally
MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent
// MakeResultEvent creates a flat list of result events from an internal wrapped event, based on successful matchers and extracted data // MakeResultEvent creates a flat list of result events from an internal wrapped event, based on successful matchers and extracted data

View File

@ -18,6 +18,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
@ -125,8 +126,8 @@ func (request *Request) GetID() string {
} }
// ExecuteWithResults executes the protocol requests and returns results instead of writing them. // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (request *Request) ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
address, err := getAddress(input) address, err := getAddress(input.Input)
if err != nil { if err != nil {
return nil return nil
} }
@ -152,7 +153,7 @@ func (request *Request) ExecuteWithResults(input string, dynamicValues, previous
finalAddress, dataErr := expressions.EvaluateByte([]byte(request.Address), payloadValues) finalAddress, dataErr := expressions.EvaluateByte([]byte(request.Address), payloadValues)
if dataErr != nil { if dataErr != nil {
requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), dataErr) requestOptions.Output.Request(requestOptions.TemplateID, input.Input, request.Type().String(), dataErr)
requestOptions.Progress.IncrementFailedRequestsBy(1) requestOptions.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(dataErr, "could not evaluate template expressions") return errors.Wrap(dataErr, "could not evaluate template expressions")
} }
@ -164,7 +165,7 @@ func (request *Request) ExecuteWithResults(input string, dynamicValues, previous
response, err := request.tlsx.Connect(host, host, port) response, err := request.tlsx.Connect(host, host, port)
if err != nil { if err != nil {
requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err) requestOptions.Output.Request(requestOptions.TemplateID, input.Input, request.Type().String(), err)
requestOptions.Progress.IncrementFailedRequestsBy(1) requestOptions.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, "could not connect to server") return errors.Wrap(err, "could not connect to server")
} }
@ -173,12 +174,12 @@ func (request *Request) ExecuteWithResults(input string, dynamicValues, previous
gologger.Verbose().Msgf("Sent SSL request to %s", address) gologger.Verbose().Msgf("Sent SSL request to %s", address)
if requestOptions.Options.Debug || requestOptions.Options.DebugRequests || requestOptions.Options.StoreResponse { if requestOptions.Options.Debug || requestOptions.Options.DebugRequests || requestOptions.Options.StoreResponse {
msg := fmt.Sprintf("[%s] Dumped SSL request for %s", requestOptions.TemplateID, input) msg := fmt.Sprintf("[%s] Dumped SSL request for %s", requestOptions.TemplateID, input.Input)
if requestOptions.Options.Debug || requestOptions.Options.DebugRequests { if requestOptions.Options.Debug || requestOptions.Options.DebugRequests {
gologger.Debug().Str("address", input).Msg(msg) gologger.Debug().Str("address", input.Input).Msg(msg)
} }
if requestOptions.Options.StoreResponse { if requestOptions.Options.StoreResponse {
request.options.Output.WriteStoreDebugData(input, request.options.TemplateID, request.Type().String(), msg) request.options.Output.WriteStoreDebugData(input.Input, request.options.TemplateID, request.Type().String(), msg)
} }
} }
@ -219,13 +220,13 @@ func (request *Request) ExecuteWithResults(input string, dynamicValues, previous
event := eventcreator.CreateEvent(request, data, requestOptions.Options.Debug || requestOptions.Options.DebugResponse) event := eventcreator.CreateEvent(request, data, requestOptions.Options.Debug || requestOptions.Options.DebugResponse)
if requestOptions.Options.Debug || requestOptions.Options.DebugResponse || requestOptions.Options.StoreResponse { if requestOptions.Options.Debug || requestOptions.Options.DebugResponse || requestOptions.Options.StoreResponse {
msg := fmt.Sprintf("[%s] Dumped SSL response for %s", requestOptions.TemplateID, input) msg := fmt.Sprintf("[%s] Dumped SSL response for %s", requestOptions.TemplateID, input.Input)
if requestOptions.Options.Debug || requestOptions.Options.DebugResponse { if requestOptions.Options.Debug || requestOptions.Options.DebugResponse {
gologger.Debug().Msg(msg) gologger.Debug().Msg(msg)
gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, jsonDataString, requestOptions.Options.NoColor, false)) gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, jsonDataString, requestOptions.Options.NoColor, false))
} }
if requestOptions.Options.StoreResponse { if requestOptions.Options.StoreResponse {
request.options.Output.WriteStoreDebugData(input, request.options.TemplateID, request.Type().String(), fmt.Sprintf("%s\n%s", msg, jsonDataString)) request.options.Output.WriteStoreDebugData(input.Input, request.options.TemplateID, request.Type().String(), fmt.Sprintf("%s\n%s", msg, jsonDataString))
} }
} }
callback(event) callback(event)

View File

@ -8,6 +8,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/testutils"
) )
@ -27,7 +28,9 @@ func TestSSLProtocol(t *testing.T) {
require.Nil(t, err, "could not compile ssl request") require.Nil(t, err, "could not compile ssl request")
var gotEvent output.InternalEvent var gotEvent output.InternalEvent
err = request.ExecuteWithResults("google.com:443", nil, nil, func(event *output.InternalWrappedEvent) { ctxArgs := contextargs.New()
ctxArgs.Input = "google.com:443"
err = request.ExecuteWithResults(ctxArgs, nil, nil, func(event *output.InternalWrappedEvent) {
gotEvent = event.InternalEvent gotEvent = event.InternalEvent
}) })
require.Nil(t, err, "could not run ssl request") require.Nil(t, err, "could not run ssl request")

View File

@ -23,6 +23,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
@ -135,8 +136,8 @@ func (request *Request) GetID() string {
} }
// ExecuteWithResults executes the protocol requests and returns results instead of writing them. // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (request *Request) ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
hostname, err := getAddress(input) hostname, err := getAddress(input.Input)
if err != nil { if err != nil {
return err return err
} }
@ -149,13 +150,13 @@ func (request *Request) ExecuteWithResults(input string, dynamicValues, previous
if !ok { if !ok {
break break
} }
if err := request.executeRequestWithPayloads(input, hostname, value, previous, callback); err != nil { if err := request.executeRequestWithPayloads(input.Input, hostname, value, previous, callback); err != nil {
return err return err
} }
} }
} else { } else {
value := make(map[string]interface{}) value := make(map[string]interface{})
if err := request.executeRequestWithPayloads(input, hostname, value, previous, callback); err != nil { if err := request.executeRequestWithPayloads(input.Input, hostname, value, previous, callback); err != nil {
return err return err
} }
} }

View File

@ -15,6 +15,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer"
@ -86,9 +87,9 @@ func (request *Request) GetID() string {
} }
// ExecuteWithResults executes the protocol requests and returns results instead of writing them. // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (request *Request) ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
// generate variables // generate variables
variables := generateVariables(input) variables := generateVariables(input.Input)
if request.options.Options.Debug || request.options.Options.DebugRequests { if request.options.Options.Debug || request.options.Options.DebugRequests {
gologger.Debug().Msgf("Protocol request variables: \n%s\n", vardump.DumpVariables(variables)) gologger.Debug().Msgf("Protocol request variables: \n%s\n", vardump.DumpVariables(variables))

View File

@ -11,6 +11,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/writer" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/writer"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http"
) )
@ -29,10 +30,10 @@ import (
// The equality check is performed as described below - // The equality check is performed as described below -
// //
// Cases where clustering is not perfomed (request is considered different) // Cases where clustering is not perfomed (request is considered different)
// - If request contains payloads,raw,body,unsafe,req-condition,name attributes // - If request contains payloads,raw,body,unsafe,req-condition,name attributes
// - If request methods,max-redirects,cookie-reuse,redirects are not equal // - If request methods,max-redirects,cookie-reuse,redirects are not equal
// - If request paths aren't identical. // - If request paths aren't identical.
// - If request headers aren't identical // - If request headers aren't identical
// //
// If multiple requests are identified as identical, they are appended to a slice. // If multiple requests are identified as identical, they are appended to a slice.
// Finally, the engine creates a single executer with a clusteredexecuter for all templates // Finally, the engine creates a single executer with a clusteredexecuter for all templates
@ -179,7 +180,7 @@ func (e *ClusterExecuter) Requests() int {
} }
// Execute executes the protocol group and returns true or false if results were found. // Execute executes the protocol group and returns true or false if results were found.
func (e *ClusterExecuter) Execute(input string) (bool, error) { func (e *ClusterExecuter) Execute(input *contextargs.Context) (bool, error) {
var results bool var results bool
previous := make(map[string]interface{}) previous := make(map[string]interface{})
@ -207,13 +208,13 @@ func (e *ClusterExecuter) Execute(input string) (bool, error) {
} }
}) })
if err != nil && e.options.HostErrorsCache != nil { if err != nil && e.options.HostErrorsCache != nil {
e.options.HostErrorsCache.MarkFailed(input, err) e.options.HostErrorsCache.MarkFailed(input.Input, err)
} }
return results, err return results, err
} }
// ExecuteWithResults executes the protocol requests and returns results instead of writing them. // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (e *ClusterExecuter) ExecuteWithResults(input string, callback protocols.OutputEventCallback) error { func (e *ClusterExecuter) ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error {
dynamicValues := make(map[string]interface{}) dynamicValues := make(map[string]interface{})
err := e.requests.ExecuteWithResults(input, dynamicValues, nil, func(event *output.InternalWrappedEvent) { err := e.requests.ExecuteWithResults(input, dynamicValues, nil, func(event *output.InternalWrappedEvent) {
for _, operator := range e.operators { for _, operator := range e.operators {
@ -229,7 +230,7 @@ func (e *ClusterExecuter) ExecuteWithResults(input string, callback protocols.Ou
} }
}) })
if err != nil && e.options.HostErrorsCache != nil { if err != nil && e.options.HostErrorsCache != nil {
e.options.HostErrorsCache.MarkFailed(input, err) e.options.HostErrorsCache.MarkFailed(input.Input, err)
} }
return err return err
} }