diff --git a/integration_tests/workflow/dns-value-share-template-1.yaml b/integration_tests/workflow/dns-value-share-template-1.yaml new file mode 100644 index 00000000..a00e2bfc --- /dev/null +++ b/integration_tests/workflow/dns-value-share-template-1.yaml @@ -0,0 +1,17 @@ +id: dns-value-sharing-template1 + +info: + name: dns-value-sharing-template1 + author: pdteam + severity: info + +dns: + - name: "{{FQDN}}" + type: A + + extractors: + - type: regex + name: extracted + group: 1 + regex: + - "IN\tA\t(.+)" \ No newline at end of file diff --git a/integration_tests/workflow/dns-value-share-template-2.yaml b/integration_tests/workflow/dns-value-share-template-2.yaml new file mode 100644 index 00000000..d6f7c137 --- /dev/null +++ b/integration_tests/workflow/dns-value-share-template-2.yaml @@ -0,0 +1,10 @@ +id: dns-value-sharing-template2 + +info: + name: dns-value-sharing-template2 + author: pdteam + severity: info + +dns: + - name: "{{extracted}}" + type: PTR \ No newline at end of file diff --git a/integration_tests/workflow/dns-value-share-template-3.yaml b/integration_tests/workflow/dns-value-share-template-3.yaml new file mode 100644 index 00000000..7765a95f --- /dev/null +++ b/integration_tests/workflow/dns-value-share-template-3.yaml @@ -0,0 +1,19 @@ +id: value-sharing-template2 + +info: + name: value-sharing-template2 + author: pdteam + severity: info + +requests: + - raw: + - | + GET / HTTP/1.1 + Host: {{Hostname}} + + {{extracted}} + + matchers: + - type: word + words: + - "ok" \ No newline at end of file diff --git a/integration_tests/workflow/dns-value-share-workflow.yaml b/integration_tests/workflow/dns-value-share-workflow.yaml new file mode 100644 index 00000000..53ac298e --- /dev/null +++ b/integration_tests/workflow/dns-value-share-workflow.yaml @@ -0,0 +1,11 @@ +id: dns-value-sharing-workflow +info: + name: DNS Value Sharing Test + author: pdteam + severity: info + +workflows: + - template: workflow/dns-value-share-template-1.yaml + subtemplates: + - template: workflow/dns-value-share-template-2.yaml + - template: workflow/dns-value-share-template-3.yaml \ No newline at end of file diff --git a/integration_tests/workflow/http-value-share-template-1.yaml b/integration_tests/workflow/http-value-share-template-1.yaml new file mode 100644 index 00000000..a273e113 --- /dev/null +++ b/integration_tests/workflow/http-value-share-template-1.yaml @@ -0,0 +1,17 @@ +id: value-sharing-template1 + +info: + name: value-sharing-template1 + author: pdteam + severity: info + +requests: + - path: + - "{{BaseURL}}/path1" + extractors: + - type: regex + part: body + name: extracted + regex: + - 'href="(.*)"' + group: 1 \ No newline at end of file diff --git a/integration_tests/workflow/http-value-share-template-2.yaml b/integration_tests/workflow/http-value-share-template-2.yaml new file mode 100644 index 00000000..7bac99f3 --- /dev/null +++ b/integration_tests/workflow/http-value-share-template-2.yaml @@ -0,0 +1,19 @@ +id: value-sharing-template2 + +info: + name: value-sharing-template2 + author: pdteam + severity: info + +requests: + - raw: + - | + GET /path2 HTTP/1.1 + Host: {{Hostname}} + + {{extracted}} + + matchers: + - type: word + words: + - "test-value" \ No newline at end of file diff --git a/integration_tests/workflow/http-value-share-workflow.yaml b/integration_tests/workflow/http-value-share-workflow.yaml new file mode 100644 index 00000000..d7da342b --- /dev/null +++ b/integration_tests/workflow/http-value-share-workflow.yaml @@ -0,0 +1,10 @@ +id: http-value-sharing-workflow +info: + name: HTTP Value Sharing Test + author: pdteam + severity: info + +workflows: + - template: workflow/http-value-share-template-1.yaml + subtemplates: + - template: workflow/http-value-share-template-2.yaml \ No newline at end of file diff --git a/v2/cmd/integration-test/workflow.go b/v2/cmd/integration-test/workflow.go index c45976e8..3fa08523 100644 --- a/v2/cmd/integration-test/workflow.go +++ b/v2/cmd/integration-test/workflow.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "io" "net/http" "net/http/httptest" @@ -11,10 +12,12 @@ import ( ) var workflowTestcases = map[string]testutils.TestCase{ - "workflow/basic.yaml": &workflowBasic{}, - "workflow/condition-matched.yaml": &workflowConditionMatched{}, - "workflow/condition-unmatched.yaml": &workflowConditionUnmatch{}, - "workflow/matcher-name.yaml": &workflowMatcherName{}, + "workflow/basic.yaml": &workflowBasic{}, + "workflow/condition-matched.yaml": &workflowConditionMatched{}, + "workflow/condition-unmatched.yaml": &workflowConditionUnmatch{}, + "workflow/matcher-name.yaml": &workflowMatcherName{}, + "workflow/http-value-share-workflow.yaml": &workflowHttpKeyValueShare{}, + "workflow/dns-value-share-workflow.yaml": &workflowDnsKeyValueShare{}, } type workflowBasic struct{} @@ -92,3 +95,39 @@ func (h *workflowMatcherName) Execute(filePath string) error { return expectResultsCount(results, 1) } + +type workflowHttpKeyValueShare struct{} + +// Execute executes a test case and returns an error if occurred +func (h *workflowHttpKeyValueShare) Execute(filePath string) error { + router := httprouter.New() + router.GET("/path1", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + fmt.Fprintf(w, "href=\"test-value\"") + }) + router.GET("/path2", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + body, _ := io.ReadAll(r.Body) + fmt.Fprintf(w, "%s", body) + }) + ts := httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunNucleiWorkflowAndGetResults(filePath, ts.URL, debug) + if err != nil { + return err + } + + return expectResultsCount(results, 1) +} + +type workflowDnsKeyValueShare struct{} + +// Execute executes a test case and returns an error if occurred +func (h *workflowDnsKeyValueShare) Execute(filePath string) error { + results, err := testutils.RunNucleiWorkflowAndGetResults(filePath, "http://scanme.sh", debug) + if err != nil { + return err + } + + // no results - ensure that the variable sharing works + return expectResultsCount(results, 1) +} diff --git a/v2/go.mod b/v2/go.mod index c905bdb4..7b7d51d0 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -28,7 +28,6 @@ require ( github.com/projectdiscovery/cryptoutil v1.0.0 github.com/projectdiscovery/fastdialer v0.0.16-0.20220908084548-3eab0c2a02d5 github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08 - github.com/projectdiscovery/goflags v0.0.10-0.20220829035232-bc74fe1cb567 github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.2 github.com/projectdiscovery/interactsh v1.0.6-0.20220827132222-460cc6270053 @@ -36,7 +35,7 @@ require ( github.com/projectdiscovery/rawhttp v0.1.1 github.com/projectdiscovery/retryabledns v1.0.15 github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d - github.com/projectdiscovery/stringsutil v0.0.0-20220731064040-4b67f194751e + github.com/projectdiscovery/stringsutil v0.0.1 github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 @@ -51,7 +50,7 @@ require ( github.com/xanzy/go-gitlab v0.73.1 go.uber.org/atomic v1.10.0 go.uber.org/multierr v1.8.0 - golang.org/x/net v0.0.0-20220805013720-a33c5aa5df48 + golang.org/x/net v0.0.0-20221002022538-bcab6841153b golang.org/x/oauth2 v0.0.0-20220722155238-128564f6959c golang.org/x/text v0.3.7 gopkg.in/yaml.v2 v2.4.0 @@ -73,7 +72,7 @@ require ( github.com/mholt/archiver v3.1.1+incompatible github.com/mitchellh/go-homedir v1.1.0 github.com/openrdap/rdap v0.9.1-0.20191017185644-af93e7ef17b7 - github.com/projectdiscovery/fileutil v0.0.0-20220705195237-01becc2a8963 + github.com/projectdiscovery/fileutil v0.0.1 github.com/projectdiscovery/iputil v0.0.0-20220712175312-b9406f31cdd8 github.com/projectdiscovery/nvd v1.0.9 github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4 @@ -148,7 +147,7 @@ require ( github.com/mattn/go-isatty v0.0.16 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect github.com/mholt/acmez v1.0.4 // indirect - github.com/microcosm-cc/bluemonday v1.0.19 // indirect + github.com/microcosm-cc/bluemonday v1.0.20 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/onsi/ginkgo v1.16.4 // indirect @@ -195,5 +194,6 @@ require ( github.com/klauspost/compress v1.15.8 // indirect github.com/nwaples/rardecode v1.1.2 // indirect github.com/pierrec/lz4 v2.6.1+incompatible // indirect + github.com/projectdiscovery/goflags v0.1.0 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect ) diff --git a/v2/go.sum b/v2/go.sum index 8cccd5d4..4f401d53 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -469,6 +469,8 @@ github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/le github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= github.com/microcosm-cc/bluemonday v1.0.19 h1:OI7hoF5FY4pFz2VA//RN8TfM0YJ2dJcl4P4APrCWy6c= github.com/microcosm-cc/bluemonday v1.0.19/go.mod h1:QNzV2UbLK2/53oIIwTOyLUSABMkjZ4tqiyC1g/DyqxE= +github.com/microcosm-cc/bluemonday v1.0.20 h1:flpzsq4KU3QIYAYGV/szUat7H+GPOXR0B2JU5A1Wp8Y= +github.com/microcosm-cc/bluemonday v1.0.20/go.mod h1:yfBmMi8mxvaZut3Yytv+jTXRY8mxyjJ0/kQBTElld50= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= @@ -556,12 +558,16 @@ github.com/projectdiscovery/fileutil v0.0.0-20220308101036-16c79af1cf5d/go.mod h github.com/projectdiscovery/fileutil v0.0.0-20220609150212-453ac591c36c/go.mod h1:g8wsrb0S5NtEN0JgVyyPeb3FQdArx+UMESmFX94bcGY= github.com/projectdiscovery/fileutil v0.0.0-20220705195237-01becc2a8963 h1:4o97N9ftX1J3iKlIRVMPVOVZs4qbCczJvoFF2WA40t4= github.com/projectdiscovery/fileutil v0.0.0-20220705195237-01becc2a8963/go.mod h1:DaY7wmLPMleyHDCD/14YApPCDtrARY4J8Eny2ZGsG/g= +github.com/projectdiscovery/fileutil v0.0.1 h1:3K3UqCDOan3LsvWhV0nyvVuMWSwCloNPUJIGcXsi1os= +github.com/projectdiscovery/fileutil v0.0.1/go.mod h1:Oo6ZEvXmQz/xPF0YukzmwpdW2LYinWCSEmzZOQsJCLg= github.com/projectdiscovery/folderutil v0.0.0-20220215113126-add60a1e8e08 h1:m1pgJisawU7zP9lKGktOEk6KNrNAR7e4Q07Kt3ox0NM= github.com/projectdiscovery/folderutil v0.0.0-20220215113126-add60a1e8e08/go.mod h1:BMqXH4jNGByVdE2iLtKvc/6XStaiZRuCIaKv1vw9PnI= github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= github.com/projectdiscovery/goflags v0.0.8/go.mod h1:GDSkWyXa6kfQjpJu10SO64DN8lXuKXVENlBMk8N7H80= github.com/projectdiscovery/goflags v0.0.10-0.20220829035232-bc74fe1cb567 h1:0CFt+l9ZO7lT1ZPU/NCTAdRzkZZ5OH+b4FvxsJo1GhE= github.com/projectdiscovery/goflags v0.0.10-0.20220829035232-bc74fe1cb567/go.mod h1:t/dEhv2VDOzayugXZCkbkX8n+pPeVmRD+WgQRSgReeI= +github.com/projectdiscovery/goflags v0.1.0 h1:Z7sUVK8wgH6aGJWinmGQEtsn+GNf/0RQ+z1wQcpCeeA= +github.com/projectdiscovery/goflags v0.1.0/go.mod h1:/YBPA+1igSkQbwD7a91o0HUIwMDlsmQDRZL2oSYSyEQ= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI= github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY= @@ -616,6 +622,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20220422150559-b54fb5dc6833/go.mo github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3/go.mod h1:mF5sh4jTghoGWwgUb9qWi5waTFklClDbtrqtJU93awc= github.com/projectdiscovery/stringsutil v0.0.0-20220731064040-4b67f194751e h1:JBdwX+DJNq5FIJGsZCSKLGV4EhSRiPtNk22Vi+3cTDg= github.com/projectdiscovery/stringsutil v0.0.0-20220731064040-4b67f194751e/go.mod h1:32NYmKyHkKsmisAOAaWrR15lz2ysz2M8x3KMeeoRHoU= +github.com/projectdiscovery/stringsutil v0.0.1 h1:a6TCMT+D1aUsoZxNiYf9O30wiDOoLOHDwj89HBjr5BQ= +github.com/projectdiscovery/stringsutil v0.0.1/go.mod h1:TDi2LEqR3OML0BxGoMbbfAHSk5AdfHX762Oc302sgmM= github.com/projectdiscovery/tlsx v0.0.7 h1:McoDo4Ju7aetogatU4lVTRcQpxkf0qgQSoWgtkDavCk= github.com/projectdiscovery/tlsx v0.0.7/go.mod h1:/ZCk/zzyuDdXx2E511yhNIj3LERnGUXghqkASBbdh5M= github.com/projectdiscovery/urlutil v0.0.0-20210525140139-b874f06ad921 h1:EgaxpJm7+lKppfAHkFHs+S+II0lodp4Gu3leZCCkWlc= @@ -922,6 +930,9 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220728211354-c7608f3a8462/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20220805013720-a33c5aa5df48 h1:N9Vc/rorQUDes6B9CNdIxAn5jODGj2wzfrei2x4wNj4= golang.org/x/net v0.0.0-20220805013720-a33c5aa5df48/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20221002022538-bcab6841153b h1:6e93nYa3hNqAvLr0pD4PN1fFS+gKzp2zAXqrnTCstqU= +golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= diff --git a/v2/pkg/core/execute.go b/v2/pkg/core/execute.go index 434a4fff..924797fb 100644 --- a/v2/pkg/core/execute.go +++ b/v2/pkg/core/execute.go @@ -6,6 +6,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" generalTypes "github.com/projectdiscovery/nuclei/v2/pkg/types" @@ -59,7 +60,7 @@ func (e *Engine) ExecuteWithOpts(templatesList []*templates.Template, target Inp // processSelfContainedTemplates execute a self-contained template. func (e *Engine) executeSelfContainedTemplateWithInput(template *templates.Template, results *atomic.Bool) { - match, err := template.Executer.Execute("") + match, err := template.Executer.Execute(contextargs.New()) if err != nil { gologger.Warning().Msgf("[%s] Could not execute step: %s\n", e.executerOpts.Colorizer.BrightBlue(template.ID), err) } @@ -139,7 +140,9 @@ func (e *Engine) executeModelWithInput(templateType types.ProtocolType, template case types.WorkflowProtocol: match = e.executeWorkflow(value, template.CompiledWorkflow) default: - match, err = template.Executer.Execute(value) + ctxArgs := contextargs.New() + ctxArgs.Input = value + match, err = template.Executer.Execute(ctxArgs) } if err != nil { gologger.Warning().Msgf("[%s] Could not execute step: %s\n", e.executerOpts.Colorizer.BrightBlue(template.ID), err) @@ -200,7 +203,9 @@ func (e *Engine) executeModelWithInputAndResult(templateType types.ProtocolType, case types.WorkflowProtocol: match = e.executeWorkflow(value, template.CompiledWorkflow) default: - err = template.Executer.ExecuteWithResults(value, func(event *output.InternalWrappedEvent) { + ctxArgs := contextargs.New() + ctxArgs.Input = value + err = template.Executer.ExecuteWithResults(ctxArgs, func(event *output.InternalWrappedEvent) { for _, result := range event.Results { callback(result) } @@ -240,7 +245,9 @@ func (e *ChildExecuter) Execute(template *templates.Template, URL string) { wg.Add() go func(tpl *templates.Template) { - match, err := template.Executer.Execute(URL) + ctxArgs := contextargs.New() + ctxArgs.Input = URL + match, err := template.Executer.Execute(ctxArgs) if err != nil { gologger.Warning().Msgf("[%s] Could not execute step: %s\n", e.e.executerOpts.Colorizer.BrightBlue(template.ID), err) } diff --git a/v2/pkg/core/workflow_execute.go b/v2/pkg/core/workflow_execute.go index 689cae9f..99918913 100644 --- a/v2/pkg/core/workflow_execute.go +++ b/v2/pkg/core/workflow_execute.go @@ -1,26 +1,36 @@ package core import ( + "fmt" + "net/http/cookiejar" + "github.com/remeh/sizedwaitgroup" "go.uber.org/atomic" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/workflows" ) -const workflowExecutionErrorMessageTemplate = "[%s] Could not execute workflow step: %s\n" +const workflowStepExecutionError = "[%s] Could not execute workflow step: %s\n" // executeWorkflow runs a workflow on an input and returns true or false func (e *Engine) executeWorkflow(input string, w *workflows.Workflow) bool { results := &atomic.Bool{} + // at this point we should be at the start root execution of a workflow tree, hence we create global shared instances + workflowCookieJar, _ := cookiejar.New(nil) + ctxArgs := contextargs.New() + ctxArgs.Input = input + ctxArgs.CookieJar = workflowCookieJar + swg := sizedwaitgroup.New(w.Options.Options.TemplateThreads) for _, template := range w.Workflows { swg.Add() func(template *workflows.WorkflowTemplate) { - if err := e.runWorkflowStep(template, input, results, &swg, w); err != nil { - gologger.Warning().Msgf(workflowExecutionErrorMessageTemplate, template.Template, err) + if err := e.runWorkflowStep(template, ctxArgs, results, &swg, w); err != nil { + gologger.Warning().Msgf(workflowStepExecutionError, template.Template, err) } swg.Done() }(template) @@ -31,7 +41,7 @@ func (e *Engine) executeWorkflow(input string, w *workflows.Workflow) bool { // runWorkflowStep runs a workflow step for the workflow. It executes the workflow // in a recursive manner running all subtemplates and matchers. -func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, input string, results *atomic.Bool, swg *sizedwaitgroup.SizedWaitGroup, w *workflows.Workflow) error { +func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, input *contextargs.Context, results *atomic.Bool, swg *sizedwaitgroup.SizedWaitGroup, w *workflows.Workflow) error { var firstMatched bool var err error var mainErr error @@ -49,6 +59,25 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, input str if len(result.Results) > 0 { firstMatched = true } + + if result.OperatorsResult != nil && result.OperatorsResult.Extracts != nil { + for k, v := range result.OperatorsResult.Extracts { + // normalize items: + switch len(v) { + case 0, 1: + // - key:[item] => key: item + input.Set(k, v[0]) + default: + // - key:[item_0, ..., item_n] => key0:item_0, keyn:item_n + for vIdx, vVal := range v { + normalizedKIdx := fmt.Sprintf("%s%d", k, vIdx) + input.Set(normalizedKIdx, vVal) + } + // also add the original name with full slice + input.Set(k, v) + } + } + } }) } else { var matched bool @@ -59,12 +88,12 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, input str } if err != nil { if w.Options.HostErrorsCache != nil { - w.Options.HostErrorsCache.MarkFailed(input, err) + w.Options.HostErrorsCache.MarkFailed(input.Input, err) } if len(template.Executers) == 1 { mainErr = err } else { - gologger.Warning().Msgf(workflowExecutionErrorMessageTemplate, template.Template, err) + gologger.Warning().Msgf(workflowStepExecutionError, template.Template, err) } continue } @@ -82,6 +111,12 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, input str return } + if event.OperatorsResult.Extracts != nil { + for k, v := range event.OperatorsResult.Extracts { + input.Set(k, v) + } + } + for _, matcher := range template.Matchers { if !matcher.Match(event.OperatorsResult) { continue @@ -92,7 +127,7 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, input str go func(subtemplate *workflows.WorkflowTemplate) { if err := e.runWorkflowStep(subtemplate, input, results, swg, w); err != nil { - gologger.Warning().Msgf(workflowExecutionErrorMessageTemplate, subtemplate.Template, err) + gologger.Warning().Msgf(workflowStepExecutionError, subtemplate.Template, err) } swg.Done() }(subtemplate) @@ -103,7 +138,7 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, input str if len(template.Executers) == 1 { mainErr = err } else { - gologger.Warning().Msgf(workflowExecutionErrorMessageTemplate, template.Template, err) + gologger.Warning().Msgf(workflowStepExecutionError, template.Template, err) } continue } @@ -116,7 +151,7 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, input str go func(template *workflows.WorkflowTemplate) { if err := e.runWorkflowStep(template, input, results, swg, w); err != nil { - gologger.Warning().Msgf(workflowExecutionErrorMessageTemplate, template.Template, err) + gologger.Warning().Msgf(workflowStepExecutionError, template.Template, err) } swg.Done() }(subtemplate) diff --git a/v2/pkg/core/workflow_execute_test.go b/v2/pkg/core/workflow_execute_test.go index cddb067e..da911ab8 100644 --- a/v2/pkg/core/workflow_execute_test.go +++ b/v2/pkg/core/workflow_execute_test.go @@ -10,6 +10,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/progress" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/nuclei/v2/pkg/workflows" ) @@ -178,17 +179,17 @@ func (m *mockExecuter) Requests() int { } // Execute executes the protocol group and returns true or false if results were found. -func (m *mockExecuter) Execute(input string) (bool, error) { +func (m *mockExecuter) Execute(input *contextargs.Context) (bool, error) { if m.executeHook != nil { - m.executeHook(input) + m.executeHook(input.Input) } return m.result, nil } // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (m *mockExecuter) ExecuteWithResults(input string, callback protocols.OutputEventCallback) error { +func (m *mockExecuter) ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error { if m.executeHook != nil { - m.executeHook(input) + m.executeHook(input.Input) } for _, output := range m.outputs { callback(output) diff --git a/v2/pkg/protocols/common/contextargs/args.go b/v2/pkg/protocols/common/contextargs/args.go new file mode 100644 index 00000000..e3bde44a --- /dev/null +++ b/v2/pkg/protocols/common/contextargs/args.go @@ -0,0 +1,31 @@ +package contextargs + +// Args is a generic map with helpers +type Args map[string]interface{} + +// Set a key with value +func (args Args) Set(key string, value interface{}) { + args[key] = value +} + +// Get the value associated to a key +func (args Args) Get(key string) (interface{}, bool) { + value, ok := args[key] + return value, ok +} + +// Has verifies if the map contains the key +func (args Args) Has(key string) bool { + _, ok := args[key] + return ok +} + +// IsEmpty verifies if the map is empty +func (Args Args) IsEmpty() bool { + return len(Args) == 0 +} + +// create a new args map instance +func newArgs() map[string]interface{} { + return make(map[string]interface{}) +} diff --git a/v2/pkg/protocols/common/contextargs/contextargs.go b/v2/pkg/protocols/common/contextargs/contextargs.go new file mode 100644 index 00000000..863e8cd0 --- /dev/null +++ b/v2/pkg/protocols/common/contextargs/contextargs.go @@ -0,0 +1,109 @@ +package contextargs + +import ( + "net/http/cookiejar" + "sync" + + "golang.org/x/exp/maps" +) + +// Context implements a shared context struct to share information across multiple templates within a workflow +type Context struct { + // Input target for the executor + Input string + // CookieJar shared within workflow's http templates + CookieJar *cookiejar.Jar + + // Access to Args must use lock strategies to prevent data races + *sync.RWMutex + // Args is a workflow shared key-value store + args Args +} + +// Create a new contextargs instance +func New() *Context { + return &Context{} +} + +// Create a new contextargs instance with input string +func NewWithInput(input string) *Context { + return &Context{Input: input} +} + +func (ctx *Context) initialize() { + ctx.args = newArgs() + ctx.RWMutex = &sync.RWMutex{} +} + +func (ctx *Context) set(key string, value interface{}) { + ctx.Lock() + defer ctx.Unlock() + + ctx.args.Set(key, value) +} + +// Set the specific key-value pair +func (ctx *Context) Set(key string, value interface{}) { + if !ctx.isInitialized() { + ctx.initialize() + } + + ctx.set(key, value) +} + +func (ctx *Context) isInitialized() bool { + return ctx.args != nil +} + +func (ctx *Context) hasArgs() bool { + return ctx.isInitialized() && !ctx.args.IsEmpty() +} + +func (ctx *Context) get(key string) (interface{}, bool) { + ctx.RLock() + defer ctx.RUnlock() + + return ctx.args.Get(key) +} + +// Get the value with specific key if exists +func (ctx *Context) Get(key string) (interface{}, bool) { + if !ctx.hasArgs() { + return nil, false + } + + return ctx.get(key) +} + +func (ctx *Context) GetAll() Args { + if !ctx.hasArgs() { + return nil + } + + return maps.Clone(ctx.args) +} + +func (ctx *Context) ForEach(f func(string, interface{})) { + ctx.RLock() + defer ctx.RUnlock() + + for k, v := range ctx.args { + f(k, v) + } +} + +func (ctx *Context) has(key string) bool { + ctx.RLock() + defer ctx.RUnlock() + + return ctx.args.Has(key) +} + +// Has check if the key exists +func (ctx *Context) Has(key string) bool { + return ctx.hasArgs() && ctx.has(key) +} + +func (ctx *Context) HasArgs() bool { + return ctx.hasArgs() +} diff --git a/v2/pkg/protocols/common/contextargs/doc.go b/v2/pkg/protocols/common/contextargs/doc.go new file mode 100644 index 00000000..188f45dc --- /dev/null +++ b/v2/pkg/protocols/common/contextargs/doc.go @@ -0,0 +1,4 @@ +// Package contextargs implements a generic entity for shared context within workflows +// +// All templates within a workflow shares the same cookiejar and a key-value store with shared items +package contextargs diff --git a/v2/pkg/protocols/common/executer/executer.go b/v2/pkg/protocols/common/executer/executer.go index 49a32b13..22b2e9be 100644 --- a/v2/pkg/protocols/common/executer/executer.go +++ b/v2/pkg/protocols/common/executer/executer.go @@ -10,6 +10,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/writer" ) @@ -58,10 +59,15 @@ func (e *Executer) Requests() int { } // Execute executes the protocol group and returns true or false if results were found. -func (e *Executer) Execute(input string) (bool, error) { +func (e *Executer) Execute(input *contextargs.Context) (bool, error) { var results bool dynamicValues := make(map[string]interface{}) + if input.HasArgs() { + input.ForEach(func(key string, value interface{}) { + dynamicValues[key] = value + }) + } previous := make(map[string]interface{}) for _, req := range e.requests { err := req.ExecuteWithResults(input, dynamicValues, previous, func(event *output.InternalWrappedEvent) { @@ -95,9 +101,9 @@ func (e *Executer) Execute(input string) (bool, error) { }) if err != nil { if e.options.HostErrorsCache != nil { - e.options.HostErrorsCache.MarkFailed(input, err) + e.options.HostErrorsCache.MarkFailed(input.Input, err) } - gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input, err) + gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input.Input, err) } // If a match was found and stop at first match is set, break out of the loop and return if results && (e.options.StopAtFirstMatch || e.options.Options.StopAtFirstMatch) { @@ -108,8 +114,13 @@ func (e *Executer) Execute(input string) (bool, error) { } // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEventCallback) error { +func (e *Executer) ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error { dynamicValues := make(map[string]interface{}) + if input.HasArgs() { + input.ForEach(func(key string, value interface{}) { + dynamicValues[key] = value + }) + } previous := make(map[string]interface{}) var results bool @@ -136,9 +147,9 @@ func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEve }) if err != nil { if e.options.HostErrorsCache != nil { - e.options.HostErrorsCache.MarkFailed(input, err) + e.options.HostErrorsCache.MarkFailed(input.Input, err) } - gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input, err) + gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input.Input, err) } // If a match was found and stop at first match is set, break out of the loop and return if results && (e.options.StopAtFirstMatch || e.options.Options.StopAtFirstMatch) { diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index 00293089..be1c3507 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -12,6 +12,7 @@ import ( "github.com/projectdiscovery/iputil" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" @@ -30,13 +31,13 @@ func (request *Request) Type() templateTypes.ProtocolType { } // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (request *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { +func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error { // Parse the URL and return domain if URL. var domain string - if utils.IsURL(input) { - domain = extractDomain(input) + if utils.IsURL(input.Input) { + domain = extractDomain(input.Input) } else { - domain = input + domain = input.Input } var err error @@ -45,6 +46,8 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review return errors.Wrap(err, "could not build request") } vars := GenerateVariables(domain) + // merge with metadata (eg. from workflow context) + vars = generators.MergeMaps(vars, metadata) variablesMap := request.options.Variables.Evaluate(vars) vars = generators.MergeMaps(variablesMap, vars) @@ -107,7 +110,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review } } - outputEvent := request.responseToDSLMap(compiledRequest, response, input, input, traceData) + outputEvent := request.responseToDSLMap(compiledRequest, response, input.Input, input.Input, traceData) for k, v := range previous { outputEvent[k] = v } @@ -115,7 +118,6 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review outputEvent[k] = v } event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse) - // TODO: dynamic values are not supported yet dumpResponse(event, request, request.options, response.String(), domain) if request.Trace { diff --git a/v2/pkg/protocols/dns/request_test.go b/v2/pkg/protocols/dns/request_test.go index cb997b7c..9ddabe71 100644 --- a/v2/pkg/protocols/dns/request_test.go +++ b/v2/pkg/protocols/dns/request_test.go @@ -11,6 +11,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) @@ -53,7 +54,9 @@ func TestDNSExecuteWithResults(t *testing.T) { t.Run("domain-valid", func(t *testing.T) { metadata := make(output.InternalEvent) previous := make(output.InternalEvent) - err := request.ExecuteWithResults("example.com", metadata, previous, func(event *output.InternalWrappedEvent) { + ctxArgs := contextargs.New() + ctxArgs.Input = "example.com" + err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) { finalEvent = event }) require.Nil(t, err, "could not execute dns request") @@ -68,7 +71,7 @@ func TestDNSExecuteWithResults(t *testing.T) { t.Run("url-to-domain", func(t *testing.T) { metadata := make(output.InternalEvent) previous := make(output.InternalEvent) - err := request.ExecuteWithResults("https://example.com", metadata, previous, func(event *output.InternalWrappedEvent) { + err := request.ExecuteWithResults(contextargs.NewWithInput("https://example.com"), metadata, previous, func(event *output.InternalWrappedEvent) { finalEvent = event }) require.Nil(t, err, "could not execute dns request") diff --git a/v2/pkg/protocols/file/request.go b/v2/pkg/protocols/file/request.go index 20b5dade..08109416 100644 --- a/v2/pkg/protocols/file/request.go +++ b/v2/pkg/protocols/file/request.go @@ -18,6 +18,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" @@ -44,9 +45,9 @@ type FileMatch struct { var emptyResultErr = errors.New("Empty result") // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (request *Request) ExecuteWithResults(input string, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error { +func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error { wg := sizedwaitgroup.New(request.options.Options.BulkSize) - err := request.getInputPaths(input, func(filePath string) { + err := request.getInputPaths(input.Input, func(filePath string) { wg.Add() func(filePath string) { defer wg.Done() @@ -62,7 +63,7 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp // every new file in the compressed multi-file archive counts 1 request.options.Progress.AddToTotal(1) archiveFileName := filepath.Join(filePath, file.Name()) - event, fileMatches, err := request.processReader(file.ReadCloser, archiveFileName, input, file.Size(), previous) + event, fileMatches, err := request.processReader(file.ReadCloser, archiveFileName, input.Input, file.Size(), previous) if err != nil { if errors.Is(err, emptyResultErr) { // no matches but one file elaborated @@ -115,7 +116,7 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp _ = tmpFileOut.Sync() // rewind the file _, _ = tmpFileOut.Seek(0, 0) - event, fileMatches, err := request.processReader(tmpFileOut, filePath, input, fileStat.Size(), previous) + event, fileMatches, err := request.processReader(tmpFileOut, filePath, input.Input, fileStat.Size(), previous) if err != nil { if errors.Is(err, emptyResultErr) { // no matches but one file elaborated @@ -135,7 +136,7 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp default: // normal file - increments the counter by 1 request.options.Progress.AddToTotal(1) - event, fileMatches, err := request.processFile(filePath, input, previous) + event, fileMatches, err := request.processFile(filePath, input.Input, previous) if err != nil { if errors.Is(err, emptyResultErr) { // no matches but one file elaborated @@ -157,7 +158,7 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp wg.Wait() if err != nil { - request.options.Output.Request(request.options.TemplatePath, input, request.Type().String(), err) + request.options.Output.Request(request.options.TemplatePath, input.Input, request.Type().String(), err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not send file request") } diff --git a/v2/pkg/protocols/file/request_test.go b/v2/pkg/protocols/file/request_test.go index 74aea078..9de78fd4 100644 --- a/v2/pkg/protocols/file/request_test.go +++ b/v2/pkg/protocols/file/request_test.go @@ -13,6 +13,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) @@ -65,7 +66,9 @@ func TestFileExecuteWithResults(t *testing.T) { t.Run("valid", func(t *testing.T) { metadata := make(output.InternalEvent) previous := make(output.InternalEvent) - err := request.ExecuteWithResults(tempDir, metadata, previous, func(event *output.InternalWrappedEvent) { + ctxArgs := contextargs.New() + ctxArgs.Input = tempDir + err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) { finalEvent = event }) require.Nil(t, err, "could not execute file request") diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go index cdf4b169..23b529a8 100644 --- a/v2/pkg/protocols/headless/request.go +++ b/v2/pkg/protocols/headless/request.go @@ -10,6 +10,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" @@ -29,7 +30,8 @@ func (request *Request) Type() templateTypes.ProtocolType { } // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (request *Request) ExecuteWithResults(inputURL string, metadata, previous output.InternalEvent /*TODO review unused parameter*/, callback protocols.OutputEventCallback) error { +func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, previous output.InternalEvent /*TODO review unused parameter*/, callback protocols.OutputEventCallback) error { + inputURL := input.Input if request.options.Browser.UserAgent() == "" { request.options.Browser.SetUserAgent(request.compiledUserAgent) } diff --git a/v2/pkg/protocols/http/httpclientpool/clientpool.go b/v2/pkg/protocols/http/httpclientpool/clientpool.go index c7aa1616..60a284b3 100644 --- a/v2/pkg/protocols/http/httpclientpool/clientpool.go +++ b/v2/pkg/protocols/http/httpclientpool/clientpool.go @@ -59,6 +59,7 @@ func Init(options *types.Options) error { type ConnectionConfiguration struct { // DisableKeepAlive of the connection DisableKeepAlive bool + Cookiejar *cookiejar.Jar } // Configuration contains the custom configuration options for a client @@ -241,7 +242,9 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl } var jar *cookiejar.Jar - if configuration.CookieReuse { + if configuration.Connection != nil && configuration.Connection.Cookiejar != nil { + jar = configuration.Connection.Cookiejar + } else if configuration.CookieReuse { if jar, err = cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List}); err != nil { return nil, errors.Wrap(err, "could not create cookiejar") } diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index ddddc5ac..e9b34bf3 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -22,6 +22,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" @@ -45,7 +46,8 @@ func (request *Request) Type() templateTypes.ProtocolType { } // executeRaceRequest executes race condition request for a URL -func (request *Request) executeRaceRequest(reqURL string, previous output.InternalEvent, callback protocols.OutputEventCallback) error { +func (request *Request) executeRaceRequest(input *contextargs.Context, previous output.InternalEvent, callback protocols.OutputEventCallback) error { + reqURL := input.Input var generatedRequests []*generatedRequest // Requests within race condition should be dumped once and the output prefilled to allow DSL language to work @@ -98,7 +100,7 @@ func (request *Request) executeRaceRequest(reqURL string, previous output.Intern wg.Add(1) go func(httpRequest *generatedRequest) { defer wg.Done() - err := request.executeRequest(reqURL, httpRequest, previous, false, callback, 0) + err := request.executeRequest(input, httpRequest, previous, false, callback, 0) mutex.Lock() if err != nil { requestErr = multierr.Append(requestErr, err) @@ -113,7 +115,7 @@ func (request *Request) executeRaceRequest(reqURL string, previous output.Intern } // executeRaceRequest executes parallel requests for a template -func (request *Request) executeParallelHTTP(reqURL string, dynamicValues output.InternalEvent, callback protocols.OutputEventCallback) error { +func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicValues output.InternalEvent, callback protocols.OutputEventCallback) error { generator := request.newGenerator() // Workers that keeps enqueuing new requests @@ -127,7 +129,7 @@ func (request *Request) executeParallelHTTP(reqURL string, dynamicValues output. if !ok { break } - generatedHttpRequest, err := generator.Make(context.Background(), reqURL, inputData, payloads, dynamicValues) + generatedHttpRequest, err := generator.Make(context.Background(), input.Input, inputData, payloads, dynamicValues) if err != nil { if err == io.EOF { break @@ -135,8 +137,8 @@ func (request *Request) executeParallelHTTP(reqURL string, dynamicValues output. request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) return err } - if reqURL == "" { - reqURL = generatedHttpRequest.URL() + if input.Input == "" { + input.Input = generatedHttpRequest.URL() } swg.Add() go func(httpRequest *generatedRequest) { @@ -145,7 +147,7 @@ func (request *Request) executeParallelHTTP(reqURL string, dynamicValues output. request.options.RateLimiter.Take() previous := make(map[string]interface{}) - err := request.executeRequest(reqURL, httpRequest, previous, false, callback, 0) + err := request.executeRequest(input, httpRequest, previous, false, callback, 0) mutex.Lock() if err != nil { requestErr = multierr.Append(requestErr, err) @@ -159,11 +161,11 @@ func (request *Request) executeParallelHTTP(reqURL string, dynamicValues output. } // executeTurboHTTP executes turbo http request for a URL -func (request *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { +func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { generator := request.newGenerator() // need to extract the target from the url - URL, err := url.Parse(reqURL) + URL, err := url.Parse(input.Input) if err != nil { return err } @@ -194,20 +196,20 @@ func (request *Request) executeTurboHTTP(reqURL string, dynamicValues, previous if !ok { break } - generatedHttpRequest, err := generator.Make(context.Background(), reqURL, inputData, payloads, dynamicValues) + generatedHttpRequest, err := generator.Make(context.Background(), input.Input, inputData, payloads, dynamicValues) if err != nil { request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) return err } - if reqURL == "" { - reqURL = generatedHttpRequest.URL() + if input.Input == "" { + input.Input = generatedHttpRequest.URL() } generatedHttpRequest.pipelinedClient = pipeClient swg.Add() go func(httpRequest *generatedRequest) { defer swg.Done() - err := request.executeRequest(reqURL, httpRequest, previous, false, callback, 0) + err := request.executeRequest(input, httpRequest, previous, false, callback, 0) mutex.Lock() if err != nil { requestErr = multierr.Append(requestErr, err) @@ -221,24 +223,24 @@ func (request *Request) executeTurboHTTP(reqURL string, dynamicValues, previous } // ExecuteWithResults executes the final request on a URL -func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { +func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { if request.Pipeline || request.Race && request.RaceNumberRequests > 0 || request.Threads > 0 { variablesMap := request.options.Variables.Evaluate(generators.MergeMaps(dynamicValues, previous)) dynamicValues = generators.MergeMaps(variablesMap, dynamicValues) } // verify if pipeline was requested if request.Pipeline { - return request.executeTurboHTTP(reqURL, dynamicValues, previous, callback) + return request.executeTurboHTTP(input, dynamicValues, previous, callback) } // verify if a basic race condition was requested if request.Race && request.RaceNumberRequests > 0 { - return request.executeRaceRequest(reqURL, dynamicValues, callback) + return request.executeRaceRequest(input, dynamicValues, callback) } // verify if parallel elaboration was requested if request.Threads > 0 { - return request.executeParallelHTTP(reqURL, dynamicValues, callback) + return request.executeParallelHTTP(input, dynamicValues, callback) } generator := request.newGenerator() @@ -257,7 +259,7 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou ctx, cancel := context.WithTimeout(context.Background(), time.Duration(request.options.Options.Timeout)*time.Second) defer cancel() - generatedHttpRequest, err := generator.Make(ctx, reqURL, data, payloads, dynamicValue) + generatedHttpRequest, err := generator.Make(ctx, input.Input, data, payloads, dynamicValue) if err != nil { if err == io.EOF { return true, nil @@ -270,15 +272,15 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou generatedHttpRequest.interactshURLs = append(generatedHttpRequest.interactshURLs, interactURLs...) } hasInteractMarkers := interactsh.HasMarkers(data) || len(generatedHttpRequest.interactshURLs) > 0 - if reqURL == "" { - reqURL = generatedHttpRequest.URL() + if input.Input == "" { + input.Input = generatedHttpRequest.URL() } // Check if hosts keep erroring - if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(reqURL) { + if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(input.Input) { return true, nil } var gotMatches bool - err = request.executeRequest(reqURL, generatedHttpRequest, previous, hasInteractMatchers, func(event *output.InternalWrappedEvent) { + err = request.executeRequest(input, generatedHttpRequest, previous, hasInteractMatchers, func(event *output.InternalWrappedEvent) { // Add the extracts to the dynamic values if any. if event.OperatorsResult != nil { gotMatches = event.OperatorsResult.Matched @@ -303,7 +305,7 @@ func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previou } if err != nil { if request.options.HostErrorsCache != nil { - request.options.HostErrorsCache.MarkFailed(reqURL, err) + request.options.HostErrorsCache.MarkFailed(input.Input, err) } requestErr = err } @@ -347,7 +349,7 @@ const drainReqSize = int64(8 * 1024) var errStopExecution = errors.New("stop execution due to unresolved variables") // executeRequest executes the actual generated request and returns error if occurred -func (request *Request) executeRequest(reqURL string, generatedRequest *generatedRequest, previousEvent output.InternalEvent, hasInteractMatchers bool, callback protocols.OutputEventCallback, requestCount int) error { +func (request *Request) executeRequest(input *contextargs.Context, generatedRequest *generatedRequest, previousEvent output.InternalEvent, hasInteractMatchers bool, callback protocols.OutputEventCallback, requestCount int) error { request.setCustomHeaders(generatedRequest) // Try to evaluate any payloads before replacement @@ -375,7 +377,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate if !generatedRequest.original.Race { var dumpError error // TODO: dump is currently not working with post-processors - somehow it alters the signature - dumpedRequest, dumpError = dump(generatedRequest, reqURL) + dumpedRequest, dumpError = dump(generatedRequest, input.Input) if dumpError != nil { return dumpError } @@ -383,12 +385,12 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate if ignoreList := GetVariablesNamesSkipList(generatedRequest.original.Signature.Value); ignoreList != nil { if varErr := expressions.ContainsVariablesWithIgnoreList(ignoreList, dumpedRequestString); varErr != nil && !request.SkipVariablesCheck { - gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, reqURL, varErr) + gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, input.Input, varErr) return errStopExecution } } else { // Check if are there any unresolved variables. If yes, skip unless overridden by user. if varErr := expressions.ContainsUnresolvedVariables(dumpedRequestString); varErr != nil && !request.SkipVariablesCheck { - gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, reqURL, varErr) + gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, input.Input, varErr) return errStopExecution } } @@ -402,7 +404,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate if parsed, parseErr := url.Parse(formedURL); parseErr == nil { hostname = parsed.Host } - resp, err = generatedRequest.pipelinedClient.DoRaw(generatedRequest.rawRequest.Method, reqURL, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data))) + resp, err = generatedRequest.pipelinedClient.DoRaw(generatedRequest.rawRequest.Method, input.Input, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data))) } else if generatedRequest.request != nil { resp, err = generatedRequest.pipelinedClient.Dor(generatedRequest.request) } @@ -410,7 +412,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate formedURL = generatedRequest.rawRequest.FullURL // use request url as matched url if empty if formedURL == "" { - formedURL = reqURL + formedURL = input.Input } if parsed, parseErr := url.Parse(formedURL); parseErr == nil { hostname = parsed.Host @@ -420,7 +422,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate options.CustomRawBytes = generatedRequest.rawRequest.UnsafeRawBytes options.ForceReadAllBody = request.ForceReadAllBody options.SNI = request.options.Options.SNI - resp, err = generatedRequest.original.rawhttpClient.DoRawWithOptions(generatedRequest.rawRequest.Method, reqURL, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)), &options) + resp, err = generatedRequest.original.rawhttpClient.DoRawWithOptions(generatedRequest.rawRequest.Method, input.Input, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)), &options) } else { hostname = generatedRequest.request.URL.Host formedURL = generatedRequest.request.URL.String() @@ -437,18 +439,29 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate if errSignature := request.handleSignature(generatedRequest); errSignature != nil { return errSignature } - resp, err = request.httpClient.Do(generatedRequest.request) + + httpclient := request.httpClient + if input.CookieJar != nil { + connConfiguration := request.connConfiguration + connConfiguration.Connection.Cookiejar = input.CookieJar + client, err := httpclientpool.Get(request.options.Options, connConfiguration) + if err != nil { + return errors.Wrap(err, "could not get http client") + } + httpclient = client + } + resp, err = httpclient.Do(generatedRequest.request) } } // use request url as matched url if empty if formedURL == "" { - formedURL = reqURL + formedURL = input.Input } // Dump the requests containing all headers if !generatedRequest.original.Race { var dumpError error - dumpedRequest, dumpError = dump(generatedRequest, reqURL) + dumpedRequest, dumpError = dump(generatedRequest, input.Input) if dumpError != nil { return dumpError } @@ -461,7 +474,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate gologger.Print().Msgf("%s", dumpedRequestString) } if request.options.Options.StoreResponse { - request.options.Output.WriteStoreDebugData(reqURL, request.options.TemplateID, request.Type().String(), fmt.Sprintf("%s\n%s", msg, dumpedRequestString)) + request.options.Output.WriteStoreDebugData(input.Input, request.options.TemplateID, request.Type().String(), fmt.Sprintf("%s\n%s", msg, dumpedRequestString)) } } } @@ -477,7 +490,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate // If we have interactsh markers and request times out, still send // a callback event so in case we receive an interaction, correlation is possible. if hasInteractMatchers { - outputEvent := request.responseToDSLMap(&http.Response{}, reqURL, formedURL, tostring.UnsafeToString(dumpedRequest), "", "", "", 0, generatedRequest.meta) + outputEvent := request.responseToDSLMap(&http.Response{}, input.Input, formedURL, tostring.UnsafeToString(dumpedRequest), "", "", "", 0, generatedRequest.meta) if i := strings.LastIndex(hostname, ":"); i != -1 { hostname = hostname[:i] } @@ -559,7 +572,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate if response.resp == nil { continue // Skip nil responses } - matchedURL := reqURL + matchedURL := input.Input if generatedRequest.rawRequest != nil && generatedRequest.rawRequest.FullURL != "" { matchedURL = generatedRequest.rawRequest.FullURL } @@ -574,7 +587,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate } finalEvent := make(output.InternalEvent) - outputEvent := request.responseToDSLMap(response.resp, reqURL, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(response.fullResponse), tostring.UnsafeToString(response.body), tostring.UnsafeToString(response.headers), duration, generatedRequest.meta) + outputEvent := request.responseToDSLMap(response.resp, input.Input, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(response.fullResponse), tostring.UnsafeToString(response.body), tostring.UnsafeToString(response.headers), duration, generatedRequest.meta) if i := strings.LastIndex(hostname, ":"); i != -1 { hostname = hostname[:i] } @@ -610,7 +623,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate responseContentType := resp.Header.Get("Content-Type") isResponseTruncated := request.MaxSize > 0 && len(gotData) >= request.MaxSize - dumpResponse(event, request, response.fullResponse, formedURL, responseContentType, isResponseTruncated, reqURL) + dumpResponse(event, request, response.fullResponse, formedURL, responseContentType, isResponseTruncated, input.Input) callback(event) diff --git a/v2/pkg/protocols/http/request_test.go b/v2/pkg/protocols/http/request_test.go index 6cd10d10..2d93c190 100644 --- a/v2/pkg/protocols/http/request_test.go +++ b/v2/pkg/protocols/http/request_test.go @@ -14,6 +14,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) @@ -81,7 +82,9 @@ Disallow: /c`)) t.Run("test", func(t *testing.T) { metadata := make(output.InternalEvent) previous := make(output.InternalEvent) - err := request.ExecuteWithResults(ts.URL, metadata, previous, func(event *output.InternalWrappedEvent) { + ctxArgs := contextargs.New() + ctxArgs.Input = ts.URL + err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) { if event.OperatorsResult != nil && event.OperatorsResult.Matched { matchCount++ } diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index a3b39b96..b3957e8f 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -17,6 +17,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" @@ -35,17 +36,17 @@ func (request *Request) Type() templateTypes.ProtocolType { } // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (request *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { +func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { var address string var err error if request.SelfContained { address = "" } else { - address, err = getAddress(input) + address, err = getAddress(input.Input) } if err != nil { - request.options.Output.Request(request.options.TemplatePath, input, request.Type().String(), err) + request.options.Output.Request(request.options.TemplatePath, input.Input, request.Type().String(), err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not get address from url") } @@ -56,7 +57,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review variables = generators.MergeMaps(variablesMap, variables) actualAddress := replacer.Replace(kv.address, variables) - if err := request.executeAddress(variables, actualAddress, address, input, kv.tls, previous, callback); err != nil { + if err := request.executeAddress(variables, actualAddress, address, input.Input, kv.tls, previous, callback); err != nil { gologger.Warning().Msgf("Could not make network request for %s: %s\n", actualAddress, err) continue } diff --git a/v2/pkg/protocols/network/request_test.go b/v2/pkg/protocols/network/request_test.go index acfe5b12..859a61cc 100644 --- a/v2/pkg/protocols/network/request_test.go +++ b/v2/pkg/protocols/network/request_test.go @@ -16,6 +16,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) @@ -64,7 +65,9 @@ func TestNetworkExecuteWithResults(t *testing.T) { t.Run("domain-valid", func(t *testing.T) { metadata := make(output.InternalEvent) previous := make(output.InternalEvent) - err := request.ExecuteWithResults(parsed.Host, metadata, previous, func(event *output.InternalWrappedEvent) { + ctxArgs := contextargs.New() + ctxArgs.Input = parsed.Host + err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) { finalEvent = event }) require.Nil(t, err, "could not execute network request") @@ -79,7 +82,9 @@ func TestNetworkExecuteWithResults(t *testing.T) { t.Run("invalid-port-override", func(t *testing.T) { metadata := make(output.InternalEvent) previous := make(output.InternalEvent) - err := request.ExecuteWithResults("127.0.0.1:11211", metadata, previous, func(event *output.InternalWrappedEvent) { + ctxArgs := contextargs.New() + ctxArgs.Input = "127.0.0.1:11211" + err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) { finalEvent = event }) require.Nil(t, err, "could not execute network request") @@ -92,7 +97,9 @@ func TestNetworkExecuteWithResults(t *testing.T) { t.Run("hex-to-string", func(t *testing.T) { metadata := make(output.InternalEvent) previous := make(output.InternalEvent) - err := request.ExecuteWithResults(parsed.Host, metadata, previous, func(event *output.InternalWrappedEvent) { + ctxArgs := contextargs.New() + ctxArgs.Input = parsed.Host + err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) { finalEvent = event }) require.Nil(t, err, "could not execute network request") diff --git a/v2/pkg/protocols/offlinehttp/request.go b/v2/pkg/protocols/offlinehttp/request.go index 2b794b69..76b98ea5 100644 --- a/v2/pkg/protocols/offlinehttp/request.go +++ b/v2/pkg/protocols/offlinehttp/request.go @@ -13,6 +13,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring" templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" @@ -28,10 +29,10 @@ func (request *Request) Type() templateTypes.ProtocolType { } // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (request *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { +func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { wg := sizedwaitgroup.New(request.options.Options.BulkSize) - err := request.getInputPaths(input, func(data string) { + err := request.getInputPaths(input.Input, func(data string) { wg.Add() go func(data string) { @@ -97,7 +98,7 @@ func (request *Request) ExecuteWithResults(input string, metadata /*TODO review }) wg.Wait() if err != nil { - request.options.Output.Request(request.options.TemplatePath, input, "file", err) + request.options.Output.Request(request.options.TemplatePath, input.Input, "file", err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not send file request") } diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index ccac4ad1..fecc38b1 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -13,6 +13,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/progress" "github.com/projectdiscovery/nuclei/v2/pkg/projectfile" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/excludematchers" @@ -30,9 +31,9 @@ type Executer interface { // Requests returns the total number of requests the rule will perform Requests() int // Execute executes the protocol group and returns true or false if results were found. - Execute(input string) (bool, error) + Execute(input *contextargs.Context) (bool, error) // ExecuteWithResults executes the protocol requests and returns results instead of writing them. - ExecuteWithResults(input string, callback OutputEventCallback) error + ExecuteWithResults(input *contextargs.Context, callback OutputEventCallback) error } // ExecuterOptions contains the configuration options for executer clients @@ -100,7 +101,7 @@ type Request interface { // Extract performs extracting operation for an extractor on model and returns true or false. Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} // ExecuteWithResults executes the protocol requests and returns results instead of writing them. - ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback OutputEventCallback) error + ExecuteWithResults(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback OutputEventCallback) error // MakeResultEventItem creates a result event from internal wrapped event. Intended to be used by MakeResultEventItem internally MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent // MakeResultEvent creates a flat list of result events from an internal wrapped event, based on successful matchers and extracted data diff --git a/v2/pkg/protocols/ssl/ssl.go b/v2/pkg/protocols/ssl/ssl.go index dcaa55b9..8a0dbd43 100644 --- a/v2/pkg/protocols/ssl/ssl.go +++ b/v2/pkg/protocols/ssl/ssl.go @@ -18,6 +18,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" @@ -125,8 +126,8 @@ func (request *Request) GetID() string { } // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (request *Request) ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { - address, err := getAddress(input) +func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { + address, err := getAddress(input.Input) if err != nil { return nil } @@ -152,7 +153,7 @@ func (request *Request) ExecuteWithResults(input string, dynamicValues, previous finalAddress, dataErr := expressions.EvaluateByte([]byte(request.Address), payloadValues) if dataErr != nil { - requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), dataErr) + requestOptions.Output.Request(requestOptions.TemplateID, input.Input, request.Type().String(), dataErr) requestOptions.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(dataErr, "could not evaluate template expressions") } @@ -164,7 +165,7 @@ func (request *Request) ExecuteWithResults(input string, dynamicValues, previous response, err := request.tlsx.Connect(host, host, port) if err != nil { - requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err) + requestOptions.Output.Request(requestOptions.TemplateID, input.Input, request.Type().String(), err) requestOptions.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not connect to server") } @@ -173,12 +174,12 @@ func (request *Request) ExecuteWithResults(input string, dynamicValues, previous gologger.Verbose().Msgf("Sent SSL request to %s", address) if requestOptions.Options.Debug || requestOptions.Options.DebugRequests || requestOptions.Options.StoreResponse { - msg := fmt.Sprintf("[%s] Dumped SSL request for %s", requestOptions.TemplateID, input) + msg := fmt.Sprintf("[%s] Dumped SSL request for %s", requestOptions.TemplateID, input.Input) if requestOptions.Options.Debug || requestOptions.Options.DebugRequests { - gologger.Debug().Str("address", input).Msg(msg) + gologger.Debug().Str("address", input.Input).Msg(msg) } if requestOptions.Options.StoreResponse { - request.options.Output.WriteStoreDebugData(input, request.options.TemplateID, request.Type().String(), msg) + request.options.Output.WriteStoreDebugData(input.Input, request.options.TemplateID, request.Type().String(), msg) } } @@ -219,13 +220,13 @@ func (request *Request) ExecuteWithResults(input string, dynamicValues, previous event := eventcreator.CreateEvent(request, data, requestOptions.Options.Debug || requestOptions.Options.DebugResponse) if requestOptions.Options.Debug || requestOptions.Options.DebugResponse || requestOptions.Options.StoreResponse { - msg := fmt.Sprintf("[%s] Dumped SSL response for %s", requestOptions.TemplateID, input) + msg := fmt.Sprintf("[%s] Dumped SSL response for %s", requestOptions.TemplateID, input.Input) if requestOptions.Options.Debug || requestOptions.Options.DebugResponse { gologger.Debug().Msg(msg) gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, jsonDataString, requestOptions.Options.NoColor, false)) } if requestOptions.Options.StoreResponse { - request.options.Output.WriteStoreDebugData(input, request.options.TemplateID, request.Type().String(), fmt.Sprintf("%s\n%s", msg, jsonDataString)) + request.options.Output.WriteStoreDebugData(input.Input, request.options.TemplateID, request.Type().String(), fmt.Sprintf("%s\n%s", msg, jsonDataString)) } } callback(event) diff --git a/v2/pkg/protocols/ssl/ssl_test.go b/v2/pkg/protocols/ssl/ssl_test.go index 8764f0ce..469a590e 100644 --- a/v2/pkg/protocols/ssl/ssl_test.go +++ b/v2/pkg/protocols/ssl/ssl_test.go @@ -8,6 +8,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/testutils" ) @@ -27,7 +28,9 @@ func TestSSLProtocol(t *testing.T) { require.Nil(t, err, "could not compile ssl request") var gotEvent output.InternalEvent - err = request.ExecuteWithResults("google.com:443", nil, nil, func(event *output.InternalWrappedEvent) { + ctxArgs := contextargs.New() + ctxArgs.Input = "google.com:443" + err = request.ExecuteWithResults(ctxArgs, nil, nil, func(event *output.InternalWrappedEvent) { gotEvent = event.InternalEvent }) require.Nil(t, err, "could not run ssl request") diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index d95cef39..c31318aa 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -23,6 +23,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" @@ -135,8 +136,8 @@ func (request *Request) GetID() string { } // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (request *Request) ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { - hostname, err := getAddress(input) +func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { + hostname, err := getAddress(input.Input) if err != nil { return err } @@ -149,13 +150,13 @@ func (request *Request) ExecuteWithResults(input string, dynamicValues, previous if !ok { break } - if err := request.executeRequestWithPayloads(input, hostname, value, previous, callback); err != nil { + if err := request.executeRequestWithPayloads(input.Input, hostname, value, previous, callback); err != nil { return err } } } else { value := make(map[string]interface{}) - if err := request.executeRequestWithPayloads(input, hostname, value, previous, callback); err != nil { + if err := request.executeRequestWithPayloads(input.Input, hostname, value, previous, callback); err != nil { return err } } diff --git a/v2/pkg/protocols/whois/whois.go b/v2/pkg/protocols/whois/whois.go index e74bf9c3..14bbe78d 100644 --- a/v2/pkg/protocols/whois/whois.go +++ b/v2/pkg/protocols/whois/whois.go @@ -15,6 +15,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer" @@ -86,9 +87,9 @@ func (request *Request) GetID() string { } // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (request *Request) ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { +func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { // generate variables - variables := generateVariables(input) + variables := generateVariables(input.Input) if request.options.Options.Debug || request.options.Options.DebugRequests { gologger.Debug().Msgf("Protocol request variables: \n%s\n", vardump.DumpVariables(variables)) diff --git a/v2/pkg/templates/cluster.go b/v2/pkg/templates/cluster.go index 78fad59b..253c2846 100644 --- a/v2/pkg/templates/cluster.go +++ b/v2/pkg/templates/cluster.go @@ -11,6 +11,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/writer" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http" ) @@ -29,10 +30,10 @@ import ( // The equality check is performed as described below - // // Cases where clustering is not perfomed (request is considered different) -// - If request contains payloads,raw,body,unsafe,req-condition,name attributes -// - If request methods,max-redirects,cookie-reuse,redirects are not equal -// - If request paths aren't identical. -// - If request headers aren't identical +// - If request contains payloads,raw,body,unsafe,req-condition,name attributes +// - If request methods,max-redirects,cookie-reuse,redirects are not equal +// - If request paths aren't identical. +// - If request headers aren't identical // // If multiple requests are identified as identical, they are appended to a slice. // Finally, the engine creates a single executer with a clusteredexecuter for all templates @@ -179,7 +180,7 @@ func (e *ClusterExecuter) Requests() int { } // Execute executes the protocol group and returns true or false if results were found. -func (e *ClusterExecuter) Execute(input string) (bool, error) { +func (e *ClusterExecuter) Execute(input *contextargs.Context) (bool, error) { var results bool previous := make(map[string]interface{}) @@ -207,13 +208,13 @@ func (e *ClusterExecuter) Execute(input string) (bool, error) { } }) if err != nil && e.options.HostErrorsCache != nil { - e.options.HostErrorsCache.MarkFailed(input, err) + e.options.HostErrorsCache.MarkFailed(input.Input, err) } return results, err } // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (e *ClusterExecuter) ExecuteWithResults(input string, callback protocols.OutputEventCallback) error { +func (e *ClusterExecuter) ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error { dynamicValues := make(map[string]interface{}) err := e.requests.ExecuteWithResults(input, dynamicValues, nil, func(event *output.InternalWrappedEvent) { for _, operator := range e.operators { @@ -229,7 +230,7 @@ func (e *ClusterExecuter) ExecuteWithResults(input string, callback protocols.Ou } }) if err != nil && e.options.HostErrorsCache != nil { - e.options.HostErrorsCache.MarkFailed(input, err) + e.options.HostErrorsCache.MarkFailed(input.Input, err) } return err }