From 49ef5cbf16475a5ef1f5cd4b22764ba09e9a6fac Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com> Date: Wed, 13 Mar 2024 20:35:19 +0530 Subject: [PATCH] handle 1 more edgecase (#4868) * handle 1 more edgecase * add integration test for this edgecase * fix multi-http-var-sharing with integration test * add -payload-concurrency (-pc) flag * fix missing internal:true login in multiprotocol engine * fix/handle absolute invalid url parsing * support -pc & -jc in go sdk * fix missing variables in code protocol operators * add payload count parallelhttp check --- cmd/integration-test/http.go | 63 ++++++++++++++++++- cmd/nuclei/main.go | 1 + go.mod | 7 +-- go.sum | 14 ++--- .../protocols/code/py-env-var.yaml | 2 +- integration_tests/protocols/code/py-file.yaml | 2 +- .../protocols/code/py-interactsh.yaml | 2 +- .../protocols/code/py-snippet.yaml | 2 +- .../http-matcher-extractor-dy-extractor.yaml | 36 +++++++++++ .../protocols/http/http-paths.yaml | 2 +- .../http/multi-http-var-sharing.yaml | 36 +++++++++++ lib/config.go | 12 ++-- pkg/operators/operators.go | 10 ++- pkg/protocols/code/code.go | 4 ++ pkg/protocols/http/build_request.go | 7 +-- pkg/protocols/http/request.go | 11 ++-- pkg/protocols/http/utils.go | 13 +++- pkg/protocols/protocols.go | 2 +- pkg/tmplexec/exec.go | 26 +++----- pkg/tmplexec/multiproto/multi.go | 13 ++-- pkg/types/types.go | 2 + 21 files changed, 204 insertions(+), 63 deletions(-) create mode 100644 integration_tests/protocols/http/http-matcher-extractor-dy-extractor.yaml create mode 100644 integration_tests/protocols/http/multi-http-var-sharing.yaml diff --git a/cmd/integration-test/http.go b/cmd/integration-test/http.go index e1fddc04..41e1910a 100644 --- a/cmd/integration-test/http.go +++ b/cmd/integration-test/http.go @@ -80,6 +80,55 @@ var httpTestcases = []TestCaseInfo{ {Path: "protocols/http/disable-path-automerge.yaml", TestCase: &httpDisablePathAutomerge{}}, {Path: "protocols/http/http-preprocessor.yaml", TestCase: &httpPreprocessor{}}, {Path: "protocols/http/multi-request.yaml", TestCase: &httpMultiRequest{}}, + {Path: "protocols/http/http-matcher-extractor-dy-extractor.yaml", TestCase: &httpMatcherExtractorDynamicExtractor{}}, + {Path: "protocols/http/multi-http-var-sharing.yaml", TestCase: &httpMultiVarSharing{}}, +} + +type httpMultiVarSharing struct{} + +func (h *httpMultiVarSharing) Execute(filePath string) error { + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "https://scanme.sh", debug) + if err != nil { + return err + } + return expectResultsCount(results, 1) +} + +type httpMatcherExtractorDynamicExtractor struct{} + +func (h *httpMatcherExtractorDynamicExtractor) Execute(filePath string) error { + router := httprouter.New() + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + html := ` + + + Domains + +` + fmt.Fprint(w, html) + }) + router.GET("/domains", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + html := ` + + + Dynamic Extractor Test + + + + + + ` + fmt.Fprint(w, html) + }) + ts := httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) + if err != nil { + return err + } + + return expectResultsCount(results, 1) } type httpInteractshRequest struct{} @@ -776,8 +825,18 @@ func (h *httpPaths) Execute(filepath string) error { } } - if !reflect.DeepEqual(expected, actual) { - return fmt.Errorf("%8v: %v\n%-8v: %v", "expected", expected, "actual", actual) + if len(expected) > len(actual) { + actualValuesIndex := len(actual) - 1 + if actualValuesIndex < 0 { + actualValuesIndex = 0 + } + return fmt.Errorf("missing values : %v", expected[actualValuesIndex:]) + } else if len(expected) < len(actual) { + return fmt.Errorf("unexpected values : %v", actual[len(expected)-1:]) + } else { + if !reflect.DeepEqual(expected, actual) { + return fmt.Errorf("expected: %v\n\nactual: %v", expected, actual) + } } return nil } diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go index d16078db..684ea618 100644 --- a/cmd/nuclei/main.go +++ b/cmd/nuclei/main.go @@ -322,6 +322,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.IntVarP(&options.HeadlessBulkSize, "headless-bulk-size", "hbs", 10, "maximum number of headless hosts to be analyzed in parallel per template"), flagSet.IntVarP(&options.HeadlessTemplateThreads, "headless-concurrency", "headc", 10, "maximum number of headless templates to be executed in parallel"), flagSet.IntVarP(&options.JsConcurrency, "js-concurrency", "jsc", 120, "maximum number of javascript runtimes to be executed in parallel"), + flagSet.IntVarP(&options.PayloadConcurrency, "payload-concurrency", "pc", 25, "max payload concurrency for each template"), ) flagSet.CreateGroup("optimization", "Optimizations", flagSet.IntVar(&options.Timeout, "timeout", 10, "time to wait in seconds before timeout"), diff --git a/go.mod b/go.mod index ca89a0eb..18fcf143 100644 --- a/go.mod +++ b/go.mod @@ -20,12 +20,12 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.20 - github.com/projectdiscovery/fastdialer v0.0.61 + github.com/projectdiscovery/fastdialer v0.0.62 github.com/projectdiscovery/hmap v0.0.41 github.com/projectdiscovery/interactsh v1.1.9 github.com/projectdiscovery/rawhttp v0.1.40 github.com/projectdiscovery/retryabledns v1.0.58 - github.com/projectdiscovery/retryablehttp-go v1.0.50 + github.com/projectdiscovery/retryablehttp-go v1.0.51 github.com/projectdiscovery/yamldoc-go v1.0.4 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 @@ -91,7 +91,7 @@ require ( github.com/projectdiscovery/tlsx v1.1.6 github.com/projectdiscovery/uncover v1.0.7 github.com/projectdiscovery/useragent v0.0.40 - github.com/projectdiscovery/utils v0.0.82 + github.com/projectdiscovery/utils v0.0.84-0.20240312214300-d3ba70dbb9ca github.com/projectdiscovery/wappalyzergo v0.0.112 github.com/redis/go-redis/v9 v9.1.0 github.com/sashabaranov/go-openai v1.15.3 @@ -274,7 +274,6 @@ require ( github.com/tklauser/numcpus v0.6.0 // indirect github.com/trivago/tgo v1.0.7 github.com/ulikunitz/xz v0.5.11 // indirect - github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/yl2chen/cidranger v1.0.2 // indirect github.com/ysmood/goob v0.4.0 // indirect diff --git a/go.sum b/go.sum index 5c896ca7..f3305018 100644 --- a/go.sum +++ b/go.sum @@ -812,8 +812,8 @@ github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPo github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4= github.com/projectdiscovery/dsl v0.0.46 h1:zBNNzSBA1aakGY44w6KhnjJQx/zO+oW2Wx7TR8xZm/A= github.com/projectdiscovery/dsl v0.0.46/go.mod h1:eoZEJFCIT5emI00xj8HTQwHz4YwwGAD+grL3G7CDlfs= -github.com/projectdiscovery/fastdialer v0.0.61 h1:z5OzP9lRbn6fSIezgReKC3hkzRh+YX41ST9OgkVEm/s= -github.com/projectdiscovery/fastdialer v0.0.61/go.mod h1:FyxJ0m1MwB69nLmdXYqK32f3a0Pf+5YpC8wBY73baiE= +github.com/projectdiscovery/fastdialer v0.0.62 h1:Wyba2hD6ZF3S04MgCn380mC+1RXJ+dq14Yq8u2yk7ps= +github.com/projectdiscovery/fastdialer v0.0.62/go.mod h1:2baj2TRXTw+hHbKTW9IZR4dhpxCGJkq5AKL1ge5gis8= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw= github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q= @@ -852,8 +852,8 @@ github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gB github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= github.com/projectdiscovery/retryabledns v1.0.58 h1:ut1FSB9+GZ6zQIlKJFLqIz2RZs81EmkbsHTuIrWfYLE= github.com/projectdiscovery/retryabledns v1.0.58/go.mod h1:RobmKoNBgngAVE4H9REQtaLP1pa4TCyypHy1MWHT1mY= -github.com/projectdiscovery/retryablehttp-go v1.0.50 h1:QiVqNzpDFoTZm2z1oO68FEtl2VTk2kFscPetxmCK2gc= -github.com/projectdiscovery/retryablehttp-go v1.0.50/go.mod h1:yY/go76Lx/MgJmR+dSg/Xa8wOiXNjmuEdBDVDODKeDk= +github.com/projectdiscovery/retryablehttp-go v1.0.51 h1:8XMrNC8JrwvySESe2d+XWF9bq4unWqD4PUPEC4Cai8s= +github.com/projectdiscovery/retryablehttp-go v1.0.51/go.mod h1:6cdh/acYHpeYWg7+Iblh4xBRb87bC118L4G4mpvCMuA= github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us= github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= @@ -864,8 +864,8 @@ github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7 github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE= github.com/projectdiscovery/useragent v0.0.40 h1:1LUhReSGPkhqsM5n40OOC9dIoNqMGs1dyGFJcOmg2Fo= github.com/projectdiscovery/useragent v0.0.40/go.mod h1:EvK1x3s948Gtqb/XOahXcauyejCL/rSgy5d1IAvsKT4= -github.com/projectdiscovery/utils v0.0.82 h1:U//02floCSFxJluN7MP+rJSwI4Px7o454JL7ukERArI= -github.com/projectdiscovery/utils v0.0.82/go.mod h1:AbmIvy0TTlsfXxPDEMaNPVrxmqDmYiCnbGqh0TTthE4= +github.com/projectdiscovery/utils v0.0.84-0.20240312214300-d3ba70dbb9ca h1:GY9lUYDlENXPSFPJH01Bm1BfhrUF2jpnUBR+K4VPJIs= +github.com/projectdiscovery/utils v0.0.84-0.20240312214300-d3ba70dbb9ca/go.mod h1:wzMfHBq2I9oy+DEiMfUYV86g1D7eXKaQsgWnqFpmMtI= github.com/projectdiscovery/wappalyzergo v0.0.112 h1:QPpp5jmj1lqLd5mFdFKQ9VvcYhQNqyU9Mr+IB0US2zA= github.com/projectdiscovery/wappalyzergo v0.0.112/go.mod h1:hc/o+fgM8KtdpFesjfBTmHTwsR+yBd+4kYZW/DGy/x8= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= @@ -1034,8 +1034,6 @@ github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oW github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 h1:TtyC78WMafNW8QFfv3TeP3yWNDG+uxNkk9vOrnDu6JA= -github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6/go.mod h1:h8272+G2omSmi30fBXiZDMkmHuOgonplfKIKjQWzlfs= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= diff --git a/integration_tests/protocols/code/py-env-var.yaml b/integration_tests/protocols/code/py-env-var.yaml index c4b9e6c5..4ccf3648 100644 --- a/integration_tests/protocols/code/py-env-var.yaml +++ b/integration_tests/protocols/code/py-env-var.yaml @@ -20,4 +20,4 @@ code: - type: word words: - "hello from input baz" -# digest: 4a0a00473045022100b290a0c40f27573f0de9a950be13457a9bf59ade1ff2f497bf01a3b526e5db750220761942acffd6d27e2714ddaa1c73c699ccd7de48839f08cff1d6a9456bc8ff1f:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file +# digest: 4a0a0047304502207e3a5eda5f3207c3c01c820562243281926c1215224a7c80ed7528559b9f52cb022100f6ef99bb45843f481705778630f2cfd8f4d1cc3acb96392ff016f22e06aa91af:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file diff --git a/integration_tests/protocols/code/py-file.yaml b/integration_tests/protocols/code/py-file.yaml index 74203e8d..9e0b041b 100644 --- a/integration_tests/protocols/code/py-file.yaml +++ b/integration_tests/protocols/code/py-file.yaml @@ -18,4 +18,4 @@ code: - type: word words: - "hello from input" -# digest: 490a004630440220335663a6a4db720ee6276ab7179a87a6be0b4030771ec5ee82ecf6982342113602200a2570db7eb9721f6ceb1a89543fc436ee62b30d1b720c75ea3834ed3d2b64f3:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file +# digest: 4a0a004730450220069673af9bd6d6677f9529d06f5d8bd46d543089a4731ed18ee806761d75fd60022100913a3e27b0a5809baf710ba9585bf9fe729634c0e19e3e13eef70a6bd100df34:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file diff --git a/integration_tests/protocols/code/py-interactsh.yaml b/integration_tests/protocols/code/py-interactsh.yaml index eac53c3b..24e4b062 100644 --- a/integration_tests/protocols/code/py-interactsh.yaml +++ b/integration_tests/protocols/code/py-interactsh.yaml @@ -26,4 +26,4 @@ code: part: interactsh_protocol words: - "http" -# digest: 490a004630440220400892730a62fa1bbb1064e4d88eea760dbf8f01c6b630ff0f5b126fd1952839022025a6d52e730c1f1cfcbd440e6269f93489db3a77cb2a27d0f47522c0819dc8d3:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file +# digest: 490a00463044022003b8d069e3c84412729c43e33013a52ee04eabcf096d511979691d71d8e905f60220011f4475899abed4f86b4bd5e6c2423750759135206e4729826afe1ed8a44f4d:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file diff --git a/integration_tests/protocols/code/py-snippet.yaml b/integration_tests/protocols/code/py-snippet.yaml index f8a30b4f..287ca2c6 100644 --- a/integration_tests/protocols/code/py-snippet.yaml +++ b/integration_tests/protocols/code/py-snippet.yaml @@ -21,4 +21,4 @@ code: - type: word words: - "hello from input" -# digest: 490a0046304402206b14abdc0d5fc13466f5c292da9fb2a19d1b2c5e683cc052037fe367b372f82b02202c00b9acbd8106a769eb411794c567d3019433671397bf909e16b286105ed69e:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file +# digest: 4a0a00473045022100c291615cf2a8005450c17a6554e81a9cdab14743b299f0679c644247929198b502206fdacc8ab173bde2b4015340012637916bf2659f66f320fcc06b97ac639072a1:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file diff --git a/integration_tests/protocols/http/http-matcher-extractor-dy-extractor.yaml b/integration_tests/protocols/http/http-matcher-extractor-dy-extractor.yaml new file mode 100644 index 00000000..eb26d50b --- /dev/null +++ b/integration_tests/protocols/http/http-matcher-extractor-dy-extractor.yaml @@ -0,0 +1,36 @@ +id: http-matcher-extractor-dy-extractor +info: + name: HTTP matcher and extractor & dynamic extractor + description: > + Edgecase to test for a combination of matchers , extractors and dynamic extractors + author: pdteam + severity: info + +http: + - raw: + - | + GET {{BaseURL}} HTTP/1.1 + - | + GET {{absolutePath}} HTTP/1.1 + + req-condition: true + extractors: + - type: regex + internal: true + part: body_1 + name: absolutePath + regex: + - '' + group: 1 + - type: regex + internal: false + part: body_2 + name: title + regex: + - ']*>([^<]+)' + group: 1 + matchers: + - type: regex + part: body_2 + regex: + - ']*>([^<]+)' \ No newline at end of file diff --git a/integration_tests/protocols/http/http-paths.yaml b/integration_tests/protocols/http/http-paths.yaml index ae389a37..ba5be2d5 100644 --- a/integration_tests/protocols/http/http-paths.yaml +++ b/integration_tests/protocols/http/http-paths.yaml @@ -28,7 +28,7 @@ info: - "//CFIDE/wizards/common/utils.cfc" # Test all templates with FullURLs -requests: +http: - raw: # relative path without leading slash with param # If relative path does not have `/` prefix it is autocorrected diff --git a/integration_tests/protocols/http/multi-http-var-sharing.yaml b/integration_tests/protocols/http/multi-http-var-sharing.yaml new file mode 100644 index 00000000..606a0365 --- /dev/null +++ b/integration_tests/protocols/http/multi-http-var-sharing.yaml @@ -0,0 +1,36 @@ +id: multi-http-var-sharing + +info: + name: Multi HTTP var sharing + author: pdteam + severity: info + description: | + A template which has multiple HTTP requests block and variables are shared between them + +http: + - method: GET + path: + - "{{BaseURL}}" + + matchers: + - type: word + words: + - "This is test matcher text" + negative: true + internal: true + + extractors: + - type: dsl + name: ffff + dsl: + - status_code + internal: true + + - method: GET + path: + - "{{BaseURL}}/{{ffff}}" + + matchers: + - type: status + status: + - 200 \ No newline at end of file diff --git a/lib/config.go b/lib/config.go index 0c48f373..65f3fc8c 100644 --- a/lib/config.go +++ b/lib/config.go @@ -107,10 +107,12 @@ func WithInteractshOptions(opts InteractshOpts) NucleiSDKOptions { // Concurrency options type Concurrency struct { - TemplateConcurrency int // number of templates to run concurrently (per host in host-spray mode) - HostConcurrency int // number of hosts to scan concurrently (per template in template-spray mode) - HeadlessHostConcurrency int // number of hosts to scan concurrently for headless templates (per template in template-spray mode) - HeadlessTemplateConcurrency int // number of templates to run concurrently for headless templates (per host in host-spray mode) + TemplateConcurrency int // number of templates to run concurrently (per host in host-spray mode) + HostConcurrency int // number of hosts to scan concurrently (per template in template-spray mode) + HeadlessHostConcurrency int // number of hosts to scan concurrently for headless templates (per template in template-spray mode) + HeadlessTemplateConcurrency int // number of templates to run concurrently for headless templates (per host in host-spray mode) + JavascriptTemplateConcurrency int // number of templates to run concurrently for javascript templates (per host in host-spray mode) + TemplatePayloadConcurrency int // max concurrent payloads to run for a template (a good default is 25) } // WithConcurrency sets concurrency options @@ -120,6 +122,8 @@ func WithConcurrency(opts Concurrency) NucleiSDKOptions { e.opts.BulkSize = opts.HostConcurrency e.opts.HeadlessBulkSize = opts.HeadlessHostConcurrency e.opts.HeadlessTemplateThreads = opts.HeadlessTemplateConcurrency + e.opts.JsConcurrency = opts.JavascriptTemplateConcurrency + e.opts.PayloadConcurrency = opts.TemplatePayloadConcurrency return nil } } diff --git a/pkg/operators/operators.go b/pkg/operators/operators.go index a3b4fc56..1d6103fe 100644 --- a/pkg/operators/operators.go +++ b/pkg/operators/operators.go @@ -299,7 +299,7 @@ func (operators *Operators) Execute(data map[string]interface{}, match MatchFunc if len(result.DynamicValues) > 0 { return result, true } - return nil, false + return result, false } } @@ -313,6 +313,10 @@ func (operators *Operators) Execute(data map[string]interface{}, match MatchFunc // Don't print if we have matchers, and they have not matched, regardless of extractor if len(operators.Matchers) > 0 && !matches { + // if dynamic values are present then it is not a failure + if len(result.DynamicValues) > 0 { + return result, true + } return nil, false } // Write a final string of output if matcher type is @@ -320,6 +324,10 @@ func (operators *Operators) Execute(data map[string]interface{}, match MatchFunc if len(result.Extracts) > 0 || len(result.OutputExtracts) > 0 || matches { return result, true } + // if dynamic values are present then it is not a failure + if len(result.DynamicValues) > 0 { + return result, true + } return nil, false } diff --git a/pkg/protocols/code/code.go b/pkg/protocols/code/code.go index 9423d200..57eb9b88 100644 --- a/pkg/protocols/code/code.go +++ b/pkg/protocols/code/code.go @@ -192,6 +192,10 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa dataOutputString := fmtStdout(gOutput.Stdout.String()) data := make(output.InternalEvent) + // also include all request variables in result event + for _, value := range metaSrc.Variables { + data[value.Name] = value.Value + } data["type"] = request.Type().String() data["response"] = dataOutputString // response contains filtered output (eg without trailing \n) diff --git a/pkg/protocols/http/build_request.go b/pkg/protocols/http/build_request.go index b526d9d2..d8ebb397 100644 --- a/pkg/protocols/http/build_request.go +++ b/pkg/protocols/http/build_request.go @@ -97,7 +97,7 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context, } // Parse target url - parsed, err := urlutil.Parse(input.MetaInput.Input) + parsed, err := urlutil.ParseAbsoluteURL(input.MetaInput.Input, false) if err != nil { return nil, err } @@ -153,7 +153,7 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context, return r.generateRawRequest(ctx, reqData, parsed, finalVars, payloads) } - reqURL, err := urlutil.ParseURL(reqData, true) + reqURL, err := urlutil.ParseAbsoluteURL(reqData, true) if err != nil { return nil, errorutil.NewWithTag("http", "failed to parse url %v while creating http request", reqData) } @@ -291,8 +291,7 @@ func (r *requestGenerator) generateRawRequest(ctx context.Context, rawRequest st unsafeReq := &generatedRequest{rawRequest: rawRequestData, meta: generatorValues, original: r.request, interactshURLs: r.interactshURLs} return unsafeReq, nil } - - urlx, err := urlutil.ParseURL(rawRequestData.FullURL, true) + urlx, err := urlutil.ParseAbsoluteURL(rawRequestData.FullURL, true) if err != nil { return nil, errorutil.NewWithErr(err).Msgf("failed to create request with url %v got %v", rawRequestData.FullURL, err).WithTag("raw") } diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go index 9bacd6d9..09b6c43f 100644 --- a/pkg/protocols/http/request.go +++ b/pkg/protocols/http/request.go @@ -38,6 +38,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/types" "github.com/projectdiscovery/rawhttp" convUtil "github.com/projectdiscovery/utils/conversion" + errorutil "github.com/projectdiscovery/utils/errors" httpUtils "github.com/projectdiscovery/utils/http" "github.com/projectdiscovery/utils/reader" sliceutil "github.com/projectdiscovery/utils/slice" @@ -453,7 +454,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa } // verify if parallel elaboration was requested - if request.Threads > 0 { + if request.Threads > 0 && len(request.Payloads) > 0 { return request.executeParallelHTTP(input, dynamicValues, callback) } @@ -494,7 +495,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa return true, nil } var gotMatches bool - err = request.executeRequest(input, generatedHttpRequest, previous, hasInteractMatchers, func(event *output.InternalWrappedEvent) { + execReqErr := request.executeRequest(input, generatedHttpRequest, previous, hasInteractMatchers, func(event *output.InternalWrappedEvent) { // a special case where operators has interactsh matchers and multiple request are made // ex: status_code_2 , interactsh_protocol (from 1st request) etc needsRequestEvent := interactsh.HasMatchers(request.CompiledOperators) && request.NeedsRequestCondition() @@ -525,14 +526,14 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa }, generator.currentIndex) // If a variable is unresolved, skip all further requests - if errors.Is(err, errStopExecution) { + if errors.Is(execReqErr, errStopExecution) { return true, nil } - if err != nil { + if execReqErr != nil { if request.options.HostErrorsCache != nil { request.options.HostErrorsCache.MarkFailed(input.MetaInput.ID(), err) } - requestErr = err + requestErr = errorutil.NewWithErr(execReqErr).Msgf("got err while executing %v", generatedHttpRequest.URL()) } request.options.Progress.IncrementRequests() diff --git a/pkg/protocols/http/utils.go b/pkg/protocols/http/utils.go index fe893ebe..875faaf2 100644 --- a/pkg/protocols/http/utils.go +++ b/pkg/protocols/http/utils.go @@ -6,13 +6,22 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators" "github.com/projectdiscovery/rawhttp" + errorutil "github.com/projectdiscovery/utils/errors" ) // dump creates a dump of the http request in form of a byte slice func dump(req *generatedRequest, reqURL string) ([]byte, error) { if req.request != nil { - return req.request.Dump() + bin, err := req.request.Dump() + if err != nil { + return nil, errorutil.NewWithErr(err).WithTag("http").Msgf("could not dump request: %v", req.request.URL.String()) + } + return bin, nil } rawHttpOptions := &rawhttp.Options{CustomHeaders: req.rawRequest.UnsafeHeaders, CustomRawBytes: req.rawRequest.UnsafeRawBytes} - return rawhttp.DumpRequestRaw(req.rawRequest.Method, reqURL, req.rawRequest.Path, generators.ExpandMapValues(req.rawRequest.Headers), io.NopCloser(strings.NewReader(req.rawRequest.Data)), rawHttpOptions) + bin, err := rawhttp.DumpRequestRaw(req.rawRequest.Method, reqURL, req.rawRequest.Path, generators.ExpandMapValues(req.rawRequest.Headers), io.NopCloser(strings.NewReader(req.rawRequest.Data)), rawHttpOptions) + if err != nil { + return nil, errorutil.NewWithErr(err).WithTag("http").Msgf("could not dump request: %v", reqURL) + } + return bin, nil } diff --git a/pkg/protocols/protocols.go b/pkg/protocols/protocols.go index 32b09841..c8323dcb 100644 --- a/pkg/protocols/protocols.go +++ b/pkg/protocols/protocols.go @@ -129,7 +129,7 @@ func (e *ExecutorOptions) GetThreadsForNPayloadRequests(totalRequests int, curre if currentThreads > 0 { return currentThreads } else { - return e.Options.TemplateThreads + return e.Options.PayloadConcurrency } } diff --git a/pkg/tmplexec/exec.go b/pkg/tmplexec/exec.go index 5035a1ff..3e0e8fdd 100644 --- a/pkg/tmplexec/exec.go +++ b/pkg/tmplexec/exec.go @@ -3,6 +3,7 @@ package tmplexec import ( "errors" "fmt" + "runtime/debug" "strings" "sync/atomic" @@ -34,16 +35,6 @@ var _ protocols.Executer = &TemplateExecuter{} // NewTemplateExecuter creates a new request TemplateExecuter for list of requests func NewTemplateExecuter(requests []protocols.Request, options *protocols.ExecutorOptions) (*TemplateExecuter, error) { - isMultiProto := false - lastProto := "" - for _, request := range requests { - if request.Type().String() != lastProto && lastProto != "" { - isMultiProto = true - break - } - lastProto = request.Type().String() - } - e := &TemplateExecuter{requests: requests, options: options, results: &atomic.Bool{}} if options.Flow != "" { // we use a dummy input here because goal of flow executor at this point is to just check @@ -55,13 +46,11 @@ func NewTemplateExecuter(requests []protocols.Request, options *protocols.Execut } e.program = p } else { - // Review: - // multiproto engine is only used if there is more than one protocol in template - // else we use generic engine (should we use multiproto engine for single protocol with multiple requests as well ?) - if isMultiProto { - e.engine = multiproto.NewMultiProtocol(requests, options, e.results) - } else { + // only use generic if there is only 1 protocol with only 1 section + if len(requests) == 1 { e.engine = generic.NewGenericEngine(requests, options, e.results) + } else { + e.engine = multiproto.NewMultiProtocol(requests, options, e.results) } } return e, nil @@ -113,8 +102,9 @@ func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) { defer func() { // try catching unknown panics if r := recover(); r != nil { - ctx.LogError(fmt.Errorf("panic: %v", r)) - gologger.Verbose().Msgf("panic: %v", r) + stacktrace := debug.Stack() + ctx.LogError(fmt.Errorf("panic: %v\n%s", r, stacktrace)) + gologger.Verbose().Msgf("panic: %v\n%s", r, stacktrace) } }() diff --git a/pkg/tmplexec/multiproto/multi.go b/pkg/tmplexec/multiproto/multi.go index 021c8353..d164c03a 100644 --- a/pkg/tmplexec/multiproto/multi.go +++ b/pkg/tmplexec/multiproto/multi.go @@ -46,12 +46,13 @@ func (m *MultiProtocol) Compile() error { func (m *MultiProtocol) ExecuteWithResults(ctx *scan.ScanContext) error { // put all readonly args into template context m.options.GetTemplateCtx(ctx.Input.MetaInput).Merge(m.readOnlyArgs) - var finalProtoEvent *output.InternalWrappedEvent // callback to process results from all protocols multiProtoCallback := func(event *output.InternalWrappedEvent) { - if event != nil { - finalProtoEvent = event + if event == nil { + return } + // log event and generate result for the event + ctx.LogEvent(event) // export dynamic values from operators (i.e internal:true) if event.OperatorsResult != nil && len(event.OperatorsResult.DynamicValues) > 0 { for k, v := range event.OperatorsResult.DynamicValues { @@ -97,12 +98,6 @@ func (m *MultiProtocol) ExecuteWithResults(ctx *scan.ScanContext) error { return err } } - // Review: how to handle events of multiple protocols in a single template - // currently the outer callback is only executed once (for the last protocol in queue) - // due to workflow logic at https://github.com/projectdiscovery/nuclei/blob/main/pkg/protocols/common/executer/executem.go#L150 - // this causes addition of duplicated / unncessary variables with prefix template_id_all_variables - ctx.LogEvent(finalProtoEvent) - return nil } diff --git a/pkg/types/types.go b/pkg/types/types.go index e8419475..fb42ea6f 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -370,6 +370,8 @@ type Options struct { ScanID string // JsConcurrency is the number of concurrent js routines to run JsConcurrency int + // PayloadConcurrency is the number of concurrent payloads to run per template + PayloadConcurrency int } // ShouldLoadResume resume file