From e88889b2634d2eb33a8ae50766b38141e5881d90 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com> Date: Fri, 29 Mar 2024 13:31:30 +0530 Subject: [PATCH] add `-dast` flag and multiple bug fixes for dast templates (#4941) * add default get method * remove residual payload logic from old implementation * fuzz: clone current state of component * fuzz: bug fix stacking of payloads in multiple mode * improve stdout template loading stats * stdout: force display warnings if no templates are loaded * update flags in README.md * quote non-ascii chars in extractor output * aws request signature can only be used in signed & verified tmpls * deprecate request signature * remove logic related to deprecated fuzzing input * update test to use ordered params * fix interactsh-url lazy eval: #4946 * output: skip unnecessary updates when unescaping * updates as per requested changes --- README.md | 3 +- cmd/integration-test/fuzz.go | 51 ----------- cmd/nuclei/main.go | 10 ++- integration_tests/fuzz/fuzz-header-basic.yaml | 49 ----------- .../fuzz/fuzz-header-multiple.yaml | 41 --------- internal/runner/runner.go | 69 ++++++++------- lib/config.go | 14 ++- pkg/catalog/loader/loader.go | 31 ++++--- pkg/fuzz/component/body.go | 7 ++ pkg/fuzz/component/body_test.go | 4 +- pkg/fuzz/component/component.go | 2 + pkg/fuzz/component/cookie.go | 8 ++ pkg/fuzz/component/headers.go | 8 ++ pkg/fuzz/component/path.go | 8 ++ pkg/fuzz/component/query.go | 8 ++ pkg/fuzz/component/value.go | 9 ++ pkg/fuzz/dataformat/kv.go | 17 +++- pkg/fuzz/parts.go | 5 +- pkg/output/format_screen.go | 6 +- pkg/protocols/common/variables/variables.go | 6 ++ pkg/protocols/http/raw/raw.go | 1 - pkg/protocols/http/request_fuzz.go | 88 ++++++------------- pkg/templates/parser_stats.go | 19 ++-- pkg/templates/stats.go | 11 +-- pkg/templates/templates.go | 6 ++ pkg/templates/workflows.go | 4 + pkg/types/types.go | 5 +- 27 files changed, 216 insertions(+), 274 deletions(-) delete mode 100644 integration_tests/fuzz/fuzz-header-basic.yaml delete mode 100644 integration_tests/fuzz/fuzz-header-multiple.yaml diff --git a/README.md b/README.md index e231221c..07bce472 100644 --- a/README.md +++ b/README.md @@ -221,7 +221,8 @@ INTERACTSH: FUZZING: -ft, -fuzzing-type string overrides fuzzing type set in template (replace, prefix, postfix, infix) -fm, -fuzzing-mode string overrides fuzzing mode set in template (multiple, single) - -fuzz enable loading fuzzing templates + -fuzz enable loading fuzzing templates (Deprecated: use -dast instead) + -dast only run DAST templates UNCOVER: -uc, -uncover enable uncover engine diff --git a/cmd/integration-test/fuzz.go b/cmd/integration-test/fuzz.go index be2ac161..f5e71774 100644 --- a/cmd/integration-test/fuzz.go +++ b/cmd/integration-test/fuzz.go @@ -21,8 +21,6 @@ var fuzzingTestCases = []TestCaseInfo{ {Path: "fuzz/fuzz-type.yaml", TestCase: &fuzzTypeOverride{}}, {Path: "fuzz/fuzz-query.yaml", TestCase: &httpFuzzQuery{}}, {Path: "fuzz/fuzz-headless.yaml", TestCase: &HeadlessFuzzingQuery{}}, - {Path: "fuzz/fuzz-header-basic.yaml", TestCase: &FuzzHeaderBasic{}}, - {Path: "fuzz/fuzz-header-multiple.yaml", TestCase: &FuzzHeaderMultiple{}}, // for fuzzing we should prioritize adding test case related backend // logic in fuzz playground server instead of adding them here {Path: "fuzz/fuzz-query-num-replace.yaml", TestCase: &genericFuzzTestCase{expectedResults: 2}}, @@ -176,52 +174,3 @@ func (h *HeadlessFuzzingQuery) Execute(filePath string) error { } return expectResultsCount(got, 2) } - -type FuzzHeaderBasic struct{} - -// Execute executes a test case and returns an error if occurred -func (h *FuzzHeaderBasic) Execute(filePath string) error { - router := httprouter.New() - router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - host := r.Header.Get("Origin") - // redirect to different domain - w.WriteHeader(http.StatusOK) - fmt.Fprintf(w, "Click Here") - }) - ts := httptest.NewTLSServer(router) - defer ts.Close() - - got, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-fuzz") - if err != nil { - return err - } - return expectResultsCount(got, 1) -} - -type FuzzHeaderMultiple struct{} - -// Execute executes a test case and returns an error if occurred -func (h *FuzzHeaderMultiple) Execute(filePath string) error { - router := httprouter.New() - router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - host1 := r.Header.Get("Origin") - host2 := r.Header.Get("X-Forwared-For") - - fmt.Printf("host1: %s, host2: %s\n", host1, host2) - if host1 == host2 && host2 == "secret.local" { - w.WriteHeader(http.StatusOK) - fmt.Fprintf(w, "welcome! to secret admin panel") - return - } - // redirect to different domain - w.WriteHeader(http.StatusForbidden) - }) - ts := httptest.NewTLSServer(router) - defer ts.Close() - - got, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-fuzz") - if err != nil { - return err - } - return expectResultsCount(got, 1) -} diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go index 731865d7..dac48c17 100644 --- a/cmd/nuclei/main.go +++ b/cmd/nuclei/main.go @@ -186,6 +186,7 @@ func readConfig() *goflags.FlagSet { // when true updates nuclei binary to latest version var updateNucleiBinary bool var pdcpauth string + var fuzzFlag bool flagSet := goflags.NewFlagSet() flagSet.CaseSensitive = true @@ -313,7 +314,8 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.CreateGroup("fuzzing", "Fuzzing", flagSet.StringVarP(&options.FuzzingType, "fuzzing-type", "ft", "", "overrides fuzzing type set in template (replace, prefix, postfix, infix)"), flagSet.StringVarP(&options.FuzzingMode, "fuzzing-mode", "fm", "", "overrides fuzzing mode set in template (multiple, single)"), - flagSet.BoolVar(&options.FuzzTemplates, "fuzz", false, "enable loading fuzzing templates"), + flagSet.BoolVar(&fuzzFlag, "fuzz", false, "enable loading fuzzing templates (Deprecated: use -dast instead)"), + flagSet.BoolVar(&options.DAST, "dast", false, "only run DAST templates"), ) flagSet.CreateGroup("uncover", "Uncover", @@ -436,6 +438,12 @@ Additional documentation is available at: https://docs.nuclei.sh/getting-started goflags.DisableAutoConfigMigration = true _ = flagSet.Parse() + // when fuzz flag is enabled, set the dast flag to true + if fuzzFlag { + // backwards compatibility for fuzz flag + options.DAST = true + } + // api key hierarchy: cli flag > env var > .pdcp/credential file if pdcpauth == "true" { runner.AuthWithPDCP() diff --git a/integration_tests/fuzz/fuzz-header-basic.yaml b/integration_tests/fuzz/fuzz-header-basic.yaml deleted file mode 100644 index 10d2928c..00000000 --- a/integration_tests/fuzz/fuzz-header-basic.yaml +++ /dev/null @@ -1,49 +0,0 @@ -id: fuzz-header-basic - -info: - name: fuzz header basic - author: pdteam - severity: info - description: | - In this template we check for any reflection when fuzzing Origin header - -variables: - first: "{{rand_int(10000, 99999)}}" - -http: - - raw: - - | - GET /?x=aaa&y=bbb HTTP/1.1 - Host: {{Hostname}} - Origin: https://example.com - X-Fuzz-Header: 1337 - Cookie: z=aaa; bb=aaa - User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) - Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 - Accept-Language: en-US,en;q=0.9 - Connection: close - - payloads: - reflection: - - "'\"><{{first}}" - - fuzzing: - - part: header - type: replace - mode: single - keys: ["Origin"] - fuzz: - - "{{reflection}}" - - stop-at-first-match: true - matchers-condition: and - matchers: - - type: word - part: body - words: - - "{{reflection}}" - - - type: word - part: header - words: - - "text/html" \ No newline at end of file diff --git a/integration_tests/fuzz/fuzz-header-multiple.yaml b/integration_tests/fuzz/fuzz-header-multiple.yaml deleted file mode 100644 index 0a535b57..00000000 --- a/integration_tests/fuzz/fuzz-header-multiple.yaml +++ /dev/null @@ -1,41 +0,0 @@ -id: fuzz-header-multiple - -info: - name: fuzz header multiple - author: pdteam - severity: info - description: | - In this template we fuzz multiple headers with single payload - -http: - - raw: - - | - GET /?x=aaa&y=bbb HTTP/1.1 - Host: {{Hostname}} - Origin: https://example.com - X-Forwared-For: 1337 - Cookie: z=aaa; bb=aaa - User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) - Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 - Accept-Language: en-US,en;q=0.9 - Connection: close - - payloads: - reflection: - - "secret.local" - - fuzzing: - - part: header - type: replace - mode: multiple - keys: ["Origin", "X-Forwared-For"] - fuzz: - - "{{reflection}}" - - stop-at-first-match: true - matchers-condition: and - matchers: - - type: word - part: body - words: - - "admin" \ No newline at end of file diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 608b4868..309c1c95 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -475,10 +475,9 @@ func (r *Runner) RunEnumeration() error { // If using input-file flags, only load http fuzzing based templates. loaderConfig := loader.NewConfig(r.options, r.catalog, executorOpts) - if !strings.EqualFold(r.options.InputFileMode, "list") || r.options.FuzzTemplates { + if !strings.EqualFold(r.options.InputFileMode, "list") || r.options.DAST { // if input type is not list (implicitly enable fuzzing) - r.options.FuzzTemplates = true - loaderConfig.OnlyLoadHTTPFuzzing = true + r.options.DAST = true } store, err := loader.New(loaderConfig) if err != nil { @@ -640,19 +639,27 @@ func (r *Runner) displayExecutionInfo(store *loader.Store) { stats.Display(templates.SyntaxWarningStats) stats.Display(templates.SyntaxErrorStats) stats.Display(templates.RuntimeWarningsStats) - if r.options.Verbose { + tmplCount := len(store.Templates()) + workflowCount := len(store.Workflows()) + if r.options.Verbose || (tmplCount == 0 && workflowCount == 0) { // only print these stats in verbose mode - stats.DisplayAsWarning(templates.HeadlessFlagWarningStats) - stats.DisplayAsWarning(templates.CodeFlagWarningStats) - stats.DisplayAsWarning(templates.TemplatesExecutedStats) - stats.DisplayAsWarning(templates.HeadlessFlagWarningStats) - stats.DisplayAsWarning(templates.CodeFlagWarningStats) - stats.DisplayAsWarning(templates.FuzzFlagWarningStats) - stats.DisplayAsWarning(templates.TemplatesExecutedStats) + stats.ForceDisplayWarning(templates.ExcludedHeadlessTmplStats) + stats.ForceDisplayWarning(templates.ExcludedCodeTmplStats) + stats.ForceDisplayWarning(templates.ExludedDastTmplStats) + stats.ForceDisplayWarning(templates.TemplatesExcludedStats) } - stats.DisplayAsWarning(templates.UnsignedCodeWarning) + if tmplCount == 0 && workflowCount == 0 { + // if dast flag is used print explicit warning + if r.options.DAST { + gologger.DefaultLogger.Print().Msgf("[%v] No DAST templates found", aurora.BrightYellow("WRN")) + } + stats.ForceDisplayWarning(templates.SkippedCodeTmplTamperedStats) + } else { + stats.DisplayAsWarning(templates.SkippedCodeTmplTamperedStats) + } stats.ForceDisplayWarning(templates.SkippedUnsignedStats) + stats.ForceDisplayWarning(templates.SkippedRequestSignatureStats) cfg := config.DefaultConfig @@ -666,25 +673,27 @@ func (r *Runner) displayExecutionInfo(store *loader.Store) { } } - if len(store.Templates()) > 0 { - gologger.Info().Msgf("New templates added in latest release: %d", len(config.DefaultConfig.GetNewAdditions())) - gologger.Info().Msgf("Templates loaded for current scan: %d", len(store.Templates())) - } - if len(store.Workflows()) > 0 { - gologger.Info().Msgf("Workflows loaded for current scan: %d", len(store.Workflows())) - } - for k, v := range templates.SignatureStats { - value := v.Load() - if k == templates.Unsigned && value > 0 { - // adjust skipped unsigned templates via code or -dut flag - value = value - uint64(stats.GetValue(templates.SkippedUnsignedStats)) - value = value - uint64(stats.GetValue(templates.CodeFlagWarningStats)) + if tmplCount > 0 || workflowCount > 0 { + if len(store.Templates()) > 0 { + gologger.Info().Msgf("New templates added in latest release: %d", len(config.DefaultConfig.GetNewAdditions())) + gologger.Info().Msgf("Templates loaded for current scan: %d", len(store.Templates())) } - if value > 0 { - if k != templates.Unsigned { - gologger.Info().Msgf("Executing %d signed templates from %s", value, k) - } else if !r.options.Silent && !config.DefaultConfig.HideTemplateSigWarning { - gologger.Print().Msgf("[%v] Loaded %d unsigned templates for scan. Use with caution.", aurora.BrightYellow("WRN"), value) + if len(store.Workflows()) > 0 { + gologger.Info().Msgf("Workflows loaded for current scan: %d", len(store.Workflows())) + } + for k, v := range templates.SignatureStats { + value := v.Load() + if k == templates.Unsigned && value > 0 { + // adjust skipped unsigned templates via code or -dut flag + value = value - uint64(stats.GetValue(templates.SkippedUnsignedStats)) + value = value - uint64(stats.GetValue(templates.ExcludedCodeTmplStats)) + } + if value > 0 { + if k == templates.Unsigned && !r.options.Silent && !config.DefaultConfig.HideTemplateSigWarning { + gologger.Print().Msgf("[%v] Loading %d unsigned templates for scan. Use with caution.", aurora.BrightYellow("WRN"), value) + } else { + gologger.Info().Msgf("Executing %d signed templates from %s", value, k) + } } } } diff --git a/lib/config.go b/lib/config.go index 58fab00a..52a8e232 100644 --- a/lib/config.go +++ b/lib/config.go @@ -376,10 +376,18 @@ func LoadSecretsFromFile(files []string, prefetch bool) NucleiSDKOptions { } } -// EnableFuzzTemplates allows enabling template fuzzing -func EnableFuzzTemplates() NucleiSDKOptions { +// DASTMode only run DAST templates +func DASTMode() NucleiSDKOptions { return func(e *NucleiEngine) error { - e.opts.FuzzTemplates = true + e.opts.DAST = true + return nil + } +} + +// SignedTemplatesOnly only run signed templates and disabled loading all unsigned templates +func SignedTemplatesOnly() NucleiSDKOptions { + return func(e *NucleiEngine) error { + e.opts.DisableUnsignedTemplates = true return nil } } diff --git a/pkg/catalog/loader/loader.go b/pkg/catalog/loader/loader.go index 1884fa33..0dce8423 100644 --- a/pkg/catalog/loader/loader.go +++ b/pkg/catalog/loader/loader.go @@ -61,8 +61,6 @@ type Config struct { Catalog catalog.Catalog ExecutorOptions protocols.ExecutorOptions - - OnlyLoadHTTPFuzzing bool } // Store is a storage for loaded nuclei templates @@ -405,33 +403,42 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ stats.Increment(templates.SkippedUnsignedStats) continue } - if len(parsed.RequestsHeadless) > 0 && !store.config.ExecutorOptions.Options.Headless { + // if template has request signature like aws then only signed and verified templates are allowed + if parsed.UsesRequestSignature() && !parsed.Verified { + stats.Increment(templates.SkippedRequestSignatureStats) + continue + } + // DAST only templates + if store.config.ExecutorOptions.Options.DAST { + // check if the template is a DAST template + if parsed.IsFuzzing() { + loadedTemplates = append(loadedTemplates, parsed) + } + } else if len(parsed.RequestsHeadless) > 0 && !store.config.ExecutorOptions.Options.Headless { // donot include headless template in final list if headless flag is not set - stats.Increment(templates.HeadlessFlagWarningStats) + stats.Increment(templates.ExcludedHeadlessTmplStats) if config.DefaultConfig.LogAllEvents { gologger.Print().Msgf("[%v] Headless flag is required for headless template '%s'.\n", aurora.Yellow("WRN").String(), templatePath) } } else if len(parsed.RequestsCode) > 0 && !store.config.ExecutorOptions.Options.EnableCodeTemplates { // donot include 'Code' protocol custom template in final list if code flag is not set - stats.Increment(templates.CodeFlagWarningStats) + stats.Increment(templates.ExcludedCodeTmplStats) if config.DefaultConfig.LogAllEvents { gologger.Print().Msgf("[%v] Code flag is required for code protocol template '%s'.\n", aurora.Yellow("WRN").String(), templatePath) } } else if len(parsed.RequestsCode) > 0 && !parsed.Verified && len(parsed.Workflows) == 0 { // donot include unverified 'Code' protocol custom template in final list - stats.Increment(templates.UnsignedCodeWarning) + stats.Increment(templates.SkippedCodeTmplTamperedStats) // these will be skipped so increment skip counter stats.Increment(templates.SkippedUnsignedStats) if config.DefaultConfig.LogAllEvents { gologger.Print().Msgf("[%v] Tampered/Unsigned template at %v.\n", aurora.Yellow("WRN").String(), templatePath) } - } else if parsed.IsFuzzing() && !store.config.ExecutorOptions.Options.FuzzTemplates { - stats.Increment(templates.FuzzFlagWarningStats) + } else if parsed.IsFuzzing() && !store.config.ExecutorOptions.Options.DAST { + stats.Increment(templates.ExludedDastTmplStats) if config.DefaultConfig.LogAllEvents { - gologger.Print().Msgf("[%v] Fuzz flag is required for fuzzing template '%s'.\n", aurora.Yellow("WRN").String(), templatePath) + gologger.Print().Msgf("[%v] -dast flag is required for DAST template '%s'.\n", aurora.Yellow("WRN").String(), templatePath) } - } else if store.config.OnlyLoadHTTPFuzzing && !parsed.IsFuzzing() { - gologger.Warning().Msgf("Non-Fuzzing template '%s' can only be run on list input mode targets\n", templatePath) } else { loadedTemplates = append(loadedTemplates, parsed) } @@ -439,7 +446,7 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ } if err != nil { if strings.Contains(err.Error(), templates.ErrExcluded.Error()) { - stats.Increment(templates.TemplatesExecutedStats) + stats.Increment(templates.TemplatesExcludedStats) if cfg.DefaultConfig.LogAllEvents { gologger.Print().Msgf("[%v] %v\n", aurora.Yellow("WRN").String(), err.Error()) } diff --git a/pkg/fuzz/component/body.go b/pkg/fuzz/component/body.go index 32d59a01..9d5bbe66 100644 --- a/pkg/fuzz/component/body.go +++ b/pkg/fuzz/component/body.go @@ -141,3 +141,10 @@ func (b *Body) Rebuild() (*retryablehttp.Request, error) { cloned.Header.Set("Content-Length", strconv.Itoa(len(encoded))) return cloned, nil } + +func (b *Body) Clone() Component { + return &Body{ + value: b.value.Clone(), + req: b.req.Clone(context.Background()), + } +} diff --git a/pkg/fuzz/component/body_test.go b/pkg/fuzz/component/body_test.go index 42e7e405..1cfcac83 100644 --- a/pkg/fuzz/component/body_test.go +++ b/pkg/fuzz/component/body_test.go @@ -4,11 +4,11 @@ import ( "bytes" "io" "mime/multipart" - "net/url" "strings" "testing" "github.com/projectdiscovery/retryablehttp-go" + urlutil "github.com/projectdiscovery/utils/url" "github.com/stretchr/testify/require" ) @@ -80,7 +80,7 @@ func TestBodyXMLComponent(t *testing.T) { } func TestBodyFormComponent(t *testing.T) { - formData := url.Values{} + formData := urlutil.NewOrderedParams() formData.Set("key1", "value1") formData.Set("key2", "value2") diff --git a/pkg/fuzz/component/component.go b/pkg/fuzz/component/component.go index 5a8279c4..a15ac285 100644 --- a/pkg/fuzz/component/component.go +++ b/pkg/fuzz/component/component.go @@ -46,6 +46,8 @@ type Component interface { // Rebuild returns a new request with the // component rebuilt Rebuild() (*retryablehttp.Request, error) + // Clones current state of this component + Clone() Component } const ( diff --git a/pkg/fuzz/component/cookie.go b/pkg/fuzz/component/cookie.go index b50087e3..77667c74 100644 --- a/pkg/fuzz/component/cookie.go +++ b/pkg/fuzz/component/cookie.go @@ -99,6 +99,14 @@ func (c *Cookie) Rebuild() (*retryablehttp.Request, error) { return cloned, nil } +// Clone clones current state of this component +func (c *Cookie) Clone() Component { + return &Cookie{ + value: c.value.Clone(), + req: c.req.Clone(context.Background()), + } +} + // A list of cookies that are essential to the request and // must not be fuzzed. var defaultIgnoredCookieKeys = map[string]struct{}{ diff --git a/pkg/fuzz/component/headers.go b/pkg/fuzz/component/headers.go index 9f921bee..fa0f5e11 100644 --- a/pkg/fuzz/component/headers.go +++ b/pkg/fuzz/component/headers.go @@ -102,6 +102,14 @@ func (q *Header) Rebuild() (*retryablehttp.Request, error) { return cloned, nil } +// Clones current state of this component +func (q *Header) Clone() Component { + return &Header{ + value: q.value.Clone(), + req: q.req.Clone(context.Background()), + } +} + // A list of headers that are essential to the request and // must not be fuzzed. var defaultIgnoredHeaderKeys = map[string]struct{}{ diff --git a/pkg/fuzz/component/path.go b/pkg/fuzz/component/path.go index 9521a089..9d2e7580 100644 --- a/pkg/fuzz/component/path.go +++ b/pkg/fuzz/component/path.go @@ -83,3 +83,11 @@ func (q *Path) Rebuild() (*retryablehttp.Request, error) { } return cloned, nil } + +// Clones current state to a new component +func (q *Path) Clone() Component { + return &Path{ + value: q.value.Clone(), + req: q.req.Clone(context.Background()), + } +} diff --git a/pkg/fuzz/component/query.go b/pkg/fuzz/component/query.go index 3fb2f350..3dc07e6c 100644 --- a/pkg/fuzz/component/query.go +++ b/pkg/fuzz/component/query.go @@ -92,3 +92,11 @@ func (q *Query) Rebuild() (*retryablehttp.Request, error) { cloned.Update() return cloned, nil } + +// Clones current state to a new component +func (q *Query) Clone() Component { + return &Query{ + value: q.value.Clone(), + req: q.req.Clone(context.Background()), + } +} diff --git a/pkg/fuzz/component/value.go b/pkg/fuzz/component/value.go index 190dcc9d..ad2044cf 100644 --- a/pkg/fuzz/component/value.go +++ b/pkg/fuzz/component/value.go @@ -36,6 +36,15 @@ func NewValue(data string) *Value { return v } +// Clones current state of this value +func (v *Value) Clone() *Value { + return &Value{ + data: v.data, + parsed: v.parsed.Clone(), + dataFormat: v.dataFormat, + } +} + // String returns the string representation of the value func (v *Value) String() string { return v.data diff --git a/pkg/fuzz/dataformat/kv.go b/pkg/fuzz/dataformat/kv.go index 83d227c3..72bd0da6 100644 --- a/pkg/fuzz/dataformat/kv.go +++ b/pkg/fuzz/dataformat/kv.go @@ -1,6 +1,9 @@ package dataformat -import mapsutil "github.com/projectdiscovery/utils/maps" +import ( + mapsutil "github.com/projectdiscovery/utils/maps" + "golang.org/x/exp/maps" +) // KV is a key-value struct // that is implemented or used by fuzzing package @@ -14,6 +17,18 @@ type KV struct { OrderedMap *mapsutil.OrderedMap[string, any] } +// Clones the current state of the KV struct +func (kv *KV) Clone() KV { + newKV := KV{} + if kv.OrderedMap == nil { + newKV.Map = maps.Clone(kv.Map) + return newKV + } + clonedOrderedMap := kv.OrderedMap.Clone() + newKV.OrderedMap = &clonedOrderedMap + return newKV +} + // IsNIL returns true if the KV struct is nil func (kv *KV) IsNIL() bool { return kv.Map == nil && kv.OrderedMap == nil diff --git a/pkg/fuzz/parts.go b/pkg/fuzz/parts.go index 9b24446b..2796a7dc 100644 --- a/pkg/fuzz/parts.go +++ b/pkg/fuzz/parts.go @@ -34,12 +34,13 @@ func (rule *Rule) checkRuleApplicableOnComponent(component component.Component) // executePartComponent executes this rule on a given component and payload func (rule *Rule) executePartComponent(input *ExecuteRuleInput, payload ValueOrKeyValue, ruleComponent component.Component) error { + // Note: component needs to be cloned because they contain values copied by reference if payload.IsKV() { // for kv fuzzing - return rule.executePartComponentOnKV(input, payload, ruleComponent) + return rule.executePartComponentOnKV(input, payload, ruleComponent.Clone()) } else { // for value only fuzzing - return rule.executePartComponentOnValues(input, payload.Value, ruleComponent) + return rule.executePartComponentOnValues(input, payload.Value, ruleComponent.Clone()) } } diff --git a/pkg/output/format_screen.go b/pkg/output/format_screen.go index d468981d..2d6310df 100644 --- a/pkg/output/format_screen.go +++ b/pkg/output/format_screen.go @@ -59,8 +59,12 @@ func (w *StandardWriter) formatScreen(output *ResultEvent) []byte { for i, item := range output.ExtractedResults { // trim trailing space + // quote non-ascii and non printable characters and then + // unquote quotes (`"`) for readability item = strings.TrimSpace(item) - item = strings.ReplaceAll(item, "\n", "\\n") // only replace newlines + item = strconv.QuoteToASCII(item) + item = strings.ReplaceAll(item, `\"`, `"`) + builder.WriteString(w.aurora.BrightCyan(item).String()) if i != len(output.ExtractedResults)-1 { diff --git a/pkg/protocols/common/variables/variables.go b/pkg/protocols/common/variables/variables.go index cc8278e0..131098f9 100644 --- a/pkg/protocols/common/variables/variables.go +++ b/pkg/protocols/common/variables/variables.go @@ -135,6 +135,12 @@ func (variables *Variable) checkForLazyEval() bool { return } } + // this is a hotfix and not the best way to do it + // will be refactored once we move scan state to scanContext (see: https://github.com/projectdiscovery/nuclei/issues/4631) + if strings.Contains(types.ToString(value), "interactsh-url") { + variables.LazyEval = true + return + } }) return variables.LazyEval } diff --git a/pkg/protocols/http/raw/raw.go b/pkg/protocols/http/raw/raw.go index a1eb544a..a1e4c574 100644 --- a/pkg/protocols/http/raw/raw.go +++ b/pkg/protocols/http/raw/raw.go @@ -283,7 +283,6 @@ func (r *Request) ApplyAuthStrategy(strategy authx.AuthStrategy) { gologger.Error().Msgf("auth strategy failed to parse url: %s got %v", r.FullURL, err) return } - _ = parsed for _, p := range s.Data.Params { parsed.Params.Add(p.Key, p.Value) } diff --git a/pkg/protocols/http/request_fuzz.go b/pkg/protocols/http/request_fuzz.go index b9f22131..6e2e4a74 100644 --- a/pkg/protocols/http/request_fuzz.go +++ b/pkg/protocols/http/request_fuzz.go @@ -8,6 +8,7 @@ package http import ( "context" "fmt" + "net/http" "strings" "github.com/pkg/errors" @@ -23,13 +24,14 @@ import ( protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils" "github.com/projectdiscovery/nuclei/v3/pkg/types" "github.com/projectdiscovery/retryablehttp-go" + urlutil "github.com/projectdiscovery/utils/url" ) // executeFuzzingRule executes fuzzing request for a URL // TODO: // 1. use SPMHandler and rewrite stop at first match logic here // 2. use scanContext instead of contextargs.Context -func (request *Request) executeFuzzingRule(input *contextargs.Context, _ output.InternalEvent, callback protocols.OutputEventCallback) error { +func (request *Request) executeFuzzingRule(input *contextargs.Context, previous output.InternalEvent, callback protocols.OutputEventCallback) error { // methdology: // to check applicablity of rule, we first try to execute it with one value // if it is applicable, we execute all requests @@ -46,29 +48,20 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, _ output. return nil } - // Iterate through all requests for template and queue them for fuzzing - generator := request.newGenerator(true) - - // this will generate next value along with request it is meant to be used with - currRequest, payloads, result := generator.nextValue() - if !result && input.MetaInput.ReqResp == nil { - // this case is only true if input is not a full http request - return fmt.Errorf("no values to generate requests") + if input.MetaInput.Input == "" && input.MetaInput.ReqResp == nil { + return errors.New("empty input provided for fuzzing") } - // if it is a full http request obtained from target file + // ==== fuzzing when full HTTP request is provided ===== + if input.MetaInput.ReqResp != nil { - // Note: in case of full http request, we only need to build it once - // and then reuse it for all requests and completely abandon the request - // returned by generator - _ = currRequest - generated, err := input.MetaInput.ReqResp.BuildRequest() + baseRequest, err := input.MetaInput.ReqResp.BuildRequest() if err != nil { return errors.Wrap(err, "fuzz: could not build request obtained from target file") } - input.MetaInput.Input = generated.URL.String() + input.MetaInput.Input = baseRequest.URL.String() // execute with one value first to checks its applicability - err = request.executePayloadUsingRules(input, payloads, generated, callback) + err = request.executeAllFuzzingRules(input, previous, baseRequest, callback) if err != nil { // in case of any error, return it if fuzz.IsErrRuleNotApplicable(err) { @@ -79,36 +72,25 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, _ output. if errors.Is(err, errStopExecution) { return err } - gologger.Verbose().Msgf("[%s] fuzz: inital payload request execution failed : %s\n", request.options.TemplateID, err) - } - - // if it is applicable, execute all requests - for { - _, payloads, result := generator.nextValue() - if !result { - break - } - err = request.executePayloadUsingRules(input, payloads, generated, callback) - if err != nil { - // continue to next request since this is payload specific - gologger.Verbose().Msgf("[%s] fuzz: payload request execution failed : %s\n", request.options.TemplateID, err) - continue - } + gologger.Verbose().Msgf("[%s] fuzz: payload request execution failed : %s\n", request.options.TemplateID, err) } return nil } // ==== fuzzing when only URL is provided ===== - generated, err := generator.Make(context.Background(), input, currRequest, payloads, nil) + // we need to use this url instead of input + inputx := input.Clone() + parsed, err := urlutil.ParseAbsoluteURL(input.MetaInput.Input, true) + if err != nil { + return errors.Wrap(err, "fuzz: could not parse input url") + } + baseRequest, err := retryablehttp.NewRequestFromURL(http.MethodGet, parsed, nil) if err != nil { return errors.Wrap(err, "fuzz: could not build request from url") } - // we need to use this url instead of input - inputx := input.Clone() - inputx.MetaInput.Input = generated.request.URL.String() // execute with one value first to checks its applicability - err = request.executePayloadUsingRules(inputx, generated.dynamicValues, generated.request, callback) + err = request.executeAllFuzzingRules(inputx, previous, baseRequest, callback) if err != nil { // in case of any error, return it if fuzz.IsErrRuleNotApplicable(err) { @@ -119,34 +101,13 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, _ output. if errors.Is(err, errStopExecution) { return err } - gologger.Verbose().Msgf("[%s] fuzz: inital payload request execution failed : %s\n", request.options.TemplateID, err) - } - - // continue to next request since this is payload specific - for { - currRequest, payloads, result = generator.nextValue() - if !result { - break - } - generated, err := generator.Make(context.Background(), input, currRequest, payloads, nil) - if err != nil { - return errors.Wrap(err, "fuzz: could not build request from url") - } - // we need to use this url instead of input - inputx := input.Clone() - inputx.MetaInput.Input = generated.request.URL.String() - // execute with one value first to checks its applicability - err = request.executePayloadUsingRules(inputx, generated.dynamicValues, generated.request, callback) - if err != nil { - gologger.Verbose().Msgf("[%s] fuzz: payload request execution failed : %s\n", request.options.TemplateID, err) - continue - } + gologger.Verbose().Msgf("[%s] fuzz: payload request execution failed : %s\n", request.options.TemplateID, err) } return nil } -// executePayloadUsingRules executes a payload using rules with given payload i.e values -func (request *Request) executePayloadUsingRules(input *contextargs.Context, values map[string]interface{}, baseRequest *retryablehttp.Request, callback protocols.OutputEventCallback) error { +// executeAllFuzzingRules executes all fuzzing rules defined in template for a given base request +func (request *Request) executeAllFuzzingRules(input *contextargs.Context, values map[string]interface{}, baseRequest *retryablehttp.Request, callback protocols.OutputEventCallback) error { applicable := false for _, rule := range request.Fuzzing { err := rule.Execute(&fuzz.ExecuteRuleInput{ @@ -156,7 +117,7 @@ func (request *Request) executePayloadUsingRules(input *contextargs.Context, val return request.executeGeneratedFuzzingRequest(gr, input, callback) }, Values: values, - BaseRequest: baseRequest, + BaseRequest: baseRequest.Clone(context.TODO()), }) if err == nil { applicable = true @@ -295,6 +256,9 @@ func (request *Request) filterDataMap(input *contextargs.Context) map[string]int return true }) m["header"] = sb.String() + } else { + // add default method value + m["method"] = http.MethodGet } // dump if svd is enabled diff --git a/pkg/templates/parser_stats.go b/pkg/templates/parser_stats.go index 473439b6..29060103 100644 --- a/pkg/templates/parser_stats.go +++ b/pkg/templates/parser_stats.go @@ -1,13 +1,14 @@ package templates const ( - SyntaxWarningStats = "syntax-warnings" - SyntaxErrorStats = "syntax-errors" - RuntimeWarningsStats = "runtime-warnings" - UnsignedCodeWarning = "unsigned-warnings" - HeadlessFlagWarningStats = "headless-flag-missing-warnings" - TemplatesExecutedStats = "templates-executed" - CodeFlagWarningStats = "code-flag-missing-warnings" - FuzzFlagWarningStats = "fuzz-flag-missing-warnings" - SkippedUnsignedStats = "skipped-unsigned-stats" // tracks loading of unsigned templates + SyntaxWarningStats = "syntax-warnings" + SyntaxErrorStats = "syntax-errors" + RuntimeWarningsStats = "runtime-warnings" + SkippedCodeTmplTamperedStats = "unsigned-warnings" + ExcludedHeadlessTmplStats = "headless-flag-missing-warnings" + TemplatesExcludedStats = "templates-executed" + ExcludedCodeTmplStats = "code-flag-missing-warnings" + ExludedDastTmplStats = "fuzz-flag-missing-warnings" + SkippedUnsignedStats = "skipped-unsigned-stats" // tracks loading of unsigned templates + SkippedRequestSignatureStats = "skipped-request-signature-stats" ) diff --git a/pkg/templates/stats.go b/pkg/templates/stats.go index 25a61e6e..fe3d4ca3 100644 --- a/pkg/templates/stats.go +++ b/pkg/templates/stats.go @@ -6,10 +6,11 @@ func init() { stats.NewEntry(SyntaxWarningStats, "Found %d templates with syntax warning (use -validate flag for further examination)") stats.NewEntry(SyntaxErrorStats, "Found %d templates with syntax error (use -validate flag for further examination)") stats.NewEntry(RuntimeWarningsStats, "Found %d templates with runtime error (use -validate flag for further examination)") - stats.NewEntry(UnsignedCodeWarning, "Found %d unsigned or tampered code template (carefully examine before using it & use -sign flag to sign them)") - stats.NewEntry(HeadlessFlagWarningStats, "Excluded %d headless template[s] (disabled as default), use -headless option to run headless templates.") - stats.NewEntry(CodeFlagWarningStats, "Excluded %d code template[s] (disabled as default), use -code option to run code templates.") - stats.NewEntry(TemplatesExecutedStats, "Excluded %d template[s] with known weak matchers / tags excluded from default run using .nuclei-ignore") - stats.NewEntry(FuzzFlagWarningStats, "Excluded %d fuzz template[s] (disabled as default), use -fuzz option to run fuzz templates.") + stats.NewEntry(SkippedCodeTmplTamperedStats, "Found %d unsigned or tampered code template (carefully examine before using it & use -sign flag to sign them)") + stats.NewEntry(ExcludedHeadlessTmplStats, "Excluded %d headless template[s] (disabled as default), use -headless option to run headless templates.") + stats.NewEntry(ExcludedCodeTmplStats, "Excluded %d code template[s] (disabled as default), use -code option to run code templates.") + stats.NewEntry(TemplatesExcludedStats, "Excluded %d template[s] with known weak matchers / tags excluded from default run using .nuclei-ignore") + stats.NewEntry(ExludedDastTmplStats, "Excluded %d dast template[s] (disabled as default), use -dast option to run dast templates.") stats.NewEntry(SkippedUnsignedStats, "Skipping %d unsigned template[s]") + stats.NewEntry(SkippedRequestSignatureStats, "Skipping %d templates, HTTP Request signatures can only be used in Signed & Verified templates.") } diff --git a/pkg/templates/templates.go b/pkg/templates/templates.go index 68f9324a..01dbb06a 100644 --- a/pkg/templates/templates.go +++ b/pkg/templates/templates.go @@ -132,6 +132,7 @@ type Template struct { // description: | // Signature is the request signature method + // WARNING: 'signature' will be deprecated and will be removed in a future release. Prefer using 'code' protocol for writing cloud checks // values: // - "AWS" Signature http.SignatureTypeHolder `yaml:"signature,omitempty" json:"signature,omitempty" jsonschema:"title=signature is the http request signature method,description=Signature is the HTTP Request signature Method,enum=AWS,deprecated=true"` @@ -214,6 +215,11 @@ func (template *Template) IsFuzzing() bool { return false } +// UsesRequestSignature returns true if the template uses a request signature like AWS +func (template *Template) UsesRequestSignature() bool { + return template.Signature.Value.String() != "" +} + // HasCodeProtocol returns true if the template has a code protocol section func (template *Template) HasCodeProtocol() bool { return len(template.RequestsCode) > 0 diff --git a/pkg/templates/workflows.go b/pkg/templates/workflows.go index c402ce76..6f6d5d3d 100644 --- a/pkg/templates/workflows.go +++ b/pkg/templates/workflows.go @@ -86,6 +86,10 @@ func parseWorkflowTemplate(workflow *workflows.WorkflowTemplate, preprocessor Pr stats.Increment(SkippedUnsignedStats) continue } + if template.UsesRequestSignature() && !template.Verified { + stats.Increment(SkippedRequestSignatureStats) + continue + } if len(template.RequestsCode) > 0 { if !options.Options.EnableCodeTemplates { diff --git a/pkg/types/types.go b/pkg/types/types.go index 45a8788a..ce783809 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -372,9 +372,6 @@ type Options struct { ScanID string // JsConcurrency is the number of concurrent js routines to run JsConcurrency int - // Fuzz enabled execution of fuzzing templates - // Note: when Fuzz is enabled other templates will not be executed - FuzzTemplates bool // SecretsFile is file containing secrets for nuclei SecretsFile goflags.StringSlice // PreFetchSecrets pre-fetches the secrets from the auth provider @@ -385,6 +382,8 @@ type Options struct { SkipFormatValidation bool // PayloadConcurrency is the number of concurrent payloads to run per template PayloadConcurrency int + // Dast only runs DAST templates + DAST bool } // ShouldLoadResume resume file