From 255032f4f2f37e53118b2f57a3e0f0e82191378a Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com> Date: Mon, 1 Apr 2024 19:18:21 +0530 Subject: [PATCH] pre-condition in code , fuzz and other misc updates (#4966) * fuzz: rename 'filters' -> 'pre-condition' * code proto: pre-condition + integration test * feat: dsl document generator * update dsl page header * fix lint error * add js defined helper funcs in docs * remove panic recovery unless its for third party(go-rod,goja) * handle dynamic values flattening edgecase in flow+multiprotocol * fix order of kv in form-data (failing test) * fix template loading counters * Revert "handle dynamic values flattening edgecase in flow+multiprotocol" This reverts commit 58fdd4faf7df5d654b46a9585011f614d5c98aa4. * fix flow iteration using 'iterate' --- .gitignore | 2 + Makefile | 5 +- cmd/integration-test/code.go | 17 ++ go.mod | 2 +- .../flow/iterate-one-value-flow.yaml | 5 +- .../fuzz/fuzz-body-generic-sqli.yaml | 2 +- .../fuzz/fuzz-body-json-sqli.yaml | 2 +- .../fuzz/fuzz-body-multipart-form-sqli.yaml | 2 +- .../fuzz/fuzz-body-params-sqli.yaml | 2 +- .../fuzz/fuzz-body-xml-sqli.yaml | 2 +- .../fuzz/fuzz-cookie-error-sqli.yaml | 2 +- .../fuzz/fuzz-host-header-injection.yaml | 2 +- integration_tests/fuzz/fuzz-path-sqli.yaml | 2 +- .../fuzz/fuzz-query-num-replace.yaml | 4 +- .../protocols/code/pre-condition.yaml | 26 ++ internal/runner/runner.go | 7 +- pkg/catalog/loader/loader.go | 22 +- pkg/fuzz/dataformat/form.go | 12 +- pkg/fuzz/execute.go | 5 - pkg/input/types/http.go | 5 - pkg/js/devtools/scrapefuncs/main.go | 201 +++++++------ pkg/js/global/helpers.go | 37 +++ pkg/js/gojs/set.go | 4 + pkg/protocols/code/code.go | 86 +++++- pkg/protocols/code/helpers.go | 272 ++++++++++++++++++ pkg/protocols/http/http.go | 19 +- pkg/protocols/http/request.go | 15 - pkg/protocols/http/request_fuzz.go | 6 +- pkg/templates/compile.go | 7 +- pkg/templates/templates.go | 3 +- pkg/templates/workflows.go | 11 + pkg/tmplexec/exec.go | 9 - pkg/tmplexec/flow/flow_executor.go | 8 - pkg/tmplexec/flow/flow_internal.go | 8 +- pkg/tmplexec/flow/vm.go | 93 +++--- 35 files changed, 676 insertions(+), 231 deletions(-) create mode 100644 integration_tests/protocols/code/pre-condition.yaml create mode 100644 pkg/protocols/code/helpers.go diff --git a/.gitignore b/.gitignore index 57d2daa0..65815d8d 100644 --- a/.gitignore +++ b/.gitignore @@ -35,4 +35,6 @@ pkg/protocols/headless/engine/.cache **/*-cache /fuzzplayground integration_tests/fuzzplayground +/dsl.md + diff --git a/Makefile b/Makefile index f6950b31..dadbe620 100644 --- a/Makefile +++ b/Makefile @@ -47,4 +47,7 @@ fuzzplayground: memogen: $(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "memogen" cmd/memogen/memogen.go ./memogen -src pkg/js/libs -tpl cmd/memogen/function.tpl - +dsl-docs: + rm -f dsl.md scrapefuncs 2>/dev/null + $(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "scrapefuncs" pkg/js/devtools/scrapefuncs/main.go + ./scrapefuncs -out dsl.md diff --git a/cmd/integration-test/code.go b/cmd/integration-test/code.go index 320eadf2..94c28ea0 100644 --- a/cmd/integration-test/code.go +++ b/cmd/integration-test/code.go @@ -23,6 +23,7 @@ var codeTestCases = []TestCaseInfo{ {Path: "protocols/code/py-nosig.yaml", TestCase: &codePyNoSig{}, DisableOn: isCodeDisabled}, {Path: "protocols/code/py-interactsh.yaml", TestCase: &codeSnippet{}, DisableOn: isCodeDisabled}, {Path: "protocols/code/ps1-snippet.yaml", TestCase: &codeSnippet{}, DisableOn: func() bool { return !osutils.IsWindows() || isCodeDisabled() }}, + {Path: "protocols/code/pre-condition.yaml", TestCase: &codePreCondition{}, DisableOn: isCodeDisabled}, } const ( @@ -94,6 +95,22 @@ func (h *codeSnippet) Execute(filePath string) error { return expectResultsCount(results, 1) } +type codePreCondition struct{} + +// Execute executes a test case and returns an error if occurred +func (h *codePreCondition) Execute(filePath string) error { + results, err := testutils.RunNucleiArgsWithEnvAndGetResults(debug, getEnvValues(), "-t", filePath, "-u", "input", "-code") + if err != nil { + return err + } + if osutils.IsLinux() { + return expectResultsCount(results, 1) + } else { + return expectResultsCount(results, 0) + + } +} + type codeFile struct{} // Execute executes a test case and returns an error if occurred diff --git a/go.mod b/go.mod index c82a4826..8718c045 100644 --- a/go.mod +++ b/go.mod @@ -97,7 +97,6 @@ require ( github.com/projectdiscovery/utils v0.0.85 github.com/projectdiscovery/wappalyzergo v0.0.112 github.com/redis/go-redis/v9 v9.1.0 - github.com/sashabaranov/go-openai v1.15.3 github.com/seh-msft/burpxml v1.0.1 github.com/stretchr/testify v1.9.0 github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706 @@ -205,6 +204,7 @@ require ( github.com/projectdiscovery/stringsutil v0.0.2 // indirect github.com/quic-go/quic-go v0.40.1 // indirect github.com/refraction-networking/utls v1.6.1 // indirect + github.com/sashabaranov/go-openai v1.15.3 // indirect github.com/shirou/gopsutil v3.21.11+incompatible // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/sirupsen/logrus v1.9.3 // indirect diff --git a/integration_tests/flow/iterate-one-value-flow.yaml b/integration_tests/flow/iterate-one-value-flow.yaml index d45eaa58..9ab6633c 100644 --- a/integration_tests/flow/iterate-one-value-flow.yaml +++ b/integration_tests/flow/iterate-one-value-flow.yaml @@ -4,10 +4,13 @@ info: name: Test Flow Iterate One Value Flow author: pdteam severity: info + description: | + If length of template.extracted variable is not know, i.e it could be an array of 1 or more values, then iterate function + should be used to iterate over values because nuclei by default converts array to string if it has only 1 value. flow: | http(1) - for(let value of template.extracted){ + for(let value of iterate(template.extracted)){ set("value", value) http(2) } diff --git a/integration_tests/fuzz/fuzz-body-generic-sqli.yaml b/integration_tests/fuzz/fuzz-body-generic-sqli.yaml index 83ca4962..6ae25a7a 100644 --- a/integration_tests/fuzz/fuzz-body-generic-sqli.yaml +++ b/integration_tests/fuzz/fuzz-body-generic-sqli.yaml @@ -10,7 +10,7 @@ info: and performs fuzzing on the value of every key http: - - filters: + - pre-condition: - type: dsl dsl: - method != "GET" diff --git a/integration_tests/fuzz/fuzz-body-json-sqli.yaml b/integration_tests/fuzz/fuzz-body-json-sqli.yaml index 187ce1b4..5dfeca45 100644 --- a/integration_tests/fuzz/fuzz-body-json-sqli.yaml +++ b/integration_tests/fuzz/fuzz-body-json-sqli.yaml @@ -10,7 +10,7 @@ info: Note: this is example template, and payloads/matchers need to be modified appropriately. http: - - filters: + - pre-condition: - type: dsl dsl: - method != "GET" diff --git a/integration_tests/fuzz/fuzz-body-multipart-form-sqli.yaml b/integration_tests/fuzz/fuzz-body-multipart-form-sqli.yaml index dcca2e18..c7e8fc9d 100644 --- a/integration_tests/fuzz/fuzz-body-multipart-form-sqli.yaml +++ b/integration_tests/fuzz/fuzz-body-multipart-form-sqli.yaml @@ -10,7 +10,7 @@ info: Note: this is example template, and payloads/matchers need to be modified appropriately. http: - - filters: + - pre-condition: - type: dsl dsl: - method != "GET" diff --git a/integration_tests/fuzz/fuzz-body-params-sqli.yaml b/integration_tests/fuzz/fuzz-body-params-sqli.yaml index 2fdd1742..b090feb2 100644 --- a/integration_tests/fuzz/fuzz-body-params-sqli.yaml +++ b/integration_tests/fuzz/fuzz-body-params-sqli.yaml @@ -10,7 +10,7 @@ info: Note: this is example template, and payloads/matchers need to be modified appropriately. http: - - filters: + - pre-condition: - type: dsl dsl: - method != "GET" diff --git a/integration_tests/fuzz/fuzz-body-xml-sqli.yaml b/integration_tests/fuzz/fuzz-body-xml-sqli.yaml index 8ac62842..ae77efd3 100644 --- a/integration_tests/fuzz/fuzz-body-xml-sqli.yaml +++ b/integration_tests/fuzz/fuzz-body-xml-sqli.yaml @@ -10,7 +10,7 @@ info: Note: this is example template, and payloads/matchers need to be modified appropriately. http: - - filters: + - pre-condition: - type: dsl dsl: - method != "GET" diff --git a/integration_tests/fuzz/fuzz-cookie-error-sqli.yaml b/integration_tests/fuzz/fuzz-cookie-error-sqli.yaml index 86bb2a16..3e457800 100644 --- a/integration_tests/fuzz/fuzz-cookie-error-sqli.yaml +++ b/integration_tests/fuzz/fuzz-cookie-error-sqli.yaml @@ -9,7 +9,7 @@ info: Note: this is example template, and payloads/matchers need to be modified appropriately. http: - - filters: + - pre-condition: - type: dsl dsl: - 'method == "GET"' diff --git a/integration_tests/fuzz/fuzz-host-header-injection.yaml b/integration_tests/fuzz/fuzz-host-header-injection.yaml index cda22235..551398de 100644 --- a/integration_tests/fuzz/fuzz-host-header-injection.yaml +++ b/integration_tests/fuzz/fuzz-host-header-injection.yaml @@ -10,7 +10,7 @@ variables: domain: "oast.fun" http: - - filters: + - pre-condition: - type: dsl dsl: - 'method == "GET"' diff --git a/integration_tests/fuzz/fuzz-path-sqli.yaml b/integration_tests/fuzz/fuzz-path-sqli.yaml index e034dec3..e23098d9 100644 --- a/integration_tests/fuzz/fuzz-path-sqli.yaml +++ b/integration_tests/fuzz/fuzz-path-sqli.yaml @@ -11,7 +11,7 @@ info: Note: this is example template, and payloads/matchers need to be modified appropriately. http: - - filters: + - pre-condition: - type: dsl dsl: - 'method == "GET"' diff --git a/integration_tests/fuzz/fuzz-query-num-replace.yaml b/integration_tests/fuzz/fuzz-query-num-replace.yaml index 90f3a393..dc628336 100644 --- a/integration_tests/fuzz/fuzz-query-num-replace.yaml +++ b/integration_tests/fuzz/fuzz-query-num-replace.yaml @@ -7,7 +7,7 @@ info: description: Query Value Fuzzing using Fuzzing Rules http: - - filters: + - pre-condition: - type: dsl dsl: - 'len(query) > 0' @@ -16,7 +16,7 @@ http: part: path words: - /blog/post - filters-condition: and + pre-condition-operator: and payloads: nums: diff --git a/integration_tests/protocols/code/pre-condition.yaml b/integration_tests/protocols/code/pre-condition.yaml new file mode 100644 index 00000000..a61b4f90 --- /dev/null +++ b/integration_tests/protocols/code/pre-condition.yaml @@ -0,0 +1,26 @@ +id: pre-condition-code + +info: + name: example code template + author: pdteam + severity: info + + +self-contained: true + +variables: + OAST: "{{interactsh-url}}" + +code: + - pre-condition: IsLinux() + engine: + - sh + - bash + source: | + echo "$OAST" | base64 + + matchers: + - type: dsl + dsl: + - true +# digest: 4a0a00473045022100c7215ce9f11e6a51c193bb54643a05cdd1cde18a3abb6c9983c5c7524d3ff03002203d93581c81d3ad5db463570cbbd2bdee529328d32a5b00e037610c211e448cef:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 5598e774..ba82073f 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -688,14 +688,9 @@ func (r *Runner) displayExecutionInfo(store *loader.Store) { } for k, v := range templates.SignatureStats { value := v.Load() - if k == templates.Unsigned && value > 0 { - // adjust skipped unsigned templates via code or -dut flag - value = value - uint64(stats.GetValue(templates.SkippedUnsignedStats)) - value = value - uint64(stats.GetValue(templates.ExcludedCodeTmplStats)) - } if value > 0 { if k == templates.Unsigned && !r.options.Silent && !config.DefaultConfig.HideTemplateSigWarning { - gologger.Print().Msgf("[%v] Loading %d unsigned templates for scan. Use with caution.", aurora.BrightYellow("WRN"), value) + gologger.Print().Msgf("[%v] Executing %d unsigned templates for scan. Use with caution.", aurora.BrightYellow("WRN"), value) } else { gologger.Info().Msgf("Executing %d signed templates from %s", value, k) } diff --git a/pkg/catalog/loader/loader.go b/pkg/catalog/loader/loader.go index 0dce8423..711008f2 100644 --- a/pkg/catalog/loader/loader.go +++ b/pkg/catalog/loader/loader.go @@ -13,7 +13,6 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v3/pkg/catalog" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" - cfg "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader/filter" "github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v3/pkg/protocols" @@ -387,6 +386,21 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ templatePathMap := store.pathFilter.Match(includedTemplates) loadedTemplates := make([]*templates.Template, 0, len(templatePathMap)) + + loadTemplate := func(tmpl *templates.Template) { + loadedTemplates = append(loadedTemplates, tmpl) + // increment signed/unsigned counters + if tmpl.Verified { + if tmpl.TemplateVerifier == "" { + templates.SignatureStats[templates.PDVerifier].Add(1) + } else { + templates.SignatureStats[tmpl.TemplateVerifier].Add(1) + } + } else { + templates.SignatureStats[templates.Unsigned].Add(1) + } + } + for templatePath := range templatePathMap { loaded, err := store.config.ExecutorOptions.Parser.LoadTemplate(templatePath, store.tagFilter, tags, store.config.Catalog) if loaded || store.pathFilter.MatchIncluded(templatePath) { @@ -412,7 +426,7 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ if store.config.ExecutorOptions.Options.DAST { // check if the template is a DAST template if parsed.IsFuzzing() { - loadedTemplates = append(loadedTemplates, parsed) + loadTemplate(parsed) } } else if len(parsed.RequestsHeadless) > 0 && !store.config.ExecutorOptions.Options.Headless { // donot include headless template in final list if headless flag is not set @@ -440,14 +454,14 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ gologger.Print().Msgf("[%v] -dast flag is required for DAST template '%s'.\n", aurora.Yellow("WRN").String(), templatePath) } } else { - loadedTemplates = append(loadedTemplates, parsed) + loadTemplate(parsed) } } } if err != nil { if strings.Contains(err.Error(), templates.ErrExcluded.Error()) { stats.Increment(templates.TemplatesExcludedStats) - if cfg.DefaultConfig.LogAllEvents { + if config.DefaultConfig.LogAllEvents { gologger.Print().Msgf("[%v] %v\n", aurora.Yellow("WRN").String(), err.Error()) } continue diff --git a/pkg/fuzz/dataformat/form.go b/pkg/fuzz/dataformat/form.go index 9a9c7b61..ab335299 100644 --- a/pkg/fuzz/dataformat/form.go +++ b/pkg/fuzz/dataformat/form.go @@ -2,7 +2,6 @@ package dataformat import ( "fmt" - "net/url" "regexp" "strconv" "strings" @@ -116,13 +115,11 @@ func (f *Form) Encode(data KV) (string, error) { // Decode decodes the data from Form format func (f *Form) Decode(data string) (KV, error) { - parsed, err := url.ParseQuery(data) - if err != nil { - return KV{}, err - } + ordered_params := urlutil.NewOrderedParams() + ordered_params.Merge(data) values := mapsutil.NewOrderedMap[string, any]() - for key, value := range parsed { + ordered_params.Iterate(func(key string, value []string) bool { if len(value) == 1 { values.Set(key, value[0]) } else { @@ -134,7 +131,8 @@ func (f *Form) Decode(data string) (KV, error) { } values.Set(key, value[len(value)-1]) } - } + return true + }) return KVOrderedMap(&values), nil } diff --git a/pkg/fuzz/execute.go b/pkg/fuzz/execute.go index 3a86ef7e..6e1a7a6a 100644 --- a/pkg/fuzz/execute.go +++ b/pkg/fuzz/execute.go @@ -64,11 +64,6 @@ type GeneratedRequest struct { // Input is not thread safe and should not be shared between concurrent // goroutines. func (rule *Rule) Execute(input *ExecuteRuleInput) (err error) { - defer func() { - if r := recover(); r != nil { - err = fmt.Errorf("got panic while executing rule: %v", r) - } - }() if !rule.isInputURLValid(input.Input) { return ErrRuleNotApplicable.Msgf("invalid input url: %v", input.Input.MetaInput.Input) } diff --git a/pkg/input/types/http.go b/pkg/input/types/http.go index 62b25249..a407d6a0 100644 --- a/pkg/input/types/http.go +++ b/pkg/input/types/http.go @@ -207,11 +207,6 @@ func (hr *HttpResponse) Clone() *HttpResponse { // and returns the request and response object // Note: it currently does not parse response and is meant to be added manually since its a optional field func ParseRawRequest(raw string) (rr *RequestResponse, err error) { - defer func() { - if r := recover(); r != nil { - err = fmt.Errorf("panic: %v", r) - } - }() protoReader := textproto.NewReader(bufio.NewReader(strings.NewReader(raw))) methodLine, err := protoReader.ReadLine() if err != nil { diff --git a/pkg/js/devtools/scrapefuncs/main.go b/pkg/js/devtools/scrapefuncs/main.go index 04c62f91..3ecbe6de 100644 --- a/pkg/js/devtools/scrapefuncs/main.go +++ b/pkg/js/devtools/scrapefuncs/main.go @@ -1,113 +1,92 @@ package main import ( - "bytes" - "context" "flag" "fmt" "go/ast" "go/parser" "go/token" - "log" "os" + "path/filepath" + "sort" "strings" - filutil "github.com/projectdiscovery/utils/file" - "github.com/sashabaranov/go-openai" + mapsutil "github.com/projectdiscovery/utils/maps" + "golang.org/x/exp/maps" ) -var sysprompt = ` -data present after ---raw data--- contains raw data extracted by a parser and contains information about function ---- example --- -Name: log -Signatures: "log(msg string)" -Signatures: "log(msg map[string]interface{})" -Description: log prints given input to stdout with [JS] prefix for debugging purposes ---- end example --- -Here Name is name of function , signature[s] is actual function declaration and description is description of function -using this data for every such function generate a abstract implementation of function in javascript along with jsdoc annotations ---- example expected output--- -/** - * log prints given input to stdout with [JS] prefix for debugging purposes - * log(msg string) - * log(msg map[string]interface{}) - * @function - * @param {string} msg - The message to print. - */ -function log(msg) { - // implemented in go -}; ---- instructions --- -ACT as helpful coding assistant and do the same for all functions present in data -` - -const userPrompt = ` ----raw data--- -{{source}} ----new javascript--- -` - var ( - dir string - key string - keyfile string - out string + dir string + out string ) +type DSLHelperFunc struct { + Name string + Description string + Signatures []string +} + +var pkg2NameMapping = map[string]string{ + "code": "Code Protocol", + "javascript": "JavaScript Protocol", + "global": "Javascript Runtime", + "compiler": "Javascript Runtime", + "flow": "Template Flow", +} + +var preferredOrder = []string{"Javascript Runtime", "Template Flow", "Code Protocol", "JavaScript Protocol"} + func main() { - flag.StringVar(&dir, "dir", "pkg/js/global", "directory to process") - flag.StringVar(&key, "key", "", "openai api key") - flag.StringVar(&keyfile, "keyfile", "", "openai api key file") - flag.StringVar(&out, "out", "", "output js file with declarations of all global functions") + flag.StringVar(&dir, "dir", "pkg/", "directory to process") + flag.StringVar(&out, "out", "", "output markdown file with helper file declarations") flag.Parse() - finalKey := "" - if key != "" { - key = finalKey - } - if keyfile != "" && filutil.FileExists(keyfile) { - data, err := os.ReadFile(keyfile) - if err != nil { - log.Fatal(err) + dirList := []string{} + + if err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { + if d.IsDir() { + dirList = append(dirList, path) } - finalKey = string(data) + return nil + }); err != nil { + panic(err) } - if key := os.Getenv("OPENAI_API_KEY"); key != "" { - finalKey = key + pkgs := map[string]*ast.Package{} + + for _, dir := range dirList { + fset := token.NewFileSet() + pkgss, err := parser.ParseDir(fset, dir, nil, 0) + if err != nil { + fmt.Println(err) + return + } + pkgs = mapsutil.Merge(pkgs, pkgss) } - if finalKey == "" { - log.Fatal("openai api key is not set") - } - llm := openai.NewClient(finalKey) - var buff bytes.Buffer - - fset := token.NewFileSet() - pkgs, err := parser.ParseDir(fset, dir, nil, 0) - if err != nil { - fmt.Println(err) - return - } + dslHelpers := map[string][]DSLHelperFunc{} for _, pkg := range pkgs { - for _, file := range pkg.Files { + for fname, file := range pkg.Files { ast.Inspect(file, func(n ast.Node) bool { switch x := n.(type) { case *ast.CallExpr: if sel, ok := x.Fun.(*ast.SelectorExpr); ok { if sel.Sel.Name == "RegisterFuncWithSignature" { + hf := DSLHelperFunc{} for _, arg := range x.Args { if kv, ok := arg.(*ast.CompositeLit); ok { for _, elt := range kv.Elts { if kv, ok := elt.(*ast.KeyValueExpr); ok { key := kv.Key.(*ast.Ident).Name switch key { - case "Name", "Description": - buff.WriteString(fmt.Sprintf("%s: %s\n", key, strings.Trim(kv.Value.(*ast.BasicLit).Value, `"`))) + case "Name": + hf.Name = strings.Trim(kv.Value.(*ast.BasicLit).Value, `"`) + case "Description": + hf.Description = strings.Trim(kv.Value.(*ast.BasicLit).Value, `"`) case "Signatures": if comp, ok := kv.Value.(*ast.CompositeLit); ok { for _, signature := range comp.Elts { - buff.WriteString(fmt.Sprintf("%s: %s\n", key, signature.(*ast.BasicLit).Value)) + hf.Signatures = append(hf.Signatures, strings.Trim(signature.(*ast.BasicLit).Value, `"`)) } } } @@ -115,7 +94,17 @@ func main() { } } } - buff.WriteString("\n") + if hf.Name != "" { + identifier := pkg2NameMapping[pkg.Name] + if identifier == "" { + identifier = pkg.Name + " (" + filepath.Dir(fname) + ")" + } + + if dslHelpers[identifier] == nil { + dslHelpers[identifier] = []DSLHelperFunc{} + } + dslHelpers[identifier] = append(dslHelpers[identifier], hf) + } } } } @@ -124,32 +113,54 @@ func main() { } } - fmt.Printf("[+] Scraped %d functions\n\n", strings.Count(buff.String(), "Name:")) - fmt.Println(buff.String()) - - fmt.Printf("[+] Generating jsdoc for all functions\n\n") - resp, err := llm.CreateChatCompletion(context.TODO(), openai.ChatCompletionRequest{ - Model: "gpt-4", - Messages: []openai.ChatCompletionMessage{ - {Role: "system", Content: sysprompt}, - {Role: "user", Content: strings.ReplaceAll(userPrompt, "{{source}}", buff.String())}, - }, - Temperature: 0.1, - }) - if err != nil { - fmt.Println(err) - return + // DSL Helper functions stats + for pkg, funcs := range dslHelpers { + fmt.Printf("Found %d DSL Helper functions in package %s\n", len(funcs), pkg) } - if len(resp.Choices) == 0 { - fmt.Println("no choices returned") - return - } - data := resp.Choices[0].Message.Content - - fmt.Println(data) + // Generate Markdown tables with ## as package name if out != "" { - if err := os.WriteFile(out, []byte(data), 0600); err != nil { + var sb strings.Builder + sb.WriteString(`--- +title: "Javascript Helper Functions" +description: "Available JS Helper Functions that can be used in global js runtime & protocol specific helpers." +icon: "function" +iconType: "solid" +--- + + +`) + + actualKeys := maps.Keys(dslHelpers) + sort.Slice(actualKeys, func(i, j int) bool { + for _, preferredKey := range preferredOrder { + if actualKeys[i] == preferredKey { + return true + } + if actualKeys[j] == preferredKey { + return false + } + } + return actualKeys[i] < actualKeys[j] + }) + + for _, v := range actualKeys { + pkg := v + funcs := dslHelpers[pkg] + sb.WriteString("## " + pkg + "\n\n") + sb.WriteString("| Name | Description | Signatures |\n") + sb.WriteString("|------|-------------|------------|\n") + for _, f := range funcs { + sigSlice := []string{} + for _, sig := range f.Signatures { + sigSlice = append(sigSlice, "`"+sig+"`") + } + sb.WriteString(fmt.Sprintf("| %s | %s | %s |\n", f.Name, f.Description, strings.Join(sigSlice, ", "))) + } + sb.WriteString("\n") + } + + if err := os.WriteFile(out, []byte(sb.String()), 0644); err != nil { fmt.Println(err) return } diff --git a/pkg/js/global/helpers.go b/pkg/js/global/helpers.go index 1dae2af0..5510d7ae 100644 --- a/pkg/js/global/helpers.go +++ b/pkg/js/global/helpers.go @@ -38,3 +38,40 @@ func registerAdditionalHelpers(runtime *goja.Runtime) { }, }) } + +func init() { + // these are dummy functions we use trigger documentation generation + // actual definations are in exports.js + _ = gojs.RegisterFuncWithSignature(nil, gojs.FuncOpts{ + Name: "to_json", + Signatures: []string{ + "to_json(any) object", + }, + Description: "Converts a given object to JSON", + }) + + _ = gojs.RegisterFuncWithSignature(nil, gojs.FuncOpts{ + Name: "dump_json", + Signatures: []string{ + "dump_json(any)", + }, + Description: "Prints a given object as JSON in console", + }) + + _ = gojs.RegisterFuncWithSignature(nil, gojs.FuncOpts{ + Name: "to_array", + Signatures: []string{ + "to_array(any) array", + }, + Description: "Sets/Updates objects prototype to array to enable Array.XXX functions", + }) + + _ = gojs.RegisterFuncWithSignature(nil, gojs.FuncOpts{ + Name: "hex_to_ascii", + Signatures: []string{ + "hex_to_ascii(string) string", + }, + Description: "Converts a given hex string to ascii", + }) + +} diff --git a/pkg/js/gojs/set.go b/pkg/js/gojs/set.go index 38e45d8d..9703a3c6 100644 --- a/pkg/js/gojs/set.go +++ b/pkg/js/gojs/set.go @@ -7,6 +7,7 @@ import ( var ( ErrInvalidFuncOpts = errorutil.NewWithFmt("invalid function options: %v") + ErrNilRuntime = errorutil.New("runtime is nil") ) type FuncOpts struct { @@ -23,6 +24,9 @@ func (f *FuncOpts) valid() bool { // RegisterFunc registers a function with given name, signatures and description func RegisterFuncWithSignature(runtime *goja.Runtime, opts FuncOpts) error { + if runtime == nil { + return ErrNilRuntime + } if !opts.valid() { return ErrInvalidFuncOpts.Msgf("name: %s, signatures: %v, description: %s", opts.Name, opts.Signatures, opts.Description) } diff --git a/pkg/protocols/code/code.go b/pkg/protocols/code/code.go index 57eb9b88..5193344a 100644 --- a/pkg/protocols/code/code.go +++ b/pkg/protocols/code/code.go @@ -8,11 +8,15 @@ import ( "strings" "time" + "github.com/alecthomas/chroma/quick" + "github.com/ditashi/jsbeautifier-go/jsbeautifier" + "github.com/dop251/goja" "github.com/pkg/errors" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gozero" gozerotypes "github.com/projectdiscovery/gozero/types" + "github.com/projectdiscovery/nuclei/v3/pkg/js/compiler" "github.com/projectdiscovery/nuclei/v3/pkg/operators" "github.com/projectdiscovery/nuclei/v3/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers" @@ -52,6 +56,9 @@ type Request struct { // Engine type Engine []string `yaml:"engine,omitempty" jsonschema:"title=engine,description=Engine"` // description: | + // PreCondition is a condition which is evaluated before sending the request. + PreCondition string `yaml:"pre-condition,omitempty" json:"pre-condition,omitempty" jsonschema:"title=pre-condition for the request,description=PreCondition is a condition which is evaluated before sending the request"` + // description: | // Engine Arguments Args []string `yaml:"args,omitempty" jsonschema:"title=args,description=Args"` // description: | @@ -61,9 +68,10 @@ type Request struct { // Source File/Snippet Source string `yaml:"source,omitempty" jsonschema:"title=source file/snippet,description=Source snippet"` - options *protocols.ExecutorOptions - gozero *gozero.Gozero - src *gozero.Source + options *protocols.ExecutorOptions `yaml:"-" json:"-"` + preConditionCompiled *goja.Program `yaml:"-" json:"-"` + gozero *gozero.Gozero `yaml:"-" json:"-"` + src *gozero.Source `yaml:"-" json:"-"` } // Compile compiles the request generators preparing any requests possible. @@ -110,6 +118,15 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { } request.CompiledOperators = compiled } + + // compile pre-condition if any + if request.PreCondition != "" { + preConditionCompiled, err := compiler.WrapScriptNCompile(request.PreCondition, false) + if err != nil { + return errorutil.NewWithTag(request.TemplateID, "could not compile pre-condition: %s", err) + } + request.preConditionCompiled = preConditionCompiled + } return nil } @@ -130,11 +147,6 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa return err } defer func() { - // catch any panics just in case - if r := recover(); r != nil { - gologger.Error().Msgf("[%s] Panic occurred in code protocol: %s\n", request.options.TemplateID, r) - err = fmt.Errorf("panic occurred: %s", r) - } if err := metaSrc.Cleanup(); err != nil { gologger.Warning().Msgf("%s\n", err) } @@ -160,7 +172,45 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa allvars[name] = v metaSrc.AddVariable(gozerotypes.Variable{Name: name, Value: v}) } + + // set timeout using multiplier timeout := TimeoutMultiplier * request.options.Options.Timeout + + if request.PreCondition != "" { + if request.options.Options.Debug || request.options.Options.DebugRequests { + gologger.Debug().Msgf("[%s] Executing Precondition for Code request\n", request.TemplateID) + var highlightFormatter = "terminal256" + if request.options.Options.NoColor { + highlightFormatter = "text" + } + var buff bytes.Buffer + _ = quick.Highlight(&buff, beautifyJavascript(request.PreCondition), "javascript", highlightFormatter, "monokai") + prettyPrint(request.TemplateID, buff.String()) + } + + args := compiler.NewExecuteArgs() + args.TemplateCtx = allvars + + result, err := request.options.JsCompiler.ExecuteWithOptions(request.preConditionCompiled, args, + &compiler.ExecuteOptions{ + Timeout: timeout, + Source: &request.PreCondition, + Callback: registerPreConditionFunctions, + Cleanup: cleanUpPreConditionFunctions, + }) + if err != nil { + return errorutil.NewWithTag(request.TemplateID, "could not execute pre-condition: %s", err) + } + if !result.GetSuccess() || types.ToString(result["error"]) != "" { + gologger.Warning().Msgf("[%s] Precondition for request %s was not satisfied\n", request.TemplateID, request.PreCondition) + request.options.Progress.IncrementFailedRequestsBy(1) + return nil + } + if request.options.Options.Debug || request.options.Options.DebugRequests { + gologger.Debug().Msgf("[%s] Precondition for request was satisfied\n", request.TemplateID) + } + } + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second) defer cancel() // Note: we use contextutil despite the fact that gozero accepts context as argument @@ -336,3 +386,23 @@ func interpretEnvVars(source string, vars map[string]interface{}) string { } return source } + +func beautifyJavascript(code string) string { + opts := jsbeautifier.DefaultOptions() + beautified, err := jsbeautifier.Beautify(&code, opts) + if err != nil { + return code + } + return beautified +} + +func prettyPrint(templateId string, buff string) { + lines := strings.Split(buff, "\n") + final := []string{} + for _, v := range lines { + if v != "" { + final = append(final, "\t"+v) + } + } + gologger.Debug().Msgf(" [%v] Pre-condition Code:\n\n%v\n\n", templateId, strings.Join(final, "\n")) +} diff --git a/pkg/protocols/code/helpers.go b/pkg/protocols/code/helpers.go new file mode 100644 index 00000000..f67144e7 --- /dev/null +++ b/pkg/protocols/code/helpers.go @@ -0,0 +1,272 @@ +package code + +import ( + goruntime "runtime" + + "github.com/dop251/goja" + "github.com/projectdiscovery/nuclei/v3/pkg/js/gojs" + osutils "github.com/projectdiscovery/utils/os" +) + +// registerPreConditionFunctions registers the pre-condition functions +func registerPreConditionFunctions(runtime *goja.Runtime) error { + // Note: the only reason we are not using forloop to generate these functions is because + // 'scrapefuncs' uses this function to find all dsl helper functions and document them. + + // === OS === + err := gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{ + Name: "OS", + Signatures: []string{ + "OS() string", + }, + Description: "OS returns the current OS", + FuncDecl: func() string { + return goruntime.GOOS + }, + }) + if err != nil { + return err + } + + // IsLinux checks if the current OS is Linux + err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{ + Name: "IsLinux", + Signatures: []string{ + "IsLinux() bool", + }, + Description: "IsLinux checks if the current OS is Linux", + FuncDecl: func() bool { + return osutils.IsLinux() + }, + }) + if err != nil { + return err + } + + // IsWindows checks if the current OS is Windows + err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{ + Name: "IsWindows", + Signatures: []string{ + "IsWindows() bool", + }, + Description: "IsWindows checks if the current OS is Windows", + FuncDecl: func() bool { + return osutils.IsWindows() + }, + }) + if err != nil { + return err + } + + // IsOSX checks if the current OS is OSX + err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{ + Name: "IsOSX", + Signatures: []string{ + "IsOSX() bool", + }, + Description: "IsOSX checks if the current OS is OSX", + + FuncDecl: func() bool { + return osutils.IsOSX() + }, + }) + if err != nil { + return err + } + + // IsAndroid checks if the current OS is Android + err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{ + Name: "IsAndroid", + Signatures: []string{ + "IsAndroid() bool", + }, + Description: "IsAndroid checks if the current OS is Android", + FuncDecl: func() bool { + return osutils.IsAndroid() + }, + }) + if err != nil { + return err + } + + // IsIOS checks if the current OS is IOS + err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{ + Name: "IsIOS", + Signatures: []string{ + "IsIOS() bool", + }, + Description: "IsIOS checks if the current OS is IOS", + FuncDecl: func() bool { + return osutils.IsIOS() + }, + }) + if err != nil { + return err + } + + // IsJS checks if the current OS is JS + err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{ + Name: "IsJS", + Signatures: []string{ + "IsJS() bool", + }, + Description: "IsJS checks if the current OS is JS", + FuncDecl: func() bool { + return osutils.IsJS() + }, + }) + if err != nil { + return err + } + + // IsFreeBSD checks if the current OS is FreeBSD + err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{ + Name: "IsFreeBSD", + Signatures: []string{ + "IsFreeBSD() bool", + }, + Description: "IsFreeBSD checks if the current OS is FreeBSD", + FuncDecl: func() bool { + return osutils.IsFreeBSD() + }, + }) + if err != nil { + return err + } + + // IsOpenBSD checks if the current OS is OpenBSD + err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{ + Name: "IsOpenBSD", + Signatures: []string{ + "IsOpenBSD() bool", + }, + Description: "IsOpenBSD checks if the current OS is OpenBSD", + FuncDecl: func() bool { + return osutils.IsOpenBSD() + }, + }) + if err != nil { + return err + } + + // IsSolaris checks if the current OS is Solaris + err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{ + Name: "IsSolaris", + Signatures: []string{ + "IsSolaris() bool", + }, + Description: "IsSolaris checks if the current OS is Solaris", + FuncDecl: func() bool { + return osutils.IsSolaris() + }, + }) + if err != nil { + return err + } + + // === Arch === + err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{ + Name: "Arch", + Signatures: []string{ + "Arch() string", + }, + Description: "Arch returns the current architecture", + FuncDecl: func() string { + return goruntime.GOARCH + }, + }) + if err != nil { + return err + } + + err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{ + Name: "Is386", + Signatures: []string{ + "Is386() bool", + }, + Description: "Is386 checks if the current architecture is 386", + FuncDecl: func() bool { + return osutils.Is386() + }, + }) + if err != nil { + return err + } + + err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{ + Name: "IsAmd64", + Signatures: []string{ + "IsAmd64() bool", + }, + Description: "IsAmd64 checks if the current architecture is Amd64", + FuncDecl: func() bool { + return osutils.IsAmd64() + }, + }) + if err != nil { + return err + } + + err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{ + Name: "IsARM", + Signatures: []string{ + "IsARM() bool", + }, + Description: "IsArm checks if the current architecture is Arm", + FuncDecl: func() bool { + return osutils.IsARM() + }, + }) + if err != nil { + return err + } + + err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{ + Name: "IsARM64", + Signatures: []string{ + "IsARM64() bool", + }, + Description: "IsArm64 checks if the current architecture is Arm64", + FuncDecl: func() bool { + return osutils.IsARM64() + }, + }) + if err != nil { + return err + } + + err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{ + Name: "IsWasm", + Signatures: []string{ + "IsWasm() bool", + }, + Description: "IsWasm checks if the current architecture is Wasm", + FuncDecl: func() bool { + return osutils.IsWasm() + }, + }) + if err != nil { + return err + } + + return nil +} + +func cleanUpPreConditionFunctions(runtime *goja.Runtime) { + _ = runtime.GlobalObject().Delete("OS") + _ = runtime.GlobalObject().Delete("IsLinux") + _ = runtime.GlobalObject().Delete("IsWindows") + _ = runtime.GlobalObject().Delete("IsOSX") + _ = runtime.GlobalObject().Delete("IsAndroid") + _ = runtime.GlobalObject().Delete("IsIOS") + _ = runtime.GlobalObject().Delete("IsJS") + _ = runtime.GlobalObject().Delete("IsFreeBSD") + _ = runtime.GlobalObject().Delete("IsOpenBSD") + _ = runtime.GlobalObject().Delete("IsSolaris") + _ = runtime.GlobalObject().Delete("Arch") + _ = runtime.GlobalObject().Delete("Is386") + _ = runtime.GlobalObject().Delete("IsAmd64") + _ = runtime.GlobalObject().Delete("IsARM") + _ = runtime.GlobalObject().Delete("IsARM64") + _ = runtime.GlobalObject().Delete("IsWasm") +} diff --git a/pkg/protocols/http/http.go b/pkg/protocols/http/http.go index 5ed5b639..51e5a36d 100644 --- a/pkg/protocols/http/http.go +++ b/pkg/protocols/http/http.go @@ -211,13 +211,12 @@ type Request struct { // DisablePathAutomerge disables merging target url path with raw request path DisablePathAutomerge bool `yaml:"disable-path-automerge,omitempty" json:"disable-path-automerge,omitempty" jsonschema:"title=disable auto merging of path,description=Disable merging target url path with raw request path"` // description: | - // Filter is matcher-like field to check if fuzzing should be performed on this request or not - FuzzingFilter []*matchers.Matcher `yaml:"filters,omitempty" json:"filter,omitempty" jsonschema:"title=filter for fuzzing,description=Filter is matcher-like field to check if fuzzing should be performed on this request or not"` + // Fuzz PreCondition is matcher-like field to check if fuzzing should be performed on this request or not + FuzzPreCondition []*matchers.Matcher `yaml:"pre-condition,omitempty" json:"pre-condition,omitempty" jsonschema:"title=pre-condition for fuzzing/dast,description=PreCondition is matcher-like field to check if fuzzing should be performed on this request or not"` // description: | - // Filter condition is the condition to apply on the filter (AND/OR). Default is OR - FuzzingFilterCondition string `yaml:"filters-condition,omitempty" json:"filter-condition,omitempty" jsonschema:"title=condition between the filters,description=Conditions between the filters,enum=and,enum=or"` - // cached variables that may be used along with request. - fuzzingFilterCondition matchers.ConditionType `yaml:"-" json:"-"` + // FuzzPreConditionOperator is the operator between multiple PreConditions for fuzzing Default is OR + FuzzPreConditionOperator string `yaml:"pre-condition-operator,omitempty" json:"pre-condition-operator,omitempty" jsonschema:"title=condition between the filters,description=Operator to use between multiple per-conditions,enum=and,enum=or"` + fuzzPreConditionOperator matchers.ConditionType `yaml:"-" json:"-"` } // Options returns executer options for http request @@ -326,13 +325,13 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { // === fuzzing filters ===== // - if request.FuzzingFilterCondition != "" { - request.fuzzingFilterCondition = matchers.ConditionTypes[request.FuzzingFilterCondition] + if request.FuzzPreConditionOperator != "" { + request.fuzzPreConditionOperator = matchers.ConditionTypes[request.FuzzPreConditionOperator] } else { - request.fuzzingFilterCondition = matchers.ORCondition + request.fuzzPreConditionOperator = matchers.ORCondition } - for _, filter := range request.FuzzingFilter { + for _, filter := range request.FuzzPreCondition { if err := filter.CompileMatchers(); err != nil { return errors.Wrap(err, "could not compile matcher") } diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go index ae05479b..a942c5b1 100644 --- a/pkg/protocols/http/request.go +++ b/pkg/protocols/http/request.go @@ -137,11 +137,6 @@ func (request *Request) executeRaceRequest(input *contextargs.Context, previous // execute http request go func(httpRequest *generatedRequest) { defer spmHandler.Release() - defer func() { - if r := recover(); r != nil { - gologger.Verbose().Msgf("[%s] Recovered from panic: %v\n", request.options.TemplateID, r) - } - }() if spmHandler.FoundFirstMatch() { // stop sending more requests condition is met return @@ -218,11 +213,6 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV spmHandler.Acquire() go func(httpRequest *generatedRequest) { defer spmHandler.Release() - defer func() { - if r := recover(); r != nil { - gologger.Verbose().Msgf("[%s] Recovered from panic: %v\n", request.options.TemplateID, r) - } - }() if spmHandler.FoundFirstMatch() { return } @@ -319,11 +309,6 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu spmHandler.Acquire() go func(httpRequest *generatedRequest) { defer spmHandler.Release() - defer func() { - if r := recover(); r != nil { - gologger.Verbose().Msgf("[%s] Recovered from panic: %v\n", request.options.TemplateID, r) - } - }() if spmHandler.FoundFirstMatch() { // skip if first match is found return diff --git a/pkg/protocols/http/request_fuzz.go b/pkg/protocols/http/request_fuzz.go index 75470599..d6a29c18 100644 --- a/pkg/protocols/http/request_fuzz.go +++ b/pkg/protocols/http/request_fuzz.go @@ -194,11 +194,11 @@ func (request *Request) executeGeneratedFuzzingRequest(gr fuzz.GeneratedRequest, // ShouldFuzzTarget checks if given target should be fuzzed or not using `filter` field in template func (request *Request) ShouldFuzzTarget(input *contextargs.Context) bool { - if len(request.FuzzingFilter) == 0 { + if len(request.FuzzPreCondition) == 0 { return true } status := []bool{} - for index, filter := range request.FuzzingFilter { + for index, filter := range request.FuzzPreCondition { isMatch, _ := request.Match(request.filterDataMap(input), filter) status = append(status, isMatch) if request.options.Options.MatcherStatus { @@ -209,7 +209,7 @@ func (request *Request) ShouldFuzzTarget(input *contextargs.Context) bool { return true } var matched bool - if request.fuzzingFilterCondition == matchers.ANDCondition { + if request.fuzzPreConditionOperator == matchers.ANDCondition { matched = operators.EvalBoolSlice(status, true) } else { matched = operators.EvalBoolSlice(status, false) diff --git a/pkg/templates/compile.go b/pkg/templates/compile.go index eda128c7..b821fcc6 100644 --- a/pkg/templates/compile.go +++ b/pkg/templates/compile.go @@ -35,7 +35,8 @@ var ( ) const ( - Unsigned = "unsigned" + Unsigned = "unsigned" + PDVerifier = "projectdiscovery/nuclei-templates" ) func init() { @@ -272,7 +273,6 @@ func ParseTemplateFromReader(reader io.Reader, preprocessor Preprocessor, option if config.DefaultConfig.LogAllEvents { gologger.DefaultLogger.Print().Msgf("[%v] Template %s is not signed or tampered\n", aurora.Yellow("WRN").String(), template.ID) } - SignatureStats[Unsigned].Add(1) } return template, nil } @@ -293,7 +293,6 @@ func ParseTemplateFromReader(reader io.Reader, preprocessor Preprocessor, option if config.DefaultConfig.LogAllEvents { gologger.DefaultLogger.Print().Msgf("[%v] Template %s is not signed or tampered\n", aurora.Yellow("WRN").String(), template.ID) } - SignatureStats[Unsigned].Add(1) } generatedConstants := map[string]interface{}{} @@ -399,7 +398,7 @@ func parseTemplate(data []byte, options protocols.ExecutorOptions) (*Template, e for _, verifier = range signer.DefaultTemplateVerifiers { template.Verified, _ = verifier.Verify(data, template) if template.Verified { - SignatureStats[verifier.Identifier()].Add(1) + template.TemplateVerifier = verifier.Identifier() break } } diff --git a/pkg/templates/templates.go b/pkg/templates/templates.go index 01dbb06a..6403c276 100644 --- a/pkg/templates/templates.go +++ b/pkg/templates/templates.go @@ -154,7 +154,8 @@ type Template struct { // Verified defines if the template signature is digitally verified Verified bool `yaml:"-" json:"-"` - + // TemplateVerifier is identifier verifier used to verify the template (default nuclei-templates have projectdiscovery/nuclei-templates) + TemplateVerifier string `yaml:"-" json:"-"` // RequestsQueue contains all template requests in order (both protocol & request order) RequestsQueue []protocols.Request `yaml:"-" json:"-"` diff --git a/pkg/templates/workflows.go b/pkg/templates/workflows.go index 6f6d5d3d..bafb7aaf 100644 --- a/pkg/templates/workflows.go +++ b/pkg/templates/workflows.go @@ -101,6 +101,17 @@ func parseWorkflowTemplate(workflow *workflows.WorkflowTemplate, preprocessor Pr continue } } + + // increment signed/unsigned counters + if template.Verified { + if template.TemplateVerifier == "" { + SignatureStats[PDVerifier].Add(1) + } else { + SignatureStats[template.TemplateVerifier].Add(1) + } + } else { + SignatureStats[Unsigned].Add(1) + } workflowTemplates = append(workflowTemplates, template) } diff --git a/pkg/tmplexec/exec.go b/pkg/tmplexec/exec.go index 3e0e8fdd..d5c2aaee 100644 --- a/pkg/tmplexec/exec.go +++ b/pkg/tmplexec/exec.go @@ -3,7 +3,6 @@ package tmplexec import ( "errors" "fmt" - "runtime/debug" "strings" "sync/atomic" @@ -99,14 +98,6 @@ func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) { // since it is of no use after scan is completed (regardless of success or failure) e.options.RemoveTemplateCtx(ctx.Input.MetaInput) }() - defer func() { - // try catching unknown panics - if r := recover(); r != nil { - stacktrace := debug.Stack() - ctx.LogError(fmt.Errorf("panic: %v\n%s", r, stacktrace)) - gologger.Verbose().Msgf("panic: %v\n%s", r, stacktrace) - } - }() var lastMatcherEvent *output.InternalWrappedEvent writeFailureCallback := func(event *output.InternalWrappedEvent, matcherStatus bool) { diff --git a/pkg/tmplexec/flow/flow_executor.go b/pkg/tmplexec/flow/flow_executor.go index 33821ac0..31aa9f0f 100644 --- a/pkg/tmplexec/flow/flow_executor.go +++ b/pkg/tmplexec/flow/flow_executor.go @@ -8,7 +8,6 @@ import ( "sync/atomic" "github.com/dop251/goja" - "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v3/pkg/protocols" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v3/pkg/scan" @@ -175,13 +174,6 @@ func (f *FlowExecutor) Compile() error { // ExecuteWithResults executes the flow and returns results func (f *FlowExecutor) ExecuteWithResults(ctx *scan.ScanContext) error { - defer func() { - if e := recover(); e != nil { - f.ctx.LogError(fmt.Errorf("panic occurred while executing target %v with flow: %v", ctx.Input.MetaInput.Input, e)) - gologger.Error().Label(f.options.TemplateID).Msgf("panic occurred while executing target %v with flow: %v", ctx.Input.MetaInput.Input, e) - } - }() - f.ctx.Input = ctx.Input // -----Load all types of variables----- // add all input args to template context diff --git a/pkg/tmplexec/flow/flow_internal.go b/pkg/tmplexec/flow/flow_internal.go index b7f589d3..6e82bd96 100644 --- a/pkg/tmplexec/flow/flow_internal.go +++ b/pkg/tmplexec/flow/flow_internal.go @@ -96,10 +96,12 @@ func (f *FlowExecutor) protocolResultCallback(req protocols.Request, matcherStat if len(v) == 1 { // add it to flatten keys list so it will be flattened to a string later f.flattenKeys = append(f.flattenKeys, k) + // flatten and convert it to string + f.options.GetTemplateCtx(f.ctx.Input.MetaInput).Set(k, v[0]) + } else { + // keep it as slice + f.options.GetTemplateCtx(f.ctx.Input.MetaInput).Set(k, v) } - // always preserve extracted value type - f.options.GetTemplateCtx(f.ctx.Input.MetaInput).Set(k, v) - } } } else if !result.HasOperatorResult() && !hasOperators(req.GetCompiledOperators()) { diff --git a/pkg/tmplexec/flow/vm.go b/pkg/tmplexec/flow/vm.go index 41d6c392..2e22bd8e 100644 --- a/pkg/tmplexec/flow/vm.go +++ b/pkg/tmplexec/flow/vm.go @@ -7,6 +7,7 @@ import ( "github.com/dop251/goja" "github.com/logrusorgru/aurora" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v3/pkg/js/gojs" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump" "github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/flow/builtin" @@ -49,46 +50,68 @@ var gojapool = &sync.Pool{ } func registerBuiltins(runtime *goja.Runtime) { - _ = runtime.Set("log", func(call goja.FunctionCall) goja.Value { - // TODO: verify string interpolation and handle multiple args - arg := call.Argument(0).Export() - switch value := arg.(type) { - case string: - gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), value) - case map[string]interface{}: - gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), vardump.DumpVariables(value)) - default: - gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), value) - } - return call.Argument(0) // return the same value + + _ = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{ + Name: "log", + Description: "Logs a given object/message to stdout (only for debugging purposes)", + Signatures: []string{ + "log(obj any) any", + }, + FuncDecl: func(call goja.FunctionCall) goja.Value { + arg := call.Argument(0).Export() + switch value := arg.(type) { + case string: + gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), value) + case map[string]interface{}: + gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), vardump.DumpVariables(value)) + default: + gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), value) + } + return call.Argument(0) // return the same value + }, }) - _ = runtime.Set("iterate", func(call goja.FunctionCall) goja.Value { - allVars := []any{} - for _, v := range call.Arguments { - if v.Export() == nil { - continue - } - if v.ExportType().Kind() == reflect.Slice { - // convert []datatype to []interface{} - // since it cannot be type asserted to []interface{} directly - rfValue := reflect.ValueOf(v.Export()) - for i := 0; i < rfValue.Len(); i++ { - allVars = append(allVars, rfValue.Index(i).Interface()) + _ = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{ + Name: "iterate", + Description: "Normalizes and Iterates over all arguments (can be a string,array,null etc) and returns an array of objects\nNote: If the object type is unknown(i.e could be a string or array) iterate should be used and it will always return an array of strings", + Signatures: []string{ + "iterate(...any) []any", + }, + FuncDecl: func(call goja.FunctionCall) goja.Value { + allVars := []any{} + for _, v := range call.Arguments { + if v.Export() == nil { + continue + } + if v.ExportType().Kind() == reflect.Slice { + // convert []datatype to []interface{} + // since it cannot be type asserted to []interface{} directly + rfValue := reflect.ValueOf(v.Export()) + for i := 0; i < rfValue.Len(); i++ { + allVars = append(allVars, rfValue.Index(i).Interface()) + } + } else { + allVars = append(allVars, v.Export()) } - } else { - allVars = append(allVars, v.Export()) } - } - return runtime.ToValue(allVars) + return runtime.ToValue(allVars) + }, }) - _ = runtime.Set("Dedupe", func(call goja.ConstructorCall) *goja.Object { - d := builtin.NewDedupe(runtime) - obj := call.This - // register these methods - _ = obj.Set("Add", d.Add) - _ = obj.Set("Values", d.Values) - return nil + _ = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{ + Name: "Dedupe", + Description: "De-duplicates given values and returns a new array of unique values", + Signatures: []string{ + "new Dedupe()", + }, + FuncDecl: func(call goja.ConstructorCall) *goja.Object { + d := builtin.NewDedupe(runtime) + obj := call.This + // register these methods + _ = obj.Set("Add", d.Add) + _ = obj.Set("Values", d.Values) + return nil + }, }) + }