pre-condition in code , fuzz and other misc updates (#4966)

* fuzz: rename 'filters' -> 'pre-condition'

* code proto: pre-condition + integration test

* feat: dsl document generator

* update dsl page header

* fix lint error

* add js defined helper funcs in docs

* remove panic recovery unless its for third party(go-rod,goja)

* handle dynamic values flattening edgecase in flow+multiprotocol

* fix order of kv in form-data (failing test)

* fix template loading counters

* Revert "handle dynamic values flattening edgecase in flow+multiprotocol"

This reverts commit 58fdd4faf7df5d654b46a9585011f614d5c98aa4.

* fix flow iteration using 'iterate'
dev
Tarun Koyalwar 2024-04-01 19:18:21 +05:30 committed by GitHub
parent 1d8b10be2a
commit 255032f4f2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
35 changed files with 676 additions and 231 deletions

2
.gitignore vendored
View File

@ -35,4 +35,6 @@ pkg/protocols/headless/engine/.cache
**/*-cache **/*-cache
/fuzzplayground /fuzzplayground
integration_tests/fuzzplayground integration_tests/fuzzplayground
/dsl.md

View File

@ -47,4 +47,7 @@ fuzzplayground:
memogen: memogen:
$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "memogen" cmd/memogen/memogen.go $(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "memogen" cmd/memogen/memogen.go
./memogen -src pkg/js/libs -tpl cmd/memogen/function.tpl ./memogen -src pkg/js/libs -tpl cmd/memogen/function.tpl
dsl-docs:
rm -f dsl.md scrapefuncs 2>/dev/null
$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "scrapefuncs" pkg/js/devtools/scrapefuncs/main.go
./scrapefuncs -out dsl.md

View File

@ -23,6 +23,7 @@ var codeTestCases = []TestCaseInfo{
{Path: "protocols/code/py-nosig.yaml", TestCase: &codePyNoSig{}, DisableOn: isCodeDisabled}, {Path: "protocols/code/py-nosig.yaml", TestCase: &codePyNoSig{}, DisableOn: isCodeDisabled},
{Path: "protocols/code/py-interactsh.yaml", TestCase: &codeSnippet{}, DisableOn: isCodeDisabled}, {Path: "protocols/code/py-interactsh.yaml", TestCase: &codeSnippet{}, DisableOn: isCodeDisabled},
{Path: "protocols/code/ps1-snippet.yaml", TestCase: &codeSnippet{}, DisableOn: func() bool { return !osutils.IsWindows() || isCodeDisabled() }}, {Path: "protocols/code/ps1-snippet.yaml", TestCase: &codeSnippet{}, DisableOn: func() bool { return !osutils.IsWindows() || isCodeDisabled() }},
{Path: "protocols/code/pre-condition.yaml", TestCase: &codePreCondition{}, DisableOn: isCodeDisabled},
} }
const ( const (
@ -94,6 +95,22 @@ func (h *codeSnippet) Execute(filePath string) error {
return expectResultsCount(results, 1) return expectResultsCount(results, 1)
} }
type codePreCondition struct{}
// Execute executes a test case and returns an error if occurred
func (h *codePreCondition) Execute(filePath string) error {
results, err := testutils.RunNucleiArgsWithEnvAndGetResults(debug, getEnvValues(), "-t", filePath, "-u", "input", "-code")
if err != nil {
return err
}
if osutils.IsLinux() {
return expectResultsCount(results, 1)
} else {
return expectResultsCount(results, 0)
}
}
type codeFile struct{} type codeFile struct{}
// Execute executes a test case and returns an error if occurred // Execute executes a test case and returns an error if occurred

2
go.mod
View File

@ -97,7 +97,6 @@ require (
github.com/projectdiscovery/utils v0.0.85 github.com/projectdiscovery/utils v0.0.85
github.com/projectdiscovery/wappalyzergo v0.0.112 github.com/projectdiscovery/wappalyzergo v0.0.112
github.com/redis/go-redis/v9 v9.1.0 github.com/redis/go-redis/v9 v9.1.0
github.com/sashabaranov/go-openai v1.15.3
github.com/seh-msft/burpxml v1.0.1 github.com/seh-msft/burpxml v1.0.1
github.com/stretchr/testify v1.9.0 github.com/stretchr/testify v1.9.0
github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706 github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706
@ -205,6 +204,7 @@ require (
github.com/projectdiscovery/stringsutil v0.0.2 // indirect github.com/projectdiscovery/stringsutil v0.0.2 // indirect
github.com/quic-go/quic-go v0.40.1 // indirect github.com/quic-go/quic-go v0.40.1 // indirect
github.com/refraction-networking/utls v1.6.1 // indirect github.com/refraction-networking/utls v1.6.1 // indirect
github.com/sashabaranov/go-openai v1.15.3 // indirect
github.com/shirou/gopsutil v3.21.11+incompatible // indirect github.com/shirou/gopsutil v3.21.11+incompatible // indirect
github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect github.com/sirupsen/logrus v1.9.3 // indirect

View File

@ -4,10 +4,13 @@ info:
name: Test Flow Iterate One Value Flow name: Test Flow Iterate One Value Flow
author: pdteam author: pdteam
severity: info severity: info
description: |
If length of template.extracted variable is not know, i.e it could be an array of 1 or more values, then iterate function
should be used to iterate over values because nuclei by default converts array to string if it has only 1 value.
flow: | flow: |
http(1) http(1)
for(let value of template.extracted){ for(let value of iterate(template.extracted)){
set("value", value) set("value", value)
http(2) http(2)
} }

View File

@ -10,7 +10,7 @@ info:
and performs fuzzing on the value of every key and performs fuzzing on the value of every key
http: http:
- filters: - pre-condition:
- type: dsl - type: dsl
dsl: dsl:
- method != "GET" - method != "GET"

View File

@ -10,7 +10,7 @@ info:
Note: this is example template, and payloads/matchers need to be modified appropriately. Note: this is example template, and payloads/matchers need to be modified appropriately.
http: http:
- filters: - pre-condition:
- type: dsl - type: dsl
dsl: dsl:
- method != "GET" - method != "GET"

View File

@ -10,7 +10,7 @@ info:
Note: this is example template, and payloads/matchers need to be modified appropriately. Note: this is example template, and payloads/matchers need to be modified appropriately.
http: http:
- filters: - pre-condition:
- type: dsl - type: dsl
dsl: dsl:
- method != "GET" - method != "GET"

View File

@ -10,7 +10,7 @@ info:
Note: this is example template, and payloads/matchers need to be modified appropriately. Note: this is example template, and payloads/matchers need to be modified appropriately.
http: http:
- filters: - pre-condition:
- type: dsl - type: dsl
dsl: dsl:
- method != "GET" - method != "GET"

View File

@ -10,7 +10,7 @@ info:
Note: this is example template, and payloads/matchers need to be modified appropriately. Note: this is example template, and payloads/matchers need to be modified appropriately.
http: http:
- filters: - pre-condition:
- type: dsl - type: dsl
dsl: dsl:
- method != "GET" - method != "GET"

View File

@ -9,7 +9,7 @@ info:
Note: this is example template, and payloads/matchers need to be modified appropriately. Note: this is example template, and payloads/matchers need to be modified appropriately.
http: http:
- filters: - pre-condition:
- type: dsl - type: dsl
dsl: dsl:
- 'method == "GET"' - 'method == "GET"'

View File

@ -10,7 +10,7 @@ variables:
domain: "oast.fun" domain: "oast.fun"
http: http:
- filters: - pre-condition:
- type: dsl - type: dsl
dsl: dsl:
- 'method == "GET"' - 'method == "GET"'

View File

@ -11,7 +11,7 @@ info:
Note: this is example template, and payloads/matchers need to be modified appropriately. Note: this is example template, and payloads/matchers need to be modified appropriately.
http: http:
- filters: - pre-condition:
- type: dsl - type: dsl
dsl: dsl:
- 'method == "GET"' - 'method == "GET"'

View File

@ -7,7 +7,7 @@ info:
description: Query Value Fuzzing using Fuzzing Rules description: Query Value Fuzzing using Fuzzing Rules
http: http:
- filters: - pre-condition:
- type: dsl - type: dsl
dsl: dsl:
- 'len(query) > 0' - 'len(query) > 0'
@ -16,7 +16,7 @@ http:
part: path part: path
words: words:
- /blog/post - /blog/post
filters-condition: and pre-condition-operator: and
payloads: payloads:
nums: nums:

View File

@ -0,0 +1,26 @@
id: pre-condition-code
info:
name: example code template
author: pdteam
severity: info
self-contained: true
variables:
OAST: "{{interactsh-url}}"
code:
- pre-condition: IsLinux()
engine:
- sh
- bash
source: |
echo "$OAST" | base64
matchers:
- type: dsl
dsl:
- true
# digest: 4a0a00473045022100c7215ce9f11e6a51c193bb54643a05cdd1cde18a3abb6c9983c5c7524d3ff03002203d93581c81d3ad5db463570cbbd2bdee529328d32a5b00e037610c211e448cef:4a3eb6b4988d95847d4203be25ed1d46

View File

@ -688,14 +688,9 @@ func (r *Runner) displayExecutionInfo(store *loader.Store) {
} }
for k, v := range templates.SignatureStats { for k, v := range templates.SignatureStats {
value := v.Load() value := v.Load()
if k == templates.Unsigned && value > 0 {
// adjust skipped unsigned templates via code or -dut flag
value = value - uint64(stats.GetValue(templates.SkippedUnsignedStats))
value = value - uint64(stats.GetValue(templates.ExcludedCodeTmplStats))
}
if value > 0 { if value > 0 {
if k == templates.Unsigned && !r.options.Silent && !config.DefaultConfig.HideTemplateSigWarning { if k == templates.Unsigned && !r.options.Silent && !config.DefaultConfig.HideTemplateSigWarning {
gologger.Print().Msgf("[%v] Loading %d unsigned templates for scan. Use with caution.", aurora.BrightYellow("WRN"), value) gologger.Print().Msgf("[%v] Executing %d unsigned templates for scan. Use with caution.", aurora.BrightYellow("WRN"), value)
} else { } else {
gologger.Info().Msgf("Executing %d signed templates from %s", value, k) gologger.Info().Msgf("Executing %d signed templates from %s", value, k)
} }

View File

@ -13,7 +13,6 @@ import (
"github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog" "github.com/projectdiscovery/nuclei/v3/pkg/catalog"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
cfg "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader/filter" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader/filter"
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols" "github.com/projectdiscovery/nuclei/v3/pkg/protocols"
@ -387,6 +386,21 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ
templatePathMap := store.pathFilter.Match(includedTemplates) templatePathMap := store.pathFilter.Match(includedTemplates)
loadedTemplates := make([]*templates.Template, 0, len(templatePathMap)) loadedTemplates := make([]*templates.Template, 0, len(templatePathMap))
loadTemplate := func(tmpl *templates.Template) {
loadedTemplates = append(loadedTemplates, tmpl)
// increment signed/unsigned counters
if tmpl.Verified {
if tmpl.TemplateVerifier == "" {
templates.SignatureStats[templates.PDVerifier].Add(1)
} else {
templates.SignatureStats[tmpl.TemplateVerifier].Add(1)
}
} else {
templates.SignatureStats[templates.Unsigned].Add(1)
}
}
for templatePath := range templatePathMap { for templatePath := range templatePathMap {
loaded, err := store.config.ExecutorOptions.Parser.LoadTemplate(templatePath, store.tagFilter, tags, store.config.Catalog) loaded, err := store.config.ExecutorOptions.Parser.LoadTemplate(templatePath, store.tagFilter, tags, store.config.Catalog)
if loaded || store.pathFilter.MatchIncluded(templatePath) { if loaded || store.pathFilter.MatchIncluded(templatePath) {
@ -412,7 +426,7 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ
if store.config.ExecutorOptions.Options.DAST { if store.config.ExecutorOptions.Options.DAST {
// check if the template is a DAST template // check if the template is a DAST template
if parsed.IsFuzzing() { if parsed.IsFuzzing() {
loadedTemplates = append(loadedTemplates, parsed) loadTemplate(parsed)
} }
} else if len(parsed.RequestsHeadless) > 0 && !store.config.ExecutorOptions.Options.Headless { } else if len(parsed.RequestsHeadless) > 0 && !store.config.ExecutorOptions.Options.Headless {
// donot include headless template in final list if headless flag is not set // donot include headless template in final list if headless flag is not set
@ -440,14 +454,14 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ
gologger.Print().Msgf("[%v] -dast flag is required for DAST template '%s'.\n", aurora.Yellow("WRN").String(), templatePath) gologger.Print().Msgf("[%v] -dast flag is required for DAST template '%s'.\n", aurora.Yellow("WRN").String(), templatePath)
} }
} else { } else {
loadedTemplates = append(loadedTemplates, parsed) loadTemplate(parsed)
} }
} }
} }
if err != nil { if err != nil {
if strings.Contains(err.Error(), templates.ErrExcluded.Error()) { if strings.Contains(err.Error(), templates.ErrExcluded.Error()) {
stats.Increment(templates.TemplatesExcludedStats) stats.Increment(templates.TemplatesExcludedStats)
if cfg.DefaultConfig.LogAllEvents { if config.DefaultConfig.LogAllEvents {
gologger.Print().Msgf("[%v] %v\n", aurora.Yellow("WRN").String(), err.Error()) gologger.Print().Msgf("[%v] %v\n", aurora.Yellow("WRN").String(), err.Error())
} }
continue continue

View File

@ -2,7 +2,6 @@ package dataformat
import ( import (
"fmt" "fmt"
"net/url"
"regexp" "regexp"
"strconv" "strconv"
"strings" "strings"
@ -116,13 +115,11 @@ func (f *Form) Encode(data KV) (string, error) {
// Decode decodes the data from Form format // Decode decodes the data from Form format
func (f *Form) Decode(data string) (KV, error) { func (f *Form) Decode(data string) (KV, error) {
parsed, err := url.ParseQuery(data) ordered_params := urlutil.NewOrderedParams()
if err != nil { ordered_params.Merge(data)
return KV{}, err
}
values := mapsutil.NewOrderedMap[string, any]() values := mapsutil.NewOrderedMap[string, any]()
for key, value := range parsed { ordered_params.Iterate(func(key string, value []string) bool {
if len(value) == 1 { if len(value) == 1 {
values.Set(key, value[0]) values.Set(key, value[0])
} else { } else {
@ -134,7 +131,8 @@ func (f *Form) Decode(data string) (KV, error) {
} }
values.Set(key, value[len(value)-1]) values.Set(key, value[len(value)-1])
} }
} return true
})
return KVOrderedMap(&values), nil return KVOrderedMap(&values), nil
} }

View File

@ -64,11 +64,6 @@ type GeneratedRequest struct {
// Input is not thread safe and should not be shared between concurrent // Input is not thread safe and should not be shared between concurrent
// goroutines. // goroutines.
func (rule *Rule) Execute(input *ExecuteRuleInput) (err error) { func (rule *Rule) Execute(input *ExecuteRuleInput) (err error) {
defer func() {
if r := recover(); r != nil {
err = fmt.Errorf("got panic while executing rule: %v", r)
}
}()
if !rule.isInputURLValid(input.Input) { if !rule.isInputURLValid(input.Input) {
return ErrRuleNotApplicable.Msgf("invalid input url: %v", input.Input.MetaInput.Input) return ErrRuleNotApplicable.Msgf("invalid input url: %v", input.Input.MetaInput.Input)
} }

View File

@ -207,11 +207,6 @@ func (hr *HttpResponse) Clone() *HttpResponse {
// and returns the request and response object // and returns the request and response object
// Note: it currently does not parse response and is meant to be added manually since its a optional field // Note: it currently does not parse response and is meant to be added manually since its a optional field
func ParseRawRequest(raw string) (rr *RequestResponse, err error) { func ParseRawRequest(raw string) (rr *RequestResponse, err error) {
defer func() {
if r := recover(); r != nil {
err = fmt.Errorf("panic: %v", r)
}
}()
protoReader := textproto.NewReader(bufio.NewReader(strings.NewReader(raw))) protoReader := textproto.NewReader(bufio.NewReader(strings.NewReader(raw)))
methodLine, err := protoReader.ReadLine() methodLine, err := protoReader.ReadLine()
if err != nil { if err != nil {

View File

@ -1,113 +1,92 @@
package main package main
import ( import (
"bytes"
"context"
"flag" "flag"
"fmt" "fmt"
"go/ast" "go/ast"
"go/parser" "go/parser"
"go/token" "go/token"
"log"
"os" "os"
"path/filepath"
"sort"
"strings" "strings"
filutil "github.com/projectdiscovery/utils/file" mapsutil "github.com/projectdiscovery/utils/maps"
"github.com/sashabaranov/go-openai" "golang.org/x/exp/maps"
) )
var sysprompt = `
data present after ---raw data--- contains raw data extracted by a parser and contains information about function
--- example ---
Name: log
Signatures: "log(msg string)"
Signatures: "log(msg map[string]interface{})"
Description: log prints given input to stdout with [JS] prefix for debugging purposes
--- end example ---
Here Name is name of function , signature[s] is actual function declaration and description is description of function
using this data for every such function generate a abstract implementation of function in javascript along with jsdoc annotations
--- example expected output---
/**
* log prints given input to stdout with [JS] prefix for debugging purposes
* log(msg string)
* log(msg map[string]interface{})
* @function
* @param {string} msg - The message to print.
*/
function log(msg) {
// implemented in go
};
--- instructions ---
ACT as helpful coding assistant and do the same for all functions present in data
`
const userPrompt = `
---raw data---
{{source}}
---new javascript---
`
var ( var (
dir string dir string
key string out string
keyfile string
out string
) )
type DSLHelperFunc struct {
Name string
Description string
Signatures []string
}
var pkg2NameMapping = map[string]string{
"code": "Code Protocol",
"javascript": "JavaScript Protocol",
"global": "Javascript Runtime",
"compiler": "Javascript Runtime",
"flow": "Template Flow",
}
var preferredOrder = []string{"Javascript Runtime", "Template Flow", "Code Protocol", "JavaScript Protocol"}
func main() { func main() {
flag.StringVar(&dir, "dir", "pkg/js/global", "directory to process") flag.StringVar(&dir, "dir", "pkg/", "directory to process")
flag.StringVar(&key, "key", "", "openai api key") flag.StringVar(&out, "out", "", "output markdown file with helper file declarations")
flag.StringVar(&keyfile, "keyfile", "", "openai api key file")
flag.StringVar(&out, "out", "", "output js file with declarations of all global functions")
flag.Parse() flag.Parse()
finalKey := "" dirList := []string{}
if key != "" {
key = finalKey if err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error {
} if d.IsDir() {
if keyfile != "" && filutil.FileExists(keyfile) { dirList = append(dirList, path)
data, err := os.ReadFile(keyfile)
if err != nil {
log.Fatal(err)
} }
finalKey = string(data) return nil
}); err != nil {
panic(err)
} }
if key := os.Getenv("OPENAI_API_KEY"); key != "" { pkgs := map[string]*ast.Package{}
finalKey = key
for _, dir := range dirList {
fset := token.NewFileSet()
pkgss, err := parser.ParseDir(fset, dir, nil, 0)
if err != nil {
fmt.Println(err)
return
}
pkgs = mapsutil.Merge(pkgs, pkgss)
} }
if finalKey == "" { dslHelpers := map[string][]DSLHelperFunc{}
log.Fatal("openai api key is not set")
}
llm := openai.NewClient(finalKey)
var buff bytes.Buffer
fset := token.NewFileSet()
pkgs, err := parser.ParseDir(fset, dir, nil, 0)
if err != nil {
fmt.Println(err)
return
}
for _, pkg := range pkgs { for _, pkg := range pkgs {
for _, file := range pkg.Files { for fname, file := range pkg.Files {
ast.Inspect(file, func(n ast.Node) bool { ast.Inspect(file, func(n ast.Node) bool {
switch x := n.(type) { switch x := n.(type) {
case *ast.CallExpr: case *ast.CallExpr:
if sel, ok := x.Fun.(*ast.SelectorExpr); ok { if sel, ok := x.Fun.(*ast.SelectorExpr); ok {
if sel.Sel.Name == "RegisterFuncWithSignature" { if sel.Sel.Name == "RegisterFuncWithSignature" {
hf := DSLHelperFunc{}
for _, arg := range x.Args { for _, arg := range x.Args {
if kv, ok := arg.(*ast.CompositeLit); ok { if kv, ok := arg.(*ast.CompositeLit); ok {
for _, elt := range kv.Elts { for _, elt := range kv.Elts {
if kv, ok := elt.(*ast.KeyValueExpr); ok { if kv, ok := elt.(*ast.KeyValueExpr); ok {
key := kv.Key.(*ast.Ident).Name key := kv.Key.(*ast.Ident).Name
switch key { switch key {
case "Name", "Description": case "Name":
buff.WriteString(fmt.Sprintf("%s: %s\n", key, strings.Trim(kv.Value.(*ast.BasicLit).Value, `"`))) hf.Name = strings.Trim(kv.Value.(*ast.BasicLit).Value, `"`)
case "Description":
hf.Description = strings.Trim(kv.Value.(*ast.BasicLit).Value, `"`)
case "Signatures": case "Signatures":
if comp, ok := kv.Value.(*ast.CompositeLit); ok { if comp, ok := kv.Value.(*ast.CompositeLit); ok {
for _, signature := range comp.Elts { for _, signature := range comp.Elts {
buff.WriteString(fmt.Sprintf("%s: %s\n", key, signature.(*ast.BasicLit).Value)) hf.Signatures = append(hf.Signatures, strings.Trim(signature.(*ast.BasicLit).Value, `"`))
} }
} }
} }
@ -115,7 +94,17 @@ func main() {
} }
} }
} }
buff.WriteString("\n") if hf.Name != "" {
identifier := pkg2NameMapping[pkg.Name]
if identifier == "" {
identifier = pkg.Name + " (" + filepath.Dir(fname) + ")"
}
if dslHelpers[identifier] == nil {
dslHelpers[identifier] = []DSLHelperFunc{}
}
dslHelpers[identifier] = append(dslHelpers[identifier], hf)
}
} }
} }
} }
@ -124,32 +113,54 @@ func main() {
} }
} }
fmt.Printf("[+] Scraped %d functions\n\n", strings.Count(buff.String(), "Name:")) // DSL Helper functions stats
fmt.Println(buff.String()) for pkg, funcs := range dslHelpers {
fmt.Printf("Found %d DSL Helper functions in package %s\n", len(funcs), pkg)
fmt.Printf("[+] Generating jsdoc for all functions\n\n")
resp, err := llm.CreateChatCompletion(context.TODO(), openai.ChatCompletionRequest{
Model: "gpt-4",
Messages: []openai.ChatCompletionMessage{
{Role: "system", Content: sysprompt},
{Role: "user", Content: strings.ReplaceAll(userPrompt, "{{source}}", buff.String())},
},
Temperature: 0.1,
})
if err != nil {
fmt.Println(err)
return
} }
if len(resp.Choices) == 0 {
fmt.Println("no choices returned")
return
}
data := resp.Choices[0].Message.Content
fmt.Println(data)
// Generate Markdown tables with ## as package name
if out != "" { if out != "" {
if err := os.WriteFile(out, []byte(data), 0600); err != nil { var sb strings.Builder
sb.WriteString(`---
title: "Javascript Helper Functions"
description: "Available JS Helper Functions that can be used in global js runtime & protocol specific helpers."
icon: "function"
iconType: "solid"
---
`)
actualKeys := maps.Keys(dslHelpers)
sort.Slice(actualKeys, func(i, j int) bool {
for _, preferredKey := range preferredOrder {
if actualKeys[i] == preferredKey {
return true
}
if actualKeys[j] == preferredKey {
return false
}
}
return actualKeys[i] < actualKeys[j]
})
for _, v := range actualKeys {
pkg := v
funcs := dslHelpers[pkg]
sb.WriteString("## " + pkg + "\n\n")
sb.WriteString("| Name | Description | Signatures |\n")
sb.WriteString("|------|-------------|------------|\n")
for _, f := range funcs {
sigSlice := []string{}
for _, sig := range f.Signatures {
sigSlice = append(sigSlice, "`"+sig+"`")
}
sb.WriteString(fmt.Sprintf("| %s | %s | %s |\n", f.Name, f.Description, strings.Join(sigSlice, ", ")))
}
sb.WriteString("\n")
}
if err := os.WriteFile(out, []byte(sb.String()), 0644); err != nil {
fmt.Println(err) fmt.Println(err)
return return
} }

View File

@ -38,3 +38,40 @@ func registerAdditionalHelpers(runtime *goja.Runtime) {
}, },
}) })
} }
func init() {
// these are dummy functions we use trigger documentation generation
// actual definations are in exports.js
_ = gojs.RegisterFuncWithSignature(nil, gojs.FuncOpts{
Name: "to_json",
Signatures: []string{
"to_json(any) object",
},
Description: "Converts a given object to JSON",
})
_ = gojs.RegisterFuncWithSignature(nil, gojs.FuncOpts{
Name: "dump_json",
Signatures: []string{
"dump_json(any)",
},
Description: "Prints a given object as JSON in console",
})
_ = gojs.RegisterFuncWithSignature(nil, gojs.FuncOpts{
Name: "to_array",
Signatures: []string{
"to_array(any) array",
},
Description: "Sets/Updates objects prototype to array to enable Array.XXX functions",
})
_ = gojs.RegisterFuncWithSignature(nil, gojs.FuncOpts{
Name: "hex_to_ascii",
Signatures: []string{
"hex_to_ascii(string) string",
},
Description: "Converts a given hex string to ascii",
})
}

View File

@ -7,6 +7,7 @@ import (
var ( var (
ErrInvalidFuncOpts = errorutil.NewWithFmt("invalid function options: %v") ErrInvalidFuncOpts = errorutil.NewWithFmt("invalid function options: %v")
ErrNilRuntime = errorutil.New("runtime is nil")
) )
type FuncOpts struct { type FuncOpts struct {
@ -23,6 +24,9 @@ func (f *FuncOpts) valid() bool {
// RegisterFunc registers a function with given name, signatures and description // RegisterFunc registers a function with given name, signatures and description
func RegisterFuncWithSignature(runtime *goja.Runtime, opts FuncOpts) error { func RegisterFuncWithSignature(runtime *goja.Runtime, opts FuncOpts) error {
if runtime == nil {
return ErrNilRuntime
}
if !opts.valid() { if !opts.valid() {
return ErrInvalidFuncOpts.Msgf("name: %s, signatures: %v, description: %s", opts.Name, opts.Signatures, opts.Description) return ErrInvalidFuncOpts.Msgf("name: %s, signatures: %v, description: %s", opts.Name, opts.Signatures, opts.Description)
} }

View File

@ -8,11 +8,15 @@ import (
"strings" "strings"
"time" "time"
"github.com/alecthomas/chroma/quick"
"github.com/ditashi/jsbeautifier-go/jsbeautifier"
"github.com/dop251/goja"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/gozero" "github.com/projectdiscovery/gozero"
gozerotypes "github.com/projectdiscovery/gozero/types" gozerotypes "github.com/projectdiscovery/gozero/types"
"github.com/projectdiscovery/nuclei/v3/pkg/js/compiler"
"github.com/projectdiscovery/nuclei/v3/pkg/operators" "github.com/projectdiscovery/nuclei/v3/pkg/operators"
"github.com/projectdiscovery/nuclei/v3/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v3/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers"
@ -52,6 +56,9 @@ type Request struct {
// Engine type // Engine type
Engine []string `yaml:"engine,omitempty" jsonschema:"title=engine,description=Engine"` Engine []string `yaml:"engine,omitempty" jsonschema:"title=engine,description=Engine"`
// description: | // description: |
// PreCondition is a condition which is evaluated before sending the request.
PreCondition string `yaml:"pre-condition,omitempty" json:"pre-condition,omitempty" jsonschema:"title=pre-condition for the request,description=PreCondition is a condition which is evaluated before sending the request"`
// description: |
// Engine Arguments // Engine Arguments
Args []string `yaml:"args,omitempty" jsonschema:"title=args,description=Args"` Args []string `yaml:"args,omitempty" jsonschema:"title=args,description=Args"`
// description: | // description: |
@ -61,9 +68,10 @@ type Request struct {
// Source File/Snippet // Source File/Snippet
Source string `yaml:"source,omitempty" jsonschema:"title=source file/snippet,description=Source snippet"` Source string `yaml:"source,omitempty" jsonschema:"title=source file/snippet,description=Source snippet"`
options *protocols.ExecutorOptions options *protocols.ExecutorOptions `yaml:"-" json:"-"`
gozero *gozero.Gozero preConditionCompiled *goja.Program `yaml:"-" json:"-"`
src *gozero.Source gozero *gozero.Gozero `yaml:"-" json:"-"`
src *gozero.Source `yaml:"-" json:"-"`
} }
// Compile compiles the request generators preparing any requests possible. // Compile compiles the request generators preparing any requests possible.
@ -110,6 +118,15 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
} }
request.CompiledOperators = compiled request.CompiledOperators = compiled
} }
// compile pre-condition if any
if request.PreCondition != "" {
preConditionCompiled, err := compiler.WrapScriptNCompile(request.PreCondition, false)
if err != nil {
return errorutil.NewWithTag(request.TemplateID, "could not compile pre-condition: %s", err)
}
request.preConditionCompiled = preConditionCompiled
}
return nil return nil
} }
@ -130,11 +147,6 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
return err return err
} }
defer func() { defer func() {
// catch any panics just in case
if r := recover(); r != nil {
gologger.Error().Msgf("[%s] Panic occurred in code protocol: %s\n", request.options.TemplateID, r)
err = fmt.Errorf("panic occurred: %s", r)
}
if err := metaSrc.Cleanup(); err != nil { if err := metaSrc.Cleanup(); err != nil {
gologger.Warning().Msgf("%s\n", err) gologger.Warning().Msgf("%s\n", err)
} }
@ -160,7 +172,45 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
allvars[name] = v allvars[name] = v
metaSrc.AddVariable(gozerotypes.Variable{Name: name, Value: v}) metaSrc.AddVariable(gozerotypes.Variable{Name: name, Value: v})
} }
// set timeout using multiplier
timeout := TimeoutMultiplier * request.options.Options.Timeout timeout := TimeoutMultiplier * request.options.Options.Timeout
if request.PreCondition != "" {
if request.options.Options.Debug || request.options.Options.DebugRequests {
gologger.Debug().Msgf("[%s] Executing Precondition for Code request\n", request.TemplateID)
var highlightFormatter = "terminal256"
if request.options.Options.NoColor {
highlightFormatter = "text"
}
var buff bytes.Buffer
_ = quick.Highlight(&buff, beautifyJavascript(request.PreCondition), "javascript", highlightFormatter, "monokai")
prettyPrint(request.TemplateID, buff.String())
}
args := compiler.NewExecuteArgs()
args.TemplateCtx = allvars
result, err := request.options.JsCompiler.ExecuteWithOptions(request.preConditionCompiled, args,
&compiler.ExecuteOptions{
Timeout: timeout,
Source: &request.PreCondition,
Callback: registerPreConditionFunctions,
Cleanup: cleanUpPreConditionFunctions,
})
if err != nil {
return errorutil.NewWithTag(request.TemplateID, "could not execute pre-condition: %s", err)
}
if !result.GetSuccess() || types.ToString(result["error"]) != "" {
gologger.Warning().Msgf("[%s] Precondition for request %s was not satisfied\n", request.TemplateID, request.PreCondition)
request.options.Progress.IncrementFailedRequestsBy(1)
return nil
}
if request.options.Options.Debug || request.options.Options.DebugRequests {
gologger.Debug().Msgf("[%s] Precondition for request was satisfied\n", request.TemplateID)
}
}
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second) ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second)
defer cancel() defer cancel()
// Note: we use contextutil despite the fact that gozero accepts context as argument // Note: we use contextutil despite the fact that gozero accepts context as argument
@ -336,3 +386,23 @@ func interpretEnvVars(source string, vars map[string]interface{}) string {
} }
return source return source
} }
func beautifyJavascript(code string) string {
opts := jsbeautifier.DefaultOptions()
beautified, err := jsbeautifier.Beautify(&code, opts)
if err != nil {
return code
}
return beautified
}
func prettyPrint(templateId string, buff string) {
lines := strings.Split(buff, "\n")
final := []string{}
for _, v := range lines {
if v != "" {
final = append(final, "\t"+v)
}
}
gologger.Debug().Msgf(" [%v] Pre-condition Code:\n\n%v\n\n", templateId, strings.Join(final, "\n"))
}

View File

@ -0,0 +1,272 @@
package code
import (
goruntime "runtime"
"github.com/dop251/goja"
"github.com/projectdiscovery/nuclei/v3/pkg/js/gojs"
osutils "github.com/projectdiscovery/utils/os"
)
// registerPreConditionFunctions registers the pre-condition functions
func registerPreConditionFunctions(runtime *goja.Runtime) error {
// Note: the only reason we are not using forloop to generate these functions is because
// 'scrapefuncs' uses this function to find all dsl helper functions and document them.
// === OS ===
err := gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{
Name: "OS",
Signatures: []string{
"OS() string",
},
Description: "OS returns the current OS",
FuncDecl: func() string {
return goruntime.GOOS
},
})
if err != nil {
return err
}
// IsLinux checks if the current OS is Linux
err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{
Name: "IsLinux",
Signatures: []string{
"IsLinux() bool",
},
Description: "IsLinux checks if the current OS is Linux",
FuncDecl: func() bool {
return osutils.IsLinux()
},
})
if err != nil {
return err
}
// IsWindows checks if the current OS is Windows
err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{
Name: "IsWindows",
Signatures: []string{
"IsWindows() bool",
},
Description: "IsWindows checks if the current OS is Windows",
FuncDecl: func() bool {
return osutils.IsWindows()
},
})
if err != nil {
return err
}
// IsOSX checks if the current OS is OSX
err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{
Name: "IsOSX",
Signatures: []string{
"IsOSX() bool",
},
Description: "IsOSX checks if the current OS is OSX",
FuncDecl: func() bool {
return osutils.IsOSX()
},
})
if err != nil {
return err
}
// IsAndroid checks if the current OS is Android
err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{
Name: "IsAndroid",
Signatures: []string{
"IsAndroid() bool",
},
Description: "IsAndroid checks if the current OS is Android",
FuncDecl: func() bool {
return osutils.IsAndroid()
},
})
if err != nil {
return err
}
// IsIOS checks if the current OS is IOS
err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{
Name: "IsIOS",
Signatures: []string{
"IsIOS() bool",
},
Description: "IsIOS checks if the current OS is IOS",
FuncDecl: func() bool {
return osutils.IsIOS()
},
})
if err != nil {
return err
}
// IsJS checks if the current OS is JS
err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{
Name: "IsJS",
Signatures: []string{
"IsJS() bool",
},
Description: "IsJS checks if the current OS is JS",
FuncDecl: func() bool {
return osutils.IsJS()
},
})
if err != nil {
return err
}
// IsFreeBSD checks if the current OS is FreeBSD
err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{
Name: "IsFreeBSD",
Signatures: []string{
"IsFreeBSD() bool",
},
Description: "IsFreeBSD checks if the current OS is FreeBSD",
FuncDecl: func() bool {
return osutils.IsFreeBSD()
},
})
if err != nil {
return err
}
// IsOpenBSD checks if the current OS is OpenBSD
err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{
Name: "IsOpenBSD",
Signatures: []string{
"IsOpenBSD() bool",
},
Description: "IsOpenBSD checks if the current OS is OpenBSD",
FuncDecl: func() bool {
return osutils.IsOpenBSD()
},
})
if err != nil {
return err
}
// IsSolaris checks if the current OS is Solaris
err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{
Name: "IsSolaris",
Signatures: []string{
"IsSolaris() bool",
},
Description: "IsSolaris checks if the current OS is Solaris",
FuncDecl: func() bool {
return osutils.IsSolaris()
},
})
if err != nil {
return err
}
// === Arch ===
err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{
Name: "Arch",
Signatures: []string{
"Arch() string",
},
Description: "Arch returns the current architecture",
FuncDecl: func() string {
return goruntime.GOARCH
},
})
if err != nil {
return err
}
err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{
Name: "Is386",
Signatures: []string{
"Is386() bool",
},
Description: "Is386 checks if the current architecture is 386",
FuncDecl: func() bool {
return osutils.Is386()
},
})
if err != nil {
return err
}
err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{
Name: "IsAmd64",
Signatures: []string{
"IsAmd64() bool",
},
Description: "IsAmd64 checks if the current architecture is Amd64",
FuncDecl: func() bool {
return osutils.IsAmd64()
},
})
if err != nil {
return err
}
err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{
Name: "IsARM",
Signatures: []string{
"IsARM() bool",
},
Description: "IsArm checks if the current architecture is Arm",
FuncDecl: func() bool {
return osutils.IsARM()
},
})
if err != nil {
return err
}
err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{
Name: "IsARM64",
Signatures: []string{
"IsARM64() bool",
},
Description: "IsArm64 checks if the current architecture is Arm64",
FuncDecl: func() bool {
return osutils.IsARM64()
},
})
if err != nil {
return err
}
err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{
Name: "IsWasm",
Signatures: []string{
"IsWasm() bool",
},
Description: "IsWasm checks if the current architecture is Wasm",
FuncDecl: func() bool {
return osutils.IsWasm()
},
})
if err != nil {
return err
}
return nil
}
func cleanUpPreConditionFunctions(runtime *goja.Runtime) {
_ = runtime.GlobalObject().Delete("OS")
_ = runtime.GlobalObject().Delete("IsLinux")
_ = runtime.GlobalObject().Delete("IsWindows")
_ = runtime.GlobalObject().Delete("IsOSX")
_ = runtime.GlobalObject().Delete("IsAndroid")
_ = runtime.GlobalObject().Delete("IsIOS")
_ = runtime.GlobalObject().Delete("IsJS")
_ = runtime.GlobalObject().Delete("IsFreeBSD")
_ = runtime.GlobalObject().Delete("IsOpenBSD")
_ = runtime.GlobalObject().Delete("IsSolaris")
_ = runtime.GlobalObject().Delete("Arch")
_ = runtime.GlobalObject().Delete("Is386")
_ = runtime.GlobalObject().Delete("IsAmd64")
_ = runtime.GlobalObject().Delete("IsARM")
_ = runtime.GlobalObject().Delete("IsARM64")
_ = runtime.GlobalObject().Delete("IsWasm")
}

View File

@ -211,13 +211,12 @@ type Request struct {
// DisablePathAutomerge disables merging target url path with raw request path // DisablePathAutomerge disables merging target url path with raw request path
DisablePathAutomerge bool `yaml:"disable-path-automerge,omitempty" json:"disable-path-automerge,omitempty" jsonschema:"title=disable auto merging of path,description=Disable merging target url path with raw request path"` DisablePathAutomerge bool `yaml:"disable-path-automerge,omitempty" json:"disable-path-automerge,omitempty" jsonschema:"title=disable auto merging of path,description=Disable merging target url path with raw request path"`
// description: | // description: |
// Filter is matcher-like field to check if fuzzing should be performed on this request or not // Fuzz PreCondition is matcher-like field to check if fuzzing should be performed on this request or not
FuzzingFilter []*matchers.Matcher `yaml:"filters,omitempty" json:"filter,omitempty" jsonschema:"title=filter for fuzzing,description=Filter is matcher-like field to check if fuzzing should be performed on this request or not"` FuzzPreCondition []*matchers.Matcher `yaml:"pre-condition,omitempty" json:"pre-condition,omitempty" jsonschema:"title=pre-condition for fuzzing/dast,description=PreCondition is matcher-like field to check if fuzzing should be performed on this request or not"`
// description: | // description: |
// Filter condition is the condition to apply on the filter (AND/OR). Default is OR // FuzzPreConditionOperator is the operator between multiple PreConditions for fuzzing Default is OR
FuzzingFilterCondition string `yaml:"filters-condition,omitempty" json:"filter-condition,omitempty" jsonschema:"title=condition between the filters,description=Conditions between the filters,enum=and,enum=or"` FuzzPreConditionOperator string `yaml:"pre-condition-operator,omitempty" json:"pre-condition-operator,omitempty" jsonschema:"title=condition between the filters,description=Operator to use between multiple per-conditions,enum=and,enum=or"`
// cached variables that may be used along with request. fuzzPreConditionOperator matchers.ConditionType `yaml:"-" json:"-"`
fuzzingFilterCondition matchers.ConditionType `yaml:"-" json:"-"`
} }
// Options returns executer options for http request // Options returns executer options for http request
@ -326,13 +325,13 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
// === fuzzing filters ===== // // === fuzzing filters ===== //
if request.FuzzingFilterCondition != "" { if request.FuzzPreConditionOperator != "" {
request.fuzzingFilterCondition = matchers.ConditionTypes[request.FuzzingFilterCondition] request.fuzzPreConditionOperator = matchers.ConditionTypes[request.FuzzPreConditionOperator]
} else { } else {
request.fuzzingFilterCondition = matchers.ORCondition request.fuzzPreConditionOperator = matchers.ORCondition
} }
for _, filter := range request.FuzzingFilter { for _, filter := range request.FuzzPreCondition {
if err := filter.CompileMatchers(); err != nil { if err := filter.CompileMatchers(); err != nil {
return errors.Wrap(err, "could not compile matcher") return errors.Wrap(err, "could not compile matcher")
} }

View File

@ -137,11 +137,6 @@ func (request *Request) executeRaceRequest(input *contextargs.Context, previous
// execute http request // execute http request
go func(httpRequest *generatedRequest) { go func(httpRequest *generatedRequest) {
defer spmHandler.Release() defer spmHandler.Release()
defer func() {
if r := recover(); r != nil {
gologger.Verbose().Msgf("[%s] Recovered from panic: %v\n", request.options.TemplateID, r)
}
}()
if spmHandler.FoundFirstMatch() { if spmHandler.FoundFirstMatch() {
// stop sending more requests condition is met // stop sending more requests condition is met
return return
@ -218,11 +213,6 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
spmHandler.Acquire() spmHandler.Acquire()
go func(httpRequest *generatedRequest) { go func(httpRequest *generatedRequest) {
defer spmHandler.Release() defer spmHandler.Release()
defer func() {
if r := recover(); r != nil {
gologger.Verbose().Msgf("[%s] Recovered from panic: %v\n", request.options.TemplateID, r)
}
}()
if spmHandler.FoundFirstMatch() { if spmHandler.FoundFirstMatch() {
return return
} }
@ -319,11 +309,6 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
spmHandler.Acquire() spmHandler.Acquire()
go func(httpRequest *generatedRequest) { go func(httpRequest *generatedRequest) {
defer spmHandler.Release() defer spmHandler.Release()
defer func() {
if r := recover(); r != nil {
gologger.Verbose().Msgf("[%s] Recovered from panic: %v\n", request.options.TemplateID, r)
}
}()
if spmHandler.FoundFirstMatch() { if spmHandler.FoundFirstMatch() {
// skip if first match is found // skip if first match is found
return return

View File

@ -194,11 +194,11 @@ func (request *Request) executeGeneratedFuzzingRequest(gr fuzz.GeneratedRequest,
// ShouldFuzzTarget checks if given target should be fuzzed or not using `filter` field in template // ShouldFuzzTarget checks if given target should be fuzzed or not using `filter` field in template
func (request *Request) ShouldFuzzTarget(input *contextargs.Context) bool { func (request *Request) ShouldFuzzTarget(input *contextargs.Context) bool {
if len(request.FuzzingFilter) == 0 { if len(request.FuzzPreCondition) == 0 {
return true return true
} }
status := []bool{} status := []bool{}
for index, filter := range request.FuzzingFilter { for index, filter := range request.FuzzPreCondition {
isMatch, _ := request.Match(request.filterDataMap(input), filter) isMatch, _ := request.Match(request.filterDataMap(input), filter)
status = append(status, isMatch) status = append(status, isMatch)
if request.options.Options.MatcherStatus { if request.options.Options.MatcherStatus {
@ -209,7 +209,7 @@ func (request *Request) ShouldFuzzTarget(input *contextargs.Context) bool {
return true return true
} }
var matched bool var matched bool
if request.fuzzingFilterCondition == matchers.ANDCondition { if request.fuzzPreConditionOperator == matchers.ANDCondition {
matched = operators.EvalBoolSlice(status, true) matched = operators.EvalBoolSlice(status, true)
} else { } else {
matched = operators.EvalBoolSlice(status, false) matched = operators.EvalBoolSlice(status, false)

View File

@ -35,7 +35,8 @@ var (
) )
const ( const (
Unsigned = "unsigned" Unsigned = "unsigned"
PDVerifier = "projectdiscovery/nuclei-templates"
) )
func init() { func init() {
@ -272,7 +273,6 @@ func ParseTemplateFromReader(reader io.Reader, preprocessor Preprocessor, option
if config.DefaultConfig.LogAllEvents { if config.DefaultConfig.LogAllEvents {
gologger.DefaultLogger.Print().Msgf("[%v] Template %s is not signed or tampered\n", aurora.Yellow("WRN").String(), template.ID) gologger.DefaultLogger.Print().Msgf("[%v] Template %s is not signed or tampered\n", aurora.Yellow("WRN").String(), template.ID)
} }
SignatureStats[Unsigned].Add(1)
} }
return template, nil return template, nil
} }
@ -293,7 +293,6 @@ func ParseTemplateFromReader(reader io.Reader, preprocessor Preprocessor, option
if config.DefaultConfig.LogAllEvents { if config.DefaultConfig.LogAllEvents {
gologger.DefaultLogger.Print().Msgf("[%v] Template %s is not signed or tampered\n", aurora.Yellow("WRN").String(), template.ID) gologger.DefaultLogger.Print().Msgf("[%v] Template %s is not signed or tampered\n", aurora.Yellow("WRN").String(), template.ID)
} }
SignatureStats[Unsigned].Add(1)
} }
generatedConstants := map[string]interface{}{} generatedConstants := map[string]interface{}{}
@ -399,7 +398,7 @@ func parseTemplate(data []byte, options protocols.ExecutorOptions) (*Template, e
for _, verifier = range signer.DefaultTemplateVerifiers { for _, verifier = range signer.DefaultTemplateVerifiers {
template.Verified, _ = verifier.Verify(data, template) template.Verified, _ = verifier.Verify(data, template)
if template.Verified { if template.Verified {
SignatureStats[verifier.Identifier()].Add(1) template.TemplateVerifier = verifier.Identifier()
break break
} }
} }

View File

@ -154,7 +154,8 @@ type Template struct {
// Verified defines if the template signature is digitally verified // Verified defines if the template signature is digitally verified
Verified bool `yaml:"-" json:"-"` Verified bool `yaml:"-" json:"-"`
// TemplateVerifier is identifier verifier used to verify the template (default nuclei-templates have projectdiscovery/nuclei-templates)
TemplateVerifier string `yaml:"-" json:"-"`
// RequestsQueue contains all template requests in order (both protocol & request order) // RequestsQueue contains all template requests in order (both protocol & request order)
RequestsQueue []protocols.Request `yaml:"-" json:"-"` RequestsQueue []protocols.Request `yaml:"-" json:"-"`

View File

@ -101,6 +101,17 @@ func parseWorkflowTemplate(workflow *workflows.WorkflowTemplate, preprocessor Pr
continue continue
} }
} }
// increment signed/unsigned counters
if template.Verified {
if template.TemplateVerifier == "" {
SignatureStats[PDVerifier].Add(1)
} else {
SignatureStats[template.TemplateVerifier].Add(1)
}
} else {
SignatureStats[Unsigned].Add(1)
}
workflowTemplates = append(workflowTemplates, template) workflowTemplates = append(workflowTemplates, template)
} }

View File

@ -3,7 +3,6 @@ package tmplexec
import ( import (
"errors" "errors"
"fmt" "fmt"
"runtime/debug"
"strings" "strings"
"sync/atomic" "sync/atomic"
@ -99,14 +98,6 @@ func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) {
// since it is of no use after scan is completed (regardless of success or failure) // since it is of no use after scan is completed (regardless of success or failure)
e.options.RemoveTemplateCtx(ctx.Input.MetaInput) e.options.RemoveTemplateCtx(ctx.Input.MetaInput)
}() }()
defer func() {
// try catching unknown panics
if r := recover(); r != nil {
stacktrace := debug.Stack()
ctx.LogError(fmt.Errorf("panic: %v\n%s", r, stacktrace))
gologger.Verbose().Msgf("panic: %v\n%s", r, stacktrace)
}
}()
var lastMatcherEvent *output.InternalWrappedEvent var lastMatcherEvent *output.InternalWrappedEvent
writeFailureCallback := func(event *output.InternalWrappedEvent, matcherStatus bool) { writeFailureCallback := func(event *output.InternalWrappedEvent, matcherStatus bool) {

View File

@ -8,7 +8,6 @@ import (
"sync/atomic" "sync/atomic"
"github.com/dop251/goja" "github.com/dop251/goja"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols" "github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v3/pkg/scan" "github.com/projectdiscovery/nuclei/v3/pkg/scan"
@ -175,13 +174,6 @@ func (f *FlowExecutor) Compile() error {
// ExecuteWithResults executes the flow and returns results // ExecuteWithResults executes the flow and returns results
func (f *FlowExecutor) ExecuteWithResults(ctx *scan.ScanContext) error { func (f *FlowExecutor) ExecuteWithResults(ctx *scan.ScanContext) error {
defer func() {
if e := recover(); e != nil {
f.ctx.LogError(fmt.Errorf("panic occurred while executing target %v with flow: %v", ctx.Input.MetaInput.Input, e))
gologger.Error().Label(f.options.TemplateID).Msgf("panic occurred while executing target %v with flow: %v", ctx.Input.MetaInput.Input, e)
}
}()
f.ctx.Input = ctx.Input f.ctx.Input = ctx.Input
// -----Load all types of variables----- // -----Load all types of variables-----
// add all input args to template context // add all input args to template context

View File

@ -96,10 +96,12 @@ func (f *FlowExecutor) protocolResultCallback(req protocols.Request, matcherStat
if len(v) == 1 { if len(v) == 1 {
// add it to flatten keys list so it will be flattened to a string later // add it to flatten keys list so it will be flattened to a string later
f.flattenKeys = append(f.flattenKeys, k) f.flattenKeys = append(f.flattenKeys, k)
// flatten and convert it to string
f.options.GetTemplateCtx(f.ctx.Input.MetaInput).Set(k, v[0])
} else {
// keep it as slice
f.options.GetTemplateCtx(f.ctx.Input.MetaInput).Set(k, v)
} }
// always preserve extracted value type
f.options.GetTemplateCtx(f.ctx.Input.MetaInput).Set(k, v)
} }
} }
} else if !result.HasOperatorResult() && !hasOperators(req.GetCompiledOperators()) { } else if !result.HasOperatorResult() && !hasOperators(req.GetCompiledOperators()) {

View File

@ -7,6 +7,7 @@ import (
"github.com/dop251/goja" "github.com/dop251/goja"
"github.com/logrusorgru/aurora" "github.com/logrusorgru/aurora"
"github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v3/pkg/js/gojs"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
"github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/flow/builtin" "github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/flow/builtin"
@ -49,46 +50,68 @@ var gojapool = &sync.Pool{
} }
func registerBuiltins(runtime *goja.Runtime) { func registerBuiltins(runtime *goja.Runtime) {
_ = runtime.Set("log", func(call goja.FunctionCall) goja.Value {
// TODO: verify string interpolation and handle multiple args _ = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{
arg := call.Argument(0).Export() Name: "log",
switch value := arg.(type) { Description: "Logs a given object/message to stdout (only for debugging purposes)",
case string: Signatures: []string{
gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), value) "log(obj any) any",
case map[string]interface{}: },
gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), vardump.DumpVariables(value)) FuncDecl: func(call goja.FunctionCall) goja.Value {
default: arg := call.Argument(0).Export()
gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), value) switch value := arg.(type) {
} case string:
return call.Argument(0) // return the same value gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), value)
case map[string]interface{}:
gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), vardump.DumpVariables(value))
default:
gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), value)
}
return call.Argument(0) // return the same value
},
}) })
_ = runtime.Set("iterate", func(call goja.FunctionCall) goja.Value { _ = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{
allVars := []any{} Name: "iterate",
for _, v := range call.Arguments { Description: "Normalizes and Iterates over all arguments (can be a string,array,null etc) and returns an array of objects\nNote: If the object type is unknown(i.e could be a string or array) iterate should be used and it will always return an array of strings",
if v.Export() == nil { Signatures: []string{
continue "iterate(...any) []any",
} },
if v.ExportType().Kind() == reflect.Slice { FuncDecl: func(call goja.FunctionCall) goja.Value {
// convert []datatype to []interface{} allVars := []any{}
// since it cannot be type asserted to []interface{} directly for _, v := range call.Arguments {
rfValue := reflect.ValueOf(v.Export()) if v.Export() == nil {
for i := 0; i < rfValue.Len(); i++ { continue
allVars = append(allVars, rfValue.Index(i).Interface()) }
if v.ExportType().Kind() == reflect.Slice {
// convert []datatype to []interface{}
// since it cannot be type asserted to []interface{} directly
rfValue := reflect.ValueOf(v.Export())
for i := 0; i < rfValue.Len(); i++ {
allVars = append(allVars, rfValue.Index(i).Interface())
}
} else {
allVars = append(allVars, v.Export())
} }
} else {
allVars = append(allVars, v.Export())
} }
} return runtime.ToValue(allVars)
return runtime.ToValue(allVars) },
}) })
_ = runtime.Set("Dedupe", func(call goja.ConstructorCall) *goja.Object { _ = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{
d := builtin.NewDedupe(runtime) Name: "Dedupe",
obj := call.This Description: "De-duplicates given values and returns a new array of unique values",
// register these methods Signatures: []string{
_ = obj.Set("Add", d.Add) "new Dedupe()",
_ = obj.Set("Values", d.Values) },
return nil FuncDecl: func(call goja.ConstructorCall) *goja.Object {
d := builtin.NewDedupe(runtime)
obj := call.This
// register these methods
_ = obj.Set("Add", d.Add)
_ = obj.Set("Values", d.Values)
return nil
},
}) })
} }