mirror of https://github.com/daffainfo/nuclei.git
support for dynamic variables in template context (multi protocol execution) (#3672)
* multi proto request genesis * adds template context dynamic vars * feat: proto level resp variables * remove proto prefix hacky logic * implement template ctx args * remove old var name logic * improve AddTemplateVars func * add multi proto comments+docs * vardump with sorted keys * fix race condition in ctx args * default initialize ctx args * use generic map * index variables with multiple values * fix nil cookies * use synclock map * fix build failure * fix lint error * resolve merge conflicts * multi proto: add unit+ integration tests * fix unit tests * Issue 3339 headless fuzz (#3790) * Basic headless fuzzing * Remove debug statements * Add integration tests * Update template * Fix recognize payload value in matcher * Update tempalte * use req.SetURL() --------- Co-authored-by: Tarun Koyalwar <tarun@projectdiscovery.io> * Auto Generate Syntax Docs + JSONSchema [Fri Jun 9 00:23:32 UTC 2023] 🤖 * Add headless header and status matchers (#3794) * add headless header and status matchers * rename headers as header * add integration test for header+status * fix typo --------- Co-authored-by: Shubham Rasal <shubham@projectdiscovery.io> Co-authored-by: GitHub Action <action@github.com> Co-authored-by: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com>dev
parent
b4e4715d36
commit
e1d3f474a4
|
@ -1585,6 +1585,8 @@ Appears in:
|
|||
|
||||
- <code><a href="#httprequest">http.Request</a>.fuzzing</code>
|
||||
|
||||
- <code><a href="#headlessrequest">headless.Request</a>.fuzzing</code>
|
||||
|
||||
|
||||
|
||||
|
||||
|
@ -2717,6 +2719,19 @@ StopAtFirstMatch stops the execution of the requests and template as soon as a m
|
|||
|
||||
<hr />
|
||||
|
||||
<div class="dd">
|
||||
|
||||
<code>fuzzing</code> <i>[]<a href="#fuzzrule">fuzz.Rule</a></i>
|
||||
|
||||
</div>
|
||||
<div class="dt">
|
||||
|
||||
Fuzzing describes schema to fuzz headless requests
|
||||
|
||||
</div>
|
||||
|
||||
<hr />
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
|
@ -0,0 +1,31 @@
|
|||
id: headless-query-fuzzing
|
||||
|
||||
info:
|
||||
name: Example Query Fuzzing
|
||||
author: pdteam
|
||||
severity: info
|
||||
|
||||
headless:
|
||||
- steps:
|
||||
- action: navigate
|
||||
args:
|
||||
url: "{{BaseURL}}"
|
||||
- action: waitload
|
||||
|
||||
payloads:
|
||||
redirect:
|
||||
- "blog.com"
|
||||
- "portal.com"
|
||||
|
||||
fuzzing:
|
||||
- part: query
|
||||
mode: single
|
||||
type: replace
|
||||
fuzz:
|
||||
- "https://{{redirect}}"
|
||||
|
||||
matchers:
|
||||
- type: word
|
||||
part: body
|
||||
words:
|
||||
- "{{redirect}}"
|
|
@ -0,0 +1,24 @@
|
|||
id: headless-header-status-test
|
||||
|
||||
info:
|
||||
name: headless header + status test
|
||||
author: pdteam
|
||||
severity: info
|
||||
|
||||
headless:
|
||||
- steps:
|
||||
- args:
|
||||
url: "{{BaseURL}}"
|
||||
action: navigate
|
||||
- action: waitload
|
||||
|
||||
matchers-condition: and
|
||||
matchers:
|
||||
- type: word
|
||||
part: header
|
||||
words:
|
||||
- text/plain
|
||||
|
||||
- type: status
|
||||
status:
|
||||
- 200
|
|
@ -0,0 +1,29 @@
|
|||
id: dns-http-dynamic-values
|
||||
|
||||
info:
|
||||
name: multi protocol request with dynamic values
|
||||
author: pdteam
|
||||
severity: info
|
||||
|
||||
dns:
|
||||
- name: "{{FQDN}}" # DNS Request
|
||||
type: cname
|
||||
|
||||
extractors:
|
||||
- type: dsl
|
||||
name: blogid
|
||||
dsl:
|
||||
- trim_suffix(cname,'.ghost.io.')
|
||||
internal: true
|
||||
|
||||
http:
|
||||
- method: GET # http request
|
||||
path:
|
||||
- "{{BaseURL}}"
|
||||
|
||||
matchers:
|
||||
- type: dsl
|
||||
dsl:
|
||||
- contains(body,'ProjectDiscovery.io') # check for http string
|
||||
- blogid == 'projectdiscovery' # check for cname (extracted information from dns response)
|
||||
condition: and
|
|
@ -0,0 +1,30 @@
|
|||
id: dns-ssl-http-with-variables
|
||||
|
||||
info:
|
||||
name: multi protocol request with dynamic values
|
||||
author: pdteam
|
||||
severity: info
|
||||
|
||||
|
||||
variables:
|
||||
cname_filtered: '{{trim_suffix(dns_cname,".ghost.io.")}}'
|
||||
|
||||
dns:
|
||||
- name: "{{FQDN}}" # DNS Request
|
||||
type: cname
|
||||
|
||||
ssl:
|
||||
- address: "{{Hostname}}" # ssl request
|
||||
|
||||
http:
|
||||
- method: GET # http request
|
||||
path:
|
||||
- "{{BaseURL}}"
|
||||
|
||||
matchers:
|
||||
- type: dsl
|
||||
dsl:
|
||||
- contains(http_body,'ProjectDiscovery.io') # check for http string
|
||||
- cname_filtered == 'projectdiscovery' # check for cname (extracted information from dns response)
|
||||
- ssl_subject_cn == 'blog.projectdiscovery.io'
|
||||
condition: and
|
|
@ -0,0 +1,26 @@
|
|||
id: dns-ssl-http-proto-prefix
|
||||
|
||||
info:
|
||||
name: multi protocol request with dynamic values
|
||||
author: pdteam
|
||||
severity: info
|
||||
|
||||
dns:
|
||||
- name: "{{FQDN}}" # DNS Request
|
||||
type: cname
|
||||
|
||||
ssl:
|
||||
- address: "{{Hostname}}" # ssl request
|
||||
|
||||
http:
|
||||
- method: GET # http request
|
||||
path:
|
||||
- "{{BaseURL}}"
|
||||
|
||||
matchers:
|
||||
- type: dsl
|
||||
dsl:
|
||||
- contains(http_body,'ProjectDiscovery.io') # check for http string
|
||||
- trim_suffix(dns_cname,'.ghost.io.') == 'projectdiscovery' # check for cname (extracted information from dns response)
|
||||
- ssl_subject_cn == 'blog.projectdiscovery.io'
|
||||
condition: and
|
|
@ -372,6 +372,72 @@
|
|||
"title": "type of the matcher",
|
||||
"description": "Type of the matcher"
|
||||
},
|
||||
"fuzz.Rule": {
|
||||
"properties": {
|
||||
"type": {
|
||||
"enum": [
|
||||
"replace",
|
||||
"prefix",
|
||||
"postfix",
|
||||
"infix"
|
||||
],
|
||||
"type": "string",
|
||||
"title": "type of rule",
|
||||
"description": "Type of fuzzing rule to perform"
|
||||
},
|
||||
"part": {
|
||||
"enum": [
|
||||
"query"
|
||||
],
|
||||
"type": "string",
|
||||
"title": "part of rule",
|
||||
"description": "Part of request rule to fuzz"
|
||||
},
|
||||
"mode": {
|
||||
"enum": [
|
||||
"single",
|
||||
"multiple"
|
||||
],
|
||||
"type": "string",
|
||||
"title": "mode of rule",
|
||||
"description": "Mode of request rule to fuzz"
|
||||
},
|
||||
"keys": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array",
|
||||
"title": "keys of parameters to fuzz",
|
||||
"description": "Keys of parameters to fuzz"
|
||||
},
|
||||
"keys-regex": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array",
|
||||
"title": "keys regex to fuzz",
|
||||
"description": "Regex of parameter keys to fuzz"
|
||||
},
|
||||
"values": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array",
|
||||
"title": "values regex to fuzz",
|
||||
"description": "Regex of parameter values to fuzz"
|
||||
},
|
||||
"fuzz": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array",
|
||||
"title": "payloads of fuzz rule",
|
||||
"description": "Payloads to perform fuzzing substitutions with"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"type": "object"
|
||||
},
|
||||
"generators.AttackTypeHolder": {
|
||||
"enum": [
|
||||
"batteringram",
|
||||
|
@ -653,6 +719,14 @@
|
|||
"type": "string",
|
||||
"title": "condition between the matchers",
|
||||
"description": "Conditions between the matchers"
|
||||
},
|
||||
"fuzzing": {
|
||||
"items": {
|
||||
"$ref": "#/definitions/fuzz.Rule"
|
||||
},
|
||||
"type": "array",
|
||||
"title": "fuzzin rules for http fuzzing",
|
||||
"description": "Fuzzing describes rule schema to fuzz headless requests"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
|
@ -953,72 +1027,6 @@
|
|||
"title": "type of the signature",
|
||||
"description": "Type of the signature"
|
||||
},
|
||||
"fuzz.Rule": {
|
||||
"properties": {
|
||||
"type": {
|
||||
"enum": [
|
||||
"replace",
|
||||
"prefix",
|
||||
"postfix",
|
||||
"infix"
|
||||
],
|
||||
"type": "string",
|
||||
"title": "type of rule",
|
||||
"description": "Type of fuzzing rule to perform"
|
||||
},
|
||||
"part": {
|
||||
"enum": [
|
||||
"query"
|
||||
],
|
||||
"type": "string",
|
||||
"title": "part of rule",
|
||||
"description": "Part of request rule to fuzz"
|
||||
},
|
||||
"mode": {
|
||||
"enum": [
|
||||
"single",
|
||||
"multiple"
|
||||
],
|
||||
"type": "string",
|
||||
"title": "mode of rule",
|
||||
"description": "Mode of request rule to fuzz"
|
||||
},
|
||||
"keys": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array",
|
||||
"title": "keys of parameters to fuzz",
|
||||
"description": "Keys of parameters to fuzz"
|
||||
},
|
||||
"keys-regex": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array",
|
||||
"title": "keys regex to fuzz",
|
||||
"description": "Regex of parameter keys to fuzz"
|
||||
},
|
||||
"values": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array",
|
||||
"title": "values regex to fuzz",
|
||||
"description": "Regex of parameter values to fuzz"
|
||||
},
|
||||
"fuzz": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array",
|
||||
"title": "payloads of fuzz rule",
|
||||
"description": "Payloads to perform fuzzing substitutions with"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"type": "object"
|
||||
},
|
||||
"network.Input": {
|
||||
"properties": {
|
||||
"data": {
|
||||
|
|
|
@ -13,9 +13,10 @@ import (
|
|||
)
|
||||
|
||||
var fuzzingTestCases = map[string]testutils.TestCase{
|
||||
"fuzz/fuzz-mode.yaml": &fuzzModeOverride{},
|
||||
"fuzz/fuzz-type.yaml": &fuzzTypeOverride{},
|
||||
"fuzz/fuzz-query.yaml": &httpFuzzQuery{},
|
||||
"fuzz/fuzz-mode.yaml": &fuzzModeOverride{},
|
||||
"fuzz/fuzz-type.yaml": &fuzzTypeOverride{},
|
||||
"fuzz/fuzz-query.yaml": &httpFuzzQuery{},
|
||||
"fuzz/fuzz-headless.yaml": &HeadlessFuzzingQuery{},
|
||||
}
|
||||
|
||||
type httpFuzzQuery struct{}
|
||||
|
@ -126,3 +127,23 @@ func (h *fuzzTypeOverride) Execute(filePath string) error {
|
|||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// HeadlessFuzzingQuery tests fuzzing is working not in headless mode
|
||||
type HeadlessFuzzingQuery struct{}
|
||||
|
||||
// Execute executes a test case and returns an error if occurred
|
||||
func (h *HeadlessFuzzingQuery) Execute(filePath string) error {
|
||||
router := httprouter.New()
|
||||
router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
|
||||
resp := fmt.Sprintf("<html><body>%s</body></html>", r.URL.Query().Get("url"))
|
||||
fmt.Fprint(w, resp)
|
||||
})
|
||||
ts := httptest.NewTLSServer(router)
|
||||
defer ts.Close()
|
||||
|
||||
got, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL+"?url=https://scanme.sh", debug, "-headless")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return expectResultsCount(got, 2)
|
||||
}
|
||||
|
|
|
@ -11,12 +11,13 @@ import (
|
|||
)
|
||||
|
||||
var headlessTestcases = map[string]testutils.TestCase{
|
||||
"headless/headless-basic.yaml": &headlessBasic{},
|
||||
"headless/headless-header-action.yaml": &headlessHeaderActions{},
|
||||
"headless/headless-extract-values.yaml": &headlessExtractValues{},
|
||||
"headless/headless-payloads.yaml": &headlessPayloads{},
|
||||
"headless/variables.yaml": &headlessVariables{},
|
||||
"headless/file-upload.yaml": &headlessFileUpload{},
|
||||
"headless/headless-basic.yaml": &headlessBasic{},
|
||||
"headless/headless-header-action.yaml": &headlessHeaderActions{},
|
||||
"headless/headless-extract-values.yaml": &headlessExtractValues{},
|
||||
"headless/headless-payloads.yaml": &headlessPayloads{},
|
||||
"headless/variables.yaml": &headlessVariables{},
|
||||
"headless/file-upload.yaml": &headlessFileUpload{},
|
||||
"headless/headless-header-status-test.yaml": &headlessHeaderStatus{},
|
||||
}
|
||||
|
||||
type headlessBasic struct{}
|
||||
|
@ -158,3 +159,15 @@ func (h *headlessFileUpload) Execute(filePath string) error {
|
|||
|
||||
return expectResultsCount(results, 1)
|
||||
}
|
||||
|
||||
type headlessHeaderStatus struct{}
|
||||
|
||||
// Execute executes a test case and returns an error if occurred
|
||||
func (h *headlessHeaderStatus) Execute(filePath string) error {
|
||||
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "https://scanme.sh", debug, "-headless")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return expectResultsCount(results, 1)
|
||||
}
|
||||
|
|
|
@ -39,6 +39,7 @@ var (
|
|||
"offlineHttp": offlineHttpTestcases,
|
||||
"customConfigDir": customConfigDirTestCases,
|
||||
"fuzzing": fuzzingTestCases,
|
||||
"multi": multiProtoTestcases,
|
||||
}
|
||||
|
||||
// For debug purposes
|
||||
|
|
|
@ -0,0 +1,20 @@
|
|||
package main
|
||||
|
||||
import "github.com/projectdiscovery/nuclei/v2/pkg/testutils"
|
||||
|
||||
var multiProtoTestcases = map[string]testutils.TestCase{
|
||||
"multi/dynamic-values.yaml": &multiProtoDynamicExtractor{},
|
||||
"multi/evaluate-variables.yaml": &multiProtoDynamicExtractor{}, // Not a typo execution is same as above testcase
|
||||
"multi/exported-response-vars.yaml": &multiProtoDynamicExtractor{}, // Not a typo execution is same as above testcase
|
||||
}
|
||||
|
||||
type multiProtoDynamicExtractor struct{}
|
||||
|
||||
// Execute executes a test case and returns an error if occurred
|
||||
func (h *multiProtoDynamicExtractor) Execute(templatePath string) error {
|
||||
results, err := testutils.RunNucleiTemplateAndGetResults(templatePath, "blog.projectdiscovery.io", debug)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return expectResultsCount(results, 1)
|
||||
}
|
|
@ -38,7 +38,7 @@ func (matcher *Matcher) CompileMatchers() error {
|
|||
}
|
||||
|
||||
// By default, match on body if user hasn't provided any specific items
|
||||
if matcher.Part == "" {
|
||||
if matcher.Part == "" && matcher.GetType() != DSLMatcher {
|
||||
matcher.Part = "body"
|
||||
}
|
||||
|
||||
|
|
|
@ -1,31 +0,0 @@
|
|||
package contextargs
|
||||
|
||||
// Args is a generic map with helpers
|
||||
type Args map[string]interface{}
|
||||
|
||||
// Set a key with value
|
||||
func (args Args) Set(key string, value interface{}) {
|
||||
args[key] = value
|
||||
}
|
||||
|
||||
// Get the value associated to a key
|
||||
func (args Args) Get(key string) (interface{}, bool) {
|
||||
value, ok := args[key]
|
||||
return value, ok
|
||||
}
|
||||
|
||||
// Has verifies if the map contains the key
|
||||
func (args Args) Has(key string) bool {
|
||||
_, ok := args[key]
|
||||
return ok
|
||||
}
|
||||
|
||||
// IsEmpty verifies if the map is empty
|
||||
func (Args Args) IsEmpty() bool {
|
||||
return len(Args) == 0
|
||||
}
|
||||
|
||||
// create a new args map instance
|
||||
func newArgs() map[string]interface{} {
|
||||
return make(map[string]interface{})
|
||||
}
|
|
@ -2,9 +2,9 @@ package contextargs
|
|||
|
||||
import (
|
||||
"net/http/cookiejar"
|
||||
"sync"
|
||||
|
||||
"golang.org/x/exp/maps"
|
||||
"github.com/projectdiscovery/gologger"
|
||||
maputils "github.com/projectdiscovery/utils/maps"
|
||||
)
|
||||
|
||||
// Context implements a shared context struct to share information across multiple templates within a workflow
|
||||
|
@ -14,106 +14,68 @@ type Context struct {
|
|||
|
||||
// CookieJar shared within workflow's http templates
|
||||
CookieJar *cookiejar.Jar
|
||||
|
||||
// Access to Args must use lock strategies to prevent data races
|
||||
*sync.RWMutex
|
||||
// Args is a workflow shared key-value store
|
||||
args Args
|
||||
args maputils.SyncLockMap[string, interface{}]
|
||||
}
|
||||
|
||||
// Create a new contextargs instance
|
||||
func New() *Context {
|
||||
return &Context{MetaInput: &MetaInput{}}
|
||||
return NewWithInput("")
|
||||
}
|
||||
|
||||
// Create a new contextargs instance with input string
|
||||
func NewWithInput(input string) *Context {
|
||||
return &Context{MetaInput: &MetaInput{Input: input}}
|
||||
}
|
||||
|
||||
func (ctx *Context) initialize() {
|
||||
ctx.args = newArgs()
|
||||
ctx.RWMutex = &sync.RWMutex{}
|
||||
}
|
||||
|
||||
func (ctx *Context) set(key string, value interface{}) {
|
||||
ctx.Lock()
|
||||
defer ctx.Unlock()
|
||||
|
||||
ctx.args.Set(key, value)
|
||||
jar, err := cookiejar.New(nil)
|
||||
if err != nil {
|
||||
gologger.Error().Msgf("Could not create cookie jar: %s\n", err)
|
||||
}
|
||||
return &Context{MetaInput: &MetaInput{Input: input}, CookieJar: jar, args: maputils.SyncLockMap[string, interface{}]{
|
||||
Map: make(map[string]interface{}),
|
||||
}}
|
||||
}
|
||||
|
||||
// Set the specific key-value pair
|
||||
func (ctx *Context) Set(key string, value interface{}) {
|
||||
if !ctx.isInitialized() {
|
||||
ctx.initialize()
|
||||
if err := ctx.args.Set(key, value); err != nil {
|
||||
gologger.Error().Msgf("contextargs: could not set key: %s\n", err)
|
||||
}
|
||||
|
||||
ctx.set(key, value)
|
||||
}
|
||||
|
||||
func (ctx *Context) isInitialized() bool {
|
||||
return ctx.args != nil
|
||||
}
|
||||
|
||||
func (ctx *Context) hasArgs() bool {
|
||||
return ctx.isInitialized() && !ctx.args.IsEmpty()
|
||||
}
|
||||
|
||||
func (ctx *Context) get(key string) (interface{}, bool) {
|
||||
ctx.RLock()
|
||||
defer ctx.RUnlock()
|
||||
|
||||
return ctx.args.Get(key)
|
||||
}
|
||||
|
||||
// Get the value with specific key if exists
|
||||
func (ctx *Context) Get(key string) (interface{}, bool) {
|
||||
if !ctx.hasArgs() {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
return ctx.get(key)
|
||||
return ctx.args.Get(key)
|
||||
}
|
||||
|
||||
func (ctx *Context) GetAll() Args {
|
||||
if !ctx.hasArgs() {
|
||||
return nil
|
||||
}
|
||||
|
||||
return maps.Clone(ctx.args)
|
||||
func (ctx *Context) GetAll() maputils.Map[string, interface{}] {
|
||||
return ctx.args.GetAll()
|
||||
}
|
||||
|
||||
func (ctx *Context) ForEach(f func(string, interface{})) {
|
||||
ctx.RLock()
|
||||
defer ctx.RUnlock()
|
||||
|
||||
for k, v := range ctx.args {
|
||||
f(k, v)
|
||||
func (ctx *Context) ForEach(f func(string, interface{}) error) {
|
||||
if err := ctx.args.Iterate(f); err != nil {
|
||||
gologger.Error().Msgf("contextargs: could not iterate: %s\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (ctx *Context) has(key string) bool {
|
||||
ctx.RLock()
|
||||
defer ctx.RUnlock()
|
||||
|
||||
return ctx.args.Has(key)
|
||||
// Merge merges the map into the contextargs
|
||||
func (ctx *Context) Merge(m map[string]interface{}) {
|
||||
if err := ctx.args.Merge(m); err != nil {
|
||||
gologger.Error().Msgf("contextargs: could not merge: %s\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Has check if the key exists
|
||||
func (ctx *Context) Has(key string) bool {
|
||||
return ctx.hasArgs() && ctx.has(key)
|
||||
return ctx.args.Has(key)
|
||||
}
|
||||
|
||||
func (ctx *Context) HasArgs() bool {
|
||||
return ctx.hasArgs()
|
||||
return !ctx.args.IsEmpty()
|
||||
}
|
||||
|
||||
func (ctx *Context) Clone() *Context {
|
||||
newCtx := &Context{
|
||||
MetaInput: ctx.MetaInput.Clone(),
|
||||
RWMutex: ctx.RWMutex,
|
||||
args: ctx.args,
|
||||
args: *ctx.args.Clone(),
|
||||
CookieJar: ctx.CookieJar,
|
||||
}
|
||||
return newCtx
|
||||
|
|
|
@ -65,8 +65,9 @@ func (e *Executer) Execute(input *contextargs.Context) (bool, error) {
|
|||
|
||||
dynamicValues := make(map[string]interface{})
|
||||
if input.HasArgs() {
|
||||
input.ForEach(func(key string, value interface{}) {
|
||||
input.ForEach(func(key string, value interface{}) error {
|
||||
dynamicValues[key] = value
|
||||
return nil
|
||||
})
|
||||
}
|
||||
previous := make(map[string]interface{})
|
||||
|
@ -79,6 +80,10 @@ func (e *Executer) Execute(input *contextargs.Context) (bool, error) {
|
|||
}
|
||||
|
||||
err := req.ExecuteWithResults(inputItem, dynamicValues, previous, func(event *output.InternalWrappedEvent) {
|
||||
if event == nil {
|
||||
// ideally this should never happen since protocol exits on error and callback is not called
|
||||
return
|
||||
}
|
||||
ID := req.GetID()
|
||||
if ID != "" {
|
||||
builder := &strings.Builder{}
|
||||
|
@ -125,8 +130,9 @@ func (e *Executer) Execute(input *contextargs.Context) (bool, error) {
|
|||
func (e *Executer) ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error {
|
||||
dynamicValues := make(map[string]interface{})
|
||||
if input.HasArgs() {
|
||||
input.ForEach(func(key string, value interface{}) {
|
||||
input.ForEach(func(key string, value interface{}) error {
|
||||
dynamicValues[key] = value
|
||||
return nil
|
||||
})
|
||||
}
|
||||
previous := make(map[string]interface{})
|
||||
|
|
|
@ -73,10 +73,7 @@ func (rule *Rule) buildQueryInput(input *ExecuteRuleInput, parsed *urlutil.URL,
|
|||
req.Header.Set("User-Agent", uarand.GetRandom())
|
||||
} else {
|
||||
req = input.BaseRequest.Clone(context.TODO())
|
||||
//TODO: abstract below 3 lines with `req.UpdateURL(xx *urlutil.URL)`
|
||||
req.URL = parsed
|
||||
req.Request.URL = parsed.URL
|
||||
req.Update()
|
||||
req.SetURL(parsed)
|
||||
}
|
||||
request := GeneratedRequest{
|
||||
Request: req,
|
|
@ -5,6 +5,7 @@ import (
|
|||
"strings"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/types"
|
||||
mapsutil "github.com/projectdiscovery/utils/maps"
|
||||
)
|
||||
|
||||
// EnableVarDump enables var dump for debugging optionally
|
||||
|
@ -21,7 +22,11 @@ func DumpVariables(data map[string]interface{}) string {
|
|||
buffer.Grow(len(data) * 78) // grow buffer to an approximate size
|
||||
|
||||
builder := &strings.Builder{}
|
||||
for k, v := range data {
|
||||
// sort keys for deterministic output
|
||||
keys := mapsutil.GetSortedKeys(data)
|
||||
|
||||
for _, k := range keys {
|
||||
v := data[k]
|
||||
valueString := types.ToString(v)
|
||||
|
||||
counter++
|
||||
|
|
|
@ -53,7 +53,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
|
|||
// optionvars are vars passed from CLI or env variables
|
||||
optionVars := generators.BuildPayloadFromOptions(request.options.Options)
|
||||
// merge with metadata (eg. from workflow context)
|
||||
vars = generators.MergeMaps(vars, metadata, optionVars)
|
||||
vars = generators.MergeMaps(vars, metadata, optionVars, request.options.TemplateCtx.GetAll())
|
||||
variablesMap := request.options.Variables.Evaluate(vars)
|
||||
vars = generators.MergeMaps(vars, variablesMap, request.options.Constants)
|
||||
|
||||
|
@ -149,12 +149,17 @@ func (request *Request) execute(domain string, metadata, previous output.Interna
|
|||
|
||||
// Create the output event
|
||||
outputEvent := request.responseToDSLMap(compiledRequest, response, domain, question, traceData)
|
||||
// expose response variables in proto_var format
|
||||
// this is no-op if the template is not a multi protocol template
|
||||
request.options.AddTemplateVars(request.Type(), outputEvent)
|
||||
for k, v := range previous {
|
||||
outputEvent[k] = v
|
||||
}
|
||||
for k, v := range vars {
|
||||
outputEvent[k] = v
|
||||
}
|
||||
// add variables from template context before matching/extraction
|
||||
outputEvent = generators.MergeMaps(outputEvent, request.options.TemplateCtx.GetAll())
|
||||
event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse)
|
||||
|
||||
dumpResponse(event, request, request.options, response.String(), question)
|
||||
|
|
|
@ -19,6 +19,7 @@ import (
|
|||
"github.com/projectdiscovery/nuclei/v2/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
|
||||
templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
|
||||
|
@ -233,6 +234,8 @@ func (request *Request) findMatchesWithReader(reader io.Reader, input, filePath
|
|||
for k, v := range previous {
|
||||
dslMap[k] = v
|
||||
}
|
||||
// add template context variables to DSL map
|
||||
dslMap = generators.MergeMaps(dslMap, request.options.TemplateCtx.GetAll())
|
||||
discardEvent := eventcreator.CreateEvent(request, dslMap, isResponseDebug)
|
||||
newOpResult := discardEvent.OperatorsResult
|
||||
if newOpResult != nil {
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
package engine
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
|
@ -78,10 +79,19 @@ func (i *Instance) Run(baseURL *url.URL, actions []*Action, payloads map[string]
|
|||
return nil, nil, err
|
||||
}
|
||||
|
||||
//FIXME: this is a hack, make sure to fix this in the future. See: https://github.com/go-rod/rod/issues/188
|
||||
var e proto.NetworkResponseReceived
|
||||
wait := page.WaitEvent(&e)
|
||||
|
||||
data, err := createdPage.ExecuteActions(baseURL, actions)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
wait()
|
||||
data["header"] = headersToString(e.Response.Headers)
|
||||
data["status_code"] = fmt.Sprint(e.Response.Status)
|
||||
|
||||
return data, createdPage, nil
|
||||
}
|
||||
|
||||
|
@ -178,3 +188,15 @@ func containsAnyModificationActionType(actionTypes ...ActionType) bool {
|
|||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// headersToString converts network headers to string
|
||||
func headersToString(headers proto.NetworkHeaders) string {
|
||||
builder := &strings.Builder{}
|
||||
for header, value := range headers {
|
||||
builder.WriteString(header)
|
||||
builder.WriteString(": ")
|
||||
builder.WriteString(value.String())
|
||||
builder.WriteRune('\n')
|
||||
}
|
||||
return builder.String()
|
||||
}
|
||||
|
|
|
@ -7,6 +7,7 @@ import (
|
|||
useragent "github.com/projectdiscovery/nuclei/v2/pkg/model/types/userAgent"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/fuzz"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine"
|
||||
fileutil "github.com/projectdiscovery/utils/file"
|
||||
|
@ -54,6 +55,9 @@ type Request struct {
|
|||
// cache any variables that may be needed for operation.
|
||||
options *protocols.ExecutorOptions
|
||||
generator *generators.PayloadGenerator
|
||||
|
||||
// Fuzzing describes schema to fuzz headless requests
|
||||
Fuzzing []*fuzz.Rule `yaml:"fuzzing,omitempty" json:"fuzzing,omitempty" jsonschema:"title=fuzzin rules for http fuzzing,description=Fuzzing describes rule schema to fuzz headless requests"`
|
||||
}
|
||||
|
||||
// RequestPartDefinitions contains a mapping of request part definitions and their
|
||||
|
@ -129,6 +133,21 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
|
|||
request.CompiledOperators = compiled
|
||||
}
|
||||
request.options = options
|
||||
|
||||
if len(request.Fuzzing) > 0 {
|
||||
for _, rule := range request.Fuzzing {
|
||||
if fuzzingMode := options.Options.FuzzingMode; fuzzingMode != "" {
|
||||
rule.Mode = fuzzingMode
|
||||
}
|
||||
if fuzzingType := options.Options.FuzzingType; fuzzingType != "" {
|
||||
rule.Type = fuzzingType
|
||||
}
|
||||
if err := rule.Compile(request.generator, request.options); err != nil {
|
||||
return errors.Wrap(err, "could not compile fuzzing rule")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
package headless
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/model"
|
||||
|
@ -20,6 +21,12 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat
|
|||
}
|
||||
|
||||
switch matcher.GetType() {
|
||||
case matchers.StatusMatcher:
|
||||
statusCode, ok := getStatusCode(data)
|
||||
if !ok {
|
||||
return false, []string{}
|
||||
}
|
||||
return matcher.Result(matcher.MatchStatusCode(statusCode)), []string{}
|
||||
case matchers.SizeMatcher:
|
||||
return matcher.Result(matcher.MatchSize(len(itemStr))), []string{}
|
||||
case matchers.WordsMatcher:
|
||||
|
@ -34,6 +41,24 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat
|
|||
return false, []string{}
|
||||
}
|
||||
|
||||
func getStatusCode(data map[string]interface{}) (int, bool) {
|
||||
statusCodeValue, ok := data["status_code"]
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
statusCodeStr, ok := statusCodeValue.(string)
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
statusCode, err := strconv.Atoi(statusCodeStr)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
return statusCode, true
|
||||
}
|
||||
|
||||
// Extract performs extracting operation for an extractor on model and returns true or false.
|
||||
func (request *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} {
|
||||
itemStr, ok := request.getMatchPart(extractor.Part, data)
|
||||
|
@ -58,6 +83,8 @@ func (request *Request) getMatchPart(part string, data output.InternalEvent) (st
|
|||
part = "data"
|
||||
case "history":
|
||||
part = "history"
|
||||
case "header":
|
||||
part = "header"
|
||||
}
|
||||
|
||||
item, ok := data[part]
|
||||
|
@ -70,12 +97,14 @@ func (request *Request) getMatchPart(part string, data output.InternalEvent) (st
|
|||
}
|
||||
|
||||
// responseToDSLMap converts a headless response to a map for use in DSL matching
|
||||
func (request *Request) responseToDSLMap(resp, req, host, matched string, history string) output.InternalEvent {
|
||||
func (request *Request) responseToDSLMap(resp, headers, status_code, req, host, matched string, history string) output.InternalEvent {
|
||||
return output.InternalEvent{
|
||||
"host": host,
|
||||
"matched": matched,
|
||||
"req": req,
|
||||
"data": resp,
|
||||
"header": headers,
|
||||
"status_code": status_code,
|
||||
"history": history,
|
||||
"type": request.Type().String(),
|
||||
"template-id": request.options.TemplateID,
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
package headless
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
@ -12,6 +13,7 @@ import (
|
|||
"github.com/projectdiscovery/nuclei/v2/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/fuzz"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
|
||||
|
@ -19,6 +21,7 @@ import (
|
|||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/vardump"
|
||||
protocolutils "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils"
|
||||
templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
|
||||
urlutil "github.com/projectdiscovery/utils/url"
|
||||
)
|
||||
|
||||
var _ protocols.Request = &Request{}
|
||||
|
@ -39,7 +42,8 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
|
|||
|
||||
vars := protocolutils.GenerateVariablesWithContextArgs(input, false)
|
||||
payloads := generators.BuildPayloadFromOptions(request.options.Options)
|
||||
values := generators.MergeMaps(vars, metadata, payloads)
|
||||
// add templatecontext variables to varMap
|
||||
values := generators.MergeMaps(vars, metadata, payloads, request.options.TemplateCtx.GetAll())
|
||||
variablesMap := request.options.Variables.Evaluate(values)
|
||||
payloads = generators.MergeMaps(variablesMap, payloads, request.options.Constants)
|
||||
|
||||
|
@ -51,7 +55,10 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
|
|||
gotmatches = results.OperatorsResult.Matched
|
||||
}
|
||||
}
|
||||
|
||||
// verify if fuzz elaboration was requested
|
||||
if len(request.Fuzzing) > 0 {
|
||||
return request.executeFuzzingRule(inputURL, payloads, previous, wrappedCallback)
|
||||
}
|
||||
if request.generator != nil {
|
||||
iterator := request.generator.NewIterator()
|
||||
for {
|
||||
|
@ -129,7 +136,10 @@ func (request *Request) executeRequestWithPayloads(inputURL string, payloads map
|
|||
responseBody, _ = html.HTML()
|
||||
}
|
||||
|
||||
outputEvent := request.responseToDSLMap(responseBody, reqBuilder.String(), inputURL, inputURL, page.DumpHistory())
|
||||
outputEvent := request.responseToDSLMap(responseBody, out["header"], out["status_code"], reqBuilder.String(), inputURL, inputURL, page.DumpHistory())
|
||||
// add response fields to template context and merge templatectx variables to output event
|
||||
request.options.AddTemplateVars(request.Type(), outputEvent)
|
||||
outputEvent = generators.MergeMaps(outputEvent, request.options.TemplateCtx.GetAll())
|
||||
for k, v := range out {
|
||||
outputEvent[k] = v
|
||||
}
|
||||
|
@ -166,3 +176,38 @@ func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.
|
|||
gologger.Debug().Msgf("[%s] Dumped Headless response for %s\n\n%s", requestOptions.TemplateID, input, highlightedResponse)
|
||||
}
|
||||
}
|
||||
|
||||
// executeFuzzingRule executes a fuzzing rule in the template request
|
||||
func (request *Request) executeFuzzingRule(inputURL string, payloads map[string]interface{}, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
|
||||
// check for operator matches by wrapping callback
|
||||
gotmatches := false
|
||||
fuzzRequestCallback := func(gr fuzz.GeneratedRequest) bool {
|
||||
if gotmatches && (request.StopAtFirstMatch || request.options.Options.StopAtFirstMatch || request.options.StopAtFirstMatch) {
|
||||
return true
|
||||
}
|
||||
if err := request.executeRequestWithPayloads(gr.Request.URL.String(), gr.DynamicValues, previous, callback); err != nil {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
parsedURL, err := urlutil.Parse(inputURL)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "could not parse url")
|
||||
}
|
||||
for _, rule := range request.Fuzzing {
|
||||
err := rule.Execute(&fuzz.ExecuteRuleInput{
|
||||
URL: parsedURL,
|
||||
Callback: fuzzRequestCallback,
|
||||
Values: payloads,
|
||||
BaseRequest: nil,
|
||||
})
|
||||
if err == io.EOF {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "could not execute rule")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
@ -70,6 +70,9 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context,
|
|||
// value of `reqData` depends on the type of request specified in template
|
||||
// 1. If request is raw request = reqData contains raw request (i.e http request dump)
|
||||
// 2. If request is Normal ( simply put not a raw request) (Ex: with placeholders `path`) = reqData contains relative path
|
||||
|
||||
// add template context values to dynamicValues (this takes care of self-contained and other types of requests)
|
||||
dynamicValues = generators.MergeMaps(dynamicValues, r.request.options.TemplateCtx.GetAll())
|
||||
if r.request.SelfContained {
|
||||
return r.makeSelfContainedRequest(ctx, reqData, payloads, dynamicValues)
|
||||
}
|
||||
|
|
|
@ -11,8 +11,8 @@ import (
|
|||
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/fuzz"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/fuzz"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool"
|
||||
"github.com/projectdiscovery/rawhttp"
|
||||
"github.com/projectdiscovery/retryablehttp-go"
|
||||
|
|
|
@ -24,12 +24,12 @@ import (
|
|||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/fuzz"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/fuzz"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/signer"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/signerpool"
|
||||
|
@ -717,6 +717,9 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
|
|||
finalEvent := make(output.InternalEvent)
|
||||
|
||||
outputEvent := request.responseToDSLMap(response.resp, input.MetaInput.Input, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(response.fullResponse), tostring.UnsafeToString(response.body), tostring.UnsafeToString(response.headers), duration, generatedRequest.meta)
|
||||
// add response fields to template context and merge templatectx variables to output event
|
||||
request.options.AddTemplateVars(request.Type(), outputEvent)
|
||||
outputEvent = generators.MergeMaps(outputEvent, request.options.TemplateCtx.GetAll())
|
||||
if i := strings.LastIndex(hostname, ":"); i != -1 {
|
||||
hostname = hostname[:i]
|
||||
}
|
||||
|
|
|
@ -0,0 +1,53 @@
|
|||
## multi protocol execution
|
||||
|
||||
### Implementation
|
||||
when template is unmarshalled, if it uses more than one protocol, it will be converted to a multi protocol
|
||||
and the order of the protocols will be preserved as they were in the template and are stored in Request.Queue
|
||||
when template is compiled , we iterate over queue and compile all the requests in the queue
|
||||
|
||||
### Execution
|
||||
when multi protocol template is executed , all protocol requests present in Queue are executed in order
|
||||
and dynamic values extracted are added to template context.
|
||||
|
||||
- Protocol Responses
|
||||
apart from extracted `internal:true` values response fields/values of protocol are added to template context at `ExecutorOptions.TemplateCtx`
|
||||
which takes care of sync and other issues if any. all response fields are prefixed with template type prefix ex: `ssl_subject_dn`
|
||||
|
||||
### Other Methods
|
||||
Such templates are usually used when a particular vulnerability requires more than one protocol to be executed
|
||||
and in such cases the final result is core of the logic hence all methods such as
|
||||
Ex: MakeResultEventItem, MakeResultEvent, GetCompiledOperators
|
||||
are not implemented in multi protocol and just call the same method on last protocol in queue
|
||||
|
||||
|
||||
### Adding New Protocol to multi protocol execution logic
|
||||
while logic/implementation of multi protocol execution is abstracted. it requires 3 statements to be added in newly implemented protocol
|
||||
to make response fields of that protocol available to global context
|
||||
|
||||
- Add `request.options.TemplateCtx.GetAll()` to variablesMap in `ExecuteWithResults` Method just above `request.options.Variables.Evaluate`
|
||||
```go
|
||||
// example
|
||||
values := generators.MergeMaps(payloadValues, hostnameVariables, request.options.TemplateCtx.GetAll())
|
||||
variablesMap := request.options.Variables.Evaluate(values)
|
||||
```
|
||||
|
||||
- Add all response fields to template context just after response map is available
|
||||
```go
|
||||
outputEvent := request.responseToDSLMap(compiledRequest, response, domain, question, traceData)
|
||||
// expose response variables in proto_var format
|
||||
// this is no-op if the template is not a multi protocol template
|
||||
request.options.AddTemplateVars(request.Type(), outputEvent)
|
||||
```
|
||||
|
||||
- Append all available template context values to outputEvent
|
||||
```go
|
||||
// add variables from template context before matching/extraction
|
||||
outputEvent = generators.MergeMaps(outputEvent, request.options.TemplateCtx.GetAll())
|
||||
```
|
||||
|
||||
adding these 3 statements takes care of all logic related to multi protocol execution
|
||||
|
||||
### Exceptions
|
||||
- statements 1 & 2 are intentionally skipped in `file` protocol to avoid redundant data
|
||||
- file/dir input paths don't contain variables or are used in path (yet)
|
||||
- since files are processed by scanning each line. adding statement 2 will unintenionally load all file(s) data
|
|
@ -0,0 +1,6 @@
|
|||
package multi
|
||||
|
||||
// multi is a wrapper protocol Request that allows multiple protocols requests to be executed
|
||||
// multi protocol is just a wrapper so it should/does not include any protocol specific code
|
||||
|
||||
|
|
@ -0,0 +1,172 @@
|
|||
package multi
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/model"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
|
||||
errorutil "github.com/projectdiscovery/utils/errors"
|
||||
)
|
||||
|
||||
var _ protocols.Request = &Request{}
|
||||
|
||||
// refer doc.go for package description , limitations etc
|
||||
|
||||
// Request contains a multi protocol request
|
||||
type Request struct {
|
||||
// description: |
|
||||
// ID is the unique id for the template.
|
||||
//
|
||||
// #### Good IDs
|
||||
//
|
||||
// A good ID uniquely identifies what the requests in the template
|
||||
// are doing. Let's say you have a template that identifies a git-config
|
||||
// file on the webservers, a good name would be `git-config-exposure`. Another
|
||||
// example name is `azure-apps-nxdomain-takeover`.
|
||||
// examples:
|
||||
// - name: ID Example
|
||||
// value: "\"CVE-2021-19520\""
|
||||
ID string `yaml:"id" json:"id" jsonschema:"title=id of the template,description=The Unique ID for the template,example=cve-2021-19520,pattern=^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$"`
|
||||
// description: |
|
||||
// Info contains metadata information about the template.
|
||||
// examples:
|
||||
// - value: exampleInfoStructure
|
||||
Info model.Info `yaml:"info" json:"info" jsonschema:"title=info for the template,description=Info contains metadata for the template"`
|
||||
|
||||
// Queue is queue of all protocols present in the template
|
||||
Queue []protocols.Request `yaml:"-" json:"-"`
|
||||
// request executor options
|
||||
options *protocols.ExecutorOptions `yaml:"-" json:"-"`
|
||||
}
|
||||
|
||||
// getLastRequest returns the last request in the queue
|
||||
func (r *Request) getLastRequest() protocols.Request {
|
||||
if len(r.Queue) == 0 {
|
||||
return nil
|
||||
}
|
||||
return r.Queue[len(r.Queue)-1]
|
||||
}
|
||||
|
||||
// Requests returns the total number of requests template will send
|
||||
func (r *Request) Requests() int {
|
||||
var count int
|
||||
for _, protocol := range r.Queue {
|
||||
count += protocol.Requests()
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
// Compile compiles the protocol request for further execution.
|
||||
func (r *Request) Compile(executerOptions *protocols.ExecutorOptions) error {
|
||||
r.options = executerOptions
|
||||
r.options.TemplateCtx = contextargs.New()
|
||||
r.options.ProtocolType = types.MultiProtocol
|
||||
for _, protocol := range r.Queue {
|
||||
if err := protocol.Compile(r.options); err != nil {
|
||||
return errorutil.NewWithErr(err).Msgf("failed to compile protocol %s", protocol.Type())
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetID returns the unique template ID
|
||||
func (r *Request) GetID() string {
|
||||
return r.ID
|
||||
}
|
||||
|
||||
// Match executes matcher on model and returns true or false (used for clustering if request supports clustering)
|
||||
func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) {
|
||||
return protocols.MakeDefaultMatchFunc(data, matcher)
|
||||
}
|
||||
|
||||
// Extract performs extracting operation for an extractor on model and returns true or false (used for clustering if request supports clustering)
|
||||
func (r *Request) Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} {
|
||||
return protocols.MakeDefaultExtractFunc(data, matcher)
|
||||
}
|
||||
|
||||
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
|
||||
func (r *Request) ExecuteWithResults(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
|
||||
var finalProtoEvent *output.InternalWrappedEvent
|
||||
// callback to process results from all protocols
|
||||
multiProtoCallback := func(event *output.InternalWrappedEvent) {
|
||||
finalProtoEvent = event
|
||||
// export dynamic values from operators (i.e internal:true)
|
||||
if event.OperatorsResult != nil && len(event.OperatorsResult.DynamicValues) > 0 {
|
||||
for k, v := range event.OperatorsResult.DynamicValues {
|
||||
// TBD: iterate-all is only supported in `http` protocol
|
||||
// we either need to add support for iterate-all in other protocols or implement a different logic (specific to template context)
|
||||
// currently if dynamic value array only contains one value we replace it with the value
|
||||
if len(v) == 1 {
|
||||
r.options.TemplateCtx.Set(k, v[0])
|
||||
} else {
|
||||
// Note: if extracted value contains multiple values then they can be accessed by indexing
|
||||
// ex: if values are dynamic = []string{"a","b","c"} then they are available as
|
||||
// dynamic = "a" , dynamic1 = "b" , dynamic2 = "c"
|
||||
// we intentionally omit first index for unknown situations (where no of extracted values are not known)
|
||||
for i, val := range v {
|
||||
if i == 0 {
|
||||
r.options.TemplateCtx.Set(k, val)
|
||||
} else {
|
||||
r.options.TemplateCtx.Set(k+strconv.Itoa(i), val)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// template context: contains values extracted using `internal` extractor from previous protocols
|
||||
// these values are extracted from each protocol in queue and are passed to next protocol in queue
|
||||
// instead of adding seperator field to handle such cases these values are appended to `dynamicValues` (which are meant to be used in workflows)
|
||||
// this makes it possible to use multi protocol templates in workflows
|
||||
// Note: internal extractor values take precedence over dynamicValues from workflows (i.e other templates in workflow)
|
||||
|
||||
// execute all protocols in the queue
|
||||
for _, req := range r.Queue {
|
||||
err := req.ExecuteWithResults(input, dynamicValues, previous, multiProtoCallback)
|
||||
// if error skip execution of next protocols
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// Review: how to handle events of multiple protocols in a single template
|
||||
// currently the outer callback is only executed once (for the last protocol in queue)
|
||||
// due to workflow logic at https://github.com/projectdiscovery/nuclei/blob/main/v2/pkg/protocols/common/executer/executer.go#L150
|
||||
// this causes addition of duplicated / unncessary variables with prefix template_id_all_variables
|
||||
callback(finalProtoEvent)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// MakeResultEventItem creates a result event from internal wrapped event. Intended to be used by MakeResultEventItem internally
|
||||
func (r *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
|
||||
if r.getLastRequest() == nil {
|
||||
return nil
|
||||
}
|
||||
return r.getLastRequest().MakeResultEventItem(wrapped)
|
||||
}
|
||||
|
||||
// MakeResultEvent creates a flat list of result events from an internal wrapped event, based on successful matchers and extracted data
|
||||
func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent {
|
||||
return protocols.MakeDefaultResultEvent(r.getLastRequest(), wrapped)
|
||||
}
|
||||
|
||||
// GetCompiledOperators returns a list of the compiled operators
|
||||
func (r *Request) GetCompiledOperators() []*operators.Operators {
|
||||
last := r.getLastRequest()
|
||||
if last == nil {
|
||||
return nil
|
||||
}
|
||||
return last.GetCompiledOperators()
|
||||
}
|
||||
|
||||
// Type returns the type of the protocol request
|
||||
func (r *Request) Type() types.ProtocolType {
|
||||
return types.MultiProtocol
|
||||
}
|
|
@ -0,0 +1,75 @@
|
|||
package multi_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/parsers"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/progress"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
|
||||
"github.com/projectdiscovery/ratelimit"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
var executerOpts protocols.ExecutorOptions
|
||||
|
||||
func setup() {
|
||||
options := testutils.DefaultOptions
|
||||
testutils.Init(options)
|
||||
progressImpl, _ := progress.NewStatsTicker(0, false, false, false, false, 0)
|
||||
|
||||
executerOpts = protocols.ExecutorOptions{
|
||||
Output: testutils.NewMockOutputWriter(),
|
||||
Options: options,
|
||||
Progress: progressImpl,
|
||||
ProjectFile: nil,
|
||||
IssuesClient: nil,
|
||||
Browser: nil,
|
||||
Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory),
|
||||
RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second),
|
||||
}
|
||||
workflowLoader, err := parsers.NewLoader(&executerOpts)
|
||||
if err != nil {
|
||||
log.Fatalf("Could not create workflow loader: %s\n", err)
|
||||
}
|
||||
executerOpts.WorkflowLoader = workflowLoader
|
||||
}
|
||||
|
||||
func TestMultiProtoWithDynamicExtractor(t *testing.T) {
|
||||
setup()
|
||||
Template, err := templates.Parse("testcases/multiprotodynamic.yaml", nil, executerOpts)
|
||||
require.Nil(t, err, "could not parse template")
|
||||
|
||||
require.Equal(t, 2, len(Template.MultiProtoRequest.Queue))
|
||||
|
||||
err = Template.MultiProtoRequest.Compile(&executerOpts)
|
||||
require.Nil(t, err, "could not compile template")
|
||||
|
||||
gotresults, err := Template.Executer.Execute(contextargs.NewWithInput("blog.projectdiscovery.io"))
|
||||
require.Nil(t, err, "could not execute template")
|
||||
require.True(t, gotresults)
|
||||
}
|
||||
|
||||
func TestMultiProtoWithProtoPrefix(t *testing.T) {
|
||||
setup()
|
||||
Template, err := templates.Parse("testcases/multiprotowithprefix.yaml", nil, executerOpts)
|
||||
require.Nil(t, err, "could not parse template")
|
||||
|
||||
require.Equal(t, 3, len(Template.MultiProtoRequest.Queue))
|
||||
|
||||
err = Template.MultiProtoRequest.Compile(&executerOpts)
|
||||
require.Nil(t, err, "could not compile template")
|
||||
|
||||
require.True(t, len(Template.MultiProtoRequest.GetCompiledOperators()) > 0, "could not compile operators")
|
||||
|
||||
gotresults, err := Template.Executer.Execute(contextargs.NewWithInput("blog.projectdiscovery.io"))
|
||||
require.Nil(t, err, "could not execute template")
|
||||
require.True(t, gotresults)
|
||||
}
|
|
@ -0,0 +1,29 @@
|
|||
id: dns-http-dynamic-values
|
||||
|
||||
info:
|
||||
name: multi protocol request with dynamic values
|
||||
author: pdteam
|
||||
severity: info
|
||||
|
||||
dns:
|
||||
- name: "{{FQDN}}" # DNS Request
|
||||
type: cname
|
||||
|
||||
extractors:
|
||||
- type: dsl
|
||||
name: blogid
|
||||
dsl:
|
||||
- trim_suffix(cname,'.ghost.io.')
|
||||
internal: true
|
||||
|
||||
http:
|
||||
- method: GET # http request
|
||||
path:
|
||||
- "{{BaseURL}}"
|
||||
|
||||
matchers:
|
||||
- type: dsl
|
||||
dsl:
|
||||
- contains(body,'ProjectDiscovery.io') # check for http string
|
||||
- blogid == 'projectdiscovery' # check for cname (extracted information from dns response)
|
||||
condition: and
|
|
@ -0,0 +1,26 @@
|
|||
id: dns-http-proto-prefix
|
||||
|
||||
info:
|
||||
name: multi protocol request with dynamic values
|
||||
author: pdteam
|
||||
severity: info
|
||||
|
||||
dns:
|
||||
- name: "{{FQDN}}" # DNS Request
|
||||
type: cname
|
||||
|
||||
ssl:
|
||||
- address: "{{Hostname}}" # ssl request
|
||||
|
||||
http:
|
||||
- method: GET # http request
|
||||
path:
|
||||
- "{{BaseURL}}"
|
||||
|
||||
matchers:
|
||||
- type: dsl
|
||||
dsl:
|
||||
- contains(http_body,'ProjectDiscovery.io') # check for http string
|
||||
- trim_suffix(dns_cname,'.ghost.io.') == 'projectdiscovery' # check for cname (extracted information from dns response)
|
||||
- ssl_subject_cn == 'blog.projectdiscovery.io'
|
||||
condition: and
|
|
@ -54,6 +54,8 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
|
|||
return errors.Wrap(err, "could not get address from url")
|
||||
}
|
||||
variables := protocolutils.GenerateVariables(address, false, nil)
|
||||
// add template ctx variables to varMap
|
||||
variables = generators.MergeMaps(variables, request.options.TemplateCtx.GetAll())
|
||||
variablesMap := request.options.Variables.Evaluate(variables)
|
||||
variables = generators.MergeMaps(variablesMap, variables, request.options.Constants)
|
||||
|
||||
|
@ -269,6 +271,9 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
|
|||
|
||||
response := responseBuilder.String()
|
||||
outputEvent := request.responseToDSLMap(reqBuilder.String(), string(final[:n]), response, input, actualAddress)
|
||||
// add response fields to template context and merge templatectx variables to output event
|
||||
request.options.AddTemplateVars(request.Type(), outputEvent)
|
||||
outputEvent = generators.MergeMaps(outputEvent, request.options.TemplateCtx.GetAll())
|
||||
outputEvent["ip"] = request.dialer.GetDialedIP(hostname)
|
||||
if request.options.StopAtFirstMatch {
|
||||
outputEvent["stop-at-first-match"] = true
|
||||
|
|
|
@ -14,6 +14,7 @@ import (
|
|||
"github.com/projectdiscovery/nuclei/v2/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring"
|
||||
templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
|
||||
|
@ -87,6 +88,9 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata
|
|||
}
|
||||
|
||||
outputEvent := request.responseToDSLMap(resp, data, data, data, tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(body), headersToString(resp.Header), 0, nil)
|
||||
// add response fields to template context and merge templatectx variables to output event
|
||||
request.options.AddTemplateVars(request.Type(), outputEvent)
|
||||
outputEvent = generators.MergeMaps(outputEvent, request.options.TemplateCtx.GetAll())
|
||||
outputEvent["ip"] = ""
|
||||
for k, v := range previous {
|
||||
outputEvent[k] = v
|
||||
|
|
|
@ -2,6 +2,7 @@ package protocols
|
|||
|
||||
import (
|
||||
"github.com/projectdiscovery/ratelimit"
|
||||
stringsutil "github.com/projectdiscovery/utils/strings"
|
||||
|
||||
"github.com/logrusorgru/aurora"
|
||||
|
||||
|
@ -85,6 +86,41 @@ type ExecutorOptions struct {
|
|||
Colorizer aurora.Aurora
|
||||
WorkflowLoader model.WorkflowLoader
|
||||
ResumeCfg *types.ResumeCfg
|
||||
// TemplateContext (contains all variables that are templatescoped i.e multi protocol)
|
||||
// only used in case of multi protocol templates
|
||||
TemplateCtx *contextargs.Context
|
||||
// ProtocolType is the type of the template
|
||||
ProtocolType templateTypes.ProtocolType
|
||||
}
|
||||
|
||||
// AddTemplateVars adds vars to template context with given template type as prefix
|
||||
// this method is no-op if template is not multi protocol
|
||||
func (e *ExecutorOptions) AddTemplateVars(templateType templateTypes.ProtocolType, vars map[string]interface{}) {
|
||||
if e.ProtocolType != templateTypes.MultiProtocol {
|
||||
// no-op if not multi protocol template
|
||||
return
|
||||
}
|
||||
for k, v := range vars {
|
||||
if !stringsutil.EqualFoldAny(k, "template-id", "template-info", "template-path") {
|
||||
if templateType < templateTypes.InvalidProtocol {
|
||||
k = templateType.String() + "_" + k
|
||||
}
|
||||
e.TemplateCtx.Set(k, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// AddTemplateVar adds given var to template context with given template type as prefix
|
||||
// this method is no-op if template is not multi protocol
|
||||
func (e *ExecutorOptions) AddTemplateVar(prefix, key string, value interface{}) {
|
||||
if e.ProtocolType != templateTypes.MultiProtocol {
|
||||
// no-op if not multi protocol template
|
||||
return
|
||||
}
|
||||
if prefix != "" {
|
||||
key = prefix + "_" + key
|
||||
}
|
||||
e.TemplateCtx.Set(key, value)
|
||||
}
|
||||
|
||||
// Copy returns a copy of the executeroptions structure
|
||||
|
|
|
@ -24,7 +24,6 @@ import (
|
|||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/vardump"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils"
|
||||
protocolutils "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils"
|
||||
templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/types"
|
||||
|
@ -188,7 +187,8 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
|
|||
payloadValues["Port"] = port
|
||||
|
||||
hostnameVariables := protocolutils.GenerateDNSVariables(hostname)
|
||||
values := generators.MergeMaps(payloadValues, hostnameVariables)
|
||||
// add template context variables to varMap
|
||||
values := generators.MergeMaps(payloadValues, hostnameVariables, request.options.TemplateCtx.GetAll())
|
||||
variablesMap := request.options.Variables.Evaluate(values)
|
||||
payloadValues = generators.MergeMaps(variablesMap, payloadValues, request.options.Constants)
|
||||
|
||||
|
@ -267,10 +267,11 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
|
|||
// if field is not exported f.IsZero() , f.Value() will panic
|
||||
continue
|
||||
}
|
||||
tag := utils.CleanStructFieldJSONTag(f.Tag("json"))
|
||||
tag := protocolutils.CleanStructFieldJSONTag(f.Tag("json"))
|
||||
if tag == "" || f.IsZero() {
|
||||
continue
|
||||
}
|
||||
request.options.AddTemplateVar(request.Type().String(), tag, f.Value())
|
||||
data[tag] = f.Value()
|
||||
}
|
||||
|
||||
|
@ -285,13 +286,16 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
|
|||
// if field is not exported f.IsZero() , f.Value() will panic
|
||||
continue
|
||||
}
|
||||
tag := utils.CleanStructFieldJSONTag(f.Tag("json"))
|
||||
tag := protocolutils.CleanStructFieldJSONTag(f.Tag("json"))
|
||||
if tag == "" || f.IsZero() {
|
||||
continue
|
||||
}
|
||||
request.options.AddTemplateVar(request.Type().String(), tag, f.Value())
|
||||
data[tag] = f.Value()
|
||||
}
|
||||
|
||||
// add response fields ^ to template context and merge templatectx variables to output event
|
||||
data = generators.MergeMaps(data, request.options.TemplateCtx.GetAll())
|
||||
event := eventcreator.CreateEvent(request, data, requestOptions.Options.Debug || requestOptions.Options.DebugResponse)
|
||||
if requestOptions.Options.Debug || requestOptions.Options.DebugResponse || requestOptions.Options.StoreResponse {
|
||||
msg := fmt.Sprintf("[%s] Dumped SSL response for %s", requestOptions.TemplateID, input.MetaInput.Input)
|
||||
|
|
|
@ -175,7 +175,8 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam
|
|||
}
|
||||
defaultVars := protocolutils.GenerateVariables(parsed, false, nil)
|
||||
optionVars := generators.BuildPayloadFromOptions(request.options.Options)
|
||||
variables := request.options.Variables.Evaluate(generators.MergeMaps(defaultVars, optionVars, dynamicValues))
|
||||
// add templatecontext variables to varMap
|
||||
variables := request.options.Variables.Evaluate(generators.MergeMaps(defaultVars, optionVars, dynamicValues, request.options.TemplateCtx.GetAll()))
|
||||
payloadValues := generators.MergeMaps(variables, defaultVars, optionVars, dynamicValues, request.options.Constants)
|
||||
|
||||
requestOptions := request.options
|
||||
|
@ -254,12 +255,6 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam
|
|||
gologger.Verbose().Msgf("Sent Websocket request to %s", input)
|
||||
|
||||
data := make(map[string]interface{})
|
||||
for k, v := range previous {
|
||||
data[k] = v
|
||||
}
|
||||
for k, v := range events {
|
||||
data[k] = v
|
||||
}
|
||||
|
||||
data["type"] = request.Type().String()
|
||||
data["success"] = "true"
|
||||
|
@ -269,6 +264,17 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam
|
|||
data["matched"] = addressToDial
|
||||
data["ip"] = request.dialer.GetDialedIP(hostname)
|
||||
|
||||
// add response fields to template context and merge templatectx variables to output event
|
||||
request.options.AddTemplateVars(request.Type(), data)
|
||||
data = generators.MergeMaps(data, request.options.TemplateCtx.GetAll())
|
||||
|
||||
for k, v := range previous {
|
||||
data[k] = v
|
||||
}
|
||||
for k, v := range events {
|
||||
data[k] = v
|
||||
}
|
||||
|
||||
event := eventcreator.CreateEventWithAdditionalOptions(request, data, requestOptions.Options.Debug || requestOptions.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) {
|
||||
internalWrappedEvent.OperatorsResult.PayloadValues = payloadValues
|
||||
})
|
||||
|
|
|
@ -90,7 +90,8 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
|
|||
// generate variables
|
||||
defaultVars := protocolutils.GenerateVariables(input.MetaInput.Input, false, nil)
|
||||
optionVars := generators.BuildPayloadFromOptions(request.options.Options)
|
||||
vars := request.options.Variables.Evaluate(generators.MergeMaps(defaultVars, optionVars, dynamicValues))
|
||||
// add templatectx variables to varMap
|
||||
vars := request.options.Variables.Evaluate(generators.MergeMaps(defaultVars, optionVars, dynamicValues, request.options.TemplateCtx.GetAll()))
|
||||
|
||||
variables := generators.MergeMaps(vars, defaultVars, optionVars, dynamicValues, request.options.Constants)
|
||||
|
||||
|
@ -132,6 +133,10 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
|
|||
data["host"] = query
|
||||
data["response"] = jsonDataString
|
||||
|
||||
// add response fields to template context and merge templatectx variables to output event
|
||||
request.options.AddTemplateVars(request.Type(), data)
|
||||
data = generators.MergeMaps(data, request.options.TemplateCtx.GetAll())
|
||||
|
||||
event := eventcreator.CreateEvent(request, data, request.options.Options.Debug || request.options.Options.DebugResponse)
|
||||
if request.options.Options.Debug || request.options.Options.DebugResponse {
|
||||
gologger.Debug().Msgf("[%s] Dumped WHOIS response for %s", request.options.TemplateID, query)
|
||||
|
|
|
@ -10,6 +10,7 @@ import (
|
|||
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/executer"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/offlinehttp"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/templates/cache"
|
||||
|
@ -122,30 +123,31 @@ func (template *Template) compileProtocolRequests(options protocols.ExecutorOpti
|
|||
|
||||
var requests []protocols.Request
|
||||
|
||||
if len(template.RequestsDNS) > 0 {
|
||||
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsDNS)...)
|
||||
}
|
||||
if len(template.RequestsFile) > 0 {
|
||||
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsFile)...)
|
||||
}
|
||||
if len(template.RequestsNetwork) > 0 {
|
||||
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsNetwork)...)
|
||||
}
|
||||
if len(template.RequestsHTTP) > 0 {
|
||||
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsHTTP)...)
|
||||
}
|
||||
if len(template.RequestsHeadless) > 0 && options.Options.Headless {
|
||||
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsHeadless)...)
|
||||
}
|
||||
if len(template.RequestsSSL) > 0 {
|
||||
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsSSL)...)
|
||||
}
|
||||
if len(template.RequestsWebsocket) > 0 {
|
||||
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsWebsocket)...)
|
||||
}
|
||||
if len(template.RequestsWHOIS) > 0 {
|
||||
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsWHOIS)...)
|
||||
if len(template.MultiProtoRequest.Queue) > 0 {
|
||||
template.MultiProtoRequest.ID = template.ID
|
||||
template.MultiProtoRequest.Info = template.Info
|
||||
requests = append(requests, &template.MultiProtoRequest)
|
||||
} else {
|
||||
switch {
|
||||
case len(template.RequestsDNS) > 0:
|
||||
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsDNS)...)
|
||||
case len(template.RequestsFile) > 0:
|
||||
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsFile)...)
|
||||
case len(template.RequestsNetwork) > 0:
|
||||
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsNetwork)...)
|
||||
case len(template.RequestsHTTP) > 0:
|
||||
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsHTTP)...)
|
||||
case len(template.RequestsHeadless) > 0 && options.Options.Headless:
|
||||
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsHeadless)...)
|
||||
case len(template.RequestsSSL) > 0:
|
||||
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsSSL)...)
|
||||
case len(template.RequestsWebsocket) > 0:
|
||||
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsWebsocket)...)
|
||||
case len(template.RequestsWHOIS) > 0:
|
||||
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsWHOIS)...)
|
||||
}
|
||||
}
|
||||
|
||||
template.Executer = executer.NewExecuter(requests, &options)
|
||||
return nil
|
||||
}
|
||||
|
@ -232,8 +234,10 @@ func ParseTemplateFromReader(reader io.Reader, preprocessor Preprocessor, option
|
|||
options.Variables = template.Variables
|
||||
}
|
||||
|
||||
// create empty context args for template scope
|
||||
options.TemplateCtx = contextargs.New()
|
||||
options.ProtocolType = template.Type()
|
||||
options.Constants = template.Constants
|
||||
|
||||
// If no requests, and it is also not a workflow, return error.
|
||||
if template.Requests() == 0 {
|
||||
return nil, fmt.Errorf("no requests defined for %s", template.ID)
|
||||
|
|
|
@ -25,6 +25,7 @@ import (
|
|||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/variables"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/workflows"
|
||||
"github.com/projectdiscovery/ratelimit"
|
||||
|
@ -53,7 +54,6 @@ func setup() {
|
|||
log.Fatalf("Could not create workflow loader: %s\n", err)
|
||||
}
|
||||
executerOpts.WorkflowLoader = workflowLoader
|
||||
|
||||
}
|
||||
|
||||
func Test_ParseFromURL(t *testing.T) {
|
||||
|
@ -197,3 +197,16 @@ func Test_WrongTemplate(t *testing.T) {
|
|||
require.Nil(t, got, "could not parse template")
|
||||
require.ErrorContains(t, err, "no requests defined ")
|
||||
}
|
||||
|
||||
func Test_Multiprotocol(t *testing.T) {
|
||||
setup()
|
||||
got, err := templates.Parse("tests/multiproto.yaml", nil, executerOpts)
|
||||
require.Nil(t, err, "could not parse template")
|
||||
require.Equal(t, 3, got.Requests())
|
||||
require.Equal(t, types.MultiProtocol, got.Type())
|
||||
|
||||
got, err = templates.Parse("tests/multiproto.json", nil, executerOpts)
|
||||
require.Nil(t, err, "could not parse template")
|
||||
require.Equal(t, 3, got.Requests())
|
||||
require.Equal(t, types.MultiProtocol, got.Type())
|
||||
}
|
||||
|
|
|
@ -12,6 +12,7 @@ import (
|
|||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/file"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/multi"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/network"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/ssl"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/websocket"
|
||||
|
@ -55,6 +56,7 @@ type Template struct {
|
|||
// examples:
|
||||
// - value: exampleNormalHTTPRequest
|
||||
// RequestsWithHTTP is placeholder(internal) only, and should not be used instead use RequestsHTTP
|
||||
// Deprecated: Use RequestsHTTP instead.
|
||||
RequestsWithHTTP []*http.Request `yaml:"http,omitempty" json:"http,omitempty" jsonschema:"title=http requests to make,description=HTTP requests to make for the template"`
|
||||
// description: |
|
||||
// DNS contains the dns request to make in the template
|
||||
|
@ -77,6 +79,7 @@ type Template struct {
|
|||
// examples:
|
||||
// - value: exampleNormalNetworkRequest
|
||||
// RequestsWithTCP is placeholder(internal) only, and should not be used instead use RequestsNetwork
|
||||
// Deprecated: Use RequestsNetwork instead.
|
||||
RequestsWithTCP []*network.Request `yaml:"tcp,omitempty" json:"tcp,omitempty" jsonschema:"title=network(tcp) requests to make,description=Network requests to make for the template"`
|
||||
// description: |
|
||||
// Headless contains the headless request to make in the template.
|
||||
|
@ -126,24 +129,16 @@ type Template struct {
|
|||
|
||||
// Verified defines if the template signature is digitally verified
|
||||
Verified bool `yaml:"-" json:"-"`
|
||||
}
|
||||
|
||||
// TemplateProtocols is a list of accepted template protocols
|
||||
var TemplateProtocols = []string{
|
||||
"dns",
|
||||
"file",
|
||||
"http",
|
||||
"headless",
|
||||
"network",
|
||||
"workflow",
|
||||
"ssl",
|
||||
"websocket",
|
||||
"whois",
|
||||
// MultiProtoRequest (Internal) contains multi protocol request if multiple protocols are used
|
||||
MultiProtoRequest multi.Request `yaml:"-" json:"-"`
|
||||
}
|
||||
|
||||
// Type returns the type of the template
|
||||
func (template *Template) Type() types.ProtocolType {
|
||||
switch {
|
||||
case len(template.MultiProtoRequest.Queue) > 0:
|
||||
return types.MultiProtocol
|
||||
case len(template.RequestsDNS) > 0:
|
||||
return types.DNSProtocol
|
||||
case len(template.RequestsFile) > 0:
|
||||
|
@ -200,7 +195,62 @@ func (template *Template) UnmarshalYAML(unmarshal func(interface{}) error) error
|
|||
if len(alias.RequestsWithTCP) > 0 {
|
||||
template.RequestsNetwork = alias.RequestsWithTCP
|
||||
}
|
||||
return validate.New().Struct(template)
|
||||
err = validate.New().Struct(template)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// check if the template contains a multi protocols
|
||||
if template.isMultiProtocol() {
|
||||
var tempmap yaml.MapSlice
|
||||
err = unmarshal(&tempmap)
|
||||
if err != nil {
|
||||
return errorutil.NewWithErr(err).Msgf("failed to unmarshal multi protocol template %s", template.ID)
|
||||
}
|
||||
arr := []string{}
|
||||
for _, v := range tempmap {
|
||||
key, ok := v.Key.(string)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
arr = append(arr, key)
|
||||
}
|
||||
// add protocols to the protocol stack (the idea is to preserve the order of the protocols)
|
||||
template.addProtocolsToQueue(arr...)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Internal function to create a protocol stack from a template if the template is a multi protocol template
|
||||
func (template *Template) addProtocolsToQueue(keys ...string) {
|
||||
for _, key := range keys {
|
||||
switch key {
|
||||
case types.DNSProtocol.String():
|
||||
template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsDNS)...)
|
||||
case types.FileProtocol.String():
|
||||
template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsFile)...)
|
||||
case types.HTTPProtocol.String():
|
||||
template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsHTTP)...)
|
||||
case types.HeadlessProtocol.String():
|
||||
template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsHeadless)...)
|
||||
case types.NetworkProtocol.String():
|
||||
template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsNetwork)...)
|
||||
case types.SSLProtocol.String():
|
||||
template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsSSL)...)
|
||||
case types.WebsocketProtocol.String():
|
||||
template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsWebsocket)...)
|
||||
case types.WHOISProtocol.String():
|
||||
template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsWHOIS)...)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// isMultiProtocol checks if the template is a multi protocol template
|
||||
func (template *Template) isMultiProtocol() bool {
|
||||
counter := len(template.RequestsDNS) + len(template.RequestsFile) +
|
||||
len(template.RequestsHTTP) + len(template.RequestsHeadless) +
|
||||
len(template.RequestsNetwork) + len(template.RequestsSSL) +
|
||||
len(template.RequestsWebsocket) + len(template.RequestsWHOIS)
|
||||
return counter > 1
|
||||
}
|
||||
|
||||
// MarshalJSON forces recursive struct validation during marshal operation
|
||||
|
@ -219,5 +269,22 @@ func (template *Template) UnmarshalJSON(data []byte) error {
|
|||
return err
|
||||
}
|
||||
*template = Template(*alias)
|
||||
return validate.New().Struct(template)
|
||||
err = validate.New().Struct(template)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// check if template contains multiple protocols
|
||||
if template.isMultiProtocol() {
|
||||
var tempMap map[string]interface{}
|
||||
err = json.Unmarshal(data, &tempMap)
|
||||
if err != nil {
|
||||
return errorutil.NewWithErr(err).Msgf("failed to unmarshal multi protocol template %s", template.ID)
|
||||
}
|
||||
arr := []string{}
|
||||
for k := range tempMap {
|
||||
arr = append(arr, k)
|
||||
}
|
||||
template.addProtocolsToQueue(arr...)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
@ -677,6 +677,10 @@ func init() {
|
|||
TypeName: "http.Request",
|
||||
FieldName: "fuzzing",
|
||||
},
|
||||
{
|
||||
TypeName: "headless.Request",
|
||||
FieldName: "fuzzing",
|
||||
},
|
||||
}
|
||||
FUZZRuleDoc.Fields = make([]encoder.Doc, 7)
|
||||
FUZZRuleDoc.Fields[0].Name = "type"
|
||||
|
@ -1203,7 +1207,7 @@ func init() {
|
|||
Value: "Headless response received from client (default)",
|
||||
},
|
||||
}
|
||||
HEADLESSRequestDoc.Fields = make([]encoder.Doc, 7)
|
||||
HEADLESSRequestDoc.Fields = make([]encoder.Doc, 8)
|
||||
HEADLESSRequestDoc.Fields[0].Name = "id"
|
||||
HEADLESSRequestDoc.Fields[0].Type = "string"
|
||||
HEADLESSRequestDoc.Fields[0].Note = ""
|
||||
|
@ -1239,6 +1243,11 @@ func init() {
|
|||
HEADLESSRequestDoc.Fields[6].Note = ""
|
||||
HEADLESSRequestDoc.Fields[6].Description = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found."
|
||||
HEADLESSRequestDoc.Fields[6].Comments[encoder.LineComment] = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found."
|
||||
HEADLESSRequestDoc.Fields[7].Name = "fuzzing"
|
||||
HEADLESSRequestDoc.Fields[7].Type = "[]fuzz.Rule"
|
||||
HEADLESSRequestDoc.Fields[7].Note = ""
|
||||
HEADLESSRequestDoc.Fields[7].Description = "Fuzzing describes schema to fuzz headless requests"
|
||||
HEADLESSRequestDoc.Fields[7].Comments[encoder.LineComment] = " Fuzzing describes schema to fuzz headless requests"
|
||||
|
||||
ENGINEActionDoc.Type = "engine.Action"
|
||||
ENGINEActionDoc.Comments[encoder.LineComment] = " Action is an action taken by the browser to reach a navigation"
|
||||
|
|
|
@ -0,0 +1,41 @@
|
|||
{
|
||||
"id": "nuclei-multi-protocol",
|
||||
"info": {
|
||||
"name": "multi protocol support",
|
||||
"author": "pdteam",
|
||||
"severity": "info"
|
||||
},
|
||||
"dns": [
|
||||
{
|
||||
"name": "{{FQDN}}",
|
||||
"type": "cname"
|
||||
}
|
||||
],
|
||||
"ssl": [
|
||||
{
|
||||
"address": "{{Hostname}}"
|
||||
}
|
||||
],
|
||||
"http": [
|
||||
{
|
||||
"method": "GET",
|
||||
"path": [
|
||||
"{{BaseURL}}"
|
||||
],
|
||||
"headers": {
|
||||
"Host": "{{ssl_subject_cn}}",
|
||||
"Metadata": "{{ssl_cipher}}"
|
||||
},
|
||||
"matchers": [
|
||||
{
|
||||
"type": "dsl",
|
||||
"dsl": [
|
||||
"http_status_code == 404",
|
||||
"contains(dns_cname, 'github.io')"
|
||||
],
|
||||
"condition": "and"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
|
@ -0,0 +1,30 @@
|
|||
id: nuclei-multi-protocol
|
||||
|
||||
info:
|
||||
name: multi protocol support
|
||||
author: pdteam
|
||||
severity: info
|
||||
|
||||
dns:
|
||||
- name: "{{FQDN}}" # dns request
|
||||
type: cname
|
||||
|
||||
ssl:
|
||||
- address: "{{Hostname}}" # ssl request
|
||||
|
||||
http:
|
||||
- method: GET
|
||||
path:
|
||||
- "{{BaseURL}}" # http request
|
||||
|
||||
headers:
|
||||
Host: "{{ssl_subject_cn}}" # host extracted from ssl request
|
||||
Metadata: "{{ssl_cipher}}"
|
||||
|
||||
matchers:
|
||||
- type: dsl
|
||||
dsl:
|
||||
# - contains(http_body,'File not found') # check for http string
|
||||
- http_status_code == 404
|
||||
- contains(dns_cname, 'github.io') # check for cname
|
||||
condition: and
|
|
@ -36,6 +36,8 @@ const (
|
|||
WebsocketProtocol
|
||||
// name:whois
|
||||
WHOISProtocol
|
||||
// name: multi
|
||||
MultiProtocol
|
||||
limit
|
||||
InvalidProtocol
|
||||
)
|
||||
|
@ -52,6 +54,7 @@ var protocolMappings = map[ProtocolType]string{
|
|||
SSLProtocol: "ssl",
|
||||
WebsocketProtocol: "websocket",
|
||||
WHOISProtocol: "whois",
|
||||
MultiProtocol: "multi",
|
||||
}
|
||||
|
||||
func GetSupportedProtocolTypes() ProtocolTypes {
|
||||
|
|
|
@ -16,6 +16,7 @@ import (
|
|||
"github.com/projectdiscovery/nuclei/v2/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/progress"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/types"
|
||||
)
|
||||
|
@ -90,6 +91,7 @@ func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) *protoco
|
|||
Browser: nil,
|
||||
Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory),
|
||||
RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second),
|
||||
TemplateCtx: contextargs.New(),
|
||||
}
|
||||
return executerOpts
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue