support for dynamic variables in template context (multi protocol execution) (#3672)

* multi proto request genesis

* adds template context dynamic vars

* feat: proto level resp variables

* remove proto prefix hacky logic

* implement template ctx args

* remove old var name logic

* improve AddTemplateVars func

* add multi proto comments+docs

* vardump with sorted keys

* fix race condition in ctx args

* default initialize ctx args

* use generic map

* index variables with multiple values

* fix nil cookies

* use synclock map

* fix build failure

* fix lint error

* resolve merge conflicts

* multi proto: add unit+ integration tests

* fix unit tests

* Issue 3339 headless fuzz (#3790)

* Basic headless fuzzing

* Remove debug statements

* Add integration tests

* Update template

* Fix recognize payload value in matcher

* Update tempalte

* use req.SetURL()

---------

Co-authored-by: Tarun Koyalwar <tarun@projectdiscovery.io>

* Auto Generate Syntax Docs + JSONSchema [Fri Jun  9 00:23:32 UTC 2023] 🤖

* Add headless header and status matchers (#3794)

* add headless header and status matchers

* rename headers as header

* add integration test for header+status

* fix typo

---------

Co-authored-by: Shubham Rasal <shubham@projectdiscovery.io>
Co-authored-by: GitHub Action <action@github.com>
Co-authored-by: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com>
dev
Tarun Koyalwar 2023-06-09 19:52:56 +05:30 committed by GitHub
parent b4e4715d36
commit e1d3f474a4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
52 changed files with 1114 additions and 238 deletions

View File

@ -1585,6 +1585,8 @@ Appears in:
- <code><a href="#httprequest">http.Request</a>.fuzzing</code>
- <code><a href="#headlessrequest">headless.Request</a>.fuzzing</code>
@ -2717,6 +2719,19 @@ StopAtFirstMatch stops the execution of the requests and template as soon as a m
<hr />
<div class="dd">
<code>fuzzing</code> <i>[]<a href="#fuzzrule">fuzz.Rule</a></i>
</div>
<div class="dt">
Fuzzing describes schema to fuzz headless requests
</div>
<hr />

View File

@ -0,0 +1,31 @@
id: headless-query-fuzzing
info:
name: Example Query Fuzzing
author: pdteam
severity: info
headless:
- steps:
- action: navigate
args:
url: "{{BaseURL}}"
- action: waitload
payloads:
redirect:
- "blog.com"
- "portal.com"
fuzzing:
- part: query
mode: single
type: replace
fuzz:
- "https://{{redirect}}"
matchers:
- type: word
part: body
words:
- "{{redirect}}"

View File

@ -0,0 +1,24 @@
id: headless-header-status-test
info:
name: headless header + status test
author: pdteam
severity: info
headless:
- steps:
- args:
url: "{{BaseURL}}"
action: navigate
- action: waitload
matchers-condition: and
matchers:
- type: word
part: header
words:
- text/plain
- type: status
status:
- 200

View File

@ -0,0 +1,29 @@
id: dns-http-dynamic-values
info:
name: multi protocol request with dynamic values
author: pdteam
severity: info
dns:
- name: "{{FQDN}}" # DNS Request
type: cname
extractors:
- type: dsl
name: blogid
dsl:
- trim_suffix(cname,'.ghost.io.')
internal: true
http:
- method: GET # http request
path:
- "{{BaseURL}}"
matchers:
- type: dsl
dsl:
- contains(body,'ProjectDiscovery.io') # check for http string
- blogid == 'projectdiscovery' # check for cname (extracted information from dns response)
condition: and

View File

@ -0,0 +1,30 @@
id: dns-ssl-http-with-variables
info:
name: multi protocol request with dynamic values
author: pdteam
severity: info
variables:
cname_filtered: '{{trim_suffix(dns_cname,".ghost.io.")}}'
dns:
- name: "{{FQDN}}" # DNS Request
type: cname
ssl:
- address: "{{Hostname}}" # ssl request
http:
- method: GET # http request
path:
- "{{BaseURL}}"
matchers:
- type: dsl
dsl:
- contains(http_body,'ProjectDiscovery.io') # check for http string
- cname_filtered == 'projectdiscovery' # check for cname (extracted information from dns response)
- ssl_subject_cn == 'blog.projectdiscovery.io'
condition: and

View File

@ -0,0 +1,26 @@
id: dns-ssl-http-proto-prefix
info:
name: multi protocol request with dynamic values
author: pdteam
severity: info
dns:
- name: "{{FQDN}}" # DNS Request
type: cname
ssl:
- address: "{{Hostname}}" # ssl request
http:
- method: GET # http request
path:
- "{{BaseURL}}"
matchers:
- type: dsl
dsl:
- contains(http_body,'ProjectDiscovery.io') # check for http string
- trim_suffix(dns_cname,'.ghost.io.') == 'projectdiscovery' # check for cname (extracted information from dns response)
- ssl_subject_cn == 'blog.projectdiscovery.io'
condition: and

View File

@ -372,6 +372,72 @@
"title": "type of the matcher",
"description": "Type of the matcher"
},
"fuzz.Rule": {
"properties": {
"type": {
"enum": [
"replace",
"prefix",
"postfix",
"infix"
],
"type": "string",
"title": "type of rule",
"description": "Type of fuzzing rule to perform"
},
"part": {
"enum": [
"query"
],
"type": "string",
"title": "part of rule",
"description": "Part of request rule to fuzz"
},
"mode": {
"enum": [
"single",
"multiple"
],
"type": "string",
"title": "mode of rule",
"description": "Mode of request rule to fuzz"
},
"keys": {
"items": {
"type": "string"
},
"type": "array",
"title": "keys of parameters to fuzz",
"description": "Keys of parameters to fuzz"
},
"keys-regex": {
"items": {
"type": "string"
},
"type": "array",
"title": "keys regex to fuzz",
"description": "Regex of parameter keys to fuzz"
},
"values": {
"items": {
"type": "string"
},
"type": "array",
"title": "values regex to fuzz",
"description": "Regex of parameter values to fuzz"
},
"fuzz": {
"items": {
"type": "string"
},
"type": "array",
"title": "payloads of fuzz rule",
"description": "Payloads to perform fuzzing substitutions with"
}
},
"additionalProperties": false,
"type": "object"
},
"generators.AttackTypeHolder": {
"enum": [
"batteringram",
@ -653,6 +719,14 @@
"type": "string",
"title": "condition between the matchers",
"description": "Conditions between the matchers"
},
"fuzzing": {
"items": {
"$ref": "#/definitions/fuzz.Rule"
},
"type": "array",
"title": "fuzzin rules for http fuzzing",
"description": "Fuzzing describes rule schema to fuzz headless requests"
}
},
"additionalProperties": false,
@ -953,72 +1027,6 @@
"title": "type of the signature",
"description": "Type of the signature"
},
"fuzz.Rule": {
"properties": {
"type": {
"enum": [
"replace",
"prefix",
"postfix",
"infix"
],
"type": "string",
"title": "type of rule",
"description": "Type of fuzzing rule to perform"
},
"part": {
"enum": [
"query"
],
"type": "string",
"title": "part of rule",
"description": "Part of request rule to fuzz"
},
"mode": {
"enum": [
"single",
"multiple"
],
"type": "string",
"title": "mode of rule",
"description": "Mode of request rule to fuzz"
},
"keys": {
"items": {
"type": "string"
},
"type": "array",
"title": "keys of parameters to fuzz",
"description": "Keys of parameters to fuzz"
},
"keys-regex": {
"items": {
"type": "string"
},
"type": "array",
"title": "keys regex to fuzz",
"description": "Regex of parameter keys to fuzz"
},
"values": {
"items": {
"type": "string"
},
"type": "array",
"title": "values regex to fuzz",
"description": "Regex of parameter values to fuzz"
},
"fuzz": {
"items": {
"type": "string"
},
"type": "array",
"title": "payloads of fuzz rule",
"description": "Payloads to perform fuzzing substitutions with"
}
},
"additionalProperties": false,
"type": "object"
},
"network.Input": {
"properties": {
"data": {

View File

@ -13,9 +13,10 @@ import (
)
var fuzzingTestCases = map[string]testutils.TestCase{
"fuzz/fuzz-mode.yaml": &fuzzModeOverride{},
"fuzz/fuzz-type.yaml": &fuzzTypeOverride{},
"fuzz/fuzz-query.yaml": &httpFuzzQuery{},
"fuzz/fuzz-mode.yaml": &fuzzModeOverride{},
"fuzz/fuzz-type.yaml": &fuzzTypeOverride{},
"fuzz/fuzz-query.yaml": &httpFuzzQuery{},
"fuzz/fuzz-headless.yaml": &HeadlessFuzzingQuery{},
}
type httpFuzzQuery struct{}
@ -126,3 +127,23 @@ func (h *fuzzTypeOverride) Execute(filePath string) error {
}
return nil
}
// HeadlessFuzzingQuery tests fuzzing is working not in headless mode
type HeadlessFuzzingQuery struct{}
// Execute executes a test case and returns an error if occurred
func (h *HeadlessFuzzingQuery) Execute(filePath string) error {
router := httprouter.New()
router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
resp := fmt.Sprintf("<html><body>%s</body></html>", r.URL.Query().Get("url"))
fmt.Fprint(w, resp)
})
ts := httptest.NewTLSServer(router)
defer ts.Close()
got, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL+"?url=https://scanme.sh", debug, "-headless")
if err != nil {
return err
}
return expectResultsCount(got, 2)
}

View File

@ -11,12 +11,13 @@ import (
)
var headlessTestcases = map[string]testutils.TestCase{
"headless/headless-basic.yaml": &headlessBasic{},
"headless/headless-header-action.yaml": &headlessHeaderActions{},
"headless/headless-extract-values.yaml": &headlessExtractValues{},
"headless/headless-payloads.yaml": &headlessPayloads{},
"headless/variables.yaml": &headlessVariables{},
"headless/file-upload.yaml": &headlessFileUpload{},
"headless/headless-basic.yaml": &headlessBasic{},
"headless/headless-header-action.yaml": &headlessHeaderActions{},
"headless/headless-extract-values.yaml": &headlessExtractValues{},
"headless/headless-payloads.yaml": &headlessPayloads{},
"headless/variables.yaml": &headlessVariables{},
"headless/file-upload.yaml": &headlessFileUpload{},
"headless/headless-header-status-test.yaml": &headlessHeaderStatus{},
}
type headlessBasic struct{}
@ -158,3 +159,15 @@ func (h *headlessFileUpload) Execute(filePath string) error {
return expectResultsCount(results, 1)
}
type headlessHeaderStatus struct{}
// Execute executes a test case and returns an error if occurred
func (h *headlessHeaderStatus) Execute(filePath string) error {
results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "https://scanme.sh", debug, "-headless")
if err != nil {
return err
}
return expectResultsCount(results, 1)
}

View File

@ -39,6 +39,7 @@ var (
"offlineHttp": offlineHttpTestcases,
"customConfigDir": customConfigDirTestCases,
"fuzzing": fuzzingTestCases,
"multi": multiProtoTestcases,
}
// For debug purposes

View File

@ -0,0 +1,20 @@
package main
import "github.com/projectdiscovery/nuclei/v2/pkg/testutils"
var multiProtoTestcases = map[string]testutils.TestCase{
"multi/dynamic-values.yaml": &multiProtoDynamicExtractor{},
"multi/evaluate-variables.yaml": &multiProtoDynamicExtractor{}, // Not a typo execution is same as above testcase
"multi/exported-response-vars.yaml": &multiProtoDynamicExtractor{}, // Not a typo execution is same as above testcase
}
type multiProtoDynamicExtractor struct{}
// Execute executes a test case and returns an error if occurred
func (h *multiProtoDynamicExtractor) Execute(templatePath string) error {
results, err := testutils.RunNucleiTemplateAndGetResults(templatePath, "blog.projectdiscovery.io", debug)
if err != nil {
return err
}
return expectResultsCount(results, 1)
}

View File

@ -38,7 +38,7 @@ func (matcher *Matcher) CompileMatchers() error {
}
// By default, match on body if user hasn't provided any specific items
if matcher.Part == "" {
if matcher.Part == "" && matcher.GetType() != DSLMatcher {
matcher.Part = "body"
}

View File

@ -1,31 +0,0 @@
package contextargs
// Args is a generic map with helpers
type Args map[string]interface{}
// Set a key with value
func (args Args) Set(key string, value interface{}) {
args[key] = value
}
// Get the value associated to a key
func (args Args) Get(key string) (interface{}, bool) {
value, ok := args[key]
return value, ok
}
// Has verifies if the map contains the key
func (args Args) Has(key string) bool {
_, ok := args[key]
return ok
}
// IsEmpty verifies if the map is empty
func (Args Args) IsEmpty() bool {
return len(Args) == 0
}
// create a new args map instance
func newArgs() map[string]interface{} {
return make(map[string]interface{})
}

View File

@ -2,9 +2,9 @@ package contextargs
import (
"net/http/cookiejar"
"sync"
"golang.org/x/exp/maps"
"github.com/projectdiscovery/gologger"
maputils "github.com/projectdiscovery/utils/maps"
)
// Context implements a shared context struct to share information across multiple templates within a workflow
@ -14,106 +14,68 @@ type Context struct {
// CookieJar shared within workflow's http templates
CookieJar *cookiejar.Jar
// Access to Args must use lock strategies to prevent data races
*sync.RWMutex
// Args is a workflow shared key-value store
args Args
args maputils.SyncLockMap[string, interface{}]
}
// Create a new contextargs instance
func New() *Context {
return &Context{MetaInput: &MetaInput{}}
return NewWithInput("")
}
// Create a new contextargs instance with input string
func NewWithInput(input string) *Context {
return &Context{MetaInput: &MetaInput{Input: input}}
}
func (ctx *Context) initialize() {
ctx.args = newArgs()
ctx.RWMutex = &sync.RWMutex{}
}
func (ctx *Context) set(key string, value interface{}) {
ctx.Lock()
defer ctx.Unlock()
ctx.args.Set(key, value)
jar, err := cookiejar.New(nil)
if err != nil {
gologger.Error().Msgf("Could not create cookie jar: %s\n", err)
}
return &Context{MetaInput: &MetaInput{Input: input}, CookieJar: jar, args: maputils.SyncLockMap[string, interface{}]{
Map: make(map[string]interface{}),
}}
}
// Set the specific key-value pair
func (ctx *Context) Set(key string, value interface{}) {
if !ctx.isInitialized() {
ctx.initialize()
if err := ctx.args.Set(key, value); err != nil {
gologger.Error().Msgf("contextargs: could not set key: %s\n", err)
}
ctx.set(key, value)
}
func (ctx *Context) isInitialized() bool {
return ctx.args != nil
}
func (ctx *Context) hasArgs() bool {
return ctx.isInitialized() && !ctx.args.IsEmpty()
}
func (ctx *Context) get(key string) (interface{}, bool) {
ctx.RLock()
defer ctx.RUnlock()
return ctx.args.Get(key)
}
// Get the value with specific key if exists
func (ctx *Context) Get(key string) (interface{}, bool) {
if !ctx.hasArgs() {
return nil, false
}
return ctx.get(key)
return ctx.args.Get(key)
}
func (ctx *Context) GetAll() Args {
if !ctx.hasArgs() {
return nil
}
return maps.Clone(ctx.args)
func (ctx *Context) GetAll() maputils.Map[string, interface{}] {
return ctx.args.GetAll()
}
func (ctx *Context) ForEach(f func(string, interface{})) {
ctx.RLock()
defer ctx.RUnlock()
for k, v := range ctx.args {
f(k, v)
func (ctx *Context) ForEach(f func(string, interface{}) error) {
if err := ctx.args.Iterate(f); err != nil {
gologger.Error().Msgf("contextargs: could not iterate: %s\n", err)
}
}
func (ctx *Context) has(key string) bool {
ctx.RLock()
defer ctx.RUnlock()
return ctx.args.Has(key)
// Merge merges the map into the contextargs
func (ctx *Context) Merge(m map[string]interface{}) {
if err := ctx.args.Merge(m); err != nil {
gologger.Error().Msgf("contextargs: could not merge: %s\n", err)
}
}
// Has check if the key exists
func (ctx *Context) Has(key string) bool {
return ctx.hasArgs() && ctx.has(key)
return ctx.args.Has(key)
}
func (ctx *Context) HasArgs() bool {
return ctx.hasArgs()
return !ctx.args.IsEmpty()
}
func (ctx *Context) Clone() *Context {
newCtx := &Context{
MetaInput: ctx.MetaInput.Clone(),
RWMutex: ctx.RWMutex,
args: ctx.args,
args: *ctx.args.Clone(),
CookieJar: ctx.CookieJar,
}
return newCtx

View File

@ -65,8 +65,9 @@ func (e *Executer) Execute(input *contextargs.Context) (bool, error) {
dynamicValues := make(map[string]interface{})
if input.HasArgs() {
input.ForEach(func(key string, value interface{}) {
input.ForEach(func(key string, value interface{}) error {
dynamicValues[key] = value
return nil
})
}
previous := make(map[string]interface{})
@ -79,6 +80,10 @@ func (e *Executer) Execute(input *contextargs.Context) (bool, error) {
}
err := req.ExecuteWithResults(inputItem, dynamicValues, previous, func(event *output.InternalWrappedEvent) {
if event == nil {
// ideally this should never happen since protocol exits on error and callback is not called
return
}
ID := req.GetID()
if ID != "" {
builder := &strings.Builder{}
@ -125,8 +130,9 @@ func (e *Executer) Execute(input *contextargs.Context) (bool, error) {
func (e *Executer) ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error {
dynamicValues := make(map[string]interface{})
if input.HasArgs() {
input.ForEach(func(key string, value interface{}) {
input.ForEach(func(key string, value interface{}) error {
dynamicValues[key] = value
return nil
})
}
previous := make(map[string]interface{})

View File

@ -73,10 +73,7 @@ func (rule *Rule) buildQueryInput(input *ExecuteRuleInput, parsed *urlutil.URL,
req.Header.Set("User-Agent", uarand.GetRandom())
} else {
req = input.BaseRequest.Clone(context.TODO())
//TODO: abstract below 3 lines with `req.UpdateURL(xx *urlutil.URL)`
req.URL = parsed
req.Request.URL = parsed.URL
req.Update()
req.SetURL(parsed)
}
request := GeneratedRequest{
Request: req,

View File

@ -5,6 +5,7 @@ import (
"strings"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
mapsutil "github.com/projectdiscovery/utils/maps"
)
// EnableVarDump enables var dump for debugging optionally
@ -21,7 +22,11 @@ func DumpVariables(data map[string]interface{}) string {
buffer.Grow(len(data) * 78) // grow buffer to an approximate size
builder := &strings.Builder{}
for k, v := range data {
// sort keys for deterministic output
keys := mapsutil.GetSortedKeys(data)
for _, k := range keys {
v := data[k]
valueString := types.ToString(v)
counter++

View File

@ -53,7 +53,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
// optionvars are vars passed from CLI or env variables
optionVars := generators.BuildPayloadFromOptions(request.options.Options)
// merge with metadata (eg. from workflow context)
vars = generators.MergeMaps(vars, metadata, optionVars)
vars = generators.MergeMaps(vars, metadata, optionVars, request.options.TemplateCtx.GetAll())
variablesMap := request.options.Variables.Evaluate(vars)
vars = generators.MergeMaps(vars, variablesMap, request.options.Constants)
@ -149,12 +149,17 @@ func (request *Request) execute(domain string, metadata, previous output.Interna
// Create the output event
outputEvent := request.responseToDSLMap(compiledRequest, response, domain, question, traceData)
// expose response variables in proto_var format
// this is no-op if the template is not a multi protocol template
request.options.AddTemplateVars(request.Type(), outputEvent)
for k, v := range previous {
outputEvent[k] = v
}
for k, v := range vars {
outputEvent[k] = v
}
// add variables from template context before matching/extraction
outputEvent = generators.MergeMaps(outputEvent, request.options.TemplateCtx.GetAll())
event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse)
dumpResponse(event, request, request.options, response.String(), question)

View File

@ -19,6 +19,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
@ -233,6 +234,8 @@ func (request *Request) findMatchesWithReader(reader io.Reader, input, filePath
for k, v := range previous {
dslMap[k] = v
}
// add template context variables to DSL map
dslMap = generators.MergeMaps(dslMap, request.options.TemplateCtx.GetAll())
discardEvent := eventcreator.CreateEvent(request, dslMap, isResponseDebug)
newOpResult := discardEvent.OperatorsResult
if newOpResult != nil {

View File

@ -1,6 +1,7 @@
package engine
import (
"fmt"
"net/url"
"strings"
"sync"
@ -78,10 +79,19 @@ func (i *Instance) Run(baseURL *url.URL, actions []*Action, payloads map[string]
return nil, nil, err
}
//FIXME: this is a hack, make sure to fix this in the future. See: https://github.com/go-rod/rod/issues/188
var e proto.NetworkResponseReceived
wait := page.WaitEvent(&e)
data, err := createdPage.ExecuteActions(baseURL, actions)
if err != nil {
return nil, nil, err
}
wait()
data["header"] = headersToString(e.Response.Headers)
data["status_code"] = fmt.Sprint(e.Response.Status)
return data, createdPage, nil
}
@ -178,3 +188,15 @@ func containsAnyModificationActionType(actionTypes ...ActionType) bool {
}
return false
}
// headersToString converts network headers to string
func headersToString(headers proto.NetworkHeaders) string {
builder := &strings.Builder{}
for header, value := range headers {
builder.WriteString(header)
builder.WriteString(": ")
builder.WriteString(value.String())
builder.WriteRune('\n')
}
return builder.String()
}

View File

@ -7,6 +7,7 @@ import (
useragent "github.com/projectdiscovery/nuclei/v2/pkg/model/types/userAgent"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/fuzz"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine"
fileutil "github.com/projectdiscovery/utils/file"
@ -54,6 +55,9 @@ type Request struct {
// cache any variables that may be needed for operation.
options *protocols.ExecutorOptions
generator *generators.PayloadGenerator
// Fuzzing describes schema to fuzz headless requests
Fuzzing []*fuzz.Rule `yaml:"fuzzing,omitempty" json:"fuzzing,omitempty" jsonschema:"title=fuzzin rules for http fuzzing,description=Fuzzing describes rule schema to fuzz headless requests"`
}
// RequestPartDefinitions contains a mapping of request part definitions and their
@ -129,6 +133,21 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
request.CompiledOperators = compiled
}
request.options = options
if len(request.Fuzzing) > 0 {
for _, rule := range request.Fuzzing {
if fuzzingMode := options.Options.FuzzingMode; fuzzingMode != "" {
rule.Mode = fuzzingMode
}
if fuzzingType := options.Options.FuzzingType; fuzzingType != "" {
rule.Type = fuzzingType
}
if err := rule.Compile(request.generator, request.options); err != nil {
return errors.Wrap(err, "could not compile fuzzing rule")
}
}
}
return nil
}

View File

@ -1,6 +1,7 @@
package headless
import (
"strconv"
"time"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
@ -20,6 +21,12 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat
}
switch matcher.GetType() {
case matchers.StatusMatcher:
statusCode, ok := getStatusCode(data)
if !ok {
return false, []string{}
}
return matcher.Result(matcher.MatchStatusCode(statusCode)), []string{}
case matchers.SizeMatcher:
return matcher.Result(matcher.MatchSize(len(itemStr))), []string{}
case matchers.WordsMatcher:
@ -34,6 +41,24 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat
return false, []string{}
}
func getStatusCode(data map[string]interface{}) (int, bool) {
statusCodeValue, ok := data["status_code"]
if !ok {
return 0, false
}
statusCodeStr, ok := statusCodeValue.(string)
if !ok {
return 0, false
}
statusCode, err := strconv.Atoi(statusCodeStr)
if err != nil {
return 0, false
}
return statusCode, true
}
// Extract performs extracting operation for an extractor on model and returns true or false.
func (request *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} {
itemStr, ok := request.getMatchPart(extractor.Part, data)
@ -58,6 +83,8 @@ func (request *Request) getMatchPart(part string, data output.InternalEvent) (st
part = "data"
case "history":
part = "history"
case "header":
part = "header"
}
item, ok := data[part]
@ -70,12 +97,14 @@ func (request *Request) getMatchPart(part string, data output.InternalEvent) (st
}
// responseToDSLMap converts a headless response to a map for use in DSL matching
func (request *Request) responseToDSLMap(resp, req, host, matched string, history string) output.InternalEvent {
func (request *Request) responseToDSLMap(resp, headers, status_code, req, host, matched string, history string) output.InternalEvent {
return output.InternalEvent{
"host": host,
"matched": matched,
"req": req,
"data": resp,
"header": headers,
"status_code": status_code,
"history": history,
"type": request.Type().String(),
"template-id": request.options.TemplateID,

View File

@ -1,6 +1,7 @@
package headless
import (
"io"
"net/url"
"strings"
"time"
@ -12,6 +13,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/fuzz"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
@ -19,6 +21,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/vardump"
protocolutils "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils"
templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
urlutil "github.com/projectdiscovery/utils/url"
)
var _ protocols.Request = &Request{}
@ -39,7 +42,8 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
vars := protocolutils.GenerateVariablesWithContextArgs(input, false)
payloads := generators.BuildPayloadFromOptions(request.options.Options)
values := generators.MergeMaps(vars, metadata, payloads)
// add templatecontext variables to varMap
values := generators.MergeMaps(vars, metadata, payloads, request.options.TemplateCtx.GetAll())
variablesMap := request.options.Variables.Evaluate(values)
payloads = generators.MergeMaps(variablesMap, payloads, request.options.Constants)
@ -51,7 +55,10 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
gotmatches = results.OperatorsResult.Matched
}
}
// verify if fuzz elaboration was requested
if len(request.Fuzzing) > 0 {
return request.executeFuzzingRule(inputURL, payloads, previous, wrappedCallback)
}
if request.generator != nil {
iterator := request.generator.NewIterator()
for {
@ -129,7 +136,10 @@ func (request *Request) executeRequestWithPayloads(inputURL string, payloads map
responseBody, _ = html.HTML()
}
outputEvent := request.responseToDSLMap(responseBody, reqBuilder.String(), inputURL, inputURL, page.DumpHistory())
outputEvent := request.responseToDSLMap(responseBody, out["header"], out["status_code"], reqBuilder.String(), inputURL, inputURL, page.DumpHistory())
// add response fields to template context and merge templatectx variables to output event
request.options.AddTemplateVars(request.Type(), outputEvent)
outputEvent = generators.MergeMaps(outputEvent, request.options.TemplateCtx.GetAll())
for k, v := range out {
outputEvent[k] = v
}
@ -166,3 +176,38 @@ func dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.
gologger.Debug().Msgf("[%s] Dumped Headless response for %s\n\n%s", requestOptions.TemplateID, input, highlightedResponse)
}
}
// executeFuzzingRule executes a fuzzing rule in the template request
func (request *Request) executeFuzzingRule(inputURL string, payloads map[string]interface{}, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
// check for operator matches by wrapping callback
gotmatches := false
fuzzRequestCallback := func(gr fuzz.GeneratedRequest) bool {
if gotmatches && (request.StopAtFirstMatch || request.options.Options.StopAtFirstMatch || request.options.StopAtFirstMatch) {
return true
}
if err := request.executeRequestWithPayloads(gr.Request.URL.String(), gr.DynamicValues, previous, callback); err != nil {
return false
}
return true
}
parsedURL, err := urlutil.Parse(inputURL)
if err != nil {
return errors.Wrap(err, "could not parse url")
}
for _, rule := range request.Fuzzing {
err := rule.Execute(&fuzz.ExecuteRuleInput{
URL: parsedURL,
Callback: fuzzRequestCallback,
Values: payloads,
BaseRequest: nil,
})
if err == io.EOF {
return nil
}
if err != nil {
return errors.Wrap(err, "could not execute rule")
}
}
return nil
}

View File

@ -70,6 +70,9 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context,
// value of `reqData` depends on the type of request specified in template
// 1. If request is raw request = reqData contains raw request (i.e http request dump)
// 2. If request is Normal ( simply put not a raw request) (Ex: with placeholders `path`) = reqData contains relative path
// add template context values to dynamicValues (this takes care of self-contained and other types of requests)
dynamicValues = generators.MergeMaps(dynamicValues, r.request.options.TemplateCtx.GetAll())
if r.request.SelfContained {
return r.makeSelfContainedRequest(ctx, reqData, payloads, dynamicValues)
}

View File

@ -11,8 +11,8 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/fuzz"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/fuzz"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool"
"github.com/projectdiscovery/rawhttp"
"github.com/projectdiscovery/retryablehttp-go"

View File

@ -24,12 +24,12 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/fuzz"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/fuzz"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/signer"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/signerpool"
@ -717,6 +717,9 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
finalEvent := make(output.InternalEvent)
outputEvent := request.responseToDSLMap(response.resp, input.MetaInput.Input, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(response.fullResponse), tostring.UnsafeToString(response.body), tostring.UnsafeToString(response.headers), duration, generatedRequest.meta)
// add response fields to template context and merge templatectx variables to output event
request.options.AddTemplateVars(request.Type(), outputEvent)
outputEvent = generators.MergeMaps(outputEvent, request.options.TemplateCtx.GetAll())
if i := strings.LastIndex(hostname, ":"); i != -1 {
hostname = hostname[:i]
}

View File

@ -0,0 +1,53 @@
## multi protocol execution
### Implementation
when template is unmarshalled, if it uses more than one protocol, it will be converted to a multi protocol
and the order of the protocols will be preserved as they were in the template and are stored in Request.Queue
when template is compiled , we iterate over queue and compile all the requests in the queue
### Execution
when multi protocol template is executed , all protocol requests present in Queue are executed in order
and dynamic values extracted are added to template context.
- Protocol Responses
apart from extracted `internal:true` values response fields/values of protocol are added to template context at `ExecutorOptions.TemplateCtx`
which takes care of sync and other issues if any. all response fields are prefixed with template type prefix ex: `ssl_subject_dn`
### Other Methods
Such templates are usually used when a particular vulnerability requires more than one protocol to be executed
and in such cases the final result is core of the logic hence all methods such as
Ex: MakeResultEventItem, MakeResultEvent, GetCompiledOperators
are not implemented in multi protocol and just call the same method on last protocol in queue
### Adding New Protocol to multi protocol execution logic
while logic/implementation of multi protocol execution is abstracted. it requires 3 statements to be added in newly implemented protocol
to make response fields of that protocol available to global context
- Add `request.options.TemplateCtx.GetAll()` to variablesMap in `ExecuteWithResults` Method just above `request.options.Variables.Evaluate`
```go
// example
values := generators.MergeMaps(payloadValues, hostnameVariables, request.options.TemplateCtx.GetAll())
variablesMap := request.options.Variables.Evaluate(values)
```
- Add all response fields to template context just after response map is available
```go
outputEvent := request.responseToDSLMap(compiledRequest, response, domain, question, traceData)
// expose response variables in proto_var format
// this is no-op if the template is not a multi protocol template
request.options.AddTemplateVars(request.Type(), outputEvent)
```
- Append all available template context values to outputEvent
```go
// add variables from template context before matching/extraction
outputEvent = generators.MergeMaps(outputEvent, request.options.TemplateCtx.GetAll())
```
adding these 3 statements takes care of all logic related to multi protocol execution
### Exceptions
- statements 1 & 2 are intentionally skipped in `file` protocol to avoid redundant data
- file/dir input paths don't contain variables or are used in path (yet)
- since files are processed by scanning each line. adding statement 2 will unintenionally load all file(s) data

View File

@ -0,0 +1,6 @@
package multi
// multi is a wrapper protocol Request that allows multiple protocols requests to be executed
// multi protocol is just a wrapper so it should/does not include any protocol specific code

View File

@ -0,0 +1,172 @@
package multi
import (
"strconv"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
errorutil "github.com/projectdiscovery/utils/errors"
)
var _ protocols.Request = &Request{}
// refer doc.go for package description , limitations etc
// Request contains a multi protocol request
type Request struct {
// description: |
// ID is the unique id for the template.
//
// #### Good IDs
//
// A good ID uniquely identifies what the requests in the template
// are doing. Let's say you have a template that identifies a git-config
// file on the webservers, a good name would be `git-config-exposure`. Another
// example name is `azure-apps-nxdomain-takeover`.
// examples:
// - name: ID Example
// value: "\"CVE-2021-19520\""
ID string `yaml:"id" json:"id" jsonschema:"title=id of the template,description=The Unique ID for the template,example=cve-2021-19520,pattern=^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$"`
// description: |
// Info contains metadata information about the template.
// examples:
// - value: exampleInfoStructure
Info model.Info `yaml:"info" json:"info" jsonschema:"title=info for the template,description=Info contains metadata for the template"`
// Queue is queue of all protocols present in the template
Queue []protocols.Request `yaml:"-" json:"-"`
// request executor options
options *protocols.ExecutorOptions `yaml:"-" json:"-"`
}
// getLastRequest returns the last request in the queue
func (r *Request) getLastRequest() protocols.Request {
if len(r.Queue) == 0 {
return nil
}
return r.Queue[len(r.Queue)-1]
}
// Requests returns the total number of requests template will send
func (r *Request) Requests() int {
var count int
for _, protocol := range r.Queue {
count += protocol.Requests()
}
return count
}
// Compile compiles the protocol request for further execution.
func (r *Request) Compile(executerOptions *protocols.ExecutorOptions) error {
r.options = executerOptions
r.options.TemplateCtx = contextargs.New()
r.options.ProtocolType = types.MultiProtocol
for _, protocol := range r.Queue {
if err := protocol.Compile(r.options); err != nil {
return errorutil.NewWithErr(err).Msgf("failed to compile protocol %s", protocol.Type())
}
}
return nil
}
// GetID returns the unique template ID
func (r *Request) GetID() string {
return r.ID
}
// Match executes matcher on model and returns true or false (used for clustering if request supports clustering)
func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) {
return protocols.MakeDefaultMatchFunc(data, matcher)
}
// Extract performs extracting operation for an extractor on model and returns true or false (used for clustering if request supports clustering)
func (r *Request) Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} {
return protocols.MakeDefaultExtractFunc(data, matcher)
}
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (r *Request) ExecuteWithResults(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
var finalProtoEvent *output.InternalWrappedEvent
// callback to process results from all protocols
multiProtoCallback := func(event *output.InternalWrappedEvent) {
finalProtoEvent = event
// export dynamic values from operators (i.e internal:true)
if event.OperatorsResult != nil && len(event.OperatorsResult.DynamicValues) > 0 {
for k, v := range event.OperatorsResult.DynamicValues {
// TBD: iterate-all is only supported in `http` protocol
// we either need to add support for iterate-all in other protocols or implement a different logic (specific to template context)
// currently if dynamic value array only contains one value we replace it with the value
if len(v) == 1 {
r.options.TemplateCtx.Set(k, v[0])
} else {
// Note: if extracted value contains multiple values then they can be accessed by indexing
// ex: if values are dynamic = []string{"a","b","c"} then they are available as
// dynamic = "a" , dynamic1 = "b" , dynamic2 = "c"
// we intentionally omit first index for unknown situations (where no of extracted values are not known)
for i, val := range v {
if i == 0 {
r.options.TemplateCtx.Set(k, val)
} else {
r.options.TemplateCtx.Set(k+strconv.Itoa(i), val)
}
}
}
}
}
}
// template context: contains values extracted using `internal` extractor from previous protocols
// these values are extracted from each protocol in queue and are passed to next protocol in queue
// instead of adding seperator field to handle such cases these values are appended to `dynamicValues` (which are meant to be used in workflows)
// this makes it possible to use multi protocol templates in workflows
// Note: internal extractor values take precedence over dynamicValues from workflows (i.e other templates in workflow)
// execute all protocols in the queue
for _, req := range r.Queue {
err := req.ExecuteWithResults(input, dynamicValues, previous, multiProtoCallback)
// if error skip execution of next protocols
if err != nil {
return err
}
}
// Review: how to handle events of multiple protocols in a single template
// currently the outer callback is only executed once (for the last protocol in queue)
// due to workflow logic at https://github.com/projectdiscovery/nuclei/blob/main/v2/pkg/protocols/common/executer/executer.go#L150
// this causes addition of duplicated / unncessary variables with prefix template_id_all_variables
callback(finalProtoEvent)
return nil
}
// MakeResultEventItem creates a result event from internal wrapped event. Intended to be used by MakeResultEventItem internally
func (r *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
if r.getLastRequest() == nil {
return nil
}
return r.getLastRequest().MakeResultEventItem(wrapped)
}
// MakeResultEvent creates a flat list of result events from an internal wrapped event, based on successful matchers and extracted data
func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent {
return protocols.MakeDefaultResultEvent(r.getLastRequest(), wrapped)
}
// GetCompiledOperators returns a list of the compiled operators
func (r *Request) GetCompiledOperators() []*operators.Operators {
last := r.getLastRequest()
if last == nil {
return nil
}
return last.GetCompiledOperators()
}
// Type returns the type of the protocol request
func (r *Request) Type() types.ProtocolType {
return types.MultiProtocol
}

View File

@ -0,0 +1,75 @@
package multi_test
import (
"context"
"log"
"testing"
"time"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk"
"github.com/projectdiscovery/nuclei/v2/pkg/parsers"
"github.com/projectdiscovery/nuclei/v2/pkg/progress"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
"github.com/projectdiscovery/ratelimit"
"github.com/stretchr/testify/require"
)
var executerOpts protocols.ExecutorOptions
func setup() {
options := testutils.DefaultOptions
testutils.Init(options)
progressImpl, _ := progress.NewStatsTicker(0, false, false, false, false, 0)
executerOpts = protocols.ExecutorOptions{
Output: testutils.NewMockOutputWriter(),
Options: options,
Progress: progressImpl,
ProjectFile: nil,
IssuesClient: nil,
Browser: nil,
Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory),
RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second),
}
workflowLoader, err := parsers.NewLoader(&executerOpts)
if err != nil {
log.Fatalf("Could not create workflow loader: %s\n", err)
}
executerOpts.WorkflowLoader = workflowLoader
}
func TestMultiProtoWithDynamicExtractor(t *testing.T) {
setup()
Template, err := templates.Parse("testcases/multiprotodynamic.yaml", nil, executerOpts)
require.Nil(t, err, "could not parse template")
require.Equal(t, 2, len(Template.MultiProtoRequest.Queue))
err = Template.MultiProtoRequest.Compile(&executerOpts)
require.Nil(t, err, "could not compile template")
gotresults, err := Template.Executer.Execute(contextargs.NewWithInput("blog.projectdiscovery.io"))
require.Nil(t, err, "could not execute template")
require.True(t, gotresults)
}
func TestMultiProtoWithProtoPrefix(t *testing.T) {
setup()
Template, err := templates.Parse("testcases/multiprotowithprefix.yaml", nil, executerOpts)
require.Nil(t, err, "could not parse template")
require.Equal(t, 3, len(Template.MultiProtoRequest.Queue))
err = Template.MultiProtoRequest.Compile(&executerOpts)
require.Nil(t, err, "could not compile template")
require.True(t, len(Template.MultiProtoRequest.GetCompiledOperators()) > 0, "could not compile operators")
gotresults, err := Template.Executer.Execute(contextargs.NewWithInput("blog.projectdiscovery.io"))
require.Nil(t, err, "could not execute template")
require.True(t, gotresults)
}

View File

@ -0,0 +1,29 @@
id: dns-http-dynamic-values
info:
name: multi protocol request with dynamic values
author: pdteam
severity: info
dns:
- name: "{{FQDN}}" # DNS Request
type: cname
extractors:
- type: dsl
name: blogid
dsl:
- trim_suffix(cname,'.ghost.io.')
internal: true
http:
- method: GET # http request
path:
- "{{BaseURL}}"
matchers:
- type: dsl
dsl:
- contains(body,'ProjectDiscovery.io') # check for http string
- blogid == 'projectdiscovery' # check for cname (extracted information from dns response)
condition: and

View File

@ -0,0 +1,26 @@
id: dns-http-proto-prefix
info:
name: multi protocol request with dynamic values
author: pdteam
severity: info
dns:
- name: "{{FQDN}}" # DNS Request
type: cname
ssl:
- address: "{{Hostname}}" # ssl request
http:
- method: GET # http request
path:
- "{{BaseURL}}"
matchers:
- type: dsl
dsl:
- contains(http_body,'ProjectDiscovery.io') # check for http string
- trim_suffix(dns_cname,'.ghost.io.') == 'projectdiscovery' # check for cname (extracted information from dns response)
- ssl_subject_cn == 'blog.projectdiscovery.io'
condition: and

View File

@ -54,6 +54,8 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
return errors.Wrap(err, "could not get address from url")
}
variables := protocolutils.GenerateVariables(address, false, nil)
// add template ctx variables to varMap
variables = generators.MergeMaps(variables, request.options.TemplateCtx.GetAll())
variablesMap := request.options.Variables.Evaluate(variables)
variables = generators.MergeMaps(variablesMap, variables, request.options.Constants)
@ -269,6 +271,9 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
response := responseBuilder.String()
outputEvent := request.responseToDSLMap(reqBuilder.String(), string(final[:n]), response, input, actualAddress)
// add response fields to template context and merge templatectx variables to output event
request.options.AddTemplateVars(request.Type(), outputEvent)
outputEvent = generators.MergeMaps(outputEvent, request.options.TemplateCtx.GetAll())
outputEvent["ip"] = request.dialer.GetDialedIP(hostname)
if request.options.StopAtFirstMatch {
outputEvent["stop-at-first-match"] = true

View File

@ -14,6 +14,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring"
templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
@ -87,6 +88,9 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata
}
outputEvent := request.responseToDSLMap(resp, data, data, data, tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(body), headersToString(resp.Header), 0, nil)
// add response fields to template context and merge templatectx variables to output event
request.options.AddTemplateVars(request.Type(), outputEvent)
outputEvent = generators.MergeMaps(outputEvent, request.options.TemplateCtx.GetAll())
outputEvent["ip"] = ""
for k, v := range previous {
outputEvent[k] = v

View File

@ -2,6 +2,7 @@ package protocols
import (
"github.com/projectdiscovery/ratelimit"
stringsutil "github.com/projectdiscovery/utils/strings"
"github.com/logrusorgru/aurora"
@ -85,6 +86,41 @@ type ExecutorOptions struct {
Colorizer aurora.Aurora
WorkflowLoader model.WorkflowLoader
ResumeCfg *types.ResumeCfg
// TemplateContext (contains all variables that are templatescoped i.e multi protocol)
// only used in case of multi protocol templates
TemplateCtx *contextargs.Context
// ProtocolType is the type of the template
ProtocolType templateTypes.ProtocolType
}
// AddTemplateVars adds vars to template context with given template type as prefix
// this method is no-op if template is not multi protocol
func (e *ExecutorOptions) AddTemplateVars(templateType templateTypes.ProtocolType, vars map[string]interface{}) {
if e.ProtocolType != templateTypes.MultiProtocol {
// no-op if not multi protocol template
return
}
for k, v := range vars {
if !stringsutil.EqualFoldAny(k, "template-id", "template-info", "template-path") {
if templateType < templateTypes.InvalidProtocol {
k = templateType.String() + "_" + k
}
e.TemplateCtx.Set(k, v)
}
}
}
// AddTemplateVar adds given var to template context with given template type as prefix
// this method is no-op if template is not multi protocol
func (e *ExecutorOptions) AddTemplateVar(prefix, key string, value interface{}) {
if e.ProtocolType != templateTypes.MultiProtocol {
// no-op if not multi protocol template
return
}
if prefix != "" {
key = prefix + "_" + key
}
e.TemplateCtx.Set(key, value)
}
// Copy returns a copy of the executeroptions structure

View File

@ -24,7 +24,6 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/vardump"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils"
protocolutils "github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils"
templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
@ -188,7 +187,8 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
payloadValues["Port"] = port
hostnameVariables := protocolutils.GenerateDNSVariables(hostname)
values := generators.MergeMaps(payloadValues, hostnameVariables)
// add template context variables to varMap
values := generators.MergeMaps(payloadValues, hostnameVariables, request.options.TemplateCtx.GetAll())
variablesMap := request.options.Variables.Evaluate(values)
payloadValues = generators.MergeMaps(variablesMap, payloadValues, request.options.Constants)
@ -267,10 +267,11 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
// if field is not exported f.IsZero() , f.Value() will panic
continue
}
tag := utils.CleanStructFieldJSONTag(f.Tag("json"))
tag := protocolutils.CleanStructFieldJSONTag(f.Tag("json"))
if tag == "" || f.IsZero() {
continue
}
request.options.AddTemplateVar(request.Type().String(), tag, f.Value())
data[tag] = f.Value()
}
@ -285,13 +286,16 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
// if field is not exported f.IsZero() , f.Value() will panic
continue
}
tag := utils.CleanStructFieldJSONTag(f.Tag("json"))
tag := protocolutils.CleanStructFieldJSONTag(f.Tag("json"))
if tag == "" || f.IsZero() {
continue
}
request.options.AddTemplateVar(request.Type().String(), tag, f.Value())
data[tag] = f.Value()
}
// add response fields ^ to template context and merge templatectx variables to output event
data = generators.MergeMaps(data, request.options.TemplateCtx.GetAll())
event := eventcreator.CreateEvent(request, data, requestOptions.Options.Debug || requestOptions.Options.DebugResponse)
if requestOptions.Options.Debug || requestOptions.Options.DebugResponse || requestOptions.Options.StoreResponse {
msg := fmt.Sprintf("[%s] Dumped SSL response for %s", requestOptions.TemplateID, input.MetaInput.Input)

View File

@ -175,7 +175,8 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam
}
defaultVars := protocolutils.GenerateVariables(parsed, false, nil)
optionVars := generators.BuildPayloadFromOptions(request.options.Options)
variables := request.options.Variables.Evaluate(generators.MergeMaps(defaultVars, optionVars, dynamicValues))
// add templatecontext variables to varMap
variables := request.options.Variables.Evaluate(generators.MergeMaps(defaultVars, optionVars, dynamicValues, request.options.TemplateCtx.GetAll()))
payloadValues := generators.MergeMaps(variables, defaultVars, optionVars, dynamicValues, request.options.Constants)
requestOptions := request.options
@ -254,12 +255,6 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam
gologger.Verbose().Msgf("Sent Websocket request to %s", input)
data := make(map[string]interface{})
for k, v := range previous {
data[k] = v
}
for k, v := range events {
data[k] = v
}
data["type"] = request.Type().String()
data["success"] = "true"
@ -269,6 +264,17 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam
data["matched"] = addressToDial
data["ip"] = request.dialer.GetDialedIP(hostname)
// add response fields to template context and merge templatectx variables to output event
request.options.AddTemplateVars(request.Type(), data)
data = generators.MergeMaps(data, request.options.TemplateCtx.GetAll())
for k, v := range previous {
data[k] = v
}
for k, v := range events {
data[k] = v
}
event := eventcreator.CreateEventWithAdditionalOptions(request, data, requestOptions.Options.Debug || requestOptions.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) {
internalWrappedEvent.OperatorsResult.PayloadValues = payloadValues
})

View File

@ -90,7 +90,8 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
// generate variables
defaultVars := protocolutils.GenerateVariables(input.MetaInput.Input, false, nil)
optionVars := generators.BuildPayloadFromOptions(request.options.Options)
vars := request.options.Variables.Evaluate(generators.MergeMaps(defaultVars, optionVars, dynamicValues))
// add templatectx variables to varMap
vars := request.options.Variables.Evaluate(generators.MergeMaps(defaultVars, optionVars, dynamicValues, request.options.TemplateCtx.GetAll()))
variables := generators.MergeMaps(vars, defaultVars, optionVars, dynamicValues, request.options.Constants)
@ -132,6 +133,10 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
data["host"] = query
data["response"] = jsonDataString
// add response fields to template context and merge templatectx variables to output event
request.options.AddTemplateVars(request.Type(), data)
data = generators.MergeMaps(data, request.options.TemplateCtx.GetAll())
event := eventcreator.CreateEvent(request, data, request.options.Options.Debug || request.options.Options.DebugResponse)
if request.options.Options.Debug || request.options.Options.DebugResponse {
gologger.Debug().Msgf("[%s] Dumped WHOIS response for %s", request.options.TemplateID, query)

View File

@ -10,6 +10,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/executer"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/offlinehttp"
"github.com/projectdiscovery/nuclei/v2/pkg/templates/cache"
@ -122,30 +123,31 @@ func (template *Template) compileProtocolRequests(options protocols.ExecutorOpti
var requests []protocols.Request
if len(template.RequestsDNS) > 0 {
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsDNS)...)
}
if len(template.RequestsFile) > 0 {
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsFile)...)
}
if len(template.RequestsNetwork) > 0 {
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsNetwork)...)
}
if len(template.RequestsHTTP) > 0 {
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsHTTP)...)
}
if len(template.RequestsHeadless) > 0 && options.Options.Headless {
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsHeadless)...)
}
if len(template.RequestsSSL) > 0 {
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsSSL)...)
}
if len(template.RequestsWebsocket) > 0 {
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsWebsocket)...)
}
if len(template.RequestsWHOIS) > 0 {
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsWHOIS)...)
if len(template.MultiProtoRequest.Queue) > 0 {
template.MultiProtoRequest.ID = template.ID
template.MultiProtoRequest.Info = template.Info
requests = append(requests, &template.MultiProtoRequest)
} else {
switch {
case len(template.RequestsDNS) > 0:
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsDNS)...)
case len(template.RequestsFile) > 0:
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsFile)...)
case len(template.RequestsNetwork) > 0:
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsNetwork)...)
case len(template.RequestsHTTP) > 0:
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsHTTP)...)
case len(template.RequestsHeadless) > 0 && options.Options.Headless:
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsHeadless)...)
case len(template.RequestsSSL) > 0:
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsSSL)...)
case len(template.RequestsWebsocket) > 0:
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsWebsocket)...)
case len(template.RequestsWHOIS) > 0:
requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsWHOIS)...)
}
}
template.Executer = executer.NewExecuter(requests, &options)
return nil
}
@ -232,8 +234,10 @@ func ParseTemplateFromReader(reader io.Reader, preprocessor Preprocessor, option
options.Variables = template.Variables
}
// create empty context args for template scope
options.TemplateCtx = contextargs.New()
options.ProtocolType = template.Type()
options.Constants = template.Constants
// If no requests, and it is also not a workflow, return error.
if template.Requests() == 0 {
return nil, fmt.Errorf("no requests defined for %s", template.ID)

View File

@ -25,6 +25,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/variables"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
"github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/workflows"
"github.com/projectdiscovery/ratelimit"
@ -53,7 +54,6 @@ func setup() {
log.Fatalf("Could not create workflow loader: %s\n", err)
}
executerOpts.WorkflowLoader = workflowLoader
}
func Test_ParseFromURL(t *testing.T) {
@ -197,3 +197,16 @@ func Test_WrongTemplate(t *testing.T) {
require.Nil(t, got, "could not parse template")
require.ErrorContains(t, err, "no requests defined ")
}
func Test_Multiprotocol(t *testing.T) {
setup()
got, err := templates.Parse("tests/multiproto.yaml", nil, executerOpts)
require.Nil(t, err, "could not parse template")
require.Equal(t, 3, got.Requests())
require.Equal(t, types.MultiProtocol, got.Type())
got, err = templates.Parse("tests/multiproto.json", nil, executerOpts)
require.Nil(t, err, "could not parse template")
require.Equal(t, 3, got.Requests())
require.Equal(t, types.MultiProtocol, got.Type())
}

View File

@ -12,6 +12,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/file"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/multi"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/network"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/ssl"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/websocket"
@ -55,6 +56,7 @@ type Template struct {
// examples:
// - value: exampleNormalHTTPRequest
// RequestsWithHTTP is placeholder(internal) only, and should not be used instead use RequestsHTTP
// Deprecated: Use RequestsHTTP instead.
RequestsWithHTTP []*http.Request `yaml:"http,omitempty" json:"http,omitempty" jsonschema:"title=http requests to make,description=HTTP requests to make for the template"`
// description: |
// DNS contains the dns request to make in the template
@ -77,6 +79,7 @@ type Template struct {
// examples:
// - value: exampleNormalNetworkRequest
// RequestsWithTCP is placeholder(internal) only, and should not be used instead use RequestsNetwork
// Deprecated: Use RequestsNetwork instead.
RequestsWithTCP []*network.Request `yaml:"tcp,omitempty" json:"tcp,omitempty" jsonschema:"title=network(tcp) requests to make,description=Network requests to make for the template"`
// description: |
// Headless contains the headless request to make in the template.
@ -126,24 +129,16 @@ type Template struct {
// Verified defines if the template signature is digitally verified
Verified bool `yaml:"-" json:"-"`
}
// TemplateProtocols is a list of accepted template protocols
var TemplateProtocols = []string{
"dns",
"file",
"http",
"headless",
"network",
"workflow",
"ssl",
"websocket",
"whois",
// MultiProtoRequest (Internal) contains multi protocol request if multiple protocols are used
MultiProtoRequest multi.Request `yaml:"-" json:"-"`
}
// Type returns the type of the template
func (template *Template) Type() types.ProtocolType {
switch {
case len(template.MultiProtoRequest.Queue) > 0:
return types.MultiProtocol
case len(template.RequestsDNS) > 0:
return types.DNSProtocol
case len(template.RequestsFile) > 0:
@ -200,7 +195,62 @@ func (template *Template) UnmarshalYAML(unmarshal func(interface{}) error) error
if len(alias.RequestsWithTCP) > 0 {
template.RequestsNetwork = alias.RequestsWithTCP
}
return validate.New().Struct(template)
err = validate.New().Struct(template)
if err != nil {
return err
}
// check if the template contains a multi protocols
if template.isMultiProtocol() {
var tempmap yaml.MapSlice
err = unmarshal(&tempmap)
if err != nil {
return errorutil.NewWithErr(err).Msgf("failed to unmarshal multi protocol template %s", template.ID)
}
arr := []string{}
for _, v := range tempmap {
key, ok := v.Key.(string)
if !ok {
continue
}
arr = append(arr, key)
}
// add protocols to the protocol stack (the idea is to preserve the order of the protocols)
template.addProtocolsToQueue(arr...)
}
return nil
}
// Internal function to create a protocol stack from a template if the template is a multi protocol template
func (template *Template) addProtocolsToQueue(keys ...string) {
for _, key := range keys {
switch key {
case types.DNSProtocol.String():
template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsDNS)...)
case types.FileProtocol.String():
template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsFile)...)
case types.HTTPProtocol.String():
template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsHTTP)...)
case types.HeadlessProtocol.String():
template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsHeadless)...)
case types.NetworkProtocol.String():
template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsNetwork)...)
case types.SSLProtocol.String():
template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsSSL)...)
case types.WebsocketProtocol.String():
template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsWebsocket)...)
case types.WHOISProtocol.String():
template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsWHOIS)...)
}
}
}
// isMultiProtocol checks if the template is a multi protocol template
func (template *Template) isMultiProtocol() bool {
counter := len(template.RequestsDNS) + len(template.RequestsFile) +
len(template.RequestsHTTP) + len(template.RequestsHeadless) +
len(template.RequestsNetwork) + len(template.RequestsSSL) +
len(template.RequestsWebsocket) + len(template.RequestsWHOIS)
return counter > 1
}
// MarshalJSON forces recursive struct validation during marshal operation
@ -219,5 +269,22 @@ func (template *Template) UnmarshalJSON(data []byte) error {
return err
}
*template = Template(*alias)
return validate.New().Struct(template)
err = validate.New().Struct(template)
if err != nil {
return err
}
// check if template contains multiple protocols
if template.isMultiProtocol() {
var tempMap map[string]interface{}
err = json.Unmarshal(data, &tempMap)
if err != nil {
return errorutil.NewWithErr(err).Msgf("failed to unmarshal multi protocol template %s", template.ID)
}
arr := []string{}
for k := range tempMap {
arr = append(arr, k)
}
template.addProtocolsToQueue(arr...)
}
return nil
}

View File

@ -677,6 +677,10 @@ func init() {
TypeName: "http.Request",
FieldName: "fuzzing",
},
{
TypeName: "headless.Request",
FieldName: "fuzzing",
},
}
FUZZRuleDoc.Fields = make([]encoder.Doc, 7)
FUZZRuleDoc.Fields[0].Name = "type"
@ -1203,7 +1207,7 @@ func init() {
Value: "Headless response received from client (default)",
},
}
HEADLESSRequestDoc.Fields = make([]encoder.Doc, 7)
HEADLESSRequestDoc.Fields = make([]encoder.Doc, 8)
HEADLESSRequestDoc.Fields[0].Name = "id"
HEADLESSRequestDoc.Fields[0].Type = "string"
HEADLESSRequestDoc.Fields[0].Note = ""
@ -1239,6 +1243,11 @@ func init() {
HEADLESSRequestDoc.Fields[6].Note = ""
HEADLESSRequestDoc.Fields[6].Description = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found."
HEADLESSRequestDoc.Fields[6].Comments[encoder.LineComment] = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found."
HEADLESSRequestDoc.Fields[7].Name = "fuzzing"
HEADLESSRequestDoc.Fields[7].Type = "[]fuzz.Rule"
HEADLESSRequestDoc.Fields[7].Note = ""
HEADLESSRequestDoc.Fields[7].Description = "Fuzzing describes schema to fuzz headless requests"
HEADLESSRequestDoc.Fields[7].Comments[encoder.LineComment] = " Fuzzing describes schema to fuzz headless requests"
ENGINEActionDoc.Type = "engine.Action"
ENGINEActionDoc.Comments[encoder.LineComment] = " Action is an action taken by the browser to reach a navigation"

View File

@ -0,0 +1,41 @@
{
"id": "nuclei-multi-protocol",
"info": {
"name": "multi protocol support",
"author": "pdteam",
"severity": "info"
},
"dns": [
{
"name": "{{FQDN}}",
"type": "cname"
}
],
"ssl": [
{
"address": "{{Hostname}}"
}
],
"http": [
{
"method": "GET",
"path": [
"{{BaseURL}}"
],
"headers": {
"Host": "{{ssl_subject_cn}}",
"Metadata": "{{ssl_cipher}}"
},
"matchers": [
{
"type": "dsl",
"dsl": [
"http_status_code == 404",
"contains(dns_cname, 'github.io')"
],
"condition": "and"
}
]
}
]
}

View File

@ -0,0 +1,30 @@
id: nuclei-multi-protocol
info:
name: multi protocol support
author: pdteam
severity: info
dns:
- name: "{{FQDN}}" # dns request
type: cname
ssl:
- address: "{{Hostname}}" # ssl request
http:
- method: GET
path:
- "{{BaseURL}}" # http request
headers:
Host: "{{ssl_subject_cn}}" # host extracted from ssl request
Metadata: "{{ssl_cipher}}"
matchers:
- type: dsl
dsl:
# - contains(http_body,'File not found') # check for http string
- http_status_code == 404
- contains(dns_cname, 'github.io') # check for cname
condition: and

View File

@ -36,6 +36,8 @@ const (
WebsocketProtocol
// name:whois
WHOISProtocol
// name: multi
MultiProtocol
limit
InvalidProtocol
)
@ -52,6 +54,7 @@ var protocolMappings = map[ProtocolType]string{
SSLProtocol: "ssl",
WebsocketProtocol: "websocket",
WHOISProtocol: "whois",
MultiProtocol: "multi",
}
func GetSupportedProtocolTypes() ProtocolTypes {

View File

@ -16,6 +16,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/progress"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
)
@ -90,6 +91,7 @@ func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) *protoco
Browser: nil,
Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory),
RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second),
TemplateCtx: contextargs.New(),
}
return executerOpts
}