daffainfo
/
nuclei
mirror of https://github.com/daffainfo/nuclei.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #479 from projectdiscovery/iceman-refactor 

Multi-request condition support
dev
PD-Team 2021-01-16 15:46:53 +05:30 committed by GitHub
parent be9cf1179a 827fd99f01
commit ba3cab41c0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 99 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
v2/pkg/protocols/common/clusterer/executer.go
View File

@ -60,7 +60,7 @@ func (e *Executer) Execute(input string) (bool, error) {
var results bool
dynamicValues := make(map[string]interface{})
err := e.requests.ExecuteWithResults(input, dynamicValues, func(event *output.InternalWrappedEvent) {
err := e.requests.ExecuteWithResults(input, dynamicValues, nil, func(event *output.InternalWrappedEvent) {
for _, operator := range e.operators {
result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract)
if matched && result != nil {
@ -85,7 +85,7 @@ func (e *Executer) Execute(input string) (bool, error) {
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEventCallback) error {
dynamicValues := make(map[string]interface{})
_ = e.requests.ExecuteWithResults(input, dynamicValues, func(event *output.InternalWrappedEvent) {
_ = e.requests.ExecuteWithResults(input, dynamicValues, nil, func(event *output.InternalWrappedEvent) {
for _, operator := range e.operators {
result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract)
if matched && result != nil {

31
v2/pkg/protocols/common/executer/executer.go
View File

@ -1,6 +1,8 @@
package executer
import (
"strings"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
)
@ -43,8 +45,20 @@ func (e *Executer) Execute(input string) (bool, error) {
var results bool
dynamicValues := make(map[string]interface{})
previous := make(map[string]interface{})
for _, req := range e.requests {
err := req.ExecuteWithResults(input, dynamicValues, func(event *output.InternalWrappedEvent) {
err := req.ExecuteWithResults(input, dynamicValues, previous, func(event *output.InternalWrappedEvent) {
ID := req.GetID()
if ID != "" {
builder := &strings.Builder{}
for k, v := range event.InternalEvent {
builder.WriteString(ID)
builder.WriteString("_")
builder.WriteString(k)
previous[builder.String()] = v
builder.Reset()
}
}
if event.OperatorsResult == nil {
return
}
@ -64,8 +78,21 @@ func (e *Executer) Execute(input string) (bool, error) {
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEventCallback) error {
dynamicValues := make(map[string]interface{})
previous := make(map[string]interface{})
for _, req := range e.requests {
_ = req.ExecuteWithResults(input, dynamicValues, func(event *output.InternalWrappedEvent) {
_ = req.ExecuteWithResults(input, dynamicValues, previous, func(event *output.InternalWrappedEvent) {
ID := req.GetID()
if ID != "" {
builder := &strings.Builder{}
for k, v := range event.InternalEvent {
builder.WriteString(ID)
builder.WriteString("_")
builder.WriteString(k)
previous[builder.String()] = v
builder.Reset()
}
}
if event.OperatorsResult == nil {
return
}

7
v2/pkg/protocols/dns/dns.go
View File

@ -14,6 +14,8 @@ import (
// Request contains a DNS protocol request to be made from a template
type Request struct {
ID string `yaml:"id"`
// Recursion specifies whether to recurse all the answers.
Recursion bool `yaml:"recursion"`
// Path contains the path/s for the request
@ -38,6 +40,11 @@ type Request struct {
options *protocols.ExecuterOptions
}
// GetID returns the unique ID of the request if any.
func (r *Request) GetID() string {
return r.ID
}
// Compile compiles the protocol request for further execution.
func (r *Request) Compile(options *protocols.ExecuterOptions) error {
// Create a dns client for the class

11
v2/pkg/protocols/dns/request.go
View File

@ -14,7 +14,7 @@ import (
var _ protocols.Request = &Request{}
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent, callback protocols.OutputEventCallback) error {
func (r *Request) ExecuteWithResults(input string, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
// Parse the URL and return domain if URL.
var domain string
if isURL(input) {
@ -52,11 +52,14 @@ func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent
gologger.Debug().Msgf("[%s] Dumped DNS response for %s", r.options.TemplateID, domain)
fmt.Fprintf(os.Stderr, "%s\n", resp.String())
}
ouputEvent := r.responseToDSLMap(compiledRequest, resp, input, input)
outputEvent := r.responseToDSLMap(compiledRequest, resp, input, input)
for k, v := range previous {
outputEvent[k] = v
}
event := &output.InternalWrappedEvent{InternalEvent: ouputEvent}
event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
if r.CompiledOperators != nil {
result, ok := r.CompiledOperators.Execute(ouputEvent, r.Match, r.Extract)
result, ok := r.CompiledOperators.Execute(outputEvent, r.Match, r.Extract)
if ok && result != nil {
event.OperatorsResult = result
event.Results = r.MakeResultEvent(event)

7
v2/pkg/protocols/file/file.go
View File

@ -8,6 +8,8 @@ import (
// Request contains a File matching mechanism for local disk operations.
type Request struct {
ID string `yaml:"id"`
// MaxSize is the maximum size of the file to run request on.
// By default, nuclei will process 5MB files and not go more than that.
// It can be set to much lower or higher depending on use.
@ -35,6 +37,11 @@ type Request struct {
// defaultDenylist is the default list of extensions to be denied
var defaultDenylist = []string{".3g2", ".3gp", ".7z", ".apk", ".arj", ".avi", ".axd", ".bmp", ".css", ".csv", ".deb", ".dll", ".doc", ".drv", ".eot", ".exe", ".flv", ".gif", ".gifv", ".gz", ".h264", ".ico", ".iso", ".jar", ".jpeg", ".jpg", ".lock", ".m4a", ".m4v", ".map", ".mkv", ".mov", ".mp3", ".mp4", ".mpeg", ".mpg", ".msi", ".ogg", ".ogm", ".ogv", ".otf", ".pdf", ".pkg", ".png", ".ppt", ".psd", ".rar", ".rm", ".rpm", ".svg", ".swf", ".sys", ".tar.gz", ".tar", ".tif", ".tiff", ".ttf", ".txt", ".vob", ".wav", ".webm", ".wmv", ".woff", ".woff2", ".xcf", ".xls", ".xlsx", ".zip"}
// GetID returns the unique ID of the request if any.
func (r *Request) GetID() string {
return r.ID
}
// Compile compiles the protocol request for further execution.
func (r *Request) Compile(options *protocols.ExecuterOptions) error {
if len(r.Matchers) > 0 || len(r.Extractors) > 0 {

11
v2/pkg/protocols/file/request.go
View File

@ -16,7 +16,7 @@ import (
var _ protocols.Request = &Request{}
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent, callback protocols.OutputEventCallback) error {
func (r *Request) ExecuteWithResults(input string, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
wg := sizedwaitgroup.New(r.options.Options.RateLimit)
err := r.getInputPaths(input, func(data string) {
@ -54,11 +54,14 @@ func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent
fmt.Fprintf(os.Stderr, "%s\n", dataStr)
}
gologger.Verbose().Msgf("[%s] Sent FILE request to %s", r.options.TemplateID, data)
ouputEvent := r.responseToDSLMap(dataStr, input, data)
outputEvent := r.responseToDSLMap(dataStr, input, data)
for k, v := range previous {
outputEvent[k] = v
}
event := &output.InternalWrappedEvent{InternalEvent: ouputEvent}
event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
if r.CompiledOperators != nil {
result, ok := r.CompiledOperators.Execute(ouputEvent, r.Match, r.Extract)
result, ok := r.CompiledOperators.Execute(outputEvent, r.Match, r.Extract)
if ok && result != nil {
event.OperatorsResult = result
event.Results = r.MakeResultEvent(event)

7
v2/pkg/protocols/http/http.go
View File

@ -14,6 +14,8 @@ import (
// Request contains a http request to be made from a template
type Request struct {
ID string `yaml:"id"`
// Name is the name of the request
Name string `yaml:"Name"`
// AttackType is the attack type
@ -71,6 +73,11 @@ type Request struct {
rawhttpClient *rawhttp.Client
}
// GetID returns the unique ID of the request if any.
func (r *Request) GetID() string {
return r.ID
}
// Compile compiles the protocol request for further execution.
func (r *Request) Compile(options *protocols.ExecuterOptions) error {
client, err := httpclientpool.Get(options.Options, &httpclientpool.Configuration{

27
v2/pkg/protocols/http/request.go
View File

@ -29,7 +29,7 @@ import (
const defaultMaxWorkers = 150
// executeRaceRequest executes race condition request for a URL
func (e *Request) executeRaceRequest(reqURL string, dynamicValues map[string]interface{}, callback protocols.OutputEventCallback) error {
func (e *Request) executeRaceRequest(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
generator := e.newGenerator()
maxWorkers := e.RaceNumberRequests
@ -45,7 +45,7 @@ func (e *Request) executeRaceRequest(reqURL string, dynamicValues map[string]int
for i := 0; i < e.RaceNumberRequests; i++ {
swg.Add()
go func(httpRequest *generatedRequest) {
err := e.executeRequest(reqURL, httpRequest, dynamicValues, callback)
err := e.executeRequest(reqURL, httpRequest, dynamicValues, previous, callback)
mutex.Lock()
if err != nil {
requestErr = multierr.Append(requestErr, err)
@ -59,7 +59,7 @@ func (e *Request) executeRaceRequest(reqURL string, dynamicValues map[string]int
}
// executeRaceRequest executes race condition request for a URL
func (e *Request) executeParallelHTTP(reqURL string, dynamicValues map[string]interface{}, callback protocols.OutputEventCallback) error {
func (e *Request) executeParallelHTTP(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
generator := e.newGenerator()
// Workers that keeps enqueuing new requests
@ -82,7 +82,7 @@ func (e *Request) executeParallelHTTP(reqURL string, dynamicValues map[string]in
defer swg.Done()
e.options.RateLimiter.Take()
err := e.executeRequest(reqURL, httpRequest, dynamicValues, callback)
err := e.executeRequest(reqURL, httpRequest, dynamicValues, previous, callback)
mutex.Lock()
if err != nil {
requestErr = multierr.Append(requestErr, err)
@ -96,7 +96,7 @@ func (e *Request) executeParallelHTTP(reqURL string, dynamicValues map[string]in
}
// executeRaceRequest executes race condition request for a URL
func (e *Request) executeTurboHTTP(reqURL string, dynamicValues map[string]interface{}, callback protocols.OutputEventCallback) error {
func (e *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
generator := e.newGenerator()
// need to extract the target from the url
@ -141,7 +141,7 @@ func (e *Request) executeTurboHTTP(reqURL string, dynamicValues map[string]inter
go func(httpRequest *generatedRequest) {
defer swg.Done()
err := e.executeRequest(reqURL, httpRequest, dynamicValues, callback)
err := e.executeRequest(reqURL, httpRequest, dynamicValues, previous, callback)
mutex.Lock()
if err != nil {
requestErr = multierr.Append(requestErr, err)
@ -155,20 +155,20 @@ func (e *Request) executeTurboHTTP(reqURL string, dynamicValues map[string]inter
}
// ExecuteWithResults executes the final request on a URL
func (r *Request) ExecuteWithResults(reqURL string, dynamicValues output.InternalEvent, callback protocols.OutputEventCallback) error {
func (r *Request) ExecuteWithResults(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
// verify if pipeline was requested
if r.Pipeline {
return r.executeTurboHTTP(reqURL, dynamicValues, callback)
return r.executeTurboHTTP(reqURL, dynamicValues, previous, callback)
}
// verify if a basic race condition was requested
if r.Race && r.RaceNumberRequests > 0 {
return r.executeRaceRequest(reqURL, dynamicValues, callback)
return r.executeRaceRequest(reqURL, dynamicValues, previous, callback)
}
// verify if parallel elaboration was requested
if r.Threads > 0 {
return r.executeParallelHTTP(reqURL, dynamicValues, callback)
return r.executeParallelHTTP(reqURL, dynamicValues, previous, callback)
}
generator := r.newGenerator()
@ -186,7 +186,7 @@ func (r *Request) ExecuteWithResults(reqURL string, dynamicValues output.Interna
var gotOutput bool
r.options.RateLimiter.Take()
err = r.executeRequest(reqURL, request, dynamicValues, func(event *output.InternalWrappedEvent) {
err = r.executeRequest(reqURL, request, dynamicValues, previous, func(event *output.InternalWrappedEvent) {
// Add the extracts to the dynamic values if any.
if event.OperatorsResult != nil {
gotOutput = true
@ -208,7 +208,7 @@ func (r *Request) ExecuteWithResults(reqURL string, dynamicValues output.Interna
}
// executeRequest executes the actual generated request and returns error if occured
func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynamicvalues map[string]interface{}, callback protocols.OutputEventCallback) error {
func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynamicvalues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
// Add User-Agent value randomly to the customHeaders slice if `random-agent` flag is given
if r.options.Options.RandomAgent {
builder := &strings.Builder{}
@ -337,6 +337,9 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynam
}
outputEvent := r.responseToDSLMap(resp, reqURL, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(data), headersToString(resp.Header), duration, request.meta)
outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname)
for k, v := range previous {
outputEvent[k] = v
}
event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
if r.CompiledOperators != nil {

7
v2/pkg/protocols/network/network.go
View File

@ -13,6 +13,8 @@ import (
// Request contains a Network protocol request to be made from a template
type Request struct {
ID string `yaml:"id"`
// Address is the address to send requests to (host:port combos generally)
Address []string `yaml:"host"`
addresses []keyValue
@ -45,6 +47,11 @@ type Input struct {
Type string `yaml:"type"`
}
// GetID returns the unique ID of the request if any.
func (r *Request) GetID() string {
return r.ID
}
// Compile compiles the protocol request for further execution.
func (r *Request) Compile(options *protocols.ExecuterOptions) error {
var err error

9
v2/pkg/protocols/network/request.go
View File

@ -20,7 +20,7 @@ import (
var _ protocols.Request = &Request{}
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent, callback protocols.OutputEventCallback) error {
func (r *Request) ExecuteWithResults(input string, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
address, err := getAddress(input)
if err != nil {
r.options.Output.Request(r.options.TemplateID, input, "network", err)
@ -38,7 +38,7 @@ func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent
actualAddress = net.JoinHostPort(actualAddress, kv.value)
}
err = r.executeAddress(actualAddress, address, input, callback)
err = r.executeAddress(actualAddress, address, input, previous, callback)
if err != nil {
gologger.Verbose().Lable("ERR").Msgf("Could not make network request for %s: %s\n", actualAddress, err)
continue
@ -48,7 +48,7 @@ func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent
}
// executeAddress executes the request for an address
func (r *Request) executeAddress(actualAddress, address, input string, callback protocols.OutputEventCallback) error {
func (r *Request) executeAddress(actualAddress, address, input string, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
if !strings.Contains(actualAddress, ":") {
err := errors.New("no port provided in network protocol request")
r.options.Output.Request(r.options.TemplateID, address, "network", err)
@ -125,6 +125,9 @@ func (r *Request) executeAddress(actualAddress, address, input string, callback
}
outputEvent := r.responseToDSLMap(reqBuilder.String(), resp, input, actualAddress)
outputEvent["ip"] = r.dialer.GetDialedIP(hostname)
for k, v := range previous {
outputEvent[k] = v
}
event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
if r.CompiledOperators != nil {

6
v2/pkg/protocols/protocols.go
View File

@ -51,12 +51,16 @@ type Request interface {
Compile(options *ExecuterOptions) error
// Requests returns the total number of requests the rule will perform
Requests() int
// GetID returns the ID for the request if any. IDs are used for multi-request
// condition matching. So, two requests can be sent and their match can
// be evaluated from the third request by using the IDs for both requests.
GetID() string
// Match performs matching operation for a matcher on model and returns true or false.
Match(data map[string]interface{}, matcher *matchers.Matcher) bool
// Extract performs extracting operation for a extractor on model and returns true or false.
Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{}
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
ExecuteWithResults(input string, metadata output.InternalEvent, callback OutputEventCallback) error
ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback OutputEventCallback) error
}
// OutputEventCallback is a callback event for any results found during scanning.