daffainfo
/
nuclei
mirror of https://github.com/daffainfo/nuclei.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #463 from projectdiscovery/network-protocol-add 

Network protocol support for nuclei
dev
Ice3man 2021-01-01 05:23:42 -08:00 committed by GitHub
parent 65e14e1c91 590f8042b9
commit c6e617a29d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 530 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
v2/internal/runner/config.go
View File

@ -93,28 +93,6 @@ func (r *Runner) readNucleiIgnoreFile() {
}
}
// checkIfInNucleiIgnore checks if a path falls under nuclei-ignore rules.
func (r *Runner) checkIfInNucleiIgnore(item string) bool {
if r.templatesConfig == nil {
return false
}
for _, paths := range r.templatesConfig.IgnorePaths {
// If we have a path to ignore, check if it's in the item.
if paths[len(paths)-1] == '/' {
if strings.Contains(item, paths) {
return true
}
continue
}
// Check for file based extension in ignores
if strings.HasSuffix(item, paths) {
return true
}
}
return false
}
// getIgnoreFilePath returns the ignore file path for the runner
func (r *Runner) getIgnoreFilePath() string {
defIgnoreFilePath := path.Join(r.templatesConfig.TemplatesDirectory, nucleiIgnoreFile)

1
v2/internal/runner/processor.go
View File

@ -18,6 +18,7 @@ func (r *Runner) processTemplateWithList(template *templates.Template) bool {
r.hostMap.Scan(func(k, _ []byte) error {
URL := string(k)
wg.Add()
go func(URL string) {
defer wg.Done()

4
v2/internal/runner/runner.go
View File

@ -16,6 +16,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/projectfile"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns/dnsclientpool"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
"github.com/remeh/sizedwaitgroup"
@ -277,5 +278,8 @@ func (r *Runner) initializeProtocols() error {
if err := httpclientpool.Init(r.options); err != nil {
return err
}
if err := networkclientpool.Init(r.options); err != nil {
return err
}
return nil
}

1
v2/pkg/output/output.go
View File

@ -161,7 +161,6 @@ func (w *StandardWriter) Request(templateID, url, requestType string, err error)
return
}
w.traceMutex.Lock()
//nolint:errcheck // We don't need to do anything here
_ = w.traceFile.Write(data)
w.traceMutex.Unlock()
}

1
v2/pkg/protocols/http/raw/raw.go
View File

@ -47,7 +47,6 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) {
break
}
//nolint:gomnd // this is not a magic number
p := strings.SplitN(line, ":", 2)
key = p[0]
if len(p) > 1 {

75
v2/pkg/protocols/http/request.go
View File

@ -161,23 +161,23 @@ func (e *Request) executeTurboHTTP(reqURL string, dynamicValues map[string]inter
}
// ExecuteWithResults executes the final request on a URL
func (e *Request) ExecuteWithResults(reqURL string, dynamicValues map[string]interface{}) ([]*output.InternalWrappedEvent, error) {
func (r *Request) ExecuteWithResults(reqURL string, dynamicValues map[string]interface{}) ([]*output.InternalWrappedEvent, error) {
// verify if pipeline was requested
if e.Pipeline {
return e.executeTurboHTTP(reqURL, dynamicValues)
if r.Pipeline {
return r.executeTurboHTTP(reqURL, dynamicValues)
}
// verify if a basic race condition was requested
if e.Race && e.RaceNumberRequests > 0 {
return e.executeRaceRequest(reqURL, dynamicValues)
if r.Race && r.RaceNumberRequests > 0 {
return r.executeRaceRequest(reqURL, dynamicValues)
}
// verify if parallel elaboration was requested
if e.Threads > 0 {
return e.executeParallelHTTP(reqURL, dynamicValues)
if r.Threads > 0 {
return r.executeParallelHTTP(reqURL, dynamicValues)
}
generator := e.newGenerator()
generator := r.newGenerator()
var requestErr error
var outputs []*output.InternalWrappedEvent
@ -187,21 +187,21 @@ func (e *Request) ExecuteWithResults(reqURL string, dynamicValues map[string]int
break
}
if err != nil {
e.options.Progress.DecrementRequests(int64(generator.Total()))
r.options.Progress.DecrementRequests(int64(generator.Total()))
return nil, err
}
e.options.RateLimiter.Take()
output, err := e.executeRequest(reqURL, request, dynamicValues)
r.options.RateLimiter.Take()
output, err := r.executeRequest(reqURL, request, dynamicValues)
if err != nil {
requestErr = multierr.Append(requestErr, err)
} else {
outputs = append(outputs, output...)
}
e.options.Progress.IncrementRequests()
r.options.Progress.IncrementRequests()
if request.original.options.Options.StopAtFirstMatch && len(output) > 0 {
e.options.Progress.DecrementRequests(int64(generator.Total()))
r.options.Progress.DecrementRequests(int64(generator.Total()))
break
}
}
@ -209,16 +209,15 @@ func (e *Request) ExecuteWithResults(reqURL string, dynamicValues map[string]int
}
// executeRequest executes the actual generated request and returns error if occured
func (e *Request) executeRequest(reqURL string, request *generatedRequest, dynamicvalues map[string]interface{}) ([]*output.InternalWrappedEvent, error) {
func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynamicvalues map[string]interface{}) ([]*output.InternalWrappedEvent, error) {
// Add User-Agent value randomly to the customHeaders slice if `random-agent` flag is given
if e.options.Options.RandomAgent {
if r.options.Options.RandomAgent {
builder := &strings.Builder{}
builder.WriteString("User-Agent: ")
// nolint:errcheck // ignoring error
builder.WriteString(uarand.GetRandom())
e.customHeaders.Set(builder.String())
r.customHeaders.Set(builder.String())
}
e.setCustomHeaders(request)
r.setCustomHeaders(request)
var (
resp *http.Response
@ -226,14 +225,14 @@ func (e *Request) executeRequest(reqURL string, request *generatedRequest, dynam
dumpedRequest []byte
fromcache bool
)
if e.options.Options.Debug || e.options.ProjectFile != nil {
if r.options.Options.Debug || r.options.ProjectFile != nil {
dumpedRequest, err = dump(request, reqURL)
if err != nil {
return nil, err
}
}
if e.options.Options.Debug {
gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", e.options.TemplateID, reqURL)
if r.options.Options.Debug {
gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", r.options.TemplateID, reqURL)
fmt.Fprintf(os.Stderr, "%s", string(dumpedRequest))
}
@ -245,23 +244,23 @@ func (e *Request) executeRequest(reqURL string, request *generatedRequest, dynam
// burp uses "\r\n" as new line character
request.rawRequest.Data = strings.ReplaceAll(request.rawRequest.Data, "\n", "\r\n")
options := request.original.rawhttpClient.Options
options.AutomaticContentLength = !e.DisableAutoContentLength
options.AutomaticHostHeader = !e.DisableAutoHostname
options.FollowRedirects = e.Redirects
options.AutomaticContentLength = !r.DisableAutoContentLength
options.AutomaticHostHeader = !r.DisableAutoHostname
options.FollowRedirects = r.Redirects
resp, err = request.original.rawhttpClient.DoRawWithOptions(request.rawRequest.Method, reqURL, request.rawRequest.Path, generators.ExpandMapValues(request.rawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.rawRequest.Data)), options)
} else {
// if nuclei-project is available check if the request was already sent previously
if e.options.ProjectFile != nil {
if r.options.ProjectFile != nil {
// if unavailable fail silently
fromcache = true
// nolint:bodyclose // false positive the response is generated at runtime
resp, err = e.options.ProjectFile.Get(dumpedRequest)
resp, err = r.options.ProjectFile.Get(dumpedRequest)
if err != nil {
fromcache = false
}
}
if resp == nil {
resp, err = e.httpClient.Do(request.request)
resp, err = r.httpClient.Do(request.request)
}
}
if err != nil {
@ -269,17 +268,17 @@ func (e *Request) executeRequest(reqURL string, request *generatedRequest, dynam
_, _ = io.Copy(ioutil.Discard, resp.Body)
resp.Body.Close()
}
e.options.Output.Request(e.options.TemplateID, reqURL, "http", err)
e.options.Progress.DecrementRequests(1)
r.options.Output.Request(r.options.TemplateID, reqURL, "http", err)
r.options.Progress.DecrementRequests(1)
return nil, err
}
gologger.Verbose().Msgf("Sent request to %s", reqURL)
e.options.Output.Request(e.options.TemplateID, reqURL, "http", err)
r.options.Output.Request(r.options.TemplateID, reqURL, "http", err)
duration := time.Since(timeStart)
// Dump response - Step 1 - Decompression not yet handled
var dumpedResponse []byte
if e.options.Options.Debug {
if r.options.Options.Debug {
var dumpErr error
dumpedResponse, dumpErr = httputil.DumpResponse(resp, true)
if dumpErr != nil {
@ -305,15 +304,15 @@ func (e *Request) executeRequest(reqURL string, request *generatedRequest, dynam
}
// Dump response - step 2 - replace gzip body with deflated one or with itself (NOP operation)
if e.options.Options.Debug {
if r.options.Options.Debug {
dumpedResponse = bytes.ReplaceAll(dumpedResponse, dataOrig, data)
gologger.Info().Msgf("[%s] Dumped HTTP response for %s\n\n", e.options.TemplateID, reqURL)
gologger.Info().Msgf("[%s] Dumped HTTP response for %s\n\n", r.options.TemplateID, reqURL)
fmt.Fprintf(os.Stderr, "%s\n", string(dumpedResponse))
}
// if nuclei-project is enabled store the response if not previously done
if e.options.ProjectFile != nil && !fromcache {
err := e.options.ProjectFile.Set(dumpedRequest, resp, data)
if r.options.ProjectFile != nil && !fromcache {
err := r.options.ProjectFile.Set(dumpedRequest, resp, data)
if err != nil {
return nil, errors.Wrap(err, "could not store in project file")
}
@ -346,11 +345,11 @@ func (e *Request) executeRequest(reqURL string, request *generatedRequest, dynam
if request.request != nil {
matchedURL = request.request.URL.String()
}
ouputEvent := e.responseToDSLMap(resp, reqURL, matchedURL, unsafeToString(dumpedRequest), unsafeToString(dumpedResponse), unsafeToString(data), headersToString(resp.Header), duration, request.meta)
ouputEvent := r.responseToDSLMap(resp, reqURL, matchedURL, unsafeToString(dumpedRequest), unsafeToString(dumpedResponse), unsafeToString(data), headersToString(resp.Header), duration, request.meta)
event := []*output.InternalWrappedEvent{{InternalEvent: ouputEvent}}
if e.CompiledOperators != nil {
result, ok := e.Operators.Execute(ouputEvent, e.Match, e.Extract)
if r.CompiledOperators != nil {
result, ok := r.Operators.Execute(ouputEvent, r.Match, r.Extract)
if !ok {
return nil, nil
}

90
v2/pkg/protocols/network/executer.go Normal file
View File

@ -0,0 +1,90 @@
package network
import (
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
)
// Executer executes a group of requests for a protocol
type Executer struct {
requests []*Request
options *protocols.ExecuterOptions
}
var _ protocols.Executer = &Executer{}
// NewExecuter creates a new request executer for list of requests
func NewExecuter(requests []*Request, options *protocols.ExecuterOptions) *Executer {
return &Executer{requests: requests, options: options}
}
// Compile compiles the execution generators preparing any requests possible.
func (e *Executer) Compile() error {
for _, request := range e.requests {
err := request.Compile(e.options)
if err != nil {
return err
}
}
return nil
}
// Requests returns the total number of requests the rule will perform
func (e *Executer) Requests() int {
var count int
for _, request := range e.requests {
count += int(request.Requests())
}
return count
}
// Execute executes the protocol group and returns true or false if results were found.
func (e *Executer) Execute(input string) (bool, error) {
var results bool
for _, req := range e.requests {
events, err := req.ExecuteWithResults(input, nil)
if err != nil {
return false, err
}
if events == nil {
return false, nil
}
// If we have a result field, we should add a result to slice.
for _, event := range events {
if event.OperatorsResult == nil {
continue
}
for _, result := range req.makeResultEvent(event) {
results = true
e.options.Output.Write(result)
}
}
}
return results, nil
}
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (e *Executer) ExecuteWithResults(input string) ([]*output.InternalWrappedEvent, error) {
var results []*output.InternalWrappedEvent
for _, req := range e.requests {
events, err := req.ExecuteWithResults(input, nil)
if err != nil {
return nil, err
}
if events == nil {
return nil, nil
}
for _, event := range events {
if event.OperatorsResult == nil {
continue
}
event.Results = req.makeResultEvent(event)
}
results = append(results, events...)
}
return results, nil
}

84
v2/pkg/protocols/network/network.go Normal file
View File

@ -0,0 +1,84 @@
package network
import (
"net"
"strings"
"github.com/pkg/errors"
"github.com/projectdiscovery/fastdialer/fastdialer"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/network/networkclientpool"
)
// Request contains a Network protocol request to be made from a template
type Request struct {
// Address is the address to send requests to (host:port combos generally)
Address []string `yaml:"host"`
addresses []keyValue
// Payload is the payload to send for the network request
Inputs []*Input `yaml:"inputs"`
// ReadSize is the size of response to read (1024 if not provided by default)
ReadSize int `yaml:"read-size"`
// Operators for the current request go here.
operators.Operators `yaml:",inline"`
CompiledOperators *operators.Operators
// cache any variables that may be needed for operation.
dialer *fastdialer.Dialer
options *protocols.ExecuterOptions
}
// keyValue is a key value pair
type keyValue struct {
key string
value string
}
// Input is the input to send on the network
type Input struct {
// Data is the data to send as the input
Data string `yaml:"data"`
// Type is the type of input - hex, text.
Type string `yaml:"type"`
}
// Compile compiles the protocol request for further execution.
func (r *Request) Compile(options *protocols.ExecuterOptions) error {
var err error
for _, address := range r.Address {
if strings.Contains(address, ":") {
addressHost, addressPort, err := net.SplitHostPort(address)
if err != nil {
return errors.Wrap(err, "could not parse address")
}
r.addresses = append(r.addresses, keyValue{key: addressHost, value: addressPort})
} else {
r.addresses = append(r.addresses, keyValue{key: address})
}
}
// Create a client for the class
client, err := networkclientpool.Get(options.Options, &networkclientpool.Configuration{})
if err != nil {
return errors.Wrap(err, "could not get network client")
}
r.dialer = client
if len(r.Matchers) > 0 || len(r.Extractors) > 0 {
compiled := &r.Operators
if err := compiled.Compile(); err != nil {
return errors.Wrap(err, "could not compile operators")
}
r.CompiledOperators = compiled
}
r.options = options
return nil
}
// Requests returns the total number of requests the YAML rule will perform
func (r *Request) Requests() int {
return len(r.addresses)
}

38
v2/pkg/protocols/network/networkclientpool/clientpool.go Normal file
View File

@ -0,0 +1,38 @@
package networkclientpool
import (
"github.com/pkg/errors"
"github.com/projectdiscovery/fastdialer/fastdialer"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
)
var (
normalClient *fastdialer.Dialer
)
// Init initializes the clientpool implementation
func Init(options *types.Options) error {
// Don't create clients if already created in past.
if normalClient != nil {
return nil
}
dialer, err := fastdialer.NewDialer(fastdialer.DefaultOptions)
if err != nil {
return errors.Wrap(err, "could not create dialer")
}
normalClient = dialer
return nil
}
// Configuration contains the custom configuration options for a client
type Configuration struct{}
// Hash returns the hash of the configuration to allow client pooling
func (c *Configuration) Hash() string {
return ""
}
// Get creates or gets a client for the protocol based on custom configuration
func Get(options *types.Options, configuration *Configuration) (*fastdialer.Dialer, error) {
return normalClient, nil
}

108
v2/pkg/protocols/network/operators.go Normal file
View File

@ -0,0 +1,108 @@
package network
import (
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
)
// Match matches a generic data response again a given matcher
func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool {
partString := matcher.Part
switch partString {
case "body", "all", "":
partString = "data"
}
item, ok := data[partString]
if !ok {
return false
}
itemStr := types.ToString(item)
switch matcher.GetType() {
case matchers.SizeMatcher:
return matcher.Result(matcher.MatchSize(len(itemStr)))
case matchers.WordsMatcher:
return matcher.Result(matcher.MatchWords(itemStr))
case matchers.RegexMatcher:
return matcher.Result(matcher.MatchRegex(itemStr))
case matchers.BinaryMatcher:
return matcher.Result(matcher.MatchBinary(itemStr))
case matchers.DSLMatcher:
return matcher.Result(matcher.MatchDSL(data))
}
return false
}
// Extract performs extracting operation for a extractor on model and returns true or false.
func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} {
part, ok := data[extractor.Part]
if !ok {
return nil
}
partString := part.(string)
switch partString {
case "body", "all":
partString = "data"
}
item, ok := data[partString]
if !ok {
return nil
}
itemStr := types.ToString(item)
switch extractor.GetType() {
case extractors.RegexExtractor:
return extractor.ExtractRegex(itemStr)
case extractors.KValExtractor:
return extractor.ExtractKval(data)
}
return nil
}
// responseToDSLMap converts a DNS response to a map for use in DSL matching
func (r *Request) responseToDSLMap(req, resp string, host, matched string) output.InternalEvent {
data := make(output.InternalEvent, 4)
// Some data regarding the request metadata
data["host"] = host
data["matched"] = matched
if r.options.Options.JSONRequests {
data["request"] = req
}
data["data"] = resp
return data
}
// makeResultEvent creates a result event from internal wrapped event
func (r *Request) makeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent {
results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1)
data := output.ResultEvent{
TemplateID: r.options.TemplateID,
Info: r.options.TemplateInfo,
Type: "network",
Host: wrapped.InternalEvent["host"].(string),
Matched: wrapped.InternalEvent["matched"].(string),
ExtractedResults: wrapped.OperatorsResult.OutputExtracts,
}
if r.options.Options.JSONRequests {
data.Request = wrapped.InternalEvent["request"].(string)
data.Response = wrapped.InternalEvent["data"].(string)
}
// If we have multiple matchers with names, write each of them separately.
if len(wrapped.OperatorsResult.Matches) > 0 {
for k := range wrapped.OperatorsResult.Matches {
data.MatcherName = k
results = append(results, &data)
}
} else {
results = append(results, &data)
}
return results
}

146
v2/pkg/protocols/network/request.go Normal file
View File

@ -0,0 +1,146 @@
package network
import (
"context"
"encoding/hex"
"fmt"
"net"
"net/url"
"os"
"strings"
"time"
"github.com/pkg/errors"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer"
)
var _ protocols.Request = &Request{}
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent) ([]*output.InternalWrappedEvent, error) {
address, err := getAddress(input)
if err != nil {
r.options.Output.Request(r.options.TemplateID, input, "network", err)
r.options.Progress.DecrementRequests(1)
return nil, errors.Wrap(err, "could not get address from url")
}
var outputs []*output.InternalWrappedEvent
for _, kv := range r.addresses {
replacer := replacer.New(map[string]interface{}{"Hostname": address})
actualAddress := replacer.Replace(kv.key)
if kv.value != "" {
if strings.Contains(address, ":") {
actualAddress, _, _ = net.SplitHostPort(actualAddress)
}
actualAddress = net.JoinHostPort(actualAddress, kv.value)
}
output, err := r.executeAddress(actualAddress, address, input)
if err != nil {
gologger.Verbose().Lable("ERR").Msgf("Could not make network request for %s: %s\n", actualAddress, err)
continue
}
outputs = append(outputs, output...)
}
return outputs, nil
}
// executeAddress executes the request for an address
func (r *Request) executeAddress(actualAddress, address, input string) ([]*output.InternalWrappedEvent, error) {
if !strings.Contains(actualAddress, ":") {
err := errors.New("no port provided in network protocol request")
r.options.Output.Request(r.options.TemplateID, address, "network", err)
r.options.Progress.DecrementRequests(1)
return nil, err
}
conn, err := r.dialer.Dial(context.Background(), "tcp", actualAddress)
if err != nil {
r.options.Output.Request(r.options.TemplateID, address, "network", err)
r.options.Progress.DecrementRequests(1)
return nil, errors.Wrap(err, "could not connect to server request")
}
defer conn.Close()
conn.SetReadDeadline(time.Now().Add(5 * time.Second))
reqBuilder := &strings.Builder{}
for _, input := range r.Inputs {
var data []byte
switch input.Type {
case "hex":
data, err = hex.DecodeString(input.Data)
default:
data = []byte(input.Data)
}
if err != nil {
r.options.Output.Request(r.options.TemplateID, address, "network", err)
r.options.Progress.DecrementRequests(1)
return nil, errors.Wrap(err, "could not write request to server")
}
reqBuilder.Grow(len(input.Data))
reqBuilder.WriteString(input.Data)
_, err = conn.Write(data)
if err != nil {
r.options.Output.Request(r.options.TemplateID, address, "network", err)
r.options.Progress.DecrementRequests(1)
return nil, errors.Wrap(err, "could not write request to server")
}
r.options.Progress.IncrementRequests()
}
if err != nil {
r.options.Output.Request(r.options.TemplateID, address, "network", err)
r.options.Progress.DecrementRequests(1)
return nil, errors.Wrap(err, "could not write request to server")
}
if r.options.Options.Debug {
gologger.Info().Str("address", actualAddress).Msgf("[%s] Dumped Network request for %s", r.options.TemplateID, actualAddress)
fmt.Fprintf(os.Stderr, "%s\n", reqBuilder.String())
}
r.options.Output.Request(r.options.TemplateID, actualAddress, "network", err)
gologger.Verbose().Msgf("[%s] Sent Network request to %s", r.options.TemplateID, actualAddress)
bufferSize := 1024
if r.ReadSize != 0 {
bufferSize = r.ReadSize
}
buffer := make([]byte, bufferSize)
n, _ := conn.Read(buffer)
resp := string(buffer[:n])
if r.options.Options.Debug {
gologger.Debug().Msgf("[%s] Dumped Network response for %s", r.options.TemplateID, actualAddress)
fmt.Fprintf(os.Stderr, "%s\n", resp)
}
ouputEvent := r.responseToDSLMap(reqBuilder.String(), resp, input, actualAddress)
event := []*output.InternalWrappedEvent{{InternalEvent: ouputEvent}}
if r.CompiledOperators != nil {
result, ok := r.Operators.Execute(ouputEvent, r.Match, r.Extract)
if !ok {
return nil, nil
}
event[0].OperatorsResult = result
}
return event, nil
}
// getAddress returns the address of the host to make request to
func getAddress(toTest string) (string, error) {
if strings.Contains(toTest, "://") {
parsed, err := url.Parse(toTest)
if err != nil {
return "", err
}
toTest = parsed.Host
}
return toTest, nil
}

24
v2/pkg/templates/compile.go
View File

@ -8,6 +8,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/network"
"github.com/projectdiscovery/nuclei/v2/pkg/workflows"
"gopkg.in/yaml.v2"
)
@ -32,12 +33,8 @@ func Parse(file string, options *protocols.ExecuterOptions) (*Template, error) {
options.TemplateInfo = template.Info
options.TemplatePath = file
// We don't support both http and dns in a single template
if len(template.RequestsDNS) > 0 && len(template.RequestsHTTP) > 0 {
return nil, fmt.Errorf("both http and dns requests for %s", template.ID)
}
// If no requests, and it is also not a workflow, return error.
if len(template.RequestsDNS)+len(template.RequestsHTTP)+len(template.Workflows) == 0 {
if len(template.RequestsDNS)+len(template.RequestsHTTP)+len(template.RequestsNetwork)+len(template.Workflows) == 0 {
return nil, fmt.Errorf("no requests defined for %s", template.ID)
}
@ -52,12 +49,6 @@ func Parse(file string, options *protocols.ExecuterOptions) (*Template, error) {
}
// Compile the requests found
for _, request := range template.RequestsDNS {
template.TotalRequests += request.Requests()
}
for _, request := range template.RequestsHTTP {
template.TotalRequests += request.Requests()
}
if len(template.RequestsDNS) > 0 {
template.Executer = dns.NewExecuter(template.RequestsDNS, options)
err = template.Executer.Compile()
@ -66,14 +57,15 @@ func Parse(file string, options *protocols.ExecuterOptions) (*Template, error) {
template.Executer = http.NewExecuter(template.RequestsHTTP, options)
err = template.Executer.Compile()
}
if len(template.RequestsNetwork) > 0 {
template.Executer = network.NewExecuter(template.RequestsNetwork, options)
err = template.Executer.Compile()
}
template.TotalRequests += template.Executer.Requests()
if err != nil {
return nil, errors.Wrap(err, "could not compile request")
}
if template.Executer != nil {
if err := template.Executer.Compile(); err != nil {
return nil, errors.Wrap(err, "could not compile template executer")
}
}
return template, nil
}

3
v2/pkg/templates/templates.go
View File

@ -4,6 +4,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/network"
"github.com/projectdiscovery/nuclei/v2/pkg/workflows"
)
@ -17,6 +18,8 @@ type Template struct {
RequestsHTTP []*http.Request `yaml:"requests,omitempty"`
// RequestsDNS contains the dns request to make in the template
RequestsDNS []*dns.Request `yaml:"dns,omitempty"`
// RequestsNetwork contains the network request to make in the template
RequestsNetwork []*network.Request `yaml:"network,omitempty"`
// Workflows is a yaml based workflow declaration code.
workflows.Workflow `yaml:",inline"`
CompiledWorkflow *workflows.Workflow

22
v2/pkg/workflows/execute_test.go
View File

@ -10,7 +10,7 @@ import (
func TestWorkflowsSimple(t *testing.T) {
workflow := &Workflow{Workflows: []*WorkflowTemplate{
{executer: &mockExecuter{result: true}},
{Executer: &mockExecuter{result: true}},
}}
matched, err := workflow.RunWorkflow("https://test.com")
@ -21,10 +21,10 @@ func TestWorkflowsSimple(t *testing.T) {
func TestWorkflowsSimpleMultiple(t *testing.T) {
var firstInput, secondInput string
workflow := &Workflow{Workflows: []*WorkflowTemplate{
{executer: &mockExecuter{result: true, executeHook: func(input string) {
{Executer: &mockExecuter{result: true, executeHook: func(input string) {
firstInput = input
}}},
{executer: &mockExecuter{result: true, executeHook: func(input string) {
{Executer: &mockExecuter{result: true, executeHook: func(input string) {
secondInput = input
}}},
}}
@ -40,11 +40,11 @@ func TestWorkflowsSimpleMultiple(t *testing.T) {
func TestWorkflowsSubtemplates(t *testing.T) {
var firstInput, secondInput string
workflow := &Workflow{Workflows: []*WorkflowTemplate{
{executer: &mockExecuter{result: true, executeHook: func(input string) {
{Executer: &mockExecuter{result: true, executeHook: func(input string) {
firstInput = input
}},
Subtemplates: []*WorkflowTemplate{
{executer: &mockExecuter{result: true, executeHook: func(input string) {
{Executer: &mockExecuter{result: true, executeHook: func(input string) {
secondInput = input
}}},
}},
@ -61,11 +61,11 @@ func TestWorkflowsSubtemplates(t *testing.T) {
func TestWorkflowsSubtemplatesNoMatch(t *testing.T) {
var firstInput, secondInput string
workflow := &Workflow{Workflows: []*WorkflowTemplate{
{executer: &mockExecuter{result: false, executeHook: func(input string) {
{Executer: &mockExecuter{result: false, executeHook: func(input string) {
firstInput = input
}},
Subtemplates: []*WorkflowTemplate{
{executer: &mockExecuter{result: true, executeHook: func(input string) {
{Executer: &mockExecuter{result: true, executeHook: func(input string) {
secondInput = input
}}},
}},
@ -82,7 +82,7 @@ func TestWorkflowsSubtemplatesNoMatch(t *testing.T) {
func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) {
var firstInput, secondInput string
workflow := &Workflow{Workflows: []*WorkflowTemplate{
{executer: &mockExecuter{result: true, executeHook: func(input string) {
{Executer: &mockExecuter{result: true, executeHook: func(input string) {
firstInput = input
}, outputs: []*output.InternalWrappedEvent{
{OperatorsResult: &operators.Result{
@ -92,7 +92,7 @@ func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) {
}},
Matchers: []*Matcher{
{Name: "tomcat", Subtemplates: []*WorkflowTemplate{
{executer: &mockExecuter{result: true, executeHook: func(input string) {
{Executer: &mockExecuter{result: true, executeHook: func(input string) {
secondInput = input
}}},
}},
@ -111,7 +111,7 @@ func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) {
func TestWorkflowsSubtemplatesWithMatcherNoMatch(t *testing.T) {
var firstInput, secondInput string
workflow := &Workflow{Workflows: []*WorkflowTemplate{
{executer: &mockExecuter{result: true, executeHook: func(input string) {
{Executer: &mockExecuter{result: true, executeHook: func(input string) {
firstInput = input
}, outputs: []*output.InternalWrappedEvent{
{OperatorsResult: &operators.Result{
@ -121,7 +121,7 @@ func TestWorkflowsSubtemplatesWithMatcherNoMatch(t *testing.T) {
}},
Matchers: []*Matcher{
{Name: "apache", Subtemplates: []*WorkflowTemplate{
{executer: &mockExecuter{result: true, executeHook: func(input string) {
{Executer: &mockExecuter{result: true, executeHook: func(input string) {
secondInput = input
}}},
}},