daffainfo
/
nuclei
mirror of https://github.com/daffainfo/nuclei.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

MIsc

dev
Ice3man543 2020-12-26 02:09:16 +05:30
parent 5bd3438b4f
commit 164a67353b
10 changed files with 231 additions and 403 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
v2/pkg/atomicboolean/bool.go
View File

@ -1,42 +0,0 @@
package atomicboolean
import (
"sync"
)
type AtomBool struct {
sync.RWMutex
flag bool
}
func New() *AtomBool {
return &AtomBool{}
}
func (b *AtomBool) Or(value bool) {
b.Lock()
defer b.Unlock()
b.flag = b.flag || value
}
func (b *AtomBool) And(value bool) {
b.Lock()
defer b.Unlock()
b.flag = b.flag && value
}
func (b *AtomBool) Set(value bool) {
b.Lock()
defer b.Unlock()
b.flag = value
}
func (b *AtomBool) Get() bool {
b.RLock()
defer b.RUnlock() //nolint
return b.flag
}

51
v2/pkg/executer/executer_dns.go
View File

@ -1,51 +0,0 @@
package executer
import (
"github.com/projectdiscovery/nuclei/v2/internal/progress"
"github.com/projectdiscovery/nuclei/v2/pkg/requests"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
)
// DNSExecuter is a client for performing a DNS request
// for a template.
type DNSExecuter struct {
template *templates.Template
}
// DNSOptions contains configuration options for the DNS executer.
type DNSOptions struct {
Template *templates.Template
DNSRequest *requests.DNSRequest
}
// NewDNSExecuter creates a new DNS executer from a template
// and a DNS request query.
func NewDNSExecuter(options *DNSOptions) *DNSExecuter {
executer := &DNSExecuter{
debug: options.Debug,
noMeta: options.NoMeta,
jsonOutput: options.JSON,
traceLog: options.TraceLog,
jsonRequest: options.JSONRequests,
dnsClient: dnsClient,
vhost: options.VHost,
template: options.Template,
dnsRequest: options.DNSRequest,
writer: options.Writer,
coloredOutput: options.ColoredOutput,
colorizer: options.Colorizer,
decolorizer: options.Decolorizer,
ratelimiter: options.RateLimiter,
}
return executer
}
// ExecuteDNS executes the DNS request on a URL
func (e *DNSExecuter) ExecuteDNS(p *progress.Progress, reqURL string) *Result {
return result
}
// Close closes the dns executer for a template.
func (e *DNSExecuter) Close() {}

106
v2/pkg/executer/output_dns.go
View File

@ -1,106 +0,0 @@
package executer
import (
"strings"
"github.com/miekg/dns"
jsoniter "github.com/json-iterator/go"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
)
// writeOutputDNS writes dns output to streams
// nolint:interfacer // dns.Msg is out of current scope
func (e *DNSExecuter) writeOutputDNS(domain string, req, resp *dns.Msg, matcher *matchers.Matcher, extractorResults []string) {
if e.jsonOutput {
output := make(jsonOutput)
output["matched"] = domain
if !e.noMeta {
output["template"] = e.template.ID
output["type"] = "dns"
output["host"] = domain
for k, v := range e.template.Info {
output[k] = v
}
if matcher != nil && len(matcher.Name) > 0 {
output["matcher_name"] = matcher.Name
}
if len(extractorResults) > 0 {
output["extracted_results"] = extractorResults
}
if e.jsonRequest {
output["request"] = req.String()
output["response"] = resp.String()
}
}
data, err := jsoniter.Marshal(output)
if err != nil {
gologger.Warningf("Could not marshal json output: %s\n", err)
}
gologger.Silentf("%s", string(data))
if e.writer != nil {
if err := e.writer.Write(data); err != nil {
gologger.Errorf("Could not write output data: %s\n", err)
return
}
}
return
}
builder := &strings.Builder{}
colorizer := e.colorizer
if !e.noMeta {
builder.WriteRune('[')
builder.WriteString(colorizer.Colorizer.BrightGreen(e.template.ID).String())
if matcher != nil && len(matcher.Name) > 0 {
builder.WriteString(":")
builder.WriteString(colorizer.Colorizer.BrightGreen(matcher.Name).Bold().String())
}
builder.WriteString("] [")
builder.WriteString(colorizer.Colorizer.BrightBlue("dns").String())
builder.WriteString("] ")
if e.template.Info["severity"] != "" {
builder.WriteString("[")
builder.WriteString(colorizer.GetColorizedSeverity(e.template.Info["severity"]))
builder.WriteString("] ")
}
}
builder.WriteString(domain)
// If any extractors, write the results
if len(extractorResults) > 0 && !e.noMeta {
builder.WriteString(" [")
for i, result := range extractorResults {
builder.WriteString(colorizer.Colorizer.BrightCyan(result).String())
if i != len(extractorResults)-1 {
builder.WriteRune(',')
}
}
builder.WriteString("]")
}
builder.WriteRune('\n')
// Write output to screen as well as any output file
message := builder.String()
gologger.Silentf("%s", message)
if e.writer != nil {
if e.coloredOutput {
message = e.decolorizer.ReplaceAllString(message, "")
}
if err := e.writer.WriteString(message); err != nil {
gologger.Errorf("Could not write output data: %s\n", err)
return
}
}
}

1
v2/pkg/output/output.go
View File

@ -48,6 +48,7 @@ type InternalEvent map[string]interface{}
// InternalWrappedEvent is a wrapped event with operators result added to it.
type InternalWrappedEvent struct {
InternalEvent InternalEvent
Results []*ResultEvent
OperatorsResult *operators.Result
}

16
v2/pkg/protocols/dns/executer.go
View File

@ -21,7 +21,8 @@ func NewExecuter(requests []*Request, options *protocols.ExecuterOptions) *Execu
// Compile compiles the execution generators preparing any requests possible.
func (e *Executer) Compile() error {
for _, request := range e.requests {
if err := request.Compile(e.options); err != nil {
err := request.Compile(e.options)
if err != nil {
return err
}
}
@ -46,6 +47,9 @@ func (e *Executer) Execute(input string) (bool, error) {
if err != nil {
return false, err
}
if events == nil {
return false, nil
}
// If we have a result field, we should add a result to slice.
for _, event := range events {
@ -63,20 +67,24 @@ func (e *Executer) Execute(input string) (bool, error) {
}
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (e *Executer) ExecuteWithResults(input string) ([]*output.ResultEvent, error) {
var results []*output.ResultEvent
func (e *Executer) ExecuteWithResults(input string) ([]*output.InternalWrappedEvent, error) {
var results []*output.InternalWrappedEvent
for _, req := range e.requests {
events, err := req.ExecuteWithResults(input, nil)
if err != nil {
return nil, err
}
if events == nil {
return nil, nil
}
for _, event := range events {
if event.OperatorsResult == nil {
continue
}
results = append(results, req.makeResultEvent(event)...)
event.Results = req.makeResultEvent(event)
}
results = append(results, events...)
}
return results, nil
}

7
v2/pkg/protocols/dns/operators.go
View File

@ -79,13 +79,16 @@ func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Ext
}
// responseToDSLMap converts a DNS response to a map for use in DSL matching
func responseToDSLMap(req, resp *dns.Msg, host, matched string) output.InternalEvent {
func (r *Request) responseToDSLMap(req, resp *dns.Msg, host, matched string) output.InternalEvent {
data := make(output.InternalEvent, 8)
// Some data regarding the request metadata
data["host"] = host
data["matched"] = matched
data["request"] = req.String()
if r.options.Options.JSONRequests {
data["request"] = req.String()
}
data["rcode"] = resp.Rcode
buffer := &bytes.Buffer{}

2
v2/pkg/protocols/dns/request.go
View File

@ -52,7 +52,7 @@ func (r *Request) ExecuteWithResults(input string, metadata output.InternalEvent
gologger.Debug().Msgf("[%s] Dumped DNS response for %s", r.options.TemplateID, domain)
fmt.Fprintf(os.Stderr, "%s\n", resp.String())
}
ouputEvent := responseToDSLMap(compiledRequest, resp, input, input)
ouputEvent := r.responseToDSLMap(compiledRequest, resp, input, input)
event := []*output.InternalWrappedEvent{{InternalEvent: ouputEvent}}
if r.Operators != nil {

211
v2/pkg/protocols/http/build_request.go Normal file
View File

@ -0,0 +1,211 @@
package http
import (
"context"
"io"
"io/ioutil"
"net/http"
"net/url"
"regexp"
"strings"
"time"
"github.com/Knetic/govaluate"
"github.com/projectdiscovery/nuclei/pkg/generators"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer"
"github.com/projectdiscovery/nuclei/v2/pkg/syncedreadcloser"
)
// MakeHTTPRequest makes the HTTP request
func (r *Request) MakeHTTPRequest(baseURL string, dynamicValues map[string]interface{}, data string) (*HTTPRequest, error) {
ctx := context.Background()
parsed, err := url.Parse(baseURL)
if err != nil {
return nil, err
}
hostname := parsed.Host
values := generators.MergeMaps(dynamicValues, map[string]interface{}{
"BaseURL": baseURLWithTemplatePrefs(data, parsed),
"Hostname": hostname,
})
// if data contains \n it's a raw request
if strings.Contains(data, "\n") {
return r.makeHTTPRequestFromRaw(ctx, baseURL, data, values)
}
return r.makeHTTPRequestFromModel(ctx, data, values)
}
// MakeHTTPRequestFromModel creates a *http.Request from a request template
func (r *Request) makeHTTPRequestFromModel(ctx context.Context, data string, values map[string]interface{}) (*HTTPRequest, error) {
replacer := newReplacer(values)
URL := replacer.Replace(data)
// Build a request on the specified URL
req, err := http.NewRequestWithContext(ctx, r.Method, URL, nil)
if err != nil {
return nil, err
}
request, err := r.fillRequest(req, values)
if err != nil {
return nil, err
}
return &HTTPRequest{Request: request}, nil
}
// InitGenerator initializes the generator
func (r *Request) InitGenerator() {
r.gsfm = NewGeneratorFSM(r.attackType, r.Payloads, r.Path, r.Raw)
}
// CreateGenerator creates the generator
func (r *Request) CreateGenerator(reqURL string) {
r.gsfm.Add(reqURL)
}
// HasGenerator check if an URL has a generator
func (r *Request) HasGenerator(reqURL string) bool {
return r.gsfm.Has(reqURL)
}
// ReadOne reads and return a generator by URL
func (r *Request) ReadOne(reqURL string) {
r.gsfm.ReadOne(reqURL)
}
// makeHTTPRequestFromRaw creates a *http.Request from a raw request
func (r *Request) makeHTTPRequestFromRaw(ctx context.Context, baseURL, data string, values map[string]interface{}) (*HTTPRequest, error) {
// Add trailing line
data += "\n"
if len(r.Payloads) > 0 {
r.gsfm.InitOrSkip(baseURL)
r.ReadOne(baseURL)
payloads, err := r.GetPayloadsValues(baseURL)
if err != nil {
return nil, err
}
return r.handleRawWithPaylods(ctx, data, baseURL, values, payloads)
}
// otherwise continue with normal flow
return r.handleRawWithPaylods(ctx, data, baseURL, values, nil)
}
func (r *Request) handleRawWithPaylods(ctx context.Context, raw, baseURL string, values, genValues map[string]interface{}) (*HTTPRequest, error) {
baseValues := generators.CopyMap(values)
finValues := generators.MergeMaps(baseValues, genValues)
replacer := newReplacer(finValues)
// Replace the dynamic variables in the URL if any
raw = replacer.Replace(raw)
dynamicValues := make(map[string]interface{})
// find all potentials tokens between {{}}
var re = regexp.MustCompile(`(?m)\{\{[^}]+\}\}`)
for _, match := range re.FindAllString(raw, -1) {
// check if the match contains a dynamic variable
expr := generators.TrimDelimiters(match)
compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, generators.HelperFunctions())
if err != nil {
return nil, err
}
result, err := compiled.Evaluate(finValues)
if err != nil {
return nil, err
}
dynamicValues[expr] = result
}
// replace dynamic values
dynamicReplacer := newReplacer(dynamicValues)
raw = dynamicReplacer.Replace(raw)
rawRequest, err := r.parseRawRequest(raw, baseURL)
if err != nil {
return nil, err
}
// rawhttp
if r.Unsafe {
unsafeReq := &HTTPRequest{
RawRequest: rawRequest,
Meta: genValues,
AutomaticHostHeader: !r.DisableAutoHostname,
AutomaticContentLengthHeader: !r.DisableAutoContentLength,
Unsafe: true,
FollowRedirects: r.Redirects,
}
return unsafeReq, nil
}
// retryablehttp
var body io.ReadCloser
body = ioutil.NopCloser(strings.NewReader(rawRequest.Data))
if r.Race {
// More or less this ensures that all requests hit the endpoint at the same approximated time
// Todo: sync internally upon writing latest request byte
body = syncedreadcloser.NewOpenGateWithTimeout(body, time.Duration(two)*time.Second)
}
req, err := http.NewRequestWithContext(ctx, rawRequest.Method, rawRequest.FullURL, body)
if err != nil {
return nil, err
}
// copy headers
for key, value := range rawRequest.Headers {
req.Header[key] = []string{value}
}
request, err := r.fillRequest(req, values)
if err != nil {
return nil, err
}
return &HTTPRequest{Request: request, Meta: genValues}, nil
}
func (r *Request) fillRequest(req *http.Request, values map[string]interface{}) (*retryablehttp.Request, error) {
replacer := replacer.New(values)
// Set the header values requested
for header, value := range r.Headers {
req.Header[header] = []string{replacer.Replace(value)}
}
// In case of multiple threads the underlying connection should remain open to allow reuse
if r.Threads <= 0 && req.Header.Get("Connection") == "" {
req.Close = true
}
// Check if the user requested a request body
if r.Body != "" {
req.Body = ioutil.NopCloser(strings.NewReader(r.Body))
}
setHeader(req, "User-Agent", "Nuclei - Open-source project (github.com/projectdiscovery/nuclei)")
// raw requests are left untouched
if len(r.Raw) > 0 {
return retryablehttp.FromRequest(req)
}
setHeader(req, "Accept", "*/*")
setHeader(req, "Accept-Language", "en")
return retryablehttp.FromRequest(req)
}
// setHeader sets some headers only if the header wasn't supplied by the user
func setHeader(req *http.Request, name, value string) {
if _, ok := req.Header[name]; !ok {
req.Header.Set(name, value)
}
}

2
v2/pkg/protocols/protocols.go
View File

@ -18,7 +18,7 @@ type Executer interface {
// Execute executes the protocol group and returns true or false if results were found.
Execute(input string) (bool, error)
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
ExecuteWithResults(input string) ([]*output.ResultEvent, error)
ExecuteWithResults(input string) ([]*output.InternalWrappedEvent, error)
}
// ExecuterOptions contains the configuration options for executer clients

196
v2/pkg/requests/bulk-http-request.go
View File

@ -1,21 +1,15 @@
package requests
import (
"context"
"fmt"
"io"
"io/ioutil"
"net"
"net/http"
"net/url"
"regexp"
"strings"
"time"
"github.com/Knetic/govaluate"
"github.com/projectdiscovery/nuclei/v2/pkg/generators"
"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/syncedreadcloser"
"github.com/projectdiscovery/rawhttp"
retryablehttp "github.com/projectdiscovery/retryablehttp-go"
)
@ -52,196 +46,6 @@ func (r *BulkHTTPRequest) GetRequestCount() int64 {
return int64(r.gsfm.Total())
}
// MakeHTTPRequest makes the HTTP request
func (r *BulkHTTPRequest) MakeHTTPRequest(baseURL string, dynamicValues map[string]interface{}, data string) (*HTTPRequest, error) {
ctx := context.Background()
parsed, err := url.Parse(baseURL)
if err != nil {
return nil, err
}
hostname := parsed.Host
values := generators.MergeMaps(dynamicValues, map[string]interface{}{
"BaseURL": baseURLWithTemplatePrefs(data, parsed),
"Hostname": hostname,
})
// if data contains \n it's a raw request
if strings.Contains(data, "\n") {
return r.makeHTTPRequestFromRaw(ctx, baseURL, data, values)
}
return r.makeHTTPRequestFromModel(ctx, data, values)
}
// MakeHTTPRequestFromModel creates a *http.Request from a request template
func (r *BulkHTTPRequest) makeHTTPRequestFromModel(ctx context.Context, data string, values map[string]interface{}) (*HTTPRequest, error) {
replacer := newReplacer(values)
URL := replacer.Replace(data)
// Build a request on the specified URL
req, err := http.NewRequestWithContext(ctx, r.Method, URL, nil)
if err != nil {
return nil, err
}
request, err := r.fillRequest(req, values)
if err != nil {
return nil, err
}
return &HTTPRequest{Request: request}, nil
}
// InitGenerator initializes the generator
func (r *BulkHTTPRequest) InitGenerator() {
r.gsfm = NewGeneratorFSM(r.attackType, r.Payloads, r.Path, r.Raw)
}
// CreateGenerator creates the generator
func (r *BulkHTTPRequest) CreateGenerator(reqURL string) {
r.gsfm.Add(reqURL)
}
// HasGenerator check if an URL has a generator
func (r *BulkHTTPRequest) HasGenerator(reqURL string) bool {
return r.gsfm.Has(reqURL)
}
// ReadOne reads and return a generator by URL
func (r *BulkHTTPRequest) ReadOne(reqURL string) {
r.gsfm.ReadOne(reqURL)
}
// makeHTTPRequestFromRaw creates a *http.Request from a raw request
func (r *BulkHTTPRequest) makeHTTPRequestFromRaw(ctx context.Context, baseURL, data string, values map[string]interface{}) (*HTTPRequest, error) {
// Add trailing line
data += "\n"
if len(r.Payloads) > 0 {
r.gsfm.InitOrSkip(baseURL)
r.ReadOne(baseURL)
payloads, err := r.GetPayloadsValues(baseURL)
if err != nil {
return nil, err
}
return r.handleRawWithPaylods(ctx, data, baseURL, values, payloads)
}
// otherwise continue with normal flow
return r.handleRawWithPaylods(ctx, data, baseURL, values, nil)
}
func (r *BulkHTTPRequest) handleRawWithPaylods(ctx context.Context, raw, baseURL string, values, genValues map[string]interface{}) (*HTTPRequest, error) {
baseValues := generators.CopyMap(values)
finValues := generators.MergeMaps(baseValues, genValues)
replacer := newReplacer(finValues)
// Replace the dynamic variables in the URL if any
raw = replacer.Replace(raw)
dynamicValues := make(map[string]interface{})
// find all potentials tokens between {{}}
var re = regexp.MustCompile(`(?m)\{\{[^}]+\}\}`)
for _, match := range re.FindAllString(raw, -1) {
// check if the match contains a dynamic variable
expr := generators.TrimDelimiters(match)
compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, generators.HelperFunctions())
if err != nil {
return nil, err
}
result, err := compiled.Evaluate(finValues)
if err != nil {
return nil, err
}
dynamicValues[expr] = result
}
// replace dynamic values
dynamicReplacer := newReplacer(dynamicValues)
raw = dynamicReplacer.Replace(raw)
rawRequest, err := r.parseRawRequest(raw, baseURL)
if err != nil {
return nil, err
}
// rawhttp
if r.Unsafe {
unsafeReq := &HTTPRequest{
RawRequest: rawRequest,
Meta: genValues,
AutomaticHostHeader: !r.DisableAutoHostname,
AutomaticContentLengthHeader: !r.DisableAutoContentLength,
Unsafe: true,
FollowRedirects: r.Redirects,
}
return unsafeReq, nil
}
// retryablehttp
var body io.ReadCloser
body = ioutil.NopCloser(strings.NewReader(rawRequest.Data))
if r.Race {
// More or less this ensures that all requests hit the endpoint at the same approximated time
// Todo: sync internally upon writing latest request byte
body = syncedreadcloser.NewOpenGateWithTimeout(body, time.Duration(two)*time.Second)
}
req, err := http.NewRequestWithContext(ctx, rawRequest.Method, rawRequest.FullURL, body)
if err != nil {
return nil, err
}
// copy headers
for key, value := range rawRequest.Headers {
req.Header[key] = []string{value}
}
request, err := r.fillRequest(req, values)
if err != nil {
return nil, err
}
return &HTTPRequest{Request: request, Meta: genValues}, nil
}
func (r *BulkHTTPRequest) fillRequest(req *http.Request, values map[string]interface{}) (*retryablehttp.Request, error) {
replacer := newReplacer(values)
// Set the header values requested
for header, value := range r.Headers {
req.Header[header] = []string{replacer.Replace(value)}
}
// In case of multiple threads the underlying connection should remain open to allow reuse
if r.Threads <= 0 && req.Header.Get("Connection") == "" {
req.Close = true
}
// Check if the user requested a request body
if r.Body != "" {
req.Body = ioutil.NopCloser(strings.NewReader(r.Body))
}
setHeader(req, "User-Agent", "Nuclei - Open-source project (github.com/projectdiscovery/nuclei)")
// raw requests are left untouched
if len(r.Raw) > 0 {
return retryablehttp.FromRequest(req)
}
setHeader(req, "Accept", "*/*")
setHeader(req, "Accept-Language", "en")
return retryablehttp.FromRequest(req)
}
// HTTPRequest is the basic HTTP request
type HTTPRequest struct {
Request *retryablehttp.Request