Misc work on extractors + compat

dev
Ice3man543 2020-12-24 12:13:18 +05:30
parent 10642c6c77
commit 5153647e0f
12 changed files with 177 additions and 211 deletions

View File

@ -3,61 +3,31 @@ package executer
import (
"fmt"
"os"
"regexp"
"strings"
"github.com/pkg/errors"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/internal/bufwriter"
"github.com/projectdiscovery/nuclei/v2/internal/progress"
"github.com/projectdiscovery/nuclei/v2/internal/tracelog"
"github.com/projectdiscovery/nuclei/v2/pkg/colorizer"
"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/requests"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
retryabledns "github.com/projectdiscovery/retryabledns"
"go.uber.org/ratelimit"
)
// DNSExecuter is a client for performing a DNS request
// for a template.
type DNSExecuter struct {
dnsClient *retryabledns.Client
template *templates.Template
dnsRequest *requests.DNSRequest
ratelimiter ratelimit.Limiter
}
// DefaultResolvers contains the list of resolvers known to be trusted.
var DefaultResolvers = []string{
"1.1.1.1:53", // Cloudflare
"1.0.0.1:53", // Cloudflare
"8.8.8.8:53", // Google
"8.8.4.4:53", // Google
template *templates.Template
}
// DNSOptions contains configuration options for the DNS executer.
type DNSOptions struct {
ColoredOutput bool
Debug bool
JSON bool
JSONRequests bool
NoMeta bool
VHost bool
TraceLog tracelog.Log
Template *templates.Template
DNSRequest *requests.DNSRequest
Writer *bufwriter.Writer
Colorizer colorizer.NucleiColorizer
Decolorizer *regexp.Regexp
RateLimiter ratelimit.Limiter
Template *templates.Template
DNSRequest *requests.DNSRequest
}
// NewDNSExecuter creates a new DNS executer from a template
// and a DNS request query.
func NewDNSExecuter(options *DNSOptions) *DNSExecuter {
dnsClient := retryabledns.New(DefaultResolvers, options.DNSRequest.Retries)
executer := &DNSExecuter{
debug: options.Debug,

View File

@ -18,7 +18,6 @@ import (
"github.com/corpix/uarand"
"github.com/pkg/errors"
"github.com/projectdiscovery/fastdialer/fastdialer"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/internal/bufwriter"
"github.com/projectdiscovery/nuclei/v2/internal/progress"
@ -74,8 +73,6 @@ type HTTPOptions struct {
BulkHTTPRequest *requests.BulkHTTPRequest
CookieJar *cookiejar.Jar
PF *projetctfile.ProjectFile
RateLimiter ratelimit.Limiter
Dialer *fastdialer.Dialer
}
// NewHTTPExecuter creates a new HTTP executer from a template
@ -93,7 +90,6 @@ func NewHTTPExecuter(options *HTTPOptions) (*HTTPExecuter, error) {
// Create the HTTP Client
client := makeHTTPClient(proxyURL, options)
// nolint:bodyclose // false positive there is no body to close yet
client.CheckRetry = retryablehttp.HostSprayRetryPolicy()
if options.CookieJar != nil {
client.HTTPClient.Jar = options.CookieJar
@ -105,9 +101,6 @@ func NewHTTPExecuter(options *HTTPOptions) (*HTTPExecuter, error) {
client.HTTPClient.Jar = jar
}
// initiate raw http client
rawClient := rawhttp.NewClient(rawhttp.DefaultOptions)
executer := &HTTPExecuter{
debug: options.Debug,
jsonOutput: options.JSON,
@ -257,17 +250,6 @@ func (e *HTTPExecuter) ExecuteTurboHTTP(reqURL string) *Result {
return result
}
pipeOptions := rawhttp.DefaultPipelineOptions
pipeOptions.Host = URL.Host
pipeOptions.MaxConnections = 1
if e.bulkHTTPRequest.PipelineConcurrentConnections > 0 {
pipeOptions.MaxConnections = e.bulkHTTPRequest.PipelineConcurrentConnections
}
if e.bulkHTTPRequest.PipelineRequestsPerConnection > 0 {
pipeOptions.MaxPendingRequests = e.bulkHTTPRequest.PipelineRequestsPerConnection
}
pipeclient := rawhttp.NewPipelineClient(pipeOptions)
// defaultMaxWorkers should be a sufficient value to keep queues always full
maxWorkers := defaultMaxWorkers
// in case the queue is bigger increase the workers

View File

@ -18,8 +18,8 @@ import (
"github.com/Knetic/govaluate"
"github.com/projectdiscovery/nuclei/v2/internal/collaborator"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
"github.com/spaolacci/murmur3"
"github.com/spf13/cast"
)
const (
@ -35,110 +35,110 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
functions := make(map[string]govaluate.ExpressionFunction)
functions["len"] = func(args ...interface{}) (interface{}, error) {
length := len(cast.ToString(args[0]))
length := len(types.ToString(args[0]))
return float64(length), nil
}
functions["toupper"] = func(args ...interface{}) (interface{}, error) {
return strings.ToUpper(cast.ToString(args[0])), nil
return strings.ToUpper(types.ToString(args[0])), nil
}
functions["tolower"] = func(args ...interface{}) (interface{}, error) {
return strings.ToLower(cast.ToString(args[0])), nil
return strings.ToLower(types.ToString(args[0])), nil
}
functions["replace"] = func(args ...interface{}) (interface{}, error) {
return strings.ReplaceAll(cast.ToString(args[0]), cast.ToString(args[1]), cast.ToString(args[2])), nil
return strings.ReplaceAll(types.ToString(args[0]), types.ToString(args[1]), types.ToString(args[2])), nil
}
functions["replace_regex"] = func(args ...interface{}) (interface{}, error) {
compiled, err := regexp.Compile(cast.ToString(args[1]))
compiled, err := regexp.Compile(types.ToString(args[1]))
if err != nil {
return nil, err
}
return compiled.ReplaceAllString(cast.ToString(args[0]), cast.ToString(args[2])), nil
return compiled.ReplaceAllString(types.ToString(args[0]), types.ToString(args[2])), nil
}
functions["trim"] = func(args ...interface{}) (interface{}, error) {
return strings.Trim(cast.ToString(args[0]), cast.ToString(args[2])), nil
return strings.Trim(types.ToString(args[0]), types.ToString(args[2])), nil
}
functions["trimleft"] = func(args ...interface{}) (interface{}, error) {
return strings.TrimLeft(cast.ToString(args[0]), cast.ToString(args[1])), nil
return strings.TrimLeft(types.ToString(args[0]), types.ToString(args[1])), nil
}
functions["trimright"] = func(args ...interface{}) (interface{}, error) {
return strings.TrimRight(cast.ToString(args[0]), cast.ToString(args[1])), nil
return strings.TrimRight(types.ToString(args[0]), types.ToString(args[1])), nil
}
functions["trimspace"] = func(args ...interface{}) (interface{}, error) {
return strings.TrimSpace(cast.ToString(args[0])), nil
return strings.TrimSpace(types.ToString(args[0])), nil
}
functions["trimprefix"] = func(args ...interface{}) (interface{}, error) {
return strings.TrimPrefix(cast.ToString(args[0]), cast.ToString(args[1])), nil
return strings.TrimPrefix(types.ToString(args[0]), types.ToString(args[1])), nil
}
functions["trimsuffix"] = func(args ...interface{}) (interface{}, error) {
return strings.TrimSuffix(cast.ToString(args[0]), cast.ToString(args[1])), nil
return strings.TrimSuffix(types.ToString(args[0]), types.ToString(args[1])), nil
}
functions["reverse"] = func(args ...interface{}) (interface{}, error) {
return reverseString(cast.ToString(args[0])), nil
return reverseString(types.ToString(args[0])), nil
}
// encoding
functions["base64"] = func(args ...interface{}) (interface{}, error) {
sEnc := base64.StdEncoding.EncodeToString([]byte(cast.ToString(args[0])))
sEnc := base64.StdEncoding.EncodeToString([]byte(types.ToString(args[0])))
return sEnc, nil
}
// python encodes to base64 with lines of 76 bytes terminated by new line "\n"
functions["base64_py"] = func(args ...interface{}) (interface{}, error) {
sEnc := base64.StdEncoding.EncodeToString([]byte(cast.ToString(args[0])))
sEnc := base64.StdEncoding.EncodeToString([]byte(types.ToString(args[0])))
return insertInto(sEnc, 76, '\n'), nil
}
functions["base64_decode"] = func(args ...interface{}) (interface{}, error) {
return base64.StdEncoding.DecodeString(cast.ToString(args[0]))
return base64.StdEncoding.DecodeString(types.ToString(args[0]))
}
functions["url_encode"] = func(args ...interface{}) (interface{}, error) {
return url.PathEscape(cast.ToString(args[0])), nil
return url.PathEscape(types.ToString(args[0])), nil
}
functions["url_decode"] = func(args ...interface{}) (interface{}, error) {
return url.PathUnescape(cast.ToString(args[0]))
return url.PathUnescape(types.ToString(args[0]))
}
functions["hex_encode"] = func(args ...interface{}) (interface{}, error) {
return hex.EncodeToString([]byte(cast.ToString(args[0]))), nil
return hex.EncodeToString([]byte(types.ToString(args[0]))), nil
}
functions["hex_decode"] = func(args ...interface{}) (interface{}, error) {
hx, _ := hex.DecodeString(cast.ToString(args[0]))
hx, _ := hex.DecodeString(types.ToString(args[0]))
return string(hx), nil
}
functions["html_escape"] = func(args ...interface{}) (interface{}, error) {
return html.EscapeString(cast.ToString(args[0])), nil
return html.EscapeString(types.ToString(args[0])), nil
}
functions["html_unescape"] = func(args ...interface{}) (interface{}, error) {
return html.UnescapeString(cast.ToString(args[0])), nil
return html.UnescapeString(types.ToString(args[0])), nil
}
// hashing
functions["md5"] = func(args ...interface{}) (interface{}, error) {
hash := md5.Sum([]byte(cast.ToString(args[0])))
hash := md5.Sum([]byte(types.ToString(args[0])))
return hex.EncodeToString(hash[:]), nil
}
functions["sha256"] = func(args ...interface{}) (interface{}, error) {
h := sha256.New()
_, err := h.Write([]byte(cast.ToString(args[0])))
_, err := h.Write([]byte(types.ToString(args[0])))
if err != nil {
return nil, err
@ -148,7 +148,7 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
functions["sha1"] = func(args ...interface{}) (interface{}, error) {
h := sha1.New()
_, err := h.Write([]byte(cast.ToString(args[0])))
_, err := h.Write([]byte(types.ToString(args[0])))
if err != nil {
return nil, err
@ -157,20 +157,20 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
}
functions["mmh3"] = func(args ...interface{}) (interface{}, error) {
return fmt.Sprintf("%d", int32(murmur3.Sum32WithSeed([]byte(cast.ToString(args[0])), 0))), nil
return fmt.Sprintf("%d", int32(murmur3.Sum32WithSeed([]byte(types.ToString(args[0])), 0))), nil
}
// search
functions["contains"] = func(args ...interface{}) (interface{}, error) {
return strings.Contains(cast.ToString(args[0]), cast.ToString(args[1])), nil
return strings.Contains(types.ToString(args[0]), types.ToString(args[1])), nil
}
functions["regex"] = func(args ...interface{}) (interface{}, error) {
compiled, err := regexp.Compile(cast.ToString(args[0]))
compiled, err := regexp.Compile(types.ToString(args[0]))
if err != nil {
return nil, err
}
return compiled.MatchString(cast.ToString(args[1])), nil
return compiled.MatchString(types.ToString(args[1])), nil
}
// random generators
@ -178,10 +178,10 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
chars := letters + numbers
bad := ""
if len(args) >= 1 {
chars = cast.ToString(args[0])
chars = types.ToString(args[0])
}
if len(args) >= withCutSetArgsSize {
bad = cast.ToString(args[1])
bad = types.ToString(args[1])
}
chars = trimAll(chars, bad)
return chars[rand.Intn(len(chars))], nil
@ -196,10 +196,10 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
l = args[0].(int)
}
if len(args) >= withCutSetArgsSize {
bad = cast.ToString(args[1])
bad = types.ToString(args[1])
}
if len(args) >= withBaseRandArgsSize {
base = cast.ToString(args[2])
base = types.ToString(args[2])
}
base = trimAll(base, bad)
return randSeq(base, l), nil
@ -214,7 +214,7 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
l = args[0].(int)
}
if len(args) >= withCutSetArgsSize {
bad = cast.ToString(args[1])
bad = types.ToString(args[1])
}
chars = trimAll(chars, bad)
return randSeq(chars, l), nil
@ -229,7 +229,7 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
l = args[0].(int)
}
if len(args) >= withCutSetArgsSize {
bad = cast.ToString(args[1])
bad = types.ToString(args[1])
}
chars = trimAll(chars, bad)
return randSeq(chars, l), nil
@ -244,7 +244,7 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
l = args[0].(int)
}
if len(args) >= withCutSetArgsSize {
bad = cast.ToString(args[1])
bad = types.ToString(args[1])
}
chars = trimAll(chars, bad)
return randSeq(chars, l), nil
@ -273,7 +273,7 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction {
// Collaborator
functions["collab"] = func(args ...interface{}) (interface{}, error) {
// check if collaborator contains a specific pattern
return collaborator.DefaultCollaborator.Has(cast.ToString(args[0])), nil
return collaborator.DefaultCollaborator.Has(types.ToString(args[0])), nil
}
return functions
}

View File

@ -25,14 +25,8 @@ func (e *Extractor) CompileExtractors() error {
}
// Setup the part of the request to match, if any.
if e.Part != "" {
e.part, ok = PartTypes[e.Part]
if !ok {
return fmt.Errorf("unknown matcher part specified: %s", e.Part)
}
} else {
e.part = BodyPart
if e.Part == "" {
e.Part = "body"
}
return nil
}

View File

@ -1,52 +1,21 @@
package extractors
import (
"net/http"
import "github.com/projectdiscovery/nuclei/v2/pkg/types"
"github.com/miekg/dns"
)
// Extract extracts data from an output structure based on user options
func (e *Extractor) Extract(data map[string]interface{}) map[string]struct{} {
part, ok := data[e.Part]
if !ok {
return nil
}
partString := types.ToString(part)
// Extract extracts response from the parts of request using a regex
func (e *Extractor) Extract(resp *http.Response, body, headers string) map[string]struct{} {
switch e.extractorType {
case RegexExtractor:
if e.part == BodyPart {
return e.extractRegex(body)
} else if e.part == HeaderPart {
return e.extractRegex(headers)
} else {
matches := e.extractRegex(headers)
if len(matches) > 0 {
return matches
}
return e.extractRegex(body)
}
return e.extractRegex(partString)
case KValExtractor:
if e.part == HeaderPart {
return e.extractKVal(resp)
}
matches := e.extractKVal(resp)
if len(matches) > 0 {
return matches
}
return e.extractCookieKVal(resp)
return e.extractKVal(data)
}
return nil
}
// ExtractDNS extracts response from dns message using a regex
// nolint:interfacer // dns.Msg is out of current scope
func (e *Extractor) ExtractDNS(msg *dns.Msg) map[string]struct{} {
switch e.extractorType {
case RegexExtractor:
return e.extractRegex(msg.String())
case KValExtractor:
}
return nil
}
@ -57,39 +26,34 @@ func (e *Extractor) extractRegex(corpus string) map[string]struct{} {
groupPlusOne := e.RegexGroup + 1
for _, regex := range e.regexCompiled {
matches := regex.FindAllStringSubmatch(corpus, -1)
for _, match := range matches {
if len(match) >= groupPlusOne {
results[match[e.RegexGroup]] = struct{}{}
if len(match) < groupPlusOne {
continue
}
matchString := match[e.RegexGroup]
if _, ok := results[matchString]; !ok {
results[matchString] = struct{}{}
}
}
}
return results
}
// extractKVal extracts text from http response
func (e *Extractor) extractKVal(r *http.Response) map[string]struct{} {
// extractKVal extracts key value pairs from a data map
func (e *Extractor) extractKVal(data map[string]interface{}) map[string]struct{} {
results := make(map[string]struct{})
for _, k := range e.KVal {
for _, v := range r.Header.Values(k) {
results[v] = struct{}{}
item, ok := data[k]
if !ok {
continue
}
itemString := types.ToString(item)
if _, ok := results[itemString]; !ok {
results[itemString] = struct{}{}
}
}
return results
}
// extractCookieKVal extracts text from cookies
func (e *Extractor) extractCookieKVal(r *http.Response) map[string]struct{} {
results := make(map[string]struct{})
for _, k := range e.KVal {
for _, cookie := range r.Cookies() {
if cookie.Name == k {
results[cookie.Value] = struct{}{}
}
}
}
return results
}

View File

@ -25,8 +25,6 @@ type Extractor struct {
//
// By default, matching is performed in request body.
Part string `yaml:"part,omitempty"`
// part is the part of the request to match
part Part
// Internal defines if this is used internally
Internal bool `yaml:"internal,omitempty"`
}
@ -46,27 +44,3 @@ var ExtractorTypes = map[string]ExtractorType{
"regex": RegexExtractor,
"kval": KValExtractor,
}
// Part is the part of the request to match
type Part int
const (
// BodyPart matches body of the response.
BodyPart Part = iota + 1
// HeaderPart matches headers of the response.
HeaderPart
// AllPart matches both response body and headers of the response.
AllPart
)
// PartTypes is an table for conversion of part type from string.
var PartTypes = map[string]Part{
"body": BodyPart,
"header": HeaderPart,
"all": AllPart,
}
// GetPart returns the part of the matcher
func (e *Extractor) GetPart() Part {
return e.part
}

View File

@ -17,9 +17,9 @@ func (m *Matcher) CompileMatchers() error {
if !ok {
return fmt.Errorf("unknown matcher type specified: %s", m.Type)
}
// By default, match on all if user hasn't provided any specific items
// By default, match on body if user hasn't provided any specific items
if m.Part == "" {
m.Part = "all"
m.Part = "body"
}
// Compile the regexes

View File

@ -4,7 +4,7 @@ import (
"bytes"
"errors"
"github.com/spf13/cast"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
)
// formatScreen formats the output for showing on screen.
@ -52,7 +52,7 @@ func (w *StandardWriter) formatScreen(output Event) ([]byte, error) {
if ok {
builder.WriteString(" [")
extractorResults := cast.ToStringSlice(extractedResults)
extractorResults := types.ToStringSlice(extractedResults)
for i, item := range extractorResults {
builder.WriteString(w.aurora.BrightCyan(item).String())
@ -68,7 +68,7 @@ func (w *StandardWriter) formatScreen(output Event) ([]byte, error) {
if ok {
builder.WriteString(" [")
metaResults := cast.ToStringMap(metaResults)
metaResults := types.ToStringMap(metaResults)
var first = true
for name, value := range metaResults {
@ -79,7 +79,7 @@ func (w *StandardWriter) formatScreen(output Event) ([]byte, error) {
builder.WriteString(w.aurora.BrightYellow(name).String())
builder.WriteRune('=')
builder.WriteString(w.aurora.BrightYellow(cast.ToString(value)).String())
builder.WriteString(w.aurora.BrightYellow(types.ToString(value)).String())
}
builder.WriteString("]")
}

View File

@ -35,7 +35,9 @@ func responseToDSLMap(msg *dns.Msg) map[string]interface{} {
data["ns"] = buffer.String()
buffer.Reset()
data["raw"] = msg.String()
rawData := msg.String()
data["raw"] = rawData
data["body"] = rawData // Use rawdata as body for dns responses matching
data["status_code"] = msg.Rcode
return data
}

View File

@ -9,7 +9,7 @@ import (
// responseToDSLMap converts a HTTP response to a map for use in DSL matching
func responseToDSLMap(resp *http.Response, body, headers string, duration time.Duration, extra map[string]interface{}) map[string]interface{} {
data := make(map[string]interface{}, len(extra)+6+len(resp.Header))
data := make(map[string]interface{}, len(extra)+6+len(resp.Header)+len(resp.Cookies()))
for k, v := range extra {
data[k] = v
}
@ -18,14 +18,20 @@ func responseToDSLMap(resp *http.Response, body, headers string, duration time.D
data["status_code"] = resp.StatusCode
data["body"] = body
for _, cookie := range resp.Cookies() {
data[cookie.Name] = cookie.Value
}
for k, v := range resp.Header {
k = strings.ToLower(strings.TrimSpace(strings.ReplaceAll(k, "-", "_")))
data[k] = strings.Join(v, " ")
}
data["headers"] = headers
data["header"] = headers
data["all_headers"] = headers
if r, err := httputil.DumpResponse(resp, true); err == nil {
data["raw"] = string(r)
rawString := string(r)
data["raw"] = rawString
data["all"] = rawString
}
data["duration"] = duration.Seconds()
return data

View File

@ -279,24 +279,9 @@ func baseURLWithTemplatePrefs(data string, parsedURL *url.URL) string {
parsedURL.Host = hostname
}
}
return parsedURL.String()
}
// CustomHeaders valid for all requests
type CustomHeaders []string
// String returns just a label
func (c *CustomHeaders) String() string {
return "Custom Global Headers"
}
// Set a new global header
func (c *CustomHeaders) Set(value string) error {
*c = append(*c, value)
return nil
}
// Next returns the next generator by URL
func (r *BulkHTTPRequest) Next(reqURL string) bool {
return r.gsfm.Next(reqURL)

View File

@ -0,0 +1,89 @@
// Taken from https://github.com/spf13/cast.
package types
import (
"fmt"
"strconv"
"strings"
)
// ToString converts an interface to string in a quick way
func ToString(data interface{}) string {
switch s := data.(type) {
case string:
return s
case bool:
return strconv.FormatBool(s)
case float64:
return strconv.FormatFloat(s, 'f', -1, 64)
case float32:
return strconv.FormatFloat(float64(s), 'f', -1, 32)
case int:
return strconv.Itoa(s)
case int64:
return strconv.FormatInt(s, 10)
case int32:
return strconv.Itoa(int(s))
case int16:
return strconv.FormatInt(int64(s), 10)
case int8:
return strconv.FormatInt(int64(s), 10)
case uint:
return strconv.FormatUint(uint64(s), 10)
case uint64:
return strconv.FormatUint(uint64(s), 10)
case uint32:
return strconv.FormatUint(uint64(s), 10)
case uint16:
return strconv.FormatUint(uint64(s), 10)
case uint8:
return strconv.FormatUint(uint64(s), 10)
case []byte:
return string(s)
case fmt.Stringer:
return s.String()
case error:
return s.Error()
default:
return fmt.Sprintf("%v", data)
}
}
// ToStringSlice casts an interface to a []string type.
func ToStringSlice(i interface{}) []string {
var a []string
switch v := i.(type) {
case []interface{}:
for _, u := range v {
a = append(a, ToString(u))
}
return a
case []string:
return v
case string:
return strings.Fields(v)
case interface{}:
return []string{ToString(v)}
default:
return nil
}
}
// ToStringMap casts an interface to a map[string]interface{} type.
func ToStringMap(i interface{}) map[string]interface{} {
var m = map[string]interface{}{}
switch v := i.(type) {
case map[interface{}]interface{}:
for k, val := range v {
m[ToString(k)] = val
}
return m
case map[string]interface{}:
return v
default:
return nil
}
}