daffainfo
/
nuclei
mirror of https://github.com/daffainfo/nuclei.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

More refactoring of nuclei packages

dev
Ice3man543 2020-12-24 20:47:41 +05:30
parent 2b50d99c0c
commit 60789f4ba2
9 changed files with 116 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
v2/pkg/executer/executer_dns.go
View File

@ -8,7 +8,6 @@ import (
"github.com/pkg/errors"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/internal/progress"
"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/requests"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
)
@ -95,45 +94,6 @@ func (e *DNSExecuter) ExecuteDNS(p *progress.Progress, reqURL string) *Result {
fmt.Fprintf(os.Stderr, "%s\n", resp.String())
}
matcherCondition := e.dnsRequest.GetMatchersCondition()
for _, matcher := range e.dnsRequest.Matchers {
// Check if the matcher matched
if !matcher.MatchDNS(resp) {
// If the condition is AND we haven't matched, return.
if matcherCondition == matchers.ANDCondition {
return result
}
} else {
// If the matcher has matched, and its an OR
// write the first output then move to next matcher.
if matcherCondition == matchers.ORCondition && len(e.dnsRequest.Extractors) == 0 {
e.writeOutputDNS(domain, compiledRequest, resp, matcher, nil)
result.GotResults = true
}
}
}
// All matchers have successfully completed so now start with the
// next task which is extraction of input from matchers.
var extractorResults []string
for _, extractor := range e.dnsRequest.Extractors {
for match := range extractor.ExtractDNS(resp) {
if !extractor.Internal {
extractorResults = append(extractorResults, match)
}
}
}
// Write a final string of output if matcher type is
// AND or if we have extractors for the mechanism too.
if len(e.dnsRequest.Extractors) > 0 || matcherCondition == matchers.ANDCondition {
e.writeOutputDNS(domain, compiledRequest, resp, nil, extractorResults)
result.GotResults = true
}
return result
}

25
v2/pkg/operators/extractors/extract.go
View File

@ -2,25 +2,8 @@ package extractors
import "github.com/projectdiscovery/nuclei/v2/pkg/types"
// Extract extracts data from an output structure based on user options
func (e *Extractor) Extract(data map[string]interface{}) map[string]struct{} {
part, ok := data[e.Part]
if !ok {
return nil
}
partString := types.ToString(part)
switch e.extractorType {
case RegexExtractor:
return e.extractRegex(partString)
case KValExtractor:
return e.extractKVal(data)
}
return nil
}
// extractRegex extracts text from a corpus and returns it
func (e *Extractor) extractRegex(corpus string) map[string]struct{} {
// ExtractRegex extracts text from a corpus and returns it
func (e *Extractor) ExtractRegex(corpus string) map[string]struct{} {
results := make(map[string]struct{})
groupPlusOne := e.RegexGroup + 1
@ -41,8 +24,8 @@ func (e *Extractor) extractRegex(corpus string) map[string]struct{} {
return results
}
// extractKVal extracts key value pairs from a data map
func (e *Extractor) extractKVal(data map[string]interface{}) map[string]struct{} {
// ExtractKval extracts key value pairs from a data map
func (e *Extractor) ExtractKval(data map[string]interface{}) map[string]struct{} {
results := make(map[string]struct{})
for _, k := range e.KVal {

5
v2/pkg/operators/extractors/extractors.go
View File

@ -44,3 +44,8 @@ var ExtractorTypes = map[string]ExtractorType{
"regex": RegexExtractor,
"kval": KValExtractor,
}
// GetType returns the type of the matcher
func (e *Extractor) GetType() ExtractorType {
return e.extractorType
}

55
v2/pkg/operators/matchers/match.go
View File

@ -5,37 +5,8 @@ import (
"strings"
)
// Match matches a generic data response again a given matcher
func (m *Matcher) Match(data map[string]interface{}) bool {
part, ok := data[m.Part]
if !ok {
return false
}
partString := part.(string)
switch m.matcherType {
case StatusMatcher:
statusCode, ok := data["status_code"]
if !ok {
return false
}
return m.isNegative(m.matchStatusCode(statusCode.(int)))
case SizeMatcher:
return m.isNegative(m.matchSizeCode(len(partString)))
case WordsMatcher:
return m.isNegative(m.matchWords(partString))
case RegexMatcher:
return m.isNegative(m.matchRegex(partString))
case BinaryMatcher:
return m.isNegative(m.matchBinary(partString))
case DSLMatcher:
return m.isNegative(m.matchDSL(data))
}
return false
}
// matchStatusCode matches a status code check against an HTTP Response
func (m *Matcher) matchStatusCode(statusCode int) bool {
// MatchStatusCode matches a status code check against a corpus
func (m *Matcher) MatchStatusCode(statusCode int) bool {
// Iterate over all the status codes accepted as valid
//
// Status codes don't support AND conditions.
@ -50,8 +21,8 @@ func (m *Matcher) matchStatusCode(statusCode int) bool {
return false
}
// matchStatusCode matches a size check against an HTTP Response
func (m *Matcher) matchSizeCode(length int) bool {
// MatchSize matches a size check against a corpus
func (m *Matcher) MatchSize(length int) bool {
// Iterate over all the sizes accepted as valid
//
// Sizes codes don't support AND conditions.
@ -66,8 +37,8 @@ func (m *Matcher) matchSizeCode(length int) bool {
return false
}
// matchWords matches a word check against an HTTP Response/Headers.
func (m *Matcher) matchWords(corpus string) bool {
// MatchWords matches a word check against a corpus.
func (m *Matcher) MatchWords(corpus string) bool {
// Iterate over all the words accepted as valid
for i, word := range m.Words {
// Continue if the word doesn't match
@ -94,8 +65,8 @@ func (m *Matcher) matchWords(corpus string) bool {
return false
}
// matchRegex matches a regex check against an HTTP Response/Headers.
func (m *Matcher) matchRegex(corpus string) bool {
// MatchRegex matches a regex check against a corpus
func (m *Matcher) MatchRegex(corpus string) bool {
// Iterate over all the regexes accepted as valid
for i, regex := range m.regexCompiled {
// Continue if the regex doesn't match
@ -122,8 +93,8 @@ func (m *Matcher) matchRegex(corpus string) bool {
return false
}
// matchWords matches a word check against an HTTP Response/Headers.
func (m *Matcher) matchBinary(corpus string) bool {
// MatchBinary matches a binary check against a corpus
func (m *Matcher) MatchBinary(corpus string) bool {
// Iterate over all the words accepted as valid
for i, binary := range m.Binary {
// Continue if the word doesn't match
@ -151,11 +122,11 @@ func (m *Matcher) matchBinary(corpus string) bool {
return false
}
// matchDSL matches on a generic map result
func (m *Matcher) matchDSL(mp map[string]interface{}) bool {
// MatchDSL matches on a generic map result
func (m *Matcher) MatchDSL(data map[string]interface{}) bool {
// Iterate over all the expressions accepted as valid
for i, expression := range m.dslCompiled {
result, err := expression.Evaluate(mp)
result, err := expression.Evaluate(data)
if err != nil {
continue
}

10
v2/pkg/operators/matchers/match_test.go
View File

@ -9,22 +9,22 @@ import (
func TestANDCondition(t *testing.T) {
m := &Matcher{condition: ANDCondition, Words: []string{"a", "b"}}
matched := m.matchWords("a b")
matched := m.MatchWords("a b")
require.True(t, matched, "Could not match valid AND condition")
matched = m.matchWords("b")
matched = m.MatchWords("b")
require.False(t, matched, "Could match invalid AND condition")
}
func TestORCondition(t *testing.T) {
m := &Matcher{condition: ORCondition, Words: []string{"a", "b"}}
matched := m.matchWords("a b")
matched := m.MatchWords("a b")
require.True(t, matched, "Could not match valid OR condition")
matched = m.matchWords("b")
matched = m.MatchWords("b")
require.True(t, matched, "Could not match valid OR condition")
matched = m.matchWords("c")
matched = m.MatchWords("c")
require.False(t, matched, "Could match invalid OR condition")
}

10
v2/pkg/operators/matchers/matchers.go
View File

@ -88,11 +88,15 @@ var ConditionTypes = map[string]ConditionType{
"or": ORCondition,
}
// isNegative reverts the results of the match if the matcher
// is of type negative.
func (m *Matcher) isNegative(data bool) bool {
// Result reverts the results of the match if the matcher is of type negative.
func (m *Matcher) Result(data bool) bool {
if m.Negative {
return !data
}
return data
}
// GetType returns the type of the matcher
func (m *Matcher) GetType() MatcherType {
return m.matcherType
}

20
v2/pkg/operators/operators.go
View File

@ -3,6 +3,7 @@ package operators
import (
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
)
// Operators contains the operators that can be applied on protocols
@ -31,12 +32,20 @@ type Result struct {
Matches map[string]struct{}
// Extracts contains all the data extracted from inputs
Extracts map[string][]string
// OutputExtracts is the list of extracts to be displayed on screen.
OutputExtracts []string
// DynamicValues contains any dynamic values to be templated
DynamicValues map[string]string
}
// MatchFunc performs matching operation for a matcher on model and returns true or false.
type MatchFunc func(data map[string]interface{}, matcher *matchers.Matcher) bool
// ExtractFunc performs extracting operation for a extractor on model and returns true or false.
type ExtractFunc func(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{}
// Execute executes the operators on data and returns a result structure
func (r *Operators) Execute(data map[string]interface{}) (*Result, bool) {
func (r *Operators) Execute(data output.Event, match MatchFunc, extract ExtractFunc) (*Result, bool) {
matcherCondition := r.GetMatchersCondition()
result := &Result{
@ -46,7 +55,7 @@ func (r *Operators) Execute(data map[string]interface{}) (*Result, bool) {
}
for _, matcher := range r.Matchers {
// Check if the matcher matched
if !matcher.Match(data) {
if !match(data, matcher) {
// If the condition is AND we haven't matched, try next request.
if matcherCondition == matchers.ANDCondition {
return nil, false
@ -62,9 +71,10 @@ func (r *Operators) Execute(data map[string]interface{}) (*Result, bool) {
// All matchers have successfully completed so now start with the
// next task which is extraction of input from matchers.
var extractorResults, outputExtractorResults []string
for _, extractor := range r.Extractors {
for match := range extractor.Extract(data) {
var extractorResults []string
for match := range extract(data, extractor) {
extractorResults = append(extractorResults, match)
if extractor.Internal {
@ -72,7 +82,7 @@ func (r *Operators) Execute(data map[string]interface{}) (*Result, bool) {
result.DynamicValues[extractor.Name] = match
}
} else {
outputExtractorResults = append(outputExtractorResults, match)
result.OutputExtracts = append(result.OutputExtracts, match)
}
}
result.Extracts[extractor.Name] = extractorResults

63
v2/pkg/protocols/http/matchers.go
View File

@ -5,8 +5,69 @@ import (
"net/http/httputil"
"strings"
"time"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
)
// Match matches a generic data response again a given matcher
func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool {
part, ok := data[matcher.Part]
if !ok {
return false
}
partString := part.(string)
switch partString {
case "header":
partString = "all_headers"
case "all":
partString = "raw"
}
switch matcher.GetType() {
case matchers.StatusMatcher:
statusCode, ok := data["status_code"]
if !ok {
return false
}
return matcher.Result(matcher.MatchStatusCode(statusCode.(int)))
case matchers.SizeMatcher:
return matcher.Result(matcher.MatchSize(len(partString)))
case matchers.WordsMatcher:
return matcher.Result(matcher.MatchWords(partString))
case matchers.RegexMatcher:
return matcher.Result(matcher.MatchRegex(partString))
case matchers.BinaryMatcher:
return matcher.Result(matcher.MatchBinary(partString))
case matchers.DSLMatcher:
return matcher.Result(matcher.MatchDSL(data))
}
return false
}
// Extract performs extracting operation for a extractor on model and returns true or false.
func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} {
part, ok := data[extractor.Part]
if !ok {
return nil
}
partString := part.(string)
switch partString {
case "header":
partString = "all_headers"
case "all":
partString = "raw"
}
switch extractor.GetType() {
case extractors.RegexExtractor:
return extractor.ExtractRegex(partString)
case extractors.KValExtractor:
return extractor.ExtractKval(data)
}
return nil
}
// responseToDSLMap converts a HTTP response to a map for use in DSL matching
func responseToDSLMap(resp *http.Response, body, headers string, duration time.Duration, extra map[string]interface{}) map[string]interface{} {
data := make(map[string]interface{}, len(extra)+6+len(resp.Header)+len(resp.Cookies()))
@ -25,13 +86,11 @@ func responseToDSLMap(resp *http.Response, body, headers string, duration time.D
k = strings.ToLower(strings.TrimSpace(strings.ReplaceAll(k, "-", "_")))
data[k] = strings.Join(v, " ")
}
data["header"] = headers
data["all_headers"] = headers
if r, err := httputil.DumpResponse(resp, true); err == nil {
rawString := string(r)
data["raw"] = rawString
data["all"] = rawString
}
data["duration"] = duration.Seconds()
return data

6
v2/pkg/protocols/protocols.go
View File

@ -1,6 +1,8 @@
package protocols
import (
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
"go.uber.org/ratelimit"
@ -12,6 +14,10 @@ type Executer interface {
Compile(options ExecuterOptions) error
// Requests returns the total number of requests the rule will perform
Requests() int64
// Match performs matching operation for a matcher on model and returns true or false.
Match(data map[string]interface{}, matcher *matchers.Matcher) bool
// Extract performs extracting operation for a extractor on model and returns true or false.
Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{}
// Execute executes the protocol requests and returns an output event channel.
Execute(input string) (bool, error)
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.