package operators import ( "fmt" "strconv" "strings" "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v3/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/excludematchers" sliceutil "github.com/projectdiscovery/utils/slice" ) // Operators contains the operators that can be applied on protocols type Operators struct { // description: | // Matchers contains the detection mechanism for the request to identify // whether the request was successful by doing pattern matching // on request/responses. // // Multiple matchers can be combined with `matcher-condition` flag // which accepts either `and` or `or` as argument. Matchers []*matchers.Matcher `yaml:"matchers,omitempty" json:"matchers,omitempty" jsonschema:"title=matchers to run on response,description=Detection mechanism to identify whether the request was successful by doing pattern matching"` // description: | // Extractors contains the extraction mechanism for the request to identify // and extract parts of the response. Extractors []*extractors.Extractor `yaml:"extractors,omitempty" json:"extractors,omitempty" jsonschema:"title=extractors to run on response,description=Extractors contains the extraction mechanism for the request to identify and extract parts of the response"` // description: | // MatchersCondition is the condition between the matchers. Default is OR. // values: // - "and" // - "or" MatchersCondition string `yaml:"matchers-condition,omitempty" json:"matchers-condition,omitempty" jsonschema:"title=condition between the matchers,description=Conditions between the matchers,enum=and,enum=or"` // cached variables that may be used along with request. matchersCondition matchers.ConditionType // TemplateID is the ID of the template for matcher TemplateID string `json:"-" yaml:"-" jsonschema:"-"` // ExcludeMatchers is a list of excludeMatchers items ExcludeMatchers *excludematchers.ExcludeMatchers `json:"-" yaml:"-" jsonschema:"-"` } // Compile compiles the operators as well as their corresponding matchers and extractors func (operators *Operators) Compile() error { if operators.MatchersCondition != "" { operators.matchersCondition = matchers.ConditionTypes[operators.MatchersCondition] } else { operators.matchersCondition = matchers.ORCondition } for _, matcher := range operators.Matchers { if err := matcher.CompileMatchers(); err != nil { return errors.Wrap(err, "could not compile matcher") } } for _, extractor := range operators.Extractors { if err := extractor.CompileExtractors(); err != nil { return errors.Wrap(err, "could not compile extractor") } } return nil } // GetMatchersCondition returns the condition for the matchers func (operators *Operators) GetMatchersCondition() matchers.ConditionType { return operators.matchersCondition } // Result is a result structure created from operators running on data. type Result struct { // Matched is true if any matchers matched Matched bool // Extracted is true if any result type values were extracted Extracted bool // Matches is a map of matcher names that we matched Matches map[string][]string // Extracts contains all the data extracted from inputs Extracts map[string][]string // OutputExtracts is the list of extracts to be displayed on screen. OutputExtracts []string outputUnique map[string]struct{} // DynamicValues contains any dynamic values to be templated DynamicValues map[string][]string // PayloadValues contains payload values provided by user. (Optional) PayloadValues map[string]interface{} // Optional lineCounts for file protocol LineCount string // Operators is reference to operators that generated this result (Read-Only) Operators *Operators } func (result *Result) HasMatch(name string) bool { return result.hasItem(name, result.Matches) } func (result *Result) HasExtract(name string) bool { return result.hasItem(name, result.Extracts) } func (result *Result) hasItem(name string, m map[string][]string) bool { for matchName := range m { if strings.EqualFold(name, matchName) { return true } } return false } // MakeDynamicValuesCallback takes an input dynamic values map and calls // the callback function with all variations of the data in input in form // of map[string]string (interface{}). func MakeDynamicValuesCallback(input map[string][]string, iterateAllValues bool, callback func(map[string]interface{}) bool) { output := make(map[string]interface{}, len(input)) if !iterateAllValues { for k, v := range input { if len(v) > 0 { output[k] = v[0] } } callback(output) return } inputIndex := make(map[string]int, len(input)) var maxValue int for _, v := range input { if len(v) > maxValue { maxValue = len(v) } } for i := 0; i < maxValue; i++ { for k, v := range input { if len(v) == 0 { continue } if len(v) == 1 { output[k] = v[0] continue } if gotIndex, ok := inputIndex[k]; !ok { inputIndex[k] = 0 output[k] = v[0] } else { newIndex := gotIndex + 1 if newIndex >= len(v) { output[k] = v[len(v)-1] continue } output[k] = v[newIndex] inputIndex[k] = newIndex } } // skip if the callback says so if callback(output) { return } } } // Merge merges a result structure into the other. func (r *Result) Merge(result *Result) { if !r.Matched && result.Matched { r.Matched = result.Matched } if !r.Extracted && result.Extracted { r.Extracted = result.Extracted } for k, v := range result.Matches { r.Matches[k] = sliceutil.Dedupe(append(r.Matches[k], v...)) } for k, v := range result.Extracts { r.Extracts[k] = sliceutil.Dedupe(append(r.Extracts[k], v...)) } r.outputUnique = make(map[string]struct{}) output := r.OutputExtracts r.OutputExtracts = make([]string, 0, len(output)) for _, v := range output { if _, ok := r.outputUnique[v]; !ok { r.outputUnique[v] = struct{}{} r.OutputExtracts = append(r.OutputExtracts, v) } } for _, v := range result.OutputExtracts { if _, ok := r.outputUnique[v]; !ok { r.outputUnique[v] = struct{}{} r.OutputExtracts = append(r.OutputExtracts, v) } } for k, v := range result.DynamicValues { if _, ok := r.DynamicValues[k]; !ok { r.DynamicValues[k] = v } else { r.DynamicValues[k] = sliceutil.Dedupe(append(r.DynamicValues[k], v...)) } } for k, v := range result.PayloadValues { r.PayloadValues[k] = v } } // MatchFunc performs matching operation for a matcher on model and returns true or false. type MatchFunc func(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) // ExtractFunc performs extracting operation for an extractor on model and returns true or false. type ExtractFunc func(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} // Execute executes the operators on data and returns a result structure func (operators *Operators) Execute(data map[string]interface{}, match MatchFunc, extract ExtractFunc, isDebug bool) (*Result, bool) { matcherCondition := operators.GetMatchersCondition() var matches bool result := &Result{ Matches: make(map[string][]string), Extracts: make(map[string][]string), DynamicValues: make(map[string][]string), outputUnique: make(map[string]struct{}), Operators: operators, } // state variable to check if all extractors are internal var allInternalExtractors bool = true // Start with the extractors first and evaluate them. for _, extractor := range operators.Extractors { if !extractor.Internal && allInternalExtractors { allInternalExtractors = false } var extractorResults []string for match := range extract(data, extractor) { extractorResults = append(extractorResults, match) if extractor.Internal { if data, ok := result.DynamicValues[extractor.Name]; !ok { result.DynamicValues[extractor.Name] = []string{match} } else { result.DynamicValues[extractor.Name] = append(data, match) } } else { if _, ok := result.outputUnique[match]; !ok { result.OutputExtracts = append(result.OutputExtracts, match) result.outputUnique[match] = struct{}{} } } } if len(extractorResults) > 0 && !extractor.Internal && extractor.Name != "" { result.Extracts[extractor.Name] = extractorResults } // update data with whatever was extracted doesn't matter if it is internal or not (skip unless it empty) if len(extractorResults) > 0 { data[extractor.Name] = getExtractedValue(extractorResults) } } // expose dynamic values to same request matchers if len(result.DynamicValues) > 0 { dataDynamicValues := make(map[string]interface{}) for dynName, dynValues := range result.DynamicValues { if len(dynValues) > 1 { for dynIndex, dynValue := range dynValues { dynKeyName := fmt.Sprintf("%s%d", dynName, dynIndex) dataDynamicValues[dynKeyName] = dynValue } dataDynamicValues[dynName] = dynValues } else { dataDynamicValues[dynName] = dynValues[0] } } data = generators.MergeMaps(data, dataDynamicValues) } for matcherIndex, matcher := range operators.Matchers { // Skip matchers that are in the blocklist if operators.ExcludeMatchers != nil { if operators.ExcludeMatchers.Match(operators.TemplateID, matcher.Name) { continue } } if isMatch, matched := match(data, matcher); isMatch { if isDebug { // matchers without an explicit name or with AND condition should only be made visible if debug is enabled matcherName := getMatcherName(matcher, matcherIndex) result.Matches[matcherName] = matched } else { // if it's a "named" matcher with OR condition, then display it if matcherCondition == matchers.ORCondition && matcher.Name != "" { result.Matches[matcher.Name] = matched } } matches = true } else if matcherCondition == matchers.ANDCondition { if len(result.DynamicValues) > 0 { return result, true } return nil, false } } result.Matched = matches result.Extracted = len(result.OutputExtracts) > 0 if len(result.DynamicValues) > 0 && allInternalExtractors { // only return early if all extractors are internal // if some are internal and some are not then followthrough return result, true } // Don't print if we have matchers, and they have not matched, regardless of extractor if len(operators.Matchers) > 0 && !matches { return nil, false } // Write a final string of output if matcher type is // AND or if we have extractors for the mechanism too. if len(result.Extracts) > 0 || len(result.OutputExtracts) > 0 || matches { return result, true } return nil, false } func getMatcherName(matcher *matchers.Matcher, matcherIndex int) string { if matcher.Name != "" { return matcher.Name } else { return matcher.Type.String() + "-" + strconv.Itoa(matcherIndex+1) // making the index start from 1 to be more readable } } // ExecuteInternalExtractors executes internal dynamic extractors func (operators *Operators) ExecuteInternalExtractors(data map[string]interface{}, extract ExtractFunc) map[string]interface{} { dynamicValues := make(map[string]interface{}) // Start with the extractors first and evaluate them. for _, extractor := range operators.Extractors { if !extractor.Internal { continue } for match := range extract(data, extractor) { if _, ok := dynamicValues[extractor.Name]; !ok { dynamicValues[extractor.Name] = match } } } return dynamicValues } // IsEmpty determines if the operator has matchers or extractors func (operators *Operators) IsEmpty() bool { return operators.Len() == 0 } // Len calculates the sum of the number of matchers and extractors func (operators *Operators) Len() int { return len(operators.Matchers) + len(operators.Extractors) } // getExtractedValue takes array of extracted values if it only has one value // then it is flattened and returned as a string else original type is returned func getExtractedValue(values []string) any { if len(values) == 1 { return values[0] } else { return values } }