daffainfo
/
nuclei
mirror of https://github.com/daffainfo/nuclei.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' of https://github.com/projectdiscovery/nuclei into yamldoc

dev
Ice3man543 2021-08-20 15:13:22 +05:30
parent 232c0d9e0e d7fc8cd99c
commit e34e784756
14 changed files with 358 additions and 273 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
v2/internal/runner/templates.go
View File

@ -1,42 +1,18 @@
package runner
import (
"bytes"
"fmt"
"io/ioutil"
"os"
"strings"
"github.com/karrick/godirwalk"
"gopkg.in/yaml.v2"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/internal/severity"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
"github.com/projectdiscovery/nuclei/v2/pkg/parsers"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
)
// parseTemplateFile returns the parsed template file
func (r *Runner) parseTemplateFile(file string) (*templates.Template, error) {
f, err := os.Open(file)
if err != nil {
return nil, err
}
defer f.Close()
data, err := ioutil.ReadAll(f)
if err != nil {
return nil, err
}
template := &templates.Template{}
err = yaml.NewDecoder(bytes.NewReader(data)).Decode(template)
if err != nil {
return nil, err
}
return template, nil
}
func (r *Runner) templateLogMsg(id, name, author string, templateSeverity severity.Severity) string {
// Display the message for the template
return fmt.Sprintf("[%s] %s (%s) [%s]",
@ -70,7 +46,7 @@ func appendAtSignToAuthors(author string) string {
}
func (r *Runner) logAvailableTemplate(tplPath string) {
t, err := r.parseTemplateFile(tplPath)
t, err := parsers.ParseTemplate(tplPath)
if err != nil {
gologger.Error().Msgf("Could not parse file '%s': %s\n", tplPath, err)
} else {

99
v2/pkg/catalog/loader/filter/tag_filter.go
View File

@ -19,15 +19,12 @@ type TagFilter struct {
// ErrExcluded is returned for excluded templates
var ErrExcluded = errors.New("the template was excluded")
// Match takes a tag and whether the template was matched from user
// input and returns true or false using a tag filter.
//
// If the tag was specified in deny list, it will not return true
// unless it is explicitly specified by user in includeTags which is the
// matchAllows section.
//
// It returns true if the tag is specified, or false.
func (tagFilter *TagFilter) Match(templateTags, templateAuthors []string, templateSeverity severity.Severity) (bool, error) {
// Match filters templates based on user provided tags, authors, extraTags and severity.
// If the template contains tags specified in the deny list, it will not be matched
// unless it is explicitly specified by user using the includeTags (matchAllows field).
// Matching rule: (tag1 OR tag2...) AND (author1 OR author2...) AND (severity1 OR severity2...) AND (extraTags1 OR extraTags2...)
// Returns true if the template matches the filter criteria, false otherwise.
func (tagFilter *TagFilter) Match(templateTags, templateAuthors []string, templateSeverity severity.Severity, extraTags []string) (bool, error) {
for _, templateTag := range templateTags {
_, blocked := tagFilter.block[templateTag]
_, allowed := tagFilter.matchAllows[templateTag]
@ -37,30 +34,45 @@ func (tagFilter *TagFilter) Match(templateTags, templateAuthors []string, templa
}
}
if !isTagMatch(templateTags, tagFilter) {
if !isExtraTagMatch(extraTags, templateTags) {
return false, nil
}
if !isAuthorMatch(templateAuthors, tagFilter) {
if !isTagMatch(tagFilter, templateTags) {
return false, nil
}
if len(tagFilter.severities) > 0 {
if _, ok := tagFilter.severities[templateSeverity]; !ok {
if !isAuthorMatch(tagFilter, templateAuthors) {
return false, nil
}
if !isSeverityMatch(tagFilter, templateSeverity) {
return false, nil
}
return true, nil
}
func isAuthorMatch(templateAuthors []string, tagFilter *TagFilter) bool {
func isSeverityMatch(tagFilter *TagFilter, templateSeverity severity.Severity) bool {
if len(tagFilter.severities) == 0 {
return true
}
if _, ok := tagFilter.severities[templateSeverity]; ok {
return true
}
return false
}
func isAuthorMatch(tagFilter *TagFilter, templateAuthors []string) bool {
if len(tagFilter.authors) == 0 {
return true
}
for _, templateAuthor := range templateAuthors {
if _, ok := tagFilter.authors[templateAuthor]; ok {
templateAuthorMap := toMap(templateAuthors)
for requiredAuthor := range tagFilter.authors {
if _, ok := templateAuthorMap[requiredAuthor]; ok {
return true
}
}
@ -68,7 +80,22 @@ func isAuthorMatch(templateAuthors []string, tagFilter *TagFilter) bool {
return false
}
func isTagMatch(templateTags []string, tagFilter *TagFilter) bool {
func isExtraTagMatch(extraTags []string, templateTags []string) bool {
if len(extraTags) == 0 {
return true
}
templatesTagMap := toMap(templateTags)
for _, extraTag := range extraTags {
if _, ok := templatesTagMap[extraTag]; ok {
return true
}
}
return false
}
func isTagMatch(tagFilter *TagFilter, templateTags []string) bool {
if len(tagFilter.allowedTags) == 0 {
return true
}
@ -82,42 +109,6 @@ func isTagMatch(templateTags []string, tagFilter *TagFilter) bool {
return false
}
// MatchWithWorkflowTags takes an addition list of allowed tags and returns true if the match was successful.
func (tagFilter *TagFilter) MatchWithWorkflowTags(templateTags, templateAuthors []string, templateSeverity severity.Severity, workflowTags []string) (bool, error) {
for _, templateTag := range templateTags {
_, blocked := tagFilter.block[templateTag]
_, allowed := tagFilter.matchAllows[templateTag]
if blocked && !allowed { // the whitelist has precedence over the blacklist
return false, ErrExcluded
}
}
templatesTagMap := toMap(templateTags)
for _, workflowTag := range workflowTags {
if _, ok := templatesTagMap[workflowTag]; !ok {
return false, nil
}
}
if len(tagFilter.authors) > 0 {
templateAuthorTagMap := toMap(templateAuthors)
for requiredAuthor := range tagFilter.authors {
if _, ok := templateAuthorTagMap[requiredAuthor]; !ok {
return false, nil
}
}
}
if len(tagFilter.severities) > 0 {
if _, ok := tagFilter.severities[templateSeverity]; !ok {
return false, nil
}
}
return true, nil
}
type Config struct {
Tags []string
ExcludeTags []string
@ -193,7 +184,7 @@ func splitCommaTrim(value string) []string {
}
func toMap(slice []string) map[string]struct{} {
result := make(map[string]struct{})
result := make(map[string]struct{}, len(slice))
for _, value := range slice {
if _, ok := result[value]; !ok {
result[value] = struct{}{}

24
v2/pkg/catalog/loader/filter/tag_filter_test.go
View File

@ -15,11 +15,11 @@ func TestTagBasedFilter(t *testing.T) {
filter := New(config)
t.Run("true", func(t *testing.T) {
matched, _ := filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low)
matched, _ := filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil)
require.True(t, matched, "could not get correct match")
})
t.Run("false", func(t *testing.T) {
matched, _ := filter.Match([]string{"consul"}, []string{"pdteam"}, severity.Low)
matched, _ := filter.Match([]string{"consul"}, []string{"pdteam"}, severity.Low, nil)
require.False(t, matched, "could not get correct match")
})
t.Run("not-match-excludes", func(t *testing.T) {
@ -27,7 +27,7 @@ func TestTagBasedFilter(t *testing.T) {
ExcludeTags: []string{"dos"},
}
filter := New(config)
matched, err := filter.Match([]string{"dos"}, []string{"pdteam"}, severity.Low)
matched, err := filter.Match([]string{"dos"}, []string{"pdteam"}, severity.Low, nil)
require.False(t, matched, "could not get correct match")
require.Equal(t, ErrExcluded, err, "could not get correct error")
})
@ -38,7 +38,7 @@ func TestTagBasedFilter(t *testing.T) {
IncludeTags: []string{"fuzz"},
}
filter := New(config)
matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low)
matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil)
require.Nil(t, err, "could not get match")
require.True(t, matched, "could not get correct match")
})
@ -48,7 +48,7 @@ func TestTagBasedFilter(t *testing.T) {
ExcludeTags: []string{"fuzz"},
}
filter := New(config)
matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low)
matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil)
require.Nil(t, err, "could not get match")
require.True(t, matched, "could not get correct match")
})
@ -57,7 +57,7 @@ func TestTagBasedFilter(t *testing.T) {
Authors: []string{"pdteam"},
}
filter := New(config)
matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low)
matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil)
require.True(t, matched, "could not get correct match")
})
t.Run("match-severity", func(t *testing.T) {
@ -65,7 +65,7 @@ func TestTagBasedFilter(t *testing.T) {
Severities: severity.Severities{severity.High},
}
filter := New(config)
matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High)
matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil)
require.True(t, matched, "could not get correct match")
})
t.Run("match-exclude-with-tags", func(t *testing.T) {
@ -74,7 +74,7 @@ func TestTagBasedFilter(t *testing.T) {
ExcludeTags: []string{"another"},
}
filter := New(config)
matched, _ := filter.Match([]string{"another"}, []string{"pdteam"}, severity.High)
matched, _ := filter.Match([]string{"another"}, []string{"pdteam"}, severity.High, nil)
require.False(t, matched, "could not get correct match")
})
t.Run("match-conditions", func(t *testing.T) {
@ -84,13 +84,13 @@ func TestTagBasedFilter(t *testing.T) {
Severities: severity.Severities{severity.High},
}
filter := New(config)
matched, _ := filter.Match([]string{"jira"}, []string{"pdteam"}, severity.High)
matched, _ := filter.Match([]string{"jira"}, []string{"pdteam"}, severity.High, nil)
require.True(t, matched, "could not get correct match")
matched, _ = filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low)
matched, _ = filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil)
require.False(t, matched, "could not get correct match")
matched, _ = filter.Match([]string{"jira"}, []string{"random"}, severity.Low)
matched, _ = filter.Match([]string{"jira"}, []string{"random"}, severity.Low, nil)
require.False(t, matched, "could not get correct match")
matched, _ = filter.Match([]string{"consul"}, []string{"random"}, severity.Low)
matched, _ = filter.Match([]string{"consul"}, []string{"random"}, severity.Low, nil)
require.False(t, matched, "could not get correct match")
})
}

114
v2/pkg/catalog/loader/loader.go
View File

@ -70,103 +70,89 @@ func New(config *Config) (*Store, error) {
}
// Templates returns all the templates in the store
func (s *Store) Templates() []*templates.Template {
return s.templates
func (store *Store) Templates() []*templates.Template {
return store.templates
}
// Workflows returns all the workflows in the store
func (s *Store) Workflows() []*templates.Template {
return s.workflows
func (store *Store) Workflows() []*templates.Template {
return store.workflows
}
// RegisterPreprocessor allows a custom preprocessor to be passed to the store to run against templates
func (s *Store) RegisterPreprocessor(preprocessor templates.Preprocessor) {
s.preprocessor = preprocessor
func (store *Store) RegisterPreprocessor(preprocessor templates.Preprocessor) {
store.preprocessor = preprocessor
}
// Load loads all the templates from a store, performs filtering and returns
// the complete compiled templates for a nuclei execution configuration.
func (s *Store) Load() {
s.templates = s.LoadTemplates(s.finalTemplates)
s.workflows = s.LoadWorkflows(s.config.Workflows)
func (store *Store) Load() {
store.templates = store.LoadTemplates(store.finalTemplates)
store.workflows = store.LoadWorkflows(store.config.Workflows)
}
// ValidateTemplates takes a list of templates and validates them
// erroring out on discovering any faulty templates.
func (s *Store) ValidateTemplates(templatesList, workflowsList []string) bool {
includedTemplates := s.config.Catalog.GetTemplatesPath(templatesList)
includedWorkflows := s.config.Catalog.GetTemplatesPath(workflowsList)
templatesMap := s.pathFilter.Match(includedTemplates)
workflowsMap := s.pathFilter.Match(includedWorkflows)
func (store *Store) ValidateTemplates(templatesList, workflowsList []string) bool {
templatePaths := store.config.Catalog.GetTemplatesPath(templatesList)
workflowPaths := store.config.Catalog.GetTemplatesPath(workflowsList)
filteredTemplatePaths := store.pathFilter.Match(templatePaths)
filteredWorkflowPaths := store.pathFilter.Match(workflowPaths)
notErrored := true
for k := range templatesMap {
_, err := s.loadTemplate(k, false)
if err != nil {
errorValidationFunc := func(message string, template string, err error) {
if strings.Contains(err.Error(), "cannot create template executer") {
continue
return
}
if err == filter.ErrExcluded {
continue
return
}
notErrored = false
gologger.Error().Msgf("Error occurred loading template %s: %s\n", k, err)
continue
gologger.Error().Msgf(message, template, err)
}
_, err = templates.Parse(k, s.preprocessor, s.config.ExecutorOptions)
for templatePath := range filteredTemplatePaths {
_, err := parsers.LoadTemplate(templatePath, store.tagFilter, nil)
if err != nil {
if strings.Contains(err.Error(), "cannot create template executer") {
errorValidationFunc("Error occurred loading template %s: %s\n", templatePath, err)
continue
}
if err == filter.ErrExcluded {
continue
}
notErrored = false
gologger.Error().Msgf("Error occurred parsing template %s: %s\n", k, err)
}
}
for k := range workflowsMap {
_, err := s.loadTemplate(k, true)
_, err = templates.Parse(templatePath, store.preprocessor, store.config.ExecutorOptions)
if err != nil {
if strings.Contains(err.Error(), "cannot create template executer") {
errorValidationFunc("Error occurred parsing template %s: %s\n", templatePath, err)
continue
}
if err == filter.ErrExcluded {
continue
}
notErrored = false
gologger.Error().Msgf("Error occurred loading workflow %s: %s\n", k, err)
}
_, err = templates.Parse(k, s.preprocessor, s.config.ExecutorOptions)
for workflowPath := range filteredWorkflowPaths {
_, err := parsers.LoadWorkflow(workflowPath, store.tagFilter)
if err != nil {
if strings.Contains(err.Error(), "cannot create template executer") {
errorValidationFunc("Error occurred loading workflow %s: %s\n", workflowPath, err)
continue
}
if err == filter.ErrExcluded {
_, err = templates.Parse(workflowPath, store.preprocessor, store.config.ExecutorOptions)
if err != nil {
errorValidationFunc("Error occurred parsing workflow %s: %s\n", workflowPath, err)
continue
}
notErrored = false
gologger.Error().Msgf("Error occurred parsing workflow %s: %s\n", k, err)
}
}
return notErrored
}
// LoadTemplates takes a list of templates and returns paths for them
func (s *Store) LoadTemplates(templatesList []string) []*templates.Template {
includedTemplates := s.config.Catalog.GetTemplatesPath(templatesList)
templatesMap := s.pathFilter.Match(includedTemplates)
func (store *Store) LoadTemplates(templatesList []string) []*templates.Template {
includedTemplates := store.config.Catalog.GetTemplatesPath(templatesList)
templatePathMap := store.pathFilter.Match(includedTemplates)
loadedTemplates := make([]*templates.Template, 0, len(templatesMap))
for k := range templatesMap {
loaded, err := s.loadTemplate(k, false)
loadedTemplates := make([]*templates.Template, 0, len(templatePathMap))
for templatePath := range templatePathMap {
loaded, err := parsers.LoadTemplate(templatePath, store.tagFilter, nil)
if err != nil {
gologger.Warning().Msgf("Could not load template %s: %s\n", k, err)
gologger.Warning().Msgf("Could not load template %s: %s\n", templatePath, err)
}
if loaded {
parsed, err := templates.Parse(k, s.preprocessor, s.config.ExecutorOptions)
parsed, err := templates.Parse(templatePath, store.preprocessor, store.config.ExecutorOptions)
if err != nil {
gologger.Warning().Msgf("Could not parse template %s: %s\n", k, err)
gologger.Warning().Msgf("Could not parse template %s: %s\n", templatePath, err)
} else if parsed != nil {
loadedTemplates = append(loadedTemplates, parsed)
}
@ -176,20 +162,20 @@ func (s *Store) LoadTemplates(templatesList []string) []*templates.Template {
}
// LoadWorkflows takes a list of workflows and returns paths for them
func (s *Store) LoadWorkflows(workflowsList []string) []*templates.Template {
includedWorkflows := s.config.Catalog.GetTemplatesPath(workflowsList)
workflowsMap := s.pathFilter.Match(includedWorkflows)
func (store *Store) LoadWorkflows(workflowsList []string) []*templates.Template {
includedWorkflows := store.config.Catalog.GetTemplatesPath(workflowsList)
workflowPathMap := store.pathFilter.Match(includedWorkflows)
loadedWorkflows := make([]*templates.Template, 0, len(workflowsMap))
for k := range workflowsMap {
loaded, err := s.loadTemplate(k, true)
loadedWorkflows := make([]*templates.Template, 0, len(workflowPathMap))
for workflowPath := range workflowPathMap {
loaded, err := parsers.LoadWorkflow(workflowPath, store.tagFilter)
if err != nil {
gologger.Warning().Msgf("Could not load workflow %s: %s\n", k, err)
gologger.Warning().Msgf("Could not load workflow %s: %s\n", workflowPath, err)
}
if loaded {
parsed, err := templates.Parse(k, s.preprocessor, s.config.ExecutorOptions)
parsed, err := templates.Parse(workflowPath, store.preprocessor, store.config.ExecutorOptions)
if err != nil {
gologger.Warning().Msgf("Could not parse workflow %s: %s\n", k, err)
gologger.Warning().Msgf("Could not parse workflow %s: %s\n", workflowPath, err)
} else if parsed != nil {
loadedWorkflows = append(loadedWorkflows, parsed)
}
@ -197,7 +183,3 @@ func (s *Store) LoadWorkflows(workflowsList []string) []*templates.Template {
}
return loadedWorkflows
}
func (s *Store) loadTemplate(templatePath string, isWorkflow bool) (bool, error) {
return parsers.Load(templatePath, isWorkflow, nil, s.tagFilter) // TODO consider separating template and workflow loading logic
}

13
v2/pkg/model/model.go
View File

@ -61,6 +61,13 @@ type Info struct {
// - high
// - critical
SeverityHolder severity.SeverityHolder `json:"severity,omitempty" yaml:"severity,omitempty"`
// description: |
// AdditionalFields regarding metadata of the template.
//
// examples:
// - value: >
// map[string]string{"customField1":"customValue1"}
AdditionalFields map[string]string `json:"additional-fields,omitempty" yaml:"additional-fields,omitempty"`
}
// StringSlice represents a single (in-lined) or multiple string value(s).
@ -86,13 +93,17 @@ func (stringSlice StringSlice) ToSlice() []string {
}
}
func (stringSlice StringSlice) String() string {
return strings.Join(stringSlice.ToSlice(), ", ")
}
func (stringSlice *StringSlice) UnmarshalYAML(unmarshal func(interface{}) error) error {
marshalledSlice, err := marshalStringToSlice(unmarshal)
if err != nil {
return err
}
result := make([]string, len(marshalledSlice))
result := make([]string, 0, len(marshalledSlice))
//nolint:gosimple,nolintlint //cannot be replaced with result = append(result, slices...) because the values are being normalized
for _, value := range marshalledSlice {
result = append(result, strings.ToLower(strings.TrimSpace(value))) // TODO do we need to introduce RawStringSlice and/or NormalizedStringSlices?

63
v2/pkg/model/model_test.go
View File

@ -2,11 +2,13 @@ package model
import (
"encoding/json"
"strings"
"testing"
"gopkg.in/yaml.v2"
"github.com/projectdiscovery/nuclei/v2/internal/severity"
"github.com/stretchr/testify/assert"
"gopkg.in/yaml.v2"
)
func TestInfoJsonMarshal(t *testing.T) {
@ -52,3 +54,62 @@ severity: high
`
assert.Equal(t, expected, string(result))
}
func TestUnmarshal(t *testing.T) {
templateName := "Test Template"
authors := []string{"forgedhallpass", "ice3man"}
tags := []string{"cve", "misc"}
references := []string{"http://test.com", "http://domain.com"}
dynamicKey1 := "customDynamicKey1"
dynamicKey2 := "customDynamicKey2"
dynamicKeysMap := map[string]string{
dynamicKey1: "customDynamicValue1",
dynamicKey2: "customDynamicValue2",
}
assertUnmarshalledTemplateInfo := func(t *testing.T, yamlPayload string) Info {
info := Info{}
err := yaml.Unmarshal([]byte(yamlPayload), &info)
assert.Nil(t, err)
assert.Equal(t, info.Name, templateName)
assert.Equal(t, info.Authors.ToSlice(), authors)
assert.Equal(t, info.Tags.ToSlice(), tags)
assert.Equal(t, info.SeverityHolder.Severity, severity.Critical)
assert.Equal(t, info.Reference.ToSlice(), references)
assert.Equal(t, info.AdditionalFields, dynamicKeysMap)
return info
}
yamlPayload1 := `
name: ` + templateName + `
author: ` + strings.Join(authors, ", ") + `
tags: ` + strings.Join(tags, ", ") + `
severity: critical
reference: ` + strings.Join(references, ", ") + `
additional-fields:
` + dynamicKey1 + `: ` + dynamicKeysMap[dynamicKey1] + `
` + dynamicKey2 + `: ` + dynamicKeysMap[dynamicKey2] + `
`
yamlPayload2 := `
name: ` + templateName + `
author:
- ` + authors[0] + `
- ` + authors[1] + `
tags:
- ` + tags[0] + `
- ` + tags[1] + `
severity: critical
reference:
- ` + references[0] + ` # comments are not unmarshalled
- ` + references[1] + `
additional-fields:
` + dynamicKey1 + `: ` + dynamicKeysMap[dynamicKey1] + `
` + dynamicKey2 + `: ` + dynamicKeysMap[dynamicKey2] + `
`
info1 := assertUnmarshalledTemplateInfo(t, yamlPayload1)
info2 := assertUnmarshalledTemplateInfo(t, yamlPayload2)
assert.Equal(t, info1, info2)
}

10
v2/pkg/model/worflow_loader.go
View File

@ -1,10 +1,12 @@
package model
// TODO shouldn't this rather be TemplateLoader?
// WorkflowLoader is a loader interface required for workflow initialization.
type WorkflowLoader interface {
// ListTags lists a list of templates for tags from the provided templates directory
ListTags(workflowTags []string) []string
// GetTemplatePathsByTags returns a list of template paths based on the provided tags from the templates directory
GetTemplatePathsByTags(tags []string) []string
// ListTemplates takes a list of templates and returns paths for them
ListTemplates(templatesList []string, noValidate bool) []string
// GetTemplatePaths takes a list of templates and returns paths for them
GetTemplatePaths(templatesList []string, noValidate bool) []string
}

68
v2/pkg/parsers/parser.go
View File

@ -1,13 +1,14 @@
package parsers
import (
"bytes"
"fmt"
"io/ioutil"
"os"
"regexp"
"gopkg.in/yaml.v2"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
@ -16,49 +17,57 @@ import (
const mandatoryFieldMissingTemplate = "mandatory '%s' field is missing"
// Load loads a template by parsing metadata and running all tag and path based filters on the template.
func Load(templatePath string, isWorkflow bool, workflowTags []string, tagFilter *filter.TagFilter) (bool, error) {
template, templateParseError := parseTemplate(templatePath)
// LoadTemplate returns true if the template is valid and matches the filtering criteria.
func LoadTemplate(templatePath string, tagFilter *filter.TagFilter, extraTags []string) (bool, error) {
template, templateParseError := ParseTemplate(templatePath)
if templateParseError != nil {
return false, templateParseError
}
if len(template.Workflows) > 0 {
return false, nil
}
templateInfo := template.Info
if validationError := validateMandatoryInfoFields(&templateInfo); validationError != nil {
return false, validationError
}
if len(template.Workflows) > 0 {
if isWorkflow {
return true, nil // if a workflow is declared and this template is a workflow, then load
} else { //nolint:indent-error-flow,revive // preferred: readability and extensibility
return false, nil // if a workflow is declared and this template is not a workflow then do not load
}
} else if isWorkflow {
return false, nil // if no workflows are declared and this template is a workflow then do not load
} else { // if workflows are not declared and the template is not a workflow then parse it
return isInfoMetadataMatch(tagFilter, &templateInfo, workflowTags)
}
return isTemplateInfoMetadataMatch(tagFilter, &templateInfo, extraTags)
}
func isInfoMetadataMatch(tagFilter *filter.TagFilter, templateInfo *model.Info, workflowTags []string) (bool, error) {
// LoadWorkflow returns true if the workflow is valid and matches the filtering criteria.
func LoadWorkflow(templatePath string, tagFilter *filter.TagFilter) (bool, error) {
template, templateParseError := ParseTemplate(templatePath)
if templateParseError != nil {
return false, templateParseError
}
templateInfo := template.Info
if len(template.Workflows) > 0 {
if validationError := validateMandatoryInfoFields(&templateInfo); validationError != nil {
return false, validationError
}
return isTemplateInfoMetadataMatch(tagFilter, &templateInfo, nil) // we don't want workflows to be loaded by tags
}
return false, nil
}
func isTemplateInfoMetadataMatch(tagFilter *filter.TagFilter, templateInfo *model.Info, extraTags []string) (bool, error) {
templateTags := templateInfo.Tags.ToSlice()
templateAuthors := templateInfo.Authors.ToSlice()
templateSeverity := templateInfo.SeverityHolder.Severity
var match bool
var err error
if len(workflowTags) == 0 {
match, err = tagFilter.Match(templateTags, templateAuthors, templateSeverity)
} else {
match, err = tagFilter.MatchWithWorkflowTags(templateTags, templateAuthors, templateSeverity, workflowTags)
}
match, err := tagFilter.Match(templateTags, templateAuthors, templateSeverity, extraTags)
if err == filter.ErrExcluded {
return false, filter.ErrExcluded
}
return match, nil
return match, err
}
func validateMandatoryInfoFields(info *model.Info) error {
@ -76,7 +85,10 @@ func validateMandatoryInfoFields(info *model.Info) error {
return nil
}
func parseTemplate(templatePath string) (*templates.Template, error) {
var fieldErrorRegexp = regexp.MustCompile(`not found in`)
// ParseTemplate parses a template and returns a *templates.Template structure
func ParseTemplate(templatePath string) (*templates.Template, error) {
f, err := os.Open(templatePath)
if err != nil {
return nil, err
@ -89,8 +101,12 @@ func parseTemplate(templatePath string) (*templates.Template, error) {
}
template := &templates.Template{}
err = yaml.NewDecoder(bytes.NewReader(data)).Decode(template)
err = yaml.UnmarshalStrict(data, template)
if err != nil {
if fieldErrorRegexp.MatchString(err.Error()) {
gologger.Warning().Msgf("Unrecognized fields in template %s: %s", templatePath, err)
return template, nil
}
return nil, err
}
return template, nil

30
v2/pkg/parsers/workflow_loader.go
View File

@ -29,35 +29,33 @@ func NewLoader(options *protocols.ExecuterOptions) (model.WorkflowLoader, error)
return &workflowLoader{pathFilter: pathFilter, tagFilter: tagFilter, options: options}, nil
}
// ListTags lists a list of templates for tags from the provided templates directory
func (w *workflowLoader) ListTags(workflowTags []string) []string {
func (w *workflowLoader) GetTemplatePathsByTags(templateTags []string) []string {
includedTemplates := w.options.Catalog.GetTemplatesPath([]string{w.options.Options.TemplatesDirectory})
templatesMap := w.pathFilter.Match(includedTemplates)
templatePathMap := w.pathFilter.Match(includedTemplates)
loadedTemplates := make([]string, 0, len(templatesMap))
for k := range templatesMap {
loaded, err := Load(k, false, workflowTags, w.tagFilter)
loadedTemplates := make([]string, 0, len(templatePathMap))
for templatePath := range templatePathMap {
loaded, err := LoadTemplate(templatePath, w.tagFilter, templateTags)
if err != nil {
gologger.Warning().Msgf("Could not load template %s: %s\n", k, err)
gologger.Warning().Msgf("Could not load template %s: %s\n", templatePath, err)
} else if loaded {
loadedTemplates = append(loadedTemplates, k)
loadedTemplates = append(loadedTemplates, templatePath)
}
}
return loadedTemplates
}
// ListTemplates takes a list of templates and returns paths for them
func (w *workflowLoader) ListTemplates(templatesList []string, noValidate bool) []string {
func (w *workflowLoader) GetTemplatePaths(templatesList []string, noValidate bool) []string {
includedTemplates := w.options.Catalog.GetTemplatesPath(templatesList)
templatesMap := w.pathFilter.Match(includedTemplates)
templatesPathMap := w.pathFilter.Match(includedTemplates)
loadedTemplates := make([]string, 0, len(templatesMap))
for k := range templatesMap {
matched, err := Load(k, false, nil, w.tagFilter)
loadedTemplates := make([]string, 0, len(templatesPathMap))
for templatePath := range templatesPathMap {
matched, err := LoadTemplate(templatePath, w.tagFilter, nil)
if err != nil {
gologger.Warning().Msgf("Could not load template %s: %s\n", k, err)
gologger.Warning().Msgf("Could not load template %s: %s\n", templatePath, err)
} else if matched || noValidate {
loadedTemplates = append(loadedTemplates, k)
loadedTemplates = append(loadedTemplates, templatePath)
}
}
return loadedTemplates

50
v2/pkg/reporting/format/format.go
View File

@ -119,9 +119,8 @@ func MarkdownDescription(event *output.ResultEvent) string { // TODO remove the
reference := event.Info.Reference
if !reference.IsEmpty() {
builder.WriteString("\nReference: \n")
builder.WriteString("\nReferences: \n")
/*TODO couldn't the following code replace the logic below?
referenceSlice := reference.ToSlice()
for i, item := range referenceSlice {
builder.WriteString("- ")
@ -129,23 +128,6 @@ func MarkdownDescription(event *output.ResultEvent) string { // TODO remove the
if len(referenceSlice)-1 != i {
builder.WriteString("\n")
}
}*/
switch value := reference.Value.(type) {
case string:
if !strings.HasPrefix(value, "-") {
builder.WriteString("- ")
}
builder.WriteString(value)
case []interface{}:
slice := types.ToStringSlice(value)
for i, item := range slice {
builder.WriteString("- ")
builder.WriteString(item)
if len(slice)-1 != i {
builder.WriteString("\n")
}
}
}
}
@ -171,23 +153,25 @@ func GetMatchedTemplate(event *output.ResultEvent) string {
}
func ToMarkdownTableString(templateInfo *model.Info) string {
fields := map[string]string{
"Name": templateInfo.Name,
"Authors": sliceToString(templateInfo.Authors),
"Tags": sliceToString(templateInfo.Tags),
"Description": templateInfo.Description,
"Severity": templateInfo.SeverityHolder.Severity.String(),
}
fields := utils.NewEmptyInsertionOrderedStringMap(5)
fields.Set("Name", templateInfo.Name)
fields.Set("Authors", templateInfo.Authors.String())
fields.Set("Tags", templateInfo.Tags.String())
fields.Set("Severity", templateInfo.SeverityHolder.Severity.String())
fields.Set("Description", templateInfo.Description)
builder := &bytes.Buffer{}
for k, v := range fields {
if utils.IsNotBlank(v) {
builder.WriteString(fmt.Sprintf("| %s | %s |\n", k, v))
toMarkDownTable := func(insertionOrderedStringMap *utils.InsertionOrderedStringMap) {
insertionOrderedStringMap.ForEach(func(key string, value string) {
if utils.IsNotBlank(value) {
builder.WriteString(fmt.Sprintf("| %s | %s |\n", key, value))
}
}
return builder.String()
})
}
func sliceToString(stringSlice model.StringSlice) string {
return strings.Join(stringSlice.ToSlice(), ", ")
toMarkDownTable(fields)
toMarkDownTable(utils.NewInsertionOrderedStringMap(templateInfo.AdditionalFields))
return builder.String()
}

45
v2/pkg/reporting/format/format_test.go Normal file
View File

@ -0,0 +1,45 @@
package format
import (
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/projectdiscovery/nuclei/v2/internal/severity"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
)
func TestToMarkdownTableString(t *testing.T) {
info := model.Info{
Name: "Test Template Name",
Authors: model.StringSlice{Value: []string{"forgedhallpass", "ice3man"}},
Description: "Test description",
SeverityHolder: severity.SeverityHolder{Severity: severity.High},
Tags: model.StringSlice{Value: []string{"cve", "misc"}},
Reference: model.StringSlice{Value: "reference1"},
AdditionalFields: map[string]string{
"customDynamicKey1": "customDynamicValue1",
"customDynamicKey2": "customDynamicValue2",
},
}
result := ToMarkdownTableString(&info)
expectedOrderedAttributes := `| Name | Test Template Name |
| Authors | forgedhallpass, ice3man |
| Tags | cve, misc |
| Severity | high |
| Description | Test description |`
expectedDynamicAttributes := []string{
"| customDynamicKey1 | customDynamicValue1 |",
"| customDynamicKey2 | customDynamicValue2 |",
"", // the expected result ends in a new line (\n)
}
actualAttributeSlice := strings.Split(result, "\n")
dynamicAttributeIndex := len(actualAttributeSlice) - len(expectedDynamicAttributes)
assert.Equal(t, strings.Split(expectedOrderedAttributes, "\n"), actualAttributeSlice[:dynamicAttributeIndex]) // the first part of the result is ordered
assert.ElementsMatch(t, expectedDynamicAttributes, actualAttributeSlice[dynamicAttributeIndex:]) // dynamic parameters are not ordered
}

20
v2/pkg/reporting/trackers/jira/jira.go
View File

@ -181,9 +181,8 @@ func jiraFormatDescription(event *output.ResultEvent) string { // TODO remove th
reference := event.Info.Reference
if !reference.IsEmpty() {
builder.WriteString("\nReference: \n")
builder.WriteString("\nReferences: \n")
/*TODO couldn't the following code replace the logic below?
referenceSlice := reference.ToSlice()
for i, item := range referenceSlice {
builder.WriteString("- ")
@ -191,23 +190,6 @@ func jiraFormatDescription(event *output.ResultEvent) string { // TODO remove th
if len(referenceSlice)-1 != i {
builder.WriteString("\n")
}
}*/
switch v := reference.Value.(type) {
case string:
if !strings.HasPrefix(v, "-") {
builder.WriteString("- ")
}
builder.WriteString(v)
case []interface{}:
slice := types.ToStringSlice(v)
for i, item := range slice {
builder.WriteString("- ")
builder.WriteString(item)
if len(slice)-1 != i {
builder.WriteString("\n")
}
}
}
}
builder.WriteString("\n---\nGenerated by [Nuclei|https://github.com/projectdiscovery/nuclei]")

8
v2/pkg/templates/workflows.go
View File

@ -48,11 +48,11 @@ func parseWorkflow(preprocessor Preprocessor, workflow *workflows.WorkflowTempla
func parseWorkflowTemplate(workflow *workflows.WorkflowTemplate, preprocessor Preprocessor, options *protocols.ExecuterOptions, loader model.WorkflowLoader, noValidate bool) error {
var paths []string
workflowTags := workflow.Tags
if !workflowTags.IsEmpty() {
paths = loader.ListTags(workflowTags.ToSlice())
subTemplateTags := workflow.Tags
if !subTemplateTags.IsEmpty() {
paths = loader.GetTemplatePathsByTags(subTemplateTags.ToSlice())
} else {
paths = loader.ListTemplates([]string{workflow.Template}, noValidate)
paths = loader.GetTemplatePaths([]string{workflow.Template}, noValidate)
}
if len(paths) == 0 {
return nil

37
v2/pkg/utils/insertion_ordered_map.go Normal file
View File

@ -0,0 +1,37 @@
package utils
type InsertionOrderedStringMap struct {
keys []string `yaml:"-"`
values map[string]string
}
func NewEmptyInsertionOrderedStringMap(size int) *InsertionOrderedStringMap {
return &InsertionOrderedStringMap{
keys: make([]string, 0, size),
values: make(map[string]string, size),
}
}
func NewInsertionOrderedStringMap(stringMap map[string]string) *InsertionOrderedStringMap {
result := NewEmptyInsertionOrderedStringMap(len(stringMap))
for k, v := range stringMap {
result.Set(k, v)
}
return result
}
func (insertionOrderedStringMap *InsertionOrderedStringMap) ForEach(fn func(key string, data string)) {
for _, key := range insertionOrderedStringMap.keys {
fn(key, insertionOrderedStringMap.values[key])
}
}
func (insertionOrderedStringMap *InsertionOrderedStringMap) Set(key string, value string) {
_, present := insertionOrderedStringMap.values[key]
insertionOrderedStringMap.values[key] = value
if !present {
insertionOrderedStringMap.keys = append(insertionOrderedStringMap.keys, key)
}
}