From 6541b04f4c5a2d8e74ae1f0c5282033065cd9f9d Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 25 Oct 2021 23:24:42 +0530 Subject: [PATCH 1/3] Added new type and exclude-type flag --- v2/cmd/nuclei/main.go | 2 + v2/internal/runner/runner.go | 2 + v2/pkg/catalog/loader/filter/tag_filter.go | 39 ++++++++++++++- .../catalog/loader/filter/tag_filter_test.go | 49 +++++++++++++------ v2/pkg/catalog/loader/loader.go | 4 ++ v2/pkg/parsers/parser.go | 6 +-- v2/pkg/templates/templates.go | 18 +++++++ v2/pkg/types/types.go | 4 ++ 8 files changed, 104 insertions(+), 20 deletions(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index d2f8ebb9..99ef6558 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -68,6 +68,8 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.StringSliceVarP(&options.ExcludedTemplates, "exclude-templates", "et", []string{}, "template or template directory paths to exclude"), flagSet.VarP(&options.Severities, "severity", "s", fmt.Sprintf("Templates to run based on severity. Possible values: %s", severity.GetSupportedSeverities().String())), flagSet.VarP(&options.ExcludeSeverities, "exclude-severity", "es", fmt.Sprintf("Templates to exclude based on severity. Possible values: %s", severity.GetSupportedSeverities().String())), + flagSet.NormalizedStringSliceVarP(&options.Protocols, "type", "tp", []string{}, "protocol types to be executed (http,dns,headless,network,file,etc)"), + flagSet.NormalizedStringSliceVarP(&options.ExcludeProtocols, "exclude-type", "etype", []string{}, "protocol types to not be executed (http,dns,file,etc"), flagSet.NormalizedStringSliceVarP(&options.Author, "author", "a", []string{}, "execute templates that are (co-)created by the specified authors"), ) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index b3f80695..c4c550ea 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -376,6 +376,8 @@ func (r *Runner) RunEnumeration() error { ExcludeSeverities: r.options.ExcludeSeverities, IncludeTags: r.options.IncludeTags, TemplatesDirectory: r.options.TemplatesDirectory, + Types: r.options.Protocols, + ExcludeTypes: r.options.ExcludeProtocols, Catalog: r.catalog, ExecutorOptions: executerOpts, } diff --git a/v2/pkg/catalog/loader/filter/tag_filter.go b/v2/pkg/catalog/loader/filter/tag_filter.go index ed8722c9..c33e96fd 100644 --- a/v2/pkg/catalog/loader/filter/tag_filter.go +++ b/v2/pkg/catalog/loader/filter/tag_filter.go @@ -15,6 +15,8 @@ type TagFilter struct { authors map[string]struct{} block map[string]struct{} matchAllows map[string]struct{} + types map[string]struct{} + excludeTypes map[string]struct{} } // ErrExcluded is returned for excluded templates @@ -25,7 +27,7 @@ var ErrExcluded = errors.New("the template was excluded") // unless it is explicitly specified by user using the includeTags (matchAllows field). // Matching rule: (tag1 OR tag2...) AND (author1 OR author2...) AND (severity1 OR severity2...) AND (extraTags1 OR extraTags2...) // Returns true if the template matches the filter criteria, false otherwise. -func (tagFilter *TagFilter) Match(templateTags, templateAuthors []string, templateSeverity severity.Severity, extraTags []string) (bool, error) { +func (tagFilter *TagFilter) Match(templateTags, templateAuthors []string, templateSeverity severity.Severity, extraTags []string, templateType string) (bool, error) { for _, templateTag := range templateTags { _, blocked := tagFilter.block[templateTag] _, allowed := tagFilter.matchAllows[templateTag] @@ -51,6 +53,9 @@ func (tagFilter *TagFilter) Match(templateTags, templateAuthors []string, templa return false, nil } + if !isTemplateTypeMatch(tagFilter, templateType) { + return false, nil + } return true, nil } @@ -116,6 +121,24 @@ func isTagMatch(tagFilter *TagFilter, templateTags []string) bool { return false } +func isTemplateTypeMatch(tagFilter *TagFilter, templateType string) bool { + if (len(tagFilter.excludeTypes) == 0 && len(tagFilter.types) == 0) || templateType == "" { + return true + } + + included := true + if len(tagFilter.types) > 0 { + _, included = tagFilter.types[templateType] + } + + excluded := false + if len(tagFilter.excludeTypes) > 0 { + _, excluded = tagFilter.excludeTypes[templateType] + } + + return included && !excluded +} + type Config struct { Tags []string ExcludeTags []string @@ -123,6 +146,8 @@ type Config struct { Severities severity.Severities ExcludeSeverities severity.Severities IncludeTags []string + Types []string + ExcludeTypes []string } // New returns a tag filter for nuclei tag based execution @@ -136,6 +161,8 @@ func New(config *Config) *TagFilter { excludeSeverities: make(map[severity.Severity]struct{}), block: make(map[string]struct{}), matchAllows: make(map[string]struct{}), + types: make(map[string]struct{}), + excludeTypes: make(map[string]struct{}), } for _, tag := range config.ExcludeTags { for _, val := range splitCommaTrim(tag) { @@ -177,6 +204,16 @@ func New(config *Config) *TagFilter { delete(filter.block, val) } } + for _, tag := range config.Types { + if _, ok := filter.types[tag]; !ok { + filter.types[tag] = struct{}{} + } + } + for _, tag := range config.ExcludeTypes { + if _, ok := filter.excludeTypes[tag]; !ok { + filter.excludeTypes[tag] = struct{}{} + } + } return filter } diff --git a/v2/pkg/catalog/loader/filter/tag_filter_test.go b/v2/pkg/catalog/loader/filter/tag_filter_test.go index 22d18b18..7599c27b 100644 --- a/v2/pkg/catalog/loader/filter/tag_filter_test.go +++ b/v2/pkg/catalog/loader/filter/tag_filter_test.go @@ -15,19 +15,19 @@ func TestTagBasedFilter(t *testing.T) { }) t.Run("true", func(t *testing.T) { - matched, _ := filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil) + matched, _ := filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil, "") require.True(t, matched, "could not get correct match") }) t.Run("false", func(t *testing.T) { - matched, _ := filter.Match([]string{"consul"}, []string{"pdteam"}, severity.Low, nil) + matched, _ := filter.Match([]string{"consul"}, []string{"pdteam"}, severity.Low, nil, "") require.False(t, matched, "could not get correct match") }) t.Run("match-extra-tags-positive", func(t *testing.T) { - matched, _ := filter.Match([]string{"cves", "vuln"}, []string{"pdteam"}, severity.Low, []string{"vuln"}) + matched, _ := filter.Match([]string{"cves", "vuln"}, []string{"pdteam"}, severity.Low, []string{"vuln"}, "") require.True(t, matched, "could not get correct match") }) t.Run("match-extra-tags-negative", func(t *testing.T) { - matched, _ := filter.Match([]string{"cves"}, []string{"pdteam"}, severity.Low, []string{"vuln"}) + matched, _ := filter.Match([]string{"cves"}, []string{"pdteam"}, severity.Low, []string{"vuln"}, "") require.False(t, matched, "could not get correct match") }) } @@ -36,7 +36,7 @@ func TestTagBasedFilter(t *testing.T) { filter := New(&Config{ ExcludeTags: []string{"dos"}, }) - matched, err := filter.Match([]string{"dos"}, []string{"pdteam"}, severity.Low, nil) + matched, err := filter.Match([]string{"dos"}, []string{"pdteam"}, severity.Low, nil, "") require.False(t, matched, "could not get correct match") require.Equal(t, ErrExcluded, err, "could not get correct error") }) @@ -46,7 +46,7 @@ func TestTagBasedFilter(t *testing.T) { ExcludeTags: []string{"dos", "fuzz"}, IncludeTags: []string{"fuzz"}, }) - matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil) + matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, "") require.Nil(t, err, "could not get match") require.True(t, matched, "could not get correct match") }) @@ -55,7 +55,7 @@ func TestTagBasedFilter(t *testing.T) { Tags: []string{"fuzz"}, ExcludeTags: []string{"fuzz"}, }) - matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil) + matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, "") require.Nil(t, err, "could not get match") require.True(t, matched, "could not get correct match") }) @@ -63,24 +63,24 @@ func TestTagBasedFilter(t *testing.T) { filter := New(&Config{ Authors: []string{"pdteam"}, }) - matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil) + matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, "") require.True(t, matched, "could not get correct match") }) t.Run("match-severity", func(t *testing.T) { filter := New(&Config{ Severities: severity.Severities{severity.High}, }) - matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil) + matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, "") require.True(t, matched, "could not get correct match") }) t.Run("match-exclude-severity", func(t *testing.T) { filter := New(&Config{ ExcludeSeverities: severity.Severities{severity.Low}, }) - matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil) + matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, "") require.True(t, matched, "could not get correct match") - matched, _ = filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil) + matched, _ = filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, "") require.False(t, matched, "could not get correct match") }) t.Run("match-exclude-with-tags", func(t *testing.T) { @@ -88,7 +88,7 @@ func TestTagBasedFilter(t *testing.T) { Tags: []string{"tag"}, ExcludeTags: []string{"another"}, }) - matched, _ := filter.Match([]string{"another"}, []string{"pdteam"}, severity.High, nil) + matched, _ := filter.Match([]string{"another"}, []string{"pdteam"}, severity.High, nil, "") require.False(t, matched, "could not get correct match") }) t.Run("match-conditions", func(t *testing.T) { @@ -97,16 +97,33 @@ func TestTagBasedFilter(t *testing.T) { Tags: []string{"jira"}, Severities: severity.Severities{severity.High}, }) - matched, _ := filter.Match([]string{"jira", "cve"}, []string{"pdteam", "someOtherUser"}, severity.High, nil) + matched, _ := filter.Match([]string{"jira", "cve"}, []string{"pdteam", "someOtherUser"}, severity.High, nil, "") require.True(t, matched, "could not get correct match") - matched, _ = filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil) + matched, _ = filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil, "") require.False(t, matched, "could not get correct match") - matched, _ = filter.Match([]string{"jira"}, []string{"random"}, severity.Low, nil) + matched, _ = filter.Match([]string{"jira"}, []string{"random"}, severity.Low, nil, "") require.False(t, matched, "could not get correct match") - matched, _ = filter.Match([]string{"consul"}, []string{"random"}, severity.Low, nil) + matched, _ = filter.Match([]string{"consul"}, []string{"random"}, severity.Low, nil, "") + require.False(t, matched, "could not get correct match") + }) + t.Run("match-type", func(t *testing.T) { + filter := New(&Config{ + Types: []string{"http"}, + }) + matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, "http") + require.True(t, matched, "could not get correct match") + }) + t.Run("match-exclude-type", func(t *testing.T) { + filter := New(&Config{ + ExcludeTypes: []string{"http"}, + }) + matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, "dns") + require.True(t, matched, "could not get correct match") + + matched, _ = filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, "http") require.False(t, matched, "could not get correct match") }) } diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go index eac22b4a..fe725e2e 100644 --- a/v2/pkg/catalog/loader/loader.go +++ b/v2/pkg/catalog/loader/loader.go @@ -21,6 +21,8 @@ type Config struct { Tags []string ExcludeTags []string + Types []string + ExcludeTypes []string Authors []string Severities severity.Severities ExcludeSeverities severity.Severities @@ -56,6 +58,8 @@ func New(config *Config) (*Store, error) { Severities: config.Severities, ExcludeSeverities: config.ExcludeSeverities, IncludeTags: config.IncludeTags, + Types: config.Types, + ExcludeTypes: config.ExcludeTypes, }), pathFilter: filter.NewPathFilter(&filter.PathFilterConfig{ IncludedTemplates: config.IncludeTemplates, diff --git a/v2/pkg/parsers/parser.go b/v2/pkg/parsers/parser.go index 7cd0c7b9..9e201fb0 100644 --- a/v2/pkg/parsers/parser.go +++ b/v2/pkg/parsers/parser.go @@ -39,7 +39,7 @@ func LoadTemplate(templatePath string, tagFilter *filter.TagFilter, extraTags [] return false, validationError } - return isTemplateInfoMetadataMatch(tagFilter, &template.Info, extraTags) + return isTemplateInfoMetadataMatch(tagFilter, &template.Info, extraTags, template.Type()) } // LoadWorkflow returns true if the workflow is valid and matches the filtering criteria. @@ -59,12 +59,12 @@ func LoadWorkflow(templatePath string) (bool, error) { return false, nil } -func isTemplateInfoMetadataMatch(tagFilter *filter.TagFilter, templateInfo *model.Info, extraTags []string) (bool, error) { +func isTemplateInfoMetadataMatch(tagFilter *filter.TagFilter, templateInfo *model.Info, extraTags []string, templateType string) (bool, error) { templateTags := templateInfo.Tags.ToSlice() templateAuthors := templateInfo.Authors.ToSlice() templateSeverity := templateInfo.SeverityHolder.Severity - match, err := tagFilter.Match(templateTags, templateAuthors, templateSeverity, extraTags) + match, err := tagFilter.Match(templateTags, templateAuthors, templateSeverity, extraTags, templateType) if err == filter.ErrExcluded { return false, filter.ErrExcluded diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index 61cb49ff..1e84def6 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -73,3 +73,21 @@ type Template struct { Path string `yaml:"-" json:"-"` } + +// Type returns the type of the template +func (t *Template) Type() string { + switch { + case len(t.RequestsDNS) > 0: + return "dns" + case len(t.RequestsFile) > 0: + return "file" + case len(t.RequestsHTTP) > 0: + return "http" + case len(t.RequestsHeadless) > 0: + return "headless" + case len(t.RequestsNetwork) > 0: + return "network" + default: + return "" + } +} diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index d258f424..36f66503 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -29,6 +29,10 @@ type Options struct { Severities severity.Severities // ExcludeSeverities specifies severities to exclude ExcludeSeverities severity.Severities + // Protocols contains the protocols to be allowed executed + Protocols goflags.NormalizedStringSlice + // ExcludeProtocols contains protocols to not be executed + ExcludeProtocols goflags.NormalizedStringSlice // Author filters templates based on their author and only run the matching ones. Author goflags.NormalizedStringSlice // IncludeTags includes specified tags to be run even while being in denylist From 533fb3f1080a395a7799969dfc5ff71ca0cdaa93 Mon Sep 17 00:00:00 2001 From: sandeep Date: Thu, 28 Oct 2021 22:02:22 +0530 Subject: [PATCH 2/3] misc flag update --- v2/cmd/nuclei/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 99ef6558..85f0ecf3 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -68,8 +68,8 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.StringSliceVarP(&options.ExcludedTemplates, "exclude-templates", "et", []string{}, "template or template directory paths to exclude"), flagSet.VarP(&options.Severities, "severity", "s", fmt.Sprintf("Templates to run based on severity. Possible values: %s", severity.GetSupportedSeverities().String())), flagSet.VarP(&options.ExcludeSeverities, "exclude-severity", "es", fmt.Sprintf("Templates to exclude based on severity. Possible values: %s", severity.GetSupportedSeverities().String())), - flagSet.NormalizedStringSliceVarP(&options.Protocols, "type", "tp", []string{}, "protocol types to be executed (http,dns,headless,network,file,etc)"), - flagSet.NormalizedStringSliceVarP(&options.ExcludeProtocols, "exclude-type", "etype", []string{}, "protocol types to not be executed (http,dns,file,etc"), + flagSet.NormalizedStringSliceVarP(&options.Protocols, "type", "pt", []string{}, "protocol types to be executed (http,dns,headless,network,file,etc)"), + flagSet.NormalizedStringSliceVarP(&options.ExcludeProtocols, "exclude-type", "ept", []string{}, "protocol types to not be executed (http,dns,file,etc"), flagSet.NormalizedStringSliceVarP(&options.Author, "author", "a", []string{}, "execute templates that are (co-)created by the specified authors"), ) From 47949c0b522ce305aa50a924273809a468acd8d5 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 3 Nov 2021 17:18:35 +0530 Subject: [PATCH 3/3] Use separate type enum for protocol types --- v2/cmd/nuclei/main.go | 5 +- v2/internal/runner/runner.go | 4 +- v2/pkg/catalog/loader/filter/tag_filter.go | 26 +-- .../catalog/loader/filter/tag_filter_test.go | 43 ++--- v2/pkg/catalog/loader/loader.go | 9 +- v2/pkg/parsers/parser.go | 3 +- v2/pkg/templates/templates.go | 17 +- v2/pkg/templates/types/types.go | 154 ++++++++++++++++++ v2/pkg/types/types.go | 5 +- 9 files changed, 216 insertions(+), 50 deletions(-) create mode 100644 v2/pkg/templates/types/types.go diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 99ef6558..3d657024 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -9,6 +9,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/internal/runner" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" + templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) @@ -68,8 +69,8 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.StringSliceVarP(&options.ExcludedTemplates, "exclude-templates", "et", []string{}, "template or template directory paths to exclude"), flagSet.VarP(&options.Severities, "severity", "s", fmt.Sprintf("Templates to run based on severity. Possible values: %s", severity.GetSupportedSeverities().String())), flagSet.VarP(&options.ExcludeSeverities, "exclude-severity", "es", fmt.Sprintf("Templates to exclude based on severity. Possible values: %s", severity.GetSupportedSeverities().String())), - flagSet.NormalizedStringSliceVarP(&options.Protocols, "type", "tp", []string{}, "protocol types to be executed (http,dns,headless,network,file,etc)"), - flagSet.NormalizedStringSliceVarP(&options.ExcludeProtocols, "exclude-type", "etype", []string{}, "protocol types to not be executed (http,dns,file,etc"), + flagSet.VarP(&options.Protocols, "type", "tp", fmt.Sprintf("protocol types to be executed. Possible values: %s", templateTypes.GetSupportedProtocolTypes())), + flagSet.VarP(&options.ExcludeProtocols, "exclude-type", "etype", fmt.Sprintf("protocol types to not be executed. Possible values: %s", templateTypes.GetSupportedProtocolTypes())), flagSet.NormalizedStringSliceVarP(&options.Author, "author", "a", []string{}, "execute templates that are (co-)created by the specified authors"), ) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index c4c550ea..78ed7f13 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -376,8 +376,8 @@ func (r *Runner) RunEnumeration() error { ExcludeSeverities: r.options.ExcludeSeverities, IncludeTags: r.options.IncludeTags, TemplatesDirectory: r.options.TemplatesDirectory, - Types: r.options.Protocols, - ExcludeTypes: r.options.ExcludeProtocols, + Protocols: r.options.Protocols, + ExcludeProtocols: r.options.ExcludeProtocols, Catalog: r.catalog, ExecutorOptions: executerOpts, } diff --git a/v2/pkg/catalog/loader/filter/tag_filter.go b/v2/pkg/catalog/loader/filter/tag_filter.go index c33e96fd..1420003e 100644 --- a/v2/pkg/catalog/loader/filter/tag_filter.go +++ b/v2/pkg/catalog/loader/filter/tag_filter.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" + "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" ) // TagFilter is used to filter nuclei templates for tag based execution @@ -15,8 +16,8 @@ type TagFilter struct { authors map[string]struct{} block map[string]struct{} matchAllows map[string]struct{} - types map[string]struct{} - excludeTypes map[string]struct{} + types map[types.ProtocolType]struct{} + excludeTypes map[types.ProtocolType]struct{} } // ErrExcluded is returned for excluded templates @@ -27,7 +28,7 @@ var ErrExcluded = errors.New("the template was excluded") // unless it is explicitly specified by user using the includeTags (matchAllows field). // Matching rule: (tag1 OR tag2...) AND (author1 OR author2...) AND (severity1 OR severity2...) AND (extraTags1 OR extraTags2...) // Returns true if the template matches the filter criteria, false otherwise. -func (tagFilter *TagFilter) Match(templateTags, templateAuthors []string, templateSeverity severity.Severity, extraTags []string, templateType string) (bool, error) { +func (tagFilter *TagFilter) Match(templateTags, templateAuthors []string, templateSeverity severity.Severity, extraTags []string, templateType types.ProtocolType) (bool, error) { for _, templateTag := range templateTags { _, blocked := tagFilter.block[templateTag] _, allowed := tagFilter.matchAllows[templateTag] @@ -121,8 +122,11 @@ func isTagMatch(tagFilter *TagFilter, templateTags []string) bool { return false } -func isTemplateTypeMatch(tagFilter *TagFilter, templateType string) bool { - if (len(tagFilter.excludeTypes) == 0 && len(tagFilter.types) == 0) || templateType == "" { +func isTemplateTypeMatch(tagFilter *TagFilter, templateType types.ProtocolType) bool { + if len(tagFilter.excludeTypes) == 0 && len(tagFilter.types) == 0 { + return true + } + if templateType.String() == "" || templateType == types.InvalidProtocol { return true } @@ -146,8 +150,8 @@ type Config struct { Severities severity.Severities ExcludeSeverities severity.Severities IncludeTags []string - Types []string - ExcludeTypes []string + Protocols types.ProtocolTypes + ExcludeProtocols types.ProtocolTypes } // New returns a tag filter for nuclei tag based execution @@ -161,8 +165,8 @@ func New(config *Config) *TagFilter { excludeSeverities: make(map[severity.Severity]struct{}), block: make(map[string]struct{}), matchAllows: make(map[string]struct{}), - types: make(map[string]struct{}), - excludeTypes: make(map[string]struct{}), + types: make(map[types.ProtocolType]struct{}), + excludeTypes: make(map[types.ProtocolType]struct{}), } for _, tag := range config.ExcludeTags { for _, val := range splitCommaTrim(tag) { @@ -204,12 +208,12 @@ func New(config *Config) *TagFilter { delete(filter.block, val) } } - for _, tag := range config.Types { + for _, tag := range config.Protocols { if _, ok := filter.types[tag]; !ok { filter.types[tag] = struct{}{} } } - for _, tag := range config.ExcludeTypes { + for _, tag := range config.ExcludeProtocols { if _, ok := filter.excludeTypes[tag]; !ok { filter.excludeTypes[tag] = struct{}{} } diff --git a/v2/pkg/catalog/loader/filter/tag_filter_test.go b/v2/pkg/catalog/loader/filter/tag_filter_test.go index 7599c27b..0758f6d4 100644 --- a/v2/pkg/catalog/loader/filter/tag_filter_test.go +++ b/v2/pkg/catalog/loader/filter/tag_filter_test.go @@ -6,6 +6,7 @@ import ( "github.com/stretchr/testify/require" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" + "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" ) func TestTagBasedFilter(t *testing.T) { @@ -15,19 +16,19 @@ func TestTagBasedFilter(t *testing.T) { }) t.Run("true", func(t *testing.T) { - matched, _ := filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil, "") + matched, _ := filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol) require.True(t, matched, "could not get correct match") }) t.Run("false", func(t *testing.T) { - matched, _ := filter.Match([]string{"consul"}, []string{"pdteam"}, severity.Low, nil, "") + matched, _ := filter.Match([]string{"consul"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol) require.False(t, matched, "could not get correct match") }) t.Run("match-extra-tags-positive", func(t *testing.T) { - matched, _ := filter.Match([]string{"cves", "vuln"}, []string{"pdteam"}, severity.Low, []string{"vuln"}, "") + matched, _ := filter.Match([]string{"cves", "vuln"}, []string{"pdteam"}, severity.Low, []string{"vuln"}, types.HTTPProtocol) require.True(t, matched, "could not get correct match") }) t.Run("match-extra-tags-negative", func(t *testing.T) { - matched, _ := filter.Match([]string{"cves"}, []string{"pdteam"}, severity.Low, []string{"vuln"}, "") + matched, _ := filter.Match([]string{"cves"}, []string{"pdteam"}, severity.Low, []string{"vuln"}, types.HTTPProtocol) require.False(t, matched, "could not get correct match") }) } @@ -36,7 +37,7 @@ func TestTagBasedFilter(t *testing.T) { filter := New(&Config{ ExcludeTags: []string{"dos"}, }) - matched, err := filter.Match([]string{"dos"}, []string{"pdteam"}, severity.Low, nil, "") + matched, err := filter.Match([]string{"dos"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol) require.False(t, matched, "could not get correct match") require.Equal(t, ErrExcluded, err, "could not get correct error") }) @@ -46,7 +47,7 @@ func TestTagBasedFilter(t *testing.T) { ExcludeTags: []string{"dos", "fuzz"}, IncludeTags: []string{"fuzz"}, }) - matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, "") + matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol) require.Nil(t, err, "could not get match") require.True(t, matched, "could not get correct match") }) @@ -55,7 +56,7 @@ func TestTagBasedFilter(t *testing.T) { Tags: []string{"fuzz"}, ExcludeTags: []string{"fuzz"}, }) - matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, "") + matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol) require.Nil(t, err, "could not get match") require.True(t, matched, "could not get correct match") }) @@ -63,24 +64,24 @@ func TestTagBasedFilter(t *testing.T) { filter := New(&Config{ Authors: []string{"pdteam"}, }) - matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, "") + matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol) require.True(t, matched, "could not get correct match") }) t.Run("match-severity", func(t *testing.T) { filter := New(&Config{ Severities: severity.Severities{severity.High}, }) - matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, "") + matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, types.HTTPProtocol) require.True(t, matched, "could not get correct match") }) t.Run("match-exclude-severity", func(t *testing.T) { filter := New(&Config{ ExcludeSeverities: severity.Severities{severity.Low}, }) - matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, "") + matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, types.HTTPProtocol) require.True(t, matched, "could not get correct match") - matched, _ = filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, "") + matched, _ = filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol) require.False(t, matched, "could not get correct match") }) t.Run("match-exclude-with-tags", func(t *testing.T) { @@ -88,7 +89,7 @@ func TestTagBasedFilter(t *testing.T) { Tags: []string{"tag"}, ExcludeTags: []string{"another"}, }) - matched, _ := filter.Match([]string{"another"}, []string{"pdteam"}, severity.High, nil, "") + matched, _ := filter.Match([]string{"another"}, []string{"pdteam"}, severity.High, nil, types.HTTPProtocol) require.False(t, matched, "could not get correct match") }) t.Run("match-conditions", func(t *testing.T) { @@ -97,33 +98,33 @@ func TestTagBasedFilter(t *testing.T) { Tags: []string{"jira"}, Severities: severity.Severities{severity.High}, }) - matched, _ := filter.Match([]string{"jira", "cve"}, []string{"pdteam", "someOtherUser"}, severity.High, nil, "") + matched, _ := filter.Match([]string{"jira", "cve"}, []string{"pdteam", "someOtherUser"}, severity.High, nil, types.HTTPProtocol) require.True(t, matched, "could not get correct match") - matched, _ = filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil, "") + matched, _ = filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol) require.False(t, matched, "could not get correct match") - matched, _ = filter.Match([]string{"jira"}, []string{"random"}, severity.Low, nil, "") + matched, _ = filter.Match([]string{"jira"}, []string{"random"}, severity.Low, nil, types.HTTPProtocol) require.False(t, matched, "could not get correct match") - matched, _ = filter.Match([]string{"consul"}, []string{"random"}, severity.Low, nil, "") + matched, _ = filter.Match([]string{"consul"}, []string{"random"}, severity.Low, nil, types.HTTPProtocol) require.False(t, matched, "could not get correct match") }) t.Run("match-type", func(t *testing.T) { filter := New(&Config{ - Types: []string{"http"}, + Protocols: []types.ProtocolType{types.HTTPProtocol}, }) - matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, "http") + matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, types.HTTPProtocol) require.True(t, matched, "could not get correct match") }) t.Run("match-exclude-type", func(t *testing.T) { filter := New(&Config{ - ExcludeTypes: []string{"http"}, + ExcludeProtocols: []types.ProtocolType{types.HTTPProtocol}, }) - matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, "dns") + matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, types.DNSProtocol) require.True(t, matched, "could not get correct match") - matched, _ = filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, "http") + matched, _ = filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol) require.False(t, matched, "could not get correct match") }) } diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go index fe725e2e..604a7da5 100644 --- a/v2/pkg/catalog/loader/loader.go +++ b/v2/pkg/catalog/loader/loader.go @@ -10,6 +10,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/parsers" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/templates" + "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" ) // Config contains the configuration options for the loader @@ -21,8 +22,8 @@ type Config struct { Tags []string ExcludeTags []string - Types []string - ExcludeTypes []string + Protocols types.ProtocolTypes + ExcludeProtocols types.ProtocolTypes Authors []string Severities severity.Severities ExcludeSeverities severity.Severities @@ -58,8 +59,8 @@ func New(config *Config) (*Store, error) { Severities: config.Severities, ExcludeSeverities: config.ExcludeSeverities, IncludeTags: config.IncludeTags, - Types: config.Types, - ExcludeTypes: config.ExcludeTypes, + Protocols: config.Protocols, + ExcludeProtocols: config.ExcludeProtocols, }), pathFilter: filter.NewPathFilter(&filter.PathFilterConfig{ IncludedTemplates: config.IncludeTemplates, diff --git a/v2/pkg/parsers/parser.go b/v2/pkg/parsers/parser.go index 9e201fb0..e26e7488 100644 --- a/v2/pkg/parsers/parser.go +++ b/v2/pkg/parsers/parser.go @@ -14,6 +14,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/templates/cache" + "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" "github.com/projectdiscovery/nuclei/v2/pkg/utils" "github.com/projectdiscovery/nuclei/v2/pkg/utils/stats" ) @@ -59,7 +60,7 @@ func LoadWorkflow(templatePath string) (bool, error) { return false, nil } -func isTemplateInfoMetadataMatch(tagFilter *filter.TagFilter, templateInfo *model.Info, extraTags []string, templateType string) (bool, error) { +func isTemplateInfoMetadataMatch(tagFilter *filter.TagFilter, templateInfo *model.Info, extraTags []string, templateType types.ProtocolType) (bool, error) { templateTags := templateInfo.Tags.ToSlice() templateAuthors := templateInfo.Authors.ToSlice() templateSeverity := templateInfo.SeverityHolder.Severity diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index 1e84def6..ff9e527f 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -9,6 +9,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network" + "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" "github.com/projectdiscovery/nuclei/v2/pkg/workflows" ) @@ -75,19 +76,21 @@ type Template struct { } // Type returns the type of the template -func (t *Template) Type() string { +func (t *Template) Type() types.ProtocolType { switch { case len(t.RequestsDNS) > 0: - return "dns" + return types.DNSProtocol case len(t.RequestsFile) > 0: - return "file" + return types.FileProtocol case len(t.RequestsHTTP) > 0: - return "http" + return types.HTTPProtocol case len(t.RequestsHeadless) > 0: - return "headless" + return types.HeadlessProtocol case len(t.RequestsNetwork) > 0: - return "network" + return types.NetworkProtocol + case t.CompiledWorkflow != nil: + return types.WorkflowProtocol default: - return "" + return types.InvalidProtocol } } diff --git a/v2/pkg/templates/types/types.go b/v2/pkg/templates/types/types.go new file mode 100644 index 00000000..f82bac5b --- /dev/null +++ b/v2/pkg/templates/types/types.go @@ -0,0 +1,154 @@ +package types + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/alecthomas/jsonschema" + "github.com/pkg/errors" + "github.com/projectdiscovery/goflags" + "github.com/projectdiscovery/nuclei/v2/pkg/model/types/stringslice" +) + +// ProtocolType is the type of the request protocol specified +type ProtocolType int + +// Supported values for the ProtocolType +const ( + DNSProtocol ProtocolType = iota + 1 + FileProtocol + HTTPProtocol + HeadlessProtocol + NetworkProtocol + WorkflowProtocol + limit + InvalidProtocol +) + +// ExtractorTypes is a table for conversion of extractor type from string. +var protocolMappings = map[ProtocolType]string{ + InvalidProtocol: "invalid", + DNSProtocol: "dns", + FileProtocol: "file", + HTTPProtocol: "http", + HeadlessProtocol: "headless", + NetworkProtocol: "network", + WorkflowProtocol: "workflow", +} + +func GetSupportedProtocolTypes() ProtocolTypes { + var result []ProtocolType + for index := ProtocolType(1); index < limit; index++ { + result = append(result, index) + } + return result +} + +func toProtocolType(valueToMap string) (ProtocolType, error) { + normalizedValue := normalizeValue(valueToMap) + for key, currentValue := range protocolMappings { + if normalizedValue == currentValue { + return key, nil + } + } + return -1, errors.New("Invalid protocol type: " + valueToMap) +} + +func normalizeValue(value string) string { + return strings.TrimSpace(strings.ToLower(value)) +} + +func (t ProtocolType) String() string { + return protocolMappings[t] +} + +// TypeHolder is used to hold internal type of the protocol +type TypeHolder struct { + ProtocolType ProtocolType +} + +func (holder TypeHolder) JSONSchemaType() *jsonschema.Type { + gotType := &jsonschema.Type{ + Type: "string", + Title: "type of the protocol", + Description: "Type of the protocol", + } + for _, types := range GetSupportedProtocolTypes() { + gotType.Enum = append(gotType.Enum, types.String()) + } + return gotType +} + +func (holder *TypeHolder) UnmarshalYAML(unmarshal func(interface{}) error) error { + var marshalledTypes string + if err := unmarshal(&marshalledTypes); err != nil { + return err + } + + computedType, err := toProtocolType(marshalledTypes) + if err != nil { + return err + } + + holder.ProtocolType = computedType + return nil +} + +func (holder *TypeHolder) MarshalJSON() ([]byte, error) { + return json.Marshal(holder.ProtocolType.String()) +} + +func (holder TypeHolder) MarshalYAML() (interface{}, error) { + return holder.ProtocolType.String(), nil +} + +type ProtocolTypes []ProtocolType + +func (protocolTypes *ProtocolTypes) Set(values string) error { + inputTypes, err := goflags.ToNormalizedStringSlice(values) + if err != nil { + return err + } + + for _, inputType := range inputTypes { + if err := setProtocolType(protocolTypes, inputType); err != nil { + return err + } + } + return nil +} + +func (protocolTypes *ProtocolTypes) UnmarshalYAML(unmarshal func(interface{}) error) error { + var stringSliceValue stringslice.StringSlice + if err := unmarshal(&stringSliceValue); err != nil { + return err + } + + stringSLice := stringSliceValue.ToSlice() + var result = make(ProtocolTypes, 0, len(stringSLice)) + for _, typeString := range stringSLice { + if err := setProtocolType(&result, typeString); err != nil { + return err + } + } + *protocolTypes = result + return nil +} + +func (protocolTypes ProtocolTypes) String() string { + var stringTypes []string + for _, t := range protocolTypes { + stringTypes = append(stringTypes, t.String()) + } + return strings.Join(stringTypes, ", ") +} + +func setProtocolType(protocolTypes *ProtocolTypes, value string) error { + computedType, err := toProtocolType(value) + if err != nil { + return fmt.Errorf("'%s' is not a valid extract type", value) + } + *protocolTypes = append(*protocolTypes, computedType) + return nil +} diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 36f66503..fc65e3a1 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -3,6 +3,7 @@ package types import ( "github.com/projectdiscovery/goflags" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" + "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" ) // Options contains the configuration options for nuclei scanner. @@ -30,9 +31,9 @@ type Options struct { // ExcludeSeverities specifies severities to exclude ExcludeSeverities severity.Severities // Protocols contains the protocols to be allowed executed - Protocols goflags.NormalizedStringSlice + Protocols types.ProtocolTypes // ExcludeProtocols contains protocols to not be executed - ExcludeProtocols goflags.NormalizedStringSlice + ExcludeProtocols types.ProtocolTypes // Author filters templates based on their author and only run the matching ones. Author goflags.NormalizedStringSlice // IncludeTags includes specified tags to be run even while being in denylist