daffainfo
/
nuclei
mirror of https://github.com/daffainfo/nuclei.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1186 from projectdiscovery/protocol-types 

Added new type and exclude-type flag
dev
Sandeep Singh 2021-11-03 18:47:26 +05:30 committed by GitHub
parent 2224880d89 956fb9296c
commit 0f897a3977
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 270 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
v2/cmd/nuclei/main.go
View File

@ -9,6 +9,7 @@ import (
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/internal/runner"
"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
)
@ -70,6 +71,8 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.StringSliceVarP(&options.ExcludedTemplates, "exclude-templates", "et", []string{}, "template or template directory paths to exclude"),
flagSet.VarP(&options.Severities, "severity", "s", fmt.Sprintf("Templates to run based on severity. Possible values: %s", severity.GetSupportedSeverities().String())),
flagSet.VarP(&options.ExcludeSeverities, "exclude-severity", "es", fmt.Sprintf("Templates to exclude based on severity. Possible values: %s", severity.GetSupportedSeverities().String())),
flagSet.VarP(&options.Protocols, "type", "pt", fmt.Sprintf("protocol types to be executed. Possible values: %s", templateTypes.GetSupportedProtocolTypes())),
flagSet.VarP(&options.ExcludeProtocols, "exclude-type", "ept", fmt.Sprintf("protocol types to not be executed. Possible values: %s", templateTypes.GetSupportedProtocolTypes())),
flagSet.NormalizedStringSliceVarP(&options.Author, "author", "a", []string{}, "execute templates that are (co-)created by the specified authors"),
)

2
v2/internal/runner/runner.go
View File

@ -378,6 +378,8 @@ func (r *Runner) RunEnumeration() error {
ExcludeSeverities: r.options.ExcludeSeverities,
IncludeTags: r.options.IncludeTags,
TemplatesDirectory: r.options.TemplatesDirectory,
Protocols: r.options.Protocols,
ExcludeProtocols: r.options.ExcludeProtocols,
Catalog: r.catalog,
ExecutorOptions: executerOpts,
}

43
v2/pkg/catalog/loader/filter/tag_filter.go
View File

@ -5,6 +5,7 @@ import (
"strings"
"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
)
// TagFilter is used to filter nuclei templates for tag based execution
@ -15,6 +16,8 @@ type TagFilter struct {
authors map[string]struct{}
block map[string]struct{}
matchAllows map[string]struct{}
types map[types.ProtocolType]struct{}
excludeTypes map[types.ProtocolType]struct{}
}
// ErrExcluded is returned for excluded templates
@ -25,7 +28,7 @@ var ErrExcluded = errors.New("the template was excluded")
// unless it is explicitly specified by user using the includeTags (matchAllows field).
// Matching rule: (tag1 OR tag2...) AND (author1 OR author2...) AND (severity1 OR severity2...) AND (extraTags1 OR extraTags2...)
// Returns true if the template matches the filter criteria, false otherwise.
func (tagFilter *TagFilter) Match(templateTags, templateAuthors []string, templateSeverity severity.Severity, extraTags []string) (bool, error) {
func (tagFilter *TagFilter) Match(templateTags, templateAuthors []string, templateSeverity severity.Severity, extraTags []string, templateType types.ProtocolType) (bool, error) {
for _, templateTag := range templateTags {
_, blocked := tagFilter.block[templateTag]
_, allowed := tagFilter.matchAllows[templateTag]
@ -51,6 +54,9 @@ func (tagFilter *TagFilter) Match(templateTags, templateAuthors []string, templa
return false, nil
}
if !isTemplateTypeMatch(tagFilter, templateType) {
return false, nil
}
return true, nil
}
@ -116,6 +122,27 @@ func isTagMatch(tagFilter *TagFilter, templateTags []string) bool {
return false
}
func isTemplateTypeMatch(tagFilter *TagFilter, templateType types.ProtocolType) bool {
if len(tagFilter.excludeTypes) == 0 && len(tagFilter.types) == 0 {
return true
}
if templateType.String() == "" || templateType == types.InvalidProtocol {
return true
}
included := true
if len(tagFilter.types) > 0 {
_, included = tagFilter.types[templateType]
}
excluded := false
if len(tagFilter.excludeTypes) > 0 {
_, excluded = tagFilter.excludeTypes[templateType]
}
return included && !excluded
}
type Config struct {
Tags []string
ExcludeTags []string
@ -123,6 +150,8 @@ type Config struct {
Severities severity.Severities
ExcludeSeverities severity.Severities
IncludeTags []string
Protocols types.ProtocolTypes
ExcludeProtocols types.ProtocolTypes
}
// New returns a tag filter for nuclei tag based execution
@ -136,6 +165,8 @@ func New(config *Config) *TagFilter {
excludeSeverities: make(map[severity.Severity]struct{}),
block: make(map[string]struct{}),
matchAllows: make(map[string]struct{}),
types: make(map[types.ProtocolType]struct{}),
excludeTypes: make(map[types.ProtocolType]struct{}),
}
for _, tag := range config.ExcludeTags {
for _, val := range splitCommaTrim(tag) {
@ -177,6 +208,16 @@ func New(config *Config) *TagFilter {
delete(filter.block, val)
}
}
for _, tag := range config.Protocols {
if _, ok := filter.types[tag]; !ok {
filter.types[tag] = struct{}{}
}
}
for _, tag := range config.ExcludeProtocols {
if _, ok := filter.excludeTypes[tag]; !ok {
filter.excludeTypes[tag] = struct{}{}
}
}
return filter
}

50
v2/pkg/catalog/loader/filter/tag_filter_test.go
View File

@ -6,6 +6,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
)
func TestTagBasedFilter(t *testing.T) {
@ -15,19 +16,19 @@ func TestTagBasedFilter(t *testing.T) {
})
t.Run("true", func(t *testing.T) {
matched, _ := filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil)
matched, _ := filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol)
require.True(t, matched, "could not get correct match")
})
t.Run("false", func(t *testing.T) {
matched, _ := filter.Match([]string{"consul"}, []string{"pdteam"}, severity.Low, nil)
matched, _ := filter.Match([]string{"consul"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol)
require.False(t, matched, "could not get correct match")
})
t.Run("match-extra-tags-positive", func(t *testing.T) {
matched, _ := filter.Match([]string{"cves", "vuln"}, []string{"pdteam"}, severity.Low, []string{"vuln"})
matched, _ := filter.Match([]string{"cves", "vuln"}, []string{"pdteam"}, severity.Low, []string{"vuln"}, types.HTTPProtocol)
require.True(t, matched, "could not get correct match")
})
t.Run("match-extra-tags-negative", func(t *testing.T) {
matched, _ := filter.Match([]string{"cves"}, []string{"pdteam"}, severity.Low, []string{"vuln"})
matched, _ := filter.Match([]string{"cves"}, []string{"pdteam"}, severity.Low, []string{"vuln"}, types.HTTPProtocol)
require.False(t, matched, "could not get correct match")
})
}
@ -36,7 +37,7 @@ func TestTagBasedFilter(t *testing.T) {
filter := New(&Config{
ExcludeTags: []string{"dos"},
})
matched, err := filter.Match([]string{"dos"}, []string{"pdteam"}, severity.Low, nil)
matched, err := filter.Match([]string{"dos"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol)
require.False(t, matched, "could not get correct match")
require.Equal(t, ErrExcluded, err, "could not get correct error")
})
@ -46,7 +47,7 @@ func TestTagBasedFilter(t *testing.T) {
ExcludeTags: []string{"dos", "fuzz"},
IncludeTags: []string{"fuzz"},
})
matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil)
matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol)
require.Nil(t, err, "could not get match")
require.True(t, matched, "could not get correct match")
})
@ -55,7 +56,7 @@ func TestTagBasedFilter(t *testing.T) {
Tags: []string{"fuzz"},
ExcludeTags: []string{"fuzz"},
})
matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil)
matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol)
require.Nil(t, err, "could not get match")
require.True(t, matched, "could not get correct match")
})
@ -63,24 +64,24 @@ func TestTagBasedFilter(t *testing.T) {
filter := New(&Config{
Authors: []string{"pdteam"},
})
matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil)
matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol)
require.True(t, matched, "could not get correct match")
})
t.Run("match-severity", func(t *testing.T) {
filter := New(&Config{
Severities: severity.Severities{severity.High},
})
matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil)
matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, types.HTTPProtocol)
require.True(t, matched, "could not get correct match")
})
t.Run("match-exclude-severity", func(t *testing.T) {
filter := New(&Config{
ExcludeSeverities: severity.Severities{severity.Low},
})
matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil)
matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, types.HTTPProtocol)
require.True(t, matched, "could not get correct match")
matched, _ = filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil)
matched, _ = filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol)
require.False(t, matched, "could not get correct match")
})
t.Run("match-exclude-with-tags", func(t *testing.T) {
@ -88,7 +89,7 @@ func TestTagBasedFilter(t *testing.T) {
Tags: []string{"tag"},
ExcludeTags: []string{"another"},
})
matched, _ := filter.Match([]string{"another"}, []string{"pdteam"}, severity.High, nil)
matched, _ := filter.Match([]string{"another"}, []string{"pdteam"}, severity.High, nil, types.HTTPProtocol)
require.False(t, matched, "could not get correct match")
})
t.Run("match-conditions", func(t *testing.T) {
@ -97,16 +98,33 @@ func TestTagBasedFilter(t *testing.T) {
Tags: []string{"jira"},
Severities: severity.Severities{severity.High},
})
matched, _ := filter.Match([]string{"jira", "cve"}, []string{"pdteam", "someOtherUser"}, severity.High, nil)
matched, _ := filter.Match([]string{"jira", "cve"}, []string{"pdteam", "someOtherUser"}, severity.High, nil, types.HTTPProtocol)
require.True(t, matched, "could not get correct match")
matched, _ = filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil)
matched, _ = filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol)
require.False(t, matched, "could not get correct match")
matched, _ = filter.Match([]string{"jira"}, []string{"random"}, severity.Low, nil)
matched, _ = filter.Match([]string{"jira"}, []string{"random"}, severity.Low, nil, types.HTTPProtocol)
require.False(t, matched, "could not get correct match")
matched, _ = filter.Match([]string{"consul"}, []string{"random"}, severity.Low, nil)
matched, _ = filter.Match([]string{"consul"}, []string{"random"}, severity.Low, nil, types.HTTPProtocol)
require.False(t, matched, "could not get correct match")
})
t.Run("match-type", func(t *testing.T) {
filter := New(&Config{
Protocols: []types.ProtocolType{types.HTTPProtocol},
})
matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, types.HTTPProtocol)
require.True(t, matched, "could not get correct match")
})
t.Run("match-exclude-type", func(t *testing.T) {
filter := New(&Config{
ExcludeProtocols: []types.ProtocolType{types.HTTPProtocol},
})
matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil, types.DNSProtocol)
require.True(t, matched, "could not get correct match")
matched, _ = filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil, types.HTTPProtocol)
require.False(t, matched, "could not get correct match")
})
}

5
v2/pkg/catalog/loader/loader.go
View File

@ -10,6 +10,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/parsers"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
"github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
)
// Config contains the configuration options for the loader
@ -23,6 +24,8 @@ type Config struct {
Tags []string
ExcludeTags []string
Protocols types.ProtocolTypes
ExcludeProtocols types.ProtocolTypes
Authors []string
Severities severity.Severities
ExcludeSeverities severity.Severities
@ -59,6 +62,8 @@ func New(config *Config) (*Store, error) {
Severities: config.Severities,
ExcludeSeverities: config.ExcludeSeverities,
IncludeTags: config.IncludeTags,
Protocols: config.Protocols,
ExcludeProtocols: config.ExcludeProtocols,
}),
pathFilter: filter.NewPathFilter(&filter.PathFilterConfig{
IncludedTemplates: config.IncludeTemplates,

7
v2/pkg/parsers/parser.go
View File

@ -14,6 +14,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
"github.com/projectdiscovery/nuclei/v2/pkg/templates/cache"
"github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
"github.com/projectdiscovery/nuclei/v2/pkg/utils"
"github.com/projectdiscovery/nuclei/v2/pkg/utils/stats"
)
@ -39,7 +40,7 @@ func LoadTemplate(templatePath string, tagFilter *filter.TagFilter, extraTags []
return false, validationError
}
return isTemplateInfoMetadataMatch(tagFilter, &template.Info, extraTags)
return isTemplateInfoMetadataMatch(tagFilter, &template.Info, extraTags, template.Type())
}
// LoadWorkflow returns true if the workflow is valid and matches the filtering criteria.
@ -59,12 +60,12 @@ func LoadWorkflow(templatePath string) (bool, error) {
return false, nil
}
func isTemplateInfoMetadataMatch(tagFilter *filter.TagFilter, templateInfo *model.Info, extraTags []string) (bool, error) {
func isTemplateInfoMetadataMatch(tagFilter *filter.TagFilter, templateInfo *model.Info, extraTags []string, templateType types.ProtocolType) (bool, error) {
templateTags := templateInfo.Tags.ToSlice()
templateAuthors := templateInfo.Authors.ToSlice()
templateSeverity := templateInfo.SeverityHolder.Severity
match, err := tagFilter.Match(templateTags, templateAuthors, templateSeverity, extraTags)
match, err := tagFilter.Match(templateTags, templateAuthors, templateSeverity, extraTags, templateType)
if err == filter.ErrExcluded {
return false, filter.ErrExcluded

21
v2/pkg/templates/templates.go
View File

@ -9,6 +9,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/network"
"github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
"github.com/projectdiscovery/nuclei/v2/pkg/workflows"
)
@ -73,3 +74,23 @@ type Template struct {
Path string `yaml:"-" json:"-"`
}
// Type returns the type of the template
func (t *Template) Type() types.ProtocolType {
switch {
case len(t.RequestsDNS) > 0:
return types.DNSProtocol
case len(t.RequestsFile) > 0:
return types.FileProtocol
case len(t.RequestsHTTP) > 0:
return types.HTTPProtocol
case len(t.RequestsHeadless) > 0:
return types.HeadlessProtocol
case len(t.RequestsNetwork) > 0:
return types.NetworkProtocol
case t.CompiledWorkflow != nil:
return types.WorkflowProtocol
default:
return types.InvalidProtocol
}
}

154
v2/pkg/templates/types/types.go Normal file
View File

@ -0,0 +1,154 @@
package types
import (
"encoding/json"
"fmt"
"strings"
"github.com/alecthomas/jsonschema"
"github.com/pkg/errors"
"github.com/projectdiscovery/goflags"
"github.com/projectdiscovery/nuclei/v2/pkg/model/types/stringslice"
)
// ProtocolType is the type of the request protocol specified
type ProtocolType int
// Supported values for the ProtocolType
const (
DNSProtocol ProtocolType = iota + 1
FileProtocol
HTTPProtocol
HeadlessProtocol
NetworkProtocol
WorkflowProtocol
limit
InvalidProtocol
)
// ExtractorTypes is a table for conversion of extractor type from string.
var protocolMappings = map[ProtocolType]string{
InvalidProtocol: "invalid",
DNSProtocol: "dns",
FileProtocol: "file",
HTTPProtocol: "http",
HeadlessProtocol: "headless",
NetworkProtocol: "network",
WorkflowProtocol: "workflow",
}
func GetSupportedProtocolTypes() ProtocolTypes {
var result []ProtocolType
for index := ProtocolType(1); index < limit; index++ {
result = append(result, index)
}
return result
}
func toProtocolType(valueToMap string) (ProtocolType, error) {
normalizedValue := normalizeValue(valueToMap)
for key, currentValue := range protocolMappings {
if normalizedValue == currentValue {
return key, nil
}
}
return -1, errors.New("Invalid protocol type: " + valueToMap)
}
func normalizeValue(value string) string {
return strings.TrimSpace(strings.ToLower(value))
}
func (t ProtocolType) String() string {
return protocolMappings[t]
}
// TypeHolder is used to hold internal type of the protocol
type TypeHolder struct {
ProtocolType ProtocolType
}
func (holder TypeHolder) JSONSchemaType() *jsonschema.Type {
gotType := &jsonschema.Type{
Type: "string",
Title: "type of the protocol",
Description: "Type of the protocol",
}
for _, types := range GetSupportedProtocolTypes() {
gotType.Enum = append(gotType.Enum, types.String())
}
return gotType
}
func (holder *TypeHolder) UnmarshalYAML(unmarshal func(interface{}) error) error {
var marshalledTypes string
if err := unmarshal(&marshalledTypes); err != nil {
return err
}
computedType, err := toProtocolType(marshalledTypes)
if err != nil {
return err
}
holder.ProtocolType = computedType
return nil
}
func (holder *TypeHolder) MarshalJSON() ([]byte, error) {
return json.Marshal(holder.ProtocolType.String())
}
func (holder TypeHolder) MarshalYAML() (interface{}, error) {
return holder.ProtocolType.String(), nil
}
type ProtocolTypes []ProtocolType
func (protocolTypes *ProtocolTypes) Set(values string) error {
inputTypes, err := goflags.ToNormalizedStringSlice(values)
if err != nil {
return err
}
for _, inputType := range inputTypes {
if err := setProtocolType(protocolTypes, inputType); err != nil {
return err
}
}
return nil
}
func (protocolTypes *ProtocolTypes) UnmarshalYAML(unmarshal func(interface{}) error) error {
var stringSliceValue stringslice.StringSlice
if err := unmarshal(&stringSliceValue); err != nil {
return err
}
stringSLice := stringSliceValue.ToSlice()
var result = make(ProtocolTypes, 0, len(stringSLice))
for _, typeString := range stringSLice {
if err := setProtocolType(&result, typeString); err != nil {
return err
}
}
*protocolTypes = result
return nil
}
func (protocolTypes ProtocolTypes) String() string {
var stringTypes []string
for _, t := range protocolTypes {
stringTypes = append(stringTypes, t.String())
}
return strings.Join(stringTypes, ", ")
}
func setProtocolType(protocolTypes *ProtocolTypes, value string) error {
computedType, err := toProtocolType(value)
if err != nil {
return fmt.Errorf("'%s' is not a valid extract type", value)
}
*protocolTypes = append(*protocolTypes, computedType)
return nil
}

5
v2/pkg/types/types.go
View File

@ -3,6 +3,7 @@ package types
import (
"github.com/projectdiscovery/goflags"
"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
)
// Options contains the configuration options for nuclei scanner.
@ -33,6 +34,10 @@ type Options struct {
Severities severity.Severities
// ExcludeSeverities specifies severities to exclude
ExcludeSeverities severity.Severities
// Protocols contains the protocols to be allowed executed
Protocols types.ProtocolTypes
// ExcludeProtocols contains protocols to not be executed
ExcludeProtocols types.ProtocolTypes
// Author filters templates based on their author and only run the matching ones.
Author goflags.NormalizedStringSlice
// IncludeTags includes specified tags to be run even while being in denylist