From 7669e9781acaf5c2625641c52d0dab90fab0576e Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Wed, 30 Jun 2021 18:39:01 +0530 Subject: [PATCH 01/11] Rework template loading into individual module + better tags and filters --- v2/cmd/nuclei/main.go | 5 +- v2/internal/runner/banner.go | 20 +-- v2/internal/runner/config.go | 126 ------------------- v2/internal/runner/options.go | 5 +- v2/internal/runner/runner.go | 62 ++++------ v2/internal/runner/templates.go | 41 ------ v2/internal/runner/update_test.go | 5 +- v2/pkg/catalog/catalogue.go | 6 - v2/pkg/catalog/config/config.go | 125 +++++++++++++++++++ v2/pkg/catalog/find.go | 5 +- v2/pkg/catalog/ignore.go | 58 --------- v2/pkg/catalog/ignore_test.go | 40 ------ v2/pkg/catalog/loader/filter.go | 117 +++++++++++++++++ v2/pkg/catalog/loader/filter_test.go | 65 ++++++++++ v2/pkg/catalog/loader/loader.go | 179 +++++++++++++++++++++++++++ v2/pkg/catalog/loader/loader_test.go | 43 +++++++ v2/pkg/templates/compile.go | 17 --- v2/pkg/types/types.go | 9 +- 18 files changed, 585 insertions(+), 343 deletions(-) create mode 100644 v2/pkg/catalog/config/config.go delete mode 100644 v2/pkg/catalog/ignore.go delete mode 100644 v2/pkg/catalog/ignore_test.go create mode 100644 v2/pkg/catalog/loader/filter.go create mode 100644 v2/pkg/catalog/loader/filter_test.go create mode 100644 v2/pkg/catalog/loader/loader.go create mode 100644 v2/pkg/catalog/loader/loader_test.go diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index c1459ceb..f402b0af 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -42,7 +42,10 @@ based on templates offering massive extensibility and ease of use.`) set.StringSliceVarP(&options.Templates, "templates", "t", []string{}, "Templates to run, supports single and multiple templates using directory.") set.StringSliceVarP(&options.Workflows, "workflows", "w", []string{}, "Workflows to run for nuclei") set.StringSliceVarP(&options.ExcludedTemplates, "exclude", "et", []string{}, "Templates to exclude, supports single and multiple templates using directory.") - set.StringSliceVarP(&options.Severity, "severity", "impact", []string{}, "Templates to run based on severity, supports single and multiple severity.") + set.StringSliceVarP(&options.Severity, "severity", "impact", []string{}, "Templates to run based on severity") + set.StringSliceVar(&options.Author, "author", []string{}, "Templates to run based on author") + set.StringSliceVar(&options.IncludeTemplates, "include-templates", []string{}, "Templates to force run even if they are in denylist") + set.StringSliceVar(&options.IncludeTags, "include-tags", []string{}, "Tags to force run even if they are in denylist") set.StringVarP(&options.Targets, "list", "l", "", "List of URLs to run templates on") set.StringVarP(&options.Output, "output", "o", "", "File to write output to (optional)") set.StringVar(&options.ProxyURL, "proxy-url", "", "URL of the proxy server") diff --git a/v2/internal/runner/banner.go b/v2/internal/runner/banner.go index 66594a2b..f5fd503e 100644 --- a/v2/internal/runner/banner.go +++ b/v2/internal/runner/banner.go @@ -1,17 +1,23 @@ package runner -import "github.com/projectdiscovery/gologger" +import ( + "fmt" -const banner = ` + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" +) + +var banner string + +func init() { + banner = fmt.Sprintf(` __ _ ____ __ _______/ /__ (_) / __ \/ / / / ___/ / _ \/ / / / / / /_/ / /__/ / __/ / - /_/ /_/\__,_/\___/_/\___/_/ v2.3.8 -` - -// Version is the current version of nuclei -const Version = `2.3.8` + /_/ /_/\__,_/\___/_/\___/_/ %s +`, config.Version) +} // showBanner is used to show the banner to the user func showBanner() { diff --git a/v2/internal/runner/config.go b/v2/internal/runner/config.go index 38643b36..75c10db6 100644 --- a/v2/internal/runner/config.go +++ b/v2/internal/runner/config.go @@ -1,127 +1 @@ package runner - -import ( - "os" - "path" - "regexp" - "time" - - jsoniter "github.com/json-iterator/go" - "github.com/projectdiscovery/gologger" - "gopkg.in/yaml.v2" -) - -// nucleiConfig contains some configuration options for nuclei -type nucleiConfig struct { - TemplatesDirectory string `json:"templates-directory,omitempty"` - CurrentVersion string `json:"current-version,omitempty"` - LastChecked time.Time `json:"last-checked,omitempty"` - IgnoreURL string `json:"ignore-url,omitempty"` - NucleiVersion string `json:"nuclei-version,omitempty"` - LastCheckedIgnore time.Time `json:"last-checked-ignore,omitempty"` - // IgnorePaths ignores all the paths listed unless specified manually - IgnorePaths []string `json:"ignore-paths,omitempty"` -} - -// nucleiConfigFilename is the filename of nuclei configuration file. -const nucleiConfigFilename = ".templates-config.json" - -var reVersion = regexp.MustCompile(`\d+\.\d+\.\d+`) - -// readConfiguration reads the nuclei configuration file from disk. -func readConfiguration() (*nucleiConfig, error) { - home, err := os.UserHomeDir() - if err != nil { - return nil, err - } - configDir := path.Join(home, "/.config", "/nuclei") - _ = os.MkdirAll(configDir, os.ModePerm) - - templatesConfigFile := path.Join(configDir, nucleiConfigFilename) - file, err := os.Open(templatesConfigFile) - if err != nil { - return nil, err - } - defer file.Close() - - config := &nucleiConfig{} - err = jsoniter.NewDecoder(file).Decode(config) - if err != nil { - return nil, err - } - return config, nil -} - -// readConfiguration reads the nuclei configuration file from disk. -func (r *Runner) writeConfiguration(config *nucleiConfig) error { - home, err := os.UserHomeDir() - if err != nil { - return err - } - configDir := path.Join(home, "/.config", "/nuclei") - _ = os.MkdirAll(configDir, os.ModePerm) - - if config.IgnoreURL == "" { - config.IgnoreURL = "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore" - } - config.LastChecked = time.Now() - config.LastCheckedIgnore = time.Now() - config.NucleiVersion = Version - templatesConfigFile := path.Join(configDir, nucleiConfigFilename) - file, err := os.OpenFile(templatesConfigFile, os.O_WRONLY|os.O_CREATE, 0777) - if err != nil { - return err - } - defer file.Close() - - err = jsoniter.NewEncoder(file).Encode(config) - if err != nil { - return err - } - return nil -} - -const nucleiIgnoreFile = ".nuclei-ignore" - -type ignoreFile struct { - Tags []string `yaml:"tags"` - Files []string `yaml:"files"` -} - -// readNucleiIgnoreFile reads the nuclei ignore file marking it in map -func (r *Runner) readNucleiIgnoreFile() { - file, err := os.Open(r.getIgnoreFilePath()) - if err != nil { - gologger.Error().Msgf("Could not read nuclei-ignore file: %s\n", err) - return - } - defer file.Close() - - ignore := &ignoreFile{} - if err := yaml.NewDecoder(file).Decode(ignore); err != nil { - gologger.Error().Msgf("Could not parse nuclei-ignore file: %s\n", err) - return - } - r.options.ExcludeTags = append(r.options.ExcludeTags, ignore.Tags...) - r.templatesConfig.IgnorePaths = append(r.templatesConfig.IgnorePaths, ignore.Files...) -} - -// getIgnoreFilePath returns the ignore file path for the runner -func (r *Runner) getIgnoreFilePath() string { - var defIgnoreFilePath string - - home, err := os.UserHomeDir() - if err == nil { - configDir := path.Join(home, "/.config", "/nuclei") - _ = os.MkdirAll(configDir, os.ModePerm) - - defIgnoreFilePath = path.Join(configDir, nucleiIgnoreFile) - return defIgnoreFilePath - } - cwd, err := os.Getwd() - if err != nil { - return defIgnoreFilePath - } - cwdIgnoreFilePath := path.Join(cwd, nucleiIgnoreFile) - return cwdIgnoreFilePath -} diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 61864cbf..811b148a 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -10,6 +10,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/formatter" "github.com/projectdiscovery/gologger/levels" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) @@ -26,11 +27,11 @@ func ParseOptions(options *types.Options) { showBanner() if options.Version { - gologger.Info().Msgf("Current Version: %s\n", Version) + gologger.Info().Msgf("Current Version: %s\n", config.Version) os.Exit(0) } if options.TemplatesVersion { - config, err := readConfiguration() + config, err := config.ReadConfiguration() if err != nil { gologger.Fatal().Msgf("Could not read template configuration: %s\n", err) } diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 1eff8bea..4b70bc57 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -11,9 +11,10 @@ import ( "github.com/logrusorgru/aurora" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/hmap/store/hybrid" - "github.com/projectdiscovery/nuclei/v2/internal/collaborator" "github.com/projectdiscovery/nuclei/v2/internal/colorizer" "github.com/projectdiscovery/nuclei/v2/pkg/catalog" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/progress" "github.com/projectdiscovery/nuclei/v2/pkg/projectfile" @@ -40,7 +41,7 @@ type Runner struct { output output.Writer interactsh *interactsh.Client inputCount int64 - templatesConfig *nucleiConfig + templatesConfig *config.Config options *types.Options projectFile *projectfile.ProjectFile catalog *catalog.Catalog @@ -69,11 +70,6 @@ func New(options *types.Options) (*Runner, error) { } runner.catalog = catalog.New(runner.options.TemplatesDirectory) - // Read nucleiignore file if given a templateconfig - if runner.templatesConfig != nil { - runner.readNucleiIgnoreFile() - runner.catalog.AppendIgnore(runner.templatesConfig.IgnorePaths) - } var reportingOptions *reporting.Options if options.ReportingConfig != "" { file, err := os.Open(options.ReportingConfig) @@ -227,11 +223,6 @@ func New(options *types.Options) (*Runner, error) { } } - // Enable Polling - if options.BurpCollaboratorBiid != "" { - collaborator.DefaultCollaborator.Collab.AddBIID(options.BurpCollaboratorBiid) - } - if options.RateLimit > 0 { runner.ratelimiter = ratelimit.New(options.RateLimit) } else { @@ -257,10 +248,7 @@ func (r *Runner) Close() { func (r *Runner) RunEnumeration() { defer r.Close() - // If we have no templates, run on whole template directory with provided tags - if len(r.options.Templates) == 0 && len(r.options.Workflows) == 0 && !r.options.NewTemplates && (len(r.options.Tags) > 0 || len(r.options.ExcludeTags) > 0) { - r.options.Templates = append(r.options.Templates, r.options.TemplatesDirectory) - } + // If user asked for new templates to be executed, collect the list from template directory. if r.options.NewTemplates { templatesLoaded, err := r.readNewTemplatesFile() if err != nil { @@ -268,32 +256,32 @@ func (r *Runner) RunEnumeration() { } r.options.Templates = append(r.options.Templates, templatesLoaded...) } - includedTemplates := r.catalog.GetTemplatesPath(r.options.Templates, false) - excludedTemplates := r.catalog.GetTemplatesPath(r.options.ExcludedTemplates, true) - // defaults to all templates - allTemplates := includedTemplates + ignoreFile := config.ReadIgnoreFile() + r.options.ExcludeTags = append(r.options.ExcludeTags, ignoreFile.Tags...) + r.options.ExcludedTemplates = append(r.options.ExcludedTemplates, ignoreFile.Files...) - if len(excludedTemplates) > 0 { - excludedMap := make(map[string]struct{}, len(excludedTemplates)) - for _, excl := range excludedTemplates { - excludedMap[excl] = struct{}{} - } - // rebuild list with only non-excluded templates - allTemplates = []string{} - - for _, incl := range includedTemplates { - if _, found := excludedMap[incl]; !found { - allTemplates = append(allTemplates, incl) - } else { - gologger.Warning().Msgf("Excluding '%s'", incl) - } - } + loaderConfig := &loader.Config{ + Templates: r.options.Templates, + Workflows: r.options.Workflows, + ExcludeTemplates: r.options.ExcludedTemplates, + Tags: r.options.Tags, + ExcludeTags: r.options.ExcludeTags, + IncludeTemplates: r.options.IncludeTemplates, + Authors: r.options.Author, + Severities: r.options.Severity, + IncludeTags: r.options.IncludeTags, + TemplatesDirectory: r.options.TemplatesDirectory, + Catalog: r.catalog, + } + store, err := loader.New(loaderConfig) + if err != nil { + gologger.Fatal().Msgf("Could not load templates from config: %s\n", err) } // pre-parse all the templates, apply filters finalTemplates := []*templates.Template{} - workflowPaths := r.catalog.GetTemplatesPath(r.options.Workflows, false) + workflowPaths := r.catalog.GetTemplatesPath(r.options.Workflows) availableTemplates, _ := r.getParsedTemplatesFor(allTemplates, r.options.Severity, false) availableWorkflows, workflowCount := r.getParsedTemplatesFor(workflowPaths, r.options.Severity, true) @@ -364,8 +352,6 @@ func (r *Runner) RunEnumeration() { results := &atomic.Bool{} wgtemplates := sizedwaitgroup.New(r.options.TemplateThreads) - // Starts polling or ignore - collaborator.DefaultCollaborator.Poll() // tracks global progress and captures stdout/stderr until p.Wait finishes r.progress.Init(r.inputCount, templateCount, totalRequests) diff --git a/v2/internal/runner/templates.go b/v2/internal/runner/templates.go index e24d3460..9c6fe4ed 100644 --- a/v2/internal/runner/templates.go +++ b/v2/internal/runner/templates.go @@ -12,47 +12,6 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/types" ) -// getParsedTemplatesFor parse the specified templates and returns a slice of the parsable ones, optionally filtered -// by severity, along with a flag indicating if workflows are present. -func (r *Runner) getParsedTemplatesFor(templatePaths, severities []string, workflows bool) (parsedTemplates map[string]*templates.Template, workflowCount int) { - filterBySeverity := len(severities) > 0 - - if !workflows { - gologger.Info().Msgf("Loading templates...") - } else { - gologger.Info().Msgf("Loading workflows...") - } - - parsedTemplates = make(map[string]*templates.Template) - for _, match := range templatePaths { - t, err := r.parseTemplateFile(match) - if err != nil { - gologger.Warning().Msgf("Could not parse file '%s': %s\n", match, err) - continue - } - if t == nil { - continue - } - if len(t.Workflows) == 0 && workflows { - continue // don't print if user only wants to run workflows - } - if len(t.Workflows) > 0 && !workflows { - continue // don't print workflow if user only wants to run templates - } - if len(t.Workflows) > 0 { - workflowCount++ - } - sev := strings.ToLower(types.ToString(t.Info["severity"])) - if !filterBySeverity || hasMatchingSeverity(sev, severities) { - parsedTemplates[t.ID] = t - gologger.Info().Msgf("%s\n", r.templateLogMsg(t.ID, types.ToString(t.Info["name"]), types.ToString(t.Info["author"]), sev)) - } else { - gologger.Warning().Msgf("Excluding template %s due to severity filter (%s not in [%s])", t.ID, sev, severities) - } - } - return parsedTemplates, workflowCount -} - // parseTemplateFile returns the parsed template file func (r *Runner) parseTemplateFile(file string) (*templates.Template, error) { executerOpts := protocols.ExecuterOptions{ diff --git a/v2/internal/runner/update_test.go b/v2/internal/runner/update_test.go index 8ee4e947..73cf2b9a 100644 --- a/v2/internal/runner/update_test.go +++ b/v2/internal/runner/update_test.go @@ -15,6 +15,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/internal/testutils" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/stretchr/testify/require" ) @@ -41,7 +42,7 @@ func TestDownloadReleaseAndUnzipAddition(t *testing.T) { require.Nil(t, err, "could not create temp directory") defer os.RemoveAll(templatesDirectory) - r := &Runner{templatesConfig: &nucleiConfig{TemplatesDirectory: templatesDirectory}} + r := &Runner{templatesConfig: &config.Config{TemplatesDirectory: templatesDirectory}} results, err := r.downloadReleaseAndUnzip(context.Background(), "1.0.0", ts.URL) require.Nil(t, err, "could not download release and unzip") @@ -94,7 +95,7 @@ func TestDownloadReleaseAndUnzipDeletion(t *testing.T) { require.Nil(t, err, "could not create temp directory") defer os.RemoveAll(templatesDirectory) - r := &Runner{templatesConfig: &nucleiConfig{TemplatesDirectory: templatesDirectory}} + r := &Runner{templatesConfig: &config.Config{TemplatesDirectory: templatesDirectory}} results, err := r.downloadReleaseAndUnzip(context.Background(), "1.0.0", ts.URL) require.Nil(t, err, "could not download release and unzip") diff --git a/v2/pkg/catalog/catalogue.go b/v2/pkg/catalog/catalogue.go index c085e5c2..38a58d7c 100644 --- a/v2/pkg/catalog/catalogue.go +++ b/v2/pkg/catalog/catalogue.go @@ -2,7 +2,6 @@ package catalog // Catalog is a template catalog helper implementation type Catalog struct { - ignoreFiles []string templatesDirectory string } @@ -11,8 +10,3 @@ func New(directory string) *Catalog { catalog := &Catalog{templatesDirectory: directory} return catalog } - -// AppendIgnore appends to the catalog store ignore list. -func (c *Catalog) AppendIgnore(list []string) { - c.ignoreFiles = append(c.ignoreFiles, list...) -} diff --git a/v2/pkg/catalog/config/config.go b/v2/pkg/catalog/config/config.go new file mode 100644 index 00000000..4acfc534 --- /dev/null +++ b/v2/pkg/catalog/config/config.go @@ -0,0 +1,125 @@ +package config + +import ( + "os" + "path" + "regexp" + "time" + + jsoniter "github.com/json-iterator/go" + "github.com/projectdiscovery/gologger" + "gopkg.in/yaml.v2" +) + +// Config contains the internal nuclei engine configuration +type Config struct { + TemplatesDirectory string `json:"templates-directory,omitempty"` + CurrentVersion string `json:"current-version,omitempty"` + LastChecked time.Time `json:"last-checked,omitempty"` + IgnoreURL string `json:"ignore-url,omitempty"` + NucleiVersion string `json:"nuclei-version,omitempty"` + LastCheckedIgnore time.Time `json:"last-checked-ignore,omitempty"` +} + +// nucleiConfigFilename is the filename of nuclei configuration file. +const nucleiConfigFilename = ".templates-config.json" + +var reVersion = regexp.MustCompile(`\d+\.\d+\.\d+`) + +// Version is the current version of nuclei +const Version = `2.3.8` + +var ( + homeDir string + configDir string + templatesConfigFile string +) + +func init() { + homeDir, _ = os.UserHomeDir() + configDir = path.Join(homeDir, "/.config", "/nuclei") + _ = os.MkdirAll(configDir, os.ModePerm) + templatesConfigFile = path.Join(configDir, nucleiConfigFilename) +} + +// readConfiguration reads the nuclei configuration file from disk. +func ReadConfiguration() (*Config, error) { + file, err := os.Open(templatesConfigFile) + if err != nil { + return nil, err + } + defer file.Close() + + config := &Config{} + err = jsoniter.NewDecoder(file).Decode(config) + if err != nil { + return nil, err + } + return config, nil +} + +// WriteConfiguration writes the updated nuclei configuration to disk +func WriteConfiguration(config *Config) error { + if config.IgnoreURL == "" { + config.IgnoreURL = "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore" + } + config.LastChecked = time.Now() + config.LastCheckedIgnore = time.Now() + config.NucleiVersion = Version + file, err := os.OpenFile(templatesConfigFile, os.O_WRONLY|os.O_CREATE, 0777) + if err != nil { + return err + } + defer file.Close() + + err = jsoniter.NewEncoder(file).Encode(config) + if err != nil { + return err + } + return nil +} + +const nucleiIgnoreFile = ".nuclei-ignore" + +// IgnoreFile is an internal nuclei template blocking configuration file +type IgnoreFile struct { + Tags []string `yaml:"tags"` + Files []string `yaml:"files"` +} + +// ReadIgnoreFile reads the nuclei ignore file returning blocked tags and paths +func ReadIgnoreFile() IgnoreFile { + file, err := os.Open(getIgnoreFilePath()) + if err != nil { + gologger.Error().Msgf("Could not read nuclei-ignore file: %s\n", err) + return IgnoreFile{} + } + defer file.Close() + + ignore := IgnoreFile{} + if err := yaml.NewDecoder(file).Decode(&ignore); err != nil { + gologger.Error().Msgf("Could not parse nuclei-ignore file: %s\n", err) + return IgnoreFile{} + } + return ignore +} + +// getIgnoreFilePath returns the ignore file path for the runner +func getIgnoreFilePath() string { + var defIgnoreFilePath string + + home, err := os.UserHomeDir() + if err == nil { + configDir := path.Join(home, "/.config", "/nuclei") + _ = os.MkdirAll(configDir, os.ModePerm) + + defIgnoreFilePath = path.Join(configDir, nucleiIgnoreFile) + return defIgnoreFilePath + } + cwd, err := os.Getwd() + if err != nil { + return defIgnoreFilePath + } + cwdIgnoreFilePath := path.Join(cwd, nucleiIgnoreFile) + return cwdIgnoreFilePath +} diff --git a/v2/pkg/catalog/find.go b/v2/pkg/catalog/find.go index 4529d2e5..f0a54b63 100644 --- a/v2/pkg/catalog/find.go +++ b/v2/pkg/catalog/find.go @@ -12,7 +12,7 @@ import ( ) // GetTemplatesPath returns a list of absolute paths for the provided template list. -func (c *Catalog) GetTemplatesPath(definitions []string, noCheckIgnore bool) []string { +func (c *Catalog) GetTemplatesPath(definitions []string) []string { // keeps track of processed dirs and files processed := make(map[string]bool) allTemplates := []string{} @@ -23,9 +23,6 @@ func (c *Catalog) GetTemplatesPath(definitions []string, noCheckIgnore bool) []s gologger.Error().Msgf("Could not find template '%s': %s\n", t, err) } for _, path := range paths { - if !noCheckIgnore && c.checkIfInNucleiIgnore(path) { - continue - } if _, ok := processed[path]; !ok { processed[path] = true allTemplates = append(allTemplates, path) diff --git a/v2/pkg/catalog/ignore.go b/v2/pkg/catalog/ignore.go deleted file mode 100644 index c3ecb991..00000000 --- a/v2/pkg/catalog/ignore.go +++ /dev/null @@ -1,58 +0,0 @@ -package catalog - -import ( - "strings" - - "github.com/projectdiscovery/gologger" -) - -// checkIfInNucleiIgnore checks if a path falls under nuclei-ignore rules. -func (c *Catalog) checkIfInNucleiIgnore(item string) bool { - if c.templatesDirectory == "" { - return false - } - - matched := false - for _, paths := range c.ignoreFiles { - if !strings.HasSuffix(paths, ".yaml") { - if strings.HasSuffix(strings.TrimSuffix(item, "/"), strings.TrimSuffix(paths, "/")) { - matched = true - break - } - } else if strings.HasSuffix(item, paths) { - matched = true - break - } - } - if matched { - gologger.Warning().Msgf("Excluding %s due to nuclei-ignore filter", item) - return true - } - return false -} - -// ignoreFilesWithExcludes ignores results with exclude paths -func (c *Catalog) ignoreFilesWithExcludes(results, excluded []string) []string { - var templates []string - - for _, result := range results { - matched := false - for _, paths := range excluded { - if !strings.HasSuffix(paths, ".yaml") { - if strings.HasSuffix(strings.TrimSuffix(result, "/"), strings.TrimSuffix(paths, "/")) { - matched = true - break - } - } else if strings.HasSuffix(result, paths) { - matched = true - break - } - } - if !matched { - templates = append(templates, result) - } else { - gologger.Error().Msgf("Excluding %s due to excludes filter", result) - } - } - return templates -} diff --git a/v2/pkg/catalog/ignore_test.go b/v2/pkg/catalog/ignore_test.go deleted file mode 100644 index 845696a4..00000000 --- a/v2/pkg/catalog/ignore_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package catalog - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestIgnoreFilesIgnore(t *testing.T) { - c := &Catalog{ - ignoreFiles: []string{"workflows/", "cves/2020/cve-2020-5432.yaml"}, - templatesDirectory: "test", - } - tests := []struct { - path string - ignore bool - }{ - {"workflows/", true}, - {"misc", false}, - {"cves/", false}, - {"cves/2020/cve-2020-5432.yaml", true}, - {"/Users/test/nuclei-templates/workflows/", true}, - {"/Users/test/nuclei-templates/misc", false}, - {"/Users/test/nuclei-templates/cves/", false}, - {"/Users/test/nuclei-templates/cves/2020/cve-2020-5432.yaml", true}, - } - for _, test := range tests { - require.Equal(t, test.ignore, c.checkIfInNucleiIgnore(test.path), fmt.Sprintf("could not ignore file correctly: %v", test)) - } -} - -func TestExcludeFilesIgnore(t *testing.T) { - c := &Catalog{} - excludes := []string{"workflows/", "cves/2020/cve-2020-5432.yaml"} - paths := []string{"/Users/test/nuclei-templates/workflows/", "/Users/test/nuclei-templates/cves/2020/cve-2020-5432.yaml", "/Users/test/nuclei-templates/workflows/test-workflow.yaml", "/Users/test/nuclei-templates/cves/"} - - data := c.ignoreFilesWithExcludes(paths, excludes) - require.Equal(t, []string{"/Users/test/nuclei-templates/workflows/test-workflow.yaml", "/Users/test/nuclei-templates/cves/"}, data, "could not exclude correct files") -} diff --git a/v2/pkg/catalog/loader/filter.go b/v2/pkg/catalog/loader/filter.go new file mode 100644 index 00000000..0f3caf44 --- /dev/null +++ b/v2/pkg/catalog/loader/filter.go @@ -0,0 +1,117 @@ +package loader + +import "strings" + +// tagFilter is used to filter nuclei tag based execution +type tagFilter struct { + allowedTags map[string]struct{} + severities map[string]struct{} + authors map[string]struct{} + block map[string]struct{} + matchAllows map[string]struct{} +} + +// match takes a tag and whether the template was matched from user +// input and returns true or false using a tag filter. +// +// If the tag was specified in deny list, it will not return true +// unless it is explicitly specified by user in includeTags which is the +// matchAllows section. +// +// It returns true if the tag is specified, or false. +func (t *tagFilter) match(tag, author, severity string, templateMatched bool) bool { + _, ok := t.block[tag] + if ok { + if _, allowOk := t.matchAllows[tag]; allowOk && templateMatched { + return true + } + return false + } + matchedAny := false + if len(t.allowedTags) > 0 { + _, ok = t.allowedTags[tag] + if !ok { + return false + } + matchedAny = true + } + if len(t.authors) > 0 { + _, ok = t.authors[author] + if !ok { + return false + } + matchedAny = true + } + if len(t.severities) > 0 { + _, ok = t.severities[severity] + if !ok { + return false + } + matchedAny = true + } + if len(t.allowedTags) == 0 && len(t.authors) == 0 && len(t.severities) == 0 { + return true + } + return matchedAny +} + +// createTagFilter returns a tag filter for nuclei tag based execution +// +// It takes into account Tags, Severities, Authors, IncludeTags, ExcludeTags. +func (config *Config) createTagFilter() *tagFilter { + filter := &tagFilter{ + allowedTags: make(map[string]struct{}), + authors: make(map[string]struct{}), + severities: make(map[string]struct{}), + block: make(map[string]struct{}), + matchAllows: make(map[string]struct{}), + } + for _, tag := range config.Tags { + for _, val := range splitCommaTrim(tag) { + if _, ok := filter.allowedTags[val]; !ok { + filter.allowedTags[val] = struct{}{} + } + } + } + for _, tag := range config.Severities { + for _, val := range splitCommaTrim(tag) { + if _, ok := filter.severities[val]; !ok { + filter.severities[val] = struct{}{} + } + } + } + for _, tag := range config.Authors { + for _, val := range splitCommaTrim(tag) { + if _, ok := filter.authors[val]; !ok { + filter.authors[val] = struct{}{} + } + } + } + for _, tag := range config.IncludeTags { + for _, val := range splitCommaTrim(tag) { + if _, ok := filter.matchAllows[val]; !ok { + filter.matchAllows[val] = struct{}{} + } + } + } + for _, tag := range config.ExcludeTags { + for _, val := range splitCommaTrim(tag) { + if _, ok := filter.block[val]; !ok { + filter.block[val] = struct{}{} + } + } + } + return filter +} + +func splitCommaTrim(value string) []string { + if !strings.Contains(value, ",") { + return []string{value} + } + splitted := strings.Split(value, ",") + final := make([]string, len(splitted)) + for i, value := range splitted { + final[i] = strings.TrimSpace(value) + } + return final +} diff --git a/v2/pkg/catalog/loader/filter_test.go b/v2/pkg/catalog/loader/filter_test.go new file mode 100644 index 00000000..2535d112 --- /dev/null +++ b/v2/pkg/catalog/loader/filter_test.go @@ -0,0 +1,65 @@ +package loader + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestTagBasedFilter(t *testing.T) { + config := &Config{ + Tags: []string{"cves", "2021", "jira"}, + } + filter := config.createTagFilter() + + t.Run("true", func(t *testing.T) { + require.True(t, filter.match("jira", "pdteam", "low", false), "could not get correct match") + }) + t.Run("false", func(t *testing.T) { + require.False(t, filter.match("consul", "pdteam", "low", false), "could not get correct match") + }) + t.Run("not-match-excludes", func(t *testing.T) { + config := &Config{ + Tags: []string{"cves", "dos"}, + ExcludeTags: []string{"dos"}, + } + filter := config.createTagFilter() + require.False(t, filter.match("jira", "pdteam", "low", false), "could not get correct match") + }) + t.Run("match-includes", func(t *testing.T) { + config := &Config{ + Tags: []string{"cves", "fuzz"}, + ExcludeTags: []string{"dos", "fuzz"}, + IncludeTags: []string{"fuzz"}, + } + + filter := config.createTagFilter() + require.False(t, filter.match("fuzz", "pdteam", "low", false), "could not get correct match") + }) + t.Run("match-author", func(t *testing.T) { + config := &Config{ + Authors: []string{"pdteam"}, + } + filter := config.createTagFilter() + require.True(t, filter.match("fuzz", "pdteam", "low", false), "could not get correct match") + }) + t.Run("match-severity", func(t *testing.T) { + config := &Config{ + Severities: []string{"high"}, + } + filter := config.createTagFilter() + require.True(t, filter.match("fuzz", "pdteam", "high", false), "could not get correct match") + }) + t.Run("match-conditions", func(t *testing.T) { + config := &Config{ + Authors: []string{"pdteam"}, + Tags: []string{"jira"}, + Severities: []string{"high"}, + } + filter := config.createTagFilter() + require.True(t, filter.match("jira", "pdteam", "high", false), "could not get correct match") + require.False(t, filter.match("jira", "pdteam", "low", false), "could not get correct match") + require.False(t, filter.match("jira", "random", "low", false), "could not get correct match") + require.False(t, filter.match("consul", "random", "low", false), "could not get correct match") + }) +} diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go new file mode 100644 index 00000000..b08cadb8 --- /dev/null +++ b/v2/pkg/catalog/loader/loader.go @@ -0,0 +1,179 @@ +package loader + +import ( + "bytes" + "errors" + "io/ioutil" + "os" + "strings" + + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog" + "github.com/projectdiscovery/nuclei/v2/pkg/templates" + "github.com/projectdiscovery/nuclei/v2/pkg/types" + "gopkg.in/yaml.v2" +) + +// Config contains the configuration options for the loader +type Config struct { + Templates []string + Workflows []string + ExcludeTemplates []string + IncludeTemplates []string + + Tags []string + ExcludeTags []string + Authors []string + Severities []string + IncludeTags []string + + Catalog *catalog.Catalog + TemplatesDirectory string +} + +// Store is a storage for loaded nuclei templates +type Store struct { + tagFilter *tagFilter + config *Config + finalTemplates []string + templateMatched bool +} + +// New creates a new template store based on provided configuration +func New(config *Config) (*Store, error) { + // Create a tag filter based on provided configuration + store := &Store{ + config: config, + tagFilter: config.createTagFilter(), + } + + // hasFindByMetadata := config.hasFindByMetadata() + // Handle a case with no templates or workflows, where we use base directory + if len(config.Templates) == 0 && len(config.Workflows) == 0 { + config.Templates = append(config.Templates, config.TemplatesDirectory) + } else { + store.templateMatched = true + } + store.finalTemplates = append(store.finalTemplates, config.Templates...) + return store, nil +} + +// Load loads all the templates from a store, performs filtering and returns +// the complete compiled templates for a nuclei execution configuration. +func (s *Store) Load() (templates []string, workflows []string) { + includedTemplates := s.config.Catalog.GetTemplatesPath(s.config.Templates) + includedWorkflows := s.config.Catalog.GetTemplatesPath(s.config.Workflows) + excludedTemplates := s.config.Catalog.GetTemplatesPath(s.config.ExcludeTemplates) + alwaysIncludeTemplates := s.config.Catalog.GetTemplatesPath(s.config.IncludeTemplates) + + alwaysIncludedTemplatesMap := make(map[string]struct{}) + for _, tpl := range alwaysIncludeTemplates { + alwaysIncludedTemplatesMap[tpl] = struct{}{} + } + + templatesMap := make(map[string]struct{}) + for _, tpl := range includedTemplates { + templatesMap[tpl] = struct{}{} + } + for _, template := range excludedTemplates { + if _, ok := alwaysIncludedTemplatesMap[template]; ok { + continue + } else { + delete(templatesMap, template) + } + } + + for k := range templatesMap { + loaded, err := s.loadTemplateParseMetadata(k, false) + if err != nil { + gologger.Warning().Msgf("Could not load template %s: %s\n", k, err) + } + if loaded { + templates = append(templates, k) + } + } + + workflowsMap := make(map[string]struct{}) + for _, tpl := range includedWorkflows { + workflowsMap[tpl] = struct{}{} + } + for _, template := range excludedTemplates { + if _, ok := alwaysIncludedTemplatesMap[template]; ok { + continue + } else { + delete(templatesMap, template) + } + } + for k := range workflowsMap { + loaded, err := s.loadTemplateParseMetadata(k, true) + if err != nil { + gologger.Warning().Msgf("Could not load workflow %s: %s\n", k, err) + } + if loaded { + workflows = append(workflows, k) + } + } + return templates, workflows +} + +// loadTemplateParseMetadata loads a template by parsing metadata and running +// all tag and path based filters on the template. +func (s *Store) loadTemplateParseMetadata(templatePath string, workflow bool) (bool, error) { + f, err := os.Open(templatePath) + if err != nil { + return false, err + } + defer f.Close() + + data, err := ioutil.ReadAll(f) + if err != nil { + return false, err + } + + template := &templates.Template{} + err = yaml.NewDecoder(bytes.NewReader(data)).Decode(template) + if err != nil { + return false, err + } + if _, ok := template.Info["name"]; !ok { + return false, errors.New("no template name field provided") + } + author, ok := template.Info["author"] + if !ok { + return false, errors.New("no template author field provided") + } + severity, ok := template.Info["severity"] + if !ok { + return false, errors.New("no template severity field provided") + } + templateTags, ok := template.Info["tags"] + if !ok { + templateTags = "" + } + tagStr := types.ToString(templateTags) + + tags := strings.Split(tagStr, ",") + severityStr := types.ToString(severity) + authors := strings.Split(types.ToString(author), ",") + + matched := false +mainLoop: + for _, tag := range tags { + for _, author := range authors { + if !matched && s.tagFilter.match(strings.TrimSpace(tag), strings.TrimSpace(author), severityStr, s.templateMatched) { + matched = true + break mainLoop + } + } + } + if !matched { + return false, nil + } + if len(template.Workflows) == 0 && workflow { + return false, nil + } + if len(template.Workflows) > 0 && !workflow { + return false, nil + } + return true, nil +} diff --git a/v2/pkg/catalog/loader/loader_test.go b/v2/pkg/catalog/loader/loader_test.go new file mode 100644 index 00000000..8d586eae --- /dev/null +++ b/v2/pkg/catalog/loader/loader_test.go @@ -0,0 +1,43 @@ +package loader + +import ( + "testing" + + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" + "github.com/stretchr/testify/require" +) + +func TestLoadTemplates(t *testing.T) { + config, err := config.ReadConfiguration() + require.Nil(t, err, "could not read configuration") + + store, err := New(&Config{ + Templates: []string{"cves/CVE-2021-21315.yaml"}, + }) + require.Nil(t, err, "could not load templates") + require.Equal(t, []string{"cves/CVE-2021-21315.yaml"}, store.finalTemplates, "could not get correct templates") + + t.Run("blank", func(t *testing.T) { + store, err := New(&Config{ + TemplatesDirectory: config.TemplatesDirectory, + }) + require.Nil(t, err, "could not load templates") + require.Equal(t, []string{config.TemplatesDirectory}, store.finalTemplates, "could not get correct templates") + }) + t.Run("only-tags", func(t *testing.T) { + store, err := New(&Config{ + Tags: []string{"cves"}, + TemplatesDirectory: config.TemplatesDirectory, + }) + require.Nil(t, err, "could not load templates") + require.Equal(t, []string{config.TemplatesDirectory}, store.finalTemplates, "could not get correct templates") + }) + t.Run("tags-with-path", func(t *testing.T) { + store, err := New(&Config{ + Tags: []string{"cves"}, + TemplatesDirectory: config.TemplatesDirectory, + }) + require.Nil(t, err, "could not load templates") + require.Equal(t, []string{config.TemplatesDirectory}, store.finalTemplates, "could not get correct templates") + }) +} diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 16b6ff21..9484befb 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -12,7 +12,6 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/executer" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/offlinehttp" - "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/nuclei/v2/pkg/workflows" "gopkg.in/yaml.v2" ) @@ -45,22 +44,6 @@ func Parse(filePath string, options protocols.ExecuterOptions) (*Template, error if _, ok := template.Info["author"]; !ok { return nil, errors.New("no template author field provided") } - templateTags, ok := template.Info["tags"] - if !ok { - templateTags = "" - } - matchWithTags := false - if len(options.Options.Tags) > 0 { - if err := matchTemplateWithTags(types.ToString(templateTags), types.ToString(template.Info["severity"]), options.Options.Tags); err != nil { - return nil, fmt.Errorf("tags filter not matched %s", templateTags) - } - matchWithTags = true - } - if len(options.Options.ExcludeTags) > 0 && !matchWithTags { - if err := matchTemplateWithTags(types.ToString(templateTags), types.ToString(template.Info["severity"]), options.Options.ExcludeTags); err == nil { - return nil, fmt.Errorf("exclude-tags filter matched %s", templateTags) - } - } // Setting up variables regarding template metadata options.TemplateID = template.ID diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 89cb701e..b26385eb 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -19,7 +19,14 @@ type Options struct { // CustomHeaders is the list of custom global headers to send with each request. CustomHeaders goflags.StringSlice // Severity filters templates based on their severity and only run the matching ones. - Severity goflags.StringSlice + Severity goflags.StringSlice + // Author filters templates based on their author and only run the matching ones. + Author goflags.StringSlice + // IncludeTags includes specified tags to be run even while being in denylist + IncludeTags goflags.StringSlice + // IncludeTemplates includes specified templates to be run even while being in denylist + IncludeTemplates goflags.StringSlice + InternalResolversList []string // normalized from resolvers flag as well as file provided. // BurpCollaboratorBiid is the Burp Collaborator BIID for polling interactions. BurpCollaboratorBiid string From dff76e9cd2539219b6632e403c2e93114559fcb0 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 1 Jul 2021 14:36:40 +0530 Subject: [PATCH 02/11] Loader rewriter working poc --- v2/internal/runner/options.go | 7 --- v2/internal/runner/runner.go | 88 ++++++++++++++++++++++++------ v2/internal/runner/templates.go | 60 ++++++--------------- v2/internal/runner/update.go | 89 +++++++++++++++++++++++++++---- v2/internal/runner/update_test.go | 1 - v2/pkg/catalog/config/config.go | 16 +++--- v2/pkg/catalog/loader/loader.go | 36 ++++++++++--- v2/pkg/templates/compile.go | 46 ---------------- v2/pkg/templates/compile_test.go | 41 -------------- 9 files changed, 210 insertions(+), 174 deletions(-) delete mode 100644 v2/pkg/templates/compile_test.go diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 811b148a..ab026a83 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -81,13 +81,6 @@ func validateOptions(options *types.Options) error { return errors.New("both verbose and silent mode specified") } - if !options.TemplateList { - // Check if a list of templates was provided and it exists - if len(options.Templates) == 0 && !options.NewTemplates && len(options.Workflows) == 0 && len(options.Tags) == 0 && !options.UpdateTemplates { - return errors.New("no template/templates provided") - } - } - // Validate proxy options if provided err := validateProxyURL(options.ProxyURL, "invalid http proxy format (It should be http://username:password@host:port)") if err != nil { diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 4b70bc57..b14d2d7c 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -260,6 +260,17 @@ func (r *Runner) RunEnumeration() { r.options.ExcludeTags = append(r.options.ExcludeTags, ignoreFile.Tags...) r.options.ExcludedTemplates = append(r.options.ExcludedTemplates, ignoreFile.Files...) + executerOpts := protocols.ExecuterOptions{ + Output: r.output, + Options: r.options, + Progress: r.progress, + Catalog: r.catalog, + IssuesClient: r.issuesClient, + RateLimiter: r.ratelimiter, + Interactsh: r.interactsh, + ProjectFile: r.projectFile, + Browser: r.browser, + } loaderConfig := &loader.Config{ Templates: r.options.Templates, Workflows: r.options.Workflows, @@ -272,21 +283,60 @@ func (r *Runner) RunEnumeration() { IncludeTags: r.options.IncludeTags, TemplatesDirectory: r.options.TemplatesDirectory, Catalog: r.catalog, + ExecutorOptions: executerOpts, } store, err := loader.New(loaderConfig) if err != nil { gologger.Fatal().Msgf("Could not load templates from config: %s\n", err) } + store.Load() + + builder := &strings.Builder{} + if r.templatesConfig != nil && r.templatesConfig.NucleiLatestVersion != "" { + builder.WriteString(" (") + + if config.Version == r.templatesConfig.NucleiLatestVersion { + builder.WriteString(r.colorizer.Green("latest").String()) + } else { + builder.WriteString(r.colorizer.Red("outdated").String()) + } + builder.WriteString(")") + } + messageStr := builder.String() + builder.Reset() + + gologger.Info().Msgf("Using Nuclei Engine %s%s", config.Version, messageStr) + + if r.templatesConfig != nil && r.templatesConfig.NucleiTemplatesLatestVersion != "" { + builder.WriteString(" (") + + if r.templatesConfig.CurrentVersion == r.templatesConfig.NucleiTemplatesLatestVersion { + builder.WriteString(r.colorizer.Green("latest").String()) + } else { + builder.WriteString(r.colorizer.Red("outdated").String()) + } + builder.WriteString(")") + } + messageStr = builder.String() + builder.Reset() + + gologger.Info().Msgf("Using Nuclei Templates %s%s", r.templatesConfig.CurrentVersion, messageStr) + + if r.interactsh != nil { + gologger.Info().Msgf("Using Interactsh Server %s", r.options.InteractshURL) + } + if len(store.Templates()) > 0 { + gologger.Info().Msgf("Running Nuclei Templates (%d)", len(store.Templates())) + } + if len(store.Workflows()) > 0 { + gologger.Info().Msgf("Running Nuclei Workflows (%d)", len(store.Workflows())) + } // pre-parse all the templates, apply filters finalTemplates := []*templates.Template{} - workflowPaths := r.catalog.GetTemplatesPath(r.options.Workflows) - availableTemplates, _ := r.getParsedTemplatesFor(allTemplates, r.options.Severity, false) - availableWorkflows, workflowCount := r.getParsedTemplatesFor(workflowPaths, r.options.Severity, true) - var unclusteredRequests int64 = 0 - for _, template := range availableTemplates { + for _, template := range store.Templates() { // workflows will dynamically adjust the totals while running, as // it can't be know in advance which requests will be called if len(template.Workflows) > 0 { @@ -295,9 +345,21 @@ func (r *Runner) RunEnumeration() { unclusteredRequests += int64(template.TotalRequests) * r.inputCount } - originalTemplatesCount := len(availableTemplates) + if r.options.Verbose { + for _, template := range store.Templates() { + r.logAvailableTemplate(template.Path) + } + for _, template := range store.Workflows() { + r.logAvailableTemplate(template.Path) + } + } + templatesMap := make(map[string]*templates.Template) + for _, v := range store.Templates() { + templatesMap[v.ID] = v + } + originalTemplatesCount := len(store.Templates()) clusterCount := 0 - clusters := clusterer.Cluster(availableTemplates) + clusters := clusterer.Cluster(templatesMap) for _, cluster := range clusters { if len(cluster) > 1 && !r.options.OfflineHTTP { executerOpts := protocols.ExecuterOptions{ @@ -324,7 +386,7 @@ func (r *Runner) RunEnumeration() { finalTemplates = append(finalTemplates, cluster...) } } - for _, workflows := range availableWorkflows { + for _, workflows := range store.Workflows() { finalTemplates = append(finalTemplates, workflows) } @@ -336,20 +398,16 @@ func (r *Runner) RunEnumeration() { totalRequests += int64(t.TotalRequests) * r.inputCount } if totalRequests < unclusteredRequests { - gologger.Info().Msgf("Reduced %d requests to %d (%d templates clustered)", unclusteredRequests, totalRequests, clusterCount) + gologger.Info().Msgf("Reduced %d requests (%d templates clustered)", unclusteredRequests-totalRequests, clusterCount) } - templateCount := originalTemplatesCount + len(availableWorkflows) + workflowCount := len(store.Workflows()) + templateCount := originalTemplatesCount + workflowCount // 0 matches means no templates were found in directory if templateCount == 0 { gologger.Fatal().Msgf("Error, no templates were found.\n") } - gologger.Info().Msgf("Using %s rules (%s templates, %s workflows)", - r.colorizer.Bold(templateCount).String(), - r.colorizer.Bold(templateCount-workflowCount).String(), - r.colorizer.Bold(workflowCount).String()) - results := &atomic.Bool{} wgtemplates := sizedwaitgroup.New(r.options.TemplateThreads) diff --git a/v2/internal/runner/templates.go b/v2/internal/runner/templates.go index 9c6fe4ed..29724a31 100644 --- a/v2/internal/runner/templates.go +++ b/v2/internal/runner/templates.go @@ -1,36 +1,36 @@ package runner import ( + "bytes" "fmt" + "io/ioutil" "os" "strings" "github.com/karrick/godirwalk" "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/types" + "gopkg.in/yaml.v2" ) // parseTemplateFile returns the parsed template file func (r *Runner) parseTemplateFile(file string) (*templates.Template, error) { - executerOpts := protocols.ExecuterOptions{ - Output: r.output, - Options: r.options, - Progress: r.progress, - Catalog: r.catalog, - IssuesClient: r.issuesClient, - RateLimiter: r.ratelimiter, - Interactsh: r.interactsh, - ProjectFile: r.projectFile, - Browser: r.browser, - } - template, err := templates.Parse(file, executerOpts) + f, err := os.Open(file) if err != nil { return nil, err } - if template == nil { - return nil, nil + defer f.Close() + + data, err := ioutil.ReadAll(f) + if err != nil { + return nil, err + } + + template := &templates.Template{} + err = yaml.NewDecoder(bytes.NewReader(data)).Decode(template) + if err != nil { + return nil, err } return template, nil } @@ -52,7 +52,7 @@ func (r *Runner) logAvailableTemplate(tplPath string) { if err != nil { gologger.Error().Msgf("Could not parse file '%s': %s\n", tplPath, err) } else { - gologger.Print().Msgf("%s\n", r.templateLogMsg(t.ID, types.ToString(t.Info["name"]), types.ToString(t.Info["author"]), types.ToString(t.Info["severity"]))) + gologger.Info().Msgf("%s\n", r.templateLogMsg(t.ID, types.ToString(t.Info["name"]), types.ToString(t.Info["author"]), types.ToString(t.Info["severity"]))) } } @@ -89,38 +89,12 @@ func (r *Runner) listAvailableTemplates() { } } -func hasMatchingSeverity(templateSeverity string, allowedSeverities []string) bool { - for _, s := range allowedSeverities { - finalSeverities := []string{} - if strings.Contains(s, ",") { - finalSeverities = strings.Split(s, ",") - } else { - finalSeverities = append(finalSeverities, s) - } - - for _, sev := range finalSeverities { - sev = strings.ToLower(sev) - if sev != "" && strings.HasPrefix(templateSeverity, sev) { - return true - } - } - } - return false -} - func directoryWalker(fsPath string, callback func(fsPath string, d *godirwalk.Dirent) error) error { - err := godirwalk.Walk(fsPath, &godirwalk.Options{ + return godirwalk.Walk(fsPath, &godirwalk.Options{ Callback: callback, ErrorCallback: func(fsPath string, err error) godirwalk.ErrorAction { return godirwalk.SkipNode }, Unsorted: true, }) - - // directory couldn't be walked - if err != nil { - return err - } - - return nil } diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go index 7e09cf73..196bdc8e 100644 --- a/v2/internal/runner/update.go +++ b/v2/internal/runner/update.go @@ -7,6 +7,7 @@ import ( "context" "crypto/md5" "encoding/hex" + "encoding/json" "fmt" "io" "io/ioutil" @@ -14,6 +15,7 @@ import ( "os" "path" "path/filepath" + "regexp" "strconv" "strings" "time" @@ -23,6 +25,7 @@ import ( "github.com/olekukonko/tablewriter" "github.com/pkg/errors" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" ) const ( @@ -30,6 +33,13 @@ const ( repoName = "nuclei-templates" ) +const nucleiIgnoreFile = ".nuclei-ignore" + +// nucleiConfigFilename is the filename of nuclei configuration file. +const nucleiConfigFilename = ".templates-config.json" + +var reVersion = regexp.MustCompile(`\d+\.\d+\.\d+`) + // updateTemplates checks if the default list of nuclei-templates // exist in the users home directory, if not the latest revision // is downloaded from github. @@ -46,7 +56,7 @@ func (r *Runner) updateTemplates() error { templatesConfigFile := path.Join(configDir, nucleiConfigFilename) if _, statErr := os.Stat(templatesConfigFile); !os.IsNotExist(statErr) { - config, readErr := readConfiguration() + config, readErr := config.ReadConfiguration() if err != nil { return readErr } @@ -55,12 +65,12 @@ func (r *Runner) updateTemplates() error { ignoreURL := "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore" if r.templatesConfig == nil { - currentConfig := &nucleiConfig{ + currentConfig := &config.Config{ TemplatesDirectory: path.Join(home, "nuclei-templates"), IgnoreURL: ignoreURL, - NucleiVersion: Version, + NucleiVersion: config.Version, } - if writeErr := r.writeConfiguration(currentConfig); writeErr != nil { + if writeErr := config.WriteConfiguration(currentConfig, false, false); writeErr != nil { return errors.Wrap(writeErr, "could not write template configuration") } r.templatesConfig = currentConfig @@ -68,12 +78,19 @@ func (r *Runner) updateTemplates() error { // Check if last checked for nuclei-ignore is more than 1 hours. // and if true, run the check. + // + // Also at the same time fetch latest version from github to do outdated nuclei + // and templates check. + checkedIgnore := false if r.templatesConfig == nil || time.Since(r.templatesConfig.LastCheckedIgnore) > 1*time.Hour || r.options.UpdateTemplates { + r.fetchLatestVersionsFromGithub() + if r.templatesConfig != nil && r.templatesConfig.IgnoreURL != "" { ignoreURL = r.templatesConfig.IgnoreURL } gologger.Verbose().Msgf("Downloading config file from %s", ignoreURL) + checkedIgnore = true ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) req, reqErr := http.NewRequestWithContext(ctx, http.MethodGet, ignoreURL, nil) if reqErr == nil { @@ -106,7 +123,7 @@ func (r *Runner) updateTemplates() error { } // Use custom location if user has given a template directory - r.templatesConfig = &nucleiConfig{ + r.templatesConfig = &config.Config{ TemplatesDirectory: path.Join(home, "nuclei-templates"), } if r.options.TemplatesDirectory != "" && r.options.TemplatesDirectory != path.Join(home, "nuclei-templates") { @@ -120,13 +137,14 @@ func (r *Runner) updateTemplates() error { } gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory) + r.fetchLatestVersionsFromGithub() // also fetch latest versions _, err = r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL()) if err != nil { return err } r.templatesConfig.CurrentVersion = version.String() - err = r.writeConfiguration(r.templatesConfig) + err = config.WriteConfiguration(r.templatesConfig, true, checkedIgnore) if err != nil { return err } @@ -162,13 +180,13 @@ func (r *Runner) updateTemplates() error { if version.EQ(oldVersion) { gologger.Info().Msgf("Your nuclei-templates are up to date: v%s\n", oldVersion.String()) - return r.writeConfiguration(r.templatesConfig) + return config.WriteConfiguration(r.templatesConfig, false, checkedIgnore) } if version.GT(oldVersion) { if !r.options.UpdateTemplates { gologger.Warning().Msgf("Your current nuclei-templates v%s are outdated. Latest is v%s\n", oldVersion, version.String()) - return r.writeConfiguration(r.templatesConfig) + return config.WriteConfiguration(r.templatesConfig, false, checkedIgnore) } if r.options.TemplatesDirectory != "" { @@ -177,11 +195,12 @@ func (r *Runner) updateTemplates() error { r.templatesConfig.CurrentVersion = version.String() gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory) + r.fetchLatestVersionsFromGithub() _, err = r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL()) if err != nil { return err } - err = r.writeConfiguration(r.templatesConfig) + err = config.WriteConfiguration(r.templatesConfig, true, checkedIgnore) if err != nil { return err } @@ -461,3 +480,55 @@ func (r *Runner) printUpdateChangelog(results *templateUpdateResults, version st } table.Render() } + +// fetchLatestVersionsFromGithub fetches latest versions of nuclei repos from github +func (r *Runner) fetchLatestVersionsFromGithub() { + nucleiLatest, err := r.githubFetchLatestTagRepo("projectdiscovery/nuclei") + if err != nil { + gologger.Warning().Msgf("Could not fetch latest nuclei release: %s", err) + } + templatesLatest, err := r.githubFetchLatestTagRepo("projectdiscovery/nuclei-templates") + if err != nil { + gologger.Warning().Msgf("Could not fetch latest nuclei-templates release: %s", err) + } + if r.templatesConfig != nil { + r.templatesConfig.NucleiLatestVersion = nucleiLatest + r.templatesConfig.NucleiTemplatesLatestVersion = templatesLatest + } +} + +type githubTagData struct { + Name string +} + +// githubFetchLatestTagRepo fetches latest tag from github +func (r *Runner) githubFetchLatestTagRepo(repo string) (string, error) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + url := fmt.Sprintf("https://api.github.com/repos/%s/tags", repo) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return "", err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", err + } + + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", err + } + + var tags []githubTagData + err = json.Unmarshal(body, &tags) + if err != nil { + return "", err + } + if len(tags) == 0 { + return "", fmt.Errorf("no tags found for %s", repo) + } + return tags[0].Name, nil +} diff --git a/v2/internal/runner/update_test.go b/v2/internal/runner/update_test.go index 73cf2b9a..43ed84ce 100644 --- a/v2/internal/runner/update_test.go +++ b/v2/internal/runner/update_test.go @@ -43,7 +43,6 @@ func TestDownloadReleaseAndUnzipAddition(t *testing.T) { defer os.RemoveAll(templatesDirectory) r := &Runner{templatesConfig: &config.Config{TemplatesDirectory: templatesDirectory}} - results, err := r.downloadReleaseAndUnzip(context.Background(), "1.0.0", ts.URL) require.Nil(t, err, "could not download release and unzip") require.Equal(t, "base.yaml", results.additions[0], "could not get correct base addition") diff --git a/v2/pkg/catalog/config/config.go b/v2/pkg/catalog/config/config.go index 4acfc534..a616c319 100644 --- a/v2/pkg/catalog/config/config.go +++ b/v2/pkg/catalog/config/config.go @@ -3,7 +3,6 @@ package config import ( "os" "path" - "regexp" "time" jsoniter "github.com/json-iterator/go" @@ -19,13 +18,14 @@ type Config struct { IgnoreURL string `json:"ignore-url,omitempty"` NucleiVersion string `json:"nuclei-version,omitempty"` LastCheckedIgnore time.Time `json:"last-checked-ignore,omitempty"` + + NucleiLatestVersion string `json:"nuclei-latest-version"` + NucleiTemplatesLatestVersion string `json:"nuclei-templates-latest-version"` } // nucleiConfigFilename is the filename of nuclei configuration file. const nucleiConfigFilename = ".templates-config.json" -var reVersion = regexp.MustCompile(`\d+\.\d+\.\d+`) - // Version is the current version of nuclei const Version = `2.3.8` @@ -59,12 +59,16 @@ func ReadConfiguration() (*Config, error) { } // WriteConfiguration writes the updated nuclei configuration to disk -func WriteConfiguration(config *Config) error { +func WriteConfiguration(config *Config, checked, checkedIgnore bool) error { if config.IgnoreURL == "" { config.IgnoreURL = "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore" } - config.LastChecked = time.Now() - config.LastCheckedIgnore = time.Now() + if checked { + config.LastChecked = time.Now() + } + if checkedIgnore { + config.LastCheckedIgnore = time.Now() + } config.NucleiVersion = Version file, err := os.OpenFile(templatesConfigFile, os.O_WRONLY|os.O_CREATE, 0777) if err != nil { diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go index b08cadb8..eed2f90c 100644 --- a/v2/pkg/catalog/loader/loader.go +++ b/v2/pkg/catalog/loader/loader.go @@ -9,6 +9,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/catalog" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/types" "gopkg.in/yaml.v2" @@ -28,6 +29,7 @@ type Config struct { IncludeTags []string Catalog *catalog.Catalog + ExecutorOptions protocols.ExecuterOptions TemplatesDirectory string } @@ -37,6 +39,9 @@ type Store struct { config *Config finalTemplates []string templateMatched bool + + templates []*templates.Template + workflows []*templates.Template } // New creates a new template store based on provided configuration @@ -47,7 +52,6 @@ func New(config *Config) (*Store, error) { tagFilter: config.createTagFilter(), } - // hasFindByMetadata := config.hasFindByMetadata() // Handle a case with no templates or workflows, where we use base directory if len(config.Templates) == 0 && len(config.Workflows) == 0 { config.Templates = append(config.Templates, config.TemplatesDirectory) @@ -55,13 +59,24 @@ func New(config *Config) (*Store, error) { store.templateMatched = true } store.finalTemplates = append(store.finalTemplates, config.Templates...) + return store, nil } +// Templates returns all the templates in the store +func (s *Store) Templates() []*templates.Template { + return s.templates +} + +// Workflows returns all the workflows in the store +func (s *Store) Workflows() []*templates.Template { + return s.workflows +} + // Load loads all the templates from a store, performs filtering and returns // the complete compiled templates for a nuclei execution configuration. -func (s *Store) Load() (templates []string, workflows []string) { - includedTemplates := s.config.Catalog.GetTemplatesPath(s.config.Templates) +func (s *Store) Load() { + includedTemplates := s.config.Catalog.GetTemplatesPath(s.finalTemplates) includedWorkflows := s.config.Catalog.GetTemplatesPath(s.config.Workflows) excludedTemplates := s.config.Catalog.GetTemplatesPath(s.config.ExcludeTemplates) alwaysIncludeTemplates := s.config.Catalog.GetTemplatesPath(s.config.IncludeTemplates) @@ -89,7 +104,12 @@ func (s *Store) Load() (templates []string, workflows []string) { gologger.Warning().Msgf("Could not load template %s: %s\n", k, err) } if loaded { - templates = append(templates, k) + parsed, err := templates.Parse(k, s.config.ExecutorOptions) + if err != nil { + gologger.Warning().Msgf("Could not parse template %s: %s\n", k, err) + } else if parsed != nil { + s.templates = append(s.templates, parsed) + } } } @@ -110,10 +130,14 @@ func (s *Store) Load() (templates []string, workflows []string) { gologger.Warning().Msgf("Could not load workflow %s: %s\n", k, err) } if loaded { - workflows = append(workflows, k) + parsed, err := templates.Parse(k, s.config.ExecutorOptions) + if err != nil { + gologger.Warning().Msgf("Could not parse workflow %s: %s\n", k, err) + } else if parsed != nil { + s.workflows = append(s.workflows, parsed) + } } } - return templates, workflows } // loadTemplateParseMetadata loads a template by parsing metadata and running diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 9484befb..5c0a6253 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -189,49 +189,3 @@ func (t *Template) parseWorkflowTemplate(workflow *workflows.WorkflowTemplate, o } return nil } - -// matchTemplateWithTags matches if the template matches a tag -func matchTemplateWithTags(tags, severity string, tagsInput []string) error { - actualTags := strings.Split(tags, ",") - if severity != "" { - actualTags = append(actualTags, severity) // also add severity to tag - } - - matched := false -mainLoop: - for _, t := range tagsInput { - commaTags := strings.Split(t, ",") - for _, tag := range commaTags { - tag = strings.TrimSpace(tag) - key, value := getKeyValue(tag) - - for _, templTag := range actualTags { - templTag = strings.TrimSpace(templTag) - tKey, tValue := getKeyValue(templTag) - - if strings.EqualFold(key, tKey) && strings.EqualFold(value, tValue) { - matched = true - break mainLoop - } - } - } - } - if !matched { - return errors.New("could not match template tags with input") - } - return nil -} - -// getKeyValue returns key value pair for a data string -func getKeyValue(data string) (key, value string) { - if strings.Contains(data, ":") { - parts := strings.SplitN(data, ":", 2) - if len(parts) == 2 { - key, value = parts[0], parts[1] - } - } - if value == "" { - value = data - } - return key, value -} diff --git a/v2/pkg/templates/compile_test.go b/v2/pkg/templates/compile_test.go deleted file mode 100644 index 6ce605f7..00000000 --- a/v2/pkg/templates/compile_test.go +++ /dev/null @@ -1,41 +0,0 @@ -package templates - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestMatchTemplateWithTags(t *testing.T) { - err := matchTemplateWithTags("php,linux,symfony", "", []string{"php"}) - require.Nil(t, err, "could not get php tag from input slice") - - err = matchTemplateWithTags("lang:php,os:linux,cms:symfony", "", []string{"cms:symfony"}) - require.Nil(t, err, "could not get php tag from input key value") - - err = matchTemplateWithTags("lang:php,os:linux,symfony", "", []string{"cms:symfony"}) - require.NotNil(t, err, "could get key value tag from input key value") - - err = matchTemplateWithTags("lang:php,os:linux,cms:jira", "", []string{"cms:symfony"}) - require.NotNil(t, err, "could get key value tag from input key value") - - t.Run("space", func(t *testing.T) { - err = matchTemplateWithTags("lang:php, os:linux, cms:symfony", "", []string{"cms:symfony"}) - require.Nil(t, err, "could get key value tag from input key value with space") - }) - - t.Run("comma-tags", func(t *testing.T) { - err = matchTemplateWithTags("lang:php,os:linux,cms:symfony", "", []string{"test,cms:symfony"}) - require.Nil(t, err, "could get key value tag from input key value with comma") - }) - - t.Run("severity", func(t *testing.T) { - err = matchTemplateWithTags("lang:php,os:linux,cms:symfony", "low", []string{"low"}) - require.Nil(t, err, "could get key value tag for severity") - }) - - t.Run("blank-tags", func(t *testing.T) { - err = matchTemplateWithTags("", "low", []string{"jira"}) - require.NotNil(t, err, "could get value tag for blank severity") - }) -} From ba3804107ed22a8554bfe1c3bf25943593328b6c Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 1 Jul 2021 16:16:23 +0530 Subject: [PATCH 03/11] Fix some exclude conditions for tags --- v2/pkg/catalog/loader/filter.go | 24 +++++++++++++-------- v2/pkg/catalog/loader/filter_test.go | 31 +++++++++++++++++++--------- v2/pkg/catalog/loader/loader.go | 10 ++++++--- 3 files changed, 43 insertions(+), 22 deletions(-) diff --git a/v2/pkg/catalog/loader/filter.go b/v2/pkg/catalog/loader/filter.go index 0f3caf44..742b9dcd 100644 --- a/v2/pkg/catalog/loader/filter.go +++ b/v2/pkg/catalog/loader/filter.go @@ -1,6 +1,9 @@ package loader -import "strings" +import ( + "errors" + "strings" +) // tagFilter is used to filter nuclei tag based execution type tagFilter struct { @@ -11,6 +14,9 @@ type tagFilter struct { matchAllows map[string]struct{} } +// ErrExcluded is returned for execluded templates +var ErrExcluded = errors.New("the template was excluded") + // match takes a tag and whether the template was matched from user // input and returns true or false using a tag filter. // @@ -19,40 +25,40 @@ type tagFilter struct { // matchAllows section. // // It returns true if the tag is specified, or false. -func (t *tagFilter) match(tag, author, severity string, templateMatched bool) bool { +func (t *tagFilter) match(tag, author, severity string, templateMatched bool) (bool, error) { _, ok := t.block[tag] if ok { if _, allowOk := t.matchAllows[tag]; allowOk && templateMatched { - return true + return true, nil } - return false + return false, ErrExcluded } matchedAny := false if len(t.allowedTags) > 0 { _, ok = t.allowedTags[tag] if !ok { - return false + return false, nil } matchedAny = true } if len(t.authors) > 0 { _, ok = t.authors[author] if !ok { - return false + return false, nil } matchedAny = true } if len(t.severities) > 0 { _, ok = t.severities[severity] if !ok { - return false + return false, nil } matchedAny = true } if len(t.allowedTags) == 0 && len(t.authors) == 0 && len(t.severities) == 0 { - return true + return true, nil } - return matchedAny + return matchedAny, nil } // createTagFilter returns a tag filter for nuclei tag based execution diff --git a/v2/pkg/catalog/loader/filter_test.go b/v2/pkg/catalog/loader/filter_test.go index 2535d112..4e9c6c2e 100644 --- a/v2/pkg/catalog/loader/filter_test.go +++ b/v2/pkg/catalog/loader/filter_test.go @@ -13,10 +13,12 @@ func TestTagBasedFilter(t *testing.T) { filter := config.createTagFilter() t.Run("true", func(t *testing.T) { - require.True(t, filter.match("jira", "pdteam", "low", false), "could not get correct match") + matched, _ := filter.match("jira", "pdteam", "low", false) + require.True(t, matched, "could not get correct match") }) t.Run("false", func(t *testing.T) { - require.False(t, filter.match("consul", "pdteam", "low", false), "could not get correct match") + matched, _ := filter.match("consul", "pdteam", "low", false) + require.False(t, matched, "could not get correct match") }) t.Run("not-match-excludes", func(t *testing.T) { config := &Config{ @@ -24,7 +26,9 @@ func TestTagBasedFilter(t *testing.T) { ExcludeTags: []string{"dos"}, } filter := config.createTagFilter() - require.False(t, filter.match("jira", "pdteam", "low", false), "could not get correct match") + matched, err := filter.match("dos", "pdteam", "low", false) + require.False(t, matched, "could not get correct match") + require.Equal(t, ErrExcluded, err, "could not get correct error") }) t.Run("match-includes", func(t *testing.T) { config := &Config{ @@ -34,21 +38,24 @@ func TestTagBasedFilter(t *testing.T) { } filter := config.createTagFilter() - require.False(t, filter.match("fuzz", "pdteam", "low", false), "could not get correct match") + matched, _ := filter.match("fuzz", "pdteam", "low", false) + require.False(t, matched, "could not get correct match") }) t.Run("match-author", func(t *testing.T) { config := &Config{ Authors: []string{"pdteam"}, } filter := config.createTagFilter() - require.True(t, filter.match("fuzz", "pdteam", "low", false), "could not get correct match") + matched, _ := filter.match("fuzz", "pdteam", "low", false) + require.True(t, matched, "could not get correct match") }) t.Run("match-severity", func(t *testing.T) { config := &Config{ Severities: []string{"high"}, } filter := config.createTagFilter() - require.True(t, filter.match("fuzz", "pdteam", "high", false), "could not get correct match") + matched, _ := filter.match("fuzz", "pdteam", "high", false) + require.True(t, matched, "could not get correct match") }) t.Run("match-conditions", func(t *testing.T) { config := &Config{ @@ -57,9 +64,13 @@ func TestTagBasedFilter(t *testing.T) { Severities: []string{"high"}, } filter := config.createTagFilter() - require.True(t, filter.match("jira", "pdteam", "high", false), "could not get correct match") - require.False(t, filter.match("jira", "pdteam", "low", false), "could not get correct match") - require.False(t, filter.match("jira", "random", "low", false), "could not get correct match") - require.False(t, filter.match("consul", "random", "low", false), "could not get correct match") + matched, _ := filter.match("jira", "pdteam", "high", false) + require.True(t, matched, "could not get correct match") + matched, _ = filter.match("jira", "pdteam", "low", false) + require.False(t, matched, "could not get correct match") + matched, _ = filter.match("jira", "random", "low", false) + require.False(t, matched, "could not get correct match") + matched, _ = filter.match("consul", "random", "low", false) + require.False(t, matched, "could not get correct match") }) } diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go index eed2f90c..561426bd 100644 --- a/v2/pkg/catalog/loader/loader.go +++ b/v2/pkg/catalog/loader/loader.go @@ -129,6 +129,7 @@ func (s *Store) Load() { if err != nil { gologger.Warning().Msgf("Could not load workflow %s: %s\n", k, err) } + if loaded { parsed, err := templates.Parse(k, s.config.ExecutorOptions) if err != nil { @@ -181,12 +182,15 @@ func (s *Store) loadTemplateParseMetadata(templatePath string, workflow bool) (b authors := strings.Split(types.ToString(author), ",") matched := false -mainLoop: + for _, tag := range tags { for _, author := range authors { - if !matched && s.tagFilter.match(strings.TrimSpace(tag), strings.TrimSpace(author), severityStr, s.templateMatched) { + match, err := s.tagFilter.match(strings.TrimSpace(tag), strings.TrimSpace(author), severityStr, s.templateMatched) + if err == ErrExcluded { + return false, ErrExcluded + } + if !matched && match && err == nil { matched = true - break mainLoop } } } From 6bc0b343548b3f5e4c26f587eb1bd159f21d8615 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 1 Jul 2021 16:29:26 +0530 Subject: [PATCH 04/11] Fixed include condition logic --- v2/pkg/catalog/loader/filter.go | 19 ++++++++++--------- v2/pkg/catalog/loader/filter_test.go | 23 ++++++++++++----------- v2/pkg/catalog/loader/loader.go | 14 ++++++-------- 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/v2/pkg/catalog/loader/filter.go b/v2/pkg/catalog/loader/filter.go index 742b9dcd..cf6b92ee 100644 --- a/v2/pkg/catalog/loader/filter.go +++ b/v2/pkg/catalog/loader/filter.go @@ -25,10 +25,10 @@ var ErrExcluded = errors.New("the template was excluded") // matchAllows section. // // It returns true if the tag is specified, or false. -func (t *tagFilter) match(tag, author, severity string, templateMatched bool) (bool, error) { +func (t *tagFilter) match(tag, author, severity string) (bool, error) { _, ok := t.block[tag] if ok { - if _, allowOk := t.matchAllows[tag]; allowOk && templateMatched { + if _, allowOk := t.matchAllows[tag]; allowOk { return true, nil } return false, ErrExcluded @@ -93,13 +93,6 @@ func (config *Config) createTagFilter() *tagFilter { } } } - for _, tag := range config.IncludeTags { - for _, val := range splitCommaTrim(tag) { - if _, ok := filter.matchAllows[val]; !ok { - filter.matchAllows[val] = struct{}{} - } - } - } for _, tag := range config.ExcludeTags { for _, val := range splitCommaTrim(tag) { if _, ok := filter.block[val]; !ok { @@ -107,6 +100,14 @@ func (config *Config) createTagFilter() *tagFilter { } } } + for _, tag := range config.IncludeTags { + for _, val := range splitCommaTrim(tag) { + if _, ok := filter.matchAllows[val]; !ok { + filter.matchAllows[val] = struct{}{} + } + delete(filter.block, val) + } + } return filter } diff --git a/v2/pkg/catalog/loader/filter_test.go b/v2/pkg/catalog/loader/filter_test.go index 4e9c6c2e..a0df7e95 100644 --- a/v2/pkg/catalog/loader/filter_test.go +++ b/v2/pkg/catalog/loader/filter_test.go @@ -13,11 +13,11 @@ func TestTagBasedFilter(t *testing.T) { filter := config.createTagFilter() t.Run("true", func(t *testing.T) { - matched, _ := filter.match("jira", "pdteam", "low", false) + matched, _ := filter.match("jira", "pdteam", "low") require.True(t, matched, "could not get correct match") }) t.Run("false", func(t *testing.T) { - matched, _ := filter.match("consul", "pdteam", "low", false) + matched, _ := filter.match("consul", "pdteam", "low") require.False(t, matched, "could not get correct match") }) t.Run("not-match-excludes", func(t *testing.T) { @@ -26,7 +26,7 @@ func TestTagBasedFilter(t *testing.T) { ExcludeTags: []string{"dos"}, } filter := config.createTagFilter() - matched, err := filter.match("dos", "pdteam", "low", false) + matched, err := filter.match("dos", "pdteam", "low") require.False(t, matched, "could not get correct match") require.Equal(t, ErrExcluded, err, "could not get correct error") }) @@ -38,15 +38,16 @@ func TestTagBasedFilter(t *testing.T) { } filter := config.createTagFilter() - matched, _ := filter.match("fuzz", "pdteam", "low", false) - require.False(t, matched, "could not get correct match") + matched, err := filter.match("fuzz", "pdteam", "low") + require.Nil(t, err, "could not get match") + require.True(t, matched, "could not get correct match") }) t.Run("match-author", func(t *testing.T) { config := &Config{ Authors: []string{"pdteam"}, } filter := config.createTagFilter() - matched, _ := filter.match("fuzz", "pdteam", "low", false) + matched, _ := filter.match("fuzz", "pdteam", "low") require.True(t, matched, "could not get correct match") }) t.Run("match-severity", func(t *testing.T) { @@ -54,7 +55,7 @@ func TestTagBasedFilter(t *testing.T) { Severities: []string{"high"}, } filter := config.createTagFilter() - matched, _ := filter.match("fuzz", "pdteam", "high", false) + matched, _ := filter.match("fuzz", "pdteam", "high") require.True(t, matched, "could not get correct match") }) t.Run("match-conditions", func(t *testing.T) { @@ -64,13 +65,13 @@ func TestTagBasedFilter(t *testing.T) { Severities: []string{"high"}, } filter := config.createTagFilter() - matched, _ := filter.match("jira", "pdteam", "high", false) + matched, _ := filter.match("jira", "pdteam", "high") require.True(t, matched, "could not get correct match") - matched, _ = filter.match("jira", "pdteam", "low", false) + matched, _ = filter.match("jira", "pdteam", "low") require.False(t, matched, "could not get correct match") - matched, _ = filter.match("jira", "random", "low", false) + matched, _ = filter.match("jira", "random", "low") require.False(t, matched, "could not get correct match") - matched, _ = filter.match("consul", "random", "low", false) + matched, _ = filter.match("consul", "random", "low") require.False(t, matched, "could not get correct match") }) } diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go index 561426bd..6e46873e 100644 --- a/v2/pkg/catalog/loader/loader.go +++ b/v2/pkg/catalog/loader/loader.go @@ -35,10 +35,9 @@ type Config struct { // Store is a storage for loaded nuclei templates type Store struct { - tagFilter *tagFilter - config *Config - finalTemplates []string - templateMatched bool + tagFilter *tagFilter + config *Config + finalTemplates []string templates []*templates.Template workflows []*templates.Template @@ -55,8 +54,6 @@ func New(config *Config) (*Store, error) { // Handle a case with no templates or workflows, where we use base directory if len(config.Templates) == 0 && len(config.Workflows) == 0 { config.Templates = append(config.Templates, config.TemplatesDirectory) - } else { - store.templateMatched = true } store.finalTemplates = append(store.finalTemplates, config.Templates...) @@ -169,8 +166,9 @@ func (s *Store) loadTemplateParseMetadata(templatePath string, workflow bool) (b } severity, ok := template.Info["severity"] if !ok { - return false, errors.New("no template severity field provided") + severity = "" } + templateTags, ok := template.Info["tags"] if !ok { templateTags = "" @@ -185,7 +183,7 @@ func (s *Store) loadTemplateParseMetadata(templatePath string, workflow bool) (b for _, tag := range tags { for _, author := range authors { - match, err := s.tagFilter.match(strings.TrimSpace(tag), strings.TrimSpace(author), severityStr, s.templateMatched) + match, err := s.tagFilter.match(strings.TrimSpace(tag), strings.TrimSpace(author), severityStr) if err == ErrExcluded { return false, ErrExcluded } From e440ab5cbab1aa63a018fb37e0e4699789a73ce1 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 1 Jul 2021 16:39:00 +0530 Subject: [PATCH 05/11] Fixed nuclei/templates version check --- v2/internal/runner/update.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go index 196bdc8e..64891297 100644 --- a/v2/internal/runner/update.go +++ b/v2/internal/runner/update.go @@ -530,5 +530,5 @@ func (r *Runner) githubFetchLatestTagRepo(repo string) (string, error) { if len(tags) == 0 { return "", fmt.Errorf("no tags found for %s", repo) } - return tags[0].Name, nil + return strings.TrimPrefix(tags[0].Name, "v"), nil } From d58180d588c1714df7a8787bbb12e9199ef1610c Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 1 Jul 2021 18:22:08 +0530 Subject: [PATCH 06/11] Added vv flag to display more extra verbose information --- v2/cmd/nuclei/main.go | 1 + v2/internal/runner/runner.go | 2 +- v2/pkg/types/types.go | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index f402b0af..20c2a29b 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -96,6 +96,7 @@ based on templates offering massive extensibility and ease of use.`) set.IntVar(&options.InteractionsEviction, "interactions-eviction", 60, "Number of seconds to wait before evicting requests from cache") set.IntVar(&options.InteractionsPollDuration, "interactions-poll-duration", 5, "Number of seconds before each interaction poll request") set.IntVar(&options.InteractionsColldownPeriod, "interactions-cooldown-period", 5, "Extra time for interaction polling before exiting") + set.BoolVar(&options.VerboseVerbose, "vv", false, "Display Extra Verbose Information") _ = set.Parse() if cfgFile != "" { diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index b14d2d7c..40bc3c3c 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -345,7 +345,7 @@ func (r *Runner) RunEnumeration() { unclusteredRequests += int64(template.TotalRequests) * r.inputCount } - if r.options.Verbose { + if r.options.VerboseVerbose { for _, template := range store.Templates() { r.logAvailableTemplate(template.Path) } diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index b26385eb..2662a1e4 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -107,7 +107,8 @@ type Options struct { // Version specifies if we should just show version and exit Version bool // Verbose flag indicates whether to show verbose output or not - Verbose bool + Verbose bool + VerboseVerbose bool // No-Color disables the colored output. NoColor bool // UpdateTemplates updates the templates installed at startup From 98e82c5f112daf4cd67d3978ae10779de7311131 Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Thu, 1 Jul 2021 20:35:40 +0530 Subject: [PATCH 07/11] Update config.go --- v2/pkg/catalog/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/catalog/config/config.go b/v2/pkg/catalog/config/config.go index a616c319..b6d2c157 100644 --- a/v2/pkg/catalog/config/config.go +++ b/v2/pkg/catalog/config/config.go @@ -27,7 +27,7 @@ type Config struct { const nucleiConfigFilename = ".templates-config.json" // Version is the current version of nuclei -const Version = `2.3.8` +const Version = `2.4.0-dev` var ( homeDir string From 12deece54b9434609fbbe8621435249021987fd4 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 1 Jul 2021 20:57:22 +0530 Subject: [PATCH 08/11] Fixed bug with .nuclei-config check --- v2/internal/runner/update.go | 6 +++++- v2/pkg/catalog/config/config.go | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go index 64891297..a120a6b9 100644 --- a/v2/internal/runner/update.go +++ b/v2/internal/runner/update.go @@ -108,7 +108,10 @@ func (r *Runner) updateTemplates() error { _ = ioutil.WriteFile(path.Join(configDir, nucleiIgnoreFile), data, 0644) } if r.templatesConfig != nil { - r.templatesConfig.LastCheckedIgnore = time.Now() + err = config.WriteConfiguration(r.templatesConfig, false, true) + if err != nil { + gologger.Warning().Msgf("Could not get ignore-file from %s: %s", ignoreURL, err) + } } } } @@ -502,6 +505,7 @@ type githubTagData struct { } // githubFetchLatestTagRepo fetches latest tag from github +// This function was half written by github copilot AI :D. func (r *Runner) githubFetchLatestTagRepo(repo string) (string, error) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() diff --git a/v2/pkg/catalog/config/config.go b/v2/pkg/catalog/config/config.go index a616c319..b8ab4623 100644 --- a/v2/pkg/catalog/config/config.go +++ b/v2/pkg/catalog/config/config.go @@ -70,7 +70,7 @@ func WriteConfiguration(config *Config, checked, checkedIgnore bool) error { config.LastCheckedIgnore = time.Now() } config.NucleiVersion = Version - file, err := os.OpenFile(templatesConfigFile, os.O_WRONLY|os.O_CREATE, 0777) + file, err := os.OpenFile(templatesConfigFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0777) if err != nil { return err } From d57d1ecae277a1efac605b88e5078fae5fee25b1 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 1 Jul 2021 21:02:57 +0530 Subject: [PATCH 09/11] Fixed filter allow condition with tag and etags --- v2/pkg/catalog/loader/filter.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/v2/pkg/catalog/loader/filter.go b/v2/pkg/catalog/loader/filter.go index cf6b92ee..17f0142b 100644 --- a/v2/pkg/catalog/loader/filter.go +++ b/v2/pkg/catalog/loader/filter.go @@ -26,6 +26,14 @@ var ErrExcluded = errors.New("the template was excluded") // // It returns true if the tag is specified, or false. func (t *tagFilter) match(tag, author, severity string) (bool, error) { + matchedAny := false + if len(t.allowedTags) > 0 { + _, ok := t.allowedTags[tag] + if !ok { + return false, nil + } + matchedAny = true + } _, ok := t.block[tag] if ok { if _, allowOk := t.matchAllows[tag]; allowOk { @@ -33,14 +41,6 @@ func (t *tagFilter) match(tag, author, severity string) (bool, error) { } return false, ErrExcluded } - matchedAny := false - if len(t.allowedTags) > 0 { - _, ok = t.allowedTags[tag] - if !ok { - return false, nil - } - matchedAny = true - } if len(t.authors) > 0 { _, ok = t.authors[author] if !ok { @@ -77,6 +77,7 @@ func (config *Config) createTagFilter() *tagFilter { if _, ok := filter.allowedTags[val]; !ok { filter.allowedTags[val] = struct{}{} } + delete(filter.block, val) } } for _, tag := range config.Severities { From cddb0cb8af74a680646228db2179ec2bd00d7c85 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 1 Jul 2021 21:09:33 +0530 Subject: [PATCH 10/11] Fixed exclude filter for tags --- v2/pkg/catalog/loader/filter.go | 14 +++++++------- v2/pkg/catalog/loader/filter_test.go | 11 ++++++++++- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/v2/pkg/catalog/loader/filter.go b/v2/pkg/catalog/loader/filter.go index 17f0142b..fc71f8eb 100644 --- a/v2/pkg/catalog/loader/filter.go +++ b/v2/pkg/catalog/loader/filter.go @@ -72,12 +72,11 @@ func (config *Config) createTagFilter() *tagFilter { block: make(map[string]struct{}), matchAllows: make(map[string]struct{}), } - for _, tag := range config.Tags { + for _, tag := range config.ExcludeTags { for _, val := range splitCommaTrim(tag) { - if _, ok := filter.allowedTags[val]; !ok { - filter.allowedTags[val] = struct{}{} + if _, ok := filter.block[val]; !ok { + filter.block[val] = struct{}{} } - delete(filter.block, val) } } for _, tag := range config.Severities { @@ -94,11 +93,12 @@ func (config *Config) createTagFilter() *tagFilter { } } } - for _, tag := range config.ExcludeTags { + for _, tag := range config.Tags { for _, val := range splitCommaTrim(tag) { - if _, ok := filter.block[val]; !ok { - filter.block[val] = struct{}{} + if _, ok := filter.allowedTags[val]; !ok { + filter.allowedTags[val] = struct{}{} } + delete(filter.block, val) } } for _, tag := range config.IncludeTags { diff --git a/v2/pkg/catalog/loader/filter_test.go b/v2/pkg/catalog/loader/filter_test.go index a0df7e95..af0a857b 100644 --- a/v2/pkg/catalog/loader/filter_test.go +++ b/v2/pkg/catalog/loader/filter_test.go @@ -22,7 +22,6 @@ func TestTagBasedFilter(t *testing.T) { }) t.Run("not-match-excludes", func(t *testing.T) { config := &Config{ - Tags: []string{"cves", "dos"}, ExcludeTags: []string{"dos"}, } filter := config.createTagFilter() @@ -42,6 +41,16 @@ func TestTagBasedFilter(t *testing.T) { require.Nil(t, err, "could not get match") require.True(t, matched, "could not get correct match") }) + t.Run("match-includes", func(t *testing.T) { + config := &Config{ + Tags: []string{"fuzz"}, + ExcludeTags: []string{"fuzz"}, + } + filter := config.createTagFilter() + matched, err := filter.match("fuzz", "pdteam", "low") + require.Nil(t, err, "could not get match") + require.True(t, matched, "could not get correct match") + }) t.Run("match-author", func(t *testing.T) { config := &Config{ Authors: []string{"pdteam"}, From 714aec12191914e8c36c49fcf432ad200687d213 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Fri, 2 Jul 2021 15:46:42 +0530 Subject: [PATCH 11/11] Fixed lint tests --- v2/pkg/catalog/loader/loader_test.go | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/v2/pkg/catalog/loader/loader_test.go b/v2/pkg/catalog/loader/loader_test.go index 8d586eae..b9f6ab43 100644 --- a/v2/pkg/catalog/loader/loader_test.go +++ b/v2/pkg/catalog/loader/loader_test.go @@ -3,41 +3,38 @@ package loader import ( "testing" - "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/stretchr/testify/require" ) func TestLoadTemplates(t *testing.T) { - config, err := config.ReadConfiguration() - require.Nil(t, err, "could not read configuration") - store, err := New(&Config{ Templates: []string{"cves/CVE-2021-21315.yaml"}, }) require.Nil(t, err, "could not load templates") require.Equal(t, []string{"cves/CVE-2021-21315.yaml"}, store.finalTemplates, "could not get correct templates") + templatesDirectory := "/test" t.Run("blank", func(t *testing.T) { store, err := New(&Config{ - TemplatesDirectory: config.TemplatesDirectory, + TemplatesDirectory: templatesDirectory, }) require.Nil(t, err, "could not load templates") - require.Equal(t, []string{config.TemplatesDirectory}, store.finalTemplates, "could not get correct templates") + require.Equal(t, []string{templatesDirectory}, store.finalTemplates, "could not get correct templates") }) t.Run("only-tags", func(t *testing.T) { store, err := New(&Config{ Tags: []string{"cves"}, - TemplatesDirectory: config.TemplatesDirectory, + TemplatesDirectory: templatesDirectory, }) require.Nil(t, err, "could not load templates") - require.Equal(t, []string{config.TemplatesDirectory}, store.finalTemplates, "could not get correct templates") + require.Equal(t, []string{templatesDirectory}, store.finalTemplates, "could not get correct templates") }) t.Run("tags-with-path", func(t *testing.T) { store, err := New(&Config{ Tags: []string{"cves"}, - TemplatesDirectory: config.TemplatesDirectory, + TemplatesDirectory: templatesDirectory, }) require.Nil(t, err, "could not load templates") - require.Equal(t, []string{config.TemplatesDirectory}, store.finalTemplates, "could not get correct templates") + require.Equal(t, []string{templatesDirectory}, store.finalTemplates, "could not get correct templates") }) }