daffainfo
/
nuclei
mirror of https://github.com/daffainfo/nuclei.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #799 from projectdiscovery/loader-fix 

[wip] Nuclei Templates Loader Rewrite
dev
Ice3man 2021-07-02 15:56:13 +05:30 committed by GitHub
parent f61d420093 3dbf51f1a3
commit c731e7f927
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 817 additions and 497 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
v2/cmd/nuclei/main.go
View File

@ -42,7 +42,10 @@ based on templates offering massive extensibility and ease of use.`)
set.StringSliceVarP(&options.Templates, "templates", "t", []string{}, "Templates to run, supports single and multiple templates using directory.")
set.StringSliceVarP(&options.Workflows, "workflows", "w", []string{}, "Workflows to run for nuclei")
set.StringSliceVarP(&options.ExcludedTemplates, "exclude", "et", []string{}, "Templates to exclude, supports single and multiple templates using directory.")
set.StringSliceVarP(&options.Severity, "severity", "impact", []string{}, "Templates to run based on severity, supports single and multiple severity.")
set.StringSliceVarP(&options.Severity, "severity", "impact", []string{}, "Templates to run based on severity")
set.StringSliceVar(&options.Author, "author", []string{}, "Templates to run based on author")
set.StringSliceVar(&options.IncludeTemplates, "include-templates", []string{}, "Templates to force run even if they are in denylist")
set.StringSliceVar(&options.IncludeTags, "include-tags", []string{}, "Tags to force run even if they are in denylist")
set.StringVarP(&options.Targets, "list", "l", "", "List of URLs to run templates on")
set.StringVarP(&options.Output, "output", "o", "", "File to write output to (optional)")
set.StringVar(&options.ProxyURL, "proxy-url", "", "URL of the proxy server")
@ -92,6 +95,7 @@ based on templates offering massive extensibility and ease of use.`)
set.IntVar(&options.InteractionsEviction, "interactions-eviction", 60, "Number of seconds to wait before evicting requests from cache")
set.IntVar(&options.InteractionsPollDuration, "interactions-poll-duration", 5, "Number of seconds before each interaction poll request")
set.IntVar(&options.InteractionsColldownPeriod, "interactions-cooldown-period", 5, "Extra time for interaction polling before exiting")
set.BoolVar(&options.VerboseVerbose, "vv", false, "Display Extra Verbose Information")
_ = set.Parse()
if cfgFile != "" {

20
v2/internal/runner/banner.go
View File

@ -1,17 +1,23 @@
package runner
import "github.com/projectdiscovery/gologger"
import (
"fmt"
const banner = `
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
)
var banner string
func init() {
banner = fmt.Sprintf(`
__ _
____ __ _______/ /__ (_)
/ __ \/ / / / ___/ / _ \/ /
/ / / / /_/ / /__/ / __/ /
/_/ /_/\__,_/\___/_/\___/_/ v2.3.8
`
// Version is the current version of nuclei
const Version = `2.3.8`
/_/ /_/\__,_/\___/_/\___/_/ %s
`, config.Version)
}
// showBanner is used to show the banner to the user
func showBanner() {

126
v2/internal/runner/config.go
View File

@ -1,127 +1 @@
package runner
import (
"os"
"path"
"regexp"
"time"
jsoniter "github.com/json-iterator/go"
"github.com/projectdiscovery/gologger"
"gopkg.in/yaml.v2"
)
// nucleiConfig contains some configuration options for nuclei
type nucleiConfig struct {
TemplatesDirectory string `json:"templates-directory,omitempty"`
CurrentVersion string `json:"current-version,omitempty"`
LastChecked time.Time `json:"last-checked,omitempty"`
IgnoreURL string `json:"ignore-url,omitempty"`
NucleiVersion string `json:"nuclei-version,omitempty"`
LastCheckedIgnore time.Time `json:"last-checked-ignore,omitempty"`
// IgnorePaths ignores all the paths listed unless specified manually
IgnorePaths []string `json:"ignore-paths,omitempty"`
}
// nucleiConfigFilename is the filename of nuclei configuration file.
const nucleiConfigFilename = ".templates-config.json"
var reVersion = regexp.MustCompile(`\d+\.\d+\.\d+`)
// readConfiguration reads the nuclei configuration file from disk.
func readConfiguration() (*nucleiConfig, error) {
home, err := os.UserHomeDir()
if err != nil {
return nil, err
}
configDir := path.Join(home, "/.config", "/nuclei")
_ = os.MkdirAll(configDir, os.ModePerm)
templatesConfigFile := path.Join(configDir, nucleiConfigFilename)
file, err := os.Open(templatesConfigFile)
if err != nil {
return nil, err
}
defer file.Close()
config := &nucleiConfig{}
err = jsoniter.NewDecoder(file).Decode(config)
if err != nil {
return nil, err
}
return config, nil
}
// readConfiguration reads the nuclei configuration file from disk.
func (r *Runner) writeConfiguration(config *nucleiConfig) error {
home, err := os.UserHomeDir()
if err != nil {
return err
}
configDir := path.Join(home, "/.config", "/nuclei")
_ = os.MkdirAll(configDir, os.ModePerm)
if config.IgnoreURL == "" {
config.IgnoreURL = "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore"
}
config.LastChecked = time.Now()
config.LastCheckedIgnore = time.Now()
config.NucleiVersion = Version
templatesConfigFile := path.Join(configDir, nucleiConfigFilename)
file, err := os.OpenFile(templatesConfigFile, os.O_WRONLY|os.O_CREATE, 0777)
if err != nil {
return err
}
defer file.Close()
err = jsoniter.NewEncoder(file).Encode(config)
if err != nil {
return err
}
return nil
}
const nucleiIgnoreFile = ".nuclei-ignore"
type ignoreFile struct {
Tags []string `yaml:"tags"`
Files []string `yaml:"files"`
}
// readNucleiIgnoreFile reads the nuclei ignore file marking it in map
func (r *Runner) readNucleiIgnoreFile() {
file, err := os.Open(r.getIgnoreFilePath())
if err != nil {
gologger.Error().Msgf("Could not read nuclei-ignore file: %s\n", err)
return
}
defer file.Close()
ignore := &ignoreFile{}
if err := yaml.NewDecoder(file).Decode(ignore); err != nil {
gologger.Error().Msgf("Could not parse nuclei-ignore file: %s\n", err)
return
}
r.options.ExcludeTags = append(r.options.ExcludeTags, ignore.Tags...)
r.templatesConfig.IgnorePaths = append(r.templatesConfig.IgnorePaths, ignore.Files...)
}
// getIgnoreFilePath returns the ignore file path for the runner
func (r *Runner) getIgnoreFilePath() string {
var defIgnoreFilePath string
home, err := os.UserHomeDir()
if err == nil {
configDir := path.Join(home, "/.config", "/nuclei")
_ = os.MkdirAll(configDir, os.ModePerm)
defIgnoreFilePath = path.Join(configDir, nucleiIgnoreFile)
return defIgnoreFilePath
}
cwd, err := os.Getwd()
if err != nil {
return defIgnoreFilePath
}
cwdIgnoreFilePath := path.Join(cwd, nucleiIgnoreFile)
return cwdIgnoreFilePath
}

12
v2/internal/runner/options.go
View File

@ -10,6 +10,7 @@ import (
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/gologger/formatter"
"github.com/projectdiscovery/gologger/levels"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
)
@ -26,11 +27,11 @@ func ParseOptions(options *types.Options) {
showBanner()
if options.Version {
gologger.Info().Msgf("Current Version: %s\n", Version)
gologger.Info().Msgf("Current Version: %s\n", config.Version)
os.Exit(0)
}
if options.TemplatesVersion {
config, err := readConfiguration()
config, err := config.ReadConfiguration()
if err != nil {
gologger.Fatal().Msgf("Could not read template configuration: %s\n", err)
}
@ -80,13 +81,6 @@ func validateOptions(options *types.Options) error {
return errors.New("both verbose and silent mode specified")
}
if !options.TemplateList {
// Check if a list of templates was provided and it exists
if len(options.Templates) == 0 && !options.NewTemplates && len(options.Workflows) == 0 && len(options.Tags) == 0 && !options.UpdateTemplates {
return errors.New("no template/templates provided")
}
}
// Validate proxy options if provided
err := validateProxyURL(options.ProxyURL, "invalid http proxy format (It should be http://username:password@host:port)")
if err != nil {

132
v2/internal/runner/runner.go
View File

@ -13,6 +13,8 @@ import (
"github.com/projectdiscovery/hmap/store/hybrid"
"github.com/projectdiscovery/nuclei/v2/internal/colorizer"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/progress"
"github.com/projectdiscovery/nuclei/v2/pkg/projectfile"
@ -39,7 +41,7 @@ type Runner struct {
output output.Writer
interactsh *interactsh.Client
inputCount int64
templatesConfig *nucleiConfig
templatesConfig *config.Config
options *types.Options
projectFile *projectfile.ProjectFile
catalog *catalog.Catalog
@ -68,11 +70,6 @@ func New(options *types.Options) (*Runner, error) {
}
runner.catalog = catalog.New(runner.options.TemplatesDirectory)
// Read nucleiignore file if given a templateconfig
if runner.templatesConfig != nil {
runner.readNucleiIgnoreFile()
runner.catalog.AppendIgnore(runner.templatesConfig.IgnorePaths)
}
var reportingOptions *reporting.Options
if options.ReportingConfig != "" {
file, err := os.Open(options.ReportingConfig)
@ -251,10 +248,7 @@ func (r *Runner) Close() {
func (r *Runner) RunEnumeration() {
defer r.Close()
// If we have no templates, run on whole template directory with provided tags
if len(r.options.Templates) == 0 && len(r.options.Workflows) == 0 && !r.options.NewTemplates && (len(r.options.Tags) > 0 || len(r.options.ExcludeTags) > 0) {
r.options.Templates = append(r.options.Templates, r.options.TemplatesDirectory)
}
// If user asked for new templates to be executed, collect the list from template directory.
if r.options.NewTemplates {
templatesLoaded, err := r.readNewTemplatesFile()
if err != nil {
@ -262,37 +256,87 @@ func (r *Runner) RunEnumeration() {
}
r.options.Templates = append(r.options.Templates, templatesLoaded...)
}
includedTemplates := r.catalog.GetTemplatesPath(r.options.Templates, false)
excludedTemplates := r.catalog.GetTemplatesPath(r.options.ExcludedTemplates, true)
// defaults to all templates
allTemplates := includedTemplates
ignoreFile := config.ReadIgnoreFile()
r.options.ExcludeTags = append(r.options.ExcludeTags, ignoreFile.Tags...)
r.options.ExcludedTemplates = append(r.options.ExcludedTemplates, ignoreFile.Files...)
if len(excludedTemplates) > 0 {
excludedMap := make(map[string]struct{}, len(excludedTemplates))
for _, excl := range excludedTemplates {
excludedMap[excl] = struct{}{}
executerOpts := protocols.ExecuterOptions{
Output: r.output,
Options: r.options,
Progress: r.progress,
Catalog: r.catalog,
IssuesClient: r.issuesClient,
RateLimiter: r.ratelimiter,
Interactsh: r.interactsh,
ProjectFile: r.projectFile,
Browser: r.browser,
}
// rebuild list with only non-excluded templates
allTemplates = []string{}
loaderConfig := &loader.Config{
Templates: r.options.Templates,
Workflows: r.options.Workflows,
ExcludeTemplates: r.options.ExcludedTemplates,
Tags: r.options.Tags,
ExcludeTags: r.options.ExcludeTags,
IncludeTemplates: r.options.IncludeTemplates,
Authors: r.options.Author,
Severities: r.options.Severity,
IncludeTags: r.options.IncludeTags,
TemplatesDirectory: r.options.TemplatesDirectory,
Catalog: r.catalog,
ExecutorOptions: executerOpts,
}
store, err := loader.New(loaderConfig)
if err != nil {
gologger.Fatal().Msgf("Could not load templates from config: %s\n", err)
}
store.Load()
for _, incl := range includedTemplates {
if _, found := excludedMap[incl]; !found {
allTemplates = append(allTemplates, incl)
builder := &strings.Builder{}
if r.templatesConfig != nil && r.templatesConfig.NucleiLatestVersion != "" {
builder.WriteString(" (")
if config.Version == r.templatesConfig.NucleiLatestVersion {
builder.WriteString(r.colorizer.Green("latest").String())
} else {
gologger.Warning().Msgf("Excluding '%s'", incl)
builder.WriteString(r.colorizer.Red("outdated").String())
}
builder.WriteString(")")
}
messageStr := builder.String()
builder.Reset()
gologger.Info().Msgf("Using Nuclei Engine %s%s", config.Version, messageStr)
if r.templatesConfig != nil && r.templatesConfig.NucleiTemplatesLatestVersion != "" {
builder.WriteString(" (")
if r.templatesConfig.CurrentVersion == r.templatesConfig.NucleiTemplatesLatestVersion {
builder.WriteString(r.colorizer.Green("latest").String())
} else {
builder.WriteString(r.colorizer.Red("outdated").String())
}
builder.WriteString(")")
}
messageStr = builder.String()
builder.Reset()
gologger.Info().Msgf("Using Nuclei Templates %s%s", r.templatesConfig.CurrentVersion, messageStr)
if r.interactsh != nil {
gologger.Info().Msgf("Using Interactsh Server %s", r.options.InteractshURL)
}
if len(store.Templates()) > 0 {
gologger.Info().Msgf("Running Nuclei Templates (%d)", len(store.Templates()))
}
if len(store.Workflows()) > 0 {
gologger.Info().Msgf("Running Nuclei Workflows (%d)", len(store.Workflows()))
}
// pre-parse all the templates, apply filters
finalTemplates := []*templates.Template{}
workflowPaths := r.catalog.GetTemplatesPath(r.options.Workflows, false)
availableTemplates, _ := r.getParsedTemplatesFor(allTemplates, r.options.Severity, false)
availableWorkflows, workflowCount := r.getParsedTemplatesFor(workflowPaths, r.options.Severity, true)
var unclusteredRequests int64
for _, template := range availableTemplates {
var unclusteredRequests int64 = 0
for _, template := range store.Templates() {
// workflows will dynamically adjust the totals while running, as
// it can't be know in advance which requests will be called
if len(template.Workflows) > 0 {
@ -301,9 +345,21 @@ func (r *Runner) RunEnumeration() {
unclusteredRequests += int64(template.TotalRequests) * r.inputCount
}
originalTemplatesCount := len(availableTemplates)
if r.options.VerboseVerbose {
for _, template := range store.Templates() {
r.logAvailableTemplate(template.Path)
}
for _, template := range store.Workflows() {
r.logAvailableTemplate(template.Path)
}
}
templatesMap := make(map[string]*templates.Template)
for _, v := range store.Templates() {
templatesMap[v.ID] = v
}
originalTemplatesCount := len(store.Templates())
clusterCount := 0
clusters := clusterer.Cluster(availableTemplates)
clusters := clusterer.Cluster(templatesMap)
for _, cluster := range clusters {
if len(cluster) > 1 && !r.options.OfflineHTTP {
executerOpts := protocols.ExecuterOptions{
@ -330,7 +386,7 @@ func (r *Runner) RunEnumeration() {
finalTemplates = append(finalTemplates, cluster...)
}
}
for _, workflows := range availableWorkflows {
for _, workflows := range store.Workflows() {
finalTemplates = append(finalTemplates, workflows)
}
@ -342,20 +398,16 @@ func (r *Runner) RunEnumeration() {
totalRequests += int64(t.TotalRequests) * r.inputCount
}
if totalRequests < unclusteredRequests {
gologger.Info().Msgf("Reduced %d requests to %d (%d templates clustered)", unclusteredRequests, totalRequests, clusterCount)
gologger.Info().Msgf("Reduced %d requests (%d templates clustered)", unclusteredRequests-totalRequests, clusterCount)
}
templateCount := originalTemplatesCount + len(availableWorkflows)
workflowCount := len(store.Workflows())
templateCount := originalTemplatesCount + workflowCount
// 0 matches means no templates were found in directory
if templateCount == 0 {
gologger.Fatal().Msgf("Error, no templates were found.\n")
}
gologger.Info().Msgf("Using %s rules (%s templates, %s workflows)",
r.colorizer.Bold(templateCount).String(),
r.colorizer.Bold(templateCount-workflowCount).String(),
r.colorizer.Bold(workflowCount).String())
results := &atomic.Bool{}
wgtemplates := sizedwaitgroup.New(r.options.TemplateThreads)

101
v2/internal/runner/templates.go
View File

@ -1,77 +1,36 @@
package runner
import (
"bytes"
"fmt"
"io/ioutil"
"os"
"strings"
"github.com/karrick/godirwalk"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
"gopkg.in/yaml.v2"
)
// getParsedTemplatesFor parse the specified templates and returns a slice of the parsable ones, optionally filtered
// by severity, along with a flag indicating if workflows are present.
func (r *Runner) getParsedTemplatesFor(templatePaths, severities []string, workflows bool) (parsedTemplates map[string]*templates.Template, workflowCount int) {
filterBySeverity := len(severities) > 0
if !workflows {
gologger.Info().Msgf("Loading templates...")
} else {
gologger.Info().Msgf("Loading workflows...")
}
parsedTemplates = make(map[string]*templates.Template)
for _, match := range templatePaths {
t, err := r.parseTemplateFile(match)
if err != nil {
gologger.Warning().Msgf("Could not parse file '%s': %s\n", match, err)
continue
}
if t == nil {
continue
}
if len(t.Workflows) == 0 && workflows {
continue // don't print if user only wants to run workflows
}
if len(t.Workflows) > 0 && !workflows {
continue // don't print workflow if user only wants to run templates
}
if len(t.Workflows) > 0 {
workflowCount++
}
sev := strings.ToLower(types.ToString(t.Info["severity"]))
if !filterBySeverity || hasMatchingSeverity(sev, severities) {
parsedTemplates[t.ID] = t
gologger.Info().Msgf("%s\n", r.templateLogMsg(t.ID, types.ToString(t.Info["name"]), types.ToString(t.Info["author"]), sev))
} else {
gologger.Warning().Msgf("Excluding template %s due to severity filter (%s not in [%s])", t.ID, sev, severities)
}
}
return parsedTemplates, workflowCount
}
// parseTemplateFile returns the parsed template file
func (r *Runner) parseTemplateFile(file string) (*templates.Template, error) {
executerOpts := protocols.ExecuterOptions{
Output: r.output,
Options: r.options,
Progress: r.progress,
Catalog: r.catalog,
IssuesClient: r.issuesClient,
RateLimiter: r.ratelimiter,
Interactsh: r.interactsh,
ProjectFile: r.projectFile,
Browser: r.browser,
}
template, err := templates.Parse(file, executerOpts)
f, err := os.Open(file)
if err != nil {
return nil, err
}
if template == nil {
return nil, nil
defer f.Close()
data, err := ioutil.ReadAll(f)
if err != nil {
return nil, err
}
template := &templates.Template{}
err = yaml.NewDecoder(bytes.NewReader(data)).Decode(template)
if err != nil {
return nil, err
}
return template, nil
}
@ -93,7 +52,7 @@ func (r *Runner) logAvailableTemplate(tplPath string) {
if err != nil {
gologger.Error().Msgf("Could not parse file '%s': %s\n", tplPath, err)
} else {
gologger.Print().Msgf("%s\n", r.templateLogMsg(t.ID, types.ToString(t.Info["name"]), types.ToString(t.Info["author"]), types.ToString(t.Info["severity"])))
gologger.Info().Msgf("%s\n", r.templateLogMsg(t.ID, types.ToString(t.Info["name"]), types.ToString(t.Info["author"]), types.ToString(t.Info["severity"])))
}
}
@ -130,38 +89,12 @@ func (r *Runner) listAvailableTemplates() {
}
}
func hasMatchingSeverity(templateSeverity string, allowedSeverities []string) bool {
for _, s := range allowedSeverities {
finalSeverities := []string{}
if strings.Contains(s, ",") {
finalSeverities = strings.Split(s, ",")
} else {
finalSeverities = append(finalSeverities, s)
}
for _, sev := range finalSeverities {
sev = strings.ToLower(sev)
if sev != "" && strings.HasPrefix(templateSeverity, sev) {
return true
}
}
}
return false
}
func directoryWalker(fsPath string, callback func(fsPath string, d *godirwalk.Dirent) error) error {
err := godirwalk.Walk(fsPath, &godirwalk.Options{
return godirwalk.Walk(fsPath, &godirwalk.Options{
Callback: callback,
ErrorCallback: func(fsPath string, err error) godirwalk.ErrorAction {
return godirwalk.SkipNode
},
Unsorted: true,
})
// directory couldn't be walked
if err != nil {
return err
}
return nil
}

95
v2/internal/runner/update.go
View File

@ -7,6 +7,7 @@ import (
"context"
"crypto/md5"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"io/ioutil"
@ -14,6 +15,7 @@ import (
"os"
"path"
"path/filepath"
"regexp"
"strconv"
"strings"
"time"
@ -23,6 +25,7 @@ import (
"github.com/olekukonko/tablewriter"
"github.com/pkg/errors"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
)
const (
@ -30,6 +33,13 @@ const (
repoName = "nuclei-templates"
)
const nucleiIgnoreFile = ".nuclei-ignore"
// nucleiConfigFilename is the filename of nuclei configuration file.
const nucleiConfigFilename = ".templates-config.json"
var reVersion = regexp.MustCompile(`\d+\.\d+\.\d+`)
// updateTemplates checks if the default list of nuclei-templates
// exist in the users home directory, if not the latest revision
// is downloaded from github.
@ -46,7 +56,7 @@ func (r *Runner) updateTemplates() error {
templatesConfigFile := path.Join(configDir, nucleiConfigFilename)
if _, statErr := os.Stat(templatesConfigFile); !os.IsNotExist(statErr) {
config, readErr := readConfiguration()
config, readErr := config.ReadConfiguration()
if err != nil {
return readErr
}
@ -55,12 +65,12 @@ func (r *Runner) updateTemplates() error {
ignoreURL := "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore"
if r.templatesConfig == nil {
currentConfig := &nucleiConfig{
currentConfig := &config.Config{
TemplatesDirectory: path.Join(home, "nuclei-templates"),
IgnoreURL: ignoreURL,
NucleiVersion: Version,
NucleiVersion: config.Version,
}
if writeErr := r.writeConfiguration(currentConfig); writeErr != nil {
if writeErr := config.WriteConfiguration(currentConfig, false, false); writeErr != nil {
return errors.Wrap(writeErr, "could not write template configuration")
}
r.templatesConfig = currentConfig
@ -68,12 +78,19 @@ func (r *Runner) updateTemplates() error {
// Check if last checked for nuclei-ignore is more than 1 hours.
// and if true, run the check.
//
// Also at the same time fetch latest version from github to do outdated nuclei
// and templates check.
checkedIgnore := false
if r.templatesConfig == nil || time.Since(r.templatesConfig.LastCheckedIgnore) > 1*time.Hour || r.options.UpdateTemplates {
r.fetchLatestVersionsFromGithub()
if r.templatesConfig != nil && r.templatesConfig.IgnoreURL != "" {
ignoreURL = r.templatesConfig.IgnoreURL
}
gologger.Verbose().Msgf("Downloading config file from %s", ignoreURL)
checkedIgnore = true
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
req, reqErr := http.NewRequestWithContext(ctx, http.MethodGet, ignoreURL, nil)
if reqErr == nil {
@ -91,7 +108,10 @@ func (r *Runner) updateTemplates() error {
_ = ioutil.WriteFile(path.Join(configDir, nucleiIgnoreFile), data, 0644)
}
if r.templatesConfig != nil {
r.templatesConfig.LastCheckedIgnore = time.Now()
err = config.WriteConfiguration(r.templatesConfig, false, true)
if err != nil {
gologger.Warning().Msgf("Could not get ignore-file from %s: %s", ignoreURL, err)
}
}
}
}
@ -106,7 +126,7 @@ func (r *Runner) updateTemplates() error {
}
// Use custom location if user has given a template directory
r.templatesConfig = &nucleiConfig{
r.templatesConfig = &config.Config{
TemplatesDirectory: path.Join(home, "nuclei-templates"),
}
if r.options.TemplatesDirectory != "" && r.options.TemplatesDirectory != path.Join(home, "nuclei-templates") {
@ -120,13 +140,14 @@ func (r *Runner) updateTemplates() error {
}
gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory)
r.fetchLatestVersionsFromGithub() // also fetch latest versions
_, err = r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL())
if err != nil {
return err
}
r.templatesConfig.CurrentVersion = version.String()
err = r.writeConfiguration(r.templatesConfig)
err = config.WriteConfiguration(r.templatesConfig, true, checkedIgnore)
if err != nil {
return err
}
@ -162,13 +183,13 @@ func (r *Runner) updateTemplates() error {
if version.EQ(oldVersion) {
gologger.Info().Msgf("Your nuclei-templates are up to date: v%s\n", oldVersion.String())
return r.writeConfiguration(r.templatesConfig)
return config.WriteConfiguration(r.templatesConfig, false, checkedIgnore)
}
if version.GT(oldVersion) {
if !r.options.UpdateTemplates {
gologger.Warning().Msgf("Your current nuclei-templates v%s are outdated. Latest is v%s\n", oldVersion, version.String())
return r.writeConfiguration(r.templatesConfig)
return config.WriteConfiguration(r.templatesConfig, false, checkedIgnore)
}
if r.options.TemplatesDirectory != "" {
@ -177,11 +198,12 @@ func (r *Runner) updateTemplates() error {
r.templatesConfig.CurrentVersion = version.String()
gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory)
r.fetchLatestVersionsFromGithub()
_, err = r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL())
if err != nil {
return err
}
err = r.writeConfiguration(r.templatesConfig)
err = config.WriteConfiguration(r.templatesConfig, true, checkedIgnore)
if err != nil {
return err
}
@ -461,3 +483,56 @@ func (r *Runner) printUpdateChangelog(results *templateUpdateResults, version st
}
table.Render()
}
// fetchLatestVersionsFromGithub fetches latest versions of nuclei repos from github
func (r *Runner) fetchLatestVersionsFromGithub() {
nucleiLatest, err := r.githubFetchLatestTagRepo("projectdiscovery/nuclei")
if err != nil {
gologger.Warning().Msgf("Could not fetch latest nuclei release: %s", err)
}
templatesLatest, err := r.githubFetchLatestTagRepo("projectdiscovery/nuclei-templates")
if err != nil {
gologger.Warning().Msgf("Could not fetch latest nuclei-templates release: %s", err)
}
if r.templatesConfig != nil {
r.templatesConfig.NucleiLatestVersion = nucleiLatest
r.templatesConfig.NucleiTemplatesLatestVersion = templatesLatest
}
}
type githubTagData struct {
Name string
}
// githubFetchLatestTagRepo fetches latest tag from github
// This function was half written by github copilot AI :D.
func (r *Runner) githubFetchLatestTagRepo(repo string) (string, error) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
url := fmt.Sprintf("https://api.github.com/repos/%s/tags", repo)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return "", err
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return "", err
}
var tags []githubTagData
err = json.Unmarshal(body, &tags)
if err != nil {
return "", err
}
if len(tags) == 0 {
return "", fmt.Errorf("no tags found for %s", repo)
}
return strings.TrimPrefix(tags[0].Name, "v"), nil
}

6
v2/internal/runner/update_test.go
View File

@ -15,6 +15,7 @@ import (
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/internal/testutils"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
"github.com/stretchr/testify/require"
)
@ -41,8 +42,7 @@ func TestDownloadReleaseAndUnzipAddition(t *testing.T) {
require.Nil(t, err, "could not create temp directory")
defer os.RemoveAll(templatesDirectory)
r := &Runner{templatesConfig: &nucleiConfig{TemplatesDirectory: templatesDirectory}}
r := &Runner{templatesConfig: &config.Config{TemplatesDirectory: templatesDirectory}}
results, err := r.downloadReleaseAndUnzip(context.Background(), "1.0.0", ts.URL)
require.Nil(t, err, "could not download release and unzip")
require.Equal(t, "base.yaml", results.additions[0], "could not get correct base addition")
@ -94,7 +94,7 @@ func TestDownloadReleaseAndUnzipDeletion(t *testing.T) {
require.Nil(t, err, "could not create temp directory")
defer os.RemoveAll(templatesDirectory)
r := &Runner{templatesConfig: &nucleiConfig{TemplatesDirectory: templatesDirectory}}
r := &Runner{templatesConfig: &config.Config{TemplatesDirectory: templatesDirectory}}
results, err := r.downloadReleaseAndUnzip(context.Background(), "1.0.0", ts.URL)
require.Nil(t, err, "could not download release and unzip")

6
v2/pkg/catalog/catalogue.go
View File

@ -2,7 +2,6 @@ package catalog
// Catalog is a template catalog helper implementation
type Catalog struct {
ignoreFiles []string
templatesDirectory string
}
@ -11,8 +10,3 @@ func New(directory string) *Catalog {
catalog := &Catalog{templatesDirectory: directory}
return catalog
}
// AppendIgnore appends to the catalog store ignore list.
func (c *Catalog) AppendIgnore(list []string) {
c.ignoreFiles = append(c.ignoreFiles, list...)
}

129
v2/pkg/catalog/config/config.go Normal file
View File

@ -0,0 +1,129 @@
package config
import (
"os"
"path"
"time"
jsoniter "github.com/json-iterator/go"
"github.com/projectdiscovery/gologger"
"gopkg.in/yaml.v2"
)
// Config contains the internal nuclei engine configuration
type Config struct {
TemplatesDirectory string `json:"templates-directory,omitempty"`
CurrentVersion string `json:"current-version,omitempty"`
LastChecked time.Time `json:"last-checked,omitempty"`
IgnoreURL string `json:"ignore-url,omitempty"`
NucleiVersion string `json:"nuclei-version,omitempty"`
LastCheckedIgnore time.Time `json:"last-checked-ignore,omitempty"`
NucleiLatestVersion string `json:"nuclei-latest-version"`
NucleiTemplatesLatestVersion string `json:"nuclei-templates-latest-version"`
}
// nucleiConfigFilename is the filename of nuclei configuration file.
const nucleiConfigFilename = ".templates-config.json"
// Version is the current version of nuclei
const Version = `2.4.0-dev`
var (
homeDir string
configDir string
templatesConfigFile string
)
func init() {
homeDir, _ = os.UserHomeDir()
configDir = path.Join(homeDir, "/.config", "/nuclei")
_ = os.MkdirAll(configDir, os.ModePerm)
templatesConfigFile = path.Join(configDir, nucleiConfigFilename)
}
// readConfiguration reads the nuclei configuration file from disk.
func ReadConfiguration() (*Config, error) {
file, err := os.Open(templatesConfigFile)
if err != nil {
return nil, err
}
defer file.Close()
config := &Config{}
err = jsoniter.NewDecoder(file).Decode(config)
if err != nil {
return nil, err
}
return config, nil
}
// WriteConfiguration writes the updated nuclei configuration to disk
func WriteConfiguration(config *Config, checked, checkedIgnore bool) error {
if config.IgnoreURL == "" {
config.IgnoreURL = "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore"
}
if checked {
config.LastChecked = time.Now()
}
if checkedIgnore {
config.LastCheckedIgnore = time.Now()
}
config.NucleiVersion = Version
file, err := os.OpenFile(templatesConfigFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0777)
if err != nil {
return err
}
defer file.Close()
err = jsoniter.NewEncoder(file).Encode(config)
if err != nil {
return err
}
return nil
}
const nucleiIgnoreFile = ".nuclei-ignore"
// IgnoreFile is an internal nuclei template blocking configuration file
type IgnoreFile struct {
Tags []string `yaml:"tags"`
Files []string `yaml:"files"`
}
// ReadIgnoreFile reads the nuclei ignore file returning blocked tags and paths
func ReadIgnoreFile() IgnoreFile {
file, err := os.Open(getIgnoreFilePath())
if err != nil {
gologger.Error().Msgf("Could not read nuclei-ignore file: %s\n", err)
return IgnoreFile{}
}
defer file.Close()
ignore := IgnoreFile{}
if err := yaml.NewDecoder(file).Decode(&ignore); err != nil {
gologger.Error().Msgf("Could not parse nuclei-ignore file: %s\n", err)
return IgnoreFile{}
}
return ignore
}
// getIgnoreFilePath returns the ignore file path for the runner
func getIgnoreFilePath() string {
var defIgnoreFilePath string
home, err := os.UserHomeDir()
if err == nil {
configDir := path.Join(home, "/.config", "/nuclei")
_ = os.MkdirAll(configDir, os.ModePerm)
defIgnoreFilePath = path.Join(configDir, nucleiIgnoreFile)
return defIgnoreFilePath
}
cwd, err := os.Getwd()
if err != nil {
return defIgnoreFilePath
}
cwdIgnoreFilePath := path.Join(cwd, nucleiIgnoreFile)
return cwdIgnoreFilePath
}

5
v2/pkg/catalog/find.go
View File

@ -12,7 +12,7 @@ import (
)
// GetTemplatesPath returns a list of absolute paths for the provided template list.
func (c *Catalog) GetTemplatesPath(definitions []string, noCheckIgnore bool) []string {
func (c *Catalog) GetTemplatesPath(definitions []string) []string {
// keeps track of processed dirs and files
processed := make(map[string]bool)
allTemplates := []string{}
@ -23,9 +23,6 @@ func (c *Catalog) GetTemplatesPath(definitions []string, noCheckIgnore bool) []s
gologger.Error().Msgf("Could not find template '%s': %s\n", t, err)
}
for _, path := range paths {
if !noCheckIgnore && c.checkIfInNucleiIgnore(path) {
continue
}
if _, ok := processed[path]; !ok {
processed[path] = true
allTemplates = append(allTemplates, path)

58
v2/pkg/catalog/ignore.go
View File

@ -1,58 +0,0 @@
package catalog
import (
"strings"
"github.com/projectdiscovery/gologger"
)
// checkIfInNucleiIgnore checks if a path falls under nuclei-ignore rules.
func (c *Catalog) checkIfInNucleiIgnore(item string) bool {
if c.templatesDirectory == "" {
return false
}
matched := false
for _, paths := range c.ignoreFiles {
if !strings.HasSuffix(paths, ".yaml") {
if strings.HasSuffix(strings.TrimSuffix(item, "/"), strings.TrimSuffix(paths, "/")) {
matched = true
break
}
} else if strings.HasSuffix(item, paths) {
matched = true
break
}
}
if matched {
gologger.Warning().Msgf("Excluding %s due to nuclei-ignore filter", item)
return true
}
return false
}
// ignoreFilesWithExcludes ignores results with exclude paths
func (c *Catalog) ignoreFilesWithExcludes(results, excluded []string) []string {
var templates []string
for _, result := range results {
matched := false
for _, paths := range excluded {
if !strings.HasSuffix(paths, ".yaml") {
if strings.HasSuffix(strings.TrimSuffix(result, "/"), strings.TrimSuffix(paths, "/")) {
matched = true
break
}
} else if strings.HasSuffix(result, paths) {
matched = true
break
}
}
if !matched {
templates = append(templates, result)
} else {
gologger.Error().Msgf("Excluding %s due to excludes filter", result)
}
}
return templates
}

40
v2/pkg/catalog/ignore_test.go
View File

@ -1,40 +0,0 @@
package catalog
import (
"fmt"
"testing"
"github.com/stretchr/testify/require"
)
func TestIgnoreFilesIgnore(t *testing.T) {
c := &Catalog{
ignoreFiles: []string{"workflows/", "cves/2020/cve-2020-5432.yaml"},
templatesDirectory: "test",
}
tests := []struct {
path string
ignore bool
}{
{"workflows/", true},
{"misc", false},
{"cves/", false},
{"cves/2020/cve-2020-5432.yaml", true},
{"/Users/test/nuclei-templates/workflows/", true},
{"/Users/test/nuclei-templates/misc", false},
{"/Users/test/nuclei-templates/cves/", false},
{"/Users/test/nuclei-templates/cves/2020/cve-2020-5432.yaml", true},
}
for _, test := range tests {
require.Equal(t, test.ignore, c.checkIfInNucleiIgnore(test.path), fmt.Sprintf("could not ignore file correctly: %v", test))
}
}
func TestExcludeFilesIgnore(t *testing.T) {
c := &Catalog{}
excludes := []string{"workflows/", "cves/2020/cve-2020-5432.yaml"}
paths := []string{"/Users/test/nuclei-templates/workflows/", "/Users/test/nuclei-templates/cves/2020/cve-2020-5432.yaml", "/Users/test/nuclei-templates/workflows/test-workflow.yaml", "/Users/test/nuclei-templates/cves/"}
data := c.ignoreFilesWithExcludes(paths, excludes)
require.Equal(t, []string{"/Users/test/nuclei-templates/workflows/test-workflow.yaml", "/Users/test/nuclei-templates/cves/"}, data, "could not exclude correct files")
}

125
v2/pkg/catalog/loader/filter.go Normal file
View File

@ -0,0 +1,125 @@
package loader
import (
"errors"
"strings"
)
// tagFilter is used to filter nuclei tag based execution
type tagFilter struct {
allowedTags map[string]struct{}
severities map[string]struct{}
authors map[string]struct{}
block map[string]struct{}
matchAllows map[string]struct{}
}
// ErrExcluded is returned for execluded templates
var ErrExcluded = errors.New("the template was excluded")
// match takes a tag and whether the template was matched from user
// input and returns true or false using a tag filter.
//
// If the tag was specified in deny list, it will not return true
// unless it is explicitly specified by user in includeTags which is the
// matchAllows section.
//
// It returns true if the tag is specified, or false.
func (t *tagFilter) match(tag, author, severity string) (bool, error) {
matchedAny := false
if len(t.allowedTags) > 0 {
_, ok := t.allowedTags[tag]
if !ok {
return false, nil
}
matchedAny = true
}
_, ok := t.block[tag]
if ok {
if _, allowOk := t.matchAllows[tag]; allowOk {
return true, nil
}
return false, ErrExcluded
}
if len(t.authors) > 0 {
_, ok = t.authors[author]
if !ok {
return false, nil
}
matchedAny = true
}
if len(t.severities) > 0 {
_, ok = t.severities[severity]
if !ok {
return false, nil
}
matchedAny = true
}
if len(t.allowedTags) == 0 && len(t.authors) == 0 && len(t.severities) == 0 {
return true, nil
}
return matchedAny, nil
}
// createTagFilter returns a tag filter for nuclei tag based execution
//
// It takes into account Tags, Severities, Authors, IncludeTags, ExcludeTags.
func (config *Config) createTagFilter() *tagFilter {
filter := &tagFilter{
allowedTags: make(map[string]struct{}),
authors: make(map[string]struct{}),
severities: make(map[string]struct{}),
block: make(map[string]struct{}),
matchAllows: make(map[string]struct{}),
}
for _, tag := range config.ExcludeTags {
for _, val := range splitCommaTrim(tag) {
if _, ok := filter.block[val]; !ok {
filter.block[val] = struct{}{}
}
}
}
for _, tag := range config.Severities {
for _, val := range splitCommaTrim(tag) {
if _, ok := filter.severities[val]; !ok {
filter.severities[val] = struct{}{}
}
}
}
for _, tag := range config.Authors {
for _, val := range splitCommaTrim(tag) {
if _, ok := filter.authors[val]; !ok {
filter.authors[val] = struct{}{}
}
}
}
for _, tag := range config.Tags {
for _, val := range splitCommaTrim(tag) {
if _, ok := filter.allowedTags[val]; !ok {
filter.allowedTags[val] = struct{}{}
}
delete(filter.block, val)
}
}
for _, tag := range config.IncludeTags {
for _, val := range splitCommaTrim(tag) {
if _, ok := filter.matchAllows[val]; !ok {
filter.matchAllows[val] = struct{}{}
}
delete(filter.block, val)
}
}
return filter
}
func splitCommaTrim(value string) []string {
if !strings.Contains(value, ",") {
return []string{value}
}
splitted := strings.Split(value, ",")
final := make([]string, len(splitted))
for i, value := range splitted {
final[i] = strings.TrimSpace(value)
}
return final
}

86
v2/pkg/catalog/loader/filter_test.go Normal file
View File

@ -0,0 +1,86 @@
package loader
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestTagBasedFilter(t *testing.T) {
config := &Config{
Tags: []string{"cves", "2021", "jira"},
}
filter := config.createTagFilter()
t.Run("true", func(t *testing.T) {
matched, _ := filter.match("jira", "pdteam", "low")
require.True(t, matched, "could not get correct match")
})
t.Run("false", func(t *testing.T) {
matched, _ := filter.match("consul", "pdteam", "low")
require.False(t, matched, "could not get correct match")
})
t.Run("not-match-excludes", func(t *testing.T) {
config := &Config{
ExcludeTags: []string{"dos"},
}
filter := config.createTagFilter()
matched, err := filter.match("dos", "pdteam", "low")
require.False(t, matched, "could not get correct match")
require.Equal(t, ErrExcluded, err, "could not get correct error")
})
t.Run("match-includes", func(t *testing.T) {
config := &Config{
Tags: []string{"cves", "fuzz"},
ExcludeTags: []string{"dos", "fuzz"},
IncludeTags: []string{"fuzz"},
}
filter := config.createTagFilter()
matched, err := filter.match("fuzz", "pdteam", "low")
require.Nil(t, err, "could not get match")
require.True(t, matched, "could not get correct match")
})
t.Run("match-includes", func(t *testing.T) {
config := &Config{
Tags: []string{"fuzz"},
ExcludeTags: []string{"fuzz"},
}
filter := config.createTagFilter()
matched, err := filter.match("fuzz", "pdteam", "low")
require.Nil(t, err, "could not get match")
require.True(t, matched, "could not get correct match")
})
t.Run("match-author", func(t *testing.T) {
config := &Config{
Authors: []string{"pdteam"},
}
filter := config.createTagFilter()
matched, _ := filter.match("fuzz", "pdteam", "low")
require.True(t, matched, "could not get correct match")
})
t.Run("match-severity", func(t *testing.T) {
config := &Config{
Severities: []string{"high"},
}
filter := config.createTagFilter()
matched, _ := filter.match("fuzz", "pdteam", "high")
require.True(t, matched, "could not get correct match")
})
t.Run("match-conditions", func(t *testing.T) {
config := &Config{
Authors: []string{"pdteam"},
Tags: []string{"jira"},
Severities: []string{"high"},
}
filter := config.createTagFilter()
matched, _ := filter.match("jira", "pdteam", "high")
require.True(t, matched, "could not get correct match")
matched, _ = filter.match("jira", "pdteam", "low")
require.False(t, matched, "could not get correct match")
matched, _ = filter.match("jira", "random", "low")
require.False(t, matched, "could not get correct match")
matched, _ = filter.match("consul", "random", "low")
require.False(t, matched, "could not get correct match")
})
}

205
v2/pkg/catalog/loader/loader.go Normal file
View File

@ -0,0 +1,205 @@
package loader
import (
"bytes"
"errors"
"io/ioutil"
"os"
"strings"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
"gopkg.in/yaml.v2"
)
// Config contains the configuration options for the loader
type Config struct {
Templates []string
Workflows []string
ExcludeTemplates []string
IncludeTemplates []string
Tags []string
ExcludeTags []string
Authors []string
Severities []string
IncludeTags []string
Catalog *catalog.Catalog
ExecutorOptions protocols.ExecuterOptions
TemplatesDirectory string
}
// Store is a storage for loaded nuclei templates
type Store struct {
tagFilter *tagFilter
config *Config
finalTemplates []string
templates []*templates.Template
workflows []*templates.Template
}
// New creates a new template store based on provided configuration
func New(config *Config) (*Store, error) {
// Create a tag filter based on provided configuration
store := &Store{
config: config,
tagFilter: config.createTagFilter(),
}
// Handle a case with no templates or workflows, where we use base directory
if len(config.Templates) == 0 && len(config.Workflows) == 0 {
config.Templates = append(config.Templates, config.TemplatesDirectory)
}
store.finalTemplates = append(store.finalTemplates, config.Templates...)
return store, nil
}
// Templates returns all the templates in the store
func (s *Store) Templates() []*templates.Template {
return s.templates
}
// Workflows returns all the workflows in the store
func (s *Store) Workflows() []*templates.Template {
return s.workflows
}
// Load loads all the templates from a store, performs filtering and returns
// the complete compiled templates for a nuclei execution configuration.
func (s *Store) Load() {
includedTemplates := s.config.Catalog.GetTemplatesPath(s.finalTemplates)
includedWorkflows := s.config.Catalog.GetTemplatesPath(s.config.Workflows)
excludedTemplates := s.config.Catalog.GetTemplatesPath(s.config.ExcludeTemplates)
alwaysIncludeTemplates := s.config.Catalog.GetTemplatesPath(s.config.IncludeTemplates)
alwaysIncludedTemplatesMap := make(map[string]struct{})
for _, tpl := range alwaysIncludeTemplates {
alwaysIncludedTemplatesMap[tpl] = struct{}{}
}
templatesMap := make(map[string]struct{})
for _, tpl := range includedTemplates {
templatesMap[tpl] = struct{}{}
}
for _, template := range excludedTemplates {
if _, ok := alwaysIncludedTemplatesMap[template]; ok {
continue
} else {
delete(templatesMap, template)
}
}
for k := range templatesMap {
loaded, err := s.loadTemplateParseMetadata(k, false)
if err != nil {
gologger.Warning().Msgf("Could not load template %s: %s\n", k, err)
}
if loaded {
parsed, err := templates.Parse(k, s.config.ExecutorOptions)
if err != nil {
gologger.Warning().Msgf("Could not parse template %s: %s\n", k, err)
} else if parsed != nil {
s.templates = append(s.templates, parsed)
}
}
}
workflowsMap := make(map[string]struct{})
for _, tpl := range includedWorkflows {
workflowsMap[tpl] = struct{}{}
}
for _, template := range excludedTemplates {
if _, ok := alwaysIncludedTemplatesMap[template]; ok {
continue
} else {
delete(templatesMap, template)
}
}
for k := range workflowsMap {
loaded, err := s.loadTemplateParseMetadata(k, true)
if err != nil {
gologger.Warning().Msgf("Could not load workflow %s: %s\n", k, err)
}
if loaded {
parsed, err := templates.Parse(k, s.config.ExecutorOptions)
if err != nil {
gologger.Warning().Msgf("Could not parse workflow %s: %s\n", k, err)
} else if parsed != nil {
s.workflows = append(s.workflows, parsed)
}
}
}
}
// loadTemplateParseMetadata loads a template by parsing metadata and running
// all tag and path based filters on the template.
func (s *Store) loadTemplateParseMetadata(templatePath string, workflow bool) (bool, error) {
f, err := os.Open(templatePath)
if err != nil {
return false, err
}
defer f.Close()
data, err := ioutil.ReadAll(f)
if err != nil {
return false, err
}
template := &templates.Template{}
err = yaml.NewDecoder(bytes.NewReader(data)).Decode(template)
if err != nil {
return false, err
}
if _, ok := template.Info["name"]; !ok {
return false, errors.New("no template name field provided")
}
author, ok := template.Info["author"]
if !ok {
return false, errors.New("no template author field provided")
}
severity, ok := template.Info["severity"]
if !ok {
severity = ""
}
templateTags, ok := template.Info["tags"]
if !ok {
templateTags = ""
}
tagStr := types.ToString(templateTags)
tags := strings.Split(tagStr, ",")
severityStr := types.ToString(severity)
authors := strings.Split(types.ToString(author), ",")
matched := false
for _, tag := range tags {
for _, author := range authors {
match, err := s.tagFilter.match(strings.TrimSpace(tag), strings.TrimSpace(author), severityStr)
if err == ErrExcluded {
return false, ErrExcluded
}
if !matched && match && err == nil {
matched = true
}
}
}
if !matched {
return false, nil
}
if len(template.Workflows) == 0 && workflow {
return false, nil
}
if len(template.Workflows) > 0 && !workflow {
return false, nil
}
return true, nil
}

40
v2/pkg/catalog/loader/loader_test.go Normal file
View File

@ -0,0 +1,40 @@
package loader
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestLoadTemplates(t *testing.T) {
store, err := New(&Config{
Templates: []string{"cves/CVE-2021-21315.yaml"},
})
require.Nil(t, err, "could not load templates")
require.Equal(t, []string{"cves/CVE-2021-21315.yaml"}, store.finalTemplates, "could not get correct templates")
templatesDirectory := "/test"
t.Run("blank", func(t *testing.T) {
store, err := New(&Config{
TemplatesDirectory: templatesDirectory,
})
require.Nil(t, err, "could not load templates")
require.Equal(t, []string{templatesDirectory}, store.finalTemplates, "could not get correct templates")
})
t.Run("only-tags", func(t *testing.T) {
store, err := New(&Config{
Tags: []string{"cves"},
TemplatesDirectory: templatesDirectory,
})
require.Nil(t, err, "could not load templates")
require.Equal(t, []string{templatesDirectory}, store.finalTemplates, "could not get correct templates")
})
t.Run("tags-with-path", func(t *testing.T) {
store, err := New(&Config{
Tags: []string{"cves"},
TemplatesDirectory: templatesDirectory,
})
require.Nil(t, err, "could not load templates")
require.Equal(t, []string{templatesDirectory}, store.finalTemplates, "could not get correct templates")
})
}

63
v2/pkg/templates/compile.go
View File

@ -12,7 +12,6 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/executer"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/offlinehttp"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
"github.com/projectdiscovery/nuclei/v2/pkg/workflows"
"gopkg.in/yaml.v2"
)
@ -45,22 +44,6 @@ func Parse(filePath string, options protocols.ExecuterOptions) (*Template, error
if _, ok := template.Info["author"]; !ok {
return nil, errors.New("no template author field provided")
}
templateTags, ok := template.Info["tags"]
if !ok {
templateTags = ""
}
matchWithTags := false
if len(options.Options.Tags) > 0 {
if err := matchTemplateWithTags(types.ToString(templateTags), types.ToString(template.Info["severity"]), options.Options.Tags); err != nil {
return nil, fmt.Errorf("tags filter not matched %s", templateTags)
}
matchWithTags = true
}
if len(options.Options.ExcludeTags) > 0 && !matchWithTags {
if err := matchTemplateWithTags(types.ToString(templateTags), types.ToString(template.Info["severity"]), options.Options.ExcludeTags); err == nil {
return nil, fmt.Errorf("exclude-tags filter matched %s", templateTags)
}
}
// Setting up variables regarding template metadata
options.TemplateID = template.ID
@ -206,49 +189,3 @@ func (t *Template) parseWorkflowTemplate(workflow *workflows.WorkflowTemplate, o
}
return nil
}
// matchTemplateWithTags matches if the template matches a tag
func matchTemplateWithTags(tags, severity string, tagsInput []string) error {
actualTags := strings.Split(tags, ",")
if severity != "" {
actualTags = append(actualTags, severity) // also add severity to tag
}
matched := false
mainLoop:
for _, t := range tagsInput {
commaTags := strings.Split(t, ",")
for _, tag := range commaTags {
tag = strings.TrimSpace(tag)
key, value := getKeyValue(tag)
for _, templTag := range actualTags {
templTag = strings.TrimSpace(templTag)
tKey, tValue := getKeyValue(templTag)
if strings.EqualFold(key, tKey) && strings.EqualFold(value, tValue) {
matched = true
break mainLoop
}
}
}
}
if !matched {
return errors.New("could not match template tags with input")
}
return nil
}
// getKeyValue returns key value pair for a data string
func getKeyValue(data string) (key, value string) {
if strings.Contains(data, ":") {
parts := strings.SplitN(data, ":", 2)
if len(parts) == 2 {
key, value = parts[0], parts[1]
}
}
if value == "" {
value = data
}
return key, value
}

41
v2/pkg/templates/compile_test.go
View File

@ -1,41 +0,0 @@
package templates
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestMatchTemplateWithTags(t *testing.T) {
err := matchTemplateWithTags("php,linux,symfony", "", []string{"php"})
require.Nil(t, err, "could not get php tag from input slice")
err = matchTemplateWithTags("lang:php,os:linux,cms:symfony", "", []string{"cms:symfony"})
require.Nil(t, err, "could not get php tag from input key value")
err = matchTemplateWithTags("lang:php,os:linux,symfony", "", []string{"cms:symfony"})
require.NotNil(t, err, "could get key value tag from input key value")
err = matchTemplateWithTags("lang:php,os:linux,cms:jira", "", []string{"cms:symfony"})
require.NotNil(t, err, "could get key value tag from input key value")
t.Run("space", func(t *testing.T) {
err = matchTemplateWithTags("lang:php, os:linux, cms:symfony", "", []string{"cms:symfony"})
require.Nil(t, err, "could get key value tag from input key value with space")
})
t.Run("comma-tags", func(t *testing.T) {
err = matchTemplateWithTags("lang:php,os:linux,cms:symfony", "", []string{"test,cms:symfony"})
require.Nil(t, err, "could get key value tag from input key value with comma")
})
t.Run("severity", func(t *testing.T) {
err = matchTemplateWithTags("lang:php,os:linux,cms:symfony", "low", []string{"low"})
require.Nil(t, err, "could get key value tag for severity")
})
t.Run("blank-tags", func(t *testing.T) {
err = matchTemplateWithTags("", "low", []string{"jira"})
require.NotNil(t, err, "could get value tag for blank severity")
})
}

8
v2/pkg/types/types.go
View File

@ -20,6 +20,13 @@ type Options struct {
CustomHeaders goflags.StringSlice
// Severity filters templates based on their severity and only run the matching ones.
Severity goflags.StringSlice
// Author filters templates based on their author and only run the matching ones.
Author goflags.StringSlice
// IncludeTags includes specified tags to be run even while being in denylist
IncludeTags goflags.StringSlice
// IncludeTemplates includes specified templates to be run even while being in denylist
IncludeTemplates goflags.StringSlice
InternalResolversList []string // normalized from resolvers flag as well as file provided.
// ProjectPath allows nuclei to use a user defined project folder
ProjectPath string
@ -99,6 +106,7 @@ type Options struct {
Version bool
// Verbose flag indicates whether to show verbose output or not
Verbose bool
VerboseVerbose bool
// No-Color disables the colored output.
NoColor bool
// UpdateTemplates updates the templates installed at startup