mirror of https://github.com/daffainfo/nuclei.git
Added new stats counters + validation fixes
Ice3man543
parent
d33f6eb502
commit
c3e32f5e15
|
|
@ -13,6 +13,7 @@ require (
|
|||
github.com/bluele/gcache v0.0.2
|
||||
github.com/c4milo/unpackit v0.1.0 // indirect
|
||||
github.com/corpix/uarand v0.1.1
|
||||
github.com/davecgh/go-spew v1.1.1
|
||||
github.com/go-rod/rod v0.91.1
|
||||
github.com/google/go-github v17.0.0+incompatible
|
||||
github.com/gosuri/uilive v0.0.4 // indirect
|
||||
|
|
@ -23,6 +24,7 @@ require (
|
|||
github.com/karlseguin/ccache v2.0.3+incompatible
|
||||
github.com/karrick/godirwalk v1.16.1
|
||||
github.com/logrusorgru/aurora v2.0.3+incompatible
|
||||
github.com/mattn/go-runewidth v0.0.13 // indirect
|
||||
github.com/miekg/dns v1.1.43
|
||||
github.com/olekukonko/tablewriter v0.0.5
|
||||
github.com/owenrumney/go-sarif v1.0.11
|
||||
|
|
@ -58,5 +60,7 @@ require (
|
|||
go.uber.org/ratelimit v0.2.0
|
||||
golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d
|
||||
golang.org/x/oauth2 v0.0.0-20210817223510-7df4dd6e12ab
|
||||
golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf // indirect
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0
|
||||
)
|
||||
|
|
|
|||
12
v2/go.sum
12
v2/go.sum
|
|
@ -228,6 +228,7 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO
|
|||
github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 h1:i462o439ZjprVSFSZLZxcsoAe592sZB1rci2Z8j4wdk=
|
||||
github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA=
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
|
||||
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
|
||||
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
|
||||
github.com/itchyny/go-flags v1.5.0/go.mod h1:lenkYuCobuxLBAd/HGFE4LRoW8D3B6iXRQfWYJ+MNbA=
|
||||
github.com/itchyny/gojq v0.12.4 h1:8zgOZWMejEWCLjbF/1mWY7hY7QEARm7dtuhC6Bp4R8o=
|
||||
|
|
@ -274,6 +275,7 @@ github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczG
|
|||
github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
|
||||
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
|
||||
github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
|
||||
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
|
||||
github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
||||
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
||||
|
|
@ -281,6 +283,8 @@ github.com/mattn/go-isatty v0.0.13 h1:qdl+GuBjcsKKDco5BsxPJlId98mSWNKqYA+Co0SC1y
|
|||
github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
|
||||
github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
|
||||
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
|
||||
github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
|
||||
github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
|
||||
github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
|
||||
github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
|
||||
|
|
@ -359,6 +363,8 @@ github.com/projectdiscovery/yamldoc-go v1.0.2/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6n
|
|||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E=
|
||||
github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo=
|
||||
github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
|
||||
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/rogpeppe/go-internal v1.6.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
|
||||
|
|
@ -501,6 +507,7 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
|
|||
golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.4.0 h1:8pl+sMODzuvGJkmj2W4kZihvVb5mKm8pB/X44PIQHv8=
|
||||
golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
|
|
@ -608,6 +615,8 @@ golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7w
|
|||
golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I=
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf h1:2ucpDCmfkl8Bd/FsLtiD653Wf96cW37s+iGx93zsu4k=
|
||||
golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
|
|
@ -666,6 +675,7 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc
|
|||
golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
|
||||
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.0.0-20210101214203-2dba1e4ea05c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY=
|
||||
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
|
|
@ -696,6 +706,8 @@ google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww
|
|||
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc=
|
||||
google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
|
||||
google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ import (
|
|||
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/types"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/utils"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/utils/stats"
|
||||
)
|
||||
|
||||
// Runner is a client for running the enumeration process.
|
||||
|
|
@ -339,14 +340,23 @@ func (r *Runner) RunEnumeration() error {
|
|||
if err != nil {
|
||||
return errors.Wrap(err, "could not load templates from config")
|
||||
}
|
||||
store.Load()
|
||||
|
||||
if r.options.Validate {
|
||||
if err := store.ValidateTemplates(r.options.Templates, r.options.Workflows); err != nil {
|
||||
return err
|
||||
}
|
||||
gologger.Info().Msgf("All templates validated successfully\n")
|
||||
if stats.GetValue("syntax-warnings") == 0 && stats.GetValue("syntax-errors") == 0 {
|
||||
gologger.Info().Msgf("All templates validated successfully\n")
|
||||
} else {
|
||||
return errors.New("encountered errors while performing template validation")
|
||||
}
|
||||
return nil // exit
|
||||
}
|
||||
store.Load()
|
||||
|
||||
// Display stats for any loaded templates syntax warnings or errors
|
||||
stats.Display("syntax-warnings")
|
||||
stats.Display("syntax-errors")
|
||||
|
||||
builder := &strings.Builder{}
|
||||
if r.templatesConfig != nil && r.templatesConfig.NucleiLatestVersion != "" {
|
||||
|
|
|
|||
|
|
@ -4,14 +4,17 @@ import (
|
|||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"regexp"
|
||||
|
||||
"gopkg.in/yaml.v2"
|
||||
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/model"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/templates/cache"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/utils"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/utils/stats"
|
||||
)
|
||||
|
||||
const mandatoryFieldMissingTemplate = "mandatory '%s' field is missing"
|
||||
|
|
@ -85,8 +88,21 @@ func validateMandatoryInfoFields(info *model.Info) error {
|
|||
|
||||
var parsedTemplatesCache *cache.Templates
|
||||
|
||||
var fieldErrorRegexp = regexp.MustCompile(`not found in`)
|
||||
|
||||
func init() {
|
||||
parsedTemplatesCache = cache.New()
|
||||
|
||||
stats.NewEntry("syntax-warnings", "Got %d syntax warnings for the loaded templates")
|
||||
stats.NewEntry("syntax-errors", "Got %d syntax errors for the loaded templates")
|
||||
}
|
||||
|
||||
type ErrValidationFailure struct {
|
||||
Err error
|
||||
}
|
||||
|
||||
func (e ErrValidationFailure) Error() string {
|
||||
return fmt.Sprintf("validation error: %v\n", e.Err)
|
||||
}
|
||||
|
||||
// ParseTemplate parses a template and returns a *templates.Template structure
|
||||
|
|
@ -107,9 +123,15 @@ func ParseTemplate(templatePath string) (*templates.Template, error) {
|
|||
}
|
||||
|
||||
template := &templates.Template{}
|
||||
err = yaml.Unmarshal(data, template)
|
||||
err = yaml.UnmarshalStrict(data, template)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
errString := err.Error()
|
||||
if !fieldErrorRegexp.MatchString(errString) {
|
||||
stats.Increment("syntax-errors")
|
||||
return nil, err
|
||||
}
|
||||
stats.Increment("syntax-warnings")
|
||||
gologger.Warning().Msgf("Syntax warnings for template %s: %s", templatePath, err)
|
||||
}
|
||||
parsedTemplatesCache.Store(templatePath, template, nil)
|
||||
return template, nil
|
||||
|
|
|
|||
|
|
@ -4,13 +4,11 @@ import (
|
|||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"gopkg.in/yaml.v2"
|
||||
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/executer"
|
||||
|
|
@ -21,7 +19,6 @@ import (
|
|||
|
||||
var (
|
||||
ErrCreateTemplateExecutor = errors.New("cannot create template executer")
|
||||
fieldErrorRegexp = regexp.MustCompile(`not found in`)
|
||||
)
|
||||
|
||||
var parsedTemplatesCache *cache.Templates
|
||||
|
|
@ -56,12 +53,9 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute
|
|||
data = preprocessor.Process(data)
|
||||
}
|
||||
|
||||
err = yaml.UnmarshalStrict(data, template)
|
||||
err = yaml.Unmarshal(data, template)
|
||||
if err != nil {
|
||||
if !fieldErrorRegexp.MatchString(err.Error()) {
|
||||
return nil, err
|
||||
}
|
||||
gologger.Warning().Msgf("Unrecognized fields in template %s: %s", filePath, err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if utils.IsBlank(template.Info.Name) {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,3 @@
|
|||
// Package stats provides a storage mechanism for storing
|
||||
// and display vital statistics of the engine at various durations.
|
||||
package stats
|
||||
|
|
@ -0,0 +1,102 @@
|
|||
package stats
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/projectdiscovery/gologger"
|
||||
)
|
||||
|
||||
// Storage is a storage for storing statistics information
|
||||
// about the nuclei engine displaying it at user-defined intervals.
|
||||
type Storage struct {
|
||||
data map[string]*storageDataItem
|
||||
mutex *sync.RWMutex
|
||||
}
|
||||
|
||||
type storageDataItem struct {
|
||||
description string
|
||||
value int64
|
||||
}
|
||||
|
||||
var Default *Storage
|
||||
|
||||
func init() {
|
||||
Default = New()
|
||||
}
|
||||
|
||||
// NewEntry creates a new entry in the storage object
|
||||
func NewEntry(name, description string) {
|
||||
Default.NewEntry(name, description)
|
||||
}
|
||||
|
||||
// Increment incrmements the value for a name string
|
||||
func Increment(name string) {
|
||||
Default.Increment(name)
|
||||
}
|
||||
|
||||
// Display displays the stats for a name
|
||||
func Display(name string) {
|
||||
Default.Display(name)
|
||||
}
|
||||
|
||||
// GetValue returns the value for a set variable
|
||||
func GetValue(name string) int64 {
|
||||
return Default.GetValue(name)
|
||||
}
|
||||
|
||||
// New creates a new storage object
|
||||
func New() *Storage {
|
||||
return &Storage{data: make(map[string]*storageDataItem), mutex: &sync.RWMutex{}}
|
||||
}
|
||||
|
||||
// NewEntry creates a new entry in the storage object
|
||||
func (s *Storage) NewEntry(name, description string) {
|
||||
s.mutex.Lock()
|
||||
s.data[name] = &storageDataItem{description: description, value: 0}
|
||||
s.mutex.Unlock()
|
||||
}
|
||||
|
||||
// Increment incrmements the value for a name string
|
||||
func (s *Storage) Increment(name string) {
|
||||
s.mutex.RLock()
|
||||
data, ok := s.data[name]
|
||||
if !ok {
|
||||
s.mutex.RUnlock()
|
||||
return
|
||||
}
|
||||
s.mutex.RUnlock()
|
||||
|
||||
atomic.AddInt64(&data.value, 1)
|
||||
}
|
||||
|
||||
// Display displays the stats for a name
|
||||
func (s *Storage) Display(name string) {
|
||||
s.mutex.RLock()
|
||||
data, ok := s.data[name]
|
||||
if !ok {
|
||||
s.mutex.RUnlock()
|
||||
return
|
||||
}
|
||||
s.mutex.RUnlock()
|
||||
|
||||
dataValue := atomic.LoadInt64(&data.value)
|
||||
if dataValue == 0 {
|
||||
return // don't show for nil stats
|
||||
}
|
||||
gologger.Info().Msgf(data.description, dataValue)
|
||||
}
|
||||
|
||||
// GetValue returns the value for a set variable
|
||||
func (s *Storage) GetValue(name string) int64 {
|
||||
s.mutex.RLock()
|
||||
data, ok := s.data[name]
|
||||
if !ok {
|
||||
s.mutex.RUnlock()
|
||||
return 0
|
||||
}
|
||||
s.mutex.RUnlock()
|
||||
|
||||
dataValue := atomic.LoadInt64(&data.value)
|
||||
return dataValue
|
||||
}
|
||||
Loading…
Reference in New Issue