daffainfo
/
nuclei
mirror of https://github.com/daffainfo/nuclei.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1517 from projectdiscovery/automatic-workflows 

Added initial automatic workflow implementation
dev
Sandeep Singh 2022-03-17 19:04:40 +05:30 committed by GitHub
parent 4b7ebd0399 428841d946
commit f94372acf3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 442 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
v2/cmd/nuclei/main.go
View File

@ -90,8 +90,8 @@ on extensive configurability, massive extensibility and ease of use.`)
)
createGroup(flagSet, "templates", "Templates",
flagSet.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "run only new templates added in latest nuclei-templates release"),
flagSet.BoolVarP(&options.AutomaticScan, "automatic-scan", "as", false, "automatic web scan using wappalyzer technology detection to tags mapping"),
flagSet.FileNormalizedOriginalStringSliceVarP(&options.Templates, "templates", "t", []string{}, "list of template or template directory to run (comma-separated, file)"),
flagSet.FileNormalizedOriginalStringSliceVarP(&options.TemplateURLs, "template-url", "tu", []string{}, "list of template urls to run (comma-separated, file)"),
flagSet.FileNormalizedOriginalStringSliceVarP(&options.Workflows, "workflows", "w", []string{}, "list of workflow or workflow directory to run (comma-separated, file)"),

1
v2/go.mod
View File

@ -75,6 +75,7 @@ require (
github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3
github.com/projectdiscovery/nvd v1.0.9-0.20220314070650-d4a214c1f87d
github.com/projectdiscovery/sliceutil v0.0.0-20220225084130-8392ac12fa6d
github.com/projectdiscovery/wappalyzergo v0.0.25
github.com/stretchr/testify v1.7.1
github.com/zmap/zcrypto v0.0.0-20211005224000-2d0ffdec8a9b
)

2
v2/go.sum
View File

@ -486,6 +486,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20210823090203-2f5f137e8e1d/go.mo
github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
github.com/projectdiscovery/stringsutil v0.0.0-20220119085121-22513a958700 h1:L7Vb5AdzIV1Xs088Nvslfhh/piKP9gjTxjxfiqnd4mk=
github.com/projectdiscovery/stringsutil v0.0.0-20220119085121-22513a958700/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
github.com/projectdiscovery/wappalyzergo v0.0.25 h1:7C//STQwq0DPExjpXS9EmjxnKb3WWkI1S4MSTog+7+M=
github.com/projectdiscovery/wappalyzergo v0.0.25/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM=
github.com/projectdiscovery/yamldoc-go v1.0.2/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24=
github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6 h1:DvWRQpw7Ib2CRL3ogYm/BWM+X0UGPfz1n9Ix9YKgFM8=
github.com/projectdiscovery/yamldoc-go v1.0.3-0.20211126104922-00d2c6bb43b6/go.mod h1:8OfZj8p/axkUM/TJoS/O9LDjj/S8u17rxRbqluE9CU4=

75
v2/internal/runner/runner.go
View File

@ -13,6 +13,7 @@ import (
"github.com/logrusorgru/aurora"
"github.com/pkg/errors"
"go.uber.org/atomic"
"go.uber.org/ratelimit"
"github.com/projectdiscovery/gologger"
@ -22,12 +23,12 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader"
"github.com/projectdiscovery/nuclei/v2/pkg/core"
"github.com/projectdiscovery/nuclei/v2/pkg/core/inputs/hybrid"
"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/parsers"
"github.com/projectdiscovery/nuclei/v2/pkg/progress"
"github.com/projectdiscovery/nuclei/v2/pkg/projectfile"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/automaticscan"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
@ -54,7 +55,6 @@ type Runner struct {
progress progress.Progress
colorizer aurora.Aurora
issuesClient *reporting.Client
addColor func(severity.Severity) string
hmapInputProvider *hybrid.Input
browser *engine.Browser
ratelimiter ratelimit.Limiter
@ -112,7 +112,8 @@ func New(options *types.Options) (*Runner, error) {
// output coloring
useColor := !options.NoColor
runner.colorizer = aurora.NewAurora(useColor)
runner.addColor = colorizer.New(runner.colorizer)
templates.Colorizer = runner.colorizer
templates.SeverityColorizer = colorizer.New(runner.colorizer)
if options.TemplateList {
runner.listAvailableTemplates()
@ -338,6 +339,52 @@ func (r *Runner) RunEnumeration() error {
r.displayExecutionInfo(store)
var results *atomic.Bool
if r.options.AutomaticScan {
results, err = r.executeSmartWorkflowInput(executerOpts, store, engine)
} else {
results, err = r.executeTemplatesInput(store, engine)
}
if r.interactsh != nil {
matched := r.interactsh.Close()
if matched {
results.CAS(false, true)
}
}
r.progress.Stop()
if r.issuesClient != nil {
r.issuesClient.Close()
}
if !results.Load() {
gologger.Info().Msgf("No results found. Better luck next time!")
}
if r.browser != nil {
r.browser.Close()
}
return err
}
func (r *Runner) executeSmartWorkflowInput(executerOpts protocols.ExecuterOptions, store *loader.Store, engine *core.Engine) (*atomic.Bool, error) {
r.progress.Init(r.hmapInputProvider.Count(), 0, 0)
service, err := automaticscan.New(automaticscan.Options{
ExecuterOpts: executerOpts,
Store: store,
Engine: engine,
Target: r.hmapInputProvider,
})
if err != nil {
return nil, errors.Wrap(err, "could not create smart workflow service")
}
service.Execute()
result := &atomic.Bool{}
result.Store(service.Close())
return result, nil
}
func (r *Runner) executeTemplatesInput(store *loader.Store, engine *core.Engine) (*atomic.Bool, error) {
var unclusteredRequests int64
for _, template := range store.Templates() {
// workflows will dynamically adjust the totals while running, as
@ -378,32 +425,14 @@ func (r *Runner) RunEnumeration() error {
// 0 matches means no templates were found in directory
if templateCount == 0 {
return errors.New("no valid templates were found")
return nil, errors.New("no valid templates were found")
}
// tracks global progress and captures stdout/stderr until p.Wait finishes
r.progress.Init(r.hmapInputProvider.Count(), templateCount, totalRequests)
results := engine.ExecuteWithOpts(finalTemplates, r.hmapInputProvider, true)
if r.interactsh != nil {
matched := r.interactsh.Close()
if matched {
results.CAS(false, true)
}
}
r.progress.Stop()
if r.issuesClient != nil {
r.issuesClient.Close()
}
if !results.Load() {
gologger.Info().Msgf("No results found. Better luck next time!")
}
if r.browser != nil {
r.browser.Close()
}
return nil
return results, nil
}
// displayExecutionInfo displays misc info about the nuclei engine execution

31
v2/internal/runner/templates.go
View File

@ -1,50 +1,23 @@
package runner
import (
"fmt"
"os"
"strings"
"github.com/karrick/godirwalk"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v2/pkg/parsers"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
)
func (r *Runner) templateLogMsg(id, name string, authors []string, templateSeverity severity.Severity) string {
// Display the message for the template
return fmt.Sprintf("[%s] %s (%s) [%s]",
r.colorizer.BrightBlue(id).String(),
r.colorizer.Bold(name).String(),
r.colorizer.BrightYellow(appendAtSignToAuthors(authors)).String(),
r.addColor(templateSeverity))
}
// appendAtSignToAuthors appends @ before each author and returns the final string
func appendAtSignToAuthors(authors []string) string {
if len(authors) == 0 {
return "@none"
}
values := make([]string, 0, len(authors))
for _, k := range authors {
if !strings.HasPrefix(k, "@") {
values = append(values, fmt.Sprintf("@%s", k))
} else {
values = append(values, k)
}
}
return strings.Join(values, ",")
}
func (r *Runner) logAvailableTemplate(tplPath string) {
t, err := parsers.ParseTemplate(tplPath)
if err != nil {
gologger.Error().Msgf("Could not parse file '%s': %s\n", tplPath, err)
} else {
gologger.Print().Msgf("%s\n", r.templateLogMsg(t.ID,
gologger.Print().Msgf("%s\n", templates.TemplateLogMessage(t.ID,
types.ToString(t.Info.Name),
t.Info.Authors.ToSlice(),
t.Info.SeverityHolder.Severity))

25
v2/pkg/catalog/loader/loader.go
View File

@ -295,3 +295,28 @@ func (store *Store) LoadWorkflows(workflowsList []string) []*templates.Template
}
return loadedWorkflows
}
// LoadTemplatesWithTags takes a list of templates and extra tags
// returning templates that match.
func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templates.Template {
includedTemplates := store.config.Catalog.GetTemplatesPath(templatesList)
templatePathMap := store.pathFilter.Match(includedTemplates)
loadedTemplates := make([]*templates.Template, 0, len(templatePathMap))
for templatePath := range templatePathMap {
loaded, err := parsers.LoadTemplate(templatePath, store.tagFilter, tags)
if err != nil {
gologger.Warning().Msgf("Could not load template %s: %s\n", templatePath, err)
}
if loaded {
parsed, err := templates.Parse(templatePath, store.preprocessor, store.config.ExecutorOptions)
if err != nil {
stats.Increment(parsers.RuntimeWarningsStats)
gologger.Warning().Msgf("Could not parse template %s: %s\n", templatePath, err)
} else if parsed != nil {
loadedTemplates = append(loadedTemplates, parsed)
}
}
}
return loadedTemplates
}

5
v2/pkg/core/engine.go
View File

@ -57,3 +57,8 @@ func (e *Engine) SetExecuterOptions(options protocols.ExecuterOptions) {
func (e *Engine) ExecuterOptions() protocols.ExecuterOptions {
return e.executerOpts
}
// WorkPool returns the worker pool for the engine
func (e *Engine) WorkPool() *WorkPool {
return e.workPool
}

101
v2/pkg/core/execute.go
View File

@ -5,6 +5,7 @@ import (
"go.uber.org/atomic"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
"github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
generalTypes "github.com/projectdiscovery/nuclei/v2/pkg/types"
@ -155,3 +156,103 @@ func (e *Engine) executeModelWithInput(templateType types.ProtocolType, template
currentInfo.Completed = true
currentInfo.Unlock()
}
// ExecuteWithResults a list of templates with results
func (e *Engine) ExecuteWithResults(templatesList []*templates.Template, target InputProvider, callback func(*output.ResultEvent)) *atomic.Bool {
results := &atomic.Bool{}
for _, template := range templatesList {
templateType := template.Type()
var wg *sizedwaitgroup.SizedWaitGroup
if templateType == types.HeadlessProtocol {
wg = e.workPool.Headless
} else {
wg = e.workPool.Default
}
wg.Add()
go func(tpl *templates.Template) {
e.executeModelWithInputAndResult(templateType, tpl, target, results, callback)
wg.Done()
}(template)
}
e.workPool.Wait()
return results
}
// executeModelWithInputAndResult executes a type of template with input and result
func (e *Engine) executeModelWithInputAndResult(templateType types.ProtocolType, template *templates.Template, target InputProvider, results *atomic.Bool, callback func(*output.ResultEvent)) {
wg := e.workPool.InputPool(templateType)
target.Scan(func(scannedValue string) {
// Skip if the host has had errors
if e.executerOpts.HostErrorsCache != nil && e.executerOpts.HostErrorsCache.Check(scannedValue) {
return
}
wg.WaitGroup.Add()
go func(value string) {
defer wg.WaitGroup.Done()
var match bool
var err error
switch templateType {
case types.WorkflowProtocol:
match = e.executeWorkflow(value, template.CompiledWorkflow)
default:
err = template.Executer.ExecuteWithResults(value, func(event *output.InternalWrappedEvent) {
for _, result := range event.Results {
callback(result)
}
})
}
if err != nil {
gologger.Warning().Msgf("[%s] Could not execute step: %s\n", e.executerOpts.Colorizer.BrightBlue(template.ID), err)
}
results.CAS(false, match)
}(scannedValue)
})
wg.WaitGroup.Wait()
}
type ChildExecuter struct {
e *Engine
results *atomic.Bool
}
// Close closes the executer returning bool results
func (e *ChildExecuter) Close() *atomic.Bool {
e.e.workPool.Wait()
return e.results
}
// Execute executes a template and URLs
func (e *ChildExecuter) Execute(template *templates.Template, URL string) {
templateType := template.Type()
var wg *sizedwaitgroup.SizedWaitGroup
if templateType == types.HeadlessProtocol {
wg = e.e.workPool.Headless
} else {
wg = e.e.workPool.Default
}
wg.Add()
go func(tpl *templates.Template) {
match, err := template.Executer.Execute(URL)
if err != nil {
gologger.Warning().Msgf("[%s] Could not execute step: %s\n", e.e.executerOpts.Colorizer.BrightBlue(template.ID), err)
}
e.results.CAS(false, match)
wg.Done()
}(template)
}
// ExecuteWithOpts executes with the full options
func (e *Engine) ChildExecuter() *ChildExecuter {
return &ChildExecuter{
e: e,
results: &atomic.Bool{},
}
}

17
v2/pkg/protocols/common/automaticscan/doc.go Normal file
View File

@ -0,0 +1,17 @@
// Package automaticscan implements automatic technology based template
// execution for a nuclei instance.
//
// First wappalyzer based technology detection is performed and templates
// are executed based on the results found. The results of wappalyzer
// technology detection are lowercased and split on space characters in the name,
// which are then used as tags for the execution of the templates.
//
// Example -
// "Amazon Web Services,Jenkins,Atlassian Jira" -> "amazon,web,services,jenkins,atlassian,jira".
//
// Wappalyzergo (https://github.com/projectdiscovery/wappalyzergo) is used for wappalyzer tech
// detection.
//
// The logic is very simple and can be further improved to increase the coverage of
// this mode of nuclei exection.
package automaticscan

189
v2/pkg/protocols/common/automaticscan/workflow.go Normal file
View File

@ -0,0 +1,189 @@
package automaticscan
import (
"io"
"io/ioutil"
"net/http"
"strings"
"github.com/corpix/uarand"
"github.com/pkg/errors"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader"
"github.com/projectdiscovery/nuclei/v2/pkg/core"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
"github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
"github.com/projectdiscovery/retryablehttp-go"
wappalyzer "github.com/projectdiscovery/wappalyzergo"
)
// Service is a service for automatic automatic scan execution
type Service struct {
opts protocols.ExecuterOptions
store *loader.Store
engine *core.Engine
target core.InputProvider
wappalyzer *wappalyzer.Wappalyze
childExecuter *core.ChildExecuter
httpclient *retryablehttp.Client
results bool
allTemplates []string
}
// Options contains configuration options for automatic scan service
type Options struct {
ExecuterOpts protocols.ExecuterOptions
Store *loader.Store
Engine *core.Engine
Target core.InputProvider
}
// New takes options and returns a new smart workflow service
func New(opts Options) (*Service, error) {
wappalyzer, err := wappalyzer.New()
if err != nil {
return nil, err
}
// Collect path for default directories we want to look for templates in
var allTemplates []string
for _, directory := range defaultTemplatesDirectories {
templates, err := opts.ExecuterOpts.Catalog.GetTemplatePath(directory)
if err != nil {
return nil, errors.Wrap(err, "could not get templates in directory")
}
allTemplates = append(allTemplates, templates...)
}
childExecuter := opts.Engine.ChildExecuter()
httpclient, err := httpclientpool.Get(opts.ExecuterOpts.Options, &httpclientpool.Configuration{
Connection: &httpclientpool.ConnectionConfiguration{DisableKeepAlive: true},
})
if err != nil {
return nil, errors.Wrap(err, "could not get http client")
}
return &Service{
opts: opts.ExecuterOpts,
store: opts.Store,
engine: opts.Engine,
target: opts.Target,
wappalyzer: wappalyzer,
allTemplates: allTemplates,
childExecuter: childExecuter,
httpclient: httpclient,
}, nil
}
// Close closes the service
func (s *Service) Close() bool {
results := s.childExecuter.Close()
if results.Load() {
s.results = true
}
return s.results
}
// Execute performs the execution of smart workflows on provided input
func (s *Service) Execute() {
if err := s.executeWappalyzerTechDetection(); err != nil {
gologger.Error().Msgf("Could not execute wappalyzer based detection: %s", err)
}
}
var (
defaultTemplatesDirectories = []string{"cves/", "default-logins/", "dns/", "exposures/", "miscellaneous/", "misconfiguration/", "network/", "takeovers/", "vulnerabilities/"}
)
const maxDefaultBody = 2 * 1024 * 1024
// executeWappalyzerTechDetection implements the logic to run the wappalyzer
// technologies detection on inputs which returns tech.
//
// The returned tags are then used for further execution.
func (s *Service) executeWappalyzerTechDetection() error {
gologger.Info().Msgf("Executing wappalyzer based tech detection on input urls")
// Iterate through each target making http request and identifying fingerprints
inputPool := s.engine.WorkPool().InputPool(types.HTTPProtocol)
s.target.Scan(func(value string) {
inputPool.WaitGroup.Add()
go func(input string) {
defer inputPool.WaitGroup.Done()
s.processWappalyzerInputPair(input)
}(value)
})
inputPool.WaitGroup.Wait()
return nil
}
func (s *Service) processWappalyzerInputPair(input string) {
req, err := retryablehttp.NewRequest(http.MethodGet, input, nil)
if err != nil {
return
}
req.Header.Set("User-Agent", uarand.GetRandom())
resp, err := s.httpclient.Do(req)
if err != nil {
if resp != nil {
resp.Body.Close()
}
return
}
reader := io.LimitReader(resp.Body, maxDefaultBody)
data, err := ioutil.ReadAll(reader)
if err != nil {
resp.Body.Close()
return
}
resp.Body.Close()
fingerprints := s.wappalyzer.Fingerprint(resp.Header, data)
items := make([]string, 0, len(fingerprints))
for k := range fingerprints {
if strings.Contains(k, " ") {
parts := strings.Split(strings.ToLower(k), " ")
items = append(items, parts...)
} else {
items = append(items, strings.ToLower(k))
}
}
if len(items) == 0 {
return
}
uniqueTags := uniqueSlice(items)
templatesList := s.store.LoadTemplatesWithTags(s.allTemplates, uniqueTags)
gologger.Info().Msgf("Executing tags (%v) for host %s (%d templates)", strings.Join(uniqueTags, ","), input, len(templatesList))
for _, t := range templatesList {
s.opts.Progress.AddToTotal(int64(t.Executer.Requests()))
if s.opts.Options.VerboseVerbose {
gologger.Print().Msgf("%s\n", templates.TemplateLogMessage(t.ID,
t.Info.Name,
t.Info.Authors.ToSlice(),
t.Info.SeverityHolder.Severity))
}
s.childExecuter.Execute(t, input)
}
}
func uniqueSlice(slice []string) []string {
data := make(map[string]struct{}, len(slice))
for _, item := range slice {
if _, ok := data[item]; !ok {
data[item] = struct{}{}
}
}
finalSlice := make([]string, 0, len(data))
for item := range data {
finalSlice = append(finalSlice, item)
}
return finalSlice
}

44
v2/pkg/templates/log.go Normal file
View File

@ -0,0 +1,44 @@
package templates
import (
"fmt"
"strings"
"github.com/logrusorgru/aurora"
"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
)
var (
Colorizer aurora.Aurora
SeverityColorizer func(severity.Severity) string
)
// TemplateLogMessage returns a beautified log string for a template
func TemplateLogMessage(id, name string, authors []string, templateSeverity severity.Severity) string {
if Colorizer == nil || SeverityColorizer == nil {
return ""
}
// Display the message for the template
return fmt.Sprintf("[%s] %s (%s) [%s]",
Colorizer.BrightBlue(id).String(),
Colorizer.Bold(name).String(),
Colorizer.BrightYellow(appendAtSignToAuthors(authors)).String(),
SeverityColorizer(templateSeverity))
}
// appendAtSignToAuthors appends @ before each author and returns the final string
func appendAtSignToAuthors(authors []string) string {
if len(authors) == 0 {
return "@none"
}
values := make([]string, 0, len(authors))
for _, k := range authors {
if !strings.HasPrefix(k, "@") {
values = append(values, fmt.Sprintf("@%s", k))
} else {
values = append(values, k)
}
}
return strings.Join(values, ",")
}

2
v2/internal/runner/templates_test.go → v2/pkg/templates/log_test.go
View File

@ -1,4 +1,4 @@
package runner
package templates
import (
"testing"

2
v2/pkg/types/types.go
View File

@ -147,6 +147,8 @@ type Options struct {
DebugResponse bool
// LeaveDefaultPorts skips normalization of default ports
LeaveDefaultPorts bool
// AutomaticScan enables automatic tech based template execution
AutomaticScan bool
// Silent suppresses any extra text and only writes found URLs on screen.
Silent bool
// Version specifies if we should just show version and exit