nuclei/v2/internal/runner/runner.go

316 lines
8.3 KiB
Go
Raw Normal View History

package runner
import (
"bufio"
"os"
"regexp"
"strings"
2020-08-23 18:46:18 +00:00
"github.com/logrusorgru/aurora"
"github.com/pkg/errors"
2020-11-06 02:15:27 +00:00
"github.com/projectdiscovery/fastdialer/fastdialer"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/hmap/store/hybrid"
2020-09-10 11:02:01 +00:00
"github.com/projectdiscovery/nuclei/v2/internal/bufwriter"
2020-07-23 18:19:19 +00:00
"github.com/projectdiscovery/nuclei/v2/internal/progress"
"github.com/projectdiscovery/nuclei/v2/internal/tracelog"
2020-07-24 16:12:16 +00:00
"github.com/projectdiscovery/nuclei/v2/pkg/atomicboolean"
2020-10-23 08:13:34 +00:00
"github.com/projectdiscovery/nuclei/v2/pkg/collaborator"
"github.com/projectdiscovery/nuclei/v2/pkg/colorizer"
2020-10-18 01:09:24 +00:00
"github.com/projectdiscovery/nuclei/v2/pkg/projectfile"
2020-07-01 10:47:24 +00:00
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
"github.com/projectdiscovery/nuclei/v2/pkg/workflows"
2020-10-18 01:09:24 +00:00
"github.com/remeh/sizedwaitgroup"
"go.uber.org/ratelimit"
)
// Runner is a client for running the enumeration process.
type Runner struct {
2020-07-23 18:19:19 +00:00
inputCount int64
traceLog tracelog.Log
2020-07-23 18:19:19 +00:00
2020-04-04 12:51:05 +00:00
// output is the output file to write if any
2020-09-11 16:05:29 +00:00
output *bufwriter.Writer
2020-06-24 22:23:37 +00:00
templatesConfig *nucleiConfig
2020-04-04 12:51:05 +00:00
// options contains configuration options for runner
options *Options
2020-07-23 18:19:19 +00:00
2020-10-18 01:09:24 +00:00
pf *projectfile.ProjectFile
2020-10-15 21:39:00 +00:00
2020-07-23 18:19:19 +00:00
// progress tracking
progress *progress.Progress
// output coloring
colorizer colorizer.NucleiColorizer
decolorizer *regexp.Regexp
2020-10-23 23:27:46 +00:00
// rate limiter
ratelimiter ratelimit.Limiter
// input deduplication
2020-11-18 22:54:33 +00:00
hm *hybrid.HybridMap
2020-11-06 02:15:27 +00:00
dialer *fastdialer.Dialer
}
// New creates a new client for running enumeration process.
func New(options *Options) (*Runner, error) {
runner := &Runner{
traceLog: &tracelog.NoopLogger{},
options: options,
}
if options.TraceLogFile != "" {
fileLog, err := tracelog.NewFileLogger(options.TraceLogFile)
if err != nil {
return nil, errors.Wrap(err, "could not create file trace logger")
}
runner.traceLog = fileLog
}
2020-04-04 11:42:29 +00:00
2020-06-24 22:23:37 +00:00
if err := runner.updateTemplates(); err != nil {
gologger.Labelf("Could not update templates: %s\n", err)
2020-06-24 22:23:37 +00:00
}
// output coloring
useColor := !options.NoColor
runner.colorizer = *colorizer.NewNucleiColorizer(aurora.NewAurora(useColor))
if useColor {
// compile a decolorization regex to cleanup file output messages
runner.decolorizer = regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`)
}
if options.TemplateList {
runner.listAvailableTemplates()
os.Exit(0)
}
if (len(options.Templates) == 0 || (options.Targets == "" && !options.Stdin && options.Target == "")) && options.UpdateTemplates {
2020-06-24 22:23:37 +00:00
os.Exit(0)
}
2020-08-23 18:46:18 +00:00
// Read nucleiignore file if given a templateconfig
if runner.templatesConfig != nil {
runner.readNucleiIgnoreFile()
}
2020-06-24 22:23:37 +00:00
if hm, err := hybrid.New(hybrid.DefaultDiskOptions); err != nil {
gologger.Fatalf("Could not create temporary input file: %s\n", err)
} else {
runner.hm = hm
}
runner.inputCount = 0
dupeCount := 0
// Handle single target
if options.Target != "" {
runner.inputCount++
2020-11-20 10:12:06 +00:00
// nolint:errcheck // ignoring error
runner.hm.Set(options.Target, nil)
}
// Handle stdin
if options.Stdin {
scanner := bufio.NewScanner(os.Stdin)
for scanner.Scan() {
url := strings.TrimSpace(scanner.Text())
// skip empty lines
if url == "" {
continue
}
// skip dupes
if _, ok := runner.hm.Get(url); ok {
dupeCount++
continue
}
runner.inputCount++
2020-11-20 10:12:06 +00:00
// nolint:errcheck // ignoring error
runner.hm.Set(url, nil)
}
2020-07-23 18:19:19 +00:00
}
// Handle taget file
if options.Targets != "" {
input, err := os.Open(options.Targets)
if err != nil {
gologger.Fatalf("Could not open targets file '%s': %s\n", options.Targets, err)
}
scanner := bufio.NewScanner(input)
for scanner.Scan() {
url := strings.TrimSpace(scanner.Text())
// skip empty lines
if url == "" {
continue
}
// skip dupes
if _, ok := runner.hm.Get(url); ok {
dupeCount++
continue
}
runner.inputCount++
2020-11-20 10:12:06 +00:00
// nolint:errcheck // ignoring error
runner.hm.Set(url, nil)
}
input.Close()
}
if dupeCount > 0 {
gologger.Labelf("Supplied input was automatically deduplicated (%d removed).", dupeCount)
2020-07-23 18:19:19 +00:00
}
2020-04-04 12:51:05 +00:00
// Create the output file if asked
if options.Output != "" {
2020-10-30 12:06:05 +00:00
output, errBufWriter := bufwriter.New(options.Output)
if errBufWriter != nil {
gologger.Fatalf("Could not create output file '%s': %s\n", options.Output, errBufWriter)
2020-04-04 12:51:05 +00:00
}
runner.output = output
}
2020-07-23 18:19:19 +00:00
// Creates the progress tracking object
runner.progress = progress.NewProgress(options.EnableProgressBar)
2020-07-23 18:19:19 +00:00
2020-10-17 00:10:47 +00:00
// create project file if requested or load existing one
if options.Project {
2020-10-30 12:06:05 +00:00
var projectFileErr error
runner.pf, projectFileErr = projectfile.New(&projectfile.Options{Path: options.ProjectPath, Cleanup: options.ProjectPath == ""})
if projectFileErr != nil {
return nil, projectFileErr
2020-10-15 21:39:00 +00:00
}
}
2020-10-23 08:13:34 +00:00
// Enable Polling
if options.BurpCollaboratorBiid != "" {
collaborator.DefaultCollaborator.Collab.AddBIID(options.BurpCollaboratorBiid)
}
2020-10-23 23:27:46 +00:00
// Create Dialer
var err error
2020-11-06 02:15:27 +00:00
runner.dialer, err = fastdialer.NewDialer(fastdialer.DefaultOptions)
2020-10-23 23:27:46 +00:00
if err != nil {
return nil, err
}
if options.RateLimit > 0 {
runner.ratelimiter = ratelimit.New(options.RateLimit)
} else {
runner.ratelimiter = ratelimit.NewUnlimited()
}
return runner, nil
}
// Close releases all the resources and cleans up
2020-04-04 12:51:05 +00:00
func (r *Runner) Close() {
2020-09-11 16:05:29 +00:00
if r.output != nil {
r.output.Close()
}
r.hm.Close()
2020-10-18 01:09:24 +00:00
if r.pf != nil {
r.pf.Close()
2020-10-15 21:39:00 +00:00
}
2020-04-04 12:51:05 +00:00
}
// RunEnumeration sets up the input layer for giving input nuclei.
// binary and runs the actual enumeration
func (r *Runner) RunEnumeration() {
2020-08-02 13:48:10 +00:00
// resolves input templates definitions and any optional exclusion
includedTemplates := r.getTemplatesFor(r.options.Templates)
excludedTemplates := r.getTemplatesFor(r.options.ExcludedTemplates)
// defaults to all templates
allTemplates := includedTemplates
2020-08-02 13:48:10 +00:00
if len(excludedTemplates) > 0 {
excludedMap := make(map[string]struct{}, len(excludedTemplates))
for _, excl := range excludedTemplates {
excludedMap[excl] = struct{}{}
}
// rebuild list with only non-excluded templates
allTemplates = []string{}
2020-08-02 13:48:10 +00:00
for _, incl := range includedTemplates {
if _, found := excludedMap[incl]; !found {
allTemplates = append(allTemplates, incl)
} else {
gologger.Warningf("Excluding '%s'", incl)
}
}
}
2020-08-02 16:33:55 +00:00
// pre-parse all the templates, apply filters
availableTemplates, workflowCount := r.getParsedTemplatesFor(allTemplates, r.options.Severity)
templateCount := len(availableTemplates)
hasWorkflows := workflowCount > 0
// 0 matches means no templates were found in directory
2020-08-02 16:33:55 +00:00
if templateCount == 0 {
gologger.Fatalf("Error, no templates were found.\n")
}
2020-08-02 16:33:55 +00:00
gologger.Infof("Using %s rules (%s templates, %s workflows)",
r.colorizer.Colorizer.Bold(templateCount).String(),
r.colorizer.Colorizer.Bold(templateCount-workflowCount).String(),
r.colorizer.Colorizer.Bold(workflowCount).String())
2020-07-23 18:19:19 +00:00
2020-07-25 21:22:09 +00:00
// precompute total request count
var totalRequests int64 = 0
2020-08-02 16:33:55 +00:00
for _, t := range availableTemplates {
switch av := t.(type) {
case *templates.Template:
totalRequests += (av.GetHTTPRequestCount() + av.GetDNSRequestCount()) * r.inputCount
2020-07-26 22:00:06 +00:00
case *workflows.Workflow:
// workflows will dynamically adjust the totals while running, as
// it can't be know in advance which requests will be called
} // nolint:wsl // comment
}
2020-07-23 18:19:19 +00:00
2020-10-17 00:10:47 +00:00
results := atomicboolean.New()
wgtemplates := sizedwaitgroup.New(r.options.TemplateThreads)
2020-10-23 08:13:34 +00:00
// Starts polling or ignore
collaborator.DefaultCollaborator.Poll()
2020-07-24 16:12:16 +00:00
if r.inputCount == 0 {
gologger.Errorf("Could not find any valid input URLs.")
} else if totalRequests > 0 || hasWorkflows {
// tracks global progress and captures stdout/stderr until p.Wait finishes
2020-08-02 16:33:55 +00:00
p := r.progress
p.Init(r.inputCount, templateCount, totalRequests)
2020-08-02 16:33:55 +00:00
for _, t := range availableTemplates {
2020-10-17 00:10:47 +00:00
wgtemplates.Add()
2020-08-02 16:33:55 +00:00
go func(template interface{}) {
defer wgtemplates.Done()
switch tt := template.(type) {
case *templates.Template:
for _, request := range tt.RequestsDNS {
2020-10-09 21:11:07 +00:00
results.Or(r.processTemplateWithList(p, tt, request))
}
for _, request := range tt.BulkRequestsHTTP {
2020-10-09 21:11:07 +00:00
results.Or(r.processTemplateWithList(p, tt, request))
}
case *workflows.Workflow:
results.Or(r.processWorkflowWithList(p, template.(*workflows.Workflow)))
2020-06-26 12:37:55 +00:00
}
2020-08-02 16:33:55 +00:00
}(t)
}
2020-07-24 16:12:16 +00:00
wgtemplates.Wait()
p.Stop()
}
2020-07-23 18:19:19 +00:00
2020-07-24 16:12:16 +00:00
if !results.Get() {
if r.output != nil {
r.output.Close()
2020-09-10 11:02:01 +00:00
os.Remove(r.options.Output)
}
gologger.Infof("No results found. Happy hacking!")
}
}