From 6d68b09863b17fdc1e13162b86f5b0268b170f44 Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Sat, 4 Jul 2020 23:00:11 +0200 Subject: [PATCH 01/51] Initial progress bar implementation --- go.mod | 19 ----- go.sum | 71 ------------------ v2/go.mod | 5 +- v2/internal/progress/progress.go | 86 ++++++++++++++++++++++ v2/internal/progress/stdcapture.go | 112 +++++++++++++++++++++++++++++ v2/internal/runner/runner.go | 12 +++- v2/pkg/executor/executer_http.go | 44 +++++++++++- v2/pkg/workflows/var.go | 6 +- 8 files changed, 259 insertions(+), 96 deletions(-) create mode 100644 v2/internal/progress/progress.go create mode 100644 v2/internal/progress/stdcapture.go diff --git a/go.mod b/go.mod index 21b12431..14e48f74 100644 --- a/go.mod +++ b/go.mod @@ -1,22 +1,3 @@ module github.com/projectdiscovery/nuclei go 1.14 - -require ( - github.com/Knetic/govaluate v3.0.0+incompatible - github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 - github.com/blang/semver v3.5.1+incompatible - github.com/d5/tengo v1.24.8 - github.com/d5/tengo/v2 v2.6.0 - github.com/google/go-github v17.0.0+incompatible - github.com/google/go-github/v32 v32.0.0 - github.com/json-iterator/go v1.1.10 - github.com/karrick/godirwalk v1.15.6 - github.com/miekg/dns v1.1.29 - github.com/pkg/errors v0.9.1 - github.com/projectdiscovery/gologger v1.0.0 - github.com/projectdiscovery/retryabledns v1.0.4 - github.com/projectdiscovery/retryablehttp-go v1.0.1 - golang.org/x/net v0.0.0-20200528225125-3c3fba18258b - gopkg.in/yaml.v2 v2.3.0 -) diff --git a/go.sum b/go.sum index f1241330..e69de29b 100644 --- a/go.sum +++ b/go.sum @@ -1,71 +0,0 @@ -github.com/Knetic/govaluate v1.5.0 h1:L4MyqdJSld9xr2eZcZHCWLfeIX2SBjqrwIKG1pcm/+4= -github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f6s9Tn1Tt7/WTEg= -github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= -github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0S6Vi7/lbWECcX0j45yZReDZ56BQsrVBOEEY= -github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= -github.com/blang/semver v1.1.0 h1:ol1rO7QQB5uy7umSNV7VAmLugfLRD+17sYJujRNYPhg= -github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= -github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/d5/tengo v1.24.8 h1:PRJ+NWt7ae/9sSbIfThOBTkPSvNV+dwYoBAvwfNgNJY= -github.com/d5/tengo v1.24.8/go.mod h1:VhLq8Q2QFhCIJO3NhvM934qOThykMqJi9y9Siqd1ocQ= -github.com/d5/tengo/v2 v2.6.0 h1:D0cJtpiBzaLJ/Smv6nnUc/LIfO46oKwDx85NZtIRNRI= -github.com/d5/tengo/v2 v2.6.0/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= -github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= -github.com/google/go-github/v32 v32.0.0 h1:q74KVb22spUq0U5HqZ9VCYqQz8YRuOtL/39ZnfwO+NM= -github.com/google/go-github/v32 v32.0.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI= -github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= -github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/karrick/godirwalk v1.15.6 h1:Yf2mmR8TJy+8Fa0SuQVto5SYap6IF7lNVX4Jdl8G1qA= -github.com/karrick/godirwalk v1.15.6/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= -github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 h1:bqDmpDG49ZRnB5PcgP0RXtQvnMSgIF14M7CBd2shtXs= -github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= -github.com/miekg/dns v1.1.29 h1:xHBEhR+t5RzcFJjBLJlax2daXOrTYtr9z4WdKEfWFzg= -github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/projectdiscovery/gologger v1.0.0 h1:XAQ8kHeVKXMjY4rLGh7eT5+oHU077BNEvs7X6n+vu1s= -github.com/projectdiscovery/gologger v1.0.0/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= -github.com/projectdiscovery/retryabledns v1.0.4 h1:0Va7qHlWQsIXjRLISTjzfN3tnJmHYDudY05Nu3IJd60= -github.com/projectdiscovery/retryabledns v1.0.4/go.mod h1:/UzJn4I+cPdQl6pKiiQfvVAT636YZvJQYZhYhGB0dUQ= -github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNwshuwptuBVYWw9lx8RE= -github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200528225125-3c3fba18258b h1:IYiJPiJfzktmDAO1HQiwjMjwjlYKHAL7KzeD544RJPs= -golang.org/x/net v0.0.0-20200528225125-3c3fba18258b/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/v2/go.mod b/v2/go.mod index bffaf20f..97475806 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -6,17 +6,18 @@ require ( github.com/Knetic/govaluate v3.0.0+incompatible github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 github.com/blang/semver v3.5.1+incompatible - github.com/d5/tengo v1.24.8 github.com/d5/tengo/v2 v2.6.0 - github.com/google/go-github v17.0.0+incompatible github.com/google/go-github/v32 v32.0.0 github.com/json-iterator/go v1.1.10 github.com/karrick/godirwalk v1.15.6 + github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 github.com/miekg/dns v1.1.29 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/gologger v1.0.0 github.com/projectdiscovery/retryabledns v1.0.4 github.com/projectdiscovery/retryablehttp-go v1.0.1 + github.com/stretchr/testify v1.5.1 + github.com/vbauerster/mpb/v5 v5.2.2 golang.org/x/net v0.0.0-20200528225125-3c3fba18258b gopkg.in/yaml.v2 v2.3.0 ) diff --git a/v2/internal/progress/progress.go b/v2/internal/progress/progress.go new file mode 100644 index 00000000..289c7d57 --- /dev/null +++ b/v2/internal/progress/progress.go @@ -0,0 +1,86 @@ +package progress + +import ( + "fmt" + "github.com/logrusorgru/aurora" + "github.com/vbauerster/mpb/v5" + "github.com/vbauerster/mpb/v5/cwriter" + "github.com/vbauerster/mpb/v5/decor" + "io" + "os" + "regexp" + "strconv" + "strings" + "sync" +) + +type Progress struct { + progress *mpb.Progress + captureData *captureData + termWidth int +} + +func NewProgress(group *sync.WaitGroup) *Progress { + w := cwriter.New(os.Stdout) + tw, err := w.GetWidth() + if err != nil { + panic("Couldn't determine available terminal width.") + } + + p := &Progress{ + progress: mpb.New( + mpb.WithWaitGroup(group), + mpb.WithOutput(os.Stderr), + mpb.PopCompletedMode(), + ), + termWidth: tw, + } + return p +} + +func (p *Progress) NewBar(name string, total int64, URL string) *mpb.Bar { + return p.progress.AddBar( + total, + mpb.BarNoPop(), + mpb.PrependDecorators( + decor.Name("[" + aurora.Green(URL).String() + " / " + aurora.Magenta(name).String() + "]"), + decor.CountersNoUnit(aurora.Blue(" %d/%d").String()), + decor.NewPercentage(aurora.Bold("%d").String(), decor.WCSyncSpace), + ), + mpb.AppendDecorators( + decor.EwmaSpeed(0, aurora.Yellow("%.2f req/s ").String(), 60), + decor.OnComplete( + decor.EwmaETA(decor.ET_STYLE_GO, 60), aurora.Bold("done!").String(), + ), + ), + ) +} + +func (p *Progress) Wait() { + p.progress.Wait() +} + +// + +func (p *Progress) StartStdCapture() { + p.captureData = startStdCapture() +} + +func (p *Progress) StopStdCaptureAndShow() { + stopStdCapture(p.captureData) + for _, captured := range p.captureData.Data { + var r = regexp.MustCompile("(.{" + strconv.Itoa(p.termWidth) + "})") + multiline := r.ReplaceAllString(captured, "$1\n") + arr := strings.Split(multiline, "\n") + + for _, msg := range arr { + p.progress.Add(0, makeLogBar(msg)).SetTotal(0, true) + } + } +} + +func makeLogBar(msg string) mpb.BarFiller { + return mpb.BarFillerFunc(func(w io.Writer, _ int, st decor.Statistics) { + fmt.Fprintf(w, msg) + }) +} diff --git a/v2/internal/progress/stdcapture.go b/v2/internal/progress/stdcapture.go new file mode 100644 index 00000000..3df05e97 --- /dev/null +++ b/v2/internal/progress/stdcapture.go @@ -0,0 +1,112 @@ +package progress + +/** + Inspired by the https://github.com/PumpkinSeed/cage module + */ +import ( + "bytes" + "context" + "io" + "os" + "strings" + "sync" + "time" +) + +type captureData struct { + backupStdout *os.File + writerStdout *os.File + backupStderr *os.File + writerStderr *os.File + + data string + channel chan string + + sync sync.WaitGroup + + Data []string +} + +var( + mutex = &sync.Mutex{} +) + +func startStdCapture() *captureData { + mutex.Lock() + + rStdout, wStdout, errStdout := os.Pipe() + if errStdout != nil { + panic(errStdout) + } + + rStderr, wStderr, errStderr := os.Pipe() + if errStderr != nil { + panic(errStderr) + } + + c := &captureData{ + backupStdout: os.Stdout, + writerStdout: wStdout, + + backupStderr: os.Stderr, + writerStderr: wStderr, + + channel: make(chan string), + } + + os.Stdout = c.writerStdout + os.Stderr = c.writerStderr + + c.sync.Add(2) + + go func( wg *sync.WaitGroup, out chan string, readerStdout *os.File, readerStderr *os.File) { + defer wg.Done() + + var bufStdout bytes.Buffer + _, _ = io.Copy(&bufStdout, readerStdout) + if bufStdout.Len() > 0 { + out <- bufStdout.String() + } + + var bufStderr bytes.Buffer + _, _ = io.Copy(&bufStderr, readerStderr) + if bufStderr.Len() > 0 { + out <- bufStderr.String() + } + }(&c.sync, c.channel, rStdout, rStderr) + + ctx, _ := context.WithTimeout(context.Background(), 50 * time.Millisecond) + //defer cancel() + + go func(ctx context.Context, wg *sync.WaitGroup, c *captureData) { + defer wg.Done() + + select { + case out := <-c.channel: + c.data += out + case <-ctx.Done(): + break + } + }(ctx, &c.sync, c) + + return c +} + +func stopStdCapture(c *captureData) { + _ = c.writerStdout.Close() + _ = c.writerStderr.Close() + + c.sync.Wait() + + close(c.channel) + + os.Stdout = c.backupStdout + os.Stderr = c.backupStderr + + c.Data = strings.Split(c.data, "\n") + if c.Data[len(c.Data)-1] == "" { + c.Data = c.Data[:len(c.Data)-1] + } + + mutex.Unlock() +} diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 4d2713d0..1d87f0b2 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "github.com/projectdiscovery/nuclei/v2/internal/progress" "io" "io/ioutil" "os" @@ -268,6 +269,9 @@ func (r *Runner) processTemplateWithList(template *templates.Template, request i limiter := make(chan struct{}, r.options.Threads) wg := &sync.WaitGroup{} + // track progress + p := progress.NewProgress(wg) + scanner := bufio.NewScanner(reader) for scanner.Scan() { text := scanner.Text() @@ -281,20 +285,24 @@ func (r *Runner) processTemplateWithList(template *templates.Template, request i var err error if httpExecutor != nil { - err = httpExecutor.ExecuteHTTP(URL) + err = httpExecutor.ExecuteHTTP(p, URL) } if dnsExecutor != nil { err = dnsExecutor.ExecuteDNS(URL) } if err != nil { + p.StartStdCapture() gologger.Warningf("Could not execute step: %s\n", err) + p.StopStdCaptureAndShow() } <-limiter wg.Done() }(text) } close(limiter) - wg.Wait() + + // Wait for both the WaitGroup and all the bars to complete + p.Wait() // See if we got any results from the executors var results bool diff --git a/v2/pkg/executor/executer_http.go b/v2/pkg/executor/executer_http.go index c743d47b..3986d4d5 100644 --- a/v2/pkg/executor/executer_http.go +++ b/v2/pkg/executor/executer_http.go @@ -4,6 +4,7 @@ import ( "bufio" "crypto/tls" "fmt" + "github.com/projectdiscovery/nuclei/v2/internal/progress" "io" "io/ioutil" "net/http" @@ -92,30 +93,46 @@ func (e *HTTPExecutor) GotResults() bool { return true } +func (e *HTTPExecutor) GetRequestCount() int64 { + return int64( len(e.httpRequest.Raw) | len(e.httpRequest.Path) ) +} + // ExecuteHTTP executes the HTTP request on a URL -func (e *HTTPExecutor) ExecuteHTTP(URL string) error { +func (e *HTTPExecutor) ExecuteHTTP(p *progress.Progress, URL string) error { // Compile each request for the template based on the URL compiledRequest, err := e.httpRequest.MakeHTTPRequest(URL) if err != nil { return errors.Wrap(err, "could not make http request") } + // track progress + bar := p.NewBar(e.template.ID, e.GetRequestCount(), URL) + // Send the request to the target servers mainLoop: for compiledRequest := range compiledRequest { + start := time.Now() + if compiledRequest.Error != nil { + bar.Abort(true) return errors.Wrap(err, "could not make http request") } e.setCustomHeaders(compiledRequest) req := compiledRequest.Request if e.debug { + p.StartStdCapture() gologger.Infof("Dumped HTTP request for %s (%s)\n\n", URL, e.template.ID) + p.StopStdCaptureAndShow() + dumpedRequest, err := httputil.DumpRequest(req.Request, true) if err != nil { + bar.Abort(true) return errors.Wrap(err, "could not dump http request") } + p.StartStdCapture() fmt.Fprintf(os.Stderr, "%s", string(dumpedRequest)) + p.StopStdCaptureAndShow() } resp, err := e.httpClient.Do(req) @@ -123,22 +140,30 @@ mainLoop: if resp != nil { resp.Body.Close() } + bar.Abort(true) return errors.Wrap(err, "could not make http request") } if e.debug { + p.StartStdCapture() gologger.Infof("Dumped HTTP response for %s (%s)\n\n", URL, e.template.ID) + p.StopStdCaptureAndShow() + dumpedResponse, err := httputil.DumpResponse(resp, true) if err != nil { + bar.Abort(true) return errors.Wrap(err, "could not dump http response") } + p.StartStdCapture() fmt.Fprintf(os.Stderr, "%s\n", string(dumpedResponse)) + p.StopStdCaptureAndShow() } data, err := ioutil.ReadAll(resp.Body) if err != nil { io.Copy(ioutil.Discard, resp.Body) resp.Body.Close() + bar.Abort(true) return errors.Wrap(err, "could not read http body") } resp.Body.Close() @@ -147,6 +172,7 @@ mainLoop: // so in case we have to manually do it data, err = requests.HandleDecompression(compiledRequest.Request, data) if err != nil { + bar.Abort(true) return errors.Wrap(err, "could not decompress http body") } @@ -166,13 +192,19 @@ mainLoop: if !matcher.Match(resp, body, headers) { // If the condition is AND we haven't matched, try next request. if matcherCondition == matchers.ANDCondition { + bar.IncrBy(1) + bar.DecoratorEwmaUpdate(time.Since(start)) continue mainLoop } } else { // If the matcher has matched, and its an OR // write the first output then move to next matcher. if matcherCondition == matchers.ORCondition && len(e.httpRequest.Extractors) == 0 { + // capture stdout and emit it via a mpb.BarFiller + p.StartStdCapture() e.writeOutputHTTP(compiledRequest, matcher, nil) + p.StopStdCaptureAndShow() + atomic.CompareAndSwapUint32(&e.results, 0, 1) } } @@ -194,16 +226,26 @@ mainLoop: // Write a final string of output if matcher type is // AND or if we have extractors for the mechanism too. if len(e.httpRequest.Extractors) > 0 || matcherCondition == matchers.ANDCondition { + // capture stdout and emit it via a mpb.BarFiller + p.StartStdCapture() e.writeOutputHTTP(compiledRequest, nil, extractorResults) + p.StopStdCaptureAndShow() + atomic.CompareAndSwapUint32(&e.results, 0, 1) } + + bar.Increment() + bar.DecoratorEwmaUpdate(time.Since(start)) } + p.StartStdCapture() gologger.Verbosef("Sent HTTP request to %s\n", "http-request", URL) + p.StopStdCaptureAndShow() return nil } + // Close closes the http executor for a template. func (e *HTTPExecutor) Close() { e.outputMutex.Lock() diff --git a/v2/pkg/workflows/var.go b/v2/pkg/workflows/var.go index 5d8897e4..80791cca 100644 --- a/v2/pkg/workflows/var.go +++ b/v2/pkg/workflows/var.go @@ -3,6 +3,7 @@ package workflows import ( "github.com/d5/tengo/v2" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/internal/progress" "github.com/projectdiscovery/nuclei/v2/pkg/executor" ) @@ -33,6 +34,9 @@ func (n *NucleiVar) CanCall() bool { func (n *NucleiVar) Call(args ...tengo.Object) (ret tengo.Object, err error) { var gotResult bool + // track progress + p := progress.NewProgress(nil) + for _, template := range n.Templates { if template.HTTPOptions != nil { for _, request := range template.HTTPOptions.Template.RequestsHTTP { @@ -42,7 +46,7 @@ func (n *NucleiVar) Call(args ...tengo.Object) (ret tengo.Object, err error) { gologger.Warningf("Could not compile request for template '%s': %s\n", template.HTTPOptions.Template.ID, err) continue } - err = httpExecutor.ExecuteHTTP(n.URL) + err = httpExecutor.ExecuteHTTP(p, n.URL) if err != nil { gologger.Warningf("Could not send request for template '%s': %s\n", template.HTTPOptions.Template.ID, err) continue From 9afd9bc4c2f3e4f04fb66317c0e3116e7a1459da Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Sun, 5 Jul 2020 17:17:04 +0200 Subject: [PATCH 02/51] Refactoring to compute workload in advance --- v2/internal/runner/runner.go | 93 ++++++++++++++++---------------- v2/pkg/executor/executer_http.go | 4 -- v2/pkg/requests/http-request.go | 5 ++ 3 files changed, 50 insertions(+), 52 deletions(-) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 1d87f0b2..710f58e8 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -5,24 +5,26 @@ import ( "context" "errors" "fmt" + "github.com/d5/tengo/v2" + "github.com/karrick/godirwalk" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/internal/progress" + "github.com/projectdiscovery/nuclei/v2/pkg/executor" + "github.com/projectdiscovery/nuclei/v2/pkg/requests" + "github.com/projectdiscovery/nuclei/v2/pkg/templates" + "github.com/projectdiscovery/nuclei/v2/pkg/workflows" "io" "io/ioutil" "os" "strings" "sync" - - "github.com/d5/tengo/v2" - "github.com/karrick/godirwalk" - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v2/pkg/executor" - "github.com/projectdiscovery/nuclei/v2/pkg/requests" - "github.com/projectdiscovery/nuclei/v2/pkg/templates" - "github.com/projectdiscovery/nuclei/v2/pkg/workflows" ) // Runner is a client for running the enumeration process. type Runner struct { + input *os.File + inputCount int64 + // output is the output file to write if any output *os.File outputMutex *sync.Mutex @@ -70,6 +72,25 @@ func New(options *Options) (*Runner, error) { tempInput.Close() } + // Setup input, handle a list of hosts as argument + var err error + if options.Targets != "" { + runner.input, err = os.Open(options.Targets) + } else if options.Stdin || options.Target != "" { + runner.input, err = os.Open(runner.tempFile) + } + if err != nil { + gologger.Fatalf("Could not open targets file '%s': %s\n", options.Targets, err) + } + + // Precompute total number of targets + scanner := bufio.NewScanner(runner.input) + runner.inputCount = 0 + for scanner.Scan() { + runner.inputCount++ + } + runner.input.Seek(0,0) + // Create the output file if asked if options.Output != "" { output, err := os.Create(options.Output) @@ -84,9 +105,18 @@ func New(options *Options) (*Runner, error) { // Close releases all the resources and cleans up func (r *Runner) Close() { r.output.Close() + r.input.Close() os.Remove(r.tempFile) } +func (r *Runner) getHTTPRequestsCount(t *templates.Template) int64 { + var count int64 = 0 + for _, request := range t.RequestsHTTP { + count += request.GetRequestCount() + } + return count +} + // RunEnumeration sets up the input layer for giving input nuclei. // binary and runs the actual enumeration func (r *Runner) RunEnumeration() { @@ -196,30 +226,11 @@ func (r *Runner) RunEnumeration() { } gologger.Infof("No results found for the template. Happy hacking!") } - return + } -// processTemplate processes a template and runs the enumeration on all the targets -func (r *Runner) processTemplateRequest(template *templates.Template, request interface{}) bool { - var file *os.File - var err error - - // Handle a list of hosts as argument - if r.options.Targets != "" { - file, err = os.Open(r.options.Targets) - } else if r.options.Stdin || r.options.Target != "" { - file, err = os.Open(r.tempFile) - } - if err != nil { - gologger.Fatalf("Could not open targets file '%s': %s\n", r.options.Targets, err) - } - results := r.processTemplateWithList(template, request, file) - file.Close() - return results -} - -// processDomain processes the list with a template -func (r *Runner) processTemplateWithList(template *templates.Template, request interface{}, reader io.Reader) bool { +// processTemplateWithList processes a template and runs the enumeration on all the targets +func (r *Runner) processTemplateWithList(p *progress.Progress, template *templates.Template, request interface{}) bool { // Display the message for the template message := fmt.Sprintf("[%s] Loaded template %s (@%s)", template.ID, template.Info.Name, template.Info.Author) if template.Info.Severity != "" { @@ -269,10 +280,8 @@ func (r *Runner) processTemplateWithList(template *templates.Template, request i limiter := make(chan struct{}, r.options.Threads) wg := &sync.WaitGroup{} - // track progress - p := progress.NewProgress(wg) - - scanner := bufio.NewScanner(reader) + r.input.Seek(0, 0) + scanner := bufio.NewScanner(r.input) for scanner.Scan() { text := scanner.Text() if text == "" { @@ -319,20 +328,8 @@ func (r *Runner) processTemplateWithList(template *templates.Template, request i // ProcessWorkflowWithList coming from stdin or list of targets func (r *Runner) ProcessWorkflowWithList(workflow *workflows.Workflow) { - var file *os.File - var err error - // Handle a list of hosts as argument - if r.options.Targets != "" { - file, err = os.Open(r.options.Targets) - } else if r.options.Stdin { - file, err = os.Open(r.tempFile) - } - if err != nil { - gologger.Fatalf("Could not open targets file '%s': %s\n", r.options.Targets, err) - } - defer file.Close() - - scanner := bufio.NewScanner(file) + r.input.Seek(0, 0) + scanner := bufio.NewScanner(r.input) for scanner.Scan() { text := scanner.Text() if text == "" { diff --git a/v2/pkg/executor/executer_http.go b/v2/pkg/executor/executer_http.go index 3986d4d5..9cbefa8e 100644 --- a/v2/pkg/executor/executer_http.go +++ b/v2/pkg/executor/executer_http.go @@ -93,10 +93,6 @@ func (e *HTTPExecutor) GotResults() bool { return true } -func (e *HTTPExecutor) GetRequestCount() int64 { - return int64( len(e.httpRequest.Raw) | len(e.httpRequest.Path) ) -} - // ExecuteHTTP executes the HTTP request on a URL func (e *HTTPExecutor) ExecuteHTTP(p *progress.Progress, URL string) error { // Compile each request for the template based on the URL diff --git a/v2/pkg/requests/http-request.go b/v2/pkg/requests/http-request.go index b6f28342..525afa83 100644 --- a/v2/pkg/requests/http-request.go +++ b/v2/pkg/requests/http-request.go @@ -72,6 +72,11 @@ func (r *HTTPRequest) SetAttackType(attack generators.Type) { r.attackType = attack } +// Returns the total number of requests the YAML rule will perform +func (r *HTTPRequest) GetRequestCount() int64 { + return int64( len(r.Raw) | len(r.Path) ) +} + // MakeHTTPRequest creates a *http.Request from a request configuration func (r *HTTPRequest) MakeHTTPRequest(baseURL string) (chan *CompiledHTTP, error) { parsed, err := url.Parse(baseURL) From e59ac01c6567261bd271bef872999f4dc2bab768 Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Sun, 5 Jul 2020 17:22:21 +0200 Subject: [PATCH 03/51] Experimental single progress bar --- v2/internal/progress/progress.go | 9 +++++++-- v2/internal/runner/runner.go | 32 ++++++++++++++++++++++++++------ v2/pkg/executor/executer_http.go | 23 ++++++++++------------- 3 files changed, 43 insertions(+), 21 deletions(-) diff --git a/v2/internal/progress/progress.go b/v2/internal/progress/progress.go index 289c7d57..59b0b446 100644 --- a/v2/internal/progress/progress.go +++ b/v2/internal/progress/progress.go @@ -16,6 +16,7 @@ import ( type Progress struct { progress *mpb.Progress + Bar *mpb.Bar captureData *captureData termWidth int } @@ -34,16 +35,20 @@ func NewProgress(group *sync.WaitGroup) *Progress { mpb.PopCompletedMode(), ), termWidth: tw, + Bar: nil, } return p } -func (p *Progress) NewBar(name string, total int64, URL string) *mpb.Bar { +func (p *Progress) NewBar(name string, total int64) *mpb.Bar { + barname := "[" + aurora.Green(name).String() + "]" + return p.progress.AddBar( total, mpb.BarNoPop(), + //mpb.BarQueueAfter(p.Bar), mpb.PrependDecorators( - decor.Name("[" + aurora.Green(URL).String() + " / " + aurora.Magenta(name).String() + "]"), + decor.Name(barname), decor.CountersNoUnit(aurora.Blue(" %d/%d").String()), decor.NewPercentage(aurora.Bold("%d").String(), decor.WCSyncSpace), ), diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 710f58e8..01ed081b 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -133,6 +133,9 @@ func (r *Runner) RunEnumeration() { r.options.Templates = newPath } + // track progress + p := progress.NewProgress(nil) + // Single yaml provided if strings.HasSuffix(r.options.Templates, ".yaml") { t, err := r.parse(r.options.Templates) @@ -140,13 +143,17 @@ func (r *Runner) RunEnumeration() { case *templates.Template: var results bool template := t.(*templates.Template) + + p.Bar = p.NewBar(template.ID, r.inputCount * r.getHTTPRequestsCount(template)) + // process http requests for _, request := range template.RequestsHTTP { - results = r.processTemplateRequest(template, request) + results = r.processTemplateWithList(p, template, request) } + // process dns requests for _, request := range template.RequestsDNS { - dnsResults := r.processTemplateRequest(template, request) + dnsResults := r.processTemplateWithList(p, template, request) if !results { results = dnsResults } @@ -200,22 +207,28 @@ func (r *Runner) RunEnumeration() { case *templates.Template: template := t.(*templates.Template) for _, request := range template.RequestsDNS { - dnsResults := r.processTemplateRequest(template, request) + dnsResults := r.processTemplateWithList(p, template, request) if dnsResults { results = dnsResults } } + + p.Bar = p.NewBar(template.ID, r.inputCount * r.getHTTPRequestsCount(template)) + for _, request := range template.RequestsHTTP { - httpResults := r.processTemplateRequest(template, request) + httpResults := r.processTemplateWithList(p, template, request) if httpResults { results = httpResults } } + case *workflows.Workflow: workflow := t.(*workflows.Workflow) r.ProcessWorkflowWithList(workflow) default: + p.StartStdCapture() gologger.Errorf("Could not parse file '%s': %s\n", r.options.Templates, err) + p.StopStdCaptureAndShow() } } if !results { @@ -224,9 +237,13 @@ func (r *Runner) RunEnumeration() { r.output.Close() os.Remove(outputFile) } + p.StartStdCapture() gologger.Infof("No results found for the template. Happy hacking!") + p.StopStdCaptureAndShow() } + p.Wait() + return } // processTemplateWithList processes a template and runs the enumeration on all the targets @@ -236,7 +253,9 @@ func (r *Runner) processTemplateWithList(p *progress.Progress, template *templat if template.Info.Severity != "" { message += " [" + template.Info.Severity + "]" } + p.StartStdCapture() gologger.Infof("%s\n", message) + p.StopStdCaptureAndShow() var writer *bufio.Writer if r.output != nil { @@ -273,7 +292,9 @@ func (r *Runner) processTemplateWithList(p *progress.Progress, template *templat }) } if err != nil { + p.StartStdCapture() gologger.Warningf("Could not create http client: %s\n", err) + p.StopStdCaptureAndShow() return false } @@ -310,8 +331,7 @@ func (r *Runner) processTemplateWithList(p *progress.Progress, template *templat } close(limiter) - // Wait for both the WaitGroup and all the bars to complete - p.Wait() + wg.Wait() // See if we got any results from the executors var results bool diff --git a/v2/pkg/executor/executer_http.go b/v2/pkg/executor/executer_http.go index 9cbefa8e..77c3a9da 100644 --- a/v2/pkg/executor/executer_http.go +++ b/v2/pkg/executor/executer_http.go @@ -101,16 +101,13 @@ func (e *HTTPExecutor) ExecuteHTTP(p *progress.Progress, URL string) error { return errors.Wrap(err, "could not make http request") } - // track progress - bar := p.NewBar(e.template.ID, e.GetRequestCount(), URL) - // Send the request to the target servers mainLoop: for compiledRequest := range compiledRequest { start := time.Now() if compiledRequest.Error != nil { - bar.Abort(true) + p.Bar.Abort(true) return errors.Wrap(err, "could not make http request") } e.setCustomHeaders(compiledRequest) @@ -123,7 +120,7 @@ mainLoop: dumpedRequest, err := httputil.DumpRequest(req.Request, true) if err != nil { - bar.Abort(true) + p.Bar.Abort(true) return errors.Wrap(err, "could not dump http request") } p.StartStdCapture() @@ -136,7 +133,7 @@ mainLoop: if resp != nil { resp.Body.Close() } - bar.Abort(true) + p.Bar.Abort(true) return errors.Wrap(err, "could not make http request") } @@ -147,7 +144,7 @@ mainLoop: dumpedResponse, err := httputil.DumpResponse(resp, true) if err != nil { - bar.Abort(true) + p.Bar.Abort(true) return errors.Wrap(err, "could not dump http response") } p.StartStdCapture() @@ -159,7 +156,7 @@ mainLoop: if err != nil { io.Copy(ioutil.Discard, resp.Body) resp.Body.Close() - bar.Abort(true) + p.Bar.Abort(true) return errors.Wrap(err, "could not read http body") } resp.Body.Close() @@ -168,7 +165,7 @@ mainLoop: // so in case we have to manually do it data, err = requests.HandleDecompression(compiledRequest.Request, data) if err != nil { - bar.Abort(true) + p.Bar.Abort(true) return errors.Wrap(err, "could not decompress http body") } @@ -188,8 +185,8 @@ mainLoop: if !matcher.Match(resp, body, headers) { // If the condition is AND we haven't matched, try next request. if matcherCondition == matchers.ANDCondition { - bar.IncrBy(1) - bar.DecoratorEwmaUpdate(time.Since(start)) + p.Bar.IncrBy(1) + p.Bar.DecoratorEwmaUpdate(time.Since(start)) continue mainLoop } } else { @@ -230,8 +227,8 @@ mainLoop: atomic.CompareAndSwapUint32(&e.results, 0, 1) } - bar.Increment() - bar.DecoratorEwmaUpdate(time.Since(start)) + p.Bar.Increment() + p.Bar.DecoratorEwmaUpdate(time.Since(start)) } p.StartStdCapture() From 683de150bdcb3673b8cb9ad41901bcb83beb8f3f Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Sun, 5 Jul 2020 20:11:53 +0200 Subject: [PATCH 04/51] Remove any completed bar so far --- v2/internal/progress/progress.go | 1 + 1 file changed, 1 insertion(+) diff --git a/v2/internal/progress/progress.go b/v2/internal/progress/progress.go index 59b0b446..e520f135 100644 --- a/v2/internal/progress/progress.go +++ b/v2/internal/progress/progress.go @@ -47,6 +47,7 @@ func (p *Progress) NewBar(name string, total int64) *mpb.Bar { total, mpb.BarNoPop(), //mpb.BarQueueAfter(p.Bar), + mpb.BarRemoveOnComplete(), mpb.PrependDecorators( decor.Name(barname), decor.CountersNoUnit(aurora.Blue(" %d/%d").String()), From 02bf957c1385e8d7424cd34598dec0e5d5cd2e38 Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Sun, 5 Jul 2020 21:39:04 +0200 Subject: [PATCH 05/51] Better handling of the timeout context --- v2/internal/progress/stdcapture.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/v2/internal/progress/stdcapture.go b/v2/internal/progress/stdcapture.go index 3df05e97..ac24b52a 100644 --- a/v2/internal/progress/stdcapture.go +++ b/v2/internal/progress/stdcapture.go @@ -75,19 +75,19 @@ func startStdCapture() *captureData { } }(&c.sync, c.channel, rStdout, rStderr) - ctx, _ := context.WithTimeout(context.Background(), 50 * time.Millisecond) - //defer cancel() - - go func(ctx context.Context, wg *sync.WaitGroup, c *captureData) { - defer wg.Done() + go func(wg *sync.WaitGroup, c *captureData) { + ctx, cancel := context.WithTimeout(context.Background(), 10 * time.Millisecond) + defer cancel() select { case out := <-c.channel: c.data += out + wg.Done() case <-ctx.Done(): + wg.Done() break } - }(ctx, &c.sync, c) + }(&c.sync, c) return c } From d9031156348905722c7327638379cf49d55adc3e Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Sun, 5 Jul 2020 22:46:34 +0200 Subject: [PATCH 06/51] Use simple average instead of ewma-based, reduce cpu/mem footprint --- v2/internal/progress/progress.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/internal/progress/progress.go b/v2/internal/progress/progress.go index e520f135..a615e4de 100644 --- a/v2/internal/progress/progress.go +++ b/v2/internal/progress/progress.go @@ -54,9 +54,9 @@ func (p *Progress) NewBar(name string, total int64) *mpb.Bar { decor.NewPercentage(aurora.Bold("%d").String(), decor.WCSyncSpace), ), mpb.AppendDecorators( - decor.EwmaSpeed(0, aurora.Yellow("%.2f req/s ").String(), 60), + decor.AverageSpeed(0, aurora.Yellow("%.2f req/s ").String()), decor.OnComplete( - decor.EwmaETA(decor.ET_STYLE_GO, 60), aurora.Bold("done!").String(), + decor.AverageETA(decor.ET_STYLE_GO), aurora.Bold("done!").String(), ), ), ) From 3a4d7ba3e132b37854f0f0a9d3f0e76f9582fb3a Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Sun, 5 Jul 2020 22:47:07 +0200 Subject: [PATCH 07/51] No need to abort the only running bar anymore --- v2/pkg/executor/executer_http.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/v2/pkg/executor/executer_http.go b/v2/pkg/executor/executer_http.go index 77c3a9da..f531b7ac 100644 --- a/v2/pkg/executor/executer_http.go +++ b/v2/pkg/executor/executer_http.go @@ -107,7 +107,6 @@ mainLoop: start := time.Now() if compiledRequest.Error != nil { - p.Bar.Abort(true) return errors.Wrap(err, "could not make http request") } e.setCustomHeaders(compiledRequest) @@ -120,7 +119,6 @@ mainLoop: dumpedRequest, err := httputil.DumpRequest(req.Request, true) if err != nil { - p.Bar.Abort(true) return errors.Wrap(err, "could not dump http request") } p.StartStdCapture() @@ -133,7 +131,6 @@ mainLoop: if resp != nil { resp.Body.Close() } - p.Bar.Abort(true) return errors.Wrap(err, "could not make http request") } @@ -144,7 +141,6 @@ mainLoop: dumpedResponse, err := httputil.DumpResponse(resp, true) if err != nil { - p.Bar.Abort(true) return errors.Wrap(err, "could not dump http response") } p.StartStdCapture() @@ -156,7 +152,6 @@ mainLoop: if err != nil { io.Copy(ioutil.Discard, resp.Body) resp.Body.Close() - p.Bar.Abort(true) return errors.Wrap(err, "could not read http body") } resp.Body.Close() @@ -165,7 +160,6 @@ mainLoop: // so in case we have to manually do it data, err = requests.HandleDecompression(compiledRequest.Request, data) if err != nil { - p.Bar.Abort(true) return errors.Wrap(err, "could not decompress http body") } From 20758a14a5975f9c9d46014bb7255914e2bf3b60 Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Sun, 5 Jul 2020 22:48:05 +0200 Subject: [PATCH 08/51] Use shortcut method --- v2/pkg/executor/executer_http.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/executor/executer_http.go b/v2/pkg/executor/executer_http.go index f531b7ac..aac62f1a 100644 --- a/v2/pkg/executor/executer_http.go +++ b/v2/pkg/executor/executer_http.go @@ -179,7 +179,7 @@ mainLoop: if !matcher.Match(resp, body, headers) { // If the condition is AND we haven't matched, try next request. if matcherCondition == matchers.ANDCondition { - p.Bar.IncrBy(1) + p.Bar.Increment() p.Bar.DecoratorEwmaUpdate(time.Since(start)) continue mainLoop } From 6afd9490606231d4f3327fb753aa20203337eb4d Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Sun, 5 Jul 2020 23:21:20 +0200 Subject: [PATCH 09/51] Clean up --- v2/internal/progress/progress.go | 1 - 1 file changed, 1 deletion(-) diff --git a/v2/internal/progress/progress.go b/v2/internal/progress/progress.go index a615e4de..2e92bb40 100644 --- a/v2/internal/progress/progress.go +++ b/v2/internal/progress/progress.go @@ -46,7 +46,6 @@ func (p *Progress) NewBar(name string, total int64) *mpb.Bar { return p.progress.AddBar( total, mpb.BarNoPop(), - //mpb.BarQueueAfter(p.Bar), mpb.BarRemoveOnComplete(), mpb.PrependDecorators( decor.Name(barname), From d03fbd1932304955a279ebb2daf9d51a5d8cb06a Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Sun, 5 Jul 2020 23:38:58 +0200 Subject: [PATCH 10/51] Encapsulate behavior and clean up --- v2/internal/progress/progress.go | 15 ++++++++++----- v2/internal/runner/runner.go | 4 ++-- v2/pkg/executor/executer_http.go | 8 ++------ 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/v2/internal/progress/progress.go b/v2/internal/progress/progress.go index 2e92bb40..cf3091a8 100644 --- a/v2/internal/progress/progress.go +++ b/v2/internal/progress/progress.go @@ -16,7 +16,7 @@ import ( type Progress struct { progress *mpb.Progress - Bar *mpb.Bar + bar *mpb.Bar captureData *captureData termWidth int } @@ -35,15 +35,13 @@ func NewProgress(group *sync.WaitGroup) *Progress { mpb.PopCompletedMode(), ), termWidth: tw, - Bar: nil, } return p } -func (p *Progress) NewBar(name string, total int64) *mpb.Bar { +func (p *Progress) SetupProgressBar(name string, total int64) *mpb.Bar { barname := "[" + aurora.Green(name).String() + "]" - - return p.progress.AddBar( + bar := p.progress.AddBar( total, mpb.BarNoPop(), mpb.BarRemoveOnComplete(), @@ -59,6 +57,13 @@ func (p *Progress) NewBar(name string, total int64) *mpb.Bar { ), ), ) + + p.bar = bar + return bar +} + +func (p *Progress) Update() { + p.bar.Increment() } func (p *Progress) Wait() { diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 01ed081b..57706cc6 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -144,7 +144,7 @@ func (r *Runner) RunEnumeration() { var results bool template := t.(*templates.Template) - p.Bar = p.NewBar(template.ID, r.inputCount * r.getHTTPRequestsCount(template)) + p.SetupProgressBar(template.ID, r.inputCount * r.getHTTPRequestsCount(template)) // process http requests for _, request := range template.RequestsHTTP { @@ -213,7 +213,7 @@ func (r *Runner) RunEnumeration() { } } - p.Bar = p.NewBar(template.ID, r.inputCount * r.getHTTPRequestsCount(template)) + p.SetupProgressBar(template.ID, r.inputCount * r.getHTTPRequestsCount(template)) for _, request := range template.RequestsHTTP { httpResults := r.processTemplateWithList(p, template, request) diff --git a/v2/pkg/executor/executer_http.go b/v2/pkg/executor/executer_http.go index aac62f1a..5de13370 100644 --- a/v2/pkg/executor/executer_http.go +++ b/v2/pkg/executor/executer_http.go @@ -104,8 +104,6 @@ func (e *HTTPExecutor) ExecuteHTTP(p *progress.Progress, URL string) error { // Send the request to the target servers mainLoop: for compiledRequest := range compiledRequest { - start := time.Now() - if compiledRequest.Error != nil { return errors.Wrap(err, "could not make http request") } @@ -179,8 +177,7 @@ mainLoop: if !matcher.Match(resp, body, headers) { // If the condition is AND we haven't matched, try next request. if matcherCondition == matchers.ANDCondition { - p.Bar.Increment() - p.Bar.DecoratorEwmaUpdate(time.Since(start)) + p.Update() continue mainLoop } } else { @@ -221,8 +218,7 @@ mainLoop: atomic.CompareAndSwapUint32(&e.results, 0, 1) } - p.Bar.Increment() - p.Bar.DecoratorEwmaUpdate(time.Since(start)) + p.Update() } p.StartStdCapture() From 0b87f339b98a5698dca2deac3e8fff9ab40dede4 Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Sun, 5 Jul 2020 23:39:29 +0200 Subject: [PATCH 11/51] Update .gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 6a65b8cf..f78e3f1f 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -cmd/nuclei/nuclei* +v2/cmd/nuclei/nuclei* +.idea \ No newline at end of file From d7b440a4ac561da5faf1b776a671d3583da48ebf Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Sun, 5 Jul 2020 23:52:47 +0200 Subject: [PATCH 12/51] Disambiguate error message --- v2/pkg/executor/executer_http.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/pkg/executor/executer_http.go b/v2/pkg/executor/executer_http.go index 5de13370..664fd44c 100644 --- a/v2/pkg/executor/executer_http.go +++ b/v2/pkg/executor/executer_http.go @@ -98,14 +98,14 @@ func (e *HTTPExecutor) ExecuteHTTP(p *progress.Progress, URL string) error { // Compile each request for the template based on the URL compiledRequest, err := e.httpRequest.MakeHTTPRequest(URL) if err != nil { - return errors.Wrap(err, "could not make http request") + return errors.Wrap(err, "could not compile http request") } // Send the request to the target servers mainLoop: for compiledRequest := range compiledRequest { if compiledRequest.Error != nil { - return errors.Wrap(err, "could not make http request") + return errors.Wrap(err, "error in compiled http request") } e.setCustomHeaders(compiledRequest) req := compiledRequest.Request @@ -129,7 +129,7 @@ mainLoop: if resp != nil { resp.Body.Close() } - return errors.Wrap(err, "could not make http request") + return errors.Wrap(err, "could not issue http request") } if e.debug { From 6f894b718a8798359af1fb733d812999417c6295 Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Mon, 6 Jul 2020 00:09:58 +0200 Subject: [PATCH 13/51] Abort on error, bar will be properly recreated --- v2/internal/progress/progress.go | 4 ++++ v2/pkg/executor/executer_http.go | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/v2/internal/progress/progress.go b/v2/internal/progress/progress.go index cf3091a8..b9a6e9d0 100644 --- a/v2/internal/progress/progress.go +++ b/v2/internal/progress/progress.go @@ -66,6 +66,10 @@ func (p *Progress) Update() { p.bar.Increment() } +func (p *Progress) Abort() { + p.bar.Abort(true) +} + func (p *Progress) Wait() { p.progress.Wait() } diff --git a/v2/pkg/executor/executer_http.go b/v2/pkg/executor/executer_http.go index 664fd44c..59c7a67a 100644 --- a/v2/pkg/executor/executer_http.go +++ b/v2/pkg/executor/executer_http.go @@ -105,6 +105,7 @@ func (e *HTTPExecutor) ExecuteHTTP(p *progress.Progress, URL string) error { mainLoop: for compiledRequest := range compiledRequest { if compiledRequest.Error != nil { + p.Abort() return errors.Wrap(err, "error in compiled http request") } e.setCustomHeaders(compiledRequest) @@ -117,6 +118,7 @@ mainLoop: dumpedRequest, err := httputil.DumpRequest(req.Request, true) if err != nil { + p.Abort() return errors.Wrap(err, "could not dump http request") } p.StartStdCapture() @@ -129,6 +131,7 @@ mainLoop: if resp != nil { resp.Body.Close() } + p.Abort() return errors.Wrap(err, "could not issue http request") } @@ -139,6 +142,7 @@ mainLoop: dumpedResponse, err := httputil.DumpResponse(resp, true) if err != nil { + p.Abort() return errors.Wrap(err, "could not dump http response") } p.StartStdCapture() @@ -150,6 +154,7 @@ mainLoop: if err != nil { io.Copy(ioutil.Discard, resp.Body) resp.Body.Close() + p.Abort() return errors.Wrap(err, "could not read http body") } resp.Body.Close() @@ -158,6 +163,7 @@ mainLoop: // so in case we have to manually do it data, err = requests.HandleDecompression(compiledRequest.Request, data) if err != nil { + p.Abort() return errors.Wrap(err, "could not decompress http body") } From 9e1b39549f04516cef3ee0716fef78932fa15d1d Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Tue, 7 Jul 2020 22:39:43 +0200 Subject: [PATCH 14/51] Minor refactoring, track count of aborted requests for better progress --- v2/internal/progress/progress.go | 13 +++++++++++-- v2/internal/runner/runner.go | 31 +++++++++++++++---------------- v2/pkg/executor/executer_http.go | 16 ++++++++++------ v2/pkg/templates/templates.go | 8 ++++++++ 4 files changed, 44 insertions(+), 24 deletions(-) diff --git a/v2/internal/progress/progress.go b/v2/internal/progress/progress.go index b9a6e9d0..73f43d17 100644 --- a/v2/internal/progress/progress.go +++ b/v2/internal/progress/progress.go @@ -17,6 +17,8 @@ import ( type Progress struct { progress *mpb.Progress bar *mpb.Bar + total int64 + initialTotal int64 captureData *captureData termWidth int } @@ -59,6 +61,8 @@ func (p *Progress) SetupProgressBar(name string, total int64) *mpb.Bar { ) p.bar = bar + p.total = total + p.initialTotal = total return bar } @@ -66,11 +70,16 @@ func (p *Progress) Update() { p.bar.Increment() } -func (p *Progress) Abort() { - p.bar.Abort(true) +func (p *Progress) Abort(remaining int64) { + p.total -= remaining + p.bar.SetTotal(p.total, false) } func (p *Progress) Wait() { + if p.initialTotal != p.total { + p.bar.SetTotal(p.total, true) + } + p.progress.Wait() } diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 57706cc6..949ad247 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -109,14 +109,6 @@ func (r *Runner) Close() { os.Remove(r.tempFile) } -func (r *Runner) getHTTPRequestsCount(t *templates.Template) int64 { - var count int64 = 0 - for _, request := range t.RequestsHTTP { - count += request.GetRequestCount() - } - return count -} - // RunEnumeration sets up the input layer for giving input nuclei. // binary and runs the actual enumeration func (r *Runner) RunEnumeration() { @@ -133,18 +125,19 @@ func (r *Runner) RunEnumeration() { r.options.Templates = newPath } - // track progress - p := progress.NewProgress(nil) - // Single yaml provided if strings.HasSuffix(r.options.Templates, ".yaml") { t, err := r.parse(r.options.Templates) + + // track progress + p := progress.NewProgress(nil) + switch t.(type) { case *templates.Template: var results bool template := t.(*templates.Template) - p.SetupProgressBar(template.ID, r.inputCount * r.getHTTPRequestsCount(template)) + p.SetupProgressBar(template.ID, r.inputCount * template.GetHTTPRequestsCount()) // process http requests for _, request := range template.RequestsHTTP { @@ -159,6 +152,8 @@ func (r *Runner) RunEnumeration() { } } + p.Wait() + if !results { if r.output != nil { outputFile := r.output.Name() @@ -200,6 +195,9 @@ func (r *Runner) RunEnumeration() { gologger.Fatalf("Error, no templates found in directory: '%s'\n", r.options.Templates) } + // track progress + p := progress.NewProgress(nil) + var results bool for _, match := range matches { t, err := r.parse(match) @@ -213,7 +211,7 @@ func (r *Runner) RunEnumeration() { } } - p.SetupProgressBar(template.ID, r.inputCount * r.getHTTPRequestsCount(template)) + p.SetupProgressBar(template.ID, r.inputCount * template.GetHTTPRequestsCount()) for _, request := range template.RequestsHTTP { httpResults := r.processTemplateWithList(p, template, request) @@ -231,18 +229,19 @@ func (r *Runner) RunEnumeration() { p.StopStdCaptureAndShow() } } + p.Wait() + if !results { if r.output != nil { outputFile := r.output.Name() r.output.Close() os.Remove(outputFile) } - p.StartStdCapture() + //p.StartStdCapture() gologger.Infof("No results found for the template. Happy hacking!") - p.StopStdCaptureAndShow() + //p.StopStdCaptureAndShow() } - p.Wait() return } diff --git a/v2/pkg/executor/executer_http.go b/v2/pkg/executor/executer_http.go index 59c7a67a..8ce0b7f1 100644 --- a/v2/pkg/executor/executer_http.go +++ b/v2/pkg/executor/executer_http.go @@ -101,11 +101,13 @@ func (e *HTTPExecutor) ExecuteHTTP(p *progress.Progress, URL string) error { return errors.Wrap(err, "could not compile http request") } + remaining := e.template.GetHTTPRequestsCount() + // Send the request to the target servers mainLoop: for compiledRequest := range compiledRequest { if compiledRequest.Error != nil { - p.Abort() + p.Abort(remaining) return errors.Wrap(err, "error in compiled http request") } e.setCustomHeaders(compiledRequest) @@ -118,7 +120,7 @@ mainLoop: dumpedRequest, err := httputil.DumpRequest(req.Request, true) if err != nil { - p.Abort() + p.Abort(remaining) return errors.Wrap(err, "could not dump http request") } p.StartStdCapture() @@ -131,7 +133,7 @@ mainLoop: if resp != nil { resp.Body.Close() } - p.Abort() + p.Abort(remaining) return errors.Wrap(err, "could not issue http request") } @@ -142,7 +144,7 @@ mainLoop: dumpedResponse, err := httputil.DumpResponse(resp, true) if err != nil { - p.Abort() + p.Abort(remaining) return errors.Wrap(err, "could not dump http response") } p.StartStdCapture() @@ -154,7 +156,7 @@ mainLoop: if err != nil { io.Copy(ioutil.Discard, resp.Body) resp.Body.Close() - p.Abort() + p.Abort(remaining) return errors.Wrap(err, "could not read http body") } resp.Body.Close() @@ -163,7 +165,7 @@ mainLoop: // so in case we have to manually do it data, err = requests.HandleDecompression(compiledRequest.Request, data) if err != nil { - p.Abort() + p.Abort(remaining) return errors.Wrap(err, "could not decompress http body") } @@ -184,6 +186,7 @@ mainLoop: // If the condition is AND we haven't matched, try next request. if matcherCondition == matchers.ANDCondition { p.Update() + remaining-- continue mainLoop } } else { @@ -225,6 +228,7 @@ mainLoop: } p.Update() + remaining-- } p.StartStdCapture() diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index eebbf6ce..58000720 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -25,3 +25,11 @@ type Info struct { // Severity optionally describes the severity of the template Severity string `yaml:"severity,omitempty"` } + +func (t* Template) GetHTTPRequestsCount() int64 { + var count int64 = 0 + for _, request := range t.RequestsHTTP { + count += request.GetRequestCount() + } + return count +} \ No newline at end of file From bd274cf1d60e77330f74655bbf8b370aa8df4df3 Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Tue, 7 Jul 2020 22:42:34 +0200 Subject: [PATCH 15/51] Remove unused code --- v2/internal/progress/stdcapture.go | 1 - 1 file changed, 1 deletion(-) diff --git a/v2/internal/progress/stdcapture.go b/v2/internal/progress/stdcapture.go index ac24b52a..1243c68b 100644 --- a/v2/internal/progress/stdcapture.go +++ b/v2/internal/progress/stdcapture.go @@ -85,7 +85,6 @@ func startStdCapture() *captureData { wg.Done() case <-ctx.Done(): wg.Done() - break } }(&c.sync, c) From df501136a6a02f97442a73b8ff74143c1ddc6613 Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Tue, 7 Jul 2020 23:11:33 +0200 Subject: [PATCH 16/51] Abort single request --- v2/pkg/executor/executer_http.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/v2/pkg/executor/executer_http.go b/v2/pkg/executor/executer_http.go index 658e0bb7..04200f01 100644 --- a/v2/pkg/executor/executer_http.go +++ b/v2/pkg/executor/executer_http.go @@ -132,9 +132,10 @@ mainLoop: if resp != nil { resp.Body.Close() } - //p.Abort(remaining) - //return errors.Wrap(err, "could not issue http request") + p.Abort(1) + p.StartStdCapture() gologger.Warningf("Could not do request: %s\n", err) + p.StopStdCaptureAndShow() continue } From 333809f3d63b54eb60cd42d7a5018469d759a365 Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Wed, 8 Jul 2020 23:13:05 +0200 Subject: [PATCH 17/51] Safer stdio capturing --- v2/internal/progress/progress.go | 4 ++ v2/internal/progress/stdcapture.go | 85 +++++++++++++----------------- 2 files changed, 40 insertions(+), 49 deletions(-) diff --git a/v2/internal/progress/progress.go b/v2/internal/progress/progress.go index 73f43d17..1ced39ea 100644 --- a/v2/internal/progress/progress.go +++ b/v2/internal/progress/progress.go @@ -21,6 +21,7 @@ type Progress struct { initialTotal int64 captureData *captureData termWidth int + mutex *sync.Mutex } func NewProgress(group *sync.WaitGroup) *Progress { @@ -37,6 +38,7 @@ func NewProgress(group *sync.WaitGroup) *Progress { mpb.PopCompletedMode(), ), termWidth: tw, + mutex: &sync.Mutex{}, } return p } @@ -86,6 +88,7 @@ func (p *Progress) Wait() { // func (p *Progress) StartStdCapture() { + p.mutex.Lock() p.captureData = startStdCapture() } @@ -100,6 +103,7 @@ func (p *Progress) StopStdCaptureAndShow() { p.progress.Add(0, makeLogBar(msg)).SetTotal(0, true) } } + p.mutex.Unlock() } func makeLogBar(msg string) mpb.BarFiller { diff --git a/v2/internal/progress/stdcapture.go b/v2/internal/progress/stdcapture.go index 1243c68b..5dffae65 100644 --- a/v2/internal/progress/stdcapture.go +++ b/v2/internal/progress/stdcapture.go @@ -5,7 +5,6 @@ package progress */ import ( "bytes" - "context" "io" "os" "strings" @@ -19,21 +18,18 @@ type captureData struct { backupStderr *os.File writerStderr *os.File - data string - channel chan string + dataStdout string + dataStderr string + outStdout chan []byte + outStderr chan []byte - sync sync.WaitGroup + //sync sync.WaitGroup - Data []string + DataStdOut []string + DataStdErr []string } -var( - mutex = &sync.Mutex{} -) - func startStdCapture() *captureData { - mutex.Lock() - rStdout, wStdout, errStdout := os.Pipe() if errStdout != nil { panic(errStdout) @@ -51,42 +47,23 @@ func startStdCapture() *captureData { backupStderr: os.Stderr, writerStderr: wStderr, - channel: make(chan string), + outStdout: make(chan []byte), + outStderr: make(chan []byte), } os.Stdout = c.writerStdout os.Stderr = c.writerStderr - c.sync.Add(2) - - go func( wg *sync.WaitGroup, out chan string, readerStdout *os.File, readerStderr *os.File) { - defer wg.Done() - - var bufStdout bytes.Buffer - _, _ = io.Copy(&bufStdout, readerStdout) - if bufStdout.Len() > 0 { - out <- bufStdout.String() + stdCopy := func(out chan<- []byte, reader *os.File) { + var buffer bytes.Buffer + _, _ = io.Copy(&buffer, reader) + if buffer.Len() > 0 { + out <- buffer.Bytes() } + } - var bufStderr bytes.Buffer - _, _ = io.Copy(&bufStderr, readerStderr) - if bufStderr.Len() > 0 { - out <- bufStderr.String() - } - }(&c.sync, c.channel, rStdout, rStderr) - - go func(wg *sync.WaitGroup, c *captureData) { - ctx, cancel := context.WithTimeout(context.Background(), 10 * time.Millisecond) - defer cancel() - - select { - case out := <-c.channel: - c.data += out - wg.Done() - case <-ctx.Done(): - wg.Done() - } - }(&c.sync, c) + go stdCopy(c.outStdout, rStdout) + go stdCopy(c.outStderr, rStderr) return c } @@ -95,17 +72,27 @@ func stopStdCapture(c *captureData) { _ = c.writerStdout.Close() _ = c.writerStderr.Close() - c.sync.Wait() + var wg sync.WaitGroup - close(c.channel) + stdRead := func(in <-chan []byte, outString *string, outArray *[]string) { + defer wg.Done() + + select { + case out := <-in: + *outString = string(out) + *outArray = strings.Split(*outString, "\n") + if (*outArray)[len(*outArray)-1] == "" { + *outArray = (*outArray)[:len(*outArray)-1] + } + case <-time.After(50 * time.Millisecond): + } + } + + wg.Add(2) + go stdRead(c.outStdout, &c.dataStdout, &c.DataStdOut) + go stdRead(c.outStderr, &c.dataStderr, &c.DataStdErr) + wg.Wait() os.Stdout = c.backupStdout os.Stderr = c.backupStderr - - c.Data = strings.Split(c.data, "\n") - if c.Data[len(c.Data)-1] == "" { - c.Data = c.Data[:len(c.Data)-1] - } - - mutex.Unlock() } From 6c43aab488c3d0fa6f17e9c43e34b1e552c6a5a3 Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Wed, 8 Jul 2020 23:13:53 +0200 Subject: [PATCH 18/51] Try use multiple mpb.Progress for distinct stdout/stderr output --- v2/internal/progress/progress.go | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/v2/internal/progress/progress.go b/v2/internal/progress/progress.go index 1ced39ea..a046f7b4 100644 --- a/v2/internal/progress/progress.go +++ b/v2/internal/progress/progress.go @@ -16,6 +16,7 @@ import ( type Progress struct { progress *mpb.Progress + progress_stdout *mpb.Progress bar *mpb.Bar total int64 initialTotal int64 @@ -37,6 +38,13 @@ func NewProgress(group *sync.WaitGroup) *Progress { mpb.WithOutput(os.Stderr), mpb.PopCompletedMode(), ), + + progress_stdout: mpb.New( + mpb.WithWaitGroup(group), + mpb.WithOutput(os.Stdout), + //mpb.PopCompletedMode(), + ), + termWidth: tw, mutex: &sync.Mutex{}, } @@ -83,6 +91,7 @@ func (p *Progress) Wait() { } p.progress.Wait() + p.progress_stdout.Wait() } // @@ -94,7 +103,20 @@ func (p *Progress) StartStdCapture() { func (p *Progress) StopStdCaptureAndShow() { stopStdCapture(p.captureData) - for _, captured := range p.captureData.Data { + + // stdout + for _, captured := range p.captureData.DataStdOut { + var r = regexp.MustCompile("(.{" + strconv.Itoa(p.termWidth) + "})") + multiline := r.ReplaceAllString(captured, "$1\n") + arr := strings.Split(multiline, "\n") + + for _, msg := range arr { + p.progress_stdout.Add(0, makeLogBar(msg)).SetTotal(0, true) + } + } + + // stderr + for _, captured := range p.captureData.DataStdErr { var r = regexp.MustCompile("(.{" + strconv.Itoa(p.termWidth) + "})") multiline := r.ReplaceAllString(captured, "$1\n") arr := strings.Split(multiline, "\n") @@ -103,6 +125,7 @@ func (p *Progress) StopStdCaptureAndShow() { p.progress.Add(0, makeLogBar(msg)).SetTotal(0, true) } } + p.mutex.Unlock() } From 0ff138a477f313e344cff12bd3ece8473ac65ae9 Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Thu, 9 Jul 2020 20:57:24 +0200 Subject: [PATCH 19/51] Do not use mpb.Progress for logging This will cause sync issues with very fast output and will defeat the purpose of logging. Instead, buffer both stdout/stderr and show their output at the end. --- v2/internal/progress/progress.go | 62 +++++++++--------------------- v2/internal/progress/stdcapture.go | 30 ++++++--------- v2/internal/runner/runner.go | 14 ++++--- v2/pkg/executor/executer_http.go | 16 ++++---- 4 files changed, 48 insertions(+), 74 deletions(-) diff --git a/v2/internal/progress/progress.go b/v2/internal/progress/progress.go index a046f7b4..612d8482 100644 --- a/v2/internal/progress/progress.go +++ b/v2/internal/progress/progress.go @@ -6,30 +6,28 @@ import ( "github.com/vbauerster/mpb/v5" "github.com/vbauerster/mpb/v5/cwriter" "github.com/vbauerster/mpb/v5/decor" - "io" "os" - "regexp" - "strconv" "strings" "sync" ) type Progress struct { progress *mpb.Progress - progress_stdout *mpb.Progress bar *mpb.Bar total int64 initialTotal int64 captureData *captureData termWidth int mutex *sync.Mutex + stdout *strings.Builder + stderr *strings.Builder } func NewProgress(group *sync.WaitGroup) *Progress { - w := cwriter.New(os.Stdout) + w := cwriter.New(os.Stderr) tw, err := w.GetWidth() if err != nil { - panic("Couldn't determine available terminal width.") + tw = 80 } p := &Progress{ @@ -38,15 +36,10 @@ func NewProgress(group *sync.WaitGroup) *Progress { mpb.WithOutput(os.Stderr), mpb.PopCompletedMode(), ), - - progress_stdout: mpb.New( - mpb.WithWaitGroup(group), - mpb.WithOutput(os.Stdout), - //mpb.PopCompletedMode(), - ), - termWidth: tw, mutex: &sync.Mutex{}, + stdout: &strings.Builder{}, + stderr: &strings.Builder{}, } return p } @@ -89,9 +82,7 @@ func (p *Progress) Wait() { if p.initialTotal != p.total { p.bar.SetTotal(p.total, true) } - p.progress.Wait() - p.progress_stdout.Wait() } // @@ -101,36 +92,21 @@ func (p *Progress) StartStdCapture() { p.captureData = startStdCapture() } -func (p *Progress) StopStdCaptureAndShow() { +func (p *Progress) StopStdCapture() { stopStdCapture(p.captureData) - - // stdout - for _, captured := range p.captureData.DataStdOut { - var r = regexp.MustCompile("(.{" + strconv.Itoa(p.termWidth) + "})") - multiline := r.ReplaceAllString(captured, "$1\n") - arr := strings.Split(multiline, "\n") - - for _, msg := range arr { - p.progress_stdout.Add(0, makeLogBar(msg)).SetTotal(0, true) - } - } - - // stderr - for _, captured := range p.captureData.DataStdErr { - var r = regexp.MustCompile("(.{" + strconv.Itoa(p.termWidth) + "})") - multiline := r.ReplaceAllString(captured, "$1\n") - arr := strings.Split(multiline, "\n") - - for _, msg := range arr { - p.progress.Add(0, makeLogBar(msg)).SetTotal(0, true) - } - } - + p.stdout.Write(p.captureData.DataStdOut.Bytes()) + p.stderr.Write(p.captureData.DataStdErr.Bytes()) p.mutex.Unlock() } -func makeLogBar(msg string) mpb.BarFiller { - return mpb.BarFillerFunc(func(w io.Writer, _ int, st decor.Statistics) { - fmt.Fprintf(w, msg) - }) +func (p *Progress) ShowStdOut() { + if p.stdout.Len() > 0 { + fmt.Fprint(os.Stdout, p.stdout.String()) + } +} + +func (p *Progress) ShowStdErr() { + if p.stderr.Len() > 0 { + fmt.Fprint(os.Stderr, p.stderr.String()) + } } diff --git a/v2/internal/progress/stdcapture.go b/v2/internal/progress/stdcapture.go index 5dffae65..1d8a0719 100644 --- a/v2/internal/progress/stdcapture.go +++ b/v2/internal/progress/stdcapture.go @@ -7,9 +7,7 @@ import ( "bytes" "io" "os" - "strings" "sync" - "time" ) type captureData struct { @@ -18,15 +16,11 @@ type captureData struct { backupStderr *os.File writerStderr *os.File - dataStdout string - dataStderr string + DataStdOut *bytes.Buffer + DataStdErr *bytes.Buffer + outStdout chan []byte outStderr chan []byte - - //sync sync.WaitGroup - - DataStdOut []string - DataStdErr []string } func startStdCapture() *captureData { @@ -49,6 +43,9 @@ func startStdCapture() *captureData { outStdout: make(chan []byte), outStderr: make(chan []byte), + + DataStdOut: &bytes.Buffer{}, + DataStdErr: &bytes.Buffer{}, } os.Stdout = c.writerStdout @@ -74,23 +71,20 @@ func stopStdCapture(c *captureData) { var wg sync.WaitGroup - stdRead := func(in <-chan []byte, outString *string, outArray *[]string) { + stdRead := func(in <-chan []byte, outData *bytes.Buffer) { defer wg.Done() select { case out := <-in: - *outString = string(out) - *outArray = strings.Split(*outString, "\n") - if (*outArray)[len(*outArray)-1] == "" { - *outArray = (*outArray)[:len(*outArray)-1] - } - case <-time.After(50 * time.Millisecond): + outData.Write(out) + default: + //case <-time.After(10 * time.Millisecond): } } wg.Add(2) - go stdRead(c.outStdout, &c.dataStdout, &c.DataStdOut) - go stdRead(c.outStderr, &c.dataStderr, &c.DataStdErr) + go stdRead(c.outStdout, c.DataStdOut) + go stdRead(c.outStderr, c.DataStdErr) wg.Wait() os.Stdout = c.backupStdout diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index cf9dd88e..ce3b812a 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -153,6 +153,8 @@ func (r *Runner) RunEnumeration() { } p.Wait() + p.ShowStdErr() + p.ShowStdOut() if !results { if r.output != nil { @@ -226,10 +228,12 @@ func (r *Runner) RunEnumeration() { default: p.StartStdCapture() gologger.Errorf("Could not parse file '%s': %s\n", r.options.Templates, err) - p.StopStdCaptureAndShow() + p.StopStdCapture() } } p.Wait() + p.ShowStdErr() + p.ShowStdOut() if !results { if r.output != nil { @@ -239,7 +243,7 @@ func (r *Runner) RunEnumeration() { } //p.StartStdCapture() gologger.Infof("No results found for the template. Happy hacking!") - //p.StopStdCaptureAndShow() + //p.StopStdCapture() } return @@ -254,7 +258,7 @@ func (r *Runner) processTemplateWithList(p *progress.Progress, template *templat } p.StartStdCapture() gologger.Infof("%s\n", message) - p.StopStdCaptureAndShow() + p.StopStdCapture() var writer *bufio.Writer if r.output != nil { @@ -293,7 +297,7 @@ func (r *Runner) processTemplateWithList(p *progress.Progress, template *templat if err != nil { p.StartStdCapture() gologger.Warningf("Could not create http client: %s\n", err) - p.StopStdCaptureAndShow() + p.StopStdCapture() return false } @@ -322,7 +326,7 @@ func (r *Runner) processTemplateWithList(p *progress.Progress, template *templat if err != nil { p.StartStdCapture() gologger.Warningf("Could not execute step: %s\n", err) - p.StopStdCaptureAndShow() + p.StopStdCapture() } <-limiter wg.Done() diff --git a/v2/pkg/executor/executer_http.go b/v2/pkg/executor/executer_http.go index 04200f01..9678a791 100644 --- a/v2/pkg/executor/executer_http.go +++ b/v2/pkg/executor/executer_http.go @@ -115,7 +115,7 @@ mainLoop: if e.debug { p.StartStdCapture() gologger.Infof("Dumped HTTP request for %s (%s)\n\n", URL, e.template.ID) - p.StopStdCaptureAndShow() + p.StopStdCapture() dumpedRequest, err := httputil.DumpRequest(req.Request, true) if err != nil { @@ -124,7 +124,7 @@ mainLoop: } p.StartStdCapture() fmt.Fprintf(os.Stderr, "%s", string(dumpedRequest)) - p.StopStdCaptureAndShow() + p.StopStdCapture() } resp, err := e.httpClient.Do(req) @@ -135,14 +135,14 @@ mainLoop: p.Abort(1) p.StartStdCapture() gologger.Warningf("Could not do request: %s\n", err) - p.StopStdCaptureAndShow() + p.StopStdCapture() continue } if e.debug { p.StartStdCapture() gologger.Infof("Dumped HTTP response for %s (%s)\n\n", URL, e.template.ID) - p.StopStdCaptureAndShow() + p.StopStdCapture() dumpedResponse, err := httputil.DumpResponse(resp, true) if err != nil { @@ -151,7 +151,7 @@ mainLoop: } p.StartStdCapture() fmt.Fprintf(os.Stderr, "%s\n", string(dumpedResponse)) - p.StopStdCaptureAndShow() + p.StopStdCapture() } data, err := ioutil.ReadAll(resp.Body) @@ -193,7 +193,7 @@ mainLoop: // capture stdout and emit it via a mpb.BarFiller p.StartStdCapture() e.writeOutputHTTP(compiledRequest, matcher, nil) - p.StopStdCaptureAndShow() + p.StopStdCapture() atomic.CompareAndSwapUint32(&e.results, 0, 1) } @@ -216,7 +216,7 @@ mainLoop: // capture stdout and emit it via a mpb.BarFiller p.StartStdCapture() e.writeOutputHTTP(compiledRequest, nil, extractorResults) - p.StopStdCaptureAndShow() + p.StopStdCapture() atomic.CompareAndSwapUint32(&e.results, 0, 1) } @@ -227,7 +227,7 @@ mainLoop: p.StartStdCapture() gologger.Verbosef("Sent HTTP request to %s\n", "http-request", URL) - p.StopStdCaptureAndShow() + p.StopStdCapture() return nil } From cbb8b11784d9a4e30ce26ca78f86f9e9a9e0547a Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Thu, 9 Jul 2020 21:18:06 +0200 Subject: [PATCH 20/51] Use atomic operations to alter and track the progress --- v2/internal/progress/progress.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/internal/progress/progress.go b/v2/internal/progress/progress.go index 612d8482..41b11710 100644 --- a/v2/internal/progress/progress.go +++ b/v2/internal/progress/progress.go @@ -74,8 +74,8 @@ func (p *Progress) Update() { } func (p *Progress) Abort(remaining int64) { - p.total -= remaining - p.bar.SetTotal(p.total, false) + atomic.AddInt64(&p.total, -remaining) + p.bar.SetTotal(atomic.LoadInt64(&p.total), false) } func (p *Progress) Wait() { From 81eed093d236bb2f89c448d4e66992fde15bab67 Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Thu, 9 Jul 2020 21:20:00 +0200 Subject: [PATCH 21/51] Minor refactoring, rename mutex --- v2/internal/progress/progress.go | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/v2/internal/progress/progress.go b/v2/internal/progress/progress.go index 41b11710..25637aa8 100644 --- a/v2/internal/progress/progress.go +++ b/v2/internal/progress/progress.go @@ -9,18 +9,19 @@ import ( "os" "strings" "sync" + "sync/atomic" ) type Progress struct { - progress *mpb.Progress - bar *mpb.Bar - total int64 - initialTotal int64 - captureData *captureData - termWidth int - mutex *sync.Mutex - stdout *strings.Builder - stderr *strings.Builder + progress *mpb.Progress + bar *mpb.Bar + total int64 + initialTotal int64 + captureData *captureData + termWidth int + stdCaptureMutex *sync.Mutex + stdout *strings.Builder + stderr *strings.Builder } func NewProgress(group *sync.WaitGroup) *Progress { @@ -36,10 +37,10 @@ func NewProgress(group *sync.WaitGroup) *Progress { mpb.WithOutput(os.Stderr), mpb.PopCompletedMode(), ), - termWidth: tw, - mutex: &sync.Mutex{}, - stdout: &strings.Builder{}, - stderr: &strings.Builder{}, + termWidth: tw, + stdCaptureMutex: &sync.Mutex{}, + stdout: &strings.Builder{}, + stderr: &strings.Builder{}, } return p } @@ -88,7 +89,7 @@ func (p *Progress) Wait() { // func (p *Progress) StartStdCapture() { - p.mutex.Lock() + p.stdCaptureMutex.Lock() p.captureData = startStdCapture() } @@ -96,7 +97,7 @@ func (p *Progress) StopStdCapture() { stopStdCapture(p.captureData) p.stdout.Write(p.captureData.DataStdOut.Bytes()) p.stderr.Write(p.captureData.DataStdErr.Bytes()) - p.mutex.Unlock() + p.stdCaptureMutex.Unlock() } func (p *Progress) ShowStdOut() { From e5949c8eba2ead5a710fe1db2f6d01de96609f00 Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Thu, 9 Jul 2020 21:21:37 +0200 Subject: [PATCH 22/51] Precompute total number of HTTP requests when using multiple templates --- v2/internal/runner/runner.go | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index ce3b812a..dceadbcf 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -200,6 +200,23 @@ func (r *Runner) RunEnumeration() { // track progress p := progress.NewProgress(nil) + // precompute request count + var totalRequests int64 = 0 + for _, match := range matches { + t, err := r.parse(match) + switch t.(type) { + case *templates.Template: + template := t.(*templates.Template) + totalRequests += template.GetHTTPRequestsCount() + default: + p.StartStdCapture() + gologger.Errorf("Could not parse file '%s': %s\n", r.options.Templates, err) + p.StopStdCapture() + } + } + + p.SetupProgressBar("Multiple templates", r.inputCount * totalRequests) + var results bool for _, match := range matches { t, err := r.parse(match) @@ -213,8 +230,6 @@ func (r *Runner) RunEnumeration() { } } - p.SetupProgressBar(template.ID, r.inputCount * template.GetHTTPRequestsCount()) - for _, request := range template.RequestsHTTP { httpResults := r.processTemplateWithList(p, template, request) if httpResults { From 2fe3d354c332da2850e630433f22fb68cdf89bb1 Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Fri, 10 Jul 2020 22:14:39 +0200 Subject: [PATCH 23/51] Group url/host-bound log messages to avoid segmentation in the output. This is the simple and memory-friendly way. The alternative would be to build a map[host]strings.Builder to allow for seamless per-host output construction and output, but memory usage would probably be higher even with a pool of builders. --- v2/pkg/executor/executer_http.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/v2/pkg/executor/executer_http.go b/v2/pkg/executor/executer_http.go index 9678a791..96660b96 100644 --- a/v2/pkg/executor/executer_http.go +++ b/v2/pkg/executor/executer_http.go @@ -113,16 +113,13 @@ mainLoop: req := compiledRequest.Request if e.debug { - p.StartStdCapture() - gologger.Infof("Dumped HTTP request for %s (%s)\n\n", URL, e.template.ID) - p.StopStdCapture() - dumpedRequest, err := httputil.DumpRequest(req.Request, true) if err != nil { p.Abort(remaining) return errors.Wrap(err, "could not dump http request") } p.StartStdCapture() + gologger.Infof("Dumped HTTP request for %s (%s)\n\n", URL, e.template.ID) fmt.Fprintf(os.Stderr, "%s", string(dumpedRequest)) p.StopStdCapture() } @@ -140,16 +137,13 @@ mainLoop: } if e.debug { - p.StartStdCapture() - gologger.Infof("Dumped HTTP response for %s (%s)\n\n", URL, e.template.ID) - p.StopStdCapture() - dumpedResponse, err := httputil.DumpResponse(resp, true) if err != nil { p.Abort(remaining) return errors.Wrap(err, "could not dump http response") } p.StartStdCapture() + gologger.Infof("Dumped HTTP response for %s (%s)\n\n", URL, e.template.ID) fmt.Fprintf(os.Stderr, "%s\n", string(dumpedResponse)) p.StopStdCapture() } From 002daadf46813665cf23ef6292f998cdcae21630 Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Fri, 10 Jul 2020 23:42:49 +0200 Subject: [PATCH 24/51] Try avoid using time.After when reading captured output. --- v2/internal/progress/stdcapture.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/v2/internal/progress/stdcapture.go b/v2/internal/progress/stdcapture.go index 1d8a0719..09070894 100644 --- a/v2/internal/progress/stdcapture.go +++ b/v2/internal/progress/stdcapture.go @@ -57,6 +57,7 @@ func startStdCapture() *captureData { if buffer.Len() > 0 { out <- buffer.Bytes() } + close(out) } go stdCopy(c.outStdout, rStdout) @@ -74,11 +75,13 @@ func stopStdCapture(c *captureData) { stdRead := func(in <-chan []byte, outData *bytes.Buffer) { defer wg.Done() - select { - case out := <-in: - outData.Write(out) - default: - //case <-time.After(10 * time.Millisecond): + for { + out, more := <-in + if more { + outData.Write(out) + } else { + return + } } } From 97901f36b45df1770b2e129efe4587d75cbc26e7 Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Sat, 11 Jul 2020 22:57:44 +0200 Subject: [PATCH 25/51] Refactor code, add meaningful comments --- v2/go.sum | 13 +++ v2/internal/progress/bar.go | 27 +++++++ v2/internal/progress/doc.go | 2 + v2/internal/progress/progress.go | 133 ++++++++++++++++++++++--------- v2/internal/runner/runner.go | 19 +++-- v2/pkg/executor/executer_http.go | 12 +-- 6 files changed, 153 insertions(+), 53 deletions(-) create mode 100644 v2/internal/progress/bar.go create mode 100644 v2/internal/progress/doc.go diff --git a/v2/go.sum b/v2/go.sum index 752972ef..c05d7692 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -1,5 +1,9 @@ github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f6s9Tn1Tt7/WTEg= github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM= +github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0S6Vi7/lbWECcX0j45yZReDZ56BQsrVBOEEY= github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= @@ -24,6 +28,10 @@ github.com/karrick/godirwalk v1.15.6 h1:Yf2mmR8TJy+8Fa0SuQVto5SYap6IF7lNVX4Jdl8G github.com/karrick/godirwalk v1.15.6/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 h1:bqDmpDG49ZRnB5PcgP0RXtQvnMSgIF14M7CBd2shtXs= github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/miekg/dns v1.1.29 h1:xHBEhR+t5RzcFJjBLJlax2daXOrTYtr9z4WdKEfWFzg= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= @@ -42,6 +50,8 @@ github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/vbauerster/mpb/v5 v5.2.2 h1:zIICVOm+XD+uV6crpSORaL6I0Q1WqOdvxZTp+r3L9cw= +github.com/vbauerster/mpb/v5 v5.2.2/go.mod h1:W5Fvgw4dm3/0NhqzV8j6EacfuTe5SvnzBRwiXxDR9ww= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -57,8 +67,11 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 h1:DYfZAGf2WMFjMxbgTjaC+2HC7NkNAQs+6Q8b9WEB/F4= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/v2/internal/progress/bar.go b/v2/internal/progress/bar.go new file mode 100644 index 00000000..32774d51 --- /dev/null +++ b/v2/internal/progress/bar.go @@ -0,0 +1,27 @@ +package progress + +import ( + "github.com/vbauerster/mpb/v5" + "sync/atomic" +) + +// Represents a single progress bar +type Bar struct { + bar *mpb.Bar + total int64 + initialTotal int64 +} + +// Drops the specified number of requests from the progress bar total. +// This may be the case when uncompleted requests are encountered and shouldn't be part of the total count. +func (b *Bar) Drop(count int64) { + atomic.AddInt64(&b.total, -count) + b.bar.SetTotal(atomic.LoadInt64(&b.total), false) +} + +// Ensures that a progress bar's total count is up-to-date if during an enumeration there were uncompleted requests. +func (b *Bar) finish() { + if b.initialTotal != b.total { + b.bar.SetTotal(b.total, true) + } +} diff --git a/v2/internal/progress/doc.go b/v2/internal/progress/doc.go new file mode 100644 index 00000000..1a4999e5 --- /dev/null +++ b/v2/internal/progress/doc.go @@ -0,0 +1,2 @@ +// Tracks enumeration progress information and implements visible tracking with one or more progress bars. +package progress diff --git a/v2/internal/progress/progress.go b/v2/internal/progress/progress.go index 25637aa8..109b54fe 100644 --- a/v2/internal/progress/progress.go +++ b/v2/internal/progress/progress.go @@ -9,14 +9,14 @@ import ( "os" "strings" "sync" - "sync/atomic" ) +// Encapsulates progress tracking. type Progress struct { - progress *mpb.Progress - bar *mpb.Bar - total int64 - initialTotal int64 + progress *mpb.Progress + barTemplate *Bar + barGlobal *Bar + captureData *captureData termWidth int stdCaptureMutex *sync.Mutex @@ -24,6 +24,7 @@ type Progress struct { stderr *strings.Builder } +// Creates and returns a new progress tracking object. func NewProgress(group *sync.WaitGroup) *Progress { w := cwriter.New(os.Stderr) tw, err := w.GetWidth() @@ -45,54 +46,110 @@ func NewProgress(group *sync.WaitGroup) *Progress { return p } -func (p *Progress) SetupProgressBar(name string, total int64) *mpb.Bar { - barname := "[" + aurora.Green(name).String() + "]" - bar := p.progress.AddBar( +// Creates and returns a progress bar that tracks request progress for a specific template. +func (p *Progress) SetupTemplateProgressbar(templateIndex int, templateCount int, name string, requestCount int64) { + barName := "[" + aurora.Green(name).String() + "]" + + if templateIndex > -1 && templateCount > -1 { + barName = aurora.Sprintf("[%d/%d] ", aurora.Bold(aurora.Cyan(templateIndex)), aurora.Cyan(templateCount)) + barName + } + + bar := p.setupProgressbar(barName, requestCount) + + if p.barTemplate != nil { + // ensure any previous bar has finished and aborted requests have also been considered + p.barTemplate.finish() + } + + p.barTemplate = &Bar{ + bar: bar, + total: requestCount, + initialTotal: requestCount, + } +} + +// Creates and returns a progress bar that tracks all the requests progress. +// This is only useful when multiple templates are processed within the same run. +func (p *Progress) SetupGlobalProgressbar(hostCount int64, templateCount int, requestCount int64) { + hostPlural := "host" + if hostCount > 1 { + hostPlural = "hosts" + } + + barName := "[" + aurora.Sprintf( + aurora.Cyan("%d templates, %d %s"), + aurora.Bold(aurora.Cyan(templateCount)), + aurora.Bold(aurora.Cyan(hostCount)), + hostPlural) + "]" + + bar := p.setupProgressbar(barName, requestCount) + + p.barGlobal = &Bar{ + bar: bar, + total: requestCount, + initialTotal: requestCount, + } +} + +// Update progress tracking information and increments the request counter by one unit. +// If a global progress bar is present it will be updated as well. +func (p *Progress) Update() { + p.barTemplate.bar.Increment() + + if p.barGlobal != nil { + p.barGlobal.bar.Increment() + } +} + +// Drops the specified number of requests from the progress bar total. +// This may be the case when uncompleted requests are encountered and shouldn't be part of the total count. +// If a global progress bar is present it will be updated as well. +func (p *Progress) Drop(count int64) { + p.barTemplate.Drop(count) + + if p.barGlobal != nil { + p.barGlobal.Drop(count) + } +} + +// Ensures that a progress bar's total count is up-to-date if during an enumeration there were uncompleted requests and +// wait for all the progress bars to finish. +// If a global progress bar is present it will be updated as well. +func (p *Progress) Wait() { + p.barTemplate.finish() + + if p.barGlobal != nil { + p.barGlobal.finish() + } + + p.progress.Wait() +} + +// Creates and returns a progress bar. +func (p *Progress) setupProgressbar(name string, total int64) *mpb.Bar { + return p.progress.AddBar( total, mpb.BarNoPop(), mpb.BarRemoveOnComplete(), mpb.PrependDecorators( - decor.Name(barname), - decor.CountersNoUnit(aurora.Blue(" %d/%d").String()), + decor.Name(name, decor.WCSyncSpaceR), + decor.CountersNoUnit(aurora.Blue(" %d/%d").String(), decor.WCSyncSpace), decor.NewPercentage(aurora.Bold("%d").String(), decor.WCSyncSpace), ), mpb.AppendDecorators( - decor.AverageSpeed(0, aurora.Yellow("%.2f req/s ").String()), - decor.OnComplete( - decor.AverageETA(decor.ET_STYLE_GO), aurora.Bold("done!").String(), - ), + decor.AverageSpeed(0, aurora.Yellow("%.2f r/s ").String(), decor.WCSyncSpace), + decor.AverageETA(decor.ET_STYLE_GO, decor.WCSyncSpace), ), ) - - p.bar = bar - p.total = total - p.initialTotal = total - return bar } -func (p *Progress) Update() { - p.bar.Increment() -} - -func (p *Progress) Abort(remaining int64) { - atomic.AddInt64(&p.total, -remaining) - p.bar.SetTotal(atomic.LoadInt64(&p.total), false) -} - -func (p *Progress) Wait() { - if p.initialTotal != p.total { - p.bar.SetTotal(p.total, true) - } - p.progress.Wait() -} - -// - +// Starts capturing stdout and stderr instead of producing visual output that may interfere with the progress bars. func (p *Progress) StartStdCapture() { p.stdCaptureMutex.Lock() p.captureData = startStdCapture() } +// Stops capturing stdout and stderr and store both output to be shown later. func (p *Progress) StopStdCapture() { stopStdCapture(p.captureData) p.stdout.Write(p.captureData.DataStdOut.Bytes()) @@ -100,12 +157,14 @@ func (p *Progress) StopStdCapture() { p.stdCaptureMutex.Unlock() } +// Writes the captured stdout data to stdout, if any. func (p *Progress) ShowStdOut() { if p.stdout.Len() > 0 { fmt.Fprint(os.Stdout, p.stdout.String()) } } +// Writes the captured stderr data to stderr, if any. func (p *Progress) ShowStdErr() { if p.stderr.Len() > 0 { fmt.Fprint(os.Stderr, p.stderr.String()) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index dceadbcf..7859f74f 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -137,7 +137,7 @@ func (r *Runner) RunEnumeration() { var results bool template := t.(*templates.Template) - p.SetupProgressBar(template.ID, r.inputCount * template.GetHTTPRequestsCount()) + p.SetupTemplateProgressbar(-1, -1, template.ID, r.inputCount * template.GetHTTPRequestsCount()) // process http requests for _, request := range template.RequestsHTTP { @@ -197,11 +197,9 @@ func (r *Runner) RunEnumeration() { gologger.Fatalf("Error, no templates found in directory: '%s'\n", r.options.Templates) } - // track progress - p := progress.NewProgress(nil) - // precompute request count var totalRequests int64 = 0 + var totalTemplates int = len(matches) for _, match := range matches { t, err := r.parse(match) switch t.(type) { @@ -209,20 +207,23 @@ func (r *Runner) RunEnumeration() { template := t.(*templates.Template) totalRequests += template.GetHTTPRequestsCount() default: - p.StartStdCapture() gologger.Errorf("Could not parse file '%s': %s\n", r.options.Templates, err) - p.StopStdCapture() } } - p.SetupProgressBar("Multiple templates", r.inputCount * totalRequests) + // track progress + p := progress.NewProgress(nil) + p.SetupGlobalProgressbar(r.inputCount, len(matches), r.inputCount * totalRequests) var results bool - for _, match := range matches { + for i, match := range matches { t, err := r.parse(match) switch t.(type) { case *templates.Template: template := t.(*templates.Template) + + p.SetupTemplateProgressbar(i, totalTemplates, template.ID, r.inputCount * template.GetHTTPRequestsCount()) + for _, request := range template.RequestsDNS { dnsResults := r.processTemplateWithList(p, template, request) if dnsResults { @@ -256,9 +257,7 @@ func (r *Runner) RunEnumeration() { r.output.Close() os.Remove(outputFile) } - //p.StartStdCapture() gologger.Infof("No results found for the template. Happy hacking!") - //p.StopStdCapture() } return diff --git a/v2/pkg/executor/executer_http.go b/v2/pkg/executor/executer_http.go index 96660b96..a3a60bb4 100644 --- a/v2/pkg/executor/executer_http.go +++ b/v2/pkg/executor/executer_http.go @@ -106,7 +106,7 @@ func (e *HTTPExecutor) ExecuteHTTP(p *progress.Progress, URL string) error { mainLoop: for compiledRequest := range compiledRequest { if compiledRequest.Error != nil { - p.Abort(remaining) + p.Drop(remaining) return errors.Wrap(err, "error in compiled http request") } e.setCustomHeaders(compiledRequest) @@ -115,7 +115,7 @@ mainLoop: if e.debug { dumpedRequest, err := httputil.DumpRequest(req.Request, true) if err != nil { - p.Abort(remaining) + p.Drop(remaining) return errors.Wrap(err, "could not dump http request") } p.StartStdCapture() @@ -129,7 +129,7 @@ mainLoop: if resp != nil { resp.Body.Close() } - p.Abort(1) + p.Drop(1) p.StartStdCapture() gologger.Warningf("Could not do request: %s\n", err) p.StopStdCapture() @@ -139,7 +139,7 @@ mainLoop: if e.debug { dumpedResponse, err := httputil.DumpResponse(resp, true) if err != nil { - p.Abort(remaining) + p.Drop(remaining) return errors.Wrap(err, "could not dump http response") } p.StartStdCapture() @@ -152,7 +152,7 @@ mainLoop: if err != nil { io.Copy(ioutil.Discard, resp.Body) resp.Body.Close() - p.Abort(remaining) + p.Drop(remaining) return errors.Wrap(err, "could not read http body") } resp.Body.Close() @@ -161,7 +161,7 @@ mainLoop: // so in case we have to manually do it data, err = requests.HandleDecompression(compiledRequest.Request, data) if err != nil { - p.Abort(remaining) + p.Drop(remaining) return errors.Wrap(err, "could not decompress http body") } From de82c1617e127d9addb45b874b96f59569fd31e9 Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Sat, 11 Jul 2020 23:50:59 +0200 Subject: [PATCH 26/51] Remove unused code --- v2/internal/progress/progress.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/v2/internal/progress/progress.go b/v2/internal/progress/progress.go index 109b54fe..42b2ed50 100644 --- a/v2/internal/progress/progress.go +++ b/v2/internal/progress/progress.go @@ -4,7 +4,6 @@ import ( "fmt" "github.com/logrusorgru/aurora" "github.com/vbauerster/mpb/v5" - "github.com/vbauerster/mpb/v5/cwriter" "github.com/vbauerster/mpb/v5/decor" "os" "strings" @@ -18,7 +17,6 @@ type Progress struct { barGlobal *Bar captureData *captureData - termWidth int stdCaptureMutex *sync.Mutex stdout *strings.Builder stderr *strings.Builder @@ -26,19 +24,12 @@ type Progress struct { // Creates and returns a new progress tracking object. func NewProgress(group *sync.WaitGroup) *Progress { - w := cwriter.New(os.Stderr) - tw, err := w.GetWidth() - if err != nil { - tw = 80 - } - p := &Progress{ progress: mpb.New( mpb.WithWaitGroup(group), mpb.WithOutput(os.Stderr), mpb.PopCompletedMode(), ), - termWidth: tw, stdCaptureMutex: &sync.Mutex{}, stdout: &strings.Builder{}, stderr: &strings.Builder{}, From e877d613e436912ff4c588c197f85dd410c59a9d Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Sat, 11 Jul 2020 23:52:45 +0200 Subject: [PATCH 27/51] More clear comment wording --- v2/internal/progress/progress.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/internal/progress/progress.go b/v2/internal/progress/progress.go index 42b2ed50..dc4dce02 100644 --- a/v2/internal/progress/progress.go +++ b/v2/internal/progress/progress.go @@ -48,7 +48,7 @@ func (p *Progress) SetupTemplateProgressbar(templateIndex int, templateCount int bar := p.setupProgressbar(barName, requestCount) if p.barTemplate != nil { - // ensure any previous bar has finished and aborted requests have also been considered + // ensure any previous bar has finished and dropped requests have also been considered p.barTemplate.finish() } From d0174c047c4b92309ae096172469d0a4b1a7a124 Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Sun, 12 Jul 2020 18:09:29 +0200 Subject: [PATCH 28/51] Respect color output user flag, remove unused Progress WaitGroup --- v2/internal/progress/progress.go | 31 ++++++++++++++++++++----------- v2/internal/runner/runner.go | 17 ++++++++++++----- v2/pkg/workflows/var.go | 2 +- 3 files changed, 33 insertions(+), 17 deletions(-) diff --git a/v2/internal/progress/progress.go b/v2/internal/progress/progress.go index dc4dce02..2ae847d6 100644 --- a/v2/internal/progress/progress.go +++ b/v2/internal/progress/progress.go @@ -20,29 +20,34 @@ type Progress struct { stdCaptureMutex *sync.Mutex stdout *strings.Builder stderr *strings.Builder + + colorizer aurora.Aurora } // Creates and returns a new progress tracking object. -func NewProgress(group *sync.WaitGroup) *Progress { +func NewProgress(noColor bool) *Progress { p := &Progress{ progress: mpb.New( - mpb.WithWaitGroup(group), mpb.WithOutput(os.Stderr), mpb.PopCompletedMode(), ), stdCaptureMutex: &sync.Mutex{}, stdout: &strings.Builder{}, stderr: &strings.Builder{}, + colorizer: aurora.NewAurora(!noColor), } return p } // Creates and returns a progress bar that tracks request progress for a specific template. func (p *Progress) SetupTemplateProgressbar(templateIndex int, templateCount int, name string, requestCount int64) { - barName := "[" + aurora.Green(name).String() + "]" + color := p.colorizer + barName := "[" + color.Green(name).String() + "]" if templateIndex > -1 && templateCount > -1 { - barName = aurora.Sprintf("[%d/%d] ", aurora.Bold(aurora.Cyan(templateIndex)), aurora.Cyan(templateCount)) + barName + barName = color.Sprintf("[%d/%d] ", + color.Bold(color.Cyan(templateIndex)), + color.Cyan(templateCount)) + barName } bar := p.setupProgressbar(barName, requestCount) @@ -62,15 +67,17 @@ func (p *Progress) SetupTemplateProgressbar(templateIndex int, templateCount int // Creates and returns a progress bar that tracks all the requests progress. // This is only useful when multiple templates are processed within the same run. func (p *Progress) SetupGlobalProgressbar(hostCount int64, templateCount int, requestCount int64) { + color := p.colorizer + hostPlural := "host" if hostCount > 1 { hostPlural = "hosts" } - barName := "[" + aurora.Sprintf( - aurora.Cyan("%d templates, %d %s"), - aurora.Bold(aurora.Cyan(templateCount)), - aurora.Bold(aurora.Cyan(hostCount)), + barName := "[" + color.Sprintf( + color.Cyan("%d templates, %d %s"), + color.Bold(color.Cyan(templateCount)), + color.Bold(color.Cyan(hostCount)), hostPlural) + "]" bar := p.setupProgressbar(barName, requestCount) @@ -118,17 +125,19 @@ func (p *Progress) Wait() { // Creates and returns a progress bar. func (p *Progress) setupProgressbar(name string, total int64) *mpb.Bar { + color := p.colorizer return p.progress.AddBar( total, mpb.BarNoPop(), mpb.BarRemoveOnComplete(), mpb.PrependDecorators( decor.Name(name, decor.WCSyncSpaceR), - decor.CountersNoUnit(aurora.Blue(" %d/%d").String(), decor.WCSyncSpace), - decor.NewPercentage(aurora.Bold("%d").String(), decor.WCSyncSpace), + decor.CountersNoUnit(color.Blue(" %d/%d").String(), decor.WCSyncSpace), + decor.NewPercentage(color.Bold("%d").String(), decor.WCSyncSpace), ), mpb.AppendDecorators( - decor.AverageSpeed(0, aurora.Yellow("%.2f r/s ").String(), decor.WCSyncSpace), + decor.AverageSpeed(0, color.Yellow("%.2f r/s ").String(), decor.WCSyncSpace), + decor.Elapsed(decor.ET_STYLE_GO, decor.WCSyncSpace), decor.AverageETA(decor.ET_STYLE_GO, decor.WCSyncSpace), ), ) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 7859f74f..1f293192 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -33,6 +33,9 @@ type Runner struct { templatesConfig *nucleiConfig // options contains configuration options for runner options *Options + + // progress tracking + progress *progress.Progress } // New creates a new client for running enumeration process. @@ -99,6 +102,10 @@ func New(options *Options) (*Runner, error) { } runner.output = output } + + // Creates the progress tracking object + runner.progress = progress.NewProgress(runner.options.NoColor) + return runner, nil } @@ -125,18 +132,18 @@ func (r *Runner) RunEnumeration() { r.options.Templates = newPath } + p := r.progress + // Single yaml provided if strings.HasSuffix(r.options.Templates, ".yaml") { t, err := r.parse(r.options.Templates) - // track progress - p := progress.NewProgress(nil) - switch t.(type) { case *templates.Template: var results bool template := t.(*templates.Template) + // track single template progress p.SetupTemplateProgressbar(-1, -1, template.ID, r.inputCount * template.GetHTTPRequestsCount()) // process http requests @@ -211,8 +218,7 @@ func (r *Runner) RunEnumeration() { } } - // track progress - p := progress.NewProgress(nil) + // track global progress p.SetupGlobalProgressbar(r.inputCount, len(matches), r.inputCount * totalRequests) var results bool @@ -222,6 +228,7 @@ func (r *Runner) RunEnumeration() { case *templates.Template: template := t.(*templates.Template) + // track template progress p.SetupTemplateProgressbar(i, totalTemplates, template.ID, r.inputCount * template.GetHTTPRequestsCount()) for _, request := range template.RequestsDNS { diff --git a/v2/pkg/workflows/var.go b/v2/pkg/workflows/var.go index 80791cca..ad61f525 100644 --- a/v2/pkg/workflows/var.go +++ b/v2/pkg/workflows/var.go @@ -35,7 +35,7 @@ func (n *NucleiVar) Call(args ...tengo.Object) (ret tengo.Object, err error) { var gotResult bool // track progress - p := progress.NewProgress(nil) + p := progress.NewProgress(false) for _, template := range n.Templates { if template.HTTPOptions != nil { From 362fee27a447d012fcaaec51d471c990a27f10b7 Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Thu, 23 Jul 2020 20:19:19 +0200 Subject: [PATCH 29/51] Initial update to latest updates --- v2/go.mod | 2 + v2/go.sum | 12 + v2/internal/runner/runner.go | 121 +++++++---- v2/pkg/executer/executer_http.go | 30 ++- v2/pkg/executor/executer_http.go | 313 --------------------------- v2/pkg/requests/bulk-http-request.go | 20 +- v2/pkg/templates/templates.go | 2 +- v2/pkg/workflows/var.go | 5 +- 8 files changed, 141 insertions(+), 364 deletions(-) delete mode 100644 v2/pkg/executor/executer_http.go diff --git a/v2/go.mod b/v2/go.mod index 92142e4c..a96311c5 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -11,11 +11,13 @@ require ( github.com/google/go-github/v32 v32.1.0 github.com/json-iterator/go v1.1.10 github.com/karrick/godirwalk v1.15.6 + github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 github.com/miekg/dns v1.1.30 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/gologger v1.0.0 github.com/projectdiscovery/retryabledns v1.0.4 github.com/projectdiscovery/retryablehttp-go v1.0.1 + github.com/vbauerster/mpb/v5 v5.2.4 golang.org/x/net v0.0.0-20200707034311-ab3426394381 gopkg.in/yaml.v2 v2.3.0 ) diff --git a/v2/go.sum b/v2/go.sum index b2a1cf82..5fc827d1 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -1,6 +1,10 @@ github.com/Knetic/govaluate v1.5.0 h1:L4MyqdJSld9xr2eZcZHCWLfeIX2SBjqrwIKG1pcm/+4= github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f6s9Tn1Tt7/WTEg= github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM= +github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0S6Vi7/lbWECcX0j45yZReDZ56BQsrVBOEEY= github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= github.com/blang/semver v1.1.0 h1:ol1rO7QQB5uy7umSNV7VAmLugfLRD+17sYJujRNYPhg= @@ -26,6 +30,8 @@ github.com/karrick/godirwalk v1.15.6 h1:Yf2mmR8TJy+8Fa0SuQVto5SYap6IF7lNVX4Jdl8G github.com/karrick/godirwalk v1.15.6/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 h1:bqDmpDG49ZRnB5PcgP0RXtQvnMSgIF14M7CBd2shtXs= github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.30 h1:Qww6FseFn8PRfw07jueqIXqodm0JKiiKuK0DeXSqfyo= github.com/miekg/dns v1.1.30/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= @@ -45,6 +51,10 @@ github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/vbauerster/mpb v1.1.3 h1:IRgic8VFaURXkW0VxDLkNOiNaAgtw0okB2YIaVvJDI4= +github.com/vbauerster/mpb v3.4.0+incompatible h1:mfiiYw87ARaeRW6x5gWwYRUawxaW1tLAD8IceomUCNw= +github.com/vbauerster/mpb/v5 v5.2.4 h1:PLP8vv75RcEgxGoJVtKaRD2FHSxEmIV/u4ZuOrfO8Qg= +github.com/vbauerster/mpb/v5 v5.2.4/go.mod h1:K4iCHQp5sWnmAgEn+uW1sAxSilctb4JPAGXx49jV+Aw= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= @@ -63,6 +73,8 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 5a29fe2d..0feb6b47 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -16,6 +16,7 @@ import ( "github.com/d5/tengo/v2/stdlib" "github.com/karrick/godirwalk" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/internal/progress" "github.com/projectdiscovery/nuclei/v2/pkg/executer" "github.com/projectdiscovery/nuclei/v2/pkg/requests" "github.com/projectdiscovery/nuclei/v2/pkg/templates" @@ -24,6 +25,9 @@ import ( // Runner is a client for running the enumeration process. type Runner struct { + input *os.File + inputCount int64 + // output is the output file to write if any output *os.File outputMutex *sync.Mutex @@ -32,6 +36,9 @@ type Runner struct { templatesConfig *nucleiConfig // options contains configuration options for runner options *Options + + // progress tracking + progress *progress.Progress } // New creates a new client for running enumeration process. @@ -71,6 +78,25 @@ func New(options *Options) (*Runner, error) { tempInput.Close() } + // Setup input, handle a list of hosts as argument + var err error + if options.Targets != "" { + runner.input, err = os.Open(options.Targets) + } else if options.Stdin || options.Target != "" { + runner.input, err = os.Open(runner.tempFile) + } + if err != nil { + gologger.Fatalf("Could not open targets file '%s': %s\n", options.Targets, err) + } + + // Precompute total number of targets + scanner := bufio.NewScanner(runner.input) + runner.inputCount = 0 + for scanner.Scan() { + runner.inputCount++ + } + runner.input.Seek(0, 0) + // Create the output file if asked if options.Output != "" { output, err := os.Create(options.Output) @@ -79,12 +105,17 @@ func New(options *Options) (*Runner, error) { } runner.output = output } + + // Creates the progress tracking object + runner.progress = progress.NewProgress(runner.options.NoColor) + return runner, nil } // Close releases all the resources and cleans up func (r *Runner) Close() { r.output.Close() + r.input.Close() os.Remove(r.tempFile) } @@ -191,21 +222,51 @@ func (r *Runner) RunEnumeration() { gologger.Fatalf("Error, no templates were found.\n") } + p := r.progress + templateCount := len(allTemplates) + isSingleTemplate := templateCount == 1 + + if !isSingleTemplate { + // precompute request count + var totalRequests int64 = 0 + for _, match := range allTemplates { + t, err := r.parse(match) + switch t.(type) { + case *templates.Template: + template := t.(*templates.Template) + totalRequests += template.GetHTTPRequestsCount() + default: + gologger.Errorf("Could not parse file '%s': %s\n", match, err) + } + } + + // track global progress + p.SetupGlobalProgressbar(r.inputCount, templateCount, r.inputCount*totalRequests) + } + // run with the specified templates var results bool - for _, match := range allTemplates { + for i, match := range allTemplates { t, err := r.parse(match) switch t.(type) { case *templates.Template: template := t.(*templates.Template) + + if isSingleTemplate { + // track single template progress + p.SetupTemplateProgressbar(-1, -1, template.ID, r.inputCount*template.GetHTTPRequestsCount()) + } else { + p.SetupTemplateProgressbar(i, templateCount, template.ID, r.inputCount * template.GetHTTPRequestsCount()) + } + for _, request := range template.RequestsDNS { - dnsResults := r.processTemplateRequest(template, request) + dnsResults := r.processTemplateWithList(p, template, request) if dnsResults { results = dnsResults } } for _, request := range template.BulkRequestsHTTP { - httpResults := r.processTemplateRequest(template, request) + httpResults := r.processTemplateWithList(p, template, request) if httpResults { results = httpResults } @@ -214,9 +275,15 @@ func (r *Runner) RunEnumeration() { workflow := t.(*workflows.Workflow) r.ProcessWorkflowWithList(workflow) default: + p.StartStdCapture() gologger.Errorf("Could not parse file '%s': %s\n", match, err) + p.StopStdCapture() } } + p.Wait() + p.ShowStdErr() + p.ShowStdOut() + if !results { if r.output != nil { outputFile := r.output.Name() @@ -228,33 +295,16 @@ func (r *Runner) RunEnumeration() { return } -// processTemplate processes a template and runs the enumeration on all the targets -func (r *Runner) processTemplateRequest(template *templates.Template, request interface{}) bool { - var file *os.File - var err error - - // Handle a list of hosts as argument - if r.options.Targets != "" { - file, err = os.Open(r.options.Targets) - } else if r.options.Stdin || r.options.Target != "" { - file, err = os.Open(r.tempFile) - } - if err != nil { - gologger.Fatalf("Could not open targets file '%s': %s\n", r.options.Targets, err) - } - results := r.processTemplateWithList(template, request, file) - file.Close() - return results -} - -// processDomain processes the list with a template -func (r *Runner) processTemplateWithList(template *templates.Template, request interface{}, reader io.Reader) bool { +// processTemplateWithList processes a template and runs the enumeration on all the targets +func (r *Runner) processTemplateWithList(p *progress.Progress, template *templates.Template, request interface{}) bool { // Display the message for the template message := fmt.Sprintf("[%s] Loaded template %s (@%s)", template.ID, template.Info.Name, template.Info.Author) if template.Info.Severity != "" { message += " [" + template.Info.Severity + "]" } + p.StartStdCapture() gologger.Infof("%s\n", message) + p.StopStdCapture() var writer *bufio.Writer if r.output != nil { @@ -300,7 +350,8 @@ func (r *Runner) processTemplateWithList(template *templates.Template, request i limiter := make(chan struct{}, r.options.Threads) wg := &sync.WaitGroup{} - scanner := bufio.NewScanner(reader) + r.input.Seek(0, 0) + scanner := bufio.NewScanner(r.input) for scanner.Scan() { text := scanner.Text() if text == "" { @@ -313,13 +364,15 @@ func (r *Runner) processTemplateWithList(template *templates.Template, request i var result executer.Result if httpExecuter != nil { - result = httpExecuter.ExecuteHTTP(URL) + result = httpExecuter.ExecuteHTTP(p, URL) } if dnsExecuter != nil { result = dnsExecuter.ExecuteDNS(URL) } if result.Error != nil { + p.StartStdCapture() gologger.Warningf("Could not execute step: %s\n", result.Error) + p.StopStdCapture() } <-limiter wg.Done() @@ -343,20 +396,8 @@ func (r *Runner) processTemplateWithList(template *templates.Template, request i // ProcessWorkflowWithList coming from stdin or list of targets func (r *Runner) ProcessWorkflowWithList(workflow *workflows.Workflow) { - var file *os.File - var err error - // Handle a list of hosts as argument - if r.options.Targets != "" { - file, err = os.Open(r.options.Targets) - } else if r.options.Stdin { - file, err = os.Open(r.tempFile) - } - if err != nil { - gologger.Fatalf("Could not open targets file '%s': %s\n", r.options.Targets, err) - } - defer file.Close() - - scanner := bufio.NewScanner(file) + r.input.Seek(0, 0) + scanner := bufio.NewScanner(r.input) for scanner.Scan() { text := scanner.Text() if text == "" { diff --git a/v2/pkg/executer/executer_http.go b/v2/pkg/executer/executer_http.go index b4a5d934..8f43fa04 100644 --- a/v2/pkg/executer/executer_http.go +++ b/v2/pkg/executer/executer_http.go @@ -17,6 +17,7 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/internal/progress" "github.com/projectdiscovery/nuclei/v2/pkg/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/requests" "github.com/projectdiscovery/nuclei/v2/pkg/templates" @@ -99,45 +100,54 @@ func NewHTTPExecuter(options *HTTPOptions) (*HTTPExecuter, error) { } // ExecuteHTTP executes the HTTP request on a URL -func (e *HTTPExecuter) ExecuteHTTP(URL string) (result Result) { +func (e *HTTPExecuter) ExecuteHTTP(p *progress.Progress, URL string) (result Result) { result.Matches = make(map[string]interface{}) result.Extractions = make(map[string]interface{}) dynamicvalues := make(map[string]interface{}) e.bulkHttpRequest.Reset() + remaining := e.template.GetHTTPRequestsCount() + for e.bulkHttpRequest.Next() && !result.Done { httpRequest, err := e.bulkHttpRequest.MakeHTTPRequest(URL, dynamicvalues, e.bulkHttpRequest.Current()) if err != nil { - result.Error = errors.Wrap(err, "could not make http request") + result.Error = errors.Wrap(err, "could not build http request") return } - err = e.handleHTTP(URL, httpRequest, dynamicvalues, &result) + err = e.handleHTTP(p, URL, httpRequest, dynamicvalues, &result) if err != nil { - result.Error = errors.Wrap(err, "could not make http request") + result.Error = errors.Wrap(err, "could not handle http request") + p.Drop(remaining) return } e.bulkHttpRequest.Increment() + p.Update() + remaining-- } + p.StartStdCapture() gologger.Verbosef("Sent HTTP request to %s\n", "http-request", URL) + p.StopStdCapture() return } -func (e *HTTPExecuter) handleHTTP(URL string, request *requests.HttpRequest, dynamicvalues map[string]interface{}, result *Result) error { +func (e *HTTPExecuter) handleHTTP(p *progress.Progress, URL string, request *requests.HttpRequest, dynamicvalues map[string]interface{}, result *Result) error { e.setCustomHeaders(request) req := request.Request if e.debug { - gologger.Infof("Dumped HTTP request for %s (%s)\n\n", URL, e.template.ID) dumpedRequest, err := httputil.DumpRequest(req.Request, true) if err != nil { return errors.Wrap(err, "could not make http request") } + p.StartStdCapture() + gologger.Infof("Dumped HTTP request for %s (%s)\n\n", URL, e.template.ID) fmt.Fprintf(os.Stderr, "%s", string(dumpedRequest)) + p.StopStdCapture() } resp, err := e.httpClient.Do(req) if err != nil { @@ -148,12 +158,14 @@ func (e *HTTPExecuter) handleHTTP(URL string, request *requests.HttpRequest, dyn } if e.debug { - gologger.Infof("Dumped HTTP response for %s (%s)\n\n", URL, e.template.ID) dumpedResponse, err := httputil.DumpResponse(resp, true) if err != nil { return errors.Wrap(err, "could not dump http response") } + p.StartStdCapture() + gologger.Infof("Dumped HTTP response for %s (%s)\n\n", URL, e.template.ID) fmt.Fprintf(os.Stderr, "%s\n", string(dumpedResponse)) + p.StopStdCapture() } data, err := ioutil.ReadAll(resp.Body) @@ -190,7 +202,9 @@ func (e *HTTPExecuter) handleHTTP(URL string, request *requests.HttpRequest, dyn result.Matches[matcher.Name] = nil // probably redundant but ensures we snapshot current payload values when matchers are valid result.Meta = request.Meta + p.StartStdCapture() e.writeOutputHTTP(request, resp, body, matcher, nil) + p.StopStdCapture() e.Results = true } } @@ -214,7 +228,9 @@ func (e *HTTPExecuter) handleHTTP(URL string, request *requests.HttpRequest, dyn // Write a final string of output if matcher type is // AND or if we have extractors for the mechanism too. if len(e.bulkHttpRequest.Extractors) > 0 || matcherCondition == matchers.ANDCondition { + p.StartStdCapture() e.writeOutputHTTP(request, resp, body, nil, extractorResults) + p.StopStdCapture() e.Results = true } diff --git a/v2/pkg/executor/executer_http.go b/v2/pkg/executor/executer_http.go deleted file mode 100644 index a3a60bb4..00000000 --- a/v2/pkg/executor/executer_http.go +++ /dev/null @@ -1,313 +0,0 @@ -package executor - -import ( - "bufio" - "crypto/tls" - "fmt" - "github.com/projectdiscovery/nuclei/v2/internal/progress" - "io" - "io/ioutil" - "net/http" - "net/http/httputil" - "net/url" - "os" - "strings" - "sync" - "sync/atomic" - "time" - - "github.com/pkg/errors" - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v2/pkg/matchers" - "github.com/projectdiscovery/nuclei/v2/pkg/requests" - "github.com/projectdiscovery/nuclei/v2/pkg/templates" - "github.com/projectdiscovery/retryablehttp-go" - "golang.org/x/net/proxy" -) - -// HTTPExecutor is client for performing HTTP requests -// for a template. -type HTTPExecutor struct { - debug bool - results uint32 - jsonOutput bool - httpClient *retryablehttp.Client - template *templates.Template - httpRequest *requests.HTTPRequest - writer *bufio.Writer - outputMutex *sync.Mutex - customHeaders requests.CustomHeaders -} - -// HTTPOptions contains configuration options for the HTTP executor. -type HTTPOptions struct { - Template *templates.Template - HTTPRequest *requests.HTTPRequest - Writer *bufio.Writer - Timeout int - Retries int - ProxyURL string - ProxySocksURL string - Debug bool - JSON bool - CustomHeaders requests.CustomHeaders -} - -// NewHTTPExecutor creates a new HTTP executor from a template -// and a HTTP request query. -func NewHTTPExecutor(options *HTTPOptions) (*HTTPExecutor, error) { - var proxyURL *url.URL - var err error - - if options.ProxyURL != "" { - proxyURL, err = url.Parse(options.ProxyURL) - } - if err != nil { - return nil, err - } - - // Create the HTTP Client - client := makeHTTPClient(proxyURL, options) - client.CheckRetry = retryablehttp.HostSprayRetryPolicy() - - executer := &HTTPExecutor{ - debug: options.Debug, - jsonOutput: options.JSON, - results: 0, - httpClient: client, - template: options.Template, - httpRequest: options.HTTPRequest, - outputMutex: &sync.Mutex{}, - writer: options.Writer, - customHeaders: options.CustomHeaders, - } - return executer, nil -} - -// GotResults returns true if there were any results for the executor -func (e *HTTPExecutor) GotResults() bool { - if atomic.LoadUint32(&e.results) == 0 { - return false - } - return true -} - -// ExecuteHTTP executes the HTTP request on a URL -func (e *HTTPExecutor) ExecuteHTTP(p *progress.Progress, URL string) error { - // Compile each request for the template based on the URL - compiledRequest, err := e.httpRequest.MakeHTTPRequest(URL) - if err != nil { - return errors.Wrap(err, "could not compile http request") - } - - remaining := e.template.GetHTTPRequestsCount() - - // Send the request to the target servers -mainLoop: - for compiledRequest := range compiledRequest { - if compiledRequest.Error != nil { - p.Drop(remaining) - return errors.Wrap(err, "error in compiled http request") - } - e.setCustomHeaders(compiledRequest) - req := compiledRequest.Request - - if e.debug { - dumpedRequest, err := httputil.DumpRequest(req.Request, true) - if err != nil { - p.Drop(remaining) - return errors.Wrap(err, "could not dump http request") - } - p.StartStdCapture() - gologger.Infof("Dumped HTTP request for %s (%s)\n\n", URL, e.template.ID) - fmt.Fprintf(os.Stderr, "%s", string(dumpedRequest)) - p.StopStdCapture() - } - - resp, err := e.httpClient.Do(req) - if err != nil { - if resp != nil { - resp.Body.Close() - } - p.Drop(1) - p.StartStdCapture() - gologger.Warningf("Could not do request: %s\n", err) - p.StopStdCapture() - continue - } - - if e.debug { - dumpedResponse, err := httputil.DumpResponse(resp, true) - if err != nil { - p.Drop(remaining) - return errors.Wrap(err, "could not dump http response") - } - p.StartStdCapture() - gologger.Infof("Dumped HTTP response for %s (%s)\n\n", URL, e.template.ID) - fmt.Fprintf(os.Stderr, "%s\n", string(dumpedResponse)) - p.StopStdCapture() - } - - data, err := ioutil.ReadAll(resp.Body) - if err != nil { - io.Copy(ioutil.Discard, resp.Body) - resp.Body.Close() - p.Drop(remaining) - return errors.Wrap(err, "could not read http body") - } - resp.Body.Close() - - // net/http doesn't automatically decompress the response body if an encoding has been specified by the user in the request - // so in case we have to manually do it - data, err = requests.HandleDecompression(compiledRequest.Request, data) - if err != nil { - p.Drop(remaining) - return errors.Wrap(err, "could not decompress http body") - } - - // Convert response body from []byte to string with zero copy - body := unsafeToString(data) - - var headers string - matcherCondition := e.httpRequest.GetMatchersCondition() - for _, matcher := range e.httpRequest.Matchers { - headers = headersToString(resp.Header) - // Check if the matcher matched - if !matcher.Match(resp, body, headers) { - // If the condition is AND we haven't matched, try next request. - if matcherCondition == matchers.ANDCondition { - p.Update() - remaining-- - continue mainLoop - } - } else { - // If the matcher has matched, and its an OR - // write the first output then move to next matcher. - if matcherCondition == matchers.ORCondition && len(e.httpRequest.Extractors) == 0 { - // capture stdout and emit it via a mpb.BarFiller - p.StartStdCapture() - e.writeOutputHTTP(compiledRequest, matcher, nil) - p.StopStdCapture() - - atomic.CompareAndSwapUint32(&e.results, 0, 1) - } - } - } - - // All matchers have successfully completed so now start with the - // next task which is extraction of input from matchers. - var extractorResults []string - for _, extractor := range e.httpRequest.Extractors { - headers = headersToString(resp.Header) - for match := range extractor.Extract(body, headers) { - extractorResults = append(extractorResults, match) - } - } - - // Write a final string of output if matcher type is - // AND or if we have extractors for the mechanism too. - if len(e.httpRequest.Extractors) > 0 || matcherCondition == matchers.ANDCondition { - // capture stdout and emit it via a mpb.BarFiller - p.StartStdCapture() - e.writeOutputHTTP(compiledRequest, nil, extractorResults) - p.StopStdCapture() - - atomic.CompareAndSwapUint32(&e.results, 0, 1) - } - - p.Update() - remaining-- - } - - p.StartStdCapture() - gologger.Verbosef("Sent HTTP request to %s\n", "http-request", URL) - p.StopStdCapture() - - return nil -} - - -// Close closes the http executor for a template. -func (e *HTTPExecutor) Close() { - e.outputMutex.Lock() - e.writer.Flush() - e.outputMutex.Unlock() -} - -// makeHTTPClient creates a http client -func makeHTTPClient(proxyURL *url.URL, options *HTTPOptions) *retryablehttp.Client { - retryablehttpOptions := retryablehttp.DefaultOptionsSpraying - retryablehttpOptions.RetryWaitMax = 10 * time.Second - retryablehttpOptions.RetryMax = options.Retries - followRedirects := options.HTTPRequest.Redirects - maxRedirects := options.HTTPRequest.MaxRedirects - - transport := &http.Transport{ - MaxIdleConnsPerHost: -1, - TLSClientConfig: &tls.Config{ - Renegotiation: tls.RenegotiateOnceAsClient, - InsecureSkipVerify: true, - }, - DisableKeepAlives: true, - } - - // Attempts to overwrite the dial function with the socks proxied version - if options.ProxySocksURL != "" { - var proxyAuth *proxy.Auth - socksURL, err := url.Parse(options.ProxySocksURL) - if err == nil { - proxyAuth = &proxy.Auth{} - proxyAuth.User = socksURL.User.Username() - proxyAuth.Password, _ = socksURL.User.Password() - } - dialer, err := proxy.SOCKS5("tcp", fmt.Sprintf("%s:%s", socksURL.Hostname(), socksURL.Port()), proxyAuth, proxy.Direct) - if err == nil { - transport.Dial = dialer.Dial - } - } - - if proxyURL != nil { - transport.Proxy = http.ProxyURL(proxyURL) - } - return retryablehttp.NewWithHTTPClient(&http.Client{ - Transport: transport, - Timeout: time.Duration(options.Timeout) * time.Second, - CheckRedirect: makeCheckRedirectFunc(followRedirects, maxRedirects), - }, retryablehttpOptions) -} - -type checkRedirectFunc func(_ *http.Request, requests []*http.Request) error - -func makeCheckRedirectFunc(followRedirects bool, maxRedirects int) checkRedirectFunc { - return func(_ *http.Request, requests []*http.Request) error { - if !followRedirects { - return http.ErrUseLastResponse - } - if maxRedirects == 0 { - if len(requests) > 10 { - return http.ErrUseLastResponse - } - return nil - } - if len(requests) > maxRedirects { - return http.ErrUseLastResponse - } - return nil - } -} - -func (e *HTTPExecutor) setCustomHeaders(r *requests.CompiledHTTP) { - for _, customHeader := range e.customHeaders { - // This should be pre-computed somewhere and done only once - tokens := strings.Split(customHeader, ":") - // if it's an invalid header skip it - if len(tokens) < 2 { - continue - } - - headerName, headerValue := tokens[0], strings.Join(tokens[1:], "") - headerName = strings.TrimSpace(headerName) - headerValue = strings.TrimSpace(headerValue) - r.Request.Header.Set(headerName, headerValue) - } -} diff --git a/v2/pkg/requests/bulk-http-request.go b/v2/pkg/requests/bulk-http-request.go index 0046bfda..68156cc4 100644 --- a/v2/pkg/requests/bulk-http-request.go +++ b/v2/pkg/requests/bulk-http-request.go @@ -8,6 +8,7 @@ import ( "net/url" "regexp" "strings" + "sync" "github.com/Knetic/govaluate" "github.com/projectdiscovery/nuclei/v2/pkg/extractors" @@ -55,6 +56,7 @@ type BulkHTTPRequest struct { Raw []string `yaml:"raw,omitempty"` positionPath int positionRaw int + positionMutex sync.Mutex generator func(payloads map[string][]string) (out chan map[string]interface{}) currentPayloads map[string]interface{} basePayloads map[string][]string @@ -83,8 +85,8 @@ func (r *BulkHTTPRequest) SetAttackType(attack generators.Type) { } // Returns the total number of requests the YAML rule will perform -func (r *HTTPRequest) GetRequestCount() int64 { - return int64( len(r.Raw) | len(r.Path) ) +func (r *BulkHTTPRequest) GetRequestCount() int64 { + return int64(len(r.Raw) | len(r.Path)) } func (r *BulkHTTPRequest) MakeHTTPRequest(baseURL string, dynamicValues map[string]interface{}, data string) (*HttpRequest, error) { @@ -369,19 +371,30 @@ func (r *BulkHTTPRequest) parseRawRequest(request string, baseURL string) (*RawR } func (r *BulkHTTPRequest) Next() bool { + r.positionMutex.Lock() + defer r.positionMutex.Unlock() + if r.positionPath+r.positionRaw >= len(r.Path)+len(r.Raw) { return false } return true } func (r *BulkHTTPRequest) Position() int { + r.positionMutex.Lock() + defer r.positionMutex.Unlock() + return r.positionPath + r.positionRaw } func (r *BulkHTTPRequest) Reset() { + r.positionMutex.Lock() r.positionPath = 0 r.positionRaw = 0 + r.positionMutex.Unlock() } func (r *BulkHTTPRequest) Current() string { + r.positionMutex.Lock() + defer r.positionMutex.Unlock() + if r.positionPath < len(r.Path) && len(r.Path) != 0 { return r.Path[r.positionPath] } @@ -393,6 +406,9 @@ func (r *BulkHTTPRequest) Total() int { } func (r *BulkHTTPRequest) Increment() { + r.positionMutex.Lock() + defer r.positionMutex.Unlock() + if len(r.Path) > 0 && r.positionPath < len(r.Path) { r.positionPath++ return diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index f495be6c..73efd990 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -30,7 +30,7 @@ type Info struct { func (t* Template) GetHTTPRequestsCount() int64 { var count int64 = 0 - for _, request := range t.RequestsHTTP { + for _, request := range t.BulkRequestsHTTP { count += request.GetRequestCount() } return count diff --git a/v2/pkg/workflows/var.go b/v2/pkg/workflows/var.go index c40c93bd..e46b6869 100644 --- a/v2/pkg/workflows/var.go +++ b/v2/pkg/workflows/var.go @@ -51,6 +51,9 @@ func (n *NucleiVar) Call(args ...tengo.Object) (ret tengo.Object, err error) { externalVars = iterableToMap(args[1]) } + // track progress + p := progress.NewProgress(false) + var gotResult bool for _, template := range n.Templates { if template.HTTPOptions != nil { @@ -65,7 +68,7 @@ func (n *NucleiVar) Call(args ...tengo.Object) (ret tengo.Object, err error) { gologger.Warningf("Could not compile request for template '%s': %s\n", template.HTTPOptions.Template.ID, err) continue } - result := httpExecuter.ExecuteHTTP(n.URL) + result := httpExecuter.ExecuteHTTP(p, n.URL) if result.Error != nil { gologger.Warningf("Could not send request for template '%s': %s\n", template.HTTPOptions.Template.ID, result.Error) continue From 8aecbeb82117f99f02766d812b28b1f52663a2e8 Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Fri, 24 Jul 2020 22:30:15 +0200 Subject: [PATCH 30/51] Experimental support for new parallelism --- v2/internal/progress/bar.go | 7 +++- v2/internal/progress/progress.go | 68 +++++++++++++++----------------- v2/internal/runner/runner.go | 23 +++++------ v2/pkg/executer/executer_http.go | 4 +- 4 files changed, 52 insertions(+), 50 deletions(-) diff --git a/v2/internal/progress/bar.go b/v2/internal/progress/bar.go index 32774d51..2dee4f29 100644 --- a/v2/internal/progress/bar.go +++ b/v2/internal/progress/bar.go @@ -14,7 +14,7 @@ type Bar struct { // Drops the specified number of requests from the progress bar total. // This may be the case when uncompleted requests are encountered and shouldn't be part of the total count. -func (b *Bar) Drop(count int64) { +func (b *Bar) drop(count int64) { atomic.AddInt64(&b.total, -count) b.bar.SetTotal(atomic.LoadInt64(&b.total), false) } @@ -25,3 +25,8 @@ func (b *Bar) finish() { b.bar.SetTotal(b.total, true) } } + +// Update progress tracking information and increments the request counter by one unit. +func (b *Bar) increment() { + b.bar.Increment() +} \ No newline at end of file diff --git a/v2/internal/progress/progress.go b/v2/internal/progress/progress.go index 2ae847d6..44871c87 100644 --- a/v2/internal/progress/progress.go +++ b/v2/internal/progress/progress.go @@ -13,8 +13,8 @@ import ( // Encapsulates progress tracking. type Progress struct { progress *mpb.Progress - barTemplate *Bar - barGlobal *Bar + bars map[string]*Bar + gbar *Bar captureData *captureData stdCaptureMutex *sync.Mutex @@ -35,29 +35,22 @@ func NewProgress(noColor bool) *Progress { stdout: &strings.Builder{}, stderr: &strings.Builder{}, colorizer: aurora.NewAurora(!noColor), + bars: make(map[string]*Bar), } return p } // Creates and returns a progress bar that tracks request progress for a specific template. -func (p *Progress) SetupTemplateProgressbar(templateIndex int, templateCount int, name string, requestCount int64) { +func (p *Progress) SetupTemplateProgressbar(templateId string, requestCount int64, priority int) { + if p.bars[templateId] != nil { + panic(fmt.Sprintf("A progressbar is already bound to [%s].", templateId)) + } + color := p.colorizer - barName := "[" + color.Green(name).String() + "]" + barName := "[" + color.Green(templateId).String() + "]" + bar := p.setupProgressbar(barName, requestCount, priority) - if templateIndex > -1 && templateCount > -1 { - barName = color.Sprintf("[%d/%d] ", - color.Bold(color.Cyan(templateIndex)), - color.Cyan(templateCount)) + barName - } - - bar := p.setupProgressbar(barName, requestCount) - - if p.barTemplate != nil { - // ensure any previous bar has finished and dropped requests have also been considered - p.barTemplate.finish() - } - - p.barTemplate = &Bar{ + p.bars[templateId] = &Bar{ bar: bar, total: requestCount, initialTotal: requestCount, @@ -67,6 +60,10 @@ func (p *Progress) SetupTemplateProgressbar(templateIndex int, templateCount int // Creates and returns a progress bar that tracks all the requests progress. // This is only useful when multiple templates are processed within the same run. func (p *Progress) SetupGlobalProgressbar(hostCount int64, templateCount int, requestCount int64) { + if p.gbar != nil { + panic("A global progressbar is already present.") + } + color := p.colorizer hostPlural := "host" @@ -80,9 +77,9 @@ func (p *Progress) SetupGlobalProgressbar(hostCount int64, templateCount int, re color.Bold(color.Cyan(hostCount)), hostPlural) + "]" - bar := p.setupProgressbar(barName, requestCount) + bar := p.setupProgressbar(barName, requestCount, 0) - p.barGlobal = &Bar{ + p.gbar = &Bar{ bar: bar, total: requestCount, initialTotal: requestCount, @@ -91,22 +88,20 @@ func (p *Progress) SetupGlobalProgressbar(hostCount int64, templateCount int, re // Update progress tracking information and increments the request counter by one unit. // If a global progress bar is present it will be updated as well. -func (p *Progress) Update() { - p.barTemplate.bar.Increment() - - if p.barGlobal != nil { - p.barGlobal.bar.Increment() +func (p *Progress) Update(templateId string) { + p.bars[templateId].increment() + if p.gbar != nil { + p.gbar.increment() } } // Drops the specified number of requests from the progress bar total. // This may be the case when uncompleted requests are encountered and shouldn't be part of the total count. // If a global progress bar is present it will be updated as well. -func (p *Progress) Drop(count int64) { - p.barTemplate.Drop(count) - - if p.barGlobal != nil { - p.barGlobal.Drop(count) +func (p *Progress) Drop(templateId string, count int64) { + p.bars[templateId].drop(count) + if p.gbar != nil { + p.gbar.drop(count) } } @@ -114,20 +109,21 @@ func (p *Progress) Drop(count int64) { // wait for all the progress bars to finish. // If a global progress bar is present it will be updated as well. func (p *Progress) Wait() { - p.barTemplate.finish() - - if p.barGlobal != nil { - p.barGlobal.finish() + for _, bar := range p.bars { + bar.finish() + } + if p.gbar != nil { + p.gbar.finish() } - p.progress.Wait() } // Creates and returns a progress bar. -func (p *Progress) setupProgressbar(name string, total int64) *mpb.Bar { +func (p *Progress) setupProgressbar(name string, total int64, priority int) *mpb.Bar { color := p.colorizer return p.progress.AddBar( total, + mpb.BarPriority(priority), mpb.BarNoPop(), mpb.BarRemoveOnComplete(), mpb.PrependDecorators( diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index e6b416ab..75dd24d9 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -238,23 +238,30 @@ func (r *Runner) RunEnumeration() { switch t.(type) { case *templates.Template: template := t.(*templates.Template) - p.SetupTemplateProgressbar(-1, -1, template.ID, r.inputCount*template.GetHTTPRequestsCount()) + p.SetupTemplateProgressbar(template.ID, r.inputCount*template.GetHTTPRequestsCount(), 0) default: gologger.Errorf("Could not parse file '%s': %s\n", match, err) } } else { var totalRequests int64 = 0 - for _, match := range allTemplates { + parsedTemplates := []string{} + + for i, match := range allTemplates { t, err := r.parse(match) switch t.(type) { case *templates.Template: template := t.(*templates.Template) totalRequests += template.GetHTTPRequestsCount() + p.SetupTemplateProgressbar(template.ID, r.inputCount*template.GetHTTPRequestsCount(), i+1) + parsedTemplates = append(parsedTemplates, match) default: gologger.Errorf("Could not parse file '%s': %s\n", match, err) } } + // ensure only successfully parsed templates are processed + allTemplates = parsedTemplates + // track global progress p.SetupGlobalProgressbar(r.inputCount, templateCount, r.inputCount*totalRequests) } @@ -264,20 +271,14 @@ func (r *Runner) RunEnumeration() { results atomicboolean.AtomBool ) - for i, match := range allTemplates { + for _, match := range allTemplates { wgtemplates.Add(1) - - go func(match string, templateIndex int) { + go func(match string) { defer wgtemplates.Done() t, err := r.parse(match) switch t.(type) { case *templates.Template: template := t.(*templates.Template) - - if !isSingleTemplate { - p.SetupTemplateProgressbar(templateIndex, templateCount, template.ID, r.inputCount * template.GetHTTPRequestsCount()) - } - for _, request := range template.RequestsDNS { results.Or(r.processTemplateWithList(p, template, request)) } @@ -292,7 +293,7 @@ func (r *Runner) RunEnumeration() { gologger.Errorf("Could not parse file '%s': %s\n", match, err) p.StopStdCapture() } - }(match, i) + }(match) } wgtemplates.Wait() diff --git a/v2/pkg/executer/executer_http.go b/v2/pkg/executer/executer_http.go index aedc6b17..31f3421e 100644 --- a/v2/pkg/executer/executer_http.go +++ b/v2/pkg/executer/executer_http.go @@ -119,12 +119,12 @@ func (e *HTTPExecuter) ExecuteHTTP(p *progress.Progress, URL string) (result Res err = e.handleHTTP(p, URL, httpRequest, dynamicvalues, &result) if err != nil { result.Error = errors.Wrap(err, "could not handle http request") - p.Drop(remaining) + p.Drop(e.template.ID, remaining) return } e.bulkHttpRequest.Increment(URL) - p.Update() + p.Update(e.template.ID) remaining-- } From 3bfdd0c381ffb7787c13d245232b30479679a2f2 Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Sat, 25 Jul 2020 00:46:18 +0200 Subject: [PATCH 31/51] Increment progress instead of altering totals, wrap more std streams --- v2/internal/progress/bar.go | 13 ++----------- v2/internal/progress/progress.go | 29 +++++++++++------------------ v2/internal/runner/runner.go | 2 ++ v2/pkg/executer/executer_http.go | 1 + 4 files changed, 16 insertions(+), 29 deletions(-) diff --git a/v2/internal/progress/bar.go b/v2/internal/progress/bar.go index 2dee4f29..24663b07 100644 --- a/v2/internal/progress/bar.go +++ b/v2/internal/progress/bar.go @@ -2,7 +2,6 @@ package progress import ( "github.com/vbauerster/mpb/v5" - "sync/atomic" ) // Represents a single progress bar @@ -15,18 +14,10 @@ type Bar struct { // Drops the specified number of requests from the progress bar total. // This may be the case when uncompleted requests are encountered and shouldn't be part of the total count. func (b *Bar) drop(count int64) { - atomic.AddInt64(&b.total, -count) - b.bar.SetTotal(atomic.LoadInt64(&b.total), false) -} - -// Ensures that a progress bar's total count is up-to-date if during an enumeration there were uncompleted requests. -func (b *Bar) finish() { - if b.initialTotal != b.total { - b.bar.SetTotal(b.total, true) - } + b.bar.IncrInt64(count) } // Update progress tracking information and increments the request counter by one unit. func (b *Bar) increment() { b.bar.Increment() -} \ No newline at end of file +} diff --git a/v2/internal/progress/progress.go b/v2/internal/progress/progress.go index 44871c87..b6f60169 100644 --- a/v2/internal/progress/progress.go +++ b/v2/internal/progress/progress.go @@ -12,16 +12,14 @@ import ( // Encapsulates progress tracking. type Progress struct { - progress *mpb.Progress - bars map[string]*Bar - gbar *Bar - + progress *mpb.Progress + bars map[string]*Bar + gbar *Bar captureData *captureData stdCaptureMutex *sync.Mutex stdout *strings.Builder stderr *strings.Builder - - colorizer aurora.Aurora + colorizer aurora.Aurora } // Creates and returns a new progress tracking object. @@ -34,8 +32,8 @@ func NewProgress(noColor bool) *Progress { stdCaptureMutex: &sync.Mutex{}, stdout: &strings.Builder{}, stderr: &strings.Builder{}, - colorizer: aurora.NewAurora(!noColor), - bars: make(map[string]*Bar), + colorizer: aurora.NewAurora(!noColor), + bars: make(map[string]*Bar), } return p } @@ -47,7 +45,7 @@ func (p *Progress) SetupTemplateProgressbar(templateId string, requestCount int6 } color := p.colorizer - barName := "[" + color.Green(templateId).String() + "]" + barName := color.Green(templateId).String() bar := p.setupProgressbar(barName, requestCount, priority) p.bars[templateId] = &Bar{ @@ -71,11 +69,11 @@ func (p *Progress) SetupGlobalProgressbar(hostCount int64, templateCount int, re hostPlural = "hosts" } - barName := "[" + color.Sprintf( + barName := color.Sprintf( color.Cyan("%d templates, %d %s"), color.Bold(color.Cyan(templateCount)), color.Bold(color.Cyan(hostCount)), - hostPlural) + "]" + hostPlural) bar := p.setupProgressbar(barName, requestCount, 0) @@ -109,25 +107,20 @@ func (p *Progress) Drop(templateId string, count int64) { // wait for all the progress bars to finish. // If a global progress bar is present it will be updated as well. func (p *Progress) Wait() { - for _, bar := range p.bars { - bar.finish() - } - if p.gbar != nil { - p.gbar.finish() - } p.progress.Wait() } // Creates and returns a progress bar. func (p *Progress) setupProgressbar(name string, total int64, priority int) *mpb.Bar { color := p.colorizer + return p.progress.AddBar( total, mpb.BarPriority(priority), mpb.BarNoPop(), mpb.BarRemoveOnComplete(), mpb.PrependDecorators( - decor.Name(name, decor.WCSyncSpaceR), + decor.Name(fmt.Sprintf("[%s]", name), decor.WCSyncSpaceR), decor.CountersNoUnit(color.Blue(" %d/%d").String(), decor.WCSyncSpace), decor.NewPercentage(color.Bold("%d").String(), decor.WCSyncSpace), ), diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 75dd24d9..6f686c46 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -360,7 +360,9 @@ func (r *Runner) processTemplateWithList(p *progress.Progress, template *templat }) } if err != nil { + p.StartStdCapture() gologger.Warningf("Could not create http client: %s\n", err) + p.StopStdCapture() return false } diff --git a/v2/pkg/executer/executer_http.go b/v2/pkg/executer/executer_http.go index 31f3421e..0be9d416 100644 --- a/v2/pkg/executer/executer_http.go +++ b/v2/pkg/executer/executer_http.go @@ -113,6 +113,7 @@ func (e *HTTPExecuter) ExecuteHTTP(p *progress.Progress, URL string) (result Res httpRequest, err := e.bulkHttpRequest.MakeHTTPRequest(URL, dynamicvalues, e.bulkHttpRequest.Current(URL)) if err != nil { result.Error = errors.Wrap(err, "could not build http request") + p.Drop(e.template.ID, remaining) return } From 88e683c0b51ad735c36ad7fd79b6e3cd72794dfd Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Sat, 25 Jul 2020 14:56:20 +0200 Subject: [PATCH 32/51] Always use a global progress bar, protect input file reading --- v2/go.mod | 3 +++ v2/go.sum | 15 +++++++----- v2/internal/runner/runner.go | 46 ++++++++++++++++-------------------- 3 files changed, 32 insertions(+), 32 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index b3e01084..622be1bb 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -16,7 +16,10 @@ require ( github.com/projectdiscovery/gologger v1.0.0 github.com/projectdiscovery/retryabledns v1.0.4 github.com/projectdiscovery/retryablehttp-go v1.0.1 + github.com/stretchr/testify v1.5.1 github.com/vbauerster/mpb/v5 v5.2.4 + golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 // indirect golang.org/x/net v0.0.0-20200707034311-ab3426394381 + golang.org/x/sys v0.0.0-20200724161237-0e2f3a69832c // indirect gopkg.in/yaml.v2 v2.3.0 ) diff --git a/v2/go.sum b/v2/go.sum index e70d632c..01fc7adb 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -1,4 +1,3 @@ -github.com/Knetic/govaluate v1.5.0 h1:L4MyqdJSld9xr2eZcZHCWLfeIX2SBjqrwIKG1pcm/+4= github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f6s9Tn1Tt7/WTEg= github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM= @@ -7,16 +6,14 @@ github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpH github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0S6Vi7/lbWECcX0j45yZReDZ56BQsrVBOEEY= github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= -github.com/blang/semver v1.1.0 h1:ol1rO7QQB5uy7umSNV7VAmLugfLRD+17sYJujRNYPhg= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/d5/tengo v1.24.8 h1:PRJ+NWt7ae/9sSbIfThOBTkPSvNV+dwYoBAvwfNgNJY= github.com/d5/tengo/v2 v2.6.0 h1:D0cJtpiBzaLJ/Smv6nnUc/LIfO46oKwDx85NZtIRNRI= github.com/d5/tengo/v2 v2.6.0/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= github.com/google/go-github/v32 v32.1.0 h1:GWkQOdXqviCPx7Q7Fj+KyPoGm4SwHRh8rheoPhd27II= github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= @@ -39,6 +36,7 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLD github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/projectdiscovery/gologger v1.0.0 h1:XAQ8kHeVKXMjY4rLGh7eT5+oHU077BNEvs7X6n+vu1s= github.com/projectdiscovery/gologger v1.0.0/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= @@ -48,15 +46,16 @@ github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNw github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/vbauerster/mpb v1.1.3 h1:IRgic8VFaURXkW0VxDLkNOiNaAgtw0okB2YIaVvJDI4= -github.com/vbauerster/mpb v3.4.0+incompatible h1:mfiiYw87ARaeRW6x5gWwYRUawxaW1tLAD8IceomUCNw= github.com/vbauerster/mpb/v5 v5.2.4 h1:PLP8vv75RcEgxGoJVtKaRD2FHSxEmIV/u4ZuOrfO8Qg= github.com/vbauerster/mpb/v5 v5.2.4/go.mod h1:K4iCHQp5sWnmAgEn+uW1sAxSilctb4JPAGXx49jV+Aw= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 h1:DZhuSZLsGlFL4CmhA8BcRA0mnthyA/nZ00AqCUo7vHg= +golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -65,6 +64,7 @@ golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -73,10 +73,13 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20u golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200724161237-0e2f3a69832c h1:UIcGWL6/wpCfyGuJnRFJRurA+yj8RrW7Q6x2YMCXt6c= +golang.org/x/sys v0.0.0-20200724161237-0e2f3a69832c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 6f686c46..f583d8c7 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -28,6 +28,7 @@ import ( type Runner struct { input *os.File inputCount int64 + inputMutex *sync.Mutex // output is the output file to write if any output *os.File @@ -46,6 +47,7 @@ type Runner struct { // New creates a new client for running enumeration process. func New(options *Options) (*Runner, error) { runner := &Runner{ + inputMutex: &sync.Mutex{}, outputMutex: &sync.Mutex{}, options: options, } @@ -229,43 +231,32 @@ func (r *Runner) RunEnumeration() { // progress tracking p := r.progress templateCount := len(allTemplates) - isSingleTemplate := templateCount == 1 // precompute total request count if we are executing more than one template - if isSingleTemplate { - match := allTemplates[0] + var totalRequests int64 = 0 + barIndex := 0 + parsedTemplates := []string{} + + for _, match := range allTemplates { t, err := r.parse(match) switch t.(type) { case *templates.Template: + barIndex++ template := t.(*templates.Template) - p.SetupTemplateProgressbar(template.ID, r.inputCount*template.GetHTTPRequestsCount(), 0) + totalRequests += template.GetHTTPRequestsCount() + p.SetupTemplateProgressbar(template.ID, r.inputCount*template.GetHTTPRequestsCount(), barIndex) + parsedTemplates = append(parsedTemplates, match) default: gologger.Errorf("Could not parse file '%s': %s\n", match, err) } - } else { - var totalRequests int64 = 0 - parsedTemplates := []string{} - - for i, match := range allTemplates { - t, err := r.parse(match) - switch t.(type) { - case *templates.Template: - template := t.(*templates.Template) - totalRequests += template.GetHTTPRequestsCount() - p.SetupTemplateProgressbar(template.ID, r.inputCount*template.GetHTTPRequestsCount(), i+1) - parsedTemplates = append(parsedTemplates, match) - default: - gologger.Errorf("Could not parse file '%s': %s\n", match, err) - } - } - - // ensure only successfully parsed templates are processed - allTemplates = parsedTemplates - - // track global progress - p.SetupGlobalProgressbar(r.inputCount, templateCount, r.inputCount*totalRequests) } + // ensure only successfully parsed templates are processed + allTemplates = parsedTemplates + + // track global progress + p.SetupGlobalProgressbar(r.inputCount, templateCount, r.inputCount*totalRequests) + var ( wgtemplates sync.WaitGroup results atomicboolean.AtomBool @@ -367,6 +358,8 @@ func (r *Runner) processTemplateWithList(p *progress.Progress, template *templat } var wg sync.WaitGroup + + r.inputMutex.Lock() r.input.Seek(0, 0) scanner := bufio.NewScanner(r.input) for scanner.Scan() { @@ -396,6 +389,7 @@ func (r *Runner) processTemplateWithList(p *progress.Progress, template *templat <-r.limiter }(text) } + r.inputMutex.Unlock() wg.Wait() From dad381e66eef1fc17a7ee81d30266eddfa8e6972 Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Sat, 25 Jul 2020 19:58:17 +0200 Subject: [PATCH 33/51] Cleanup pluralization --- v2/internal/progress/progress.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/v2/internal/progress/progress.go b/v2/internal/progress/progress.go index b6f60169..14560e00 100644 --- a/v2/internal/progress/progress.go +++ b/v2/internal/progress/progress.go @@ -55,6 +55,13 @@ func (p *Progress) SetupTemplateProgressbar(templateId string, requestCount int6 } } +func pluralize(count int64, singular, plural string) string { + if count > 1 { + return plural + } + return singular +} + // Creates and returns a progress bar that tracks all the requests progress. // This is only useful when multiple templates are processed within the same run. func (p *Progress) SetupGlobalProgressbar(hostCount int64, templateCount int, requestCount int64) { @@ -64,16 +71,12 @@ func (p *Progress) SetupGlobalProgressbar(hostCount int64, templateCount int, re color := p.colorizer - hostPlural := "host" - if hostCount > 1 { - hostPlural = "hosts" - } - barName := color.Sprintf( - color.Cyan("%d templates, %d %s"), + color.Cyan("%d %s, %d %s"), color.Bold(color.Cyan(templateCount)), + pluralize(int64(templateCount), "template", "templates"), color.Bold(color.Cyan(hostCount)), - hostPlural) + pluralize(hostCount, "host", "hosts")) bar := p.setupProgressbar(barName, requestCount, 0) From 436e7223d8d9bdedd5c9c7c237b8d91ed22450bd Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Sat, 25 Jul 2020 20:29:12 +0200 Subject: [PATCH 34/51] Update output coloring, set a maximum length for template names --- v2/internal/progress/progress.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/v2/internal/progress/progress.go b/v2/internal/progress/progress.go index 14560e00..fa1d3b97 100644 --- a/v2/internal/progress/progress.go +++ b/v2/internal/progress/progress.go @@ -45,7 +45,14 @@ func (p *Progress) SetupTemplateProgressbar(templateId string, requestCount int6 } color := p.colorizer - barName := color.Green(templateId).String() + uiBarName := templateId + + const MaxLen = 40 + if len(uiBarName) > MaxLen { + uiBarName = uiBarName[:MaxLen] + ".." + } + + barName := color.BrightYellow(uiBarName).String() bar := p.setupProgressbar(barName, requestCount, priority) p.bars[templateId] = &Bar{ @@ -124,11 +131,11 @@ func (p *Progress) setupProgressbar(name string, total int64, priority int) *mpb mpb.BarRemoveOnComplete(), mpb.PrependDecorators( decor.Name(fmt.Sprintf("[%s]", name), decor.WCSyncSpaceR), - decor.CountersNoUnit(color.Blue(" %d/%d").String(), decor.WCSyncSpace), + decor.CountersNoUnit(color.BrightBlue(" %d/%d").String(), decor.WCSyncSpace), decor.NewPercentage(color.Bold("%d").String(), decor.WCSyncSpace), ), mpb.AppendDecorators( - decor.AverageSpeed(0, color.Yellow("%.2f r/s ").String(), decor.WCSyncSpace), + decor.AverageSpeed(0, color.BrightBlue("%.2f r/s ").String(), decor.WCSyncSpace), decor.Elapsed(decor.ET_STYLE_GO, decor.WCSyncSpace), decor.AverageETA(decor.ET_STYLE_GO, decor.WCSyncSpace), ), From d8e69cacf20653ec6e04373863ba62a79728a925 Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Sat, 25 Jul 2020 21:03:18 +0200 Subject: [PATCH 35/51] Pad bar name with spaces (approx. size due to escape codes) --- v2/internal/progress/progress.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/v2/internal/progress/progress.go b/v2/internal/progress/progress.go index fa1d3b97..b40f1d2f 100644 --- a/v2/internal/progress/progress.go +++ b/v2/internal/progress/progress.go @@ -52,8 +52,8 @@ func (p *Progress) SetupTemplateProgressbar(templateId string, requestCount int6 uiBarName = uiBarName[:MaxLen] + ".." } - barName := color.BrightYellow(uiBarName).String() - bar := p.setupProgressbar(barName, requestCount, priority) + uiBarName = fmt.Sprintf(fmt.Sprintf("%%-%ds", MaxLen), "[" + color.BrightYellow(uiBarName).String() + "]") + bar := p.setupProgressbar(uiBarName, requestCount, priority) p.bars[templateId] = &Bar{ bar: bar, @@ -62,13 +62,6 @@ func (p *Progress) SetupTemplateProgressbar(templateId string, requestCount int6 } } -func pluralize(count int64, singular, plural string) string { - if count > 1 { - return plural - } - return singular -} - // Creates and returns a progress bar that tracks all the requests progress. // This is only useful when multiple templates are processed within the same run. func (p *Progress) SetupGlobalProgressbar(hostCount int64, templateCount int, requestCount int64) { @@ -85,7 +78,7 @@ func (p *Progress) SetupGlobalProgressbar(hostCount int64, templateCount int, re color.Bold(color.Cyan(hostCount)), pluralize(hostCount, "host", "hosts")) - bar := p.setupProgressbar(barName, requestCount, 0) + bar := p.setupProgressbar("[" + barName + "]", requestCount, 0) p.gbar = &Bar{ bar: bar, @@ -94,6 +87,13 @@ func (p *Progress) SetupGlobalProgressbar(hostCount int64, templateCount int, re } } +func pluralize(count int64, singular, plural string) string { + if count > 1 { + return plural + } + return singular +} + // Update progress tracking information and increments the request counter by one unit. // If a global progress bar is present it will be updated as well. func (p *Progress) Update(templateId string) { @@ -130,7 +130,7 @@ func (p *Progress) setupProgressbar(name string, total int64, priority int) *mpb mpb.BarNoPop(), mpb.BarRemoveOnComplete(), mpb.PrependDecorators( - decor.Name(fmt.Sprintf("[%s]", name), decor.WCSyncSpaceR), + decor.Name(name, decor.WCSyncSpaceR), decor.CountersNoUnit(color.BrightBlue(" %d/%d").String(), decor.WCSyncSpace), decor.NewPercentage(color.Bold("%d").String(), decor.WCSyncSpace), ), From 65d7246b2f1b6bbcc6fc329df9b5d4d987ea1a39 Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Sat, 25 Jul 2020 22:15:45 +0200 Subject: [PATCH 36/51] Use custom formatter for percentage, ensure fixed string size is used --- v2/internal/progress/progress.go | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/v2/internal/progress/progress.go b/v2/internal/progress/progress.go index b40f1d2f..ab5010b2 100644 --- a/v2/internal/progress/progress.go +++ b/v2/internal/progress/progress.go @@ -132,7 +132,7 @@ func (p *Progress) setupProgressbar(name string, total int64, priority int) *mpb mpb.PrependDecorators( decor.Name(name, decor.WCSyncSpaceR), decor.CountersNoUnit(color.BrightBlue(" %d/%d").String(), decor.WCSyncSpace), - decor.NewPercentage(color.Bold("%d").String(), decor.WCSyncSpace), + customFormattedPercentage(color.Bold("%3.0f%%").String(), decor.WCSyncSpace), ), mpb.AppendDecorators( decor.AverageSpeed(0, color.BrightBlue("%.2f r/s ").String(), decor.WCSyncSpace), @@ -142,6 +142,26 @@ func (p *Progress) setupProgressbar(name string, total int64, priority int) *mpb ) } +// Helper function to calculate percentage +func computePercentage(total, current int64, width int) float64 { + if total <= 0 { + return 0 + } + if current >= total { + return float64(width) + } + return float64(int64(width)*current) / float64(total) +} + +// Percentage decorator with custom formatting +func customFormattedPercentage(format string, wcc ...decor.WC) decor.Decorator { + f := func(s decor.Statistics) string { + p := computePercentage(s.Total, s.Current, 100) + return fmt.Sprintf(format, p) + } + return decor.Any(f, wcc...) +} + // Starts capturing stdout and stderr instead of producing visual output that may interfere with the progress bars. func (p *Progress) StartStdCapture() { p.stdCaptureMutex.Lock() From 48cf65b56336506b011a260bd3f3d32722b3c10b Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Sat, 25 Jul 2020 23:02:24 +0200 Subject: [PATCH 37/51] Automatically de-duplicate supplied user input --- v2/internal/runner/runner.go | 40 +++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 7dcf58ef..712eb826 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -26,9 +26,8 @@ import ( // Runner is a client for running the enumeration process. type Runner struct { - input *os.File + input string inputCount int64 - inputMutex *sync.Mutex // output is the output file to write if any output *os.File @@ -47,7 +46,6 @@ type Runner struct { // New creates a new client for running enumeration process. func New(options *Options) (*Runner, error) { runner := &Runner{ - inputMutex: &sync.Mutex{}, outputMutex: &sync.Mutex{}, options: options, } @@ -84,22 +82,37 @@ func New(options *Options) (*Runner, error) { // Setup input, handle a list of hosts as argument var err error + var input *os.File if options.Targets != "" { - runner.input, err = os.Open(options.Targets) + input, err = os.Open(options.Targets) } else if options.Stdin || options.Target != "" { - runner.input, err = os.Open(runner.tempFile) + input, err = os.Open(runner.tempFile) } if err != nil { gologger.Fatalf("Could not open targets file '%s': %s\n", options.Targets, err) } - // Precompute total number of targets - scanner := bufio.NewScanner(runner.input) + // Deduplicate input and pre-compute total number of targets + var usedInput = make(map[string]bool) + dupeCount := 0 + sb := strings.Builder{} + scanner := bufio.NewScanner(input) runner.inputCount = 0 for scanner.Scan() { - runner.inputCount++ + url := scanner.Text() + if _, ok := usedInput[url]; !ok { + usedInput[url] = true + runner.inputCount++ + sb.WriteString(url) + sb.WriteString("\n") + } else { + dupeCount++ + } + } + runner.input = sb.String() + if dupeCount > 0 { + gologger.Labelf("Supplied input was automatically deduplicated (%d removed).", dupeCount) } - runner.input.Seek(0, 0) // Create the output file if asked if options.Output != "" { @@ -121,7 +134,6 @@ func New(options *Options) (*Runner, error) { // Close releases all the resources and cleans up func (r *Runner) Close() { r.output.Close() - r.input.Close() os.Remove(r.tempFile) } @@ -361,9 +373,7 @@ func (r *Runner) processTemplateWithList(p *progress.Progress, template *templat var wg sync.WaitGroup - r.inputMutex.Lock() - r.input.Seek(0, 0) - scanner := bufio.NewScanner(r.input) + scanner := bufio.NewScanner(strings.NewReader(r.input)) for scanner.Scan() { text := scanner.Text() if text == "" { @@ -393,7 +403,6 @@ func (r *Runner) processTemplateWithList(p *progress.Progress, template *templat <-r.limiter }(text) } - r.inputMutex.Unlock() wg.Wait() @@ -403,8 +412,7 @@ func (r *Runner) processTemplateWithList(p *progress.Progress, template *templat // ProcessWorkflowWithList coming from stdin or list of targets func (r *Runner) ProcessWorkflowWithList(workflow *workflows.Workflow) { - r.input.Seek(0, 0) - scanner := bufio.NewScanner(r.input) + scanner := bufio.NewScanner(strings.NewReader(r.input)) for scanner.Scan() { text := scanner.Text() if text == "" { From a9560336f401a61c9b65a392dadde41fc4f1a90d Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Sat, 25 Jul 2020 23:13:58 +0200 Subject: [PATCH 38/51] Refactoring and unused code removal --- v2/internal/progress/bar.go | 23 ----------------------- v2/internal/progress/progress.go | 30 +++++++++--------------------- 2 files changed, 9 insertions(+), 44 deletions(-) delete mode 100644 v2/internal/progress/bar.go diff --git a/v2/internal/progress/bar.go b/v2/internal/progress/bar.go deleted file mode 100644 index 24663b07..00000000 --- a/v2/internal/progress/bar.go +++ /dev/null @@ -1,23 +0,0 @@ -package progress - -import ( - "github.com/vbauerster/mpb/v5" -) - -// Represents a single progress bar -type Bar struct { - bar *mpb.Bar - total int64 - initialTotal int64 -} - -// Drops the specified number of requests from the progress bar total. -// This may be the case when uncompleted requests are encountered and shouldn't be part of the total count. -func (b *Bar) drop(count int64) { - b.bar.IncrInt64(count) -} - -// Update progress tracking information and increments the request counter by one unit. -func (b *Bar) increment() { - b.bar.Increment() -} diff --git a/v2/internal/progress/progress.go b/v2/internal/progress/progress.go index ab5010b2..3bfd861e 100644 --- a/v2/internal/progress/progress.go +++ b/v2/internal/progress/progress.go @@ -13,8 +13,8 @@ import ( // Encapsulates progress tracking. type Progress struct { progress *mpb.Progress - bars map[string]*Bar - gbar *Bar + bars map[string]*mpb.Bar + gbar *mpb.Bar captureData *captureData stdCaptureMutex *sync.Mutex stdout *strings.Builder @@ -33,7 +33,7 @@ func NewProgress(noColor bool) *Progress { stdout: &strings.Builder{}, stderr: &strings.Builder{}, colorizer: aurora.NewAurora(!noColor), - bars: make(map[string]*Bar), + bars: make(map[string]*mpb.Bar), } return p } @@ -53,13 +53,7 @@ func (p *Progress) SetupTemplateProgressbar(templateId string, requestCount int6 } uiBarName = fmt.Sprintf(fmt.Sprintf("%%-%ds", MaxLen), "[" + color.BrightYellow(uiBarName).String() + "]") - bar := p.setupProgressbar(uiBarName, requestCount, priority) - - p.bars[templateId] = &Bar{ - bar: bar, - total: requestCount, - initialTotal: requestCount, - } + p.bars[templateId] = p.setupProgressbar(uiBarName, requestCount, priority) } // Creates and returns a progress bar that tracks all the requests progress. @@ -78,13 +72,7 @@ func (p *Progress) SetupGlobalProgressbar(hostCount int64, templateCount int, re color.Bold(color.Cyan(hostCount)), pluralize(hostCount, "host", "hosts")) - bar := p.setupProgressbar("[" + barName + "]", requestCount, 0) - - p.gbar = &Bar{ - bar: bar, - total: requestCount, - initialTotal: requestCount, - } + p.gbar = p.setupProgressbar("[" + barName + "]", requestCount, 0) } func pluralize(count int64, singular, plural string) string { @@ -97,9 +85,9 @@ func pluralize(count int64, singular, plural string) string { // Update progress tracking information and increments the request counter by one unit. // If a global progress bar is present it will be updated as well. func (p *Progress) Update(templateId string) { - p.bars[templateId].increment() + p.bars[templateId].Increment() if p.gbar != nil { - p.gbar.increment() + p.gbar.Increment() } } @@ -107,9 +95,9 @@ func (p *Progress) Update(templateId string) { // This may be the case when uncompleted requests are encountered and shouldn't be part of the total count. // If a global progress bar is present it will be updated as well. func (p *Progress) Drop(templateId string, count int64) { - p.bars[templateId].drop(count) + p.bars[templateId].IncrInt64(count) if p.gbar != nil { - p.gbar.drop(count) + p.gbar.IncrInt64(count) } } From b33a2b6d1682092cc0c2298d17c028dd6de89036 Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Sat, 25 Jul 2020 23:22:09 +0200 Subject: [PATCH 39/51] Update outdated comment --- v2/internal/runner/runner.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 712eb826..fd16d314 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -244,7 +244,7 @@ func (r *Runner) RunEnumeration() { p := r.progress templateCount := len(allTemplates) - // precompute total request count if we are executing more than one template + // precompute total request count var totalRequests int64 = 0 barIndex := 0 parsedTemplates := []string{} @@ -256,6 +256,7 @@ func (r *Runner) RunEnumeration() { barIndex++ template := t.(*templates.Template) totalRequests += template.GetHTTPRequestsCount() + // track per-template progress p.SetupTemplateProgressbar(template.ID, r.inputCount*template.GetHTTPRequestsCount(), barIndex) parsedTemplates = append(parsedTemplates, match) default: From 32e20b13e6c0fc5cc19bcac7719977dfb5c61fed Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Sun, 26 Jul 2020 15:10:03 +0200 Subject: [PATCH 40/51] Use single progressbar, tweak styling --- v2/internal/progress/progress.go | 65 +++++--------------------------- v2/internal/runner/runner.go | 4 +- v2/pkg/executer/executer_http.go | 8 ++-- 3 files changed, 14 insertions(+), 63 deletions(-) diff --git a/v2/internal/progress/progress.go b/v2/internal/progress/progress.go index 3bfd861e..bfde7d9d 100644 --- a/v2/internal/progress/progress.go +++ b/v2/internal/progress/progress.go @@ -13,7 +13,6 @@ import ( // Encapsulates progress tracking. type Progress struct { progress *mpb.Progress - bars map[string]*mpb.Bar gbar *mpb.Bar captureData *captureData stdCaptureMutex *sync.Mutex @@ -33,32 +32,13 @@ func NewProgress(noColor bool) *Progress { stdout: &strings.Builder{}, stderr: &strings.Builder{}, colorizer: aurora.NewAurora(!noColor), - bars: make(map[string]*mpb.Bar), } return p } -// Creates and returns a progress bar that tracks request progress for a specific template. -func (p *Progress) SetupTemplateProgressbar(templateId string, requestCount int64, priority int) { - if p.bars[templateId] != nil { - panic(fmt.Sprintf("A progressbar is already bound to [%s].", templateId)) - } - - color := p.colorizer - uiBarName := templateId - - const MaxLen = 40 - if len(uiBarName) > MaxLen { - uiBarName = uiBarName[:MaxLen] + ".." - } - - uiBarName = fmt.Sprintf(fmt.Sprintf("%%-%ds", MaxLen), "[" + color.BrightYellow(uiBarName).String() + "]") - p.bars[templateId] = p.setupProgressbar(uiBarName, requestCount, priority) -} - // Creates and returns a progress bar that tracks all the requests progress. // This is only useful when multiple templates are processed within the same run. -func (p *Progress) SetupGlobalProgressbar(hostCount int64, templateCount int, requestCount int64) { +func (p *Progress) InitProgressbar(hostCount int64, templateCount int, requestCount int64) { if p.gbar != nil { panic("A global progressbar is already present.") } @@ -72,7 +52,7 @@ func (p *Progress) SetupGlobalProgressbar(hostCount int64, templateCount int, re color.Bold(color.Cyan(hostCount)), pluralize(hostCount, "host", "hosts")) - p.gbar = p.setupProgressbar("[" + barName + "]", requestCount, 0) + p.gbar = p.setupProgressbar("["+barName+"]", requestCount, 0) } func pluralize(count int64, singular, plural string) string { @@ -83,22 +63,15 @@ func pluralize(count int64, singular, plural string) string { } // Update progress tracking information and increments the request counter by one unit. -// If a global progress bar is present it will be updated as well. -func (p *Progress) Update(templateId string) { - p.bars[templateId].Increment() - if p.gbar != nil { - p.gbar.Increment() - } +func (p *Progress) Update() { + p.gbar.Increment() } // Drops the specified number of requests from the progress bar total. // This may be the case when uncompleted requests are encountered and shouldn't be part of the total count. -// If a global progress bar is present it will be updated as well. -func (p *Progress) Drop(templateId string, count int64) { - p.bars[templateId].IncrInt64(count) - if p.gbar != nil { - p.gbar.IncrInt64(count) - } +func (p *Progress) Drop(count int64) { + // mimic dropping by incrementing the completed requests + p.gbar.IncrInt64(count) } // Ensures that a progress bar's total count is up-to-date if during an enumeration there were uncompleted requests and @@ -120,36 +93,16 @@ func (p *Progress) setupProgressbar(name string, total int64, priority int) *mpb mpb.PrependDecorators( decor.Name(name, decor.WCSyncSpaceR), decor.CountersNoUnit(color.BrightBlue(" %d/%d").String(), decor.WCSyncSpace), - customFormattedPercentage(color.Bold("%3.0f%%").String(), decor.WCSyncSpace), + decor.NewPercentage(color.Bold("%d").String(), decor.WCSyncSpace), ), mpb.AppendDecorators( - decor.AverageSpeed(0, color.BrightBlue("%.2f r/s ").String(), decor.WCSyncSpace), + decor.AverageSpeed(0, color.BrightYellow("%.2f").Bold().String() + color.BrightYellow("r/s").String(), decor.WCSyncSpace), decor.Elapsed(decor.ET_STYLE_GO, decor.WCSyncSpace), decor.AverageETA(decor.ET_STYLE_GO, decor.WCSyncSpace), ), ) } -// Helper function to calculate percentage -func computePercentage(total, current int64, width int) float64 { - if total <= 0 { - return 0 - } - if current >= total { - return float64(width) - } - return float64(int64(width)*current) / float64(total) -} - -// Percentage decorator with custom formatting -func customFormattedPercentage(format string, wcc ...decor.WC) decor.Decorator { - f := func(s decor.Statistics) string { - p := computePercentage(s.Total, s.Current, 100) - return fmt.Sprintf(format, p) - } - return decor.Any(f, wcc...) -} - // Starts capturing stdout and stderr instead of producing visual output that may interfere with the progress bars. func (p *Progress) StartStdCapture() { p.stdCaptureMutex.Lock() diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index fd16d314..7de35024 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -256,8 +256,6 @@ func (r *Runner) RunEnumeration() { barIndex++ template := t.(*templates.Template) totalRequests += template.GetHTTPRequestsCount() - // track per-template progress - p.SetupTemplateProgressbar(template.ID, r.inputCount*template.GetHTTPRequestsCount(), barIndex) parsedTemplates = append(parsedTemplates, match) default: gologger.Errorf("Could not parse file '%s': %s\n", match, err) @@ -268,7 +266,7 @@ func (r *Runner) RunEnumeration() { allTemplates = parsedTemplates // track global progress - p.SetupGlobalProgressbar(r.inputCount, templateCount, r.inputCount*totalRequests) + p.InitProgressbar(r.inputCount, templateCount, r.inputCount*totalRequests) var ( wgtemplates sync.WaitGroup diff --git a/v2/pkg/executer/executer_http.go b/v2/pkg/executer/executer_http.go index 03167276..792dabf4 100644 --- a/v2/pkg/executer/executer_http.go +++ b/v2/pkg/executer/executer_http.go @@ -110,7 +110,7 @@ func (e *HTTPExecuter) ExecuteHTTP(p *progress.Progress, URL string) (result Res if e.bulkHttpRequest.HasGenerator(URL) { return } - + remaining := e.template.GetHTTPRequestsCount() e.bulkHttpRequest.CreateGenerator(URL) @@ -118,19 +118,19 @@ func (e *HTTPExecuter) ExecuteHTTP(p *progress.Progress, URL string) (result Res httpRequest, err := e.bulkHttpRequest.MakeHTTPRequest(URL, dynamicvalues, e.bulkHttpRequest.Current(URL)) if err != nil { result.Error = errors.Wrap(err, "could not build http request") - p.Drop(e.template.ID, remaining) + p.Drop(remaining) return } err = e.handleHTTP(p, URL, httpRequest, dynamicvalues, &result) if err != nil { result.Error = errors.Wrap(err, "could not handle http request") - p.Drop(e.template.ID, remaining) + p.Drop(remaining) return } e.bulkHttpRequest.Increment(URL) - p.Update(e.template.ID) + p.Update() remaining-- } From 3cc79c2c490ca67df6d52d6894a1936dee3bedfd Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Sun, 26 Jul 2020 15:35:26 +0200 Subject: [PATCH 41/51] Properly close file after use --- v2/internal/runner/runner.go | 1 + 1 file changed, 1 insertion(+) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 7de35024..4f2cd443 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -109,6 +109,7 @@ func New(options *Options) (*Runner, error) { dupeCount++ } } + input.Close() runner.input = sb.String() if dupeCount > 0 { gologger.Labelf("Supplied input was automatically deduplicated (%d removed).", dupeCount) From 536e0dbf5cf507f0507dd5a5cefee21dfb0d0260 Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Sun, 26 Jul 2020 15:42:20 +0200 Subject: [PATCH 42/51] Remove old go mod files --- go.mod | 3 --- go.sum | 0 2 files changed, 3 deletions(-) delete mode 100644 go.mod delete mode 100644 go.sum diff --git a/go.mod b/go.mod deleted file mode 100644 index 14e48f74..00000000 --- a/go.mod +++ /dev/null @@ -1,3 +0,0 @@ -module github.com/projectdiscovery/nuclei - -go 1.14 diff --git a/go.sum b/go.sum deleted file mode 100644 index e69de29b..00000000 From 4d8131c8d8afb0c11003926b51e9ef9e181ca65b Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Sun, 26 Jul 2020 16:36:01 +0200 Subject: [PATCH 43/51] Add support for DNS requests --- v2/internal/runner/runner.go | 5 +++-- v2/pkg/executer/executer_dns.go | 15 ++++++++++++++- v2/pkg/executer/executer_http.go | 2 +- v2/pkg/requests/dns-request.go | 5 +++++ v2/pkg/templates/templates.go | 10 +++++++++- v2/pkg/workflows/var.go | 2 +- 6 files changed, 33 insertions(+), 6 deletions(-) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 09b485da..5533d442 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -290,7 +290,8 @@ func (r *Runner) RunEnumeration() { case *templates.Template: barIndex++ template := t.(*templates.Template) - totalRequests += template.GetHTTPRequestsCount() + totalRequests += template.GetHTTPRequestCount() + totalRequests += template.GetDNSRequestCount() parsedTemplates = append(parsedTemplates, match) default: gologger.Errorf("Could not parse file '%s': %s\n", match, err) @@ -426,7 +427,7 @@ func (r *Runner) processTemplateWithList(p *progress.Progress, template *templat globalresult.Or(result.GotResults) } if dnsExecuter != nil { - result = dnsExecuter.ExecuteDNS(URL) + result = dnsExecuter.ExecuteDNS(p, URL) globalresult.Or(result.GotResults) } if result.Error != nil { diff --git a/v2/pkg/executer/executer_dns.go b/v2/pkg/executer/executer_dns.go index 4c9a3474..1542df9d 100644 --- a/v2/pkg/executer/executer_dns.go +++ b/v2/pkg/executer/executer_dns.go @@ -3,6 +3,7 @@ package executer import ( "bufio" "fmt" + "github.com/projectdiscovery/nuclei/v2/internal/progress" "os" "sync" @@ -62,7 +63,7 @@ func NewDNSExecuter(options *DNSOptions) *DNSExecuter { } // ExecuteDNS executes the DNS request on a URL -func (e *DNSExecuter) ExecuteDNS(URL string) (result Result) { +func (e *DNSExecuter) ExecuteDNS(p *progress.Progress, URL string) (result Result) { // Parse the URL and return domain if URL. var domain string if isURL(URL) { @@ -75,26 +76,34 @@ func (e *DNSExecuter) ExecuteDNS(URL string) (result Result) { compiledRequest, err := e.dnsRequest.MakeDNSRequest(domain) if err != nil { result.Error = errors.Wrap(err, "could not make dns request") + p.Drop(1) return } if e.debug { + p.StartStdCapture() gologger.Infof("Dumped DNS request for %s (%s)\n\n", URL, e.template.ID) fmt.Fprintf(os.Stderr, "%s\n", compiledRequest.String()) + p.StopStdCapture() } // Send the request to the target servers resp, err := e.dnsClient.Do(compiledRequest) if err != nil { result.Error = errors.Wrap(err, "could not send dns request") + p.Drop(1) return } + p.Update() + gologger.Verbosef("Sent DNS request to %s\n", "dns-request", URL) if e.debug { + p.StartStdCapture() gologger.Infof("Dumped DNS response for %s (%s)\n\n", URL, e.template.ID) fmt.Fprintf(os.Stderr, "%s\n", resp.String()) + p.StopStdCapture() } matcherCondition := e.dnsRequest.GetMatchersCondition() @@ -109,7 +118,9 @@ func (e *DNSExecuter) ExecuteDNS(URL string) (result Result) { // If the matcher has matched, and its an OR // write the first output then move to next matcher. if matcherCondition == matchers.ORCondition && len(e.dnsRequest.Extractors) == 0 { + p.StartStdCapture() e.writeOutputDNS(domain, matcher, nil) + p.StopStdCapture() result.GotResults = true } } @@ -129,7 +140,9 @@ func (e *DNSExecuter) ExecuteDNS(URL string) (result Result) { // Write a final string of output if matcher type is // AND or if we have extractors for the mechanism too. if len(e.dnsRequest.Extractors) > 0 || matcherCondition == matchers.ANDCondition { + p.StartStdCapture() e.writeOutputDNS(domain, nil, extractorResults) + p.StopStdCapture() } return diff --git a/v2/pkg/executer/executer_http.go b/v2/pkg/executer/executer_http.go index 792dabf4..c39d8a8b 100644 --- a/v2/pkg/executer/executer_http.go +++ b/v2/pkg/executer/executer_http.go @@ -111,7 +111,7 @@ func (e *HTTPExecuter) ExecuteHTTP(p *progress.Progress, URL string) (result Res return } - remaining := e.template.GetHTTPRequestsCount() + remaining := e.template.GetHTTPRequestCount() e.bulkHttpRequest.CreateGenerator(URL) for e.bulkHttpRequest.Next(URL) && !result.Done { diff --git a/v2/pkg/requests/dns-request.go b/v2/pkg/requests/dns-request.go index ba695c2d..1cb1b3fa 100644 --- a/v2/pkg/requests/dns-request.go +++ b/v2/pkg/requests/dns-request.go @@ -42,6 +42,11 @@ func (r *DNSRequest) SetMatchersCondition(condition matchers.ConditionType) { r.matchersCondition = condition } +// Returns the total number of requests the YAML rule will perform +func (r *DNSRequest) GetRequestCount() int64 { + return 1 +} + // MakeDNSRequest creates a *dns.Request from a request template func (r *DNSRequest) MakeDNSRequest(domain string) (*dns.Msg, error) { domain = dns.Fqdn(domain) diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index 73efd990..6842f359 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -28,10 +28,18 @@ type Info struct { Description string `yaml:"description,omitempty"` } -func (t* Template) GetHTTPRequestsCount() int64 { +func (t* Template) GetHTTPRequestCount() int64 { var count int64 = 0 for _, request := range t.BulkRequestsHTTP { count += request.GetRequestCount() } return count +} + +func (t *Template) GetDNSRequestCount() int64 { + var count int64 = 0 + for _, request := range t.RequestsDNS { + count += request.GetRequestCount() + } + return count } \ No newline at end of file diff --git a/v2/pkg/workflows/var.go b/v2/pkg/workflows/var.go index 7a28b642..1638be1c 100644 --- a/v2/pkg/workflows/var.go +++ b/v2/pkg/workflows/var.go @@ -86,7 +86,7 @@ func (n *NucleiVar) Call(args ...tengo.Object) (ret tengo.Object, err error) { for _, request := range template.DNSOptions.Template.RequestsDNS { template.DNSOptions.DNSRequest = request dnsExecuter := executer.NewDNSExecuter(template.DNSOptions) - result := dnsExecuter.ExecuteDNS(n.URL) + result := dnsExecuter.ExecuteDNS(p,n.URL) if result.Error != nil { gologger.Warningf("Could not compile request for template '%s': %s\n", template.HTTPOptions.Template.ID, result.Error) continue From baa1715c449d757398201ead027a34f494d97e14 Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Sun, 26 Jul 2020 16:43:53 +0200 Subject: [PATCH 44/51] Better input sanitization, skip empty lines --- v2/internal/runner/runner.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 5533d442..b18fc7f4 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -93,7 +93,7 @@ func New(options *Options) (*Runner, error) { gologger.Fatalf("Could not open targets file '%s': %s\n", options.Targets, err) } - // Deduplicate input and pre-compute total number of targets + // Sanitize input and pre-compute total number of targets var usedInput = make(map[string]bool) dupeCount := 0 sb := strings.Builder{} @@ -101,6 +101,11 @@ func New(options *Options) (*Runner, error) { runner.inputCount = 0 for scanner.Scan() { url := scanner.Text() + // skip empty lines + if len(url) == 0 { + continue + } + // deduplication if _, ok := usedInput[url]; !ok { usedInput[url] = true runner.inputCount++ @@ -450,9 +455,6 @@ func (r *Runner) ProcessWorkflowWithList(workflow *workflows.Workflow) { scanner := bufio.NewScanner(strings.NewReader(r.input)) for scanner.Scan() { text := scanner.Text() - if text == "" { - continue - } if err := r.ProcessWorkflow(workflow, text); err != nil { gologger.Warningf("Could not run workflow for %s: %s\n", text, err) } From 660c8420c1112fd32442cce060437a0040ffe127 Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Sun, 26 Jul 2020 16:57:19 +0200 Subject: [PATCH 45/51] Remove unused code --- v2/internal/runner/runner.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index b18fc7f4..28c23e57 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -286,14 +286,12 @@ func (r *Runner) RunEnumeration() { // precompute total request count var totalRequests int64 = 0 - barIndex := 0 parsedTemplates := []string{} for _, match := range allTemplates { t, err := r.parse(match) switch t.(type) { case *templates.Template: - barIndex++ template := t.(*templates.Template) totalRequests += template.GetHTTPRequestCount() totalRequests += template.GetDNSRequestCount() From 06cffee6aa5a409206c906cd04da5f05dd795d34 Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Mon, 27 Jul 2020 00:00:06 +0200 Subject: [PATCH 46/51] Add support for workflows --- v2/internal/progress/progress.go | 24 ++++++++++++++++++++++-- v2/internal/runner/runner.go | 27 +++++++++++++++------------ v2/pkg/executer/executer_http.go | 2 +- v2/pkg/workflows/var.go | 16 ++++++++++++---- 4 files changed, 50 insertions(+), 19 deletions(-) diff --git a/v2/internal/progress/progress.go b/v2/internal/progress/progress.go index bfde7d9d..1b28c1a3 100644 --- a/v2/internal/progress/progress.go +++ b/v2/internal/progress/progress.go @@ -14,6 +14,9 @@ import ( type Progress struct { progress *mpb.Progress gbar *mpb.Bar + total int64 + initialTotal int64 + totalMutex *sync.Mutex captureData *captureData stdCaptureMutex *sync.Mutex stdout *strings.Builder @@ -28,6 +31,7 @@ func NewProgress(noColor bool) *Progress { mpb.WithOutput(os.Stderr), mpb.PopCompletedMode(), ), + totalMutex: &sync.Mutex{}, stdCaptureMutex: &sync.Mutex{}, stdout: &strings.Builder{}, stderr: &strings.Builder{}, @@ -62,6 +66,14 @@ func pluralize(count int64, singular, plural string) string { return singular } +// Update total progress request count +func (p *Progress) AddToTotal(delta int64) { + p.totalMutex.Lock() + p.total += delta + p.gbar.SetTotal(p.total, false) + p.totalMutex.Unlock() +} + // Update progress tracking information and increments the request counter by one unit. func (p *Progress) Update() { p.gbar.Increment() @@ -72,12 +84,17 @@ func (p *Progress) Update() { func (p *Progress) Drop(count int64) { // mimic dropping by incrementing the completed requests p.gbar.IncrInt64(count) + } // Ensures that a progress bar's total count is up-to-date if during an enumeration there were uncompleted requests and // wait for all the progress bars to finish. -// If a global progress bar is present it will be updated as well. func (p *Progress) Wait() { + p.totalMutex.Lock() + if p.initialTotal != p.total { + p.gbar.SetTotal(p.total, true) + } + p.totalMutex.Unlock() p.progress.Wait() } @@ -85,6 +102,9 @@ func (p *Progress) Wait() { func (p *Progress) setupProgressbar(name string, total int64, priority int) *mpb.Bar { color := p.colorizer + p.total = total + p.initialTotal = total + return p.progress.AddBar( total, mpb.BarPriority(priority), @@ -96,7 +116,7 @@ func (p *Progress) setupProgressbar(name string, total int64, priority int) *mpb decor.NewPercentage(color.Bold("%d").String(), decor.WCSyncSpace), ), mpb.AppendDecorators( - decor.AverageSpeed(0, color.BrightYellow("%.2f").Bold().String() + color.BrightYellow("r/s").String(), decor.WCSyncSpace), + decor.AverageSpeed(0, color.BrightYellow("%.2f").Bold().String()+color.BrightYellow("r/s").String(), decor.WCSyncSpace), decor.Elapsed(decor.ET_STYLE_GO, decor.WCSyncSpace), decor.AverageETA(decor.ET_STYLE_GO, decor.WCSyncSpace), ), diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 28c23e57..decb6292 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -282,10 +282,10 @@ func (r *Runner) RunEnumeration() { // progress tracking p := r.progress - templateCount := len(allTemplates) // precompute total request count var totalRequests int64 = 0 + templateCount := len(allTemplates) parsedTemplates := []string{} for _, match := range allTemplates { @@ -293,8 +293,11 @@ func (r *Runner) RunEnumeration() { switch t.(type) { case *templates.Template: template := t.(*templates.Template) - totalRequests += template.GetHTTPRequestCount() - totalRequests += template.GetDNSRequestCount() + totalRequests += template.GetHTTPRequestCount() + template.GetDNSRequestCount() + parsedTemplates = append(parsedTemplates, match) + case *workflows.Workflow: + // workflows will dynamically adjust the totals while running, as + // it can't be know in advance which requests will be called parsedTemplates = append(parsedTemplates, match) default: gologger.Errorf("Could not parse file '%s': %s\n", match, err) @@ -328,7 +331,7 @@ func (r *Runner) RunEnumeration() { } case *workflows.Workflow: workflow := t.(*workflows.Workflow) - r.ProcessWorkflowWithList(workflow) + r.ProcessWorkflowWithList(p, workflow) default: p.StartStdCapture() gologger.Errorf("Could not parse file '%s': %s\n", match, err) @@ -401,6 +404,7 @@ func (r *Runner) processTemplateWithList(p *progress.Progress, template *templat }) } if err != nil { + p.Drop(request.(*requests.BulkHTTPRequest).GetRequestCount()) p.StartStdCapture() gologger.Warningf("Could not create http client: %s\n", err) p.StopStdCapture() @@ -414,9 +418,6 @@ func (r *Runner) processTemplateWithList(p *progress.Progress, template *templat scanner := bufio.NewScanner(strings.NewReader(r.input)) for scanner.Scan() { text := scanner.Text() - if text == "" { - continue - } r.limiter <- struct{}{} wg.Add(1) @@ -449,18 +450,18 @@ func (r *Runner) processTemplateWithList(p *progress.Progress, template *templat } // ProcessWorkflowWithList coming from stdin or list of targets -func (r *Runner) ProcessWorkflowWithList(workflow *workflows.Workflow) { +func (r *Runner) ProcessWorkflowWithList(p *progress.Progress, workflow *workflows.Workflow) { scanner := bufio.NewScanner(strings.NewReader(r.input)) for scanner.Scan() { text := scanner.Text() - if err := r.ProcessWorkflow(workflow, text); err != nil { + if err := r.ProcessWorkflow(p, workflow, text); err != nil { gologger.Warningf("Could not run workflow for %s: %s\n", text, err) } } } // ProcessWorkflow towards an URL -func (r *Runner) ProcessWorkflow(workflow *workflows.Workflow, URL string) error { +func (r *Runner) ProcessWorkflow(p *progress.Progress, workflow *workflows.Workflow, URL string) error { script := tengo.NewScript([]byte(workflow.Logic)) script.SetImports(stdlib.GetModuleMap(stdlib.AllModuleNames()...)) var jar *cookiejar.Jar @@ -481,7 +482,9 @@ func (r *Runner) ProcessWorkflow(workflow *workflows.Workflow, URL string) error // Check if the template is an absolute path or relative path. // If the path is absolute, use it. Otherwise, if r.isRelative(value) { + p.StartStdCapture() newPath, err := r.resolvePath(value) + p.StopStdCapture() if err != nil { return err } @@ -495,7 +498,7 @@ func (r *Runner) ProcessWorkflow(workflow *workflows.Workflow, URL string) error if err != nil { return err } - template := &workflows.Template{} + template := &workflows.Template{Progress: p} if len(t.BulkRequestsHTTP) > 0 { template.HTTPOptions = &executer.HTTPOptions{ Debug: r.options.Debug, @@ -546,7 +549,7 @@ func (r *Runner) ProcessWorkflow(workflow *workflows.Workflow, URL string) error if err != nil { return err } - template := &workflows.Template{} + template := &workflows.Template{Progress: p} if len(t.BulkRequestsHTTP) > 0 { template.HTTPOptions = &executer.HTTPOptions{ Debug: r.options.Debug, diff --git a/v2/pkg/executer/executer_http.go b/v2/pkg/executer/executer_http.go index c39d8a8b..896b5a40 100644 --- a/v2/pkg/executer/executer_http.go +++ b/v2/pkg/executer/executer_http.go @@ -111,7 +111,7 @@ func (e *HTTPExecuter) ExecuteHTTP(p *progress.Progress, URL string) (result Res return } - remaining := e.template.GetHTTPRequestCount() + remaining := e.bulkHttpRequest.GetRequestCount() e.bulkHttpRequest.CreateGenerator(URL) for e.bulkHttpRequest.Next(URL) && !result.Done { diff --git a/v2/pkg/workflows/var.go b/v2/pkg/workflows/var.go index 1638be1c..a4dd3677 100644 --- a/v2/pkg/workflows/var.go +++ b/v2/pkg/workflows/var.go @@ -24,6 +24,7 @@ type NucleiVar struct { type Template struct { HTTPOptions *executer.HTTPOptions DNSOptions *executer.DNSOptions + Progress *progress.Progress } // TypeName of the variable @@ -52,12 +53,11 @@ func (n *NucleiVar) Call(args ...tengo.Object) (ret tengo.Object, err error) { externalVars = iterableToMap(args[1]) } - // track progress - p := progress.NewProgress(false) - var gotResult atomicboolean.AtomBool for _, template := range n.Templates { + p := template.Progress if template.HTTPOptions != nil { + p.AddToTotal(template.HTTPOptions.Template.GetHTTPRequestCount()) for _, request := range template.HTTPOptions.Template.BulkRequestsHTTP { // apply externally supplied payloads if any request.Headers = generators.MergeMapsWithStrings(request.Headers, headers) @@ -66,12 +66,17 @@ func (n *NucleiVar) Call(args ...tengo.Object) (ret tengo.Object, err error) { template.HTTPOptions.BulkHttpRequest = request httpExecuter, err := executer.NewHTTPExecuter(template.HTTPOptions) if err != nil { + p.Drop(request.GetRequestCount()) + p.StartStdCapture() gologger.Warningf("Could not compile request for template '%s': %s\n", template.HTTPOptions.Template.ID, err) + p.StopStdCapture() continue } result := httpExecuter.ExecuteHTTP(p, n.URL) if result.Error != nil { + p.StartStdCapture() gologger.Warningf("Could not send request for template '%s': %s\n", template.HTTPOptions.Template.ID, result.Error) + p.StopStdCapture() continue } @@ -83,12 +88,15 @@ func (n *NucleiVar) Call(args ...tengo.Object) (ret tengo.Object, err error) { } if template.DNSOptions != nil { + p.AddToTotal(template.DNSOptions.Template.GetDNSRequestCount()) for _, request := range template.DNSOptions.Template.RequestsDNS { template.DNSOptions.DNSRequest = request dnsExecuter := executer.NewDNSExecuter(template.DNSOptions) - result := dnsExecuter.ExecuteDNS(p,n.URL) + result := dnsExecuter.ExecuteDNS(p, n.URL) if result.Error != nil { + p.StartStdCapture() gologger.Warningf("Could not compile request for template '%s': %s\n", template.HTTPOptions.Template.ID, result.Error) + p.StopStdCapture() continue } From 8718d52546d56845e98871d0b5ddfb7f808a7d2e Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Mon, 27 Jul 2020 18:47:23 +0200 Subject: [PATCH 47/51] Ensure input URLs and requests are present before tracking any progress --- v2/internal/runner/runner.go | 74 +++++++++++++++++++----------------- 1 file changed, 39 insertions(+), 35 deletions(-) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 8dc5983d..5d04d764 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -293,7 +293,7 @@ func (r *Runner) RunEnumeration() { switch t.(type) { case *templates.Template: template := t.(*templates.Template) - totalRequests += template.GetHTTPRequestCount() + template.GetDNSRequestCount() + totalRequests += (template.GetHTTPRequestCount() + template.GetDNSRequestCount()) * r.inputCount parsedTemplates = append(parsedTemplates, match) case *workflows.Workflow: // workflows will dynamically adjust the totals while running, as @@ -304,46 +304,50 @@ func (r *Runner) RunEnumeration() { } } - // ensure only successfully parsed templates are processed - allTemplates = parsedTemplates - - // track global progress - p.InitProgressbar(r.inputCount, templateCount, r.inputCount*totalRequests) - var ( wgtemplates sync.WaitGroup results atomicboolean.AtomBool ) - for _, match := range allTemplates { - wgtemplates.Add(1) - go func(match string) { - defer wgtemplates.Done() - t, err := r.parse(match) - switch t.(type) { - case *templates.Template: - template := t.(*templates.Template) - for _, request := range template.RequestsDNS { - results.Or(r.processTemplateWithList(p, template, request)) - } - for _, request := range template.BulkRequestsHTTP { - results.Or(r.processTemplateWithList(p, template, request)) - } - case *workflows.Workflow: - workflow := t.(*workflows.Workflow) - r.ProcessWorkflowWithList(p, workflow) - default: - p.StartStdCapture() - gologger.Errorf("Could not parse file '%s': %s\n", match, err) - p.StopStdCapture() - } - }(match) - } + if r.inputCount == 0 { + gologger.Errorf("Could not find any valid input URLs.") + } else if totalRequests > 0 { + // ensure only successfully parsed templates are processed + allTemplates = parsedTemplates - wgtemplates.Wait() - p.Wait() - p.ShowStdErr() - p.ShowStdOut() + // track global progress + p.InitProgressbar(r.inputCount, templateCount, totalRequests) + + for _, match := range allTemplates { + wgtemplates.Add(1) + go func(match string) { + defer wgtemplates.Done() + t, err := r.parse(match) + switch t.(type) { + case *templates.Template: + template := t.(*templates.Template) + for _, request := range template.RequestsDNS { + results.Or(r.processTemplateWithList(p, template, request)) + } + for _, request := range template.BulkRequestsHTTP { + results.Or(r.processTemplateWithList(p, template, request)) + } + case *workflows.Workflow: + workflow := t.(*workflows.Workflow) + r.ProcessWorkflowWithList(p, workflow) + default: + p.StartStdCapture() + gologger.Errorf("Could not parse file '%s': %s\n", match, err) + p.StopStdCapture() + } + }(match) + } + + wgtemplates.Wait() + p.Wait() + p.ShowStdErr() + p.ShowStdOut() + } if !results.Get() { if r.output != nil { From 53b280cf326854b75efa7b73afdf6b2a75c00bba Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Mon, 27 Jul 2020 20:38:48 +0200 Subject: [PATCH 48/51] Ensure workflows can be run alone as well --- v2/internal/runner/runner.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 5d04d764..cb40035d 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -285,7 +285,7 @@ func (r *Runner) RunEnumeration() { // precompute total request count var totalRequests int64 = 0 - templateCount := len(allTemplates) + hasWorkflows := false parsedTemplates := []string{} for _, match := range allTemplates { @@ -299,11 +299,16 @@ func (r *Runner) RunEnumeration() { // workflows will dynamically adjust the totals while running, as // it can't be know in advance which requests will be called parsedTemplates = append(parsedTemplates, match) + hasWorkflows = true default: gologger.Errorf("Could not parse file '%s': %s\n", match, err) } } + // ensure only successfully parsed templates are processed + allTemplates = parsedTemplates + templateCount := len(allTemplates) + var ( wgtemplates sync.WaitGroup results atomicboolean.AtomBool @@ -311,9 +316,7 @@ func (r *Runner) RunEnumeration() { if r.inputCount == 0 { gologger.Errorf("Could not find any valid input URLs.") - } else if totalRequests > 0 { - // ensure only successfully parsed templates are processed - allTemplates = parsedTemplates + } else if totalRequests > 0 || hasWorkflows { // track global progress p.InitProgressbar(r.inputCount, templateCount, totalRequests) @@ -459,7 +462,9 @@ func (r *Runner) ProcessWorkflowWithList(p *progress.Progress, workflow *workflo for scanner.Scan() { text := scanner.Text() if err := r.ProcessWorkflow(p, workflow, text); err != nil { + p.StartStdCapture() gologger.Warningf("Could not run workflow for %s: %s\n", text, err) + p.StopStdCapture() } } } From 6209b25d9af6498b4f23cef6ce9b30b39b76e47e Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Mon, 27 Jul 2020 20:39:13 +0200 Subject: [PATCH 49/51] Ensure an empty progressbar is aborted --- v2/internal/progress/progress.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/v2/internal/progress/progress.go b/v2/internal/progress/progress.go index 1b28c1a3..e74590d4 100644 --- a/v2/internal/progress/progress.go +++ b/v2/internal/progress/progress.go @@ -91,7 +91,9 @@ func (p *Progress) Drop(count int64) { // wait for all the progress bars to finish. func (p *Progress) Wait() { p.totalMutex.Lock() - if p.initialTotal != p.total { + if p.total == 0 { + p.gbar.Abort(true) + } else if p.initialTotal != p.total { p.gbar.SetTotal(p.total, true) } p.totalMutex.Unlock() From 24c53628ea4453b8cf080e4d3114f9e087bb6c7d Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Mon, 27 Jul 2020 21:15:38 +0200 Subject: [PATCH 50/51] Capture additional DNS logging --- v2/pkg/executer/executer_dns.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/v2/pkg/executer/executer_dns.go b/v2/pkg/executer/executer_dns.go index 1542df9d..92686798 100644 --- a/v2/pkg/executer/executer_dns.go +++ b/v2/pkg/executer/executer_dns.go @@ -97,7 +97,9 @@ func (e *DNSExecuter) ExecuteDNS(p *progress.Progress, URL string) (result Resul p.Update() + p.StartStdCapture() gologger.Verbosef("Sent DNS request to %s\n", "dns-request", URL) + p.StopStdCapture() if e.debug { p.StartStdCapture() From 9d2539f85603f057d89d1829f791c1b499581e32 Mon Sep 17 00:00:00 2001 From: Manuel Bua Date: Mon, 27 Jul 2020 21:35:24 +0200 Subject: [PATCH 51/51] Makes stdio capturing global This dramatically lower chances of reaching max open files limit as it works with two os.Pipe only, but it may be sub-optimal in some cases such as with the `-debug` switch because there are no guarded writes anymore when using `fmt.Fprintf` directly, such as when dumping request or responses. --- v2/internal/runner/runner.go | 15 +++------------ v2/pkg/executer/executer_dns.go | 10 ---------- v2/pkg/executer/executer_http.go | 10 ---------- v2/pkg/workflows/var.go | 6 ------ 4 files changed, 3 insertions(+), 38 deletions(-) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 0b84b12a..b313e845 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -320,6 +320,7 @@ func (r *Runner) RunEnumeration() { // track global progress p.InitProgressbar(r.inputCount, templateCount, totalRequests) + p.StartStdCapture() for _, match := range allTemplates { wgtemplates.Add(1) @@ -339,15 +340,15 @@ func (r *Runner) RunEnumeration() { workflow := t.(*workflows.Workflow) r.ProcessWorkflowWithList(p, workflow) default: - p.StartStdCapture() gologger.Errorf("Could not parse file '%s': %s\n", match, err) - p.StopStdCapture() } }(match) } wgtemplates.Wait() p.Wait() + + p.StopStdCapture() p.ShowStdErr() p.ShowStdOut() } @@ -370,9 +371,7 @@ func (r *Runner) processTemplateWithList(p *progress.Progress, template *templat if template.Info.Severity != "" { message += " [" + template.Info.Severity + "]" } - p.StartStdCapture() gologger.Infof("%s\n", message) - p.StopStdCapture() var writer *bufio.Writer if r.output != nil { @@ -412,9 +411,7 @@ func (r *Runner) processTemplateWithList(p *progress.Progress, template *templat } if err != nil { p.Drop(request.(*requests.BulkHTTPRequest).GetRequestCount()) - p.StartStdCapture() gologger.Warningf("Could not create http client: %s\n", err) - p.StopStdCapture() return false } @@ -442,9 +439,7 @@ func (r *Runner) processTemplateWithList(p *progress.Progress, template *templat globalresult.Or(result.GotResults) } if result.Error != nil { - p.StartStdCapture() gologger.Warningf("Could not execute step: %s\n", result.Error) - p.StopStdCapture() } <-r.limiter }(text) @@ -469,9 +464,7 @@ func (r *Runner) ProcessWorkflowWithList(p *progress.Progress, workflow *workflo defer wg.Done() if err := r.ProcessWorkflow(p, workflow, text); err != nil { - p.StartStdCapture() gologger.Warningf("Could not run workflow for %s: %s\n", text, err) - p.StopStdCapture() } <-r.limiter }(text) @@ -502,9 +495,7 @@ func (r *Runner) ProcessWorkflow(p *progress.Progress, workflow *workflows.Workf // Check if the template is an absolute path or relative path. // If the path is absolute, use it. Otherwise, if r.isRelative(value) { - p.StartStdCapture() newPath, err := r.resolvePath(value) - p.StopStdCapture() if err != nil { newPath, err = r.resolvePathWithBaseFolder(filepath.Dir(workflow.GetPath()), value) if err != nil { diff --git a/v2/pkg/executer/executer_dns.go b/v2/pkg/executer/executer_dns.go index 92686798..9334a2c5 100644 --- a/v2/pkg/executer/executer_dns.go +++ b/v2/pkg/executer/executer_dns.go @@ -81,10 +81,8 @@ func (e *DNSExecuter) ExecuteDNS(p *progress.Progress, URL string) (result Resul } if e.debug { - p.StartStdCapture() gologger.Infof("Dumped DNS request for %s (%s)\n\n", URL, e.template.ID) fmt.Fprintf(os.Stderr, "%s\n", compiledRequest.String()) - p.StopStdCapture() } // Send the request to the target servers @@ -97,15 +95,11 @@ func (e *DNSExecuter) ExecuteDNS(p *progress.Progress, URL string) (result Resul p.Update() - p.StartStdCapture() gologger.Verbosef("Sent DNS request to %s\n", "dns-request", URL) - p.StopStdCapture() if e.debug { - p.StartStdCapture() gologger.Infof("Dumped DNS response for %s (%s)\n\n", URL, e.template.ID) fmt.Fprintf(os.Stderr, "%s\n", resp.String()) - p.StopStdCapture() } matcherCondition := e.dnsRequest.GetMatchersCondition() @@ -120,9 +114,7 @@ func (e *DNSExecuter) ExecuteDNS(p *progress.Progress, URL string) (result Resul // If the matcher has matched, and its an OR // write the first output then move to next matcher. if matcherCondition == matchers.ORCondition && len(e.dnsRequest.Extractors) == 0 { - p.StartStdCapture() e.writeOutputDNS(domain, matcher, nil) - p.StopStdCapture() result.GotResults = true } } @@ -142,9 +134,7 @@ func (e *DNSExecuter) ExecuteDNS(p *progress.Progress, URL string) (result Resul // Write a final string of output if matcher type is // AND or if we have extractors for the mechanism too. if len(e.dnsRequest.Extractors) > 0 || matcherCondition == matchers.ANDCondition { - p.StartStdCapture() e.writeOutputDNS(domain, nil, extractorResults) - p.StopStdCapture() } return diff --git a/v2/pkg/executer/executer_http.go b/v2/pkg/executer/executer_http.go index 896b5a40..20c309bb 100644 --- a/v2/pkg/executer/executer_http.go +++ b/v2/pkg/executer/executer_http.go @@ -134,9 +134,7 @@ func (e *HTTPExecuter) ExecuteHTTP(p *progress.Progress, URL string) (result Res remaining-- } - p.StartStdCapture() gologger.Verbosef("Sent HTTP request to %s\n", "http-request", URL) - p.StopStdCapture() return } @@ -150,10 +148,8 @@ func (e *HTTPExecuter) handleHTTP(p *progress.Progress, URL string, request *req if err != nil { return errors.Wrap(err, "could not make http request") } - p.StartStdCapture() gologger.Infof("Dumped HTTP request for %s (%s)\n\n", URL, e.template.ID) fmt.Fprintf(os.Stderr, "%s", string(dumpedRequest)) - p.StopStdCapture() } resp, err := e.httpClient.Do(req) if err != nil { @@ -168,10 +164,8 @@ func (e *HTTPExecuter) handleHTTP(p *progress.Progress, URL string, request *req if err != nil { return errors.Wrap(err, "could not dump http response") } - p.StartStdCapture() gologger.Infof("Dumped HTTP response for %s (%s)\n\n", URL, e.template.ID) fmt.Fprintf(os.Stderr, "%s\n", string(dumpedResponse)) - p.StopStdCapture() } data, err := ioutil.ReadAll(resp.Body) @@ -208,9 +202,7 @@ func (e *HTTPExecuter) handleHTTP(p *progress.Progress, URL string, request *req result.Matches[matcher.Name] = nil // probably redundant but ensures we snapshot current payload values when matchers are valid result.Meta = request.Meta - p.StartStdCapture() e.writeOutputHTTP(request, resp, body, matcher, nil) - p.StopStdCapture() result.GotResults = true } } @@ -237,9 +229,7 @@ func (e *HTTPExecuter) handleHTTP(p *progress.Progress, URL string, request *req // Write a final string of output if matcher type is // AND or if we have extractors for the mechanism too. if len(outputExtractorResults) > 0 || matcherCondition == matchers.ANDCondition { - p.StartStdCapture() e.writeOutputHTTP(request, resp, body, nil, outputExtractorResults) - p.StopStdCapture() result.GotResults = true } diff --git a/v2/pkg/workflows/var.go b/v2/pkg/workflows/var.go index bfab45c3..77758393 100644 --- a/v2/pkg/workflows/var.go +++ b/v2/pkg/workflows/var.go @@ -67,16 +67,12 @@ func (n *NucleiVar) Call(args ...tengo.Object) (ret tengo.Object, err error) { httpExecuter, err := executer.NewHTTPExecuter(template.HTTPOptions) if err != nil { p.Drop(request.GetRequestCount()) - p.StartStdCapture() gologger.Warningf("Could not compile request for template '%s': %s\n", template.HTTPOptions.Template.ID, err) - p.StopStdCapture() continue } result := httpExecuter.ExecuteHTTP(p, n.URL) if result.Error != nil { - p.StartStdCapture() gologger.Warningf("Could not send request for template '%s': %s\n", template.HTTPOptions.Template.ID, result.Error) - p.StopStdCapture() continue } @@ -94,9 +90,7 @@ func (n *NucleiVar) Call(args ...tengo.Object) (ret tengo.Object, err error) { dnsExecuter := executer.NewDNSExecuter(template.DNSOptions) result := dnsExecuter.ExecuteDNS(p, n.URL) if result.Error != nil { - p.StartStdCapture() gologger.Warningf("Could not compile request for template '%s': %s\n", template.HTTPOptions.Template.ID, result.Error) - p.StopStdCapture() continue }