Merge pull request #149 from manuelbua/enumeration-progressbar

Enumeration progressbar

.gitignore

@@ -1,2 +1,3 @@
 cmd/nuclei/nuclei*
 v2/cmd/nuclei/nuclei
+.idea

v2/go.mod

@@ -10,12 +10,16 @@ require (
 	github.com/google/go-github/v32 v32.1.0
 	github.com/json-iterator/go v1.1.10
 	github.com/karrick/godirwalk v1.15.6
-	github.com/labstack/gommon v0.3.0
+	github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381
 	github.com/miekg/dns v1.1.30
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/gologger v1.0.0
 	github.com/projectdiscovery/retryabledns v1.0.4
 	github.com/projectdiscovery/retryablehttp-go v1.0.1
+	github.com/stretchr/testify v1.5.1
+	github.com/vbauerster/mpb/v5 v5.2.4
+	golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 // indirect
 	golang.org/x/net v0.0.0-20200707034311-ab3426394381
+	golang.org/x/sys v0.0.0-20200724161237-0e2f3a69832c // indirect
 	gopkg.in/yaml.v2 v2.3.0
 )

v2/go.sum

@@ -1,18 +1,19 @@
-github.com/Knetic/govaluate v1.5.0 h1:L4MyqdJSld9xr2eZcZHCWLfeIX2SBjqrwIKG1pcm/+4=
 github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f6s9Tn1Tt7/WTEg=
 github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
+github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM=
+github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
+github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8=
+github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
 github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0S6Vi7/lbWECcX0j45yZReDZ56BQsrVBOEEY=
 github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg=
-github.com/blang/semver v1.1.0 h1:ol1rO7QQB5uy7umSNV7VAmLugfLRD+17sYJujRNYPhg=
 github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
-github.com/d5/tengo v1.24.8 h1:PRJ+NWt7ae/9sSbIfThOBTkPSvNV+dwYoBAvwfNgNJY=
 github.com/d5/tengo/v2 v2.6.0 h1:D0cJtpiBzaLJ/Smv6nnUc/LIfO46oKwDx85NZtIRNRI=
 github.com/d5/tengo/v2 v2.6.0/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
 github.com/google/go-github/v32 v32.1.0 h1:GWkQOdXqviCPx7Q7Fj+KyPoGm4SwHRh8rheoPhd27II=
 github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI=
 github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
@@ -22,15 +23,10 @@ github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/karrick/godirwalk v1.15.6 h1:Yf2mmR8TJy+8Fa0SuQVto5SYap6IF7lNVX4Jdl8G1qA=
 github.com/karrick/godirwalk v1.15.6/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
-github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0=
-github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
 github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 h1:bqDmpDG49ZRnB5PcgP0RXtQvnMSgIF14M7CBd2shtXs=
 github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
-github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
+github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
-github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
-github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
-github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg=
-github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
 github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
 github.com/miekg/dns v1.1.30 h1:Qww6FseFn8PRfw07jueqIXqodm0JKiiKuK0DeXSqfyo=
 github.com/miekg/dns v1.1.30/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
@@ -40,6 +36,7 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLD
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/projectdiscovery/gologger v1.0.0 h1:XAQ8kHeVKXMjY4rLGh7eT5+oHU077BNEvs7X6n+vu1s=
 github.com/projectdiscovery/gologger v1.0.0/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
@@ -49,16 +46,16 @@ github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNw
 github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
+github.com/vbauerster/mpb/v5 v5.2.4 h1:PLP8vv75RcEgxGoJVtKaRD2FHSxEmIV/u4ZuOrfO8Qg=
-github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/vbauerster/mpb/v5 v5.2.4/go.mod h1:K4iCHQp5sWnmAgEn+uW1sAxSilctb4JPAGXx49jV+Aw=
-github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8=
-github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 h1:DZhuSZLsGlFL4CmhA8BcRA0mnthyA/nZ00AqCUo7vHg=
+golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -67,18 +64,22 @@ golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo=
+golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200724161237-0e2f3a69832c h1:UIcGWL6/wpCfyGuJnRFJRurA+yj8RrW7Q6x2YMCXt6c=
+golang.org/x/sys v0.0.0-20200724161237-0e2f3a69832c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=

v2/internal/progress/doc.go

@@ -0,0 +1,2 @@
+// Tracks enumeration progress information and implements visible tracking with one or more progress bars.
+package progress

v2/internal/progress/progress.go

@@ -0,0 +1,154 @@
+package progress
+
+import (
+	"fmt"
+	"github.com/logrusorgru/aurora"
+	"github.com/vbauerster/mpb/v5"
+	"github.com/vbauerster/mpb/v5/decor"
+	"os"
+	"strings"
+	"sync"
+)
+
+// Encapsulates progress tracking.
+type Progress struct {
+	progress        *mpb.Progress
+	gbar            *mpb.Bar
+	total           int64
+	initialTotal    int64
+	totalMutex      *sync.Mutex
+	captureData     *captureData
+	stdCaptureMutex *sync.Mutex
+	stdout          *strings.Builder
+	stderr          *strings.Builder
+	colorizer       aurora.Aurora
+}
+
+// Creates and returns a new progress tracking object.
+func NewProgress(noColor bool) *Progress {
+	p := &Progress{
+		progress: mpb.New(
+			mpb.WithOutput(os.Stderr),
+			mpb.PopCompletedMode(),
+		),
+		totalMutex:      &sync.Mutex{},
+		stdCaptureMutex: &sync.Mutex{},
+		stdout:          &strings.Builder{},
+		stderr:          &strings.Builder{},
+		colorizer:       aurora.NewAurora(!noColor),
+	}
+	return p
+}
+
+// Creates and returns a progress bar that tracks all the requests progress.
+// This is only useful when multiple templates are processed within the same run.
+func (p *Progress) InitProgressbar(hostCount int64, templateCount int, requestCount int64) {
+	if p.gbar != nil {
+		panic("A global progressbar is already present.")
+	}
+
+	color := p.colorizer
+
+	barName := color.Sprintf(
+		color.Cyan("%d %s, %d %s"),
+		color.Bold(color.Cyan(templateCount)),
+		pluralize(int64(templateCount), "template", "templates"),
+		color.Bold(color.Cyan(hostCount)),
+		pluralize(hostCount, "host", "hosts"))
+
+	p.gbar = p.setupProgressbar("["+barName+"]", requestCount, 0)
+}
+
+func pluralize(count int64, singular, plural string) string {
+	if count > 1 {
+		return plural
+	}
+	return singular
+}
+
+// Update total progress request count
+func (p *Progress) AddToTotal(delta int64) {
+	p.totalMutex.Lock()
+	p.total += delta
+	p.gbar.SetTotal(p.total, false)
+	p.totalMutex.Unlock()
+}
+
+// Update progress tracking information and increments the request counter by one unit.
+func (p *Progress) Update() {
+	p.gbar.Increment()
+}
+
+// Drops the specified number of requests from the progress bar total.
+// This may be the case when uncompleted requests are encountered and shouldn't be part of the total count.
+func (p *Progress) Drop(count int64) {
+	// mimic dropping by incrementing the completed requests
+	p.gbar.IncrInt64(count)
+
+}
+
+// Ensures that a progress bar's total count is up-to-date if during an enumeration there were uncompleted requests and
+// wait for all the progress bars to finish.
+func (p *Progress) Wait() {
+	p.totalMutex.Lock()
+	if p.total == 0 {
+		p.gbar.Abort(true)
+	} else if p.initialTotal != p.total {
+		p.gbar.SetTotal(p.total, true)
+	}
+	p.totalMutex.Unlock()
+	p.progress.Wait()
+}
+
+// Creates and returns a progress bar.
+func (p *Progress) setupProgressbar(name string, total int64, priority int) *mpb.Bar {
+	color := p.colorizer
+
+	p.total = total
+	p.initialTotal = total
+
+	return p.progress.AddBar(
+		total,
+		mpb.BarPriority(priority),
+		mpb.BarNoPop(),
+		mpb.BarRemoveOnComplete(),
+		mpb.PrependDecorators(
+			decor.Name(name, decor.WCSyncSpaceR),
+			decor.CountersNoUnit(color.BrightBlue(" %d/%d").String(), decor.WCSyncSpace),
+			decor.NewPercentage(color.Bold("%d").String(), decor.WCSyncSpace),
+		),
+		mpb.AppendDecorators(
+			decor.AverageSpeed(0, color.BrightYellow("%.2f").Bold().String()+color.BrightYellow("r/s").String(), decor.WCSyncSpace),
+			decor.Elapsed(decor.ET_STYLE_GO, decor.WCSyncSpace),
+			decor.AverageETA(decor.ET_STYLE_GO, decor.WCSyncSpace),
+		),
+	)
+}
+
+// Starts capturing stdout and stderr instead of producing visual output that may interfere with the progress bars.
+func (p *Progress) StartStdCapture() {
+	p.stdCaptureMutex.Lock()
+	p.captureData = startStdCapture()
+}
+
+// Stops capturing stdout and stderr and store both output to be shown later.
+func (p *Progress) StopStdCapture() {
+	stopStdCapture(p.captureData)
+	p.stdout.Write(p.captureData.DataStdOut.Bytes())
+	p.stderr.Write(p.captureData.DataStdErr.Bytes())
+	p.stdCaptureMutex.Unlock()
+}
+
+// Writes the captured stdout data to stdout, if any.
+func (p *Progress) ShowStdOut() {
+	if p.stdout.Len() > 0 {
+		fmt.Fprint(os.Stdout, p.stdout.String())
+	}
+}
+
+// Writes the captured stderr data to stderr, if any.
+func (p *Progress) ShowStdErr() {
+	if p.stderr.Len() > 0 {
+		fmt.Fprint(os.Stderr, p.stderr.String())
+	}
+}

v2/internal/progress/stdcapture.go

@@ -0,0 +1,95 @@
+package progress
+
+/**
+  Inspired by the https://github.com/PumpkinSeed/cage module
+ */
+import (
+	"bytes"
+	"io"
+	"os"
+	"sync"
+)
+
+type captureData struct {
+	backupStdout *os.File
+	writerStdout *os.File
+	backupStderr *os.File
+	writerStderr *os.File
+
+	DataStdOut *bytes.Buffer
+	DataStdErr *bytes.Buffer
+
+	outStdout  chan []byte
+	outStderr  chan []byte
+}
+
+func startStdCapture() *captureData {
+	rStdout, wStdout, errStdout := os.Pipe()
+	if errStdout != nil {
+		panic(errStdout)
+	}
+
+	rStderr, wStderr, errStderr := os.Pipe()
+	if errStderr != nil {
+		panic(errStderr)
+	}
+
+	c := &captureData{
+		backupStdout: os.Stdout,
+		writerStdout: wStdout,
+
+		backupStderr: os.Stderr,
+		writerStderr: wStderr,
+
+		outStdout: make(chan []byte),
+		outStderr: make(chan []byte),
+
+		DataStdOut: &bytes.Buffer{},
+		DataStdErr: &bytes.Buffer{},
+	}
+
+	os.Stdout = c.writerStdout
+	os.Stderr = c.writerStderr
+
+	stdCopy := func(out chan<- []byte, reader *os.File) {
+		var buffer bytes.Buffer
+		_, _ = io.Copy(&buffer, reader)
+		if buffer.Len() > 0 {
+			out <- buffer.Bytes()
+		}
+		close(out)
+	}
+
+	go stdCopy(c.outStdout, rStdout)
+	go stdCopy(c.outStderr, rStderr)
+
+	return c
+}
+
+func stopStdCapture(c *captureData) {
+	_ = c.writerStdout.Close()
+	_ = c.writerStderr.Close()
+
+	var wg sync.WaitGroup
+
+	stdRead := func(in <-chan []byte, outData *bytes.Buffer) {
+		defer wg.Done()
+
+		for {
+			out, more := <-in
+			if more {
+				outData.Write(out)
+			} else {
+				return
+			}
+		}
+	}
+
+	wg.Add(2)
+	go stdRead(c.outStdout, c.DataStdOut)
+	go stdRead(c.outStderr, c.DataStdErr)
+	wg.Wait()
+
+	os.Stdout = c.backupStdout
+	os.Stderr = c.backupStderr
+}

v2/internal/runner/runner.go

@@ -17,6 +17,7 @@ import (
 	"github.com/d5/tengo/v2/stdlib"
 	"github.com/karrick/godirwalk"
 	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/internal/progress"
 	"github.com/projectdiscovery/nuclei/v2/pkg/atomicboolean"
 	"github.com/projectdiscovery/nuclei/v2/pkg/executer"
 	"github.com/projectdiscovery/nuclei/v2/pkg/requests"
@@ -26,6 +27,9 @@ import (
 
 // Runner is a client for running the enumeration process.
 type Runner struct {
+	input      string
+	inputCount int64
+
 	// output is the output file to write if any
 	output      *os.File
 	outputMutex *sync.Mutex
@@ -35,6 +39,9 @@ type Runner struct {
 	// options contains configuration options for runner
 	options *Options
 	limiter chan struct{}
+
+	// progress tracking
+	progress *progress.Progress
 }
 
 // New creates a new client for running enumeration process.
@@ -74,6 +81,46 @@ func New(options *Options) (*Runner, error) {
 		tempInput.Close()
 	}
 
+	// Setup input, handle a list of hosts as argument
+	var err error
+	var input *os.File
+	if options.Targets != "" {
+		input, err = os.Open(options.Targets)
+	} else if options.Stdin || options.Target != "" {
+		input, err = os.Open(runner.tempFile)
+	}
+	if err != nil {
+		gologger.Fatalf("Could not open targets file '%s': %s\n", options.Targets, err)
+	}
+
+	// Sanitize input and pre-compute total number of targets
+	var usedInput = make(map[string]bool)
+	dupeCount := 0
+	sb := strings.Builder{}
+	scanner := bufio.NewScanner(input)
+	runner.inputCount = 0
+	for scanner.Scan() {
+		url := scanner.Text()
+		// skip empty lines
+		if len(url) == 0 {
+			continue
+		}
+		// deduplication
+		if _, ok := usedInput[url]; !ok {
+			usedInput[url] = true
+			runner.inputCount++
+			sb.WriteString(url)
+			sb.WriteString("\n")
+		} else {
+			dupeCount++
+		}
+	}
+	input.Close()
+	runner.input = sb.String()
+	if dupeCount > 0 {
+		gologger.Labelf("Supplied input was automatically deduplicated (%d removed).", dupeCount)
+	}
+
 	// Create the output file if asked
 	if options.Output != "" {
 		output, err := os.Create(options.Output)
@@ -83,6 +130,9 @@ func New(options *Options) (*Runner, error) {
 		runner.output = output
 	}
 
+	// Creates the progress tracking object
+	runner.progress = progress.NewProgress(runner.options.NoColor)
+
 	runner.limiter = make(chan struct{}, options.Threads)
 
 	return runner, nil
@@ -230,35 +280,78 @@ func (r *Runner) RunEnumeration() {
 		gologger.Fatalf("Error, no templates were found.\n")
 	}
 
+	// progress tracking
+	p := r.progress
+
+	// precompute total request count
+	var totalRequests int64 = 0
+	hasWorkflows := false
+	parsedTemplates := []string{}
+
+	for _, match := range allTemplates {
+		t, err := r.parse(match)
+		switch t.(type) {
+		case *templates.Template:
+			template := t.(*templates.Template)
+			totalRequests += (template.GetHTTPRequestCount() + template.GetDNSRequestCount()) * r.inputCount
+			parsedTemplates = append(parsedTemplates, match)
+		case *workflows.Workflow:
+			// workflows will dynamically adjust the totals while running, as
+			// it can't be know in advance which requests will be called
+			parsedTemplates = append(parsedTemplates, match)
+			hasWorkflows = true
+		default:
+			gologger.Errorf("Could not parse file '%s': %s\n", match, err)
+		}
+	}
+
+	// ensure only successfully parsed templates are processed
+	allTemplates = parsedTemplates
+	templateCount := len(allTemplates)
+
 	var (
 		wgtemplates sync.WaitGroup
 		results     atomicboolean.AtomBool
 	)
 
-	for _, match := range allTemplates {
+	if r.inputCount == 0 {
-		wgtemplates.Add(1)
+		gologger.Errorf("Could not find any valid input URLs.")
-		go func(match string) {
+	} else if totalRequests > 0 || hasWorkflows {
-			defer wgtemplates.Done()
-			t, err := r.parse(match)
-			switch t.(type) {
-			case *templates.Template:
-				template := t.(*templates.Template)
-				for _, request := range template.RequestsDNS {
-					results.Or(r.processTemplateRequest(template, request))
-				}
-				for _, request := range template.BulkRequestsHTTP {
-					results.Or(r.processTemplateRequest(template, request))
-				}
-			case *workflows.Workflow:
-				workflow := t.(*workflows.Workflow)
-				r.ProcessWorkflowWithList(workflow)
-			default:
-				gologger.Errorf("Could not parse file '%s': %s\n", match, err)
-			}
-		}(match)
-	}
 
-	wgtemplates.Wait()
+		// track global progress
+		p.InitProgressbar(r.inputCount, templateCount, totalRequests)
+		p.StartStdCapture()
+
+		for _, match := range allTemplates {
+			wgtemplates.Add(1)
+			go func(match string) {
+				defer wgtemplates.Done()
+				t, err := r.parse(match)
+				switch t.(type) {
+				case *templates.Template:
+					template := t.(*templates.Template)
+					for _, request := range template.RequestsDNS {
+						results.Or(r.processTemplateWithList(p, template, request))
+					}
+					for _, request := range template.BulkRequestsHTTP {
+						results.Or(r.processTemplateWithList(p, template, request))
+					}
+				case *workflows.Workflow:
+					workflow := t.(*workflows.Workflow)
+					r.ProcessWorkflowWithList(p, workflow)
+				default:
+					gologger.Errorf("Could not parse file '%s': %s\n", match, err)
+				}
+			}(match)
+		}
+
+		wgtemplates.Wait()
+		p.Wait()
+
+		p.StopStdCapture()
+		p.ShowStdErr()
+		p.ShowStdOut()
+	}
 
 	if !results.Get() {
 		if r.output != nil {
@@ -271,27 +364,8 @@ func (r *Runner) RunEnumeration() {
 	return
 }
 
-// processTemplate processes a template and runs the enumeration on all the targets
+// processTemplateWithList processes a template and runs the enumeration on all the targets
-func (r *Runner) processTemplateRequest(template *templates.Template, request interface{}) bool {
+func (r *Runner) processTemplateWithList(p *progress.Progress, template *templates.Template, request interface{}) bool {
-	var file *os.File
-	var err error
-
-	// Handle a list of hosts as argument
-	if r.options.Targets != "" {
-		file, err = os.Open(r.options.Targets)
-	} else if r.options.Stdin || r.options.Target != "" {
-		file, err = os.Open(r.tempFile)
-	}
-	if err != nil {
-		gologger.Fatalf("Could not open targets file '%s': %s\n", r.options.Targets, err)
-	}
-	defer file.Close()
-
-	return r.processTemplateWithList(template, request, file)
-}
-
-// processDomain processes the list with a template
-func (r *Runner) processTemplateWithList(template *templates.Template, request interface{}, reader io.Reader) bool {
 	// Display the message for the template
 	message := fmt.Sprintf("[%s] Loaded template %s (@%s)", template.ID, template.Info.Name, template.Info.Author)
 	if template.Info.Severity != "" {
@@ -336,6 +410,7 @@ func (r *Runner) processTemplateWithList(template *templates.Template, request i
 		})
 	}
 	if err != nil {
+		p.Drop(request.(*requests.BulkHTTPRequest).GetRequestCount())
 		gologger.Warningf("Could not create http client: %s\n", err)
 		return false
 	}
@@ -343,12 +418,10 @@ func (r *Runner) processTemplateWithList(template *templates.Template, request i
 	var globalresult atomicboolean.AtomBool
 
 	var wg sync.WaitGroup
-	scanner := bufio.NewScanner(reader)
+
+	scanner := bufio.NewScanner(strings.NewReader(r.input))
 	for scanner.Scan() {
 		text := scanner.Text()
-		if text == "" {
-			continue
-		}
 
 		r.limiter <- struct{}{}
 		wg.Add(1)
@@ -358,11 +431,11 @@ func (r *Runner) processTemplateWithList(template *templates.Template, request i
 			var result executer.Result
 
 			if httpExecuter != nil {
-				result = httpExecuter.ExecuteHTTP(URL)
+				result = httpExecuter.ExecuteHTTP(p, URL)
 				globalresult.Or(result.GotResults)
 			}
 			if dnsExecuter != nil {
-				result = dnsExecuter.ExecuteDNS(URL)
+				result = dnsExecuter.ExecuteDNS(p, URL)
 				globalresult.Or(result.GotResults)
 			}
 			if result.Error != nil {
@@ -379,35 +452,18 @@ func (r *Runner) processTemplateWithList(template *templates.Template, request i
 }
 
 // ProcessWorkflowWithList coming from stdin or list of targets
-func (r *Runner) ProcessWorkflowWithList(workflow *workflows.Workflow) {
+func (r *Runner) ProcessWorkflowWithList(p *progress.Progress, workflow *workflows.Workflow) {
-	var file *os.File
-	var err error
-	// Handle a list of hosts as argument
-	if r.options.Targets != "" {
-		file, err = os.Open(r.options.Targets)
-	} else if r.options.Stdin || r.options.Target != "" {
-		file, err = os.Open(r.tempFile)
-	}
-	if err != nil {
-		gologger.Fatalf("Could not open targets file '%s': %s\n", r.options.Targets, err)
-	}
-	defer file.Close()
-
 	var wg sync.WaitGroup
-	scanner := bufio.NewScanner(file)
+	scanner := bufio.NewScanner(strings.NewReader(r.input))
 	for scanner.Scan() {
 		text := scanner.Text()
-		if text == "" {
-			continue
-		}
-
 		r.limiter <- struct{}{}
 		wg.Add(1)
 
 		go func(URL string) {
 			defer wg.Done()
 
-			if err := r.ProcessWorkflow(workflow, text); err != nil {
+			if err := r.ProcessWorkflow(p, workflow, text); err != nil {
 				gologger.Warningf("Could not run workflow for %s: %s\n", text, err)
 			}
 			<-r.limiter
@@ -418,7 +474,7 @@ func (r *Runner) ProcessWorkflowWithList(workflow *workflows.Workflow) {
 }
 
 // ProcessWorkflow towards an URL
-func (r *Runner) ProcessWorkflow(workflow *workflows.Workflow, URL string) error {
+func (r *Runner) ProcessWorkflow(p *progress.Progress, workflow *workflows.Workflow, URL string) error {
 	script := tengo.NewScript([]byte(workflow.Logic))
 	script.SetImports(stdlib.GetModuleMap(stdlib.AllModuleNames()...))
 	var jar *cookiejar.Jar
@@ -456,7 +512,7 @@ func (r *Runner) ProcessWorkflow(workflow *workflows.Workflow, URL string) error
 			if err != nil {
 				return err
 			}
-			template := &workflows.Template{}
+			template := &workflows.Template{Progress: p}
 			if len(t.BulkRequestsHTTP) > 0 {
 				template.HTTPOptions = &executer.HTTPOptions{
 					Debug:         r.options.Debug,
@@ -507,7 +563,7 @@ func (r *Runner) ProcessWorkflow(workflow *workflows.Workflow, URL string) error
 				if err != nil {
 					return err
 				}
-				template := &workflows.Template{}
+				template := &workflows.Template{Progress: p}
 				if len(t.BulkRequestsHTTP) > 0 {
 					template.HTTPOptions = &executer.HTTPOptions{
 						Debug:         r.options.Debug,

v2/pkg/executer/executer_dns.go

@@ -3,6 +3,7 @@ package executer
 import (
 	"bufio"
 	"fmt"
+	"github.com/projectdiscovery/nuclei/v2/internal/progress"
 	"os"
 	"sync"
 
@@ -62,7 +63,7 @@ func NewDNSExecuter(options *DNSOptions) *DNSExecuter {
 }
 
 // ExecuteDNS executes the DNS request on a URL
-func (e *DNSExecuter) ExecuteDNS(URL string) (result Result) {
+func (e *DNSExecuter) ExecuteDNS(p *progress.Progress, URL string) (result Result) {
 	// Parse the URL and return domain if URL.
 	var domain string
 	if isURL(URL) {
@@ -75,6 +76,7 @@ func (e *DNSExecuter) ExecuteDNS(URL string) (result Result) {
 	compiledRequest, err := e.dnsRequest.MakeDNSRequest(domain)
 	if err != nil {
 		result.Error = errors.Wrap(err, "could not make dns request")
+		p.Drop(1)
 		return
 	}
 
@@ -87,9 +89,12 @@ func (e *DNSExecuter) ExecuteDNS(URL string) (result Result) {
 	resp, err := e.dnsClient.Do(compiledRequest)
 	if err != nil {
 		result.Error = errors.Wrap(err, "could not send dns request")
+		p.Drop(1)
 		return
 	}
 
+	p.Update()
+
 	gologger.Verbosef("Sent DNS request to %s\n", "dns-request", URL)
 
 	if e.debug {

v2/pkg/executer/executer_http.go

@@ -17,6 +17,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/internal/progress"
 	"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
 	"github.com/projectdiscovery/nuclei/v2/pkg/requests"
 	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
@@ -100,7 +101,7 @@ func NewHTTPExecuter(options *HTTPOptions) (*HTTPExecuter, error) {
 }
 
 // ExecuteHTTP executes the HTTP request on a URL
-func (e *HTTPExecuter) ExecuteHTTP(URL string) (result Result) {
+func (e *HTTPExecuter) ExecuteHTTP(p *progress.Progress, URL string) (result Result) {
 	result.Matches = make(map[string]interface{})
 	result.Extractions = make(map[string]interface{})
 	dynamicvalues := make(map[string]interface{})
@@ -110,21 +111,27 @@ func (e *HTTPExecuter) ExecuteHTTP(URL string) (result Result) {
 		return
 	}
 
+	remaining := e.bulkHttpRequest.GetRequestCount()
+
 	e.bulkHttpRequest.CreateGenerator(URL)
 	for e.bulkHttpRequest.Next(URL) && !result.Done {
 		httpRequest, err := e.bulkHttpRequest.MakeHTTPRequest(URL, dynamicvalues, e.bulkHttpRequest.Current(URL))
 		if err != nil {
-			result.Error = errors.Wrap(err, "could not make http request")
+			result.Error = errors.Wrap(err, "could not build http request")
+			p.Drop(remaining)
 			return
 		}
 
-		err = e.handleHTTP(URL, httpRequest, dynamicvalues, &result)
+		err = e.handleHTTP(p, URL, httpRequest, dynamicvalues, &result)
 		if err != nil {
-			result.Error = errors.Wrap(err, "could not make http request")
+			result.Error = errors.Wrap(err, "could not handle http request")
+			p.Drop(remaining)
 			return
 		}
 
 		e.bulkHttpRequest.Increment(URL)
+		p.Update()
+		remaining--
 	}
 
 	gologger.Verbosef("Sent HTTP request to %s\n", "http-request", URL)
@@ -132,16 +139,16 @@ func (e *HTTPExecuter) ExecuteHTTP(URL string) (result Result) {
 	return
 }
 
-func (e *HTTPExecuter) handleHTTP(URL string, request *requests.HttpRequest, dynamicvalues map[string]interface{}, result *Result) error {
+func (e *HTTPExecuter) handleHTTP(p *progress.Progress, URL string, request *requests.HttpRequest, dynamicvalues map[string]interface{}, result *Result) error {
 	e.setCustomHeaders(request)
 	req := request.Request
 
 	if e.debug {
-		gologger.Infof("Dumped HTTP request for %s (%s)\n\n", URL, e.template.ID)
 		dumpedRequest, err := httputil.DumpRequest(req.Request, true)
 		if err != nil {
 			return errors.Wrap(err, "could not make http request")
 		}
+		gologger.Infof("Dumped HTTP request for %s (%s)\n\n", URL, e.template.ID)
 		fmt.Fprintf(os.Stderr, "%s", string(dumpedRequest))
 	}
 	resp, err := e.httpClient.Do(req)
@@ -153,11 +160,11 @@ func (e *HTTPExecuter) handleHTTP(URL string, request *requests.HttpRequest, dyn
 	}
 
 	if e.debug {
-		gologger.Infof("Dumped HTTP response for %s (%s)\n\n", URL, e.template.ID)
 		dumpedResponse, err := httputil.DumpResponse(resp, true)
 		if err != nil {
 			return errors.Wrap(err, "could not dump http response")
 		}
+		gologger.Infof("Dumped HTTP response for %s (%s)\n\n", URL, e.template.ID)
 		fmt.Fprintf(os.Stderr, "%s\n", string(dumpedResponse))
 	}
 

v2/pkg/requests/bulk-http-request.go

@@ -76,6 +76,11 @@ func (r *BulkHTTPRequest) SetAttackType(attack generators.Type) {
 	r.attackType = attack
 }
 
+// Returns the total number of requests the YAML rule will perform
+func (r *BulkHTTPRequest) GetRequestCount() int64 {
+	return int64(len(r.Raw) | len(r.Path))
+}
+
 func (r *BulkHTTPRequest) MakeHTTPRequest(baseURL string, dynamicValues map[string]interface{}, data string) (*HttpRequest, error) {
 	parsed, err := url.Parse(baseURL)
 	if err != nil {

v2/pkg/requests/dns-request.go

@@ -42,6 +42,11 @@ func (r *DNSRequest) SetMatchersCondition(condition matchers.ConditionType) {
 	r.matchersCondition = condition
 }
 
+// Returns the total number of requests the YAML rule will perform
+func (r *DNSRequest) GetRequestCount() int64 {
+	return 1
+}
+
 // MakeDNSRequest creates a *dns.Request from a request template
 func (r *DNSRequest) MakeDNSRequest(domain string) (*dns.Msg, error) {
 	domain = dns.Fqdn(domain)

v2/pkg/templates/templates.go

@@ -27,3 +27,19 @@ type Info struct {
 	// Description optionally describes the template.
 	Description string `yaml:"description,omitempty"`
 }
+
+func (t* Template) GetHTTPRequestCount() int64 {
+	var count int64 = 0
+	for _, request := range t.BulkRequestsHTTP {
+		count += request.GetRequestCount()
+	}
+	return count
+}
+
+func (t *Template) GetDNSRequestCount() int64 {
+	var count int64 = 0
+	for _, request := range t.RequestsDNS {
+		count += request.GetRequestCount()
+	}
+	return count
+}

v2/pkg/workflows/var.go

@@ -5,6 +5,7 @@ import (
 
 	tengo "github.com/d5/tengo/v2"
 	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v2/internal/progress"
 	"github.com/projectdiscovery/nuclei/v2/pkg/atomicboolean"
 	"github.com/projectdiscovery/nuclei/v2/pkg/executer"
 	"github.com/projectdiscovery/nuclei/v2/pkg/generators"
@@ -23,6 +24,7 @@ type NucleiVar struct {
 type Template struct {
 	HTTPOptions *executer.HTTPOptions
 	DNSOptions  *executer.DNSOptions
+	Progress    *progress.Progress
 }
 
 // TypeName of the variable
@@ -53,7 +55,9 @@ func (n *NucleiVar) Call(args ...tengo.Object) (ret tengo.Object, err error) {
 
 	var gotResult atomicboolean.AtomBool
 	for _, template := range n.Templates {
+		p := template.Progress
 		if template.HTTPOptions != nil {
+			p.AddToTotal(template.HTTPOptions.Template.GetHTTPRequestCount())
 			for _, request := range template.HTTPOptions.Template.BulkRequestsHTTP {
 				// apply externally supplied payloads if any
 				request.Headers = generators.MergeMapsWithStrings(request.Headers, headers)
@@ -62,10 +66,11 @@ func (n *NucleiVar) Call(args ...tengo.Object) (ret tengo.Object, err error) {
 				template.HTTPOptions.BulkHttpRequest = request
 				httpExecuter, err := executer.NewHTTPExecuter(template.HTTPOptions)
 				if err != nil {
+					p.Drop(request.GetRequestCount())
 					gologger.Warningf("Could not compile request for template '%s': %s\n", template.HTTPOptions.Template.ID, err)
 					continue
 				}
-				result := httpExecuter.ExecuteHTTP(n.URL)
+				result := httpExecuter.ExecuteHTTP(p, n.URL)
 				if result.Error != nil {
 					gologger.Warningf("Could not send request for template '%s': %s\n", template.HTTPOptions.Template.ID, result.Error)
 					continue
@@ -79,10 +84,11 @@ func (n *NucleiVar) Call(args ...tengo.Object) (ret tengo.Object, err error) {
 		}
 
 		if template.DNSOptions != nil {
+			p.AddToTotal(template.DNSOptions.Template.GetDNSRequestCount())
 			for _, request := range template.DNSOptions.Template.RequestsDNS {
 				template.DNSOptions.DNSRequest = request
 				dnsExecuter := executer.NewDNSExecuter(template.DNSOptions)
-				result := dnsExecuter.ExecuteDNS(n.URL)
+				result := dnsExecuter.ExecuteDNS(p, n.URL)
 				if result.Error != nil {
 					gologger.Warningf("Could not compile request for template '%s': %s\n", template.HTTPOptions.Template.ID, result.Error)
 					continue