diff --git a/.gitignore b/.gitignore index 4c07d426..3eed20ea 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ cmd/nuclei/nuclei* v2/cmd/nuclei/nuclei +.idea diff --git a/v2/go.mod b/v2/go.mod index 28235ebc..622be1bb 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -10,12 +10,16 @@ require ( github.com/google/go-github/v32 v32.1.0 github.com/json-iterator/go v1.1.10 github.com/karrick/godirwalk v1.15.6 - github.com/labstack/gommon v0.3.0 + github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 github.com/miekg/dns v1.1.30 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/gologger v1.0.0 github.com/projectdiscovery/retryabledns v1.0.4 github.com/projectdiscovery/retryablehttp-go v1.0.1 + github.com/stretchr/testify v1.5.1 + github.com/vbauerster/mpb/v5 v5.2.4 + golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 // indirect golang.org/x/net v0.0.0-20200707034311-ab3426394381 + golang.org/x/sys v0.0.0-20200724161237-0e2f3a69832c // indirect gopkg.in/yaml.v2 v2.3.0 ) diff --git a/v2/go.sum b/v2/go.sum index 478a7ef0..01fc7adb 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -1,18 +1,19 @@ -github.com/Knetic/govaluate v1.5.0 h1:L4MyqdJSld9xr2eZcZHCWLfeIX2SBjqrwIKG1pcm/+4= github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f6s9Tn1Tt7/WTEg= github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM= +github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0S6Vi7/lbWECcX0j45yZReDZ56BQsrVBOEEY= github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= -github.com/blang/semver v1.1.0 h1:ol1rO7QQB5uy7umSNV7VAmLugfLRD+17sYJujRNYPhg= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/d5/tengo v1.24.8 h1:PRJ+NWt7ae/9sSbIfThOBTkPSvNV+dwYoBAvwfNgNJY= github.com/d5/tengo/v2 v2.6.0 h1:D0cJtpiBzaLJ/Smv6nnUc/LIfO46oKwDx85NZtIRNRI= github.com/d5/tengo/v2 v2.6.0/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= github.com/google/go-github/v32 v32.1.0 h1:GWkQOdXqviCPx7Q7Fj+KyPoGm4SwHRh8rheoPhd27II= github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= @@ -22,15 +23,10 @@ github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/karrick/godirwalk v1.15.6 h1:Yf2mmR8TJy+8Fa0SuQVto5SYap6IF7lNVX4Jdl8G1qA= github.com/karrick/godirwalk v1.15.6/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= -github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0= -github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 h1:bqDmpDG49ZRnB5PcgP0RXtQvnMSgIF14M7CBd2shtXs= github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= -github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= -github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg= -github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.30 h1:Qww6FseFn8PRfw07jueqIXqodm0JKiiKuK0DeXSqfyo= github.com/miekg/dns v1.1.30/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= @@ -40,6 +36,7 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLD github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/projectdiscovery/gologger v1.0.0 h1:XAQ8kHeVKXMjY4rLGh7eT5+oHU077BNEvs7X6n+vu1s= github.com/projectdiscovery/gologger v1.0.0/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= @@ -49,16 +46,16 @@ github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNw github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= -github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8= -github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/vbauerster/mpb/v5 v5.2.4 h1:PLP8vv75RcEgxGoJVtKaRD2FHSxEmIV/u4ZuOrfO8Qg= +github.com/vbauerster/mpb/v5 v5.2.4/go.mod h1:K4iCHQp5sWnmAgEn+uW1sAxSilctb4JPAGXx49jV+Aw= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 h1:DZhuSZLsGlFL4CmhA8BcRA0mnthyA/nZ00AqCUo7vHg= +golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -67,18 +64,22 @@ golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200724161237-0e2f3a69832c h1:UIcGWL6/wpCfyGuJnRFJRurA+yj8RrW7Q6x2YMCXt6c= +golang.org/x/sys v0.0.0-20200724161237-0e2f3a69832c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= diff --git a/v2/internal/progress/doc.go b/v2/internal/progress/doc.go new file mode 100644 index 00000000..1a4999e5 --- /dev/null +++ b/v2/internal/progress/doc.go @@ -0,0 +1,2 @@ +// Tracks enumeration progress information and implements visible tracking with one or more progress bars. +package progress diff --git a/v2/internal/progress/progress.go b/v2/internal/progress/progress.go new file mode 100644 index 00000000..e74590d4 --- /dev/null +++ b/v2/internal/progress/progress.go @@ -0,0 +1,154 @@ +package progress + +import ( + "fmt" + "github.com/logrusorgru/aurora" + "github.com/vbauerster/mpb/v5" + "github.com/vbauerster/mpb/v5/decor" + "os" + "strings" + "sync" +) + +// Encapsulates progress tracking. +type Progress struct { + progress *mpb.Progress + gbar *mpb.Bar + total int64 + initialTotal int64 + totalMutex *sync.Mutex + captureData *captureData + stdCaptureMutex *sync.Mutex + stdout *strings.Builder + stderr *strings.Builder + colorizer aurora.Aurora +} + +// Creates and returns a new progress tracking object. +func NewProgress(noColor bool) *Progress { + p := &Progress{ + progress: mpb.New( + mpb.WithOutput(os.Stderr), + mpb.PopCompletedMode(), + ), + totalMutex: &sync.Mutex{}, + stdCaptureMutex: &sync.Mutex{}, + stdout: &strings.Builder{}, + stderr: &strings.Builder{}, + colorizer: aurora.NewAurora(!noColor), + } + return p +} + +// Creates and returns a progress bar that tracks all the requests progress. +// This is only useful when multiple templates are processed within the same run. +func (p *Progress) InitProgressbar(hostCount int64, templateCount int, requestCount int64) { + if p.gbar != nil { + panic("A global progressbar is already present.") + } + + color := p.colorizer + + barName := color.Sprintf( + color.Cyan("%d %s, %d %s"), + color.Bold(color.Cyan(templateCount)), + pluralize(int64(templateCount), "template", "templates"), + color.Bold(color.Cyan(hostCount)), + pluralize(hostCount, "host", "hosts")) + + p.gbar = p.setupProgressbar("["+barName+"]", requestCount, 0) +} + +func pluralize(count int64, singular, plural string) string { + if count > 1 { + return plural + } + return singular +} + +// Update total progress request count +func (p *Progress) AddToTotal(delta int64) { + p.totalMutex.Lock() + p.total += delta + p.gbar.SetTotal(p.total, false) + p.totalMutex.Unlock() +} + +// Update progress tracking information and increments the request counter by one unit. +func (p *Progress) Update() { + p.gbar.Increment() +} + +// Drops the specified number of requests from the progress bar total. +// This may be the case when uncompleted requests are encountered and shouldn't be part of the total count. +func (p *Progress) Drop(count int64) { + // mimic dropping by incrementing the completed requests + p.gbar.IncrInt64(count) + +} + +// Ensures that a progress bar's total count is up-to-date if during an enumeration there were uncompleted requests and +// wait for all the progress bars to finish. +func (p *Progress) Wait() { + p.totalMutex.Lock() + if p.total == 0 { + p.gbar.Abort(true) + } else if p.initialTotal != p.total { + p.gbar.SetTotal(p.total, true) + } + p.totalMutex.Unlock() + p.progress.Wait() +} + +// Creates and returns a progress bar. +func (p *Progress) setupProgressbar(name string, total int64, priority int) *mpb.Bar { + color := p.colorizer + + p.total = total + p.initialTotal = total + + return p.progress.AddBar( + total, + mpb.BarPriority(priority), + mpb.BarNoPop(), + mpb.BarRemoveOnComplete(), + mpb.PrependDecorators( + decor.Name(name, decor.WCSyncSpaceR), + decor.CountersNoUnit(color.BrightBlue(" %d/%d").String(), decor.WCSyncSpace), + decor.NewPercentage(color.Bold("%d").String(), decor.WCSyncSpace), + ), + mpb.AppendDecorators( + decor.AverageSpeed(0, color.BrightYellow("%.2f").Bold().String()+color.BrightYellow("r/s").String(), decor.WCSyncSpace), + decor.Elapsed(decor.ET_STYLE_GO, decor.WCSyncSpace), + decor.AverageETA(decor.ET_STYLE_GO, decor.WCSyncSpace), + ), + ) +} + +// Starts capturing stdout and stderr instead of producing visual output that may interfere with the progress bars. +func (p *Progress) StartStdCapture() { + p.stdCaptureMutex.Lock() + p.captureData = startStdCapture() +} + +// Stops capturing stdout and stderr and store both output to be shown later. +func (p *Progress) StopStdCapture() { + stopStdCapture(p.captureData) + p.stdout.Write(p.captureData.DataStdOut.Bytes()) + p.stderr.Write(p.captureData.DataStdErr.Bytes()) + p.stdCaptureMutex.Unlock() +} + +// Writes the captured stdout data to stdout, if any. +func (p *Progress) ShowStdOut() { + if p.stdout.Len() > 0 { + fmt.Fprint(os.Stdout, p.stdout.String()) + } +} + +// Writes the captured stderr data to stderr, if any. +func (p *Progress) ShowStdErr() { + if p.stderr.Len() > 0 { + fmt.Fprint(os.Stderr, p.stderr.String()) + } +} diff --git a/v2/internal/progress/stdcapture.go b/v2/internal/progress/stdcapture.go new file mode 100644 index 00000000..09070894 --- /dev/null +++ b/v2/internal/progress/stdcapture.go @@ -0,0 +1,95 @@ +package progress + +/** + Inspired by the https://github.com/PumpkinSeed/cage module + */ +import ( + "bytes" + "io" + "os" + "sync" +) + +type captureData struct { + backupStdout *os.File + writerStdout *os.File + backupStderr *os.File + writerStderr *os.File + + DataStdOut *bytes.Buffer + DataStdErr *bytes.Buffer + + outStdout chan []byte + outStderr chan []byte +} + +func startStdCapture() *captureData { + rStdout, wStdout, errStdout := os.Pipe() + if errStdout != nil { + panic(errStdout) + } + + rStderr, wStderr, errStderr := os.Pipe() + if errStderr != nil { + panic(errStderr) + } + + c := &captureData{ + backupStdout: os.Stdout, + writerStdout: wStdout, + + backupStderr: os.Stderr, + writerStderr: wStderr, + + outStdout: make(chan []byte), + outStderr: make(chan []byte), + + DataStdOut: &bytes.Buffer{}, + DataStdErr: &bytes.Buffer{}, + } + + os.Stdout = c.writerStdout + os.Stderr = c.writerStderr + + stdCopy := func(out chan<- []byte, reader *os.File) { + var buffer bytes.Buffer + _, _ = io.Copy(&buffer, reader) + if buffer.Len() > 0 { + out <- buffer.Bytes() + } + close(out) + } + + go stdCopy(c.outStdout, rStdout) + go stdCopy(c.outStderr, rStderr) + + return c +} + +func stopStdCapture(c *captureData) { + _ = c.writerStdout.Close() + _ = c.writerStderr.Close() + + var wg sync.WaitGroup + + stdRead := func(in <-chan []byte, outData *bytes.Buffer) { + defer wg.Done() + + for { + out, more := <-in + if more { + outData.Write(out) + } else { + return + } + } + } + + wg.Add(2) + go stdRead(c.outStdout, c.DataStdOut) + go stdRead(c.outStderr, c.DataStdErr) + wg.Wait() + + os.Stdout = c.backupStdout + os.Stderr = c.backupStderr +} diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 558a06f5..b313e845 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -17,6 +17,7 @@ import ( "github.com/d5/tengo/v2/stdlib" "github.com/karrick/godirwalk" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/internal/progress" "github.com/projectdiscovery/nuclei/v2/pkg/atomicboolean" "github.com/projectdiscovery/nuclei/v2/pkg/executer" "github.com/projectdiscovery/nuclei/v2/pkg/requests" @@ -26,6 +27,9 @@ import ( // Runner is a client for running the enumeration process. type Runner struct { + input string + inputCount int64 + // output is the output file to write if any output *os.File outputMutex *sync.Mutex @@ -35,6 +39,9 @@ type Runner struct { // options contains configuration options for runner options *Options limiter chan struct{} + + // progress tracking + progress *progress.Progress } // New creates a new client for running enumeration process. @@ -74,6 +81,46 @@ func New(options *Options) (*Runner, error) { tempInput.Close() } + // Setup input, handle a list of hosts as argument + var err error + var input *os.File + if options.Targets != "" { + input, err = os.Open(options.Targets) + } else if options.Stdin || options.Target != "" { + input, err = os.Open(runner.tempFile) + } + if err != nil { + gologger.Fatalf("Could not open targets file '%s': %s\n", options.Targets, err) + } + + // Sanitize input and pre-compute total number of targets + var usedInput = make(map[string]bool) + dupeCount := 0 + sb := strings.Builder{} + scanner := bufio.NewScanner(input) + runner.inputCount = 0 + for scanner.Scan() { + url := scanner.Text() + // skip empty lines + if len(url) == 0 { + continue + } + // deduplication + if _, ok := usedInput[url]; !ok { + usedInput[url] = true + runner.inputCount++ + sb.WriteString(url) + sb.WriteString("\n") + } else { + dupeCount++ + } + } + input.Close() + runner.input = sb.String() + if dupeCount > 0 { + gologger.Labelf("Supplied input was automatically deduplicated (%d removed).", dupeCount) + } + // Create the output file if asked if options.Output != "" { output, err := os.Create(options.Output) @@ -83,6 +130,9 @@ func New(options *Options) (*Runner, error) { runner.output = output } + // Creates the progress tracking object + runner.progress = progress.NewProgress(runner.options.NoColor) + runner.limiter = make(chan struct{}, options.Threads) return runner, nil @@ -230,35 +280,78 @@ func (r *Runner) RunEnumeration() { gologger.Fatalf("Error, no templates were found.\n") } + // progress tracking + p := r.progress + + // precompute total request count + var totalRequests int64 = 0 + hasWorkflows := false + parsedTemplates := []string{} + + for _, match := range allTemplates { + t, err := r.parse(match) + switch t.(type) { + case *templates.Template: + template := t.(*templates.Template) + totalRequests += (template.GetHTTPRequestCount() + template.GetDNSRequestCount()) * r.inputCount + parsedTemplates = append(parsedTemplates, match) + case *workflows.Workflow: + // workflows will dynamically adjust the totals while running, as + // it can't be know in advance which requests will be called + parsedTemplates = append(parsedTemplates, match) + hasWorkflows = true + default: + gologger.Errorf("Could not parse file '%s': %s\n", match, err) + } + } + + // ensure only successfully parsed templates are processed + allTemplates = parsedTemplates + templateCount := len(allTemplates) + var ( wgtemplates sync.WaitGroup results atomicboolean.AtomBool ) - for _, match := range allTemplates { - wgtemplates.Add(1) - go func(match string) { - defer wgtemplates.Done() - t, err := r.parse(match) - switch t.(type) { - case *templates.Template: - template := t.(*templates.Template) - for _, request := range template.RequestsDNS { - results.Or(r.processTemplateRequest(template, request)) - } - for _, request := range template.BulkRequestsHTTP { - results.Or(r.processTemplateRequest(template, request)) - } - case *workflows.Workflow: - workflow := t.(*workflows.Workflow) - r.ProcessWorkflowWithList(workflow) - default: - gologger.Errorf("Could not parse file '%s': %s\n", match, err) - } - }(match) - } + if r.inputCount == 0 { + gologger.Errorf("Could not find any valid input URLs.") + } else if totalRequests > 0 || hasWorkflows { - wgtemplates.Wait() + // track global progress + p.InitProgressbar(r.inputCount, templateCount, totalRequests) + p.StartStdCapture() + + for _, match := range allTemplates { + wgtemplates.Add(1) + go func(match string) { + defer wgtemplates.Done() + t, err := r.parse(match) + switch t.(type) { + case *templates.Template: + template := t.(*templates.Template) + for _, request := range template.RequestsDNS { + results.Or(r.processTemplateWithList(p, template, request)) + } + for _, request := range template.BulkRequestsHTTP { + results.Or(r.processTemplateWithList(p, template, request)) + } + case *workflows.Workflow: + workflow := t.(*workflows.Workflow) + r.ProcessWorkflowWithList(p, workflow) + default: + gologger.Errorf("Could not parse file '%s': %s\n", match, err) + } + }(match) + } + + wgtemplates.Wait() + p.Wait() + + p.StopStdCapture() + p.ShowStdErr() + p.ShowStdOut() + } if !results.Get() { if r.output != nil { @@ -271,27 +364,8 @@ func (r *Runner) RunEnumeration() { return } -// processTemplate processes a template and runs the enumeration on all the targets -func (r *Runner) processTemplateRequest(template *templates.Template, request interface{}) bool { - var file *os.File - var err error - - // Handle a list of hosts as argument - if r.options.Targets != "" { - file, err = os.Open(r.options.Targets) - } else if r.options.Stdin || r.options.Target != "" { - file, err = os.Open(r.tempFile) - } - if err != nil { - gologger.Fatalf("Could not open targets file '%s': %s\n", r.options.Targets, err) - } - defer file.Close() - - return r.processTemplateWithList(template, request, file) -} - -// processDomain processes the list with a template -func (r *Runner) processTemplateWithList(template *templates.Template, request interface{}, reader io.Reader) bool { +// processTemplateWithList processes a template and runs the enumeration on all the targets +func (r *Runner) processTemplateWithList(p *progress.Progress, template *templates.Template, request interface{}) bool { // Display the message for the template message := fmt.Sprintf("[%s] Loaded template %s (@%s)", template.ID, template.Info.Name, template.Info.Author) if template.Info.Severity != "" { @@ -336,6 +410,7 @@ func (r *Runner) processTemplateWithList(template *templates.Template, request i }) } if err != nil { + p.Drop(request.(*requests.BulkHTTPRequest).GetRequestCount()) gologger.Warningf("Could not create http client: %s\n", err) return false } @@ -343,12 +418,10 @@ func (r *Runner) processTemplateWithList(template *templates.Template, request i var globalresult atomicboolean.AtomBool var wg sync.WaitGroup - scanner := bufio.NewScanner(reader) + + scanner := bufio.NewScanner(strings.NewReader(r.input)) for scanner.Scan() { text := scanner.Text() - if text == "" { - continue - } r.limiter <- struct{}{} wg.Add(1) @@ -358,11 +431,11 @@ func (r *Runner) processTemplateWithList(template *templates.Template, request i var result executer.Result if httpExecuter != nil { - result = httpExecuter.ExecuteHTTP(URL) + result = httpExecuter.ExecuteHTTP(p, URL) globalresult.Or(result.GotResults) } if dnsExecuter != nil { - result = dnsExecuter.ExecuteDNS(URL) + result = dnsExecuter.ExecuteDNS(p, URL) globalresult.Or(result.GotResults) } if result.Error != nil { @@ -379,35 +452,18 @@ func (r *Runner) processTemplateWithList(template *templates.Template, request i } // ProcessWorkflowWithList coming from stdin or list of targets -func (r *Runner) ProcessWorkflowWithList(workflow *workflows.Workflow) { - var file *os.File - var err error - // Handle a list of hosts as argument - if r.options.Targets != "" { - file, err = os.Open(r.options.Targets) - } else if r.options.Stdin || r.options.Target != "" { - file, err = os.Open(r.tempFile) - } - if err != nil { - gologger.Fatalf("Could not open targets file '%s': %s\n", r.options.Targets, err) - } - defer file.Close() - +func (r *Runner) ProcessWorkflowWithList(p *progress.Progress, workflow *workflows.Workflow) { var wg sync.WaitGroup - scanner := bufio.NewScanner(file) + scanner := bufio.NewScanner(strings.NewReader(r.input)) for scanner.Scan() { text := scanner.Text() - if text == "" { - continue - } - r.limiter <- struct{}{} wg.Add(1) go func(URL string) { defer wg.Done() - if err := r.ProcessWorkflow(workflow, text); err != nil { + if err := r.ProcessWorkflow(p, workflow, text); err != nil { gologger.Warningf("Could not run workflow for %s: %s\n", text, err) } <-r.limiter @@ -418,7 +474,7 @@ func (r *Runner) ProcessWorkflowWithList(workflow *workflows.Workflow) { } // ProcessWorkflow towards an URL -func (r *Runner) ProcessWorkflow(workflow *workflows.Workflow, URL string) error { +func (r *Runner) ProcessWorkflow(p *progress.Progress, workflow *workflows.Workflow, URL string) error { script := tengo.NewScript([]byte(workflow.Logic)) script.SetImports(stdlib.GetModuleMap(stdlib.AllModuleNames()...)) var jar *cookiejar.Jar @@ -456,7 +512,7 @@ func (r *Runner) ProcessWorkflow(workflow *workflows.Workflow, URL string) error if err != nil { return err } - template := &workflows.Template{} + template := &workflows.Template{Progress: p} if len(t.BulkRequestsHTTP) > 0 { template.HTTPOptions = &executer.HTTPOptions{ Debug: r.options.Debug, @@ -507,7 +563,7 @@ func (r *Runner) ProcessWorkflow(workflow *workflows.Workflow, URL string) error if err != nil { return err } - template := &workflows.Template{} + template := &workflows.Template{Progress: p} if len(t.BulkRequestsHTTP) > 0 { template.HTTPOptions = &executer.HTTPOptions{ Debug: r.options.Debug, diff --git a/v2/pkg/executer/executer_dns.go b/v2/pkg/executer/executer_dns.go index 4c9a3474..9334a2c5 100644 --- a/v2/pkg/executer/executer_dns.go +++ b/v2/pkg/executer/executer_dns.go @@ -3,6 +3,7 @@ package executer import ( "bufio" "fmt" + "github.com/projectdiscovery/nuclei/v2/internal/progress" "os" "sync" @@ -62,7 +63,7 @@ func NewDNSExecuter(options *DNSOptions) *DNSExecuter { } // ExecuteDNS executes the DNS request on a URL -func (e *DNSExecuter) ExecuteDNS(URL string) (result Result) { +func (e *DNSExecuter) ExecuteDNS(p *progress.Progress, URL string) (result Result) { // Parse the URL and return domain if URL. var domain string if isURL(URL) { @@ -75,6 +76,7 @@ func (e *DNSExecuter) ExecuteDNS(URL string) (result Result) { compiledRequest, err := e.dnsRequest.MakeDNSRequest(domain) if err != nil { result.Error = errors.Wrap(err, "could not make dns request") + p.Drop(1) return } @@ -87,9 +89,12 @@ func (e *DNSExecuter) ExecuteDNS(URL string) (result Result) { resp, err := e.dnsClient.Do(compiledRequest) if err != nil { result.Error = errors.Wrap(err, "could not send dns request") + p.Drop(1) return } + p.Update() + gologger.Verbosef("Sent DNS request to %s\n", "dns-request", URL) if e.debug { diff --git a/v2/pkg/executer/executer_http.go b/v2/pkg/executer/executer_http.go index 8824dc60..20c309bb 100644 --- a/v2/pkg/executer/executer_http.go +++ b/v2/pkg/executer/executer_http.go @@ -17,6 +17,7 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/internal/progress" "github.com/projectdiscovery/nuclei/v2/pkg/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/requests" "github.com/projectdiscovery/nuclei/v2/pkg/templates" @@ -100,7 +101,7 @@ func NewHTTPExecuter(options *HTTPOptions) (*HTTPExecuter, error) { } // ExecuteHTTP executes the HTTP request on a URL -func (e *HTTPExecuter) ExecuteHTTP(URL string) (result Result) { +func (e *HTTPExecuter) ExecuteHTTP(p *progress.Progress, URL string) (result Result) { result.Matches = make(map[string]interface{}) result.Extractions = make(map[string]interface{}) dynamicvalues := make(map[string]interface{}) @@ -110,21 +111,27 @@ func (e *HTTPExecuter) ExecuteHTTP(URL string) (result Result) { return } + remaining := e.bulkHttpRequest.GetRequestCount() + e.bulkHttpRequest.CreateGenerator(URL) for e.bulkHttpRequest.Next(URL) && !result.Done { httpRequest, err := e.bulkHttpRequest.MakeHTTPRequest(URL, dynamicvalues, e.bulkHttpRequest.Current(URL)) if err != nil { - result.Error = errors.Wrap(err, "could not make http request") + result.Error = errors.Wrap(err, "could not build http request") + p.Drop(remaining) return } - err = e.handleHTTP(URL, httpRequest, dynamicvalues, &result) + err = e.handleHTTP(p, URL, httpRequest, dynamicvalues, &result) if err != nil { - result.Error = errors.Wrap(err, "could not make http request") + result.Error = errors.Wrap(err, "could not handle http request") + p.Drop(remaining) return } e.bulkHttpRequest.Increment(URL) + p.Update() + remaining-- } gologger.Verbosef("Sent HTTP request to %s\n", "http-request", URL) @@ -132,16 +139,16 @@ func (e *HTTPExecuter) ExecuteHTTP(URL string) (result Result) { return } -func (e *HTTPExecuter) handleHTTP(URL string, request *requests.HttpRequest, dynamicvalues map[string]interface{}, result *Result) error { +func (e *HTTPExecuter) handleHTTP(p *progress.Progress, URL string, request *requests.HttpRequest, dynamicvalues map[string]interface{}, result *Result) error { e.setCustomHeaders(request) req := request.Request if e.debug { - gologger.Infof("Dumped HTTP request for %s (%s)\n\n", URL, e.template.ID) dumpedRequest, err := httputil.DumpRequest(req.Request, true) if err != nil { return errors.Wrap(err, "could not make http request") } + gologger.Infof("Dumped HTTP request for %s (%s)\n\n", URL, e.template.ID) fmt.Fprintf(os.Stderr, "%s", string(dumpedRequest)) } resp, err := e.httpClient.Do(req) @@ -153,11 +160,11 @@ func (e *HTTPExecuter) handleHTTP(URL string, request *requests.HttpRequest, dyn } if e.debug { - gologger.Infof("Dumped HTTP response for %s (%s)\n\n", URL, e.template.ID) dumpedResponse, err := httputil.DumpResponse(resp, true) if err != nil { return errors.Wrap(err, "could not dump http response") } + gologger.Infof("Dumped HTTP response for %s (%s)\n\n", URL, e.template.ID) fmt.Fprintf(os.Stderr, "%s\n", string(dumpedResponse)) } diff --git a/v2/pkg/requests/bulk-http-request.go b/v2/pkg/requests/bulk-http-request.go index a85e4ab6..2e5cbe20 100644 --- a/v2/pkg/requests/bulk-http-request.go +++ b/v2/pkg/requests/bulk-http-request.go @@ -76,6 +76,11 @@ func (r *BulkHTTPRequest) SetAttackType(attack generators.Type) { r.attackType = attack } +// Returns the total number of requests the YAML rule will perform +func (r *BulkHTTPRequest) GetRequestCount() int64 { + return int64(len(r.Raw) | len(r.Path)) +} + func (r *BulkHTTPRequest) MakeHTTPRequest(baseURL string, dynamicValues map[string]interface{}, data string) (*HttpRequest, error) { parsed, err := url.Parse(baseURL) if err != nil { diff --git a/v2/pkg/requests/dns-request.go b/v2/pkg/requests/dns-request.go index ba695c2d..1cb1b3fa 100644 --- a/v2/pkg/requests/dns-request.go +++ b/v2/pkg/requests/dns-request.go @@ -42,6 +42,11 @@ func (r *DNSRequest) SetMatchersCondition(condition matchers.ConditionType) { r.matchersCondition = condition } +// Returns the total number of requests the YAML rule will perform +func (r *DNSRequest) GetRequestCount() int64 { + return 1 +} + // MakeDNSRequest creates a *dns.Request from a request template func (r *DNSRequest) MakeDNSRequest(domain string) (*dns.Msg, error) { domain = dns.Fqdn(domain) diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index 95db3abe..6842f359 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -27,3 +27,19 @@ type Info struct { // Description optionally describes the template. Description string `yaml:"description,omitempty"` } + +func (t* Template) GetHTTPRequestCount() int64 { + var count int64 = 0 + for _, request := range t.BulkRequestsHTTP { + count += request.GetRequestCount() + } + return count +} + +func (t *Template) GetDNSRequestCount() int64 { + var count int64 = 0 + for _, request := range t.RequestsDNS { + count += request.GetRequestCount() + } + return count +} \ No newline at end of file diff --git a/v2/pkg/workflows/var.go b/v2/pkg/workflows/var.go index 1f5138ab..77758393 100644 --- a/v2/pkg/workflows/var.go +++ b/v2/pkg/workflows/var.go @@ -5,6 +5,7 @@ import ( tengo "github.com/d5/tengo/v2" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/internal/progress" "github.com/projectdiscovery/nuclei/v2/pkg/atomicboolean" "github.com/projectdiscovery/nuclei/v2/pkg/executer" "github.com/projectdiscovery/nuclei/v2/pkg/generators" @@ -23,6 +24,7 @@ type NucleiVar struct { type Template struct { HTTPOptions *executer.HTTPOptions DNSOptions *executer.DNSOptions + Progress *progress.Progress } // TypeName of the variable @@ -53,7 +55,9 @@ func (n *NucleiVar) Call(args ...tengo.Object) (ret tengo.Object, err error) { var gotResult atomicboolean.AtomBool for _, template := range n.Templates { + p := template.Progress if template.HTTPOptions != nil { + p.AddToTotal(template.HTTPOptions.Template.GetHTTPRequestCount()) for _, request := range template.HTTPOptions.Template.BulkRequestsHTTP { // apply externally supplied payloads if any request.Headers = generators.MergeMapsWithStrings(request.Headers, headers) @@ -62,10 +66,11 @@ func (n *NucleiVar) Call(args ...tengo.Object) (ret tengo.Object, err error) { template.HTTPOptions.BulkHttpRequest = request httpExecuter, err := executer.NewHTTPExecuter(template.HTTPOptions) if err != nil { + p.Drop(request.GetRequestCount()) gologger.Warningf("Could not compile request for template '%s': %s\n", template.HTTPOptions.Template.ID, err) continue } - result := httpExecuter.ExecuteHTTP(n.URL) + result := httpExecuter.ExecuteHTTP(p, n.URL) if result.Error != nil { gologger.Warningf("Could not send request for template '%s': %s\n", template.HTTPOptions.Template.ID, result.Error) continue @@ -79,10 +84,11 @@ func (n *NucleiVar) Call(args ...tengo.Object) (ret tengo.Object, err error) { } if template.DNSOptions != nil { + p.AddToTotal(template.DNSOptions.Template.GetDNSRequestCount()) for _, request := range template.DNSOptions.Template.RequestsDNS { template.DNSOptions.DNSRequest = request dnsExecuter := executer.NewDNSExecuter(template.DNSOptions) - result := dnsExecuter.ExecuteDNS(n.URL) + result := dnsExecuter.ExecuteDNS(p, n.URL) if result.Error != nil { gologger.Warningf("Could not compile request for template '%s': %s\n", template.HTTPOptions.Template.ID, result.Error) continue