Experimental live results output with active progressbar

See 333809f3d6 for an early implementation of this.

v2/internal/progress/progress.go

@@ -4,8 +4,12 @@ import (
 	"fmt"
 	"github.com/logrusorgru/aurora"
 	"github.com/vbauerster/mpb/v5"
+	"github.com/vbauerster/mpb/v5/cwriter"
 	"github.com/vbauerster/mpb/v5/decor"
+	"io"
 	"os"
+	"regexp"
+	"strconv"
 	"strings"
 	"sync"
 )
@@ -22,10 +26,17 @@ type Progress struct {
 	stdout          *strings.Builder
 	stderr          *strings.Builder
 	colorizer       aurora.Aurora
+	termWidth int
 }
 
 // Creates and returns a new progress tracking object.
 func NewProgress(noColor bool) *Progress {
+	w := cwriter.New(os.Stdout)
+	tw, err := w.GetWidth()
+	if err != nil {
+		panic("Couldn't determine available terminal width.")
+	}
+
 	p := &Progress{
 		progress: mpb.New(
 			mpb.WithOutput(os.Stderr),
@@ -36,6 +47,7 @@ func NewProgress(noColor bool) *Progress {
 		stdout:          &strings.Builder{},
 		stderr:          &strings.Builder{},
 		colorizer:       aurora.NewAurora(!noColor),
+		termWidth: tw,
 	}
 	return p
 }
@@ -136,19 +148,38 @@ func (p *Progress) StopStdCapture() {
 	stopStdCapture(p.captureData)
 	p.stdout.Write(p.captureData.DataStdOut.Bytes())
 	p.stderr.Write(p.captureData.DataStdErr.Bytes())
+
+	//
+	var r = regexp.MustCompile("(.{" + strconv.Itoa(p.termWidth) + "})")
+	multiline := r.ReplaceAllString(p.stdout.String(), "$1\n")
+	arr := strings.Split(multiline, "\n")
+	for _, msg := range arr {
+		if len(msg) > 0 {
+			p.progress.Add(0, makeLogBar(msg)).SetTotal(0, true)
+		}
+	}
+	p.stdout.Reset()
+	//
+
 	p.stdCaptureMutex.Unlock()
 }
 
 // Writes the captured stdout data to stdout, if any.
 func (p *Progress) ShowStdOut() {
-	if p.stdout.Len() > 0 {
-		fmt.Fprint(os.Stdout, p.stdout.String())
-	}
+	//if p.stdout.Len() > 0 {
+	//	fmt.Fprint(os.Stdout, p.stdout.String())
+	//}
 }
 
 // Writes the captured stderr data to stderr, if any.
 func (p *Progress) ShowStdErr() {
-	if p.stderr.Len() > 0 {
-		fmt.Fprint(os.Stderr, p.stderr.String())
-	}
+	//if p.stderr.Len() > 0 {
+	//	fmt.Fprint(os.Stderr, p.stderr.String())
+	//}
 }
+
+func makeLogBar(msg string) mpb.BarFiller {
+	return mpb.BarFillerFunc(func(w io.Writer, _ int, st decor.Statistics) {
+		fmt.Fprintf(w, msg)
+	})
+}

v2/internal/runner/runner.go

@@ -341,7 +341,6 @@ func (r *Runner) RunEnumeration() {
 		// track global progress
 		if p != nil {
 			p.InitProgressbar(r.inputCount, templateCount, totalRequests)
-			p.StartStdCapture()
 		}
 
 		for _, match := range allTemplates {
@@ -372,7 +371,6 @@ func (r *Runner) RunEnumeration() {
 		if p != nil {
 			p.Wait()
 
-			p.StopStdCapture()
 			p.ShowStdErr()
 			p.ShowStdOut()
 		}
@@ -396,7 +394,9 @@ func (r *Runner) processTemplateWithList(p *progress.Progress, template *templat
 	if template.Info.Severity != "" {
 		message += " [" + template.Info.Severity + "]"
 	}
+	p.StartStdCapture()
 	gologger.Infof("%s\n", message)
+	p.StopStdCapture()
 
 	var writer *bufio.Writer
 	if r.output != nil {
@@ -444,7 +444,9 @@ func (r *Runner) processTemplateWithList(p *progress.Progress, template *templat
 		if p != nil {
 			p.Drop(request.(*requests.BulkHTTPRequest).GetRequestCount())
 		}
+		p.StartStdCapture()
 		gologger.Warningf("Could not create http client: %s\n", err)
+		p.StopStdCapture()
 		return false
 	}
 
@@ -472,7 +474,9 @@ func (r *Runner) processTemplateWithList(p *progress.Progress, template *templat
 				globalresult.Or(result.GotResults)
 			}
 			if result.Error != nil {
+				p.StartStdCapture()
 				gologger.Warningf("Could not execute step: %s\n", result.Error)
+				p.StopStdCapture()
 			}
 			<-r.limiter
 		}(text)
@@ -497,7 +501,9 @@ func (r *Runner) ProcessWorkflowWithList(p *progress.Progress, workflow *workflo
 			defer wg.Done()
 
 			if err := r.ProcessWorkflow(p, workflow, text); err != nil {
+				p.StartStdCapture()
 				gologger.Warningf("Could not run workflow for %s: %s\n", text, err)
+				p.StopStdCapture()
 			}
 			<-r.limiter
 		}(text)
@@ -528,7 +534,9 @@ func (r *Runner) ProcessWorkflow(p *progress.Progress, workflow *workflows.Workf
 		// Check if the template is an absolute path or relative path.
 		// If the path is absolute, use it. Otherwise,
 		if r.isRelative(value) {
+			p.StartStdCapture()
 			newPath, err := r.resolvePath(value)
+			p.StopStdCapture()
 			if err != nil {
 				newPath, err = r.resolvePathWithBaseFolder(filepath.Dir(workflow.GetPath()), value)
 				if err != nil {

v2/pkg/executer/executer_dns.go

@@ -96,8 +96,10 @@ func (e *DNSExecuter) ExecuteDNS(p *progress.Progress, URL string) (result Resul
 	}
 
 	if e.debug {
+		p.StartStdCapture()
 		gologger.Infof("Dumped DNS request for %s (%s)\n\n", URL, e.template.ID)
 		fmt.Fprintf(os.Stderr, "%s\n", compiledRequest.String())
+		p.StopStdCapture()
 	}
 
 	// Send the request to the target servers
@@ -117,8 +119,10 @@ func (e *DNSExecuter) ExecuteDNS(p *progress.Progress, URL string) (result Resul
 	gologger.Verbosef("Sent DNS request to %s\n", "dns-request", URL)
 
 	if e.debug {
+		p.StartStdCapture()
 		gologger.Infof("Dumped DNS response for %s (%s)\n\n", URL, e.template.ID)
 		fmt.Fprintf(os.Stderr, "%s\n", resp.String())
+		p.StopStdCapture()
 	}
 
 	matcherCondition := e.dnsRequest.GetMatchersCondition()
@@ -133,7 +137,9 @@ func (e *DNSExecuter) ExecuteDNS(p *progress.Progress, URL string) (result Resul
 			// If the matcher has matched, and its an OR
 			// write the first output then move to next matcher.
 			if matcherCondition == matchers.ORCondition && len(e.dnsRequest.Extractors) == 0 {
+				p.StartStdCapture()
 				e.writeOutputDNS(domain, matcher, nil)
+				p.StopStdCapture()
 				result.GotResults = true
 			}
 		}
@@ -153,7 +159,9 @@ func (e *DNSExecuter) ExecuteDNS(p *progress.Progress, URL string) (result Resul
 	// Write a final string of output if matcher type is
 	// AND or if we have extractors for the mechanism too.
 	if len(e.dnsRequest.Extractors) > 0 || matcherCondition == matchers.ANDCondition {
+		p.StartStdCapture()
 		e.writeOutputDNS(domain, nil, extractorResults)
+		p.StopStdCapture()
 	}
 
 	return

v2/pkg/executer/executer_http.go

@@ -152,7 +152,9 @@ func (e *HTTPExecuter) ExecuteHTTP(p *progress.Progress, URL string) (result Res
 		remaining--
 	}
 
+	p.StartStdCapture()
 	gologger.Verbosef("Sent HTTP request to %s\n", "http-request", URL)
+	p.StopStdCapture()
 
 	return
 }
@@ -166,8 +168,10 @@ func (e *HTTPExecuter) handleHTTP(p *progress.Progress, URL string, request *req
 		if err != nil {
 			return errors.Wrap(err, "could not make http request")
 		}
+		p.StartStdCapture()
 		gologger.Infof("Dumped HTTP request for %s (%s)\n\n", URL, e.template.ID)
 		fmt.Fprintf(os.Stderr, "%s", string(dumpedRequest))
+		p.StopStdCapture()
 	}
 	resp, err := e.httpClient.Do(req)
 	if err != nil {
@@ -182,8 +186,10 @@ func (e *HTTPExecuter) handleHTTP(p *progress.Progress, URL string, request *req
 		if err != nil {
 			return errors.Wrap(err, "could not dump http response")
 		}
+		p.StartStdCapture()
 		gologger.Infof("Dumped HTTP response for %s (%s)\n\n", URL, e.template.ID)
 		fmt.Fprintf(os.Stderr, "%s\n", string(dumpedResponse))
+		p.StopStdCapture()
 	}
 
 	data, err := ioutil.ReadAll(resp.Body)
@@ -220,7 +226,9 @@ func (e *HTTPExecuter) handleHTTP(p *progress.Progress, URL string, request *req
 				result.Matches[matcher.Name] = nil
 				// probably redundant but ensures we snapshot current payload values when matchers are valid
 				result.Meta = request.Meta
+				p.StartStdCapture()
 				e.writeOutputHTTP(request, resp, body, matcher, nil)
+				p.StopStdCapture()
 				result.GotResults = true
 			}
 		}
@@ -247,7 +255,9 @@ func (e *HTTPExecuter) handleHTTP(p *progress.Progress, URL string, request *req
 	// Write a final string of output if matcher type is
 	// AND or if we have extractors for the mechanism too.
 	if len(outputExtractorResults) > 0 || matcherCondition == matchers.ANDCondition {
+		p.StartStdCapture()
 		e.writeOutputHTTP(request, resp, body, nil, outputExtractorResults)
+		p.StopStdCapture()
 		result.GotResults = true
 	}
 

v2/pkg/workflows/var.go

@@ -71,12 +71,16 @@ func (n *NucleiVar) Call(args ...tengo.Object) (ret tengo.Object, err error) {
 					if p != nil {
 						p.Drop(request.GetRequestCount())
 					}
+					p.StartStdCapture()
 					gologger.Warningf("Could not compile request for template '%s': %s\n", template.HTTPOptions.Template.ID, err)
+					p.StopStdCapture()
 					continue
 				}
 				result := httpExecuter.ExecuteHTTP(p, n.URL)
 				if result.Error != nil {
+					p.StartStdCapture()
 					gologger.Warningf("Could not send request for template '%s': %s\n", template.HTTPOptions.Template.ID, result.Error)
+					p.StopStdCapture()
 					continue
 				}
 
@@ -96,7 +100,9 @@ func (n *NucleiVar) Call(args ...tengo.Object) (ret tengo.Object, err error) {
 				dnsExecuter := executer.NewDNSExecuter(template.DNSOptions)
 				result := dnsExecuter.ExecuteDNS(p, n.URL)
 				if result.Error != nil {
+					p.StartStdCapture()
 					gologger.Warningf("Could not compile request for template '%s': %s\n", template.HTTPOptions.Template.ID, result.Error)
+					p.StopStdCapture()
 					continue
 				}