Merge pull request #373 from projectdiscovery/added-no-meta-flag

Added -no-meta flag to ignore meta

.golangci.yml

@@ -59,7 +59,6 @@ linters:
     - bodyclose
     - deadcode
     - dogsled
-    - dupl
     - errcheck
     - exhaustive
     - gochecknoinits

v2/internal/runner/options.go

@@ -13,20 +13,6 @@ import (
 // Options contains the configuration options for tuning
 // the template requesting process.
 type Options struct {
-	Templates          multiStringFlag        // Signature specifies the template/templates to use
-	ExcludedTemplates  multiStringFlag        // Signature specifies the template/templates to exclude
-	CustomHeaders      requests.CustomHeaders // Custom global headers
-	Severity           string                 // Filter templates based on their severity and only run the matching ones.
-	Target             string                 // Target is a single URL/Domain to scan usng a template
-	Targets            string                 // Targets specifies the targets to scan using templates.
-	Output             string                 // Output is the file to write found subdomains to.
-	ProxyURL           string                 // ProxyURL is the URL for the proxy server
-	ProxySocksURL      string                 // ProxySocksURL is the URL for the proxy socks server
-	TemplatesDirectory string                 // TemplatesDirectory is the directory to use for storing templates
-	Threads            int                    // Thread controls the number of concurrent requests to make.
-	Timeout            int                    // Timeout is the seconds to wait for a response from the server.
-	Retries            int                    // Retries is the number of times to retry the request
-	RateLimit          int                    // Rate-Limit of requests per specified target
 	Debug              bool                   // Debug mode allows debugging request/responses for the engine
 	Silent             bool                   // Silent suppresses any extra text and only writes found URLs on screen.
 	Version            bool                   // Version specifies if we should just show version and exit
@@ -39,7 +25,22 @@ type Options struct {
 	TemplateList       bool                   // List available templates
 	Stdin              bool                   // Stdin specifies whether stdin input was given to the process
 	StopAtFirstMatch   bool                   // Stop processing template at first full match (this may break chained requests)
+	NoMeta             bool                   // Don't display metadata for the matches
 	BulkSize           int                    // Number of targets analyzed in parallel for each template
+	Threads            int                    // Thread controls the number of concurrent requests to make.
+	Timeout            int                    // Timeout is the seconds to wait for a response from the server.
+	Retries            int                    // Retries is the number of times to retry the request
+	RateLimit          int                    // Rate-Limit of requests per specified target
+	Severity           string                 // Filter templates based on their severity and only run the matching ones.
+	Target             string                 // Target is a single URL/Domain to scan usng a template
+	Targets            string                 // Targets specifies the targets to scan using templates.
+	Output             string                 // Output is the file to write found subdomains to.
+	ProxyURL           string                 // ProxyURL is the URL for the proxy server
+	ProxySocksURL      string                 // ProxySocksURL is the URL for the proxy socks server
+	TemplatesDirectory string                 // TemplatesDirectory is the directory to use for storing templates
+	Templates          multiStringFlag        // Signature specifies the template/templates to use
+	ExcludedTemplates  multiStringFlag        // Signature specifies the template/templates to exclude
+	CustomHeaders      requests.CustomHeaders // Custom global headers
 }
 
 type multiStringFlag []string
@@ -82,7 +83,7 @@ func ParseOptions() *Options {
 	flag.IntVar(&options.RateLimit, "rate-limit", -1, "Per Target Rate-Limit")
 	flag.BoolVar(&options.StopAtFirstMatch, "stop-at-first-match", false, "Stop processing http requests at first match (this may break template/workflow logic)")
 	flag.IntVar(&options.BulkSize, "bulk-size", 150, "Number of hosts analyzed in parallel per template")
-
+	flag.BoolVar(&options.NoMeta, "no-meta", false, "Don't display metadata for the matches")
 	flag.Parse()
 
 	// Check if stdin pipe was given

v2/internal/runner/processor.go

@@ -45,6 +45,7 @@ func (r *Runner) processTemplateWithList(p progress.IProgress, template *templat
 			Writer:        r.output,
 			JSON:          r.options.JSON,
 			JSONRequests:  r.options.JSONRequests,
+			NoMeta:        r.options.NoMeta,
 			ColoredOutput: !r.options.NoColor,
 			Colorizer:     r.colorizer,
 			Decolorizer:   r.decolorizer,
@@ -62,6 +63,7 @@ func (r *Runner) processTemplateWithList(p progress.IProgress, template *templat
 			CustomHeaders:    r.options.CustomHeaders,
 			JSON:             r.options.JSON,
 			JSONRequests:     r.options.JSONRequests,
+			NoMeta:           r.options.NoMeta,
 			CookieReuse:      value.CookieReuse,
 			ColoredOutput:    !r.options.NoColor,
 			Colorizer:        &r.colorizer,

v2/pkg/executer/executer_dns.go

@@ -23,6 +23,7 @@ type DNSExecuter struct {
 	debug         bool
 	jsonOutput    bool
 	jsonRequest   bool
+	noMeta        bool
 	Results       bool
 	dnsClient     *retryabledns.Client
 	template      *templates.Template
@@ -47,6 +48,7 @@ type DNSOptions struct {
 	Debug         bool
 	JSON          bool
 	JSONRequests  bool
+	NoMeta        bool
 	Template      *templates.Template
 	DNSRequest    *requests.DNSRequest
 	Writer        *bufwriter.Writer
@@ -62,6 +64,7 @@ func NewDNSExecuter(options *DNSOptions) *DNSExecuter {
 
 	executer := &DNSExecuter{
 		debug:         options.Debug,
+		noMeta:        options.NoMeta,
 		jsonOutput:    options.JSON,
 		jsonRequest:   options.JSONRequests,
 		dnsClient:     dnsClient,

v2/pkg/executer/executer_http.go

@@ -55,6 +55,7 @@ type HTTPExecuter struct {
 	Results          bool
 	jsonOutput       bool
 	jsonRequest      bool
+	noMeta           bool
 	stopAtFirstMatch bool
 }
 
@@ -74,6 +75,7 @@ type HTTPOptions struct {
 	Debug            bool
 	JSON             bool
 	JSONRequests     bool
+	NoMeta           bool
 	CookieReuse      bool
 	ColoredOutput    bool
 	StopAtFirstMatch bool
@@ -119,6 +121,7 @@ func NewHTTPExecuter(options *HTTPOptions) (*HTTPExecuter, error) {
 		debug:            options.Debug,
 		jsonOutput:       options.JSON,
 		jsonRequest:      options.JSONRequests,
+		noMeta:           options.NoMeta,
 		httpClient:       client,
 		rawHTTPClient:    rawClient,
 		template:         options.Template,

v2/pkg/executer/output_dns.go

@@ -15,21 +15,24 @@ import (
 func (e *DNSExecuter) writeOutputDNS(domain string, req, resp *dns.Msg, matcher *matchers.Matcher, extractorResults []string) {
 	if e.jsonOutput {
 		output := make(jsonOutput)
-		output["template"] = e.template.ID
-		output["type"] = "dns"
 		output["matched"] = domain
-		for k, v := range e.template.Info {
+
-			output[k] = v
+		if !e.noMeta {
-		}
+			output["template"] = e.template.ID
-		if matcher != nil && len(matcher.Name) > 0 {
+			output["type"] = "dns"
-			output["matcher_name"] = matcher.Name
+			for k, v := range e.template.Info {
-		}
+				output[k] = v
-		if len(extractorResults) > 0 {
+			}
-			output["extracted_results"] = extractorResults
+			if matcher != nil && len(matcher.Name) > 0 {
-		}
+				output["matcher_name"] = matcher.Name
-		if e.jsonRequest {
+			}
-			output["request"] = req.String()
+			if len(extractorResults) > 0 {
-			output["response"] = resp.String()
+				output["extracted_results"] = extractorResults
+			}
+			if e.jsonRequest {
+				output["request"] = req.String()
+				output["response"] = resp.String()
+			}
 		}
 
 		data, err := jsoniter.Marshal(output)
@@ -49,28 +52,29 @@ func (e *DNSExecuter) writeOutputDNS(domain string, req, resp *dns.Msg, matcher
 	builder := &strings.Builder{}
 	colorizer := e.colorizer
 
-	builder.WriteRune('[')
+	if !e.noMeta {
-	builder.WriteString(colorizer.Colorizer.BrightGreen(e.template.ID).String())
+		builder.WriteRune('[')
+		builder.WriteString(colorizer.Colorizer.BrightGreen(e.template.ID).String())
 
-	if matcher != nil && len(matcher.Name) > 0 {
+		if matcher != nil && len(matcher.Name) > 0 {
-		builder.WriteString(":")
+			builder.WriteString(":")
-		builder.WriteString(colorizer.Colorizer.BrightGreen(matcher.Name).Bold().String())
+			builder.WriteString(colorizer.Colorizer.BrightGreen(matcher.Name).Bold().String())
-	}
+		}
 
-	builder.WriteString("] [")
+		builder.WriteString("] [")
-	builder.WriteString(colorizer.Colorizer.BrightBlue("dns").String())
+		builder.WriteString(colorizer.Colorizer.BrightBlue("dns").String())
-	builder.WriteString("] ")
-
-	if e.template.Info["severity"] != "" {
-		builder.WriteString("[")
-		builder.WriteString(colorizer.GetColorizedSeverity(e.template.Info["severity"]))
 		builder.WriteString("] ")
-	}
 
+		if e.template.Info["severity"] != "" {
+			builder.WriteString("[")
+			builder.WriteString(colorizer.GetColorizedSeverity(e.template.Info["severity"]))
+			builder.WriteString("] ")
+		}
+	}
 	builder.WriteString(domain)
 
 	// If any extractors, write the results
-	if len(extractorResults) > 0 {
+	if len(extractorResults) > 0 && !e.noMeta {
 		builder.WriteString(" [")
 
 		for i, result := range extractorResults {
@@ -80,10 +84,8 @@ func (e *DNSExecuter) writeOutputDNS(domain string, req, resp *dns.Msg, matcher
 				builder.WriteRune(',')
 			}
 		}
-
 		builder.WriteString("]")
 	}
-
 	builder.WriteRune('\n')
 
 	// Write output to screen as well as any output file

v2/pkg/executer/output_http.go

@@ -14,47 +14,48 @@ import (
 // writeOutputHTTP writes http output to streams
 func (e *HTTPExecuter) writeOutputHTTP(req *requests.HTTPRequest, resp *http.Response, body string, matcher *matchers.Matcher, extractorResults []string, meta map[string]interface{}) {
 	var URL string
-	// rawhttp
 	if req.RawRequest != nil {
 		URL = req.RawRequest.FullURL
 	}
-	// retryablehttp
 	if req.Request != nil {
 		URL = req.Request.URL.String()
 	}
 
 	if e.jsonOutput {
 		output := make(jsonOutput)
-		output["template"] = e.template.ID
-		output["type"] = "http"
-		output["matched"] = URL
-		if len(meta) > 0 {
-			output["meta"] = meta
-		}
-		for k, v := range e.template.Info {
-			output[k] = v
-		}
-		if matcher != nil && len(matcher.Name) > 0 {
-			output["matcher_name"] = matcher.Name
-		}
-		if len(extractorResults) > 0 {
-			output["extracted_results"] = extractorResults
-		}
 
-		// TODO: URL should be an argument
+		output["matched"] = URL
-		if e.jsonRequest {
+		if !e.noMeta {
-			dumpedRequest, err := requests.Dump(req, URL)
+			output["template"] = e.template.ID
-			if err != nil {
+			output["type"] = "http"
-				gologger.Warningf("could not dump request: %s\n", err)
+			if len(meta) > 0 {
-			} else {
+				output["meta"] = meta
-				output["request"] = string(dumpedRequest)
+			}
+			for k, v := range e.template.Info {
+				output[k] = v
+			}
+			if matcher != nil && len(matcher.Name) > 0 {
+				output["matcher_name"] = matcher.Name
+			}
+			if len(extractorResults) > 0 {
+				output["extracted_results"] = extractorResults
 			}
 
-			dumpedResponse, err := httputil.DumpResponse(resp, false)
+			// TODO: URL should be an argument
-			if err != nil {
+			if e.jsonRequest {
-				gologger.Warningf("could not dump response: %s\n", err)
+				dumpedRequest, err := requests.Dump(req, URL)
-			} else {
+				if err != nil {
-				output["response"] = string(dumpedResponse) + body
+					gologger.Warningf("could not dump request: %s\n", err)
+				} else {
+					output["request"] = string(dumpedRequest)
+				}
+
+				dumpedResponse, err := httputil.DumpResponse(resp, false)
+				if err != nil {
+					gologger.Warningf("could not dump response: %s\n", err)
+				} else {
+					output["response"] = string(dumpedResponse) + body
+				}
 			}
 		}
 
@@ -76,28 +77,29 @@ func (e *HTTPExecuter) writeOutputHTTP(req *requests.HTTPRequest, resp *http.Res
 	builder := &strings.Builder{}
 	colorizer := e.colorizer
 
-	builder.WriteRune('[')
+	if !e.noMeta {
-	builder.WriteString(colorizer.Colorizer.BrightGreen(e.template.ID).String())
+		builder.WriteRune('[')
+		builder.WriteString(colorizer.Colorizer.BrightGreen(e.template.ID).String())
 
-	if matcher != nil && len(matcher.Name) > 0 {
+		if matcher != nil && len(matcher.Name) > 0 {
-		builder.WriteString(":")
+			builder.WriteString(":")
-		builder.WriteString(colorizer.Colorizer.BrightGreen(matcher.Name).Bold().String())
+			builder.WriteString(colorizer.Colorizer.BrightGreen(matcher.Name).Bold().String())
-	}
+		}
 
-	builder.WriteString("] [")
+		builder.WriteString("] [")
-	builder.WriteString(colorizer.Colorizer.BrightBlue("http").String())
+		builder.WriteString(colorizer.Colorizer.BrightBlue("http").String())
-	builder.WriteString("] ")
-
-	if e.template.Info["severity"] != "" {
-		builder.WriteString("[")
-		builder.WriteString(colorizer.GetColorizedSeverity(e.template.Info["severity"]))
 		builder.WriteString("] ")
-	}
 
+		if e.template.Info["severity"] != "" {
+			builder.WriteString("[")
+			builder.WriteString(colorizer.GetColorizedSeverity(e.template.Info["severity"]))
+			builder.WriteString("] ")
+		}
+	}
 	builder.WriteString(URL)
 
 	// If any extractors, write the results
-	if len(extractorResults) > 0 {
+	if len(extractorResults) > 0 && !e.noMeta {
 		builder.WriteString(" [")
 
 		for i, result := range extractorResults {
@@ -112,7 +114,7 @@ func (e *HTTPExecuter) writeOutputHTTP(req *requests.HTTPRequest, resp *http.Res
 	}
 
 	// write meta if any
-	if len(req.Meta) > 0 {
+	if len(req.Meta) > 0 && !e.noMeta {
 		builder.WriteString(" [")
 
 		var metas []string