Added timestamp optional flag + user-agent to probing (#2962)

* Added timestamp optional flag + user-agent to probing

* fix typo

* misc update

Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com>

README.md

@@ -139,7 +139,7 @@ OUTPUT:
    -json                         write output in JSONL(ines) format
    -irr, -include-rr             include request/response pairs in the JSONL output (for findings only)
    -nm, -no-meta                 disable printing result metadata in cli output
-   -nts, -no-timestamp           disable printing timestamp in cli output
+   -ts, -timestamp               enable printing timestamp in cli output
    -rdb, -report-db string       nuclei reporting database (always use this to persist report data)
    -ms, -matcher-status          display match failure status
    -me, -markdown-export string  directory to export results in markdown format

v2/cmd/nuclei/main.go

@@ -171,7 +171,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.BoolVar(&options.JSON, "json", false, "write output in JSONL(ines) format"),
 		flagSet.BoolVarP(&options.JSONRequests, "include-rr", "irr", false, "include request/response pairs in the JSONL output (for findings only)"),
 		flagSet.BoolVarP(&options.NoMeta, "no-meta", "nm", false, "disable printing result metadata in cli output"),
-		flagSet.BoolVarP(&options.NoTimestamp, "no-timestamp", "nts", false, "disable printing timestamp in cli output"),
+		flagSet.BoolVarP(&options.Timestamp, "timestamp", "ts", false, "enables printing timestamp in cli output"),
 		flagSet.StringVarP(&options.ReportingDB, "report-db", "rdb", "", "nuclei reporting database (always use this to persist report data)"),
 		flagSet.BoolVarP(&options.MatcherStatus, "matcher-status", "ms", false, "display match failure status"),
 		flagSet.StringVarP(&options.MarkdownExportDirectory, "markdown-export", "me", "", "directory to export results in markdown format"),

v2/internal/runner/inputs.go

@@ -3,9 +3,11 @@ package runner
 import (
 	"fmt"
 	"io"
+	"net/http"
 	"strings"
 	"sync/atomic"
 
+	"github.com/corpix/uarand"
 	"github.com/pkg/errors"
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/hmap/store/hybrid"
@@ -71,7 +73,13 @@ var (
 func probeURL(input string, httpclient *retryablehttp.Client) string {
 	for _, scheme := range httpSchemes {
 		formedURL := fmt.Sprintf("%s://%s", scheme, input)
-		resp, err := httpclient.Get(formedURL)
+		req, err := retryablehttp.NewRequest(http.MethodGet, formedURL, nil)
+		if err != nil {
+			continue
+		}
+		req.Header.Set("User-Agent", uarand.GetRandom())
+
+		resp, err := httpclient.Do(req)
 		if resp != nil {
 			_, _ = io.CopyN(io.Discard, resp.Body, drainReqSize)
 			resp.Body.Close()

v2/internal/runner/runner.go

@@ -186,7 +186,7 @@ func New(options *types.Options) (*Runner, error) {
 	runner.hmapInputProvider = hmapInput
 
 	// Create the output file if asked
-	outputWriter, err := output.NewStandardWriter(!options.NoColor, options.NoMeta, options.NoTimestamp, options.JSON, options.JSONRequests, options.MatcherStatus, options.StoreResponse, options.Output, options.TraceLogFile, options.ErrorLogFile, options.StoreResponseDir)
+	outputWriter, err := output.NewStandardWriter(!options.NoColor, options.NoMeta, options.Timestamp, options.JSON, options.JSONRequests, options.MatcherStatus, options.StoreResponse, options.Output, options.TraceLogFile, options.ErrorLogFile, options.StoreResponseDir)
 	if err != nil {
 		return nil, errors.Wrap(err, "could not create output file")
 	}

v2/pkg/output/format_screen.go

@@ -12,7 +12,7 @@ func (w *StandardWriter) formatScreen(output *ResultEvent) []byte {
 	builder := &bytes.Buffer{}
 
 	if !w.noMetadata {
-		if !w.noTimestamp {
+		if w.timestamp {
 			builder.WriteRune('[')
 			builder.WriteString(w.aurora.Cyan(output.Timestamp.Format("2006-01-02 15:04:05")).String())
 			builder.WriteString("] ")

v2/pkg/output/output.go

@@ -46,7 +46,7 @@ type Writer interface {
 type StandardWriter struct {
 	json             bool
 	jsonReqResp      bool
-	noTimestamp      bool
+	timestamp        bool
 	noMetadata       bool
 	matcherStatus    bool
 	mutex            *sync.Mutex
@@ -123,7 +123,7 @@ type ResultEvent struct {
 }
 
 // NewStandardWriter creates a new output writer based on user configurations
-func NewStandardWriter(colors, noMetadata, noTimestamp, json, jsonReqResp, MatcherStatus, storeResponse bool, file, traceFile string, errorFile string, storeResponseDir string) (*StandardWriter, error) {
+func NewStandardWriter(colors, noMetadata, timestamp, json, jsonReqResp, MatcherStatus, storeResponse bool, file, traceFile string, errorFile string, storeResponseDir string) (*StandardWriter, error) {
 	auroraColorizer := aurora.NewAurora(colors)
 
 	var outputFile io.WriteCloser
@@ -161,7 +161,7 @@ func NewStandardWriter(colors, noMetadata, noTimestamp, json, jsonReqResp, Match
 		jsonReqResp:      jsonReqResp,
 		noMetadata:       noMetadata,
 		matcherStatus:    MatcherStatus,
-		noTimestamp:      noTimestamp,
+		timestamp:        timestamp,
 		aurora:           auroraColorizer,
 		mutex:            &sync.Mutex{},
 		outputFile:       outputFile,

v2/pkg/protocols/http/fuzz/parts.go

@@ -68,8 +68,6 @@ func (rule *Rule) buildQueryInput(input *ExecuteRuleInput, parsed url.URL, inter
 			return err
 		}
 		req.Header.Set("User-Agent", uarand.GetRandom())
-		req.Header.Set("Accept", "*/*")
-		req.Header.Set("Accept-Language", "en")
 	} else {
 		req = input.BaseRequest.Clone(context.Background())
 		req.URL = &parsed

v2/pkg/types/types.go

@@ -216,8 +216,8 @@ type Options struct {
 	Stream bool
 	// NoMeta disables display of metadata for the matches
 	NoMeta bool
-	// NoTimestamp disables display of timestamp for the matcher
+	// Timestamp enables display of timestamp for the matcher
-	NoTimestamp bool
+	Timestamp bool
 	// Project is used to avoid sending same HTTP request multiple times
 	Project bool
 	// NewTemplates only runs newly added templates from the repository