Exclude Raw Request Payloads (#3710)

* Add command docs and CLI hook

* Add configurable exclusion from reports

* Register the CLI argument with exporter configuration

* Switch to inverted logic with JSONRequest flag

* Switch variable name for the -include-rr/-irr flag

* Remove flags from README

* Update call for -irr and -or

* convert -irr to no-op

---------

Co-authored-by: Tarun Koyalwar <tarun@projectdiscovery.io>

README.md

@@ -154,7 +154,8 @@ OUTPUT:
    -silent                       display findings only
    -nc, -no-color                disable output content coloring (ANSI escape codes)
    -j, -jsonl                    write output in JSONL(ines) format
-   -irr, -include-rr             include request/response pairs in the JSONL output (for findings only)
+   -irr, -include-rr             include request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only) [DEPRECATED]
+   -or, -omit-raw                omit request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only)
    -nm, -no-meta                 disable printing result metadata in cli output
    -ts, -timestamp               enables printing timestamp in cli output
    -rdb, -report-db string       nuclei reporting database (always use this to persist report data)

README_CN.md

@@ -134,6 +134,7 @@ Nuclei是一款注重于可配置性、可扩展性和易用性的基于模板
    -nc, -no-color                        禁用输出内容着色（ANSI转义码）
    -j, -jsonl                            输出为jsonL（ines）
    -irr, -include-rr                     在JSONL中输出对应的请求和相应（仅结果）
+   -or, -omit-raw                     
    -nm, -no-meta                         不显示匹配的元数据
    -nts, -no-timestamp                   不在输出中显示时间戳
    -rdb, -report-db string               本地的Nuclei结果数据库（始终使用该数据库保存结果）

README_ID.md

@@ -133,7 +133,8 @@ OUTPUT:
    -silent                       display findings only
    -nc, -no-color                disable output content coloring (ANSI escape codes)
    -j, -jsonl                    write output in JSONL(ines) format
-   -irr, -include-rr             include request/response pairs in the JSONL output (for findings only)
+   -irr, -include-rr             include request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only) [DEPRECATED]
+   -or, -omit-raw                omit request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only)
    -nm, -no-meta                 disable printing result metadata in cli output
    -nts, -no-timestamp           disable printing timestamp in cli output
    -rdb, -report-db string       nuclei reporting database (always use this to persist report data)

README_KR.md

@@ -130,6 +130,7 @@ OUTPUT:
    -nc, -no-color                출력 내용 색상 비활성화 (ANSI escape codes)
    -j, -jsonl                    JSONL(ines) 형식으로 출력
    -irr, -include-rr             JSONL 출력에 요청/응답 쌍 포함(결과만)
+   -or, -omit-raw                
    -nm, -no-meta                 cli 출력에서 결과 메타데이터 출력 비활성화
    -nts, -no-timestamp           cli 출력에서 결과 타임스탬프 출력 비활성화
    -rdb, -report-db string       nuclei 보고 데이터베이스(보고서 데이터를 유지하려면 항상 이것을 사용)

v2/cmd/nuclei/main.go

@@ -170,7 +170,8 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.BoolVar(&options.Silent, "silent", false, "display findings only"),
 		flagSet.BoolVarP(&options.NoColor, "no-color", "nc", false, "disable output content coloring (ANSI escape codes)"),
 		flagSet.BoolVarP(&options.JSONL, "jsonl", "j", false, "write output in JSONL(ines) format"),
-		flagSet.BoolVarP(&options.JSONRequests, "include-rr", "irr", false, "include request/response pairs in the JSONL output (for findings only)"),
+		flagSet.BoolVarP(&options.JSONRequests, "include-rr", "irr", true, "include request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only) [DEPRECATED use `-omit-raw`]"),
+		flagSet.BoolVarP(&options.OmitRawRequests, "omit-raw", "or", false, "omit request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only)"),
 		flagSet.BoolVarP(&options.NoMeta, "no-meta", "nm", false, "disable printing result metadata in cli output"),
 		flagSet.BoolVarP(&options.Timestamp, "timestamp", "ts", false, "enables printing timestamp in cli output"),
 		flagSet.StringVarP(&options.ReportingDB, "report-db", "rdb", "", "nuclei reporting database (always use this to persist report data)"),

v2/internal/runner/runner.go

@@ -340,10 +340,16 @@ func createReportingOptions(options *types.Options) (*reporting.Options, error)
 	}
 	if options.MarkdownExportDirectory != "" {
 		if reportingOptions != nil {
-			reportingOptions.MarkdownExporter = &markdown.Options{Directory: options.MarkdownExportDirectory}
+			reportingOptions.MarkdownExporter = &markdown.Options{
+				Directory:         options.MarkdownExportDirectory,
+				IncludeRawPayload: !options.OmitRawRequests,
+			}
 		} else {
 			reportingOptions = &reporting.Options{}
-			reportingOptions.MarkdownExporter = &markdown.Options{Directory: options.MarkdownExportDirectory}
+			reportingOptions.MarkdownExporter = &markdown.Options{
+				Directory:         options.MarkdownExportDirectory,
+				IncludeRawPayload: !options.OmitRawRequests,
+			}
 		}
 	}
 	if options.SarifExport != "" {
@@ -356,18 +362,30 @@ func createReportingOptions(options *types.Options) (*reporting.Options, error)
 	}
 	if options.JSONExport != "" {
 		if reportingOptions != nil {
-			reportingOptions.JSONExporter = &jsonexporter.Options{File: options.JSONExport}
+			reportingOptions.JSONExporter = &jsonexporter.Options{
+				File:              options.JSONExport,
+				IncludeRawPayload: !options.OmitRawRequests,
+			}
 		} else {
 			reportingOptions = &reporting.Options{}
-			reportingOptions.JSONExporter = &jsonexporter.Options{File: options.JSONExport}
+			reportingOptions.JSONExporter = &jsonexporter.Options{
+				File:              options.JSONExport,
+				IncludeRawPayload: !options.OmitRawRequests,
+			}
 		}
 	}
 	if options.JSONLExport != "" {
 		if reportingOptions != nil {
-			reportingOptions.JSONLExporter = &jsonl.Options{File: options.JSONLExport}
+			reportingOptions.JSONLExporter = &jsonl.Options{
+				File:              options.JSONLExport,
+				IncludeRawPayload: !options.OmitRawRequests,
+			}
 		} else {
 			reportingOptions = &reporting.Options{}
-			reportingOptions.JSONLExporter = &jsonl.Options{File: options.JSONLExport}
+			reportingOptions.JSONLExporter = &jsonl.Options{
+				File:              options.JSONLExport,
+				IncludeRawPayload: !options.OmitRawRequests,
+			}
 		}
 	}
 

v2/pkg/output/output.go

@@ -45,19 +45,19 @@ type Writer interface {
 
 // StandardWriter is a writer writing output to file and screen for results.
 type StandardWriter struct {
-	json                  bool
+	json             bool
-	jsonReqResp           bool
+	jsonReqResp      bool
-	timestamp             bool
+	timestamp        bool
-	noMetadata            bool
+	noMetadata       bool
-	matcherStatus         bool
+	matcherStatus    bool
-	mutex                 *sync.Mutex
+	mutex            *sync.Mutex
-	aurora                aurora.Aurora
+	aurora           aurora.Aurora
-	outputFile            io.WriteCloser
+	outputFile       io.WriteCloser
-	traceFile             io.WriteCloser
+	traceFile        io.WriteCloser
-	errorFile             io.WriteCloser
+	errorFile        io.WriteCloser
-	severityColors        func(severity.Severity) string
+	severityColors   func(severity.Severity) string
-	storeResponse         bool
+	storeResponse    bool
-	storeResponseDir      string
+	storeResponseDir string
 }
 
 var decolorizerRegex = regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`)
@@ -189,19 +189,19 @@ func NewStandardWriter(options *types.Options) (*StandardWriter, error) {
 	}
 
 	writer := &StandardWriter{
-		json:                  options.JSONL,
+		json:             options.JSONL,
-		jsonReqResp:           options.JSONRequests,
+		jsonReqResp:      !options.OmitRawRequests,
-		noMetadata:            options.NoMeta,
+		noMetadata:       options.NoMeta,
-		matcherStatus:         options.MatcherStatus,
+		matcherStatus:    options.MatcherStatus,
-		timestamp:             options.Timestamp,
+		timestamp:        options.Timestamp,
-		aurora:                auroraColorizer,
+		aurora:           auroraColorizer,
-		mutex:                 &sync.Mutex{},
+		mutex:            &sync.Mutex{},
-		outputFile:            outputFile,
+		outputFile:       outputFile,
-		traceFile:             traceOutput,
+		traceFile:        traceOutput,
-		errorFile:             errorOutput,
+		errorFile:        errorOutput,
-		severityColors:        colorizer.New(auroraColorizer),
+		severityColors:   colorizer.New(auroraColorizer),
-		storeResponse:         options.StoreResponse,
+		storeResponse:    options.StoreResponse,
-		storeResponseDir:      options.StoreResponseDir,
+		storeResponseDir: options.StoreResponseDir,
 	}
 	return writer, nil
 }

v2/pkg/reporting/exporters/jsonexporter/jsonexporter.go

@@ -17,7 +17,8 @@ type Exporter struct {
 // Options contains the configuration options for JSON exporter client
 type Options struct {
 	// File is the file to export found JSON result to
-	File string `yaml:"file"`
+	File              string `yaml:"file"`
+	IncludeRawPayload bool   `yaml:"include-raw-payload"`
 }
 
 // New creates a new JSON exporter integration client based on options.
@@ -36,6 +37,15 @@ func (exporter *Exporter) Export(event *output.ResultEvent) error {
 	exporter.mutex.Lock()
 	defer exporter.mutex.Unlock()
 
+	// If the IncludeRawPayload is not set, then set the request and response to an empty string in the event to avoid
+	// writing them to the list of events.
+	// This will reduce the amount of storage as well as the fields being excluded from the resulting JSON output since
+	// the property is set to "omitempty"
+	if !exporter.options.IncludeRawPayload {
+		event.Request = ""
+		event.Response = ""
+	}
+
 	// Add the event to the rows
 	exporter.rows = append(exporter.rows, *event)
 

v2/pkg/reporting/exporters/jsonl/jsonl.go

@@ -17,7 +17,8 @@ type Exporter struct {
 // Options contains the configuration options for JSONL exporter client
 type Options struct {
 	// File is the file to export found JSONL result to
-	File string `yaml:"file"`
+	File              string `yaml:"file"`
+	IncludeRawPayload bool   `yaml:"include-raw-payload"`
 }
 
 // New creates a new JSONL exporter integration client based on options.
@@ -36,6 +37,15 @@ func (exporter *Exporter) Export(event *output.ResultEvent) error {
 	exporter.mutex.Lock()
 	defer exporter.mutex.Unlock()
 
+	// If the IncludeRawPayload is not set, then set the request and response to an empty string in the event to avoid
+	// writing them to the list of events.
+	// This will reduce the amount of storage as well as the fields being excluded from the resulting JSONL output since
+	// the property is set to "omitempty"
+	if !exporter.options.IncludeRawPayload {
+		event.Request = ""
+		event.Response = ""
+	}
+
 	// Add the event to the rows
 	exporter.rows = append(exporter.rows, *event)
 

v2/pkg/reporting/exporters/markdown/markdown.go

@@ -23,7 +23,8 @@ type Exporter struct {
 // Options contains the configuration options for GitHub issue tracker client
 type Options struct {
 	// Directory is the directory to export found results to
-	Directory string `yaml:"directory"`
+	Directory         string `yaml:"directory"`
+	IncludeRawPayload bool   `yaml:"include-raw-payload"`
 }
 
 // New creates a new markdown exporter integration client based on options.
@@ -51,6 +52,15 @@ func New(options *Options) (*Exporter, error) {
 
 // Export exports a passed result event to markdown
 func (exporter *Exporter) Export(event *output.ResultEvent) error {
+	// If the IncludeRawPayload is not set, then set the request and response to an empty string in the event to avoid
+	// writing them to the list of events.
+	// This will reduce the amount of storage as well as the fields being excluded from the markdown report output since
+	// the property is set to "omitempty"
+	if !exporter.options.IncludeRawPayload {
+		event.Request = ""
+		event.Response = ""
+	}
+
 	// index file generation
 	file, err := os.OpenFile(filepath.Join(exporter.directory, indexFileName), os.O_APPEND|os.O_WRONLY, 0644)
 	if err != nil {

v2/pkg/testutils/testutils.go

@@ -36,7 +36,7 @@ var DefaultOptions = &types.Options{
 	NoColor:                    true,
 	UpdateTemplates:            false,
 	JSONL:                      false,
-	JSONRequests:               false,
+	OmitRawRequests:            false,
 	EnableProgressBar:          false,
 	TemplateList:               false,
 	Stdin:                      false,

v2/pkg/types/types.go

@@ -231,7 +231,10 @@ type Options struct {
 	// JSON writes json line output to files
 	JSONL bool
 	// JSONRequests writes requests/responses for matches in JSON output
+	// Deprecated: use OmitRawRequests instead as of now JSONRequests(include raw requests) is always true
 	JSONRequests bool
+	// OmitRawRequests omits requests/responses for matches in JSON output
+	OmitRawRequests bool
 	// JSONExport is the file to export JSON output format to
 	JSONExport string
 	// JSONLExport is the file to export JSONL output format to