daffainfo
/
nuclei
mirror of https://github.com/daffainfo/nuclei.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Exclude Raw Request Payloads (#3710) 

* Add command docs and CLI hook

* Add configurable exclusion from reports

* Register the CLI argument with exporter configuration

* Switch to inverted logic with JSONRequest flag

* Switch variable name for the -include-rr/-irr flag

* Remove flags from README

* Update call for -irr and -or

* convert -irr to no-op

---------

Co-authored-by: Tarun Koyalwar <tarun@projectdiscovery.io>
dev
Keith Chason 2023-07-04 16:37:56 -04:00 committed by GitHub
parent 1eb4c7c80c
commit b3ccb9a6e5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 95 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
README.md
View File

@ -154,7 +154,8 @@ OUTPUT:
-silent display findings only
-nc, -no-color disable output content coloring (ANSI escape codes)
-j, -jsonl write output in JSONL(ines) format
-irr, -include-rr include request/response pairs in the JSONL output (for findings only)
-irr, -include-rr include request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only) [DEPRECATED]
-or, -omit-raw omit request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only)
-nm, -no-meta disable printing result metadata in cli output
-ts, -timestamp enables printing timestamp in cli output
-rdb, -report-db string nuclei reporting database (always use this to persist report data)

1
README_CN.md
View File

@ -134,6 +134,7 @@ Nuclei是一款注重于可配置性、可扩展性和易用性的基于模板
-nc, -no-color 禁用输出内容着色（ANSI转义码）
-j, -jsonl 输出为jsonL（ines）
-irr, -include-rr 在JSONL中输出对应的请求和相应（仅结果）
-or, -omit-raw
-nm, -no-meta 不显示匹配的元数据
-nts, -no-timestamp 不在输出中显示时间戳
-rdb, -report-db string 本地的Nuclei结果数据库（始终使用该数据库保存结果）

3
README_ID.md
View File

@ -133,7 +133,8 @@ OUTPUT:
-silent display findings only
-nc, -no-color disable output content coloring (ANSI escape codes)
-j, -jsonl write output in JSONL(ines) format
-irr, -include-rr include request/response pairs in the JSONL output (for findings only)
-irr, -include-rr include request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only) [DEPRECATED]
-or, -omit-raw omit request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only)
-nm, -no-meta disable printing result metadata in cli output
-nts, -no-timestamp disable printing timestamp in cli output
-rdb, -report-db string nuclei reporting database (always use this to persist report data)

1
README_KR.md
View File

@ -130,6 +130,7 @@ OUTPUT:
-nc, -no-color 출력 내용 색상 비활성화 (ANSI escape codes)
-j, -jsonl JSONL(ines) 형식으로 출력
-irr, -include-rr JSONL 출력에 요청/응답 쌍 포함(결과만)
-or, -omit-raw
-nm, -no-meta cli 출력에서 결과 메타데이터 출력 비활성화
-nts, -no-timestamp cli 출력에서 결과 타임스탬프 출력 비활성화
-rdb, -report-db string nuclei 보고 데이터베이스(보고서 데이터를 유지하려면 항상 이것을 사용)

3
v2/cmd/nuclei/main.go
View File

@ -170,7 +170,8 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.BoolVar(&options.Silent, "silent", false, "display findings only"),
flagSet.BoolVarP(&options.NoColor, "no-color", "nc", false, "disable output content coloring (ANSI escape codes)"),
flagSet.BoolVarP(&options.JSONL, "jsonl", "j", false, "write output in JSONL(ines) format"),
flagSet.BoolVarP(&options.JSONRequests, "include-rr", "irr", false, "include request/response pairs in the JSONL output (for findings only)"),
flagSet.BoolVarP(&options.JSONRequests, "include-rr", "irr", true, "include request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only) [DEPRECATED use `-omit-raw`]"),
flagSet.BoolVarP(&options.OmitRawRequests, "omit-raw", "or", false, "omit request/response pairs in the JSON, JSONL, and Markdown outputs (for findings only)"),
flagSet.BoolVarP(&options.NoMeta, "no-meta", "nm", false, "disable printing result metadata in cli output"),
flagSet.BoolVarP(&options.Timestamp, "timestamp", "ts", false, "enables printing timestamp in cli output"),
flagSet.StringVarP(&options.ReportingDB, "report-db", "rdb", "", "nuclei reporting database (always use this to persist report data)"),

30
v2/internal/runner/runner.go
View File

@ -340,10 +340,16 @@ func createReportingOptions(options *types.Options) (*reporting.Options, error)
}
if options.MarkdownExportDirectory != "" {
if reportingOptions != nil {
reportingOptions.MarkdownExporter = &markdown.Options{Directory: options.MarkdownExportDirectory}
reportingOptions.MarkdownExporter = &markdown.Options{
Directory: options.MarkdownExportDirectory,
IncludeRawPayload: !options.OmitRawRequests,
}
} else {
reportingOptions = &reporting.Options{}
reportingOptions.MarkdownExporter = &markdown.Options{Directory: options.MarkdownExportDirectory}
reportingOptions.MarkdownExporter = &markdown.Options{
Directory: options.MarkdownExportDirectory,
IncludeRawPayload: !options.OmitRawRequests,
}
}
}
if options.SarifExport != "" {
@ -356,18 +362,30 @@ func createReportingOptions(options *types.Options) (*reporting.Options, error)
}
if options.JSONExport != "" {
if reportingOptions != nil {
reportingOptions.JSONExporter = &jsonexporter.Options{File: options.JSONExport}
reportingOptions.JSONExporter = &jsonexporter.Options{
File: options.JSONExport,
IncludeRawPayload: !options.OmitRawRequests,
}
} else {
reportingOptions = &reporting.Options{}
reportingOptions.JSONExporter = &jsonexporter.Options{File: options.JSONExport}
reportingOptions.JSONExporter = &jsonexporter.Options{
File: options.JSONExport,
IncludeRawPayload: !options.OmitRawRequests,
}
}
}
if options.JSONLExport != "" {
if reportingOptions != nil {
reportingOptions.JSONLExporter = &jsonl.Options{File: options.JSONLExport}
reportingOptions.JSONLExporter = &jsonl.Options{
File: options.JSONLExport,
IncludeRawPayload: !options.OmitRawRequests,
}
} else {
reportingOptions = &reporting.Options{}
reportingOptions.JSONLExporter = &jsonl.Options{File: options.JSONLExport}
reportingOptions.JSONLExporter = &jsonl.Options{
File: options.JSONLExport,
IncludeRawPayload: !options.OmitRawRequests,
}
}
}

52
v2/pkg/output/output.go
View File

@ -45,19 +45,19 @@ type Writer interface {
// StandardWriter is a writer writing output to file and screen for results.
type StandardWriter struct {
json bool
jsonReqResp bool
timestamp bool
noMetadata bool
matcherStatus bool
mutex *sync.Mutex
aurora aurora.Aurora
outputFile io.WriteCloser
traceFile io.WriteCloser
errorFile io.WriteCloser
severityColors func(severity.Severity) string
storeResponse bool
storeResponseDir string
json bool
jsonReqResp bool
timestamp bool
noMetadata bool
matcherStatus bool
mutex *sync.Mutex
aurora aurora.Aurora
outputFile io.WriteCloser
traceFile io.WriteCloser
errorFile io.WriteCloser
severityColors func(severity.Severity) string
storeResponse bool
storeResponseDir string
}
var decolorizerRegex = regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`)
@ -189,19 +189,19 @@ func NewStandardWriter(options *types.Options) (*StandardWriter, error) {
}
writer := &StandardWriter{
json: options.JSONL,
jsonReqResp: options.JSONRequests,
noMetadata: options.NoMeta,
matcherStatus: options.MatcherStatus,
timestamp: options.Timestamp,
aurora: auroraColorizer,
mutex: &sync.Mutex{},
outputFile: outputFile,
traceFile: traceOutput,
errorFile: errorOutput,
severityColors: colorizer.New(auroraColorizer),
storeResponse: options.StoreResponse,
storeResponseDir: options.StoreResponseDir,
json: options.JSONL,
jsonReqResp: !options.OmitRawRequests,
noMetadata: options.NoMeta,
matcherStatus: options.MatcherStatus,
timestamp: options.Timestamp,
aurora: auroraColorizer,
mutex: &sync.Mutex{},
outputFile: outputFile,
traceFile: traceOutput,
errorFile: errorOutput,
severityColors: colorizer.New(auroraColorizer),
storeResponse: options.StoreResponse,
storeResponseDir: options.StoreResponseDir,
}
return writer, nil
}

12
v2/pkg/reporting/exporters/jsonexporter/jsonexporter.go
View File

@ -17,7 +17,8 @@ type Exporter struct {
// Options contains the configuration options for JSON exporter client
type Options struct {
// File is the file to export found JSON result to
File string `yaml:"file"`
File string `yaml:"file"`
IncludeRawPayload bool `yaml:"include-raw-payload"`
}
// New creates a new JSON exporter integration client based on options.
@ -36,6 +37,15 @@ func (exporter *Exporter) Export(event *output.ResultEvent) error {
exporter.mutex.Lock()
defer exporter.mutex.Unlock()
// If the IncludeRawPayload is not set, then set the request and response to an empty string in the event to avoid
// writing them to the list of events.
// This will reduce the amount of storage as well as the fields being excluded from the resulting JSON output since
// the property is set to "omitempty"
if !exporter.options.IncludeRawPayload {
event.Request = ""
event.Response = ""
}
// Add the event to the rows
exporter.rows = append(exporter.rows, *event)

12
v2/pkg/reporting/exporters/jsonl/jsonl.go
View File

@ -17,7 +17,8 @@ type Exporter struct {
// Options contains the configuration options for JSONL exporter client
type Options struct {
// File is the file to export found JSONL result to
File string `yaml:"file"`
File string `yaml:"file"`
IncludeRawPayload bool `yaml:"include-raw-payload"`
}
// New creates a new JSONL exporter integration client based on options.
@ -36,6 +37,15 @@ func (exporter *Exporter) Export(event *output.ResultEvent) error {
exporter.mutex.Lock()
defer exporter.mutex.Unlock()
// If the IncludeRawPayload is not set, then set the request and response to an empty string in the event to avoid
// writing them to the list of events.
// This will reduce the amount of storage as well as the fields being excluded from the resulting JSONL output since
// the property is set to "omitempty"
if !exporter.options.IncludeRawPayload {
event.Request = ""
event.Response = ""
}
// Add the event to the rows
exporter.rows = append(exporter.rows, *event)

12
v2/pkg/reporting/exporters/markdown/markdown.go
View File

@ -23,7 +23,8 @@ type Exporter struct {
// Options contains the configuration options for GitHub issue tracker client
type Options struct {
// Directory is the directory to export found results to
Directory string `yaml:"directory"`
Directory string `yaml:"directory"`
IncludeRawPayload bool `yaml:"include-raw-payload"`
}
// New creates a new markdown exporter integration client based on options.
@ -51,6 +52,15 @@ func New(options *Options) (*Exporter, error) {
// Export exports a passed result event to markdown
func (exporter *Exporter) Export(event *output.ResultEvent) error {
// If the IncludeRawPayload is not set, then set the request and response to an empty string in the event to avoid
// writing them to the list of events.
// This will reduce the amount of storage as well as the fields being excluded from the markdown report output since
// the property is set to "omitempty"
if !exporter.options.IncludeRawPayload {
event.Request = ""
event.Response = ""
}
// index file generation
file, err := os.OpenFile(filepath.Join(exporter.directory, indexFileName), os.O_APPEND|os.O_WRONLY, 0644)
if err != nil {

2
v2/pkg/testutils/testutils.go
View File

@ -36,7 +36,7 @@ var DefaultOptions = &types.Options{
NoColor: true,
UpdateTemplates: false,
JSONL: false,
JSONRequests: false,
OmitRawRequests: false,
EnableProgressBar: false,
TemplateList: false,
Stdin: false,

3
v2/pkg/types/types.go
View File

@ -231,7 +231,10 @@ type Options struct {
// JSON writes json line output to files
JSONL bool
// JSONRequests writes requests/responses for matches in JSON output
// Deprecated: use OmitRawRequests instead as of now JSONRequests(include raw requests) is always true
JSONRequests bool
// OmitRawRequests omits requests/responses for matches in JSON output
OmitRawRequests bool
// JSONExport is the file to export JSON output format to
JSONExport string
// JSONLExport is the file to export JSONL output format to