daffainfo
/
nuclei
mirror of https://github.com/daffainfo/nuclei.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add additional json fields: port,ip,scheme,url (#4417) 

* add additional json fields: port,ip,scheme,url

* include host field in case of ip input
dev
Tarun Koyalwar 2023-11-28 14:26:23 +05:30 committed by GitHub
parent ce5df9cc02
commit 6e969cbd3c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 183 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
pkg/output/output.go
View File

@ -25,6 +25,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/model"
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v3/pkg/operators"
protocolUtils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/nuclei/v3/pkg/utils"
fileutil "github.com/projectdiscovery/utils/file"
@ -62,7 +63,7 @@ type StandardWriter struct {
severityColors func(severity.Severity) string
storeResponse bool
storeResponseDir string
omitTemplate bool
omitTemplate bool
DisableStdout bool
AddNewLinesOutputFile bool // by default this is only done for stdout
}
@ -132,6 +133,12 @@ type ResultEvent struct {
Type string `json:"type"`
// Host is the host input on which match was found.
Host string `json:"host,omitempty"`
// Port is port of the host input on which match was found (if applicable).
Port string `json:"port,omitempty"`
// Scheme is the scheme of the host input on which match was found (if applicable).
Scheme string `json:"scheme,omitempty"`
// URL is the Base URL of the host input on which match was found (if applicable).
URL string `json:"url,omitempty"`
// Path is the path input on which match was found.
Path string `json:"path,omitempty"`
// Matched contains the matched input in its transformed form.
@ -347,6 +354,14 @@ func (w *StandardWriter) WriteFailure(wrappedEvent *InternalWrappedEvent) error
if event["template-info"] != nil {
templateInfo = event["template-info"].(model.Info)
}
fields := protocolUtils.GetJsonFieldsFromURL(types.ToString(event["host"]))
if types.ToString(event["ip"]) != "" {
fields.Ip = types.ToString(event["ip"])
}
if types.ToString(event["path"]) != "" {
fields.Path = types.ToString(event["path"])
}
data := &ResultEvent{
Template: templatePath,
TemplateURL: templateURL,
@ -354,7 +369,12 @@ func (w *StandardWriter) WriteFailure(wrappedEvent *InternalWrappedEvent) error
TemplatePath: types.ToString(event["template-path"]),
Info: templateInfo,
Type: types.ToString(event["type"]),
Host: types.ToString(event["host"]),
Host: fields.Host,
Path: fields.Path,
Port: fields.Port,
Scheme: fields.Scheme,
URL: fields.URL,
IP: fields.Ip,
Request: types.ToString(event["request"]),
Response: types.ToString(event["response"]),
MatcherStatus: false,

9
pkg/protocols/code/code.go
View File

@ -244,12 +244,21 @@ func (request *Request) Type() templateTypes.ProtocolType {
}
func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
fields := protocolutils.GetJsonFieldsFromURL(types.ToString(wrapped.InternalEvent["input"]))
if types.ToString(wrapped.InternalEvent["ip"]) != "" {
fields.Ip = types.ToString(wrapped.InternalEvent["ip"])
}
data := &output.ResultEvent{
TemplateID: types.ToString(request.options.TemplateID),
TemplatePath: types.ToString(request.options.TemplatePath),
Info: request.options.TemplateInfo,
Type: types.ToString(wrapped.InternalEvent["type"]),
Matched: types.ToString(wrapped.InternalEvent["input"]),
Host: fields.Host,
Port: fields.Port,
Scheme: fields.Scheme,
URL: fields.URL,
IP: fields.Ip,
Metadata: wrapped.OperatorsResult.PayloadValues,
ExtractedResults: wrapped.OperatorsResult.OutputExtracts,
Timestamp: time.Now(),

16
pkg/protocols/headless/operators.go
View File

@ -10,6 +10,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
protocolUtils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
)
@ -125,17 +126,28 @@ func (request *Request) GetCompiledOperators() []*operators.Operators {
}
func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
fields := protocolUtils.GetJsonFieldsFromURL(types.ToString(wrapped.InternalEvent["host"]))
if types.ToString(wrapped.InternalEvent["ip"]) != "" {
fields.Ip = types.ToString(wrapped.InternalEvent["ip"])
}
if types.ToString(wrapped.InternalEvent["path"]) != "" {
fields.Path = types.ToString(wrapped.InternalEvent["path"])
}
data := &output.ResultEvent{
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),
Info: wrapped.InternalEvent["template-info"].(model.Info),
Type: types.ToString(wrapped.InternalEvent["type"]),
Host: types.ToString(wrapped.InternalEvent["host"]),
Host: fields.Host,
Path: fields.Path,
Port: fields.Port,
Scheme: fields.Scheme,
URL: fields.URL,
Matched: types.ToString(wrapped.InternalEvent["matched"]),
ExtractedResults: wrapped.OperatorsResult.OutputExtracts,
Timestamp: time.Now(),
MatcherStatus: true,
IP: types.ToString(wrapped.InternalEvent["ip"]),
IP: fields.Ip,
Request: types.ToString(wrapped.InternalEvent["request"]),
Response: types.ToString(wrapped.InternalEvent["data"]),
TemplateEncoded: request.options.EncodeTemplate(),

15
pkg/protocols/http/operators.go
View File

@ -149,18 +149,29 @@ func (request *Request) GetCompiledOperators() []*operators.Operators {
}
func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
fields := utils.GetJsonFieldsFromURL(types.ToString(wrapped.InternalEvent["host"]))
if types.ToString(wrapped.InternalEvent["ip"]) != "" {
fields.Ip = types.ToString(wrapped.InternalEvent["ip"])
}
if types.ToString(wrapped.InternalEvent["path"]) != "" {
fields.Path = types.ToString(wrapped.InternalEvent["path"])
}
data := &output.ResultEvent{
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),
Info: wrapped.InternalEvent["template-info"].(model.Info),
Type: types.ToString(wrapped.InternalEvent["type"]),
Host: types.ToString(wrapped.InternalEvent["host"]),
Host: fields.Host,
Port: fields.Port,
Scheme: fields.Scheme,
URL: fields.URL,
Path: fields.Path,
Matched: types.ToString(wrapped.InternalEvent["matched"]),
Metadata: wrapped.OperatorsResult.PayloadValues,
ExtractedResults: wrapped.OperatorsResult.OutputExtracts,
Timestamp: time.Now(),
MatcherStatus: true,
IP: types.ToString(wrapped.InternalEvent["ip"]),
IP: fields.Ip,
Request: types.ToString(wrapped.InternalEvent["request"]),
Response: request.truncateResponse(wrapped.InternalEvent["response"]),
CURLCommand: types.ToString(wrapped.InternalEvent["curl-command"]),

10
pkg/protocols/javascript/js.go
View File

@ -620,12 +620,18 @@ func (request *Request) getExcludePorts() string {
}
func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
fields := protocolutils.GetJsonFieldsFromURL(types.ToString(wrapped.InternalEvent["host"]))
if types.ToString(wrapped.InternalEvent["ip"]) != "" {
fields.Ip = types.ToString(wrapped.InternalEvent["ip"])
}
data := &output.ResultEvent{
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),
Info: wrapped.InternalEvent["template-info"].(model.Info),
Type: types.ToString(wrapped.InternalEvent["type"]),
Host: types.ToString(wrapped.InternalEvent["host"]),
Host: fields.Host,
Port: fields.Port,
URL: fields.URL,
Matched: types.ToString(wrapped.InternalEvent["matched"]),
Metadata: wrapped.OperatorsResult.PayloadValues,
ExtractedResults: wrapped.OperatorsResult.OutputExtracts,
@ -633,7 +639,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
MatcherStatus: true,
Request: types.ToString(wrapped.InternalEvent["request"]),
Response: types.ToString(wrapped.InternalEvent["response"]),
IP: types.ToString(wrapped.InternalEvent["ip"]),
IP: fields.Ip,
TemplateEncoded: request.options.EncodeTemplate(),
Error: types.ToString(wrapped.InternalEvent["error"]),
}

19
pkg/protocols/network/operators.go
View File

@ -9,6 +9,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
)
@ -94,18 +95,24 @@ func (request *Request) GetCompiledOperators() []*operators.Operators {
}
func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
fields := protocolutils.GetJsonFieldsFromURL(types.ToString(wrapped.InternalEvent["host"]))
if types.ToString(wrapped.InternalEvent["ip"]) != "" {
fields.Ip = types.ToString(wrapped.InternalEvent["ip"])
}
data := &output.ResultEvent{
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),
Info: wrapped.InternalEvent["template-info"].(model.Info),
Type: types.ToString(wrapped.InternalEvent["type"]),
Host: types.ToString(wrapped.InternalEvent["host"]),
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),
Info: wrapped.InternalEvent["template-info"].(model.Info),
Type: types.ToString(wrapped.InternalEvent["type"]),
Host: fields.Host,
Port: fields.Port,
URL: fields.URL,
Matched: types.ToString(wrapped.InternalEvent["matched"]),
ExtractedResults: wrapped.OperatorsResult.OutputExtracts,
Metadata: wrapped.OperatorsResult.PayloadValues,
Timestamp: time.Now(),
MatcherStatus: true,
IP: types.ToString(wrapped.InternalEvent["ip"]),
IP: fields.Ip,
Request: types.ToString(wrapped.InternalEvent["request"]),
Response: types.ToString(wrapped.InternalEvent["data"]),
TemplateEncoded: request.options.EncodeTemplate(),

14
pkg/protocols/ssl/ssl.go
View File

@ -390,18 +390,28 @@ func (request *Request) Type() templateTypes.ProtocolType {
}
func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
fields := protocolutils.GetJsonFieldsFromURL(types.ToString(wrapped.InternalEvent["host"]))
if types.ToString(wrapped.InternalEvent["ip"]) != "" {
fields.Ip = types.ToString(wrapped.InternalEvent["ip"])
}
// in case scheme is not specified , we only connect to port 443 unless custom https port was specified
// like 8443 etc
if fields.Port == "80" {
fields.Port = "443"
}
data := &output.ResultEvent{
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),
Info: wrapped.InternalEvent["template-info"].(model.Info),
Type: types.ToString(wrapped.InternalEvent["type"]),
Host: types.ToString(wrapped.InternalEvent["host"]),
Host: fields.Host,
Port: fields.Port,
Matched: types.ToString(wrapped.InternalEvent["matched"]),
Metadata: wrapped.OperatorsResult.PayloadValues,
ExtractedResults: wrapped.OperatorsResult.OutputExtracts,
Timestamp: time.Now(),
MatcherStatus: true,
IP: types.ToString(wrapped.InternalEvent["ip"]),
IP: fields.Ip,
TemplateEncoded: request.options.EncodeTemplate(),
Error: types.ToString(wrapped.InternalEvent["error"]),
}

71
pkg/protocols/utils/fields.go Normal file
View File

@ -0,0 +1,71 @@
package utils
import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
iputil "github.com/projectdiscovery/utils/ip"
urlutil "github.com/projectdiscovery/utils/url"
)
// JsonFields contains additional metadata fields for JSON output
type JsonFields struct {
Host string `json:"host,omitempty"`
Path string `json:"path,omitempty"`
Port string `json:"port,omitempty"`
Ip string `json:"ip,omitempty"`
Scheme string `json:"scheme,omitempty"`
URL string `json:"url,omitempty"`
}
// GetJsonFields returns the json fields for the request
func GetJsonFieldsFromURL(URL string) JsonFields {
parsed, err := urlutil.Parse(URL)
if err != nil {
return JsonFields{}
}
fields := JsonFields{
Port: parsed.Port(),
Scheme: parsed.Scheme,
URL: parsed.String(),
Path: parsed.Path,
}
if fields.Port == "" {
fields.Port = "80"
if fields.Scheme == "https" {
fields.Port = "443"
}
}
if iputil.IsIP(parsed.Host) {
fields.Ip = parsed.Host
}
fields.Host = parsed.Host
return fields
}
// GetJsonFieldsFromMetaInput returns the json fields for the request
func GetJsonFieldsFromMetaInput(ctx *contextargs.MetaInput) JsonFields {
input := ctx.Input
fields := JsonFields{
Ip: ctx.CustomIP,
}
parsed, err := urlutil.Parse(input)
if err != nil {
return fields
}
fields.Port = parsed.Port()
fields.Scheme = parsed.Scheme
fields.URL = parsed.String()
fields.Path = parsed.Path
if fields.Port == "" {
fields.Port = "80"
if fields.Scheme == "https" {
fields.Port = "443"
}
}
if iputil.IsIP(parsed.Host) {
fields.Ip = parsed.Host
}
fields.Host = parsed.Host
return fields
}

9
pkg/protocols/websocket/websocket.go
View File

@ -395,18 +395,23 @@ var RequestPartDefinitions = map[string]string{
}
func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
fields := protocolutils.GetJsonFieldsFromURL(types.ToString(wrapped.InternalEvent["host"]))
if types.ToString(wrapped.InternalEvent["ip"]) != "" {
fields.Ip = types.ToString(wrapped.InternalEvent["ip"])
}
data := &output.ResultEvent{
TemplateID: types.ToString(request.options.TemplateID),
TemplatePath: types.ToString(request.options.TemplatePath),
Info: request.options.TemplateInfo,
Type: types.ToString(wrapped.InternalEvent["type"]),
Host: types.ToString(wrapped.InternalEvent["host"]),
Host: fields.Host,
Port: fields.Port,
Matched: types.ToString(wrapped.InternalEvent["matched"]),
Metadata: wrapped.OperatorsResult.PayloadValues,
ExtractedResults: wrapped.OperatorsResult.OutputExtracts,
Timestamp: time.Now(),
MatcherStatus: true,
IP: types.ToString(wrapped.InternalEvent["ip"]),
IP: fields.Ip,
Request: types.ToString(wrapped.InternalEvent["request"]),
Response: types.ToString(wrapped.InternalEvent["response"]),
TemplateEncoded: request.options.EncodeTemplate(),

15
pkg/testutils/testutils.go
View File

@ -20,6 +20,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/progress"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit"
protocolUtils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/nuclei/v3/pkg/utils"
)
@ -166,6 +167,13 @@ func (m *MockOutputWriter) WriteFailure(wrappedEvent *output.InternalWrappedEven
if ti, ok := event["template-info"].(model.Info); ok {
templateInfo = ti
}
fields := protocolUtils.GetJsonFieldsFromURL(types.ToString(event["host"]))
if types.ToString(event["ip"]) != "" {
fields.Ip = types.ToString(event["ip"])
}
if types.ToString(event["path"]) != "" {
fields.Path = types.ToString(event["path"])
}
data := &output.ResultEvent{
Template: templatePath,
TemplateURL: templateURL,
@ -173,7 +181,12 @@ func (m *MockOutputWriter) WriteFailure(wrappedEvent *output.InternalWrappedEven
TemplatePath: types.ToString(event["template-path"]),
Info: templateInfo,
Type: types.ToString(event["type"]),
Host: types.ToString(event["host"]),
Path: fields.Path,
Host: fields.Host,
Port: fields.Port,
Scheme: fields.Scheme,
URL: fields.URL,
IP: fields.Ip,
Request: types.ToString(event["request"]),
Response: types.ToString(event["response"]),
MatcherStatus: false,