Work on sarif integration start

dev
Ice3man543 2021-06-05 18:01:08 +05:30
parent 849c7a45fc
commit abc7063339
19 changed files with 153 additions and 2 deletions

View File

@ -85,6 +85,7 @@ based on templates offering massive extensibility and ease of use.`)
set.IntVar(&options.PageTimeout, "page-timeout", 20, "Seconds to wait for each page in headless")
set.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "Only run newly added templates")
set.StringVarP(&options.DiskExportDirectory, "disk-export", "de", "", "Directory on disk to export reports in markdown to")
set.StringVarP(&options.SarifFile, "sarif-file", "sf", "", "File to export sarif report to")
set.BoolVar(&options.NoInteractsh, "no-interactsh", false, "Do not use interactsh server for blind interaction polling")
set.StringVar(&options.InteractshURL, "interactsh-url", "https://interact.sh", "Interactsh Server URL")
set.IntVar(&options.InteractionsCacheSize, "interactions-cache-size", 5000, "Number of requests to keep in interactions cache")

View File

@ -23,6 +23,7 @@ require (
github.com/miekg/dns v1.1.38
github.com/mitchellh/go-ps v1.0.0
github.com/olekukonko/tablewriter v0.0.5
github.com/owenrumney/go-sarif v1.0.4 // indirect
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/clistats v0.0.8
github.com/projectdiscovery/collaborator v0.0.2
@ -53,7 +54,6 @@ require (
golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 // indirect
golang.org/x/sys v0.0.0-20210218155724-8ebf48af031b // indirect
golang.org/x/text v0.3.4 // indirect
golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect
google.golang.org/appengine v1.6.7 // indirect
gopkg.in/yaml.v2 v2.4.0

View File

@ -40,6 +40,7 @@ github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF0
github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA=
github.com/andygrunwald/go-jira v1.13.0 h1:vvIImGgX32bHfoiyUwkNo+/YrPnRczNarvhLOncP6dE=
github.com/andygrunwald/go-jira v1.13.0/go.mod h1:jYi4kFDbRPZTJdJOVJO4mpMMIwdB+rcZwSO58DzPd2I=
github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo=
github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
@ -206,6 +207,8 @@ github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/owenrumney/go-sarif v1.0.4 h1:0LFC5eHP6amc/9ajM1jDiE52UfXFcl/oozay+X3KgV4=
github.com/owenrumney/go-sarif v1.0.4/go.mod h1:DXUGbHwQcCMvqcvZbxh8l/7diHsJVztOKZgmPt88RNI=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@ -266,6 +269,8 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4=
github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4=
github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI=
github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ=
github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM=
github.com/xanzy/go-gitlab v0.44.0 h1:cEiGhqu7EpFGuei2a2etAwB+x6403E5CvpLn35y+GPs=
@ -283,6 +288,8 @@ github.com/ysmood/leakless v0.6.12/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNq
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/zclconf/go-cty v1.8.2 h1:u+xZfBKgpycDnTNjPhGiTEYZS5qS/Sb5MqSfm7vzcjg=
github.com/zclconf/go-cty v1.8.2/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
@ -427,6 +434,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc=
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

View File

@ -24,6 +24,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/disk"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/sarif"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
"github.com/remeh/sizedwaitgroup"
@ -95,6 +96,14 @@ func New(options *types.Options) (*Runner, error) {
reportingOptions.DiskExporter = &disk.Options{Directory: options.DiskExportDirectory}
}
}
if options.SarifFile != "" {
if reportingOptions != nil {
reportingOptions.SarifExporter = &sarif.Options{File: options.SarifFile}
} else {
reportingOptions = &reporting.Options{}
reportingOptions.SarifExporter = &sarif.Options{File: options.SarifFile}
}
}
if reportingOptions != nil {
if client, err := reporting.New(reportingOptions, options.ReportingDB); err != nil {
gologger.Fatal().Msgf("Could not create issue reporting client: %s\n", err)

View File

@ -54,6 +54,8 @@ type InternalWrappedEvent struct {
type ResultEvent struct {
// TemplateID is the ID of the template for the result.
TemplateID string `json:"templateID"`
// TemplatePath is the path of template
TemplatePath string `json:"-"`
// Info contains information block of the template for the result.
Info map[string]interface{} `json:"info,inline"`
// MatcherName is the name of the matcher matched if any.

View File

@ -22,6 +22,7 @@ type Executer struct {
type clusteredOperator struct {
templateID string
templatePath string
templateInfo map[string]interface{}
operator *operators.Operators
}
@ -38,6 +39,7 @@ func NewExecuter(requests []*templates.Template, options *protocols.ExecuterOpti
executer.operators = append(executer.operators, &clusteredOperator{
templateID: req.ID,
templateInfo: req.Info,
templatePath: req.Options.TemplatePath,
operator: req.RequestsHTTP[0].CompiledOperators,
})
}
@ -67,6 +69,7 @@ func (e *Executer) Execute(input string) (bool, error) {
if matched && result != nil {
event.OperatorsResult = result
event.InternalEvent["template-id"] = operator.templateID
event.InternalEvent["template-path"] = operator.templatePath
event.InternalEvent["template-info"] = operator.templateInfo
event.Results = e.requests.MakeResultEvent(event)
results = true
@ -94,6 +97,7 @@ func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEve
if matched && result != nil {
event.OperatorsResult = result
event.InternalEvent["template-id"] = operator.templateID
event.InternalEvent["template-path"] = operator.templatePath
event.InternalEvent["template-info"] = operator.templateInfo
event.Results = e.requests.MakeResultEvent(event)
callback(event)

View File

@ -103,6 +103,7 @@ func (r *Request) responseToDSLMap(req, resp *dns.Msg, host, matched string) out
data["raw"] = rawData
data["template-id"] = r.options.TemplateID
data["template-info"] = r.options.TemplateInfo
data["template-path"] = r.options.TemplatePath
return data
}
@ -137,6 +138,7 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu
func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
data := &output.ResultEvent{
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),
Info: wrapped.InternalEvent["template-info"].(map[string]interface{}),
Type: "dns",
Host: types.ToString(wrapped.InternalEvent["host"]),

View File

@ -71,6 +71,7 @@ func (r *Request) responseToDSLMap(raw, host, matched string) output.InternalEve
data["raw"] = raw
data["template-id"] = r.options.TemplateID
data["template-info"] = r.options.TemplateInfo
data["template-path"] = r.options.TemplatePath
return data
}
@ -105,6 +106,7 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu
func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
data := &output.ResultEvent{
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),
Info: wrapped.InternalEvent["template-info"].(map[string]interface{}),
Type: "file",
Path: types.ToString(wrapped.InternalEvent["path"]),

View File

@ -99,6 +99,7 @@ func (b *Browser) Close() {
// headless process launch.
func (b *Browser) killChromeProcesses() {
newProcesses := b.findChromeProcesses()
for id := range newProcesses {
if _, ok := b.previouspids[id]; ok {
continue

View File

@ -49,7 +49,6 @@ func (i *Instance) Run(baseURL *url.URL, actions []*Action, timeout time.Duratio
if err != nil {
return nil, nil, err
}
go router.Run()
data, err := createdPage.ExecuteActions(baseURL, actions)
if err != nil {

View File

@ -23,6 +23,7 @@ func TestActionNavigate(t *testing.T) {
instance, err := browser.NewInstance()
require.Nil(t, err, "could not create browser instance")
defer instance.Close()
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintln(w, `

View File

@ -72,6 +72,7 @@ func (r *Request) responseToDSLMap(resp, req, host, matched string) output.Inter
data["data"] = resp
data["template-id"] = r.options.TemplateID
data["template-info"] = r.options.TemplateInfo
data["template-path"] = r.options.TemplatePath
return data
}
@ -106,6 +107,7 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu
func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
data := &output.ResultEvent{
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),
Info: wrapped.InternalEvent["template-info"].(map[string]interface{}),
Type: "headless",
Host: types.ToString(wrapped.InternalEvent["host"]),

View File

@ -105,6 +105,7 @@ func (r *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, r
data["duration"] = duration.Seconds()
data["template-id"] = r.options.TemplateID
data["template-info"] = r.options.TemplateInfo
data["template-path"] = r.options.TemplatePath
return data
}
@ -139,6 +140,7 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu
func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
data := &output.ResultEvent{
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),
Info: wrapped.InternalEvent["template-info"].(map[string]interface{}),
Type: "http",
Host: types.ToString(wrapped.InternalEvent["host"]),

View File

@ -73,6 +73,7 @@ func (r *Request) responseToDSLMap(req, resp, raw, host, matched string) output.
data["raw"] = raw // Raw is the full transaction data for network
data["template-id"] = r.options.TemplateID
data["template-info"] = r.options.TemplateInfo
data["template-path"] = r.options.TemplatePath
return data
}
@ -107,6 +108,7 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu
func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
data := &output.ResultEvent{
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),
Info: wrapped.InternalEvent["template-info"].(map[string]interface{}),
Type: "network",
Host: types.ToString(wrapped.InternalEvent["host"]),

View File

@ -101,6 +101,7 @@ func (r *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, r
data["duration"] = duration.Seconds()
data["template-id"] = r.options.TemplateID
data["template-info"] = r.options.TemplateInfo
data["template-path"] = r.options.TemplatePath
return data
}
@ -135,6 +136,7 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu
func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
data := &output.ResultEvent{
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),
Info: wrapped.InternalEvent["template-info"].(map[string]interface{}),
Type: "http",
Path: types.ToString(wrapped.InternalEvent["path"]),

View File

@ -58,3 +58,8 @@ func (i *Exporter) Export(event *output.ResultEvent) error {
err := ioutil.WriteFile(path.Join(i.directory, finalFilename), data, 0644)
return err
}
// Close closes the exporter after operation
func (i *Exporter) Close() error {
return nil
}

View File

@ -0,0 +1,91 @@
package sarif
import (
"os"
"sync"
"github.com/owenrumney/go-sarif/sarif"
"github.com/pkg/errors"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/format"
)
type Exporter struct {
sarif *sarif.Report
run *sarif.Run
mutex *sync.Mutex
options *Options
}
// Options contains the configuration options for sarif exporter client
type Options struct {
// File is the file to export found sarif result to
File string `yaml:"file"`
}
// New creates a new disk exporter integration client based on options.
func New(options *Options) (*Exporter, error) {
report, err := sarif.New(sarif.Version210)
if err != nil {
return nil, errors.Wrap(err, "could not create sarif exporter")
}
run := sarif.NewRun("nuclei", "https://github.com/projectdiscovery/nuclei")
return &Exporter{options: options, sarif: report, run: run, mutex: &sync.Mutex{}}, nil
}
// Export exports a passed result event to disk
func (i *Exporter) Export(event *output.ResultEvent) error {
i.mutex.Lock()
defer i.mutex.Unlock()
description := format.MarkdownDescription(event)
var ruleDescription string
if d, ok := event.Info["description"]; ok {
ruleDescription = d.(string)
}
var ruleSeverity string
if s, ok := event.Info["severity"]; ok {
ruleSeverity = s.(string)
}
var ruleName string
if s, ok := event.Info["name"]; ok {
ruleName = s.(string)
}
var sarifSeverity string
switch ruleSeverity {
case "info":
sarifSeverity = "none"
case "low", "medium":
sarifSeverity = "warning"
case "high", "critical":
sarifSeverity = "error"
}
_ = i.run.AddRule(event.TemplateID).
WithDescription(ruleName).
WithFullDescription(sarif.NewMultiformatMessageString(ruleDescription)).
WithHelp(ruleDescription)
_ = i.run.AddResult(event.TemplateID).
WithMessage(sarif.NewMessage().WithText(description)).
WithLevel(sarifSeverity).
WithLocation(sarif.NewLocation().WithMessage(sarif.NewMessage().WithText(event.Host)).WithPhysicalLocation(
sarif.NewPhysicalLocation().
WithArtifactLocation(sarif.NewArtifactLocation().WithUri(event.Type)).
WithRegion(sarif.NewRegion().WithStartColumn(1).WithStartLine(1).WithEndLine(1).WithEndColumn(1)),
))
return nil
}
// Close closes the exporter after operation
func (i *Exporter) Close() error {
i.sarif.AddRun(i.run)
file, err := os.Create(i.options.File)
if err != nil {
return errors.Wrap(err, "could not create sarif output file")
}
defer file.Close()
return i.sarif.Write(file)
}

View File

@ -7,6 +7,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/dedupe"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/disk"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/sarif"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/github"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/gitlab"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/jira"
@ -28,6 +29,8 @@ type Options struct {
Jira *jira.Options `yaml:"jira"`
// DiskExporter contains configuration options for Disk Exporter Module
DiskExporter *disk.Options `yaml:"disk"`
// SarifExporter contains configuration options for Sarif Exporter Module
SarifExporter *sarif.Options `yaml:"sarif"`
}
// Filter filters the received event and decides whether to perform
@ -79,6 +82,8 @@ type Tracker interface {
// Exporter is an interface implemented by an issue exporter
type Exporter interface {
// Close closes the exporter after operation
Close() error
// Export exports an issue to an exporter
Export(event *output.ResultEvent) error
}
@ -129,6 +134,13 @@ func New(options *Options, db string) (*Client, error) {
}
client.exporters = append(client.exporters, exporter)
}
if options.SarifExporter != nil {
exporter, err := sarif.New(options.SarifExporter)
if err != nil {
return nil, errors.Wrap(err, "could not create exporting client")
}
client.exporters = append(client.exporters, exporter)
}
storage, err := dedupe.New(db)
if err != nil {
return nil, err
@ -140,6 +152,9 @@ func New(options *Options, db string) (*Client, error) {
// Close closes the issue tracker reporting client
func (c *Client) Close() {
c.dedupe.Close()
for _, exporter := range c.exporters {
exporter.Close()
}
}
// CreateIssue creates an issue in the tracker

View File

@ -47,6 +47,8 @@ type Options struct {
ReportingConfig string
// DiskExportDirectory is the directory to export reports in markdown on disk to
DiskExportDirectory string
// SarifFile is the file to export sarif output format to
SarifFile string
// ResolversFile is a file containing resolvers for nuclei.
ResolversFile string
// StatsInterval is the number of seconds to display stats after