From 68dd65fd38e105067fb541c713f401cf6fe2b358 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Wed, 26 May 2021 00:07:59 +0200 Subject: [PATCH 01/25] Deprecating golint --- .github/workflows/build.yaml | 4 ++-- .golangci.yml | 11 ++--------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 2020dd10..20c71a09 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -13,10 +13,10 @@ jobs: - name: Checkout code uses: actions/checkout@v2 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v2.5.2 + uses: golangci/golangci-lint-action with: # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. - version: v1.33 + version: latest args: --timeout 5m working-directory: v2/ diff --git a/.golangci.yml b/.golangci.yml index 6c88a19f..9a2621ab 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -66,7 +66,6 @@ linters: - gocritic - gofmt - goimports - - golint #- gomnd - goprintffuncname - gosimple @@ -89,6 +88,7 @@ linters: - unused - varcheck - whitespace + - revive # don't enable: # - depguard @@ -105,11 +105,4 @@ linters: # - nestif # - prealloc # - testpackage - # - wsl - -# golangci.com configuration -# https://github.com/golangci/golangci/wiki/Configuration -service: - golangci-lint-version: 1.33.x # use the fixed version to not introduce new linters unexpectedly - prepare: - - echo "here I can run custom commands, but no preparation needed for this repo" + # - wsl \ No newline at end of file From 72ddf043219f9f871546260af23f957e214a2653 Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Thu, 27 May 2021 10:38:30 +0530 Subject: [PATCH 02/25] Adding security information --- SECURITY.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000..fc8033f4 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,5 @@ +# Security Policy + +## Reporting a Vulnerability + +DO NOT CREATE AN ISSUE to report a security problem. Instead, please send an email to security@projectdiscovery.io and we will acknowledge it within 3 working days. From cb5688ba5112a5d860ca63a072563466b61baa9c Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Wed, 2 Jun 2021 14:07:49 +0530 Subject: [PATCH 03/25] Testing docker multi platform images --- .github/workflows/dockerhub-push.yml | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/.github/workflows/dockerhub-push.yml b/.github/workflows/dockerhub-push.yml index 8a93961a..9c3d4800 100644 --- a/.github/workflows/dockerhub-push.yml +++ b/.github/workflows/dockerhub-push.yml @@ -1,19 +1,17 @@ -# dockerhub-push pushes docker build to dockerhub automatically -# on the creation of a new release -name: Publish to Dockerhub on creation of a new release -on: - release: - types: [published] +name: ◎ Docker Push +on: + workflow_dispatch: + jobs: update: runs-on: ubuntu-latest steps: - - uses: actions/checkout@master + uses: actions/checkout@v2 - name: Publish to Dockerhub Registry - #pre: echo ::save-state name=RELEASE_VERSION::$(echo ${GITHUB_REF:10}) - uses: elgohr/Publish-Docker-Github-Action@master + uses: dawidd6/action-docker-publish-changed@v3 with: name: projectdiscovery/nuclei - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - tags: "latest" #"latest,${{ env.STATE_RELEASE_VERSION }}" + username: ${{secrets.DOCKER_USERNAME}} + password: ${{secrets.DOCKER_PASSWORD}} + platforms: linux/amd64,linux/arm64,linux/arm + tag: latest \ No newline at end of file From 791b025309817ff3913724e9cfd482d08759ec56 Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Wed, 2 Jun 2021 14:16:58 +0530 Subject: [PATCH 04/25] Update dockerhub-push.yml --- .github/workflows/dockerhub-push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerhub-push.yml b/.github/workflows/dockerhub-push.yml index 9c3d4800..1e235e7a 100644 --- a/.github/workflows/dockerhub-push.yml +++ b/.github/workflows/dockerhub-push.yml @@ -6,7 +6,7 @@ jobs: update: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + - uses: actions/checkout@v2 - name: Publish to Dockerhub Registry uses: dawidd6/action-docker-publish-changed@v3 with: From 5a6303265d7ff207f54ab733c1ce396b7fa1c69a Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 3 Jun 2021 10:26:41 +0530 Subject: [PATCH 05/25] Fixed #745 crash with http clustering module The crash is mitigated in two ways. - An additional check in http clustering module for req condition (no clustering with conditions) - Clustered Executer also now passes a non-nil map to mitigate the crash if for some reason we reach there. --- v2/pkg/protocols/common/clusterer/executer.go | 3 ++- v2/pkg/protocols/http/cluster.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/v2/pkg/protocols/common/clusterer/executer.go b/v2/pkg/protocols/common/clusterer/executer.go index 84207ab1..134ddc67 100644 --- a/v2/pkg/protocols/common/clusterer/executer.go +++ b/v2/pkg/protocols/common/clusterer/executer.go @@ -60,8 +60,9 @@ func (e *Executer) Requests() int { func (e *Executer) Execute(input string) (bool, error) { var results bool + previous := make(map[string]interface{}) dynamicValues := make(map[string]interface{}) - err := e.requests.ExecuteWithResults(input, dynamicValues, nil, func(event *output.InternalWrappedEvent) { + err := e.requests.ExecuteWithResults(input, dynamicValues, previous, func(event *output.InternalWrappedEvent) { for _, operator := range e.operators { result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract) if matched && result != nil { diff --git a/v2/pkg/protocols/http/cluster.go b/v2/pkg/protocols/http/cluster.go index 68107c9e..bdfa0735 100644 --- a/v2/pkg/protocols/http/cluster.go +++ b/v2/pkg/protocols/http/cluster.go @@ -10,7 +10,7 @@ import ( // are similar enough to be considered one and can be checked by // just adding the matcher/extractors for the request and the correct IDs. func (r *Request) CanCluster(other *Request) bool { - if len(r.Payloads) > 0 || len(r.Raw) > 0 || len(r.Body) > 0 || r.Unsafe { + if len(r.Payloads) > 0 || len(r.Raw) > 0 || len(r.Body) > 0 || r.Unsafe || r.ReqCondition || r.Name != "" { return false } if r.Method != other.Method || From 38abe40db8d486855f4fe2cadcf028a57ad4ba11 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Thu, 3 Jun 2021 10:49:13 +0530 Subject: [PATCH 06/25] Fixed #739 by adding normal request support to pipelining --- v2/go.mod | 2 +- v2/go.sum | 4 ++++ v2/pkg/protocols/http/request.go | 14 +++++++++----- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 793998ff..8ef803c2 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -31,7 +31,7 @@ require ( github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.1 github.com/projectdiscovery/interactsh v0.0.3 - github.com/projectdiscovery/rawhttp v0.0.6 + github.com/projectdiscovery/rawhttp v0.0.7 github.com/projectdiscovery/retryabledns v1.0.10 github.com/projectdiscovery/retryablehttp-go v1.0.1 github.com/remeh/sizedwaitgroup v1.0.0 diff --git a/v2/go.sum b/v2/go.sum index 10c1e785..66378737 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -228,6 +228,10 @@ github.com/projectdiscovery/interactsh v0.0.3 h1:PUkWk+NzSyd5glMqfORmuqizhsd7c3W github.com/projectdiscovery/interactsh v0.0.3/go.mod h1:dWnKO14d2FLP3kLhI9DecEsiAC/aZiJoUBGFjGhDskY= github.com/projectdiscovery/rawhttp v0.0.6 h1:HbgPB1eKXQVV5F9sq0Uxflm95spWFyZYD8dgFpeOC9M= github.com/projectdiscovery/rawhttp v0.0.6/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= +github.com/projectdiscovery/rawhttp v0.0.7-0.20210603051339-3582c0c9ea3e h1:it83hXLxlmjf9/nwwVt+yMN1wwOvWMFITXVFVvxYYqs= +github.com/projectdiscovery/rawhttp v0.0.7-0.20210603051339-3582c0c9ea3e/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= +github.com/projectdiscovery/rawhttp v0.0.7 h1:5m4peVgjbl7gqDcRYMTVEuX+Xs/nh76ohTkkvufucLg= +github.com/projectdiscovery/rawhttp v0.0.7/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= github.com/projectdiscovery/retryabledns v1.0.7/go.mod h1:/UzJn4I+cPdQl6pKiiQfvVAT636YZvJQYZhYhGB0dUQ= github.com/projectdiscovery/retryabledns v1.0.10 h1:xJZ2aKoqrNg/OZEw1+4+QIOH40V/WkZDYY1ZZc+uphE= github.com/projectdiscovery/retryabledns v1.0.10/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4= diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 6dc361fa..d3f774a5 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -117,7 +117,7 @@ func (r *Request) executeParallelHTTP(reqURL string, dynamicValues, previous out return requestErr } -// executeRaceRequest executes turbo http request for a URL +// executeTurboHTTP executes turbo http request for a URL func (r *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { generator := r.newGenerator() @@ -277,11 +277,15 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ var hostname string timeStart := time.Now() if request.original.Pipeline { - formedURL = request.rawRequest.FullURL - if parsed, parseErr := url.Parse(formedURL); parseErr == nil { - hostname = parsed.Host + if request.rawRequest != nil { + formedURL = request.rawRequest.FullURL + if parsed, parseErr := url.Parse(formedURL); parseErr == nil { + hostname = parsed.Host + } + resp, err = request.pipelinedClient.DoRaw(request.rawRequest.Method, reqURL, request.rawRequest.Path, generators.ExpandMapValues(request.rawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.rawRequest.Data))) + } else if request.request != nil { + resp, err = request.pipelinedClient.Dor(request.request) } - resp, err = request.pipelinedClient.DoRaw(request.rawRequest.Method, reqURL, request.rawRequest.Path, generators.ExpandMapValues(request.rawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.rawRequest.Data))) } else if request.original.Unsafe && request.rawRequest != nil { formedURL = request.rawRequest.FullURL if parsed, parseErr := url.Parse(formedURL); parseErr == nil { From 19157597a0b3bf0c89a9bd25e5dda9021c031574 Mon Sep 17 00:00:00 2001 From: ganoes Date: Fri, 4 Jun 2021 13:11:09 +0200 Subject: [PATCH 07/25] Added support of Jira on-prem server --- v2/cmd/nuclei/issue-tracker-config.yaml | 8 ++++--- v2/pkg/reporting/trackers/jira/jira.go | 30 ++++++++++++++++++++----- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/v2/cmd/nuclei/issue-tracker-config.yaml b/v2/cmd/nuclei/issue-tracker-config.yaml index f69342c7..3fffce49 100644 --- a/v2/cmd/nuclei/issue-tracker-config.yaml +++ b/v2/cmd/nuclei/issue-tracker-config.yaml @@ -32,14 +32,16 @@ # issue-label: "" # jira contains configuration options for jira issue tracker -#jira: +#jira: +# # Cloud is the boolean which tells if Jira instance is running in the cloud or on-prem version is used +# cloud: true # # URL is the jira application url # url: "" -# # account-id is the account-id of the jira user +# # account-id is the account-id of the jira user or username in case of on-prem Jira # account-id: "" # # email is the email of the user for jira instance # email: "" -# # token is the token for jira instance. +# # token is the token for jira instance or password in case of on-prem Jira # token: "" # # project-name is the name of the project. # project-name: "" diff --git a/v2/pkg/reporting/trackers/jira/jira.go b/v2/pkg/reporting/trackers/jira/jira.go index 0c68b4ce..3e7f65ee 100644 --- a/v2/pkg/reporting/trackers/jira/jira.go +++ b/v2/pkg/reporting/trackers/jira/jira.go @@ -20,6 +20,8 @@ type Integration struct { // Options contains the configuration options for jira client type Options struct { + // Cloud value is set to true when Jira cloud is used + Cloud bool `yaml:"cloud"` // URL is the URL of the jira server URL string `yaml:"url"` // AccountID is the accountID of the jira user. @@ -36,8 +38,12 @@ type Options struct { // New creates a new issue tracker integration client based on options. func New(options *Options) (*Integration, error) { + username := options.Email + if !options.Cloud { + username = options.AccountID + } tp := jira.BasicAuthTransport{ - Username: options.Email, + Username: username, Password: options.Token, } jiraClient, err := jira.NewClient(tp.Client(), options.URL) @@ -51,15 +57,27 @@ func New(options *Options) (*Integration, error) { func (i *Integration) CreateIssue(event *output.ResultEvent) error { summary := format.Summary(event) - issueData := &jira.Issue{ - Fields: &jira.IssueFields{ - Assignee: &jira.User{AccountID: i.options.AccountID}, - Reporter: &jira.User{AccountID: i.options.AccountID}, + fields := &jira.IssueFields{ + Assignee: &jira.User{AccountID: i.options.AccountID}, + Reporter: &jira.User{AccountID: i.options.AccountID}, + Description: jiraFormatDescription(event), + Type: jira.IssueType{Name: i.options.IssueType}, + Project: jira.Project{Key: i.options.ProjectName}, + Summary: summary, + } + // On-prem version of Jira server does not use AccountID + if !i.options.Cloud { + fields = &jira.IssueFields{ + Assignee: &jira.User{Name: i.options.AccountID}, Description: jiraFormatDescription(event), Type: jira.IssueType{Name: i.options.IssueType}, Project: jira.Project{Key: i.options.ProjectName}, Summary: summary, - }, + } + } + + issueData := &jira.Issue{ + Fields: fields, } _, resp, err := i.jira.Issue.Create(issueData) if err != nil { From abc7063339e86c1c00e6787e082121dc959517ce Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Sat, 5 Jun 2021 18:01:08 +0530 Subject: [PATCH 08/25] Work on sarif integration start --- v2/cmd/nuclei/main.go | 1 + v2/go.mod | 2 +- v2/go.sum | 9 ++ v2/internal/runner/runner.go | 9 ++ v2/pkg/output/output.go | 2 + v2/pkg/protocols/common/clusterer/executer.go | 4 + v2/pkg/protocols/dns/operators.go | 2 + v2/pkg/protocols/file/operators.go | 2 + v2/pkg/protocols/headless/engine/engine.go | 1 + v2/pkg/protocols/headless/engine/page.go | 1 - .../headless/engine/page_actions_test.go | 1 + v2/pkg/protocols/headless/operators.go | 2 + v2/pkg/protocols/http/operators.go | 2 + v2/pkg/protocols/network/operators.go | 2 + v2/pkg/protocols/offlinehttp/operators.go | 2 + v2/pkg/reporting/exporters/disk/disk.go | 5 + v2/pkg/reporting/exporters/sarif/sarif.go | 91 +++++++++++++++++++ v2/pkg/reporting/reporting.go | 15 +++ v2/pkg/types/types.go | 2 + 19 files changed, 153 insertions(+), 2 deletions(-) create mode 100644 v2/pkg/reporting/exporters/sarif/sarif.go diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 574da4fc..0ce15511 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -85,6 +85,7 @@ based on templates offering massive extensibility and ease of use.`) set.IntVar(&options.PageTimeout, "page-timeout", 20, "Seconds to wait for each page in headless") set.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "Only run newly added templates") set.StringVarP(&options.DiskExportDirectory, "disk-export", "de", "", "Directory on disk to export reports in markdown to") + set.StringVarP(&options.SarifFile, "sarif-file", "sf", "", "File to export sarif report to") set.BoolVar(&options.NoInteractsh, "no-interactsh", false, "Do not use interactsh server for blind interaction polling") set.StringVar(&options.InteractshURL, "interactsh-url", "https://interact.sh", "Interactsh Server URL") set.IntVar(&options.InteractionsCacheSize, "interactions-cache-size", 5000, "Number of requests to keep in interactions cache") diff --git a/v2/go.mod b/v2/go.mod index 793998ff..34d191cd 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -23,6 +23,7 @@ require ( github.com/miekg/dns v1.1.38 github.com/mitchellh/go-ps v1.0.0 github.com/olekukonko/tablewriter v0.0.5 + github.com/owenrumney/go-sarif v1.0.4 // indirect github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.8 github.com/projectdiscovery/collaborator v0.0.2 @@ -53,7 +54,6 @@ require ( golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 // indirect golang.org/x/sys v0.0.0-20210218155724-8ebf48af031b // indirect - golang.org/x/text v0.3.4 // indirect golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect google.golang.org/appengine v1.6.7 // indirect gopkg.in/yaml.v2 v2.4.0 diff --git a/v2/go.sum b/v2/go.sum index 10c1e785..ad6ac302 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -40,6 +40,7 @@ github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF0 github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= github.com/andygrunwald/go-jira v1.13.0 h1:vvIImGgX32bHfoiyUwkNo+/YrPnRczNarvhLOncP6dE= github.com/andygrunwald/go-jira v1.13.0/go.mod h1:jYi4kFDbRPZTJdJOVJO4mpMMIwdB+rcZwSO58DzPd2I= +github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -206,6 +207,8 @@ github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+ github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/owenrumney/go-sarif v1.0.4 h1:0LFC5eHP6amc/9ajM1jDiE52UfXFcl/oozay+X3KgV4= +github.com/owenrumney/go-sarif v1.0.4/go.mod h1:DXUGbHwQcCMvqcvZbxh8l/7diHsJVztOKZgmPt88RNI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -266,6 +269,8 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= +github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ= github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM= github.com/xanzy/go-gitlab v0.44.0 h1:cEiGhqu7EpFGuei2a2etAwB+x6403E5CvpLn35y+GPs= @@ -283,6 +288,8 @@ github.com/ysmood/leakless v0.6.12/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNq github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/zclconf/go-cty v1.8.2 h1:u+xZfBKgpycDnTNjPhGiTEYZS5qS/Sb5MqSfm7vzcjg= +github.com/zclconf/go-cty v1.8.2/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -427,6 +434,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 40a965f2..45e7dce4 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -24,6 +24,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine" "github.com/projectdiscovery/nuclei/v2/pkg/reporting" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/disk" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/sarif" "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/remeh/sizedwaitgroup" @@ -95,6 +96,14 @@ func New(options *types.Options) (*Runner, error) { reportingOptions.DiskExporter = &disk.Options{Directory: options.DiskExportDirectory} } } + if options.SarifFile != "" { + if reportingOptions != nil { + reportingOptions.SarifExporter = &sarif.Options{File: options.SarifFile} + } else { + reportingOptions = &reporting.Options{} + reportingOptions.SarifExporter = &sarif.Options{File: options.SarifFile} + } + } if reportingOptions != nil { if client, err := reporting.New(reportingOptions, options.ReportingDB); err != nil { gologger.Fatal().Msgf("Could not create issue reporting client: %s\n", err) diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index 2fac10fd..bd72c894 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -54,6 +54,8 @@ type InternalWrappedEvent struct { type ResultEvent struct { // TemplateID is the ID of the template for the result. TemplateID string `json:"templateID"` + // TemplatePath is the path of template + TemplatePath string `json:"-"` // Info contains information block of the template for the result. Info map[string]interface{} `json:"info,inline"` // MatcherName is the name of the matcher matched if any. diff --git a/v2/pkg/protocols/common/clusterer/executer.go b/v2/pkg/protocols/common/clusterer/executer.go index 84207ab1..45621bbe 100644 --- a/v2/pkg/protocols/common/clusterer/executer.go +++ b/v2/pkg/protocols/common/clusterer/executer.go @@ -22,6 +22,7 @@ type Executer struct { type clusteredOperator struct { templateID string + templatePath string templateInfo map[string]interface{} operator *operators.Operators } @@ -38,6 +39,7 @@ func NewExecuter(requests []*templates.Template, options *protocols.ExecuterOpti executer.operators = append(executer.operators, &clusteredOperator{ templateID: req.ID, templateInfo: req.Info, + templatePath: req.Options.TemplatePath, operator: req.RequestsHTTP[0].CompiledOperators, }) } @@ -67,6 +69,7 @@ func (e *Executer) Execute(input string) (bool, error) { if matched && result != nil { event.OperatorsResult = result event.InternalEvent["template-id"] = operator.templateID + event.InternalEvent["template-path"] = operator.templatePath event.InternalEvent["template-info"] = operator.templateInfo event.Results = e.requests.MakeResultEvent(event) results = true @@ -94,6 +97,7 @@ func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEve if matched && result != nil { event.OperatorsResult = result event.InternalEvent["template-id"] = operator.templateID + event.InternalEvent["template-path"] = operator.templatePath event.InternalEvent["template-info"] = operator.templateInfo event.Results = e.requests.MakeResultEvent(event) callback(event) diff --git a/v2/pkg/protocols/dns/operators.go b/v2/pkg/protocols/dns/operators.go index 860a38f8..5d18c4d5 100644 --- a/v2/pkg/protocols/dns/operators.go +++ b/v2/pkg/protocols/dns/operators.go @@ -103,6 +103,7 @@ func (r *Request) responseToDSLMap(req, resp *dns.Msg, host, matched string) out data["raw"] = rawData data["template-id"] = r.options.TemplateID data["template-info"] = r.options.TemplateInfo + data["template-path"] = r.options.TemplatePath return data } @@ -137,6 +138,7 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), + TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(map[string]interface{}), Type: "dns", Host: types.ToString(wrapped.InternalEvent["host"]), diff --git a/v2/pkg/protocols/file/operators.go b/v2/pkg/protocols/file/operators.go index 7b3b59fe..e7afdaef 100644 --- a/v2/pkg/protocols/file/operators.go +++ b/v2/pkg/protocols/file/operators.go @@ -71,6 +71,7 @@ func (r *Request) responseToDSLMap(raw, host, matched string) output.InternalEve data["raw"] = raw data["template-id"] = r.options.TemplateID data["template-info"] = r.options.TemplateInfo + data["template-path"] = r.options.TemplatePath return data } @@ -105,6 +106,7 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), + TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(map[string]interface{}), Type: "file", Path: types.ToString(wrapped.InternalEvent["path"]), diff --git a/v2/pkg/protocols/headless/engine/engine.go b/v2/pkg/protocols/headless/engine/engine.go index 6c8d613e..ea85f975 100644 --- a/v2/pkg/protocols/headless/engine/engine.go +++ b/v2/pkg/protocols/headless/engine/engine.go @@ -99,6 +99,7 @@ func (b *Browser) Close() { // headless process launch. func (b *Browser) killChromeProcesses() { newProcesses := b.findChromeProcesses() + for id := range newProcesses { if _, ok := b.previouspids[id]; ok { continue diff --git a/v2/pkg/protocols/headless/engine/page.go b/v2/pkg/protocols/headless/engine/page.go index 185eb33c..16eae572 100644 --- a/v2/pkg/protocols/headless/engine/page.go +++ b/v2/pkg/protocols/headless/engine/page.go @@ -49,7 +49,6 @@ func (i *Instance) Run(baseURL *url.URL, actions []*Action, timeout time.Duratio if err != nil { return nil, nil, err } - go router.Run() data, err := createdPage.ExecuteActions(baseURL, actions) if err != nil { diff --git a/v2/pkg/protocols/headless/engine/page_actions_test.go b/v2/pkg/protocols/headless/engine/page_actions_test.go index e89fe535..658c3290 100644 --- a/v2/pkg/protocols/headless/engine/page_actions_test.go +++ b/v2/pkg/protocols/headless/engine/page_actions_test.go @@ -23,6 +23,7 @@ func TestActionNavigate(t *testing.T) { instance, err := browser.NewInstance() require.Nil(t, err, "could not create browser instance") + defer instance.Close() ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(w, ` diff --git a/v2/pkg/protocols/headless/operators.go b/v2/pkg/protocols/headless/operators.go index f1845a89..265dc95d 100644 --- a/v2/pkg/protocols/headless/operators.go +++ b/v2/pkg/protocols/headless/operators.go @@ -72,6 +72,7 @@ func (r *Request) responseToDSLMap(resp, req, host, matched string) output.Inter data["data"] = resp data["template-id"] = r.options.TemplateID data["template-info"] = r.options.TemplateInfo + data["template-path"] = r.options.TemplatePath return data } @@ -106,6 +107,7 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), + TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(map[string]interface{}), Type: "headless", Host: types.ToString(wrapped.InternalEvent["host"]), diff --git a/v2/pkg/protocols/http/operators.go b/v2/pkg/protocols/http/operators.go index 0c1fcb99..77438f04 100644 --- a/v2/pkg/protocols/http/operators.go +++ b/v2/pkg/protocols/http/operators.go @@ -105,6 +105,7 @@ func (r *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, r data["duration"] = duration.Seconds() data["template-id"] = r.options.TemplateID data["template-info"] = r.options.TemplateInfo + data["template-path"] = r.options.TemplatePath return data } @@ -139,6 +140,7 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), + TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(map[string]interface{}), Type: "http", Host: types.ToString(wrapped.InternalEvent["host"]), diff --git a/v2/pkg/protocols/network/operators.go b/v2/pkg/protocols/network/operators.go index a0a95ede..d515e686 100644 --- a/v2/pkg/protocols/network/operators.go +++ b/v2/pkg/protocols/network/operators.go @@ -73,6 +73,7 @@ func (r *Request) responseToDSLMap(req, resp, raw, host, matched string) output. data["raw"] = raw // Raw is the full transaction data for network data["template-id"] = r.options.TemplateID data["template-info"] = r.options.TemplateInfo + data["template-path"] = r.options.TemplatePath return data } @@ -107,6 +108,7 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), + TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(map[string]interface{}), Type: "network", Host: types.ToString(wrapped.InternalEvent["host"]), diff --git a/v2/pkg/protocols/offlinehttp/operators.go b/v2/pkg/protocols/offlinehttp/operators.go index c71376c9..d60cf72f 100644 --- a/v2/pkg/protocols/offlinehttp/operators.go +++ b/v2/pkg/protocols/offlinehttp/operators.go @@ -101,6 +101,7 @@ func (r *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, r data["duration"] = duration.Seconds() data["template-id"] = r.options.TemplateID data["template-info"] = r.options.TemplateInfo + data["template-path"] = r.options.TemplatePath return data } @@ -135,6 +136,7 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), + TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(map[string]interface{}), Type: "http", Path: types.ToString(wrapped.InternalEvent["path"]), diff --git a/v2/pkg/reporting/exporters/disk/disk.go b/v2/pkg/reporting/exporters/disk/disk.go index 35c1d25b..7ff73739 100644 --- a/v2/pkg/reporting/exporters/disk/disk.go +++ b/v2/pkg/reporting/exporters/disk/disk.go @@ -58,3 +58,8 @@ func (i *Exporter) Export(event *output.ResultEvent) error { err := ioutil.WriteFile(path.Join(i.directory, finalFilename), data, 0644) return err } + +// Close closes the exporter after operation +func (i *Exporter) Close() error { + return nil +} diff --git a/v2/pkg/reporting/exporters/sarif/sarif.go b/v2/pkg/reporting/exporters/sarif/sarif.go new file mode 100644 index 00000000..141e1713 --- /dev/null +++ b/v2/pkg/reporting/exporters/sarif/sarif.go @@ -0,0 +1,91 @@ +package sarif + +import ( + "os" + "sync" + + "github.com/owenrumney/go-sarif/sarif" + "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/format" +) + +type Exporter struct { + sarif *sarif.Report + run *sarif.Run + mutex *sync.Mutex + + options *Options +} + +// Options contains the configuration options for sarif exporter client +type Options struct { + // File is the file to export found sarif result to + File string `yaml:"file"` +} + +// New creates a new disk exporter integration client based on options. +func New(options *Options) (*Exporter, error) { + report, err := sarif.New(sarif.Version210) + if err != nil { + return nil, errors.Wrap(err, "could not create sarif exporter") + } + run := sarif.NewRun("nuclei", "https://github.com/projectdiscovery/nuclei") + return &Exporter{options: options, sarif: report, run: run, mutex: &sync.Mutex{}}, nil +} + +// Export exports a passed result event to disk +func (i *Exporter) Export(event *output.ResultEvent) error { + i.mutex.Lock() + defer i.mutex.Unlock() + + description := format.MarkdownDescription(event) + + var ruleDescription string + if d, ok := event.Info["description"]; ok { + ruleDescription = d.(string) + } + var ruleSeverity string + if s, ok := event.Info["severity"]; ok { + ruleSeverity = s.(string) + } + var ruleName string + if s, ok := event.Info["name"]; ok { + ruleName = s.(string) + } + + var sarifSeverity string + switch ruleSeverity { + case "info": + sarifSeverity = "none" + case "low", "medium": + sarifSeverity = "warning" + case "high", "critical": + sarifSeverity = "error" + } + _ = i.run.AddRule(event.TemplateID). + WithDescription(ruleName). + WithFullDescription(sarif.NewMultiformatMessageString(ruleDescription)). + WithHelp(ruleDescription) + _ = i.run.AddResult(event.TemplateID). + WithMessage(sarif.NewMessage().WithText(description)). + WithLevel(sarifSeverity). + WithLocation(sarif.NewLocation().WithMessage(sarif.NewMessage().WithText(event.Host)).WithPhysicalLocation( + sarif.NewPhysicalLocation(). + WithArtifactLocation(sarif.NewArtifactLocation().WithUri(event.Type)). + WithRegion(sarif.NewRegion().WithStartColumn(1).WithStartLine(1).WithEndLine(1).WithEndColumn(1)), + )) + return nil +} + +// Close closes the exporter after operation +func (i *Exporter) Close() error { + i.sarif.AddRun(i.run) + + file, err := os.Create(i.options.File) + if err != nil { + return errors.Wrap(err, "could not create sarif output file") + } + defer file.Close() + return i.sarif.Write(file) +} diff --git a/v2/pkg/reporting/reporting.go b/v2/pkg/reporting/reporting.go index e33600bb..7eaf2602 100644 --- a/v2/pkg/reporting/reporting.go +++ b/v2/pkg/reporting/reporting.go @@ -7,6 +7,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/dedupe" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/disk" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/sarif" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/github" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/gitlab" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/jira" @@ -28,6 +29,8 @@ type Options struct { Jira *jira.Options `yaml:"jira"` // DiskExporter contains configuration options for Disk Exporter Module DiskExporter *disk.Options `yaml:"disk"` + // SarifExporter contains configuration options for Sarif Exporter Module + SarifExporter *sarif.Options `yaml:"sarif"` } // Filter filters the received event and decides whether to perform @@ -79,6 +82,8 @@ type Tracker interface { // Exporter is an interface implemented by an issue exporter type Exporter interface { + // Close closes the exporter after operation + Close() error // Export exports an issue to an exporter Export(event *output.ResultEvent) error } @@ -129,6 +134,13 @@ func New(options *Options, db string) (*Client, error) { } client.exporters = append(client.exporters, exporter) } + if options.SarifExporter != nil { + exporter, err := sarif.New(options.SarifExporter) + if err != nil { + return nil, errors.Wrap(err, "could not create exporting client") + } + client.exporters = append(client.exporters, exporter) + } storage, err := dedupe.New(db) if err != nil { return nil, err @@ -140,6 +152,9 @@ func New(options *Options, db string) (*Client, error) { // Close closes the issue tracker reporting client func (c *Client) Close() { c.dedupe.Close() + for _, exporter := range c.exporters { + exporter.Close() + } } // CreateIssue creates an issue in the tracker diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index d9b40700..1ae84d9b 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -47,6 +47,8 @@ type Options struct { ReportingConfig string // DiskExportDirectory is the directory to export reports in markdown on disk to DiskExportDirectory string + // SarifFile is the file to export sarif output format to + SarifFile string // ResolversFile is a file containing resolvers for nuclei. ResolversFile string // StatsInterval is the number of seconds to display stats after From d3b2c8fee00046554571d6591d1e41261eaa4087 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Sat, 5 Jun 2021 20:06:23 +0530 Subject: [PATCH 09/25] Improved output format --- v2/pkg/reporting/exporters/sarif/sarif.go | 143 +++++++++++++++++++--- 1 file changed, 123 insertions(+), 20 deletions(-) diff --git a/v2/pkg/reporting/exporters/sarif/sarif.go b/v2/pkg/reporting/exporters/sarif/sarif.go index 141e1713..742f6345 100644 --- a/v2/pkg/reporting/exporters/sarif/sarif.go +++ b/v2/pkg/reporting/exporters/sarif/sarif.go @@ -1,13 +1,17 @@ package sarif import ( + "bytes" "os" + "path" + "strings" "sync" "github.com/owenrumney/go-sarif/sarif" "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/format" + "github.com/projectdiscovery/nuclei/v2/pkg/types" ) type Exporter struct { @@ -15,6 +19,7 @@ type Exporter struct { run *sarif.Run mutex *sync.Mutex + home string options *Options } @@ -30,43 +35,58 @@ func New(options *Options) (*Exporter, error) { if err != nil { return nil, errors.Wrap(err, "could not create sarif exporter") } + home, err := os.UserHomeDir() + if err != nil { + return nil, errors.Wrap(err, "could not get home dir") + } + templatePath := path.Join(home, "nuclei-templates") + run := sarif.NewRun("nuclei", "https://github.com/projectdiscovery/nuclei") - return &Exporter{options: options, sarif: report, run: run, mutex: &sync.Mutex{}}, nil + return &Exporter{options: options, home: templatePath, sarif: report, run: run, mutex: &sync.Mutex{}}, nil } // Export exports a passed result event to disk func (i *Exporter) Export(event *output.ResultEvent) error { i.mutex.Lock() defer i.mutex.Unlock() + templatePath := strings.TrimPrefix(event.TemplatePath, i.home) - description := format.MarkdownDescription(event) + description := getSarifResultMessage(event, templatePath) + sarifSeverity := getSarifSeverity(event) + sarifRuleHelpURIs := getSarifRuleHelpURIFromReferences(event) - var ruleDescription string - if d, ok := event.Info["description"]; ok { - ruleDescription = d.(string) - } - var ruleSeverity string - if s, ok := event.Info["severity"]; ok { - ruleSeverity = s.(string) - } var ruleName string if s, ok := event.Info["name"]; ok { ruleName = s.(string) } - var sarifSeverity string - switch ruleSeverity { - case "info": - sarifSeverity = "none" - case "low", "medium": - sarifSeverity = "warning" - case "high", "critical": - sarifSeverity = "error" + var templateURL string + if strings.HasPrefix(event.TemplatePath, i.home) { + templateURL = "https://github.com/projectdiscovery/nuclei-templates/blob/master" + templatePath } + + var ruleDescription string + if d, ok := event.Info["description"]; ok { + ruleDescription = d.(string) + } + builder := &strings.Builder{} + builder.WriteString(ruleDescription) + if sarifRuleHelpURIs != "" { + builder.WriteString("\nReferences: \n") + builder.WriteString(sarifRuleHelpURIs) + } + if templateURL != "" { + builder.WriteString("\nTemplate URL: ") + builder.WriteString(templateURL) + } + ruleHelp := builder.String() + _ = i.run.AddRule(event.TemplateID). WithDescription(ruleName). - WithFullDescription(sarif.NewMultiformatMessageString(ruleDescription)). - WithHelp(ruleDescription) + WithHelp(ruleHelp). + WithHelpURI(templateURL). + WithFullDescription(sarif.NewMultiformatMessageString(sarifRuleHelpURIs)) + _ = i.run.AddResult(event.TemplateID). WithMessage(sarif.NewMessage().WithText(description)). WithLevel(sarifSeverity). @@ -78,6 +98,89 @@ func (i *Exporter) Export(event *output.ResultEvent) error { return nil } +// getSarifSeverity returns the sarif severity +func getSarifSeverity(event *output.ResultEvent) string { + var ruleSeverity string + if s, ok := event.Info["severity"]; ok { + ruleSeverity = s.(string) + } + + switch ruleSeverity { + case "info": + return "none" + case "low", "medium": + return "warning" + case "high", "critical": + return "error" + default: + return "none" + } +} + +// getSarifRuleHelpURIFromReferences returns the sarif rule help uri +func getSarifRuleHelpURIFromReferences(event *output.ResultEvent) string { + if d, ok := event.Info["reference"]; ok { + switch v := d.(type) { + case string: + return v + case []interface{}: + slice := types.ToStringSlice(v) + return strings.Join(slice, "\n") + } + } + return "" +} + +// getSarifResultMessage gets a sarif result message from event +func getSarifResultMessage(event *output.ResultEvent, templatePath string) string { + template := format.GetMatchedTemplate(event) + builder := &bytes.Buffer{} + + builder.WriteString(template) + builder.WriteString(" matched at ") + builder.WriteString(event.Host) + builder.WriteString(" (") + builder.WriteString(strings.ToUpper(event.Type)) + builder.WriteString(") => ") + builder.WriteString(event.Matched) + + if len(event.ExtractedResults) > 0 || len(event.Metadata) > 0 { + if len(event.ExtractedResults) > 0 { + builder.WriteString(" **Extracted results**:\n\n") + for _, v := range event.ExtractedResults { + builder.WriteString("- ") + builder.WriteString(v) + builder.WriteString("\n") + } + builder.WriteString("\n") + } + if len(event.Metadata) > 0 { + builder.WriteString(" **Metadata**:\n\n") + for k, v := range event.Metadata { + builder.WriteString("- ") + builder.WriteString(k) + builder.WriteString(": ") + builder.WriteString(types.ToString(v)) + builder.WriteString("\n") + } + builder.WriteString("\n") + } + } + if event.Interaction != nil { + builder.WriteString("**Interaction Data**\n---\n") + builder.WriteString(event.Interaction.Protocol) + } + + builder.WriteString(" To Reproduce - `nuclei -t ") + builder.WriteString(strings.TrimPrefix(templatePath, "/")) + builder.WriteString(" -target \"") + builder.WriteString(event.Host) + builder.WriteString("\"`") + + data := builder.String() + return data +} + // Close closes the exporter after operation func (i *Exporter) Close() error { i.sarif.AddRun(i.run) From e09f16b68c4a96d65a3b513e88c4c6dceca3a229 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Sat, 5 Jun 2021 20:08:52 +0530 Subject: [PATCH 10/25] Misc --- v2/pkg/reporting/exporters/sarif/sarif.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/v2/pkg/reporting/exporters/sarif/sarif.go b/v2/pkg/reporting/exporters/sarif/sarif.go index 742f6345..a5040aea 100644 --- a/v2/pkg/reporting/exporters/sarif/sarif.go +++ b/v2/pkg/reporting/exporters/sarif/sarif.go @@ -14,6 +14,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/types" ) +// Exporter is an exporter for nuclei sarif output format. type Exporter struct { sarif *sarif.Report run *sarif.Run @@ -45,10 +46,8 @@ func New(options *Options) (*Exporter, error) { return &Exporter{options: options, home: templatePath, sarif: report, run: run, mutex: &sync.Mutex{}}, nil } -// Export exports a passed result event to disk +// Export exports a passed result event to sarif structure func (i *Exporter) Export(event *output.ResultEvent) error { - i.mutex.Lock() - defer i.mutex.Unlock() templatePath := strings.TrimPrefix(event.TemplatePath, i.home) description := getSarifResultMessage(event, templatePath) @@ -81,6 +80,9 @@ func (i *Exporter) Export(event *output.ResultEvent) error { } ruleHelp := builder.String() + i.mutex.Lock() + defer i.mutex.Unlock() + _ = i.run.AddRule(event.TemplateID). WithDescription(ruleName). WithHelp(ruleHelp). @@ -183,6 +185,9 @@ func getSarifResultMessage(event *output.ResultEvent, templatePath string) strin // Close closes the exporter after operation func (i *Exporter) Close() error { + i.mutex.Lock() + defer i.mutex.Unlock() + i.sarif.AddRun(i.run) file, err := os.Create(i.options.File) From 1977d41a1e1aec1a284f76672bf3db77431c3cd4 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Sat, 5 Jun 2021 21:42:04 +0530 Subject: [PATCH 11/25] Misc --- v2/pkg/reporting/exporters/sarif/sarif.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/v2/pkg/reporting/exporters/sarif/sarif.go b/v2/pkg/reporting/exporters/sarif/sarif.go index a5040aea..4c3b9ebd 100644 --- a/v2/pkg/reporting/exporters/sarif/sarif.go +++ b/v2/pkg/reporting/exporters/sarif/sarif.go @@ -148,24 +148,24 @@ func getSarifResultMessage(event *output.ResultEvent, templatePath string) strin if len(event.ExtractedResults) > 0 || len(event.Metadata) > 0 { if len(event.ExtractedResults) > 0 { - builder.WriteString(" **Extracted results**:\n\n") + builder.WriteString(" **Extracted results**:

") for _, v := range event.ExtractedResults { builder.WriteString("- ") builder.WriteString(v) - builder.WriteString("\n") + builder.WriteString("
") } - builder.WriteString("\n") + builder.WriteString("
") } if len(event.Metadata) > 0 { - builder.WriteString(" **Metadata**:\n\n") + builder.WriteString(" **Metadata**:
") for k, v := range event.Metadata { builder.WriteString("- ") builder.WriteString(k) builder.WriteString(": ") builder.WriteString(types.ToString(v)) - builder.WriteString("\n") + builder.WriteString("
") } - builder.WriteString("\n") + builder.WriteString("
") } } if event.Interaction != nil { @@ -173,7 +173,7 @@ func getSarifResultMessage(event *output.ResultEvent, templatePath string) strin builder.WriteString(event.Interaction.Protocol) } - builder.WriteString(" To Reproduce - `nuclei -t ") + builder.WriteString("
To Reproduce - `nuclei -t ") builder.WriteString(strings.TrimPrefix(templatePath, "/")) builder.WriteString(" -target \"") builder.WriteString(event.Host) From d9bb1393d34aba583c3caece02088bbe157f4c1b Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Sat, 5 Jun 2021 23:00:59 +0530 Subject: [PATCH 12/25] Improvements to sarif report --- v2/pkg/protocols/common/clusterer/executer.go | 2 +- v2/pkg/reporting/exporters/sarif/sarif.go | 119 +++++------------- v2/pkg/reporting/format/format.go | 18 +++ v2/pkg/reporting/trackers/jira/jira.go | 18 +++ v2/pkg/templates/compile.go | 1 + v2/pkg/templates/templates.go | 2 + 6 files changed, 70 insertions(+), 90 deletions(-) diff --git a/v2/pkg/protocols/common/clusterer/executer.go b/v2/pkg/protocols/common/clusterer/executer.go index 45621bbe..9c676f77 100644 --- a/v2/pkg/protocols/common/clusterer/executer.go +++ b/v2/pkg/protocols/common/clusterer/executer.go @@ -39,7 +39,7 @@ func NewExecuter(requests []*templates.Template, options *protocols.ExecuterOpti executer.operators = append(executer.operators, &clusteredOperator{ templateID: req.ID, templateInfo: req.Info, - templatePath: req.Options.TemplatePath, + templatePath: req.Path, operator: req.RequestsHTTP[0].CompiledOperators, }) } diff --git a/v2/pkg/reporting/exporters/sarif/sarif.go b/v2/pkg/reporting/exporters/sarif/sarif.go index 4c3b9ebd..f9895cff 100644 --- a/v2/pkg/reporting/exporters/sarif/sarif.go +++ b/v2/pkg/reporting/exporters/sarif/sarif.go @@ -1,7 +1,9 @@ package sarif import ( - "bytes" + "crypto/sha1" + "encoding/hex" + "io/ioutil" "os" "path" "strings" @@ -11,7 +13,6 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/format" - "github.com/projectdiscovery/nuclei/v2/pkg/types" ) // Exporter is an exporter for nuclei sarif output format. @@ -20,8 +21,9 @@ type Exporter struct { run *sarif.Run mutex *sync.Mutex - home string - options *Options + home string + tempFile string + options *Options } // Options contains the configuration options for sarif exporter client @@ -36,6 +38,15 @@ func New(options *Options) (*Exporter, error) { if err != nil { return nil, errors.Wrap(err, "could not create sarif exporter") } + tempFile, err := ioutil.TempFile("", "sarif-test-*") + if err != nil { + return nil, errors.Wrap(err, "could not create sarif temp file") + } + defer tempFile.Close() + + tempFile.WriteString("github.com/projectdiscovery/nuclei Scan Result") + tempFileName := tempFile.Name() + home, err := os.UserHomeDir() if err != nil { return nil, errors.Wrap(err, "could not get home dir") @@ -43,16 +54,19 @@ func New(options *Options) (*Exporter, error) { templatePath := path.Join(home, "nuclei-templates") run := sarif.NewRun("nuclei", "https://github.com/projectdiscovery/nuclei") - return &Exporter{options: options, home: templatePath, sarif: report, run: run, mutex: &sync.Mutex{}}, nil + return &Exporter{options: options, tempFile: tempFileName, home: templatePath, sarif: report, run: run, mutex: &sync.Mutex{}}, nil } // Export exports a passed result event to sarif structure func (i *Exporter) Export(event *output.ResultEvent) error { templatePath := strings.TrimPrefix(event.TemplatePath, i.home) - description := getSarifResultMessage(event, templatePath) + h := sha1.New() + h.Write([]byte(event.Host)) + templateID := event.TemplateID + "-" + hex.EncodeToString(h.Sum(nil)) + + fullDescription := format.MarkdownDescription(event) sarifSeverity := getSarifSeverity(event) - sarifRuleHelpURIs := getSarifRuleHelpURIFromReferences(event) var ruleName string if s, ok := event.Info["name"]; ok { @@ -68,33 +82,22 @@ func (i *Exporter) Export(event *output.ResultEvent) error { if d, ok := event.Info["description"]; ok { ruleDescription = d.(string) } - builder := &strings.Builder{} - builder.WriteString(ruleDescription) - if sarifRuleHelpURIs != "" { - builder.WriteString("\nReferences: \n") - builder.WriteString(sarifRuleHelpURIs) - } - if templateURL != "" { - builder.WriteString("\nTemplate URL: ") - builder.WriteString(templateURL) - } - ruleHelp := builder.String() i.mutex.Lock() defer i.mutex.Unlock() - _ = i.run.AddRule(event.TemplateID). + _ = i.run.AddRule(templateID). WithDescription(ruleName). - WithHelp(ruleHelp). + WithHelp(fullDescription). WithHelpURI(templateURL). - WithFullDescription(sarif.NewMultiformatMessageString(sarifRuleHelpURIs)) + WithFullDescription(sarif.NewMultiformatMessageString(ruleDescription)) - _ = i.run.AddResult(event.TemplateID). - WithMessage(sarif.NewMessage().WithText(description)). + _ = i.run.AddResult(templateID). + WithMessage(sarif.NewMessage().WithText(event.Host)). WithLevel(sarifSeverity). WithLocation(sarif.NewLocation().WithMessage(sarif.NewMessage().WithText(event.Host)).WithPhysicalLocation( sarif.NewPhysicalLocation(). - WithArtifactLocation(sarif.NewArtifactLocation().WithUri(event.Type)). + WithArtifactLocation(sarif.NewArtifactLocation().WithUri(i.tempFile)). WithRegion(sarif.NewRegion().WithStartColumn(1).WithStartLine(1).WithEndLine(1).WithEndColumn(1)), )) return nil @@ -119,77 +122,15 @@ func getSarifSeverity(event *output.ResultEvent) string { } } -// getSarifRuleHelpURIFromReferences returns the sarif rule help uri -func getSarifRuleHelpURIFromReferences(event *output.ResultEvent) string { - if d, ok := event.Info["reference"]; ok { - switch v := d.(type) { - case string: - return v - case []interface{}: - slice := types.ToStringSlice(v) - return strings.Join(slice, "\n") - } - } - return "" -} - -// getSarifResultMessage gets a sarif result message from event -func getSarifResultMessage(event *output.ResultEvent, templatePath string) string { - template := format.GetMatchedTemplate(event) - builder := &bytes.Buffer{} - - builder.WriteString(template) - builder.WriteString(" matched at ") - builder.WriteString(event.Host) - builder.WriteString(" (") - builder.WriteString(strings.ToUpper(event.Type)) - builder.WriteString(") => ") - builder.WriteString(event.Matched) - - if len(event.ExtractedResults) > 0 || len(event.Metadata) > 0 { - if len(event.ExtractedResults) > 0 { - builder.WriteString(" **Extracted results**:

") - for _, v := range event.ExtractedResults { - builder.WriteString("- ") - builder.WriteString(v) - builder.WriteString("
") - } - builder.WriteString("
") - } - if len(event.Metadata) > 0 { - builder.WriteString(" **Metadata**:
") - for k, v := range event.Metadata { - builder.WriteString("- ") - builder.WriteString(k) - builder.WriteString(": ") - builder.WriteString(types.ToString(v)) - builder.WriteString("
") - } - builder.WriteString("
") - } - } - if event.Interaction != nil { - builder.WriteString("**Interaction Data**\n---\n") - builder.WriteString(event.Interaction.Protocol) - } - - builder.WriteString("
To Reproduce - `nuclei -t ") - builder.WriteString(strings.TrimPrefix(templatePath, "/")) - builder.WriteString(" -target \"") - builder.WriteString(event.Host) - builder.WriteString("\"`") - - data := builder.String() - return data -} - // Close closes the exporter after operation func (i *Exporter) Close() error { i.mutex.Lock() defer i.mutex.Unlock() i.sarif.AddRun(i.run) - + if len(i.run.Results) == 0 { + return nil // do not write when no results + } file, err := os.Create(i.options.File) if err != nil { return errors.Wrap(err, "could not create sarif output file") diff --git a/v2/pkg/reporting/format/format.go b/v2/pkg/reporting/format/format.go index 153b6c24..5743fcb2 100644 --- a/v2/pkg/reporting/format/format.go +++ b/v2/pkg/reporting/format/format.go @@ -110,6 +110,24 @@ func MarkdownDescription(event *output.ResultEvent) string { builder.WriteString("\n```\n") } } + if d, ok := event.Info["reference"]; ok { + builder.WriteString("\nReference: \n") + + switch v := d.(type) { + case string: + builder.WriteString("- ") + builder.WriteString(v) + case []interface{}: + slice := types.ToStringSlice(v) + for i, item := range slice { + builder.WriteString("- ") + builder.WriteString(item) + if len(slice)-1 != i { + builder.WriteString("\n") + } + } + } + } builder.WriteString("\n---\nGenerated by [Nuclei](https://github.com/projectdiscovery/nuclei)") data := builder.String() diff --git a/v2/pkg/reporting/trackers/jira/jira.go b/v2/pkg/reporting/trackers/jira/jira.go index 0c68b4ce..0cd2bbee 100644 --- a/v2/pkg/reporting/trackers/jira/jira.go +++ b/v2/pkg/reporting/trackers/jira/jira.go @@ -153,6 +153,24 @@ func jiraFormatDescription(event *output.ResultEvent) string { builder.WriteString("\n{code}\n") } } + if d, ok := event.Info["reference"]; ok { + builder.WriteString("\nReference: \n") + + switch v := d.(type) { + case string: + builder.WriteString("- ") + builder.WriteString(v) + case []interface{}: + slice := types.ToStringSlice(v) + for i, item := range slice { + builder.WriteString("- ") + builder.WriteString(item) + if len(slice)-1 != i { + builder.WriteString("\n") + } + } + } + } builder.WriteString("\n---\nGenerated by [Nuclei|https://github.com/projectdiscovery/nuclei]") data := builder.String() return data diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 4cad174b..16b6ff21 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -142,6 +142,7 @@ func Parse(filePath string, options protocols.ExecuterOptions) (*Template, error if template.Executer == nil && template.CompiledWorkflow == nil { return nil, errors.New("cannot create template executer") } + template.Path = filePath return template, nil } diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index 61037302..12c135e4 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -35,4 +35,6 @@ type Template struct { TotalRequests int `yaml:"-" json:"-"` // Executer is the actual template executor for running template requests Executer protocols.Executer `yaml:"-" json:"-"` + + Path string `yaml:"-" json:"-"` } From 1dc1b9ee73a62139f430300a81a0a00d81e4003b Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Sat, 5 Jun 2021 23:09:08 +0530 Subject: [PATCH 13/25] Print first line of running action if any --- v2/pkg/reporting/exporters/sarif/sarif.go | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/v2/pkg/reporting/exporters/sarif/sarif.go b/v2/pkg/reporting/exporters/sarif/sarif.go index f9895cff..5ac5c37b 100644 --- a/v2/pkg/reporting/exporters/sarif/sarif.go +++ b/v2/pkg/reporting/exporters/sarif/sarif.go @@ -3,7 +3,6 @@ package sarif import ( "crypto/sha1" "encoding/hex" - "io/ioutil" "os" "path" "strings" @@ -38,14 +37,6 @@ func New(options *Options) (*Exporter, error) { if err != nil { return nil, errors.Wrap(err, "could not create sarif exporter") } - tempFile, err := ioutil.TempFile("", "sarif-test-*") - if err != nil { - return nil, errors.Wrap(err, "could not create sarif temp file") - } - defer tempFile.Close() - - tempFile.WriteString("github.com/projectdiscovery/nuclei Scan Result") - tempFileName := tempFile.Name() home, err := os.UserHomeDir() if err != nil { @@ -54,7 +45,7 @@ func New(options *Options) (*Exporter, error) { templatePath := path.Join(home, "nuclei-templates") run := sarif.NewRun("nuclei", "https://github.com/projectdiscovery/nuclei") - return &Exporter{options: options, tempFile: tempFileName, home: templatePath, sarif: report, run: run, mutex: &sync.Mutex{}}, nil + return &Exporter{options: options, home: templatePath, sarif: report, run: run, mutex: &sync.Mutex{}}, nil } // Export exports a passed result event to sarif structure @@ -91,13 +82,12 @@ func (i *Exporter) Export(event *output.ResultEvent) error { WithHelp(fullDescription). WithHelpURI(templateURL). WithFullDescription(sarif.NewMultiformatMessageString(ruleDescription)) - _ = i.run.AddResult(templateID). WithMessage(sarif.NewMessage().WithText(event.Host)). WithLevel(sarifSeverity). WithLocation(sarif.NewLocation().WithMessage(sarif.NewMessage().WithText(event.Host)).WithPhysicalLocation( sarif.NewPhysicalLocation(). - WithArtifactLocation(sarif.NewArtifactLocation().WithUri(i.tempFile)). + WithArtifactLocation(sarif.NewArtifactLocation().WithUri(os.Getenv("github.action_path"))). WithRegion(sarif.NewRegion().WithStartColumn(1).WithStartLine(1).WithEndLine(1).WithEndColumn(1)), )) return nil @@ -112,13 +102,13 @@ func getSarifSeverity(event *output.ResultEvent) string { switch ruleSeverity { case "info": - return "none" + return "note" case "low", "medium": return "warning" case "high", "critical": return "error" default: - return "none" + return "note" } } From e6db12848f2c459fe92c05b29b0fb19f9b368c32 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Sat, 5 Jun 2021 23:42:37 +0530 Subject: [PATCH 14/25] Temporary fix for sarif github location issue --- v2/pkg/reporting/exporters/sarif/sarif.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/reporting/exporters/sarif/sarif.go b/v2/pkg/reporting/exporters/sarif/sarif.go index 5ac5c37b..4c222732 100644 --- a/v2/pkg/reporting/exporters/sarif/sarif.go +++ b/v2/pkg/reporting/exporters/sarif/sarif.go @@ -87,7 +87,7 @@ func (i *Exporter) Export(event *output.ResultEvent) error { WithLevel(sarifSeverity). WithLocation(sarif.NewLocation().WithMessage(sarif.NewMessage().WithText(event.Host)).WithPhysicalLocation( sarif.NewPhysicalLocation(). - WithArtifactLocation(sarif.NewArtifactLocation().WithUri(os.Getenv("github.action_path"))). + WithArtifactLocation(sarif.NewArtifactLocation().WithUri("README.md")). WithRegion(sarif.NewRegion().WithStartColumn(1).WithStartLine(1).WithEndLine(1).WithEndColumn(1)), )) return nil From eb96e701b50767ec854601af7053795f4c2779b9 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Sat, 5 Jun 2021 23:43:37 +0530 Subject: [PATCH 15/25] Fixed reference formatting in reports --- v2/pkg/reporting/format/format.go | 3 +++ v2/pkg/reporting/trackers/jira/jira.go | 3 +++ 2 files changed, 6 insertions(+) diff --git a/v2/pkg/reporting/format/format.go b/v2/pkg/reporting/format/format.go index 5743fcb2..c7327be1 100644 --- a/v2/pkg/reporting/format/format.go +++ b/v2/pkg/reporting/format/format.go @@ -44,6 +44,9 @@ func MarkdownDescription(event *output.ResultEvent) string { builder.WriteString(event.Timestamp.Format("Mon Jan 2 15:04:05 -0700 MST 2006")) builder.WriteString("\n\n**Template Information**\n\n| Key | Value |\n|---|---|\n") for k, v := range event.Info { + if k == "reference" { + continue + } builder.WriteString(fmt.Sprintf("| %s | %s |\n", k, v)) } if event.Request != "" { diff --git a/v2/pkg/reporting/trackers/jira/jira.go b/v2/pkg/reporting/trackers/jira/jira.go index 0cd2bbee..8b305205 100644 --- a/v2/pkg/reporting/trackers/jira/jira.go +++ b/v2/pkg/reporting/trackers/jira/jira.go @@ -92,6 +92,9 @@ func jiraFormatDescription(event *output.ResultEvent) string { builder.WriteString(event.Timestamp.Format("Mon Jan 2 15:04:05 -0700 MST 2006")) builder.WriteString("\n\n*Template Information*\n\n| Key | Value |\n") for k, v := range event.Info { + if k == "reference" { + continue + } builder.WriteString(fmt.Sprintf("| %s | %s |\n", k, v)) } builder.WriteString("\n*Request*\n\n{code}\n") From 5d744c7c20792c109526c77765d1aefed9aab1ee Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Sun, 6 Jun 2021 01:04:20 +0530 Subject: [PATCH 16/25] Fixed reference printing bug --- v2/pkg/reporting/format/format.go | 4 +++- v2/pkg/reporting/trackers/jira/jira.go | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/v2/pkg/reporting/format/format.go b/v2/pkg/reporting/format/format.go index c7327be1..0ce622b3 100644 --- a/v2/pkg/reporting/format/format.go +++ b/v2/pkg/reporting/format/format.go @@ -118,7 +118,9 @@ func MarkdownDescription(event *output.ResultEvent) string { switch v := d.(type) { case string: - builder.WriteString("- ") + if !strings.HasPrefix(v, "-") { + builder.WriteString("- ") + } builder.WriteString(v) case []interface{}: slice := types.ToStringSlice(v) diff --git a/v2/pkg/reporting/trackers/jira/jira.go b/v2/pkg/reporting/trackers/jira/jira.go index 8b305205..cbd91892 100644 --- a/v2/pkg/reporting/trackers/jira/jira.go +++ b/v2/pkg/reporting/trackers/jira/jira.go @@ -161,7 +161,9 @@ func jiraFormatDescription(event *output.ResultEvent) string { switch v := d.(type) { case string: - builder.WriteString("- ") + if !strings.HasPrefix(v, "-") { + builder.WriteString("- ") + } builder.WriteString(v) case []interface{}: slice := types.ToStringSlice(v) From 2c0cea1a3f3c60d73731fc9f910eabb60cb714d8 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Sun, 6 Jun 2021 02:56:36 +0530 Subject: [PATCH 17/25] Changed name of flag --- v2/cmd/nuclei/main.go | 2 +- v2/internal/runner/runner.go | 6 +++--- v2/pkg/types/types.go | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 0ce15511..6758a855 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -85,7 +85,7 @@ based on templates offering massive extensibility and ease of use.`) set.IntVar(&options.PageTimeout, "page-timeout", 20, "Seconds to wait for each page in headless") set.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "Only run newly added templates") set.StringVarP(&options.DiskExportDirectory, "disk-export", "de", "", "Directory on disk to export reports in markdown to") - set.StringVarP(&options.SarifFile, "sarif-file", "sf", "", "File to export sarif report to") + set.StringVarP(&options.SarifExport, "sarif-export", "sf", "", "File to export sarif report to") set.BoolVar(&options.NoInteractsh, "no-interactsh", false, "Do not use interactsh server for blind interaction polling") set.StringVar(&options.InteractshURL, "interactsh-url", "https://interact.sh", "Interactsh Server URL") set.IntVar(&options.InteractionsCacheSize, "interactions-cache-size", 5000, "Number of requests to keep in interactions cache") diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 45e7dce4..09bc9520 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -96,12 +96,12 @@ func New(options *types.Options) (*Runner, error) { reportingOptions.DiskExporter = &disk.Options{Directory: options.DiskExportDirectory} } } - if options.SarifFile != "" { + if options.SarifExport != "" { if reportingOptions != nil { - reportingOptions.SarifExporter = &sarif.Options{File: options.SarifFile} + reportingOptions.SarifExporter = &sarif.Options{File: options.SarifExport} } else { reportingOptions = &reporting.Options{} - reportingOptions.SarifExporter = &sarif.Options{File: options.SarifFile} + reportingOptions.SarifExporter = &sarif.Options{File: options.SarifExport} } } if reportingOptions != nil { diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 1ae84d9b..89cb701e 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -47,8 +47,8 @@ type Options struct { ReportingConfig string // DiskExportDirectory is the directory to export reports in markdown on disk to DiskExportDirectory string - // SarifFile is the file to export sarif output format to - SarifFile string + // SarifExport is the file to export sarif output format to + SarifExport string // ResolversFile is a file containing resolvers for nuclei. ResolversFile string // StatsInterval is the number of seconds to display stats after From b99f028a660349df4fb453507808dc8ac32408f2 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Sun, 6 Jun 2021 15:52:13 +0530 Subject: [PATCH 18/25] Enrich file output sarif with correct file metadata --- v2/pkg/output/output.go | 2 ++ v2/pkg/protocols/file/operators.go | 30 ++++++++++++++++++++++ v2/pkg/reporting/exporters/sarif/sarif.go | 31 ++++++++++++++++++++--- 3 files changed, 60 insertions(+), 3 deletions(-) diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index bd72c894..ffe1b180 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -84,6 +84,8 @@ type ResultEvent struct { Timestamp time.Time `json:"timestamp"` // Interaction is the full details of interactsh interaction. Interaction *server.Interaction `json:"interaction,omitempty"` + + FileToIndexPosition map[string]int `json:"-"` } // NewStandardWriter creates a new output writer based on user configurations diff --git a/v2/pkg/protocols/file/operators.go b/v2/pkg/protocols/file/operators.go index e7afdaef..674cd09c 100644 --- a/v2/pkg/protocols/file/operators.go +++ b/v2/pkg/protocols/file/operators.go @@ -1,6 +1,8 @@ package file import ( + "bufio" + "strings" "time" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" @@ -100,6 +102,33 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu data := r.makeResultEventItem(wrapped) results = append(results, data) } + raw, ok := wrapped.InternalEvent["raw"] + if !ok { + return results + } + rawStr, ok := raw.(string) + if !ok { + return results + } + + // Identify the position of match in file using a dirty hack. + for _, result := range results { + for _, extraction := range result.ExtractedResults { + scanner := bufio.NewScanner(strings.NewReader(rawStr)) + + line := 1 + for scanner.Scan() { + if strings.Contains(scanner.Text(), extraction) { + if result.FileToIndexPosition != nil { + result.FileToIndexPosition = make(map[string]int) + } + result.FileToIndexPosition[result.Matched] = line + continue + } + line++ + } + } + } return results } @@ -111,6 +140,7 @@ func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *out Type: "file", Path: types.ToString(wrapped.InternalEvent["path"]), Matched: types.ToString(wrapped.InternalEvent["matched"]), + Host: types.ToString(wrapped.InternalEvent["matched"]), ExtractedResults: wrapped.OperatorsResult.OutputExtracts, Timestamp: time.Now(), } diff --git a/v2/pkg/reporting/exporters/sarif/sarif.go b/v2/pkg/reporting/exporters/sarif/sarif.go index 4c222732..1b5d7a5a 100644 --- a/v2/pkg/reporting/exporters/sarif/sarif.go +++ b/v2/pkg/reporting/exporters/sarif/sarif.go @@ -67,6 +67,8 @@ func (i *Exporter) Export(event *output.ResultEvent) error { var templateURL string if strings.HasPrefix(event.TemplatePath, i.home) { templateURL = "https://github.com/projectdiscovery/nuclei-templates/blob/master" + templatePath + } else { + templateURL = "https://github.com/projectdiscovery/nuclei-templates" } var ruleDescription string @@ -82,14 +84,27 @@ func (i *Exporter) Export(event *output.ResultEvent) error { WithHelp(fullDescription). WithHelpURI(templateURL). WithFullDescription(sarif.NewMultiformatMessageString(ruleDescription)) - _ = i.run.AddResult(templateID). + result := i.run.AddResult(templateID). WithMessage(sarif.NewMessage().WithText(event.Host)). - WithLevel(sarifSeverity). - WithLocation(sarif.NewLocation().WithMessage(sarif.NewMessage().WithText(event.Host)).WithPhysicalLocation( + WithLevel(sarifSeverity) + + // Also write file match metadata to file + if event.Type == "file" && event.FileToIndexPosition != nil { + for file, line := range event.FileToIndexPosition { + sc, sl, el, ec := getLimitedLineAndColumns(line) + result.WithLocation(sarif.NewLocation().WithMessage(sarif.NewMessage().WithText(file)).WithPhysicalLocation( + sarif.NewPhysicalLocation(). + WithArtifactLocation(sarif.NewArtifactLocation().WithUri(file)). + WithRegion(sarif.NewRegion().WithStartColumn(sc).WithStartLine(sl).WithEndLine(el).WithEndColumn(ec)), + )) + } + } else { + result.WithLocation(sarif.NewLocation().WithMessage(sarif.NewMessage().WithText(event.Host)).WithPhysicalLocation( sarif.NewPhysicalLocation(). WithArtifactLocation(sarif.NewArtifactLocation().WithUri("README.md")). WithRegion(sarif.NewRegion().WithStartColumn(1).WithStartLine(1).WithEndLine(1).WithEndColumn(1)), )) + } return nil } @@ -112,6 +127,16 @@ func getSarifSeverity(event *output.ResultEvent) string { } } +func getLimitedLineAndColumns(line int) (int, int, int, int) { + var endline = line + + if line-3 != 0 { + endline = line + 6 + line = line - 3 + } + return 1, line, endline, 32 +} + // Close closes the exporter after operation func (i *Exporter) Close() error { i.mutex.Lock() From 7a962c8436d27036def5c1dd4be0cd8d96e55edf Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Sun, 6 Jun 2021 15:54:49 +0530 Subject: [PATCH 19/25] Correct offsets --- v2/pkg/reporting/exporters/sarif/sarif.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/v2/pkg/reporting/exporters/sarif/sarif.go b/v2/pkg/reporting/exporters/sarif/sarif.go index 1b5d7a5a..185bab20 100644 --- a/v2/pkg/reporting/exporters/sarif/sarif.go +++ b/v2/pkg/reporting/exporters/sarif/sarif.go @@ -128,10 +128,9 @@ func getSarifSeverity(event *output.ResultEvent) string { } func getLimitedLineAndColumns(line int) (int, int, int, int) { - var endline = line + endline := line + 3 - if line-3 != 0 { - endline = line + 6 + if line-3 > 0 { line = line - 3 } return 1, line, endline, 32 From 31a74a33903ca0b076554f884368f9b7b4de5b4b Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Sun, 6 Jun 2021 15:57:22 +0530 Subject: [PATCH 20/25] Fix nil pointer panic --- v2/pkg/protocols/file/operators.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/protocols/file/operators.go b/v2/pkg/protocols/file/operators.go index 674cd09c..f2a57a3d 100644 --- a/v2/pkg/protocols/file/operators.go +++ b/v2/pkg/protocols/file/operators.go @@ -119,7 +119,7 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu line := 1 for scanner.Scan() { if strings.Contains(scanner.Text(), extraction) { - if result.FileToIndexPosition != nil { + if result.FileToIndexPosition == nil { result.FileToIndexPosition = make(map[string]int) } result.FileToIndexPosition[result.Matched] = line From c913612fd369b93241d6e9e71c3bcea379e4988f Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Sun, 6 Jun 2021 16:04:06 +0530 Subject: [PATCH 21/25] Misc --- v2/pkg/reporting/exporters/sarif/sarif.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/v2/pkg/reporting/exporters/sarif/sarif.go b/v2/pkg/reporting/exporters/sarif/sarif.go index 185bab20..45750fb0 100644 --- a/v2/pkg/reporting/exporters/sarif/sarif.go +++ b/v2/pkg/reporting/exporters/sarif/sarif.go @@ -91,11 +91,10 @@ func (i *Exporter) Export(event *output.ResultEvent) error { // Also write file match metadata to file if event.Type == "file" && event.FileToIndexPosition != nil { for file, line := range event.FileToIndexPosition { - sc, sl, el, ec := getLimitedLineAndColumns(line) result.WithLocation(sarif.NewLocation().WithMessage(sarif.NewMessage().WithText(file)).WithPhysicalLocation( sarif.NewPhysicalLocation(). WithArtifactLocation(sarif.NewArtifactLocation().WithUri(file)). - WithRegion(sarif.NewRegion().WithStartColumn(sc).WithStartLine(sl).WithEndLine(el).WithEndColumn(ec)), + WithRegion(sarif.NewRegion().WithStartColumn(1).WithStartLine(line).WithEndLine(line).WithEndColumn(32)), )) } } else { From 8b52645905784a31b496aae57dccec7629328544 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Sun, 6 Jun 2021 16:12:54 +0530 Subject: [PATCH 22/25] Add name as rule help --- v2/pkg/reporting/exporters/sarif/sarif.go | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/v2/pkg/reporting/exporters/sarif/sarif.go b/v2/pkg/reporting/exporters/sarif/sarif.go index 45750fb0..e3ff74e4 100644 --- a/v2/pkg/reporting/exporters/sarif/sarif.go +++ b/v2/pkg/reporting/exporters/sarif/sarif.go @@ -91,7 +91,7 @@ func (i *Exporter) Export(event *output.ResultEvent) error { // Also write file match metadata to file if event.Type == "file" && event.FileToIndexPosition != nil { for file, line := range event.FileToIndexPosition { - result.WithLocation(sarif.NewLocation().WithMessage(sarif.NewMessage().WithText(file)).WithPhysicalLocation( + result.WithLocation(sarif.NewLocation().WithMessage(sarif.NewMessage().WithText(ruleName)).WithPhysicalLocation( sarif.NewPhysicalLocation(). WithArtifactLocation(sarif.NewArtifactLocation().WithUri(file)). WithRegion(sarif.NewRegion().WithStartColumn(1).WithStartLine(line).WithEndLine(line).WithEndColumn(32)), @@ -126,15 +126,6 @@ func getSarifSeverity(event *output.ResultEvent) string { } } -func getLimitedLineAndColumns(line int) (int, int, int, int) { - endline := line + 3 - - if line-3 > 0 { - line = line - 3 - } - return 1, line, endline, 32 -} - // Close closes the exporter after operation func (i *Exporter) Close() error { i.mutex.Lock() From 3ce48f3799a6b5a69991a685ac5cef3190267e01 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Sun, 6 Jun 2021 17:38:39 +0530 Subject: [PATCH 23/25] Misc --- v2/pkg/reporting/exporters/sarif/sarif.go | 2 +- v2/pkg/reporting/format/format.go | 4 ++-- v2/pkg/reporting/trackers/jira/jira.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/v2/pkg/reporting/exporters/sarif/sarif.go b/v2/pkg/reporting/exporters/sarif/sarif.go index e3ff74e4..c495c3e5 100644 --- a/v2/pkg/reporting/exporters/sarif/sarif.go +++ b/v2/pkg/reporting/exporters/sarif/sarif.go @@ -89,7 +89,7 @@ func (i *Exporter) Export(event *output.ResultEvent) error { WithLevel(sarifSeverity) // Also write file match metadata to file - if event.Type == "file" && event.FileToIndexPosition != nil { + if event.Type == "file" && (event.FileToIndexPosition != nil && len(event.FileToIndexPosition) > 0) { for file, line := range event.FileToIndexPosition { result.WithLocation(sarif.NewLocation().WithMessage(sarif.NewMessage().WithText(ruleName)).WithPhysicalLocation( sarif.NewPhysicalLocation(). diff --git a/v2/pkg/reporting/format/format.go b/v2/pkg/reporting/format/format.go index 0ce622b3..ec099795 100644 --- a/v2/pkg/reporting/format/format.go +++ b/v2/pkg/reporting/format/format.go @@ -63,11 +63,11 @@ func MarkdownDescription(event *output.ResultEvent) string { } else { builder.WriteString(event.Response) } - builder.WriteString("\n```\n\n") + builder.WriteString("\n```\n") } if len(event.ExtractedResults) > 0 || len(event.Metadata) > 0 { - builder.WriteString("**Extra Information**\n\n") + builder.WriteString("\n**Extra Information**\n\n") if len(event.ExtractedResults) > 0 { builder.WriteString("**Extracted results**:\n\n") for _, v := range event.ExtractedResults { diff --git a/v2/pkg/reporting/trackers/jira/jira.go b/v2/pkg/reporting/trackers/jira/jira.go index cbd91892..15e8ed67 100644 --- a/v2/pkg/reporting/trackers/jira/jira.go +++ b/v2/pkg/reporting/trackers/jira/jira.go @@ -110,7 +110,7 @@ func jiraFormatDescription(event *output.ResultEvent) string { builder.WriteString("\n{code}\n\n") if len(event.ExtractedResults) > 0 || len(event.Metadata) > 0 { - builder.WriteString("*Extra Information*\n\n") + builder.WriteString("\n*Extra Information*\n\n") if len(event.ExtractedResults) > 0 { builder.WriteString("*Extracted results*:\n\n") for _, v := range event.ExtractedResults { From 63b274ecb0e634649d424db1d3f19efdf06534a1 Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Tue, 8 Jun 2021 16:38:13 +0530 Subject: [PATCH 24/25] flag updates / improvements --- v2/cmd/nuclei/main.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 6758a855..596138f5 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -84,10 +84,10 @@ based on templates offering massive extensibility and ease of use.`) set.BoolVar(&options.SystemResolvers, "system-resolvers", false, "Use system dns resolving as error fallback") set.IntVar(&options.PageTimeout, "page-timeout", 20, "Seconds to wait for each page in headless") set.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "Only run newly added templates") - set.StringVarP(&options.DiskExportDirectory, "disk-export", "de", "", "Directory on disk to export reports in markdown to") - set.StringVarP(&options.SarifExport, "sarif-export", "sf", "", "File to export sarif report to") + set.StringVarP(&options.DiskExportDirectory, "markdown-export", "me", "", "Directory to export results in markdown format") + set.StringVarP(&options.SarifExport, "sarif-export", "se", "", "File to export results in sarif format") set.BoolVar(&options.NoInteractsh, "no-interactsh", false, "Do not use interactsh server for blind interaction polling") - set.StringVar(&options.InteractshURL, "interactsh-url", "https://interact.sh", "Interactsh Server URL") + set.StringVar(&options.InteractshURL, "interactsh-url", "https://interact.sh", "Self Hosted Interactsh Server URL") set.IntVar(&options.InteractionsCacheSize, "interactions-cache-size", 5000, "Number of requests to keep in interactions cache") set.IntVar(&options.InteractionsEviction, "interactions-eviction", 60, "Number of seconds to wait before evicting requests from cache") set.IntVar(&options.InteractionsPollDuration, "interactions-poll-duration", 5, "Number of seconds before each interaction poll request") From bd38c16e0aff9f4536bcf6e6d002ffff9e8f6155 Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Tue, 8 Jun 2021 16:46:34 +0530 Subject: [PATCH 25/25] Update banner.go --- v2/internal/runner/banner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/internal/runner/banner.go b/v2/internal/runner/banner.go index 6ee30910..7bb3f8dc 100644 --- a/v2/internal/runner/banner.go +++ b/v2/internal/runner/banner.go @@ -7,7 +7,7 @@ const banner = ` ____ __ _______/ /__ (_) / __ \/ / / / ___/ / _ \/ / / / / / /_/ / /__/ / __/ / - /_/ /_/\__,_/\___/_/\___/_/ v2.3.7 + /_/ /_/\__,_/\___/_/\___/_/ v2.3.7-dev ` // Version is the current version of nuclei