diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 2020dd10..20c71a09 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -13,10 +13,10 @@ jobs: - name: Checkout code uses: actions/checkout@v2 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v2.5.2 + uses: golangci/golangci-lint-action with: # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. - version: v1.33 + version: latest args: --timeout 5m working-directory: v2/ diff --git a/.github/workflows/dockerhub-push.yml b/.github/workflows/dockerhub-push.yml index 8a93961a..1e235e7a 100644 --- a/.github/workflows/dockerhub-push.yml +++ b/.github/workflows/dockerhub-push.yml @@ -1,19 +1,17 @@ -# dockerhub-push pushes docker build to dockerhub automatically -# on the creation of a new release -name: Publish to Dockerhub on creation of a new release -on: - release: - types: [published] +name: ◎ Docker Push +on: + workflow_dispatch: + jobs: update: runs-on: ubuntu-latest steps: - - uses: actions/checkout@master + - uses: actions/checkout@v2 - name: Publish to Dockerhub Registry - #pre: echo ::save-state name=RELEASE_VERSION::$(echo ${GITHUB_REF:10}) - uses: elgohr/Publish-Docker-Github-Action@master + uses: dawidd6/action-docker-publish-changed@v3 with: name: projectdiscovery/nuclei - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - tags: "latest" #"latest,${{ env.STATE_RELEASE_VERSION }}" + username: ${{secrets.DOCKER_USERNAME}} + password: ${{secrets.DOCKER_PASSWORD}} + platforms: linux/amd64,linux/arm64,linux/arm + tag: latest \ No newline at end of file diff --git a/.golangci.yml b/.golangci.yml index 6c88a19f..9a2621ab 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -66,7 +66,6 @@ linters: - gocritic - gofmt - goimports - - golint #- gomnd - goprintffuncname - gosimple @@ -89,6 +88,7 @@ linters: - unused - varcheck - whitespace + - revive # don't enable: # - depguard @@ -105,11 +105,4 @@ linters: # - nestif # - prealloc # - testpackage - # - wsl - -# golangci.com configuration -# https://github.com/golangci/golangci/wiki/Configuration -service: - golangci-lint-version: 1.33.x # use the fixed version to not introduce new linters unexpectedly - prepare: - - echo "here I can run custom commands, but no preparation needed for this repo" + # - wsl \ No newline at end of file diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000..fc8033f4 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,5 @@ +# Security Policy + +## Reporting a Vulnerability + +DO NOT CREATE AN ISSUE to report a security problem. Instead, please send an email to security@projectdiscovery.io and we will acknowledge it within 3 working days. diff --git a/v2/cmd/nuclei/issue-tracker-config.yaml b/v2/cmd/nuclei/issue-tracker-config.yaml index f69342c7..3fffce49 100644 --- a/v2/cmd/nuclei/issue-tracker-config.yaml +++ b/v2/cmd/nuclei/issue-tracker-config.yaml @@ -32,14 +32,16 @@ # issue-label: "" # jira contains configuration options for jira issue tracker -#jira: +#jira: +# # Cloud is the boolean which tells if Jira instance is running in the cloud or on-prem version is used +# cloud: true # # URL is the jira application url # url: "" -# # account-id is the account-id of the jira user +# # account-id is the account-id of the jira user or username in case of on-prem Jira # account-id: "" # # email is the email of the user for jira instance # email: "" -# # token is the token for jira instance. +# # token is the token for jira instance or password in case of on-prem Jira # token: "" # # project-name is the name of the project. # project-name: "" diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 574da4fc..596138f5 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -84,9 +84,10 @@ based on templates offering massive extensibility and ease of use.`) set.BoolVar(&options.SystemResolvers, "system-resolvers", false, "Use system dns resolving as error fallback") set.IntVar(&options.PageTimeout, "page-timeout", 20, "Seconds to wait for each page in headless") set.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "Only run newly added templates") - set.StringVarP(&options.DiskExportDirectory, "disk-export", "de", "", "Directory on disk to export reports in markdown to") + set.StringVarP(&options.DiskExportDirectory, "markdown-export", "me", "", "Directory to export results in markdown format") + set.StringVarP(&options.SarifExport, "sarif-export", "se", "", "File to export results in sarif format") set.BoolVar(&options.NoInteractsh, "no-interactsh", false, "Do not use interactsh server for blind interaction polling") - set.StringVar(&options.InteractshURL, "interactsh-url", "https://interact.sh", "Interactsh Server URL") + set.StringVar(&options.InteractshURL, "interactsh-url", "https://interact.sh", "Self Hosted Interactsh Server URL") set.IntVar(&options.InteractionsCacheSize, "interactions-cache-size", 5000, "Number of requests to keep in interactions cache") set.IntVar(&options.InteractionsEviction, "interactions-eviction", 60, "Number of seconds to wait before evicting requests from cache") set.IntVar(&options.InteractionsPollDuration, "interactions-poll-duration", 5, "Number of seconds before each interaction poll request") diff --git a/v2/go.mod b/v2/go.mod index 8f5daa16..4c13a5e2 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -23,6 +23,7 @@ require ( github.com/miekg/dns v1.1.38 github.com/mitchellh/go-ps v1.0.0 github.com/olekukonko/tablewriter v0.0.5 + github.com/owenrumney/go-sarif v1.0.4 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.8 github.com/projectdiscovery/collaborator v0.0.2 @@ -31,7 +32,7 @@ require ( github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.1 github.com/projectdiscovery/interactsh v0.0.3 - github.com/projectdiscovery/rawhttp v0.0.6 + github.com/projectdiscovery/rawhttp v0.0.7 github.com/projectdiscovery/retryabledns v1.0.10 github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210524224054-9fbe1f2b0727 github.com/remeh/sizedwaitgroup v1.0.0 diff --git a/v2/go.sum b/v2/go.sum index f8f415d6..0c6abf1f 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -40,6 +40,7 @@ github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF0 github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= github.com/andygrunwald/go-jira v1.13.0 h1:vvIImGgX32bHfoiyUwkNo+/YrPnRczNarvhLOncP6dE= github.com/andygrunwald/go-jira v1.13.0/go.mod h1:jYi4kFDbRPZTJdJOVJO4mpMMIwdB+rcZwSO58DzPd2I= +github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -206,6 +207,8 @@ github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+ github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/owenrumney/go-sarif v1.0.4 h1:0LFC5eHP6amc/9ajM1jDiE52UfXFcl/oozay+X3KgV4= +github.com/owenrumney/go-sarif v1.0.4/go.mod h1:DXUGbHwQcCMvqcvZbxh8l/7diHsJVztOKZgmPt88RNI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -226,8 +229,8 @@ github.com/projectdiscovery/hmap v0.0.1 h1:VAONbJw5jP+syI5smhsfkrq9XPGn4aiYy5pR6 github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0= github.com/projectdiscovery/interactsh v0.0.3 h1:PUkWk+NzSyd5glMqfORmuqizhsd7c3WdTYBOto/MQIU= github.com/projectdiscovery/interactsh v0.0.3/go.mod h1:dWnKO14d2FLP3kLhI9DecEsiAC/aZiJoUBGFjGhDskY= -github.com/projectdiscovery/rawhttp v0.0.6 h1:HbgPB1eKXQVV5F9sq0Uxflm95spWFyZYD8dgFpeOC9M= -github.com/projectdiscovery/rawhttp v0.0.6/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= +github.com/projectdiscovery/rawhttp v0.0.7 h1:5m4peVgjbl7gqDcRYMTVEuX+Xs/nh76ohTkkvufucLg= +github.com/projectdiscovery/rawhttp v0.0.7/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= github.com/projectdiscovery/retryabledns v1.0.7/go.mod h1:/UzJn4I+cPdQl6pKiiQfvVAT636YZvJQYZhYhGB0dUQ= github.com/projectdiscovery/retryabledns v1.0.10 h1:xJZ2aKoqrNg/OZEw1+4+QIOH40V/WkZDYY1ZZc+uphE= github.com/projectdiscovery/retryabledns v1.0.10/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4= @@ -267,6 +270,8 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= +github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ= github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM= github.com/xanzy/go-gitlab v0.44.0 h1:cEiGhqu7EpFGuei2a2etAwB+x6403E5CvpLn35y+GPs= @@ -284,6 +289,8 @@ github.com/ysmood/leakless v0.6.12/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNq github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/zclconf/go-cty v1.8.2 h1:u+xZfBKgpycDnTNjPhGiTEYZS5qS/Sb5MqSfm7vzcjg= +github.com/zclconf/go-cty v1.8.2/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -427,6 +434,7 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/v2/internal/runner/banner.go b/v2/internal/runner/banner.go index 6ee30910..7bb3f8dc 100644 --- a/v2/internal/runner/banner.go +++ b/v2/internal/runner/banner.go @@ -7,7 +7,7 @@ const banner = ` ____ __ _______/ /__ (_) / __ \/ / / / ___/ / _ \/ / / / / / /_/ / /__/ / __/ / - /_/ /_/\__,_/\___/_/\___/_/ v2.3.7 + /_/ /_/\__,_/\___/_/\___/_/ v2.3.7-dev ` // Version is the current version of nuclei diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 40a965f2..09bc9520 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -24,6 +24,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine" "github.com/projectdiscovery/nuclei/v2/pkg/reporting" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/disk" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/sarif" "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/remeh/sizedwaitgroup" @@ -95,6 +96,14 @@ func New(options *types.Options) (*Runner, error) { reportingOptions.DiskExporter = &disk.Options{Directory: options.DiskExportDirectory} } } + if options.SarifExport != "" { + if reportingOptions != nil { + reportingOptions.SarifExporter = &sarif.Options{File: options.SarifExport} + } else { + reportingOptions = &reporting.Options{} + reportingOptions.SarifExporter = &sarif.Options{File: options.SarifExport} + } + } if reportingOptions != nil { if client, err := reporting.New(reportingOptions, options.ReportingDB); err != nil { gologger.Fatal().Msgf("Could not create issue reporting client: %s\n", err) diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index 2fac10fd..ffe1b180 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -54,6 +54,8 @@ type InternalWrappedEvent struct { type ResultEvent struct { // TemplateID is the ID of the template for the result. TemplateID string `json:"templateID"` + // TemplatePath is the path of template + TemplatePath string `json:"-"` // Info contains information block of the template for the result. Info map[string]interface{} `json:"info,inline"` // MatcherName is the name of the matcher matched if any. @@ -82,6 +84,8 @@ type ResultEvent struct { Timestamp time.Time `json:"timestamp"` // Interaction is the full details of interactsh interaction. Interaction *server.Interaction `json:"interaction,omitempty"` + + FileToIndexPosition map[string]int `json:"-"` } // NewStandardWriter creates a new output writer based on user configurations diff --git a/v2/pkg/protocols/common/clusterer/executer.go b/v2/pkg/protocols/common/clusterer/executer.go index 84207ab1..758e99b8 100644 --- a/v2/pkg/protocols/common/clusterer/executer.go +++ b/v2/pkg/protocols/common/clusterer/executer.go @@ -22,6 +22,7 @@ type Executer struct { type clusteredOperator struct { templateID string + templatePath string templateInfo map[string]interface{} operator *operators.Operators } @@ -38,6 +39,7 @@ func NewExecuter(requests []*templates.Template, options *protocols.ExecuterOpti executer.operators = append(executer.operators, &clusteredOperator{ templateID: req.ID, templateInfo: req.Info, + templatePath: req.Path, operator: req.RequestsHTTP[0].CompiledOperators, }) } @@ -60,13 +62,15 @@ func (e *Executer) Requests() int { func (e *Executer) Execute(input string) (bool, error) { var results bool + previous := make(map[string]interface{}) dynamicValues := make(map[string]interface{}) - err := e.requests.ExecuteWithResults(input, dynamicValues, nil, func(event *output.InternalWrappedEvent) { + err := e.requests.ExecuteWithResults(input, dynamicValues, previous, func(event *output.InternalWrappedEvent) { for _, operator := range e.operators { result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract) if matched && result != nil { event.OperatorsResult = result event.InternalEvent["template-id"] = operator.templateID + event.InternalEvent["template-path"] = operator.templatePath event.InternalEvent["template-info"] = operator.templateInfo event.Results = e.requests.MakeResultEvent(event) results = true @@ -94,6 +98,7 @@ func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEve if matched && result != nil { event.OperatorsResult = result event.InternalEvent["template-id"] = operator.templateID + event.InternalEvent["template-path"] = operator.templatePath event.InternalEvent["template-info"] = operator.templateInfo event.Results = e.requests.MakeResultEvent(event) callback(event) diff --git a/v2/pkg/protocols/dns/operators.go b/v2/pkg/protocols/dns/operators.go index 860a38f8..5d18c4d5 100644 --- a/v2/pkg/protocols/dns/operators.go +++ b/v2/pkg/protocols/dns/operators.go @@ -103,6 +103,7 @@ func (r *Request) responseToDSLMap(req, resp *dns.Msg, host, matched string) out data["raw"] = rawData data["template-id"] = r.options.TemplateID data["template-info"] = r.options.TemplateInfo + data["template-path"] = r.options.TemplatePath return data } @@ -137,6 +138,7 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), + TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(map[string]interface{}), Type: "dns", Host: types.ToString(wrapped.InternalEvent["host"]), diff --git a/v2/pkg/protocols/file/operators.go b/v2/pkg/protocols/file/operators.go index 7b3b59fe..f2a57a3d 100644 --- a/v2/pkg/protocols/file/operators.go +++ b/v2/pkg/protocols/file/operators.go @@ -1,6 +1,8 @@ package file import ( + "bufio" + "strings" "time" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" @@ -71,6 +73,7 @@ func (r *Request) responseToDSLMap(raw, host, matched string) output.InternalEve data["raw"] = raw data["template-id"] = r.options.TemplateID data["template-info"] = r.options.TemplateInfo + data["template-path"] = r.options.TemplatePath return data } @@ -99,16 +102,45 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu data := r.makeResultEventItem(wrapped) results = append(results, data) } + raw, ok := wrapped.InternalEvent["raw"] + if !ok { + return results + } + rawStr, ok := raw.(string) + if !ok { + return results + } + + // Identify the position of match in file using a dirty hack. + for _, result := range results { + for _, extraction := range result.ExtractedResults { + scanner := bufio.NewScanner(strings.NewReader(rawStr)) + + line := 1 + for scanner.Scan() { + if strings.Contains(scanner.Text(), extraction) { + if result.FileToIndexPosition == nil { + result.FileToIndexPosition = make(map[string]int) + } + result.FileToIndexPosition[result.Matched] = line + continue + } + line++ + } + } + } return results } func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), + TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(map[string]interface{}), Type: "file", Path: types.ToString(wrapped.InternalEvent["path"]), Matched: types.ToString(wrapped.InternalEvent["matched"]), + Host: types.ToString(wrapped.InternalEvent["matched"]), ExtractedResults: wrapped.OperatorsResult.OutputExtracts, Timestamp: time.Now(), } diff --git a/v2/pkg/protocols/headless/engine/engine.go b/v2/pkg/protocols/headless/engine/engine.go index 6c8d613e..ea85f975 100644 --- a/v2/pkg/protocols/headless/engine/engine.go +++ b/v2/pkg/protocols/headless/engine/engine.go @@ -99,6 +99,7 @@ func (b *Browser) Close() { // headless process launch. func (b *Browser) killChromeProcesses() { newProcesses := b.findChromeProcesses() + for id := range newProcesses { if _, ok := b.previouspids[id]; ok { continue diff --git a/v2/pkg/protocols/headless/engine/page.go b/v2/pkg/protocols/headless/engine/page.go index 185eb33c..16eae572 100644 --- a/v2/pkg/protocols/headless/engine/page.go +++ b/v2/pkg/protocols/headless/engine/page.go @@ -49,7 +49,6 @@ func (i *Instance) Run(baseURL *url.URL, actions []*Action, timeout time.Duratio if err != nil { return nil, nil, err } - go router.Run() data, err := createdPage.ExecuteActions(baseURL, actions) if err != nil { diff --git a/v2/pkg/protocols/headless/engine/page_actions_test.go b/v2/pkg/protocols/headless/engine/page_actions_test.go index e89fe535..658c3290 100644 --- a/v2/pkg/protocols/headless/engine/page_actions_test.go +++ b/v2/pkg/protocols/headless/engine/page_actions_test.go @@ -23,6 +23,7 @@ func TestActionNavigate(t *testing.T) { instance, err := browser.NewInstance() require.Nil(t, err, "could not create browser instance") + defer instance.Close() ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(w, ` diff --git a/v2/pkg/protocols/headless/operators.go b/v2/pkg/protocols/headless/operators.go index f1845a89..265dc95d 100644 --- a/v2/pkg/protocols/headless/operators.go +++ b/v2/pkg/protocols/headless/operators.go @@ -72,6 +72,7 @@ func (r *Request) responseToDSLMap(resp, req, host, matched string) output.Inter data["data"] = resp data["template-id"] = r.options.TemplateID data["template-info"] = r.options.TemplateInfo + data["template-path"] = r.options.TemplatePath return data } @@ -106,6 +107,7 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), + TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(map[string]interface{}), Type: "headless", Host: types.ToString(wrapped.InternalEvent["host"]), diff --git a/v2/pkg/protocols/http/cluster.go b/v2/pkg/protocols/http/cluster.go index 68107c9e..bdfa0735 100644 --- a/v2/pkg/protocols/http/cluster.go +++ b/v2/pkg/protocols/http/cluster.go @@ -10,7 +10,7 @@ import ( // are similar enough to be considered one and can be checked by // just adding the matcher/extractors for the request and the correct IDs. func (r *Request) CanCluster(other *Request) bool { - if len(r.Payloads) > 0 || len(r.Raw) > 0 || len(r.Body) > 0 || r.Unsafe { + if len(r.Payloads) > 0 || len(r.Raw) > 0 || len(r.Body) > 0 || r.Unsafe || r.ReqCondition || r.Name != "" { return false } if r.Method != other.Method || diff --git a/v2/pkg/protocols/http/operators.go b/v2/pkg/protocols/http/operators.go index 0c1fcb99..77438f04 100644 --- a/v2/pkg/protocols/http/operators.go +++ b/v2/pkg/protocols/http/operators.go @@ -105,6 +105,7 @@ func (r *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, r data["duration"] = duration.Seconds() data["template-id"] = r.options.TemplateID data["template-info"] = r.options.TemplateInfo + data["template-path"] = r.options.TemplatePath return data } @@ -139,6 +140,7 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), + TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(map[string]interface{}), Type: "http", Host: types.ToString(wrapped.InternalEvent["host"]), diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 6dc361fa..d3f774a5 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -117,7 +117,7 @@ func (r *Request) executeParallelHTTP(reqURL string, dynamicValues, previous out return requestErr } -// executeRaceRequest executes turbo http request for a URL +// executeTurboHTTP executes turbo http request for a URL func (r *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { generator := r.newGenerator() @@ -277,11 +277,15 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ var hostname string timeStart := time.Now() if request.original.Pipeline { - formedURL = request.rawRequest.FullURL - if parsed, parseErr := url.Parse(formedURL); parseErr == nil { - hostname = parsed.Host + if request.rawRequest != nil { + formedURL = request.rawRequest.FullURL + if parsed, parseErr := url.Parse(formedURL); parseErr == nil { + hostname = parsed.Host + } + resp, err = request.pipelinedClient.DoRaw(request.rawRequest.Method, reqURL, request.rawRequest.Path, generators.ExpandMapValues(request.rawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.rawRequest.Data))) + } else if request.request != nil { + resp, err = request.pipelinedClient.Dor(request.request) } - resp, err = request.pipelinedClient.DoRaw(request.rawRequest.Method, reqURL, request.rawRequest.Path, generators.ExpandMapValues(request.rawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.rawRequest.Data))) } else if request.original.Unsafe && request.rawRequest != nil { formedURL = request.rawRequest.FullURL if parsed, parseErr := url.Parse(formedURL); parseErr == nil { diff --git a/v2/pkg/protocols/network/operators.go b/v2/pkg/protocols/network/operators.go index a0a95ede..d515e686 100644 --- a/v2/pkg/protocols/network/operators.go +++ b/v2/pkg/protocols/network/operators.go @@ -73,6 +73,7 @@ func (r *Request) responseToDSLMap(req, resp, raw, host, matched string) output. data["raw"] = raw // Raw is the full transaction data for network data["template-id"] = r.options.TemplateID data["template-info"] = r.options.TemplateInfo + data["template-path"] = r.options.TemplatePath return data } @@ -107,6 +108,7 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), + TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(map[string]interface{}), Type: "network", Host: types.ToString(wrapped.InternalEvent["host"]), diff --git a/v2/pkg/protocols/offlinehttp/operators.go b/v2/pkg/protocols/offlinehttp/operators.go index c71376c9..d60cf72f 100644 --- a/v2/pkg/protocols/offlinehttp/operators.go +++ b/v2/pkg/protocols/offlinehttp/operators.go @@ -101,6 +101,7 @@ func (r *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, r data["duration"] = duration.Seconds() data["template-id"] = r.options.TemplateID data["template-info"] = r.options.TemplateInfo + data["template-path"] = r.options.TemplatePath return data } @@ -135,6 +136,7 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), + TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), Info: wrapped.InternalEvent["template-info"].(map[string]interface{}), Type: "http", Path: types.ToString(wrapped.InternalEvent["path"]), diff --git a/v2/pkg/reporting/exporters/disk/disk.go b/v2/pkg/reporting/exporters/disk/disk.go index 35c1d25b..7ff73739 100644 --- a/v2/pkg/reporting/exporters/disk/disk.go +++ b/v2/pkg/reporting/exporters/disk/disk.go @@ -58,3 +58,8 @@ func (i *Exporter) Export(event *output.ResultEvent) error { err := ioutil.WriteFile(path.Join(i.directory, finalFilename), data, 0644) return err } + +// Close closes the exporter after operation +func (i *Exporter) Close() error { + return nil +} diff --git a/v2/pkg/reporting/exporters/sarif/sarif.go b/v2/pkg/reporting/exporters/sarif/sarif.go new file mode 100644 index 00000000..c495c3e5 --- /dev/null +++ b/v2/pkg/reporting/exporters/sarif/sarif.go @@ -0,0 +1,144 @@ +package sarif + +import ( + "crypto/sha1" + "encoding/hex" + "os" + "path" + "strings" + "sync" + + "github.com/owenrumney/go-sarif/sarif" + "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/format" +) + +// Exporter is an exporter for nuclei sarif output format. +type Exporter struct { + sarif *sarif.Report + run *sarif.Run + mutex *sync.Mutex + + home string + tempFile string + options *Options +} + +// Options contains the configuration options for sarif exporter client +type Options struct { + // File is the file to export found sarif result to + File string `yaml:"file"` +} + +// New creates a new disk exporter integration client based on options. +func New(options *Options) (*Exporter, error) { + report, err := sarif.New(sarif.Version210) + if err != nil { + return nil, errors.Wrap(err, "could not create sarif exporter") + } + + home, err := os.UserHomeDir() + if err != nil { + return nil, errors.Wrap(err, "could not get home dir") + } + templatePath := path.Join(home, "nuclei-templates") + + run := sarif.NewRun("nuclei", "https://github.com/projectdiscovery/nuclei") + return &Exporter{options: options, home: templatePath, sarif: report, run: run, mutex: &sync.Mutex{}}, nil +} + +// Export exports a passed result event to sarif structure +func (i *Exporter) Export(event *output.ResultEvent) error { + templatePath := strings.TrimPrefix(event.TemplatePath, i.home) + + h := sha1.New() + h.Write([]byte(event.Host)) + templateID := event.TemplateID + "-" + hex.EncodeToString(h.Sum(nil)) + + fullDescription := format.MarkdownDescription(event) + sarifSeverity := getSarifSeverity(event) + + var ruleName string + if s, ok := event.Info["name"]; ok { + ruleName = s.(string) + } + + var templateURL string + if strings.HasPrefix(event.TemplatePath, i.home) { + templateURL = "https://github.com/projectdiscovery/nuclei-templates/blob/master" + templatePath + } else { + templateURL = "https://github.com/projectdiscovery/nuclei-templates" + } + + var ruleDescription string + if d, ok := event.Info["description"]; ok { + ruleDescription = d.(string) + } + + i.mutex.Lock() + defer i.mutex.Unlock() + + _ = i.run.AddRule(templateID). + WithDescription(ruleName). + WithHelp(fullDescription). + WithHelpURI(templateURL). + WithFullDescription(sarif.NewMultiformatMessageString(ruleDescription)) + result := i.run.AddResult(templateID). + WithMessage(sarif.NewMessage().WithText(event.Host)). + WithLevel(sarifSeverity) + + // Also write file match metadata to file + if event.Type == "file" && (event.FileToIndexPosition != nil && len(event.FileToIndexPosition) > 0) { + for file, line := range event.FileToIndexPosition { + result.WithLocation(sarif.NewLocation().WithMessage(sarif.NewMessage().WithText(ruleName)).WithPhysicalLocation( + sarif.NewPhysicalLocation(). + WithArtifactLocation(sarif.NewArtifactLocation().WithUri(file)). + WithRegion(sarif.NewRegion().WithStartColumn(1).WithStartLine(line).WithEndLine(line).WithEndColumn(32)), + )) + } + } else { + result.WithLocation(sarif.NewLocation().WithMessage(sarif.NewMessage().WithText(event.Host)).WithPhysicalLocation( + sarif.NewPhysicalLocation(). + WithArtifactLocation(sarif.NewArtifactLocation().WithUri("README.md")). + WithRegion(sarif.NewRegion().WithStartColumn(1).WithStartLine(1).WithEndLine(1).WithEndColumn(1)), + )) + } + return nil +} + +// getSarifSeverity returns the sarif severity +func getSarifSeverity(event *output.ResultEvent) string { + var ruleSeverity string + if s, ok := event.Info["severity"]; ok { + ruleSeverity = s.(string) + } + + switch ruleSeverity { + case "info": + return "note" + case "low", "medium": + return "warning" + case "high", "critical": + return "error" + default: + return "note" + } +} + +// Close closes the exporter after operation +func (i *Exporter) Close() error { + i.mutex.Lock() + defer i.mutex.Unlock() + + i.sarif.AddRun(i.run) + if len(i.run.Results) == 0 { + return nil // do not write when no results + } + file, err := os.Create(i.options.File) + if err != nil { + return errors.Wrap(err, "could not create sarif output file") + } + defer file.Close() + return i.sarif.Write(file) +} diff --git a/v2/pkg/reporting/format/format.go b/v2/pkg/reporting/format/format.go index 153b6c24..ec099795 100644 --- a/v2/pkg/reporting/format/format.go +++ b/v2/pkg/reporting/format/format.go @@ -44,6 +44,9 @@ func MarkdownDescription(event *output.ResultEvent) string { builder.WriteString(event.Timestamp.Format("Mon Jan 2 15:04:05 -0700 MST 2006")) builder.WriteString("\n\n**Template Information**\n\n| Key | Value |\n|---|---|\n") for k, v := range event.Info { + if k == "reference" { + continue + } builder.WriteString(fmt.Sprintf("| %s | %s |\n", k, v)) } if event.Request != "" { @@ -60,11 +63,11 @@ func MarkdownDescription(event *output.ResultEvent) string { } else { builder.WriteString(event.Response) } - builder.WriteString("\n```\n\n") + builder.WriteString("\n```\n") } if len(event.ExtractedResults) > 0 || len(event.Metadata) > 0 { - builder.WriteString("**Extra Information**\n\n") + builder.WriteString("\n**Extra Information**\n\n") if len(event.ExtractedResults) > 0 { builder.WriteString("**Extracted results**:\n\n") for _, v := range event.ExtractedResults { @@ -110,6 +113,26 @@ func MarkdownDescription(event *output.ResultEvent) string { builder.WriteString("\n```\n") } } + if d, ok := event.Info["reference"]; ok { + builder.WriteString("\nReference: \n") + + switch v := d.(type) { + case string: + if !strings.HasPrefix(v, "-") { + builder.WriteString("- ") + } + builder.WriteString(v) + case []interface{}: + slice := types.ToStringSlice(v) + for i, item := range slice { + builder.WriteString("- ") + builder.WriteString(item) + if len(slice)-1 != i { + builder.WriteString("\n") + } + } + } + } builder.WriteString("\n---\nGenerated by [Nuclei](https://github.com/projectdiscovery/nuclei)") data := builder.String() diff --git a/v2/pkg/reporting/reporting.go b/v2/pkg/reporting/reporting.go index e33600bb..7eaf2602 100644 --- a/v2/pkg/reporting/reporting.go +++ b/v2/pkg/reporting/reporting.go @@ -7,6 +7,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/dedupe" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/disk" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/sarif" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/github" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/gitlab" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/jira" @@ -28,6 +29,8 @@ type Options struct { Jira *jira.Options `yaml:"jira"` // DiskExporter contains configuration options for Disk Exporter Module DiskExporter *disk.Options `yaml:"disk"` + // SarifExporter contains configuration options for Sarif Exporter Module + SarifExporter *sarif.Options `yaml:"sarif"` } // Filter filters the received event and decides whether to perform @@ -79,6 +82,8 @@ type Tracker interface { // Exporter is an interface implemented by an issue exporter type Exporter interface { + // Close closes the exporter after operation + Close() error // Export exports an issue to an exporter Export(event *output.ResultEvent) error } @@ -129,6 +134,13 @@ func New(options *Options, db string) (*Client, error) { } client.exporters = append(client.exporters, exporter) } + if options.SarifExporter != nil { + exporter, err := sarif.New(options.SarifExporter) + if err != nil { + return nil, errors.Wrap(err, "could not create exporting client") + } + client.exporters = append(client.exporters, exporter) + } storage, err := dedupe.New(db) if err != nil { return nil, err @@ -140,6 +152,9 @@ func New(options *Options, db string) (*Client, error) { // Close closes the issue tracker reporting client func (c *Client) Close() { c.dedupe.Close() + for _, exporter := range c.exporters { + exporter.Close() + } } // CreateIssue creates an issue in the tracker diff --git a/v2/pkg/reporting/trackers/jira/jira.go b/v2/pkg/reporting/trackers/jira/jira.go index 0c68b4ce..5a56ed9e 100644 --- a/v2/pkg/reporting/trackers/jira/jira.go +++ b/v2/pkg/reporting/trackers/jira/jira.go @@ -20,6 +20,8 @@ type Integration struct { // Options contains the configuration options for jira client type Options struct { + // Cloud value is set to true when Jira cloud is used + Cloud bool `yaml:"cloud"` // URL is the URL of the jira server URL string `yaml:"url"` // AccountID is the accountID of the jira user. @@ -36,8 +38,12 @@ type Options struct { // New creates a new issue tracker integration client based on options. func New(options *Options) (*Integration, error) { + username := options.Email + if !options.Cloud { + username = options.AccountID + } tp := jira.BasicAuthTransport{ - Username: options.Email, + Username: username, Password: options.Token, } jiraClient, err := jira.NewClient(tp.Client(), options.URL) @@ -51,15 +57,27 @@ func New(options *Options) (*Integration, error) { func (i *Integration) CreateIssue(event *output.ResultEvent) error { summary := format.Summary(event) - issueData := &jira.Issue{ - Fields: &jira.IssueFields{ - Assignee: &jira.User{AccountID: i.options.AccountID}, - Reporter: &jira.User{AccountID: i.options.AccountID}, + fields := &jira.IssueFields{ + Assignee: &jira.User{AccountID: i.options.AccountID}, + Reporter: &jira.User{AccountID: i.options.AccountID}, + Description: jiraFormatDescription(event), + Type: jira.IssueType{Name: i.options.IssueType}, + Project: jira.Project{Key: i.options.ProjectName}, + Summary: summary, + } + // On-prem version of Jira server does not use AccountID + if !i.options.Cloud { + fields = &jira.IssueFields{ + Assignee: &jira.User{Name: i.options.AccountID}, Description: jiraFormatDescription(event), Type: jira.IssueType{Name: i.options.IssueType}, Project: jira.Project{Key: i.options.ProjectName}, Summary: summary, - }, + } + } + + issueData := &jira.Issue{ + Fields: fields, } _, resp, err := i.jira.Issue.Create(issueData) if err != nil { @@ -92,6 +110,9 @@ func jiraFormatDescription(event *output.ResultEvent) string { builder.WriteString(event.Timestamp.Format("Mon Jan 2 15:04:05 -0700 MST 2006")) builder.WriteString("\n\n*Template Information*\n\n| Key | Value |\n") for k, v := range event.Info { + if k == "reference" { + continue + } builder.WriteString(fmt.Sprintf("| %s | %s |\n", k, v)) } builder.WriteString("\n*Request*\n\n{code}\n") @@ -107,7 +128,7 @@ func jiraFormatDescription(event *output.ResultEvent) string { builder.WriteString("\n{code}\n\n") if len(event.ExtractedResults) > 0 || len(event.Metadata) > 0 { - builder.WriteString("*Extra Information*\n\n") + builder.WriteString("\n*Extra Information*\n\n") if len(event.ExtractedResults) > 0 { builder.WriteString("*Extracted results*:\n\n") for _, v := range event.ExtractedResults { @@ -153,6 +174,26 @@ func jiraFormatDescription(event *output.ResultEvent) string { builder.WriteString("\n{code}\n") } } + if d, ok := event.Info["reference"]; ok { + builder.WriteString("\nReference: \n") + + switch v := d.(type) { + case string: + if !strings.HasPrefix(v, "-") { + builder.WriteString("- ") + } + builder.WriteString(v) + case []interface{}: + slice := types.ToStringSlice(v) + for i, item := range slice { + builder.WriteString("- ") + builder.WriteString(item) + if len(slice)-1 != i { + builder.WriteString("\n") + } + } + } + } builder.WriteString("\n---\nGenerated by [Nuclei|https://github.com/projectdiscovery/nuclei]") data := builder.String() return data diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 4cad174b..16b6ff21 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -142,6 +142,7 @@ func Parse(filePath string, options protocols.ExecuterOptions) (*Template, error if template.Executer == nil && template.CompiledWorkflow == nil { return nil, errors.New("cannot create template executer") } + template.Path = filePath return template, nil } diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index 61037302..12c135e4 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -35,4 +35,6 @@ type Template struct { TotalRequests int `yaml:"-" json:"-"` // Executer is the actual template executor for running template requests Executer protocols.Executer `yaml:"-" json:"-"` + + Path string `yaml:"-" json:"-"` } diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index d9b40700..89cb701e 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -47,6 +47,8 @@ type Options struct { ReportingConfig string // DiskExportDirectory is the directory to export reports in markdown on disk to DiskExportDirectory string + // SarifExport is the file to export sarif output format to + SarifExport string // ResolversFile is a file containing resolvers for nuclei. ResolversFile string // StatsInterval is the number of seconds to display stats after