Merge branch 'dev' into bugfix-709-http2

dev
Mzack9999 2021-06-11 17:20:40 +02:00
commit 29e2ee7a0e
31 changed files with 355 additions and 49 deletions

View File

@ -13,10 +13,10 @@ jobs:
- name: Checkout code
uses: actions/checkout@v2
- name: Run golangci-lint
uses: golangci/golangci-lint-action@v2.5.2
uses: golangci/golangci-lint-action
with:
# Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
version: v1.33
version: latest
args: --timeout 5m
working-directory: v2/

View File

@ -1,19 +1,17 @@
# dockerhub-push pushes docker build to dockerhub automatically
# on the creation of a new release
name: Publish to Dockerhub on creation of a new release
on:
release:
types: [published]
name: ◎ Docker Push
on:
workflow_dispatch:
jobs:
update:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@master
- uses: actions/checkout@v2
- name: Publish to Dockerhub Registry
#pre: echo ::save-state name=RELEASE_VERSION::$(echo ${GITHUB_REF:10})
uses: elgohr/Publish-Docker-Github-Action@master
uses: dawidd6/action-docker-publish-changed@v3
with:
name: projectdiscovery/nuclei
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
tags: "latest" #"latest,${{ env.STATE_RELEASE_VERSION }}"
username: ${{secrets.DOCKER_USERNAME}}
password: ${{secrets.DOCKER_PASSWORD}}
platforms: linux/amd64,linux/arm64,linux/arm
tag: latest

View File

@ -66,7 +66,6 @@ linters:
- gocritic
- gofmt
- goimports
- golint
#- gomnd
- goprintffuncname
- gosimple
@ -89,6 +88,7 @@ linters:
- unused
- varcheck
- whitespace
- revive
# don't enable:
# - depguard
@ -105,11 +105,4 @@ linters:
# - nestif
# - prealloc
# - testpackage
# - wsl
# golangci.com configuration
# https://github.com/golangci/golangci/wiki/Configuration
service:
golangci-lint-version: 1.33.x # use the fixed version to not introduce new linters unexpectedly
prepare:
- echo "here I can run custom commands, but no preparation needed for this repo"
# - wsl

5
SECURITY.md Normal file
View File

@ -0,0 +1,5 @@
# Security Policy
## Reporting a Vulnerability
DO NOT CREATE AN ISSUE to report a security problem. Instead, please send an email to security@projectdiscovery.io and we will acknowledge it within 3 working days.

View File

@ -32,14 +32,16 @@
# issue-label: ""
# jira contains configuration options for jira issue tracker
#jira:
#jira:
# # Cloud is the boolean which tells if Jira instance is running in the cloud or on-prem version is used
# cloud: true
# # URL is the jira application url
# url: ""
# # account-id is the account-id of the jira user
# # account-id is the account-id of the jira user or username in case of on-prem Jira
# account-id: ""
# # email is the email of the user for jira instance
# email: ""
# # token is the token for jira instance.
# # token is the token for jira instance or password in case of on-prem Jira
# token: ""
# # project-name is the name of the project.
# project-name: ""

View File

@ -84,9 +84,10 @@ based on templates offering massive extensibility and ease of use.`)
set.BoolVar(&options.SystemResolvers, "system-resolvers", false, "Use system dns resolving as error fallback")
set.IntVar(&options.PageTimeout, "page-timeout", 20, "Seconds to wait for each page in headless")
set.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "Only run newly added templates")
set.StringVarP(&options.DiskExportDirectory, "disk-export", "de", "", "Directory on disk to export reports in markdown to")
set.StringVarP(&options.DiskExportDirectory, "markdown-export", "me", "", "Directory to export results in markdown format")
set.StringVarP(&options.SarifExport, "sarif-export", "se", "", "File to export results in sarif format")
set.BoolVar(&options.NoInteractsh, "no-interactsh", false, "Do not use interactsh server for blind interaction polling")
set.StringVar(&options.InteractshURL, "interactsh-url", "https://interact.sh", "Interactsh Server URL")
set.StringVar(&options.InteractshURL, "interactsh-url", "https://interact.sh", "Self Hosted Interactsh Server URL")
set.IntVar(&options.InteractionsCacheSize, "interactions-cache-size", 5000, "Number of requests to keep in interactions cache")
set.IntVar(&options.InteractionsEviction, "interactions-eviction", 60, "Number of seconds to wait before evicting requests from cache")
set.IntVar(&options.InteractionsPollDuration, "interactions-poll-duration", 5, "Number of seconds before each interaction poll request")

View File

@ -23,6 +23,7 @@ require (
github.com/miekg/dns v1.1.38
github.com/mitchellh/go-ps v1.0.0
github.com/olekukonko/tablewriter v0.0.5
github.com/owenrumney/go-sarif v1.0.4
github.com/pkg/errors v0.9.1
github.com/projectdiscovery/clistats v0.0.8
github.com/projectdiscovery/collaborator v0.0.2
@ -31,7 +32,7 @@ require (
github.com/projectdiscovery/gologger v1.1.4
github.com/projectdiscovery/hmap v0.0.1
github.com/projectdiscovery/interactsh v0.0.3
github.com/projectdiscovery/rawhttp v0.0.6
github.com/projectdiscovery/rawhttp v0.0.7
github.com/projectdiscovery/retryabledns v1.0.10
github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210524224054-9fbe1f2b0727
github.com/remeh/sizedwaitgroup v1.0.0

View File

@ -40,6 +40,7 @@ github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF0
github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA=
github.com/andygrunwald/go-jira v1.13.0 h1:vvIImGgX32bHfoiyUwkNo+/YrPnRczNarvhLOncP6dE=
github.com/andygrunwald/go-jira v1.13.0/go.mod h1:jYi4kFDbRPZTJdJOVJO4mpMMIwdB+rcZwSO58DzPd2I=
github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo=
github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
@ -206,6 +207,8 @@ github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/owenrumney/go-sarif v1.0.4 h1:0LFC5eHP6amc/9ajM1jDiE52UfXFcl/oozay+X3KgV4=
github.com/owenrumney/go-sarif v1.0.4/go.mod h1:DXUGbHwQcCMvqcvZbxh8l/7diHsJVztOKZgmPt88RNI=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@ -226,8 +229,8 @@ github.com/projectdiscovery/hmap v0.0.1 h1:VAONbJw5jP+syI5smhsfkrq9XPGn4aiYy5pR6
github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0=
github.com/projectdiscovery/interactsh v0.0.3 h1:PUkWk+NzSyd5glMqfORmuqizhsd7c3WdTYBOto/MQIU=
github.com/projectdiscovery/interactsh v0.0.3/go.mod h1:dWnKO14d2FLP3kLhI9DecEsiAC/aZiJoUBGFjGhDskY=
github.com/projectdiscovery/rawhttp v0.0.6 h1:HbgPB1eKXQVV5F9sq0Uxflm95spWFyZYD8dgFpeOC9M=
github.com/projectdiscovery/rawhttp v0.0.6/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
github.com/projectdiscovery/rawhttp v0.0.7 h1:5m4peVgjbl7gqDcRYMTVEuX+Xs/nh76ohTkkvufucLg=
github.com/projectdiscovery/rawhttp v0.0.7/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0=
github.com/projectdiscovery/retryabledns v1.0.7/go.mod h1:/UzJn4I+cPdQl6pKiiQfvVAT636YZvJQYZhYhGB0dUQ=
github.com/projectdiscovery/retryabledns v1.0.10 h1:xJZ2aKoqrNg/OZEw1+4+QIOH40V/WkZDYY1ZZc+uphE=
github.com/projectdiscovery/retryabledns v1.0.10/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4=
@ -267,6 +270,8 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4=
github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4=
github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI=
github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ=
github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM=
github.com/xanzy/go-gitlab v0.44.0 h1:cEiGhqu7EpFGuei2a2etAwB+x6403E5CvpLn35y+GPs=
@ -284,6 +289,8 @@ github.com/ysmood/leakless v0.6.12/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNq
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/zclconf/go-cty v1.8.2 h1:u+xZfBKgpycDnTNjPhGiTEYZS5qS/Sb5MqSfm7vzcjg=
github.com/zclconf/go-cty v1.8.2/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
@ -427,6 +434,7 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

View File

@ -7,7 +7,7 @@ const banner = `
____ __ _______/ /__ (_)
/ __ \/ / / / ___/ / _ \/ /
/ / / / /_/ / /__/ / __/ /
/_/ /_/\__,_/\___/_/\___/_/ v2.3.7
/_/ /_/\__,_/\___/_/\___/_/ v2.3.7-dev
`
// Version is the current version of nuclei

View File

@ -24,6 +24,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/disk"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/sarif"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
"github.com/remeh/sizedwaitgroup"
@ -95,6 +96,14 @@ func New(options *types.Options) (*Runner, error) {
reportingOptions.DiskExporter = &disk.Options{Directory: options.DiskExportDirectory}
}
}
if options.SarifExport != "" {
if reportingOptions != nil {
reportingOptions.SarifExporter = &sarif.Options{File: options.SarifExport}
} else {
reportingOptions = &reporting.Options{}
reportingOptions.SarifExporter = &sarif.Options{File: options.SarifExport}
}
}
if reportingOptions != nil {
if client, err := reporting.New(reportingOptions, options.ReportingDB); err != nil {
gologger.Fatal().Msgf("Could not create issue reporting client: %s\n", err)

View File

@ -54,6 +54,8 @@ type InternalWrappedEvent struct {
type ResultEvent struct {
// TemplateID is the ID of the template for the result.
TemplateID string `json:"templateID"`
// TemplatePath is the path of template
TemplatePath string `json:"-"`
// Info contains information block of the template for the result.
Info map[string]interface{} `json:"info,inline"`
// MatcherName is the name of the matcher matched if any.
@ -82,6 +84,8 @@ type ResultEvent struct {
Timestamp time.Time `json:"timestamp"`
// Interaction is the full details of interactsh interaction.
Interaction *server.Interaction `json:"interaction,omitempty"`
FileToIndexPosition map[string]int `json:"-"`
}
// NewStandardWriter creates a new output writer based on user configurations

View File

@ -22,6 +22,7 @@ type Executer struct {
type clusteredOperator struct {
templateID string
templatePath string
templateInfo map[string]interface{}
operator *operators.Operators
}
@ -38,6 +39,7 @@ func NewExecuter(requests []*templates.Template, options *protocols.ExecuterOpti
executer.operators = append(executer.operators, &clusteredOperator{
templateID: req.ID,
templateInfo: req.Info,
templatePath: req.Path,
operator: req.RequestsHTTP[0].CompiledOperators,
})
}
@ -60,13 +62,15 @@ func (e *Executer) Requests() int {
func (e *Executer) Execute(input string) (bool, error) {
var results bool
previous := make(map[string]interface{})
dynamicValues := make(map[string]interface{})
err := e.requests.ExecuteWithResults(input, dynamicValues, nil, func(event *output.InternalWrappedEvent) {
err := e.requests.ExecuteWithResults(input, dynamicValues, previous, func(event *output.InternalWrappedEvent) {
for _, operator := range e.operators {
result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract)
if matched && result != nil {
event.OperatorsResult = result
event.InternalEvent["template-id"] = operator.templateID
event.InternalEvent["template-path"] = operator.templatePath
event.InternalEvent["template-info"] = operator.templateInfo
event.Results = e.requests.MakeResultEvent(event)
results = true
@ -94,6 +98,7 @@ func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEve
if matched && result != nil {
event.OperatorsResult = result
event.InternalEvent["template-id"] = operator.templateID
event.InternalEvent["template-path"] = operator.templatePath
event.InternalEvent["template-info"] = operator.templateInfo
event.Results = e.requests.MakeResultEvent(event)
callback(event)

View File

@ -103,6 +103,7 @@ func (r *Request) responseToDSLMap(req, resp *dns.Msg, host, matched string) out
data["raw"] = rawData
data["template-id"] = r.options.TemplateID
data["template-info"] = r.options.TemplateInfo
data["template-path"] = r.options.TemplatePath
return data
}
@ -137,6 +138,7 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu
func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
data := &output.ResultEvent{
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),
Info: wrapped.InternalEvent["template-info"].(map[string]interface{}),
Type: "dns",
Host: types.ToString(wrapped.InternalEvent["host"]),

View File

@ -1,6 +1,8 @@
package file
import (
"bufio"
"strings"
"time"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
@ -71,6 +73,7 @@ func (r *Request) responseToDSLMap(raw, host, matched string) output.InternalEve
data["raw"] = raw
data["template-id"] = r.options.TemplateID
data["template-info"] = r.options.TemplateInfo
data["template-path"] = r.options.TemplatePath
return data
}
@ -99,16 +102,45 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu
data := r.makeResultEventItem(wrapped)
results = append(results, data)
}
raw, ok := wrapped.InternalEvent["raw"]
if !ok {
return results
}
rawStr, ok := raw.(string)
if !ok {
return results
}
// Identify the position of match in file using a dirty hack.
for _, result := range results {
for _, extraction := range result.ExtractedResults {
scanner := bufio.NewScanner(strings.NewReader(rawStr))
line := 1
for scanner.Scan() {
if strings.Contains(scanner.Text(), extraction) {
if result.FileToIndexPosition == nil {
result.FileToIndexPosition = make(map[string]int)
}
result.FileToIndexPosition[result.Matched] = line
continue
}
line++
}
}
}
return results
}
func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
data := &output.ResultEvent{
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),
Info: wrapped.InternalEvent["template-info"].(map[string]interface{}),
Type: "file",
Path: types.ToString(wrapped.InternalEvent["path"]),
Matched: types.ToString(wrapped.InternalEvent["matched"]),
Host: types.ToString(wrapped.InternalEvent["matched"]),
ExtractedResults: wrapped.OperatorsResult.OutputExtracts,
Timestamp: time.Now(),
}

View File

@ -99,6 +99,7 @@ func (b *Browser) Close() {
// headless process launch.
func (b *Browser) killChromeProcesses() {
newProcesses := b.findChromeProcesses()
for id := range newProcesses {
if _, ok := b.previouspids[id]; ok {
continue

View File

@ -49,7 +49,6 @@ func (i *Instance) Run(baseURL *url.URL, actions []*Action, timeout time.Duratio
if err != nil {
return nil, nil, err
}
go router.Run()
data, err := createdPage.ExecuteActions(baseURL, actions)
if err != nil {

View File

@ -23,6 +23,7 @@ func TestActionNavigate(t *testing.T) {
instance, err := browser.NewInstance()
require.Nil(t, err, "could not create browser instance")
defer instance.Close()
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintln(w, `

View File

@ -72,6 +72,7 @@ func (r *Request) responseToDSLMap(resp, req, host, matched string) output.Inter
data["data"] = resp
data["template-id"] = r.options.TemplateID
data["template-info"] = r.options.TemplateInfo
data["template-path"] = r.options.TemplatePath
return data
}
@ -106,6 +107,7 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu
func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
data := &output.ResultEvent{
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),
Info: wrapped.InternalEvent["template-info"].(map[string]interface{}),
Type: "headless",
Host: types.ToString(wrapped.InternalEvent["host"]),

View File

@ -10,7 +10,7 @@ import (
// are similar enough to be considered one and can be checked by
// just adding the matcher/extractors for the request and the correct IDs.
func (r *Request) CanCluster(other *Request) bool {
if len(r.Payloads) > 0 || len(r.Raw) > 0 || len(r.Body) > 0 || r.Unsafe {
if len(r.Payloads) > 0 || len(r.Raw) > 0 || len(r.Body) > 0 || r.Unsafe || r.ReqCondition || r.Name != "" {
return false
}
if r.Method != other.Method ||

View File

@ -105,6 +105,7 @@ func (r *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, r
data["duration"] = duration.Seconds()
data["template-id"] = r.options.TemplateID
data["template-info"] = r.options.TemplateInfo
data["template-path"] = r.options.TemplatePath
return data
}
@ -139,6 +140,7 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu
func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
data := &output.ResultEvent{
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),
Info: wrapped.InternalEvent["template-info"].(map[string]interface{}),
Type: "http",
Host: types.ToString(wrapped.InternalEvent["host"]),

View File

@ -117,7 +117,7 @@ func (r *Request) executeParallelHTTP(reqURL string, dynamicValues, previous out
return requestErr
}
// executeRaceRequest executes turbo http request for a URL
// executeTurboHTTP executes turbo http request for a URL
func (r *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
generator := r.newGenerator()
@ -277,11 +277,15 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ
var hostname string
timeStart := time.Now()
if request.original.Pipeline {
formedURL = request.rawRequest.FullURL
if parsed, parseErr := url.Parse(formedURL); parseErr == nil {
hostname = parsed.Host
if request.rawRequest != nil {
formedURL = request.rawRequest.FullURL
if parsed, parseErr := url.Parse(formedURL); parseErr == nil {
hostname = parsed.Host
}
resp, err = request.pipelinedClient.DoRaw(request.rawRequest.Method, reqURL, request.rawRequest.Path, generators.ExpandMapValues(request.rawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.rawRequest.Data)))
} else if request.request != nil {
resp, err = request.pipelinedClient.Dor(request.request)
}
resp, err = request.pipelinedClient.DoRaw(request.rawRequest.Method, reqURL, request.rawRequest.Path, generators.ExpandMapValues(request.rawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.rawRequest.Data)))
} else if request.original.Unsafe && request.rawRequest != nil {
formedURL = request.rawRequest.FullURL
if parsed, parseErr := url.Parse(formedURL); parseErr == nil {

View File

@ -73,6 +73,7 @@ func (r *Request) responseToDSLMap(req, resp, raw, host, matched string) output.
data["raw"] = raw // Raw is the full transaction data for network
data["template-id"] = r.options.TemplateID
data["template-info"] = r.options.TemplateInfo
data["template-path"] = r.options.TemplatePath
return data
}
@ -107,6 +108,7 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu
func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
data := &output.ResultEvent{
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),
Info: wrapped.InternalEvent["template-info"].(map[string]interface{}),
Type: "network",
Host: types.ToString(wrapped.InternalEvent["host"]),

View File

@ -101,6 +101,7 @@ func (r *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, r
data["duration"] = duration.Seconds()
data["template-id"] = r.options.TemplateID
data["template-info"] = r.options.TemplateInfo
data["template-path"] = r.options.TemplatePath
return data
}
@ -135,6 +136,7 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu
func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
data := &output.ResultEvent{
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),
Info: wrapped.InternalEvent["template-info"].(map[string]interface{}),
Type: "http",
Path: types.ToString(wrapped.InternalEvent["path"]),

View File

@ -58,3 +58,8 @@ func (i *Exporter) Export(event *output.ResultEvent) error {
err := ioutil.WriteFile(path.Join(i.directory, finalFilename), data, 0644)
return err
}
// Close closes the exporter after operation
func (i *Exporter) Close() error {
return nil
}

View File

@ -0,0 +1,144 @@
package sarif
import (
"crypto/sha1"
"encoding/hex"
"os"
"path"
"strings"
"sync"
"github.com/owenrumney/go-sarif/sarif"
"github.com/pkg/errors"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/format"
)
// Exporter is an exporter for nuclei sarif output format.
type Exporter struct {
sarif *sarif.Report
run *sarif.Run
mutex *sync.Mutex
home string
tempFile string
options *Options
}
// Options contains the configuration options for sarif exporter client
type Options struct {
// File is the file to export found sarif result to
File string `yaml:"file"`
}
// New creates a new disk exporter integration client based on options.
func New(options *Options) (*Exporter, error) {
report, err := sarif.New(sarif.Version210)
if err != nil {
return nil, errors.Wrap(err, "could not create sarif exporter")
}
home, err := os.UserHomeDir()
if err != nil {
return nil, errors.Wrap(err, "could not get home dir")
}
templatePath := path.Join(home, "nuclei-templates")
run := sarif.NewRun("nuclei", "https://github.com/projectdiscovery/nuclei")
return &Exporter{options: options, home: templatePath, sarif: report, run: run, mutex: &sync.Mutex{}}, nil
}
// Export exports a passed result event to sarif structure
func (i *Exporter) Export(event *output.ResultEvent) error {
templatePath := strings.TrimPrefix(event.TemplatePath, i.home)
h := sha1.New()
h.Write([]byte(event.Host))
templateID := event.TemplateID + "-" + hex.EncodeToString(h.Sum(nil))
fullDescription := format.MarkdownDescription(event)
sarifSeverity := getSarifSeverity(event)
var ruleName string
if s, ok := event.Info["name"]; ok {
ruleName = s.(string)
}
var templateURL string
if strings.HasPrefix(event.TemplatePath, i.home) {
templateURL = "https://github.com/projectdiscovery/nuclei-templates/blob/master" + templatePath
} else {
templateURL = "https://github.com/projectdiscovery/nuclei-templates"
}
var ruleDescription string
if d, ok := event.Info["description"]; ok {
ruleDescription = d.(string)
}
i.mutex.Lock()
defer i.mutex.Unlock()
_ = i.run.AddRule(templateID).
WithDescription(ruleName).
WithHelp(fullDescription).
WithHelpURI(templateURL).
WithFullDescription(sarif.NewMultiformatMessageString(ruleDescription))
result := i.run.AddResult(templateID).
WithMessage(sarif.NewMessage().WithText(event.Host)).
WithLevel(sarifSeverity)
// Also write file match metadata to file
if event.Type == "file" && (event.FileToIndexPosition != nil && len(event.FileToIndexPosition) > 0) {
for file, line := range event.FileToIndexPosition {
result.WithLocation(sarif.NewLocation().WithMessage(sarif.NewMessage().WithText(ruleName)).WithPhysicalLocation(
sarif.NewPhysicalLocation().
WithArtifactLocation(sarif.NewArtifactLocation().WithUri(file)).
WithRegion(sarif.NewRegion().WithStartColumn(1).WithStartLine(line).WithEndLine(line).WithEndColumn(32)),
))
}
} else {
result.WithLocation(sarif.NewLocation().WithMessage(sarif.NewMessage().WithText(event.Host)).WithPhysicalLocation(
sarif.NewPhysicalLocation().
WithArtifactLocation(sarif.NewArtifactLocation().WithUri("README.md")).
WithRegion(sarif.NewRegion().WithStartColumn(1).WithStartLine(1).WithEndLine(1).WithEndColumn(1)),
))
}
return nil
}
// getSarifSeverity returns the sarif severity
func getSarifSeverity(event *output.ResultEvent) string {
var ruleSeverity string
if s, ok := event.Info["severity"]; ok {
ruleSeverity = s.(string)
}
switch ruleSeverity {
case "info":
return "note"
case "low", "medium":
return "warning"
case "high", "critical":
return "error"
default:
return "note"
}
}
// Close closes the exporter after operation
func (i *Exporter) Close() error {
i.mutex.Lock()
defer i.mutex.Unlock()
i.sarif.AddRun(i.run)
if len(i.run.Results) == 0 {
return nil // do not write when no results
}
file, err := os.Create(i.options.File)
if err != nil {
return errors.Wrap(err, "could not create sarif output file")
}
defer file.Close()
return i.sarif.Write(file)
}

View File

@ -44,6 +44,9 @@ func MarkdownDescription(event *output.ResultEvent) string {
builder.WriteString(event.Timestamp.Format("Mon Jan 2 15:04:05 -0700 MST 2006"))
builder.WriteString("\n\n**Template Information**\n\n| Key | Value |\n|---|---|\n")
for k, v := range event.Info {
if k == "reference" {
continue
}
builder.WriteString(fmt.Sprintf("| %s | %s |\n", k, v))
}
if event.Request != "" {
@ -60,11 +63,11 @@ func MarkdownDescription(event *output.ResultEvent) string {
} else {
builder.WriteString(event.Response)
}
builder.WriteString("\n```\n\n")
builder.WriteString("\n```\n")
}
if len(event.ExtractedResults) > 0 || len(event.Metadata) > 0 {
builder.WriteString("**Extra Information**\n\n")
builder.WriteString("\n**Extra Information**\n\n")
if len(event.ExtractedResults) > 0 {
builder.WriteString("**Extracted results**:\n\n")
for _, v := range event.ExtractedResults {
@ -110,6 +113,26 @@ func MarkdownDescription(event *output.ResultEvent) string {
builder.WriteString("\n```\n")
}
}
if d, ok := event.Info["reference"]; ok {
builder.WriteString("\nReference: \n")
switch v := d.(type) {
case string:
if !strings.HasPrefix(v, "-") {
builder.WriteString("- ")
}
builder.WriteString(v)
case []interface{}:
slice := types.ToStringSlice(v)
for i, item := range slice {
builder.WriteString("- ")
builder.WriteString(item)
if len(slice)-1 != i {
builder.WriteString("\n")
}
}
}
}
builder.WriteString("\n---\nGenerated by [Nuclei](https://github.com/projectdiscovery/nuclei)")
data := builder.String()

View File

@ -7,6 +7,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/dedupe"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/disk"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/sarif"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/github"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/gitlab"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/jira"
@ -28,6 +29,8 @@ type Options struct {
Jira *jira.Options `yaml:"jira"`
// DiskExporter contains configuration options for Disk Exporter Module
DiskExporter *disk.Options `yaml:"disk"`
// SarifExporter contains configuration options for Sarif Exporter Module
SarifExporter *sarif.Options `yaml:"sarif"`
}
// Filter filters the received event and decides whether to perform
@ -79,6 +82,8 @@ type Tracker interface {
// Exporter is an interface implemented by an issue exporter
type Exporter interface {
// Close closes the exporter after operation
Close() error
// Export exports an issue to an exporter
Export(event *output.ResultEvent) error
}
@ -129,6 +134,13 @@ func New(options *Options, db string) (*Client, error) {
}
client.exporters = append(client.exporters, exporter)
}
if options.SarifExporter != nil {
exporter, err := sarif.New(options.SarifExporter)
if err != nil {
return nil, errors.Wrap(err, "could not create exporting client")
}
client.exporters = append(client.exporters, exporter)
}
storage, err := dedupe.New(db)
if err != nil {
return nil, err
@ -140,6 +152,9 @@ func New(options *Options, db string) (*Client, error) {
// Close closes the issue tracker reporting client
func (c *Client) Close() {
c.dedupe.Close()
for _, exporter := range c.exporters {
exporter.Close()
}
}
// CreateIssue creates an issue in the tracker

View File

@ -20,6 +20,8 @@ type Integration struct {
// Options contains the configuration options for jira client
type Options struct {
// Cloud value is set to true when Jira cloud is used
Cloud bool `yaml:"cloud"`
// URL is the URL of the jira server
URL string `yaml:"url"`
// AccountID is the accountID of the jira user.
@ -36,8 +38,12 @@ type Options struct {
// New creates a new issue tracker integration client based on options.
func New(options *Options) (*Integration, error) {
username := options.Email
if !options.Cloud {
username = options.AccountID
}
tp := jira.BasicAuthTransport{
Username: options.Email,
Username: username,
Password: options.Token,
}
jiraClient, err := jira.NewClient(tp.Client(), options.URL)
@ -51,15 +57,27 @@ func New(options *Options) (*Integration, error) {
func (i *Integration) CreateIssue(event *output.ResultEvent) error {
summary := format.Summary(event)
issueData := &jira.Issue{
Fields: &jira.IssueFields{
Assignee: &jira.User{AccountID: i.options.AccountID},
Reporter: &jira.User{AccountID: i.options.AccountID},
fields := &jira.IssueFields{
Assignee: &jira.User{AccountID: i.options.AccountID},
Reporter: &jira.User{AccountID: i.options.AccountID},
Description: jiraFormatDescription(event),
Type: jira.IssueType{Name: i.options.IssueType},
Project: jira.Project{Key: i.options.ProjectName},
Summary: summary,
}
// On-prem version of Jira server does not use AccountID
if !i.options.Cloud {
fields = &jira.IssueFields{
Assignee: &jira.User{Name: i.options.AccountID},
Description: jiraFormatDescription(event),
Type: jira.IssueType{Name: i.options.IssueType},
Project: jira.Project{Key: i.options.ProjectName},
Summary: summary,
},
}
}
issueData := &jira.Issue{
Fields: fields,
}
_, resp, err := i.jira.Issue.Create(issueData)
if err != nil {
@ -92,6 +110,9 @@ func jiraFormatDescription(event *output.ResultEvent) string {
builder.WriteString(event.Timestamp.Format("Mon Jan 2 15:04:05 -0700 MST 2006"))
builder.WriteString("\n\n*Template Information*\n\n| Key | Value |\n")
for k, v := range event.Info {
if k == "reference" {
continue
}
builder.WriteString(fmt.Sprintf("| %s | %s |\n", k, v))
}
builder.WriteString("\n*Request*\n\n{code}\n")
@ -107,7 +128,7 @@ func jiraFormatDescription(event *output.ResultEvent) string {
builder.WriteString("\n{code}\n\n")
if len(event.ExtractedResults) > 0 || len(event.Metadata) > 0 {
builder.WriteString("*Extra Information*\n\n")
builder.WriteString("\n*Extra Information*\n\n")
if len(event.ExtractedResults) > 0 {
builder.WriteString("*Extracted results*:\n\n")
for _, v := range event.ExtractedResults {
@ -153,6 +174,26 @@ func jiraFormatDescription(event *output.ResultEvent) string {
builder.WriteString("\n{code}\n")
}
}
if d, ok := event.Info["reference"]; ok {
builder.WriteString("\nReference: \n")
switch v := d.(type) {
case string:
if !strings.HasPrefix(v, "-") {
builder.WriteString("- ")
}
builder.WriteString(v)
case []interface{}:
slice := types.ToStringSlice(v)
for i, item := range slice {
builder.WriteString("- ")
builder.WriteString(item)
if len(slice)-1 != i {
builder.WriteString("\n")
}
}
}
}
builder.WriteString("\n---\nGenerated by [Nuclei|https://github.com/projectdiscovery/nuclei]")
data := builder.String()
return data

View File

@ -142,6 +142,7 @@ func Parse(filePath string, options protocols.ExecuterOptions) (*Template, error
if template.Executer == nil && template.CompiledWorkflow == nil {
return nil, errors.New("cannot create template executer")
}
template.Path = filePath
return template, nil
}

View File

@ -35,4 +35,6 @@ type Template struct {
TotalRequests int `yaml:"-" json:"-"`
// Executer is the actual template executor for running template requests
Executer protocols.Executer `yaml:"-" json:"-"`
Path string `yaml:"-" json:"-"`
}

View File

@ -47,6 +47,8 @@ type Options struct {
ReportingConfig string
// DiskExportDirectory is the directory to export reports in markdown on disk to
DiskExportDirectory string
// SarifExport is the file to export sarif output format to
SarifExport string
// ResolversFile is a file containing resolvers for nuclei.
ResolversFile string
// StatsInterval is the number of seconds to display stats after