Merge pull request #638 from projectdiscovery/fix-bugs

v2.3.1 bug fixes
dev
PD-Team 2021-03-22 17:21:18 +05:30 committed by GitHub
commit 29b99fecac
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 260 additions and 69 deletions

View File

@ -24,9 +24,7 @@ func main() {
if err != nil { if err != nil {
gologger.Fatal().Msgf("Could not create runner: %s\n", err) gologger.Fatal().Msgf("Could not create runner: %s\n", err)
} }
nucleiRunner.RunEnumeration() nucleiRunner.RunEnumeration()
nucleiRunner.Close()
} }
func readConfig() { func readConfig() {
@ -77,7 +75,7 @@ based on templates offering massive extensibility and ease of use.`)
set.BoolVar(&options.OfflineHTTP, "passive", false, "Enable Passive HTTP response processing mode") set.BoolVar(&options.OfflineHTTP, "passive", false, "Enable Passive HTTP response processing mode")
set.StringVarP(&options.BurpCollaboratorBiid, "burp-collaborator-biid", "biid", "", "Burp Collaborator BIID") set.StringVarP(&options.BurpCollaboratorBiid, "burp-collaborator-biid", "biid", "", "Burp Collaborator BIID")
set.StringVarP(&options.ReportingConfig, "report-config", "rc", "", "Nuclei Reporting Module configuration file") set.StringVarP(&options.ReportingConfig, "report-config", "rc", "", "Nuclei Reporting Module configuration file")
set.StringVarP(&options.ReportingDB, "report-db", "rdb", "", "Local Nuclei Reporting Database") set.StringVarP(&options.ReportingDB, "report-db", "rdb", "", "Local Nuclei Reporting Database (Always use this to persistent report data)")
set.StringSliceVar(&options.Tags, "tags", []string{}, "Tags to execute templates for") set.StringSliceVar(&options.Tags, "tags", []string{}, "Tags to execute templates for")
set.StringSliceVarP(&options.ExcludeTags, "exclude-tags", "etags", []string{}, "Exclude templates with the provided tags") set.StringSliceVarP(&options.ExcludeTags, "exclude-tags", "etags", []string{}, "Exclude templates with the provided tags")
set.StringVarP(&options.ResolversFile, "resolvers", "r", "", "File containing resolver list for nuclei") set.StringVarP(&options.ResolversFile, "resolvers", "r", "", "File containing resolver list for nuclei")
@ -87,6 +85,7 @@ based on templates offering massive extensibility and ease of use.`)
set.BoolVar(&options.SystemResolvers, "system-resolvers", false, "Use system dns resolving as error fallback") set.BoolVar(&options.SystemResolvers, "system-resolvers", false, "Use system dns resolving as error fallback")
set.IntVar(&options.PageTimeout, "page-timeout", 20, "Seconds to wait for each page in headless") set.IntVar(&options.PageTimeout, "page-timeout", 20, "Seconds to wait for each page in headless")
set.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "Only run newly added templates") set.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "Only run newly added templates")
set.StringVarP(&options.DiskExportDirectory, "disk-export", "de", "", "Directory on disk to export reports in markdown to")
_ = set.Parse() _ = set.Parse()
if cfgFile != "" { if cfgFile != "" {

View File

@ -26,7 +26,7 @@ require (
github.com/projectdiscovery/clistats v0.0.8 github.com/projectdiscovery/clistats v0.0.8
github.com/projectdiscovery/collaborator v0.0.2 github.com/projectdiscovery/collaborator v0.0.2
github.com/projectdiscovery/fastdialer v0.0.8 github.com/projectdiscovery/fastdialer v0.0.8
github.com/projectdiscovery/goflags v0.0.3 github.com/projectdiscovery/goflags v0.0.4
github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/gologger v1.1.4
github.com/projectdiscovery/hmap v0.0.1 github.com/projectdiscovery/hmap v0.0.1
github.com/projectdiscovery/rawhttp v0.0.6 github.com/projectdiscovery/rawhttp v0.0.6

View File

@ -201,8 +201,8 @@ github.com/projectdiscovery/collaborator v0.0.2 h1:BSiMlWM3NvuKbpedn6fIjjEo5b7q5
github.com/projectdiscovery/collaborator v0.0.2/go.mod h1:J1z0fC7Svutz3LJqoRyTHA3F0Suh4livmkYv8MnKw20= github.com/projectdiscovery/collaborator v0.0.2/go.mod h1:J1z0fC7Svutz3LJqoRyTHA3F0Suh4livmkYv8MnKw20=
github.com/projectdiscovery/fastdialer v0.0.8 h1:mEMc8bfXV5hc1PUEkJiUnR5imYQe6+839Zezd5jLkc0= github.com/projectdiscovery/fastdialer v0.0.8 h1:mEMc8bfXV5hc1PUEkJiUnR5imYQe6+839Zezd5jLkc0=
github.com/projectdiscovery/fastdialer v0.0.8/go.mod h1:AuaV0dzrNeBLHqjNnzpFSnTXnHGIZAlGQE+WUMmSIW4= github.com/projectdiscovery/fastdialer v0.0.8/go.mod h1:AuaV0dzrNeBLHqjNnzpFSnTXnHGIZAlGQE+WUMmSIW4=
github.com/projectdiscovery/goflags v0.0.3 h1:5s9qblVIP/dQt7Mr3PMvZLvekEyioOS5oZtZ6ncLQHA= github.com/projectdiscovery/goflags v0.0.4 h1:fWKLMAr3KmPlZxE1b54pfei+vGIUJn9q6aM7woZIbCY=
github.com/projectdiscovery/goflags v0.0.3/go.mod h1:Ae1mJ5MIIqjys0lFe3GiMZ10Z8VLaxkYJ1ySA4Zv8HA= github.com/projectdiscovery/goflags v0.0.4/go.mod h1:Ae1mJ5MIIqjys0lFe3GiMZ10Z8VLaxkYJ1ySA4Zv8HA=
github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI= github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI=
github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY= github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY=
github.com/projectdiscovery/hmap v0.0.1 h1:VAONbJw5jP+syI5smhsfkrq9XPGn4aiYy5pR6KR1wog= github.com/projectdiscovery/hmap v0.0.1 h1:VAONbJw5jP+syI5smhsfkrq9XPGn4aiYy5pR6KR1wog=

View File

@ -16,13 +16,15 @@ type nucleiConfig struct {
TemplatesDirectory string `json:"templates-directory,omitempty"` TemplatesDirectory string `json:"templates-directory,omitempty"`
CurrentVersion string `json:"current-version,omitempty"` CurrentVersion string `json:"current-version,omitempty"`
LastChecked time.Time `json:"last-checked,omitempty"` LastChecked time.Time `json:"last-checked,omitempty"`
IgnoreURL string `json:"ignore-url,omitempty"`
NucleiVersion string `json:"nuclei-version,omitempty"`
LastCheckedIgnore time.Time `json:"last-checked-ignore,omitempty"`
// IgnorePaths ignores all the paths listed unless specified manually // IgnorePaths ignores all the paths listed unless specified manually
IgnorePaths []string `json:"ignore-paths,omitempty"` IgnorePaths []string `json:"ignore-paths,omitempty"`
} }
// nucleiConfigFilename is the filename of nuclei configuration file. // nucleiConfigFilename is the filename of nuclei configuration file.
const nucleiConfigFilename = ".nuclei-config.json" const nucleiConfigFilename = ".templates-config.json"
var reVersion = regexp.MustCompile(`\d+\.\d+\.\d+`) var reVersion = regexp.MustCompile(`\d+\.\d+\.\d+`)
@ -32,8 +34,10 @@ func readConfiguration() (*nucleiConfig, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
configDir := path.Join(home, "/.config", "/nuclei")
_ = os.MkdirAll(configDir, os.ModePerm)
templatesConfigFile := path.Join(home, nucleiConfigFilename) templatesConfigFile := path.Join(configDir, nucleiConfigFilename)
file, err := os.Open(templatesConfigFile) file, err := os.Open(templatesConfigFile)
if err != nil { if err != nil {
return nil, err return nil, err
@ -54,9 +58,16 @@ func (r *Runner) writeConfiguration(config *nucleiConfig) error {
if err != nil { if err != nil {
return err return err
} }
configDir := path.Join(home, "/.config", "/nuclei")
_ = os.MkdirAll(configDir, os.ModePerm)
if config.IgnoreURL == "" {
config.IgnoreURL = "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore"
}
config.LastChecked = time.Now() config.LastChecked = time.Now()
templatesConfigFile := path.Join(home, nucleiConfigFilename) config.LastCheckedIgnore = time.Now()
config.NucleiVersion = Version
templatesConfigFile := path.Join(configDir, nucleiConfigFilename)
file, err := os.OpenFile(templatesConfigFile, os.O_WRONLY|os.O_CREATE, 0777) file, err := os.OpenFile(templatesConfigFile, os.O_WRONLY|os.O_CREATE, 0777)
if err != nil { if err != nil {
return err return err
@ -95,7 +106,15 @@ func (r *Runner) readNucleiIgnoreFile() {
// getIgnoreFilePath returns the ignore file path for the runner // getIgnoreFilePath returns the ignore file path for the runner
func (r *Runner) getIgnoreFilePath() string { func (r *Runner) getIgnoreFilePath() string {
defIgnoreFilePath := path.Join(r.templatesConfig.TemplatesDirectory, nucleiIgnoreFile) var defIgnoreFilePath string
home, err := os.UserHomeDir()
if err == nil {
configDir := path.Join(home, "/.config", "/nuclei")
_ = os.MkdirAll(configDir, os.ModePerm)
defIgnoreFilePath = path.Join(configDir, nucleiIgnoreFile)
}
cwd, err := os.Getwd() cwd, err := os.Getwd()
if err != nil { if err != nil {

View File

@ -19,13 +19,15 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/clusterer" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/clusterer"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues" "github.com/projectdiscovery/nuclei/v2/pkg/reporting"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/disk"
"github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/templates"
"github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/nuclei/v2/pkg/types"
"github.com/remeh/sizedwaitgroup" "github.com/remeh/sizedwaitgroup"
"github.com/rs/xid" "github.com/rs/xid"
"go.uber.org/atomic" "go.uber.org/atomic"
"go.uber.org/ratelimit" "go.uber.org/ratelimit"
"gopkg.in/yaml.v2"
) )
// Runner is a client for running the enumeration process. // Runner is a client for running the enumeration process.
@ -39,7 +41,7 @@ type Runner struct {
catalog *catalog.Catalog catalog *catalog.Catalog
progress progress.Progress progress progress.Progress
colorizer aurora.Aurora colorizer aurora.Aurora
issuesClient *issues.Client issuesClient *reporting.Client
severityColors *colorizer.Colorizer severityColors *colorizer.Colorizer
browser *engine.Browser browser *engine.Browser
ratelimiter ratelimit.Limiter ratelimiter ratelimit.Limiter
@ -66,13 +68,36 @@ func New(options *types.Options) (*Runner, error) {
} }
runner.catalog = catalog.New(runner.options.TemplatesDirectory) runner.catalog = catalog.New(runner.options.TemplatesDirectory)
var reportingOptions *reporting.Options
if options.ReportingConfig != "" { if options.ReportingConfig != "" {
if client, err := issues.New(options.ReportingConfig, options.ReportingDB); err != nil { file, err := os.Open(options.ReportingConfig)
if err != nil {
gologger.Fatal().Msgf("Could not open reporting config file: %s\n", err)
}
reportingOptions = &reporting.Options{}
if parseErr := yaml.NewDecoder(file).Decode(reportingOptions); parseErr != nil {
file.Close()
gologger.Fatal().Msgf("Could not parse reporting config file: %s\n", parseErr)
}
file.Close()
}
if options.DiskExportDirectory != "" {
if reportingOptions != nil {
reportingOptions.DiskExporter = &disk.Options{Directory: options.DiskExportDirectory}
} else {
reportingOptions = &reporting.Options{}
reportingOptions.DiskExporter = &disk.Options{Directory: options.DiskExportDirectory}
}
}
if reportingOptions != nil {
if client, err := reporting.New(reportingOptions, options.ReportingDB); err != nil {
gologger.Fatal().Msgf("Could not create issue reporting client: %s\n", err) gologger.Fatal().Msgf("Could not create issue reporting client: %s\n", err)
} else { } else {
runner.issuesClient = client runner.issuesClient = client
} }
} }
// output coloring // output coloring
useColor := !options.NoColor useColor := !options.NoColor
runner.colorizer = aurora.NewAurora(useColor) runner.colorizer = aurora.NewAurora(useColor)
@ -197,6 +222,8 @@ func (r *Runner) Close() {
// RunEnumeration sets up the input layer for giving input nuclei. // RunEnumeration sets up the input layer for giving input nuclei.
// binary and runs the actual enumeration // binary and runs the actual enumeration
func (r *Runner) RunEnumeration() { func (r *Runner) RunEnumeration() {
defer r.Close()
// If we have no templates, run on whole template directory with provided tags // If we have no templates, run on whole template directory with provided tags
if len(r.options.Templates) == 0 && len(r.options.Workflows) == 0 && !r.options.NewTemplates && (len(r.options.Tags) > 0 || len(r.options.ExcludeTags) > 0) { if len(r.options.Templates) == 0 && len(r.options.Workflows) == 0 && !r.options.NewTemplates && (len(r.options.Tags) > 0 || len(r.options.ExcludeTags) > 0) {
r.options.Templates = append(r.options.Templates, r.options.TemplatesDirectory) r.options.Templates = append(r.options.Templates, r.options.TemplatesDirectory)

View File

@ -41,6 +41,8 @@ func (r *Runner) updateTemplates() error {
if err != nil { if err != nil {
return err return err
} }
configDir := path.Join(home, "/.config", "/nuclei")
_ = os.MkdirAll(configDir, os.ModePerm)
templatesConfigFile := path.Join(home, nucleiConfigFilename) templatesConfigFile := path.Join(home, nucleiConfigFilename)
if _, statErr := os.Stat(templatesConfigFile); !os.IsNotExist(statErr) { if _, statErr := os.Stat(templatesConfigFile); !os.IsNotExist(statErr) {
@ -51,15 +53,61 @@ func (r *Runner) updateTemplates() error {
r.templatesConfig = config r.templatesConfig = config
} }
ignoreURL := "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore"
if r.templatesConfig == nil {
currentConfig := &nucleiConfig{
TemplatesDirectory: path.Join(home, "nuclei-templates"),
IgnoreURL: ignoreURL,
NucleiVersion: Version,
}
if writeErr := r.writeConfiguration(currentConfig); writeErr != nil {
return errors.Wrap(writeErr, "could not write template configuration")
}
r.templatesConfig = currentConfig
}
// Check if last checked for nuclei-ignore is more than 1 hours.
// and if true, run the check.
if r.templatesConfig == nil || time.Since(r.templatesConfig.LastCheckedIgnore) > 1*time.Hour || r.options.UpdateTemplates {
if r.templatesConfig != nil && r.templatesConfig.IgnoreURL == "" {
ignoreURL = r.templatesConfig.IgnoreURL
}
gologger.Verbose().Msgf("Downloading config file from %s", ignoreURL)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
req, reqErr := http.NewRequestWithContext(ctx, http.MethodGet, ignoreURL, nil)
if reqErr == nil {
resp, httpGet := http.DefaultClient.Do(req)
if httpGet != nil {
if resp != nil && resp.Body != nil {
resp.Body.Close()
}
gologger.Warning().Msgf("Could not get ignore-file from %s: %s", ignoreURL, err)
} else {
data, _ := ioutil.ReadAll(resp.Body)
resp.Body.Close()
if len(data) > 0 {
_ = ioutil.WriteFile(path.Join(configDir, nucleiIgnoreFile), data, 0644)
}
if r.templatesConfig != nil {
r.templatesConfig.LastCheckedIgnore = time.Now()
}
}
}
cancel()
}
ctx := context.Background() ctx := context.Background()
if r.templatesConfig == nil || (r.options.TemplatesDirectory != "" && r.templatesConfig.TemplatesDirectory != r.options.TemplatesDirectory) { if r.templatesConfig.CurrentVersion == "" || (r.options.TemplatesDirectory != "" && r.templatesConfig.TemplatesDirectory != r.options.TemplatesDirectory) {
if !r.options.UpdateTemplates { if !r.options.UpdateTemplates {
gologger.Warning().Msgf("nuclei-templates are not installed, use update-templates flag.\n") gologger.Warning().Msgf("nuclei-templates are not installed (or indexed), use update-templates flag.\n")
return nil return nil
} }
// Use custom location if user has given a template directory // Use custom location if user has given a template directory
r.templatesConfig = &nucleiConfig{TemplatesDirectory: path.Join(home, "nuclei-templates")} r.templatesConfig = &nucleiConfig{
TemplatesDirectory: path.Join(home, "nuclei-templates"),
}
if r.options.TemplatesDirectory != "" && r.options.TemplatesDirectory != path.Join(home, "nuclei-templates") { if r.options.TemplatesDirectory != "" && r.options.TemplatesDirectory != path.Join(home, "nuclei-templates") {
r.templatesConfig.TemplatesDirectory = r.options.TemplatesDirectory r.templatesConfig.TemplatesDirectory = r.options.TemplatesDirectory
} }
@ -132,7 +180,6 @@ func (r *Runner) updateTemplates() error {
if err != nil { if err != nil {
return err return err
} }
err = r.writeConfiguration(r.templatesConfig) err = r.writeConfiguration(r.templatesConfig)
if err != nil { if err != nil {
return err return err

View File

@ -183,7 +183,7 @@ func (p *StatsTicker) getMetrics() map[string]interface{} {
results["templates"] = clistats.String(templates) results["templates"] = clistats.String(templates)
hosts, _ := p.stats.GetStatic("hosts") hosts, _ := p.stats.GetStatic("hosts")
results["hosts"] = clistats.String(hosts) results["hosts"] = clistats.String(hosts)
matched, _ := p.stats.GetStatic("matched") matched, _ := p.stats.GetCounter("matched")
results["matched"] = clistats.String(matched) results["matched"] = clistats.String(matched)
requests, _ := p.stats.GetCounter("requests") requests, _ := p.stats.GetCounter("requests")
results["requests"] = clistats.String(requests) results["requests"] = clistats.String(requests)

View File

@ -9,7 +9,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/progress" "github.com/projectdiscovery/nuclei/v2/pkg/progress"
"github.com/projectdiscovery/nuclei/v2/pkg/projectfile" "github.com/projectdiscovery/nuclei/v2/pkg/projectfile"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues" "github.com/projectdiscovery/nuclei/v2/pkg/reporting"
"github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/nuclei/v2/pkg/types"
"go.uber.org/ratelimit" "go.uber.org/ratelimit"
) )
@ -39,7 +39,7 @@ type ExecuterOptions struct {
// Options contains configuration options for the executer. // Options contains configuration options for the executer.
Options *types.Options Options *types.Options
// IssuesClient is a client for nuclei issue tracker reporting // IssuesClient is a client for nuclei issue tracker reporting
IssuesClient *issues.Client IssuesClient *reporting.Client
// Progress is a progress client for scan reporting // Progress is a progress client for scan reporting
Progress progress.Progress Progress progress.Progress
// RateLimiter is a rate-limiter for limiting sent number of requests. // RateLimiter is a rate-limiter for limiting sent number of requests.

View File

@ -0,0 +1,60 @@
package disk
import (
"bytes"
"io/ioutil"
"os"
"path"
"strings"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/format"
)
type Exporter struct {
directory string
options *Options
}
// Options contains the configuration options for github issue tracker client
type Options struct {
// Directory is the directory to export found results to
Directory string `yaml:"directory"`
}
// New creates a new disk exporter integration client based on options.
func New(options *Options) (*Exporter, error) {
directory := options.Directory
if options.Directory == "" {
dir, err := os.Getwd()
if err != nil {
return nil, err
}
directory = dir
}
_ = os.MkdirAll(directory, os.ModePerm)
return &Exporter{options: options, directory: directory}, nil
}
// Export exports a passed result event to disk
func (i *Exporter) Export(event *output.ResultEvent) error {
summary := format.Summary(event)
description := format.MarkdownDescription(event)
filenameBuilder := &strings.Builder{}
filenameBuilder.WriteString(event.TemplateID)
filenameBuilder.WriteString("-")
filenameBuilder.WriteString(strings.ReplaceAll(strings.ReplaceAll(event.Matched, "/", "_"), ":", "_"))
filenameBuilder.WriteString(".md")
finalFilename := filenameBuilder.String()
dataBuilder := &bytes.Buffer{}
dataBuilder.WriteString("### ")
dataBuilder.WriteString(summary)
dataBuilder.WriteString("\n---\n")
dataBuilder.WriteString(description)
data := dataBuilder.Bytes()
err := ioutil.WriteFile(path.Join(i.directory, finalFilename), data, 0644)
return err
}

View File

@ -46,11 +46,22 @@ func MarkdownDescription(event *output.ResultEvent) string {
for k, v := range event.Info { for k, v := range event.Info {
builder.WriteString(fmt.Sprintf("| %s | %s |\n", k, v)) builder.WriteString(fmt.Sprintf("| %s | %s |\n", k, v))
} }
builder.WriteString("\n**Request**\n\n```\n") if event.Request != "" {
builder.WriteString("\n**Request**\n\n```http\n")
builder.WriteString(event.Request) builder.WriteString(event.Request)
builder.WriteString("\n```\n\n<details><summary>**Response**</summary>\n\n```\n") builder.WriteString("\n```\n")
}
if event.Response != "" {
builder.WriteString("\n**Response**\n\n```http\n")
// If the response is larger than 5 kb, truncate it before writing.
if len(event.Response) > 5*1024 {
builder.WriteString(event.Response[:5*1024])
builder.WriteString(".... Truncated ....")
} else {
builder.WriteString(event.Response) builder.WriteString(event.Response)
}
builder.WriteString("\n```\n\n") builder.WriteString("\n```\n\n")
}
if len(event.ExtractedResults) > 0 || len(event.Metadata) > 0 { if len(event.ExtractedResults) > 0 || len(event.Metadata) > 0 {
builder.WriteString("**Extra Information**\n\n") builder.WriteString("**Extra Information**\n\n")
@ -75,6 +86,7 @@ func MarkdownDescription(event *output.ResultEvent) string {
builder.WriteString("\n") builder.WriteString("\n")
} }
} }
builder.WriteString("\n---\nGenerated by [Nuclei](https://github.com/projectdiscovery/nuclei)")
data := builder.String() data := builder.String()
return data return data
} }

View File

@ -1,17 +1,17 @@
package issues package reporting
import ( import (
"os"
"strings" "strings"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues/dedupe" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/dedupe"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues/github" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/disk"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues/gitlab" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/github"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues/jira" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/gitlab"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/jira"
"github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/nuclei/v2/pkg/types"
"gopkg.in/yaml.v2" "go.uber.org/multierr"
) )
// Options is a configuration file for nuclei reporting module // Options is a configuration file for nuclei reporting module
@ -26,6 +26,8 @@ type Options struct {
Gitlab *gitlab.Options `yaml:"gitlab"` Gitlab *gitlab.Options `yaml:"gitlab"`
// Jira contains configuration options for Jira Issue Tracker // Jira contains configuration options for Jira Issue Tracker
Jira *jira.Options `yaml:"jira"` Jira *jira.Options `yaml:"jira"`
// DiskExporter contains configuration options for Disk Exporter Module
DiskExporter *disk.Options `yaml:"disk"`
} }
// Filter filters the received event and decides whether to perform // Filter filters the received event and decides whether to perform
@ -75,25 +77,22 @@ type Tracker interface {
CreateIssue(event *output.ResultEvent) error CreateIssue(event *output.ResultEvent) error
} }
// Exporter is an interface implemented by an issue exporter
type Exporter interface {
// Export exports an issue to an exporter
Export(event *output.ResultEvent) error
}
// Client is a client for nuclei issue tracking module // Client is a client for nuclei issue tracking module
type Client struct { type Client struct {
tracker Tracker trackers []Tracker
exporters []Exporter
options *Options options *Options
dedupe *dedupe.Storage dedupe *dedupe.Storage
} }
// New creates a new nuclei issue tracker reporting client // New creates a new nuclei issue tracker reporting client
func New(config, db string) (*Client, error) { func New(options *Options, db string) (*Client, error) {
file, err := os.Open(config)
if err != nil {
return nil, errors.Wrap(err, "could not open reporting config file")
}
defer file.Close()
options := &Options{}
if parseErr := yaml.NewDecoder(file).Decode(options); parseErr != nil {
return nil, parseErr
}
if options.AllowList != nil { if options.AllowList != nil {
options.AllowList.Compile() options.AllowList.Compile()
} }
@ -101,27 +100,41 @@ func New(config, db string) (*Client, error) {
options.DenyList.Compile() options.DenyList.Compile()
} }
var tracker Tracker client := &Client{options: options}
if options.Github != nil { if options.Github != nil {
tracker, err = github.New(options.Github) tracker, err := github.New(options.Github)
}
if options.Gitlab != nil {
tracker, err = gitlab.New(options.Gitlab)
}
if options.Jira != nil {
tracker, err = jira.New(options.Jira)
}
if err != nil { if err != nil {
return nil, errors.Wrap(err, "could not create reporting client") return nil, errors.Wrap(err, "could not create reporting client")
} }
if tracker == nil { client.trackers = append(client.trackers, tracker)
return nil, errors.New("no issue tracker configuration found") }
if options.Gitlab != nil {
tracker, err := gitlab.New(options.Gitlab)
if err != nil {
return nil, errors.Wrap(err, "could not create reporting client")
}
client.trackers = append(client.trackers, tracker)
}
if options.Jira != nil {
tracker, err := jira.New(options.Jira)
if err != nil {
return nil, errors.Wrap(err, "could not create reporting client")
}
client.trackers = append(client.trackers, tracker)
}
if options.DiskExporter != nil {
exporter, err := disk.New(options.DiskExporter)
if err != nil {
return nil, errors.Wrap(err, "could not create exporting client")
}
client.exporters = append(client.exporters, exporter)
} }
storage, err := dedupe.New(db) storage, err := dedupe.New(db)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return &Client{tracker: tracker, dedupe: storage, options: options}, nil client.dedupe = storage
return client, nil
} }
// Close closes the issue tracker reporting client // Close closes the issue tracker reporting client
@ -138,15 +151,20 @@ func (c *Client) CreateIssue(event *output.ResultEvent) error {
return nil return nil
} }
found, err := c.dedupe.Index(event) unique, err := c.dedupe.Index(event)
if err != nil { if unique {
_ = c.tracker.CreateIssue(event) for _, tracker := range c.trackers {
if trackerErr := tracker.CreateIssue(event); trackerErr != nil {
err = multierr.Append(err, trackerErr)
}
}
for _, exporter := range c.exporters {
if exportErr := exporter.Export(event); exportErr != nil {
err = multierr.Append(err, exportErr)
}
}
}
return err return err
}
if found {
return c.tracker.CreateIssue(event)
}
return nil
} }
func stringSliceContains(slice []string, item string) bool { func stringSliceContains(slice []string, item string) bool {

View File

@ -9,7 +9,7 @@ import (
"github.com/google/go-github/github" "github.com/google/go-github/github"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues/format" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/format"
) )
// Integration is a client for a issue tracker integration // Integration is a client for a issue tracker integration

View File

@ -2,7 +2,7 @@ package gitlab
import ( import (
"github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues/format" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/format"
"github.com/xanzy/go-gitlab" "github.com/xanzy/go-gitlab"
) )

View File

@ -8,7 +8,7 @@ import (
jira "github.com/andygrunwald/go-jira" jira "github.com/andygrunwald/go-jira"
"github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues/format" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/format"
"github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/nuclei/v2/pkg/types"
) )
@ -97,7 +97,13 @@ func jiraFormatDescription(event *output.ResultEvent) string {
builder.WriteString("\n*Request*\n\n{code}\n") builder.WriteString("\n*Request*\n\n{code}\n")
builder.WriteString(event.Request) builder.WriteString(event.Request)
builder.WriteString("\n{code}\n\n*Response*\n\n{code}\n") builder.WriteString("\n{code}\n\n*Response*\n\n{code}\n")
// If the response is larger than 5 kb, truncate it before writing.
if len(event.Response) > 5*1024 {
builder.WriteString(event.Response[:5*1024])
builder.WriteString(".... Truncated ....")
} else {
builder.WriteString(event.Response) builder.WriteString(event.Response)
}
builder.WriteString("\n{code}\n\n") builder.WriteString("\n{code}\n\n")
if len(event.ExtractedResults) > 0 || len(event.Metadata) > 0 { if len(event.ExtractedResults) > 0 || len(event.Metadata) > 0 {
@ -123,6 +129,7 @@ func jiraFormatDescription(event *output.ResultEvent) string {
builder.WriteString("\n") builder.WriteString("\n")
} }
} }
builder.WriteString("\n---\nGenerated by [Nuclei|https://github.com/projectdiscovery/nuclei]")
data := builder.String() data := builder.String()
return data return data
} }

View File

@ -43,6 +43,8 @@ type Options struct {
ReportingDB string ReportingDB string
// ReportingConfig is the config file for nuclei reporting module // ReportingConfig is the config file for nuclei reporting module
ReportingConfig string ReportingConfig string
// DiskExportDirectory is the directory to export reports in markdown on disk to
DiskExportDirectory string
// ResolversFile is a file containing resolvers for nuclei. // ResolversFile is a file containing resolvers for nuclei.
ResolversFile string ResolversFile string
// StatsInterval is the number of seconds to display stats after // StatsInterval is the number of seconds to display stats after