From c6e7847c4eadf3687a3eb9e6a443a2333a0cf4f8 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 22 Mar 2021 12:18:05 +0530 Subject: [PATCH 01/10] Fixed matched now showing in metrics endpoint --- v2/pkg/progress/progress.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/progress/progress.go b/v2/pkg/progress/progress.go index 623870ae..0ec2626f 100644 --- a/v2/pkg/progress/progress.go +++ b/v2/pkg/progress/progress.go @@ -183,7 +183,7 @@ func (p *StatsTicker) getMetrics() map[string]interface{} { results["templates"] = clistats.String(templates) hosts, _ := p.stats.GetStatic("hosts") results["hosts"] = clistats.String(hosts) - matched, _ := p.stats.GetStatic("matched") + matched, _ := p.stats.GetCounter("matched") results["matched"] = clistats.String(matched) requests, _ := p.stats.GetCounter("requests") results["requests"] = clistats.String(requests) From f6a480f0b48d010f86ff51fcab3307dbce13facd Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 22 Mar 2021 14:03:05 +0530 Subject: [PATCH 02/10] Added disk exporters + changed some reporting modules around --- v2/cmd/nuclei/main.go | 3 +- v2/go.mod | 2 +- v2/go.sum | 4 +- v2/internal/runner/runner.go | 31 +++++- v2/pkg/protocols/protocols.go | 4 +- .../reporting/{issues => }/dedupe/dedupe.go | 0 .../{issues => }/dedupe/dedupe_test.go | 0 v2/pkg/reporting/exporters/disk/disk.go | 91 ++++++++++++++++++ .../reporting/{issues => }/format/format.go | 20 +++- .../{issues/issues.go => reporting.go} | 96 +++++++++++-------- .../{issues => trackers}/github/github.go | 2 +- .../{issues => trackers}/gitlab/gitlab.go | 2 +- .../{issues => trackers}/jira/jira.go | 2 +- v2/pkg/types/types.go | 2 + 14 files changed, 203 insertions(+), 56 deletions(-) rename v2/pkg/reporting/{issues => }/dedupe/dedupe.go (100%) rename v2/pkg/reporting/{issues => }/dedupe/dedupe_test.go (100%) create mode 100644 v2/pkg/reporting/exporters/disk/disk.go rename v2/pkg/reporting/{issues => }/format/format.go (84%) rename v2/pkg/reporting/{issues/issues.go => reporting.go} (59%) rename v2/pkg/reporting/{issues => trackers}/github/github.go (96%) rename v2/pkg/reporting/{issues => trackers}/gitlab/gitlab.go (96%) rename v2/pkg/reporting/{issues => trackers}/jira/jira.go (98%) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 9acf0195..b1764eff 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -77,7 +77,7 @@ based on templates offering massive extensibility and ease of use.`) set.BoolVar(&options.OfflineHTTP, "passive", false, "Enable Passive HTTP response processing mode") set.StringVarP(&options.BurpCollaboratorBiid, "burp-collaborator-biid", "biid", "", "Burp Collaborator BIID") set.StringVarP(&options.ReportingConfig, "report-config", "rc", "", "Nuclei Reporting Module configuration file") - set.StringVarP(&options.ReportingDB, "report-db", "rdb", "", "Local Nuclei Reporting Database") + set.StringVarP(&options.ReportingDB, "report-db", "rdb", "", "Local Nuclei Reporting Database (Always use this to persistent report data)") set.StringSliceVar(&options.Tags, "tags", []string{}, "Tags to execute templates for") set.StringSliceVarP(&options.ExcludeTags, "exclude-tags", "etags", []string{}, "Exclude templates with the provided tags") set.StringVarP(&options.ResolversFile, "resolvers", "r", "", "File containing resolver list for nuclei") @@ -87,6 +87,7 @@ based on templates offering massive extensibility and ease of use.`) set.BoolVar(&options.SystemResolvers, "system-resolvers", false, "Use system dns resolving as error fallback") set.IntVar(&options.PageTimeout, "page-timeout", 20, "Seconds to wait for each page in headless") set.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "Only run newly added templates") + set.StringVarP(&options.DiskExportDirectory, "disk-export", "de", "", "Directory on disk to export reports in markdown to") _ = set.Parse() if cfgFile != "" { diff --git a/v2/go.mod b/v2/go.mod index ba44e52c..972c641e 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -26,7 +26,7 @@ require ( github.com/projectdiscovery/clistats v0.0.8 github.com/projectdiscovery/collaborator v0.0.2 github.com/projectdiscovery/fastdialer v0.0.8 - github.com/projectdiscovery/goflags v0.0.3 + github.com/projectdiscovery/goflags v0.0.4 github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.1 github.com/projectdiscovery/rawhttp v0.0.6 diff --git a/v2/go.sum b/v2/go.sum index e82c0e49..44e195ed 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -201,8 +201,8 @@ github.com/projectdiscovery/collaborator v0.0.2 h1:BSiMlWM3NvuKbpedn6fIjjEo5b7q5 github.com/projectdiscovery/collaborator v0.0.2/go.mod h1:J1z0fC7Svutz3LJqoRyTHA3F0Suh4livmkYv8MnKw20= github.com/projectdiscovery/fastdialer v0.0.8 h1:mEMc8bfXV5hc1PUEkJiUnR5imYQe6+839Zezd5jLkc0= github.com/projectdiscovery/fastdialer v0.0.8/go.mod h1:AuaV0dzrNeBLHqjNnzpFSnTXnHGIZAlGQE+WUMmSIW4= -github.com/projectdiscovery/goflags v0.0.3 h1:5s9qblVIP/dQt7Mr3PMvZLvekEyioOS5oZtZ6ncLQHA= -github.com/projectdiscovery/goflags v0.0.3/go.mod h1:Ae1mJ5MIIqjys0lFe3GiMZ10Z8VLaxkYJ1ySA4Zv8HA= +github.com/projectdiscovery/goflags v0.0.4 h1:fWKLMAr3KmPlZxE1b54pfei+vGIUJn9q6aM7woZIbCY= +github.com/projectdiscovery/goflags v0.0.4/go.mod h1:Ae1mJ5MIIqjys0lFe3GiMZ10Z8VLaxkYJ1ySA4Zv8HA= github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI= github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY= github.com/projectdiscovery/hmap v0.0.1 h1:VAONbJw5jP+syI5smhsfkrq9XPGn4aiYy5pR6KR1wog= diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index c423a8be..9a40d88b 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -19,13 +19,15 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/clusterer" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine" - "github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/disk" "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/remeh/sizedwaitgroup" "github.com/rs/xid" "go.uber.org/atomic" "go.uber.org/ratelimit" + "gopkg.in/yaml.v2" ) // Runner is a client for running the enumeration process. @@ -39,7 +41,7 @@ type Runner struct { catalog *catalog.Catalog progress progress.Progress colorizer aurora.Aurora - issuesClient *issues.Client + issuesClient *reporting.Client severityColors *colorizer.Colorizer browser *engine.Browser ratelimiter ratelimit.Limiter @@ -66,13 +68,36 @@ func New(options *types.Options) (*Runner, error) { } runner.catalog = catalog.New(runner.options.TemplatesDirectory) + var reportingOptions *reporting.Options if options.ReportingConfig != "" { - if client, err := issues.New(options.ReportingConfig, options.ReportingDB); err != nil { + file, err := os.Open(options.ReportingConfig) + if err != nil { + gologger.Fatal().Msgf("Could not open reporting config file: %s\n", err) + } + + reportingOptions = &reporting.Options{} + if parseErr := yaml.NewDecoder(file).Decode(options); parseErr != nil { + file.Close() + gologger.Fatal().Msgf("Could not parse reporting config file: %s\n", parseErr) + } + file.Close() + } + if options.DiskExportDirectory != "" { + if reportingOptions != nil { + reportingOptions.DiskExporter = &disk.Options{Directory: options.DiskExportDirectory} + } else { + reportingOptions = &reporting.Options{} + reportingOptions.DiskExporter = &disk.Options{Directory: options.DiskExportDirectory} + } + } + if reportingOptions != nil { + if client, err := reporting.New(reportingOptions, options.ReportingDB); err != nil { gologger.Fatal().Msgf("Could not create issue reporting client: %s\n", err) } else { runner.issuesClient = client } } + // output coloring useColor := !options.NoColor runner.colorizer = aurora.NewAurora(useColor) diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 41c44a20..d4e89a02 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -9,7 +9,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/progress" "github.com/projectdiscovery/nuclei/v2/pkg/projectfile" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine" - "github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting" "github.com/projectdiscovery/nuclei/v2/pkg/types" "go.uber.org/ratelimit" ) @@ -39,7 +39,7 @@ type ExecuterOptions struct { // Options contains configuration options for the executer. Options *types.Options // IssuesClient is a client for nuclei issue tracker reporting - IssuesClient *issues.Client + IssuesClient *reporting.Client // Progress is a progress client for scan reporting Progress progress.Progress // RateLimiter is a rate-limiter for limiting sent number of requests. diff --git a/v2/pkg/reporting/issues/dedupe/dedupe.go b/v2/pkg/reporting/dedupe/dedupe.go similarity index 100% rename from v2/pkg/reporting/issues/dedupe/dedupe.go rename to v2/pkg/reporting/dedupe/dedupe.go diff --git a/v2/pkg/reporting/issues/dedupe/dedupe_test.go b/v2/pkg/reporting/dedupe/dedupe_test.go similarity index 100% rename from v2/pkg/reporting/issues/dedupe/dedupe_test.go rename to v2/pkg/reporting/dedupe/dedupe_test.go diff --git a/v2/pkg/reporting/exporters/disk/disk.go b/v2/pkg/reporting/exporters/disk/disk.go new file mode 100644 index 00000000..21f4a484 --- /dev/null +++ b/v2/pkg/reporting/exporters/disk/disk.go @@ -0,0 +1,91 @@ +package disk + +import ( + "bytes" + "crypto/sha1" + "fmt" + "io" + "io/ioutil" + "net/url" + "os" + "path" + "strings" + + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/format" + "github.com/segmentio/ksuid" +) + +type Exporter struct { + directory string + options *Options +} + +// Options contains the configuration options for github issue tracker client +type Options struct { + // Directory is the directory to export found results to + Directory string `yaml:"directory"` +} + +// New creates a new disk exporter integration client based on options. +func New(options *Options) (*Exporter, error) { + directory := options.Directory + if options.Directory == "" { + dir, err := os.Getwd() + if err != nil { + return nil, err + } + directory = dir + } + _ = os.MkdirAll(directory, os.ModePerm) + return &Exporter{options: options, directory: directory}, nil +} + +// Export exports a passed result event to disk +func (i *Exporter) Export(event *output.ResultEvent) error { + summary := format.Summary(event) + description := format.MarkdownDescription(event) + + var filename string + if outputFile := baseFilenameFromURL(event.Matched, event.Type); outputFile != "" { + filename = outputFile + } else { + filename = ksuid.New().String() + } + + filenameBuilder := &strings.Builder{} + filenameBuilder.WriteString(filename) + filenameBuilder.WriteString(".md") + finalFilename := filenameBuilder.String() + + dataBuilder := &bytes.Buffer{} + dataBuilder.WriteString("### ") + dataBuilder.WriteString(summary) + dataBuilder.WriteString("\n---\n") + dataBuilder.WriteString(description) + data := dataBuilder.Bytes() + + err := ioutil.WriteFile(path.Join(i.directory, finalFilename), data, 0644) + return err +} + +// Taken from https://github.com/michenriksen/aquatone/blob/854a5d56fbb7a00b2e5ec80d443026c7a4ced798/core/session.go#L215 +func baseFilenameFromURL(stru, protocol string) string { + u, err := url.Parse(stru) + if err != nil { + return "" + } + + h := sha1.New() + _, _ = io.WriteString(h, u.Path) + _, _ = io.WriteString(h, u.RawQuery) + _, _ = io.WriteString(h, u.Fragment) + + pathHash := fmt.Sprintf("%x", h.Sum(nil))[0:16] + host := strings.Replace(u.Host, ":", "__", 1) + if u.Scheme == "" { + u.Scheme = protocol + } + filename := fmt.Sprintf("%s__%s__%s", u.Scheme, strings.ReplaceAll(host, ".", "_"), pathHash) + return strings.ToLower(filename) +} diff --git a/v2/pkg/reporting/issues/format/format.go b/v2/pkg/reporting/format/format.go similarity index 84% rename from v2/pkg/reporting/issues/format/format.go rename to v2/pkg/reporting/format/format.go index 1544ad51..1a823c3a 100644 --- a/v2/pkg/reporting/issues/format/format.go +++ b/v2/pkg/reporting/format/format.go @@ -46,11 +46,21 @@ func MarkdownDescription(event *output.ResultEvent) string { for k, v := range event.Info { builder.WriteString(fmt.Sprintf("| %s | %s |\n", k, v)) } - builder.WriteString("\n**Request**\n\n```\n") - builder.WriteString(event.Request) - builder.WriteString("\n```\n\n
**Response**\n\n```\n") - builder.WriteString(event.Response) - builder.WriteString("\n```\n\n") + if event.Request != "" { + builder.WriteString("\n**Request**\n\n```\n") + builder.WriteString(event.Request) + } + if event.Response != "" { + builder.WriteString("\n```\n\n**Response**\n\n```\n") + // If the response is larger than 5 kb, truncate it before writing. + if len(event.Response) > 5*1024 { + builder.WriteString(event.Response[:5*1024]) + builder.WriteString(".... Truncated ....") + } else { + builder.WriteString(event.Response) + } + builder.WriteString("\n```\n\n") + } if len(event.ExtractedResults) > 0 || len(event.Metadata) > 0 { builder.WriteString("**Extra Information**\n\n") diff --git a/v2/pkg/reporting/issues/issues.go b/v2/pkg/reporting/reporting.go similarity index 59% rename from v2/pkg/reporting/issues/issues.go rename to v2/pkg/reporting/reporting.go index 66f78150..e33600bb 100644 --- a/v2/pkg/reporting/issues/issues.go +++ b/v2/pkg/reporting/reporting.go @@ -1,17 +1,17 @@ -package issues +package reporting import ( - "os" "strings" "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/output" - "github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues/dedupe" - "github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues/github" - "github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues/gitlab" - "github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues/jira" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/dedupe" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/disk" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/github" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/gitlab" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/jira" "github.com/projectdiscovery/nuclei/v2/pkg/types" - "gopkg.in/yaml.v2" + "go.uber.org/multierr" ) // Options is a configuration file for nuclei reporting module @@ -26,6 +26,8 @@ type Options struct { Gitlab *gitlab.Options `yaml:"gitlab"` // Jira contains configuration options for Jira Issue Tracker Jira *jira.Options `yaml:"jira"` + // DiskExporter contains configuration options for Disk Exporter Module + DiskExporter *disk.Options `yaml:"disk"` } // Filter filters the received event and decides whether to perform @@ -75,25 +77,22 @@ type Tracker interface { CreateIssue(event *output.ResultEvent) error } +// Exporter is an interface implemented by an issue exporter +type Exporter interface { + // Export exports an issue to an exporter + Export(event *output.ResultEvent) error +} + // Client is a client for nuclei issue tracking module type Client struct { - tracker Tracker - options *Options - dedupe *dedupe.Storage + trackers []Tracker + exporters []Exporter + options *Options + dedupe *dedupe.Storage } // New creates a new nuclei issue tracker reporting client -func New(config, db string) (*Client, error) { - file, err := os.Open(config) - if err != nil { - return nil, errors.Wrap(err, "could not open reporting config file") - } - defer file.Close() - - options := &Options{} - if parseErr := yaml.NewDecoder(file).Decode(options); parseErr != nil { - return nil, parseErr - } +func New(options *Options, db string) (*Client, error) { if options.AllowList != nil { options.AllowList.Compile() } @@ -101,27 +100,41 @@ func New(config, db string) (*Client, error) { options.DenyList.Compile() } - var tracker Tracker + client := &Client{options: options} if options.Github != nil { - tracker, err = github.New(options.Github) + tracker, err := github.New(options.Github) + if err != nil { + return nil, errors.Wrap(err, "could not create reporting client") + } + client.trackers = append(client.trackers, tracker) } if options.Gitlab != nil { - tracker, err = gitlab.New(options.Gitlab) + tracker, err := gitlab.New(options.Gitlab) + if err != nil { + return nil, errors.Wrap(err, "could not create reporting client") + } + client.trackers = append(client.trackers, tracker) } if options.Jira != nil { - tracker, err = jira.New(options.Jira) + tracker, err := jira.New(options.Jira) + if err != nil { + return nil, errors.Wrap(err, "could not create reporting client") + } + client.trackers = append(client.trackers, tracker) } - if err != nil { - return nil, errors.Wrap(err, "could not create reporting client") - } - if tracker == nil { - return nil, errors.New("no issue tracker configuration found") + if options.DiskExporter != nil { + exporter, err := disk.New(options.DiskExporter) + if err != nil { + return nil, errors.Wrap(err, "could not create exporting client") + } + client.exporters = append(client.exporters, exporter) } storage, err := dedupe.New(db) if err != nil { return nil, err } - return &Client{tracker: tracker, dedupe: storage, options: options}, nil + client.dedupe = storage + return client, nil } // Close closes the issue tracker reporting client @@ -138,15 +151,20 @@ func (c *Client) CreateIssue(event *output.ResultEvent) error { return nil } - found, err := c.dedupe.Index(event) - if err != nil { - _ = c.tracker.CreateIssue(event) - return err + unique, err := c.dedupe.Index(event) + if unique { + for _, tracker := range c.trackers { + if trackerErr := tracker.CreateIssue(event); trackerErr != nil { + err = multierr.Append(err, trackerErr) + } + } + for _, exporter := range c.exporters { + if exportErr := exporter.Export(event); exportErr != nil { + err = multierr.Append(err, exportErr) + } + } } - if found { - return c.tracker.CreateIssue(event) - } - return nil + return err } func stringSliceContains(slice []string, item string) bool { diff --git a/v2/pkg/reporting/issues/github/github.go b/v2/pkg/reporting/trackers/github/github.go similarity index 96% rename from v2/pkg/reporting/issues/github/github.go rename to v2/pkg/reporting/trackers/github/github.go index a7443634..89f6dceb 100644 --- a/v2/pkg/reporting/issues/github/github.go +++ b/v2/pkg/reporting/trackers/github/github.go @@ -9,7 +9,7 @@ import ( "github.com/google/go-github/github" "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/output" - "github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues/format" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/format" ) // Integration is a client for a issue tracker integration diff --git a/v2/pkg/reporting/issues/gitlab/gitlab.go b/v2/pkg/reporting/trackers/gitlab/gitlab.go similarity index 96% rename from v2/pkg/reporting/issues/gitlab/gitlab.go rename to v2/pkg/reporting/trackers/gitlab/gitlab.go index 895507c8..33d4fff0 100644 --- a/v2/pkg/reporting/issues/gitlab/gitlab.go +++ b/v2/pkg/reporting/trackers/gitlab/gitlab.go @@ -2,7 +2,7 @@ package gitlab import ( "github.com/projectdiscovery/nuclei/v2/pkg/output" - "github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues/format" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/format" "github.com/xanzy/go-gitlab" ) diff --git a/v2/pkg/reporting/issues/jira/jira.go b/v2/pkg/reporting/trackers/jira/jira.go similarity index 98% rename from v2/pkg/reporting/issues/jira/jira.go rename to v2/pkg/reporting/trackers/jira/jira.go index 0a0bfc80..e0cf7eb3 100644 --- a/v2/pkg/reporting/issues/jira/jira.go +++ b/v2/pkg/reporting/trackers/jira/jira.go @@ -8,7 +8,7 @@ import ( jira "github.com/andygrunwald/go-jira" "github.com/projectdiscovery/nuclei/v2/pkg/output" - "github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues/format" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/format" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index dec4a3b6..53464684 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -43,6 +43,8 @@ type Options struct { ReportingDB string // ReportingConfig is the config file for nuclei reporting module ReportingConfig string + // DiskExportDirectory is the directory to export reports in markdown on disk to + DiskExportDirectory string // ResolversFile is a file containing resolvers for nuclei. ResolversFile string // StatsInterval is the number of seconds to display stats after From 6b14e9071c7042cc03e551acf886dc53fc51d075 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 22 Mar 2021 14:05:49 +0530 Subject: [PATCH 03/10] Added length limit to jira --- v2/pkg/reporting/trackers/jira/jira.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/v2/pkg/reporting/trackers/jira/jira.go b/v2/pkg/reporting/trackers/jira/jira.go index e0cf7eb3..c17b4833 100644 --- a/v2/pkg/reporting/trackers/jira/jira.go +++ b/v2/pkg/reporting/trackers/jira/jira.go @@ -97,7 +97,13 @@ func jiraFormatDescription(event *output.ResultEvent) string { builder.WriteString("\n*Request*\n\n{code}\n") builder.WriteString(event.Request) builder.WriteString("\n{code}\n\n*Response*\n\n{code}\n") - builder.WriteString(event.Response) + // If the response is larger than 5 kb, truncate it before writing. + if len(event.Response) > 5*1024 { + builder.WriteString(event.Response[:5*1024]) + builder.WriteString(".... Truncated ....") + } else { + builder.WriteString(event.Response) + } builder.WriteString("\n{code}\n\n") if len(event.ExtractedResults) > 0 || len(event.Metadata) > 0 { From fbc5b313414c0e8debb17c6c476291efd58daebc Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 22 Mar 2021 14:28:29 +0530 Subject: [PATCH 04/10] Fixed filename bug in disk exporter --- v2/pkg/reporting/exporters/disk/disk.go | 37 ++----------------------- 1 file changed, 3 insertions(+), 34 deletions(-) diff --git a/v2/pkg/reporting/exporters/disk/disk.go b/v2/pkg/reporting/exporters/disk/disk.go index 21f4a484..9bb32291 100644 --- a/v2/pkg/reporting/exporters/disk/disk.go +++ b/v2/pkg/reporting/exporters/disk/disk.go @@ -2,18 +2,13 @@ package disk import ( "bytes" - "crypto/sha1" - "fmt" - "io" "io/ioutil" - "net/url" "os" "path" "strings" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/format" - "github.com/segmentio/ksuid" ) type Exporter struct { @@ -46,15 +41,10 @@ func (i *Exporter) Export(event *output.ResultEvent) error { summary := format.Summary(event) description := format.MarkdownDescription(event) - var filename string - if outputFile := baseFilenameFromURL(event.Matched, event.Type); outputFile != "" { - filename = outputFile - } else { - filename = ksuid.New().String() - } - filenameBuilder := &strings.Builder{} - filenameBuilder.WriteString(filename) + filenameBuilder.WriteString(event.TemplateID) + filenameBuilder.WriteString("-") + filenameBuilder.WriteString(strings.ReplaceAll(event.Matched, "/", "_")) filenameBuilder.WriteString(".md") finalFilename := filenameBuilder.String() @@ -68,24 +58,3 @@ func (i *Exporter) Export(event *output.ResultEvent) error { err := ioutil.WriteFile(path.Join(i.directory, finalFilename), data, 0644) return err } - -// Taken from https://github.com/michenriksen/aquatone/blob/854a5d56fbb7a00b2e5ec80d443026c7a4ced798/core/session.go#L215 -func baseFilenameFromURL(stru, protocol string) string { - u, err := url.Parse(stru) - if err != nil { - return "" - } - - h := sha1.New() - _, _ = io.WriteString(h, u.Path) - _, _ = io.WriteString(h, u.RawQuery) - _, _ = io.WriteString(h, u.Fragment) - - pathHash := fmt.Sprintf("%x", h.Sum(nil))[0:16] - host := strings.Replace(u.Host, ":", "__", 1) - if u.Scheme == "" { - u.Scheme = protocol - } - filename := fmt.Sprintf("%s__%s__%s", u.Scheme, strings.ReplaceAll(host, ".", "_"), pathHash) - return strings.ToLower(filename) -} From 2bd2286115be638aec669db98ff91f6c833fbfa2 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 22 Mar 2021 14:36:08 +0530 Subject: [PATCH 05/10] Adding nuclei link to reports --- v2/pkg/reporting/format/format.go | 1 + v2/pkg/reporting/trackers/jira/jira.go | 1 + 2 files changed, 2 insertions(+) diff --git a/v2/pkg/reporting/format/format.go b/v2/pkg/reporting/format/format.go index 1a823c3a..e8cf08de 100644 --- a/v2/pkg/reporting/format/format.go +++ b/v2/pkg/reporting/format/format.go @@ -85,6 +85,7 @@ func MarkdownDescription(event *output.ResultEvent) string { builder.WriteString("\n") } } + builder.WriteString("\n---\nGenerated by [Nuclei](https://github.com/projectdiscovery/nuclei)") data := builder.String() return data } diff --git a/v2/pkg/reporting/trackers/jira/jira.go b/v2/pkg/reporting/trackers/jira/jira.go index c17b4833..7053a2c5 100644 --- a/v2/pkg/reporting/trackers/jira/jira.go +++ b/v2/pkg/reporting/trackers/jira/jira.go @@ -129,6 +129,7 @@ func jiraFormatDescription(event *output.ResultEvent) string { builder.WriteString("\n") } } + builder.WriteString("\n---\nGenerated by [Nuclei|https://github.com/projectdiscovery/nuclei]") data := builder.String() return data } From 6794cbf4b66fe0a7469b9bdc3baee30117f39016 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 22 Mar 2021 14:48:11 +0530 Subject: [PATCH 06/10] Bug fixes --- v2/internal/runner/runner.go | 2 +- v2/pkg/reporting/exporters/disk/disk.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 9a40d88b..efc1b4bc 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -76,7 +76,7 @@ func New(options *types.Options) (*Runner, error) { } reportingOptions = &reporting.Options{} - if parseErr := yaml.NewDecoder(file).Decode(options); parseErr != nil { + if parseErr := yaml.NewDecoder(file).Decode(reportingOptions); parseErr != nil { file.Close() gologger.Fatal().Msgf("Could not parse reporting config file: %s\n", parseErr) } diff --git a/v2/pkg/reporting/exporters/disk/disk.go b/v2/pkg/reporting/exporters/disk/disk.go index 9bb32291..35c1d25b 100644 --- a/v2/pkg/reporting/exporters/disk/disk.go +++ b/v2/pkg/reporting/exporters/disk/disk.go @@ -44,7 +44,7 @@ func (i *Exporter) Export(event *output.ResultEvent) error { filenameBuilder := &strings.Builder{} filenameBuilder.WriteString(event.TemplateID) filenameBuilder.WriteString("-") - filenameBuilder.WriteString(strings.ReplaceAll(event.Matched, "/", "_")) + filenameBuilder.WriteString(strings.ReplaceAll(strings.ReplaceAll(event.Matched, "/", "_"), ":", "_")) filenameBuilder.WriteString(".md") finalFilename := filenameBuilder.String() From bfa2dacb7b8431eb478a9da21550f78d69c596c7 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 22 Mar 2021 15:00:26 +0530 Subject: [PATCH 07/10] Misc --- v2/cmd/nuclei/main.go | 2 -- v2/internal/runner/runner.go | 2 ++ v2/pkg/reporting/format/format.go | 5 +++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index b1764eff..9e961f07 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -24,9 +24,7 @@ func main() { if err != nil { gologger.Fatal().Msgf("Could not create runner: %s\n", err) } - nucleiRunner.RunEnumeration() - nucleiRunner.Close() } func readConfig() { diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index efc1b4bc..c8238664 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -222,6 +222,8 @@ func (r *Runner) Close() { // RunEnumeration sets up the input layer for giving input nuclei. // binary and runs the actual enumeration func (r *Runner) RunEnumeration() { + defer r.Close() + // If we have no templates, run on whole template directory with provided tags if len(r.options.Templates) == 0 && len(r.options.Workflows) == 0 && !r.options.NewTemplates && (len(r.options.Tags) > 0 || len(r.options.ExcludeTags) > 0) { r.options.Templates = append(r.options.Templates, r.options.TemplatesDirectory) diff --git a/v2/pkg/reporting/format/format.go b/v2/pkg/reporting/format/format.go index e8cf08de..4654baef 100644 --- a/v2/pkg/reporting/format/format.go +++ b/v2/pkg/reporting/format/format.go @@ -47,11 +47,12 @@ func MarkdownDescription(event *output.ResultEvent) string { builder.WriteString(fmt.Sprintf("| %s | %s |\n", k, v)) } if event.Request != "" { - builder.WriteString("\n**Request**\n\n```\n") + builder.WriteString("\n**Request**\n\n```http\n") builder.WriteString(event.Request) + builder.WriteString("\n```\n") } if event.Response != "" { - builder.WriteString("\n```\n\n**Response**\n\n```\n") + builder.WriteString("\n**Response**\n\n```http\n") // If the response is larger than 5 kb, truncate it before writing. if len(event.Response) > 5*1024 { builder.WriteString(event.Response[:5*1024]) From 9521daf3aaff77d5f74f4b45e88a1b0f5557e167 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 22 Mar 2021 15:48:27 +0530 Subject: [PATCH 08/10] Fixed nuclei ignore to read from .config --- v2/internal/runner/config.go | 29 ++++++++++++++++++++++++----- v2/internal/runner/update.go | 35 +++++++++++++++++++++++++++++++++-- 2 files changed, 57 insertions(+), 7 deletions(-) diff --git a/v2/internal/runner/config.go b/v2/internal/runner/config.go index 4c786dcf..c7634388 100644 --- a/v2/internal/runner/config.go +++ b/v2/internal/runner/config.go @@ -16,13 +16,15 @@ type nucleiConfig struct { TemplatesDirectory string `json:"templates-directory,omitempty"` CurrentVersion string `json:"current-version,omitempty"` LastChecked time.Time `json:"last-checked,omitempty"` - + IgnoreURL string `json:"ignore-url,omitempty"` + NucleiVersion string `json:"nuclei-version,omitempty"` + LastCheckedIgnore time.Time `json:"last-checked-ignore,omitempty"` // IgnorePaths ignores all the paths listed unless specified manually IgnorePaths []string `json:"ignore-paths,omitempty"` } // nucleiConfigFilename is the filename of nuclei configuration file. -const nucleiConfigFilename = ".nuclei-config.json" +const nucleiConfigFilename = ".templates-config.json" var reVersion = regexp.MustCompile(`\d+\.\d+\.\d+`) @@ -32,8 +34,10 @@ func readConfiguration() (*nucleiConfig, error) { if err != nil { return nil, err } + configDir := path.Join(home, "/.config", "/nuclei") + _ = os.MkdirAll(configDir, os.ModePerm) - templatesConfigFile := path.Join(home, nucleiConfigFilename) + templatesConfigFile := path.Join(configDir, nucleiConfigFilename) file, err := os.Open(templatesConfigFile) if err != nil { return nil, err @@ -54,9 +58,16 @@ func (r *Runner) writeConfiguration(config *nucleiConfig) error { if err != nil { return err } + configDir := path.Join(home, "/.config", "/nuclei") + _ = os.MkdirAll(configDir, os.ModePerm) + if config.IgnoreURL == "" { + config.IgnoreURL = "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore" + } config.LastChecked = time.Now() - templatesConfigFile := path.Join(home, nucleiConfigFilename) + config.LastCheckedIgnore = time.Now() + config.NucleiVersion = Version + templatesConfigFile := path.Join(configDir, nucleiConfigFilename) file, err := os.OpenFile(templatesConfigFile, os.O_WRONLY|os.O_CREATE, 0777) if err != nil { return err @@ -95,7 +106,15 @@ func (r *Runner) readNucleiIgnoreFile() { // getIgnoreFilePath returns the ignore file path for the runner func (r *Runner) getIgnoreFilePath() string { - defIgnoreFilePath := path.Join(r.templatesConfig.TemplatesDirectory, nucleiIgnoreFile) + var defIgnoreFilePath string + + home, err := os.UserHomeDir() + if err == nil { + configDir := path.Join(home, "/.config", "/nuclei") + _ = os.MkdirAll(configDir, os.ModePerm) + + defIgnoreFilePath = path.Join(configDir, nucleiIgnoreFile) + } cwd, err := os.Getwd() if err != nil { diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go index 611eec4f..181f555c 100644 --- a/v2/internal/runner/update.go +++ b/v2/internal/runner/update.go @@ -41,6 +41,8 @@ func (r *Runner) updateTemplates() error { if err != nil { return err } + configDir := path.Join(home, "/.config", "/nuclei") + _ = os.MkdirAll(configDir, os.ModePerm) templatesConfigFile := path.Join(home, nucleiConfigFilename) if _, statErr := os.Stat(templatesConfigFile); !os.IsNotExist(statErr) { @@ -51,6 +53,34 @@ func (r *Runner) updateTemplates() error { r.templatesConfig = config } + // Check if last checked for nuclei-ignore is more than 1 hours. + // and if true, run the check. + if r.templatesConfig == nil || time.Since(r.templatesConfig.LastCheckedIgnore) > 1*time.Hour { + ignoreURL := "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore" + if r.templatesConfig != nil && r.templatesConfig.IgnoreURL == "" { + ignoreURL = r.templatesConfig.IgnoreURL + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + req, reqErr := http.NewRequestWithContext(ctx, http.MethodGet, ignoreURL, nil) + if reqErr == nil { + resp, httpGet := http.DefaultClient.Do(req) + if httpGet == nil { + data, _ := ioutil.ReadAll(resp.Body) + resp.Body.Close() + + if len(data) > 0 { + _ = ioutil.WriteFile(path.Join(configDir, nucleiIgnoreFile), data, 0644) + } + if r.templatesConfig != nil { + r.templatesConfig.LastCheckedIgnore = time.Now() + } + } else if resp.Body != nil { + resp.Body.Close() + } + } + cancel() + } + ctx := context.Background() if r.templatesConfig == nil || (r.options.TemplatesDirectory != "" && r.templatesConfig.TemplatesDirectory != r.options.TemplatesDirectory) { if !r.options.UpdateTemplates { @@ -59,7 +89,9 @@ func (r *Runner) updateTemplates() error { } // Use custom location if user has given a template directory - r.templatesConfig = &nucleiConfig{TemplatesDirectory: path.Join(home, "nuclei-templates")} + r.templatesConfig = &nucleiConfig{ + TemplatesDirectory: path.Join(home, "nuclei-templates"), + } if r.options.TemplatesDirectory != "" && r.options.TemplatesDirectory != path.Join(home, "nuclei-templates") { r.templatesConfig.TemplatesDirectory = r.options.TemplatesDirectory } @@ -132,7 +164,6 @@ func (r *Runner) updateTemplates() error { if err != nil { return err } - err = r.writeConfiguration(r.templatesConfig) if err != nil { return err From 363fb34520a40186ea4eb30496607a13ad8ee294 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 22 Mar 2021 17:10:58 +0530 Subject: [PATCH 09/10] Misc --- v2/internal/runner/update.go | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go index 181f555c..fe811853 100644 --- a/v2/internal/runner/update.go +++ b/v2/internal/runner/update.go @@ -53,18 +53,36 @@ func (r *Runner) updateTemplates() error { r.templatesConfig = config } + ignoreURL := "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore" + if r.templatesConfig == nil { + currentConfig := &nucleiConfig{ + TemplatesDirectory: path.Join(home, "nuclei-templates"), + IgnoreURL: ignoreURL, + NucleiVersion: Version, + } + if err := r.writeConfiguration(currentConfig); err != nil { + return errors.Wrap(err, "could not write template configuration") + } + r.templatesConfig = currentConfig + } // Check if last checked for nuclei-ignore is more than 1 hours. // and if true, run the check. - if r.templatesConfig == nil || time.Since(r.templatesConfig.LastCheckedIgnore) > 1*time.Hour { - ignoreURL := "https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/master/.nuclei-ignore" + if r.templatesConfig == nil || time.Since(r.templatesConfig.LastCheckedIgnore) > 1*time.Hour || r.options.UpdateTemplates { if r.templatesConfig != nil && r.templatesConfig.IgnoreURL == "" { ignoreURL = r.templatesConfig.IgnoreURL } + gologger.Verbose().Msgf("Downloading config file from %s", ignoreURL) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) req, reqErr := http.NewRequestWithContext(ctx, http.MethodGet, ignoreURL, nil) if reqErr == nil { resp, httpGet := http.DefaultClient.Do(req) - if httpGet == nil { + if httpGet != nil { + if resp != nil && resp.Body != nil { + resp.Body.Close() + } + gologger.Warning().Msgf("Could not get ignore-file from %s: %s", ignoreURL, err) + } else { data, _ := ioutil.ReadAll(resp.Body) resp.Body.Close() @@ -74,17 +92,15 @@ func (r *Runner) updateTemplates() error { if r.templatesConfig != nil { r.templatesConfig.LastCheckedIgnore = time.Now() } - } else if resp.Body != nil { - resp.Body.Close() } } cancel() } ctx := context.Background() - if r.templatesConfig == nil || (r.options.TemplatesDirectory != "" && r.templatesConfig.TemplatesDirectory != r.options.TemplatesDirectory) { + if r.templatesConfig.CurrentVersion == "" || (r.options.TemplatesDirectory != "" && r.templatesConfig.TemplatesDirectory != r.options.TemplatesDirectory) { if !r.options.UpdateTemplates { - gologger.Warning().Msgf("nuclei-templates are not installed, use update-templates flag.\n") + gologger.Warning().Msgf("nuclei-templates are not installed (or indexed), use update-templates flag.\n") return nil } From 893aff46f6e06c5a1ee8f14b512a1d721c364675 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 22 Mar 2021 17:17:58 +0530 Subject: [PATCH 10/10] Fixed linter --- v2/internal/runner/update.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go index fe811853..f388bcec 100644 --- a/v2/internal/runner/update.go +++ b/v2/internal/runner/update.go @@ -60,8 +60,8 @@ func (r *Runner) updateTemplates() error { IgnoreURL: ignoreURL, NucleiVersion: Version, } - if err := r.writeConfiguration(currentConfig); err != nil { - return errors.Wrap(err, "could not write template configuration") + if writeErr := r.writeConfiguration(currentConfig); writeErr != nil { + return errors.Wrap(writeErr, "could not write template configuration") } r.templatesConfig = currentConfig }