Added disk exporters + changed some reporting modules around

v2/cmd/nuclei/main.go

@@ -77,7 +77,7 @@ based on templates offering massive extensibility and ease of use.`)
 	set.BoolVar(&options.OfflineHTTP, "passive", false, "Enable Passive HTTP response processing mode")
 	set.StringVarP(&options.BurpCollaboratorBiid, "burp-collaborator-biid", "biid", "", "Burp Collaborator BIID")
 	set.StringVarP(&options.ReportingConfig, "report-config", "rc", "", "Nuclei Reporting Module configuration file")
-	set.StringVarP(&options.ReportingDB, "report-db", "rdb", "", "Local Nuclei Reporting Database")
+	set.StringVarP(&options.ReportingDB, "report-db", "rdb", "", "Local Nuclei Reporting Database (Always use this to persistent report data)")
 	set.StringSliceVar(&options.Tags, "tags", []string{}, "Tags to execute templates for")
 	set.StringSliceVarP(&options.ExcludeTags, "exclude-tags", "etags", []string{}, "Exclude templates with the provided tags")
 	set.StringVarP(&options.ResolversFile, "resolvers", "r", "", "File containing resolver list for nuclei")
@@ -87,6 +87,7 @@ based on templates offering massive extensibility and ease of use.`)
 	set.BoolVar(&options.SystemResolvers, "system-resolvers", false, "Use system dns resolving as error fallback")
 	set.IntVar(&options.PageTimeout, "page-timeout", 20, "Seconds to wait for each page in headless")
 	set.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "Only run newly added templates")
+	set.StringVarP(&options.DiskExportDirectory, "disk-export", "de", "", "Directory on disk to export reports in markdown to")
 	_ = set.Parse()
 
 	if cfgFile != "" {

v2/go.mod

@@ -26,7 +26,7 @@ require (
 	github.com/projectdiscovery/clistats v0.0.8
 	github.com/projectdiscovery/collaborator v0.0.2
 	github.com/projectdiscovery/fastdialer v0.0.8
-	github.com/projectdiscovery/goflags v0.0.3
+	github.com/projectdiscovery/goflags v0.0.4
 	github.com/projectdiscovery/gologger v1.1.4
 	github.com/projectdiscovery/hmap v0.0.1
 	github.com/projectdiscovery/rawhttp v0.0.6

v2/go.sum

@@ -201,8 +201,8 @@ github.com/projectdiscovery/collaborator v0.0.2 h1:BSiMlWM3NvuKbpedn6fIjjEo5b7q5
 github.com/projectdiscovery/collaborator v0.0.2/go.mod h1:J1z0fC7Svutz3LJqoRyTHA3F0Suh4livmkYv8MnKw20=
 github.com/projectdiscovery/fastdialer v0.0.8 h1:mEMc8bfXV5hc1PUEkJiUnR5imYQe6+839Zezd5jLkc0=
 github.com/projectdiscovery/fastdialer v0.0.8/go.mod h1:AuaV0dzrNeBLHqjNnzpFSnTXnHGIZAlGQE+WUMmSIW4=
-github.com/projectdiscovery/goflags v0.0.3 h1:5s9qblVIP/dQt7Mr3PMvZLvekEyioOS5oZtZ6ncLQHA=
-github.com/projectdiscovery/goflags v0.0.3/go.mod h1:Ae1mJ5MIIqjys0lFe3GiMZ10Z8VLaxkYJ1ySA4Zv8HA=
+github.com/projectdiscovery/goflags v0.0.4 h1:fWKLMAr3KmPlZxE1b54pfei+vGIUJn9q6aM7woZIbCY=
+github.com/projectdiscovery/goflags v0.0.4/go.mod h1:Ae1mJ5MIIqjys0lFe3GiMZ10Z8VLaxkYJ1ySA4Zv8HA=
 github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI=
 github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY=
 github.com/projectdiscovery/hmap v0.0.1 h1:VAONbJw5jP+syI5smhsfkrq9XPGn4aiYy5pR6KR1wog=

v2/internal/runner/runner.go

@@ -19,13 +19,15 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/clusterer"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine"
-	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues"
+	"github.com/projectdiscovery/nuclei/v2/pkg/reporting"
+	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/disk"
 	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 	"github.com/remeh/sizedwaitgroup"
 	"github.com/rs/xid"
 	"go.uber.org/atomic"
 	"go.uber.org/ratelimit"
+	"gopkg.in/yaml.v2"
 )
 
 // Runner is a client for running the enumeration process.
@@ -39,7 +41,7 @@ type Runner struct {
 	catalog         *catalog.Catalog
 	progress        progress.Progress
 	colorizer       aurora.Aurora
-	issuesClient    *issues.Client
+	issuesClient    *reporting.Client
 	severityColors  *colorizer.Colorizer
 	browser         *engine.Browser
 	ratelimiter     ratelimit.Limiter
@@ -66,13 +68,36 @@ func New(options *types.Options) (*Runner, error) {
 	}
 	runner.catalog = catalog.New(runner.options.TemplatesDirectory)
 
+	var reportingOptions *reporting.Options
 	if options.ReportingConfig != "" {
-		if client, err := issues.New(options.ReportingConfig, options.ReportingDB); err != nil {
+		file, err := os.Open(options.ReportingConfig)
+		if err != nil {
+			gologger.Fatal().Msgf("Could not open reporting config file: %s\n", err)
+		}
+
+		reportingOptions = &reporting.Options{}
+		if parseErr := yaml.NewDecoder(file).Decode(options); parseErr != nil {
+			file.Close()
+			gologger.Fatal().Msgf("Could not parse reporting config file: %s\n", parseErr)
+		}
+		file.Close()
+	}
+	if options.DiskExportDirectory != "" {
+		if reportingOptions != nil {
+			reportingOptions.DiskExporter = &disk.Options{Directory: options.DiskExportDirectory}
+		} else {
+			reportingOptions = &reporting.Options{}
+			reportingOptions.DiskExporter = &disk.Options{Directory: options.DiskExportDirectory}
+		}
+	}
+	if reportingOptions != nil {
+		if client, err := reporting.New(reportingOptions, options.ReportingDB); err != nil {
 			gologger.Fatal().Msgf("Could not create issue reporting client: %s\n", err)
 		} else {
 			runner.issuesClient = client
 		}
 	}
+
 	// output coloring
 	useColor := !options.NoColor
 	runner.colorizer = aurora.NewAurora(useColor)

v2/pkg/protocols/protocols.go

@@ -9,7 +9,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/progress"
 	"github.com/projectdiscovery/nuclei/v2/pkg/projectfile"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine"
-	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues"
+	"github.com/projectdiscovery/nuclei/v2/pkg/reporting"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 	"go.uber.org/ratelimit"
 )
@@ -39,7 +39,7 @@ type ExecuterOptions struct {
 	// Options contains configuration options for the executer.
 	Options *types.Options
 	// IssuesClient is a client for nuclei issue tracker reporting
-	IssuesClient *issues.Client
+	IssuesClient *reporting.Client
 	// Progress is a progress client for scan reporting
 	Progress progress.Progress
 	// RateLimiter is a rate-limiter for limiting sent number of requests.

v2/pkg/reporting/exporters/disk/disk.go

@@ -0,0 +1,91 @@
+package disk
+
+import (
+	"bytes"
+	"crypto/sha1"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/url"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/projectdiscovery/nuclei/v2/pkg/output"
+	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/format"
+	"github.com/segmentio/ksuid"
+)
+
+type Exporter struct {
+	directory string
+	options   *Options
+}
+
+// Options contains the configuration options for github issue tracker client
+type Options struct {
+	// Directory is the directory to export found results to
+	Directory string `yaml:"directory"`
+}
+
+// New creates a new disk exporter integration client based on options.
+func New(options *Options) (*Exporter, error) {
+	directory := options.Directory
+	if options.Directory == "" {
+		dir, err := os.Getwd()
+		if err != nil {
+			return nil, err
+		}
+		directory = dir
+	}
+	_ = os.MkdirAll(directory, os.ModePerm)
+	return &Exporter{options: options, directory: directory}, nil
+}
+
+// Export exports a passed result event to disk
+func (i *Exporter) Export(event *output.ResultEvent) error {
+	summary := format.Summary(event)
+	description := format.MarkdownDescription(event)
+
+	var filename string
+	if outputFile := baseFilenameFromURL(event.Matched, event.Type); outputFile != "" {
+		filename = outputFile
+	} else {
+		filename = ksuid.New().String()
+	}
+
+	filenameBuilder := &strings.Builder{}
+	filenameBuilder.WriteString(filename)
+	filenameBuilder.WriteString(".md")
+	finalFilename := filenameBuilder.String()
+
+	dataBuilder := &bytes.Buffer{}
+	dataBuilder.WriteString("### ")
+	dataBuilder.WriteString(summary)
+	dataBuilder.WriteString("\n---\n")
+	dataBuilder.WriteString(description)
+	data := dataBuilder.Bytes()
+
+	err := ioutil.WriteFile(path.Join(i.directory, finalFilename), data, 0644)
+	return err
+}
+
+// Taken from https://github.com/michenriksen/aquatone/blob/854a5d56fbb7a00b2e5ec80d443026c7a4ced798/core/session.go#L215
+func baseFilenameFromURL(stru, protocol string) string {
+	u, err := url.Parse(stru)
+	if err != nil {
+		return ""
+	}
+
+	h := sha1.New()
+	_, _ = io.WriteString(h, u.Path)
+	_, _ = io.WriteString(h, u.RawQuery)
+	_, _ = io.WriteString(h, u.Fragment)
+
+	pathHash := fmt.Sprintf("%x", h.Sum(nil))[0:16]
+	host := strings.Replace(u.Host, ":", "__", 1)
+	if u.Scheme == "" {
+		u.Scheme = protocol
+	}
+	filename := fmt.Sprintf("%s__%s__%s", u.Scheme, strings.ReplaceAll(host, ".", "_"), pathHash)
+	return strings.ToLower(filename)
+}

v2/pkg/reporting/format/format.go

@@ -46,11 +46,21 @@ func MarkdownDescription(event *output.ResultEvent) string {
 	for k, v := range event.Info {
 		builder.WriteString(fmt.Sprintf("| %s | %s |\n", k, v))
 	}
-	builder.WriteString("\n**Request**\n\n```\n")
-	builder.WriteString(event.Request)
-	builder.WriteString("\n```\n\n<details><summary>**Response**</summary>\n\n```\n")
-	builder.WriteString(event.Response)
-	builder.WriteString("\n```\n\n")
+	if event.Request != "" {
+		builder.WriteString("\n**Request**\n\n```\n")
+		builder.WriteString(event.Request)
+	}
+	if event.Response != "" {
+		builder.WriteString("\n```\n\n**Response**\n\n```\n")
+		// If the response is larger than 5 kb, truncate it before writing.
+		if len(event.Response) > 5*1024 {
+			builder.WriteString(event.Response[:5*1024])
+			builder.WriteString(".... Truncated ....")
+		} else {
+			builder.WriteString(event.Response)
+		}
+		builder.WriteString("\n```\n\n")
+	}
 
 	if len(event.ExtractedResults) > 0 || len(event.Metadata) > 0 {
 		builder.WriteString("**Extra Information**\n\n")

v2/pkg/reporting/reporting.go

@@ -1,17 +1,17 @@
-package issues
+package reporting
 
 import (
-	"os"
 	"strings"
 
 	"github.com/pkg/errors"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
-	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues/dedupe"
-	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues/github"
-	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues/gitlab"
-	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues/jira"
+	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/dedupe"
+	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/disk"
+	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/github"
+	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/gitlab"
+	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/jira"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
-	"gopkg.in/yaml.v2"
+	"go.uber.org/multierr"
 )
 
 // Options is a configuration file for nuclei reporting module
@@ -26,6 +26,8 @@ type Options struct {
 	Gitlab *gitlab.Options `yaml:"gitlab"`
 	// Jira contains configuration options for Jira Issue Tracker
 	Jira *jira.Options `yaml:"jira"`
+	// DiskExporter contains configuration options for Disk Exporter Module
+	DiskExporter *disk.Options `yaml:"disk"`
 }
 
 // Filter filters the received event and decides whether to perform
@@ -75,25 +77,22 @@ type Tracker interface {
 	CreateIssue(event *output.ResultEvent) error
 }
 
+// Exporter is an interface implemented by an issue exporter
+type Exporter interface {
+	// Export exports an issue to an exporter
+	Export(event *output.ResultEvent) error
+}
+
 // Client is a client for nuclei issue tracking module
 type Client struct {
-	tracker Tracker
-	options *Options
-	dedupe  *dedupe.Storage
+	trackers  []Tracker
+	exporters []Exporter
+	options   *Options
+	dedupe    *dedupe.Storage
 }
 
 // New creates a new nuclei issue tracker reporting client
-func New(config, db string) (*Client, error) {
-	file, err := os.Open(config)
-	if err != nil {
-		return nil, errors.Wrap(err, "could not open reporting config file")
-	}
-	defer file.Close()
-
-	options := &Options{}
-	if parseErr := yaml.NewDecoder(file).Decode(options); parseErr != nil {
-		return nil, parseErr
-	}
+func New(options *Options, db string) (*Client, error) {
 	if options.AllowList != nil {
 		options.AllowList.Compile()
 	}
@@ -101,27 +100,41 @@ func New(config, db string) (*Client, error) {
 		options.DenyList.Compile()
 	}
 
-	var tracker Tracker
+	client := &Client{options: options}
 	if options.Github != nil {
-		tracker, err = github.New(options.Github)
+		tracker, err := github.New(options.Github)
+		if err != nil {
+			return nil, errors.Wrap(err, "could not create reporting client")
+		}
+		client.trackers = append(client.trackers, tracker)
 	}
 	if options.Gitlab != nil {
-		tracker, err = gitlab.New(options.Gitlab)
+		tracker, err := gitlab.New(options.Gitlab)
+		if err != nil {
+			return nil, errors.Wrap(err, "could not create reporting client")
+		}
+		client.trackers = append(client.trackers, tracker)
 	}
 	if options.Jira != nil {
-		tracker, err = jira.New(options.Jira)
+		tracker, err := jira.New(options.Jira)
+		if err != nil {
+			return nil, errors.Wrap(err, "could not create reporting client")
+		}
+		client.trackers = append(client.trackers, tracker)
 	}
-	if err != nil {
-		return nil, errors.Wrap(err, "could not create reporting client")
-	}
-	if tracker == nil {
-		return nil, errors.New("no issue tracker configuration found")
+	if options.DiskExporter != nil {
+		exporter, err := disk.New(options.DiskExporter)
+		if err != nil {
+			return nil, errors.Wrap(err, "could not create exporting client")
+		}
+		client.exporters = append(client.exporters, exporter)
 	}
 	storage, err := dedupe.New(db)
 	if err != nil {
 		return nil, err
 	}
-	return &Client{tracker: tracker, dedupe: storage, options: options}, nil
+	client.dedupe = storage
+	return client, nil
 }
 
 // Close closes the issue tracker reporting client
@@ -138,15 +151,20 @@ func (c *Client) CreateIssue(event *output.ResultEvent) error {
 		return nil
 	}
 
-	found, err := c.dedupe.Index(event)
-	if err != nil {
-		_ = c.tracker.CreateIssue(event)
-		return err
+	unique, err := c.dedupe.Index(event)
+	if unique {
+		for _, tracker := range c.trackers {
+			if trackerErr := tracker.CreateIssue(event); trackerErr != nil {
+				err = multierr.Append(err, trackerErr)
+			}
+		}
+		for _, exporter := range c.exporters {
+			if exportErr := exporter.Export(event); exportErr != nil {
+				err = multierr.Append(err, exportErr)
+			}
+		}
 	}
-	if found {
-		return c.tracker.CreateIssue(event)
-	}
-	return nil
+	return err
 }
 
 func stringSliceContains(slice []string, item string) bool {

v2/pkg/reporting/trackers/github/github.go

@@ -9,7 +9,7 @@ import (
 	"github.com/google/go-github/github"
 	"github.com/pkg/errors"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
-	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues/format"
+	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/format"
 )
 
 // Integration is a client for a issue tracker integration

v2/pkg/reporting/trackers/gitlab/gitlab.go

@@ -2,7 +2,7 @@ package gitlab
 
 import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
-	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues/format"
+	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/format"
 	"github.com/xanzy/go-gitlab"
 )
 

v2/pkg/reporting/trackers/jira/jira.go

@@ -8,7 +8,7 @@ import (
 
 	jira "github.com/andygrunwald/go-jira"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
-	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues/format"
+	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/format"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 )
 

v2/pkg/types/types.go

@@ -43,6 +43,8 @@ type Options struct {
 	ReportingDB string
 	// ReportingConfig is the config file for nuclei reporting module
 	ReportingConfig string
+	// DiskExportDirectory is the directory to export reports in markdown on disk to
+	DiskExportDirectory string
 	// ResolversFile is a file containing resolvers for nuclei.
 	ResolversFile string
 	// StatsInterval is the number of seconds to display stats after