Implement more granular, issue tracker level filtering (#4780)

* (feat) include gitea in default config

* (feat) implement tracker level filtering in #4779

cmd/nuclei/issue-tracker-config.yaml

@@ -1,3 +1,8 @@
+# global allow/deny list. this will affect both exporters
+# as well as issue trackers. you can filter trackers with
+# a tracker level filter on top of an exporter by setting
+# allow-list/deny-list per tracker.
+#
 #allow-list:
 #  severity: high, critical
 #deny-list:
@@ -17,6 +22,15 @@
 #  project-name: test-project
 #  # issue-label is the label of the created issue type
 #  issue-label: bug
+#  # allow-list sets a tracker level filter to only create issues for templates with
+#  # these severity labels or tags (does not affect exporters. set those globally)
+#  allow-list:
+#    severity: high, critical
+#    tags: network
+#  # deny-list sets a tracker level filter to never create issues for templates with
+#  # these severity labels or tags (does not affect exporters. set those globally)
+#  deny-list:
+#    severity: low
 #  # duplicate-issue-check flag to enable duplicate tracking issue check.
 #  duplicate-issue-check: false
 #
@@ -32,6 +46,15 @@
 #  project-name: "1234"
 #  # issue-label is the label of the created issue type
 #  issue-label: bug
+#  # allow-list sets a tracker level filter to only create issues for templates with
+#  # these severity labels or tags (does not affect exporters. set those globally)
+#  allow-list:
+#    severity: high, critical
+#    tags: network
+#  # deny-list sets a tracker level filter to never create issues for templates with
+#  # these severity labels or tags (does not affect exporters. set those globally)
+#  deny-list:
+#    severity: low
 #
 # Gitea contains configuration options for a gitea issue tracker
 #gitea:
@@ -47,6 +70,15 @@
 #  issue-label: bug
 #  # severity-as-label (optional) adds the severity as a label of the created issue
 #  severity-as-label: true
+#  # allow-list sets a tracker level filter to only create issues for templates with
+#  # these severity labels or tags (does not affect exporters. set those globally)
+#  allow-list:
+#    severity: high, critical
+#    tags: network
+#  # deny-list sets a tracker level filter to never create issues for templates with
+#  # these severity labels or tags (does not affect exporters. set those globally)
+#  deny-list:
+#    severity: low
 #  # duplicate-issue-check (optional) flag to enable duplicate tracking issue check
 #  duplicate-issue-check: false
 #
@@ -71,6 +103,15 @@
 #  # SeverityAsLabel (optional) sends the severity as the label of the created issue
 #  # User custom fields for Jira Cloud instead
 #  severity-as-label: true
+#  # allow-list sets a tracker level filter to only create issues for templates with
+#  # these severity labels or tags (does not affect exporters. set those globally)
+#  allow-list:
+#    severity: high, critical
+#    tags: network
+#  # deny-list sets a tracker level filter to never create issues for templates with
+#  # these severity labels or tags (does not affect exporters. set those globally)
+#  deny-list:
+#    severity: low
 #  # Whatever your final status is that you want to use as a closed ticket - Closed, Done, Remediated, etc
 #  # When checking for duplicates, the JQL query will filter out status's that match this.
 #  # If it finds a match _and_ the ticket does have this status, a new one will be created.

pkg/reporting/options.go

@@ -7,6 +7,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/sarif"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/splunk"
+	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/filters"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/gitea"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/github"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/gitlab"
@@ -17,9 +18,9 @@ import (
 // Options is a configuration file for nuclei reporting module
 type Options struct {
 	// AllowList contains a list of allowed events for reporting module
-	AllowList *Filter `yaml:"allow-list"`
+	AllowList *filters.Filter `yaml:"allow-list"`
 	// DenyList contains a list of denied events for reporting module
-	DenyList *Filter `yaml:"deny-list"`
+	DenyList *filters.Filter `yaml:"deny-list"`
 	// GitHub contains configuration options for GitHub Issue Tracker
 	GitHub *github.Options `yaml:"github"`
 	// GitLab contains configuration options for GitLab Issue Tracker

pkg/reporting/reporting.go

@@ -12,7 +12,6 @@ import (
 
 	"errors"
 
-	"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
 	"github.com/projectdiscovery/nuclei/v3/pkg/model/types/stringslice"
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/dedupe"
@@ -20,62 +19,26 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/sarif"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/splunk"
+	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/filters"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/gitea"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/github"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/gitlab"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/jira"
 	errorutil "github.com/projectdiscovery/utils/errors"
 	fileutil "github.com/projectdiscovery/utils/file"
-	sliceutil "github.com/projectdiscovery/utils/slice"
 )
 
-// Filter filters the received event and decides whether to perform
-// reporting for it or not.
-type Filter struct {
-	Severities severity.Severities     `yaml:"severity"`
-	Tags       stringslice.StringSlice `yaml:"tags"`
-}
-
 var (
 	ErrReportingClientCreation = errors.New("could not create reporting client")
 	ErrExportClientCreation    = errors.New("could not create exporting client")
 )
 
-// GetMatch returns true if a filter matches result event
-func (filter *Filter) GetMatch(event *output.ResultEvent) bool {
-	return isSeverityMatch(event, filter) && isTagMatch(event, filter) // TODO revisit this
-}
-
-func isTagMatch(event *output.ResultEvent, filter *Filter) bool {
-	filterTags := filter.Tags
-	if filterTags.IsEmpty() {
-		return true
-	}
-
-	tags := event.Info.Tags.ToSlice()
-	for _, filterTag := range filterTags.ToSlice() {
-		if sliceutil.Contains(tags, filterTag) {
-			return true
-		}
-	}
-
-	return false
-}
-
-func isSeverityMatch(event *output.ResultEvent, filter *Filter) bool {
-	resultEventSeverity := event.Info.SeverityHolder.Severity // TODO review
-
-	if len(filter.Severities) == 0 {
-		return true
-	}
-
-	return sliceutil.Contains(filter.Severities, resultEventSeverity)
-}
-
 // Tracker is an interface implemented by an issue tracker
 type Tracker interface {
 	// CreateIssue creates an issue in the tracker
 	CreateIssue(event *output.ResultEvent) error
+	// ShouldFilter determines if the event should be filtered out
+	ShouldFilter(event *output.ResultEvent) bool
 }
 
 // Exporter is an interface implemented by an issue exporter
@@ -197,10 +160,11 @@ func CreateConfigIfNotExists() error {
 	values := stringslice.StringSlice{Value: []string{}}
 
 	options := &Options{
-		AllowList:             &Filter{Tags: values},
-		DenyList:              &Filter{Tags: values},
+		AllowList:             &filters.Filter{Tags: values},
+		DenyList:              &filters.Filter{Tags: values},
 		GitHub:                &github.Options{},
 		GitLab:                &gitlab.Options{},
+		Gitea:                 &gitea.Options{},
 		Jira:                  &jira.Options{},
 		MarkdownExporter:      &markdown.Options{},
 		SarifExporter:         &sarif.Options{},
@@ -239,6 +203,7 @@ func (c *ReportingClient) Close() {
 
 // CreateIssue creates an issue in the tracker
 func (c *ReportingClient) CreateIssue(event *output.ResultEvent) error {
+	// process global allow/deny list
 	if c.options.AllowList != nil && !c.options.AllowList.GetMatch(event) {
 		return nil
 	}
@@ -249,6 +214,10 @@ func (c *ReportingClient) CreateIssue(event *output.ResultEvent) error {
 	unique, err := c.dedupe.Index(event)
 	if unique {
 		for _, tracker := range c.trackers {
+			// process tracker specific allow/deny list
+			if tracker.ShouldFilter(event) {
+				continue
+			}
 			if trackerErr := tracker.CreateIssue(event); trackerErr != nil {
 				err = multierr.Append(err, trackerErr)
 			}

pkg/reporting/trackers/filters/filters.go

@@ -0,0 +1,47 @@
+package filters
+
+import (
+	"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
+	"github.com/projectdiscovery/nuclei/v3/pkg/model/types/stringslice"
+	"github.com/projectdiscovery/nuclei/v3/pkg/output"
+
+	sliceutil "github.com/projectdiscovery/utils/slice"
+)
+
+// Filter filters the received event and decides whether to perform
+// reporting for it or not.
+type Filter struct {
+	Severities severity.Severities     `yaml:"severity"`
+	Tags       stringslice.StringSlice `yaml:"tags"`
+}
+
+// GetMatch returns true if a filter matches result event
+func (filter *Filter) GetMatch(event *output.ResultEvent) bool {
+	return isSeverityMatch(event, filter) && isTagMatch(event, filter) // TODO revisit this
+}
+
+func isTagMatch(event *output.ResultEvent, filter *Filter) bool {
+	filterTags := filter.Tags
+	if filterTags.IsEmpty() {
+		return true
+	}
+
+	tags := event.Info.Tags.ToSlice()
+	for _, filterTag := range filterTags.ToSlice() {
+		if sliceutil.Contains(tags, filterTag) {
+			return true
+		}
+	}
+
+	return false
+}
+
+func isSeverityMatch(event *output.ResultEvent, filter *Filter) bool {
+	resultEventSeverity := event.Info.SeverityHolder.Severity // TODO review
+
+	if len(filter.Severities) == 0 {
+		return true
+	}
+
+	return sliceutil.Contains(filter.Severities, resultEventSeverity)
+}

pkg/reporting/trackers/gitea/gitea.go

@@ -10,6 +10,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown/util"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/format"
+	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/filters"
 	"github.com/projectdiscovery/retryablehttp-go"
 )
 
@@ -34,6 +35,10 @@ type Options struct {
 	// SeverityAsLabel (optional) adds the severity as the label of the created
 	// issue.
 	SeverityAsLabel bool `yaml:"severity-as-label"`
+	// AllowList contains a list of allowed events for this tracker
+	AllowList *filters.Filter `yaml:"allow-list"`
+	// DenyList contains a list of denied events for this tracker
+	DenyList *filters.Filter `yaml:"deny-list"`
 	// DuplicateIssueCheck is a bool to enable duplicate tracking issue check and update the newest
 	DuplicateIssueCheck bool `yaml:"duplicate-issue-check" default:"false"`
 
@@ -116,6 +121,19 @@ func (i *Integration) CreateIssue(event *output.ResultEvent) error {
 	return err
 }
 
+// ShouldFilter determines if an issue should be logged to this tracker
+func (i *Integration) ShouldFilter(event *output.ResultEvent) bool {
+	if i.options.AllowList != nil && i.options.AllowList.GetMatch(event) {
+		return true
+	}
+
+	if i.options.DenyList != nil && i.options.DenyList.GetMatch(event) {
+		return true
+	}
+
+	return false
+}
+
 func (i *Integration) findIssueByTitle(title string) (*gitea.Issue, error) {
 
 	issueList, _, err := i.client.ListRepoIssues(i.options.ProjectOwner, i.options.ProjectName, gitea.ListIssueOption{

pkg/reporting/trackers/github/github.go

@@ -13,6 +13,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown/util"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/format"
+	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/filters"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 	"github.com/projectdiscovery/retryablehttp-go"
 	"golang.org/x/oauth2"
@@ -41,6 +42,10 @@ type Options struct {
 	// SeverityAsLabel (optional) sends the severity as the label of the created
 	// issue.
 	SeverityAsLabel bool `yaml:"severity-as-label"`
+	// AllowList contains a list of allowed events for this tracker
+	AllowList *filters.Filter `yaml:"allow-list"`
+	// DenyList contains a list of denied events for this tracker
+	DenyList *filters.Filter `yaml:"deny-list"`
 	// DuplicateIssueCheck (optional) comments under existing finding issue
 	// instead of creating duplicates for subsequent runs.
 	DuplicateIssueCheck bool `yaml:"duplicate-issue-check"`
@@ -129,6 +134,19 @@ func (i *Integration) CreateIssue(event *output.ResultEvent) (err error) {
 	}
 }
 
+// ShouldFilter determines if an issue should be logged to this tracker
+func (i *Integration) ShouldFilter(event *output.ResultEvent) bool {
+	if i.options.AllowList != nil && i.options.AllowList.GetMatch(event) {
+		return true
+	}
+
+	if i.options.DenyList != nil && i.options.DenyList.GetMatch(event) {
+		return true
+	}
+
+	return false
+}
+
 func (i *Integration) findIssueByTitle(ctx context.Context, title string) (*github.Issue, error) {
 	req := &github.SearchOptions{
 		Sort:      "updated",

pkg/reporting/trackers/gitlab/gitlab.go

@@ -8,6 +8,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown/util"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/format"
+	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/filters"
 	"github.com/projectdiscovery/retryablehttp-go"
 )
 
@@ -33,6 +34,10 @@ type Options struct {
 	// SeverityAsLabel (optional) sends the severity as the label of the created
 	// issue.
 	SeverityAsLabel bool `yaml:"severity-as-label"`
+	// AllowList contains a list of allowed events for this tracker
+	AllowList *filters.Filter `yaml:"allow-list"`
+	// DenyList contains a list of denied events for this tracker
+	DenyList *filters.Filter `yaml:"deny-list"`
 	// DuplicateIssueCheck is a bool to enable duplicate tracking issue check and update the newest
 	DuplicateIssueCheck bool `yaml:"duplicate-issue-check" default:"false"`
 
@@ -112,3 +117,16 @@ func (i *Integration) CreateIssue(event *output.ResultEvent) error {
 
 	return err
 }
+
+// ShouldFilter determines if an issue should be logged to this tracker
+func (i *Integration) ShouldFilter(event *output.ResultEvent) bool {
+	if i.options.AllowList != nil && i.options.AllowList.GetMatch(event) {
+		return true
+	}
+
+	if i.options.DenyList != nil && i.options.DenyList.GetMatch(event) {
+		return true
+	}
+
+	return false
+}

pkg/reporting/trackers/jira/jira.go

@@ -12,6 +12,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown/util"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/format"
+	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/filters"
 	"github.com/projectdiscovery/retryablehttp-go"
 )
 
@@ -69,6 +70,10 @@ type Options struct {
 	// SeverityAsLabel (optional) sends the severity as the label of the created
 	// issue.
 	SeverityAsLabel bool `yaml:"severity-as-label" json:"severity_as_label"`
+	// AllowList contains a list of allowed events for this tracker
+	AllowList *filters.Filter `yaml:"allow-list"`
+	// DenyList contains a list of denied events for this tracker
+	DenyList *filters.Filter `yaml:"deny-list"`
 	// Severity (optional) is the severity of the issue.
 	Severity   []string              `yaml:"severity" json:"severity"`
 	HttpClient *retryablehttp.Client `yaml:"-" json:"-"`
@@ -234,3 +239,16 @@ func (i *Integration) FindExistingIssue(event *output.ResultEvent) (string, erro
 		return chunk[0].ID, nil
 	}
 }
+
+// ShouldFilter determines if an issue should be logged to this tracker
+func (i *Integration) ShouldFilter(event *output.ResultEvent) bool {
+	if i.options.AllowList != nil && i.options.AllowList.GetMatch(event) {
+		return true
+	}
+
+	if i.options.DenyList != nil && i.options.DenyList.GetMatch(event) {
+		return true
+	}
+
+	return false
+}