Merge pull request #1008 from projectdiscovery/clusterer-fixes

Clusterer fix + interactsh improvements
2021-09-10 14:27:22 +05:30 · 2021-09-10 14:27:22 +05:30 · c5f478962b
parent 802b6b4f65 c2c77de6f3
commit c5f478962b
7 changed files with 122 additions and 69 deletions
--- a/.github/workflows/publish-docs.yaml
+++ b/.github/workflows/publish-docs.yaml
@ -2,7 +2,9 @@ name: ⏰ Publish Docs

 on:
  push:
-  pull_request:
+    branches:
+      - master
+      - dev
  workflow_dispatch:

 jobs:
--- a/SYNTAX-REFERENCE.md
+++ b/SYNTAX-REFERENCE.md
@ -282,7 +282,7 @@ name: Nagios Default Credentials Check

 <div class="dd">

-<code>author</code>  <i>StringSlice</i>
+<code>author</code>  <i><a href="#stringslicestringslice">stringslice.StringSlice</a></i>

 </div>
 <div class="dt">
@ -307,7 +307,7 @@ author: <username>

 <div class="dd">

-<code>tags</code>  <i>StringSlice</i>
+<code>tags</code>  <i><a href="#stringslicestringslice">stringslice.StringSlice</a></i>

 </div>
 <div class="dt">
@ -362,7 +362,7 @@ description: Subversion ALM for the enterprise before 8.8.2 allows reflected XSS

 <div class="dd">

-<code>reference</code>  <i>StringSlice</i>
+<code>reference</code>  <i><a href="#stringslicestringslice">stringslice.StringSlice</a></i>

 </div>
 <div class="dt">
@ -389,7 +389,7 @@ reference:

 <div class="dd">

-<code>severity</code>  <i><a href="#severityseverityholder">severity.SeverityHolder</a></i>
+<code>severity</code>  <i><a href="#severityholder">severity.Holder</a></i>

 </div>
 <div class="dt">
@ -441,18 +441,38 @@ additional-fields:



-## model.StringSlice
+## stringslice.StringSlice
+StringSlice represents a single (in-lined) or multiple string value(s).
+ The unmarshaller does not automatically convert in-lined strings to []string, hence the interface{} type is required.

 Appears in:


+- <code><a href="#modelinfo">model.Info</a>.author</code>
+
+- <code><a href="#modelinfo">model.Info</a>.tags</code>
+
+- <code><a href="#modelinfo">model.Info</a>.reference</code>
+
 - <code><a href="#workflowsworkflowtemplate">workflows.WorkflowTemplate</a>.tags</code>


+```yaml
+<username>
+```
+```yaml
+# Example tags
+cve,cve2019,grafana,auth-bypass,dos
+```
+```yaml
+- https://github.com/strapi/strapi
+- https://github.com/getgrav/grav
+```



-## severity.SeverityHolder
+## severity.Holder
+Holder holds a Severity type. Required for un/marshalling purposes

 Appears in:

@ -2617,7 +2637,7 @@ template: misconfigurations/aem

 <div class="dd">

-<code>tags</code>  <i><a href="#modelstringslice">model.StringSlice</a></i>
+<code>tags</code>  <i><a href="#stringslicestringslice">stringslice.StringSlice</a></i>

 </div>
 <div class="dt">
--- a/nuclei-jsonschema.json
+++ b/nuclei-jsonschema.json
@ -2,18 +2,6 @@
  "$schema": "http://json-schema.org/draft-04/schema#",
  "$ref": "#/definitions/templates.Template",
  "definitions": {
-    "severity.SeverityHolder": {
-      "enum": [
-        "info",
-        "low",
-        "medium",
-        "high",
-        "critical"
-      ],
-      "type": "string",
-      "title": "severity of the template",
-      "description": "Seriousness of the implications of the template"
-    },
    "model.Info": {
      "properties": {
        "name": {
@ -26,12 +14,12 @@
        },
        "author": {
          "$schema": "http://json-schema.org/draft-04/schema#",
-          "$ref": "#/definitions/model.StringSlice",
+          "$ref": "#/definitions/stringslice.StringSlice",
          "title": "author of the template",
          "description": "Author is the author of the template"
        },
        "tags": {
-          "$ref": "#/definitions/model.StringSlice",
+          "$ref": "#/definitions/stringslice.StringSlice",
          "title": "tags of the template",
          "description": "Any tags for the template"
        },
@ -44,13 +32,13 @@
          ]
        },
        "reference": {
-          "$ref": "#/definitions/model.StringSlice",
+          "$ref": "#/definitions/stringslice.StringSlice",
          "title": "references for the template",
          "description": "Links relevant to the template"
        },
        "severity": {
          "$schema": "http://json-schema.org/draft-04/schema#",
-          "$ref": "#/definitions/severity.SeverityHolder"
+          "$ref": "#/definitions/severity.Holder"
        },
        "additional-fields": {
          "patternProperties": {
@ -66,7 +54,19 @@
      "additionalProperties": false,
      "type": "object"
    },
-    "model.StringSlice": {
+    "severity.Holder": {
+      "enum": [
+        "info",
+        "low",
+        "medium",
+        "high",
+        "critical"
+      ],
+      "type": "string",
+      "title": "severity of the template",
+      "description": "Seriousness of the implications of the template"
+    },
+    "stringslice.StringSlice": {
      "oneOf": [
        {
          "type": "string"
@ -885,7 +885,7 @@
          "description": "Template or directory to execute as part of workflow"
        },
        "tags": {
-          "$ref": "#/definitions/model.StringSlice",
+          "$ref": "#/definitions/stringslice.StringSlice",
          "title": "tags to execute",
          "description": "Tags to run template based on"
        },
--- a/v2/internal/runner/banner.go
+++ b/v2/internal/runner/banner.go
@ -20,6 +20,6 @@ func showBanner() {
 	gologger.Print().Msgf("%s\n", banner)
 	gologger.Print().Msgf("\t\tprojectdiscovery.io\n\n")

-	gologger.Error().Label("WRN").Msgf("Use with caution. You are responsible for your actions\n")
+	gologger.Error().Label("WRN").Msgf("Use with caution. You are responsible for your actions.\n")
 	gologger.Error().Label("WRN").Msgf("Developers assume no liability and are not responsible for any misuse or damage.\n")
 }
--- a/v2/internal/runner/runner.go
+++ b/v2/internal/runner/runner.go
@ -439,7 +439,7 @@ func (r *Runner) RunEnumeration() error {
 	}
 	templatesMap := make(map[string]*templates.Template)
 	for _, v := range store.Templates() {
-		templatesMap[v.ID] = v
+		templatesMap[v.Path] = v
 	}
 	originalTemplatesCount := len(store.Templates())
 	clusterCount := 0
@ -471,6 +471,7 @@ func (r *Runner) RunEnumeration() error {
 			finalTemplates = append(finalTemplates, cluster...)
 		}
 	}
+
 	finalTemplates = append(finalTemplates, store.Workflows()...)

 	var totalRequests int64
--- a/v2/pkg/protocols/common/interactsh/interactsh.go
+++ b/v2/pkg/protocols/common/interactsh/interactsh.go
@ -6,6 +6,7 @@ import (
 	"net/url"
 	"os"
 	"strings"
+	"sync"
 	"sync/atomic"
 	"time"

@ -35,6 +36,7 @@ type Client struct {
 	pollDuration     time.Duration
 	cooldownDuration time.Duration

+	firstTimeGroup sync.Once
 	generated      uint32 // decide to wait if we have a generated url
 	matched        bool
 }
@ -80,14 +82,6 @@ func New(options *Options) (*Client, error) {
 		return nil, errors.Wrap(err, "could not parse server url")
 	}

-	interactsh, err := client.New(&client.Options{
-		ServerURL:         options.ServerURL,
-		Token:             options.Authorization,
-		PersistentSession: false,
-	})
-	if err != nil {
-		return nil, errors.Wrap(err, "could not create client")
-	}
 	configure := ccache.Configure()
 	configure = configure.MaxSize(options.CacheSize)
 	cache := ccache.New(configure)
@ -97,7 +91,6 @@ func New(options *Options) (*Client, error) {
 	interactionsCache := ccache.New(interactionsCfg)

 	interactClient := &Client{
-		interactsh:       interactsh,
 		eviction:         options.Eviction,
 		interactions:     interactionsCache,
 		dotHostname:      "." + parsed.Host,
@ -106,21 +99,34 @@ func New(options *Options) (*Client, error) {
 		pollDuration:     options.PollDuration,
 		cooldownDuration: options.ColldownPeriod,
 	}
+	return interactClient, nil
+}

-	interactClient.interactsh.StartPolling(interactClient.pollDuration, func(interaction *server.Interaction) {
-		if options.Debug {
+func (c *Client) firstTimeInitializeClient() error {
+	interactsh, err := client.New(&client.Options{
+		ServerURL:         c.options.ServerURL,
+		Token:             c.options.Authorization,
+		PersistentSession: false,
+	})
+	if err != nil {
+		return errors.Wrap(err, "could not create client")
+	}
+	c.interactsh = interactsh
+
+	interactsh.StartPolling(c.pollDuration, func(interaction *server.Interaction) {
+		if c.options.Debug {
 			debugPrintInteraction(interaction)
 		}
-		item := interactClient.requests.Get(interaction.UniqueID)
+		item := c.requests.Get(interaction.UniqueID)
 		if item == nil {
 			// If we don't have any request for this ID, add it to temporary
 			// lru cache so we can correlate when we get an add request.
-			gotItem := interactClient.interactions.Get(interaction.UniqueID)
+			gotItem := c.interactions.Get(interaction.UniqueID)
 			if gotItem == nil {
-				interactClient.interactions.Set(interaction.UniqueID, []*server.Interaction{interaction}, defaultInteractionDuration)
+				c.interactions.Set(interaction.UniqueID, []*server.Interaction{interaction}, defaultInteractionDuration)
 			} else if items, ok := gotItem.Value().([]*server.Interaction); ok {
 				items = append(items, interaction)
-				interactClient.interactions.Set(interaction.UniqueID, items, defaultInteractionDuration)
+				c.interactions.Set(interaction.UniqueID, items, defaultInteractionDuration)
 			}
 			return
 		}
@ -128,9 +134,9 @@ func New(options *Options) (*Client, error) {
 		if !ok {
 			return
 		}
-		_ = interactClient.processInteractionForRequest(interaction, request)
+		_ = c.processInteractionForRequest(interaction, request)
 	})
-	return interactClient, nil
+	return nil
 }

 // processInteractionForRequest processes an interaction for a request
@ -170,6 +176,11 @@ func (c *Client) processInteractionForRequest(interaction *server.Interaction, d

 // URL returns a new URL that can be interacted with
 func (c *Client) URL() string {
+	c.firstTimeGroup.Do(func() {
+		if err := c.firstTimeInitializeClient(); err != nil {
+			gologger.Error().Msgf("Could not initialize interactsh client: %s", err)
+		}
+	})
 	atomic.CompareAndSwapUint32(&c.generated, 0, 1)
 	return c.interactsh.URL()
 }
@ -179,8 +190,10 @@ func (c *Client) Close() bool {
 	if c.cooldownDuration > 0 && atomic.LoadUint32(&c.generated) == 1 {
 		time.Sleep(c.cooldownDuration)
 	}
+	if c.interactsh != nil {
 		c.interactsh.StopPolling()
 		c.interactsh.Close()
+	}
 	return c.matched
 }

--- a/v2/pkg/templates/templates_doc.go
+++ b/v2/pkg/templates/templates_doc.go
@ -1,4 +1,3 @@
-// Package templates
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at http://mozilla.org/MPL/2.0/.
@ -12,8 +11,8 @@ import (
 var (
 	TemplateDoc                  encoder.Doc
 	MODELInfoDoc                 encoder.Doc
-	MODELStringSliceDoc          encoder.Doc
-	SEVERITYSeverityHolderDoc    encoder.Doc
+	STRINGSLICEStringSliceDoc    encoder.Doc
+	SEVERITYHolderDoc            encoder.Doc
 	HTTPRequestDoc               encoder.Doc
 	MATCHERSMatcherDoc           encoder.Doc
 	EXTRACTORSExtractorDoc       encoder.Doc
@ -107,14 +106,14 @@ func init() {

 	MODELInfoDoc.Fields[0].AddExample("", "Nagios Default Credentials Check")
 	MODELInfoDoc.Fields[1].Name = "author"
-	MODELInfoDoc.Fields[1].Type = "StringSlice"
+	MODELInfoDoc.Fields[1].Type = "stringslice.StringSlice"
 	MODELInfoDoc.Fields[1].Note = ""
 	MODELInfoDoc.Fields[1].Description = "Author of the template.\n\nMultiple values can also be specified separated by commas."
 	MODELInfoDoc.Fields[1].Comments[encoder.LineComment] = "Author of the template."

 	MODELInfoDoc.Fields[1].AddExample("", "<username>")
 	MODELInfoDoc.Fields[2].Name = "tags"
-	MODELInfoDoc.Fields[2].Type = "StringSlice"
+	MODELInfoDoc.Fields[2].Type = "stringslice.StringSlice"
 	MODELInfoDoc.Fields[2].Note = ""
 	MODELInfoDoc.Fields[2].Description = "Any tags for the template.\n\nMultiple values can also be specified separated by commas."
 	MODELInfoDoc.Fields[2].Comments[encoder.LineComment] = "Any tags for the template."
@ -130,14 +129,14 @@ func init() {

 	MODELInfoDoc.Fields[3].AddExample("", "Subversion ALM for the enterprise before 8.8.2 allows reflected XSS at multiple locations")
 	MODELInfoDoc.Fields[4].Name = "reference"
-	MODELInfoDoc.Fields[4].Type = "StringSlice"
+	MODELInfoDoc.Fields[4].Type = "stringslice.StringSlice"
 	MODELInfoDoc.Fields[4].Note = ""
 	MODELInfoDoc.Fields[4].Description = "References for the template.\n\nThis should contain links relevant to the template."
 	MODELInfoDoc.Fields[4].Comments[encoder.LineComment] = "References for the template."

 	MODELInfoDoc.Fields[4].AddExample("", []string{"https://github.com/strapi/strapi", "https://github.com/getgrav/grav"})
 	MODELInfoDoc.Fields[5].Name = "severity"
-	MODELInfoDoc.Fields[5].Type = "severity.SeverityHolder"
+	MODELInfoDoc.Fields[5].Type = "severity.Holder"
 	MODELInfoDoc.Fields[5].Note = ""
 	MODELInfoDoc.Fields[5].Description = "Severity of the template."
 	MODELInfoDoc.Fields[5].Comments[encoder.LineComment] = "Severity of the template."
@ -156,27 +155,45 @@ func init() {

 	MODELInfoDoc.Fields[6].AddExample("", map[string]string{"customField1": "customValue1"})

-	MODELStringSliceDoc.Type = "model.StringSlice"
-	MODELStringSliceDoc.Comments[encoder.LineComment] = ""
-	MODELStringSliceDoc.Description = ""
-	MODELStringSliceDoc.AppearsIn = []encoder.Appearance{
+	STRINGSLICEStringSliceDoc.Type = "stringslice.StringSlice"
+	STRINGSLICEStringSliceDoc.Comments[encoder.LineComment] = " StringSlice represents a single (in-lined) or multiple string value(s)."
+	STRINGSLICEStringSliceDoc.Description = "StringSlice represents a single (in-lined) or multiple string value(s).\n The unmarshaller does not automatically convert in-lined strings to []string, hence the interface{} type is required."
+
+	STRINGSLICEStringSliceDoc.AddExample("", "<username>")
+
+	STRINGSLICEStringSliceDoc.AddExample("Example tags", "cve,cve2019,grafana,auth-bypass,dos")
+
+	STRINGSLICEStringSliceDoc.AddExample("", []string{"https://github.com/strapi/strapi", "https://github.com/getgrav/grav"})
+	STRINGSLICEStringSliceDoc.AppearsIn = []encoder.Appearance{
+		{
+			TypeName:  "model.Info",
+			FieldName: "author",
+		},
+		{
+			TypeName:  "model.Info",
+			FieldName: "tags",
+		},
+		{
+			TypeName:  "model.Info",
+			FieldName: "reference",
+		},
 		{
 			TypeName:  "workflows.WorkflowTemplate",
 			FieldName: "tags",
 		},
 	}
-	MODELStringSliceDoc.Fields = make([]encoder.Doc, 0)
+	STRINGSLICEStringSliceDoc.Fields = make([]encoder.Doc, 0)

-	SEVERITYSeverityHolderDoc.Type = "severity.SeverityHolder"
-	SEVERITYSeverityHolderDoc.Comments[encoder.LineComment] = ""
-	SEVERITYSeverityHolderDoc.Description = ""
-	SEVERITYSeverityHolderDoc.AppearsIn = []encoder.Appearance{
+	SEVERITYHolderDoc.Type = "severity.Holder"
+	SEVERITYHolderDoc.Comments[encoder.LineComment] = " Holder holds a Severity type. Required for un/marshalling purposes"
+	SEVERITYHolderDoc.Description = "Holder holds a Severity type. Required for un/marshalling purposes"
+	SEVERITYHolderDoc.AppearsIn = []encoder.Appearance{
 		{
 			TypeName:  "model.Info",
 			FieldName: "severity",
 		},
 	}
-	SEVERITYSeverityHolderDoc.Fields = make([]encoder.Doc, 0)
+	SEVERITYHolderDoc.Fields = make([]encoder.Doc, 0)

 	HTTPRequestDoc.Type = "http.Request"
 	HTTPRequestDoc.Comments[encoder.LineComment] = " Request contains a http request to be made from a template"
@ -971,7 +988,7 @@ func init() {

 	WORKFLOWSWorkflowTemplateDoc.Fields[0].AddExample("A template directory", "misconfigurations/aem")
 	WORKFLOWSWorkflowTemplateDoc.Fields[1].Name = "tags"
-	WORKFLOWSWorkflowTemplateDoc.Fields[1].Type = "model.StringSlice"
+	WORKFLOWSWorkflowTemplateDoc.Fields[1].Type = "stringslice.StringSlice"
 	WORKFLOWSWorkflowTemplateDoc.Fields[1].Note = ""
 	WORKFLOWSWorkflowTemplateDoc.Fields[1].Description = "Tags to run templates based on."
 	WORKFLOWSWorkflowTemplateDoc.Fields[1].Comments[encoder.LineComment] = "Tags to run templates based on."
@ -1016,8 +1033,8 @@ func GetTemplateDoc() *encoder.FileDoc {
 		Structs: []*encoder.Doc{
 			&TemplateDoc,
 			&MODELInfoDoc,
-			&MODELStringSliceDoc,
-			&SEVERITYSeverityHolderDoc,
+			&STRINGSLICEStringSliceDoc,
+			&SEVERITYHolderDoc,
 			&HTTPRequestDoc,
 			&MATCHERSMatcherDoc,
 			&EXTRACTORSExtractorDoc,