daffainfo
/
nuclei
mirror of https://github.com/daffainfo/nuclei.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1008 from projectdiscovery/clusterer-fixes 

Clusterer fix + interactsh improvements
dev
Ice3man 2021-09-10 14:27:22 +05:30 committed by GitHub
parent 802b6b4f65 c2c77de6f3
commit c5f478962b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 122 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/publish-docs.yaml vendored
View File

@ -2,7 +2,9 @@ name: ⏰ Publish Docs
on:
push:
pull_request:
branches:
- master
- dev
workflow_dispatch:
jobs:

34
SYNTAX-REFERENCE.md
View File

@ -282,7 +282,7 @@ name: Nagios Default Credentials Check
<div class="dd">
<code>author</code> <i>StringSlice</i>
<code>author</code> <i><a href="#stringslicestringslice">stringslice.StringSlice</a></i>
</div>
<div class="dt">
@ -307,7 +307,7 @@ author: <username>
<div class="dd">
<code>tags</code> <i>StringSlice</i>
<code>tags</code> <i><a href="#stringslicestringslice">stringslice.StringSlice</a></i>
</div>
<div class="dt">
@ -362,7 +362,7 @@ description: Subversion ALM for the enterprise before 8.8.2 allows reflected XSS
<div class="dd">
<code>reference</code> <i>StringSlice</i>
<code>reference</code> <i><a href="#stringslicestringslice">stringslice.StringSlice</a></i>
</div>
<div class="dt">
@ -389,7 +389,7 @@ reference:
<div class="dd">
<code>severity</code> <i><a href="#severityseverityholder">severity.SeverityHolder</a></i>
<code>severity</code> <i><a href="#severityholder">severity.Holder</a></i>
</div>
<div class="dt">
@ -441,18 +441,38 @@ additional-fields:
## model.StringSlice
## stringslice.StringSlice
StringSlice represents a single (in-lined) or multiple string value(s).
The unmarshaller does not automatically convert in-lined strings to []string, hence the interface{} type is required.
Appears in:
- <code><a href="#modelinfo">model.Info</a>.author</code>
- <code><a href="#modelinfo">model.Info</a>.tags</code>
- <code><a href="#modelinfo">model.Info</a>.reference</code>
- <code><a href="#workflowsworkflowtemplate">workflows.WorkflowTemplate</a>.tags</code>
```yaml
<username>
```
```yaml
# Example tags
cve,cve2019,grafana,auth-bypass,dos
```
```yaml
- https://github.com/strapi/strapi
- https://github.com/getgrav/grav
```
## severity.SeverityHolder
## severity.Holder
Holder holds a Severity type. Required for un/marshalling purposes
Appears in:
@ -2617,7 +2637,7 @@ template: misconfigurations/aem
<div class="dd">
<code>tags</code> <i><a href="#modelstringslice">model.StringSlice</a></i>
<code>tags</code> <i><a href="#stringslicestringslice">stringslice.StringSlice</a></i>
</div>
<div class="dt">

36
nuclei-jsonschema.json
View File

@ -2,18 +2,6 @@
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/templates.Template",
"definitions": {
"severity.SeverityHolder": {
"enum": [
"info",
"low",
"medium",
"high",
"critical"
],
"type": "string",
"title": "severity of the template",
"description": "Seriousness of the implications of the template"
},
"model.Info": {
"properties": {
"name": {
@ -26,12 +14,12 @@
},
"author": {
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/model.StringSlice",
"$ref": "#/definitions/stringslice.StringSlice",
"title": "author of the template",
"description": "Author is the author of the template"
},
"tags": {
"$ref": "#/definitions/model.StringSlice",
"$ref": "#/definitions/stringslice.StringSlice",
"title": "tags of the template",
"description": "Any tags for the template"
},
@ -44,13 +32,13 @@
]
},
"reference": {
"$ref": "#/definitions/model.StringSlice",
"$ref": "#/definitions/stringslice.StringSlice",
"title": "references for the template",
"description": "Links relevant to the template"
},
"severity": {
"$schema": "http://json-schema.org/draft-04/schema#",
"$ref": "#/definitions/severity.SeverityHolder"
"$ref": "#/definitions/severity.Holder"
},
"additional-fields": {
"patternProperties": {
@ -66,7 +54,19 @@
"additionalProperties": false,
"type": "object"
},
"model.StringSlice": {
"severity.Holder": {
"enum": [
"info",
"low",
"medium",
"high",
"critical"
],
"type": "string",
"title": "severity of the template",
"description": "Seriousness of the implications of the template"
},
"stringslice.StringSlice": {
"oneOf": [
{
"type": "string"
@ -885,7 +885,7 @@
"description": "Template or directory to execute as part of workflow"
},
"tags": {
"$ref": "#/definitions/model.StringSlice",
"$ref": "#/definitions/stringslice.StringSlice",
"title": "tags to execute",
"description": "Tags to run template based on"
},

2
v2/internal/runner/banner.go
View File

@ -20,6 +20,6 @@ func showBanner() {
gologger.Print().Msgf("%s\n", banner)
gologger.Print().Msgf("\t\tprojectdiscovery.io\n\n")
gologger.Error().Label("WRN").Msgf("Use with caution. You are responsible for your actions\n")
gologger.Error().Label("WRN").Msgf("Use with caution. You are responsible for your actions.\n")
gologger.Error().Label("WRN").Msgf("Developers assume no liability and are not responsible for any misuse or damage.\n")
}

3
v2/internal/runner/runner.go
View File

@ -439,7 +439,7 @@ func (r *Runner) RunEnumeration() error {
}
templatesMap := make(map[string]*templates.Template)
for _, v := range store.Templates() {
templatesMap[v.ID] = v
templatesMap[v.Path] = v
}
originalTemplatesCount := len(store.Templates())
clusterCount := 0
@ -471,6 +471,7 @@ func (r *Runner) RunEnumeration() error {
finalTemplates = append(finalTemplates, cluster...)
}
}
finalTemplates = append(finalTemplates, store.Workflows()...)
var totalRequests int64

55
v2/pkg/protocols/common/interactsh/interactsh.go
View File

@ -6,6 +6,7 @@ import (
"net/url"
"os"
"strings"
"sync"
"sync/atomic"
"time"
@ -35,8 +36,9 @@ type Client struct {
pollDuration time.Duration
cooldownDuration time.Duration
generated uint32 // decide to wait if we have a generated url
matched bool
firstTimeGroup sync.Once
generated uint32 // decide to wait if we have a generated url
matched bool
}
var (
@ -80,14 +82,6 @@ func New(options *Options) (*Client, error) {
return nil, errors.Wrap(err, "could not parse server url")
}
interactsh, err := client.New(&client.Options{
ServerURL: options.ServerURL,
Token: options.Authorization,
PersistentSession: false,
})
if err != nil {
return nil, errors.Wrap(err, "could not create client")
}
configure := ccache.Configure()
configure = configure.MaxSize(options.CacheSize)
cache := ccache.New(configure)
@ -97,7 +91,6 @@ func New(options *Options) (*Client, error) {
interactionsCache := ccache.New(interactionsCfg)
interactClient := &Client{
interactsh: interactsh,
eviction: options.Eviction,
interactions: interactionsCache,
dotHostname: "." + parsed.Host,
@ -106,21 +99,34 @@ func New(options *Options) (*Client, error) {
pollDuration: options.PollDuration,
cooldownDuration: options.ColldownPeriod,
}
return interactClient, nil
}
interactClient.interactsh.StartPolling(interactClient.pollDuration, func(interaction *server.Interaction) {
if options.Debug {
func (c *Client) firstTimeInitializeClient() error {
interactsh, err := client.New(&client.Options{
ServerURL: c.options.ServerURL,
Token: c.options.Authorization,
PersistentSession: false,
})
if err != nil {
return errors.Wrap(err, "could not create client")
}
c.interactsh = interactsh
interactsh.StartPolling(c.pollDuration, func(interaction *server.Interaction) {
if c.options.Debug {
debugPrintInteraction(interaction)
}
item := interactClient.requests.Get(interaction.UniqueID)
item := c.requests.Get(interaction.UniqueID)
if item == nil {
// If we don't have any request for this ID, add it to temporary
// lru cache so we can correlate when we get an add request.
gotItem := interactClient.interactions.Get(interaction.UniqueID)
gotItem := c.interactions.Get(interaction.UniqueID)
if gotItem == nil {
interactClient.interactions.Set(interaction.UniqueID, []*server.Interaction{interaction}, defaultInteractionDuration)
c.interactions.Set(interaction.UniqueID, []*server.Interaction{interaction}, defaultInteractionDuration)
} else if items, ok := gotItem.Value().([]*server.Interaction); ok {
items = append(items, interaction)
interactClient.interactions.Set(interaction.UniqueID, items, defaultInteractionDuration)
c.interactions.Set(interaction.UniqueID, items, defaultInteractionDuration)
}
return
}
@ -128,9 +134,9 @@ func New(options *Options) (*Client, error) {
if !ok {
return
}
_ = interactClient.processInteractionForRequest(interaction, request)
_ = c.processInteractionForRequest(interaction, request)
})
return interactClient, nil
return nil
}
// processInteractionForRequest processes an interaction for a request
@ -170,6 +176,11 @@ func (c *Client) processInteractionForRequest(interaction *server.Interaction, d
// URL returns a new URL that can be interacted with
func (c *Client) URL() string {
c.firstTimeGroup.Do(func() {
if err := c.firstTimeInitializeClient(); err != nil {
gologger.Error().Msgf("Could not initialize interactsh client: %s", err)
}
})
atomic.CompareAndSwapUint32(&c.generated, 0, 1)
return c.interactsh.URL()
}
@ -179,8 +190,10 @@ func (c *Client) Close() bool {
if c.cooldownDuration > 0 && atomic.LoadUint32(&c.generated) == 1 {
time.Sleep(c.cooldownDuration)
}
c.interactsh.StopPolling()
c.interactsh.Close()
if c.interactsh != nil {
c.interactsh.StopPolling()
c.interactsh.Close()
}
return c.matched
}

57
v2/pkg/templates/templates_doc.go
View File

@ -1,4 +1,3 @@
// Package templates
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
@ -12,8 +11,8 @@ import (
var (
TemplateDoc encoder.Doc
MODELInfoDoc encoder.Doc
MODELStringSliceDoc encoder.Doc
SEVERITYSeverityHolderDoc encoder.Doc
STRINGSLICEStringSliceDoc encoder.Doc
SEVERITYHolderDoc encoder.Doc
HTTPRequestDoc encoder.Doc
MATCHERSMatcherDoc encoder.Doc
EXTRACTORSExtractorDoc encoder.Doc
@ -107,14 +106,14 @@ func init() {
MODELInfoDoc.Fields[0].AddExample("", "Nagios Default Credentials Check")
MODELInfoDoc.Fields[1].Name = "author"
MODELInfoDoc.Fields[1].Type = "StringSlice"
MODELInfoDoc.Fields[1].Type = "stringslice.StringSlice"
MODELInfoDoc.Fields[1].Note = ""
MODELInfoDoc.Fields[1].Description = "Author of the template.\n\nMultiple values can also be specified separated by commas."
MODELInfoDoc.Fields[1].Comments[encoder.LineComment] = "Author of the template."
MODELInfoDoc.Fields[1].AddExample("", "<username>")
MODELInfoDoc.Fields[2].Name = "tags"
MODELInfoDoc.Fields[2].Type = "StringSlice"
MODELInfoDoc.Fields[2].Type = "stringslice.StringSlice"
MODELInfoDoc.Fields[2].Note = ""
MODELInfoDoc.Fields[2].Description = "Any tags for the template.\n\nMultiple values can also be specified separated by commas."
MODELInfoDoc.Fields[2].Comments[encoder.LineComment] = "Any tags for the template."
@ -130,14 +129,14 @@ func init() {
MODELInfoDoc.Fields[3].AddExample("", "Subversion ALM for the enterprise before 8.8.2 allows reflected XSS at multiple locations")
MODELInfoDoc.Fields[4].Name = "reference"
MODELInfoDoc.Fields[4].Type = "StringSlice"
MODELInfoDoc.Fields[4].Type = "stringslice.StringSlice"
MODELInfoDoc.Fields[4].Note = ""
MODELInfoDoc.Fields[4].Description = "References for the template.\n\nThis should contain links relevant to the template."
MODELInfoDoc.Fields[4].Comments[encoder.LineComment] = "References for the template."
MODELInfoDoc.Fields[4].AddExample("", []string{"https://github.com/strapi/strapi", "https://github.com/getgrav/grav"})
MODELInfoDoc.Fields[5].Name = "severity"
MODELInfoDoc.Fields[5].Type = "severity.SeverityHolder"
MODELInfoDoc.Fields[5].Type = "severity.Holder"
MODELInfoDoc.Fields[5].Note = ""
MODELInfoDoc.Fields[5].Description = "Severity of the template."
MODELInfoDoc.Fields[5].Comments[encoder.LineComment] = "Severity of the template."
@ -156,27 +155,45 @@ func init() {
MODELInfoDoc.Fields[6].AddExample("", map[string]string{"customField1": "customValue1"})
MODELStringSliceDoc.Type = "model.StringSlice"
MODELStringSliceDoc.Comments[encoder.LineComment] = ""
MODELStringSliceDoc.Description = ""
MODELStringSliceDoc.AppearsIn = []encoder.Appearance{
STRINGSLICEStringSliceDoc.Type = "stringslice.StringSlice"
STRINGSLICEStringSliceDoc.Comments[encoder.LineComment] = " StringSlice represents a single (in-lined) or multiple string value(s)."
STRINGSLICEStringSliceDoc.Description = "StringSlice represents a single (in-lined) or multiple string value(s).\n The unmarshaller does not automatically convert in-lined strings to []string, hence the interface{} type is required."
STRINGSLICEStringSliceDoc.AddExample("", "<username>")
STRINGSLICEStringSliceDoc.AddExample("Example tags", "cve,cve2019,grafana,auth-bypass,dos")
STRINGSLICEStringSliceDoc.AddExample("", []string{"https://github.com/strapi/strapi", "https://github.com/getgrav/grav"})
STRINGSLICEStringSliceDoc.AppearsIn = []encoder.Appearance{
{
TypeName: "model.Info",
FieldName: "author",
},
{
TypeName: "model.Info",
FieldName: "tags",
},
{
TypeName: "model.Info",
FieldName: "reference",
},
{
TypeName: "workflows.WorkflowTemplate",
FieldName: "tags",
},
}
MODELStringSliceDoc.Fields = make([]encoder.Doc, 0)
STRINGSLICEStringSliceDoc.Fields = make([]encoder.Doc, 0)
SEVERITYSeverityHolderDoc.Type = "severity.SeverityHolder"
SEVERITYSeverityHolderDoc.Comments[encoder.LineComment] = ""
SEVERITYSeverityHolderDoc.Description = ""
SEVERITYSeverityHolderDoc.AppearsIn = []encoder.Appearance{
SEVERITYHolderDoc.Type = "severity.Holder"
SEVERITYHolderDoc.Comments[encoder.LineComment] = " Holder holds a Severity type. Required for un/marshalling purposes"
SEVERITYHolderDoc.Description = "Holder holds a Severity type. Required for un/marshalling purposes"
SEVERITYHolderDoc.AppearsIn = []encoder.Appearance{
{
TypeName: "model.Info",
FieldName: "severity",
},
}
SEVERITYSeverityHolderDoc.Fields = make([]encoder.Doc, 0)
SEVERITYHolderDoc.Fields = make([]encoder.Doc, 0)
HTTPRequestDoc.Type = "http.Request"
HTTPRequestDoc.Comments[encoder.LineComment] = " Request contains a http request to be made from a template"
@ -971,7 +988,7 @@ func init() {
WORKFLOWSWorkflowTemplateDoc.Fields[0].AddExample("A template directory", "misconfigurations/aem")
WORKFLOWSWorkflowTemplateDoc.Fields[1].Name = "tags"
WORKFLOWSWorkflowTemplateDoc.Fields[1].Type = "model.StringSlice"
WORKFLOWSWorkflowTemplateDoc.Fields[1].Type = "stringslice.StringSlice"
WORKFLOWSWorkflowTemplateDoc.Fields[1].Note = ""
WORKFLOWSWorkflowTemplateDoc.Fields[1].Description = "Tags to run templates based on."
WORKFLOWSWorkflowTemplateDoc.Fields[1].Comments[encoder.LineComment] = "Tags to run templates based on."
@ -1016,8 +1033,8 @@ func GetTemplateDoc() *encoder.FileDoc {
Structs: []*encoder.Doc{
&TemplateDoc,
&MODELInfoDoc,
&MODELStringSliceDoc,
&SEVERITYSeverityHolderDoc,
&STRINGSLICEStringSliceDoc,
&SEVERITYHolderDoc,
&HTTPRequestDoc,
&MATCHERSMatcherDoc,
&EXTRACTORSExtractorDoc,