From c256bef362781bd59c1965786f0d53575e18c570 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Fri, 3 Sep 2021 13:50:44 +0000 Subject: [PATCH 1/5] Auto Generate Syntax Docs + JSONSchema [Fri Sep 3 13:50:44 UTC 2021] :robot: --- SYNTAX-REFERENCE.md | 34 ++++++++++++++---- nuclei-jsonschema.json | 36 +++++++++---------- v2/pkg/templates/templates_doc.go | 57 ++++++++++++++++++++----------- 3 files changed, 82 insertions(+), 45 deletions(-) diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index a50d9765..91593b2c 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -282,7 +282,7 @@ name: Nagios Default Credentials Check
-author StringSlice +author stringslice.StringSlice
@@ -307,7 +307,7 @@ author:
-tags StringSlice +tags stringslice.StringSlice
@@ -362,7 +362,7 @@ description: Subversion ALM for the enterprise before 8.8.2 allows reflected XSS
-reference StringSlice +reference stringslice.StringSlice
@@ -389,7 +389,7 @@ reference:
-severity severity.SeverityHolder +severity severity.Holder
@@ -441,18 +441,38 @@ additional-fields: -## model.StringSlice +## stringslice.StringSlice +StringSlice represents a single (in-lined) or multiple string value(s). + The unmarshaller does not automatically convert in-lined strings to []string, hence the interface{} type is required. Appears in: +- model.Info.author + +- model.Info.tags + +- model.Info.reference + - workflows.WorkflowTemplate.tags +```yaml + +``` +```yaml +# Example tags +cve,cve2019,grafana,auth-bypass,dos +``` +```yaml +- https://github.com/strapi/strapi +- https://github.com/getgrav/grav +``` -## severity.SeverityHolder +## severity.Holder +Holder holds a Severity type. Required for un/marshalling purposes Appears in: @@ -2617,7 +2637,7 @@ template: misconfigurations/aem
-tags model.StringSlice +tags stringslice.StringSlice
diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json index 6092a125..00cb3bc2 100755 --- a/nuclei-jsonschema.json +++ b/nuclei-jsonschema.json @@ -2,18 +2,6 @@ "$schema": "http://json-schema.org/draft-04/schema#", "$ref": "#/definitions/templates.Template", "definitions": { - "severity.SeverityHolder": { - "enum": [ - "info", - "low", - "medium", - "high", - "critical" - ], - "type": "string", - "title": "severity of the template", - "description": "Seriousness of the implications of the template" - }, "model.Info": { "properties": { "name": { @@ -26,12 +14,12 @@ }, "author": { "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/model.StringSlice", + "$ref": "#/definitions/stringslice.StringSlice", "title": "author of the template", "description": "Author is the author of the template" }, "tags": { - "$ref": "#/definitions/model.StringSlice", + "$ref": "#/definitions/stringslice.StringSlice", "title": "tags of the template", "description": "Any tags for the template" }, @@ -44,13 +32,13 @@ ] }, "reference": { - "$ref": "#/definitions/model.StringSlice", + "$ref": "#/definitions/stringslice.StringSlice", "title": "references for the template", "description": "Links relevant to the template" }, "severity": { "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/severity.SeverityHolder" + "$ref": "#/definitions/severity.Holder" }, "additional-fields": { "patternProperties": { @@ -66,7 +54,19 @@ "additionalProperties": false, "type": "object" }, - "model.StringSlice": { + "severity.Holder": { + "enum": [ + "info", + "low", + "medium", + "high", + "critical" + ], + "type": "string", + "title": "severity of the template", + "description": "Seriousness of the implications of the template" + }, + "stringslice.StringSlice": { "oneOf": [ { "type": "string" @@ -885,7 +885,7 @@ "description": "Template or directory to execute as part of workflow" }, "tags": { - "$ref": "#/definitions/model.StringSlice", + "$ref": "#/definitions/stringslice.StringSlice", "title": "tags to execute", "description": "Tags to run template based on" }, diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index dcbc37f5..eb795ae4 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -1,4 +1,3 @@ -// Package templates // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at http://mozilla.org/MPL/2.0/. @@ -12,8 +11,8 @@ import ( var ( TemplateDoc encoder.Doc MODELInfoDoc encoder.Doc - MODELStringSliceDoc encoder.Doc - SEVERITYSeverityHolderDoc encoder.Doc + STRINGSLICEStringSliceDoc encoder.Doc + SEVERITYHolderDoc encoder.Doc HTTPRequestDoc encoder.Doc MATCHERSMatcherDoc encoder.Doc EXTRACTORSExtractorDoc encoder.Doc @@ -107,14 +106,14 @@ func init() { MODELInfoDoc.Fields[0].AddExample("", "Nagios Default Credentials Check") MODELInfoDoc.Fields[1].Name = "author" - MODELInfoDoc.Fields[1].Type = "StringSlice" + MODELInfoDoc.Fields[1].Type = "stringslice.StringSlice" MODELInfoDoc.Fields[1].Note = "" MODELInfoDoc.Fields[1].Description = "Author of the template.\n\nMultiple values can also be specified separated by commas." MODELInfoDoc.Fields[1].Comments[encoder.LineComment] = "Author of the template." MODELInfoDoc.Fields[1].AddExample("", "") MODELInfoDoc.Fields[2].Name = "tags" - MODELInfoDoc.Fields[2].Type = "StringSlice" + MODELInfoDoc.Fields[2].Type = "stringslice.StringSlice" MODELInfoDoc.Fields[2].Note = "" MODELInfoDoc.Fields[2].Description = "Any tags for the template.\n\nMultiple values can also be specified separated by commas." MODELInfoDoc.Fields[2].Comments[encoder.LineComment] = "Any tags for the template." @@ -130,14 +129,14 @@ func init() { MODELInfoDoc.Fields[3].AddExample("", "Subversion ALM for the enterprise before 8.8.2 allows reflected XSS at multiple locations") MODELInfoDoc.Fields[4].Name = "reference" - MODELInfoDoc.Fields[4].Type = "StringSlice" + MODELInfoDoc.Fields[4].Type = "stringslice.StringSlice" MODELInfoDoc.Fields[4].Note = "" MODELInfoDoc.Fields[4].Description = "References for the template.\n\nThis should contain links relevant to the template." MODELInfoDoc.Fields[4].Comments[encoder.LineComment] = "References for the template." MODELInfoDoc.Fields[4].AddExample("", []string{"https://github.com/strapi/strapi", "https://github.com/getgrav/grav"}) MODELInfoDoc.Fields[5].Name = "severity" - MODELInfoDoc.Fields[5].Type = "severity.SeverityHolder" + MODELInfoDoc.Fields[5].Type = "severity.Holder" MODELInfoDoc.Fields[5].Note = "" MODELInfoDoc.Fields[5].Description = "Severity of the template." MODELInfoDoc.Fields[5].Comments[encoder.LineComment] = "Severity of the template." @@ -156,27 +155,45 @@ func init() { MODELInfoDoc.Fields[6].AddExample("", map[string]string{"customField1": "customValue1"}) - MODELStringSliceDoc.Type = "model.StringSlice" - MODELStringSliceDoc.Comments[encoder.LineComment] = "" - MODELStringSliceDoc.Description = "" - MODELStringSliceDoc.AppearsIn = []encoder.Appearance{ + STRINGSLICEStringSliceDoc.Type = "stringslice.StringSlice" + STRINGSLICEStringSliceDoc.Comments[encoder.LineComment] = " StringSlice represents a single (in-lined) or multiple string value(s)." + STRINGSLICEStringSliceDoc.Description = "StringSlice represents a single (in-lined) or multiple string value(s).\n The unmarshaller does not automatically convert in-lined strings to []string, hence the interface{} type is required." + + STRINGSLICEStringSliceDoc.AddExample("", "") + + STRINGSLICEStringSliceDoc.AddExample("Example tags", "cve,cve2019,grafana,auth-bypass,dos") + + STRINGSLICEStringSliceDoc.AddExample("", []string{"https://github.com/strapi/strapi", "https://github.com/getgrav/grav"}) + STRINGSLICEStringSliceDoc.AppearsIn = []encoder.Appearance{ + { + TypeName: "model.Info", + FieldName: "author", + }, + { + TypeName: "model.Info", + FieldName: "tags", + }, + { + TypeName: "model.Info", + FieldName: "reference", + }, { TypeName: "workflows.WorkflowTemplate", FieldName: "tags", }, } - MODELStringSliceDoc.Fields = make([]encoder.Doc, 0) + STRINGSLICEStringSliceDoc.Fields = make([]encoder.Doc, 0) - SEVERITYSeverityHolderDoc.Type = "severity.SeverityHolder" - SEVERITYSeverityHolderDoc.Comments[encoder.LineComment] = "" - SEVERITYSeverityHolderDoc.Description = "" - SEVERITYSeverityHolderDoc.AppearsIn = []encoder.Appearance{ + SEVERITYHolderDoc.Type = "severity.Holder" + SEVERITYHolderDoc.Comments[encoder.LineComment] = " Holder holds a Severity type. Required for un/marshalling purposes" + SEVERITYHolderDoc.Description = "Holder holds a Severity type. Required for un/marshalling purposes" + SEVERITYHolderDoc.AppearsIn = []encoder.Appearance{ { TypeName: "model.Info", FieldName: "severity", }, } - SEVERITYSeverityHolderDoc.Fields = make([]encoder.Doc, 0) + SEVERITYHolderDoc.Fields = make([]encoder.Doc, 0) HTTPRequestDoc.Type = "http.Request" HTTPRequestDoc.Comments[encoder.LineComment] = " Request contains a http request to be made from a template" @@ -971,7 +988,7 @@ func init() { WORKFLOWSWorkflowTemplateDoc.Fields[0].AddExample("A template directory", "misconfigurations/aem") WORKFLOWSWorkflowTemplateDoc.Fields[1].Name = "tags" - WORKFLOWSWorkflowTemplateDoc.Fields[1].Type = "model.StringSlice" + WORKFLOWSWorkflowTemplateDoc.Fields[1].Type = "stringslice.StringSlice" WORKFLOWSWorkflowTemplateDoc.Fields[1].Note = "" WORKFLOWSWorkflowTemplateDoc.Fields[1].Description = "Tags to run templates based on." WORKFLOWSWorkflowTemplateDoc.Fields[1].Comments[encoder.LineComment] = "Tags to run templates based on." @@ -1016,8 +1033,8 @@ func GetTemplateDoc() *encoder.FileDoc { Structs: []*encoder.Doc{ &TemplateDoc, &MODELInfoDoc, - &MODELStringSliceDoc, - &SEVERITYSeverityHolderDoc, + &STRINGSLICEStringSliceDoc, + &SEVERITYHolderDoc, &HTTPRequestDoc, &MATCHERSMatcherDoc, &EXTRACTORSExtractorDoc, From 79421aa08808ece0a7144da036f28a07b408c25f Mon Sep 17 00:00:00 2001 From: Ice3man Date: Mon, 6 Sep 2021 15:04:09 +0530 Subject: [PATCH 2/5] Update publish-docs.yaml --- .github/workflows/publish-docs.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish-docs.yaml b/.github/workflows/publish-docs.yaml index 4f534d42..e114e379 100644 --- a/.github/workflows/publish-docs.yaml +++ b/.github/workflows/publish-docs.yaml @@ -2,7 +2,9 @@ name: ⏰ Publish Docs on: push: - pull_request: + branches: + - master + - dev workflow_dispatch: jobs: From c891d113db5efcb405f32c48c6b5a722c78e00ef Mon Sep 17 00:00:00 2001 From: Geeknik Labs <466878+geeknik@users.noreply.github.com> Date: Wed, 8 Sep 2021 09:10:22 -0500 Subject: [PATCH 3/5] Update banner.go Missing punctuation. --- v2/internal/runner/banner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/internal/runner/banner.go b/v2/internal/runner/banner.go index eed5277d..b75f09c3 100644 --- a/v2/internal/runner/banner.go +++ b/v2/internal/runner/banner.go @@ -20,6 +20,6 @@ func showBanner() { gologger.Print().Msgf("%s\n", banner) gologger.Print().Msgf("\t\tprojectdiscovery.io\n\n") - gologger.Error().Label("WRN").Msgf("Use with caution. You are responsible for your actions\n") + gologger.Error().Label("WRN").Msgf("Use with caution. You are responsible for your actions.\n") gologger.Error().Label("WRN").Msgf("Developers assume no liability and are not responsible for any misuse or damage.\n") } From e10c4d68ff60506264ada099da47ec62713303e2 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Fri, 10 Sep 2021 00:21:36 +0530 Subject: [PATCH 4/5] Fixed issue with duplicate ids in clusterer --- v2/internal/runner/runner.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index c5b7c31c..89d7b641 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -439,7 +439,7 @@ func (r *Runner) RunEnumeration() error { } templatesMap := make(map[string]*templates.Template) for _, v := range store.Templates() { - templatesMap[v.ID] = v + templatesMap[v.Path] = v } originalTemplatesCount := len(store.Templates()) clusterCount := 0 @@ -471,6 +471,7 @@ func (r *Runner) RunEnumeration() error { finalTemplates = append(finalTemplates, cluster...) } } + finalTemplates = append(finalTemplates, store.Workflows()...) var totalRequests int64 From c2c77de6f36cfa548d9ce905d788d8031359f988 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Fri, 10 Sep 2021 00:32:05 +0530 Subject: [PATCH 5/5] Use sync.Once for on-use initialization of interactsh --- .../protocols/common/interactsh/interactsh.go | 55 ++++++++++++------- 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/v2/pkg/protocols/common/interactsh/interactsh.go b/v2/pkg/protocols/common/interactsh/interactsh.go index da3ef497..3d1ab39b 100644 --- a/v2/pkg/protocols/common/interactsh/interactsh.go +++ b/v2/pkg/protocols/common/interactsh/interactsh.go @@ -6,6 +6,7 @@ import ( "net/url" "os" "strings" + "sync" "sync/atomic" "time" @@ -35,8 +36,9 @@ type Client struct { pollDuration time.Duration cooldownDuration time.Duration - generated uint32 // decide to wait if we have a generated url - matched bool + firstTimeGroup sync.Once + generated uint32 // decide to wait if we have a generated url + matched bool } var ( @@ -80,14 +82,6 @@ func New(options *Options) (*Client, error) { return nil, errors.Wrap(err, "could not parse server url") } - interactsh, err := client.New(&client.Options{ - ServerURL: options.ServerURL, - Token: options.Authorization, - PersistentSession: false, - }) - if err != nil { - return nil, errors.Wrap(err, "could not create client") - } configure := ccache.Configure() configure = configure.MaxSize(options.CacheSize) cache := ccache.New(configure) @@ -97,7 +91,6 @@ func New(options *Options) (*Client, error) { interactionsCache := ccache.New(interactionsCfg) interactClient := &Client{ - interactsh: interactsh, eviction: options.Eviction, interactions: interactionsCache, dotHostname: "." + parsed.Host, @@ -106,21 +99,34 @@ func New(options *Options) (*Client, error) { pollDuration: options.PollDuration, cooldownDuration: options.ColldownPeriod, } + return interactClient, nil +} - interactClient.interactsh.StartPolling(interactClient.pollDuration, func(interaction *server.Interaction) { - if options.Debug { +func (c *Client) firstTimeInitializeClient() error { + interactsh, err := client.New(&client.Options{ + ServerURL: c.options.ServerURL, + Token: c.options.Authorization, + PersistentSession: false, + }) + if err != nil { + return errors.Wrap(err, "could not create client") + } + c.interactsh = interactsh + + interactsh.StartPolling(c.pollDuration, func(interaction *server.Interaction) { + if c.options.Debug { debugPrintInteraction(interaction) } - item := interactClient.requests.Get(interaction.UniqueID) + item := c.requests.Get(interaction.UniqueID) if item == nil { // If we don't have any request for this ID, add it to temporary // lru cache so we can correlate when we get an add request. - gotItem := interactClient.interactions.Get(interaction.UniqueID) + gotItem := c.interactions.Get(interaction.UniqueID) if gotItem == nil { - interactClient.interactions.Set(interaction.UniqueID, []*server.Interaction{interaction}, defaultInteractionDuration) + c.interactions.Set(interaction.UniqueID, []*server.Interaction{interaction}, defaultInteractionDuration) } else if items, ok := gotItem.Value().([]*server.Interaction); ok { items = append(items, interaction) - interactClient.interactions.Set(interaction.UniqueID, items, defaultInteractionDuration) + c.interactions.Set(interaction.UniqueID, items, defaultInteractionDuration) } return } @@ -128,9 +134,9 @@ func New(options *Options) (*Client, error) { if !ok { return } - _ = interactClient.processInteractionForRequest(interaction, request) + _ = c.processInteractionForRequest(interaction, request) }) - return interactClient, nil + return nil } // processInteractionForRequest processes an interaction for a request @@ -170,6 +176,11 @@ func (c *Client) processInteractionForRequest(interaction *server.Interaction, d // URL returns a new URL that can be interacted with func (c *Client) URL() string { + c.firstTimeGroup.Do(func() { + if err := c.firstTimeInitializeClient(); err != nil { + gologger.Error().Msgf("Could not initialize interactsh client: %s", err) + } + }) atomic.CompareAndSwapUint32(&c.generated, 0, 1) return c.interactsh.URL() } @@ -179,8 +190,10 @@ func (c *Client) Close() bool { if c.cooldownDuration > 0 && atomic.LoadUint32(&c.generated) == 1 { time.Sleep(c.cooldownDuration) } - c.interactsh.StopPolling() - c.interactsh.Close() + if c.interactsh != nil { + c.interactsh.StopPolling() + c.interactsh.Close() + } return c.matched }