diff --git a/README.md b/README.md index 750e2a14..bfa68972 100644 --- a/README.md +++ b/README.md @@ -90,30 +90,29 @@ TARGET: -l, -list string path to file containing a list of target URLs/hosts to scan (one per line) TEMPLATES: - -tl list all available templates -t, -templates string[] template or template directory paths to include in the scan - -w, -workflows string[] list of workflows to run - -nt, -new-templates run newly added templates only + -nt, -new-templates run only new templates added in latest nuclei-templates release + -w, -workflows string[] workflow or workflow directory paths to include in the scan -validate validate the passed templates to nuclei + -tl list all available templates FILTERING: - -tags string[] execute a subset of templates that contain the provided tags - -include-tags string[] tags from the default deny list that permit executing more intrusive templates - -etags, -exclude-tags string[] exclude templates with the provided tags - -include-templates string[] templates to be executed even if they are excluded either by default or configuration - -exclude-templates, -exclude string[] template or template directory paths to exclude - -severity, -impact value[] Templates to run based on severity. Possible values: info, low, medium, high, critical - -author string[] execute templates that are (co-)created by the specified authors + -tags string[] execute a subset of templates that contain the provided tags + -etags, -exclude-tags string[] exclude templates with the provided tags + -itags, -include-tags string[] tags from the default deny list that permit executing more intrusive templates + -et, -exclude-templates string[] template or template directory paths to exclude + -it, -include-templates string[] templates to be executed even if they are excluded either by default or configuration + -s, -severity value[] Templates to run based on severity. Possible values - info,low,medium,high,critical + -es, -exclude-severity value[] Templates to exclude based on severity. Possible values - info,low,medium,high,critical + -a, -author string[] execute templates that are (co-)created by the specified authors OUTPUT: -o, -output string output file to write found issues/vulnerabilities -silent display findings only - -v, -verbose show verbose output - -vv display extra verbose information -nc, -no-color disable output content coloring (ANSI escape codes) -json write output in JSONL(ines) format -irr, -include-rr include request/response pairs in the JSONL output (for findings only) - -nm, -no-meta don't display match metadata in CLI output + -nm, -no-meta don't display match metadata -nts, -no-timestamp don't display timestamp metadata in CLI output -rdb, -report-db string local nuclei reporting database (always use this to persist report data) -me, -markdown-export string directory to export results in markdown format @@ -125,37 +124,39 @@ CONFIGURATIONS: -H, -header string[] custom headers in header:value format -V, -var value custom vars in var=value format -r, -resolvers string file containing resolver list for nuclei - -system-resolvers use system DNS resolving as error fallback + -sr, -system-resolvers use system DNS resolving as error fallback -passive enable passive HTTP response processing mode - -env-vars enable environment variables support + -ev, -env-vars enable environment variables to be used in template INTERACTSH: - -no-interactsh disable interactsh server for OOB testing - -interactsh-url string interactsh server url for self-hosted instance (default "https://interactsh.com") - -interactsh-token string authentication token for self-hosted interactsh server - -interactions-cache-size int number of requests to keep in the interactions cache (default 5000) - -interactions-eviction int number of seconds to wait before evicting requests from cache (default 60) - -interactions-poll-duration int number of seconds to wait before each interaction poll request (default 5) - -interactions-cooldown-period int extra time for interaction polling before exiting (default 5) + -iserver, -interactsh-server string interactsh server url for self-hosted instance (default "https://interactsh.com") + -itoken, -interactsh-token string authentication token for self-hosted interactsh server + -interactions-cache-size int number of requests to keep in the interactions cache (default 5000) + -interactions-eviction int number of seconds to wait before evicting requests from cache (default 60) + -interactions-poll-duration int number of seconds to wait before each interaction poll request (default 5) + -interactions-cooldown-period int extra time for interaction polling before exiting (default 5) + -ni, -no-interactsh disable interactsh server for OAST testing, exclude OAST based templates RATE-LIMIT: -rl, -rate-limit int maximum number of requests to send per second (default 150) -rlm, -rate-limit-minute int maximum number of requests to send per minute -bs, -bulk-size int maximum number of hosts to be analyzed in parallel per template (default 25) - -c, -concurrency int maximum number of templates to be executed in parallel (default 10) + -c, -concurrency int maximum number of templates to be executed in parallel (default 25) OPTIMIZATIONS: -timeout int time to wait in seconds before timeout (default 5) -retries int number of times to retry a failed request (default 1) - -max-host-error int max errors for a host before skipping from scan (default 30) + -mhe, -max-host-error int max errors for a host before skipping from scan (default 30) -project use a project folder to avoid sending same request multiple times - -project-path string set a specific project path (default "$TMPDIR/") + -project-path string set a specific project path -spm, -stop-at-first-path stop processing HTTP requests after the first match (may break template/workflow logic) + -stream Stream mode - start elaborating without sorting the input HEADLESS: - -headless enable templates that require headless browser support - -page-timeout int seconds to wait for each page in headless mode (default 20) - -show-browser show the browser on the screen when running templates with headless mode + -headless enable templates that require headless browser support + -page-timeout int seconds to wait for each page in headless mode (default 20) + -sb, -show-browser show the browser on the screen when running templates with headless mode + -sc, -system-chrome Use local installed chrome browser instead of nuclei installed DEBUG: -debug show all requests and responses @@ -163,22 +164,24 @@ DEBUG: -debug-resp show all received responses -proxy, -proxy-url string URL of the HTTP proxy server -proxy-socks-url string URL of the SOCKS proxy server - -trace-log string file to write sent requests trace log + -tlog, -trace-log string file to write sent requests trace log -version show nuclei version + -v, -verbose show verbose output + -vv display extra verbose information -tv, -templates-version shows the version of the installed nuclei-templates UPDATE: - -update update nuclei to the latest released version - -ut, -update-templates update the community templates to latest released version - -nut, -no-update-templates do not check for nuclei-templates updates - -ud, -update-directory string overwrite the default nuclei-templates directory (default "$HOME/nuclei-templates") + -update update nuclei engine to the latest released version + -ut, -update-templates update nuclei-templates to latest released version + -ud, -update-directory string overwrite the default directory to install nuclei-templates + -duc, -disable-update-check disable automatic nuclei/templates update check STATISTICS: -stats display statistics about the running scan - -stats-json write statistics data to an output file in JSONL(ines) format + -sj, -stats-json write statistics data to an output file in JSONL(ines) format -si, -stats-interval int number of seconds to wait between showing a statistics update (default 5) - -metrics expose nuclei metrics on a port - -metrics-port int port to expose nuclei metrics on (default 9092) + -m, -metrics expose nuclei metrics on a port + -mp, -metrics-port int port to expose nuclei metrics on (default 9092) ``` ### Running Nuclei diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index 30ed998f..a3fffe35 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -2392,6 +2392,31 @@ read-size: 2048 ``` + + +
+ +
+ +read-all bool + +
+
+ +ReadAll determines if the data stream should be read till the end regardless of the size + +Default value for read-all is false. + + + +Examples: + + +```yaml +read-all: false +``` + +
diff --git a/integration_tests/http/interactsh.yaml b/integration_tests/http/interactsh.yaml new file mode 100644 index 00000000..28d9c560 --- /dev/null +++ b/integration_tests/http/interactsh.yaml @@ -0,0 +1,19 @@ +id: interactsh-integration-test + +info: + name: Interactsh Integration Test + author: pdteam + severity: info + +requests: + - method: GET + path: + - "{{BaseURL}}" + headers: + url: 'http://{{interactsh-url}}' + + matchers: + - type: word + part: interactsh_protocol # Confirms the HTTP Interaction + words: + - "http" \ No newline at end of file diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json index 56253b7e..1db207d7 100755 --- a/nuclei-jsonschema.json +++ b/nuclei-jsonschema.json @@ -809,6 +809,11 @@ "title": "size of network response to read", "description": "Size of response to read at the end. Default is 1024 bytes" }, + "read-all": { + "type": "boolean", + "title": "read all response stream", + "description": "Read all response stream till the server stops sending" + }, "matchers": { "items": { "$ref": "#/definitions/matchers.Matcher" @@ -845,6 +850,7 @@ ], "properties": { "id": { + "pattern": "^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$", "type": "string", "title": "id of the template", "description": "The Unique ID for the template", diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index c3e23a7a..c8bc6acc 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -31,9 +31,36 @@ var httpTestcases = map[string]testutils.TestCase{ "http/raw-unsafe-request.yaml": &httpRawUnsafeRequest{}, "http/request-condition.yaml": &httpRequestCondition{}, "http/request-condition-new.yaml": &httpRequestCondition{}, + "http/interactsh.yaml": &httpInteractshRequest{}, "http/self-contained.yaml": &httpRequestSelContained{}, } +type httpInteractshRequest struct{} + +// Executes executes a test case and returns an error if occurred +func (h *httpInteractshRequest) Execute(filePath string) error { + router := httprouter.New() + router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + value := r.Header.Get("url") + if value != "" { + if resp, _ := http.DefaultClient.Get(value); resp != nil { + resp.Body.Close() + } + } + })) + ts := httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) + if err != nil { + return err + } + if len(results) != 1 { + return errIncorrectResultsCount(results) + } + return nil +} + type httpGetHeaders struct{} // Execute executes a test case and returns an error if occurred diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index c5170f20..d2f8ebb9 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -141,7 +141,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.StringVarP(&options.TraceLogFile, "trace-log", "tlog", "", "file to write sent requests trace log"), flagSet.BoolVar(&options.Version, "version", false, "show nuclei version"), flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "show verbose output"), - flagSet.BoolVar(&options.VerboseVerbose, "vv", false, "display extra verbose information"), + flagSet.BoolVar(&options.VerboseVerbose, "vv", false, "display templates loaded for scan"), flagSet.BoolVarP(&options.TemplatesVersion, "templates-version", "tv", false, "shows the version of the installed nuclei-templates"), ) diff --git a/v2/internal/runner/banner.go b/v2/internal/runner/banner.go index b75f09c3..56bf4ea4 100644 --- a/v2/internal/runner/banner.go +++ b/v2/internal/runner/banner.go @@ -20,6 +20,6 @@ func showBanner() { gologger.Print().Msgf("%s\n", banner) gologger.Print().Msgf("\t\tprojectdiscovery.io\n\n") - gologger.Error().Label("WRN").Msgf("Use with caution. You are responsible for your actions.\n") - gologger.Error().Label("WRN").Msgf("Developers assume no liability and are not responsible for any misuse or damage.\n") + gologger.Print().Label("WRN").Msgf("Use with caution. You are responsible for your actions.\n") + gologger.Print().Label("WRN").Msgf("Developers assume no liability and are not responsible for any misuse or damage.\n") } diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 745f15d0..59791fe6 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -22,11 +22,6 @@ func ParseOptions(options *types.Options) { // Check if stdin pipe was given options.Stdin = hasStdin() - // if VerboseVerbose is set, it implicitly enables the Verbose option as well - if options.VerboseVerbose { - options.Verbose = true - } - // Read the inputs and configure the logging configureOutput(options) @@ -127,7 +122,7 @@ func isValidURL(urlString string) bool { // configureOutput configures the output logging levels to be displayed on the screen func configureOutput(options *types.Options) { // If the user desires verbose output, show verbose output - if options.Verbose || options.VerboseVerbose { + if options.Verbose { gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose) } if options.Debug { diff --git a/v2/pkg/catalog/config/config.go b/v2/pkg/catalog/config/config.go index 41ce5361..91b153cf 100644 --- a/v2/pkg/catalog/config/config.go +++ b/v2/pkg/catalog/config/config.go @@ -26,7 +26,7 @@ type Config struct { const nucleiConfigFilename = ".templates-config.json" // Version is the current version of nuclei -const Version = `2.5.3-dev` +const Version = `2.5.4-dev` func getConfigDetails() (string, error) { homeDir, err := os.UserHomeDir() diff --git a/v2/pkg/parsers/parser.go b/v2/pkg/parsers/parser.go index b65cb79e..7cd0c7b9 100644 --- a/v2/pkg/parsers/parser.go +++ b/v2/pkg/parsers/parser.go @@ -5,6 +5,7 @@ import ( "io/ioutil" "os" "regexp" + "strings" "gopkg.in/yaml.v2" @@ -17,7 +18,10 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/utils/stats" ) -const mandatoryFieldMissingTemplate = "mandatory '%s' field is missing" +const ( + mandatoryFieldMissingTemplate = "mandatory '%s' field is missing" + invalidFieldFormatTemplate = "invalid field format for '%s' (allowed format is %s)" +) // LoadTemplate returns true if the template is valid and matches the filtering criteria. func LoadTemplate(templatePath string, tagFilter *filter.TagFilter, extraTags []string) (bool, error) { @@ -30,12 +34,12 @@ func LoadTemplate(templatePath string, tagFilter *filter.TagFilter, extraTags [] return false, nil } - templateInfo := template.Info - if validationError := validateMandatoryInfoFields(&templateInfo); validationError != nil { + if validationError := validateTemplateFields(template); validationError != nil { + stats.Increment(SyntaxErrorStats) return false, validationError } - return isTemplateInfoMetadataMatch(tagFilter, &templateInfo, extraTags) + return isTemplateInfoMetadataMatch(tagFilter, &template.Info, extraTags) } // LoadWorkflow returns true if the workflow is valid and matches the filtering criteria. @@ -45,10 +49,8 @@ func LoadWorkflow(templatePath string) (bool, error) { return false, templateParseError } - templateInfo := template.Info - if len(template.Workflows) > 0 { - if validationError := validateMandatoryInfoFields(&templateInfo); validationError != nil { + if validationError := validateTemplateFields(template); validationError != nil { return false, validationError } return true, nil @@ -71,18 +73,29 @@ func isTemplateInfoMetadataMatch(tagFilter *filter.TagFilter, templateInfo *mode return match, err } -func validateMandatoryInfoFields(info *model.Info) error { - if info == nil { - return fmt.Errorf(mandatoryFieldMissingTemplate, "info") - } +func validateTemplateFields(template *templates.Template) error { + info := template.Info + + var errors []string if utils.IsBlank(info.Name) { - return fmt.Errorf(mandatoryFieldMissingTemplate, "name") + errors = append(errors, fmt.Sprintf(mandatoryFieldMissingTemplate, "name")) } if info.Authors.IsEmpty() { - return fmt.Errorf(mandatoryFieldMissingTemplate, "author") + errors = append(errors, fmt.Sprintf(mandatoryFieldMissingTemplate, "author")) } + + if template.ID == "" { + errors = append(errors, fmt.Sprintf(mandatoryFieldMissingTemplate, "id")) + } else if !templateIDRegexp.MatchString(template.ID) { + errors = append(errors, fmt.Sprintf(invalidFieldFormatTemplate, "id", templateIDRegexp.String())) + } + + if len(errors) > 0 { + return fmt.Errorf(strings.Join(errors, ", ")) + } + return nil } @@ -90,6 +103,7 @@ var ( parsedTemplatesCache *cache.Templates ShouldValidate bool fieldErrorRegexp = regexp.MustCompile(`not found in`) + templateIDRegexp = regexp.MustCompile(`^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$`) ) const ( diff --git a/v2/pkg/parsers/parser_test.go b/v2/pkg/parsers/parser_test.go new file mode 100644 index 00000000..ef74a317 --- /dev/null +++ b/v2/pkg/parsers/parser_test.go @@ -0,0 +1,110 @@ +package parsers + +import ( + "errors" + "fmt" + "testing" + + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter" + "github.com/projectdiscovery/nuclei/v2/pkg/model" + "github.com/projectdiscovery/nuclei/v2/pkg/model/types/stringslice" + "github.com/projectdiscovery/nuclei/v2/pkg/templates" + "github.com/stretchr/testify/require" +) + +func TestLoadTemplate(t *testing.T) { + origTemplatesCache := parsedTemplatesCache + defer func() { parsedTemplatesCache = origTemplatesCache }() + + tt := []struct { + name string + template *templates.Template + templateErr error + + expectedErr error + }{ + { + name: "valid", + template: &templates.Template{ + ID: "CVE-2021-27330", + Info: model.Info{ + Name: "Valid template", + Authors: stringslice.StringSlice{Value: "Author"}, + }, + }, + }, + { + name: "emptyTemplate", + template: &templates.Template{}, + expectedErr: errors.New("mandatory 'name' field is missing, mandatory 'author' field is missing, mandatory 'id' field is missing"), + }, + { + name: "emptyNameWithInvalidID", + template: &templates.Template{ + ID: "invalid id", + Info: model.Info{ + Authors: stringslice.StringSlice{Value: "Author"}, + }, + }, + expectedErr: errors.New("mandatory 'name' field is missing, invalid field format for 'id' (allowed format is ^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$)"), + }, + } + + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + parsedTemplatesCache.Store(tc.name, tc.template, tc.templateErr) + + tagFilter := filter.New(&filter.Config{}) + success, err := LoadTemplate(tc.name, tagFilter, nil) + if tc.expectedErr == nil { + require.NoError(t, err) + require.True(t, success) + } else { + require.Equal(t, tc.expectedErr, err) + require.False(t, success) + } + }) + } + + t.Run("invalidTemplateID", func(t *testing.T) { + tt := []struct { + id string + success bool + }{ + {id: "A-B-C", success: true}, + {id: "A-B-C-1", success: true}, + {id: "CVE_2021_27330", success: true}, + {id: "ABC DEF", success: false}, + {id: "_-__AAA_", success: false}, + {id: " CVE-2021-27330", success: false}, + {id: "CVE-2021-27330 ", success: false}, + {id: "CVE-2021-27330-", success: false}, + {id: "-CVE-2021-27330-", success: false}, + {id: "CVE-2021--27330", success: false}, + {id: "CVE-2021+27330", success: false}, + } + for i, tc := range tt { + name := fmt.Sprintf("regexp%d", i) + t.Run(name, func(t *testing.T) { + template := &templates.Template{ + ID: tc.id, + Info: model.Info{ + Name: "Valid template", + Authors: stringslice.StringSlice{Value: "Author"}, + }, + } + parsedTemplatesCache.Store(name, template, nil) + + tagFilter := filter.New(&filter.Config{}) + success, err := LoadTemplate(name, tagFilter, nil) + if tc.success { + require.NoError(t, err) + require.True(t, success) + } else { + require.Equal(t, errors.New("invalid field format for 'id' (allowed format is ^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$)"), err) + require.False(t, success) + } + }) + } + }) +} diff --git a/v2/pkg/protocols/headless/engine/http_client.go b/v2/pkg/protocols/headless/engine/http_client.go index b4747d54..4c53ebee 100644 --- a/v2/pkg/protocols/headless/engine/http_client.go +++ b/v2/pkg/protocols/headless/engine/http_client.go @@ -1,13 +1,18 @@ package engine import ( + "context" "crypto/tls" + "fmt" + "net" "net/http" + "net/http/cookiejar" "net/url" "time" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v2/pkg/types" + "golang.org/x/net/proxy" ) // newhttpClient creates a new http client for headless communication with a timeout @@ -28,7 +33,35 @@ func newhttpClient(options *types.Options) *http.Client { if proxyURL, err := url.Parse(options.ProxyURL); err == nil { transport.Proxy = http.ProxyURL(proxyURL) } + } else if options.ProxySocksURL != "" { + var proxyAuth *proxy.Auth + + socksURL, proxyErr := url.Parse(options.ProxySocksURL) + if proxyErr == nil { + proxyAuth = &proxy.Auth{} + proxyAuth.User = socksURL.User.Username() + proxyAuth.Password, _ = socksURL.User.Password() + } + dialer, proxyErr := proxy.SOCKS5("tcp", fmt.Sprintf("%s:%s", socksURL.Hostname(), socksURL.Port()), proxyAuth, proxy.Direct) + dc := dialer.(interface { + DialContext(ctx context.Context, network, addr string) (net.Conn, error) + }) + if proxyErr == nil { + transport.DialContext = dc.DialContext + } } - return &http.Client{Transport: transport, Timeout: time.Duration(options.Timeout*3) * time.Second} + jar, _ := cookiejar.New(nil) + + httpclient := &http.Client{ + Transport: transport, + Timeout: time.Duration(options.Timeout*3) * time.Second, + Jar: jar, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + // the browser should follow redirects not us + return http.ErrUseLastResponse + }, + } + + return httpclient } diff --git a/v2/pkg/protocols/headless/engine/rules.go b/v2/pkg/protocols/headless/engine/rules.go index a254902e..a802a64a 100644 --- a/v2/pkg/protocols/headless/engine/rules.go +++ b/v2/pkg/protocols/headless/engine/rules.go @@ -8,6 +8,9 @@ import ( // routingRuleHandler handles proxy rule for actions related to request/response modification func (p *Page) routingRuleHandler(ctx *rod.Hijack) { + // usually browsers don't use chunked transfer encoding so we set the content-length nevertheless + ctx.Request.Req().ContentLength = int64(len(ctx.Request.Body())) + for _, rule := range p.rules { if rule.Part != "request" { continue diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 1a619c17..3451e5f3 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -384,7 +384,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate }() var curlCommand string - if !request.Unsafe && resp != nil && generatedRequest.request != nil { + if !request.Unsafe && resp != nil && generatedRequest.request != nil && resp.Request != nil { bodyBytes, _ := generatedRequest.request.BodyBytes() resp.Request.Body = ioutil.NopCloser(bytes.NewReader(bodyBytes)) command, _ := http2curl.GetCurlCommand(resp.Request) diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index a5274eff..12094ae6 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -60,6 +60,13 @@ type Request struct { // examples: // - value: "2048" ReadSize int `yaml:"read-size,omitempty" jsonschema:"title=size of network response to read,description=Size of response to read at the end. Default is 1024 bytes"` + // description: | + // ReadAll determines if the data stream should be read till the end regardless of the size + // + // Default value for read-all is false. + // examples: + // - value: false + ReadAll bool `yaml:"read-all,omitempty" jsonschema:"title=read all response stream,description=Read all response stream till the server stops sending"` // description: | // SelfContained specifies if the request is self contained. diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index acbbac9a..66da31e9 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -6,6 +6,7 @@ import ( "io" "net" "net/url" + "os" "strings" "time" @@ -200,13 +201,48 @@ func (request *Request) executeRequestWithPayloads(actualAddress, address, input if request.ReadSize != 0 { bufferSize = request.ReadSize } - final := make([]byte, bufferSize) - n, err := conn.Read(final) - if err != nil && err != io.EOF { - request.options.Output.Request(request.options.TemplateID, address, "network", err) - return errors.Wrap(err, "could not read from server") + + var ( + final []byte + n int + ) + + if request.ReadAll { + readInterval := time.NewTimer(time.Second * 1) + // stop the timer and drain the channel + closeTimer := func(t *time.Timer) { + if !t.Stop() { + <-t.C + } + } + read_socket: + for { + select { + case <-readInterval.C: + closeTimer(readInterval) + break read_socket + default: + buf := make([]byte, bufferSize) + nBuf, err := conn.Read(buf) + if err != nil && !os.IsTimeout(err) { + request.options.Output.Request(request.options.TemplateID, address, "network", err) + closeTimer(readInterval) + return errors.Wrap(err, "could not read from server") + } + responseBuilder.Write(buf[:nBuf]) + final = append(final, buf...) + n += nBuf + } + } + } else { + final = make([]byte, bufferSize) + n, err = conn.Read(final) + if err != nil && err != io.EOF { + request.options.Output.Request(request.options.TemplateID, address, "network", err) + return errors.Wrap(err, "could not read from server") + } + responseBuilder.Write(final[:n]) } - responseBuilder.Write(final[:n]) response := responseBuilder.String() outputEvent := request.responseToDSLMap(reqBuilder.String(), string(final[:n]), response, input, actualAddress) diff --git a/v2/pkg/reporting/trackers/github/github.go b/v2/pkg/reporting/trackers/github/github.go index 672f62ed..8d6cdb86 100644 --- a/v2/pkg/reporting/trackers/github/github.go +++ b/v2/pkg/reporting/trackers/github/github.go @@ -58,6 +58,9 @@ func New(options *Options) (*Integration, error) { if err != nil { return nil, errors.Wrap(err, "could not parse custom baseurl") } + if !strings.HasSuffix(parsed.Path, "/") { + parsed.Path += "/" + } client.BaseURL = parsed } return &Integration{client: client, options: options}, nil diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index a74b6737..85213aa0 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -29,7 +29,7 @@ type Template struct { // examples: // - name: ID Example // value: "\"CVE-2021-19520\"" - ID string `yaml:"id" jsonschema:"title=id of the template,description=The Unique ID for the template,example=cve-2021-19520"` + ID string `yaml:"id" jsonschema:"title=id of the template,description=The Unique ID for the template,example=cve-2021-19520,pattern=^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$"` // description: | // Info contains metadata information about the template. // examples: diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index 0b19e675..d6042dfb 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -843,7 +843,7 @@ func init() { FieldName: "network", }, } - NETWORKRequestDoc.Fields = make([]encoder.Doc, 9) + NETWORKRequestDoc.Fields = make([]encoder.Doc, 10) NETWORKRequestDoc.Fields[0].Name = "id" NETWORKRequestDoc.Fields[0].Type = "string" NETWORKRequestDoc.Fields[0].Note = "" @@ -883,22 +883,29 @@ func init() { NETWORKRequestDoc.Fields[5].Comments[encoder.LineComment] = "ReadSize is the size of response to read at the end" NETWORKRequestDoc.Fields[5].AddExample("", 2048) - NETWORKRequestDoc.Fields[6].Name = "matchers" - NETWORKRequestDoc.Fields[6].Type = "[]matchers.Matcher" + NETWORKRequestDoc.Fields[6].Name = "read-all" + NETWORKRequestDoc.Fields[6].Type = "bool" NETWORKRequestDoc.Fields[6].Note = "" - NETWORKRequestDoc.Fields[6].Description = "Matchers contains the detection mechanism for the request to identify\nwhether the request was successful by doing pattern matching\non request/responses.\n\nMultiple matchers can be combined with `matcher-condition` flag\nwhich accepts either `and` or `or` as argument." - NETWORKRequestDoc.Fields[6].Comments[encoder.LineComment] = "Matchers contains the detection mechanism for the request to identify" - NETWORKRequestDoc.Fields[7].Name = "extractors" - NETWORKRequestDoc.Fields[7].Type = "[]extractors.Extractor" + NETWORKRequestDoc.Fields[6].Description = "ReadAll determines if the data stream should be read till the end regardless of the size\n\nDefault value for read-all is false." + NETWORKRequestDoc.Fields[6].Comments[encoder.LineComment] = "ReadAll determines if the data stream should be read till the end regardless of the size" + + NETWORKRequestDoc.Fields[6].AddExample("", false) + NETWORKRequestDoc.Fields[7].Name = "matchers" + NETWORKRequestDoc.Fields[7].Type = "[]matchers.Matcher" NETWORKRequestDoc.Fields[7].Note = "" - NETWORKRequestDoc.Fields[7].Description = "Extractors contains the extraction mechanism for the request to identify\nand extract parts of the response." - NETWORKRequestDoc.Fields[7].Comments[encoder.LineComment] = "Extractors contains the extraction mechanism for the request to identify" - NETWORKRequestDoc.Fields[8].Name = "matchers-condition" - NETWORKRequestDoc.Fields[8].Type = "string" + NETWORKRequestDoc.Fields[7].Description = "Matchers contains the detection mechanism for the request to identify\nwhether the request was successful by doing pattern matching\non request/responses.\n\nMultiple matchers can be combined with `matcher-condition` flag\nwhich accepts either `and` or `or` as argument." + NETWORKRequestDoc.Fields[7].Comments[encoder.LineComment] = "Matchers contains the detection mechanism for the request to identify" + NETWORKRequestDoc.Fields[8].Name = "extractors" + NETWORKRequestDoc.Fields[8].Type = "[]extractors.Extractor" NETWORKRequestDoc.Fields[8].Note = "" - NETWORKRequestDoc.Fields[8].Description = "MatchersCondition is the condition between the matchers. Default is OR." - NETWORKRequestDoc.Fields[8].Comments[encoder.LineComment] = "MatchersCondition is the condition between the matchers. Default is OR." - NETWORKRequestDoc.Fields[8].Values = []string{ + NETWORKRequestDoc.Fields[8].Description = "Extractors contains the extraction mechanism for the request to identify\nand extract parts of the response." + NETWORKRequestDoc.Fields[8].Comments[encoder.LineComment] = "Extractors contains the extraction mechanism for the request to identify" + NETWORKRequestDoc.Fields[9].Name = "matchers-condition" + NETWORKRequestDoc.Fields[9].Type = "string" + NETWORKRequestDoc.Fields[9].Note = "" + NETWORKRequestDoc.Fields[9].Description = "MatchersCondition is the condition between the matchers. Default is OR." + NETWORKRequestDoc.Fields[9].Comments[encoder.LineComment] = "MatchersCondition is the condition between the matchers. Default is OR." + NETWORKRequestDoc.Fields[9].Values = []string{ "and", "or", }