nuclei/pkg/types/types.go

482 lines
20 KiB
Go
Raw Normal View History

package types
import (
"io"
"os"
"path/filepath"
"strings"
"time"
"github.com/projectdiscovery/goflags"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog"
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
errorutil "github.com/projectdiscovery/utils/errors"
fileutil "github.com/projectdiscovery/utils/file"
folderutil "github.com/projectdiscovery/utils/folder"
)
2021-01-27 06:16:41 +00:00
var (
// ErrNoMoreRequests is internal error to indicate that generator has no more requests to generate
ErrNoMoreRequests = io.EOF
)
// Options contains the configuration options for nuclei scanner.
type Options struct {
2021-02-26 07:43:11 +00:00
// Tags contains a list of tags to execute templates for. Multiple paths
// can be specified with -l flag and -tags can be used in combination with
// the -l flag.
2022-08-27 14:05:17 +00:00
Tags goflags.StringSlice
2021-03-13 19:45:33 +00:00
// ExcludeTags is the list of tags to exclude
2022-08-27 14:05:17 +00:00
ExcludeTags goflags.StringSlice
// Workflows specifies any workflows to run by nuclei
2022-08-27 14:05:17 +00:00
Workflows goflags.StringSlice
// WorkflowURLs specifies URLs to a list of workflows to use
2022-08-27 14:05:17 +00:00
WorkflowURLs goflags.StringSlice
2021-02-26 07:43:11 +00:00
// Templates specifies the template/templates to use
2022-08-27 14:05:17 +00:00
Templates goflags.StringSlice
// TemplateURLs specifies URLs to a list of templates to use
2022-08-27 14:05:17 +00:00
TemplateURLs goflags.StringSlice
// RemoteTemplates specifies list of allowed URLs to load remote templates from
RemoteTemplateDomainList goflags.StringSlice
2021-02-26 07:43:11 +00:00
// ExcludedTemplates specifies the template/templates to exclude
2022-08-27 14:05:17 +00:00
ExcludedTemplates goflags.StringSlice
// ExcludeMatchers is a list of matchers to exclude processing
2022-08-27 14:05:17 +00:00
ExcludeMatchers goflags.StringSlice
2021-02-26 07:43:11 +00:00
// CustomHeaders is the list of custom global headers to send with each request.
2022-08-27 14:05:17 +00:00
CustomHeaders goflags.StringSlice
// Vars is the list of custom global vars
Vars goflags.RuntimeMap
// Severities filters templates based on their severity and only run the matching ones.
Severities severity.Severities
2021-10-08 19:24:29 +00:00
// ExcludeSeverities specifies severities to exclude
ExcludeSeverities severity.Severities
2021-11-02 20:53:48 +00:00
// Authors filters templates based on their author and only run the matching ones.
2022-08-27 14:05:17 +00:00
Authors goflags.StringSlice
2021-10-25 17:54:42 +00:00
// Protocols contains the protocols to be allowed executed
Protocols types.ProtocolTypes
2021-10-25 17:54:42 +00:00
// ExcludeProtocols contains protocols to not be executed
ExcludeProtocols types.ProtocolTypes
// IncludeTags includes specified tags to be run even while being in denylist
2022-08-27 14:05:17 +00:00
IncludeTags goflags.StringSlice
// IncludeTemplates includes specified templates to be run even while being in denylist
2022-08-27 14:05:17 +00:00
IncludeTemplates goflags.StringSlice
// IncludeIds includes specified ids to be run even while being in denylist
2022-08-27 14:05:17 +00:00
IncludeIds goflags.StringSlice
// ExcludeIds contains templates ids to not be executed
2022-08-27 14:05:17 +00:00
ExcludeIds goflags.StringSlice
2021-02-26 07:43:11 +00:00
InternalResolversList []string // normalized from resolvers flag as well as file provided.
// ProjectPath allows nuclei to use a user defined project folder
ProjectPath string
2021-04-16 11:26:41 +00:00
// InteractshURL is the URL for the interactsh server.
InteractshURL string
// Interactsh Authorization header value for self-hosted servers
2021-09-06 13:40:32 +00:00
InteractshToken string
// Target URLs/Domains to scan using a template
Targets goflags.StringSlice
// ExcludeTargets URLs/Domains to exclude from scanning
ExcludeTargets goflags.StringSlice
2021-08-04 17:28:56 +00:00
// TargetsFilePath specifies the targets from a file to scan using templates.
TargetsFilePath string
2021-11-29 13:38:45 +00:00
// Resume the scan from the state stored in the resume config file
Resume string
2021-02-26 07:43:11 +00:00
// Output is the file to write found results to.
Output string
// ProxyInternal requests
ProxyInternal bool
// Show all supported DSL signatures
ListDslSignatures bool
// List of HTTP(s)/SOCKS5 proxy to use (comma separated or file input)
2022-08-27 14:05:17 +00:00
Proxy goflags.StringSlice
2021-02-26 07:43:11 +00:00
// TemplatesDirectory is the directory to use for storing templates
NewTemplatesDirectory string
2021-02-26 07:43:11 +00:00
// TraceLogFile specifies a file to write with the trace of all requests
TraceLogFile string
2021-10-30 09:39:38 +00:00
// ErrorLogFile specifies a file to write with the errors of all requests
ErrorLogFile string
2021-02-26 07:43:11 +00:00
// ReportingDB is the db for report storage as well as deduplication
ReportingDB string
// ReportingConfig is the config file for nuclei reporting module
ReportingConfig string
// MarkdownExportDirectory is the directory to export reports in Markdown format
2021-09-19 10:56:47 +00:00
MarkdownExportDirectory string
// MarkdownExportSortMode is the method to sort the markdown reports (options: severity, template, host, none)
MarkdownExportSortMode string
2021-06-05 21:26:36 +00:00
// SarifExport is the file to export sarif output format to
SarifExport string
2021-02-26 07:43:11 +00:00
// ResolversFile is a file containing resolvers for nuclei.
ResolversFile string
// StatsInterval is the number of seconds to display stats after
StatsInterval int
// MetricsPort is the port to show metrics on
MetricsPort int
2021-08-31 19:39:16 +00:00
// MaxHostError is the maximum number of errors allowed for a host
MaxHostError int
// TrackError contains additional error messages that count towards the maximum number of errors allowed for a host
TrackError goflags.StringSlice
// NoHostErrors disables host skipping after maximum number of errors
NoHostErrors bool
2021-02-26 07:43:11 +00:00
// BulkSize is the of targets analyzed in parallel for each template
BulkSize int
// TemplateThreads is the number of templates executed in parallel
TemplateThreads int
// HeadlessBulkSize is the of targets analyzed in parallel for each headless template
HeadlessBulkSize int
// HeadlessTemplateThreads is the number of headless templates executed in parallel
HeadlessTemplateThreads int
2021-02-26 07:43:11 +00:00
// Timeout is the seconds to wait for a response from the server.
Timeout int
// Retries is the number of times to retry the request
Retries int
// Rate-Limit is the maximum number of requests per specified target
RateLimit int
// Rate-Limit is the maximum number of requests per minute for specified target
RateLimitMinute int
2021-03-02 09:24:57 +00:00
// PageTimeout is the maximum time to wait for a page in seconds
2021-03-01 08:50:56 +00:00
PageTimeout int
2021-04-16 11:26:41 +00:00
// InteractionsCacheSize is the number of interaction-url->req to keep in cache at a time.
InteractionsCacheSize int
// InteractionsPollDuration is the number of seconds to wait before each interaction poll
InteractionsPollDuration int
// Eviction is the number of seconds after which to automatically discard
// interaction requests.
InteractionsEviction int
// InteractionsCoolDownPeriod is additional seconds to wait for interactions after closing
2021-04-16 11:26:41 +00:00
// of the poller.
InteractionsCoolDownPeriod int
// MaxRedirects is the maximum numbers of redirects to be followed.
MaxRedirects int
// FollowRedirects enables following redirects for http request module
FollowRedirects bool
// FollowRedirects enables following redirects for http request module only on the same host
FollowHostRedirects bool
2021-02-26 07:43:11 +00:00
// OfflineHTTP is a flag that specific offline processing of http response
// using same matchers/extractors from http protocol without the need
// to send a new request, reading responses from a file.
OfflineHTTP bool
// Force HTTP2 requests
ForceAttemptHTTP2 bool
2021-07-08 09:54:04 +00:00
// StatsJSON writes stats output in JSON format
StatsJSON bool
2021-02-26 07:43:11 +00:00
// Headless specifies whether to allow headless mode templates
Headless bool
// ShowBrowser specifies whether the show the browser in headless mode
ShowBrowser bool
// HeadlessOptionalArguments specifies optional arguments to pass to Chrome
HeadlessOptionalArguments goflags.StringSlice
// DisableClustering disables clustering of templates
DisableClustering bool
// UseInstalledChrome skips chrome install and use local instance
UseInstalledChrome bool
// SystemResolvers enables override of nuclei's DNS client opting to use system resolver stack.
2021-02-26 07:43:11 +00:00
SystemResolvers bool
// ShowActions displays a list of all headless actions
ShowActions bool
// Deprecated: Enabled by default through clistats . Metrics enables display of metrics via an http endpoint
Metrics bool
// Debug mode allows debugging request/responses for the engine
Debug bool
2021-01-12 11:48:08 +00:00
// DebugRequests mode allows debugging request for the engine
DebugRequests bool
// DebugResponse mode allows debugging response for the engine
DebugResponse bool
// DisableHTTPProbe disables http probing feature of input normalization
DisableHTTPProbe bool
// LeaveDefaultPorts skips normalization of default ports
LeaveDefaultPorts bool
// AutomaticScan enables automatic tech based template execution
AutomaticScan bool
// Silent suppresses any extra text and only writes found URLs on screen.
Silent bool
// Validate validates the templates passed to nuclei.
Validate bool
// NoStrictSyntax disables strict syntax check on nuclei templates (allows custom key-value pairs).
NoStrictSyntax bool
// Verbose flag indicates whether to show verbose output or not
Verbose bool
VerboseVerbose bool
// ShowVarDump displays variable dump
ShowVarDump bool
// No-Color disables the colored output.
NoColor bool
// UpdateTemplates updates the templates installed at startup (also used by cloud to update datasources)
UpdateTemplates bool
JSON Export Handling Updates (#3466) * Switch -json to -jsonl * Add JSON output file * Update docs for EN and ID * Fix linting issue with error wrap * Add -j flag * Fix call for short flag * Correct typo "Ciper" to "Cipher" (#3468) * migrate dsl helper functions to dsl repo (#3461) * migrate dsl pkg code to dsl repo * fix lint error * upgrade dsl dependency * upgrade deps --------- Co-authored-by: Sandeep Singh <sandeep@projectdiscovery.io> * chore(deps): bump github.com/projectdiscovery/httpx in /v2 (#3469) Bumps [github.com/projectdiscovery/httpx](https://github.com/projectdiscovery/httpx) from 1.2.7 to 1.2.9. - [Release notes](https://github.com/projectdiscovery/httpx/releases) - [Changelog](https://github.com/projectdiscovery/httpx/blob/main/.goreleaser.yml) - [Commits](https://github.com/projectdiscovery/httpx/compare/v1.2.7...v1.2.9) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/httpx dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): bump github.com/weppos/publicsuffix-go in /v2 (#3472) Bumps [github.com/weppos/publicsuffix-go](https://github.com/weppos/publicsuffix-go) from 0.20.0 to 0.30.0. - [Release notes](https://github.com/weppos/publicsuffix-go/releases) - [Changelog](https://github.com/weppos/publicsuffix-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/weppos/publicsuffix-go/compare/v0.20.0...v0.30.0) --- updated-dependencies: - dependency-name: github.com/weppos/publicsuffix-go dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): bump github.com/projectdiscovery/wappalyzergo in /v2 (#3473) Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.81 to 0.0.88. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.81...v0.0.88) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): bump github.com/projectdiscovery/hmap in /v2 (#3470) Bumps [github.com/projectdiscovery/hmap](https://github.com/projectdiscovery/hmap) from 0.0.10 to 0.0.11. - [Release notes](https://github.com/projectdiscovery/hmap/releases) - [Commits](https://github.com/projectdiscovery/hmap/compare/v0.0.10...v0.0.11) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/hmap dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * debug catalog path * use paths instead of filepath for aws path * deps update (#3477) * deps update * fixing gologger via callback * Moved `json-export` flag to the other exporters * Switch "json[-_]exporter to jsonexporter" --------- Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: Ramana Reddy <90540245+RamanaReddy0M@users.noreply.github.com> Co-authored-by: Sandeep Singh <sandeep@projectdiscovery.io> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Tarun Koyalwar <tarun@projectdiscovery.io> Co-authored-by: Mzack9999 <mzack9999@protonmail.com> Co-authored-by: shubhamrasal <shubhamdharmarasal@gmail.com>
2023-03-31 09:59:29 +00:00
// JSON writes json line output to files
JSONL bool
// JSONRequests writes requests/responses for matches in JSON output
// Deprecated: use OmitRawRequests instead as of now JSONRequests(include raw requests) is always true
JSONRequests bool
// OmitRawRequests omits requests/responses for matches in JSON output
OmitRawRequests bool
// OmitTemplate omits encoded template from JSON output
OmitTemplate bool
JSON Export Handling Updates (#3466) * Switch -json to -jsonl * Add JSON output file * Update docs for EN and ID * Fix linting issue with error wrap * Add -j flag * Fix call for short flag * Correct typo "Ciper" to "Cipher" (#3468) * migrate dsl helper functions to dsl repo (#3461) * migrate dsl pkg code to dsl repo * fix lint error * upgrade dsl dependency * upgrade deps --------- Co-authored-by: Sandeep Singh <sandeep@projectdiscovery.io> * chore(deps): bump github.com/projectdiscovery/httpx in /v2 (#3469) Bumps [github.com/projectdiscovery/httpx](https://github.com/projectdiscovery/httpx) from 1.2.7 to 1.2.9. - [Release notes](https://github.com/projectdiscovery/httpx/releases) - [Changelog](https://github.com/projectdiscovery/httpx/blob/main/.goreleaser.yml) - [Commits](https://github.com/projectdiscovery/httpx/compare/v1.2.7...v1.2.9) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/httpx dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): bump github.com/weppos/publicsuffix-go in /v2 (#3472) Bumps [github.com/weppos/publicsuffix-go](https://github.com/weppos/publicsuffix-go) from 0.20.0 to 0.30.0. - [Release notes](https://github.com/weppos/publicsuffix-go/releases) - [Changelog](https://github.com/weppos/publicsuffix-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/weppos/publicsuffix-go/compare/v0.20.0...v0.30.0) --- updated-dependencies: - dependency-name: github.com/weppos/publicsuffix-go dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): bump github.com/projectdiscovery/wappalyzergo in /v2 (#3473) Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.81 to 0.0.88. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.81...v0.0.88) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): bump github.com/projectdiscovery/hmap in /v2 (#3470) Bumps [github.com/projectdiscovery/hmap](https://github.com/projectdiscovery/hmap) from 0.0.10 to 0.0.11. - [Release notes](https://github.com/projectdiscovery/hmap/releases) - [Commits](https://github.com/projectdiscovery/hmap/compare/v0.0.10...v0.0.11) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/hmap dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * debug catalog path * use paths instead of filepath for aws path * deps update (#3477) * deps update * fixing gologger via callback * Moved `json-export` flag to the other exporters * Switch "json[-_]exporter to jsonexporter" --------- Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: Ramana Reddy <90540245+RamanaReddy0M@users.noreply.github.com> Co-authored-by: Sandeep Singh <sandeep@projectdiscovery.io> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Tarun Koyalwar <tarun@projectdiscovery.io> Co-authored-by: Mzack9999 <mzack9999@protonmail.com> Co-authored-by: shubhamrasal <shubhamdharmarasal@gmail.com>
2023-03-31 09:59:29 +00:00
// JSONExport is the file to export JSON output format to
JSONExport string
// JSONLExport is the file to export JSONL output format to
JSONLExport string
// EnableProgressBar enables progress bar
EnableProgressBar bool
// TemplateDisplay displays the template contents
TemplateDisplay bool
// TemplateList lists available templates
TemplateList bool
// HangMonitor enables nuclei hang monitoring
HangMonitor bool
// Stdin specifies whether stdin input was given to the process
Stdin bool
// StopAtFirstMatch stops processing template at first full match (this may break chained requests)
StopAtFirstMatch bool
2021-09-26 20:33:12 +00:00
// Stream the input without sorting
Stream bool
// NoMeta disables display of metadata for the matches
NoMeta bool
// Timestamp enables display of timestamp for the matcher
Timestamp bool
// Project is used to avoid sending same HTTP request multiple times
Project bool
// NewTemplates only runs newly added templates from the repository
NewTemplates bool
// NewTemplatesWithVersion runs new templates added in specific version
2022-08-27 14:05:17 +00:00
NewTemplatesWithVersion goflags.StringSlice
// NoInteractsh disables use of interactsh server for interaction polling
NoInteractsh bool
// EnvironmentVariables enables support for environment variables
EnvironmentVariables bool
// MatcherStatus displays optional status for the failed matches as well
MatcherStatus bool
// ClientCertFile client certificate file (PEM-encoded) used for authenticating against scanned hosts
ClientCertFile string
// ClientKeyFile client key file (PEM-encoded) used for authenticating against scanned hosts
ClientKeyFile string
// ClientCAFile client certificate authority file (PEM-encoded) used for authenticating against scanned hosts
ClientCAFile string
// Deprecated: Use ZTLS library
2022-01-25 19:48:21 +00:00
ZTLS bool
// AllowLocalFileAccess allows local file access from templates payloads
AllowLocalFileAccess bool
// RestrictLocalNetworkAccess restricts local network access from templates requests
RestrictLocalNetworkAccess bool
// ShowMatchLine enables display of match line number
ShowMatchLine bool
// EnablePprof enables exposing pprof runtime information with a webserver.
2022-04-27 16:19:44 +00:00
EnablePprof bool
// StoreResponse stores received response to output directory
2022-04-27 16:19:44 +00:00
StoreResponse bool
// StoreResponseDir stores received response to custom directory
StoreResponseDir string
2022-04-27 16:19:44 +00:00
// DisableRedirects disables following redirects for http request module
DisableRedirects bool
// SNI custom hostname
SNI string
// Interface to use for network scan
Interface string
// SourceIP sets custom source IP address for network requests
SourceIP string
// AttackType overrides template level attack-type configuration
AttackType string
// ResponseReadSize is the maximum size of response to read
ResponseReadSize int
// ResponseSaveSize is the maximum size of response to save
ResponseSaveSize int
// Health Check
HealthCheck bool
// Time to wait between each input read operation before closing the stream
InputReadTimeout time.Duration
// Disable stdin for input processing
DisableStdin bool
// IncludeConditions is the list of conditions templates should match
2022-08-27 14:05:17 +00:00
IncludeConditions goflags.StringSlice
Spelling (#4008) * spelling: addresses Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: asynchronous Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: basic Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: brute force Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: constant Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: disables Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: engine Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: every time Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: execution Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: false positives Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: from Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: further Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: github Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: gitlab Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: highlight Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: hygiene Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: ignore Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: input Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: item Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: itself Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: latestxxx Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: navigation Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: negative Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: nonexistent Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: occurred Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: override Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: overrides Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: payload Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: performed Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: respective Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: retrieve Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: scanlist Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: separated Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: separator Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: severity Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: source Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: strategy Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: string Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: templates Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: terminal Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: timeout Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: trailing slash Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: trailing Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: websocket Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --------- Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com>
2023-08-01 18:33:43 +00:00
// Enable uncover engine
Uncover bool
// Uncover search query
UncoverQuery goflags.StringSlice
// Uncover search engine
UncoverEngine goflags.StringSlice
// Uncover search field
UncoverField string
// Uncover search limit
UncoverLimit int
// Uncover search delay
UncoverRateLimit int
// ScanAllIPs associated to a dns record
ScanAllIPs bool
// IPVersion to scan (4,6)
IPVersion goflags.StringSlice
// PublicTemplateDisableDownload disables downloading templates from the nuclei-templates public repository
PublicTemplateDisableDownload bool
// GitHub token used to clone/pull from private repos for custom templates
Spelling (#4008) * spelling: addresses Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: asynchronous Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: basic Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: brute force Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: constant Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: disables Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: engine Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: every time Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: execution Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: false positives Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: from Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: further Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: github Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: gitlab Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: highlight Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: hygiene Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: ignore Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: input Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: item Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: itself Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: latestxxx Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: navigation Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: negative Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: nonexistent Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: occurred Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: override Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: overrides Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: payload Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: performed Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: respective Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: retrieve Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: scanlist Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: separated Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: separator Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: severity Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: source Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: strategy Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: string Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: templates Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: terminal Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: timeout Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: trailing slash Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: trailing Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> * spelling: websocket Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --------- Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com>
2023-08-01 18:33:43 +00:00
GitHubToken string
// GitHubTemplateRepo is the list of custom public/private templates GitHub repos
GitHubTemplateRepo []string
// GitHubTemplateDisableDownload disables downloading templates from custom GitHub repositories
GitHubTemplateDisableDownload bool
// GitLabServerURL is the gitlab server to use for custom templates
GitLabServerURL string
// GitLabToken used to clone/pull from private repos for custom templates
GitLabToken string
// GitLabTemplateRepositoryIDs is the comma-separated list of custom gitlab repositories IDs
GitLabTemplateRepositoryIDs []int
// GitLabTemplateDisableDownload disables downloading templates from custom GitLab repositories
GitLabTemplateDisableDownload bool
// AWS access key for downloading templates from S3 bucket
AwsAccessKey string
// AWS secret key for downloading templates from S3 bucket
AwsSecretKey string
// AWS bucket name for downloading templates from S3 bucket
AwsBucketName string
// AWS Region name where AWS S3 bucket is located
AwsRegion string
// AwsTemplateDisableDownload disables downloading templates from AWS S3 buckets
AwsTemplateDisableDownload bool
// AzureContainerName for downloading templates from Azure Blob Storage. Example: templates
AzureContainerName string
// AzureTenantID for downloading templates from Azure Blob Storage. Example: 00000000-0000-0000-0000-000000000000
AzureTenantID string
// AzureClientID for downloading templates from Azure Blob Storage. Example: 00000000-0000-0000-0000-000000000000
AzureClientID string
// AzureClientSecret for downloading templates from Azure Blob Storage. Example: 00000000-0000-0000-0000-000000000000
AzureClientSecret string
// AzureServiceURL for downloading templates from Azure Blob Storage. Example: https://XXXXXXXXXX.blob.core.windows.net/
AzureServiceURL string
// AzureTemplateDisableDownload disables downloading templates from Azure Blob Storage
AzureTemplateDisableDownload bool
2022-12-27 20:48:00 +00:00
// Scan Strategy (auto,hosts-spray,templates-spray)
ScanStrategy string
// Fuzzing Type overrides template level fuzzing-type configuration
FuzzingType string
// Fuzzing Mode overrides template level fuzzing-mode configuration
FuzzingMode string
// TlsImpersonate enables TLS impersonation
TlsImpersonate bool
Adding support for code templates (#2930) * Adding support for code templates * adding support for python, powershell and echo (test) * removing debug code * introducing command + trivial trust store mechanism * updating tests * adding basic tests * removing deprecated oracle * mod tidy * adding signature proto with debug prints * removing debug code * fixing test * fixing param order * improving test conditional build * disable file+offlinehttp+code with cloud * adding env vars * removing debug code * reorganizing test folders * adding code template test prototype with dummy priv/pub keys * bump go to 1.20 * fixing go version * fixing lint errors * adding fatal on pub-key test failure * switching to ecdsa asn1 * removing unused signature * fixing signature * adding more tests * extending core with engine args + powershell win test * adding unsigned code test * skip template signing in particular test case * improving test coverage * refactoring key names + adding already signed algo * removing debug code * fixing syntax * fixing lint issues * removing test template * fixing dns tests path * output fmt * adding interact * fixing lint issues * adding -sign cli helper * fixing nil pointer + parse inline keys * making rsa default * adding code prot. ref * moving file to correct loc * moving test * Issue 3339 headless fuzz (#3790) * Basic headless fuzzing * Remove debug statements * Add integration tests * Update template * Fix recognize payload value in matcher * Update tempalte * use req.SetURL() --------- Co-authored-by: Tarun Koyalwar <tarun@projectdiscovery.io> * Auto Generate Syntax Docs + JSONSchema [Fri Jun 9 00:23:32 UTC 2023] :robot: * Add headless header and status matchers (#3794) * add headless header and status matchers * rename headers as header * add integration test for header+status * fix typo * add retry to py-interactsh integration test --------- Co-authored-by: Sandeep Singh <sandeep@projectdiscovery.io> Co-authored-by: Shubham Rasal <shubham@projectdiscovery.io> Co-authored-by: Tarun Koyalwar <tarun@projectdiscovery.io> Co-authored-by: GitHub Action <action@github.com> Co-authored-by: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com> Co-authored-by: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com>
2023-06-09 15:24:24 +00:00
// CodeTemplateSignaturePublicKey is the custom public key used to verify the template signature (algorithm is automatically inferred from the length)
CodeTemplateSignaturePublicKey string
// CodeTemplateSignatureAlgorithm specifies the sign algorithm (rsa, ecdsa)
CodeTemplateSignatureAlgorithm string
// SignTemplates enables signing of templates
SignTemplates bool
// EnableCodeTemplates enables code templates
EnableCodeTemplates bool
// Disables cloud upload
EnableCloudUpload bool
}
2021-10-07 10:36:27 +00:00
2021-11-29 13:38:45 +00:00
// ShouldLoadResume resume file
func (options *Options) ShouldLoadResume() bool {
return options.Resume != "" && fileutil.FileExists(options.Resume)
2021-11-29 13:38:45 +00:00
}
// ShouldSaveResume file
func (options *Options) ShouldSaveResume() bool {
return true
}
// ShouldFollowHTTPRedirects determines if http redirects should be followed
func (options *Options) ShouldFollowHTTPRedirects() bool {
return options.FollowRedirects || options.FollowHostRedirects
}
// HasClientCertificates determines if any client certificate was specified
func (options *Options) HasClientCertificates() bool {
return options.ClientCertFile != "" || options.ClientCAFile != "" || options.ClientKeyFile != ""
}
// DefaultOptions returns default options for nuclei
func DefaultOptions() *Options {
return &Options{
RateLimit: 150,
BulkSize: 25,
TemplateThreads: 25,
HeadlessBulkSize: 10,
HeadlessTemplateThreads: 10,
Timeout: 5,
Retries: 1,
MaxHostError: 30,
ResponseReadSize: 10 * 1024 * 1024,
ResponseSaveSize: 1024 * 1024,
}
}
cloud templates targets sync (#2959) * Add s3 bucket template provider - Refactor the custom github template code - add interface for template provider * Validate if aws creds are passed if bucket flag - refactor s3 provider struct to take client - add function which returns the aws s3 client - update error messages * Add aws s3 bucket flags documentation in README.md - Rename the github_test.go to customTemplate_test.go * go mod update * Move template provider code to pkg/external/customtemplates dir * Added initial data_source sync to cloud * Misc * Add pagination to scan output and scan list (#2858) * Add pagination to scan output and scan list * Use time based parameters instead of page numbers * Fix linting errors * Do not check limits at client, check at server * Remove unused constant * Misc update * Removed unnecessary flags * Misc * Misc * Misc endpoint additions * Added more routes * Typo fix * Misc fixes * Misc * Misc fixes to cloud target logic + use int for IDs * Misc * Misc fixes * Misc * Misc fixes * readme update * Add JSON output support for list-scan option (#2876) * Add JSON output support for list-scan option * Fix typo in cloud JSON output description * Following changes - Update status(finished, running) to be lower-case by default - Convert status to upper-case in DisplayScanList() * Update status to be lower-case by default * Remove additional json flag, instead use existing * Merge conflict * Accomodate comment changes and restructure code Co-authored-by: Jaideep K <jaideep@one2n.in> * Use integer IDs for scan tasks * Added get-templates-targets endpoint + JSON + validation * Added target count list * misc option / description updates * Added changes as per code review * duplicate options + typo updates * Added tablewriter for tabular data writing by default * Fixed list scan endpoint * Review changes * workflow fix * Added cloud tags etc based filtering (#3070) * Added omitempty for filtering request * go mod tidy * misc format update Co-authored-by: shubhamrasal <shubhamdharmarasal@gmail.com> Co-authored-by: Ice3man <nizamulrana@gmail.com> Co-authored-by: Jaideep Khandelwal <jdk2588@gmail.com> Co-authored-by: Siddharth Shashikar <60960197+shashikarsiddharth@users.noreply.github.com> Co-authored-by: Jaideep K <jaideep@one2n.in>
2022-12-21 17:18:43 +00:00
func (options *Options) ShouldUseHostError() bool {
return options.MaxHostError > 0 && !options.NoHostErrors
cloud templates targets sync (#2959) * Add s3 bucket template provider - Refactor the custom github template code - add interface for template provider * Validate if aws creds are passed if bucket flag - refactor s3 provider struct to take client - add function which returns the aws s3 client - update error messages * Add aws s3 bucket flags documentation in README.md - Rename the github_test.go to customTemplate_test.go * go mod update * Move template provider code to pkg/external/customtemplates dir * Added initial data_source sync to cloud * Misc * Add pagination to scan output and scan list (#2858) * Add pagination to scan output and scan list * Use time based parameters instead of page numbers * Fix linting errors * Do not check limits at client, check at server * Remove unused constant * Misc update * Removed unnecessary flags * Misc * Misc * Misc endpoint additions * Added more routes * Typo fix * Misc fixes * Misc * Misc fixes to cloud target logic + use int for IDs * Misc * Misc fixes * Misc * Misc fixes * readme update * Add JSON output support for list-scan option (#2876) * Add JSON output support for list-scan option * Fix typo in cloud JSON output description * Following changes - Update status(finished, running) to be lower-case by default - Convert status to upper-case in DisplayScanList() * Update status to be lower-case by default * Remove additional json flag, instead use existing * Merge conflict * Accomodate comment changes and restructure code Co-authored-by: Jaideep K <jaideep@one2n.in> * Use integer IDs for scan tasks * Added get-templates-targets endpoint + JSON + validation * Added target count list * misc option / description updates * Added changes as per code review * duplicate options + typo updates * Added tablewriter for tabular data writing by default * Fixed list scan endpoint * Review changes * workflow fix * Added cloud tags etc based filtering (#3070) * Added omitempty for filtering request * go mod tidy * misc format update Co-authored-by: shubhamrasal <shubhamdharmarasal@gmail.com> Co-authored-by: Ice3man <nizamulrana@gmail.com> Co-authored-by: Jaideep Khandelwal <jdk2588@gmail.com> Co-authored-by: Siddharth Shashikar <60960197+shashikarsiddharth@users.noreply.github.com> Co-authored-by: Jaideep K <jaideep@one2n.in>
2022-12-21 17:18:43 +00:00
}
func (options *Options) ParseHeadlessOptionalArguments() map[string]string {
optionalArguments := make(map[string]string)
for _, v := range options.HeadlessOptionalArguments {
if argParts := strings.SplitN(v, "=", 2); len(argParts) >= 2 {
key := strings.TrimSpace(argParts[0])
value := strings.TrimSpace(argParts[1])
if key != "" && value != "" {
optionalArguments[key] = value
}
}
}
return optionalArguments
}
// LoadHelperFile loads a helper file needed for the template
// this respects the sandbox rules and only loads files from
// allowed directories
func (options *Options) LoadHelperFile(helperFile, templatePath string, catalog catalog.Catalog) (io.ReadCloser, error) {
if !options.AllowLocalFileAccess {
// if global file access is disabled try loading with restrictions
absPath, err := options.GetValidAbsPath(helperFile, templatePath)
if err != nil {
return nil, err
}
helperFile = absPath
}
f, err := os.Open(helperFile)
if err != nil {
return nil, errorutil.NewWithErr(err).Msgf("could not open file %v", helperFile)
}
return f, nil
}
// GetValidAbsPath returns absolute path of helper file if it is allowed to be loaded
// this respects the sandbox rules and only loads files from allowed directories
func (o *Options) GetValidAbsPath(helperFilePath, templatePath string) (string, error) {
// Conditions to allow helper file
// 1. If helper file is present in nuclei-templates directory
// 2. If helper file and template file are in same directory given that its not root directory
// resolve and clean helper file path
// ResolveNClean uses a custom base path instead of CWD
resolvedPath, err := fileutil.ResolveNClean(helperFilePath, config.DefaultConfig.GetTemplateDir())
if err == nil {
// As per rule 1, if helper file is present in nuclei-templates directory, allow it
if strings.HasPrefix(resolvedPath, config.DefaultConfig.GetTemplateDir()) {
return resolvedPath, nil
}
}
// CleanPath resolves using CWD and cleans the path
helperFilePath, err = fileutil.CleanPath(helperFilePath)
if err != nil {
return "", errorutil.NewWithErr(err).Msgf("could not clean helper file path %v", helperFilePath)
}
templatePath, err = fileutil.CleanPath(templatePath)
if err != nil {
return "", errorutil.NewWithErr(err).Msgf("could not clean template path %v", templatePath)
}
// As per rule 2, if template and helper file exist in same directory or helper file existed in any child dir of template dir
// and both of them are present in user home directory, allow it
// Review: should we keep this rule ? add extra option to disable this ?
if isHomeDir(helperFilePath) && isHomeDir(templatePath) && strings.HasPrefix(filepath.Dir(helperFilePath), filepath.Dir(templatePath)) {
return helperFilePath, nil
}
// all other cases are denied
return "", errorutil.New("access to helper file %v denied", helperFilePath)
}
// isRootDir checks if given is root directory
func isHomeDir(path string) bool {
homeDir := folderutil.HomeDirOrDefault("")
return strings.HasPrefix(path, homeDir)
}