From 1fbbca66f9d313c3a242fa375f6a62104724ce9d Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Wed, 9 Nov 2022 14:18:56 +0100 Subject: [PATCH] Adding support to scan all v4/v6 IPs (#2709) * Adding support to scan all v4/v6 IPs * adding tests * metainput prototype * using new signature * fixing nil pointer * adding request context with metadata * removing log instruction * fixing merge conflicts * adding clone helpers * attempting to fix ipv6 square parenthesis wrap * fixing dialed ip info * fixing syntax * fixing output ip selection * adding integration tests * disabling test due to gh ipv6 issue * using ipv4 only due to GH limited networking * extending metainput marshaling * fixing hmap key * adding test for httpx integration * fixing lint error * reworking marshaling/id-calculation * adding ip version validation * improving handling non url targets * fixing condition check --- integration_tests/http/get-all-ips.yaml | 15 ++ .../http/get-without-scheme.yaml | 15 ++ v2/cmd/integration-test/code.go | 3 +- v2/cmd/integration-test/http.go | 27 ++++ v2/cmd/nuclei/main.go | 2 + v2/cmd/tools/fuzzplayground/main.go | 4 +- v2/go.mod | 4 +- v2/go.sum | 27 +--- v2/internal/runner/enumerate.go | 5 +- v2/internal/runner/inputs.go | 11 +- v2/internal/runner/nucleicloud/types.go | 8 +- v2/internal/runner/options.go | 18 +++ v2/internal/runner/runner.go | 5 +- v2/pkg/core/engine.go | 3 +- v2/pkg/core/execute.go | 26 ++-- v2/pkg/core/inputs/hybrid/hmap.go | 129 ++++++++++++++---- v2/pkg/core/inputs/hybrid/hmap_test.go | 75 +++++++++- v2/pkg/core/inputs/hybrid/options.go | 7 + v2/pkg/core/inputs/inputs.go | 6 +- v2/pkg/core/workflow_execute.go | 6 +- v2/pkg/core/workflow_execute_test.go | 58 ++++---- .../common/automaticscan/automaticscan.go | 10 +- .../common/contextargs/contextargs.go | 19 ++- .../protocols/common/contextargs/metainput.go | 69 ++++++++++ v2/pkg/protocols/common/executer/executer.go | 24 ++-- v2/pkg/protocols/dns/request.go | 8 +- v2/pkg/protocols/dns/request_test.go | 3 +- v2/pkg/protocols/file/request.go | 10 +- v2/pkg/protocols/file/request_test.go | 3 +- v2/pkg/protocols/headless/request.go | 2 +- v2/pkg/protocols/http/request.go | 91 +++++++----- v2/pkg/protocols/http/request_test.go | 3 +- v2/pkg/protocols/network/request.go | 6 +- v2/pkg/protocols/network/request_test.go | 9 +- v2/pkg/protocols/offlinehttp/request.go | 4 +- v2/pkg/protocols/ssl/ssl.go | 22 +-- v2/pkg/protocols/ssl/ssl_test.go | 3 +- v2/pkg/protocols/websocket/websocket.go | 6 +- v2/pkg/protocols/whois/whois.go | 2 +- v2/pkg/templates/cluster.go | 4 +- v2/pkg/types/types.go | 8 +- 41 files changed, 543 insertions(+), 217 deletions(-) create mode 100644 integration_tests/http/get-all-ips.yaml create mode 100644 integration_tests/http/get-without-scheme.yaml create mode 100644 v2/pkg/core/inputs/hybrid/options.go create mode 100644 v2/pkg/protocols/common/contextargs/metainput.go diff --git a/integration_tests/http/get-all-ips.yaml b/integration_tests/http/get-all-ips.yaml new file mode 100644 index 00000000..1dba45b1 --- /dev/null +++ b/integration_tests/http/get-all-ips.yaml @@ -0,0 +1,15 @@ +id: get-all-ips + +info: + name: Basic GET Request on all IPS + author: pdteam + severity: info + +requests: + - method: GET + path: + - "{{BaseURL}}" + matchers: + - type: word + words: + - "ok" \ No newline at end of file diff --git a/integration_tests/http/get-without-scheme.yaml b/integration_tests/http/get-without-scheme.yaml new file mode 100644 index 00000000..020800aa --- /dev/null +++ b/integration_tests/http/get-without-scheme.yaml @@ -0,0 +1,15 @@ +id: get-without-scheme + +info: + name: Basic GET Request without scheme + author: pdteam + severity: info + +requests: + - method: GET + path: + - "{{BaseURL}}" + matchers: + - type: word + words: + - "ok" \ No newline at end of file diff --git a/v2/cmd/integration-test/code.go b/v2/cmd/integration-test/code.go index 4171efeb..73414d21 100644 --- a/v2/cmd/integration-test/code.go +++ b/v2/cmd/integration-test/code.go @@ -23,6 +23,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/parsers" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit" @@ -123,7 +124,7 @@ func executeNucleiAsCode(templatePath, templateURL string) ([]string, error) { } store.Load() - input := &inputs.SimpleInputProvider{Inputs: []string{templateURL}} + input := &inputs.SimpleInputProvider{Inputs: []*contextargs.MetaInput{{Input: templateURL}}} _ = engine.Execute(store.Templates(), input) engine.WorkPool().Wait() // Wait for the scan to finish diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index 441865ea..9f1f5548 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -57,6 +57,8 @@ var httpTestcases = map[string]testutils.TestCase{ "http/get-sni-unsafe.yaml": &customCLISNIUnsafe{}, "http/annotation-timeout.yaml": &annotationTimeout{}, "http/custom-attack-type.yaml": &customAttackType{}, + "http/get-all-ips.yaml": &scanAllIPS{}, + "http/get-without-scheme.yaml": &httpGetWithoutScheme{}, } type httpInteractshRequest struct{} @@ -1001,3 +1003,28 @@ func (h *customAttackType) Execute(filePath string) error { } return expectResultsCount(got, 4) } + +// Disabled as GH doesn't support ipv6 +type scanAllIPS struct{} + +// Execute executes a test case and returns an error if occurred +func (h *scanAllIPS) Execute(filePath string) error { + got, err := testutils.RunNucleiTemplateAndGetResults(filePath, "https://scanme.sh", debug, "-scan-all-ips", "-iv", "4") + if err != nil { + return err + } + // limiting test to ipv4 (GH doesn't support ipv6) + return expectResultsCount(got, 1) +} + +// ensure that ip|host are handled without http|https scheme +type httpGetWithoutScheme struct{} + +// Execute executes a test case and returns an error if occurred +func (h *httpGetWithoutScheme) Execute(filePath string) error { + got, err := testutils.RunNucleiTemplateAndGetResults(filePath, "scanme.sh", debug) + if err != nil { + return err + } + return expectResultsCount(got, 1) +} diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 64246450..cc05cdb7 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -125,6 +125,8 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.StringSliceVarP(&options.Targets, "target", "u", []string{}, "target URLs/hosts to scan", goflags.StringSliceOptions), flagSet.StringVarP(&options.TargetsFilePath, "list", "l", "", "path to file containing a list of target URLs/hosts to scan (one per line)"), flagSet.StringVar(&options.Resume, "resume", "", "Resume scan using resume.cfg (clustering will be disabled)"), + flagSet.BoolVarP(&options.ScanAllIPs, "scan-all-ips", "sa", false, "Scan all the ip's associated with dns record"), + flagSet.StringSliceVarP(&options.IPVersion, "ip-version", "iv", []string{"4"}, "IP version to scan of hostname (4,6) - (default 4)", goflags.CommaSeparatedStringSliceOptions), ) flagSet.CreateGroup("templates", "Templates", diff --git a/v2/cmd/tools/fuzzplayground/main.go b/v2/cmd/tools/fuzzplayground/main.go index 0c2d810f..4b870e0b 100644 --- a/v2/cmd/tools/fuzzplayground/main.go +++ b/v2/cmd/tools/fuzzplayground/main.go @@ -2,7 +2,7 @@ package main import ( "fmt" - "io/ioutil" + "io" "net/http" "os/exec" "strconv" @@ -66,7 +66,7 @@ func requestHandler(ctx echo.Context) error { } defer data.Body.Close() - body, _ := ioutil.ReadAll(data.Body) + body, _ := io.ReadAll(data.Body) return ctx.HTML(200, fmt.Sprintf(bodyTemplate, string(body))) } diff --git a/v2/go.mod b/v2/go.mod index 3be323da..83ac458c 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -25,8 +25,7 @@ require ( github.com/owenrumney/go-sarif/v2 v2.1.2 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.8 - github.com/projectdiscovery/cryptoutil v1.0.0 // indirect - github.com/projectdiscovery/fastdialer v0.0.17 + github.com/projectdiscovery/fastdialer v0.0.18-0.20221102102120-8e9343e8b0e0 github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08 github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.2 @@ -208,6 +207,7 @@ require ( github.com/nwaples/rardecode v1.1.2 // indirect github.com/pierrec/lz4 v2.6.1+incompatible // indirect github.com/projectdiscovery/asnmap v0.0.1 // indirect + github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 // indirect; indirectdev github.com/projectdiscovery/fileutil v0.0.3 // indirect github.com/projectdiscovery/iputil v0.0.2 // indirect github.com/projectdiscovery/sliceutil v0.0.1 // indirect diff --git a/v2/go.sum b/v2/go.sum index 7fff3577..8ad9690a 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -518,8 +518,6 @@ github.com/mholt/acmez v1.0.4/go.mod h1:qFGLZ4u+ehWINeJZjzPlsnjJBCPAADWTcIqE/7DA github.com/mholt/archiver v3.1.1+incompatible h1:1dCVxuqs0dJseYEhi5pl7MYPH9zDa1wBi7mF09cbNkU= github.com/mholt/archiver v3.1.1+incompatible/go.mod h1:Dh2dOXnSdiLxRiPoVfIr/fI1TwETms9B8CTWfeh7ROU= github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= -github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= -github.com/microcosm-cc/bluemonday v1.0.19/go.mod h1:QNzV2UbLK2/53oIIwTOyLUSABMkjZ4tqiyC1g/DyqxE= github.com/microcosm-cc/bluemonday v1.0.21 h1:dNH3e4PSyE4vNX+KlRGHT5KrSvjeUkoNPwEORjffHJg= github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= @@ -596,26 +594,21 @@ github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc h1:jqZK github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc/go.mod h1:5tNGQP9kOfW+X5+40pZP8aqPYLHs45nJkFaSHLxdeH8= github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKSfZBKWWLh20= github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg= +github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 h1:jT6f/cdOpLkp9GAfRrxk57BUjYfIrR8E+AjMv5H5U4U= github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345/go.mod h1:clhQmPnt35ziJW1AhJRKyu8aygXCSoyWj6dtmZBRjjc= -github.com/projectdiscovery/cryptoutil v1.0.0 h1:5rQfnWDthJ5ZFcqze+rmT1N7l1HJQ6EB26MrjaYB7I0= -github.com/projectdiscovery/cryptoutil v1.0.0/go.mod h1:VJvSNE8f8A1MgpjgAL2GPJSQcJa4jbdaeQJstARFrU4= github.com/projectdiscovery/fastdialer v0.0.12/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/ieD4uda2JuUA4WJ+RLee0= github.com/projectdiscovery/fastdialer v0.0.15/go.mod h1:Q28lw9oTpiZHq09uFG6YYYLUsUjsOypZ7PXWwQGBB80= -github.com/projectdiscovery/fastdialer v0.0.17 h1:wT7LinZAcyEzkfhJLhWNQrA9m79+hTd6CJ7aWTJfXjk= -github.com/projectdiscovery/fastdialer v0.0.17/go.mod h1:poZbCGYGRfRcKxU8/1rWH8EVOJ4AZn1a8+CyINHzffQ= +github.com/projectdiscovery/fastdialer v0.0.18-0.20221102102120-8e9343e8b0e0 h1:1hcFBedqq8772PxN3Lbq7Itr3N59C8ro0xaTxYkmg9s= +github.com/projectdiscovery/fastdialer v0.0.18-0.20221102102120-8e9343e8b0e0/go.mod h1:KSHL57MbR0PbdJpagiqqB0jPqO1GUcnYZT5ngAvsmqQ= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw= github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08 h1:NwD1R/du1dqrRKN3SJl9kT6tN3K9puuWFXEvYF2ihew= github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08/go.mod h1:paLCnwV8sL7ppqIwVQodQrk3F6mnWafwTDwRd7ywZwQ= github.com/projectdiscovery/fileutil v0.0.0-20210914153648-31f843feaad4/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= -github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/fileutil v0.0.0-20220308101036-16c79af1cf5d/go.mod h1:Pm0f+MWgDFMSSI9NBedNh48LyYPs8gD3Jd8DXGmp4aQ= -github.com/projectdiscovery/fileutil v0.0.0-20220609150212-453ac591c36c/go.mod h1:g8wsrb0S5NtEN0JgVyyPeb3FQdArx+UMESmFX94bcGY= -github.com/projectdiscovery/fileutil v0.0.0-20220705195237-01becc2a8963/go.mod h1:DaY7wmLPMleyHDCD/14YApPCDtrARY4J8Eny2ZGsG/g= github.com/projectdiscovery/fileutil v0.0.3 h1:GSsoey4p8ZHIRxWF2VXh4mhLr+wfEkpJwvF0Dxpn/gg= github.com/projectdiscovery/fileutil v0.0.3/go.mod h1:GLejWd3YerG3RNYD/Hk2pJlytlYRgHdkWfWUAdCH2YQ= github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= -github.com/projectdiscovery/goflags v0.0.8/go.mod h1:GDSkWyXa6kfQjpJu10SO64DN8lXuKXVENlBMk8N7H80= github.com/projectdiscovery/goflags v0.1.3 h1:dnJlg19VkDp1iYkpAod4Tv+OAngr7Mq61LMMpBQlO0M= github.com/projectdiscovery/goflags v0.1.3/go.mod h1:/7ZAoY1SVfUcGobTP5QDvGQmrpPDDlBUDIMr7c+r94Q= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= @@ -633,14 +626,12 @@ github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+ github.com/projectdiscovery/iputil v0.0.0-20210414194613-4b4d2517acf0/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A= github.com/projectdiscovery/iputil v0.0.0-20210429152401-c18a5408ca46/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A= github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3/go.mod h1:blmYJkS8lSrrx3QcmcgS2tZIxlojeVmoGeA9twslCBU= -github.com/projectdiscovery/iputil v0.0.0-20220712175312-b9406f31cdd8/go.mod h1:vHRC+9exsfSbEngMKDl0xiWqkxlLk3lHQZpbS2yFT8U= github.com/projectdiscovery/iputil v0.0.2 h1:f6IGnZF4RImJLysPSPG3D84jyTH34q3lihCFeP+eZzI= github.com/projectdiscovery/iputil v0.0.2/go.mod h1:J3Pcz1q51pi4/JL871mQztg0KOzyWDPxnPLOYJm2pVQ= github.com/projectdiscovery/mapcidr v0.0.4/go.mod h1:ALOIj6ptkWujNoX8RdQwB2mZ+kAmKuLJBq9T5gR5wG0= github.com/projectdiscovery/mapcidr v0.0.6/go.mod h1:ZEBhMmBU3laUl3g9QGTrzJku1VJOzjdFwW01f/zVVzM= github.com/projectdiscovery/mapcidr v0.0.7/go.mod h1:7CzdUdjuLVI0s33dQ33lWgjg3vPuLFw2rQzZ0RxkT00= github.com/projectdiscovery/mapcidr v0.0.8/go.mod h1:7CzdUdjuLVI0s33dQ33lWgjg3vPuLFw2rQzZ0RxkT00= -github.com/projectdiscovery/mapcidr v1.0.1/go.mod h1:/qxlpxXZQFFjHynSc9u5O0kUPzH46VskECiwLiz7/vw= github.com/projectdiscovery/mapcidr v1.0.3 h1:SGtOOEz0AxthVO7ZonMvhrJ/AQkHIXCVgyZqJdY0cAY= github.com/projectdiscovery/mapcidr v1.0.3/go.mod h1:/0lEXlu/q0t5u34vIVF6odHR+JCdD3CIHNsMXo7nwrU= github.com/projectdiscovery/networkpolicy v0.0.1/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs= @@ -661,24 +652,18 @@ github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1: github.com/projectdiscovery/retryabledns v1.0.11/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4= github.com/projectdiscovery/retryabledns v1.0.12/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4= github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a/go.mod h1:tXaLDs4n3pRZHwfa8mdXpUWe/AYDNK3HlWDjldhRbjI= -github.com/projectdiscovery/retryabledns v1.0.15/go.mod h1:3YbsQVqP7jbQ3CDmarhyVtkJaJ8XcB7S19vMeyMxZxk= github.com/projectdiscovery/retryabledns v1.0.17 h1:XKzI26UKYt2g7YLJ/EcyYmM04sfD1vurETecPEpeA1w= github.com/projectdiscovery/retryabledns v1.0.17/go.mod h1:Dyhq/f0sGmXueso0+Ah3LbJfsX4PXpBrpfiyjZZ8SDk= github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek= github.com/projectdiscovery/retryablehttp-go v1.0.2/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI= github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d h1:VR+tDkedzHIp1pGKIDcfPFt7J8KjcjxGsJvBAP6RXFQ= github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d/go.mod h1:t4buiLTB0HtI+62iHfGDqQVTv/i+8OhAKwaX93TGsFE= -github.com/projectdiscovery/sliceutil v0.0.0-20220617151003-15892688e1d6/go.mod h1:9YZb6LRjLYAvSOm65v787dwauurixSyjlqXyYa4rTTA= -github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4/go.mod h1:RxDaccMjPzIuF7F8XbdGl1yOcqxN4YPiHr9xHpfCkGI= github.com/projectdiscovery/sliceutil v0.0.1 h1:YoCqCMcdwz+gqNfW5hFY8UvNHoA6SfyBSNkVahatleg= github.com/projectdiscovery/sliceutil v0.0.1/go.mod h1:0wBmhU5uTDwMfrEZfvwH9qa5k60Q4shPVOC9E6LGsDI= github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20210823090203-2f5f137e8e1d/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= -github.com/projectdiscovery/stringsutil v0.0.0-20220422150559-b54fb5dc6833/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= -github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3/go.mod h1:mF5sh4jTghoGWwgUb9qWi5waTFklClDbtrqtJU93awc= -github.com/projectdiscovery/stringsutil v0.0.0-20220731064040-4b67f194751e/go.mod h1:32NYmKyHkKsmisAOAaWrR15lz2ysz2M8x3KMeeoRHoU= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/projectdiscovery/tlsx v0.0.9 h1:wUC8GYUIo5jd+enqE1lnEJ3Ew7m+N6eRmFBjbSJLomU= @@ -762,8 +747,6 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= -github.com/stretchr/testify v1.7.3/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= @@ -993,17 +976,14 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210414194228-064579744ee0/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220630215102-69896b714898/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20220728211354-c7608f3a8462/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= @@ -1102,7 +1082,6 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220731174439-a90be440212d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/v2/internal/runner/enumerate.go b/v2/internal/runner/enumerate.go index 15e1bfb7..865a7700 100644 --- a/v2/internal/runner/enumerate.go +++ b/v2/internal/runner/enumerate.go @@ -19,6 +19,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/core" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "go.uber.org/atomic" ) @@ -84,8 +85,8 @@ func (r *Runner) runCloudEnumeration(store *loader.Store, nostore bool) (*atomic // TODO: Add payload file and workflow support for private templates catalogChecksums := nucleicloud.ReadCatalogChecksum() - targets := make([]string, 0, r.hmapInputProvider.Count()) - r.hmapInputProvider.Scan(func(value string) bool { + targets := make([]*contextargs.MetaInput, 0, r.hmapInputProvider.Count()) + r.hmapInputProvider.Scan(func(value *contextargs.MetaInput) bool { targets = append(targets, value) return true }) diff --git a/v2/internal/runner/inputs.go b/v2/internal/runner/inputs.go index 2f3a8fda..ef646755 100644 --- a/v2/internal/runner/inputs.go +++ b/v2/internal/runner/inputs.go @@ -9,6 +9,7 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/hmap/store/hybrid" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool" "github.com/projectdiscovery/retryablehttp-go" "github.com/remeh/sizedwaitgroup" @@ -37,18 +38,18 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) { // Probe the non-standard URLs and store them in cache swg := sizedwaitgroup.New(bulkSize) count := int32(0) - r.hmapInputProvider.Scan(func(value string) bool { - if strings.HasPrefix(value, "http://") || strings.HasPrefix(value, "https://") { + r.hmapInputProvider.Scan(func(value *contextargs.MetaInput) bool { + if strings.HasPrefix(value.Input, "http://") || strings.HasPrefix(value.Input, "https://") { return true } swg.Add() - go func(input string) { + go func(input *contextargs.MetaInput) { defer swg.Done() - if result := probeURL(input, httpclient); result != "" { + if result := probeURL(input.Input, httpclient); result != "" { atomic.AddInt32(&count, 1) - _ = hm.Set(input, []byte(result)) + _ = hm.Set(input.Input, []byte(result)) } }(value) return true diff --git a/v2/internal/runner/nucleicloud/types.go b/v2/internal/runner/nucleicloud/types.go index c746fe0b..9510f966 100644 --- a/v2/internal/runner/nucleicloud/types.go +++ b/v2/internal/runner/nucleicloud/types.go @@ -1,11 +1,15 @@ package nucleicloud -import "time" +import ( + "time" + + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" +) // AddScanRequest is a nuclei scan input item. type AddScanRequest struct { // RawTargets is a list of raw target URLs for the scan. - RawTargets []string `json:"raw_targets,omitempty"` + RawTargets []*contextargs.MetaInput `json:"raw_targets,omitempty"` // PublicTemplates is a list of public templates for the scan PublicTemplates []string `json:"public_templates,omitempty"` // PrivateTemplates is a map of template-name->contents that diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 04ef4d12..b1a1fc03 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -2,6 +2,7 @@ package runner import ( "bufio" + "fmt" "io" "log" "os" @@ -135,6 +136,23 @@ func validateOptions(options *types.Options) error { } validateCertificatePaths([]string{options.ClientCertFile, options.ClientKeyFile, options.ClientCAFile}) } + + // verify that a valid ip version type was selected (4, 6) + var useIPV4, useIPV6 bool + for _, ipv := range options.IPVersion { + switch ipv { + case "4": + useIPV4 = true + case "6": + useIPV6 = true + default: + return fmt.Errorf("unsupported ip version: %s", ipv) + } + } + if !useIPV4 && !useIPV6 { + return errors.New("ipv4 and/or ipv6 must be selected") + } + return nil } diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 1220adff..13f585d4 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -37,6 +37,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/projectfile" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/automaticscan" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit" @@ -478,8 +479,8 @@ func (r *Runner) RunEnumeration() error { func (r *Runner) isInputNonHTTP() bool { var nonURLInput bool - r.hmapInputProvider.Scan(func(value string) bool { - if !strings.Contains(value, "://") { + r.hmapInputProvider.Scan(func(value *contextargs.MetaInput) bool { + if !strings.Contains(value.Input, "://") { nonURLInput = true return false } diff --git a/v2/pkg/core/engine.go b/v2/pkg/core/engine.go index d9a4f724..16ad68d7 100644 --- a/v2/pkg/core/engine.go +++ b/v2/pkg/core/engine.go @@ -2,6 +2,7 @@ package core import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) @@ -29,7 +30,7 @@ type InputProvider interface { Count() int64 // Scan iterates the input and each found item is passed to the // callback consumer. - Scan(callback func(value string) bool) + Scan(callback func(value *contextargs.MetaInput) bool) } // New returns a new Engine instance diff --git a/v2/pkg/core/execute.go b/v2/pkg/core/execute.go index 99f30a80..073fd37a 100644 --- a/v2/pkg/core/execute.go +++ b/v2/pkg/core/execute.go @@ -43,6 +43,8 @@ func (e *Engine) ExecuteWithOpts(templatesList []*templates.Template, target Inp wg.Add() go func(tpl *templates.Template) { + defer wg.Done() + switch { case tpl.SelfContained: // Self Contained requests are executed here separately @@ -51,7 +53,6 @@ func (e *Engine) ExecuteWithOpts(templatesList []*templates.Template, target Inp // All other request types are executed here e.executeModelWithInput(templateType, tpl, target, results) } - wg.Done() }(template) } e.workPool.Wait() @@ -98,7 +99,7 @@ func (e *Engine) executeModelWithInput(templateType types.ProtocolType, template currentInfo.Unlock() } - target.Scan(func(scannedValue string) bool { + target.Scan(func(scannedValue *contextargs.MetaInput) bool { // Best effort to track the host progression // skips indexes lower than the minimum in-flight at interruption time var skip bool @@ -122,12 +123,12 @@ func (e *Engine) executeModelWithInput(templateType types.ProtocolType, template currentInfo.Unlock() // Skip if the host has had errors - if e.executerOpts.HostErrorsCache != nil && e.executerOpts.HostErrorsCache.Check(scannedValue) { + if e.executerOpts.HostErrorsCache != nil && e.executerOpts.HostErrorsCache.Check(scannedValue.ID()) { return true } wg.WaitGroup.Add() - go func(index uint32, skip bool, value string) { + go func(index uint32, skip bool, value *contextargs.MetaInput) { defer wg.WaitGroup.Done() defer cleanupInFlight(index) if skip { @@ -141,7 +142,7 @@ func (e *Engine) executeModelWithInput(templateType types.ProtocolType, template match = e.executeWorkflow(value, template.CompiledWorkflow) default: ctxArgs := contextargs.New() - ctxArgs.Input = value + ctxArgs.MetaInput = value match, err = template.Executer.Execute(ctxArgs) } if err != nil { @@ -188,14 +189,14 @@ func (e *Engine) ExecuteWithResults(templatesList []*templates.Template, target func (e *Engine) executeModelWithInputAndResult(templateType types.ProtocolType, template *templates.Template, target InputProvider, results *atomic.Bool, callback func(*output.ResultEvent)) { wg := e.workPool.InputPool(templateType) - target.Scan(func(scannedValue string) bool { + target.Scan(func(scannedValue *contextargs.MetaInput) bool { // Skip if the host has had errors - if e.executerOpts.HostErrorsCache != nil && e.executerOpts.HostErrorsCache.Check(scannedValue) { + if e.executerOpts.HostErrorsCache != nil && e.executerOpts.HostErrorsCache.Check(scannedValue.ID()) { return true } wg.WaitGroup.Add() - go func(value string) { + go func(value *contextargs.MetaInput) { defer wg.WaitGroup.Done() var match bool @@ -205,7 +206,7 @@ func (e *Engine) executeModelWithInputAndResult(templateType types.ProtocolType, match = e.executeWorkflow(value, template.CompiledWorkflow) default: ctxArgs := contextargs.New() - ctxArgs.Input = value + ctxArgs.MetaInput = value err = template.Executer.ExecuteWithResults(ctxArgs, func(event *output.InternalWrappedEvent) { for _, result := range event.Results { callback(result) @@ -235,7 +236,7 @@ func (e *ChildExecuter) Close() *atomic.Bool { } // Execute executes a template and URLs -func (e *ChildExecuter) Execute(template *templates.Template, URL string) { +func (e *ChildExecuter) Execute(template *templates.Template, value *contextargs.MetaInput) { templateType := template.Type() var wg *sizedwaitgroup.SizedWaitGroup @@ -247,14 +248,15 @@ func (e *ChildExecuter) Execute(template *templates.Template, URL string) { wg.Add() go func(tpl *templates.Template) { + defer wg.Done() + ctxArgs := contextargs.New() - ctxArgs.Input = URL + ctxArgs.MetaInput = value match, err := template.Executer.Execute(ctxArgs) if err != nil { gologger.Warning().Msgf("[%s] Could not execute step: %s\n", e.e.executerOpts.Colorizer.BrightBlue(template.ID), err) } e.results.CompareAndSwap(false, match) - wg.Done() }(template) } diff --git a/v2/pkg/core/inputs/hybrid/hmap.go b/v2/pkg/core/inputs/hybrid/hmap.go index a55bdde5..c90d4ba3 100644 --- a/v2/pkg/core/inputs/hybrid/hmap.go +++ b/v2/pkg/core/inputs/hybrid/hmap.go @@ -5,6 +5,7 @@ package hybrid import ( "bufio" "io" + "net/url" "os" "strings" "time" @@ -16,13 +17,17 @@ import ( "github.com/projectdiscovery/hmap/store/hybrid" "github.com/projectdiscovery/mapcidr" asn "github.com/projectdiscovery/mapcidr/asn" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v2/pkg/types" fileutil "github.com/projectdiscovery/utils/file" iputil "github.com/projectdiscovery/utils/ip" + sliceutil "github.com/projectdiscovery/utils/slice" ) // Input is a hmap/filekv backed nuclei Input provider type Input struct { + ipOptions *ipOptions inputCount int64 dupeCount int64 hostMap *hybrid.HybridMap @@ -37,7 +42,14 @@ func New(options *types.Options) (*Input, error) { return nil, errors.Wrap(err, "could not create temporary input file") } - input := &Input{hostMap: hm} + input := &Input{ + hostMap: hm, + ipOptions: &ipOptions{ + ScanAllIPs: options.ScanAllIPs, + IPV4: sliceutil.Contains(options.IPVersion, "4"), + IPV6: sliceutil.Contains(options.IPVersion, "6"), + }, + } if options.Stream { fkvOptions := filekv.DefaultOptions if tmpFileName, err := fileutil.GetTempFileName(); err != nil { @@ -72,15 +84,14 @@ func (i *Input) Close() { func (i *Input) initializeInputSources(options *types.Options) error { // Handle targets flags for _, target := range options.Targets { - if iputil.IsCIDR(target) { + switch { + case iputil.IsCIDR(target): i.expandCIDRInputValue(target) - continue - } - if asn.IsASN(target) { + case asn.IsASN(target): i.expandASNInputValue(target) - continue + default: + i.normalizeStoreInputValue(target) } - i.normalizeStoreInputValue(target) } // Handle stdin @@ -94,8 +105,9 @@ func (i *Input) initializeInputSources(options *types.Options) error { if inputErr != nil { return errors.Wrap(inputErr, "could not open targets file") } + defer input.Close() + i.scanInputFromReader(input) - input.Close() } return nil } @@ -104,34 +116,89 @@ func (i *Input) initializeInputSources(options *types.Options) error { func (i *Input) scanInputFromReader(reader io.Reader) { scanner := bufio.NewScanner(reader) for scanner.Scan() { - if iputil.IsCIDR(scanner.Text()) { - i.expandCIDRInputValue(scanner.Text()) - continue + item := scanner.Text() + switch { + case iputil.IsCIDR(item): + i.expandCIDRInputValue(item) + case asn.IsASN(item): + i.expandASNInputValue(item) + default: + i.normalizeStoreInputValue(item) } - if asn.IsASN(scanner.Text()) { - i.expandASNInputValue(scanner.Text()) - continue - } - i.normalizeStoreInputValue(scanner.Text()) } } // normalizeStoreInputValue normalizes and stores passed input values func (i *Input) normalizeStoreInputValue(value string) { - url := strings.TrimSpace(value) - if url == "" { + URL := strings.TrimSpace(value) + if URL == "" { return } - if _, ok := i.hostMap.Get(url); ok { + metaInput := &contextargs.MetaInput{Input: URL} + keyURL, err := metaInput.MarshalString() + if err != nil { + gologger.Warning().Msgf("%s\n", err) + return + } + + if _, ok := i.hostMap.Get(keyURL); ok { i.dupeCount++ return } + switch { + case i.ipOptions.ScanAllIPs: + // we need to resolve the hostname + // check if it's an url + var host string + parsedURL, err := url.Parse(value) + if err == nil && parsedURL.Host != "" { + host = parsedURL.Host + } else { + parsedURL = nil + host = value + } + + dnsData, err := protocolstate.Dialer.GetDNSData(host) + if err == nil && (len(dnsData.A)+len(dnsData.AAAA)) > 0 { + var ips []string + if i.ipOptions.IPV4 { + ips = append(ips, dnsData.A...) + } + if i.ipOptions.IPV6 { + ips = append(ips, dnsData.AAAA...) + } + + for _, ip := range ips { + if ip == "" { + continue + } + metaInput := &contextargs.MetaInput{Input: value, CustomIP: ip} + key, err := metaInput.MarshalString() + if err != nil { + gologger.Warning().Msgf("%s\n", err) + continue + } + _ = i.hostMap.Set(key, nil) + if i.hostMapStream != nil { + _ = i.hostMapStream.Set([]byte(key), nil) + } + } + break + } + fallthrough + default: + i.setItem(keyURL) + } +} + +// setItem in the kv store +func (i *Input) setItem(k string) { i.inputCount++ - _ = i.hostMap.Set(url, nil) + _ = i.hostMap.Set(k, nil) if i.hostMapStream != nil { - _ = i.hostMapStream.Set([]byte(url), nil) + _ = i.hostMapStream.Set([]byte(k), nil) } } @@ -142,9 +209,13 @@ func (i *Input) Count() int64 { // Scan iterates the input and each found item is passed to the // callback consumer. -func (i *Input) Scan(callback func(value string) bool) { +func (i *Input) Scan(callback func(value *contextargs.MetaInput) bool) { callbackFunc := func(k, _ []byte) error { - if !callback(string(k)) { + metaInput := &contextargs.MetaInput{} + if err := metaInput.Unmarshal(string(k)); err != nil { + return err + } + if !callback(metaInput) { return io.EOF } return nil @@ -160,14 +231,20 @@ func (i *Input) Scan(callback func(value string) bool) { func (i *Input) expandCIDRInputValue(value string) { ips, _ := mapcidr.IPAddressesAsStream(value) for ip := range ips { - if _, ok := i.hostMap.Get(ip); ok { + metaInput := &contextargs.MetaInput{Input: ip} + key, err := metaInput.MarshalString() + if err != nil { + gologger.Warning().Msgf("%s\n", err) + return + } + if _, ok := i.hostMap.Get(key); ok { i.dupeCount++ continue } i.inputCount++ - _ = i.hostMap.Set(ip, nil) + _ = i.hostMap.Set(key, nil) if i.hostMapStream != nil { - _ = i.hostMapStream.Set([]byte(ip), nil) + _ = i.hostMapStream.Set([]byte(key), nil) } } } diff --git a/v2/pkg/core/inputs/hybrid/hmap_test.go b/v2/pkg/core/inputs/hybrid/hmap_test.go index 5642f85a..5b08abb0 100644 --- a/v2/pkg/core/inputs/hybrid/hmap_test.go +++ b/v2/pkg/core/inputs/hybrid/hmap_test.go @@ -6,6 +6,9 @@ import ( "testing" "github.com/projectdiscovery/hmap/store/hybrid" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" + "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/stretchr/testify/require" ) @@ -30,11 +33,73 @@ func Test_expandCIDRInputValue(t *testing.T) { input.expandCIDRInputValue(tt.cidr) // scan got := []string{} - input.hostMap.Scan(func(k, v []byte) error { - got = append(got, string(k)) + input.hostMap.Scan(func(k, _ []byte) error { + var metainput contextargs.MetaInput + if err := metainput.Unmarshal(string(k)); err != nil { + return err + } + got = append(got, metainput.Input) return nil }) require.ElementsMatch(t, tt.expected, got, "could not get correct cidrs") + input.Close() + } +} + +func Test_scanallips_normalizeStoreInputValue(t *testing.T) { + defaultOpts := types.DefaultOptions() + _ = protocolstate.Init(defaultOpts) + tests := []struct { + hostname string + ipv4 bool + ipv6 bool + expected []string + }{ + { + hostname: "scanme.sh", + ipv4: true, + ipv6: true, + expected: []string{"128.199.158.128", "2400:6180:0:d0::91:1001"}, + }, { + hostname: "scanme.sh", + ipv4: true, + expected: []string{"128.199.158.128"}, + }, { + hostname: "scanme.sh", + ipv6: true, + expected: []string{"2400:6180:0:d0::91:1001"}, + }, { + hostname: "http://scanme.sh", + ipv4: true, + ipv6: true, + expected: []string{"128.199.158.128", "2400:6180:0:d0::91:1001"}, + }, + } + for _, tt := range tests { + hm, err := hybrid.New(hybrid.DefaultDiskOptions) + require.Nil(t, err, "could not create temporary input file") + input := &Input{ + hostMap: hm, + ipOptions: &ipOptions{ + ScanAllIPs: true, + IPV4: tt.ipv4, + IPV6: tt.ipv6, + }, + } + + input.normalizeStoreInputValue(tt.hostname) + // scan + got := []string{} + input.hostMap.Scan(func(k, v []byte) error { + var metainput contextargs.MetaInput + if err := metainput.Unmarshal(string(k)); err != nil { + return err + } + got = append(got, metainput.CustomIP) + return nil + }) + require.ElementsMatch(t, tt.expected, got, "could not get correct ips") + input.Close() } } @@ -61,7 +126,11 @@ func Test_expandASNInputValue(t *testing.T) { // scan the hmap got := []string{} input.hostMap.Scan(func(k, v []byte) error { - got = append(got, string(k)) + var metainput contextargs.MetaInput + if err := metainput.Unmarshal(string(k)); err != nil { + return err + } + got = append(got, metainput.Input) return nil }) // read the expected IPs from the file diff --git a/v2/pkg/core/inputs/hybrid/options.go b/v2/pkg/core/inputs/hybrid/options.go new file mode 100644 index 00000000..8c7f13af --- /dev/null +++ b/v2/pkg/core/inputs/hybrid/options.go @@ -0,0 +1,7 @@ +package hybrid + +type ipOptions struct { + ScanAllIPs bool + IPV4 bool + IPV6 bool +} diff --git a/v2/pkg/core/inputs/inputs.go b/v2/pkg/core/inputs/inputs.go index bf46d4f5..d7bcadb5 100644 --- a/v2/pkg/core/inputs/inputs.go +++ b/v2/pkg/core/inputs/inputs.go @@ -1,7 +1,9 @@ package inputs +import "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" + type SimpleInputProvider struct { - Inputs []string + Inputs []*contextargs.MetaInput } // Count returns the number of items for input provider @@ -10,7 +12,7 @@ func (s *SimpleInputProvider) Count() int64 { } // Scan calls a callback function till the input provider is exhausted -func (s *SimpleInputProvider) Scan(callback func(value string) bool) { +func (s *SimpleInputProvider) Scan(callback func(value *contextargs.MetaInput) bool) { for _, v := range s.Inputs { if !callback(v) { return diff --git a/v2/pkg/core/workflow_execute.go b/v2/pkg/core/workflow_execute.go index 99918913..f1aea480 100644 --- a/v2/pkg/core/workflow_execute.go +++ b/v2/pkg/core/workflow_execute.go @@ -16,13 +16,13 @@ import ( const workflowStepExecutionError = "[%s] Could not execute workflow step: %s\n" // executeWorkflow runs a workflow on an input and returns true or false -func (e *Engine) executeWorkflow(input string, w *workflows.Workflow) bool { +func (e *Engine) executeWorkflow(input *contextargs.MetaInput, w *workflows.Workflow) bool { results := &atomic.Bool{} // at this point we should be at the start root execution of a workflow tree, hence we create global shared instances workflowCookieJar, _ := cookiejar.New(nil) ctxArgs := contextargs.New() - ctxArgs.Input = input + ctxArgs.MetaInput = input ctxArgs.CookieJar = workflowCookieJar swg := sizedwaitgroup.New(w.Options.Options.TemplateThreads) @@ -88,7 +88,7 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, input *co } if err != nil { if w.Options.HostErrorsCache != nil { - w.Options.HostErrorsCache.MarkFailed(input.Input, err) + w.Options.HostErrorsCache.MarkFailed(input.MetaInput.ID(), err) } if len(template.Executers) == 1 { mainErr = err diff --git a/v2/pkg/core/workflow_execute_test.go b/v2/pkg/core/workflow_execute_test.go index da911ab8..b8858ff2 100644 --- a/v2/pkg/core/workflow_execute_test.go +++ b/v2/pkg/core/workflow_execute_test.go @@ -25,7 +25,7 @@ func TestWorkflowsSimple(t *testing.T) { }} engine := &Engine{} - matched := engine.executeWorkflow("https://test.com", workflow) + matched := engine.executeWorkflow(&contextargs.MetaInput{Input: "https://test.com"}, workflow) require.True(t, matched, "could not get correct match value") } @@ -35,19 +35,19 @@ func TestWorkflowsSimpleMultiple(t *testing.T) { var firstInput, secondInput string workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{ {Executers: []*workflows.ProtocolExecuterPair{{ - Executer: &mockExecuter{result: true, executeHook: func(input string) { - firstInput = input + Executer: &mockExecuter{result: true, executeHook: func(input *contextargs.MetaInput) { + firstInput = input.Input }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, }}, {Executers: []*workflows.ProtocolExecuterPair{{ - Executer: &mockExecuter{result: true, executeHook: func(input string) { - secondInput = input + Executer: &mockExecuter{result: true, executeHook: func(input *contextargs.MetaInput) { + secondInput = input.Input }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, }}, }} engine := &Engine{} - matched := engine.executeWorkflow("https://test.com", workflow) + matched := engine.executeWorkflow(&contextargs.MetaInput{Input: "https://test.com"}, workflow) require.True(t, matched, "could not get correct match value") require.Equal(t, "https://test.com", firstInput, "could not get correct first input") @@ -60,20 +60,20 @@ func TestWorkflowsSubtemplates(t *testing.T) { var firstInput, secondInput string workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{ {Executers: []*workflows.ProtocolExecuterPair{{ - Executer: &mockExecuter{result: true, executeHook: func(input string) { - firstInput = input + Executer: &mockExecuter{result: true, executeHook: func(input *contextargs.MetaInput) { + firstInput = input.Input }, outputs: []*output.InternalWrappedEvent{ {OperatorsResult: &operators.Result{}, Results: []*output.ResultEvent{{}}}, }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, }, Subtemplates: []*workflows.WorkflowTemplate{{Executers: []*workflows.ProtocolExecuterPair{{ - Executer: &mockExecuter{result: true, executeHook: func(input string) { - secondInput = input + Executer: &mockExecuter{result: true, executeHook: func(input *contextargs.MetaInput) { + secondInput = input.Input }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, }}}}, }} engine := &Engine{} - matched := engine.executeWorkflow("https://test.com", workflow) + matched := engine.executeWorkflow(&contextargs.MetaInput{Input: "https://test.com"}, workflow) require.True(t, matched, "could not get correct match value") require.Equal(t, "https://test.com", firstInput, "could not get correct first input") @@ -86,18 +86,18 @@ func TestWorkflowsSubtemplatesNoMatch(t *testing.T) { var firstInput, secondInput string workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{ {Executers: []*workflows.ProtocolExecuterPair{{ - Executer: &mockExecuter{result: false, executeHook: func(input string) { - firstInput = input + Executer: &mockExecuter{result: false, executeHook: func(input *contextargs.MetaInput) { + firstInput = input.Input }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, }, Subtemplates: []*workflows.WorkflowTemplate{{Executers: []*workflows.ProtocolExecuterPair{{ - Executer: &mockExecuter{result: true, executeHook: func(input string) { - secondInput = input + Executer: &mockExecuter{result: true, executeHook: func(input *contextargs.MetaInput) { + secondInput = input.Input }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, }}}}, }} engine := &Engine{} - matched := engine.executeWorkflow("https://test.com", workflow) + matched := engine.executeWorkflow(&contextargs.MetaInput{Input: "https://test.com"}, workflow) require.False(t, matched, "could not get correct match value") require.Equal(t, "https://test.com", firstInput, "could not get correct first input") @@ -110,8 +110,8 @@ func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) { var firstInput, secondInput string workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{ {Executers: []*workflows.ProtocolExecuterPair{{ - Executer: &mockExecuter{result: true, executeHook: func(input string) { - firstInput = input + Executer: &mockExecuter{result: true, executeHook: func(input *contextargs.MetaInput) { + firstInput = input.Input }, outputs: []*output.InternalWrappedEvent{ {OperatorsResult: &operators.Result{ Matches: map[string][]string{"tomcat": {}}, @@ -119,14 +119,14 @@ func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) { }}, }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, }, Matchers: []*workflows.Matcher{{Name: stringslice.StringSlice{Value: "tomcat"}, Subtemplates: []*workflows.WorkflowTemplate{{Executers: []*workflows.ProtocolExecuterPair{{ - Executer: &mockExecuter{result: true, executeHook: func(input string) { - secondInput = input + Executer: &mockExecuter{result: true, executeHook: func(input *contextargs.MetaInput) { + secondInput = input.Input }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, }}}}}}, }} engine := &Engine{} - matched := engine.executeWorkflow("https://test.com", workflow) + matched := engine.executeWorkflow(&contextargs.MetaInput{Input: "https://test.com"}, workflow) require.True(t, matched, "could not get correct match value") require.Equal(t, "https://test.com", firstInput, "could not get correct first input") @@ -139,8 +139,8 @@ func TestWorkflowsSubtemplatesWithMatcherNoMatch(t *testing.T) { var firstInput, secondInput string workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{ {Executers: []*workflows.ProtocolExecuterPair{{ - Executer: &mockExecuter{result: true, executeHook: func(input string) { - firstInput = input + Executer: &mockExecuter{result: true, executeHook: func(input *contextargs.MetaInput) { + firstInput = input.Input }, outputs: []*output.InternalWrappedEvent{ {OperatorsResult: &operators.Result{ Matches: map[string][]string{"tomcat": {}}, @@ -148,14 +148,14 @@ func TestWorkflowsSubtemplatesWithMatcherNoMatch(t *testing.T) { }}, }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, }, Matchers: []*workflows.Matcher{{Name: stringslice.StringSlice{Value: "apache"}, Subtemplates: []*workflows.WorkflowTemplate{{Executers: []*workflows.ProtocolExecuterPair{{ - Executer: &mockExecuter{result: true, executeHook: func(input string) { - secondInput = input + Executer: &mockExecuter{result: true, executeHook: func(input *contextargs.MetaInput) { + secondInput = input.Input }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, }}}}}}, }} engine := &Engine{} - matched := engine.executeWorkflow("https://test.com", workflow) + matched := engine.executeWorkflow(&contextargs.MetaInput{Input: "https://test.com"}, workflow) require.False(t, matched, "could not get correct match value") require.Equal(t, "https://test.com", firstInput, "could not get correct first input") @@ -164,7 +164,7 @@ func TestWorkflowsSubtemplatesWithMatcherNoMatch(t *testing.T) { type mockExecuter struct { result bool - executeHook func(input string) + executeHook func(input *contextargs.MetaInput) outputs []*output.InternalWrappedEvent } @@ -181,7 +181,7 @@ func (m *mockExecuter) Requests() int { // Execute executes the protocol group and returns true or false if results were found. func (m *mockExecuter) Execute(input *contextargs.Context) (bool, error) { if m.executeHook != nil { - m.executeHook(input.Input) + m.executeHook(input.MetaInput) } return m.result, nil } @@ -189,7 +189,7 @@ func (m *mockExecuter) Execute(input *contextargs.Context) (bool, error) { // ExecuteWithResults executes the protocol requests and returns results instead of writing them. func (m *mockExecuter) ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error { if m.executeHook != nil { - m.executeHook(input.Input) + m.executeHook(input.MetaInput) } for _, output := range m.outputs { callback(output) diff --git a/v2/pkg/protocols/common/automaticscan/automaticscan.go b/v2/pkg/protocols/common/automaticscan/automaticscan.go index b80956f8..10bd7fe8 100644 --- a/v2/pkg/protocols/common/automaticscan/automaticscan.go +++ b/v2/pkg/protocols/common/automaticscan/automaticscan.go @@ -14,6 +14,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader" "github.com/projectdiscovery/nuclei/v2/pkg/core" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool" "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" @@ -132,11 +133,12 @@ func (s *Service) executeWappalyzerTechDetection() error { // Iterate through each target making http request and identifying fingerprints inputPool := s.engine.WorkPool().InputPool(types.HTTPProtocol) - s.target.Scan(func(value string) bool { + s.target.Scan(func(value *contextargs.MetaInput) bool { inputPool.WaitGroup.Add() - go func(input string) { + go func(input *contextargs.MetaInput) { defer inputPool.WaitGroup.Done() + s.processWappalyzerInputPair(input) }(value) return true @@ -145,8 +147,8 @@ func (s *Service) executeWappalyzerTechDetection() error { return nil } -func (s *Service) processWappalyzerInputPair(input string) { - req, err := retryablehttp.NewRequest(http.MethodGet, input, nil) +func (s *Service) processWappalyzerInputPair(input *contextargs.MetaInput) { + req, err := retryablehttp.NewRequest(http.MethodGet, input.Input, nil) if err != nil { return } diff --git a/v2/pkg/protocols/common/contextargs/contextargs.go b/v2/pkg/protocols/common/contextargs/contextargs.go index 863e8cd0..f847ca11 100644 --- a/v2/pkg/protocols/common/contextargs/contextargs.go +++ b/v2/pkg/protocols/common/contextargs/contextargs.go @@ -9,8 +9,9 @@ import ( // Context implements a shared context struct to share information across multiple templates within a workflow type Context struct { - // Input target for the executor - Input string + // Meta is the target for the executor + MetaInput *MetaInput + // CookieJar shared within workflow's http templates CookieJar *cookiejar.Jar @@ -22,12 +23,12 @@ type Context struct { // Create a new contextargs instance func New() *Context { - return &Context{} + return &Context{MetaInput: &MetaInput{}} } // Create a new contextargs instance with input string func NewWithInput(input string) *Context { - return &Context{Input: input} + return &Context{MetaInput: &MetaInput{Input: input}} } func (ctx *Context) initialize() { @@ -107,3 +108,13 @@ func (ctx *Context) Has(key string) bool { func (ctx *Context) HasArgs() bool { return ctx.hasArgs() } + +func (ctx *Context) Clone() *Context { + newCtx := &Context{ + MetaInput: ctx.MetaInput.Clone(), + RWMutex: ctx.RWMutex, + args: ctx.args, + CookieJar: ctx.CookieJar, + } + return newCtx +} diff --git a/v2/pkg/protocols/common/contextargs/metainput.go b/v2/pkg/protocols/common/contextargs/metainput.go new file mode 100644 index 00000000..f48bf7dc --- /dev/null +++ b/v2/pkg/protocols/common/contextargs/metainput.go @@ -0,0 +1,69 @@ +package contextargs + +import ( + "bytes" + "fmt" + "strings" + + jsoniter "github.com/json-iterator/go" +) + +// MetaInput represents a target with metadata (TODO: replace with https://github.com/projectdiscovery/metainput) +type MetaInput struct { + // Input represent the target + Input string + // CustomIP to use for connection + CustomIP string +} + +func (metaInput *MetaInput) marshalToBuffer() (bytes.Buffer, error) { + var b bytes.Buffer + err := jsoniter.NewEncoder(&b).Encode(metaInput) + return b, err +} + +// ID returns a unique id/hash for metainput +func (metaInput *MetaInput) ID() string { + if metaInput.CustomIP != "" { + return fmt.Sprintf("%s-%s", metaInput.Input, metaInput.CustomIP) + } + return metaInput.Input +} + +func (metaInput *MetaInput) MarshalString() (string, error) { + b, err := metaInput.marshalToBuffer() + return b.String(), err +} + +func (metaInput *MetaInput) MustMarshalString() string { + marshaled, _ := metaInput.MarshalString() + return marshaled +} + +func (metaInput *MetaInput) MarshalBytes() ([]byte, error) { + b, err := metaInput.marshalToBuffer() + return b.Bytes(), err +} + +func (metaInput *MetaInput) MustMarshalBytes() []byte { + marshaled, _ := metaInput.MarshalBytes() + return marshaled +} + +func (metaInput *MetaInput) Unmarshal(data string) error { + return jsoniter.NewDecoder(strings.NewReader(data)).Decode(metaInput) +} + +func (metaInput *MetaInput) Clone() *MetaInput { + return &MetaInput{ + Input: metaInput.Input, + CustomIP: metaInput.CustomIP, + } +} + +func (metaInput *MetaInput) PrettyPrint() string { + if metaInput.CustomIP != "" { + return fmt.Sprintf("%s [%s]", metaInput.Input, metaInput.CustomIP) + } + return metaInput.Input +} diff --git a/v2/pkg/protocols/common/executer/executer.go b/v2/pkg/protocols/common/executer/executer.go index 1d784bbb..d0307cdb 100644 --- a/v2/pkg/protocols/common/executer/executer.go +++ b/v2/pkg/protocols/common/executer/executer.go @@ -70,14 +70,14 @@ func (e *Executer) Execute(input *contextargs.Context) (bool, error) { } previous := make(map[string]interface{}) for _, req := range e.requests { - inputItem := *input - if e.options.InputHelper != nil && input.Input != "" { - if inputItem.Input = e.options.InputHelper.Transform(input.Input, req.Type()); inputItem.Input == "" { + inputItem := input.Clone() + if e.options.InputHelper != nil && input.MetaInput.Input != "" { + if inputItem.MetaInput.Input = e.options.InputHelper.Transform(input.MetaInput.Input, req.Type()); input.MetaInput.Input == "" { return false, nil } } - err := req.ExecuteWithResults(&inputItem, dynamicValues, previous, func(event *output.InternalWrappedEvent) { + err := req.ExecuteWithResults(inputItem, dynamicValues, previous, func(event *output.InternalWrappedEvent) { ID := req.GetID() if ID != "" { builder := &strings.Builder{} @@ -108,9 +108,9 @@ func (e *Executer) Execute(input *contextargs.Context) (bool, error) { }) if err != nil { if e.options.HostErrorsCache != nil { - e.options.HostErrorsCache.MarkFailed(input.Input, err) + e.options.HostErrorsCache.MarkFailed(input.MetaInput.ID(), err) } - gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input.Input, err) + gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input.MetaInput.PrettyPrint(), err) } // If a match was found and stop at first match is set, break out of the loop and return if results && (e.options.StopAtFirstMatch || e.options.Options.StopAtFirstMatch) { @@ -134,14 +134,14 @@ func (e *Executer) ExecuteWithResults(input *contextargs.Context, callback proto for _, req := range e.requests { req := req - inputItem := *input - if e.options.InputHelper != nil && input.Input != "" { - if inputItem.Input = e.options.InputHelper.Transform(input.Input, req.Type()); inputItem.Input == "" { + inputItem := input.Clone() + if e.options.InputHelper != nil && input.MetaInput.Input != "" { + if inputItem.MetaInput.Input = e.options.InputHelper.Transform(input.MetaInput.Input, req.Type()); input.MetaInput.Input == "" { return nil } } - err := req.ExecuteWithResults(&inputItem, dynamicValues, previous, func(event *output.InternalWrappedEvent) { + err := req.ExecuteWithResults(inputItem, dynamicValues, previous, func(event *output.InternalWrappedEvent) { ID := req.GetID() if ID != "" { builder := &strings.Builder{} @@ -161,9 +161,9 @@ func (e *Executer) ExecuteWithResults(input *contextargs.Context, callback proto }) if err != nil { if e.options.HostErrorsCache != nil { - e.options.HostErrorsCache.MarkFailed(input.Input, err) + e.options.HostErrorsCache.MarkFailed(input.MetaInput.ID(), err) } - gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input.Input, err) + gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input.MetaInput.PrettyPrint(), err) } // If a match was found and stop at first match is set, break out of the loop and return if results && (e.options.StopAtFirstMatch || e.options.Options.StopAtFirstMatch) { diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index f59c8a0e..a3060dcf 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -34,10 +34,10 @@ func (request *Request) Type() templateTypes.ProtocolType { func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error { // Parse the URL and return domain if URL. var domain string - if utils.IsURL(input.Input) { - domain = extractDomain(input.Input) + if utils.IsURL(input.MetaInput.Input) { + domain = extractDomain(input.MetaInput.Input) } else { - domain = input.Input + domain = input.MetaInput.Input } var err error @@ -112,7 +112,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, } } - outputEvent := request.responseToDSLMap(compiledRequest, response, input.Input, input.Input, traceData) + outputEvent := request.responseToDSLMap(compiledRequest, response, input.MetaInput.Input, input.MetaInput.Input, traceData) for k, v := range previous { outputEvent[k] = v } diff --git a/v2/pkg/protocols/dns/request_test.go b/v2/pkg/protocols/dns/request_test.go index 9ddabe71..3d602542 100644 --- a/v2/pkg/protocols/dns/request_test.go +++ b/v2/pkg/protocols/dns/request_test.go @@ -54,8 +54,7 @@ func TestDNSExecuteWithResults(t *testing.T) { t.Run("domain-valid", func(t *testing.T) { metadata := make(output.InternalEvent) previous := make(output.InternalEvent) - ctxArgs := contextargs.New() - ctxArgs.Input = "example.com" + ctxArgs := contextargs.NewWithInput("example.com") err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) { finalEvent = event }) diff --git a/v2/pkg/protocols/file/request.go b/v2/pkg/protocols/file/request.go index 4653fa44..1bf30d60 100644 --- a/v2/pkg/protocols/file/request.go +++ b/v2/pkg/protocols/file/request.go @@ -47,7 +47,7 @@ var emptyResultErr = errors.New("Empty result") // ExecuteWithResults executes the protocol requests and returns results instead of writing them. func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error { wg := sizedwaitgroup.New(request.options.Options.BulkSize) - err := request.getInputPaths(input.Input, func(filePath string) { + err := request.getInputPaths(input.MetaInput.Input, func(filePath string) { wg.Add() func(filePath string) { defer wg.Done() @@ -63,7 +63,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, // every new file in the compressed multi-file archive counts 1 request.options.Progress.AddToTotal(1) archiveFileName := filepath.Join(filePath, file.Name()) - event, fileMatches, err := request.processReader(file.ReadCloser, archiveFileName, input.Input, file.Size(), previous) + event, fileMatches, err := request.processReader(file.ReadCloser, archiveFileName, input.MetaInput.Input, file.Size(), previous) if err != nil { if errors.Is(err, emptyResultErr) { // no matches but one file elaborated @@ -116,7 +116,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, _ = tmpFileOut.Sync() // rewind the file _, _ = tmpFileOut.Seek(0, 0) - event, fileMatches, err := request.processReader(tmpFileOut, filePath, input.Input, fileStat.Size(), previous) + event, fileMatches, err := request.processReader(tmpFileOut, filePath, input.MetaInput.Input, fileStat.Size(), previous) if err != nil { if errors.Is(err, emptyResultErr) { // no matches but one file elaborated @@ -136,7 +136,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, default: // normal file - increments the counter by 1 request.options.Progress.AddToTotal(1) - event, fileMatches, err := request.processFile(filePath, input.Input, previous) + event, fileMatches, err := request.processFile(filePath, input.MetaInput.Input, previous) if err != nil { if errors.Is(err, emptyResultErr) { // no matches but one file elaborated @@ -158,7 +158,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, wg.Wait() if err != nil { - request.options.Output.Request(request.options.TemplatePath, input.Input, request.Type().String(), err) + request.options.Output.Request(request.options.TemplatePath, input.MetaInput.Input, request.Type().String(), err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not send file request") } diff --git a/v2/pkg/protocols/file/request_test.go b/v2/pkg/protocols/file/request_test.go index 9de78fd4..96a02361 100644 --- a/v2/pkg/protocols/file/request_test.go +++ b/v2/pkg/protocols/file/request_test.go @@ -66,8 +66,7 @@ func TestFileExecuteWithResults(t *testing.T) { t.Run("valid", func(t *testing.T) { metadata := make(output.InternalEvent) previous := make(output.InternalEvent) - ctxArgs := contextargs.New() - ctxArgs.Input = tempDir + ctxArgs := contextargs.NewWithInput(tempDir) err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) { finalEvent = event }) diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go index d43168c3..4d23cf6c 100644 --- a/v2/pkg/protocols/headless/request.go +++ b/v2/pkg/protocols/headless/request.go @@ -31,7 +31,7 @@ func (request *Request) Type() templateTypes.ProtocolType { // ExecuteWithResults executes the protocol requests and returns results instead of writing them. func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, previous output.InternalEvent /*TODO review unused parameter*/, callback protocols.OutputEventCallback) error { - inputURL := input.Input + inputURL := input.MetaInput.Input if request.options.Browser.UserAgent() == "" { request.options.Browser.SetUserAgent(request.compiledUserAgent) } diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 3bd818e9..e3d849a4 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -48,7 +48,7 @@ func (request *Request) Type() templateTypes.ProtocolType { // executeRaceRequest executes race condition request for a URL func (request *Request) executeRaceRequest(input *contextargs.Context, previous output.InternalEvent, callback protocols.OutputEventCallback) error { - reqURL := input.Input + reqURL := input.MetaInput.Input var generatedRequests []*generatedRequest // Requests within race condition should be dumped once and the output prefilled to allow DSL language to work @@ -59,7 +59,8 @@ func (request *Request) executeRaceRequest(input *contextargs.Context, previous if !ok { return nil } - requestForDump, err := generator.Make(context.Background(), reqURL, inputData, payloads, nil) + ctx := request.newContext(input) + requestForDump, err := generator.Make(ctx, reqURL, inputData, payloads, nil) if err != nil { return err } @@ -87,7 +88,8 @@ func (request *Request) executeRaceRequest(input *contextargs.Context, previous if !ok { break } - generatedRequest, err := generator.Make(context.Background(), reqURL, inputData, payloads, nil) + ctx := request.newContext(input) + generatedRequest, err := generator.Make(ctx, reqURL, inputData, payloads, nil) if err != nil { return err } @@ -130,7 +132,8 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV if !ok { break } - generatedHttpRequest, err := generator.Make(context.Background(), input.Input, inputData, payloads, dynamicValues) + ctx := request.newContext(input) + generatedHttpRequest, err := generator.Make(ctx, input.MetaInput.Input, inputData, payloads, dynamicValues) if err != nil { if err == io.EOF { break @@ -138,8 +141,8 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) return err } - if input.Input == "" { - input.Input = generatedHttpRequest.URL() + if input.MetaInput.Input == "" { + input.MetaInput.Input = generatedHttpRequest.URL() } swg.Add() go func(httpRequest *generatedRequest) { @@ -166,7 +169,7 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu generator := request.newGenerator(false) // need to extract the target from the url - URL, err := url.Parse(input.Input) + URL, err := url.Parse(input.MetaInput.Input) if err != nil { return err } @@ -197,13 +200,14 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu if !ok { break } - generatedHttpRequest, err := generator.Make(context.Background(), input.Input, inputData, payloads, dynamicValues) + ctx := request.newContext(input) + generatedHttpRequest, err := generator.Make(ctx, input.MetaInput.Input, inputData, payloads, dynamicValues) if err != nil { request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) return err } - if input.Input == "" { - input.Input = generatedHttpRequest.URL() + if input.MetaInput.Input == "" { + input.MetaInput.Input = generatedHttpRequest.URL() } generatedHttpRequest.pipelinedClient = pipeClient swg.Add() @@ -225,14 +229,14 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu // executeFuzzingRule executes fuzzing request for a URL func (request *Request) executeFuzzingRule(input *contextargs.Context, previous output.InternalEvent, callback protocols.OutputEventCallback) error { - parsed, err := url.Parse(input.Input) + parsed, err := url.Parse(input.MetaInput.Input) if err != nil { return errors.Wrap(err, "could not parse url") } fuzzRequestCallback := func(gr fuzz.GeneratedRequest) bool { hasInteractMatchers := interactsh.HasMatchers(request.CompiledOperators) hasInteractMarkers := len(gr.InteractURLs) > 0 - if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(input.Input) { + if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(input.MetaInput.Input) { return false } @@ -266,7 +270,7 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, previous } if requestErr != nil { if request.options.HostErrorsCache != nil { - request.options.HostErrorsCache.MarkFailed(input.Input, requestErr) + request.options.HostErrorsCache.MarkFailed(input.MetaInput.Input, requestErr) } } request.options.Progress.IncrementRequests() @@ -285,7 +289,7 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, previous if !result { break } - generated, err := generator.Make(context.Background(), input.Input, value, payloads, nil) + generated, err := generator.Make(context.Background(), input.MetaInput.Input, value, payloads, nil) if err != nil { continue } @@ -346,10 +350,11 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa request.options.RateLimiter.Take() - ctx, cancel := context.WithTimeout(context.Background(), time.Duration(request.options.Options.Timeout)*time.Second) + ctx := request.newContext(input) + ctxWithTimeout, cancel := context.WithTimeout(ctx, time.Duration(request.options.Options.Timeout)*time.Second) defer cancel() - generatedHttpRequest, err := generator.Make(ctx, input.Input, data, payloads, dynamicValue) + generatedHttpRequest, err := generator.Make(ctxWithTimeout, input.MetaInput.Input, data, payloads, dynamicValue) if err != nil { if err == io.EOF { return true, nil @@ -367,11 +372,11 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa generatedHttpRequest.interactshURLs = append(generatedHttpRequest.interactshURLs, interactURLs...) } hasInteractMarkers := interactsh.HasMarkers(data) || len(generatedHttpRequest.interactshURLs) > 0 - if input.Input == "" { - input.Input = generatedHttpRequest.URL() + if input.MetaInput.Input == "" { + input.MetaInput.Input = generatedHttpRequest.URL() } // Check if hosts keep erroring - if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(input.Input) { + if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(input.MetaInput.ID()) { return true, nil } var gotMatches bool @@ -400,7 +405,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa } if err != nil { if request.options.HostErrorsCache != nil { - request.options.HostErrorsCache.MarkFailed(input.Input, err) + request.options.HostErrorsCache.MarkFailed(input.MetaInput.ID(), err) } requestErr = err } @@ -472,7 +477,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ if !generatedRequest.original.Race { var dumpError error // TODO: dump is currently not working with post-processors - somehow it alters the signature - dumpedRequest, dumpError = dump(generatedRequest, input.Input) + dumpedRequest, dumpError = dump(generatedRequest, input.MetaInput.Input) if dumpError != nil { return dumpError } @@ -480,12 +485,12 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ if ignoreList := GetVariablesNamesSkipList(generatedRequest.original.Signature.Value); ignoreList != nil { if varErr := expressions.ContainsVariablesWithIgnoreList(ignoreList, dumpedRequestString); varErr != nil && !request.SkipVariablesCheck { - gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, input.Input, varErr) + gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, input.MetaInput.Input, varErr) return errStopExecution } } else { // Check if are there any unresolved variables. If yes, skip unless overridden by user. if varErr := expressions.ContainsUnresolvedVariables(dumpedRequestString); varErr != nil && !request.SkipVariablesCheck { - gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, input.Input, varErr) + gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, input.MetaInput.Input, varErr) return errStopExecution } } @@ -499,7 +504,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ if parsed, parseErr := url.Parse(formedURL); parseErr == nil { hostname = parsed.Host } - resp, err = generatedRequest.pipelinedClient.DoRaw(generatedRequest.rawRequest.Method, input.Input, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data))) + resp, err = generatedRequest.pipelinedClient.DoRaw(generatedRequest.rawRequest.Method, input.MetaInput.Input, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data))) } else if generatedRequest.request != nil { resp, err = generatedRequest.pipelinedClient.Dor(generatedRequest.request) } @@ -507,7 +512,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ formedURL = generatedRequest.rawRequest.FullURL // use request url as matched url if empty if formedURL == "" { - formedURL = input.Input + formedURL = input.MetaInput.Input if generatedRequest.rawRequest.Path != "" { formedURL = fmt.Sprintf("%s%s", formedURL, generatedRequest.rawRequest.Path) } @@ -520,7 +525,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ options.CustomRawBytes = generatedRequest.rawRequest.UnsafeRawBytes options.ForceReadAllBody = request.ForceReadAllBody options.SNI = request.options.Options.SNI - resp, err = generatedRequest.original.rawhttpClient.DoRawWithOptions(generatedRequest.rawRequest.Method, input.Input, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)), &options) + resp, err = generatedRequest.original.rawhttpClient.DoRawWithOptions(generatedRequest.rawRequest.Method, input.MetaInput.Input, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)), &options) } else { hostname = generatedRequest.request.URL.Host formedURL = generatedRequest.request.URL.String() @@ -553,13 +558,13 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ } // use request url as matched url if empty if formedURL == "" { - formedURL = input.Input + formedURL = input.MetaInput.Input } // Dump the requests containing all headers if !generatedRequest.original.Race { var dumpError error - dumpedRequest, dumpError = dump(generatedRequest, input.Input) + dumpedRequest, dumpError = dump(generatedRequest, input.MetaInput.Input) if dumpError != nil { return dumpError } @@ -572,7 +577,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ gologger.Print().Msgf("%s", dumpedRequestString) } if request.options.Options.StoreResponse { - request.options.Output.WriteStoreDebugData(input.Input, request.options.TemplateID, request.Type().String(), fmt.Sprintf("%s\n%s", msg, dumpedRequestString)) + request.options.Output.WriteStoreDebugData(input.MetaInput.Input, request.options.TemplateID, request.Type().String(), fmt.Sprintf("%s\n%s", msg, dumpedRequestString)) } } } @@ -588,11 +593,16 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ // If we have interactsh markers and request times out, still send // a callback event so in case we receive an interaction, correlation is possible. if hasInteractMatchers { - outputEvent := request.responseToDSLMap(&http.Response{}, input.Input, formedURL, tostring.UnsafeToString(dumpedRequest), "", "", "", 0, generatedRequest.meta) + outputEvent := request.responseToDSLMap(&http.Response{}, input.MetaInput.Input, formedURL, tostring.UnsafeToString(dumpedRequest), "", "", "", 0, generatedRequest.meta) if i := strings.LastIndex(hostname, ":"); i != -1 { hostname = hostname[:i] } - outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname) + + if input.MetaInput.CustomIP != "" { + outputEvent["ip"] = input.MetaInput.CustomIP + } else { + outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname) + } event := &output.InternalWrappedEvent{InternalEvent: outputEvent} if request.CompiledOperators != nil { @@ -672,7 +682,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ if response.resp == nil { continue // Skip nil responses } - matchedURL := input.Input + matchedURL := input.MetaInput.Input if generatedRequest.rawRequest != nil && generatedRequest.rawRequest.FullURL != "" { matchedURL = generatedRequest.rawRequest.FullURL } @@ -687,12 +697,16 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ } finalEvent := make(output.InternalEvent) - outputEvent := request.responseToDSLMap(response.resp, input.Input, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(response.fullResponse), tostring.UnsafeToString(response.body), tostring.UnsafeToString(response.headers), duration, generatedRequest.meta) + outputEvent := request.responseToDSLMap(response.resp, input.MetaInput.Input, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(response.fullResponse), tostring.UnsafeToString(response.body), tostring.UnsafeToString(response.headers), duration, generatedRequest.meta) if i := strings.LastIndex(hostname, ":"); i != -1 { hostname = hostname[:i] } outputEvent["curl-command"] = curlCommand - outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname) + if input.MetaInput.CustomIP != "" { + outputEvent["ip"] = input.MetaInput.CustomIP + } else { + outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname) + } if request.options.Interactsh != nil { request.options.Interactsh.MakePlaceholders(generatedRequest.interactshURLs, outputEvent) } @@ -723,7 +737,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ responseContentType := resp.Header.Get("Content-Type") isResponseTruncated := request.MaxSize > 0 && len(gotData) >= request.MaxSize - dumpResponse(event, request, response.fullResponse, formedURL, responseContentType, isResponseTruncated, input.Input) + dumpResponse(event, request, response.fullResponse, formedURL, responseContentType, isResponseTruncated, input.MetaInput.Input) callback(event) @@ -842,3 +856,10 @@ func (request *Request) pruneSignatureInternalValues(maps ...map[string]interfac } } } + +func (request *Request) newContext(input *contextargs.Context) context.Context { + if input.MetaInput.CustomIP != "" { + return context.WithValue(context.Background(), "ip", input.MetaInput.CustomIP) //nolint + } + return context.Background() +} diff --git a/v2/pkg/protocols/http/request_test.go b/v2/pkg/protocols/http/request_test.go index 2d93c190..01b2fd2b 100644 --- a/v2/pkg/protocols/http/request_test.go +++ b/v2/pkg/protocols/http/request_test.go @@ -82,8 +82,7 @@ Disallow: /c`)) t.Run("test", func(t *testing.T) { metadata := make(output.InternalEvent) previous := make(output.InternalEvent) - ctxArgs := contextargs.New() - ctxArgs.Input = ts.URL + ctxArgs := contextargs.NewWithInput(ts.URL) err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) { if event.OperatorsResult != nil && event.OperatorsResult.Matched { matchCount++ diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index e4a694ea..e5680154 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -43,10 +43,10 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata if request.SelfContained { address = "" } else { - address, err = getAddress(input.Input) + address, err = getAddress(input.MetaInput.Input) } if err != nil { - request.options.Output.Request(request.options.TemplatePath, input.Input, request.Type().String(), err) + request.options.Output.Request(request.options.TemplatePath, input.MetaInput.Input, request.Type().String(), err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not get address from url") } @@ -57,7 +57,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata variables = generators.MergeMaps(variablesMap, variables) actualAddress := replacer.Replace(kv.address, variables) - if err := request.executeAddress(variables, actualAddress, address, input.Input, kv.tls, previous, callback); err != nil { + if err := request.executeAddress(variables, actualAddress, address, input.MetaInput.Input, kv.tls, previous, callback); err != nil { gologger.Warning().Msgf("Could not make network request for %s: %s\n", actualAddress, err) continue } diff --git a/v2/pkg/protocols/network/request_test.go b/v2/pkg/protocols/network/request_test.go index 859a61cc..ff8b7013 100644 --- a/v2/pkg/protocols/network/request_test.go +++ b/v2/pkg/protocols/network/request_test.go @@ -65,8 +65,7 @@ func TestNetworkExecuteWithResults(t *testing.T) { t.Run("domain-valid", func(t *testing.T) { metadata := make(output.InternalEvent) previous := make(output.InternalEvent) - ctxArgs := contextargs.New() - ctxArgs.Input = parsed.Host + ctxArgs := contextargs.NewWithInput(parsed.Host) err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) { finalEvent = event }) @@ -82,8 +81,7 @@ func TestNetworkExecuteWithResults(t *testing.T) { t.Run("invalid-port-override", func(t *testing.T) { metadata := make(output.InternalEvent) previous := make(output.InternalEvent) - ctxArgs := contextargs.New() - ctxArgs.Input = "127.0.0.1:11211" + ctxArgs := contextargs.NewWithInput("127.0.0.1:11211") err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) { finalEvent = event }) @@ -97,8 +95,7 @@ func TestNetworkExecuteWithResults(t *testing.T) { t.Run("hex-to-string", func(t *testing.T) { metadata := make(output.InternalEvent) previous := make(output.InternalEvent) - ctxArgs := contextargs.New() - ctxArgs.Input = parsed.Host + ctxArgs := contextargs.NewWithInput(parsed.Host) err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) { finalEvent = event }) diff --git a/v2/pkg/protocols/offlinehttp/request.go b/v2/pkg/protocols/offlinehttp/request.go index 4d775047..7bafa3a7 100644 --- a/v2/pkg/protocols/offlinehttp/request.go +++ b/v2/pkg/protocols/offlinehttp/request.go @@ -32,7 +32,7 @@ func (request *Request) Type() templateTypes.ProtocolType { func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { wg := sizedwaitgroup.New(request.options.Options.BulkSize) - err := request.getInputPaths(input.Input, func(data string) { + err := request.getInputPaths(input.MetaInput.Input, func(data string) { wg.Add() go func(data string) { @@ -98,7 +98,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata }) wg.Wait() if err != nil { - request.options.Output.Request(request.options.TemplatePath, input.Input, "file", err) + request.options.Output.Request(request.options.TemplatePath, input.MetaInput.Input, "file", err) request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not send file request") } diff --git a/v2/pkg/protocols/ssl/ssl.go b/v2/pkg/protocols/ssl/ssl.go index 229658e8..60f15213 100644 --- a/v2/pkg/protocols/ssl/ssl.go +++ b/v2/pkg/protocols/ssl/ssl.go @@ -127,7 +127,7 @@ func (request *Request) GetID() string { // ExecuteWithResults executes the protocol requests and returns results instead of writing them. func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { - address, err := getAddress(input.Input) + address, err := getAddress(input.MetaInput.Input) if err != nil { return nil } @@ -153,7 +153,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa finalAddress, dataErr := expressions.EvaluateByte([]byte(request.Address), payloadValues) if dataErr != nil { - requestOptions.Output.Request(requestOptions.TemplateID, input.Input, request.Type().String(), dataErr) + requestOptions.Output.Request(requestOptions.TemplateID, input.MetaInput.Input, request.Type().String(), dataErr) requestOptions.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(dataErr, "could not evaluate template expressions") } @@ -165,7 +165,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa response, err := request.tlsx.Connect(host, host, port) if err != nil { - requestOptions.Output.Request(requestOptions.TemplateID, input.Input, request.Type().String(), err) + requestOptions.Output.Request(requestOptions.TemplateID, input.MetaInput.Input, request.Type().String(), err) requestOptions.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not connect to server") } @@ -174,12 +174,12 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa gologger.Verbose().Msgf("Sent SSL request to %s", address) if requestOptions.Options.Debug || requestOptions.Options.DebugRequests || requestOptions.Options.StoreResponse { - msg := fmt.Sprintf("[%s] Dumped SSL request for %s", requestOptions.TemplateID, input.Input) + msg := fmt.Sprintf("[%s] Dumped SSL request for %s", requestOptions.TemplateID, input.MetaInput.Input) if requestOptions.Options.Debug || requestOptions.Options.DebugRequests { - gologger.Debug().Str("address", input.Input).Msg(msg) + gologger.Debug().Str("address", input.MetaInput.Input).Msg(msg) } if requestOptions.Options.StoreResponse { - request.options.Output.WriteStoreDebugData(input.Input, request.options.TemplateID, request.Type().String(), msg) + request.options.Output.WriteStoreDebugData(input.MetaInput.Input, request.options.TemplateID, request.Type().String(), msg) } } @@ -192,7 +192,11 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa data["response"] = jsonDataString data["host"] = input data["matched"] = addressToDial - data["ip"] = request.dialer.GetDialedIP(hostname) + if input.MetaInput.CustomIP != "" { + data["ip"] = input.MetaInput.CustomIP + } else { + data["ip"] = request.dialer.GetDialedIP(hostname) + } data["template-path"] = requestOptions.TemplatePath data["template-id"] = requestOptions.TemplateID data["template-info"] = requestOptions.TemplateInfo @@ -220,13 +224,13 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa event := eventcreator.CreateEvent(request, data, requestOptions.Options.Debug || requestOptions.Options.DebugResponse) if requestOptions.Options.Debug || requestOptions.Options.DebugResponse || requestOptions.Options.StoreResponse { - msg := fmt.Sprintf("[%s] Dumped SSL response for %s", requestOptions.TemplateID, input.Input) + msg := fmt.Sprintf("[%s] Dumped SSL response for %s", requestOptions.TemplateID, input.MetaInput.Input) if requestOptions.Options.Debug || requestOptions.Options.DebugResponse { gologger.Debug().Msg(msg) gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, jsonDataString, requestOptions.Options.NoColor, false)) } if requestOptions.Options.StoreResponse { - request.options.Output.WriteStoreDebugData(input.Input, request.options.TemplateID, request.Type().String(), fmt.Sprintf("%s\n%s", msg, jsonDataString)) + request.options.Output.WriteStoreDebugData(input.MetaInput.Input, request.options.TemplateID, request.Type().String(), fmt.Sprintf("%s\n%s", msg, jsonDataString)) } } callback(event) diff --git a/v2/pkg/protocols/ssl/ssl_test.go b/v2/pkg/protocols/ssl/ssl_test.go index 469a590e..77c03b22 100644 --- a/v2/pkg/protocols/ssl/ssl_test.go +++ b/v2/pkg/protocols/ssl/ssl_test.go @@ -28,8 +28,7 @@ func TestSSLProtocol(t *testing.T) { require.Nil(t, err, "could not compile ssl request") var gotEvent output.InternalEvent - ctxArgs := contextargs.New() - ctxArgs.Input = "google.com:443" + ctxArgs := contextargs.NewWithInput("google.com:443") err = request.ExecuteWithResults(ctxArgs, nil, nil, func(event *output.InternalWrappedEvent) { gotEvent = event.InternalEvent }) diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index 3cd1aa3e..9b67df65 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -137,7 +137,7 @@ func (request *Request) GetID() string { // ExecuteWithResults executes the protocol requests and returns results instead of writing them. func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { - hostname, err := getAddress(input.Input) + hostname, err := getAddress(input.MetaInput.Input) if err != nil { return err } @@ -150,13 +150,13 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa if !ok { break } - if err := request.executeRequestWithPayloads(input.Input, hostname, value, previous, callback); err != nil { + if err := request.executeRequestWithPayloads(input.MetaInput.Input, hostname, value, previous, callback); err != nil { return err } } } else { value := make(map[string]interface{}) - if err := request.executeRequestWithPayloads(input.Input, hostname, value, previous, callback); err != nil { + if err := request.executeRequestWithPayloads(input.MetaInput.Input, hostname, value, previous, callback); err != nil { return err } } diff --git a/v2/pkg/protocols/whois/whois.go b/v2/pkg/protocols/whois/whois.go index 7d7fcf7a..79f94f35 100644 --- a/v2/pkg/protocols/whois/whois.go +++ b/v2/pkg/protocols/whois/whois.go @@ -85,7 +85,7 @@ func (request *Request) GetID() string { // ExecuteWithResults executes the protocol requests and returns results instead of writing them. func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { // generate variables - variables := generateVariables(input.Input) + variables := generateVariables(input.MetaInput.Input) if vardump.EnableVarDump { gologger.Debug().Msgf("Protocol request variables: \n%s\n", vardump.DumpVariables(variables)) diff --git a/v2/pkg/templates/cluster.go b/v2/pkg/templates/cluster.go index b7af09dc..3e299f8e 100644 --- a/v2/pkg/templates/cluster.go +++ b/v2/pkg/templates/cluster.go @@ -208,7 +208,7 @@ func (e *ClusterExecuter) Execute(input *contextargs.Context) (bool, error) { } }) if err != nil && e.options.HostErrorsCache != nil { - e.options.HostErrorsCache.MarkFailed(input.Input, err) + e.options.HostErrorsCache.MarkFailed(input.MetaInput.Input, err) } return results, err } @@ -230,7 +230,7 @@ func (e *ClusterExecuter) ExecuteWithResults(input *contextargs.Context, callbac } }) if err != nil && e.options.HostErrorsCache != nil { - e.options.HostErrorsCache.MarkFailed(input.Input, err) + e.options.HostErrorsCache.MarkFailed(input.MetaInput.Input, err) } return err } diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index ee886699..58bc1a29 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -272,12 +272,16 @@ type Options struct { IncludeConditions goflags.StringSlice // Custom Config Directory CustomConfigDir string + // ConfigPath contains the config path (used by healthcheck) + ConfigPath string + // ScanAllIPs associated to a dns record + ScanAllIPs bool + // IPVersion to scan (4,6) + IPVersion goflags.StringSlice // Github token used to clone/pull from private repos for custom templates GithubToken string // GithubTemplateRepo is the list of custom public/private templates github repos GithubTemplateRepo goflags.StringSlice - - ConfigPath string // Used by healthcheck } func (options *Options) AddVarPayload(key string, value interface{}) {