mirror of https://github.com/daffainfo/nuclei.git
Adding support to scan all v4/v6 IPs (#2709)
* Adding support to scan all v4/v6 IPs * adding tests * metainput prototype * using new signature * fixing nil pointer * adding request context with metadata * removing log instruction * fixing merge conflicts * adding clone helpers * attempting to fix ipv6 square parenthesis wrap * fixing dialed ip info * fixing syntax * fixing output ip selection * adding integration tests * disabling test due to gh ipv6 issue * using ipv4 only due to GH limited networking * extending metainput marshaling * fixing hmap key * adding test for httpx integration * fixing lint error * reworking marshaling/id-calculation * adding ip version validation * improving handling non url targets * fixing condition checkdev
Mzack9999
GitHub
parent
9e56451d2e
commit
1fbbca66f9
|
|
@ -0,0 +1,15 @@
|
|||
id: get-all-ips
|
||||
|
||||
info:
|
||||
name: Basic GET Request on all IPS
|
||||
author: pdteam
|
||||
severity: info
|
||||
|
||||
requests:
|
||||
- method: GET
|
||||
path:
|
||||
- "{{BaseURL}}"
|
||||
matchers:
|
||||
- type: word
|
||||
words:
|
||||
- "ok"
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
id: get-without-scheme
|
||||
|
||||
info:
|
||||
name: Basic GET Request without scheme
|
||||
author: pdteam
|
||||
severity: info
|
||||
|
||||
requests:
|
||||
- method: GET
|
||||
path:
|
||||
- "{{BaseURL}}"
|
||||
matchers:
|
||||
- type: word
|
||||
words:
|
||||
- "ok"
|
||||
|
|
@ -23,6 +23,7 @@ import (
|
|||
"github.com/projectdiscovery/nuclei/v2/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/parsers"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
|
||||
|
|
@ -123,7 +124,7 @@ func executeNucleiAsCode(templatePath, templateURL string) ([]string, error) {
|
|||
}
|
||||
store.Load()
|
||||
|
||||
input := &inputs.SimpleInputProvider{Inputs: []string{templateURL}}
|
||||
input := &inputs.SimpleInputProvider{Inputs: []*contextargs.MetaInput{{Input: templateURL}}}
|
||||
_ = engine.Execute(store.Templates(), input)
|
||||
engine.WorkPool().Wait() // Wait for the scan to finish
|
||||
|
||||
|
|
|
|||
|
|
@ -57,6 +57,8 @@ var httpTestcases = map[string]testutils.TestCase{
|
|||
"http/get-sni-unsafe.yaml": &customCLISNIUnsafe{},
|
||||
"http/annotation-timeout.yaml": &annotationTimeout{},
|
||||
"http/custom-attack-type.yaml": &customAttackType{},
|
||||
"http/get-all-ips.yaml": &scanAllIPS{},
|
||||
"http/get-without-scheme.yaml": &httpGetWithoutScheme{},
|
||||
}
|
||||
|
||||
type httpInteractshRequest struct{}
|
||||
|
|
@ -1001,3 +1003,28 @@ func (h *customAttackType) Execute(filePath string) error {
|
|||
}
|
||||
return expectResultsCount(got, 4)
|
||||
}
|
||||
|
||||
// Disabled as GH doesn't support ipv6
|
||||
type scanAllIPS struct{}
|
||||
|
||||
// Execute executes a test case and returns an error if occurred
|
||||
func (h *scanAllIPS) Execute(filePath string) error {
|
||||
got, err := testutils.RunNucleiTemplateAndGetResults(filePath, "https://scanme.sh", debug, "-scan-all-ips", "-iv", "4")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// limiting test to ipv4 (GH doesn't support ipv6)
|
||||
return expectResultsCount(got, 1)
|
||||
}
|
||||
|
||||
// ensure that ip|host are handled without http|https scheme
|
||||
type httpGetWithoutScheme struct{}
|
||||
|
||||
// Execute executes a test case and returns an error if occurred
|
||||
func (h *httpGetWithoutScheme) Execute(filePath string) error {
|
||||
got, err := testutils.RunNucleiTemplateAndGetResults(filePath, "scanme.sh", debug)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return expectResultsCount(got, 1)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -125,6 +125,8 @@ on extensive configurability, massive extensibility and ease of use.`)
|
|||
flagSet.StringSliceVarP(&options.Targets, "target", "u", []string{}, "target URLs/hosts to scan", goflags.StringSliceOptions),
|
||||
flagSet.StringVarP(&options.TargetsFilePath, "list", "l", "", "path to file containing a list of target URLs/hosts to scan (one per line)"),
|
||||
flagSet.StringVar(&options.Resume, "resume", "", "Resume scan using resume.cfg (clustering will be disabled)"),
|
||||
flagSet.BoolVarP(&options.ScanAllIPs, "scan-all-ips", "sa", false, "Scan all the ip's associated with dns record"),
|
||||
flagSet.StringSliceVarP(&options.IPVersion, "ip-version", "iv", []string{"4"}, "IP version to scan of hostname (4,6) - (default 4)", goflags.CommaSeparatedStringSliceOptions),
|
||||
)
|
||||
|
||||
flagSet.CreateGroup("templates", "Templates",
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ package main
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"io"
|
||||
"net/http"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
|
|
@ -66,7 +66,7 @@ func requestHandler(ctx echo.Context) error {
|
|||
}
|
||||
defer data.Body.Close()
|
||||
|
||||
body, _ := ioutil.ReadAll(data.Body)
|
||||
body, _ := io.ReadAll(data.Body)
|
||||
return ctx.HTML(200, fmt.Sprintf(bodyTemplate, string(body)))
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -25,8 +25,7 @@ require (
|
|||
github.com/owenrumney/go-sarif/v2 v2.1.2
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/projectdiscovery/clistats v0.0.8
|
||||
github.com/projectdiscovery/cryptoutil v1.0.0 // indirect
|
||||
github.com/projectdiscovery/fastdialer v0.0.17
|
||||
github.com/projectdiscovery/fastdialer v0.0.18-0.20221102102120-8e9343e8b0e0
|
||||
github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08
|
||||
github.com/projectdiscovery/gologger v1.1.4
|
||||
github.com/projectdiscovery/hmap v0.0.2
|
||||
|
|
@ -208,6 +207,7 @@ require (
|
|||
github.com/nwaples/rardecode v1.1.2 // indirect
|
||||
github.com/pierrec/lz4 v2.6.1+incompatible // indirect
|
||||
github.com/projectdiscovery/asnmap v0.0.1 // indirect
|
||||
github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 // indirect; indirectdev
|
||||
github.com/projectdiscovery/fileutil v0.0.3 // indirect
|
||||
github.com/projectdiscovery/iputil v0.0.2 // indirect
|
||||
github.com/projectdiscovery/sliceutil v0.0.1 // indirect
|
||||
|
|
|
|||
27
v2/go.sum
27
v2/go.sum
|
|
@ -518,8 +518,6 @@ github.com/mholt/acmez v1.0.4/go.mod h1:qFGLZ4u+ehWINeJZjzPlsnjJBCPAADWTcIqE/7DA
|
|||
github.com/mholt/archiver v3.1.1+incompatible h1:1dCVxuqs0dJseYEhi5pl7MYPH9zDa1wBi7mF09cbNkU=
|
||||
github.com/mholt/archiver v3.1.1+incompatible/go.mod h1:Dh2dOXnSdiLxRiPoVfIr/fI1TwETms9B8CTWfeh7ROU=
|
||||
github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
|
||||
github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM=
|
||||
github.com/microcosm-cc/bluemonday v1.0.19/go.mod h1:QNzV2UbLK2/53oIIwTOyLUSABMkjZ4tqiyC1g/DyqxE=
|
||||
github.com/microcosm-cc/bluemonday v1.0.21 h1:dNH3e4PSyE4vNX+KlRGHT5KrSvjeUkoNPwEORjffHJg=
|
||||
github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM=
|
||||
github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
|
||||
|
|
@ -596,26 +594,21 @@ github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc h1:jqZK
|
|||
github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc/go.mod h1:5tNGQP9kOfW+X5+40pZP8aqPYLHs45nJkFaSHLxdeH8=
|
||||
github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKSfZBKWWLh20=
|
||||
github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg=
|
||||
github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 h1:jT6f/cdOpLkp9GAfRrxk57BUjYfIrR8E+AjMv5H5U4U=
|
||||
github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345/go.mod h1:clhQmPnt35ziJW1AhJRKyu8aygXCSoyWj6dtmZBRjjc=
|
||||
github.com/projectdiscovery/cryptoutil v1.0.0 h1:5rQfnWDthJ5ZFcqze+rmT1N7l1HJQ6EB26MrjaYB7I0=
|
||||
github.com/projectdiscovery/cryptoutil v1.0.0/go.mod h1:VJvSNE8f8A1MgpjgAL2GPJSQcJa4jbdaeQJstARFrU4=
|
||||
github.com/projectdiscovery/fastdialer v0.0.12/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/ieD4uda2JuUA4WJ+RLee0=
|
||||
github.com/projectdiscovery/fastdialer v0.0.15/go.mod h1:Q28lw9oTpiZHq09uFG6YYYLUsUjsOypZ7PXWwQGBB80=
|
||||
github.com/projectdiscovery/fastdialer v0.0.17 h1:wT7LinZAcyEzkfhJLhWNQrA9m79+hTd6CJ7aWTJfXjk=
|
||||
github.com/projectdiscovery/fastdialer v0.0.17/go.mod h1:poZbCGYGRfRcKxU8/1rWH8EVOJ4AZn1a8+CyINHzffQ=
|
||||
github.com/projectdiscovery/fastdialer v0.0.18-0.20221102102120-8e9343e8b0e0 h1:1hcFBedqq8772PxN3Lbq7Itr3N59C8ro0xaTxYkmg9s=
|
||||
github.com/projectdiscovery/fastdialer v0.0.18-0.20221102102120-8e9343e8b0e0/go.mod h1:KSHL57MbR0PbdJpagiqqB0jPqO1GUcnYZT5ngAvsmqQ=
|
||||
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
|
||||
github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
|
||||
github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08 h1:NwD1R/du1dqrRKN3SJl9kT6tN3K9puuWFXEvYF2ihew=
|
||||
github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08/go.mod h1:paLCnwV8sL7ppqIwVQodQrk3F6mnWafwTDwRd7ywZwQ=
|
||||
github.com/projectdiscovery/fileutil v0.0.0-20210914153648-31f843feaad4/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0=
|
||||
github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0=
|
||||
github.com/projectdiscovery/fileutil v0.0.0-20220308101036-16c79af1cf5d/go.mod h1:Pm0f+MWgDFMSSI9NBedNh48LyYPs8gD3Jd8DXGmp4aQ=
|
||||
github.com/projectdiscovery/fileutil v0.0.0-20220609150212-453ac591c36c/go.mod h1:g8wsrb0S5NtEN0JgVyyPeb3FQdArx+UMESmFX94bcGY=
|
||||
github.com/projectdiscovery/fileutil v0.0.0-20220705195237-01becc2a8963/go.mod h1:DaY7wmLPMleyHDCD/14YApPCDtrARY4J8Eny2ZGsG/g=
|
||||
github.com/projectdiscovery/fileutil v0.0.3 h1:GSsoey4p8ZHIRxWF2VXh4mhLr+wfEkpJwvF0Dxpn/gg=
|
||||
github.com/projectdiscovery/fileutil v0.0.3/go.mod h1:GLejWd3YerG3RNYD/Hk2pJlytlYRgHdkWfWUAdCH2YQ=
|
||||
github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY=
|
||||
github.com/projectdiscovery/goflags v0.0.8/go.mod h1:GDSkWyXa6kfQjpJu10SO64DN8lXuKXVENlBMk8N7H80=
|
||||
github.com/projectdiscovery/goflags v0.1.3 h1:dnJlg19VkDp1iYkpAod4Tv+OAngr7Mq61LMMpBQlO0M=
|
||||
github.com/projectdiscovery/goflags v0.1.3/go.mod h1:/7ZAoY1SVfUcGobTP5QDvGQmrpPDDlBUDIMr7c+r94Q=
|
||||
github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
|
||||
|
|
@ -633,14 +626,12 @@ github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+
|
|||
github.com/projectdiscovery/iputil v0.0.0-20210414194613-4b4d2517acf0/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A=
|
||||
github.com/projectdiscovery/iputil v0.0.0-20210429152401-c18a5408ca46/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A=
|
||||
github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3/go.mod h1:blmYJkS8lSrrx3QcmcgS2tZIxlojeVmoGeA9twslCBU=
|
||||
github.com/projectdiscovery/iputil v0.0.0-20220712175312-b9406f31cdd8/go.mod h1:vHRC+9exsfSbEngMKDl0xiWqkxlLk3lHQZpbS2yFT8U=
|
||||
github.com/projectdiscovery/iputil v0.0.2 h1:f6IGnZF4RImJLysPSPG3D84jyTH34q3lihCFeP+eZzI=
|
||||
github.com/projectdiscovery/iputil v0.0.2/go.mod h1:J3Pcz1q51pi4/JL871mQztg0KOzyWDPxnPLOYJm2pVQ=
|
||||
github.com/projectdiscovery/mapcidr v0.0.4/go.mod h1:ALOIj6ptkWujNoX8RdQwB2mZ+kAmKuLJBq9T5gR5wG0=
|
||||
github.com/projectdiscovery/mapcidr v0.0.6/go.mod h1:ZEBhMmBU3laUl3g9QGTrzJku1VJOzjdFwW01f/zVVzM=
|
||||
github.com/projectdiscovery/mapcidr v0.0.7/go.mod h1:7CzdUdjuLVI0s33dQ33lWgjg3vPuLFw2rQzZ0RxkT00=
|
||||
github.com/projectdiscovery/mapcidr v0.0.8/go.mod h1:7CzdUdjuLVI0s33dQ33lWgjg3vPuLFw2rQzZ0RxkT00=
|
||||
github.com/projectdiscovery/mapcidr v1.0.1/go.mod h1:/qxlpxXZQFFjHynSc9u5O0kUPzH46VskECiwLiz7/vw=
|
||||
github.com/projectdiscovery/mapcidr v1.0.3 h1:SGtOOEz0AxthVO7ZonMvhrJ/AQkHIXCVgyZqJdY0cAY=
|
||||
github.com/projectdiscovery/mapcidr v1.0.3/go.mod h1:/0lEXlu/q0t5u34vIVF6odHR+JCdD3CIHNsMXo7nwrU=
|
||||
github.com/projectdiscovery/networkpolicy v0.0.1/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs=
|
||||
|
|
@ -661,24 +652,18 @@ github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:
|
|||
github.com/projectdiscovery/retryabledns v1.0.11/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4=
|
||||
github.com/projectdiscovery/retryabledns v1.0.12/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4=
|
||||
github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a/go.mod h1:tXaLDs4n3pRZHwfa8mdXpUWe/AYDNK3HlWDjldhRbjI=
|
||||
github.com/projectdiscovery/retryabledns v1.0.15/go.mod h1:3YbsQVqP7jbQ3CDmarhyVtkJaJ8XcB7S19vMeyMxZxk=
|
||||
github.com/projectdiscovery/retryabledns v1.0.17 h1:XKzI26UKYt2g7YLJ/EcyYmM04sfD1vurETecPEpeA1w=
|
||||
github.com/projectdiscovery/retryabledns v1.0.17/go.mod h1:Dyhq/f0sGmXueso0+Ah3LbJfsX4PXpBrpfiyjZZ8SDk=
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek=
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.2/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI=
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d h1:VR+tDkedzHIp1pGKIDcfPFt7J8KjcjxGsJvBAP6RXFQ=
|
||||
github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d/go.mod h1:t4buiLTB0HtI+62iHfGDqQVTv/i+8OhAKwaX93TGsFE=
|
||||
github.com/projectdiscovery/sliceutil v0.0.0-20220617151003-15892688e1d6/go.mod h1:9YZb6LRjLYAvSOm65v787dwauurixSyjlqXyYa4rTTA=
|
||||
github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4/go.mod h1:RxDaccMjPzIuF7F8XbdGl1yOcqxN4YPiHr9xHpfCkGI=
|
||||
github.com/projectdiscovery/sliceutil v0.0.1 h1:YoCqCMcdwz+gqNfW5hFY8UvNHoA6SfyBSNkVahatleg=
|
||||
github.com/projectdiscovery/sliceutil v0.0.1/go.mod h1:0wBmhU5uTDwMfrEZfvwH9qa5k60Q4shPVOC9E6LGsDI=
|
||||
github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
|
||||
github.com/projectdiscovery/stringsutil v0.0.0-20210823090203-2f5f137e8e1d/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
|
||||
github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
|
||||
github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
|
||||
github.com/projectdiscovery/stringsutil v0.0.0-20220422150559-b54fb5dc6833/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
|
||||
github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3/go.mod h1:mF5sh4jTghoGWwgUb9qWi5waTFklClDbtrqtJU93awc=
|
||||
github.com/projectdiscovery/stringsutil v0.0.0-20220731064040-4b67f194751e/go.mod h1:32NYmKyHkKsmisAOAaWrR15lz2ysz2M8x3KMeeoRHoU=
|
||||
github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
|
||||
github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0=
|
||||
github.com/projectdiscovery/tlsx v0.0.9 h1:wUC8GYUIo5jd+enqE1lnEJ3Ew7m+N6eRmFBjbSJLomU=
|
||||
|
|
@ -762,8 +747,6 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
|
|||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
|
||||
github.com/stretchr/testify v1.7.3/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
|
|
@ -993,17 +976,14 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b
|
|||
golang.org/x/net v0.0.0-20210414194228-064579744ee0/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8=
|
||||
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
|
||||
golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||
golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.0.0-20220630215102-69896b714898/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.0.0-20220728211354-c7608f3a8462/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
|
||||
golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
|
||||
golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0=
|
||||
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
|
||||
|
|
@ -1102,7 +1082,6 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
|
|||
golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220731174439-a90be440212d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
|
||||
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ import (
|
|||
"github.com/projectdiscovery/nuclei/v2/pkg/core"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
|
||||
"go.uber.org/atomic"
|
||||
)
|
||||
|
||||
|
|
@ -84,8 +85,8 @@ func (r *Runner) runCloudEnumeration(store *loader.Store, nostore bool) (*atomic
|
|||
// TODO: Add payload file and workflow support for private templates
|
||||
catalogChecksums := nucleicloud.ReadCatalogChecksum()
|
||||
|
||||
targets := make([]string, 0, r.hmapInputProvider.Count())
|
||||
r.hmapInputProvider.Scan(func(value string) bool {
|
||||
targets := make([]*contextargs.MetaInput, 0, r.hmapInputProvider.Count())
|
||||
r.hmapInputProvider.Scan(func(value *contextargs.MetaInput) bool {
|
||||
targets = append(targets, value)
|
||||
return true
|
||||
})
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import (
|
|||
"github.com/pkg/errors"
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/hmap/store/hybrid"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool"
|
||||
"github.com/projectdiscovery/retryablehttp-go"
|
||||
"github.com/remeh/sizedwaitgroup"
|
||||
|
|
@ -37,18 +38,18 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
|
|||
// Probe the non-standard URLs and store them in cache
|
||||
swg := sizedwaitgroup.New(bulkSize)
|
||||
count := int32(0)
|
||||
r.hmapInputProvider.Scan(func(value string) bool {
|
||||
if strings.HasPrefix(value, "http://") || strings.HasPrefix(value, "https://") {
|
||||
r.hmapInputProvider.Scan(func(value *contextargs.MetaInput) bool {
|
||||
if strings.HasPrefix(value.Input, "http://") || strings.HasPrefix(value.Input, "https://") {
|
||||
return true
|
||||
}
|
||||
|
||||
swg.Add()
|
||||
go func(input string) {
|
||||
go func(input *contextargs.MetaInput) {
|
||||
defer swg.Done()
|
||||
|
||||
if result := probeURL(input, httpclient); result != "" {
|
||||
if result := probeURL(input.Input, httpclient); result != "" {
|
||||
atomic.AddInt32(&count, 1)
|
||||
_ = hm.Set(input, []byte(result))
|
||||
_ = hm.Set(input.Input, []byte(result))
|
||||
}
|
||||
}(value)
|
||||
return true
|
||||
|
|
|
|||
|
|
@ -1,11 +1,15 @@
|
|||
package nucleicloud
|
||||
|
||||
import "time"
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
|
||||
)
|
||||
|
||||
// AddScanRequest is a nuclei scan input item.
|
||||
type AddScanRequest struct {
|
||||
// RawTargets is a list of raw target URLs for the scan.
|
||||
RawTargets []string `json:"raw_targets,omitempty"`
|
||||
RawTargets []*contextargs.MetaInput `json:"raw_targets,omitempty"`
|
||||
// PublicTemplates is a list of public templates for the scan
|
||||
PublicTemplates []string `json:"public_templates,omitempty"`
|
||||
// PrivateTemplates is a map of template-name->contents that
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ package runner
|
|||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
|
|
@ -135,6 +136,23 @@ func validateOptions(options *types.Options) error {
|
|||
}
|
||||
validateCertificatePaths([]string{options.ClientCertFile, options.ClientKeyFile, options.ClientCAFile})
|
||||
}
|
||||
|
||||
// verify that a valid ip version type was selected (4, 6)
|
||||
var useIPV4, useIPV6 bool
|
||||
for _, ipv := range options.IPVersion {
|
||||
switch ipv {
|
||||
case "4":
|
||||
useIPV4 = true
|
||||
case "6":
|
||||
useIPV6 = true
|
||||
default:
|
||||
return fmt.Errorf("unsupported ip version: %s", ipv)
|
||||
}
|
||||
}
|
||||
if !useIPV4 && !useIPV6 {
|
||||
return errors.New("ipv4 and/or ipv6 must be selected")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ import (
|
|||
"github.com/projectdiscovery/nuclei/v2/pkg/projectfile"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/automaticscan"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/hosterrorscache"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
|
||||
|
|
@ -478,8 +479,8 @@ func (r *Runner) RunEnumeration() error {
|
|||
|
||||
func (r *Runner) isInputNonHTTP() bool {
|
||||
var nonURLInput bool
|
||||
r.hmapInputProvider.Scan(func(value string) bool {
|
||||
if !strings.Contains(value, "://") {
|
||||
r.hmapInputProvider.Scan(func(value *contextargs.MetaInput) bool {
|
||||
if !strings.Contains(value.Input, "://") {
|
||||
nonURLInput = true
|
||||
return false
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ package core
|
|||
|
||||
import (
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/types"
|
||||
)
|
||||
|
||||
|
|
@ -29,7 +30,7 @@ type InputProvider interface {
|
|||
Count() int64
|
||||
// Scan iterates the input and each found item is passed to the
|
||||
// callback consumer.
|
||||
Scan(callback func(value string) bool)
|
||||
Scan(callback func(value *contextargs.MetaInput) bool)
|
||||
}
|
||||
|
||||
// New returns a new Engine instance
|
||||
|
|
|
|||
|
|
@ -43,6 +43,8 @@ func (e *Engine) ExecuteWithOpts(templatesList []*templates.Template, target Inp
|
|||
|
||||
wg.Add()
|
||||
go func(tpl *templates.Template) {
|
||||
defer wg.Done()
|
||||
|
||||
switch {
|
||||
case tpl.SelfContained:
|
||||
// Self Contained requests are executed here separately
|
||||
|
|
@ -51,7 +53,6 @@ func (e *Engine) ExecuteWithOpts(templatesList []*templates.Template, target Inp
|
|||
// All other request types are executed here
|
||||
e.executeModelWithInput(templateType, tpl, target, results)
|
||||
}
|
||||
wg.Done()
|
||||
}(template)
|
||||
}
|
||||
e.workPool.Wait()
|
||||
|
|
@ -98,7 +99,7 @@ func (e *Engine) executeModelWithInput(templateType types.ProtocolType, template
|
|||
currentInfo.Unlock()
|
||||
}
|
||||
|
||||
target.Scan(func(scannedValue string) bool {
|
||||
target.Scan(func(scannedValue *contextargs.MetaInput) bool {
|
||||
// Best effort to track the host progression
|
||||
// skips indexes lower than the minimum in-flight at interruption time
|
||||
var skip bool
|
||||
|
|
@ -122,12 +123,12 @@ func (e *Engine) executeModelWithInput(templateType types.ProtocolType, template
|
|||
currentInfo.Unlock()
|
||||
|
||||
// Skip if the host has had errors
|
||||
if e.executerOpts.HostErrorsCache != nil && e.executerOpts.HostErrorsCache.Check(scannedValue) {
|
||||
if e.executerOpts.HostErrorsCache != nil && e.executerOpts.HostErrorsCache.Check(scannedValue.ID()) {
|
||||
return true
|
||||
}
|
||||
|
||||
wg.WaitGroup.Add()
|
||||
go func(index uint32, skip bool, value string) {
|
||||
go func(index uint32, skip bool, value *contextargs.MetaInput) {
|
||||
defer wg.WaitGroup.Done()
|
||||
defer cleanupInFlight(index)
|
||||
if skip {
|
||||
|
|
@ -141,7 +142,7 @@ func (e *Engine) executeModelWithInput(templateType types.ProtocolType, template
|
|||
match = e.executeWorkflow(value, template.CompiledWorkflow)
|
||||
default:
|
||||
ctxArgs := contextargs.New()
|
||||
ctxArgs.Input = value
|
||||
ctxArgs.MetaInput = value
|
||||
match, err = template.Executer.Execute(ctxArgs)
|
||||
}
|
||||
if err != nil {
|
||||
|
|
@ -188,14 +189,14 @@ func (e *Engine) ExecuteWithResults(templatesList []*templates.Template, target
|
|||
func (e *Engine) executeModelWithInputAndResult(templateType types.ProtocolType, template *templates.Template, target InputProvider, results *atomic.Bool, callback func(*output.ResultEvent)) {
|
||||
wg := e.workPool.InputPool(templateType)
|
||||
|
||||
target.Scan(func(scannedValue string) bool {
|
||||
target.Scan(func(scannedValue *contextargs.MetaInput) bool {
|
||||
// Skip if the host has had errors
|
||||
if e.executerOpts.HostErrorsCache != nil && e.executerOpts.HostErrorsCache.Check(scannedValue) {
|
||||
if e.executerOpts.HostErrorsCache != nil && e.executerOpts.HostErrorsCache.Check(scannedValue.ID()) {
|
||||
return true
|
||||
}
|
||||
|
||||
wg.WaitGroup.Add()
|
||||
go func(value string) {
|
||||
go func(value *contextargs.MetaInput) {
|
||||
defer wg.WaitGroup.Done()
|
||||
|
||||
var match bool
|
||||
|
|
@ -205,7 +206,7 @@ func (e *Engine) executeModelWithInputAndResult(templateType types.ProtocolType,
|
|||
match = e.executeWorkflow(value, template.CompiledWorkflow)
|
||||
default:
|
||||
ctxArgs := contextargs.New()
|
||||
ctxArgs.Input = value
|
||||
ctxArgs.MetaInput = value
|
||||
err = template.Executer.ExecuteWithResults(ctxArgs, func(event *output.InternalWrappedEvent) {
|
||||
for _, result := range event.Results {
|
||||
callback(result)
|
||||
|
|
@ -235,7 +236,7 @@ func (e *ChildExecuter) Close() *atomic.Bool {
|
|||
}
|
||||
|
||||
// Execute executes a template and URLs
|
||||
func (e *ChildExecuter) Execute(template *templates.Template, URL string) {
|
||||
func (e *ChildExecuter) Execute(template *templates.Template, value *contextargs.MetaInput) {
|
||||
templateType := template.Type()
|
||||
|
||||
var wg *sizedwaitgroup.SizedWaitGroup
|
||||
|
|
@ -247,14 +248,15 @@ func (e *ChildExecuter) Execute(template *templates.Template, URL string) {
|
|||
|
||||
wg.Add()
|
||||
go func(tpl *templates.Template) {
|
||||
defer wg.Done()
|
||||
|
||||
ctxArgs := contextargs.New()
|
||||
ctxArgs.Input = URL
|
||||
ctxArgs.MetaInput = value
|
||||
match, err := template.Executer.Execute(ctxArgs)
|
||||
if err != nil {
|
||||
gologger.Warning().Msgf("[%s] Could not execute step: %s\n", e.e.executerOpts.Colorizer.BrightBlue(template.ID), err)
|
||||
}
|
||||
e.results.CompareAndSwap(false, match)
|
||||
wg.Done()
|
||||
}(template)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ package hybrid
|
|||
import (
|
||||
"bufio"
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
|
@ -16,13 +17,17 @@ import (
|
|||
"github.com/projectdiscovery/hmap/store/hybrid"
|
||||
"github.com/projectdiscovery/mapcidr"
|
||||
asn "github.com/projectdiscovery/mapcidr/asn"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/types"
|
||||
fileutil "github.com/projectdiscovery/utils/file"
|
||||
iputil "github.com/projectdiscovery/utils/ip"
|
||||
sliceutil "github.com/projectdiscovery/utils/slice"
|
||||
)
|
||||
|
||||
// Input is a hmap/filekv backed nuclei Input provider
|
||||
type Input struct {
|
||||
ipOptions *ipOptions
|
||||
inputCount int64
|
||||
dupeCount int64
|
||||
hostMap *hybrid.HybridMap
|
||||
|
|
@ -37,7 +42,14 @@ func New(options *types.Options) (*Input, error) {
|
|||
return nil, errors.Wrap(err, "could not create temporary input file")
|
||||
}
|
||||
|
||||
input := &Input{hostMap: hm}
|
||||
input := &Input{
|
||||
hostMap: hm,
|
||||
ipOptions: &ipOptions{
|
||||
ScanAllIPs: options.ScanAllIPs,
|
||||
IPV4: sliceutil.Contains(options.IPVersion, "4"),
|
||||
IPV6: sliceutil.Contains(options.IPVersion, "6"),
|
||||
},
|
||||
}
|
||||
if options.Stream {
|
||||
fkvOptions := filekv.DefaultOptions
|
||||
if tmpFileName, err := fileutil.GetTempFileName(); err != nil {
|
||||
|
|
@ -72,15 +84,14 @@ func (i *Input) Close() {
|
|||
func (i *Input) initializeInputSources(options *types.Options) error {
|
||||
// Handle targets flags
|
||||
for _, target := range options.Targets {
|
||||
if iputil.IsCIDR(target) {
|
||||
switch {
|
||||
case iputil.IsCIDR(target):
|
||||
i.expandCIDRInputValue(target)
|
||||
continue
|
||||
}
|
||||
if asn.IsASN(target) {
|
||||
case asn.IsASN(target):
|
||||
i.expandASNInputValue(target)
|
||||
continue
|
||||
default:
|
||||
i.normalizeStoreInputValue(target)
|
||||
}
|
||||
i.normalizeStoreInputValue(target)
|
||||
}
|
||||
|
||||
// Handle stdin
|
||||
|
|
@ -94,8 +105,9 @@ func (i *Input) initializeInputSources(options *types.Options) error {
|
|||
if inputErr != nil {
|
||||
return errors.Wrap(inputErr, "could not open targets file")
|
||||
}
|
||||
defer input.Close()
|
||||
|
||||
i.scanInputFromReader(input)
|
||||
input.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -104,34 +116,89 @@ func (i *Input) initializeInputSources(options *types.Options) error {
|
|||
func (i *Input) scanInputFromReader(reader io.Reader) {
|
||||
scanner := bufio.NewScanner(reader)
|
||||
for scanner.Scan() {
|
||||
if iputil.IsCIDR(scanner.Text()) {
|
||||
i.expandCIDRInputValue(scanner.Text())
|
||||
continue
|
||||
item := scanner.Text()
|
||||
switch {
|
||||
case iputil.IsCIDR(item):
|
||||
i.expandCIDRInputValue(item)
|
||||
case asn.IsASN(item):
|
||||
i.expandASNInputValue(item)
|
||||
default:
|
||||
i.normalizeStoreInputValue(item)
|
||||
}
|
||||
if asn.IsASN(scanner.Text()) {
|
||||
i.expandASNInputValue(scanner.Text())
|
||||
continue
|
||||
}
|
||||
i.normalizeStoreInputValue(scanner.Text())
|
||||
}
|
||||
}
|
||||
|
||||
// normalizeStoreInputValue normalizes and stores passed input values
|
||||
func (i *Input) normalizeStoreInputValue(value string) {
|
||||
url := strings.TrimSpace(value)
|
||||
if url == "" {
|
||||
URL := strings.TrimSpace(value)
|
||||
if URL == "" {
|
||||
return
|
||||
}
|
||||
|
||||
if _, ok := i.hostMap.Get(url); ok {
|
||||
metaInput := &contextargs.MetaInput{Input: URL}
|
||||
keyURL, err := metaInput.MarshalString()
|
||||
if err != nil {
|
||||
gologger.Warning().Msgf("%s\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
if _, ok := i.hostMap.Get(keyURL); ok {
|
||||
i.dupeCount++
|
||||
return
|
||||
}
|
||||
|
||||
switch {
|
||||
case i.ipOptions.ScanAllIPs:
|
||||
// we need to resolve the hostname
|
||||
// check if it's an url
|
||||
var host string
|
||||
parsedURL, err := url.Parse(value)
|
||||
if err == nil && parsedURL.Host != "" {
|
||||
host = parsedURL.Host
|
||||
} else {
|
||||
parsedURL = nil
|
||||
host = value
|
||||
}
|
||||
|
||||
dnsData, err := protocolstate.Dialer.GetDNSData(host)
|
||||
if err == nil && (len(dnsData.A)+len(dnsData.AAAA)) > 0 {
|
||||
var ips []string
|
||||
if i.ipOptions.IPV4 {
|
||||
ips = append(ips, dnsData.A...)
|
||||
}
|
||||
if i.ipOptions.IPV6 {
|
||||
ips = append(ips, dnsData.AAAA...)
|
||||
}
|
||||
|
||||
for _, ip := range ips {
|
||||
if ip == "" {
|
||||
continue
|
||||
}
|
||||
metaInput := &contextargs.MetaInput{Input: value, CustomIP: ip}
|
||||
key, err := metaInput.MarshalString()
|
||||
if err != nil {
|
||||
gologger.Warning().Msgf("%s\n", err)
|
||||
continue
|
||||
}
|
||||
_ = i.hostMap.Set(key, nil)
|
||||
if i.hostMapStream != nil {
|
||||
_ = i.hostMapStream.Set([]byte(key), nil)
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
fallthrough
|
||||
default:
|
||||
i.setItem(keyURL)
|
||||
}
|
||||
}
|
||||
|
||||
// setItem in the kv store
|
||||
func (i *Input) setItem(k string) {
|
||||
i.inputCount++
|
||||
_ = i.hostMap.Set(url, nil)
|
||||
_ = i.hostMap.Set(k, nil)
|
||||
if i.hostMapStream != nil {
|
||||
_ = i.hostMapStream.Set([]byte(url), nil)
|
||||
_ = i.hostMapStream.Set([]byte(k), nil)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -142,9 +209,13 @@ func (i *Input) Count() int64 {
|
|||
|
||||
// Scan iterates the input and each found item is passed to the
|
||||
// callback consumer.
|
||||
func (i *Input) Scan(callback func(value string) bool) {
|
||||
func (i *Input) Scan(callback func(value *contextargs.MetaInput) bool) {
|
||||
callbackFunc := func(k, _ []byte) error {
|
||||
if !callback(string(k)) {
|
||||
metaInput := &contextargs.MetaInput{}
|
||||
if err := metaInput.Unmarshal(string(k)); err != nil {
|
||||
return err
|
||||
}
|
||||
if !callback(metaInput) {
|
||||
return io.EOF
|
||||
}
|
||||
return nil
|
||||
|
|
@ -160,14 +231,20 @@ func (i *Input) Scan(callback func(value string) bool) {
|
|||
func (i *Input) expandCIDRInputValue(value string) {
|
||||
ips, _ := mapcidr.IPAddressesAsStream(value)
|
||||
for ip := range ips {
|
||||
if _, ok := i.hostMap.Get(ip); ok {
|
||||
metaInput := &contextargs.MetaInput{Input: ip}
|
||||
key, err := metaInput.MarshalString()
|
||||
if err != nil {
|
||||
gologger.Warning().Msgf("%s\n", err)
|
||||
return
|
||||
}
|
||||
if _, ok := i.hostMap.Get(key); ok {
|
||||
i.dupeCount++
|
||||
continue
|
||||
}
|
||||
i.inputCount++
|
||||
_ = i.hostMap.Set(ip, nil)
|
||||
_ = i.hostMap.Set(key, nil)
|
||||
if i.hostMapStream != nil {
|
||||
_ = i.hostMapStream.Set([]byte(ip), nil)
|
||||
_ = i.hostMapStream.Set([]byte(key), nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,9 @@ import (
|
|||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/hmap/store/hybrid"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
|
|
@ -30,11 +33,73 @@ func Test_expandCIDRInputValue(t *testing.T) {
|
|||
input.expandCIDRInputValue(tt.cidr)
|
||||
// scan
|
||||
got := []string{}
|
||||
input.hostMap.Scan(func(k, v []byte) error {
|
||||
got = append(got, string(k))
|
||||
input.hostMap.Scan(func(k, _ []byte) error {
|
||||
var metainput contextargs.MetaInput
|
||||
if err := metainput.Unmarshal(string(k)); err != nil {
|
||||
return err
|
||||
}
|
||||
got = append(got, metainput.Input)
|
||||
return nil
|
||||
})
|
||||
require.ElementsMatch(t, tt.expected, got, "could not get correct cidrs")
|
||||
input.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func Test_scanallips_normalizeStoreInputValue(t *testing.T) {
|
||||
defaultOpts := types.DefaultOptions()
|
||||
_ = protocolstate.Init(defaultOpts)
|
||||
tests := []struct {
|
||||
hostname string
|
||||
ipv4 bool
|
||||
ipv6 bool
|
||||
expected []string
|
||||
}{
|
||||
{
|
||||
hostname: "scanme.sh",
|
||||
ipv4: true,
|
||||
ipv6: true,
|
||||
expected: []string{"128.199.158.128", "2400:6180:0:d0::91:1001"},
|
||||
}, {
|
||||
hostname: "scanme.sh",
|
||||
ipv4: true,
|
||||
expected: []string{"128.199.158.128"},
|
||||
}, {
|
||||
hostname: "scanme.sh",
|
||||
ipv6: true,
|
||||
expected: []string{"2400:6180:0:d0::91:1001"},
|
||||
}, {
|
||||
hostname: "http://scanme.sh",
|
||||
ipv4: true,
|
||||
ipv6: true,
|
||||
expected: []string{"128.199.158.128", "2400:6180:0:d0::91:1001"},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
hm, err := hybrid.New(hybrid.DefaultDiskOptions)
|
||||
require.Nil(t, err, "could not create temporary input file")
|
||||
input := &Input{
|
||||
hostMap: hm,
|
||||
ipOptions: &ipOptions{
|
||||
ScanAllIPs: true,
|
||||
IPV4: tt.ipv4,
|
||||
IPV6: tt.ipv6,
|
||||
},
|
||||
}
|
||||
|
||||
input.normalizeStoreInputValue(tt.hostname)
|
||||
// scan
|
||||
got := []string{}
|
||||
input.hostMap.Scan(func(k, v []byte) error {
|
||||
var metainput contextargs.MetaInput
|
||||
if err := metainput.Unmarshal(string(k)); err != nil {
|
||||
return err
|
||||
}
|
||||
got = append(got, metainput.CustomIP)
|
||||
return nil
|
||||
})
|
||||
require.ElementsMatch(t, tt.expected, got, "could not get correct ips")
|
||||
input.Close()
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -61,7 +126,11 @@ func Test_expandASNInputValue(t *testing.T) {
|
|||
// scan the hmap
|
||||
got := []string{}
|
||||
input.hostMap.Scan(func(k, v []byte) error {
|
||||
got = append(got, string(k))
|
||||
var metainput contextargs.MetaInput
|
||||
if err := metainput.Unmarshal(string(k)); err != nil {
|
||||
return err
|
||||
}
|
||||
got = append(got, metainput.Input)
|
||||
return nil
|
||||
})
|
||||
// read the expected IPs from the file
|
||||
|
|
|
|||
|
|
@ -0,0 +1,7 @@
|
|||
package hybrid
|
||||
|
||||
type ipOptions struct {
|
||||
ScanAllIPs bool
|
||||
IPV4 bool
|
||||
IPV6 bool
|
||||
}
|
||||
|
|
@ -1,7 +1,9 @@
|
|||
package inputs
|
||||
|
||||
import "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
|
||||
|
||||
type SimpleInputProvider struct {
|
||||
Inputs []string
|
||||
Inputs []*contextargs.MetaInput
|
||||
}
|
||||
|
||||
// Count returns the number of items for input provider
|
||||
|
|
@ -10,7 +12,7 @@ func (s *SimpleInputProvider) Count() int64 {
|
|||
}
|
||||
|
||||
// Scan calls a callback function till the input provider is exhausted
|
||||
func (s *SimpleInputProvider) Scan(callback func(value string) bool) {
|
||||
func (s *SimpleInputProvider) Scan(callback func(value *contextargs.MetaInput) bool) {
|
||||
for _, v := range s.Inputs {
|
||||
if !callback(v) {
|
||||
return
|
||||
|
|
|
|||
|
|
@ -16,13 +16,13 @@ import (
|
|||
const workflowStepExecutionError = "[%s] Could not execute workflow step: %s\n"
|
||||
|
||||
// executeWorkflow runs a workflow on an input and returns true or false
|
||||
func (e *Engine) executeWorkflow(input string, w *workflows.Workflow) bool {
|
||||
func (e *Engine) executeWorkflow(input *contextargs.MetaInput, w *workflows.Workflow) bool {
|
||||
results := &atomic.Bool{}
|
||||
|
||||
// at this point we should be at the start root execution of a workflow tree, hence we create global shared instances
|
||||
workflowCookieJar, _ := cookiejar.New(nil)
|
||||
ctxArgs := contextargs.New()
|
||||
ctxArgs.Input = input
|
||||
ctxArgs.MetaInput = input
|
||||
ctxArgs.CookieJar = workflowCookieJar
|
||||
|
||||
swg := sizedwaitgroup.New(w.Options.Options.TemplateThreads)
|
||||
|
|
@ -88,7 +88,7 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, input *co
|
|||
}
|
||||
if err != nil {
|
||||
if w.Options.HostErrorsCache != nil {
|
||||
w.Options.HostErrorsCache.MarkFailed(input.Input, err)
|
||||
w.Options.HostErrorsCache.MarkFailed(input.MetaInput.ID(), err)
|
||||
}
|
||||
if len(template.Executers) == 1 {
|
||||
mainErr = err
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ func TestWorkflowsSimple(t *testing.T) {
|
|||
}}
|
||||
|
||||
engine := &Engine{}
|
||||
matched := engine.executeWorkflow("https://test.com", workflow)
|
||||
matched := engine.executeWorkflow(&contextargs.MetaInput{Input: "https://test.com"}, workflow)
|
||||
require.True(t, matched, "could not get correct match value")
|
||||
}
|
||||
|
||||
|
|
@ -35,19 +35,19 @@ func TestWorkflowsSimpleMultiple(t *testing.T) {
|
|||
var firstInput, secondInput string
|
||||
workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{
|
||||
{Executers: []*workflows.ProtocolExecuterPair{{
|
||||
Executer: &mockExecuter{result: true, executeHook: func(input string) {
|
||||
firstInput = input
|
||||
Executer: &mockExecuter{result: true, executeHook: func(input *contextargs.MetaInput) {
|
||||
firstInput = input.Input
|
||||
}}, Options: &protocols.ExecuterOptions{Progress: progressBar}},
|
||||
}},
|
||||
{Executers: []*workflows.ProtocolExecuterPair{{
|
||||
Executer: &mockExecuter{result: true, executeHook: func(input string) {
|
||||
secondInput = input
|
||||
Executer: &mockExecuter{result: true, executeHook: func(input *contextargs.MetaInput) {
|
||||
secondInput = input.Input
|
||||
}}, Options: &protocols.ExecuterOptions{Progress: progressBar}},
|
||||
}},
|
||||
}}
|
||||
|
||||
engine := &Engine{}
|
||||
matched := engine.executeWorkflow("https://test.com", workflow)
|
||||
matched := engine.executeWorkflow(&contextargs.MetaInput{Input: "https://test.com"}, workflow)
|
||||
require.True(t, matched, "could not get correct match value")
|
||||
|
||||
require.Equal(t, "https://test.com", firstInput, "could not get correct first input")
|
||||
|
|
@ -60,20 +60,20 @@ func TestWorkflowsSubtemplates(t *testing.T) {
|
|||
var firstInput, secondInput string
|
||||
workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{
|
||||
{Executers: []*workflows.ProtocolExecuterPair{{
|
||||
Executer: &mockExecuter{result: true, executeHook: func(input string) {
|
||||
firstInput = input
|
||||
Executer: &mockExecuter{result: true, executeHook: func(input *contextargs.MetaInput) {
|
||||
firstInput = input.Input
|
||||
}, outputs: []*output.InternalWrappedEvent{
|
||||
{OperatorsResult: &operators.Result{}, Results: []*output.ResultEvent{{}}},
|
||||
}}, Options: &protocols.ExecuterOptions{Progress: progressBar}},
|
||||
}, Subtemplates: []*workflows.WorkflowTemplate{{Executers: []*workflows.ProtocolExecuterPair{{
|
||||
Executer: &mockExecuter{result: true, executeHook: func(input string) {
|
||||
secondInput = input
|
||||
Executer: &mockExecuter{result: true, executeHook: func(input *contextargs.MetaInput) {
|
||||
secondInput = input.Input
|
||||
}}, Options: &protocols.ExecuterOptions{Progress: progressBar}},
|
||||
}}}},
|
||||
}}
|
||||
|
||||
engine := &Engine{}
|
||||
matched := engine.executeWorkflow("https://test.com", workflow)
|
||||
matched := engine.executeWorkflow(&contextargs.MetaInput{Input: "https://test.com"}, workflow)
|
||||
require.True(t, matched, "could not get correct match value")
|
||||
|
||||
require.Equal(t, "https://test.com", firstInput, "could not get correct first input")
|
||||
|
|
@ -86,18 +86,18 @@ func TestWorkflowsSubtemplatesNoMatch(t *testing.T) {
|
|||
var firstInput, secondInput string
|
||||
workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{
|
||||
{Executers: []*workflows.ProtocolExecuterPair{{
|
||||
Executer: &mockExecuter{result: false, executeHook: func(input string) {
|
||||
firstInput = input
|
||||
Executer: &mockExecuter{result: false, executeHook: func(input *contextargs.MetaInput) {
|
||||
firstInput = input.Input
|
||||
}}, Options: &protocols.ExecuterOptions{Progress: progressBar}},
|
||||
}, Subtemplates: []*workflows.WorkflowTemplate{{Executers: []*workflows.ProtocolExecuterPair{{
|
||||
Executer: &mockExecuter{result: true, executeHook: func(input string) {
|
||||
secondInput = input
|
||||
Executer: &mockExecuter{result: true, executeHook: func(input *contextargs.MetaInput) {
|
||||
secondInput = input.Input
|
||||
}}, Options: &protocols.ExecuterOptions{Progress: progressBar}},
|
||||
}}}},
|
||||
}}
|
||||
|
||||
engine := &Engine{}
|
||||
matched := engine.executeWorkflow("https://test.com", workflow)
|
||||
matched := engine.executeWorkflow(&contextargs.MetaInput{Input: "https://test.com"}, workflow)
|
||||
require.False(t, matched, "could not get correct match value")
|
||||
|
||||
require.Equal(t, "https://test.com", firstInput, "could not get correct first input")
|
||||
|
|
@ -110,8 +110,8 @@ func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) {
|
|||
var firstInput, secondInput string
|
||||
workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{
|
||||
{Executers: []*workflows.ProtocolExecuterPair{{
|
||||
Executer: &mockExecuter{result: true, executeHook: func(input string) {
|
||||
firstInput = input
|
||||
Executer: &mockExecuter{result: true, executeHook: func(input *contextargs.MetaInput) {
|
||||
firstInput = input.Input
|
||||
}, outputs: []*output.InternalWrappedEvent{
|
||||
{OperatorsResult: &operators.Result{
|
||||
Matches: map[string][]string{"tomcat": {}},
|
||||
|
|
@ -119,14 +119,14 @@ func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) {
|
|||
}},
|
||||
}}, Options: &protocols.ExecuterOptions{Progress: progressBar}},
|
||||
}, Matchers: []*workflows.Matcher{{Name: stringslice.StringSlice{Value: "tomcat"}, Subtemplates: []*workflows.WorkflowTemplate{{Executers: []*workflows.ProtocolExecuterPair{{
|
||||
Executer: &mockExecuter{result: true, executeHook: func(input string) {
|
||||
secondInput = input
|
||||
Executer: &mockExecuter{result: true, executeHook: func(input *contextargs.MetaInput) {
|
||||
secondInput = input.Input
|
||||
}}, Options: &protocols.ExecuterOptions{Progress: progressBar}},
|
||||
}}}}}},
|
||||
}}
|
||||
|
||||
engine := &Engine{}
|
||||
matched := engine.executeWorkflow("https://test.com", workflow)
|
||||
matched := engine.executeWorkflow(&contextargs.MetaInput{Input: "https://test.com"}, workflow)
|
||||
require.True(t, matched, "could not get correct match value")
|
||||
|
||||
require.Equal(t, "https://test.com", firstInput, "could not get correct first input")
|
||||
|
|
@ -139,8 +139,8 @@ func TestWorkflowsSubtemplatesWithMatcherNoMatch(t *testing.T) {
|
|||
var firstInput, secondInput string
|
||||
workflow := &workflows.Workflow{Options: &protocols.ExecuterOptions{Options: &types.Options{TemplateThreads: 10}}, Workflows: []*workflows.WorkflowTemplate{
|
||||
{Executers: []*workflows.ProtocolExecuterPair{{
|
||||
Executer: &mockExecuter{result: true, executeHook: func(input string) {
|
||||
firstInput = input
|
||||
Executer: &mockExecuter{result: true, executeHook: func(input *contextargs.MetaInput) {
|
||||
firstInput = input.Input
|
||||
}, outputs: []*output.InternalWrappedEvent{
|
||||
{OperatorsResult: &operators.Result{
|
||||
Matches: map[string][]string{"tomcat": {}},
|
||||
|
|
@ -148,14 +148,14 @@ func TestWorkflowsSubtemplatesWithMatcherNoMatch(t *testing.T) {
|
|||
}},
|
||||
}}, Options: &protocols.ExecuterOptions{Progress: progressBar}},
|
||||
}, Matchers: []*workflows.Matcher{{Name: stringslice.StringSlice{Value: "apache"}, Subtemplates: []*workflows.WorkflowTemplate{{Executers: []*workflows.ProtocolExecuterPair{{
|
||||
Executer: &mockExecuter{result: true, executeHook: func(input string) {
|
||||
secondInput = input
|
||||
Executer: &mockExecuter{result: true, executeHook: func(input *contextargs.MetaInput) {
|
||||
secondInput = input.Input
|
||||
}}, Options: &protocols.ExecuterOptions{Progress: progressBar}},
|
||||
}}}}}},
|
||||
}}
|
||||
|
||||
engine := &Engine{}
|
||||
matched := engine.executeWorkflow("https://test.com", workflow)
|
||||
matched := engine.executeWorkflow(&contextargs.MetaInput{Input: "https://test.com"}, workflow)
|
||||
require.False(t, matched, "could not get correct match value")
|
||||
|
||||
require.Equal(t, "https://test.com", firstInput, "could not get correct first input")
|
||||
|
|
@ -164,7 +164,7 @@ func TestWorkflowsSubtemplatesWithMatcherNoMatch(t *testing.T) {
|
|||
|
||||
type mockExecuter struct {
|
||||
result bool
|
||||
executeHook func(input string)
|
||||
executeHook func(input *contextargs.MetaInput)
|
||||
outputs []*output.InternalWrappedEvent
|
||||
}
|
||||
|
||||
|
|
@ -181,7 +181,7 @@ func (m *mockExecuter) Requests() int {
|
|||
// Execute executes the protocol group and returns true or false if results were found.
|
||||
func (m *mockExecuter) Execute(input *contextargs.Context) (bool, error) {
|
||||
if m.executeHook != nil {
|
||||
m.executeHook(input.Input)
|
||||
m.executeHook(input.MetaInput)
|
||||
}
|
||||
return m.result, nil
|
||||
}
|
||||
|
|
@ -189,7 +189,7 @@ func (m *mockExecuter) Execute(input *contextargs.Context) (bool, error) {
|
|||
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
|
||||
func (m *mockExecuter) ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error {
|
||||
if m.executeHook != nil {
|
||||
m.executeHook(input.Input)
|
||||
m.executeHook(input.MetaInput)
|
||||
}
|
||||
for _, output := range m.outputs {
|
||||
callback(output)
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import (
|
|||
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/core"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
|
||||
"github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
|
||||
|
|
@ -132,11 +133,12 @@ func (s *Service) executeWappalyzerTechDetection() error {
|
|||
// Iterate through each target making http request and identifying fingerprints
|
||||
inputPool := s.engine.WorkPool().InputPool(types.HTTPProtocol)
|
||||
|
||||
s.target.Scan(func(value string) bool {
|
||||
s.target.Scan(func(value *contextargs.MetaInput) bool {
|
||||
inputPool.WaitGroup.Add()
|
||||
|
||||
go func(input string) {
|
||||
go func(input *contextargs.MetaInput) {
|
||||
defer inputPool.WaitGroup.Done()
|
||||
|
||||
s.processWappalyzerInputPair(input)
|
||||
}(value)
|
||||
return true
|
||||
|
|
@ -145,8 +147,8 @@ func (s *Service) executeWappalyzerTechDetection() error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) processWappalyzerInputPair(input string) {
|
||||
req, err := retryablehttp.NewRequest(http.MethodGet, input, nil)
|
||||
func (s *Service) processWappalyzerInputPair(input *contextargs.MetaInput) {
|
||||
req, err := retryablehttp.NewRequest(http.MethodGet, input.Input, nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,8 +9,9 @@ import (
|
|||
|
||||
// Context implements a shared context struct to share information across multiple templates within a workflow
|
||||
type Context struct {
|
||||
// Input target for the executor
|
||||
Input string
|
||||
// Meta is the target for the executor
|
||||
MetaInput *MetaInput
|
||||
|
||||
// CookieJar shared within workflow's http templates
|
||||
CookieJar *cookiejar.Jar
|
||||
|
||||
|
|
@ -22,12 +23,12 @@ type Context struct {
|
|||
|
||||
// Create a new contextargs instance
|
||||
func New() *Context {
|
||||
return &Context{}
|
||||
return &Context{MetaInput: &MetaInput{}}
|
||||
}
|
||||
|
||||
// Create a new contextargs instance with input string
|
||||
func NewWithInput(input string) *Context {
|
||||
return &Context{Input: input}
|
||||
return &Context{MetaInput: &MetaInput{Input: input}}
|
||||
}
|
||||
|
||||
func (ctx *Context) initialize() {
|
||||
|
|
@ -107,3 +108,13 @@ func (ctx *Context) Has(key string) bool {
|
|||
func (ctx *Context) HasArgs() bool {
|
||||
return ctx.hasArgs()
|
||||
}
|
||||
|
||||
func (ctx *Context) Clone() *Context {
|
||||
newCtx := &Context{
|
||||
MetaInput: ctx.MetaInput.Clone(),
|
||||
RWMutex: ctx.RWMutex,
|
||||
args: ctx.args,
|
||||
CookieJar: ctx.CookieJar,
|
||||
}
|
||||
return newCtx
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,69 @@
|
|||
package contextargs
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
)
|
||||
|
||||
// MetaInput represents a target with metadata (TODO: replace with https://github.com/projectdiscovery/metainput)
|
||||
type MetaInput struct {
|
||||
// Input represent the target
|
||||
Input string
|
||||
// CustomIP to use for connection
|
||||
CustomIP string
|
||||
}
|
||||
|
||||
func (metaInput *MetaInput) marshalToBuffer() (bytes.Buffer, error) {
|
||||
var b bytes.Buffer
|
||||
err := jsoniter.NewEncoder(&b).Encode(metaInput)
|
||||
return b, err
|
||||
}
|
||||
|
||||
// ID returns a unique id/hash for metainput
|
||||
func (metaInput *MetaInput) ID() string {
|
||||
if metaInput.CustomIP != "" {
|
||||
return fmt.Sprintf("%s-%s", metaInput.Input, metaInput.CustomIP)
|
||||
}
|
||||
return metaInput.Input
|
||||
}
|
||||
|
||||
func (metaInput *MetaInput) MarshalString() (string, error) {
|
||||
b, err := metaInput.marshalToBuffer()
|
||||
return b.String(), err
|
||||
}
|
||||
|
||||
func (metaInput *MetaInput) MustMarshalString() string {
|
||||
marshaled, _ := metaInput.MarshalString()
|
||||
return marshaled
|
||||
}
|
||||
|
||||
func (metaInput *MetaInput) MarshalBytes() ([]byte, error) {
|
||||
b, err := metaInput.marshalToBuffer()
|
||||
return b.Bytes(), err
|
||||
}
|
||||
|
||||
func (metaInput *MetaInput) MustMarshalBytes() []byte {
|
||||
marshaled, _ := metaInput.MarshalBytes()
|
||||
return marshaled
|
||||
}
|
||||
|
||||
func (metaInput *MetaInput) Unmarshal(data string) error {
|
||||
return jsoniter.NewDecoder(strings.NewReader(data)).Decode(metaInput)
|
||||
}
|
||||
|
||||
func (metaInput *MetaInput) Clone() *MetaInput {
|
||||
return &MetaInput{
|
||||
Input: metaInput.Input,
|
||||
CustomIP: metaInput.CustomIP,
|
||||
}
|
||||
}
|
||||
|
||||
func (metaInput *MetaInput) PrettyPrint() string {
|
||||
if metaInput.CustomIP != "" {
|
||||
return fmt.Sprintf("%s [%s]", metaInput.Input, metaInput.CustomIP)
|
||||
}
|
||||
return metaInput.Input
|
||||
}
|
||||
|
|
@ -70,14 +70,14 @@ func (e *Executer) Execute(input *contextargs.Context) (bool, error) {
|
|||
}
|
||||
previous := make(map[string]interface{})
|
||||
for _, req := range e.requests {
|
||||
inputItem := *input
|
||||
if e.options.InputHelper != nil && input.Input != "" {
|
||||
if inputItem.Input = e.options.InputHelper.Transform(input.Input, req.Type()); inputItem.Input == "" {
|
||||
inputItem := input.Clone()
|
||||
if e.options.InputHelper != nil && input.MetaInput.Input != "" {
|
||||
if inputItem.MetaInput.Input = e.options.InputHelper.Transform(input.MetaInput.Input, req.Type()); input.MetaInput.Input == "" {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
|
||||
err := req.ExecuteWithResults(&inputItem, dynamicValues, previous, func(event *output.InternalWrappedEvent) {
|
||||
err := req.ExecuteWithResults(inputItem, dynamicValues, previous, func(event *output.InternalWrappedEvent) {
|
||||
ID := req.GetID()
|
||||
if ID != "" {
|
||||
builder := &strings.Builder{}
|
||||
|
|
@ -108,9 +108,9 @@ func (e *Executer) Execute(input *contextargs.Context) (bool, error) {
|
|||
})
|
||||
if err != nil {
|
||||
if e.options.HostErrorsCache != nil {
|
||||
e.options.HostErrorsCache.MarkFailed(input.Input, err)
|
||||
e.options.HostErrorsCache.MarkFailed(input.MetaInput.ID(), err)
|
||||
}
|
||||
gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input.Input, err)
|
||||
gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input.MetaInput.PrettyPrint(), err)
|
||||
}
|
||||
// If a match was found and stop at first match is set, break out of the loop and return
|
||||
if results && (e.options.StopAtFirstMatch || e.options.Options.StopAtFirstMatch) {
|
||||
|
|
@ -134,14 +134,14 @@ func (e *Executer) ExecuteWithResults(input *contextargs.Context, callback proto
|
|||
for _, req := range e.requests {
|
||||
req := req
|
||||
|
||||
inputItem := *input
|
||||
if e.options.InputHelper != nil && input.Input != "" {
|
||||
if inputItem.Input = e.options.InputHelper.Transform(input.Input, req.Type()); inputItem.Input == "" {
|
||||
inputItem := input.Clone()
|
||||
if e.options.InputHelper != nil && input.MetaInput.Input != "" {
|
||||
if inputItem.MetaInput.Input = e.options.InputHelper.Transform(input.MetaInput.Input, req.Type()); input.MetaInput.Input == "" {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
err := req.ExecuteWithResults(&inputItem, dynamicValues, previous, func(event *output.InternalWrappedEvent) {
|
||||
err := req.ExecuteWithResults(inputItem, dynamicValues, previous, func(event *output.InternalWrappedEvent) {
|
||||
ID := req.GetID()
|
||||
if ID != "" {
|
||||
builder := &strings.Builder{}
|
||||
|
|
@ -161,9 +161,9 @@ func (e *Executer) ExecuteWithResults(input *contextargs.Context, callback proto
|
|||
})
|
||||
if err != nil {
|
||||
if e.options.HostErrorsCache != nil {
|
||||
e.options.HostErrorsCache.MarkFailed(input.Input, err)
|
||||
e.options.HostErrorsCache.MarkFailed(input.MetaInput.ID(), err)
|
||||
}
|
||||
gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input.Input, err)
|
||||
gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input.MetaInput.PrettyPrint(), err)
|
||||
}
|
||||
// If a match was found and stop at first match is set, break out of the loop and return
|
||||
if results && (e.options.StopAtFirstMatch || e.options.Options.StopAtFirstMatch) {
|
||||
|
|
|
|||
|
|
@ -34,10 +34,10 @@ func (request *Request) Type() templateTypes.ProtocolType {
|
|||
func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
|
||||
// Parse the URL and return domain if URL.
|
||||
var domain string
|
||||
if utils.IsURL(input.Input) {
|
||||
domain = extractDomain(input.Input)
|
||||
if utils.IsURL(input.MetaInput.Input) {
|
||||
domain = extractDomain(input.MetaInput.Input)
|
||||
} else {
|
||||
domain = input.Input
|
||||
domain = input.MetaInput.Input
|
||||
}
|
||||
|
||||
var err error
|
||||
|
|
@ -112,7 +112,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
|
|||
}
|
||||
}
|
||||
|
||||
outputEvent := request.responseToDSLMap(compiledRequest, response, input.Input, input.Input, traceData)
|
||||
outputEvent := request.responseToDSLMap(compiledRequest, response, input.MetaInput.Input, input.MetaInput.Input, traceData)
|
||||
for k, v := range previous {
|
||||
outputEvent[k] = v
|
||||
}
|
||||
|
|
|
|||
|
|
@ -54,8 +54,7 @@ func TestDNSExecuteWithResults(t *testing.T) {
|
|||
t.Run("domain-valid", func(t *testing.T) {
|
||||
metadata := make(output.InternalEvent)
|
||||
previous := make(output.InternalEvent)
|
||||
ctxArgs := contextargs.New()
|
||||
ctxArgs.Input = "example.com"
|
||||
ctxArgs := contextargs.NewWithInput("example.com")
|
||||
err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
|
||||
finalEvent = event
|
||||
})
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ var emptyResultErr = errors.New("Empty result")
|
|||
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
|
||||
func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
|
||||
wg := sizedwaitgroup.New(request.options.Options.BulkSize)
|
||||
err := request.getInputPaths(input.Input, func(filePath string) {
|
||||
err := request.getInputPaths(input.MetaInput.Input, func(filePath string) {
|
||||
wg.Add()
|
||||
func(filePath string) {
|
||||
defer wg.Done()
|
||||
|
|
@ -63,7 +63,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
|
|||
// every new file in the compressed multi-file archive counts 1
|
||||
request.options.Progress.AddToTotal(1)
|
||||
archiveFileName := filepath.Join(filePath, file.Name())
|
||||
event, fileMatches, err := request.processReader(file.ReadCloser, archiveFileName, input.Input, file.Size(), previous)
|
||||
event, fileMatches, err := request.processReader(file.ReadCloser, archiveFileName, input.MetaInput.Input, file.Size(), previous)
|
||||
if err != nil {
|
||||
if errors.Is(err, emptyResultErr) {
|
||||
// no matches but one file elaborated
|
||||
|
|
@ -116,7 +116,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
|
|||
_ = tmpFileOut.Sync()
|
||||
// rewind the file
|
||||
_, _ = tmpFileOut.Seek(0, 0)
|
||||
event, fileMatches, err := request.processReader(tmpFileOut, filePath, input.Input, fileStat.Size(), previous)
|
||||
event, fileMatches, err := request.processReader(tmpFileOut, filePath, input.MetaInput.Input, fileStat.Size(), previous)
|
||||
if err != nil {
|
||||
if errors.Is(err, emptyResultErr) {
|
||||
// no matches but one file elaborated
|
||||
|
|
@ -136,7 +136,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
|
|||
default:
|
||||
// normal file - increments the counter by 1
|
||||
request.options.Progress.AddToTotal(1)
|
||||
event, fileMatches, err := request.processFile(filePath, input.Input, previous)
|
||||
event, fileMatches, err := request.processFile(filePath, input.MetaInput.Input, previous)
|
||||
if err != nil {
|
||||
if errors.Is(err, emptyResultErr) {
|
||||
// no matches but one file elaborated
|
||||
|
|
@ -158,7 +158,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
|
|||
|
||||
wg.Wait()
|
||||
if err != nil {
|
||||
request.options.Output.Request(request.options.TemplatePath, input.Input, request.Type().String(), err)
|
||||
request.options.Output.Request(request.options.TemplatePath, input.MetaInput.Input, request.Type().String(), err)
|
||||
request.options.Progress.IncrementFailedRequestsBy(1)
|
||||
return errors.Wrap(err, "could not send file request")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -66,8 +66,7 @@ func TestFileExecuteWithResults(t *testing.T) {
|
|||
t.Run("valid", func(t *testing.T) {
|
||||
metadata := make(output.InternalEvent)
|
||||
previous := make(output.InternalEvent)
|
||||
ctxArgs := contextargs.New()
|
||||
ctxArgs.Input = tempDir
|
||||
ctxArgs := contextargs.NewWithInput(tempDir)
|
||||
err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
|
||||
finalEvent = event
|
||||
})
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ func (request *Request) Type() templateTypes.ProtocolType {
|
|||
|
||||
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
|
||||
func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, previous output.InternalEvent /*TODO review unused parameter*/, callback protocols.OutputEventCallback) error {
|
||||
inputURL := input.Input
|
||||
inputURL := input.MetaInput.Input
|
||||
if request.options.Browser.UserAgent() == "" {
|
||||
request.options.Browser.SetUserAgent(request.compiledUserAgent)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ func (request *Request) Type() templateTypes.ProtocolType {
|
|||
|
||||
// executeRaceRequest executes race condition request for a URL
|
||||
func (request *Request) executeRaceRequest(input *contextargs.Context, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
|
||||
reqURL := input.Input
|
||||
reqURL := input.MetaInput.Input
|
||||
var generatedRequests []*generatedRequest
|
||||
|
||||
// Requests within race condition should be dumped once and the output prefilled to allow DSL language to work
|
||||
|
|
@ -59,7 +59,8 @@ func (request *Request) executeRaceRequest(input *contextargs.Context, previous
|
|||
if !ok {
|
||||
return nil
|
||||
}
|
||||
requestForDump, err := generator.Make(context.Background(), reqURL, inputData, payloads, nil)
|
||||
ctx := request.newContext(input)
|
||||
requestForDump, err := generator.Make(ctx, reqURL, inputData, payloads, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -87,7 +88,8 @@ func (request *Request) executeRaceRequest(input *contextargs.Context, previous
|
|||
if !ok {
|
||||
break
|
||||
}
|
||||
generatedRequest, err := generator.Make(context.Background(), reqURL, inputData, payloads, nil)
|
||||
ctx := request.newContext(input)
|
||||
generatedRequest, err := generator.Make(ctx, reqURL, inputData, payloads, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -130,7 +132,8 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
|
|||
if !ok {
|
||||
break
|
||||
}
|
||||
generatedHttpRequest, err := generator.Make(context.Background(), input.Input, inputData, payloads, dynamicValues)
|
||||
ctx := request.newContext(input)
|
||||
generatedHttpRequest, err := generator.Make(ctx, input.MetaInput.Input, inputData, payloads, dynamicValues)
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
break
|
||||
|
|
@ -138,8 +141,8 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
|
|||
request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
|
||||
return err
|
||||
}
|
||||
if input.Input == "" {
|
||||
input.Input = generatedHttpRequest.URL()
|
||||
if input.MetaInput.Input == "" {
|
||||
input.MetaInput.Input = generatedHttpRequest.URL()
|
||||
}
|
||||
swg.Add()
|
||||
go func(httpRequest *generatedRequest) {
|
||||
|
|
@ -166,7 +169,7 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
|
|||
generator := request.newGenerator(false)
|
||||
|
||||
// need to extract the target from the url
|
||||
URL, err := url.Parse(input.Input)
|
||||
URL, err := url.Parse(input.MetaInput.Input)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -197,13 +200,14 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
|
|||
if !ok {
|
||||
break
|
||||
}
|
||||
generatedHttpRequest, err := generator.Make(context.Background(), input.Input, inputData, payloads, dynamicValues)
|
||||
ctx := request.newContext(input)
|
||||
generatedHttpRequest, err := generator.Make(ctx, input.MetaInput.Input, inputData, payloads, dynamicValues)
|
||||
if err != nil {
|
||||
request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
|
||||
return err
|
||||
}
|
||||
if input.Input == "" {
|
||||
input.Input = generatedHttpRequest.URL()
|
||||
if input.MetaInput.Input == "" {
|
||||
input.MetaInput.Input = generatedHttpRequest.URL()
|
||||
}
|
||||
generatedHttpRequest.pipelinedClient = pipeClient
|
||||
swg.Add()
|
||||
|
|
@ -225,14 +229,14 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
|
|||
|
||||
// executeFuzzingRule executes fuzzing request for a URL
|
||||
func (request *Request) executeFuzzingRule(input *contextargs.Context, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
|
||||
parsed, err := url.Parse(input.Input)
|
||||
parsed, err := url.Parse(input.MetaInput.Input)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "could not parse url")
|
||||
}
|
||||
fuzzRequestCallback := func(gr fuzz.GeneratedRequest) bool {
|
||||
hasInteractMatchers := interactsh.HasMatchers(request.CompiledOperators)
|
||||
hasInteractMarkers := len(gr.InteractURLs) > 0
|
||||
if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(input.Input) {
|
||||
if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(input.MetaInput.Input) {
|
||||
return false
|
||||
}
|
||||
|
||||
|
|
@ -266,7 +270,7 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, previous
|
|||
}
|
||||
if requestErr != nil {
|
||||
if request.options.HostErrorsCache != nil {
|
||||
request.options.HostErrorsCache.MarkFailed(input.Input, requestErr)
|
||||
request.options.HostErrorsCache.MarkFailed(input.MetaInput.Input, requestErr)
|
||||
}
|
||||
}
|
||||
request.options.Progress.IncrementRequests()
|
||||
|
|
@ -285,7 +289,7 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, previous
|
|||
if !result {
|
||||
break
|
||||
}
|
||||
generated, err := generator.Make(context.Background(), input.Input, value, payloads, nil)
|
||||
generated, err := generator.Make(context.Background(), input.MetaInput.Input, value, payloads, nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
|
@ -346,10 +350,11 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
|
|||
|
||||
request.options.RateLimiter.Take()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(request.options.Options.Timeout)*time.Second)
|
||||
ctx := request.newContext(input)
|
||||
ctxWithTimeout, cancel := context.WithTimeout(ctx, time.Duration(request.options.Options.Timeout)*time.Second)
|
||||
defer cancel()
|
||||
|
||||
generatedHttpRequest, err := generator.Make(ctx, input.Input, data, payloads, dynamicValue)
|
||||
generatedHttpRequest, err := generator.Make(ctxWithTimeout, input.MetaInput.Input, data, payloads, dynamicValue)
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
return true, nil
|
||||
|
|
@ -367,11 +372,11 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
|
|||
generatedHttpRequest.interactshURLs = append(generatedHttpRequest.interactshURLs, interactURLs...)
|
||||
}
|
||||
hasInteractMarkers := interactsh.HasMarkers(data) || len(generatedHttpRequest.interactshURLs) > 0
|
||||
if input.Input == "" {
|
||||
input.Input = generatedHttpRequest.URL()
|
||||
if input.MetaInput.Input == "" {
|
||||
input.MetaInput.Input = generatedHttpRequest.URL()
|
||||
}
|
||||
// Check if hosts keep erroring
|
||||
if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(input.Input) {
|
||||
if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(input.MetaInput.ID()) {
|
||||
return true, nil
|
||||
}
|
||||
var gotMatches bool
|
||||
|
|
@ -400,7 +405,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
|
|||
}
|
||||
if err != nil {
|
||||
if request.options.HostErrorsCache != nil {
|
||||
request.options.HostErrorsCache.MarkFailed(input.Input, err)
|
||||
request.options.HostErrorsCache.MarkFailed(input.MetaInput.ID(), err)
|
||||
}
|
||||
requestErr = err
|
||||
}
|
||||
|
|
@ -472,7 +477,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
|
|||
if !generatedRequest.original.Race {
|
||||
var dumpError error
|
||||
// TODO: dump is currently not working with post-processors - somehow it alters the signature
|
||||
dumpedRequest, dumpError = dump(generatedRequest, input.Input)
|
||||
dumpedRequest, dumpError = dump(generatedRequest, input.MetaInput.Input)
|
||||
if dumpError != nil {
|
||||
return dumpError
|
||||
}
|
||||
|
|
@ -480,12 +485,12 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
|
|||
|
||||
if ignoreList := GetVariablesNamesSkipList(generatedRequest.original.Signature.Value); ignoreList != nil {
|
||||
if varErr := expressions.ContainsVariablesWithIgnoreList(ignoreList, dumpedRequestString); varErr != nil && !request.SkipVariablesCheck {
|
||||
gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, input.Input, varErr)
|
||||
gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, input.MetaInput.Input, varErr)
|
||||
return errStopExecution
|
||||
}
|
||||
} else { // Check if are there any unresolved variables. If yes, skip unless overridden by user.
|
||||
if varErr := expressions.ContainsUnresolvedVariables(dumpedRequestString); varErr != nil && !request.SkipVariablesCheck {
|
||||
gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, input.Input, varErr)
|
||||
gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, input.MetaInput.Input, varErr)
|
||||
return errStopExecution
|
||||
}
|
||||
}
|
||||
|
|
@ -499,7 +504,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
|
|||
if parsed, parseErr := url.Parse(formedURL); parseErr == nil {
|
||||
hostname = parsed.Host
|
||||
}
|
||||
resp, err = generatedRequest.pipelinedClient.DoRaw(generatedRequest.rawRequest.Method, input.Input, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)))
|
||||
resp, err = generatedRequest.pipelinedClient.DoRaw(generatedRequest.rawRequest.Method, input.MetaInput.Input, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)))
|
||||
} else if generatedRequest.request != nil {
|
||||
resp, err = generatedRequest.pipelinedClient.Dor(generatedRequest.request)
|
||||
}
|
||||
|
|
@ -507,7 +512,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
|
|||
formedURL = generatedRequest.rawRequest.FullURL
|
||||
// use request url as matched url if empty
|
||||
if formedURL == "" {
|
||||
formedURL = input.Input
|
||||
formedURL = input.MetaInput.Input
|
||||
if generatedRequest.rawRequest.Path != "" {
|
||||
formedURL = fmt.Sprintf("%s%s", formedURL, generatedRequest.rawRequest.Path)
|
||||
}
|
||||
|
|
@ -520,7 +525,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
|
|||
options.CustomRawBytes = generatedRequest.rawRequest.UnsafeRawBytes
|
||||
options.ForceReadAllBody = request.ForceReadAllBody
|
||||
options.SNI = request.options.Options.SNI
|
||||
resp, err = generatedRequest.original.rawhttpClient.DoRawWithOptions(generatedRequest.rawRequest.Method, input.Input, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)), &options)
|
||||
resp, err = generatedRequest.original.rawhttpClient.DoRawWithOptions(generatedRequest.rawRequest.Method, input.MetaInput.Input, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)), &options)
|
||||
} else {
|
||||
hostname = generatedRequest.request.URL.Host
|
||||
formedURL = generatedRequest.request.URL.String()
|
||||
|
|
@ -553,13 +558,13 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
|
|||
}
|
||||
// use request url as matched url if empty
|
||||
if formedURL == "" {
|
||||
formedURL = input.Input
|
||||
formedURL = input.MetaInput.Input
|
||||
}
|
||||
|
||||
// Dump the requests containing all headers
|
||||
if !generatedRequest.original.Race {
|
||||
var dumpError error
|
||||
dumpedRequest, dumpError = dump(generatedRequest, input.Input)
|
||||
dumpedRequest, dumpError = dump(generatedRequest, input.MetaInput.Input)
|
||||
if dumpError != nil {
|
||||
return dumpError
|
||||
}
|
||||
|
|
@ -572,7 +577,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
|
|||
gologger.Print().Msgf("%s", dumpedRequestString)
|
||||
}
|
||||
if request.options.Options.StoreResponse {
|
||||
request.options.Output.WriteStoreDebugData(input.Input, request.options.TemplateID, request.Type().String(), fmt.Sprintf("%s\n%s", msg, dumpedRequestString))
|
||||
request.options.Output.WriteStoreDebugData(input.MetaInput.Input, request.options.TemplateID, request.Type().String(), fmt.Sprintf("%s\n%s", msg, dumpedRequestString))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -588,11 +593,16 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
|
|||
// If we have interactsh markers and request times out, still send
|
||||
// a callback event so in case we receive an interaction, correlation is possible.
|
||||
if hasInteractMatchers {
|
||||
outputEvent := request.responseToDSLMap(&http.Response{}, input.Input, formedURL, tostring.UnsafeToString(dumpedRequest), "", "", "", 0, generatedRequest.meta)
|
||||
outputEvent := request.responseToDSLMap(&http.Response{}, input.MetaInput.Input, formedURL, tostring.UnsafeToString(dumpedRequest), "", "", "", 0, generatedRequest.meta)
|
||||
if i := strings.LastIndex(hostname, ":"); i != -1 {
|
||||
hostname = hostname[:i]
|
||||
}
|
||||
outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname)
|
||||
|
||||
if input.MetaInput.CustomIP != "" {
|
||||
outputEvent["ip"] = input.MetaInput.CustomIP
|
||||
} else {
|
||||
outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname)
|
||||
}
|
||||
|
||||
event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
|
||||
if request.CompiledOperators != nil {
|
||||
|
|
@ -672,7 +682,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
|
|||
if response.resp == nil {
|
||||
continue // Skip nil responses
|
||||
}
|
||||
matchedURL := input.Input
|
||||
matchedURL := input.MetaInput.Input
|
||||
if generatedRequest.rawRequest != nil && generatedRequest.rawRequest.FullURL != "" {
|
||||
matchedURL = generatedRequest.rawRequest.FullURL
|
||||
}
|
||||
|
|
@ -687,12 +697,16 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
|
|||
}
|
||||
finalEvent := make(output.InternalEvent)
|
||||
|
||||
outputEvent := request.responseToDSLMap(response.resp, input.Input, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(response.fullResponse), tostring.UnsafeToString(response.body), tostring.UnsafeToString(response.headers), duration, generatedRequest.meta)
|
||||
outputEvent := request.responseToDSLMap(response.resp, input.MetaInput.Input, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(response.fullResponse), tostring.UnsafeToString(response.body), tostring.UnsafeToString(response.headers), duration, generatedRequest.meta)
|
||||
if i := strings.LastIndex(hostname, ":"); i != -1 {
|
||||
hostname = hostname[:i]
|
||||
}
|
||||
outputEvent["curl-command"] = curlCommand
|
||||
outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname)
|
||||
if input.MetaInput.CustomIP != "" {
|
||||
outputEvent["ip"] = input.MetaInput.CustomIP
|
||||
} else {
|
||||
outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname)
|
||||
}
|
||||
if request.options.Interactsh != nil {
|
||||
request.options.Interactsh.MakePlaceholders(generatedRequest.interactshURLs, outputEvent)
|
||||
}
|
||||
|
|
@ -723,7 +737,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
|
|||
|
||||
responseContentType := resp.Header.Get("Content-Type")
|
||||
isResponseTruncated := request.MaxSize > 0 && len(gotData) >= request.MaxSize
|
||||
dumpResponse(event, request, response.fullResponse, formedURL, responseContentType, isResponseTruncated, input.Input)
|
||||
dumpResponse(event, request, response.fullResponse, formedURL, responseContentType, isResponseTruncated, input.MetaInput.Input)
|
||||
|
||||
callback(event)
|
||||
|
||||
|
|
@ -842,3 +856,10 @@ func (request *Request) pruneSignatureInternalValues(maps ...map[string]interfac
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (request *Request) newContext(input *contextargs.Context) context.Context {
|
||||
if input.MetaInput.CustomIP != "" {
|
||||
return context.WithValue(context.Background(), "ip", input.MetaInput.CustomIP) //nolint
|
||||
}
|
||||
return context.Background()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -82,8 +82,7 @@ Disallow: /c`))
|
|||
t.Run("test", func(t *testing.T) {
|
||||
metadata := make(output.InternalEvent)
|
||||
previous := make(output.InternalEvent)
|
||||
ctxArgs := contextargs.New()
|
||||
ctxArgs.Input = ts.URL
|
||||
ctxArgs := contextargs.NewWithInput(ts.URL)
|
||||
err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
|
||||
if event.OperatorsResult != nil && event.OperatorsResult.Matched {
|
||||
matchCount++
|
||||
|
|
|
|||
|
|
@ -43,10 +43,10 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata
|
|||
if request.SelfContained {
|
||||
address = ""
|
||||
} else {
|
||||
address, err = getAddress(input.Input)
|
||||
address, err = getAddress(input.MetaInput.Input)
|
||||
}
|
||||
if err != nil {
|
||||
request.options.Output.Request(request.options.TemplatePath, input.Input, request.Type().String(), err)
|
||||
request.options.Output.Request(request.options.TemplatePath, input.MetaInput.Input, request.Type().String(), err)
|
||||
request.options.Progress.IncrementFailedRequestsBy(1)
|
||||
return errors.Wrap(err, "could not get address from url")
|
||||
}
|
||||
|
|
@ -57,7 +57,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata
|
|||
variables = generators.MergeMaps(variablesMap, variables)
|
||||
actualAddress := replacer.Replace(kv.address, variables)
|
||||
|
||||
if err := request.executeAddress(variables, actualAddress, address, input.Input, kv.tls, previous, callback); err != nil {
|
||||
if err := request.executeAddress(variables, actualAddress, address, input.MetaInput.Input, kv.tls, previous, callback); err != nil {
|
||||
gologger.Warning().Msgf("Could not make network request for %s: %s\n", actualAddress, err)
|
||||
continue
|
||||
}
|
||||
|
|
|
|||
|
|
@ -65,8 +65,7 @@ func TestNetworkExecuteWithResults(t *testing.T) {
|
|||
t.Run("domain-valid", func(t *testing.T) {
|
||||
metadata := make(output.InternalEvent)
|
||||
previous := make(output.InternalEvent)
|
||||
ctxArgs := contextargs.New()
|
||||
ctxArgs.Input = parsed.Host
|
||||
ctxArgs := contextargs.NewWithInput(parsed.Host)
|
||||
err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
|
||||
finalEvent = event
|
||||
})
|
||||
|
|
@ -82,8 +81,7 @@ func TestNetworkExecuteWithResults(t *testing.T) {
|
|||
t.Run("invalid-port-override", func(t *testing.T) {
|
||||
metadata := make(output.InternalEvent)
|
||||
previous := make(output.InternalEvent)
|
||||
ctxArgs := contextargs.New()
|
||||
ctxArgs.Input = "127.0.0.1:11211"
|
||||
ctxArgs := contextargs.NewWithInput("127.0.0.1:11211")
|
||||
err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
|
||||
finalEvent = event
|
||||
})
|
||||
|
|
@ -97,8 +95,7 @@ func TestNetworkExecuteWithResults(t *testing.T) {
|
|||
t.Run("hex-to-string", func(t *testing.T) {
|
||||
metadata := make(output.InternalEvent)
|
||||
previous := make(output.InternalEvent)
|
||||
ctxArgs := contextargs.New()
|
||||
ctxArgs.Input = parsed.Host
|
||||
ctxArgs := contextargs.NewWithInput(parsed.Host)
|
||||
err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
|
||||
finalEvent = event
|
||||
})
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ func (request *Request) Type() templateTypes.ProtocolType {
|
|||
func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
|
||||
wg := sizedwaitgroup.New(request.options.Options.BulkSize)
|
||||
|
||||
err := request.getInputPaths(input.Input, func(data string) {
|
||||
err := request.getInputPaths(input.MetaInput.Input, func(data string) {
|
||||
wg.Add()
|
||||
|
||||
go func(data string) {
|
||||
|
|
@ -98,7 +98,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata
|
|||
})
|
||||
wg.Wait()
|
||||
if err != nil {
|
||||
request.options.Output.Request(request.options.TemplatePath, input.Input, "file", err)
|
||||
request.options.Output.Request(request.options.TemplatePath, input.MetaInput.Input, "file", err)
|
||||
request.options.Progress.IncrementFailedRequestsBy(1)
|
||||
return errors.Wrap(err, "could not send file request")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -127,7 +127,7 @@ func (request *Request) GetID() string {
|
|||
|
||||
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
|
||||
func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
|
||||
address, err := getAddress(input.Input)
|
||||
address, err := getAddress(input.MetaInput.Input)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
|
@ -153,7 +153,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
|
|||
|
||||
finalAddress, dataErr := expressions.EvaluateByte([]byte(request.Address), payloadValues)
|
||||
if dataErr != nil {
|
||||
requestOptions.Output.Request(requestOptions.TemplateID, input.Input, request.Type().String(), dataErr)
|
||||
requestOptions.Output.Request(requestOptions.TemplateID, input.MetaInput.Input, request.Type().String(), dataErr)
|
||||
requestOptions.Progress.IncrementFailedRequestsBy(1)
|
||||
return errors.Wrap(dataErr, "could not evaluate template expressions")
|
||||
}
|
||||
|
|
@ -165,7 +165,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
|
|||
|
||||
response, err := request.tlsx.Connect(host, host, port)
|
||||
if err != nil {
|
||||
requestOptions.Output.Request(requestOptions.TemplateID, input.Input, request.Type().String(), err)
|
||||
requestOptions.Output.Request(requestOptions.TemplateID, input.MetaInput.Input, request.Type().String(), err)
|
||||
requestOptions.Progress.IncrementFailedRequestsBy(1)
|
||||
return errors.Wrap(err, "could not connect to server")
|
||||
}
|
||||
|
|
@ -174,12 +174,12 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
|
|||
gologger.Verbose().Msgf("Sent SSL request to %s", address)
|
||||
|
||||
if requestOptions.Options.Debug || requestOptions.Options.DebugRequests || requestOptions.Options.StoreResponse {
|
||||
msg := fmt.Sprintf("[%s] Dumped SSL request for %s", requestOptions.TemplateID, input.Input)
|
||||
msg := fmt.Sprintf("[%s] Dumped SSL request for %s", requestOptions.TemplateID, input.MetaInput.Input)
|
||||
if requestOptions.Options.Debug || requestOptions.Options.DebugRequests {
|
||||
gologger.Debug().Str("address", input.Input).Msg(msg)
|
||||
gologger.Debug().Str("address", input.MetaInput.Input).Msg(msg)
|
||||
}
|
||||
if requestOptions.Options.StoreResponse {
|
||||
request.options.Output.WriteStoreDebugData(input.Input, request.options.TemplateID, request.Type().String(), msg)
|
||||
request.options.Output.WriteStoreDebugData(input.MetaInput.Input, request.options.TemplateID, request.Type().String(), msg)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -192,7 +192,11 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
|
|||
data["response"] = jsonDataString
|
||||
data["host"] = input
|
||||
data["matched"] = addressToDial
|
||||
data["ip"] = request.dialer.GetDialedIP(hostname)
|
||||
if input.MetaInput.CustomIP != "" {
|
||||
data["ip"] = input.MetaInput.CustomIP
|
||||
} else {
|
||||
data["ip"] = request.dialer.GetDialedIP(hostname)
|
||||
}
|
||||
data["template-path"] = requestOptions.TemplatePath
|
||||
data["template-id"] = requestOptions.TemplateID
|
||||
data["template-info"] = requestOptions.TemplateInfo
|
||||
|
|
@ -220,13 +224,13 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
|
|||
|
||||
event := eventcreator.CreateEvent(request, data, requestOptions.Options.Debug || requestOptions.Options.DebugResponse)
|
||||
if requestOptions.Options.Debug || requestOptions.Options.DebugResponse || requestOptions.Options.StoreResponse {
|
||||
msg := fmt.Sprintf("[%s] Dumped SSL response for %s", requestOptions.TemplateID, input.Input)
|
||||
msg := fmt.Sprintf("[%s] Dumped SSL response for %s", requestOptions.TemplateID, input.MetaInput.Input)
|
||||
if requestOptions.Options.Debug || requestOptions.Options.DebugResponse {
|
||||
gologger.Debug().Msg(msg)
|
||||
gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, jsonDataString, requestOptions.Options.NoColor, false))
|
||||
}
|
||||
if requestOptions.Options.StoreResponse {
|
||||
request.options.Output.WriteStoreDebugData(input.Input, request.options.TemplateID, request.Type().String(), fmt.Sprintf("%s\n%s", msg, jsonDataString))
|
||||
request.options.Output.WriteStoreDebugData(input.MetaInput.Input, request.options.TemplateID, request.Type().String(), fmt.Sprintf("%s\n%s", msg, jsonDataString))
|
||||
}
|
||||
}
|
||||
callback(event)
|
||||
|
|
|
|||
|
|
@ -28,8 +28,7 @@ func TestSSLProtocol(t *testing.T) {
|
|||
require.Nil(t, err, "could not compile ssl request")
|
||||
|
||||
var gotEvent output.InternalEvent
|
||||
ctxArgs := contextargs.New()
|
||||
ctxArgs.Input = "google.com:443"
|
||||
ctxArgs := contextargs.NewWithInput("google.com:443")
|
||||
err = request.ExecuteWithResults(ctxArgs, nil, nil, func(event *output.InternalWrappedEvent) {
|
||||
gotEvent = event.InternalEvent
|
||||
})
|
||||
|
|
|
|||
|
|
@ -137,7 +137,7 @@ func (request *Request) GetID() string {
|
|||
|
||||
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
|
||||
func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
|
||||
hostname, err := getAddress(input.Input)
|
||||
hostname, err := getAddress(input.MetaInput.Input)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -150,13 +150,13 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
|
|||
if !ok {
|
||||
break
|
||||
}
|
||||
if err := request.executeRequestWithPayloads(input.Input, hostname, value, previous, callback); err != nil {
|
||||
if err := request.executeRequestWithPayloads(input.MetaInput.Input, hostname, value, previous, callback); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
} else {
|
||||
value := make(map[string]interface{})
|
||||
if err := request.executeRequestWithPayloads(input.Input, hostname, value, previous, callback); err != nil {
|
||||
if err := request.executeRequestWithPayloads(input.MetaInput.Input, hostname, value, previous, callback); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -85,7 +85,7 @@ func (request *Request) GetID() string {
|
|||
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
|
||||
func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
|
||||
// generate variables
|
||||
variables := generateVariables(input.Input)
|
||||
variables := generateVariables(input.MetaInput.Input)
|
||||
|
||||
if vardump.EnableVarDump {
|
||||
gologger.Debug().Msgf("Protocol request variables: \n%s\n", vardump.DumpVariables(variables))
|
||||
|
|
|
|||
|
|
@ -208,7 +208,7 @@ func (e *ClusterExecuter) Execute(input *contextargs.Context) (bool, error) {
|
|||
}
|
||||
})
|
||||
if err != nil && e.options.HostErrorsCache != nil {
|
||||
e.options.HostErrorsCache.MarkFailed(input.Input, err)
|
||||
e.options.HostErrorsCache.MarkFailed(input.MetaInput.Input, err)
|
||||
}
|
||||
return results, err
|
||||
}
|
||||
|
|
@ -230,7 +230,7 @@ func (e *ClusterExecuter) ExecuteWithResults(input *contextargs.Context, callbac
|
|||
}
|
||||
})
|
||||
if err != nil && e.options.HostErrorsCache != nil {
|
||||
e.options.HostErrorsCache.MarkFailed(input.Input, err)
|
||||
e.options.HostErrorsCache.MarkFailed(input.MetaInput.Input, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -272,12 +272,16 @@ type Options struct {
|
|||
IncludeConditions goflags.StringSlice
|
||||
// Custom Config Directory
|
||||
CustomConfigDir string
|
||||
// ConfigPath contains the config path (used by healthcheck)
|
||||
ConfigPath string
|
||||
// ScanAllIPs associated to a dns record
|
||||
ScanAllIPs bool
|
||||
// IPVersion to scan (4,6)
|
||||
IPVersion goflags.StringSlice
|
||||
// Github token used to clone/pull from private repos for custom templates
|
||||
GithubToken string
|
||||
// GithubTemplateRepo is the list of custom public/private templates github repos
|
||||
GithubTemplateRepo goflags.StringSlice
|
||||
|
||||
ConfigPath string // Used by healthcheck
|
||||
}
|
||||
|
||||
func (options *Options) AddVarPayload(key string, value interface{}) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue