From e85de39f88f6cf7683d1f212260d45533d2f005b Mon Sep 17 00:00:00 2001 From: becivells <732903873@qq.com> Date: Mon, 17 May 2021 11:51:14 +0800 Subject: [PATCH 01/43] #408 Add fofa for SubFinder --- v2/pkg/passive/sources.go | 4 ++ v2/pkg/runner/config.go | 9 +++ v2/pkg/subscraping/sources/fofa/fofa.go | 74 +++++++++++++++++++++++++ v2/pkg/subscraping/types.go | 2 + 4 files changed, 89 insertions(+) create mode 100644 v2/pkg/subscraping/sources/fofa/fofa.go diff --git a/v2/pkg/passive/sources.go b/v2/pkg/passive/sources.go index 3ac0bec..ef35ec4 100644 --- a/v2/pkg/passive/sources.go +++ b/v2/pkg/passive/sources.go @@ -14,6 +14,7 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/crtsh" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdb" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdumpster" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/fofa" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/github" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hackertarget" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/intelx" @@ -110,6 +111,7 @@ var DefaultAllSources = []string{ "virustotal", "waybackarchive", "zoomeye", + "fofa", } // Agent is a struct for running passive subdomain enumeration @@ -198,6 +200,8 @@ func (a *Agent) addSources(sources []string) { a.sources[source] = &waybackarchive.Source{} case "zoomeye": a.sources[source] = &zoomeye.Source{} + case "fofa": + a.sources[source] = &fofa.Source{} } } } diff --git a/v2/pkg/runner/config.go b/v2/pkg/runner/config.go index 81ebe40..faa8420 100644 --- a/v2/pkg/runner/config.go +++ b/v2/pkg/runner/config.go @@ -46,6 +46,7 @@ type ConfigFile struct { URLScan []string `yaml:"urlscan"` Virustotal []string `yaml:"virustotal"` ZoomEye []string `yaml:"zoomeye"` + Fofa []string `yaml:"fofa"` // Version indicates the version of subfinder installed. Version string `yaml:"subfinder-version"` } @@ -194,6 +195,14 @@ func (c *ConfigFile) GetKeys() subscraping.Keys { keys.ZoomEyePassword = parts[1] } } + if len(c.Fofa) > 0 { + fofaKeys := c.Fofa[rand.Intn(len(c.Fofa))] + parts := strings.Split(fofaKeys, ":") + if len(parts) == MultipleKeyPartsLength { + keys.FofaUsername = parts[0] + keys.FofaSecret = parts[1] + } + } return keys } diff --git a/v2/pkg/subscraping/sources/fofa/fofa.go b/v2/pkg/subscraping/sources/fofa/fofa.go new file mode 100644 index 0000000..0250aee --- /dev/null +++ b/v2/pkg/subscraping/sources/fofa/fofa.go @@ -0,0 +1,74 @@ +// Package fofa logic +package fofa + +import ( + "context" + "encoding/base64" + "fmt" + "strings" + + jsoniter "github.com/json-iterator/go" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type fofaResponse struct { + Error bool `json:"error"` + ErrMsg string `json:"errmsg"` + Size int `json:"size"` + Results []string `json:"results"` +} + +// Source is the passive scraping agent +type Source struct{} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + if session.Keys.FofaUsername == "" || session.Keys.FofaSecret == "" { + return + } + + // fofa api doc https://fofa.so/static_pages/api_help + qbase64 := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("domain=\"%s\"", domain))) + resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://fofa.so/api/v1/search/all?full=true&fields=host&page=1&size=10000&email=%s&key=%s&qbase64=%s", session.Keys.FofaUsername, session.Keys.FofaSecret, qbase64)) + if err != nil && resp == nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + + var response fofaResponse + err = jsoniter.NewDecoder(resp.Body).Decode(&response) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() + return + } + resp.Body.Close() + + if response.Error { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", response.ErrMsg)} + return + } + + if response.Size > 0 { + for _, subdomain := range response.Results { + if strings.HasPrefix(strings.ToLower(subdomain), "http://") || strings.HasPrefix(strings.ToLower(subdomain), "https://") { + subdomain = subdomain[strings.Index(subdomain, "//")+2:] + } + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + } + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "fofa" +} diff --git a/v2/pkg/subscraping/types.go b/v2/pkg/subscraping/types.go index 8a0c445..12657d8 100755 --- a/v2/pkg/subscraping/types.go +++ b/v2/pkg/subscraping/types.go @@ -56,6 +56,8 @@ type Keys struct { Virustotal string `json:"virustotal"` ZoomEyeUsername string `json:"zoomeye_username"` ZoomEyePassword string `json:"zoomeye_password"` + FofaUsername string `json:"fofa_username"` + FofaSecret string `json:"fofa_password"` } // Result is a result structure returned by a source From 9b3a5b75dddd3b5891f7af2fabd1dfab7343f4b2 Mon Sep 17 00:00:00 2001 From: Samuel Campos Date: Fri, 7 May 2021 18:01:45 +0200 Subject: [PATCH 02/43] add bind IP option it can be very useful to choose one's local IP if one have several network interfaces --- README.md | 25 +++++++++++++++++++++++++ v2/pkg/passive/passive.go | 5 +++-- v2/pkg/runner/enumerate.go | 2 +- v2/pkg/runner/options.go | 4 ++++ v2/pkg/runner/validate.go | 9 +++++++++ v2/pkg/subscraping/agent.go | 10 +++++++++- 6 files changed, 51 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 32603a4..06ef3fe 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,7 @@ This will display help for the tool. Here are all the switches it supports. | Flag | Description | Example | | ---------------- | ---------------------------------------------------------- | -------------------------------------- | | -all | Use all sources (slow) for enumeration | subfinder -d uber.com -all | +| -b | IP address to be used as local bind | subfinder -b 172.16.0.1 | | -config | Configuration file for API Keys, etc | subfinder -config config.yaml | | -d | Domain to find subdomains for | subfinder -d uber.com | | -dL | File containing list of domains to enumerate | subfinder -dL hackerone-hosts.txt | @@ -210,6 +211,30 @@ https://docs.hackerone.com http://mta-sts.managed.hackerone.com ``` +If your enterprise uses source routing to choose network output, or your computer has many public network interfaces (eg: public Wi-Fi + 4G connection + Ethernet Wire + VPN), you might want to choose your output network by binding IP source. In this case, you can use `-b` option. +In the example below, we have 3 network interfaces able to communicate to the Internet through 3 different outputs. Each output is chosen by binding one source IP with `-b` option. +```sh +▶ ip addr +[...] +3: wlp3s0: mtu 1500 qdisc mq state UP group default qlen 1000 + link/ether e8:b1:fc:50:90:a0 brd ff:ff:ff:ff:ff:ff + inet 192.168.1.87/24 brd 192.168.1.255 scope global dynamic noprefixroute wlp3s0 + valid_lft 62538sec preferred_lft 62538sec +4: tun0: mtu 1500 qdisc pfifo_fast state UNKNOWN group default qlen 100 + link/none + inet 192.168.254.70 peer 192.168.254.69/32 scope global tun0 + valid_lft forever preferred_lft forever +5: enx0c5b8f279a64: mtu 1500 qdisc pfifo_fast state UP group default qlen 1000 + link/ether 0c:5b:8f:a5:63:25 brd ff:ff:ff:ff:ff:ff + inet 192.168.8.100/24 brd 192.168.8.255 scope global dynamic noprefixroute enx0c5b8f279a64 + valid_lft 86396sec preferred_lft 86396sec + +▶ subfinder -d hackerone.com -b 192.168.1.87 +▶ subfinder -d hackerone.com -b 192.168.254.70 +▶ subfinder -d hackerone.com -b 192.168.8.100 +``` + +
diff --git a/v2/pkg/passive/passive.go b/v2/pkg/passive/passive.go index 5dfbc7d..132f224 100644 --- a/v2/pkg/passive/passive.go +++ b/v2/pkg/passive/passive.go @@ -3,6 +3,7 @@ package passive import ( "context" "fmt" + "net" "sync" "time" @@ -11,11 +12,11 @@ import ( ) // EnumerateSubdomains enumerates all the subdomains for a given domain -func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, timeout int, maxEnumTime time.Duration) chan subscraping.Result { +func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, timeout int, maxEnumTime time.Duration, localIP net.IP) chan subscraping.Result { results := make(chan subscraping.Result) go func() { - session, err := subscraping.NewSession(domain, keys, timeout) + session, err := subscraping.NewSession(domain, keys, timeout, localIP) if err != nil { results <- subscraping.Result{Type: subscraping.Error, Error: fmt.Errorf("could not init passive session for %s: %s", domain, err)} } diff --git a/v2/pkg/runner/enumerate.go b/v2/pkg/runner/enumerate.go index 28bf37f..30f8142 100644 --- a/v2/pkg/runner/enumerate.go +++ b/v2/pkg/runner/enumerate.go @@ -37,7 +37,7 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain string, outpu // Run the passive subdomain enumeration now := time.Now() - passiveResults := r.passiveAgent.EnumerateSubdomains(domain, &keys, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute) + passiveResults := r.passiveAgent.EnumerateSubdomains(domain, &keys, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute, r.options.LocalIP) wg := &sync.WaitGroup{} wg.Add(1) diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index bf90615..dce3cfb 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -3,6 +3,7 @@ package runner import ( "flag" "io" + "net" "os" "path" "reflect" @@ -38,6 +39,8 @@ type Options struct { ExcludeSources string // ExcludeSources contains the comma-separated sources to not include in the enumeration process Resolvers string // Resolvers is the comma-separated resolvers to use for enumeration ResolverList string // ResolverList is a text file containing list of resolvers to use for enumeration + LocalIP net.IP // LocalIP is the IP address used as local bind + LocalIPString string // LocalIPString is the IP address in string format got from command line ConfigFile string // ConfigFile contains the location of the config file YAMLConfig ConfigFile // YAMLConfig contains the unmarshalled yaml config file @@ -75,6 +78,7 @@ func ParseOptions() *Options { flag.StringVar(&options.Resolvers, "r", "", "Comma-separated list of resolvers to use") flag.StringVar(&options.ResolverList, "rL", "", "Text file containing list of resolvers to use") flag.BoolVar(&options.RemoveWildcard, "nW", false, "Remove Wildcard & Dead Subdomains from output") + flag.StringVar(&options.LocalIPString, "b", "", "IP address to be used as local bind") flag.StringVar(&options.ConfigFile, "config", path.Join(config, "config.yaml"), "Configuration file for API Keys, etc") flag.BoolVar(&options.Version, "version", false, "Show version of subfinder") flag.Parse() diff --git a/v2/pkg/runner/validate.go b/v2/pkg/runner/validate.go index 1f833ca..8b2396d 100644 --- a/v2/pkg/runner/validate.go +++ b/v2/pkg/runner/validate.go @@ -2,6 +2,7 @@ package runner import ( "errors" + "net" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/formatter" @@ -34,6 +35,14 @@ func (options *Options) validateOptions() error { return errors.New("hostip flag must be used with RemoveWildcard option") } + // Check local IP address + if options.LocalIPString != "" { + options.LocalIP = net.ParseIP(options.LocalIPString) + if options.LocalIP == nil { + return errors.New("local bind ip is malformed") + } + } + return nil } diff --git a/v2/pkg/subscraping/agent.go b/v2/pkg/subscraping/agent.go index b4c89ec..2a38c71 100755 --- a/v2/pkg/subscraping/agent.go +++ b/v2/pkg/subscraping/agent.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "io/ioutil" + "net" "net/http" "net/url" "time" @@ -14,7 +15,13 @@ import ( ) // NewSession creates a new session object for a domain -func NewSession(domain string, keys *Keys, timeout int) (*Session, error) { +func NewSession(domain string, keys *Keys, timeout int, localIP net.IP) (*Session, error) { + dialer := &net.Dialer{ + LocalAddr: &net.TCPAddr{ + IP: localIP, + }, + } + client := &http.Client{ Transport: &http.Transport{ MaxIdleConns: 100, @@ -22,6 +29,7 @@ func NewSession(domain string, keys *Keys, timeout int) (*Session, error) { TLSClientConfig: &tls.Config{ InsecureSkipVerify: true, }, + DialContext: dialer.DialContext, }, Timeout: time.Duration(timeout) * time.Second, } From 0a162d88541c14fcc0b95fed0a62d5af5095edff Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Tue, 25 May 2021 23:59:09 +0200 Subject: [PATCH 03/43] golint => revive --- .github/workflows/build.yaml | 4 ++-- .golangci.yml | 10 ++-------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index d0d0e47..0f4e1e9 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -13,10 +13,10 @@ jobs: - name: Checkout code uses: actions/checkout@v2 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v2.5.2 + uses: golangci/golangci-lint-action@v2 with: # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. - version: v1.33 + version: latest args: --timeout 5m working-directory: v2/ diff --git a/.golangci.yml b/.golangci.yml index a16e0ce..c7b06b6 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -90,6 +90,7 @@ linters: - unused - varcheck - whitespace + - revive # don't enable: # - depguard @@ -112,11 +113,4 @@ issues: exclude-use-default: false exclude: # should have a package comment, unless it's in another file for this package (golint) - - 'in another file for this package' - -# golangci.com configuration -# https://github.com/golangci/golangci/wiki/Configuration -service: - golangci-lint-version: 1.31.x # use the fixed version to not introduce new linters unexpectedly - prepare: - - echo "here I can run custom commands, but no preparation needed for this repo" + - 'in another file for this package' \ No newline at end of file From ee65aaf46112757480a35562eed072d2597839dd Mon Sep 17 00:00:00 2001 From: mzack Date: Sat, 3 Jul 2021 13:14:45 +0200 Subject: [PATCH 04/43] switching golinter configuration to latest default by deleting custom file --- .golangci.yml | 116 -------------------------------------------------- 1 file changed, 116 deletions(-) delete mode 100644 .golangci.yml diff --git a/.golangci.yml b/.golangci.yml deleted file mode 100644 index c7b06b6..0000000 --- a/.golangci.yml +++ /dev/null @@ -1,116 +0,0 @@ -linters-settings: - dupl: - threshold: 100 - exhaustive: - default-signifies-exhaustive: false - # funlen: - # lines: 100 - # statements: 50 - goconst: - min-len: 2 - min-occurrences: 2 - gocritic: - enabled-tags: - - diagnostic - - experimental - - opinionated - - performance - - style - disabled-checks: - - dupImport # https://github.com/go-critic/go-critic/issues/845 - - ifElseChain - # gocyclo: - # min-complexity: 15 - goimports: - local-prefixes: github.com/golangci/golangci-lint - golint: - min-confidence: 0 - gomnd: - settings: - mnd: - # don't include the "operation" and "assign" - checks: argument,case,condition,return - govet: - check-shadowing: true - settings: - printf: - funcs: - - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof - - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf - - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf - - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf - # lll: - # line-length: 140 - maligned: - suggest-new: true - misspell: - locale: US - nolintlint: - allow-leading-space: true # don't require machine-readable nolint directives (i.e. with no leading space) - allow-unused: false # report any unused nolint directives - require-explanation: false # don't require an explanation for nolint directives - require-specific: false # don't require nolint directives to be specific about which linter is being skipped - -linters: - # please, do not use `enable-all`: it's deprecated and will be removed soon. - # inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint - disable-all: true - enable: - - bodyclose - - deadcode - - dogsled - - dupl - # - errcheck - - exhaustive - - gochecknoinits - - goconst - - gocritic - - gofmt - - goimports - - golint - - gomnd - - goprintffuncname - - gosimple - - govet - - ineffassign - - interfacer - - maligned - - misspell - - nakedret - - noctx - # - nolintlint - - rowserrcheck - - scopelint - - staticcheck - - structcheck - - stylecheck - - typecheck - - unconvert - - unparam - - unused - - varcheck - - whitespace - - revive - - # don't enable: - # - depguard - # - asciicheck - # - funlen - # - gochecknoglobals - # - gocognit - # - gocyclo - # - godot - # - godox - # - goerr113 - # - gosec - # - lll - # - nestif - # - prealloc - # - testpackage - # - wsl - -issues: - exclude-use-default: false - exclude: - # should have a package comment, unless it's in another file for this package (golint) - - 'in another file for this package' \ No newline at end of file From cddad2b796501d4422ad2dac45aec9b7fe0c45bf Mon Sep 17 00:00:00 2001 From: sandeep Date: Mon, 5 Jul 2021 13:59:49 +0530 Subject: [PATCH 05/43] Update types.go --- v2/pkg/subscraping/types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/subscraping/types.go b/v2/pkg/subscraping/types.go index 12657d8..6c92f4a 100755 --- a/v2/pkg/subscraping/types.go +++ b/v2/pkg/subscraping/types.go @@ -57,7 +57,7 @@ type Keys struct { ZoomEyeUsername string `json:"zoomeye_username"` ZoomEyePassword string `json:"zoomeye_password"` FofaUsername string `json:"fofa_username"` - FofaSecret string `json:"fofa_password"` + FofaSecret string `json:"fofa_secret"` } // Result is a result structure returned by a source From 7c6bd16451cdadc6bf9e461457403cda35da87b3 Mon Sep 17 00:00:00 2001 From: sandeep Date: Mon, 5 Jul 2021 14:22:00 +0530 Subject: [PATCH 06/43] Added Fofa in default list of sources to run --- v2/pkg/passive/sources.go | 1 + 1 file changed, 1 insertion(+) diff --git a/v2/pkg/passive/sources.go b/v2/pkg/passive/sources.go index ef35ec4..1f196ee 100644 --- a/v2/pkg/passive/sources.go +++ b/v2/pkg/passive/sources.go @@ -59,6 +59,7 @@ var DefaultSources = []string{ "threatcrowd", "threatminer", "virustotal", + "fofa", } // DefaultRecursiveSources contains list of default recursive sources From 44c03419c12a86b52dd5a6d08d1d643c41186039 Mon Sep 17 00:00:00 2001 From: r-romanov Date: Sun, 18 Jul 2021 16:16:58 +0300 Subject: [PATCH 07/43] improvement: reworked the Spyse integration to use API v4 and fetch more than 10 000 results --- v2/go.mod | 1 + v2/go.sum | 4 + v2/pkg/subscraping/sources/spyse/spyse.go | 107 ++++++++++++++-------- 3 files changed, 73 insertions(+), 39 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 543cf9e..9e324d5 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -11,6 +11,7 @@ require ( github.com/projectdiscovery/fdmax v0.0.3 github.com/projectdiscovery/gologger v1.1.4 github.com/rs/xid v1.3.0 + github.com/spyse-com/go-spyse v1.2.1 github.com/stretchr/testify v1.7.0 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b diff --git a/v2/go.sum b/v2/go.sum index 0c27ad7..00d9543 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -51,6 +51,8 @@ github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/z github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= +github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -87,6 +89,8 @@ github.com/projectdiscovery/retryabledns v1.0.12-0.20210419174848-eec3ac17d61e/g github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/spyse-com/go-spyse v1.2.1 h1:Za/BnLnXWY/DqZZQm2V7NQ69aJ8FgFA8vBiipf3CHC8= +github.com/spyse-com/go-spyse v1.2.1/go.mod h1:YzL0kTQIlCVTtP0Bna4I7p/sKF2rgY1cV32dq/L4oIw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= diff --git a/v2/pkg/subscraping/sources/spyse/spyse.go b/v2/pkg/subscraping/sources/spyse/spyse.go index 2ae271e..0dd7944 100644 --- a/v2/pkg/subscraping/sources/spyse/spyse.go +++ b/v2/pkg/subscraping/sources/spyse/spyse.go @@ -3,31 +3,11 @@ package spyse import ( "context" - "fmt" - "strconv" - - jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" + "github.com/spyse-com/go-spyse/pkg" ) -type resultObject struct { - Name string `json:"name"` -} - -type dataObject struct { - Items []resultObject `json:"items"` - TotalCount int `json:"total_count"` -} - -type errorObject struct { - Code string `json:"code"` - Message string `json:"message"` -} - -type spyseResult struct { - Data dataObject `json:"data"` - Error []errorObject `json:"error"` -} +const SearchMethodResultsLimit = 10000 // Source is the passive scraping agent type Source struct{} @@ -43,33 +23,82 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } - maxCount := 100 + client, err := spyse.NewClient(session.Keys.Spyse, nil) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + return + } - for offSet := 0; offSet <= maxCount; offSet += 100 { - resp, err := session.Get(ctx, fmt.Sprintf("https://api.spyse.com/v3/data/domain/subdomain?domain=%s&limit=100&offset=%s", domain, strconv.Itoa(offSet)), "", map[string]string{"Authorization": "Bearer " + session.Keys.Spyse}) + domainSvc := spyse.NewDomainService(client) + + var searchDomain = "." + domain + var subdomainsSearchParams spyse.QueryBuilder + + subdomainsSearchParams.AppendParam(spyse.QueryParam{ + Name: domainSvc.Params().Name.Name, + Operator: domainSvc.Params().Name.Operator.EndsWith, + Value: searchDomain, + }) + + totalResults, err := domainSvc.SearchCount(ctx, subdomainsSearchParams.Query) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + return + } + + if totalResults == 0 { + return + } + + accountSvc := spyse.NewAccountService(client) + + quota, err := accountSvc.Quota(context.Background()) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + return + } + + var searchResults []spyse.Domain + + // The default "Search" method returns only first 10 000 subdomains + // To obtain more than 10 000 subdomains the "Scroll" method should be using + // Note: The "Scroll" method is only available for "PRO" customers, so we need to check + // quota.IsScrollSearchEnabled param + if totalResults > SearchMethodResultsLimit && quota.IsScrollSearchEnabled { + searchResults, err := domainSvc.ScrollSearch( + ctx, subdomainsSearchParams.Query, "") if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - session.DiscardHTTPResponse(resp) return } - var response spyseResult - err = jsoniter.NewDecoder(resp.Body).Decode(&response) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() - return + for len(searchResults.Items) > 0 { + searchResults, err = domainSvc.ScrollSearch( + context.Background(), subdomainsSearchParams.Query, searchResults.SearchID) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + return + } + + for _, r := range searchResults.Items { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: r.Name} + } } - resp.Body.Close() + } else { + var limit = 100 + var offset = 0 - if response.Data.TotalCount == 0 { - return - } + for ; int64(offset) < totalResults && int64(offset) < SearchMethodResultsLimit; offset += limit { + searchResults, err = domainSvc.Search(ctx, subdomainsSearchParams.Query, limit, offset) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + return + } - maxCount = response.Data.TotalCount + for _, r := range searchResults { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: r.Name} + } - for _, hostname := range response.Data.Items { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: hostname.Name} } } }() From 85349ba5da9edeee638923fd2cf0aeb5210121f6 Mon Sep 17 00:00:00 2001 From: r-romanov Date: Wed, 21 Jul 2021 10:04:48 +0300 Subject: [PATCH 08/43] refactor: fix some linted fragments --- v2/pkg/subscraping/sources/spyse/spyse.go | 27 +++++++++++------------ 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/v2/pkg/subscraping/sources/spyse/spyse.go b/v2/pkg/subscraping/sources/spyse/spyse.go index 0dd7944..7daf48c 100644 --- a/v2/pkg/subscraping/sources/spyse/spyse.go +++ b/v2/pkg/subscraping/sources/spyse/spyse.go @@ -7,7 +7,7 @@ import ( "github.com/spyse-com/go-spyse/pkg" ) -const SearchMethodResultsLimit = 10000 +const searchMethodResultsLimit = 10000 // Source is the passive scraping agent type Source struct{} @@ -58,47 +58,46 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } - var searchResults []spyse.Domain - // The default "Search" method returns only first 10 000 subdomains // To obtain more than 10 000 subdomains the "Scroll" method should be using // Note: The "Scroll" method is only available for "PRO" customers, so we need to check // quota.IsScrollSearchEnabled param - if totalResults > SearchMethodResultsLimit && quota.IsScrollSearchEnabled { - searchResults, err := domainSvc.ScrollSearch( + if totalResults > searchMethodResultsLimit && quota.IsScrollSearchEnabled { + var scrollResponse *spyse.DomainScrollResponse + scrollResponse, err = domainSvc.ScrollSearch( ctx, subdomainsSearchParams.Query, "") if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} return } - for len(searchResults.Items) > 0 { - searchResults, err = domainSvc.ScrollSearch( - context.Background(), subdomainsSearchParams.Query, searchResults.SearchID) + for len(scrollResponse.Items) > 0 { + scrollResponse, err = domainSvc.ScrollSearch( + context.Background(), subdomainsSearchParams.Query, scrollResponse.SearchID) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} return } - for _, r := range searchResults.Items { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: r.Name} + for i := range scrollResponse.Items { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: scrollResponse.Items[i].Name} } } } else { var limit = 100 var offset = 0 + var searchResults []spyse.Domain - for ; int64(offset) < totalResults && int64(offset) < SearchMethodResultsLimit; offset += limit { + for ; int64(offset) < totalResults && int64(offset) < searchMethodResultsLimit; offset += limit { searchResults, err = domainSvc.Search(ctx, subdomainsSearchParams.Query, limit, offset) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} return } - for _, r := range searchResults { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: r.Name} + for i := range searchResults { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: searchResults[i].Name} } - } } }() From 1a60d2fcc53dfcadd036eebe1387154743ea4a69 Mon Sep 17 00:00:00 2001 From: sandeep Date: Wed, 21 Jul 2021 13:56:35 +0530 Subject: [PATCH 09/43] lint fix --- v2/pkg/subscraping/sources/spyse/spyse.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/v2/pkg/subscraping/sources/spyse/spyse.go b/v2/pkg/subscraping/sources/spyse/spyse.go index 7daf48c..fbb7501 100644 --- a/v2/pkg/subscraping/sources/spyse/spyse.go +++ b/v2/pkg/subscraping/sources/spyse/spyse.go @@ -3,8 +3,9 @@ package spyse import ( "context" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" - "github.com/spyse-com/go-spyse/pkg" + spyse "github.com/spyse-com/go-spyse/pkg" ) const searchMethodResultsLimit = 10000 From 3b13f7e30686e63fd438a3657d26da16aa0435bc Mon Sep 17 00:00:00 2001 From: Karel Date: Fri, 30 Jul 2021 11:59:38 +0200 Subject: [PATCH 10/43] Close database connection --- v2/pkg/subscraping/sources/crtsh/crtsh.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/v2/pkg/subscraping/sources/crtsh/crtsh.go b/v2/pkg/subscraping/sources/crtsh/crtsh.go index 16400fa..4875b0c 100755 --- a/v2/pkg/subscraping/sources/crtsh/crtsh.go +++ b/v2/pkg/subscraping/sources/crtsh/crtsh.go @@ -44,6 +44,8 @@ func (s *Source) getSubdomainsFromSQL(domain string, results chan subscraping.Re return 0 } + defer db.Close() + pattern := "%." + domain query := `SELECT DISTINCT ci.NAME_VALUE as domain FROM certificate_identity ci WHERE reverse(lower(ci.NAME_VALUE)) LIKE reverse(lower($1)) From 3533d5ba3fc8ebba37fc0f14c54a47f540b80802 Mon Sep 17 00:00:00 2001 From: yhy Date: Fri, 30 Jul 2021 20:38:25 +0800 Subject: [PATCH 11/43] Add Chinaz for subfinder --- v2/pkg/passive/sources.go | 4 ++ v2/pkg/runner/config.go | 4 ++ v2/pkg/subscraping/sources/chinaz/chinaz.go | 56 +++++++++++++++++++++ v2/pkg/subscraping/types.go | 1 + 4 files changed, 65 insertions(+) create mode 100644 v2/pkg/subscraping/sources/chinaz/chinaz.go diff --git a/v2/pkg/passive/sources.go b/v2/pkg/passive/sources.go index 3ac0bec..5f5c341 100644 --- a/v2/pkg/passive/sources.go +++ b/v2/pkg/passive/sources.go @@ -10,6 +10,7 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/censys" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/certspotter" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/chaos" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/chinaz" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/commoncrawl" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/crtsh" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdb" @@ -44,6 +45,7 @@ var DefaultSources = []string{ "certspotter", "censys", "chaos", + "chinaz", "crtsh", "dnsdumpster", "hackertarget", @@ -150,6 +152,8 @@ func (a *Agent) addSources(sources []string) { a.sources[source] = &certspotter.Source{} case "chaos": a.sources[source] = &chaos.Source{} + case "chinaz": + a.sources[source] = &chinaz.Source{} case "commoncrawl": a.sources[source] = &commoncrawl.Source{} case "crtsh": diff --git a/v2/pkg/runner/config.go b/v2/pkg/runner/config.go index 81ebe40..dbd0cb4 100644 --- a/v2/pkg/runner/config.go +++ b/v2/pkg/runner/config.go @@ -33,6 +33,7 @@ type ConfigFile struct { Censys []string `yaml:"censys"` Certspotter []string `yaml:"certspotter"` Chaos []string `yaml:"chaos"` + Chinaz []string `yaml:"chinaz"` DNSDB []string `yaml:"dnsdb"` GitHub []string `yaml:"github"` IntelX []string `yaml:"intelx"` @@ -135,6 +136,9 @@ func (c *ConfigFile) GetKeys() subscraping.Keys { if len(c.Chaos) > 0 { keys.Chaos = c.Chaos[rand.Intn(len(c.Chaos))] } + if len(c.Chinaz) > 0 { + keys.Chinaz = c.Chinaz[rand.Intn(len(c.Chinaz))] + } if (len(c.DNSDB)) > 0 { keys.DNSDB = c.DNSDB[rand.Intn(len(c.DNSDB))] } diff --git a/v2/pkg/subscraping/sources/chinaz/chinaz.go b/v2/pkg/subscraping/sources/chinaz/chinaz.go new file mode 100644 index 0000000..78e3cc4 --- /dev/null +++ b/v2/pkg/subscraping/sources/chinaz/chinaz.go @@ -0,0 +1,56 @@ +package chinaz +// chinaz http://my.chinaz.com/ChinazAPI/DataCenter/MyDataApi +import ( + "context" + "fmt" + jsoniter "github.com/json-iterator/go" + "github.com/yhy0/subfinder/v2/pkg/subscraping" + "io/ioutil" +) + +// Source is the passive scraping agent +type Source struct{} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + if session.Keys.Chinaz == "" { + return + } + + resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://apidatav2.chinaz.com/single/alexa?key=%s&domain=%s", session.Keys.Chinaz, domain)) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + + body, err := ioutil.ReadAll(resp.Body) + + resp.Body.Close() + + SubdomainList :=jsoniter.Get(body, "Result").Get("ContributingSubdomainList") + + if SubdomainList.ToBool() { + _data := []byte(SubdomainList.ToString()) + for i := 0 ; i< SubdomainList.Size() ; i++{ + subdomain := jsoniter.Get(_data,i,"DataUrl").ToString() + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + } + } else { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + return + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "chinaz" +} diff --git a/v2/pkg/subscraping/types.go b/v2/pkg/subscraping/types.go index 8a0c445..ac6aca2 100755 --- a/v2/pkg/subscraping/types.go +++ b/v2/pkg/subscraping/types.go @@ -40,6 +40,7 @@ type Keys struct { CensysSecret string `json:"censysPassword"` Certspotter string `json:"certspotter"` Chaos string `json:"chaos"` + Chinaz string `json:"chinaz"` DNSDB string `json:"dnsdb"` GitHub []string `json:"github"` IntelXHost string `json:"intelXHost"` From c9f0c06aa5f1847301e5b4b68f142162998ea452 Mon Sep 17 00:00:00 2001 From: yhy <31311038+yhy0@users.noreply.github.com> Date: Fri, 30 Jul 2021 21:41:10 +0800 Subject: [PATCH 12/43] Update chinaz.go --- v2/pkg/subscraping/sources/chinaz/chinaz.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/subscraping/sources/chinaz/chinaz.go b/v2/pkg/subscraping/sources/chinaz/chinaz.go index 78e3cc4..b30e93e 100644 --- a/v2/pkg/subscraping/sources/chinaz/chinaz.go +++ b/v2/pkg/subscraping/sources/chinaz/chinaz.go @@ -4,7 +4,7 @@ import ( "context" "fmt" jsoniter "github.com/json-iterator/go" - "github.com/yhy0/subfinder/v2/pkg/subscraping" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" "io/ioutil" ) From 8cff95eac1ada7f1b2d135861f3371d10210b9b9 Mon Sep 17 00:00:00 2001 From: maldevel Date: Sat, 31 Jul 2021 09:10:39 +0300 Subject: [PATCH 13/43] dnsdumpster not working. bug fixed. --- v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go | 1 + 1 file changed, 1 insertion(+) diff --git a/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go b/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go index 2a48fe3..08f0ebf 100755 --- a/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go +++ b/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go @@ -30,6 +30,7 @@ func postForm(ctx context.Context, session *subscraping.Session, token, domain s params := url.Values{ "csrfmiddlewaretoken": {token}, "targetip": {domain}, + "user": {"free"}, } resp, err := session.HTTPRequest( From fbb0305b3d6764125750d7d19dac9b864f17db6a Mon Sep 17 00:00:00 2001 From: sandeep Date: Wed, 4 Aug 2021 16:34:19 +0530 Subject: [PATCH 14/43] workflow improvements --- .github/dependabot.yml | 5 +- .github/workflows/build-test.yml | 27 +++++++++++ .github/workflows/build.yaml | 40 ---------------- .github/workflows/codeql-analysis.yml | 38 +++++++++++++++ .../workflows/dockerhub-push-on-release.yml | 17 ------- .github/workflows/dockerhub-push.yml | 34 ++++++++++++++ .github/workflows/lint-test.yml | 19 ++++++++ .../{release.yml => release-binary.yml} | 8 ++-- v2/.goreleaser.yml | 46 +++++++++++-------- 9 files changed, 153 insertions(+), 81 deletions(-) create mode 100644 .github/workflows/build-test.yml delete mode 100644 .github/workflows/build.yaml create mode 100644 .github/workflows/codeql-analysis.yml delete mode 100644 .github/workflows/dockerhub-push-on-release.yml create mode 100644 .github/workflows/dockerhub-push.yml create mode 100644 .github/workflows/lint-test.yml rename .github/workflows/{release.yml => release-binary.yml} (85%) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 4d5617f..69d9543 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -11,6 +11,7 @@ updates: directory: "/" schedule: interval: "weekly" + target-branch: "dev" commit-message: prefix: "chore" include: "scope" @@ -20,6 +21,7 @@ updates: directory: "/" schedule: interval: "weekly" + target-branch: "dev" commit-message: prefix: "chore" include: "scope" @@ -29,6 +31,7 @@ updates: directory: "/" schedule: interval: "weekly" + target-branch: "dev" commit-message: prefix: "chore" - include: "scope" + include: "scope" \ No newline at end of file diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml new file mode 100644 index 0000000..ad0d347 --- /dev/null +++ b/.github/workflows/build-test.yml @@ -0,0 +1,27 @@ +name: 🔨 Build Test +on: + push: + pull_request: + workflow_dispatch: + + +jobs: + build: + name: Test Builds + runs-on: ubuntu-latest + steps: + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.15 + + - name: Check out code + uses: actions/checkout@v2 + + - name: Test + run: go test . + working-directory: v2/cmd/subfinder/ + + - name: Build + run: go build . + working-directory: v2/cmd/subfinder/ \ No newline at end of file diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml deleted file mode 100644 index 0f4e1e9..0000000 --- a/.github/workflows/build.yaml +++ /dev/null @@ -1,40 +0,0 @@ -name: Build - -on: - push: - branches: - - master - pull_request: - -jobs: - golangci-lint: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v2 - - name: Run golangci-lint - uses: golangci/golangci-lint-action@v2 - with: - # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. - version: latest - args: --timeout 5m - working-directory: v2/ - - build: - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@v2 - - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: 1.15 - - - name: Test - run: go test ./... - working-directory: v2/ - - - name: Build - run: go build . - working-directory: v2/cmd/subfinder/ \ No newline at end of file diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000..545cdea --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,38 @@ +name: 🚨 CodeQL Analysis + +on: + workflow_dispatch: + pull_request: + branches: + - dev + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'go' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 \ No newline at end of file diff --git a/.github/workflows/dockerhub-push-on-release.yml b/.github/workflows/dockerhub-push-on-release.yml deleted file mode 100644 index a5156cb..0000000 --- a/.github/workflows/dockerhub-push-on-release.yml +++ /dev/null @@ -1,17 +0,0 @@ -# dockerhub-push pushes docker build to dockerhub automatically -# on the creation of a new release -name: Publish to Dockerhub on creation of a new release -on: - release: - types: [published] -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@master - - name: Publish to Dockerhub Registry - uses: elgohr/Publish-Docker-Github-Action@master - with: - name: projectdiscovery/subfinder - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} diff --git a/.github/workflows/dockerhub-push.yml b/.github/workflows/dockerhub-push.yml new file mode 100644 index 0000000..ab9b9db --- /dev/null +++ b/.github/workflows/dockerhub-push.yml @@ -0,0 +1,34 @@ +name: 🌥 Docker Push + +on: + release: + types: [published] + workflow_dispatch: + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v2 + - + name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - + name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_TOKEN }} + - + name: Build and push + uses: docker/build-push-action@v2 + with: + context: . + platforms: linux/amd64,linux/arm64,linux/arm + push: true + tags: projectdiscovery/subfinder:latest \ No newline at end of file diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml new file mode 100644 index 0000000..794d073 --- /dev/null +++ b/.github/workflows/lint-test.yml @@ -0,0 +1,19 @@ +name: 🙏🏻 Lint Test +on: + push: + pull_request: + workflow_dispatch: + +jobs: + lint: + name: Lint Test + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Run golangci-lint + uses: golangci/golangci-lint-action@v2 + with: + version: latest + args: --timeout 5m + working-directory: . \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release-binary.yml similarity index 85% rename from .github/workflows/release.yml rename to .github/workflows/release-binary.yml index 8229941..6fe8c82 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release-binary.yml @@ -1,8 +1,9 @@ -name: Release +name: 🎉 Release Binary on: create: tags: - v* + workflow_dispatch: jobs: release: @@ -17,7 +18,7 @@ jobs: name: "Set up Go" uses: actions/setup-go@v2 with: - go-version: 1.15 + go-version: 1.16 - env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" @@ -26,5 +27,4 @@ jobs: with: args: "release --rm-dist" version: latest - workdir: v2/ - \ No newline at end of file + workdir: . \ No newline at end of file diff --git a/v2/.goreleaser.yml b/v2/.goreleaser.yml index 9b17be1..6006e6d 100644 --- a/v2/.goreleaser.yml +++ b/v2/.goreleaser.yml @@ -3,23 +3,31 @@ before: - go mod tidy builds: - - binary: subfinder - main: cmd/subfinder/main.go - goos: - - linux - - windows - - darwin - goarch: - - amd64 - - 386 - - arm - - arm64 - +- env: + - CGO_ENABLED=0 + goos: + - windows + - linux + - darwin + goarch: + - amd64 + - 386 + - arm + - arm64 + + ignore: + - goos: darwin + goarch: '386' + - goos: windows + goarch: 'arm' + + binary: '{{ .ProjectName }}' + main: cmd/subfinder/main.go + archives: - - id: tgz - format: tar.gz - replacements: - darwin: macOS - format_overrides: - - goos: windows - format: zip \ No newline at end of file +- format: zip + replacements: + darwin: macOS + +checksum: + algorithm: sha256 \ No newline at end of file From 6485009882dad743018e6d1b2bc0ad0d905ef21d Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Wed, 4 Aug 2021 20:00:56 +0530 Subject: [PATCH 15/43] Revert "Add C99.nl Source" --- v2/pkg/passive/passive.go | 4 +- v2/pkg/passive/sources.go | 4 -- v2/pkg/runner/config.go | 4 -- v2/pkg/runner/enumerate.go | 2 +- v2/pkg/runner/options.go | 19 ++++--- v2/pkg/subscraping/agent.go | 31 ++++-------- v2/pkg/subscraping/sources/c99/c99.go | 71 --------------------------- v2/pkg/subscraping/types.go | 1 - 8 files changed, 21 insertions(+), 115 deletions(-) delete mode 100644 v2/pkg/subscraping/sources/c99/c99.go diff --git a/v2/pkg/passive/passive.go b/v2/pkg/passive/passive.go index ace9846..5dfbc7d 100644 --- a/v2/pkg/passive/passive.go +++ b/v2/pkg/passive/passive.go @@ -11,11 +11,11 @@ import ( ) // EnumerateSubdomains enumerates all the subdomains for a given domain -func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, timeout int, maxEnumTime time.Duration) chan subscraping.Result { +func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, timeout int, maxEnumTime time.Duration) chan subscraping.Result { results := make(chan subscraping.Result) go func() { - session, err := subscraping.NewSession(domain, keys, proxy, timeout) + session, err := subscraping.NewSession(domain, keys, timeout) if err != nil { results <- subscraping.Result{Type: subscraping.Error, Error: fmt.Errorf("could not init passive session for %s: %s", domain, err)} } diff --git a/v2/pkg/passive/sources.go b/v2/pkg/passive/sources.go index a3f65e3..3ac0bec 100644 --- a/v2/pkg/passive/sources.go +++ b/v2/pkg/passive/sources.go @@ -7,7 +7,6 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/archiveis" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/binaryedge" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/bufferover" - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/c99" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/censys" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/certspotter" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/chaos" @@ -84,7 +83,6 @@ var DefaultAllSources = []string{ "archiveis", "binaryedge", "bufferover", - "c99", "censys", "certspotter", "chaos", @@ -146,8 +144,6 @@ func (a *Agent) addSources(sources []string) { a.sources[source] = &binaryedge.Source{} case "bufferover": a.sources[source] = &bufferover.Source{} - case "c99": - a.sources[source] = &c99.Source{} case "censys": a.sources[source] = &censys.Source{} case "certspotter": diff --git a/v2/pkg/runner/config.go b/v2/pkg/runner/config.go index 547ecda..81ebe40 100644 --- a/v2/pkg/runner/config.go +++ b/v2/pkg/runner/config.go @@ -30,7 +30,6 @@ type ConfigFile struct { ExcludeSources []string `yaml:"exclude-sources,omitempty"` // API keys for different sources Binaryedge []string `yaml:"binaryedge"` - C99 []string `yaml:"c99"` Censys []string `yaml:"censys"` Certspotter []string `yaml:"certspotter"` Chaos []string `yaml:"chaos"` @@ -120,9 +119,6 @@ func (c *ConfigFile) GetKeys() subscraping.Keys { if len(c.Binaryedge) > 0 { keys.Binaryedge = c.Binaryedge[rand.Intn(len(c.Binaryedge))] } - if len(c.C99) > 0 { - keys.C99 = c.C99[rand.Intn(len(c.C99))] - } if len(c.Censys) > 0 { censysKeys := c.Censys[rand.Intn(len(c.Censys))] diff --git a/v2/pkg/runner/enumerate.go b/v2/pkg/runner/enumerate.go index 84cbb03..28bf37f 100644 --- a/v2/pkg/runner/enumerate.go +++ b/v2/pkg/runner/enumerate.go @@ -37,7 +37,7 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain string, outpu // Run the passive subdomain enumeration now := time.Now() - passiveResults := r.passiveAgent.EnumerateSubdomains(domain, &keys, r.options.Proxy, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute) + passiveResults := r.passiveAgent.EnumerateSubdomains(domain, &keys, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute) wg := &sync.WaitGroup{} wg.Add(1) diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index 6525c2b..bf90615 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -32,15 +32,15 @@ type Options struct { Domain string // Domain is the domain to find subdomains for DomainsFile string // DomainsFile is the file containing list of domains to find subdomains for Output io.Writer - OutputFile string // Output is the file to write found subdomains to. - OutputDirectory string // OutputDirectory is the directory to write results to in case list of domains is given - Sources string // Sources contains a comma-separated list of sources to use for enumeration - ExcludeSources string // ExcludeSources contains the comma-separated sources to not include in the enumeration process - Resolvers string // Resolvers is the comma-separated resolvers to use for enumeration - ResolverList string // ResolverList is a text file containing list of resolvers to use for enumeration - ConfigFile string // ConfigFile contains the location of the config file - Proxy string // HTTP proxy - YAMLConfig ConfigFile // YAMLConfig contains the unmarshalled yaml config file + OutputFile string // Output is the file to write found subdomains to. + OutputDirectory string // OutputDirectory is the directory to write results to in case list of domains is given + Sources string // Sources contains a comma-separated list of sources to use for enumeration + ExcludeSources string // ExcludeSources contains the comma-separated sources to not include in the enumeration process + Resolvers string // Resolvers is the comma-separated resolvers to use for enumeration + ResolverList string // ResolverList is a text file containing list of resolvers to use for enumeration + ConfigFile string // ConfigFile contains the location of the config file + + YAMLConfig ConfigFile // YAMLConfig contains the unmarshalled yaml config file } // ParseOptions parses the command line flags provided by a user @@ -76,7 +76,6 @@ func ParseOptions() *Options { flag.StringVar(&options.ResolverList, "rL", "", "Text file containing list of resolvers to use") flag.BoolVar(&options.RemoveWildcard, "nW", false, "Remove Wildcard & Dead Subdomains from output") flag.StringVar(&options.ConfigFile, "config", path.Join(config, "config.yaml"), "Configuration file for API Keys, etc") - flag.StringVar(&options.Proxy, "http-proxy", "", "HTTP proxy to use") flag.BoolVar(&options.Version, "version", false, "Show version of subfinder") flag.Parse() diff --git a/v2/pkg/subscraping/agent.go b/v2/pkg/subscraping/agent.go index a039dfc..b4c89ec 100755 --- a/v2/pkg/subscraping/agent.go +++ b/v2/pkg/subscraping/agent.go @@ -14,29 +14,16 @@ import ( ) // NewSession creates a new session object for a domain -func NewSession(domain string, keys *Keys, proxy string, timeout int) (*Session, error) { - Transport := &http.Transport{ - MaxIdleConns: 100, - MaxIdleConnsPerHost: 100, - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - } - - // Add proxy - if proxy != "" { - proxyURL, _ := url.Parse(proxy) - if proxyURL == nil { - // Log warning but continue anyways - gologger.Warning().Msgf("Invalid proxy '%s' provided", proxy) - } else { - Transport.Proxy = http.ProxyURL(proxyURL) - } - } - +func NewSession(domain string, keys *Keys, timeout int) (*Session, error) { client := &http.Client{ - Transport: Transport, - Timeout: time.Duration(timeout) * time.Second, + Transport: &http.Transport{ + MaxIdleConns: 100, + MaxIdleConnsPerHost: 100, + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + }, + Timeout: time.Duration(timeout) * time.Second, } session := &Session{ diff --git a/v2/pkg/subscraping/sources/c99/c99.go b/v2/pkg/subscraping/sources/c99/c99.go deleted file mode 100644 index 211669f..0000000 --- a/v2/pkg/subscraping/sources/c99/c99.go +++ /dev/null @@ -1,71 +0,0 @@ -// Package c99 logic -package c99 - -import ( - "context" - "fmt" - "strings" - - jsoniter "github.com/json-iterator/go" - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" -) - -// Source is the passive scraping agent -type Source struct{} - -type dnsdbLookupResponse struct { - Success bool `json:"success"` - Subdomains []struct { - Subdomain string `json:"subdomain"` - IP string `json:"ip"` - Cloudflare bool `json:"cloudflare"` - } `json:"subdomains"` - Error string `json:"error"` -} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - - go func() { - defer close(results) - - if session.Keys.C99 == "" { - return - } - - searchURL := fmt.Sprintf("https://api.c99.nl/subdomainfinder?key=%s&domain=%s&json", session.Keys.C99, domain) - resp, err := session.SimpleGet(ctx, searchURL) - if err != nil { - session.DiscardHTTPResponse(resp) - return - } - - defer resp.Body.Close() - - var response dnsdbLookupResponse - err = jsoniter.NewDecoder(resp.Body).Decode(&response) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - return - } - - if response.Error != "" { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%v", response.Error)} - return - } - - for _, data := range response.Subdomains { - if !strings.HasPrefix(data.Subdomain, ".") { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: data.Subdomain} - } - } - }() - - return results -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "c99" -} diff --git a/v2/pkg/subscraping/types.go b/v2/pkg/subscraping/types.go index e3211be..8a0c445 100755 --- a/v2/pkg/subscraping/types.go +++ b/v2/pkg/subscraping/types.go @@ -36,7 +36,6 @@ type Session struct { // Keys contains the current API Keys we have in store type Keys struct { Binaryedge string `json:"binaryedge"` - C99 string `json:"c99"` CensysToken string `json:"censysUsername"` CensysSecret string `json:"censysPassword"` Certspotter string `json:"certspotter"` From 2bd4cb12082a1e6711ebdbdd2f00ac40cf2b3f85 Mon Sep 17 00:00:00 2001 From: Defuse Venue Date: Mon, 9 Aug 2021 16:05:05 +0900 Subject: [PATCH 16/43] Add http-proxy and C99 to README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 32603a4..c5b473e 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,7 @@ This will display help for the tool. Here are all the switches it supports. | -sources | Comma separated list of sources to use | subfinder -sources shodan,censys | | -t | Number of concurrent goroutines for resolving (default 10) | subfinder -t 100 | | -timeout | Seconds to wait before timing out (default 30) | subfinder -timeout 30 | +| -http-proxy | Http Proxy | subfinder -http-proxy http://localhost:3128 | -v | Show Verbose output | subfinder -v | | -version | Show current program version | subfinder -version | @@ -93,6 +94,7 @@ Subfinder requires **go1.14+** to install successfully. Run the following comman Subfinder will work after using the installation instructions however to configure Subfinder to work with certain services, you will need to have setup API keys. The following services do not work without an API key: - [Binaryedge](https://binaryedge.io) +- [C99](https://api.c99.nl/) - [Certspotter](https://sslmate.com/certspotter/api/) - [Censys](https://censys.io) - [Chaos](https://chaos.projectdiscovery.io) From 0f989d87e5c1e8a170c0f234e250932cfb90198d Mon Sep 17 00:00:00 2001 From: Defuse Venue Date: Mon, 9 Aug 2021 16:08:12 +0900 Subject: [PATCH 17/43] Lint README.md --- README.md | 50 +++++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index c5b473e..fe06180 100644 --- a/README.md +++ b/README.md @@ -53,31 +53,31 @@ subfinder -h ``` This will display help for the tool. Here are all the switches it supports. -| Flag | Description | Example | -| ---------------- | ---------------------------------------------------------- | -------------------------------------- | -| -all | Use all sources (slow) for enumeration | subfinder -d uber.com -all | -| -config | Configuration file for API Keys, etc | subfinder -config config.yaml | -| -d | Domain to find subdomains for | subfinder -d uber.com | -| -dL | File containing list of domains to enumerate | subfinder -dL hackerone-hosts.txt | -| -exclude-sources | List of sources to exclude from enumeration | subfinder -exclude-sources archiveis | -| -max-time | Minutes to wait for enumeration results (default 10) | subfinder -max-time 1 | -| -nC | Don't Use colors in output | subfinder -nC | -| -nW | Remove Wildcard & Dead Subdomains from output | subfinder -nW | -| -ls | List all available sources | subfinder -ls | -| -o | File to write output to (optional) | subfinder -o output.txt | -| -oD | Directory to write enumeration results to (optional) | subfinder -oD ~/outputs | -| -oI | Write output in Host,IP format | subfinder -oI | -| -oJ | Write output in JSON lines Format | subfinder -oJ | -| -r | Comma-separated list of resolvers to use | subfinder -r 1.1.1.1,1.0.0.1 | -| -rL | Text file containing list of resolvers to use | subfinder -rL resolvers.txt | -| -recursive | Enumeration recursive subdomains | subfinder -d news.yahoo.com -recursive | -| -silent | Show only subdomains in output | subfinder -silent | -| -sources | Comma separated list of sources to use | subfinder -sources shodan,censys | -| -t | Number of concurrent goroutines for resolving (default 10) | subfinder -t 100 | -| -timeout | Seconds to wait before timing out (default 30) | subfinder -timeout 30 | -| -http-proxy | Http Proxy | subfinder -http-proxy http://localhost:3128 -| -v | Show Verbose output | subfinder -v | -| -version | Show current program version | subfinder -version | +| Flag | Description | Example | +| ---------------- | ---------------------------------------------------------- | --------------------------------------------| +| -all | Use all sources (slow) for enumeration | subfinder -d uber.com -all | +| -config | Configuration file for API Keys, etc | subfinder -config config.yaml | +| -d | Domain to find subdomains for | subfinder -d uber.com | +| -dL | File containing list of domains to enumerate | subfinder -dL hackerone-hosts.txt | +| -exclude-sources | List of sources to exclude from enumeration | subfinder -exclude-sources archiveis | +| -max-time | Minutes to wait for enumeration results (default 10) | subfinder -max-time 1 | +| -nC | Don't Use colors in output | subfinder -nC | +| -nW | Remove Wildcard & Dead Subdomains from output | subfinder -nW | +| -ls | List all available sources | subfinder -ls | +| -o | File to write output to (optional) | subfinder -o output.txt | +| -oD | Directory to write enumeration results to (optional) | subfinder -oD ~/outputs | +| -oI | Write output in Host,IP format | subfinder -oI | +| -oJ | Write output in JSON lines Format | subfinder -oJ | +| -r | Comma-separated list of resolvers to use | subfinder -r 1.1.1.1,1.0.0.1 | +| -rL | Text file containing list of resolvers to use | subfinder -rL resolvers.txt | +| -recursive | Enumeration recursive subdomains | subfinder -d news.yahoo.com -recursive | +| -silent | Show only subdomains in output | subfinder -silent | +| -sources | Comma separated list of sources to use | subfinder -sources shodan,censys | +| -t | Number of concurrent goroutines for resolving (default 10) | subfinder -t 100 | +| -timeout | Seconds to wait before timing out (default 30) | subfinder -timeout 30 | +| -http-proxy | Http Proxy | subfinder -http-proxy http://localhost:3128 | +| -v | Show Verbose output | subfinder -v | +| -version | Show current program version | subfinder -version | # Installation From b3b56844b4791e8949079bd2f961db7d3982f239 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Aug 2021 14:03:04 +0000 Subject: [PATCH 18/43] chore(deps): bump golang from 1.16.6-alpine to 1.16.7-alpine Bumps golang from 1.16.6-alpine to 1.16.7-alpine. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ac84a2f..e8ba955 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build -FROM golang:1.16.6-alpine AS build-env +FROM golang:1.16.7-alpine AS build-env RUN GO111MODULE=on go get -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder # Release From 019a23763d422c533bc754f8d3d70d5ddc748993 Mon Sep 17 00:00:00 2001 From: maldevel Date: Mon, 9 Aug 2021 18:37:03 +0300 Subject: [PATCH 19/43] retrieve archiveis first page --- .../sources/archiveis/archiveis.go | 95 +++++++------------ 1 file changed, 33 insertions(+), 62 deletions(-) diff --git a/v2/pkg/subscraping/sources/archiveis/archiveis.go b/v2/pkg/subscraping/sources/archiveis/archiveis.go index c4e9d3a..c20d00c 100755 --- a/v2/pkg/subscraping/sources/archiveis/archiveis.go +++ b/v2/pkg/subscraping/sources/archiveis/archiveis.go @@ -5,74 +5,45 @@ import ( "context" "fmt" "io/ioutil" - "regexp" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) -type agent struct { - Results chan subscraping.Result - Session *subscraping.Session -} - -var reNext = regexp.MustCompile("→") - -func (a *agent) enumerate(ctx context.Context, baseURL string) { - select { - case <-ctx.Done(): - return - default: - } - - resp, err := a.Session.SimpleGet(ctx, baseURL) - if err != nil { - a.Results <- subscraping.Result{Source: "archiveis", Type: subscraping.Error, Error: err} - a.Session.DiscardHTTPResponse(resp) - return - } - - // Get the response body - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - a.Results <- subscraping.Result{Source: "archiveis", Type: subscraping.Error, Error: err} - resp.Body.Close() - return - } - - resp.Body.Close() - - src := string(body) - for _, subdomain := range a.Session.Extractor.FindAllString(src, -1) { - a.Results <- subscraping.Result{Source: "archiveis", Type: subscraping.Subdomain, Value: subdomain} - } - - match1 := reNext.FindStringSubmatch(src) - if len(match1) > 0 { - a.enumerate(ctx, match1[1]) - } -} - -// Source is the passive scraping agent type Source struct{} -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - - a := agent{ - Session: session, - Results: results, - } - - go func() { - a.enumerate(ctx, fmt.Sprintf("http://archive.is/*.%s", domain)) - close(a.Results) - }() - - return a.Results -} - -// Name returns the name of the source func (s *Source) Name() string { return "archiveis" } + +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://archive.is/*.%s", domain)) + + if err != nil { + results <- subscraping.Result{Source: "archiveis", Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + results <- subscraping.Result{Source: "archiveis", Type: subscraping.Error, Error: err} + resp.Body.Close() + return + } + + resp.Body.Close() + + src := string(body) + + for _, subdomain := range session.Extractor.FindAllString(src, -1) { + results <- subscraping.Result{Source: "archiveis", Type: subscraping.Subdomain, Value: subdomain} + } + }() + + return results +} From aad7cebc4ac9c0117cbbb83e5db69d0eb3142530 Mon Sep 17 00:00:00 2001 From: maldevel Date: Mon, 9 Aug 2021 18:55:26 +0300 Subject: [PATCH 20/43] Update archiveis.go comments added --- v2/pkg/subscraping/sources/archiveis/archiveis.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/v2/pkg/subscraping/sources/archiveis/archiveis.go b/v2/pkg/subscraping/sources/archiveis/archiveis.go index c20d00c..9a2b535 100755 --- a/v2/pkg/subscraping/sources/archiveis/archiveis.go +++ b/v2/pkg/subscraping/sources/archiveis/archiveis.go @@ -9,12 +9,15 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) +// Source is the passive scraping agent type Source struct{} +// Name returns the name of the source func (s *Source) Name() string { return "archiveis" } +// Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) From 333d59ce5199b0af939f162459f0429ed60b818b Mon Sep 17 00:00:00 2001 From: sandeep Date: Tue, 10 Aug 2021 17:16:01 +0530 Subject: [PATCH 21/43] version bump --- v2/pkg/runner/banners.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/pkg/runner/banners.go b/v2/pkg/runner/banners.go index be0a5e6..292d227 100644 --- a/v2/pkg/runner/banners.go +++ b/v2/pkg/runner/banners.go @@ -11,11 +11,11 @@ const banner = ` _______ __/ /_ / __(_)___ ____/ /__ _____ / ___/ / / / __ \/ /_/ / __ \/ __ / _ \/ ___/ (__ ) /_/ / /_/ / __/ / / / / /_/ / __/ / -/____/\__,_/_.___/_/ /_/_/ /_/\__,_/\___/_/ v2.4.8 +/____/\__,_/_.___/_/ /_/_/ /_/\__,_/\___/_/ v2.4.9 ` // Version is the current version of subfinder -const Version = `2.4.8` +const Version = `v2.4.9` // showBanner is used to show the banner to the user func showBanner() { From 9cc4fa2bf5181142fa0e2c5741073c631cca3f8d Mon Sep 17 00:00:00 2001 From: sandeep Date: Tue, 10 Aug 2021 17:20:44 +0530 Subject: [PATCH 22/43] lint workflow update --- .github/workflows/lint-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index 794d073..ad6dd61 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -16,4 +16,4 @@ jobs: with: version: latest args: --timeout 5m - working-directory: . \ No newline at end of file + working-directory: v2/ \ No newline at end of file From bba144f1b86924f1b6606ab7203616b65b3d5081 Mon Sep 17 00:00:00 2001 From: maldevel Date: Tue, 10 Aug 2021 21:20:05 +0300 Subject: [PATCH 23/43] recon.dev bug fixing, json response has changed --- v2/pkg/subscraping/agent.go | 6 +++--- v2/pkg/subscraping/sources/recon/recon.go | 10 ++++++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/v2/pkg/subscraping/agent.go b/v2/pkg/subscraping/agent.go index a039dfc..a940028 100755 --- a/v2/pkg/subscraping/agent.go +++ b/v2/pkg/subscraping/agent.go @@ -78,9 +78,9 @@ func (s *Session) HTTPRequest(ctx context.Context, method, requestURL, cookies s return nil, err } - req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36") - req.Header.Set("Accept", "*/*") - req.Header.Set("Accept-Language", "en") + req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:90.0) Gecko/20100101 Firefox/90.0") + req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8") //"*/*") + req.Header.Set("Accept-Language", "en-US,en;q=0.5") //"en") req.Header.Set("Connection", "close") if basicAuth.Username != "" || basicAuth.Password != "" { diff --git a/v2/pkg/subscraping/sources/recon/recon.go b/v2/pkg/subscraping/sources/recon/recon.go index b1539b1..ff4b9d1 100644 --- a/v2/pkg/subscraping/sources/recon/recon.go +++ b/v2/pkg/subscraping/sources/recon/recon.go @@ -10,7 +10,11 @@ import ( ) type subdomain struct { - RawDomain string `json:"rawDomain"` + Domains []string `json:"domains"` + Ip string `json:"ip"` + RawDomains []string `json:"rawDomains"` + RawPort string `json:"rawPort"` + RawIp string `json:"rawIp"` } // Source is the passive scraping agent @@ -44,7 +48,9 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se resp.Body.Close() for _, subdomain := range subdomains { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain.RawDomain} + for _, dmn := range subdomain.RawDomains { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: dmn} + } } }() From e4dcab16454947cf3fef2c266ecb25ee761b1043 Mon Sep 17 00:00:00 2001 From: maldevel Date: Tue, 10 Aug 2021 21:23:46 +0300 Subject: [PATCH 24/43] Update agent.go --- v2/pkg/subscraping/agent.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/pkg/subscraping/agent.go b/v2/pkg/subscraping/agent.go index a940028..a039dfc 100755 --- a/v2/pkg/subscraping/agent.go +++ b/v2/pkg/subscraping/agent.go @@ -78,9 +78,9 @@ func (s *Session) HTTPRequest(ctx context.Context, method, requestURL, cookies s return nil, err } - req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:90.0) Gecko/20100101 Firefox/90.0") - req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8") //"*/*") - req.Header.Set("Accept-Language", "en-US,en;q=0.5") //"en") + req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36") + req.Header.Set("Accept", "*/*") + req.Header.Set("Accept-Language", "en") req.Header.Set("Connection", "close") if basicAuth.Username != "" || basicAuth.Password != "" { From e5d9f1a58311c6cc43152d3ec5d5431a5155913d Mon Sep 17 00:00:00 2001 From: Defuse Venue Date: Wed, 11 Aug 2021 21:25:07 +0900 Subject: [PATCH 25/43] User Agent Randomization HTTP User-Agent header randomization. Adds -unsafe flag to send requests with default User-Agent header. --- v2/go.mod | 1 + v2/go.sum | 8 ++++++++ v2/pkg/passive/passive.go | 4 ++-- v2/pkg/runner/enumerate.go | 2 +- v2/pkg/runner/options.go | 2 ++ v2/pkg/subscraping/agent.go | 12 ++++++++++-- v2/pkg/subscraping/types.go | 2 ++ 7 files changed, 26 insertions(+), 5 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 543cf9e..7bd474d 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -3,6 +3,7 @@ module github.com/projectdiscovery/subfinder/v2 go 1.16 require ( + github.com/corpix/uarand v0.1.1 github.com/hako/durafmt v0.0.0-20210316092057-3a2c319c1acd github.com/json-iterator/go v1.1.10 github.com/lib/pq v1.10.0 diff --git a/v2/go.sum b/v2/go.sum index 0c27ad7..e7a7da7 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -1,8 +1,14 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic= +github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA= +github.com/corpix/uarand v0.1.1 h1:RMr1TWc9F4n5jiPDzFHtmaUXLKLNUFK0SgCLo4BhX/U= +github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -51,12 +57,14 @@ github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/z github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= diff --git a/v2/pkg/passive/passive.go b/v2/pkg/passive/passive.go index ace9846..e2aedae 100644 --- a/v2/pkg/passive/passive.go +++ b/v2/pkg/passive/passive.go @@ -11,11 +11,11 @@ import ( ) // EnumerateSubdomains enumerates all the subdomains for a given domain -func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, timeout int, maxEnumTime time.Duration) chan subscraping.Result { +func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, unsafe bool, timeout int, maxEnumTime time.Duration) chan subscraping.Result { results := make(chan subscraping.Result) go func() { - session, err := subscraping.NewSession(domain, keys, proxy, timeout) + session, err := subscraping.NewSession(domain, keys, proxy, unsafe, timeout) if err != nil { results <- subscraping.Result{Type: subscraping.Error, Error: fmt.Errorf("could not init passive session for %s: %s", domain, err)} } diff --git a/v2/pkg/runner/enumerate.go b/v2/pkg/runner/enumerate.go index 84cbb03..2e61c83 100644 --- a/v2/pkg/runner/enumerate.go +++ b/v2/pkg/runner/enumerate.go @@ -37,7 +37,7 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain string, outpu // Run the passive subdomain enumeration now := time.Now() - passiveResults := r.passiveAgent.EnumerateSubdomains(domain, &keys, r.options.Proxy, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute) + passiveResults := r.passiveAgent.EnumerateSubdomains(domain, &keys, r.options.Proxy, r.options.UnSafe, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute) wg := &sync.WaitGroup{} wg.Add(1) diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index 6525c2b..c7d9d94 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -40,6 +40,7 @@ type Options struct { ResolverList string // ResolverList is a text file containing list of resolvers to use for enumeration ConfigFile string // ConfigFile contains the location of the config file Proxy string // HTTP proxy + UnSafe bool // Send HTTP request without User-Agent header randomization YAMLConfig ConfigFile // YAMLConfig contains the unmarshalled yaml config file } @@ -77,6 +78,7 @@ func ParseOptions() *Options { flag.BoolVar(&options.RemoveWildcard, "nW", false, "Remove Wildcard & Dead Subdomains from output") flag.StringVar(&options.ConfigFile, "config", path.Join(config, "config.yaml"), "Configuration file for API Keys, etc") flag.StringVar(&options.Proxy, "http-proxy", "", "HTTP proxy to use") + flag.BoolVar(&options.UnSafe, "unsafe", false,"Send HTTP request without User-Agent header randomization") flag.BoolVar(&options.Version, "version", false, "Show version of subfinder") flag.Parse() diff --git a/v2/pkg/subscraping/agent.go b/v2/pkg/subscraping/agent.go index a039dfc..e82a596 100755 --- a/v2/pkg/subscraping/agent.go +++ b/v2/pkg/subscraping/agent.go @@ -10,11 +10,12 @@ import ( "net/url" "time" + "github.com/corpix/uarand" "github.com/projectdiscovery/gologger" ) // NewSession creates a new session object for a domain -func NewSession(domain string, keys *Keys, proxy string, timeout int) (*Session, error) { +func NewSession(domain string, keys *Keys, proxy string, unsafe bool, timeout int) (*Session, error) { Transport := &http.Transport{ MaxIdleConns: 100, MaxIdleConnsPerHost: 100, @@ -42,6 +43,7 @@ func NewSession(domain string, keys *Keys, proxy string, timeout int) (*Session, session := &Session{ Client: client, Keys: keys, + UnSafe: unsafe, } // Create a new extractor object for the current domain @@ -78,7 +80,13 @@ func (s *Session) HTTPRequest(ctx context.Context, method, requestURL, cookies s return nil, err } - req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36") + // Unsafe requests do not use user-agent randomization + if s.UnSafe { + req.Header.Set("User-Agent", "subfinder - Open-source Project (github.com/projectdiscovery/subfinder)") + } else { + req.Header.Set("User-Agent", uarand.GetRandom()) + } + req.Header.Set("Accept", "*/*") req.Header.Set("Accept-Language", "en") req.Header.Set("Connection", "close") diff --git a/v2/pkg/subscraping/types.go b/v2/pkg/subscraping/types.go index e3211be..9df42ff 100755 --- a/v2/pkg/subscraping/types.go +++ b/v2/pkg/subscraping/types.go @@ -31,6 +31,8 @@ type Session struct { Keys *Keys // Client is the current http client Client *http.Client + // Perform unsafe HTTP requests without user agent randomization + UnSafe bool } // Keys contains the current API Keys we have in store From 5bc901c47d0a22b6c1d5a2ea06ca65ad4bfdccfd Mon Sep 17 00:00:00 2001 From: Defuse Venue Date: Wed, 11 Aug 2021 21:30:15 +0900 Subject: [PATCH 26/43] Add unsafe flag to README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index fe06180..c907528 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,7 @@ This will display help for the tool. Here are all the switches it supports. | -t | Number of concurrent goroutines for resolving (default 10) | subfinder -t 100 | | -timeout | Seconds to wait before timing out (default 30) | subfinder -timeout 30 | | -http-proxy | Http Proxy | subfinder -http-proxy http://localhost:3128 | +| -unsafe | Send HTTP request without User-Agent header randomization | subfinder -unsafe | | -v | Show Verbose output | subfinder -v | | -version | Show current program version | subfinder -version | From f35adaf14356b6d6f4efe7b1241105c4829bc114 Mon Sep 17 00:00:00 2001 From: Defuse Venue Date: Wed, 11 Aug 2021 21:34:20 +0900 Subject: [PATCH 27/43] Fix lint issue --- v2/pkg/runner/options.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index c7d9d94..ce9c317 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -40,7 +40,7 @@ type Options struct { ResolverList string // ResolverList is a text file containing list of resolvers to use for enumeration ConfigFile string // ConfigFile contains the location of the config file Proxy string // HTTP proxy - UnSafe bool // Send HTTP request without User-Agent header randomization + UnSafe bool // Send HTTP request without User-Agent header randomization YAMLConfig ConfigFile // YAMLConfig contains the unmarshalled yaml config file } @@ -78,7 +78,7 @@ func ParseOptions() *Options { flag.BoolVar(&options.RemoveWildcard, "nW", false, "Remove Wildcard & Dead Subdomains from output") flag.StringVar(&options.ConfigFile, "config", path.Join(config, "config.yaml"), "Configuration file for API Keys, etc") flag.StringVar(&options.Proxy, "http-proxy", "", "HTTP proxy to use") - flag.BoolVar(&options.UnSafe, "unsafe", false,"Send HTTP request without User-Agent header randomization") + flag.BoolVar(&options.UnSafe, "unsafe", false, "Send HTTP request without User-Agent header randomization") flag.BoolVar(&options.Version, "version", false, "Show version of subfinder") flag.Parse() From 095d4295f230b4cbb26ee0bed47e63c2b0d36e67 Mon Sep 17 00:00:00 2001 From: Defuse Venue Date: Wed, 11 Aug 2021 21:42:52 +0900 Subject: [PATCH 28/43] Remove unsafe flag HTTP User-Agent should be randomized by default since subfinder uses third party apis --- README.md | 1 - v2/pkg/passive/passive.go | 4 ++-- v2/pkg/runner/enumerate.go | 2 +- v2/pkg/runner/options.go | 2 -- v2/pkg/subscraping/agent.go | 11 ++--------- v2/pkg/subscraping/types.go | 2 -- 6 files changed, 5 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index c907528..fe06180 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,6 @@ This will display help for the tool. Here are all the switches it supports. | -t | Number of concurrent goroutines for resolving (default 10) | subfinder -t 100 | | -timeout | Seconds to wait before timing out (default 30) | subfinder -timeout 30 | | -http-proxy | Http Proxy | subfinder -http-proxy http://localhost:3128 | -| -unsafe | Send HTTP request without User-Agent header randomization | subfinder -unsafe | | -v | Show Verbose output | subfinder -v | | -version | Show current program version | subfinder -version | diff --git a/v2/pkg/passive/passive.go b/v2/pkg/passive/passive.go index e2aedae..ace9846 100644 --- a/v2/pkg/passive/passive.go +++ b/v2/pkg/passive/passive.go @@ -11,11 +11,11 @@ import ( ) // EnumerateSubdomains enumerates all the subdomains for a given domain -func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, unsafe bool, timeout int, maxEnumTime time.Duration) chan subscraping.Result { +func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, timeout int, maxEnumTime time.Duration) chan subscraping.Result { results := make(chan subscraping.Result) go func() { - session, err := subscraping.NewSession(domain, keys, proxy, unsafe, timeout) + session, err := subscraping.NewSession(domain, keys, proxy, timeout) if err != nil { results <- subscraping.Result{Type: subscraping.Error, Error: fmt.Errorf("could not init passive session for %s: %s", domain, err)} } diff --git a/v2/pkg/runner/enumerate.go b/v2/pkg/runner/enumerate.go index 2e61c83..84cbb03 100644 --- a/v2/pkg/runner/enumerate.go +++ b/v2/pkg/runner/enumerate.go @@ -37,7 +37,7 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain string, outpu // Run the passive subdomain enumeration now := time.Now() - passiveResults := r.passiveAgent.EnumerateSubdomains(domain, &keys, r.options.Proxy, r.options.UnSafe, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute) + passiveResults := r.passiveAgent.EnumerateSubdomains(domain, &keys, r.options.Proxy, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute) wg := &sync.WaitGroup{} wg.Add(1) diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index ce9c317..6525c2b 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -40,7 +40,6 @@ type Options struct { ResolverList string // ResolverList is a text file containing list of resolvers to use for enumeration ConfigFile string // ConfigFile contains the location of the config file Proxy string // HTTP proxy - UnSafe bool // Send HTTP request without User-Agent header randomization YAMLConfig ConfigFile // YAMLConfig contains the unmarshalled yaml config file } @@ -78,7 +77,6 @@ func ParseOptions() *Options { flag.BoolVar(&options.RemoveWildcard, "nW", false, "Remove Wildcard & Dead Subdomains from output") flag.StringVar(&options.ConfigFile, "config", path.Join(config, "config.yaml"), "Configuration file for API Keys, etc") flag.StringVar(&options.Proxy, "http-proxy", "", "HTTP proxy to use") - flag.BoolVar(&options.UnSafe, "unsafe", false, "Send HTTP request without User-Agent header randomization") flag.BoolVar(&options.Version, "version", false, "Show version of subfinder") flag.Parse() diff --git a/v2/pkg/subscraping/agent.go b/v2/pkg/subscraping/agent.go index e82a596..6b89337 100755 --- a/v2/pkg/subscraping/agent.go +++ b/v2/pkg/subscraping/agent.go @@ -15,7 +15,7 @@ import ( ) // NewSession creates a new session object for a domain -func NewSession(domain string, keys *Keys, proxy string, unsafe bool, timeout int) (*Session, error) { +func NewSession(domain string, keys *Keys, proxy string, timeout int) (*Session, error) { Transport := &http.Transport{ MaxIdleConns: 100, MaxIdleConnsPerHost: 100, @@ -43,7 +43,6 @@ func NewSession(domain string, keys *Keys, proxy string, unsafe bool, timeout in session := &Session{ Client: client, Keys: keys, - UnSafe: unsafe, } // Create a new extractor object for the current domain @@ -80,13 +79,7 @@ func (s *Session) HTTPRequest(ctx context.Context, method, requestURL, cookies s return nil, err } - // Unsafe requests do not use user-agent randomization - if s.UnSafe { - req.Header.Set("User-Agent", "subfinder - Open-source Project (github.com/projectdiscovery/subfinder)") - } else { - req.Header.Set("User-Agent", uarand.GetRandom()) - } - + req.Header.Set("User-Agent", uarand.GetRandom()) req.Header.Set("Accept", "*/*") req.Header.Set("Accept-Language", "en") req.Header.Set("Connection", "close") diff --git a/v2/pkg/subscraping/types.go b/v2/pkg/subscraping/types.go index 9df42ff..e3211be 100755 --- a/v2/pkg/subscraping/types.go +++ b/v2/pkg/subscraping/types.go @@ -31,8 +31,6 @@ type Session struct { Keys *Keys // Client is the current http client Client *http.Client - // Perform unsafe HTTP requests without user agent randomization - UnSafe bool } // Keys contains the current API Keys we have in store From ec79e8053a8f6c0a018bad8c409ef1897b105efe Mon Sep 17 00:00:00 2001 From: Defuse Venue Date: Thu, 12 Aug 2021 08:29:58 +0900 Subject: [PATCH 29/43] Add rate limit feature --- v2/go.mod | 1 + v2/go.sum | 3 +++ v2/pkg/passive/passive.go | 4 ++-- v2/pkg/runner/enumerate.go | 2 +- v2/pkg/runner/options.go | 2 ++ v2/pkg/subscraping/agent.go | 12 +++++++++++- v2/pkg/subscraping/types.go | 4 ++++ 7 files changed, 24 insertions(+), 4 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 7bd474d..3e7c5c9 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -14,5 +14,6 @@ require ( github.com/rs/xid v1.3.0 github.com/stretchr/testify v1.7.0 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 + go.uber.org/ratelimit v0.2.0 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b ) diff --git a/v2/go.sum b/v2/go.sum index e7a7da7..1b4b092 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -3,6 +3,7 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic= github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= +github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -105,7 +106,9 @@ github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpP github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA= go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= diff --git a/v2/pkg/passive/passive.go b/v2/pkg/passive/passive.go index ace9846..91e44e9 100644 --- a/v2/pkg/passive/passive.go +++ b/v2/pkg/passive/passive.go @@ -11,11 +11,11 @@ import ( ) // EnumerateSubdomains enumerates all the subdomains for a given domain -func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, timeout int, maxEnumTime time.Duration) chan subscraping.Result { +func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, rateLimit int, timeout int, maxEnumTime time.Duration) chan subscraping.Result { results := make(chan subscraping.Result) go func() { - session, err := subscraping.NewSession(domain, keys, proxy, timeout) + session, err := subscraping.NewSession(domain, keys, proxy, rateLimit, timeout) if err != nil { results <- subscraping.Result{Type: subscraping.Error, Error: fmt.Errorf("could not init passive session for %s: %s", domain, err)} } diff --git a/v2/pkg/runner/enumerate.go b/v2/pkg/runner/enumerate.go index 84cbb03..5df3fb6 100644 --- a/v2/pkg/runner/enumerate.go +++ b/v2/pkg/runner/enumerate.go @@ -37,7 +37,7 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain string, outpu // Run the passive subdomain enumeration now := time.Now() - passiveResults := r.passiveAgent.EnumerateSubdomains(domain, &keys, r.options.Proxy, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute) + passiveResults := r.passiveAgent.EnumerateSubdomains(domain, &keys, r.options.Proxy, r.options.RateLimit, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute) wg := &sync.WaitGroup{} wg.Add(1) diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index 6525c2b..e672d0b 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -40,6 +40,7 @@ type Options struct { ResolverList string // ResolverList is a text file containing list of resolvers to use for enumeration ConfigFile string // ConfigFile contains the location of the config file Proxy string // HTTP proxy + RateLimit int // Maximum number of HTTP requests to send per second YAMLConfig ConfigFile // YAMLConfig contains the unmarshalled yaml config file } @@ -77,6 +78,7 @@ func ParseOptions() *Options { flag.BoolVar(&options.RemoveWildcard, "nW", false, "Remove Wildcard & Dead Subdomains from output") flag.StringVar(&options.ConfigFile, "config", path.Join(config, "config.yaml"), "Configuration file for API Keys, etc") flag.StringVar(&options.Proxy, "http-proxy", "", "HTTP proxy to use") + flag.IntVar(&options.RateLimit, "rate-limit", 0, "Maximum number of HTTP requests to send per second") flag.BoolVar(&options.Version, "version", false, "Show version of subfinder") flag.Parse() diff --git a/v2/pkg/subscraping/agent.go b/v2/pkg/subscraping/agent.go index 6b89337..3fafe12 100755 --- a/v2/pkg/subscraping/agent.go +++ b/v2/pkg/subscraping/agent.go @@ -12,10 +12,11 @@ import ( "github.com/corpix/uarand" "github.com/projectdiscovery/gologger" + "go.uber.org/ratelimit" ) // NewSession creates a new session object for a domain -func NewSession(domain string, keys *Keys, proxy string, timeout int) (*Session, error) { +func NewSession(domain string, keys *Keys, proxy string, rateLimit int, timeout int) (*Session, error) { Transport := &http.Transport{ MaxIdleConns: 100, MaxIdleConnsPerHost: 100, @@ -45,6 +46,13 @@ func NewSession(domain string, keys *Keys, proxy string, timeout int) (*Session, Keys: keys, } + // Initiate rate limit instance + if rateLimit > 0 { + session.RateLimiter = ratelimit.New(rateLimit) + } else { + session.RateLimiter = ratelimit.NewUnlimited() + } + // Create a new extractor object for the current domain extractor, err := NewSubdomainExtractor(domain) session.Extractor = extractor @@ -96,6 +104,8 @@ func (s *Session) HTTPRequest(ctx context.Context, method, requestURL, cookies s req.Header.Set(key, value) } + s.RateLimiter.Take() + return httpRequestWrapper(s.Client, req) } diff --git a/v2/pkg/subscraping/types.go b/v2/pkg/subscraping/types.go index e3211be..f49187a 100755 --- a/v2/pkg/subscraping/types.go +++ b/v2/pkg/subscraping/types.go @@ -4,6 +4,8 @@ import ( "context" "net/http" "regexp" + + "go.uber.org/ratelimit" ) // BasicAuth request's Authorization header @@ -31,6 +33,8 @@ type Session struct { Keys *Keys // Client is the current http client Client *http.Client + // Rate limit instance + RateLimiter ratelimit.Limiter } // Keys contains the current API Keys we have in store From f9381ceec9352301276179b8e656e75eee9ed66d Mon Sep 17 00:00:00 2001 From: Defuse Venue Date: Thu, 12 Aug 2021 08:33:18 +0900 Subject: [PATCH 30/43] Add rate-limit flag to README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index fe06180..7f55e62 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,7 @@ This will display help for the tool. Here are all the switches it supports. | -t | Number of concurrent goroutines for resolving (default 10) | subfinder -t 100 | | -timeout | Seconds to wait before timing out (default 30) | subfinder -timeout 30 | | -http-proxy | Http Proxy | subfinder -http-proxy http://localhost:3128 | +| -rate-limit | Maximum number of HTTP requests to send per second | subfinder -rate-limit 10 | | -v | Show Verbose output | subfinder -v | | -version | Show current program version | subfinder -version | From cb548cad51329a19e97ad49987512d80393bd6ed Mon Sep 17 00:00:00 2001 From: Defuse Venue Date: Thu, 12 Aug 2021 08:36:26 +0900 Subject: [PATCH 31/43] Update options.go --- v2/pkg/runner/options.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index e672d0b..9413538 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -40,7 +40,7 @@ type Options struct { ResolverList string // ResolverList is a text file containing list of resolvers to use for enumeration ConfigFile string // ConfigFile contains the location of the config file Proxy string // HTTP proxy - RateLimit int // Maximum number of HTTP requests to send per second + RateLimit int // Maximum number of HTTP requests to send per second YAMLConfig ConfigFile // YAMLConfig contains the unmarshalled yaml config file } From a1e27912719c01adb13e674706cdc3b87997a507 Mon Sep 17 00:00:00 2001 From: Defuse Venue Date: Fri, 13 Aug 2021 07:39:05 +0900 Subject: [PATCH 32/43] Change variable rateLimit to rl --- v2/pkg/passive/passive.go | 4 ++-- v2/pkg/subscraping/agent.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/v2/pkg/passive/passive.go b/v2/pkg/passive/passive.go index 91e44e9..d925701 100644 --- a/v2/pkg/passive/passive.go +++ b/v2/pkg/passive/passive.go @@ -11,11 +11,11 @@ import ( ) // EnumerateSubdomains enumerates all the subdomains for a given domain -func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, rateLimit int, timeout int, maxEnumTime time.Duration) chan subscraping.Result { +func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, rl int, timeout int, maxEnumTime time.Duration) chan subscraping.Result { results := make(chan subscraping.Result) go func() { - session, err := subscraping.NewSession(domain, keys, proxy, rateLimit, timeout) + session, err := subscraping.NewSession(domain, keys, proxy, rl, timeout) if err != nil { results <- subscraping.Result{Type: subscraping.Error, Error: fmt.Errorf("could not init passive session for %s: %s", domain, err)} } diff --git a/v2/pkg/subscraping/agent.go b/v2/pkg/subscraping/agent.go index 3fafe12..40c5a67 100755 --- a/v2/pkg/subscraping/agent.go +++ b/v2/pkg/subscraping/agent.go @@ -16,7 +16,7 @@ import ( ) // NewSession creates a new session object for a domain -func NewSession(domain string, keys *Keys, proxy string, rateLimit int, timeout int) (*Session, error) { +func NewSession(domain string, keys *Keys, proxy string, rl int, timeout int) (*Session, error) { Transport := &http.Transport{ MaxIdleConns: 100, MaxIdleConnsPerHost: 100, @@ -47,8 +47,8 @@ func NewSession(domain string, keys *Keys, proxy string, rateLimit int, timeout } // Initiate rate limit instance - if rateLimit > 0 { - session.RateLimiter = ratelimit.New(rateLimit) + if rl > 0 { + session.RateLimiter = ratelimit.New(rl) } else { session.RateLimiter = ratelimit.NewUnlimited() } From 0aac1eb68567ab68dfefc75e511ba67718473339 Mon Sep 17 00:00:00 2001 From: Defuse Venue Date: Fri, 13 Aug 2021 07:41:11 +0900 Subject: [PATCH 33/43] Fix lint --- v2/pkg/passive/passive.go | 4 ++-- v2/pkg/subscraping/agent.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/v2/pkg/passive/passive.go b/v2/pkg/passive/passive.go index d925701..0ed98d4 100644 --- a/v2/pkg/passive/passive.go +++ b/v2/pkg/passive/passive.go @@ -11,11 +11,11 @@ import ( ) // EnumerateSubdomains enumerates all the subdomains for a given domain -func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, rl int, timeout int, maxEnumTime time.Duration) chan subscraping.Result { +func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, rateLimit, timeout int, maxEnumTime time.Duration) chan subscraping.Result { results := make(chan subscraping.Result) go func() { - session, err := subscraping.NewSession(domain, keys, proxy, rl, timeout) + session, err := subscraping.NewSession(domain, keys, proxy, rateLimit, timeout) if err != nil { results <- subscraping.Result{Type: subscraping.Error, Error: fmt.Errorf("could not init passive session for %s: %s", domain, err)} } diff --git a/v2/pkg/subscraping/agent.go b/v2/pkg/subscraping/agent.go index 40c5a67..6797542 100755 --- a/v2/pkg/subscraping/agent.go +++ b/v2/pkg/subscraping/agent.go @@ -16,7 +16,7 @@ import ( ) // NewSession creates a new session object for a domain -func NewSession(domain string, keys *Keys, proxy string, rl int, timeout int) (*Session, error) { +func NewSession(domain string, keys *Keys, proxy string, rateLimit, timeout int) (*Session, error) { Transport := &http.Transport{ MaxIdleConns: 100, MaxIdleConnsPerHost: 100, @@ -47,8 +47,8 @@ func NewSession(domain string, keys *Keys, proxy string, rl int, timeout int) (* } // Initiate rate limit instance - if rl > 0 { - session.RateLimiter = ratelimit.New(rl) + if rateLimit > 0 { + session.RateLimiter = ratelimit.New(rateLimit) } else { session.RateLimiter = ratelimit.NewUnlimited() } From 60e20101421d16d83f0637f65cc444ceb46bea15 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Aug 2021 14:03:10 +0000 Subject: [PATCH 34/43] chore(deps): bump golang from 1.16.7-alpine to 1.17.0-alpine Bumps golang from 1.16.7-alpine to 1.17.0-alpine. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e8ba955..4f72364 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build -FROM golang:1.16.7-alpine AS build-env +FROM golang:1.17.0-alpine AS build-env RUN GO111MODULE=on go get -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder # Release From bd0ccd985e403df324514f2cc635cd6f98067fe4 Mon Sep 17 00:00:00 2001 From: Mikel Olasagasti Uranga Date: Tue, 31 Aug 2021 23:36:22 +0200 Subject: [PATCH 35/43] Remove executable bit from source files --- v2/pkg/subscraping/agent.go | 0 v2/pkg/subscraping/sources/archiveis/archiveis.go | 0 v2/pkg/subscraping/sources/binaryedge/binaryedge.go | 0 v2/pkg/subscraping/sources/bufferover/bufferover.go | 0 v2/pkg/subscraping/sources/certspotter/certspotter.go | 0 v2/pkg/subscraping/sources/commoncrawl/commoncrawl.go | 0 v2/pkg/subscraping/sources/crtsh/crtsh.go | 0 v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go | 0 v2/pkg/subscraping/sources/hackertarget/hackertarget.go | 0 v2/pkg/subscraping/sources/passivetotal/passivetotal.go | 0 v2/pkg/subscraping/sources/securitytrails/securitytrails.go | 0 v2/pkg/subscraping/sources/sitedossier/sitedossier.go | 0 v2/pkg/subscraping/sources/threatcrowd/threatcrowd.go | 0 v2/pkg/subscraping/sources/threatminer/threatminer.go | 0 v2/pkg/subscraping/sources/virustotal/virustotal.go | 0 v2/pkg/subscraping/sources/waybackarchive/waybackarchive.go | 0 v2/pkg/subscraping/types.go | 0 v2/pkg/subscraping/utils.go | 0 18 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 v2/pkg/subscraping/agent.go mode change 100755 => 100644 v2/pkg/subscraping/sources/archiveis/archiveis.go mode change 100755 => 100644 v2/pkg/subscraping/sources/binaryedge/binaryedge.go mode change 100755 => 100644 v2/pkg/subscraping/sources/bufferover/bufferover.go mode change 100755 => 100644 v2/pkg/subscraping/sources/certspotter/certspotter.go mode change 100755 => 100644 v2/pkg/subscraping/sources/commoncrawl/commoncrawl.go mode change 100755 => 100644 v2/pkg/subscraping/sources/crtsh/crtsh.go mode change 100755 => 100644 v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go mode change 100755 => 100644 v2/pkg/subscraping/sources/hackertarget/hackertarget.go mode change 100755 => 100644 v2/pkg/subscraping/sources/passivetotal/passivetotal.go mode change 100755 => 100644 v2/pkg/subscraping/sources/securitytrails/securitytrails.go mode change 100755 => 100644 v2/pkg/subscraping/sources/sitedossier/sitedossier.go mode change 100755 => 100644 v2/pkg/subscraping/sources/threatcrowd/threatcrowd.go mode change 100755 => 100644 v2/pkg/subscraping/sources/threatminer/threatminer.go mode change 100755 => 100644 v2/pkg/subscraping/sources/virustotal/virustotal.go mode change 100755 => 100644 v2/pkg/subscraping/sources/waybackarchive/waybackarchive.go mode change 100755 => 100644 v2/pkg/subscraping/types.go mode change 100755 => 100644 v2/pkg/subscraping/utils.go diff --git a/v2/pkg/subscraping/agent.go b/v2/pkg/subscraping/agent.go old mode 100755 new mode 100644 diff --git a/v2/pkg/subscraping/sources/archiveis/archiveis.go b/v2/pkg/subscraping/sources/archiveis/archiveis.go old mode 100755 new mode 100644 diff --git a/v2/pkg/subscraping/sources/binaryedge/binaryedge.go b/v2/pkg/subscraping/sources/binaryedge/binaryedge.go old mode 100755 new mode 100644 diff --git a/v2/pkg/subscraping/sources/bufferover/bufferover.go b/v2/pkg/subscraping/sources/bufferover/bufferover.go old mode 100755 new mode 100644 diff --git a/v2/pkg/subscraping/sources/certspotter/certspotter.go b/v2/pkg/subscraping/sources/certspotter/certspotter.go old mode 100755 new mode 100644 diff --git a/v2/pkg/subscraping/sources/commoncrawl/commoncrawl.go b/v2/pkg/subscraping/sources/commoncrawl/commoncrawl.go old mode 100755 new mode 100644 diff --git a/v2/pkg/subscraping/sources/crtsh/crtsh.go b/v2/pkg/subscraping/sources/crtsh/crtsh.go old mode 100755 new mode 100644 diff --git a/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go b/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go old mode 100755 new mode 100644 diff --git a/v2/pkg/subscraping/sources/hackertarget/hackertarget.go b/v2/pkg/subscraping/sources/hackertarget/hackertarget.go old mode 100755 new mode 100644 diff --git a/v2/pkg/subscraping/sources/passivetotal/passivetotal.go b/v2/pkg/subscraping/sources/passivetotal/passivetotal.go old mode 100755 new mode 100644 diff --git a/v2/pkg/subscraping/sources/securitytrails/securitytrails.go b/v2/pkg/subscraping/sources/securitytrails/securitytrails.go old mode 100755 new mode 100644 diff --git a/v2/pkg/subscraping/sources/sitedossier/sitedossier.go b/v2/pkg/subscraping/sources/sitedossier/sitedossier.go old mode 100755 new mode 100644 diff --git a/v2/pkg/subscraping/sources/threatcrowd/threatcrowd.go b/v2/pkg/subscraping/sources/threatcrowd/threatcrowd.go old mode 100755 new mode 100644 diff --git a/v2/pkg/subscraping/sources/threatminer/threatminer.go b/v2/pkg/subscraping/sources/threatminer/threatminer.go old mode 100755 new mode 100644 diff --git a/v2/pkg/subscraping/sources/virustotal/virustotal.go b/v2/pkg/subscraping/sources/virustotal/virustotal.go old mode 100755 new mode 100644 diff --git a/v2/pkg/subscraping/sources/waybackarchive/waybackarchive.go b/v2/pkg/subscraping/sources/waybackarchive/waybackarchive.go old mode 100755 new mode 100644 diff --git a/v2/pkg/subscraping/types.go b/v2/pkg/subscraping/types.go old mode 100755 new mode 100644 diff --git a/v2/pkg/subscraping/utils.go b/v2/pkg/subscraping/utils.go old mode 100755 new mode 100644 From 27c324a9d658708aeb10d551f0d1307ed7b4d74c Mon Sep 17 00:00:00 2001 From: Daniel Saxton Date: Sat, 4 Sep 2021 13:40:28 -0500 Subject: [PATCH 36/43] Tweak README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 32603a4..8ce2c82 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,7 @@ This will display help for the tool. Here are all the switches it supports. Subfinder requires **go1.14+** to install successfully. Run the following command to get the repo - ```sh -▶ GO111MODULE=on go get -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder +GO111MODULE=on go get -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder ``` From 0209ef13a0e65bff01953de34e70f0b30735ee1d Mon Sep 17 00:00:00 2001 From: sandeep Date: Thu, 9 Sep 2021 18:16:59 +0530 Subject: [PATCH 37/43] misc update --- .github/workflows/build-test.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index ad0d347..1d3834a 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -10,10 +10,9 @@ jobs: name: Test Builds runs-on: ubuntu-latest steps: - - name: Set up Go - uses: actions/setup-go@v2 + - uses: actions/setup-go@v2 with: - go-version: 1.15 + go-version: 1.17 - name: Check out code uses: actions/checkout@v2 From a0398d445dc1807f869933e7c0626524ef938060 Mon Sep 17 00:00:00 2001 From: sergey Date: Thu, 16 Sep 2021 16:25:58 +0300 Subject: [PATCH 38/43] improvement: updated Spyse mod to added rate-limiting process --- v2/go.mod | 5 ++++- v2/go.sum | 6 ++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 3c1a8f1..581d4a6 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -12,7 +12,7 @@ require ( github.com/projectdiscovery/fdmax v0.0.3 github.com/projectdiscovery/gologger v1.1.4 github.com/rs/xid v1.3.0 - github.com/spyse-com/go-spyse v1.2.1 + github.com/spyse-com/go-spyse v1.2.3 github.com/stretchr/testify v1.7.0 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 go.uber.org/ratelimit v0.2.0 @@ -20,9 +20,11 @@ require ( ) require ( + github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/logrusorgru/aurora v2.0.3+incompatible // indirect github.com/miekg/dns v1.1.41 // indirect + github.com/mitchellh/mapstructure v1.4.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.1 // indirect github.com/pkg/errors v0.9.1 // indirect @@ -30,4 +32,5 @@ require ( github.com/projectdiscovery/retryabledns v1.0.12-0.20210419174848-eec3ac17d61e // indirect golang.org/x/net v0.0.0-20210415231046-e915ea6b2b7d // indirect golang.org/x/sys v0.0.0-20210419170143-37df388d1f33 // indirect + golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect ) diff --git a/v2/go.sum b/v2/go.sum index 3bc2b69..eb454ad 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -98,8 +98,8 @@ github.com/projectdiscovery/retryabledns v1.0.12-0.20210419174848-eec3ac17d61e/g github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/spyse-com/go-spyse v1.2.1 h1:Za/BnLnXWY/DqZZQm2V7NQ69aJ8FgFA8vBiipf3CHC8= -github.com/spyse-com/go-spyse v1.2.1/go.mod h1:YzL0kTQIlCVTtP0Bna4I7p/sKF2rgY1cV32dq/L4oIw= +github.com/spyse-com/go-spyse v1.2.3 h1:0qo0OP5kLv0equyvI7H5pAGuDFTiFON3zXLv4BSw1yY= +github.com/spyse-com/go-spyse v1.2.3/go.mod h1:MTle/KKITU7B2oSlfpzHZPc2k+WH+n5YATh1eUje7po= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= @@ -167,6 +167,8 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= +golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= From 86bcb38f63151a6a08b6d660804f2af0df002fae Mon Sep 17 00:00:00 2001 From: sergey Date: Thu, 16 Sep 2021 17:15:18 +0300 Subject: [PATCH 39/43] improvement: removed init of account service from spyse source --- v2/pkg/subscraping/sources/spyse/spyse.go | 44 +++++++++++------------ 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/v2/pkg/subscraping/sources/spyse/spyse.go b/v2/pkg/subscraping/sources/spyse/spyse.go index fbb7501..a935fbc 100644 --- a/v2/pkg/subscraping/sources/spyse/spyse.go +++ b/v2/pkg/subscraping/sources/spyse/spyse.go @@ -51,45 +51,37 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } - accountSvc := spyse.NewAccountService(client) - - quota, err := accountSvc.Quota(context.Background()) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - return - } - // The default "Search" method returns only first 10 000 subdomains // To obtain more than 10 000 subdomains the "Scroll" method should be using // Note: The "Scroll" method is only available for "PRO" customers, so we need to check // quota.IsScrollSearchEnabled param - if totalResults > searchMethodResultsLimit && quota.IsScrollSearchEnabled { - var scrollResponse *spyse.DomainScrollResponse - scrollResponse, err = domainSvc.ScrollSearch( - ctx, subdomainsSearchParams.Query, "") - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - return - } + if totalResults > searchMethodResultsLimit && client.Account().IsScrollSearchEnabled { + var scrollID string + var scrollResults *spyse.DomainScrollResponse - for len(scrollResponse.Items) > 0 { - scrollResponse, err = domainSvc.ScrollSearch( - context.Background(), subdomainsSearchParams.Query, scrollResponse.SearchID) + for { + scrollResults, err = domainSvc.ScrollSearch(ctx, subdomainsSearchParams.Query, scrollID) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} return } + if len(scrollResults.Items) > 0 { + scrollID = scrollResults.SearchID - for i := range scrollResponse.Items { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: scrollResponse.Items[i].Name} + for i := range scrollResults.Items { + results <- subscraping.Result{ + Source: s.Name(), + Type: subscraping.Subdomain, + Value: scrollResults.Items[i].Name, + } + } } } } else { var limit = 100 - var offset = 0 var searchResults []spyse.Domain - for ; int64(offset) < totalResults && int64(offset) < searchMethodResultsLimit; offset += limit { + for offset := 0; int64(offset) < totalResults && int64(offset) < searchMethodResultsLimit; offset += limit { searchResults, err = domainSvc.Search(ctx, subdomainsSearchParams.Query, limit, offset) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -97,7 +89,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } for i := range searchResults { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: searchResults[i].Name} + results <- subscraping.Result{ + Source: s.Name(), + Type: subscraping.Subdomain, + Value: searchResults[i].Name, + } } } } From 4190e0f0e9c795bb7b55dedc1bc39484716c9426 Mon Sep 17 00:00:00 2001 From: sandeep Date: Sat, 25 Sep 2021 22:27:32 +0530 Subject: [PATCH 40/43] misc update --- README.md | 81 ++++++++----------------------------------------------- 1 file changed, 11 insertions(+), 70 deletions(-) diff --git a/README.md b/README.md index e0a121e..32b71e3 100644 --- a/README.md +++ b/README.md @@ -53,10 +53,10 @@ subfinder -h ``` This will display help for the tool. Here are all the switches it supports. -<<<<<<< HEAD | Flag | Description | Example | | ---------------- | ---------------------------------------------------------- | --------------------------------------------| | -all | Use all sources (slow) for enumeration | subfinder -d uber.com -all | +| -b | IP address to be used as local bind | subfinder -b 172.16.0.1 | | -config | Configuration file for API Keys, etc | subfinder -config config.yaml | | -d | Domain to find subdomains for | subfinder -d uber.com | | -dL | File containing list of domains to enumerate | subfinder -dL hackerone-hosts.txt | @@ -80,41 +80,14 @@ This will display help for the tool. Here are all the switches it supports. | -rate-limit | Maximum number of HTTP requests to send per second | subfinder -rate-limit 10 | | -v | Show Verbose output | subfinder -v | | -version | Show current program version | subfinder -version | -======= -| Flag | Description | Example | -| ---------------- | ---------------------------------------------------------- | -------------------------------------- | -| -all | Use all sources (slow) for enumeration | subfinder -d uber.com -all | -| -b | IP address to be used as local bind | subfinder -b 172.16.0.1 | -| -config | Configuration file for API Keys, etc | subfinder -config config.yaml | -| -d | Domain to find subdomains for | subfinder -d uber.com | -| -dL | File containing list of domains to enumerate | subfinder -dL hackerone-hosts.txt | -| -exclude-sources | List of sources to exclude from enumeration | subfinder -exclude-sources archiveis | -| -max-time | Minutes to wait for enumeration results (default 10) | subfinder -max-time 1 | -| -nC | Don't Use colors in output | subfinder -nC | -| -nW | Remove Wildcard & Dead Subdomains from output | subfinder -nW | -| -ls | List all available sources | subfinder -ls | -| -o | File to write output to (optional) | subfinder -o output.txt | -| -oD | Directory to write enumeration results to (optional) | subfinder -oD ~/outputs | -| -oI | Write output in Host,IP format | subfinder -oI | -| -oJ | Write output in JSON lines Format | subfinder -oJ | -| -r | Comma-separated list of resolvers to use | subfinder -r 1.1.1.1,1.0.0.1 | -| -rL | Text file containing list of resolvers to use | subfinder -rL resolvers.txt | -| -recursive | Enumeration recursive subdomains | subfinder -d news.yahoo.com -recursive | -| -silent | Show only subdomains in output | subfinder -silent | -| -sources | Comma separated list of sources to use | subfinder -sources shodan,censys | -| -t | Number of concurrent goroutines for resolving (default 10) | subfinder -t 100 | -| -timeout | Seconds to wait before timing out (default 30) | subfinder -timeout 30 | -| -v | Show Verbose output | subfinder -v | -| -version | Show current program version | subfinder -version | ->>>>>>> 9b3a5b75dddd3b5891f7af2fabd1dfab7343f4b2 # Installation -Subfinder requires **go1.14+** to install successfully. Run the following command to get the repo - +Subfinder requires **go1.17** to install successfully. Run the following command to get the repo - ```sh -GO111MODULE=on go get -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder +go install -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder@latest ``` @@ -194,44 +167,10 @@ The verbose flag `v` can be used to display verbose information. [bufferover] soti.croma.example.com ``` -The `-o` command can be used to specify an output file. - -```sh -▶ subfinder -d example.com -o output.txt -``` - -To run the tool on a list of domains, `-dL` option can be used. This requires a directory to write the output files. Subdomains for each domain from the list are written in a text file in the directory specified by the `-oD` flag with their name being the domain name. - -```sh -▶ cat domains.txt -hackerone.com -google.com - -▶ subfinder -dL domains.txt -oD ~/path/to/output -▶ ls ~/path/to/output - -hackerone.com.txt -google.com.txt -``` - -You can also get output in json format using `-oJ` switch. This switch saves the output in the JSON lines format. - -If you use the JSON format, or the `Host:IP` format, then it becomes mandatory for you to use the **-nW** format as resolving is essential for these output format. By default, resolving the found subdomains is disabled. - -```sh -▶ subfinder -d hackerone.com -o output.json -oJ -nW -▶ cat output.json - -{"host":"www.hackerone.com","ip":"104.16.99.52"} -{"host":"mta-sts.hackerone.com","ip":"185.199.108.153"} -{"host":"hackerone.com","ip":"104.16.100.52"} -{"host":"mta-sts.managed.hackerone.com","ip":"185.199.110.153"} -``` - The subdomains discovered can be piped to other tools too. For example, you can pipe the subdomains discovered by subfinder to httpx [httpx](https://github.com/projectdiscovery/httpx) which will then find running http servers on the host. ```sh -▶ echo hackerone.com | subfinder -silent | httpx -silent +echo hackerone.com | subfinder -silent | httpx -silent http://hackerone.com http://www.hackerone.com @@ -243,8 +182,10 @@ http://mta-sts.managed.hackerone.com If your enterprise uses source routing to choose network output, or your computer has many public network interfaces (eg: public Wi-Fi + 4G connection + Ethernet Wire + VPN), you might want to choose your output network by binding IP source. In this case, you can use `-b` option. In the example below, we have 3 network interfaces able to communicate to the Internet through 3 different outputs. Each output is chosen by binding one source IP with `-b` option. + ```sh -▶ ip addr +ip addr + [...] 3: wlp3s0: mtu 1500 qdisc mq state UP group default qlen 1000 link/ether e8:b1:fc:50:90:a0 brd ff:ff:ff:ff:ff:ff @@ -259,9 +200,9 @@ In the example below, we have 3 network interfaces able to communicate to the In inet 192.168.8.100/24 brd 192.168.8.255 scope global dynamic noprefixroute enx0c5b8f279a64 valid_lft 86396sec preferred_lft 86396sec -▶ subfinder -d hackerone.com -b 192.168.1.87 -▶ subfinder -d hackerone.com -b 192.168.254.70 -▶ subfinder -d hackerone.com -b 192.168.8.100 +subfinder -d hackerone.com -b 192.168.1.87 +subfinder -d hackerone.com -b 192.168.254.70 +subfinder -d hackerone.com -b 192.168.8.100 ``` @@ -317,7 +258,7 @@ docker run -v $HOME/.config/subfinder:/root/.config/subfinder -it projectdiscove Usage example: -``` go +```go package main import ( From 781a0411feea6c7b1a023238b6941bf0781491f3 Mon Sep 17 00:00:00 2001 From: sandeep Date: Sat, 25 Sep 2021 23:25:25 +0530 Subject: [PATCH 41/43] readme update --- README.md | 26 +++++--------------------- v2/pkg/runner/options.go | 2 +- 2 files changed, 6 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 32b71e3..fbd2030 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ This will display help for the tool. Here are all the switches it supports. | -sources | Comma separated list of sources to use | subfinder -sources shodan,censys | | -t | Number of concurrent goroutines for resolving (default 10) | subfinder -t 100 | | -timeout | Seconds to wait before timing out (default 30) | subfinder -timeout 30 | -| -http-proxy | Http Proxy | subfinder -http-proxy http://localhost:3128 | +| -proxy | HTTP proxy to use with subfinder | subfinder -proxy http://localhost:3128 | | -rate-limit | Maximum number of HTTP requests to send per second | subfinder -rate-limit 10 | | -v | Show Verbose output | subfinder -v | | -version | Show current program version | subfinder -version | @@ -95,23 +95,7 @@ go install -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder@latest Subfinder will work after using the installation instructions however to configure Subfinder to work with certain services, you will need to have setup API keys. The following services do not work without an API key: -- [Binaryedge](https://binaryedge.io) -- [C99](https://api.c99.nl/) -- [Certspotter](https://sslmate.com/certspotter/api/) -- [Censys](https://censys.io) -- [Chaos](https://chaos.projectdiscovery.io) -- [DnsDB](https://api.dnsdb.info) -- [Github](https://github.com) -- [Intelx](https://intelx.io) -- [Passivetotal](http://passivetotal.org) -- [Recon.dev](https://recon.dev) -- [Robtex](https://www.robtex.com/api/) -- [SecurityTrails](http://securitytrails.com) -- [Shodan](https://shodan.io) -- [Spyse](https://spyse.com) -- [Threatbook](https://x.threatbook.cn/en) -- [Virustotal](https://www.virustotal.com) -- [Zoomeye](https://www.zoomeye.org) +[Binaryedge](https://binaryedge.io), [C99](https://api.c99.nl/), [Certspotter](https://sslmate.com/certspotter/api/), [Chinaz](http://my.chinaz.com/ChinazAPI/DataCenter/MyDataApi), [Censys](https://censys.io), [Chaos](https://chaos.projectdiscovery.io), [DnsDB](https://api.dnsdb.info), [Fofa](https://fofa.so/static_pages/api_help), [Github](https://github.com), [Intelx](https://intelx.io), [Passivetotal](http://passivetotal.org), [Recon.dev](https://recon.dev), [Robtex](https://www.robtex.com/api/), [SecurityTrails](http://securitytrails.com), [Shodan](https://shodan.io), [Spyse](https://spyse.com), [Threatbook](https://x.threatbook.cn/en), [Virustotal](https://www.virustotal.com), [Zoomeye](https://www.zoomeye.org) Theses values are stored in the `$HOME/.config/subfinder/config.yaml` file which will be created when you run the tool for the first time. The configuration file uses the YAML format. Multiple API keys can be specified for each of these services from which one of them will be used for enumeration. @@ -149,7 +133,7 @@ github: To run the tool on a target, just use the following command. ```sh -▶ subfinder -d example.com +subfinder -d example.com ``` The verbose flag `v` can be used to display verbose information. @@ -183,7 +167,7 @@ http://mta-sts.managed.hackerone.com If your enterprise uses source routing to choose network output, or your computer has many public network interfaces (eg: public Wi-Fi + 4G connection + Ethernet Wire + VPN), you might want to choose your output network by binding IP source. In this case, you can use `-b` option. In the example below, we have 3 network interfaces able to communicate to the Internet through 3 different outputs. Each output is chosen by binding one source IP with `-b` option. -```sh +```console ip addr [...] @@ -215,7 +199,7 @@ subfinder -d hackerone.com -b 192.168.8.100 You can use the official dockerhub image at [subfinder](https://hub.docker.com/r/projectdiscovery/subfinder). Simply run - ```sh -▶ docker pull projectdiscovery/subfinder +docker pull projectdiscovery/subfinder:latest ``` The above command will pull the latest tagged release from the dockerhub repository. diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index 4ba2112..36971d9 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -82,7 +82,7 @@ func ParseOptions() *Options { flag.BoolVar(&options.RemoveWildcard, "nW", false, "Remove Wildcard & Dead Subdomains from output") flag.StringVar(&options.LocalIPString, "b", "", "IP address to be used as local bind") flag.StringVar(&options.ConfigFile, "config", path.Join(config, "config.yaml"), "Configuration file for API Keys, etc") - flag.StringVar(&options.Proxy, "http-proxy", "", "HTTP proxy to use") + flag.StringVar(&options.Proxy, "proxy", "", "HTTP proxy to use with subfinder") flag.IntVar(&options.RateLimit, "rate-limit", 0, "Maximum number of HTTP requests to send per second") flag.BoolVar(&options.Version, "version", false, "Show version of subfinder") flag.Parse() From be691365f5e62cd46b2482e9a1c6a7083d63c024 Mon Sep 17 00:00:00 2001 From: sandeep Date: Sat, 25 Sep 2021 23:36:39 +0530 Subject: [PATCH 42/43] Docker workflow update for version tagging --- .github/workflows/dockerhub-push.yml | 27 ++++++++++++++++----------- Dockerfile | 2 +- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/.github/workflows/dockerhub-push.yml b/.github/workflows/dockerhub-push.yml index ab9b9db..3ebf16d 100644 --- a/.github/workflows/dockerhub-push.yml +++ b/.github/workflows/dockerhub-push.yml @@ -9,26 +9,31 @@ jobs: docker: runs-on: ubuntu-latest steps: - - - name: Checkout + + - name: Checkout uses: actions/checkout@v2 - - - name: Set up QEMU + + - name: Get Github tag + id: meta + run: | + echo "::set-output name=tag::$(curl --silent "https://api.github.com/repos/projectdiscovery/subfinder/releases/latest" | jq -r .tag_name)" + + - name: Set up QEMU uses: docker/setup-qemu-action@v1 - - - name: Set up Docker Buildx + + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 - - - name: Login to DockerHub + + - name: Login to DockerHub uses: docker/login-action@v1 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_TOKEN }} - - - name: Build and push + + - name: Build and push uses: docker/build-push-action@v2 with: context: . platforms: linux/amd64,linux/arm64,linux/arm push: true - tags: projectdiscovery/subfinder:latest \ No newline at end of file + tags: projectdiscovery/subfinder:latest,projectdiscovery/subfinder:${{ steps.meta.outputs.tag }} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 7270f70..796402f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # Build FROM golang:1.17-alpine AS build-env -RUN GO111MODULE=on go get -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder +RUN go install -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder@latest # Release FROM alpine:3.14 From d23471b698d91dfc9f8815306ab9b0109a00c67e Mon Sep 17 00:00:00 2001 From: sandeep Date: Sat, 25 Sep 2021 23:59:22 +0530 Subject: [PATCH 43/43] readme update --- README.md | 82 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 43 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index fbd2030..5d64455 100644 --- a/README.md +++ b/README.md @@ -132,23 +132,44 @@ github: # Running Subfinder To run the tool on a target, just use the following command. + ```sh -subfinder -d example.com -``` +subfinder -d hackerone.com -The verbose flag `v` can be used to display verbose information. + __ _____ __ + _______ __/ /_ / __(_)___ ____/ /__ _____ + / ___/ / / / __ \/ /_/ / __ \/ __ / _ \/ ___/ + (__ ) /_/ / /_/ / __/ / / / / /_/ / __/ / +/____/\__,_/_.___/_/ /_/_/ /_/\__,_/\___/_/ v2.4.9 -``` -[bufferover] lutin.dima.example.com -[bufferover] izosimdima.example.com -[bufferover] glazkovdima.example.com -[bufferover] dengshima.example.com -[bufferover] wwwkima.example.com -[bufferover] proxima.example.com -[bufferover] mma.example.com -[bufferover] damidoma.example.com -[bufferover] nomerdoma.example.com -[bufferover] soti.croma.example.com + projectdiscovery.io + +Use with caution. You are responsible for your actions +Developers assume no liability and are not responsible for any misuse or damage. +By using subfinder, you also agree to the terms of the APIs used. + +[INF] Enumerating subdomains for hackerone.com + +www.hackerone.com +support.hackerone.com +links.hackerone.com +api.hackerone.com +o1.email.hackerone.com +go.hackerone.com +3d.hackerone.com +resources.hackerone.com +a.ns.hackerone.com +b.ns.hackerone.com +mta-sts.hackerone.com +docs.hackerone.com +mta-sts.forwarding.hackerone.com +gslink.hackerone.com +hackerone.com +info.hackerone.com +mta-sts.managed.hackerone.com +events.hackerone.com + +[INF] Found 18 subdomains for hackerone.com in 3 seconds 672 milliseconds ``` The subdomains discovered can be piped to other tools too. For example, you can pipe the subdomains discovered by subfinder to httpx [httpx](https://github.com/projectdiscovery/httpx) which will then find running http servers on the host. @@ -183,12 +204,11 @@ ip addr link/ether 0c:5b:8f:a5:63:25 brd ff:ff:ff:ff:ff:ff inet 192.168.8.100/24 brd 192.168.8.255 scope global dynamic noprefixroute enx0c5b8f279a64 valid_lft 86396sec preferred_lft 86396sec - -subfinder -d hackerone.com -b 192.168.1.87 -subfinder -d hackerone.com -b 192.168.254.70 -subfinder -d hackerone.com -b 192.168.8.100 ``` +```sh +subfinder -d hackerone.com -b 192.168.1.87 +``` @@ -196,38 +216,22 @@ subfinder -d hackerone.com -b 192.168.8.100 ## Subfinder with docker -You can use the official dockerhub image at [subfinder](https://hub.docker.com/r/projectdiscovery/subfinder). Simply run - +Pull the latest tagged [subfinder](https://hub.docker.com/r/projectdiscovery/subfinder) docker image: ```sh docker pull projectdiscovery/subfinder:latest ``` -The above command will pull the latest tagged release from the dockerhub repository. +Running subfinder using docker image: -If you want to build the container yourself manually, git clone the repo, then build and run the following commands - -- Clone the repo using `git clone https://github.com/projectdiscovery/subfinder.git` -- Build your docker container ```sh -docker build -t projectdiscovery/subfinder . +docker -t projectdiscovery/subfinder:latest -d hackerone.com ``` -If you are using docker, you need to first create your directory structure holding subfinder configuration file. After modifying the default config.yaml file, you can run: +Running subfinder using docker image with local config file: ```sh -▶ mkdir -p $HOME/.config/subfinder -▶ cp config.yaml $HOME/.config/subfinder/config.yaml -▶ nano $HOME/.config/subfinder/config.yaml -``` - -After that, you can pass it as a volume using the following sample command. -```sh -▶ docker run -v $HOME/.config/subfinder:/root/.config/subfinder -it projectdiscovery/subfinder -d example.com -``` - -For example, this runs the tool against example.com and output the results to your host file system: -```sh -docker run -v $HOME/.config/subfinder:/root/.config/subfinder -it projectdiscovery/subfinder -d example.com > example.com.txt +docker run -v $HOME/.config/subfinder:/root/.config/subfinder -t projectdiscovery/subfinder -d hackerone.com ```