From e85de39f88f6cf7683d1f212260d45533d2f005b Mon Sep 17 00:00:00 2001 From: becivells <732903873@qq.com> Date: Mon, 17 May 2021 11:51:14 +0800 Subject: [PATCH 01/32] #408 Add fofa for SubFinder --- v2/pkg/passive/sources.go | 4 ++ v2/pkg/runner/config.go | 9 +++ v2/pkg/subscraping/sources/fofa/fofa.go | 74 +++++++++++++++++++++++++ v2/pkg/subscraping/types.go | 2 + 4 files changed, 89 insertions(+) create mode 100644 v2/pkg/subscraping/sources/fofa/fofa.go diff --git a/v2/pkg/passive/sources.go b/v2/pkg/passive/sources.go index 3ac0bec..ef35ec4 100644 --- a/v2/pkg/passive/sources.go +++ b/v2/pkg/passive/sources.go @@ -14,6 +14,7 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/crtsh" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdb" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdumpster" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/fofa" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/github" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hackertarget" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/intelx" @@ -110,6 +111,7 @@ var DefaultAllSources = []string{ "virustotal", "waybackarchive", "zoomeye", + "fofa", } // Agent is a struct for running passive subdomain enumeration @@ -198,6 +200,8 @@ func (a *Agent) addSources(sources []string) { a.sources[source] = &waybackarchive.Source{} case "zoomeye": a.sources[source] = &zoomeye.Source{} + case "fofa": + a.sources[source] = &fofa.Source{} } } } diff --git a/v2/pkg/runner/config.go b/v2/pkg/runner/config.go index 81ebe40..faa8420 100644 --- a/v2/pkg/runner/config.go +++ b/v2/pkg/runner/config.go @@ -46,6 +46,7 @@ type ConfigFile struct { URLScan []string `yaml:"urlscan"` Virustotal []string `yaml:"virustotal"` ZoomEye []string `yaml:"zoomeye"` + Fofa []string `yaml:"fofa"` // Version indicates the version of subfinder installed. Version string `yaml:"subfinder-version"` } @@ -194,6 +195,14 @@ func (c *ConfigFile) GetKeys() subscraping.Keys { keys.ZoomEyePassword = parts[1] } } + if len(c.Fofa) > 0 { + fofaKeys := c.Fofa[rand.Intn(len(c.Fofa))] + parts := strings.Split(fofaKeys, ":") + if len(parts) == MultipleKeyPartsLength { + keys.FofaUsername = parts[0] + keys.FofaSecret = parts[1] + } + } return keys } diff --git a/v2/pkg/subscraping/sources/fofa/fofa.go b/v2/pkg/subscraping/sources/fofa/fofa.go new file mode 100644 index 0000000..0250aee --- /dev/null +++ b/v2/pkg/subscraping/sources/fofa/fofa.go @@ -0,0 +1,74 @@ +// Package fofa logic +package fofa + +import ( + "context" + "encoding/base64" + "fmt" + "strings" + + jsoniter "github.com/json-iterator/go" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type fofaResponse struct { + Error bool `json:"error"` + ErrMsg string `json:"errmsg"` + Size int `json:"size"` + Results []string `json:"results"` +} + +// Source is the passive scraping agent +type Source struct{} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + if session.Keys.FofaUsername == "" || session.Keys.FofaSecret == "" { + return + } + + // fofa api doc https://fofa.so/static_pages/api_help + qbase64 := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("domain=\"%s\"", domain))) + resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://fofa.so/api/v1/search/all?full=true&fields=host&page=1&size=10000&email=%s&key=%s&qbase64=%s", session.Keys.FofaUsername, session.Keys.FofaSecret, qbase64)) + if err != nil && resp == nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + + var response fofaResponse + err = jsoniter.NewDecoder(resp.Body).Decode(&response) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() + return + } + resp.Body.Close() + + if response.Error { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", response.ErrMsg)} + return + } + + if response.Size > 0 { + for _, subdomain := range response.Results { + if strings.HasPrefix(strings.ToLower(subdomain), "http://") || strings.HasPrefix(strings.ToLower(subdomain), "https://") { + subdomain = subdomain[strings.Index(subdomain, "//")+2:] + } + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + } + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "fofa" +} diff --git a/v2/pkg/subscraping/types.go b/v2/pkg/subscraping/types.go index 8a0c445..12657d8 100755 --- a/v2/pkg/subscraping/types.go +++ b/v2/pkg/subscraping/types.go @@ -56,6 +56,8 @@ type Keys struct { Virustotal string `json:"virustotal"` ZoomEyeUsername string `json:"zoomeye_username"` ZoomEyePassword string `json:"zoomeye_password"` + FofaUsername string `json:"fofa_username"` + FofaSecret string `json:"fofa_password"` } // Result is a result structure returned by a source From 0a162d88541c14fcc0b95fed0a62d5af5095edff Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Tue, 25 May 2021 23:59:09 +0200 Subject: [PATCH 02/32] golint => revive --- .github/workflows/build.yaml | 4 ++-- .golangci.yml | 10 ++-------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index d0d0e47..0f4e1e9 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -13,10 +13,10 @@ jobs: - name: Checkout code uses: actions/checkout@v2 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v2.5.2 + uses: golangci/golangci-lint-action@v2 with: # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. - version: v1.33 + version: latest args: --timeout 5m working-directory: v2/ diff --git a/.golangci.yml b/.golangci.yml index a16e0ce..c7b06b6 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -90,6 +90,7 @@ linters: - unused - varcheck - whitespace + - revive # don't enable: # - depguard @@ -112,11 +113,4 @@ issues: exclude-use-default: false exclude: # should have a package comment, unless it's in another file for this package (golint) - - 'in another file for this package' - -# golangci.com configuration -# https://github.com/golangci/golangci/wiki/Configuration -service: - golangci-lint-version: 1.31.x # use the fixed version to not introduce new linters unexpectedly - prepare: - - echo "here I can run custom commands, but no preparation needed for this repo" + - 'in another file for this package' \ No newline at end of file From ee65aaf46112757480a35562eed072d2597839dd Mon Sep 17 00:00:00 2001 From: mzack Date: Sat, 3 Jul 2021 13:14:45 +0200 Subject: [PATCH 03/32] switching golinter configuration to latest default by deleting custom file --- .golangci.yml | 116 -------------------------------------------------- 1 file changed, 116 deletions(-) delete mode 100644 .golangci.yml diff --git a/.golangci.yml b/.golangci.yml deleted file mode 100644 index c7b06b6..0000000 --- a/.golangci.yml +++ /dev/null @@ -1,116 +0,0 @@ -linters-settings: - dupl: - threshold: 100 - exhaustive: - default-signifies-exhaustive: false - # funlen: - # lines: 100 - # statements: 50 - goconst: - min-len: 2 - min-occurrences: 2 - gocritic: - enabled-tags: - - diagnostic - - experimental - - opinionated - - performance - - style - disabled-checks: - - dupImport # https://github.com/go-critic/go-critic/issues/845 - - ifElseChain - # gocyclo: - # min-complexity: 15 - goimports: - local-prefixes: github.com/golangci/golangci-lint - golint: - min-confidence: 0 - gomnd: - settings: - mnd: - # don't include the "operation" and "assign" - checks: argument,case,condition,return - govet: - check-shadowing: true - settings: - printf: - funcs: - - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof - - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf - - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf - - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf - # lll: - # line-length: 140 - maligned: - suggest-new: true - misspell: - locale: US - nolintlint: - allow-leading-space: true # don't require machine-readable nolint directives (i.e. with no leading space) - allow-unused: false # report any unused nolint directives - require-explanation: false # don't require an explanation for nolint directives - require-specific: false # don't require nolint directives to be specific about which linter is being skipped - -linters: - # please, do not use `enable-all`: it's deprecated and will be removed soon. - # inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint - disable-all: true - enable: - - bodyclose - - deadcode - - dogsled - - dupl - # - errcheck - - exhaustive - - gochecknoinits - - goconst - - gocritic - - gofmt - - goimports - - golint - - gomnd - - goprintffuncname - - gosimple - - govet - - ineffassign - - interfacer - - maligned - - misspell - - nakedret - - noctx - # - nolintlint - - rowserrcheck - - scopelint - - staticcheck - - structcheck - - stylecheck - - typecheck - - unconvert - - unparam - - unused - - varcheck - - whitespace - - revive - - # don't enable: - # - depguard - # - asciicheck - # - funlen - # - gochecknoglobals - # - gocognit - # - gocyclo - # - godot - # - godox - # - goerr113 - # - gosec - # - lll - # - nestif - # - prealloc - # - testpackage - # - wsl - -issues: - exclude-use-default: false - exclude: - # should have a package comment, unless it's in another file for this package (golint) - - 'in another file for this package' \ No newline at end of file From cddad2b796501d4422ad2dac45aec9b7fe0c45bf Mon Sep 17 00:00:00 2001 From: sandeep Date: Mon, 5 Jul 2021 13:59:49 +0530 Subject: [PATCH 04/32] Update types.go --- v2/pkg/subscraping/types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/subscraping/types.go b/v2/pkg/subscraping/types.go index 12657d8..6c92f4a 100755 --- a/v2/pkg/subscraping/types.go +++ b/v2/pkg/subscraping/types.go @@ -57,7 +57,7 @@ type Keys struct { ZoomEyeUsername string `json:"zoomeye_username"` ZoomEyePassword string `json:"zoomeye_password"` FofaUsername string `json:"fofa_username"` - FofaSecret string `json:"fofa_password"` + FofaSecret string `json:"fofa_secret"` } // Result is a result structure returned by a source From 7c6bd16451cdadc6bf9e461457403cda35da87b3 Mon Sep 17 00:00:00 2001 From: sandeep Date: Mon, 5 Jul 2021 14:22:00 +0530 Subject: [PATCH 05/32] Added Fofa in default list of sources to run --- v2/pkg/passive/sources.go | 1 + 1 file changed, 1 insertion(+) diff --git a/v2/pkg/passive/sources.go b/v2/pkg/passive/sources.go index ef35ec4..1f196ee 100644 --- a/v2/pkg/passive/sources.go +++ b/v2/pkg/passive/sources.go @@ -59,6 +59,7 @@ var DefaultSources = []string{ "threatcrowd", "threatminer", "virustotal", + "fofa", } // DefaultRecursiveSources contains list of default recursive sources From 44c03419c12a86b52dd5a6d08d1d643c41186039 Mon Sep 17 00:00:00 2001 From: r-romanov Date: Sun, 18 Jul 2021 16:16:58 +0300 Subject: [PATCH 06/32] improvement: reworked the Spyse integration to use API v4 and fetch more than 10 000 results --- v2/go.mod | 1 + v2/go.sum | 4 + v2/pkg/subscraping/sources/spyse/spyse.go | 107 ++++++++++++++-------- 3 files changed, 73 insertions(+), 39 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 543cf9e..9e324d5 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -11,6 +11,7 @@ require ( github.com/projectdiscovery/fdmax v0.0.3 github.com/projectdiscovery/gologger v1.1.4 github.com/rs/xid v1.3.0 + github.com/spyse-com/go-spyse v1.2.1 github.com/stretchr/testify v1.7.0 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b diff --git a/v2/go.sum b/v2/go.sum index 0c27ad7..00d9543 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -51,6 +51,8 @@ github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/z github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= +github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -87,6 +89,8 @@ github.com/projectdiscovery/retryabledns v1.0.12-0.20210419174848-eec3ac17d61e/g github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/spyse-com/go-spyse v1.2.1 h1:Za/BnLnXWY/DqZZQm2V7NQ69aJ8FgFA8vBiipf3CHC8= +github.com/spyse-com/go-spyse v1.2.1/go.mod h1:YzL0kTQIlCVTtP0Bna4I7p/sKF2rgY1cV32dq/L4oIw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= diff --git a/v2/pkg/subscraping/sources/spyse/spyse.go b/v2/pkg/subscraping/sources/spyse/spyse.go index 2ae271e..0dd7944 100644 --- a/v2/pkg/subscraping/sources/spyse/spyse.go +++ b/v2/pkg/subscraping/sources/spyse/spyse.go @@ -3,31 +3,11 @@ package spyse import ( "context" - "fmt" - "strconv" - - jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" + "github.com/spyse-com/go-spyse/pkg" ) -type resultObject struct { - Name string `json:"name"` -} - -type dataObject struct { - Items []resultObject `json:"items"` - TotalCount int `json:"total_count"` -} - -type errorObject struct { - Code string `json:"code"` - Message string `json:"message"` -} - -type spyseResult struct { - Data dataObject `json:"data"` - Error []errorObject `json:"error"` -} +const SearchMethodResultsLimit = 10000 // Source is the passive scraping agent type Source struct{} @@ -43,33 +23,82 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } - maxCount := 100 + client, err := spyse.NewClient(session.Keys.Spyse, nil) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + return + } - for offSet := 0; offSet <= maxCount; offSet += 100 { - resp, err := session.Get(ctx, fmt.Sprintf("https://api.spyse.com/v3/data/domain/subdomain?domain=%s&limit=100&offset=%s", domain, strconv.Itoa(offSet)), "", map[string]string{"Authorization": "Bearer " + session.Keys.Spyse}) + domainSvc := spyse.NewDomainService(client) + + var searchDomain = "." + domain + var subdomainsSearchParams spyse.QueryBuilder + + subdomainsSearchParams.AppendParam(spyse.QueryParam{ + Name: domainSvc.Params().Name.Name, + Operator: domainSvc.Params().Name.Operator.EndsWith, + Value: searchDomain, + }) + + totalResults, err := domainSvc.SearchCount(ctx, subdomainsSearchParams.Query) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + return + } + + if totalResults == 0 { + return + } + + accountSvc := spyse.NewAccountService(client) + + quota, err := accountSvc.Quota(context.Background()) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + return + } + + var searchResults []spyse.Domain + + // The default "Search" method returns only first 10 000 subdomains + // To obtain more than 10 000 subdomains the "Scroll" method should be using + // Note: The "Scroll" method is only available for "PRO" customers, so we need to check + // quota.IsScrollSearchEnabled param + if totalResults > SearchMethodResultsLimit && quota.IsScrollSearchEnabled { + searchResults, err := domainSvc.ScrollSearch( + ctx, subdomainsSearchParams.Query, "") if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - session.DiscardHTTPResponse(resp) return } - var response spyseResult - err = jsoniter.NewDecoder(resp.Body).Decode(&response) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() - return + for len(searchResults.Items) > 0 { + searchResults, err = domainSvc.ScrollSearch( + context.Background(), subdomainsSearchParams.Query, searchResults.SearchID) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + return + } + + for _, r := range searchResults.Items { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: r.Name} + } } - resp.Body.Close() + } else { + var limit = 100 + var offset = 0 - if response.Data.TotalCount == 0 { - return - } + for ; int64(offset) < totalResults && int64(offset) < SearchMethodResultsLimit; offset += limit { + searchResults, err = domainSvc.Search(ctx, subdomainsSearchParams.Query, limit, offset) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + return + } - maxCount = response.Data.TotalCount + for _, r := range searchResults { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: r.Name} + } - for _, hostname := range response.Data.Items { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: hostname.Name} } } }() From 85349ba5da9edeee638923fd2cf0aeb5210121f6 Mon Sep 17 00:00:00 2001 From: r-romanov Date: Wed, 21 Jul 2021 10:04:48 +0300 Subject: [PATCH 07/32] refactor: fix some linted fragments --- v2/pkg/subscraping/sources/spyse/spyse.go | 27 +++++++++++------------ 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/v2/pkg/subscraping/sources/spyse/spyse.go b/v2/pkg/subscraping/sources/spyse/spyse.go index 0dd7944..7daf48c 100644 --- a/v2/pkg/subscraping/sources/spyse/spyse.go +++ b/v2/pkg/subscraping/sources/spyse/spyse.go @@ -7,7 +7,7 @@ import ( "github.com/spyse-com/go-spyse/pkg" ) -const SearchMethodResultsLimit = 10000 +const searchMethodResultsLimit = 10000 // Source is the passive scraping agent type Source struct{} @@ -58,47 +58,46 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } - var searchResults []spyse.Domain - // The default "Search" method returns only first 10 000 subdomains // To obtain more than 10 000 subdomains the "Scroll" method should be using // Note: The "Scroll" method is only available for "PRO" customers, so we need to check // quota.IsScrollSearchEnabled param - if totalResults > SearchMethodResultsLimit && quota.IsScrollSearchEnabled { - searchResults, err := domainSvc.ScrollSearch( + if totalResults > searchMethodResultsLimit && quota.IsScrollSearchEnabled { + var scrollResponse *spyse.DomainScrollResponse + scrollResponse, err = domainSvc.ScrollSearch( ctx, subdomainsSearchParams.Query, "") if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} return } - for len(searchResults.Items) > 0 { - searchResults, err = domainSvc.ScrollSearch( - context.Background(), subdomainsSearchParams.Query, searchResults.SearchID) + for len(scrollResponse.Items) > 0 { + scrollResponse, err = domainSvc.ScrollSearch( + context.Background(), subdomainsSearchParams.Query, scrollResponse.SearchID) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} return } - for _, r := range searchResults.Items { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: r.Name} + for i := range scrollResponse.Items { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: scrollResponse.Items[i].Name} } } } else { var limit = 100 var offset = 0 + var searchResults []spyse.Domain - for ; int64(offset) < totalResults && int64(offset) < SearchMethodResultsLimit; offset += limit { + for ; int64(offset) < totalResults && int64(offset) < searchMethodResultsLimit; offset += limit { searchResults, err = domainSvc.Search(ctx, subdomainsSearchParams.Query, limit, offset) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} return } - for _, r := range searchResults { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: r.Name} + for i := range searchResults { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: searchResults[i].Name} } - } } }() From 1a60d2fcc53dfcadd036eebe1387154743ea4a69 Mon Sep 17 00:00:00 2001 From: sandeep Date: Wed, 21 Jul 2021 13:56:35 +0530 Subject: [PATCH 08/32] lint fix --- v2/pkg/subscraping/sources/spyse/spyse.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/v2/pkg/subscraping/sources/spyse/spyse.go b/v2/pkg/subscraping/sources/spyse/spyse.go index 7daf48c..fbb7501 100644 --- a/v2/pkg/subscraping/sources/spyse/spyse.go +++ b/v2/pkg/subscraping/sources/spyse/spyse.go @@ -3,8 +3,9 @@ package spyse import ( "context" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" - "github.com/spyse-com/go-spyse/pkg" + spyse "github.com/spyse-com/go-spyse/pkg" ) const searchMethodResultsLimit = 10000 From 3b13f7e30686e63fd438a3657d26da16aa0435bc Mon Sep 17 00:00:00 2001 From: Karel Date: Fri, 30 Jul 2021 11:59:38 +0200 Subject: [PATCH 09/32] Close database connection --- v2/pkg/subscraping/sources/crtsh/crtsh.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/v2/pkg/subscraping/sources/crtsh/crtsh.go b/v2/pkg/subscraping/sources/crtsh/crtsh.go index 16400fa..4875b0c 100755 --- a/v2/pkg/subscraping/sources/crtsh/crtsh.go +++ b/v2/pkg/subscraping/sources/crtsh/crtsh.go @@ -44,6 +44,8 @@ func (s *Source) getSubdomainsFromSQL(domain string, results chan subscraping.Re return 0 } + defer db.Close() + pattern := "%." + domain query := `SELECT DISTINCT ci.NAME_VALUE as domain FROM certificate_identity ci WHERE reverse(lower(ci.NAME_VALUE)) LIKE reverse(lower($1)) From 3533d5ba3fc8ebba37fc0f14c54a47f540b80802 Mon Sep 17 00:00:00 2001 From: yhy Date: Fri, 30 Jul 2021 20:38:25 +0800 Subject: [PATCH 10/32] Add Chinaz for subfinder --- v2/pkg/passive/sources.go | 4 ++ v2/pkg/runner/config.go | 4 ++ v2/pkg/subscraping/sources/chinaz/chinaz.go | 56 +++++++++++++++++++++ v2/pkg/subscraping/types.go | 1 + 4 files changed, 65 insertions(+) create mode 100644 v2/pkg/subscraping/sources/chinaz/chinaz.go diff --git a/v2/pkg/passive/sources.go b/v2/pkg/passive/sources.go index 3ac0bec..5f5c341 100644 --- a/v2/pkg/passive/sources.go +++ b/v2/pkg/passive/sources.go @@ -10,6 +10,7 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/censys" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/certspotter" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/chaos" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/chinaz" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/commoncrawl" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/crtsh" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdb" @@ -44,6 +45,7 @@ var DefaultSources = []string{ "certspotter", "censys", "chaos", + "chinaz", "crtsh", "dnsdumpster", "hackertarget", @@ -150,6 +152,8 @@ func (a *Agent) addSources(sources []string) { a.sources[source] = &certspotter.Source{} case "chaos": a.sources[source] = &chaos.Source{} + case "chinaz": + a.sources[source] = &chinaz.Source{} case "commoncrawl": a.sources[source] = &commoncrawl.Source{} case "crtsh": diff --git a/v2/pkg/runner/config.go b/v2/pkg/runner/config.go index 81ebe40..dbd0cb4 100644 --- a/v2/pkg/runner/config.go +++ b/v2/pkg/runner/config.go @@ -33,6 +33,7 @@ type ConfigFile struct { Censys []string `yaml:"censys"` Certspotter []string `yaml:"certspotter"` Chaos []string `yaml:"chaos"` + Chinaz []string `yaml:"chinaz"` DNSDB []string `yaml:"dnsdb"` GitHub []string `yaml:"github"` IntelX []string `yaml:"intelx"` @@ -135,6 +136,9 @@ func (c *ConfigFile) GetKeys() subscraping.Keys { if len(c.Chaos) > 0 { keys.Chaos = c.Chaos[rand.Intn(len(c.Chaos))] } + if len(c.Chinaz) > 0 { + keys.Chinaz = c.Chinaz[rand.Intn(len(c.Chinaz))] + } if (len(c.DNSDB)) > 0 { keys.DNSDB = c.DNSDB[rand.Intn(len(c.DNSDB))] } diff --git a/v2/pkg/subscraping/sources/chinaz/chinaz.go b/v2/pkg/subscraping/sources/chinaz/chinaz.go new file mode 100644 index 0000000..78e3cc4 --- /dev/null +++ b/v2/pkg/subscraping/sources/chinaz/chinaz.go @@ -0,0 +1,56 @@ +package chinaz +// chinaz http://my.chinaz.com/ChinazAPI/DataCenter/MyDataApi +import ( + "context" + "fmt" + jsoniter "github.com/json-iterator/go" + "github.com/yhy0/subfinder/v2/pkg/subscraping" + "io/ioutil" +) + +// Source is the passive scraping agent +type Source struct{} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + if session.Keys.Chinaz == "" { + return + } + + resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://apidatav2.chinaz.com/single/alexa?key=%s&domain=%s", session.Keys.Chinaz, domain)) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + + body, err := ioutil.ReadAll(resp.Body) + + resp.Body.Close() + + SubdomainList :=jsoniter.Get(body, "Result").Get("ContributingSubdomainList") + + if SubdomainList.ToBool() { + _data := []byte(SubdomainList.ToString()) + for i := 0 ; i< SubdomainList.Size() ; i++{ + subdomain := jsoniter.Get(_data,i,"DataUrl").ToString() + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + } + } else { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + return + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "chinaz" +} diff --git a/v2/pkg/subscraping/types.go b/v2/pkg/subscraping/types.go index 8a0c445..ac6aca2 100755 --- a/v2/pkg/subscraping/types.go +++ b/v2/pkg/subscraping/types.go @@ -40,6 +40,7 @@ type Keys struct { CensysSecret string `json:"censysPassword"` Certspotter string `json:"certspotter"` Chaos string `json:"chaos"` + Chinaz string `json:"chinaz"` DNSDB string `json:"dnsdb"` GitHub []string `json:"github"` IntelXHost string `json:"intelXHost"` From c9f0c06aa5f1847301e5b4b68f142162998ea452 Mon Sep 17 00:00:00 2001 From: yhy <31311038+yhy0@users.noreply.github.com> Date: Fri, 30 Jul 2021 21:41:10 +0800 Subject: [PATCH 11/32] Update chinaz.go --- v2/pkg/subscraping/sources/chinaz/chinaz.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/subscraping/sources/chinaz/chinaz.go b/v2/pkg/subscraping/sources/chinaz/chinaz.go index 78e3cc4..b30e93e 100644 --- a/v2/pkg/subscraping/sources/chinaz/chinaz.go +++ b/v2/pkg/subscraping/sources/chinaz/chinaz.go @@ -4,7 +4,7 @@ import ( "context" "fmt" jsoniter "github.com/json-iterator/go" - "github.com/yhy0/subfinder/v2/pkg/subscraping" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" "io/ioutil" ) From 8cff95eac1ada7f1b2d135861f3371d10210b9b9 Mon Sep 17 00:00:00 2001 From: maldevel Date: Sat, 31 Jul 2021 09:10:39 +0300 Subject: [PATCH 12/32] dnsdumpster not working. bug fixed. --- v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go | 1 + 1 file changed, 1 insertion(+) diff --git a/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go b/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go index 2a48fe3..08f0ebf 100755 --- a/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go +++ b/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go @@ -30,6 +30,7 @@ func postForm(ctx context.Context, session *subscraping.Session, token, domain s params := url.Values{ "csrfmiddlewaretoken": {token}, "targetip": {domain}, + "user": {"free"}, } resp, err := session.HTTPRequest( From fbb0305b3d6764125750d7d19dac9b864f17db6a Mon Sep 17 00:00:00 2001 From: sandeep Date: Wed, 4 Aug 2021 16:34:19 +0530 Subject: [PATCH 13/32] workflow improvements --- .github/dependabot.yml | 5 +- .github/workflows/build-test.yml | 27 +++++++++++ .github/workflows/build.yaml | 40 ---------------- .github/workflows/codeql-analysis.yml | 38 +++++++++++++++ .../workflows/dockerhub-push-on-release.yml | 17 ------- .github/workflows/dockerhub-push.yml | 34 ++++++++++++++ .github/workflows/lint-test.yml | 19 ++++++++ .../{release.yml => release-binary.yml} | 8 ++-- v2/.goreleaser.yml | 46 +++++++++++-------- 9 files changed, 153 insertions(+), 81 deletions(-) create mode 100644 .github/workflows/build-test.yml delete mode 100644 .github/workflows/build.yaml create mode 100644 .github/workflows/codeql-analysis.yml delete mode 100644 .github/workflows/dockerhub-push-on-release.yml create mode 100644 .github/workflows/dockerhub-push.yml create mode 100644 .github/workflows/lint-test.yml rename .github/workflows/{release.yml => release-binary.yml} (85%) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 4d5617f..69d9543 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -11,6 +11,7 @@ updates: directory: "/" schedule: interval: "weekly" + target-branch: "dev" commit-message: prefix: "chore" include: "scope" @@ -20,6 +21,7 @@ updates: directory: "/" schedule: interval: "weekly" + target-branch: "dev" commit-message: prefix: "chore" include: "scope" @@ -29,6 +31,7 @@ updates: directory: "/" schedule: interval: "weekly" + target-branch: "dev" commit-message: prefix: "chore" - include: "scope" + include: "scope" \ No newline at end of file diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml new file mode 100644 index 0000000..ad0d347 --- /dev/null +++ b/.github/workflows/build-test.yml @@ -0,0 +1,27 @@ +name: 🔨 Build Test +on: + push: + pull_request: + workflow_dispatch: + + +jobs: + build: + name: Test Builds + runs-on: ubuntu-latest + steps: + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.15 + + - name: Check out code + uses: actions/checkout@v2 + + - name: Test + run: go test . + working-directory: v2/cmd/subfinder/ + + - name: Build + run: go build . + working-directory: v2/cmd/subfinder/ \ No newline at end of file diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml deleted file mode 100644 index 0f4e1e9..0000000 --- a/.github/workflows/build.yaml +++ /dev/null @@ -1,40 +0,0 @@ -name: Build - -on: - push: - branches: - - master - pull_request: - -jobs: - golangci-lint: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v2 - - name: Run golangci-lint - uses: golangci/golangci-lint-action@v2 - with: - # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. - version: latest - args: --timeout 5m - working-directory: v2/ - - build: - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@v2 - - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: 1.15 - - - name: Test - run: go test ./... - working-directory: v2/ - - - name: Build - run: go build . - working-directory: v2/cmd/subfinder/ \ No newline at end of file diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000..545cdea --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,38 @@ +name: 🚨 CodeQL Analysis + +on: + workflow_dispatch: + pull_request: + branches: + - dev + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'go' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 \ No newline at end of file diff --git a/.github/workflows/dockerhub-push-on-release.yml b/.github/workflows/dockerhub-push-on-release.yml deleted file mode 100644 index a5156cb..0000000 --- a/.github/workflows/dockerhub-push-on-release.yml +++ /dev/null @@ -1,17 +0,0 @@ -# dockerhub-push pushes docker build to dockerhub automatically -# on the creation of a new release -name: Publish to Dockerhub on creation of a new release -on: - release: - types: [published] -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@master - - name: Publish to Dockerhub Registry - uses: elgohr/Publish-Docker-Github-Action@master - with: - name: projectdiscovery/subfinder - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} diff --git a/.github/workflows/dockerhub-push.yml b/.github/workflows/dockerhub-push.yml new file mode 100644 index 0000000..ab9b9db --- /dev/null +++ b/.github/workflows/dockerhub-push.yml @@ -0,0 +1,34 @@ +name: 🌥 Docker Push + +on: + release: + types: [published] + workflow_dispatch: + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v2 + - + name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - + name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_TOKEN }} + - + name: Build and push + uses: docker/build-push-action@v2 + with: + context: . + platforms: linux/amd64,linux/arm64,linux/arm + push: true + tags: projectdiscovery/subfinder:latest \ No newline at end of file diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml new file mode 100644 index 0000000..794d073 --- /dev/null +++ b/.github/workflows/lint-test.yml @@ -0,0 +1,19 @@ +name: 🙏🏻 Lint Test +on: + push: + pull_request: + workflow_dispatch: + +jobs: + lint: + name: Lint Test + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Run golangci-lint + uses: golangci/golangci-lint-action@v2 + with: + version: latest + args: --timeout 5m + working-directory: . \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release-binary.yml similarity index 85% rename from .github/workflows/release.yml rename to .github/workflows/release-binary.yml index 8229941..6fe8c82 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release-binary.yml @@ -1,8 +1,9 @@ -name: Release +name: 🎉 Release Binary on: create: tags: - v* + workflow_dispatch: jobs: release: @@ -17,7 +18,7 @@ jobs: name: "Set up Go" uses: actions/setup-go@v2 with: - go-version: 1.15 + go-version: 1.16 - env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" @@ -26,5 +27,4 @@ jobs: with: args: "release --rm-dist" version: latest - workdir: v2/ - \ No newline at end of file + workdir: . \ No newline at end of file diff --git a/v2/.goreleaser.yml b/v2/.goreleaser.yml index 9b17be1..6006e6d 100644 --- a/v2/.goreleaser.yml +++ b/v2/.goreleaser.yml @@ -3,23 +3,31 @@ before: - go mod tidy builds: - - binary: subfinder - main: cmd/subfinder/main.go - goos: - - linux - - windows - - darwin - goarch: - - amd64 - - 386 - - arm - - arm64 - +- env: + - CGO_ENABLED=0 + goos: + - windows + - linux + - darwin + goarch: + - amd64 + - 386 + - arm + - arm64 + + ignore: + - goos: darwin + goarch: '386' + - goos: windows + goarch: 'arm' + + binary: '{{ .ProjectName }}' + main: cmd/subfinder/main.go + archives: - - id: tgz - format: tar.gz - replacements: - darwin: macOS - format_overrides: - - goos: windows - format: zip \ No newline at end of file +- format: zip + replacements: + darwin: macOS + +checksum: + algorithm: sha256 \ No newline at end of file From 6485009882dad743018e6d1b2bc0ad0d905ef21d Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Wed, 4 Aug 2021 20:00:56 +0530 Subject: [PATCH 14/32] Revert "Add C99.nl Source" --- v2/pkg/passive/passive.go | 4 +- v2/pkg/passive/sources.go | 4 -- v2/pkg/runner/config.go | 4 -- v2/pkg/runner/enumerate.go | 2 +- v2/pkg/runner/options.go | 19 ++++--- v2/pkg/subscraping/agent.go | 31 ++++-------- v2/pkg/subscraping/sources/c99/c99.go | 71 --------------------------- v2/pkg/subscraping/types.go | 1 - 8 files changed, 21 insertions(+), 115 deletions(-) delete mode 100644 v2/pkg/subscraping/sources/c99/c99.go diff --git a/v2/pkg/passive/passive.go b/v2/pkg/passive/passive.go index ace9846..5dfbc7d 100644 --- a/v2/pkg/passive/passive.go +++ b/v2/pkg/passive/passive.go @@ -11,11 +11,11 @@ import ( ) // EnumerateSubdomains enumerates all the subdomains for a given domain -func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, timeout int, maxEnumTime time.Duration) chan subscraping.Result { +func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, timeout int, maxEnumTime time.Duration) chan subscraping.Result { results := make(chan subscraping.Result) go func() { - session, err := subscraping.NewSession(domain, keys, proxy, timeout) + session, err := subscraping.NewSession(domain, keys, timeout) if err != nil { results <- subscraping.Result{Type: subscraping.Error, Error: fmt.Errorf("could not init passive session for %s: %s", domain, err)} } diff --git a/v2/pkg/passive/sources.go b/v2/pkg/passive/sources.go index a3f65e3..3ac0bec 100644 --- a/v2/pkg/passive/sources.go +++ b/v2/pkg/passive/sources.go @@ -7,7 +7,6 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/archiveis" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/binaryedge" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/bufferover" - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/c99" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/censys" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/certspotter" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/chaos" @@ -84,7 +83,6 @@ var DefaultAllSources = []string{ "archiveis", "binaryedge", "bufferover", - "c99", "censys", "certspotter", "chaos", @@ -146,8 +144,6 @@ func (a *Agent) addSources(sources []string) { a.sources[source] = &binaryedge.Source{} case "bufferover": a.sources[source] = &bufferover.Source{} - case "c99": - a.sources[source] = &c99.Source{} case "censys": a.sources[source] = &censys.Source{} case "certspotter": diff --git a/v2/pkg/runner/config.go b/v2/pkg/runner/config.go index 547ecda..81ebe40 100644 --- a/v2/pkg/runner/config.go +++ b/v2/pkg/runner/config.go @@ -30,7 +30,6 @@ type ConfigFile struct { ExcludeSources []string `yaml:"exclude-sources,omitempty"` // API keys for different sources Binaryedge []string `yaml:"binaryedge"` - C99 []string `yaml:"c99"` Censys []string `yaml:"censys"` Certspotter []string `yaml:"certspotter"` Chaos []string `yaml:"chaos"` @@ -120,9 +119,6 @@ func (c *ConfigFile) GetKeys() subscraping.Keys { if len(c.Binaryedge) > 0 { keys.Binaryedge = c.Binaryedge[rand.Intn(len(c.Binaryedge))] } - if len(c.C99) > 0 { - keys.C99 = c.C99[rand.Intn(len(c.C99))] - } if len(c.Censys) > 0 { censysKeys := c.Censys[rand.Intn(len(c.Censys))] diff --git a/v2/pkg/runner/enumerate.go b/v2/pkg/runner/enumerate.go index 84cbb03..28bf37f 100644 --- a/v2/pkg/runner/enumerate.go +++ b/v2/pkg/runner/enumerate.go @@ -37,7 +37,7 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain string, outpu // Run the passive subdomain enumeration now := time.Now() - passiveResults := r.passiveAgent.EnumerateSubdomains(domain, &keys, r.options.Proxy, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute) + passiveResults := r.passiveAgent.EnumerateSubdomains(domain, &keys, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute) wg := &sync.WaitGroup{} wg.Add(1) diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index 6525c2b..bf90615 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -32,15 +32,15 @@ type Options struct { Domain string // Domain is the domain to find subdomains for DomainsFile string // DomainsFile is the file containing list of domains to find subdomains for Output io.Writer - OutputFile string // Output is the file to write found subdomains to. - OutputDirectory string // OutputDirectory is the directory to write results to in case list of domains is given - Sources string // Sources contains a comma-separated list of sources to use for enumeration - ExcludeSources string // ExcludeSources contains the comma-separated sources to not include in the enumeration process - Resolvers string // Resolvers is the comma-separated resolvers to use for enumeration - ResolverList string // ResolverList is a text file containing list of resolvers to use for enumeration - ConfigFile string // ConfigFile contains the location of the config file - Proxy string // HTTP proxy - YAMLConfig ConfigFile // YAMLConfig contains the unmarshalled yaml config file + OutputFile string // Output is the file to write found subdomains to. + OutputDirectory string // OutputDirectory is the directory to write results to in case list of domains is given + Sources string // Sources contains a comma-separated list of sources to use for enumeration + ExcludeSources string // ExcludeSources contains the comma-separated sources to not include in the enumeration process + Resolvers string // Resolvers is the comma-separated resolvers to use for enumeration + ResolverList string // ResolverList is a text file containing list of resolvers to use for enumeration + ConfigFile string // ConfigFile contains the location of the config file + + YAMLConfig ConfigFile // YAMLConfig contains the unmarshalled yaml config file } // ParseOptions parses the command line flags provided by a user @@ -76,7 +76,6 @@ func ParseOptions() *Options { flag.StringVar(&options.ResolverList, "rL", "", "Text file containing list of resolvers to use") flag.BoolVar(&options.RemoveWildcard, "nW", false, "Remove Wildcard & Dead Subdomains from output") flag.StringVar(&options.ConfigFile, "config", path.Join(config, "config.yaml"), "Configuration file for API Keys, etc") - flag.StringVar(&options.Proxy, "http-proxy", "", "HTTP proxy to use") flag.BoolVar(&options.Version, "version", false, "Show version of subfinder") flag.Parse() diff --git a/v2/pkg/subscraping/agent.go b/v2/pkg/subscraping/agent.go index a039dfc..b4c89ec 100755 --- a/v2/pkg/subscraping/agent.go +++ b/v2/pkg/subscraping/agent.go @@ -14,29 +14,16 @@ import ( ) // NewSession creates a new session object for a domain -func NewSession(domain string, keys *Keys, proxy string, timeout int) (*Session, error) { - Transport := &http.Transport{ - MaxIdleConns: 100, - MaxIdleConnsPerHost: 100, - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - } - - // Add proxy - if proxy != "" { - proxyURL, _ := url.Parse(proxy) - if proxyURL == nil { - // Log warning but continue anyways - gologger.Warning().Msgf("Invalid proxy '%s' provided", proxy) - } else { - Transport.Proxy = http.ProxyURL(proxyURL) - } - } - +func NewSession(domain string, keys *Keys, timeout int) (*Session, error) { client := &http.Client{ - Transport: Transport, - Timeout: time.Duration(timeout) * time.Second, + Transport: &http.Transport{ + MaxIdleConns: 100, + MaxIdleConnsPerHost: 100, + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + }, + Timeout: time.Duration(timeout) * time.Second, } session := &Session{ diff --git a/v2/pkg/subscraping/sources/c99/c99.go b/v2/pkg/subscraping/sources/c99/c99.go deleted file mode 100644 index 211669f..0000000 --- a/v2/pkg/subscraping/sources/c99/c99.go +++ /dev/null @@ -1,71 +0,0 @@ -// Package c99 logic -package c99 - -import ( - "context" - "fmt" - "strings" - - jsoniter "github.com/json-iterator/go" - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" -) - -// Source is the passive scraping agent -type Source struct{} - -type dnsdbLookupResponse struct { - Success bool `json:"success"` - Subdomains []struct { - Subdomain string `json:"subdomain"` - IP string `json:"ip"` - Cloudflare bool `json:"cloudflare"` - } `json:"subdomains"` - Error string `json:"error"` -} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - - go func() { - defer close(results) - - if session.Keys.C99 == "" { - return - } - - searchURL := fmt.Sprintf("https://api.c99.nl/subdomainfinder?key=%s&domain=%s&json", session.Keys.C99, domain) - resp, err := session.SimpleGet(ctx, searchURL) - if err != nil { - session.DiscardHTTPResponse(resp) - return - } - - defer resp.Body.Close() - - var response dnsdbLookupResponse - err = jsoniter.NewDecoder(resp.Body).Decode(&response) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - return - } - - if response.Error != "" { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%v", response.Error)} - return - } - - for _, data := range response.Subdomains { - if !strings.HasPrefix(data.Subdomain, ".") { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: data.Subdomain} - } - } - }() - - return results -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "c99" -} diff --git a/v2/pkg/subscraping/types.go b/v2/pkg/subscraping/types.go index e3211be..8a0c445 100755 --- a/v2/pkg/subscraping/types.go +++ b/v2/pkg/subscraping/types.go @@ -36,7 +36,6 @@ type Session struct { // Keys contains the current API Keys we have in store type Keys struct { Binaryedge string `json:"binaryedge"` - C99 string `json:"c99"` CensysToken string `json:"censysUsername"` CensysSecret string `json:"censysPassword"` Certspotter string `json:"certspotter"` From 2bd4cb12082a1e6711ebdbdd2f00ac40cf2b3f85 Mon Sep 17 00:00:00 2001 From: Defuse Venue Date: Mon, 9 Aug 2021 16:05:05 +0900 Subject: [PATCH 15/32] Add http-proxy and C99 to README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 32603a4..c5b473e 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,7 @@ This will display help for the tool. Here are all the switches it supports. | -sources | Comma separated list of sources to use | subfinder -sources shodan,censys | | -t | Number of concurrent goroutines for resolving (default 10) | subfinder -t 100 | | -timeout | Seconds to wait before timing out (default 30) | subfinder -timeout 30 | +| -http-proxy | Http Proxy | subfinder -http-proxy http://localhost:3128 | -v | Show Verbose output | subfinder -v | | -version | Show current program version | subfinder -version | @@ -93,6 +94,7 @@ Subfinder requires **go1.14+** to install successfully. Run the following comman Subfinder will work after using the installation instructions however to configure Subfinder to work with certain services, you will need to have setup API keys. The following services do not work without an API key: - [Binaryedge](https://binaryedge.io) +- [C99](https://api.c99.nl/) - [Certspotter](https://sslmate.com/certspotter/api/) - [Censys](https://censys.io) - [Chaos](https://chaos.projectdiscovery.io) From 0f989d87e5c1e8a170c0f234e250932cfb90198d Mon Sep 17 00:00:00 2001 From: Defuse Venue Date: Mon, 9 Aug 2021 16:08:12 +0900 Subject: [PATCH 16/32] Lint README.md --- README.md | 50 +++++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index c5b473e..fe06180 100644 --- a/README.md +++ b/README.md @@ -53,31 +53,31 @@ subfinder -h ``` This will display help for the tool. Here are all the switches it supports. -| Flag | Description | Example | -| ---------------- | ---------------------------------------------------------- | -------------------------------------- | -| -all | Use all sources (slow) for enumeration | subfinder -d uber.com -all | -| -config | Configuration file for API Keys, etc | subfinder -config config.yaml | -| -d | Domain to find subdomains for | subfinder -d uber.com | -| -dL | File containing list of domains to enumerate | subfinder -dL hackerone-hosts.txt | -| -exclude-sources | List of sources to exclude from enumeration | subfinder -exclude-sources archiveis | -| -max-time | Minutes to wait for enumeration results (default 10) | subfinder -max-time 1 | -| -nC | Don't Use colors in output | subfinder -nC | -| -nW | Remove Wildcard & Dead Subdomains from output | subfinder -nW | -| -ls | List all available sources | subfinder -ls | -| -o | File to write output to (optional) | subfinder -o output.txt | -| -oD | Directory to write enumeration results to (optional) | subfinder -oD ~/outputs | -| -oI | Write output in Host,IP format | subfinder -oI | -| -oJ | Write output in JSON lines Format | subfinder -oJ | -| -r | Comma-separated list of resolvers to use | subfinder -r 1.1.1.1,1.0.0.1 | -| -rL | Text file containing list of resolvers to use | subfinder -rL resolvers.txt | -| -recursive | Enumeration recursive subdomains | subfinder -d news.yahoo.com -recursive | -| -silent | Show only subdomains in output | subfinder -silent | -| -sources | Comma separated list of sources to use | subfinder -sources shodan,censys | -| -t | Number of concurrent goroutines for resolving (default 10) | subfinder -t 100 | -| -timeout | Seconds to wait before timing out (default 30) | subfinder -timeout 30 | -| -http-proxy | Http Proxy | subfinder -http-proxy http://localhost:3128 -| -v | Show Verbose output | subfinder -v | -| -version | Show current program version | subfinder -version | +| Flag | Description | Example | +| ---------------- | ---------------------------------------------------------- | --------------------------------------------| +| -all | Use all sources (slow) for enumeration | subfinder -d uber.com -all | +| -config | Configuration file for API Keys, etc | subfinder -config config.yaml | +| -d | Domain to find subdomains for | subfinder -d uber.com | +| -dL | File containing list of domains to enumerate | subfinder -dL hackerone-hosts.txt | +| -exclude-sources | List of sources to exclude from enumeration | subfinder -exclude-sources archiveis | +| -max-time | Minutes to wait for enumeration results (default 10) | subfinder -max-time 1 | +| -nC | Don't Use colors in output | subfinder -nC | +| -nW | Remove Wildcard & Dead Subdomains from output | subfinder -nW | +| -ls | List all available sources | subfinder -ls | +| -o | File to write output to (optional) | subfinder -o output.txt | +| -oD | Directory to write enumeration results to (optional) | subfinder -oD ~/outputs | +| -oI | Write output in Host,IP format | subfinder -oI | +| -oJ | Write output in JSON lines Format | subfinder -oJ | +| -r | Comma-separated list of resolvers to use | subfinder -r 1.1.1.1,1.0.0.1 | +| -rL | Text file containing list of resolvers to use | subfinder -rL resolvers.txt | +| -recursive | Enumeration recursive subdomains | subfinder -d news.yahoo.com -recursive | +| -silent | Show only subdomains in output | subfinder -silent | +| -sources | Comma separated list of sources to use | subfinder -sources shodan,censys | +| -t | Number of concurrent goroutines for resolving (default 10) | subfinder -t 100 | +| -timeout | Seconds to wait before timing out (default 30) | subfinder -timeout 30 | +| -http-proxy | Http Proxy | subfinder -http-proxy http://localhost:3128 | +| -v | Show Verbose output | subfinder -v | +| -version | Show current program version | subfinder -version | # Installation From b3b56844b4791e8949079bd2f961db7d3982f239 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Aug 2021 14:03:04 +0000 Subject: [PATCH 17/32] chore(deps): bump golang from 1.16.6-alpine to 1.16.7-alpine Bumps golang from 1.16.6-alpine to 1.16.7-alpine. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ac84a2f..e8ba955 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build -FROM golang:1.16.6-alpine AS build-env +FROM golang:1.16.7-alpine AS build-env RUN GO111MODULE=on go get -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder # Release From 019a23763d422c533bc754f8d3d70d5ddc748993 Mon Sep 17 00:00:00 2001 From: maldevel Date: Mon, 9 Aug 2021 18:37:03 +0300 Subject: [PATCH 18/32] retrieve archiveis first page --- .../sources/archiveis/archiveis.go | 95 +++++++------------ 1 file changed, 33 insertions(+), 62 deletions(-) diff --git a/v2/pkg/subscraping/sources/archiveis/archiveis.go b/v2/pkg/subscraping/sources/archiveis/archiveis.go index c4e9d3a..c20d00c 100755 --- a/v2/pkg/subscraping/sources/archiveis/archiveis.go +++ b/v2/pkg/subscraping/sources/archiveis/archiveis.go @@ -5,74 +5,45 @@ import ( "context" "fmt" "io/ioutil" - "regexp" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) -type agent struct { - Results chan subscraping.Result - Session *subscraping.Session -} - -var reNext = regexp.MustCompile("→") - -func (a *agent) enumerate(ctx context.Context, baseURL string) { - select { - case <-ctx.Done(): - return - default: - } - - resp, err := a.Session.SimpleGet(ctx, baseURL) - if err != nil { - a.Results <- subscraping.Result{Source: "archiveis", Type: subscraping.Error, Error: err} - a.Session.DiscardHTTPResponse(resp) - return - } - - // Get the response body - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - a.Results <- subscraping.Result{Source: "archiveis", Type: subscraping.Error, Error: err} - resp.Body.Close() - return - } - - resp.Body.Close() - - src := string(body) - for _, subdomain := range a.Session.Extractor.FindAllString(src, -1) { - a.Results <- subscraping.Result{Source: "archiveis", Type: subscraping.Subdomain, Value: subdomain} - } - - match1 := reNext.FindStringSubmatch(src) - if len(match1) > 0 { - a.enumerate(ctx, match1[1]) - } -} - -// Source is the passive scraping agent type Source struct{} -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - - a := agent{ - Session: session, - Results: results, - } - - go func() { - a.enumerate(ctx, fmt.Sprintf("http://archive.is/*.%s", domain)) - close(a.Results) - }() - - return a.Results -} - -// Name returns the name of the source func (s *Source) Name() string { return "archiveis" } + +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://archive.is/*.%s", domain)) + + if err != nil { + results <- subscraping.Result{Source: "archiveis", Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + results <- subscraping.Result{Source: "archiveis", Type: subscraping.Error, Error: err} + resp.Body.Close() + return + } + + resp.Body.Close() + + src := string(body) + + for _, subdomain := range session.Extractor.FindAllString(src, -1) { + results <- subscraping.Result{Source: "archiveis", Type: subscraping.Subdomain, Value: subdomain} + } + }() + + return results +} From aad7cebc4ac9c0117cbbb83e5db69d0eb3142530 Mon Sep 17 00:00:00 2001 From: maldevel Date: Mon, 9 Aug 2021 18:55:26 +0300 Subject: [PATCH 19/32] Update archiveis.go comments added --- v2/pkg/subscraping/sources/archiveis/archiveis.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/v2/pkg/subscraping/sources/archiveis/archiveis.go b/v2/pkg/subscraping/sources/archiveis/archiveis.go index c20d00c..9a2b535 100755 --- a/v2/pkg/subscraping/sources/archiveis/archiveis.go +++ b/v2/pkg/subscraping/sources/archiveis/archiveis.go @@ -9,12 +9,15 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) +// Source is the passive scraping agent type Source struct{} +// Name returns the name of the source func (s *Source) Name() string { return "archiveis" } +// Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) From 333d59ce5199b0af939f162459f0429ed60b818b Mon Sep 17 00:00:00 2001 From: sandeep Date: Tue, 10 Aug 2021 17:16:01 +0530 Subject: [PATCH 20/32] version bump --- v2/pkg/runner/banners.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/pkg/runner/banners.go b/v2/pkg/runner/banners.go index be0a5e6..292d227 100644 --- a/v2/pkg/runner/banners.go +++ b/v2/pkg/runner/banners.go @@ -11,11 +11,11 @@ const banner = ` _______ __/ /_ / __(_)___ ____/ /__ _____ / ___/ / / / __ \/ /_/ / __ \/ __ / _ \/ ___/ (__ ) /_/ / /_/ / __/ / / / / /_/ / __/ / -/____/\__,_/_.___/_/ /_/_/ /_/\__,_/\___/_/ v2.4.8 +/____/\__,_/_.___/_/ /_/_/ /_/\__,_/\___/_/ v2.4.9 ` // Version is the current version of subfinder -const Version = `2.4.8` +const Version = `v2.4.9` // showBanner is used to show the banner to the user func showBanner() { From 9cc4fa2bf5181142fa0e2c5741073c631cca3f8d Mon Sep 17 00:00:00 2001 From: sandeep Date: Tue, 10 Aug 2021 17:20:44 +0530 Subject: [PATCH 21/32] lint workflow update --- .github/workflows/lint-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index 794d073..ad6dd61 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -16,4 +16,4 @@ jobs: with: version: latest args: --timeout 5m - working-directory: . \ No newline at end of file + working-directory: v2/ \ No newline at end of file From bba144f1b86924f1b6606ab7203616b65b3d5081 Mon Sep 17 00:00:00 2001 From: maldevel Date: Tue, 10 Aug 2021 21:20:05 +0300 Subject: [PATCH 22/32] recon.dev bug fixing, json response has changed --- v2/pkg/subscraping/agent.go | 6 +++--- v2/pkg/subscraping/sources/recon/recon.go | 10 ++++++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/v2/pkg/subscraping/agent.go b/v2/pkg/subscraping/agent.go index a039dfc..a940028 100755 --- a/v2/pkg/subscraping/agent.go +++ b/v2/pkg/subscraping/agent.go @@ -78,9 +78,9 @@ func (s *Session) HTTPRequest(ctx context.Context, method, requestURL, cookies s return nil, err } - req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36") - req.Header.Set("Accept", "*/*") - req.Header.Set("Accept-Language", "en") + req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:90.0) Gecko/20100101 Firefox/90.0") + req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8") //"*/*") + req.Header.Set("Accept-Language", "en-US,en;q=0.5") //"en") req.Header.Set("Connection", "close") if basicAuth.Username != "" || basicAuth.Password != "" { diff --git a/v2/pkg/subscraping/sources/recon/recon.go b/v2/pkg/subscraping/sources/recon/recon.go index b1539b1..ff4b9d1 100644 --- a/v2/pkg/subscraping/sources/recon/recon.go +++ b/v2/pkg/subscraping/sources/recon/recon.go @@ -10,7 +10,11 @@ import ( ) type subdomain struct { - RawDomain string `json:"rawDomain"` + Domains []string `json:"domains"` + Ip string `json:"ip"` + RawDomains []string `json:"rawDomains"` + RawPort string `json:"rawPort"` + RawIp string `json:"rawIp"` } // Source is the passive scraping agent @@ -44,7 +48,9 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se resp.Body.Close() for _, subdomain := range subdomains { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain.RawDomain} + for _, dmn := range subdomain.RawDomains { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: dmn} + } } }() From e4dcab16454947cf3fef2c266ecb25ee761b1043 Mon Sep 17 00:00:00 2001 From: maldevel Date: Tue, 10 Aug 2021 21:23:46 +0300 Subject: [PATCH 23/32] Update agent.go --- v2/pkg/subscraping/agent.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/pkg/subscraping/agent.go b/v2/pkg/subscraping/agent.go index a940028..a039dfc 100755 --- a/v2/pkg/subscraping/agent.go +++ b/v2/pkg/subscraping/agent.go @@ -78,9 +78,9 @@ func (s *Session) HTTPRequest(ctx context.Context, method, requestURL, cookies s return nil, err } - req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:90.0) Gecko/20100101 Firefox/90.0") - req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8") //"*/*") - req.Header.Set("Accept-Language", "en-US,en;q=0.5") //"en") + req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36") + req.Header.Set("Accept", "*/*") + req.Header.Set("Accept-Language", "en") req.Header.Set("Connection", "close") if basicAuth.Username != "" || basicAuth.Password != "" { From e5d9f1a58311c6cc43152d3ec5d5431a5155913d Mon Sep 17 00:00:00 2001 From: Defuse Venue Date: Wed, 11 Aug 2021 21:25:07 +0900 Subject: [PATCH 24/32] User Agent Randomization HTTP User-Agent header randomization. Adds -unsafe flag to send requests with default User-Agent header. --- v2/go.mod | 1 + v2/go.sum | 8 ++++++++ v2/pkg/passive/passive.go | 4 ++-- v2/pkg/runner/enumerate.go | 2 +- v2/pkg/runner/options.go | 2 ++ v2/pkg/subscraping/agent.go | 12 ++++++++++-- v2/pkg/subscraping/types.go | 2 ++ 7 files changed, 26 insertions(+), 5 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 543cf9e..7bd474d 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -3,6 +3,7 @@ module github.com/projectdiscovery/subfinder/v2 go 1.16 require ( + github.com/corpix/uarand v0.1.1 github.com/hako/durafmt v0.0.0-20210316092057-3a2c319c1acd github.com/json-iterator/go v1.1.10 github.com/lib/pq v1.10.0 diff --git a/v2/go.sum b/v2/go.sum index 0c27ad7..e7a7da7 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -1,8 +1,14 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic= +github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA= +github.com/corpix/uarand v0.1.1 h1:RMr1TWc9F4n5jiPDzFHtmaUXLKLNUFK0SgCLo4BhX/U= +github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -51,12 +57,14 @@ github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/z github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= diff --git a/v2/pkg/passive/passive.go b/v2/pkg/passive/passive.go index ace9846..e2aedae 100644 --- a/v2/pkg/passive/passive.go +++ b/v2/pkg/passive/passive.go @@ -11,11 +11,11 @@ import ( ) // EnumerateSubdomains enumerates all the subdomains for a given domain -func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, timeout int, maxEnumTime time.Duration) chan subscraping.Result { +func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, unsafe bool, timeout int, maxEnumTime time.Duration) chan subscraping.Result { results := make(chan subscraping.Result) go func() { - session, err := subscraping.NewSession(domain, keys, proxy, timeout) + session, err := subscraping.NewSession(domain, keys, proxy, unsafe, timeout) if err != nil { results <- subscraping.Result{Type: subscraping.Error, Error: fmt.Errorf("could not init passive session for %s: %s", domain, err)} } diff --git a/v2/pkg/runner/enumerate.go b/v2/pkg/runner/enumerate.go index 84cbb03..2e61c83 100644 --- a/v2/pkg/runner/enumerate.go +++ b/v2/pkg/runner/enumerate.go @@ -37,7 +37,7 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain string, outpu // Run the passive subdomain enumeration now := time.Now() - passiveResults := r.passiveAgent.EnumerateSubdomains(domain, &keys, r.options.Proxy, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute) + passiveResults := r.passiveAgent.EnumerateSubdomains(domain, &keys, r.options.Proxy, r.options.UnSafe, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute) wg := &sync.WaitGroup{} wg.Add(1) diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index 6525c2b..c7d9d94 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -40,6 +40,7 @@ type Options struct { ResolverList string // ResolverList is a text file containing list of resolvers to use for enumeration ConfigFile string // ConfigFile contains the location of the config file Proxy string // HTTP proxy + UnSafe bool // Send HTTP request without User-Agent header randomization YAMLConfig ConfigFile // YAMLConfig contains the unmarshalled yaml config file } @@ -77,6 +78,7 @@ func ParseOptions() *Options { flag.BoolVar(&options.RemoveWildcard, "nW", false, "Remove Wildcard & Dead Subdomains from output") flag.StringVar(&options.ConfigFile, "config", path.Join(config, "config.yaml"), "Configuration file for API Keys, etc") flag.StringVar(&options.Proxy, "http-proxy", "", "HTTP proxy to use") + flag.BoolVar(&options.UnSafe, "unsafe", false,"Send HTTP request without User-Agent header randomization") flag.BoolVar(&options.Version, "version", false, "Show version of subfinder") flag.Parse() diff --git a/v2/pkg/subscraping/agent.go b/v2/pkg/subscraping/agent.go index a039dfc..e82a596 100755 --- a/v2/pkg/subscraping/agent.go +++ b/v2/pkg/subscraping/agent.go @@ -10,11 +10,12 @@ import ( "net/url" "time" + "github.com/corpix/uarand" "github.com/projectdiscovery/gologger" ) // NewSession creates a new session object for a domain -func NewSession(domain string, keys *Keys, proxy string, timeout int) (*Session, error) { +func NewSession(domain string, keys *Keys, proxy string, unsafe bool, timeout int) (*Session, error) { Transport := &http.Transport{ MaxIdleConns: 100, MaxIdleConnsPerHost: 100, @@ -42,6 +43,7 @@ func NewSession(domain string, keys *Keys, proxy string, timeout int) (*Session, session := &Session{ Client: client, Keys: keys, + UnSafe: unsafe, } // Create a new extractor object for the current domain @@ -78,7 +80,13 @@ func (s *Session) HTTPRequest(ctx context.Context, method, requestURL, cookies s return nil, err } - req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36") + // Unsafe requests do not use user-agent randomization + if s.UnSafe { + req.Header.Set("User-Agent", "subfinder - Open-source Project (github.com/projectdiscovery/subfinder)") + } else { + req.Header.Set("User-Agent", uarand.GetRandom()) + } + req.Header.Set("Accept", "*/*") req.Header.Set("Accept-Language", "en") req.Header.Set("Connection", "close") diff --git a/v2/pkg/subscraping/types.go b/v2/pkg/subscraping/types.go index e3211be..9df42ff 100755 --- a/v2/pkg/subscraping/types.go +++ b/v2/pkg/subscraping/types.go @@ -31,6 +31,8 @@ type Session struct { Keys *Keys // Client is the current http client Client *http.Client + // Perform unsafe HTTP requests without user agent randomization + UnSafe bool } // Keys contains the current API Keys we have in store From 5bc901c47d0a22b6c1d5a2ea06ca65ad4bfdccfd Mon Sep 17 00:00:00 2001 From: Defuse Venue Date: Wed, 11 Aug 2021 21:30:15 +0900 Subject: [PATCH 25/32] Add unsafe flag to README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index fe06180..c907528 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,7 @@ This will display help for the tool. Here are all the switches it supports. | -t | Number of concurrent goroutines for resolving (default 10) | subfinder -t 100 | | -timeout | Seconds to wait before timing out (default 30) | subfinder -timeout 30 | | -http-proxy | Http Proxy | subfinder -http-proxy http://localhost:3128 | +| -unsafe | Send HTTP request without User-Agent header randomization | subfinder -unsafe | | -v | Show Verbose output | subfinder -v | | -version | Show current program version | subfinder -version | From f35adaf14356b6d6f4efe7b1241105c4829bc114 Mon Sep 17 00:00:00 2001 From: Defuse Venue Date: Wed, 11 Aug 2021 21:34:20 +0900 Subject: [PATCH 26/32] Fix lint issue --- v2/pkg/runner/options.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index c7d9d94..ce9c317 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -40,7 +40,7 @@ type Options struct { ResolverList string // ResolverList is a text file containing list of resolvers to use for enumeration ConfigFile string // ConfigFile contains the location of the config file Proxy string // HTTP proxy - UnSafe bool // Send HTTP request without User-Agent header randomization + UnSafe bool // Send HTTP request without User-Agent header randomization YAMLConfig ConfigFile // YAMLConfig contains the unmarshalled yaml config file } @@ -78,7 +78,7 @@ func ParseOptions() *Options { flag.BoolVar(&options.RemoveWildcard, "nW", false, "Remove Wildcard & Dead Subdomains from output") flag.StringVar(&options.ConfigFile, "config", path.Join(config, "config.yaml"), "Configuration file for API Keys, etc") flag.StringVar(&options.Proxy, "http-proxy", "", "HTTP proxy to use") - flag.BoolVar(&options.UnSafe, "unsafe", false,"Send HTTP request without User-Agent header randomization") + flag.BoolVar(&options.UnSafe, "unsafe", false, "Send HTTP request without User-Agent header randomization") flag.BoolVar(&options.Version, "version", false, "Show version of subfinder") flag.Parse() From 095d4295f230b4cbb26ee0bed47e63c2b0d36e67 Mon Sep 17 00:00:00 2001 From: Defuse Venue Date: Wed, 11 Aug 2021 21:42:52 +0900 Subject: [PATCH 27/32] Remove unsafe flag HTTP User-Agent should be randomized by default since subfinder uses third party apis --- README.md | 1 - v2/pkg/passive/passive.go | 4 ++-- v2/pkg/runner/enumerate.go | 2 +- v2/pkg/runner/options.go | 2 -- v2/pkg/subscraping/agent.go | 11 ++--------- v2/pkg/subscraping/types.go | 2 -- 6 files changed, 5 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index c907528..fe06180 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,6 @@ This will display help for the tool. Here are all the switches it supports. | -t | Number of concurrent goroutines for resolving (default 10) | subfinder -t 100 | | -timeout | Seconds to wait before timing out (default 30) | subfinder -timeout 30 | | -http-proxy | Http Proxy | subfinder -http-proxy http://localhost:3128 | -| -unsafe | Send HTTP request without User-Agent header randomization | subfinder -unsafe | | -v | Show Verbose output | subfinder -v | | -version | Show current program version | subfinder -version | diff --git a/v2/pkg/passive/passive.go b/v2/pkg/passive/passive.go index e2aedae..ace9846 100644 --- a/v2/pkg/passive/passive.go +++ b/v2/pkg/passive/passive.go @@ -11,11 +11,11 @@ import ( ) // EnumerateSubdomains enumerates all the subdomains for a given domain -func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, unsafe bool, timeout int, maxEnumTime time.Duration) chan subscraping.Result { +func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, timeout int, maxEnumTime time.Duration) chan subscraping.Result { results := make(chan subscraping.Result) go func() { - session, err := subscraping.NewSession(domain, keys, proxy, unsafe, timeout) + session, err := subscraping.NewSession(domain, keys, proxy, timeout) if err != nil { results <- subscraping.Result{Type: subscraping.Error, Error: fmt.Errorf("could not init passive session for %s: %s", domain, err)} } diff --git a/v2/pkg/runner/enumerate.go b/v2/pkg/runner/enumerate.go index 2e61c83..84cbb03 100644 --- a/v2/pkg/runner/enumerate.go +++ b/v2/pkg/runner/enumerate.go @@ -37,7 +37,7 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain string, outpu // Run the passive subdomain enumeration now := time.Now() - passiveResults := r.passiveAgent.EnumerateSubdomains(domain, &keys, r.options.Proxy, r.options.UnSafe, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute) + passiveResults := r.passiveAgent.EnumerateSubdomains(domain, &keys, r.options.Proxy, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute) wg := &sync.WaitGroup{} wg.Add(1) diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index ce9c317..6525c2b 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -40,7 +40,6 @@ type Options struct { ResolverList string // ResolverList is a text file containing list of resolvers to use for enumeration ConfigFile string // ConfigFile contains the location of the config file Proxy string // HTTP proxy - UnSafe bool // Send HTTP request without User-Agent header randomization YAMLConfig ConfigFile // YAMLConfig contains the unmarshalled yaml config file } @@ -78,7 +77,6 @@ func ParseOptions() *Options { flag.BoolVar(&options.RemoveWildcard, "nW", false, "Remove Wildcard & Dead Subdomains from output") flag.StringVar(&options.ConfigFile, "config", path.Join(config, "config.yaml"), "Configuration file for API Keys, etc") flag.StringVar(&options.Proxy, "http-proxy", "", "HTTP proxy to use") - flag.BoolVar(&options.UnSafe, "unsafe", false, "Send HTTP request without User-Agent header randomization") flag.BoolVar(&options.Version, "version", false, "Show version of subfinder") flag.Parse() diff --git a/v2/pkg/subscraping/agent.go b/v2/pkg/subscraping/agent.go index e82a596..6b89337 100755 --- a/v2/pkg/subscraping/agent.go +++ b/v2/pkg/subscraping/agent.go @@ -15,7 +15,7 @@ import ( ) // NewSession creates a new session object for a domain -func NewSession(domain string, keys *Keys, proxy string, unsafe bool, timeout int) (*Session, error) { +func NewSession(domain string, keys *Keys, proxy string, timeout int) (*Session, error) { Transport := &http.Transport{ MaxIdleConns: 100, MaxIdleConnsPerHost: 100, @@ -43,7 +43,6 @@ func NewSession(domain string, keys *Keys, proxy string, unsafe bool, timeout in session := &Session{ Client: client, Keys: keys, - UnSafe: unsafe, } // Create a new extractor object for the current domain @@ -80,13 +79,7 @@ func (s *Session) HTTPRequest(ctx context.Context, method, requestURL, cookies s return nil, err } - // Unsafe requests do not use user-agent randomization - if s.UnSafe { - req.Header.Set("User-Agent", "subfinder - Open-source Project (github.com/projectdiscovery/subfinder)") - } else { - req.Header.Set("User-Agent", uarand.GetRandom()) - } - + req.Header.Set("User-Agent", uarand.GetRandom()) req.Header.Set("Accept", "*/*") req.Header.Set("Accept-Language", "en") req.Header.Set("Connection", "close") diff --git a/v2/pkg/subscraping/types.go b/v2/pkg/subscraping/types.go index 9df42ff..e3211be 100755 --- a/v2/pkg/subscraping/types.go +++ b/v2/pkg/subscraping/types.go @@ -31,8 +31,6 @@ type Session struct { Keys *Keys // Client is the current http client Client *http.Client - // Perform unsafe HTTP requests without user agent randomization - UnSafe bool } // Keys contains the current API Keys we have in store From ec79e8053a8f6c0a018bad8c409ef1897b105efe Mon Sep 17 00:00:00 2001 From: Defuse Venue Date: Thu, 12 Aug 2021 08:29:58 +0900 Subject: [PATCH 28/32] Add rate limit feature --- v2/go.mod | 1 + v2/go.sum | 3 +++ v2/pkg/passive/passive.go | 4 ++-- v2/pkg/runner/enumerate.go | 2 +- v2/pkg/runner/options.go | 2 ++ v2/pkg/subscraping/agent.go | 12 +++++++++++- v2/pkg/subscraping/types.go | 4 ++++ 7 files changed, 24 insertions(+), 4 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 7bd474d..3e7c5c9 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -14,5 +14,6 @@ require ( github.com/rs/xid v1.3.0 github.com/stretchr/testify v1.7.0 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 + go.uber.org/ratelimit v0.2.0 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b ) diff --git a/v2/go.sum b/v2/go.sum index e7a7da7..1b4b092 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -3,6 +3,7 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic= github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= +github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -105,7 +106,9 @@ github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpP github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA= go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= diff --git a/v2/pkg/passive/passive.go b/v2/pkg/passive/passive.go index ace9846..91e44e9 100644 --- a/v2/pkg/passive/passive.go +++ b/v2/pkg/passive/passive.go @@ -11,11 +11,11 @@ import ( ) // EnumerateSubdomains enumerates all the subdomains for a given domain -func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, timeout int, maxEnumTime time.Duration) chan subscraping.Result { +func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, rateLimit int, timeout int, maxEnumTime time.Duration) chan subscraping.Result { results := make(chan subscraping.Result) go func() { - session, err := subscraping.NewSession(domain, keys, proxy, timeout) + session, err := subscraping.NewSession(domain, keys, proxy, rateLimit, timeout) if err != nil { results <- subscraping.Result{Type: subscraping.Error, Error: fmt.Errorf("could not init passive session for %s: %s", domain, err)} } diff --git a/v2/pkg/runner/enumerate.go b/v2/pkg/runner/enumerate.go index 84cbb03..5df3fb6 100644 --- a/v2/pkg/runner/enumerate.go +++ b/v2/pkg/runner/enumerate.go @@ -37,7 +37,7 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain string, outpu // Run the passive subdomain enumeration now := time.Now() - passiveResults := r.passiveAgent.EnumerateSubdomains(domain, &keys, r.options.Proxy, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute) + passiveResults := r.passiveAgent.EnumerateSubdomains(domain, &keys, r.options.Proxy, r.options.RateLimit, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute) wg := &sync.WaitGroup{} wg.Add(1) diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index 6525c2b..e672d0b 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -40,6 +40,7 @@ type Options struct { ResolverList string // ResolverList is a text file containing list of resolvers to use for enumeration ConfigFile string // ConfigFile contains the location of the config file Proxy string // HTTP proxy + RateLimit int // Maximum number of HTTP requests to send per second YAMLConfig ConfigFile // YAMLConfig contains the unmarshalled yaml config file } @@ -77,6 +78,7 @@ func ParseOptions() *Options { flag.BoolVar(&options.RemoveWildcard, "nW", false, "Remove Wildcard & Dead Subdomains from output") flag.StringVar(&options.ConfigFile, "config", path.Join(config, "config.yaml"), "Configuration file for API Keys, etc") flag.StringVar(&options.Proxy, "http-proxy", "", "HTTP proxy to use") + flag.IntVar(&options.RateLimit, "rate-limit", 0, "Maximum number of HTTP requests to send per second") flag.BoolVar(&options.Version, "version", false, "Show version of subfinder") flag.Parse() diff --git a/v2/pkg/subscraping/agent.go b/v2/pkg/subscraping/agent.go index 6b89337..3fafe12 100755 --- a/v2/pkg/subscraping/agent.go +++ b/v2/pkg/subscraping/agent.go @@ -12,10 +12,11 @@ import ( "github.com/corpix/uarand" "github.com/projectdiscovery/gologger" + "go.uber.org/ratelimit" ) // NewSession creates a new session object for a domain -func NewSession(domain string, keys *Keys, proxy string, timeout int) (*Session, error) { +func NewSession(domain string, keys *Keys, proxy string, rateLimit int, timeout int) (*Session, error) { Transport := &http.Transport{ MaxIdleConns: 100, MaxIdleConnsPerHost: 100, @@ -45,6 +46,13 @@ func NewSession(domain string, keys *Keys, proxy string, timeout int) (*Session, Keys: keys, } + // Initiate rate limit instance + if rateLimit > 0 { + session.RateLimiter = ratelimit.New(rateLimit) + } else { + session.RateLimiter = ratelimit.NewUnlimited() + } + // Create a new extractor object for the current domain extractor, err := NewSubdomainExtractor(domain) session.Extractor = extractor @@ -96,6 +104,8 @@ func (s *Session) HTTPRequest(ctx context.Context, method, requestURL, cookies s req.Header.Set(key, value) } + s.RateLimiter.Take() + return httpRequestWrapper(s.Client, req) } diff --git a/v2/pkg/subscraping/types.go b/v2/pkg/subscraping/types.go index e3211be..f49187a 100755 --- a/v2/pkg/subscraping/types.go +++ b/v2/pkg/subscraping/types.go @@ -4,6 +4,8 @@ import ( "context" "net/http" "regexp" + + "go.uber.org/ratelimit" ) // BasicAuth request's Authorization header @@ -31,6 +33,8 @@ type Session struct { Keys *Keys // Client is the current http client Client *http.Client + // Rate limit instance + RateLimiter ratelimit.Limiter } // Keys contains the current API Keys we have in store From f9381ceec9352301276179b8e656e75eee9ed66d Mon Sep 17 00:00:00 2001 From: Defuse Venue Date: Thu, 12 Aug 2021 08:33:18 +0900 Subject: [PATCH 29/32] Add rate-limit flag to README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index fe06180..7f55e62 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,7 @@ This will display help for the tool. Here are all the switches it supports. | -t | Number of concurrent goroutines for resolving (default 10) | subfinder -t 100 | | -timeout | Seconds to wait before timing out (default 30) | subfinder -timeout 30 | | -http-proxy | Http Proxy | subfinder -http-proxy http://localhost:3128 | +| -rate-limit | Maximum number of HTTP requests to send per second | subfinder -rate-limit 10 | | -v | Show Verbose output | subfinder -v | | -version | Show current program version | subfinder -version | From cb548cad51329a19e97ad49987512d80393bd6ed Mon Sep 17 00:00:00 2001 From: Defuse Venue Date: Thu, 12 Aug 2021 08:36:26 +0900 Subject: [PATCH 30/32] Update options.go --- v2/pkg/runner/options.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index e672d0b..9413538 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -40,7 +40,7 @@ type Options struct { ResolverList string // ResolverList is a text file containing list of resolvers to use for enumeration ConfigFile string // ConfigFile contains the location of the config file Proxy string // HTTP proxy - RateLimit int // Maximum number of HTTP requests to send per second + RateLimit int // Maximum number of HTTP requests to send per second YAMLConfig ConfigFile // YAMLConfig contains the unmarshalled yaml config file } From a1e27912719c01adb13e674706cdc3b87997a507 Mon Sep 17 00:00:00 2001 From: Defuse Venue Date: Fri, 13 Aug 2021 07:39:05 +0900 Subject: [PATCH 31/32] Change variable rateLimit to rl --- v2/pkg/passive/passive.go | 4 ++-- v2/pkg/subscraping/agent.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/v2/pkg/passive/passive.go b/v2/pkg/passive/passive.go index 91e44e9..d925701 100644 --- a/v2/pkg/passive/passive.go +++ b/v2/pkg/passive/passive.go @@ -11,11 +11,11 @@ import ( ) // EnumerateSubdomains enumerates all the subdomains for a given domain -func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, rateLimit int, timeout int, maxEnumTime time.Duration) chan subscraping.Result { +func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, rl int, timeout int, maxEnumTime time.Duration) chan subscraping.Result { results := make(chan subscraping.Result) go func() { - session, err := subscraping.NewSession(domain, keys, proxy, rateLimit, timeout) + session, err := subscraping.NewSession(domain, keys, proxy, rl, timeout) if err != nil { results <- subscraping.Result{Type: subscraping.Error, Error: fmt.Errorf("could not init passive session for %s: %s", domain, err)} } diff --git a/v2/pkg/subscraping/agent.go b/v2/pkg/subscraping/agent.go index 3fafe12..40c5a67 100755 --- a/v2/pkg/subscraping/agent.go +++ b/v2/pkg/subscraping/agent.go @@ -16,7 +16,7 @@ import ( ) // NewSession creates a new session object for a domain -func NewSession(domain string, keys *Keys, proxy string, rateLimit int, timeout int) (*Session, error) { +func NewSession(domain string, keys *Keys, proxy string, rl int, timeout int) (*Session, error) { Transport := &http.Transport{ MaxIdleConns: 100, MaxIdleConnsPerHost: 100, @@ -47,8 +47,8 @@ func NewSession(domain string, keys *Keys, proxy string, rateLimit int, timeout } // Initiate rate limit instance - if rateLimit > 0 { - session.RateLimiter = ratelimit.New(rateLimit) + if rl > 0 { + session.RateLimiter = ratelimit.New(rl) } else { session.RateLimiter = ratelimit.NewUnlimited() } From 0aac1eb68567ab68dfefc75e511ba67718473339 Mon Sep 17 00:00:00 2001 From: Defuse Venue Date: Fri, 13 Aug 2021 07:41:11 +0900 Subject: [PATCH 32/32] Fix lint --- v2/pkg/passive/passive.go | 4 ++-- v2/pkg/subscraping/agent.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/v2/pkg/passive/passive.go b/v2/pkg/passive/passive.go index d925701..0ed98d4 100644 --- a/v2/pkg/passive/passive.go +++ b/v2/pkg/passive/passive.go @@ -11,11 +11,11 @@ import ( ) // EnumerateSubdomains enumerates all the subdomains for a given domain -func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, rl int, timeout int, maxEnumTime time.Duration) chan subscraping.Result { +func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, rateLimit, timeout int, maxEnumTime time.Duration) chan subscraping.Result { results := make(chan subscraping.Result) go func() { - session, err := subscraping.NewSession(domain, keys, proxy, rl, timeout) + session, err := subscraping.NewSession(domain, keys, proxy, rateLimit, timeout) if err != nil { results <- subscraping.Result{Type: subscraping.Error, Error: fmt.Errorf("could not init passive session for %s: %s", domain, err)} } diff --git a/v2/pkg/subscraping/agent.go b/v2/pkg/subscraping/agent.go index 40c5a67..6797542 100755 --- a/v2/pkg/subscraping/agent.go +++ b/v2/pkg/subscraping/agent.go @@ -16,7 +16,7 @@ import ( ) // NewSession creates a new session object for a domain -func NewSession(domain string, keys *Keys, proxy string, rl int, timeout int) (*Session, error) { +func NewSession(domain string, keys *Keys, proxy string, rateLimit, timeout int) (*Session, error) { Transport := &http.Transport{ MaxIdleConns: 100, MaxIdleConnsPerHost: 100, @@ -47,8 +47,8 @@ func NewSession(domain string, keys *Keys, proxy string, rl int, timeout int) (* } // Initiate rate limit instance - if rl > 0 { - session.RateLimiter = ratelimit.New(rl) + if rateLimit > 0 { + session.RateLimiter = ratelimit.New(rateLimit) } else { session.RateLimiter = ratelimit.NewUnlimited() }