From bba144f1b86924f1b6606ab7203616b65b3d5081 Mon Sep 17 00:00:00 2001 From: maldevel Date: Tue, 10 Aug 2021 21:20:05 +0300 Subject: [PATCH 1/2] recon.dev bug fixing, json response has changed --- v2/pkg/subscraping/agent.go | 6 +++--- v2/pkg/subscraping/sources/recon/recon.go | 10 ++++++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/v2/pkg/subscraping/agent.go b/v2/pkg/subscraping/agent.go index a039dfc..a940028 100755 --- a/v2/pkg/subscraping/agent.go +++ b/v2/pkg/subscraping/agent.go @@ -78,9 +78,9 @@ func (s *Session) HTTPRequest(ctx context.Context, method, requestURL, cookies s return nil, err } - req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36") - req.Header.Set("Accept", "*/*") - req.Header.Set("Accept-Language", "en") + req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:90.0) Gecko/20100101 Firefox/90.0") + req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8") //"*/*") + req.Header.Set("Accept-Language", "en-US,en;q=0.5") //"en") req.Header.Set("Connection", "close") if basicAuth.Username != "" || basicAuth.Password != "" { diff --git a/v2/pkg/subscraping/sources/recon/recon.go b/v2/pkg/subscraping/sources/recon/recon.go index b1539b1..ff4b9d1 100644 --- a/v2/pkg/subscraping/sources/recon/recon.go +++ b/v2/pkg/subscraping/sources/recon/recon.go @@ -10,7 +10,11 @@ import ( ) type subdomain struct { - RawDomain string `json:"rawDomain"` + Domains []string `json:"domains"` + Ip string `json:"ip"` + RawDomains []string `json:"rawDomains"` + RawPort string `json:"rawPort"` + RawIp string `json:"rawIp"` } // Source is the passive scraping agent @@ -44,7 +48,9 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se resp.Body.Close() for _, subdomain := range subdomains { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain.RawDomain} + for _, dmn := range subdomain.RawDomains { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: dmn} + } } }() From e4dcab16454947cf3fef2c266ecb25ee761b1043 Mon Sep 17 00:00:00 2001 From: maldevel Date: Tue, 10 Aug 2021 21:23:46 +0300 Subject: [PATCH 2/2] Update agent.go --- v2/pkg/subscraping/agent.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/pkg/subscraping/agent.go b/v2/pkg/subscraping/agent.go index a940028..a039dfc 100755 --- a/v2/pkg/subscraping/agent.go +++ b/v2/pkg/subscraping/agent.go @@ -78,9 +78,9 @@ func (s *Session) HTTPRequest(ctx context.Context, method, requestURL, cookies s return nil, err } - req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:90.0) Gecko/20100101 Firefox/90.0") - req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8") //"*/*") - req.Header.Set("Accept-Language", "en-US,en;q=0.5") //"en") + req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36") + req.Header.Set("Accept", "*/*") + req.Header.Set("Accept-Language", "en") req.Header.Set("Connection", "close") if basicAuth.Username != "" || basicAuth.Password != "" {