diff --git a/pkg/subscraping/sources/archiveis/archiveis.go b/pkg/subscraping/sources/archiveis/archiveis.go index 8322bed..ea56e6e 100755 --- a/pkg/subscraping/sources/archiveis/archiveis.go +++ b/pkg/subscraping/sources/archiveis/archiveis.go @@ -37,6 +37,7 @@ func (a *ArchiveIs) enumerate(ctx context.Context, baseURL string) { body, err := ioutil.ReadAll(resp.Body) if err != nil { a.Results <- subscraping.Result{Source: "archiveis", Type: subscraping.Error, Error: err} + resp.Body.Close() close(a.Results) return } diff --git a/pkg/subscraping/sources/binaryedge/binaryedge.go b/pkg/subscraping/sources/binaryedge/binaryedge.go index 1b1ba31..d71b17c 100755 --- a/pkg/subscraping/sources/binaryedge/binaryedge.go +++ b/pkg/subscraping/sources/binaryedge/binaryedge.go @@ -37,6 +37,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se err = jsoniter.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() close(results) return } @@ -82,6 +83,7 @@ func (s *Source) getSubdomains(ctx context.Context, domain string, remaining, cu err = jsoniter.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() return false } resp.Body.Close() diff --git a/pkg/subscraping/sources/bufferover/bufferover.go b/pkg/subscraping/sources/bufferover/bufferover.go index 11689e0..822c274 100755 --- a/pkg/subscraping/sources/bufferover/bufferover.go +++ b/pkg/subscraping/sources/bufferover/bufferover.go @@ -37,6 +37,7 @@ func (s *Source) getData(URL string, session *subscraping.Session, results chan body, err := ioutil.ReadAll(resp.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() return } resp.Body.Close() diff --git a/pkg/subscraping/sources/censys/censys.go b/pkg/subscraping/sources/censys/censys.go index 7d8d905..b02c23e 100644 --- a/pkg/subscraping/sources/censys/censys.go +++ b/pkg/subscraping/sources/censys/censys.go @@ -64,6 +64,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se err = jsoniter.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() close(results) return } diff --git a/pkg/subscraping/sources/certspotter/certspotter.go b/pkg/subscraping/sources/certspotter/certspotter.go index 581a68a..15c4ae7 100755 --- a/pkg/subscraping/sources/certspotter/certspotter.go +++ b/pkg/subscraping/sources/certspotter/certspotter.go @@ -37,6 +37,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se err = jsoniter.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() close(results) return } @@ -63,6 +64,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se err = jsoniter.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() close(results) return } diff --git a/pkg/subscraping/sources/certspotterold/certspotterold.go b/pkg/subscraping/sources/certspotterold/certspotterold.go index 0da2a6a..5f3115d 100755 --- a/pkg/subscraping/sources/certspotterold/certspotterold.go +++ b/pkg/subscraping/sources/certspotterold/certspotterold.go @@ -26,6 +26,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se body, err := ioutil.ReadAll(resp.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() close(results) return } diff --git a/pkg/subscraping/sources/commoncrawl/commoncrawl.go b/pkg/subscraping/sources/commoncrawl/commoncrawl.go index bbaeebb..9d5b2f5 100755 --- a/pkg/subscraping/sources/commoncrawl/commoncrawl.go +++ b/pkg/subscraping/sources/commoncrawl/commoncrawl.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "io" "io/ioutil" "strings" @@ -37,6 +38,8 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if resp.StatusCode == 500 { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: errors.New("internal server error")} + io.Copy(ioutil.Discard, resp.Body) + resp.Body.Close() close(results) return } @@ -45,6 +48,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se err = jsoniter.NewDecoder(resp.Body).Decode(&indexes) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() close(results) return } @@ -94,6 +98,7 @@ func (s *Source) getSubdomains(ctx context.Context, url string, domain string, s body, err := ioutil.ReadAll(resp.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() return false } resp.Body.Close() diff --git a/pkg/subscraping/sources/crtsh/crtsh.go b/pkg/subscraping/sources/crtsh/crtsh.go index bec2b0c..8e3f0c5 100755 --- a/pkg/subscraping/sources/crtsh/crtsh.go +++ b/pkg/subscraping/sources/crtsh/crtsh.go @@ -26,6 +26,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se body, err := ioutil.ReadAll(resp.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() close(results) return } diff --git a/pkg/subscraping/sources/digicert/digicert.go b/pkg/subscraping/sources/digicert/digicert.go index eb74658..e731250 100755 --- a/pkg/subscraping/sources/digicert/digicert.go +++ b/pkg/subscraping/sources/digicert/digicert.go @@ -26,6 +26,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se body, err := ioutil.ReadAll(resp.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() close(results) return } diff --git a/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go b/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go index f77569a..fcc7b6b 100755 --- a/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go +++ b/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go @@ -62,7 +62,7 @@ func postForm(token, domain string) (string, error) { // Now, grab the entire page in, err := ioutil.ReadAll(resp.Body) resp.Body.Close() - return string(in), nil + return string(in), err } // Source is the passive scraping agent @@ -83,6 +83,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se body, err := ioutil.ReadAll(resp.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() close(results) return } diff --git a/pkg/subscraping/sources/entrust/entrust.go b/pkg/subscraping/sources/entrust/entrust.go index 12aad1e..e042e54 100755 --- a/pkg/subscraping/sources/entrust/entrust.go +++ b/pkg/subscraping/sources/entrust/entrust.go @@ -27,6 +27,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se body, err := ioutil.ReadAll(resp.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() close(results) return } diff --git a/pkg/subscraping/sources/googleter/googleter.go b/pkg/subscraping/sources/googleter/googleter.go index 02e4ae8..93f042c 100755 --- a/pkg/subscraping/sources/googleter/googleter.go +++ b/pkg/subscraping/sources/googleter/googleter.go @@ -38,6 +38,7 @@ func (a *agent) makeRequest(token string, domain string) (string, error) { } body, err := ioutil.ReadAll(resp.Body) if err != nil { + resp.Body.Close() return "", err } resp.Body.Close() diff --git a/pkg/subscraping/sources/hackertarget/hackertarget.go b/pkg/subscraping/sources/hackertarget/hackertarget.go index 325b6bc..9e6f99c 100755 --- a/pkg/subscraping/sources/hackertarget/hackertarget.go +++ b/pkg/subscraping/sources/hackertarget/hackertarget.go @@ -27,6 +27,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se body, err := ioutil.ReadAll(resp.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() close(results) return } diff --git a/pkg/subscraping/sources/ipv4info/ipv4info.go b/pkg/subscraping/sources/ipv4info/ipv4info.go index 8a5f5c2..bd840f0 100755 --- a/pkg/subscraping/sources/ipv4info/ipv4info.go +++ b/pkg/subscraping/sources/ipv4info/ipv4info.go @@ -28,6 +28,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se body, err := ioutil.ReadAll(resp.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() close(results) return } @@ -54,6 +55,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se body, err = ioutil.ReadAll(resp.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() close(results) return } @@ -79,6 +81,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se body, err = ioutil.ReadAll(resp.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() close(results) return } @@ -104,6 +107,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se body, err = ioutil.ReadAll(resp.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() close(results) return } @@ -153,6 +157,7 @@ func (s *Source) getSubdomains(ctx context.Context, domain string, nextPage *int body, err := ioutil.ReadAll(resp.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() return false } resp.Body.Close() diff --git a/pkg/subscraping/sources/passivetotal/passivetotal.go b/pkg/subscraping/sources/passivetotal/passivetotal.go index 1960bc4..47e2bc6 100755 --- a/pkg/subscraping/sources/passivetotal/passivetotal.go +++ b/pkg/subscraping/sources/passivetotal/passivetotal.go @@ -50,6 +50,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se err = jsoniter.NewDecoder(resp.Body).Decode(&data) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() close(results) return } diff --git a/pkg/subscraping/sources/securitytrails/securitytrails.go b/pkg/subscraping/sources/securitytrails/securitytrails.go index 3ce1c28..8653ee1 100755 --- a/pkg/subscraping/sources/securitytrails/securitytrails.go +++ b/pkg/subscraping/sources/securitytrails/securitytrails.go @@ -37,6 +37,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se err = jsoniter.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() close(results) return } diff --git a/pkg/subscraping/sources/shodan/shodan.go b/pkg/subscraping/sources/shodan/shodan.go index 2c31d4a..0d32277 100644 --- a/pkg/subscraping/sources/shodan/shodan.go +++ b/pkg/subscraping/sources/shodan/shodan.go @@ -43,6 +43,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se err = jsoniter.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() close(results) return } diff --git a/pkg/subscraping/sources/sitedossier/sitedossier.go b/pkg/subscraping/sources/sitedossier/sitedossier.go index 6c51175..d24d82c 100755 --- a/pkg/subscraping/sources/sitedossier/sitedossier.go +++ b/pkg/subscraping/sources/sitedossier/sitedossier.go @@ -34,6 +34,7 @@ func (a *agent) enumerate(ctx context.Context, baseURL string) error { body, err := ioutil.ReadAll(resp.Body) if err != nil { a.results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Error, Error: err} + resp.Body.Close() close(a.results) return err } diff --git a/pkg/subscraping/sources/threatcrowd/threatcrowd.go b/pkg/subscraping/sources/threatcrowd/threatcrowd.go index 22e6ffd..f0fed81 100755 --- a/pkg/subscraping/sources/threatcrowd/threatcrowd.go +++ b/pkg/subscraping/sources/threatcrowd/threatcrowd.go @@ -27,6 +27,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se body, err := ioutil.ReadAll(resp.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() close(results) return } diff --git a/pkg/subscraping/sources/threatminer/threatminer.go b/pkg/subscraping/sources/threatminer/threatminer.go index 538fc56..2520939 100755 --- a/pkg/subscraping/sources/threatminer/threatminer.go +++ b/pkg/subscraping/sources/threatminer/threatminer.go @@ -27,6 +27,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se body, err := ioutil.ReadAll(resp.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() close(results) return } diff --git a/pkg/subscraping/sources/virustotal/virustotal.go b/pkg/subscraping/sources/virustotal/virustotal.go index 8950f1a..fb6e834 100755 --- a/pkg/subscraping/sources/virustotal/virustotal.go +++ b/pkg/subscraping/sources/virustotal/virustotal.go @@ -36,6 +36,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se err = jsoniter.NewDecoder(resp.Body).Decode(&data) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() close(results) return } diff --git a/pkg/subscraping/sources/waybackarchive/waybackarchive.go b/pkg/subscraping/sources/waybackarchive/waybackarchive.go index d34c68e..f97d744 100755 --- a/pkg/subscraping/sources/waybackarchive/waybackarchive.go +++ b/pkg/subscraping/sources/waybackarchive/waybackarchive.go @@ -27,6 +27,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se body, err := ioutil.ReadAll(pagesResp.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + pagesResp.Body.Close() close(results) return }