Close the response bodies

pkg/subscraping/sources/archiveis/archiveis.go

@@ -37,6 +37,7 @@ func (a *ArchiveIs) enumerate(ctx context.Context, baseURL string) {
 			body, err := ioutil.ReadAll(resp.Body)
 			if err != nil {
 				a.Results <- subscraping.Result{Source: "archiveis", Type: subscraping.Error, Error: err}
+				resp.Body.Close()
 				close(a.Results)
 				return
 			}

pkg/subscraping/sources/binaryedge/binaryedge.go

@@ -37,6 +37,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 		err = jsoniter.NewDecoder(resp.Body).Decode(&response)
 		if err != nil {
 			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+			resp.Body.Close()
 			close(results)
 			return
 		}
@@ -82,6 +83,7 @@ func (s *Source) getSubdomains(ctx context.Context, domain string, remaining, cu
 			err = jsoniter.NewDecoder(resp.Body).Decode(&response)
 			if err != nil {
 				results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+				resp.Body.Close()
 				return false
 			}
 			resp.Body.Close()

pkg/subscraping/sources/bufferover/bufferover.go

@@ -37,6 +37,7 @@ func (s *Source) getData(URL string, session *subscraping.Session, results chan
 	body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+		resp.Body.Close()
 		return
 	}
 	resp.Body.Close()

pkg/subscraping/sources/censys/censys.go

@@ -64,6 +64,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 			err = jsoniter.NewDecoder(resp.Body).Decode(&response)
 			if err != nil {
 				results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+				resp.Body.Close()
 				close(results)
 				return
 			}

pkg/subscraping/sources/certspotter/certspotter.go

@@ -37,6 +37,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 		err = jsoniter.NewDecoder(resp.Body).Decode(&response)
 		if err != nil {
 			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+			resp.Body.Close()
 			close(results)
 			return
 		}
@@ -63,6 +64,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 			err = jsoniter.NewDecoder(resp.Body).Decode(&response)
 			if err != nil {
 				results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+				resp.Body.Close()
 				close(results)
 				return
 			}

pkg/subscraping/sources/certspotterold/certspotterold.go

@@ -26,6 +26,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 		body, err := ioutil.ReadAll(resp.Body)
 		if err != nil {
 			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+			resp.Body.Close()
 			close(results)
 			return
 		}

pkg/subscraping/sources/commoncrawl/commoncrawl.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"strings"
 
@@ -37,6 +38,8 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 
 		if resp.StatusCode == 500 {
 			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: errors.New("internal server error")}
+			io.Copy(ioutil.Discard, resp.Body)
+			resp.Body.Close()
 			close(results)
 			return
 		}
@@ -45,6 +48,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 		err = jsoniter.NewDecoder(resp.Body).Decode(&indexes)
 		if err != nil {
 			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+			resp.Body.Close()
 			close(results)
 			return
 		}
@@ -94,6 +98,7 @@ func (s *Source) getSubdomains(ctx context.Context, url string, domain string, s
 			body, err := ioutil.ReadAll(resp.Body)
 			if err != nil {
 				results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+				resp.Body.Close()
 				return false
 			}
 			resp.Body.Close()

pkg/subscraping/sources/crtsh/crtsh.go

@@ -26,6 +26,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 		body, err := ioutil.ReadAll(resp.Body)
 		if err != nil {
 			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+			resp.Body.Close()
 			close(results)
 			return
 		}

pkg/subscraping/sources/digicert/digicert.go

@@ -26,6 +26,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 		body, err := ioutil.ReadAll(resp.Body)
 		if err != nil {
 			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+			resp.Body.Close()
 			close(results)
 			return
 		}

pkg/subscraping/sources/dnsdumpster/dnsdumpster.go

@@ -62,7 +62,7 @@ func postForm(token, domain string) (string, error) {
 	// Now, grab the entire page
 	in, err := ioutil.ReadAll(resp.Body)
 	resp.Body.Close()
-	return string(in), nil
+	return string(in), err
 }
 
 // Source is the passive scraping agent
@@ -83,6 +83,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 		body, err := ioutil.ReadAll(resp.Body)
 		if err != nil {
 			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+			resp.Body.Close()
 			close(results)
 			return
 		}

pkg/subscraping/sources/entrust/entrust.go

@@ -27,6 +27,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 		body, err := ioutil.ReadAll(resp.Body)
 		if err != nil {
 			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+			resp.Body.Close()
 			close(results)
 			return
 		}

pkg/subscraping/sources/googleter/googleter.go

@@ -38,6 +38,7 @@ func (a *agent) makeRequest(token string, domain string) (string, error) {
 	}
 	body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
+		resp.Body.Close()
 		return "", err
 	}
 	resp.Body.Close()

pkg/subscraping/sources/hackertarget/hackertarget.go

@@ -27,6 +27,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 		body, err := ioutil.ReadAll(resp.Body)
 		if err != nil {
 			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+			resp.Body.Close()
 			close(results)
 			return
 		}

pkg/subscraping/sources/ipv4info/ipv4info.go

@@ -28,6 +28,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 		body, err := ioutil.ReadAll(resp.Body)
 		if err != nil {
 			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+			resp.Body.Close()
 			close(results)
 			return
 		}
@@ -54,6 +55,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 		body, err = ioutil.ReadAll(resp.Body)
 		if err != nil {
 			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+			resp.Body.Close()
 			close(results)
 			return
 		}
@@ -79,6 +81,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 		body, err = ioutil.ReadAll(resp.Body)
 		if err != nil {
 			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+			resp.Body.Close()
 			close(results)
 			return
 		}
@@ -104,6 +107,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 		body, err = ioutil.ReadAll(resp.Body)
 		if err != nil {
 			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+			resp.Body.Close()
 			close(results)
 			return
 		}
@@ -153,6 +157,7 @@ func (s *Source) getSubdomains(ctx context.Context, domain string, nextPage *int
 			body, err := ioutil.ReadAll(resp.Body)
 			if err != nil {
 				results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+				resp.Body.Close()
 				return false
 			}
 			resp.Body.Close()

pkg/subscraping/sources/passivetotal/passivetotal.go

@@ -50,6 +50,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 		err = jsoniter.NewDecoder(resp.Body).Decode(&data)
 		if err != nil {
 			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+			resp.Body.Close()
 			close(results)
 			return
 		}

pkg/subscraping/sources/securitytrails/securitytrails.go

@@ -37,6 +37,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 		err = jsoniter.NewDecoder(resp.Body).Decode(&response)
 		if err != nil {
 			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+			resp.Body.Close()
 			close(results)
 			return
 		}

pkg/subscraping/sources/shodan/shodan.go

@@ -43,6 +43,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 			err = jsoniter.NewDecoder(resp.Body).Decode(&response)
 			if err != nil {
 				results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+				resp.Body.Close()
 				close(results)
 				return
 			}

pkg/subscraping/sources/sitedossier/sitedossier.go

@@ -34,6 +34,7 @@ func (a *agent) enumerate(ctx context.Context, baseURL string) error {
 			body, err := ioutil.ReadAll(resp.Body)
 			if err != nil {
 				a.results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Error, Error: err}
+				resp.Body.Close()
 				close(a.results)
 				return err
 			}

pkg/subscraping/sources/threatcrowd/threatcrowd.go

@@ -27,6 +27,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 		body, err := ioutil.ReadAll(resp.Body)
 		if err != nil {
 			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+			resp.Body.Close()
 			close(results)
 			return
 		}

pkg/subscraping/sources/threatminer/threatminer.go

@@ -27,6 +27,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 		body, err := ioutil.ReadAll(resp.Body)
 		if err != nil {
 			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+			resp.Body.Close()
 			close(results)
 			return
 		}

pkg/subscraping/sources/virustotal/virustotal.go

@@ -36,6 +36,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 		err = jsoniter.NewDecoder(resp.Body).Decode(&data)
 		if err != nil {
 			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+			resp.Body.Close()
 			close(results)
 			return
 		}

pkg/subscraping/sources/waybackarchive/waybackarchive.go

@@ -27,6 +27,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 		body, err := ioutil.ReadAll(pagesResp.Body)
 		if err != nil {
 			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+			pagesResp.Body.Close()
 			close(results)
 			return
 		}