Merge pull request #3381 from projectdiscovery/issue-3305-retryablehttp-api-requests

Using retryablehttp for api request

v2/cmd/cve-annotate/main.go

@@ -6,7 +6,6 @@ import (
 	"flag"
 	"fmt"
 	"log"
-	"net/http"
 	"net/url"
 	"os"
 	"regexp"
@@ -15,6 +14,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk"
 	"github.com/projectdiscovery/nvd"
+	"github.com/projectdiscovery/retryablehttp-go"
 	sliceutil "github.com/projectdiscovery/utils/slice"
 	stringsutil "github.com/projectdiscovery/utils/strings"
 	"gopkg.in/yaml.v3"
@@ -275,7 +275,7 @@ type cisaKEVData struct {
 func fetchCISAKnownExploitedVulnerabilities() error {
 	data := &cisaKEVData{}
 
-	resp, err := http.Get("https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json")
+	resp, err := retryablehttp.DefaultClient().Get("https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json")
 	if err != nil {
 		return errors.Wrap(err, "could not get cisa kev catalog")
 	}

v2/cmd/integration-test/http.go

@@ -17,6 +17,7 @@ import (
 	"gopkg.in/yaml.v2"
 
 	"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
+	"github.com/projectdiscovery/retryablehttp-go"
 	logutil "github.com/projectdiscovery/utils/log"
 	stringsutil "github.com/projectdiscovery/utils/strings"
 )
@@ -77,7 +78,7 @@ func (h *httpInteractshRequest) Execute(filePath string) error {
 	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
 		value := r.Header.Get("url")
 		if value != "" {
-			if resp, _ := http.DefaultClient.Get(value); resp != nil {
+			if resp, _ := retryablehttp.DefaultClient().Get(value); resp != nil {
 				resp.Body.Close()
 			}
 		}
@@ -101,7 +102,7 @@ func (h *httpInteractshStopAtFirstMatchRequest) Execute(filePath string) error {
 	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
 		value := r.Header.Get("url")
 		if value != "" {
-			if resp, _ := http.DefaultClient.Get(value); resp != nil {
+			if resp, _ := retryablehttp.DefaultClient().Get(value); resp != nil {
 				resp.Body.Close()
 			}
 		}

v2/cmd/tools/fuzzplayground/main.go

@@ -3,13 +3,13 @@ package main
 import (
 	"fmt"
 	"io"
-	"net/http"
 	"os/exec"
 	"strconv"
 	"strings"
 
 	"github.com/labstack/echo/v4"
 	"github.com/labstack/echo/v4/middleware"
+	"github.com/projectdiscovery/retryablehttp-go"
 )
 
 func main() {
@@ -60,7 +60,7 @@ func redirectHandler(ctx echo.Context) error {
 
 func requestHandler(ctx echo.Context) error {
 	url := ctx.QueryParam("url")
-	data, err := http.Get(url)
+	data, err := retryablehttp.DefaultClient().Get(url)
 	if err != nil {
 		return ctx.HTML(500, err.Error())
 	}

v2/internal/runner/runner.go

@@ -713,7 +713,7 @@ func (r *Runner) displayExecutionInfo(store *loader.Store) {
 }
 
 func (r *Runner) readNewTemplatesWithVersionFile(version string) ([]string, error) {
-	resp, err := http.DefaultClient.Get(fmt.Sprintf("https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/%s/.new-additions", version))
+	resp, err := retryablehttp.DefaultClient().Get(fmt.Sprintf("https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/%s/.new-additions", version))
 	if err != nil {
 		return nil, err
 	}

v2/internal/runner/update.go

@@ -29,6 +29,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/external/customtemplates"
 	client "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/updatecheck"
 	"github.com/projectdiscovery/nuclei/v2/pkg/utils"
+	"github.com/projectdiscovery/retryablehttp-go"
 	fileutil "github.com/projectdiscovery/utils/file"
 	folderutil "github.com/projectdiscovery/utils/folder"
 
@@ -308,7 +309,7 @@ func (r *Runner) downloadReleaseAndUnzip(ctx context.Context, version, downloadU
 		return nil, fmt.Errorf("failed to create HTTP request to %s: %w", downloadURL, err)
 	}
 
-	res, err := http.DefaultClient.Do(req)
+	res, err := retryablehttp.DefaultClient().Do(req)
 	if err != nil {
 		return nil, fmt.Errorf("failed to download a release file from %s: %w", downloadURL, err)
 	}

v2/pkg/catalog/loader/remote_loader.go

@@ -3,13 +3,13 @@ package loader
 import (
 	"bufio"
 	"fmt"
-	"net/http"
 	"net/url"
 	"strings"
 
 	"github.com/pkg/errors"
 
 	"github.com/projectdiscovery/nuclei/v2/pkg/utils"
+	"github.com/projectdiscovery/retryablehttp-go"
 )
 
 type ContentType string
@@ -72,7 +72,7 @@ func getRemoteContent(URL string, remoteTemplateDomainList []string, remoteConte
 		}
 		return
 	}
-	response, err := http.Get(URL)
+	response, err := retryablehttp.DefaultClient().Get(URL)
 	if err != nil {
 		remoteContentChannel <- RemoteContent{
 			Error: err,

v2/pkg/protocols/common/updatecheck/client.go

@@ -9,6 +9,7 @@ import (
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/pkg/errors"
+	"github.com/projectdiscovery/retryablehttp-go"
 )
 
 const (
@@ -74,7 +75,7 @@ func callRegisterServer(call string) (io.ReadCloser, error) {
 		query.Set("v", nucleiVersion)
 		req.URL.RawQuery = query.Encode()
 	}
-	resp, err := http.DefaultClient.Do(req)
+	resp, err := retryablehttp.DefaultClient().Do(req)
 	if err != nil {
 		return nil, errors.Wrap(err, "could not do request")
 	}

v2/pkg/utils/utils.go

@@ -3,12 +3,12 @@ package utils
 import (
 	"errors"
 	"io"
-	"net/http"
 	"net/url"
 	"strings"
 
 	"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
 	"github.com/projectdiscovery/nuclei/v2/pkg/utils/yaml"
+	"github.com/projectdiscovery/retryablehttp-go"
 	fileutil "github.com/projectdiscovery/utils/file"
 )
 
@@ -37,7 +37,7 @@ func IsURL(input string) bool {
 func ReadFromPathOrURL(templatePath string, catalog catalog.Catalog) (data []byte, err error) {
 	var reader io.Reader
 	if IsURL(templatePath) {
-		resp, err := http.Get(templatePath)
+		resp, err := retryablehttp.DefaultClient().Get(templatePath)
 		if err != nil {
 			return nil, err
 		}