Adding interact keepalive to reduce server-side id pruning (#3680)

* adding interact keepalive + improving init logic

* dep update

* go version update

* readme update

* version bump

* fixing invalid format

---------

Co-authored-by: Sandeep Singh <sandeep@projectdiscovery.io>
Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com>

.github/workflows/build-test.yml

@@ -12,7 +12,7 @@ jobs:
     name: Test Builds
     strategy:
       matrix:
-        go-version: [1.19.x]
+        go-version: [1.20.x]
         os: [ubuntu-latest, windows-latest, macOS-13]
 
     runs-on: ${{ matrix.os }}

.github/workflows/functional-test.yml

@@ -18,7 +18,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v4
         with:
-          go-version: 1.19
+          go-version: 1.20.x
 
       - name: Check out code
         uses: actions/checkout@v3

.github/workflows/lint-test.yml

@@ -15,9 +15,11 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v4
         with:
-          go-version: 1.19
+          go-version: 1.20.x
+
       - name: Checkout code
         uses: actions/checkout@v3
+
       - name: Run golangci-lint
         uses: golangci/golangci-lint-action@v3.4.0
         with:

.github/workflows/publish-docs.yaml

@@ -18,7 +18,7 @@ jobs:
       - name: "Set up Go"
         uses: actions/setup-go@v4
         with: 
-          go-version: 1.19
+          go-version: [1.20.x]
 
       - name: Generate YAML Syntax Documentation
         id: generate-docs

.github/workflows/release-binary.yml

@@ -17,7 +17,7 @@ jobs:
 
       - uses: actions/setup-go@v4
         with: 
-          go-version: 1.19
+          go-version: 1.20.x
 
       - uses: goreleaser/goreleaser-action@v4
         with: 

.github/workflows/template-validate.yml

@@ -13,7 +13,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v4
         with: 
-          go-version: 1.19
+          go-version: 1.20.x
 
       - name: Template Validation
         run: |

README.md

@@ -53,7 +53,7 @@ We have a [dedicated repository](https://github.com/projectdiscovery/nuclei-temp
 
 # Install Nuclei
 
-Nuclei requires **go1.19** to install successfully. Run the following command to install the latest version -
+Nuclei requires **go1.20** to install successfully. Run the following command to install the latest version -
 
 ```sh
 go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest

README_CN.md

@@ -52,7 +52,7 @@ Nuclei使用零误报的定制模板向目标发送请求，同时可以对主
 
 # 安装Nuclei
 
-Nuclei需要**go1.19**才能安装成功。执行下列命令安装最新版本的Nuclei
+Nuclei需要**go1.20**才能安装成功。执行下列命令安装最新版本的Nuclei
 
 ```sh
 go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest

README_ID.md

@@ -52,7 +52,7 @@ Kami memiliki [repositori khusus](https://github.com/projectdiscovery/nuclei-tem
 
 # Instalasi Nuclei
 
-Nuclei membutuhkan **go1.19** agar dapat diinstall. Jalankan perintah berikut untuk menginstal versi terbaru -
+Nuclei membutuhkan **go1.20** agar dapat diinstall. Jalankan perintah berikut untuk menginstal versi terbaru -
 
 ```sh
 go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest

README_KR.md

@@ -50,7 +50,7 @@ Nuclei는 템플릿을 기반으로 대상 간에 요청을 보내기 위해 사
 
 # 설치
 
-Nuclei를 성공적으로 설치하기 위해서 **go1.19**가 필요합니다. 다음 명령을 실행하여 최신 버전을 설치합니다.
+Nuclei를 성공적으로 설치하기 위해서 **go1.20**가 필요합니다. 다음 명령을 실행하여 최신 버전을 설치합니다.
 
 ```sh
 go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest

helm/templates/interactsh-ingress.yaml

@@ -1,12 +1,12 @@
 {{- if .Values.interactsh.ingress.enabled -}}
 {{- $fullName := include "nuclei.fullname" . -}}
 {{- $svcPort := .Values.service.port -}}
-{{- if and .Values.interactsh.ingress.className (not (semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion)) }}
+{{- if and .Values.interactsh.ingress.className (not (semverCompare ">=1.20-0" .Capabilities.KubeVersion.GitVersion)) }}
   {{- if not (hasKey .Values.interactsh.ingress.annotations "kubernetes.io/ingress.class") }}
   {{- $_ := set .Values.interactsh.ingress.annotations "kubernetes.io/ingress.class" .Values.interactsh.ingress.className}}
   {{- end }}
 {{- end }}
-{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
+{{- if semverCompare ">=1.20-0" .Capabilities.KubeVersion.GitVersion -}}
 apiVersion: networking.k8s.io/v1
 {{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
 apiVersion: networking.k8s.io/v1beta1
@@ -23,7 +23,7 @@ metadata:
     {{- toYaml . | nindent 4 }}
   {{- end }}
 spec:
-  {{- if and .Values.interactsh.ingress.className (semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion) }}
+  {{- if and .Values.interactsh.ingress.className (semverCompare ">=1.20-0" .Capabilities.KubeVersion.GitVersion) }}
   ingressClassName: {{ .Values.interactsh.ingress.className }}
   {{- end }}
   {{- if .Values.interactsh.ingress.tls }}
@@ -43,11 +43,11 @@ spec:
         paths:
           {{- range .paths }}
           - path: {{ .path }}
-            {{- if and .pathType (semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion) }}
+            {{- if and .pathType (semverCompare ">=1.20-0" $.Capabilities.KubeVersion.GitVersion) }}
             pathType: {{ .pathType }}
             {{- end }}
             backend:
-              {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
+              {{- if semverCompare ">=1.20-0" $.Capabilities.KubeVersion.GitVersion }}
               service:
                 name: {{ $fullName }}
                 port:

v2/go.mod

@@ -1,6 +1,6 @@
 module github.com/projectdiscovery/nuclei/v2
 
-go 1.19
+go 1.20
 
 require (
 	github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible
@@ -23,7 +23,7 @@ require (
 	github.com/projectdiscovery/clistats v0.0.12
 	github.com/projectdiscovery/fastdialer v0.0.26
 	github.com/projectdiscovery/hmap v0.0.11
-	github.com/projectdiscovery/interactsh v1.1.3
+	github.com/projectdiscovery/interactsh v1.1.4
 	github.com/projectdiscovery/rawhttp v0.1.11
 	github.com/projectdiscovery/retryabledns v1.0.24
 	github.com/projectdiscovery/retryablehttp-go v1.0.16
@@ -78,9 +78,9 @@ require (
 	github.com/projectdiscovery/sarif v0.0.1
 	github.com/projectdiscovery/tlsx v1.0.9
 	github.com/projectdiscovery/uncover v1.0.4
-	github.com/projectdiscovery/utils v0.0.29
+	github.com/projectdiscovery/utils v0.0.32
 	github.com/projectdiscovery/wappalyzergo v0.0.94
-	github.com/stretchr/testify v1.8.2
+	github.com/stretchr/testify v1.8.3
 	gopkg.in/src-d/go-git.v4 v4.13.1
 	gopkg.in/yaml.v3 v3.0.1
 )
@@ -122,7 +122,7 @@ require (
 	github.com/muesli/termenv v0.15.1 // indirect
 	github.com/pjbgf/sha1cd v0.3.0 // indirect
 	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
-	github.com/projectdiscovery/asnmap v1.0.3 // indirect
+	github.com/projectdiscovery/asnmap v1.0.4 // indirect
 	github.com/projectdiscovery/cdncheck v1.0.2 // indirect
 	github.com/projectdiscovery/freeport v0.0.4 // indirect
 	github.com/sashabaranov/go-openai v1.9.1 // indirect

v2/go.sum

@@ -288,7 +288,6 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
-github.com/karlseguin/ccache/v2 v2.0.8 h1:lT38cE//uyf6KcFok0rlgXtGFBWxkI6h/qg4tbFyDnA=
 github.com/kataras/jwt v0.1.8 h1:u71baOsYD22HWeSOg32tCHbczPjdCk7V4MMeJqTtmGk=
 github.com/kataras/jwt v0.1.8/go.mod h1:Q5j2IkcIHnfwy+oNY3TVWuEBJNw0ADgCcXK9CaZwV4o=
 github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
@@ -397,8 +396,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw=
 github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
-github.com/projectdiscovery/asnmap v1.0.3 h1:Jxo8bcLui9OphkABDEuuEYne0UWMvgSThReecv08FTk=
+github.com/projectdiscovery/asnmap v1.0.4 h1:dmXrls7Y0Sdeb6cLlKGwdYX8h1K9q2iYOGXioD4U2AY=
-github.com/projectdiscovery/asnmap v1.0.3/go.mod h1:tR5ZE2RxDvyPJ/w1mpmTusPeLW8rTz1ojZ5M4ATKeO4=
+github.com/projectdiscovery/asnmap v1.0.4/go.mod h1:iTLDyYsblEwYHcLiKZCRx8Et+xV7NlvgGLusANpgegc=
 github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ=
 github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss=
 github.com/projectdiscovery/cdncheck v1.0.2 h1:8bd7Ed8NgTyH4nyhf0jwRQdGIrTFsnQoPGr2shxR/xM=
@@ -421,8 +420,8 @@ github.com/projectdiscovery/hmap v0.0.11 h1:nA3qCFzWPcOw27T8PII5IWI3ZP0ys7TGCi2n
 github.com/projectdiscovery/hmap v0.0.11/go.mod h1:5sbLn2OHexvpVupStNOhusWO9jLCyEm5jcHwWB2nOkI=
 github.com/projectdiscovery/httpx v1.3.1 h1:yrpQJ/v31LLNiiSwfs0NAowtXDj4EWooKfTvCwucbZ8=
 github.com/projectdiscovery/httpx v1.3.1/go.mod h1:BIahPS6H86IArVsXV/T3SJ0CDuz2e6BIXz89eo0+IKQ=
-github.com/projectdiscovery/interactsh v1.1.3 h1:xZvIF4pYXGXAwjwbYSRiE/97kOCvTl7xiEl9LPJfmJo=
+github.com/projectdiscovery/interactsh v1.1.4 h1:1qVxJ14aG/X7TLJoK5AHnaX6I7hnbPp5R2ql1bSYzqI=
-github.com/projectdiscovery/interactsh v1.1.3/go.mod h1:UlpOGINO7wLjS2M8OhfBhfIBX0NIdAag5nhPDAz/9MQ=
+github.com/projectdiscovery/interactsh v1.1.4/go.mod h1:rM8IEm6AAm68fWWExzBHjhBWfRhFYzR9gY5emOFiZCY=
 github.com/projectdiscovery/mapcidr v1.1.1 h1:68Xvw9cKugNeAVxHE3Nl1Ej26nm1taWq6e1WPXpluc0=
 github.com/projectdiscovery/mapcidr v1.1.1/go.mod h1:yyp9ghqmmC0+r5DySgDBXE4cf2QW8SBloVESCteWiAg=
 github.com/projectdiscovery/networkpolicy v0.0.6 h1:yDvm0XCrS9HeemRrBS+J+22surzVczM94W5nHiOy/1o=
@@ -447,8 +446,8 @@ github.com/projectdiscovery/tlsx v1.0.9/go.mod h1:4vGhl8lxUWjXuyoVvnL2ObMY/FSrb9
 github.com/projectdiscovery/uncover v1.0.4 h1:d0lUNcnqDMfG4gR/WP2+UoQKLVmLE7opBiwnmI7ir/0=
 github.com/projectdiscovery/uncover v1.0.4/go.mod h1:fvSHBhj8zZFEXGOhQIpPe5j5LtDXI7adOns1iFJ5TTY=
 github.com/projectdiscovery/utils v0.0.3/go.mod h1:ne3eSlZlUKuhjHr8FfsfGcGteCzxcbJvFBx4VDBCxK0=
-github.com/projectdiscovery/utils v0.0.29 h1:B7SYDIt2OJau+8Q6v/mMf1C0TNJK1mJFprQA/z7Ygdw=
+github.com/projectdiscovery/utils v0.0.32 h1:TjUxFmRG9hiV6jauYsnIRiy08lTgGHGi8avstedNGXw=
-github.com/projectdiscovery/utils v0.0.29/go.mod h1:RpMyTA/KACEBK+Bo9mk9MgLH0Hfc+BZtSlZ4T9bjLBE=
+github.com/projectdiscovery/utils v0.0.32/go.mod h1:SaOpcZ2dJ47NE3t4R/YC2XpUyRZC6v5k8sj2TFro6+k=
 github.com/projectdiscovery/wappalyzergo v0.0.94 h1:IVRskuU95MajWCKYgvH5L67+MXDOWJDWSeBD61OsS/A=
 github.com/projectdiscovery/wappalyzergo v0.0.94/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0=
 github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
@@ -509,8 +508,9 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY=
+github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
 github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI=

v2/pkg/protocols/common/interactsh/interactsh.go

@@ -76,13 +76,15 @@ func New(options *Options) (*Client, error) {
 
 func (c *Client) poll() error {
 	if c.options.NoInteractsh {
-		return nil // do not init if disabled
+		// do not init if disabled
+		return ErrInteractshClientNotInitialized
 	}
 	interactsh, err := client.New(&client.Options{
 		ServerURL:           c.options.ServerURL,
 		Token:               c.options.Authorization,
 		DisableHTTPFallback: c.options.DisableHttpFallback,
 		HTTPClient:          c.options.HTTPClient,
+		KeepAliveInterval:   time.Minute,
 	})
 	if err != nil {
 		return errorutil.NewWithErr(err).Msgf("could not create client")
@@ -196,18 +198,17 @@ func (c *Client) AlreadyMatched(data *RequestData) bool {
 
 // URL returns a new URL that can be interacted with
 func (c *Client) URL() (string, error) {
+	// first time initialization
+	var err error
+	c.Do(func() {
+		err = c.poll()
+	})
+	if err != nil {
+		return "", errorutil.NewWithErr(err).Wrap(ErrInteractshClientNotInitialized)
+	}
+
 	if c.interactsh == nil {
-		var err error
+		return "", ErrInteractshClientNotInitialized
-		c.Do(func() {
-			err = c.poll()
-		})
-		if err != nil {
-			return "", errorutil.NewWithErr(err).Wrap(ErrInteractshClientNotInitialized)
-		}
-		// ensures interactsh is not nil
-		if c.interactsh == nil {
-			return "", ErrInteractshClientNotInitialized
-		}
 	}
 
 	c.generated.Store(true)

v2/pkg/protocols/common/randomip/randomip.go

@@ -1,11 +1,12 @@
 package randomip
 
 import (
-	"math/rand"
+	"crypto/rand"
 	"net"
 
 	"github.com/pkg/errors"
 	iputil "github.com/projectdiscovery/utils/ip"
+	randutil "github.com/projectdiscovery/utils/rand"
 )
 
 const (
@@ -16,7 +17,13 @@ func GetRandomIPWithCidr(cidrs ...string) (net.IP, error) {
 	if len(cidrs) == 0 {
 		return nil, errors.Errorf("must specify at least one cidr")
 	}
-	cidr := cidrs[rand.Intn(len(cidrs))]
+
+	randIdx, err := randutil.IntN(len(cidrs))
+	if err != nil {
+		return nil, err
+	}
+
+	cidr := cidrs[randIdx]
 
 	if !iputil.IsCIDR(cidr) {
 		return nil, errors.Errorf("%s is not a valid cidr", cidr)
@@ -56,7 +63,7 @@ func getRandomIP(ipnet *net.IPNet, size int) net.IP {
 			return ip
 		}
 
-		rand.Read(r)
+		_, _ = rand.Read(r)
 
 		for i := 0; i <= quotient; i++ {
 			if i == quotient {