From dfd4d5b855249ca9617646435ab2b2113fd71782 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Sat, 20 May 2023 22:26:13 +0200 Subject: [PATCH] Adding interact keepalive to reduce server-side id pruning (#3680) * adding interact keepalive + improving init logic * dep update * go version update * readme update * version bump * fixing invalid format --------- Co-authored-by: Sandeep Singh Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com> --- .github/workflows/build-test.yml | 2 +- .github/workflows/functional-test.yml | 2 +- .github/workflows/lint-test.yml | 4 ++- .github/workflows/publish-docs.yaml | 2 +- .github/workflows/release-binary.yml | 2 +- .github/workflows/template-validate.yml | 2 +- README.md | 2 +- README_CN.md | 2 +- README_ID.md | 2 +- README_KR.md | 2 +- helm/templates/interactsh-ingress.yaml | 10 ++++---- v2/go.mod | 10 ++++---- v2/go.sum | 16 ++++++------ .../protocols/common/interactsh/interactsh.go | 25 ++++++++++--------- v2/pkg/protocols/common/randomip/randomip.go | 13 +++++++--- 15 files changed, 53 insertions(+), 43 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index b5f06364..57ff74e6 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -12,7 +12,7 @@ jobs: name: Test Builds strategy: matrix: - go-version: [1.19.x] + go-version: [1.20.x] os: [ubuntu-latest, windows-latest, macOS-13] runs-on: ${{ matrix.os }} diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml index 5c89ca54..c029803f 100644 --- a/.github/workflows/functional-test.yml +++ b/.github/workflows/functional-test.yml @@ -18,7 +18,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v4 with: - go-version: 1.19 + go-version: 1.20.x - name: Check out code uses: actions/checkout@v3 diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index 1dd66959..e800a7a2 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -15,9 +15,11 @@ jobs: - name: Set up Go uses: actions/setup-go@v4 with: - go-version: 1.19 + go-version: 1.20.x + - name: Checkout code uses: actions/checkout@v3 + - name: Run golangci-lint uses: golangci/golangci-lint-action@v3.4.0 with: diff --git a/.github/workflows/publish-docs.yaml b/.github/workflows/publish-docs.yaml index d7b32fad..b6904d77 100644 --- a/.github/workflows/publish-docs.yaml +++ b/.github/workflows/publish-docs.yaml @@ -18,7 +18,7 @@ jobs: - name: "Set up Go" uses: actions/setup-go@v4 with: - go-version: 1.19 + go-version: [1.20.x] - name: Generate YAML Syntax Documentation id: generate-docs diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml index 3f0f43f2..4a40bf9e 100644 --- a/.github/workflows/release-binary.yml +++ b/.github/workflows/release-binary.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/setup-go@v4 with: - go-version: 1.19 + go-version: 1.20.x - uses: goreleaser/goreleaser-action@v4 with: diff --git a/.github/workflows/template-validate.yml b/.github/workflows/template-validate.yml index cc448049..5ebb7428 100644 --- a/.github/workflows/template-validate.yml +++ b/.github/workflows/template-validate.yml @@ -13,7 +13,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-go@v4 with: - go-version: 1.19 + go-version: 1.20.x - name: Template Validation run: | diff --git a/README.md b/README.md index 34d03e16..39889a4f 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ We have a [dedicated repository](https://github.com/projectdiscovery/nuclei-temp # Install Nuclei -Nuclei requires **go1.19** to install successfully. Run the following command to install the latest version - +Nuclei requires **go1.20** to install successfully. Run the following command to install the latest version - ```sh go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest diff --git a/README_CN.md b/README_CN.md index ece8ad8d..87ab7e47 100644 --- a/README_CN.md +++ b/README_CN.md @@ -52,7 +52,7 @@ Nuclei使用零误报的定制模板向目标发送请求，同时可以对主 # 安装Nuclei -Nuclei需要**go1.19**才能安装成功。执行下列命令安装最新版本的Nuclei +Nuclei需要**go1.20**才能安装成功。执行下列命令安装最新版本的Nuclei ```sh go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest diff --git a/README_ID.md b/README_ID.md index de9ed061..ac24b0d7 100644 --- a/README_ID.md +++ b/README_ID.md @@ -52,7 +52,7 @@ Kami memiliki [repositori khusus](https://github.com/projectdiscovery/nuclei-tem # Instalasi Nuclei -Nuclei membutuhkan **go1.19** agar dapat diinstall. Jalankan perintah berikut untuk menginstal versi terbaru - +Nuclei membutuhkan **go1.20** agar dapat diinstall. Jalankan perintah berikut untuk menginstal versi terbaru - ```sh go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest diff --git a/README_KR.md b/README_KR.md index 5a23858f..8ab6cfd7 100644 --- a/README_KR.md +++ b/README_KR.md @@ -50,7 +50,7 @@ Nuclei는 템플릿을 기반으로 대상 간에 요청을 보내기 위해 사 # 설치 -Nuclei를 성공적으로 설치하기 위해서 **go1.19**가 필요합니다. 다음 명령을 실행하여 최신 버전을 설치합니다. +Nuclei를 성공적으로 설치하기 위해서 **go1.20**가 필요합니다. 다음 명령을 실행하여 최신 버전을 설치합니다. ```sh go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest diff --git a/helm/templates/interactsh-ingress.yaml b/helm/templates/interactsh-ingress.yaml index 83ee56e5..9c80eb4f 100644 --- a/helm/templates/interactsh-ingress.yaml +++ b/helm/templates/interactsh-ingress.yaml @@ -1,12 +1,12 @@ {{- if .Values.interactsh.ingress.enabled -}} {{- $fullName := include "nuclei.fullname" . -}} {{- $svcPort := .Values.service.port -}} -{{- if and .Values.interactsh.ingress.className (not (semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion)) }} +{{- if and .Values.interactsh.ingress.className (not (semverCompare ">=1.20-0" .Capabilities.KubeVersion.GitVersion)) }} {{- if not (hasKey .Values.interactsh.ingress.annotations "kubernetes.io/ingress.class") }} {{- $_ := set .Values.interactsh.ingress.annotations "kubernetes.io/ingress.class" .Values.interactsh.ingress.className}} {{- end }} {{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +{{- if semverCompare ">=1.20-0" .Capabilities.KubeVersion.GitVersion -}} apiVersion: networking.k8s.io/v1 {{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} apiVersion: networking.k8s.io/v1beta1 @@ -23,7 +23,7 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: - {{- if and .Values.interactsh.ingress.className (semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion) }} + {{- if and .Values.interactsh.ingress.className (semverCompare ">=1.20-0" .Capabilities.KubeVersion.GitVersion) }} ingressClassName: {{ .Values.interactsh.ingress.className }} {{- end }} {{- if .Values.interactsh.ingress.tls }} @@ -43,11 +43,11 @@ spec: paths: {{- range .paths }} - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion) }} + {{- if and .pathType (semverCompare ">=1.20-0" $.Capabilities.KubeVersion.GitVersion) }} pathType: {{ .pathType }} {{- end }} backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + {{- if semverCompare ">=1.20-0" $.Capabilities.KubeVersion.GitVersion }} service: name: {{ $fullName }} port: diff --git a/v2/go.mod b/v2/go.mod index 772c5e51..043f3b75 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -1,6 +1,6 @@ module github.com/projectdiscovery/nuclei/v2 -go 1.19 +go 1.20 require ( github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible @@ -23,7 +23,7 @@ require ( github.com/projectdiscovery/clistats v0.0.12 github.com/projectdiscovery/fastdialer v0.0.26 github.com/projectdiscovery/hmap v0.0.11 - github.com/projectdiscovery/interactsh v1.1.3 + github.com/projectdiscovery/interactsh v1.1.4 github.com/projectdiscovery/rawhttp v0.1.11 github.com/projectdiscovery/retryabledns v1.0.24 github.com/projectdiscovery/retryablehttp-go v1.0.16 @@ -78,9 +78,9 @@ require ( github.com/projectdiscovery/sarif v0.0.1 github.com/projectdiscovery/tlsx v1.0.9 github.com/projectdiscovery/uncover v1.0.4 - github.com/projectdiscovery/utils v0.0.29 + github.com/projectdiscovery/utils v0.0.32 github.com/projectdiscovery/wappalyzergo v0.0.94 - github.com/stretchr/testify v1.8.2 + github.com/stretchr/testify v1.8.3 gopkg.in/src-d/go-git.v4 v4.13.1 gopkg.in/yaml.v3 v3.0.1 ) @@ -122,7 +122,7 @@ require ( github.com/muesli/termenv v0.15.1 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect - github.com/projectdiscovery/asnmap v1.0.3 // indirect + github.com/projectdiscovery/asnmap v1.0.4 // indirect github.com/projectdiscovery/cdncheck v1.0.2 // indirect github.com/projectdiscovery/freeport v0.0.4 // indirect github.com/sashabaranov/go-openai v1.9.1 // indirect diff --git a/v2/go.sum b/v2/go.sum index 417826b8..7a7ea9aa 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -288,7 +288,6 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/karlseguin/ccache/v2 v2.0.8 h1:lT38cE//uyf6KcFok0rlgXtGFBWxkI6h/qg4tbFyDnA= github.com/kataras/jwt v0.1.8 h1:u71baOsYD22HWeSOg32tCHbczPjdCk7V4MMeJqTtmGk= github.com/kataras/jwt v0.1.8/go.mod h1:Q5j2IkcIHnfwy+oNY3TVWuEBJNw0ADgCcXK9CaZwV4o= github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= @@ -397,8 +396,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= -github.com/projectdiscovery/asnmap v1.0.3 h1:Jxo8bcLui9OphkABDEuuEYne0UWMvgSThReecv08FTk= -github.com/projectdiscovery/asnmap v1.0.3/go.mod h1:tR5ZE2RxDvyPJ/w1mpmTusPeLW8rTz1ojZ5M4ATKeO4= +github.com/projectdiscovery/asnmap v1.0.4 h1:dmXrls7Y0Sdeb6cLlKGwdYX8h1K9q2iYOGXioD4U2AY= +github.com/projectdiscovery/asnmap v1.0.4/go.mod h1:iTLDyYsblEwYHcLiKZCRx8Et+xV7NlvgGLusANpgegc= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= github.com/projectdiscovery/cdncheck v1.0.2 h1:8bd7Ed8NgTyH4nyhf0jwRQdGIrTFsnQoPGr2shxR/xM= @@ -421,8 +420,8 @@ github.com/projectdiscovery/hmap v0.0.11 h1:nA3qCFzWPcOw27T8PII5IWI3ZP0ys7TGCi2n github.com/projectdiscovery/hmap v0.0.11/go.mod h1:5sbLn2OHexvpVupStNOhusWO9jLCyEm5jcHwWB2nOkI= github.com/projectdiscovery/httpx v1.3.1 h1:yrpQJ/v31LLNiiSwfs0NAowtXDj4EWooKfTvCwucbZ8= github.com/projectdiscovery/httpx v1.3.1/go.mod h1:BIahPS6H86IArVsXV/T3SJ0CDuz2e6BIXz89eo0+IKQ= -github.com/projectdiscovery/interactsh v1.1.3 h1:xZvIF4pYXGXAwjwbYSRiE/97kOCvTl7xiEl9LPJfmJo= -github.com/projectdiscovery/interactsh v1.1.3/go.mod h1:UlpOGINO7wLjS2M8OhfBhfIBX0NIdAag5nhPDAz/9MQ= +github.com/projectdiscovery/interactsh v1.1.4 h1:1qVxJ14aG/X7TLJoK5AHnaX6I7hnbPp5R2ql1bSYzqI= +github.com/projectdiscovery/interactsh v1.1.4/go.mod h1:rM8IEm6AAm68fWWExzBHjhBWfRhFYzR9gY5emOFiZCY= github.com/projectdiscovery/mapcidr v1.1.1 h1:68Xvw9cKugNeAVxHE3Nl1Ej26nm1taWq6e1WPXpluc0= github.com/projectdiscovery/mapcidr v1.1.1/go.mod h1:yyp9ghqmmC0+r5DySgDBXE4cf2QW8SBloVESCteWiAg= github.com/projectdiscovery/networkpolicy v0.0.6 h1:yDvm0XCrS9HeemRrBS+J+22surzVczM94W5nHiOy/1o= @@ -447,8 +446,8 @@ github.com/projectdiscovery/tlsx v1.0.9/go.mod h1:4vGhl8lxUWjXuyoVvnL2ObMY/FSrb9 github.com/projectdiscovery/uncover v1.0.4 h1:d0lUNcnqDMfG4gR/WP2+UoQKLVmLE7opBiwnmI7ir/0= github.com/projectdiscovery/uncover v1.0.4/go.mod h1:fvSHBhj8zZFEXGOhQIpPe5j5LtDXI7adOns1iFJ5TTY= github.com/projectdiscovery/utils v0.0.3/go.mod h1:ne3eSlZlUKuhjHr8FfsfGcGteCzxcbJvFBx4VDBCxK0= -github.com/projectdiscovery/utils v0.0.29 h1:B7SYDIt2OJau+8Q6v/mMf1C0TNJK1mJFprQA/z7Ygdw= -github.com/projectdiscovery/utils v0.0.29/go.mod h1:RpMyTA/KACEBK+Bo9mk9MgLH0Hfc+BZtSlZ4T9bjLBE= +github.com/projectdiscovery/utils v0.0.32 h1:TjUxFmRG9hiV6jauYsnIRiy08lTgGHGi8avstedNGXw= +github.com/projectdiscovery/utils v0.0.32/go.mod h1:SaOpcZ2dJ47NE3t4R/YC2XpUyRZC6v5k8sj2TFro6+k= github.com/projectdiscovery/wappalyzergo v0.0.94 h1:IVRskuU95MajWCKYgvH5L67+MXDOWJDWSeBD61OsS/A= github.com/projectdiscovery/wappalyzergo v0.0.94/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= @@ -509,8 +508,9 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI= diff --git a/v2/pkg/protocols/common/interactsh/interactsh.go b/v2/pkg/protocols/common/interactsh/interactsh.go index cc7630c6..3c006f66 100644 --- a/v2/pkg/protocols/common/interactsh/interactsh.go +++ b/v2/pkg/protocols/common/interactsh/interactsh.go @@ -76,13 +76,15 @@ func New(options *Options) (*Client, error) { func (c *Client) poll() error { if c.options.NoInteractsh { - return nil // do not init if disabled + // do not init if disabled + return ErrInteractshClientNotInitialized } interactsh, err := client.New(&client.Options{ ServerURL: c.options.ServerURL, Token: c.options.Authorization, DisableHTTPFallback: c.options.DisableHttpFallback, HTTPClient: c.options.HTTPClient, + KeepAliveInterval: time.Minute, }) if err != nil { return errorutil.NewWithErr(err).Msgf("could not create client") @@ -196,18 +198,17 @@ func (c *Client) AlreadyMatched(data *RequestData) bool { // URL returns a new URL that can be interacted with func (c *Client) URL() (string, error) { + // first time initialization + var err error + c.Do(func() { + err = c.poll() + }) + if err != nil { + return "", errorutil.NewWithErr(err).Wrap(ErrInteractshClientNotInitialized) + } + if c.interactsh == nil { - var err error - c.Do(func() { - err = c.poll() - }) - if err != nil { - return "", errorutil.NewWithErr(err).Wrap(ErrInteractshClientNotInitialized) - } - // ensures interactsh is not nil - if c.interactsh == nil { - return "", ErrInteractshClientNotInitialized - } + return "", ErrInteractshClientNotInitialized } c.generated.Store(true) diff --git a/v2/pkg/protocols/common/randomip/randomip.go b/v2/pkg/protocols/common/randomip/randomip.go index 3e85c2c2..69fae2c7 100644 --- a/v2/pkg/protocols/common/randomip/randomip.go +++ b/v2/pkg/protocols/common/randomip/randomip.go @@ -1,11 +1,12 @@ package randomip import ( - "math/rand" + "crypto/rand" "net" "github.com/pkg/errors" iputil "github.com/projectdiscovery/utils/ip" + randutil "github.com/projectdiscovery/utils/rand" ) const ( @@ -16,7 +17,13 @@ func GetRandomIPWithCidr(cidrs ...string) (net.IP, error) { if len(cidrs) == 0 { return nil, errors.Errorf("must specify at least one cidr") } - cidr := cidrs[rand.Intn(len(cidrs))] + + randIdx, err := randutil.IntN(len(cidrs)) + if err != nil { + return nil, err + } + + cidr := cidrs[randIdx] if !iputil.IsCIDR(cidr) { return nil, errors.Errorf("%s is not a valid cidr", cidr) @@ -56,7 +63,7 @@ func getRandomIP(ipnet *net.IPNet, size int) net.IP { return ip } - rand.Read(r) + _, _ = rand.Read(r) for i := 0; i <= quotient; i++ { if i == quotient {