Merge remote-tracking branch 'origin/dev' into code_smells

.github/workflows/publish-docs.yaml

@@ -3,7 +3,6 @@ name: ⏰ Publish Docs
 on:
   push:
     branches:
-      - master
       - dev
   workflow_dispatch:
 
@@ -28,7 +27,7 @@ jobs:
         run: |
           if ! which dstdocgen > /dev/null; then
             echo -e "Command dstdocgen not found! Installing\c"
-            go get -v github.com/projectdiscovery/yamldoc-go/cmd/docgen/dstdocgen
+            go install github.com/projectdiscovery/yamldoc-go/cmd/docgen/dstdocgen@main
           fi
           go generate pkg/templates/templates.go
           go build -o "cmd/docgen/docgen" cmd/docgen/docgen.go 
@@ -49,3 +48,4 @@ jobs:
         uses: ad-m/github-push-action@master
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
+          branch: ${{ github.ref }}

Dockerfile

@@ -1,7 +1,7 @@
 FROM golang:1.17.0-alpine as build-env
 RUN GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei
 
-FROM alpine:latest
+FROM alpine:3.14
 RUN apk add --no-cache bind-tools ca-certificates
 COPY --from=build-env /go/bin/nuclei /usr/local/bin/nuclei
 ENTRYPOINT ["nuclei"]

v2/cmd/cve-annotate/main.go

@@ -75,6 +75,10 @@ func getCVEData(client *nvd.Client, filePath, data string) {
 	}
 	severityValue := severityMatches[0][1]
 
+	// Skip if there's classification data already
+	if strings.Contains(data, "classification:") {
+		return
+	}
 	cveItem, err := client.FetchCVE(cveName)
 	if err != nil {
 		log.Printf("Could not fetch cve %s: %s\n", cveName, err)
@@ -119,7 +123,7 @@ func getCVEData(client *nvd.Client, filePath, data string) {
 		}
 	}
 	// If there is no description field, fill the description from CVE information
-	if !strings.Contains(infoBlockClean, "description:") {
+	if !strings.Contains(infoBlockClean, "description:") && len(cveItem.CVE.Description.DescriptionData) > 0 {
 		changed = true
 		newInfoBlock = newInfoBlock + fmt.Sprintf("\n  description: %s", fmt.Sprintf("%q", cveItem.CVE.Description.DescriptionData[0].Value))
 	}

v2/cmd/functional-test/testcases.txt

@@ -7,7 +7,7 @@
 {{binary}} -tags 'cve','exposure' -tags 'token','logs'
 {{binary}} -tags cve -severity high
 {{binary}} -tags cve,exposure -severity high,critical
-{{binary}} -tags cve,exposure -severity "high,critical,medium"
+{{binary}} -tags cve,exposure -severity high,critical,medium
 {{binary}} -tags cve -author geeknik
 {{binary}} -tags cve -author geeknik,pdteam
 {{binary}} -tags cve -author geeknik -severity high
@@ -19,12 +19,12 @@
 {{binary}} -tags 'cve','exposure' -tags 'token','logs'
 {{binary}} -tags cve -severity high
 {{binary}} -tags cve,exposure -severity high,critical
-{{binary}} -tags cve,exposure -severity "high,critical,medium"
+{{binary}} -tags cve,exposure -severity high,critical,medium
 {{binary}} -tags cve -author geeknik
 {{binary}} -tags cve -author geeknik,pdteam
 {{binary}} -tags cve -author geeknik -severity high
 {{binary}} -tags cve,exposure -author geeknik,pdteam -severity high,critical
-{{binary}} -tags "cve,exposure" -author "geeknik,pdteam" -severity "high,critical"
+{{binary}} -tags "cve,exposure" -author "geeknik,pdteam" -severity high,critical
 {{binary}} -tags cve -etags ssrf
 {{binary}} -tags cve,exposure -etags ssrf,config
 {{binary}} -tags cve,exposure -etags ssrf,config -severity high

v2/go.mod

@@ -1,9 +1,9 @@
 module github.com/projectdiscovery/nuclei/v2
 
-go 1.16
+go 1.17
 
 require (
-	github.com/Ice3man543/nvd v1.0.6
+	github.com/Ice3man543/nvd v1.0.8
 	github.com/Knetic/govaluate v3.0.0+incompatible
 	github.com/akrylysov/pogreb v0.10.1 // indirect
 	github.com/alecthomas/jsonschema v0.0.0-20210818095345-1014919a589c
@@ -61,6 +61,64 @@ require (
 	golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d
 	golang.org/x/oauth2 v0.0.0-20210817223510-7df4dd6e12ab
 	golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf // indirect
+	golang.org/x/text v0.3.6
 	google.golang.org/appengine v1.6.7 // indirect
 	gopkg.in/yaml.v2 v2.4.0
 )
+
+require (
+	git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a // indirect
+	github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect
+	github.com/PuerkitoBio/goquery v1.6.0 // indirect
+	github.com/StackExchange/wmi v1.2.1 // indirect
+	github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect
+	github.com/andybalholm/cascadia v1.1.0 // indirect
+	github.com/antchfx/xpath v1.1.6 // indirect
+	github.com/cespare/xxhash v1.1.0 // indirect
+	github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dgraph-io/badger v1.6.2 // indirect
+	github.com/dgraph-io/ristretto v0.0.3 // indirect
+	github.com/dimchansky/utfbom v1.1.1 // indirect
+	github.com/dsnet/compress v0.0.1 // indirect
+	github.com/dustin/go-humanize v1.0.0 // indirect
+	github.com/eggsampler/acme/v3 v3.2.1 // indirect
+	github.com/fatih/structs v1.1.0 // indirect
+	github.com/go-ole/go-ole v1.2.5 // indirect
+	github.com/golang-jwt/jwt v3.2.1+incompatible // indirect
+	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/snappy v0.0.3 // indirect
+	github.com/google/go-querystring v1.0.0 // indirect
+	github.com/google/uuid v1.2.0 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.1 // indirect
+	github.com/hashicorp/go-retryablehttp v0.6.8 // indirect
+	github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect
+	github.com/itchyny/timefmt-go v0.1.3 // indirect
+	github.com/jasonlvhit/gocron v0.0.1 // indirect
+	github.com/karlseguin/ccache/v2 v2.0.8 // indirect
+	github.com/klauspost/compress v1.4.1 // indirect
+	github.com/klauspost/cpuid v1.2.0 // indirect
+	github.com/klauspost/pgzip v1.2.5 // indirect
+	github.com/mattn/go-isatty v0.0.13 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/projectdiscovery/iputil v0.0.0-20210429152401-c18a5408ca46 // indirect
+	github.com/projectdiscovery/mapcidr v0.0.6 // indirect
+	github.com/projectdiscovery/networkpolicy v0.0.1 // indirect
+	github.com/rivo/uniseg v0.2.0 // indirect
+	github.com/tklauser/go-sysconf v0.3.7 // indirect
+	github.com/tklauser/numcpus v0.2.3 // indirect
+	github.com/trivago/tgo v1.0.7 // indirect
+	github.com/ulikunitz/xz v0.5.10 // indirect
+	github.com/valyala/bytebufferpool v1.0.0 // indirect
+	github.com/yl2chen/cidranger v1.0.2 // indirect
+	github.com/ysmood/goob v0.3.0 // indirect
+	github.com/zclconf/go-cty v1.8.4 // indirect
+	go.etcd.io/bbolt v1.3.5 // indirect
+	golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
+	google.golang.org/protobuf v1.26.0 // indirect
+	gopkg.in/corvus-ch/zbase32.v1 v1.0.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+)

v2/go.sum

@@ -37,8 +37,8 @@ github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIo
 github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/Ice3man543/nvd v1.0.6 h1:QnCqnuYAA9tY2F38oNXp/kFV5fnYq+44mmcDFhKyawc=
+github.com/Ice3man543/nvd v1.0.8 h1:2CBEgOxyWAkQocnnmEMmRtVPWooPRvcuHFLWj48EM4c=
-github.com/Ice3man543/nvd v1.0.6/go.mod h1:0DxLJk6revOcJKiZxa2K+rNF/HO1zJO97lqQtXhXfSc=
+github.com/Ice3man543/nvd v1.0.8/go.mod h1:0DxLJk6revOcJKiZxa2K+rNF/HO1zJO97lqQtXhXfSc=
 github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f6s9Tn1Tt7/WTEg=
 github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
 github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic=
@@ -101,9 +101,6 @@ github.com/corpix/uarand v0.1.1 h1:RMr1TWc9F4n5jiPDzFHtmaUXLKLNUFK0SgCLo4BhX/U=
 github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
-github.com/daehee/nvd v1.0.4 h1:qC0kJ68vAYS86v8GwBORReBhyC5yUaUzsBokxjlsT98=
-github.com/daehee/nvd v1.0.4/go.mod h1:iBRJHIdIs+ylfq8630my2eMw8kwzH4Z7qsetjJZxCzs=
-github.com/dave/dst v0.26.2 h1:lnxLAKI3tx7MgLNVDirFCsDTlTG9nKTk7GcptKcWSwY=
 github.com/dave/dst v0.26.2/go.mod h1:UMDJuIRPfyUCC78eFuB+SV/WI8oDeyFDvM/JR6NI3IU=
 github.com/dave/gopackages v0.0.0-20170318123100-46e7023ec56e/go.mod h1:i00+b/gKdIDIxuLDFob7ustLAVqhsZRk2qVZrArELGQ=
 github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg=
@@ -512,7 +509,6 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
 golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.4.0 h1:8pl+sMODzuvGJkmj2W4kZihvVb5mKm8pB/X44PIQHv8=
 golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -680,7 +676,6 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc
 golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210101214203-2dba1e4ea05c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY=
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -795,7 +790,6 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-mvdan.cc/gofumpt v0.1.1 h1:bi/1aS/5W00E2ny5q65w9SnKpWEF/UIOqDYBILpo9rA=
 mvdan.cc/gofumpt v0.1.1/go.mod h1:yXG1r1WqZVKWbVRtBWKWX9+CxGYfA51nSomhM0woR48=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=

v2/pkg/catalog/config/config.go

@@ -28,7 +28,7 @@ type Config struct {
 const nucleiConfigFilename = ".templates-config.json"
 
 // Version is the current version of nuclei
-const Version = `2.5.1-dev`
+const Version = `2.5.2-dev`
 
 func getConfigDetails() (string, error) {
 	homeDir, err := os.UserHomeDir()

v2/pkg/protocols/http/request.go

@@ -411,6 +411,18 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ
 	dumpedResponse := dumpedResponseBuilder.Bytes()
 	redirectedResponse = bytes.ReplaceAll(redirectedResponse, dataOrig, data)
 
+	// Decode gbk response content-types
+	if contentType := resp.Header.Get("Content-Type"); contentType != "" && (strings.Contains(contentType, "gbk") || strings.Contains(contentType, "gb2312")) {
+		dumpedResponse, err = decodegbk(dumpedResponse)
+		if err != nil {
+			return errors.Wrap(err, "could not gbk decode")
+		}
+		redirectedResponse, err = decodegbk(redirectedResponse)
+		if err != nil {
+			return errors.Wrap(err, "could not gbk decode")
+		}
+	}
+
 	// Dump response - step 2 - replace gzip body with deflated one or with itself (NOP operation)
 	if r.options.Options.Debug || r.options.Options.DebugResponse {
 		gologger.Info().Msgf("[%s] Dumped HTTP response for %s\n\n", r.options.TemplateID, formedURL)

v2/pkg/protocols/http/utils.go

@@ -13,6 +13,8 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring"
 	"github.com/projectdiscovery/rawhttp"
+	"golang.org/x/text/encoding/simplifiedchinese"
+	"golang.org/x/text/transform"
 )
 
 // dumpResponseWithRedirectChain dumps a http response with the
@@ -122,3 +124,14 @@ func handleDecompression(resp *http.Response, bodyOrig []byte) (bodyDec []byte,
 	}
 	return bodyDec, nil
 }
+
+// decodegbk converts GBK to UTF-8
+func decodegbk(s []byte) ([]byte, error) {
+	I := bytes.NewReader(s)
+	O := transform.NewReader(I, simplifiedchinese.GBK.NewDecoder())
+	d, e := ioutil.ReadAll(O)
+	if e != nil {
+		return nil, e
+	}
+	return d, nil
+}