From 88dc16c910cc55a138fcaba153a44f285ed4f684 Mon Sep 17 00:00:00 2001 From: mzack Date: Thu, 14 Oct 2021 00:17:24 +0200 Subject: [PATCH 1/4] Improving gbk detection coverage --- v2/go.mod | 6 +++++- v2/go.sum | 11 +++++++++++ v2/pkg/protocols/http/request.go | 3 ++- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 471e12c5..c4a34a79 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -40,7 +40,7 @@ require ( github.com/projectdiscovery/rawhttp v0.0.7 github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a github.com/projectdiscovery/retryablehttp-go v1.0.2 - github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 + github.com/projectdiscovery/stringsutil v0.0.0-20211013053023-e7b2e104d80d github.com/projectdiscovery/yamldoc-go v1.0.2 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.3.0 @@ -73,6 +73,7 @@ require ( github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/andybalholm/cascadia v1.1.0 // indirect github.com/antchfx/xpath v1.1.6 // indirect + github.com/aymerick/douceur v0.2.0 // indirect github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect @@ -86,6 +87,7 @@ require ( github.com/golang/snappy v0.0.4 // indirect github.com/google/go-querystring v1.0.0 // indirect github.com/google/uuid v1.3.0 // indirect + github.com/gorilla/css v1.0.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.1 // indirect github.com/hashicorp/go-retryablehttp v0.6.8 // indirect github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect @@ -95,6 +97,7 @@ require ( github.com/klauspost/compress v1.13.6 // indirect github.com/klauspost/pgzip v1.2.5 // indirect github.com/mattn/go-isatty v0.0.13 // indirect + github.com/microcosm-cc/bluemonday v1.0.15 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -104,6 +107,7 @@ require ( github.com/projectdiscovery/mapcidr v0.0.8 // indirect github.com/projectdiscovery/networkpolicy v0.0.1 // indirect github.com/rivo/uniseg v0.2.0 // indirect + github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect github.com/tklauser/go-sysconf v0.3.7 // indirect github.com/tklauser/numcpus v0.2.3 // indirect github.com/trivago/tgo v1.0.7 // indirect diff --git a/v2/go.sum b/v2/go.sum index e85a7fce..557a11f8 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -96,11 +96,14 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= +github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= +github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= +github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -320,6 +323,8 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= +github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= @@ -468,6 +473,8 @@ github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go. github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= +github.com/microcosm-cc/bluemonday v1.0.15 h1:J4uN+qPng9rvkBZBoBb8YGR+ijuklIMpSOZZLjYpbeY= +github.com/microcosm-cc/bluemonday v1.0.15/go.mod h1:ZLvAzeakRwrGnzQEvstVzVt3ZpqOF2+sdFr0Om+ce30= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= @@ -609,6 +616,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mo github.com/projectdiscovery/stringsutil v0.0.0-20210823090203-2f5f137e8e1d/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xbL1/7h0k6HE3RzPdYk9W/8pUxESrGWewTaZdIB5Pes= github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= +github.com/projectdiscovery/stringsutil v0.0.0-20211013053023-e7b2e104d80d h1:YBYwsm8MrSp9t7mLehyqGwUKZWB08fG+YRePQRo5iFw= +github.com/projectdiscovery/stringsutil v0.0.0-20211013053023-e7b2e104d80d/go.mod h1:JK4F9ACNPgO+Lbm80khX2q1ABInBMbwIOmbsEE61Sn4= github.com/projectdiscovery/yamldoc-go v1.0.2 h1:SKb7PHgSOXm27Zci05ba0FxpyQiu6bGEiVMEcjCK1rQ= github.com/projectdiscovery/yamldoc-go v1.0.2/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -647,6 +656,8 @@ github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI= +github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 979e0eae..bd209d7f 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -429,7 +429,8 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate redirectedResponse = bytes.ReplaceAll(redirectedResponse, dataOrig, data) // Decode gbk response content-types - if contentType := strings.ToLower(resp.Header.Get("Content-Type")); contentType != "" && (strings.Contains(contentType, "gbk") || strings.Contains(contentType, "gb2312")) { + // gb18030 supersedes gb2312 + if contentType := strings.ToLower(resp.Header.Get("Content-Type")); stringsutil.ContainsAny(contentType, "gbk", "gb2312", "gb18030") { dumpedResponse, err = decodegbk(dumpedResponse) if err != nil { return errors.Wrap(err, "could not gbk decode") From e7d632a726a66bfac93bb4b859376d3d9dcba5d0 Mon Sep 17 00:00:00 2001 From: mzack Date: Fri, 15 Oct 2021 18:17:00 +0200 Subject: [PATCH 2/4] moving gbk detection code into helper function --- v2/pkg/protocols/http/request.go | 2 +- v2/pkg/protocols/http/utils.go | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index bd209d7f..c8781c19 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -430,7 +430,7 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate // Decode gbk response content-types // gb18030 supersedes gb2312 - if contentType := strings.ToLower(resp.Header.Get("Content-Type")); stringsutil.ContainsAny(contentType, "gbk", "gb2312", "gb18030") { + if isContentTypeGbk(resp.Header.Get("Content-Type")) { dumpedResponse, err = decodegbk(dumpedResponse) if err != nil { return errors.Wrap(err, "could not gbk decode") diff --git a/v2/pkg/protocols/http/utils.go b/v2/pkg/protocols/http/utils.go index bb12b5a6..89c35a23 100644 --- a/v2/pkg/protocols/http/utils.go +++ b/v2/pkg/protocols/http/utils.go @@ -13,6 +13,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring" "github.com/projectdiscovery/rawhttp" + "github.com/projectdiscovery/stringsutil" "golang.org/x/text/encoding/simplifiedchinese" "golang.org/x/text/transform" ) @@ -135,3 +136,9 @@ func decodegbk(s []byte) ([]byte, error) { } return d, nil } + +// isContentTypeGbk checks if the content-type header is gbk +func isContentTypeGbk(contentType string) bool { + contentType = strings.ToLower(contentType) + return stringsutil.ContainsAny(contentType, "gbk", "gb2312", "gb18030") +} From 662fb6315638c7477c0f077ea16efb3580265b86 Mon Sep 17 00:00:00 2001 From: mzack Date: Sat, 16 Oct 2021 00:17:33 +0200 Subject: [PATCH 3/4] adding missing decoding on response body --- v2/pkg/protocols/http/request.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index c8781c19..6f8b187d 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -439,6 +439,12 @@ func (request *Request) executeRequest(reqURL string, generatedRequest *generate if err != nil { return errors.Wrap(err, "could not gbk decode") } + + // the uncompressed body needs to be decoded to standard utf8 + data, err = decodegbk(data) + if err != nil { + return errors.Wrap(err, "could not gbk decode") + } } // if nuclei-project is enabled store the response if not previously done From b6104527e15ae024fd77a3a89e201b2d58ba7f0d Mon Sep 17 00:00:00 2001 From: mzack Date: Sat, 16 Oct 2021 01:05:13 +0200 Subject: [PATCH 4/4] replacing go get with go install --- v2/cmd/functional-test/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/cmd/functional-test/run.sh b/v2/cmd/functional-test/run.sh index 030b2544..8f6c635c 100644 --- a/v2/cmd/functional-test/run.sh +++ b/v2/cmd/functional-test/run.sh @@ -7,7 +7,7 @@ echo 'Building Nuclei binary from current branch' go build -o nuclei_dev ../nuclei echo 'Installing latest release of nuclei' -GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei +GO111MODULE=on go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei echo 'Starting Nuclei functional test' ./functional-test -main nuclei -dev ./nuclei_dev -testcases testcases.txt \ No newline at end of file