From 038b4aa5c223f9f65624504cd160a03e82e9259c Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 2 Feb 2021 12:10:47 +0530 Subject: [PATCH 1/4] Added jira+github+gitlab issue tracker integration to nuclei --- v2/cmd/nuclei/main.go | 2 + v2/go.mod | 5 + v2/go.sum | 26 ++++ v2/internal/runner/runner.go | 25 +++- v2/internal/runner/templates.go | 13 +- v2/pkg/output/format_json.go | 3 - v2/pkg/protocols/common/clusterer/executer.go | 6 + v2/pkg/protocols/common/executer/executer.go | 5 + v2/pkg/protocols/dns/operators.go | 2 + v2/pkg/protocols/file/operators.go | 3 + v2/pkg/protocols/http/operators.go | 14 +- v2/pkg/protocols/network/operators.go | 3 + v2/pkg/protocols/protocols.go | 3 + v2/pkg/reporting/issues/dedupe/dedupe.go | 101 ++++++++++++++ v2/pkg/reporting/issues/dedupe/dedupe_test.go | 31 +++++ v2/pkg/reporting/issues/format/format.go | 96 +++++++++++++ v2/pkg/reporting/issues/github/github.go | 69 ++++++++++ v2/pkg/reporting/issues/gitlab/gitlab.go | 59 ++++++++ v2/pkg/reporting/issues/issues.go | 88 ++++++++++++ v2/pkg/reporting/issues/jira/jira.go | 128 ++++++++++++++++++ v2/pkg/types/types.go | 5 + 21 files changed, 662 insertions(+), 25 deletions(-) create mode 100644 v2/pkg/reporting/issues/dedupe/dedupe.go create mode 100644 v2/pkg/reporting/issues/dedupe/dedupe_test.go create mode 100644 v2/pkg/reporting/issues/format/format.go create mode 100644 v2/pkg/reporting/issues/github/github.go create mode 100644 v2/pkg/reporting/issues/gitlab/gitlab.go create mode 100644 v2/pkg/reporting/issues/issues.go create mode 100644 v2/pkg/reporting/issues/jira/jira.go diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index a4436a58..354f3030 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -74,6 +74,8 @@ based on templates offering massive extensibility and ease of use.`) set.BoolVarP(&options.NoMeta, "no-meta", "nm", false, "Don't display metadata for the matches") set.BoolVarP(&options.TemplatesVersion, "templates-version", "tv", false, "Shows the installed nuclei-templates version") set.StringVarP(&options.BurpCollaboratorBiid, "burp-collaborator-biid", "biid", "", "Burp Collaborator BIID") + set.StringVarP(&options.ReportingConfig, "reporting-config", "rc", "", "Nuclei Reporting Module configuration file") + set.StringVarP(&options.ReportingDirectory, "reporting-directory", "rd", "", "Nuclei Reporting Module cache directory for issue deduplication") _ = set.Parse() if cfgFile != "" { diff --git a/v2/go.mod b/v2/go.mod index 47d631a4..2d2b05f8 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -4,11 +4,13 @@ go 1.15 require ( github.com/Knetic/govaluate v3.0.0+incompatible + github.com/andygrunwald/go-jira v1.13.0 github.com/blang/semver v3.5.1+incompatible github.com/corpix/uarand v0.1.1 github.com/golang/protobuf v1.4.3 // indirect github.com/golang/snappy v0.0.2 // indirect github.com/google/go-cmp v0.5.2 // indirect + github.com/google/go-github v17.0.0+incompatible github.com/google/go-github/v32 v32.1.0 github.com/gorilla/mux v1.8.0 github.com/json-iterator/go v1.1.10 @@ -36,11 +38,14 @@ require ( github.com/spaolacci/murmur3 v1.1.0 github.com/spf13/cast v1.3.1 github.com/stretchr/testify v1.7.0 + github.com/syndtr/goleveldb v1.0.0 + github.com/xanzy/go-gitlab v0.42.0 go.uber.org/atomic v1.7.0 go.uber.org/multierr v1.6.0 go.uber.org/ratelimit v0.1.0 golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad // indirect golang.org/x/net v0.0.0-20210119194325-5f4716e94777 + golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 // indirect golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect diff --git a/v2/go.sum b/v2/go.sum index ef56f76a..008b83bc 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -4,6 +4,8 @@ github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8L github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic= github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= +github.com/andygrunwald/go-jira v1.13.0 h1:vvIImGgX32bHfoiyUwkNo+/YrPnRczNarvhLOncP6dE= +github.com/andygrunwald/go-jira v1.13.0/go.mod h1:jYi4kFDbRPZTJdJOVJO4mpMMIwdB+rcZwSO58DzPd2I= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -14,9 +16,12 @@ github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/structs v1.0.0 h1:BrX964Rv5uQ3wwS+KRUAJCBBw5PQmgJfJ6v4yly5QwU= +github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -38,12 +43,18 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI= +github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-retryablehttp v0.6.4 h1:BbgctKO892xEyOXnGiaAwIoSq1QZ/SS4AhjoAh9DnfY= +github.com/hashicorp/go-retryablehttp v0.6.4/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= @@ -71,6 +82,7 @@ github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -94,9 +106,14 @@ github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= +github.com/trivago/tgo v1.0.1 h1:bxatjJIXNIpV18bucU4Uk/LaoxvxuOlp/oowRHyncLQ= +github.com/trivago/tgo v1.0.1/go.mod h1:w4dpD+3tzNIIiIfkWWa85w5/B77tlvdZckQ+6PkFnhc= +github.com/xanzy/go-gitlab v0.42.0 h1:daNdMFnw2FG+lDRBcX+YLnKbqIKMdefVyVztMHwsFhk= +github.com/xanzy/go-gitlab v0.42.0/go.mod h1:sPLojNBn68fMUWSxIJtdVVIP8uSBYqesTfDUseX11Ug= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= @@ -104,6 +121,7 @@ go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9i go.uber.org/ratelimit v0.1.0 h1:U2AruXqeTb4Eh9sYQSTrMhH8Cb7M0Ian2ibBOnBcnAw= go.uber.org/ratelimit v0.1.0/go.mod h1:2X8KaoNd1J0lZV+PxJk/5+DGbO/tpwLR1m++a7FnB/Y= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY= @@ -117,6 +135,7 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181108082009-03003ca0c849/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -124,7 +143,10 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288 h1:JIqe8uIcRBHXDQVvZtHwp80ai3Lw3IJAeJEs55Dc1W0= +golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -145,6 +167,8 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -157,6 +181,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index f0c84562..0815a479 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -17,6 +17,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/projectfile" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/clusterer" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues" "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/remeh/sizedwaitgroup" @@ -36,6 +37,7 @@ type Runner struct { catalogue *catalogue.Catalogue progress *progress.Progress colorizer aurora.Aurora + issuesClient *issues.Client severityColors *colorizer.Colorizer ratelimiter ratelimit.Limiter } @@ -54,6 +56,13 @@ func New(options *types.Options) (*Runner, error) { } runner.catalogue = catalogue.New(runner.options.TemplatesDirectory) + if options.ReportingConfig != "" { + if client, err := issues.New(options.ReportingConfig, options.ReportingDirectory); err != nil { + gologger.Fatal().Msgf("Could not create issue reporting client: %s\n", err) + } else { + runner.issuesClient = client + } + } // output coloring useColor := !options.NoColor runner.colorizer = aurora.NewAurora(useColor) @@ -202,12 +211,13 @@ func (r *Runner) RunEnumeration() { } executerOpts := &protocols.ExecuterOptions{ - Output: r.output, - Options: r.options, - Progress: r.progress, - Catalogue: r.catalogue, - RateLimiter: r.ratelimiter, - ProjectFile: r.projectFile, + Output: r.output, + Options: r.options, + Progress: r.progress, + Catalogue: r.catalogue, + IssuesClient: r.issuesClient, + RateLimiter: r.ratelimiter, + ProjectFile: r.projectFile, } // pre-parse all the templates, apply filters finalTemplates := []*templates.Template{} @@ -292,6 +302,9 @@ func (r *Runner) RunEnumeration() { r.progress.Stop() } + if r.issuesClient != nil { + r.issuesClient.Close() + } if !results.Load() { if r.output != nil { r.output.Close() diff --git a/v2/internal/runner/templates.go b/v2/internal/runner/templates.go index 15620d11..d00cf6e4 100644 --- a/v2/internal/runner/templates.go +++ b/v2/internal/runner/templates.go @@ -43,12 +43,13 @@ func (r *Runner) getParsedTemplatesFor(templatePaths []string, severities []stri // parseTemplateFile returns the parsed template file func (r *Runner) parseTemplateFile(file string) (*templates.Template, error) { executerOpts := &protocols.ExecuterOptions{ - Output: r.output, - Options: r.options, - Progress: r.progress, - Catalogue: r.catalogue, - RateLimiter: r.ratelimiter, - ProjectFile: r.projectFile, + Output: r.output, + Options: r.options, + Progress: r.progress, + Catalogue: r.catalogue, + RateLimiter: r.ratelimiter, + IssuesClient: r.issuesClient, + ProjectFile: r.projectFile, } template, err := templates.Parse(file, executerOpts) if err != nil { diff --git a/v2/pkg/output/format_json.go b/v2/pkg/output/format_json.go index 385bb616..814bab1f 100644 --- a/v2/pkg/output/format_json.go +++ b/v2/pkg/output/format_json.go @@ -1,13 +1,10 @@ package output import ( - "time" - jsoniter "github.com/json-iterator/go" ) // formatJSON formats the output for json based formatting func (w *StandardWriter) formatJSON(output *ResultEvent) ([]byte, error) { - output.Timestamp = time.Now() return jsoniter.Marshal(output) } diff --git a/v2/pkg/protocols/common/clusterer/executer.go b/v2/pkg/protocols/common/clusterer/executer.go index cef218a3..5f518071 100644 --- a/v2/pkg/protocols/common/clusterer/executer.go +++ b/v2/pkg/protocols/common/clusterer/executer.go @@ -1,6 +1,7 @@ package clusterer import ( + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" @@ -70,6 +71,11 @@ func (e *Executer) Execute(input string) (bool, error) { event.Results = e.requests.MakeResultEvent(event) results = true for _, r := range event.Results { + if e.options.IssuesClient != nil { + if err := e.options.IssuesClient.CreateIssue(r); err != nil { + gologger.Warning().Msgf("Could not create issue on tracker: %s", err) + } + } e.options.Output.Write(r) e.options.Progress.IncrementMatched() } diff --git a/v2/pkg/protocols/common/executer/executer.go b/v2/pkg/protocols/common/executer/executer.go index 50e0c3b7..caf52ffa 100644 --- a/v2/pkg/protocols/common/executer/executer.go +++ b/v2/pkg/protocols/common/executer/executer.go @@ -64,6 +64,11 @@ func (e *Executer) Execute(input string) (bool, error) { return } for _, result := range event.Results { + if e.options.IssuesClient != nil { + if err := e.options.IssuesClient.CreateIssue(result); err != nil { + gologger.Warning().Msgf("Could not create issue on tracker: %s", err) + } + } results = true e.options.Output.Write(result) e.options.Progress.IncrementMatched() diff --git a/v2/pkg/protocols/dns/operators.go b/v2/pkg/protocols/dns/operators.go index ed9f812e..49bff20a 100644 --- a/v2/pkg/protocols/dns/operators.go +++ b/v2/pkg/protocols/dns/operators.go @@ -2,6 +2,7 @@ package dns import ( "bytes" + "time" "github.com/miekg/dns" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" @@ -154,6 +155,7 @@ func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *out Host: wrapped.InternalEvent["host"].(string), Matched: wrapped.InternalEvent["matched"].(string), ExtractedResults: wrapped.OperatorsResult.OutputExtracts, + Timestamp: time.Now(), } if r.options.Options.JSONRequests { data.Request = wrapped.InternalEvent["request"].(string) diff --git a/v2/pkg/protocols/file/operators.go b/v2/pkg/protocols/file/operators.go index 58b06689..1d00eb00 100644 --- a/v2/pkg/protocols/file/operators.go +++ b/v2/pkg/protocols/file/operators.go @@ -1,6 +1,8 @@ package file import ( + "time" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" @@ -113,6 +115,7 @@ func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *out Host: wrapped.InternalEvent["host"].(string), Matched: wrapped.InternalEvent["matched"].(string), ExtractedResults: wrapped.OperatorsResult.OutputExtracts, + Timestamp: time.Now(), } if r.options.Options.JSONRequests { data.Response = wrapped.InternalEvent["raw"].(string) diff --git a/v2/pkg/protocols/http/operators.go b/v2/pkg/protocols/http/operators.go index 5b59a772..122efdce 100644 --- a/v2/pkg/protocols/http/operators.go +++ b/v2/pkg/protocols/http/operators.go @@ -2,7 +2,6 @@ package http import ( "net/http" - "net/http/httputil" "strings" "time" @@ -86,10 +85,9 @@ func (r *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, r data["host"] = host data["matched"] = matched - if r.options.Options.JSONRequests { - data["request"] = rawReq - data["response"] = rawResp - } + data["request"] = rawReq + data["raw"] = rawResp + data["response"] = rawResp data["content_length"] = resp.ContentLength data["status_code"] = resp.StatusCode @@ -103,11 +101,6 @@ func (r *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, r data[k] = strings.Join(v, " ") } data["all_headers"] = headers - - if r, err := httputil.DumpResponse(resp, true); err == nil { - rawString := string(r) - data["raw"] = rawString - } data["duration"] = duration.Seconds() data["template-id"] = r.options.TemplateID data["template-info"] = r.options.TemplateInfo @@ -152,6 +145,7 @@ func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *out Metadata: wrapped.OperatorsResult.PayloadValues, ExtractedResults: wrapped.OperatorsResult.OutputExtracts, IP: wrapped.InternalEvent["ip"].(string), + Timestamp: time.Now(), } if r.options.Options.JSONRequests { data.Request = wrapped.InternalEvent["request"].(string) diff --git a/v2/pkg/protocols/network/operators.go b/v2/pkg/protocols/network/operators.go index 2638228e..ffb6974e 100644 --- a/v2/pkg/protocols/network/operators.go +++ b/v2/pkg/protocols/network/operators.go @@ -1,6 +1,8 @@ package network import ( + "time" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" @@ -117,6 +119,7 @@ func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *out Matched: wrapped.InternalEvent["matched"].(string), ExtractedResults: wrapped.OperatorsResult.OutputExtracts, IP: wrapped.InternalEvent["ip"].(string), + Timestamp: time.Now(), } if r.options.Options.JSONRequests { data.Request = wrapped.InternalEvent["request"].(string) diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 5a40a54d..f2417dc3 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -7,6 +7,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/projectfile" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues" "github.com/projectdiscovery/nuclei/v2/pkg/types" "go.uber.org/ratelimit" ) @@ -35,6 +36,8 @@ type ExecuterOptions struct { Output output.Writer // Options contains configuration options for the executer. Options *types.Options + // IssuesClient is a client for nuclei issue tracker reporting + IssuesClient *issues.Client // Progress is a progress client for scan reporting Progress *progress.Progress // RateLimiter is a rate-limiter for limiting sent number of requests. diff --git a/v2/pkg/reporting/issues/dedupe/dedupe.go b/v2/pkg/reporting/issues/dedupe/dedupe.go new file mode 100644 index 00000000..7df95c2c --- /dev/null +++ b/v2/pkg/reporting/issues/dedupe/dedupe.go @@ -0,0 +1,101 @@ +// Package dedupe implements deduplication layer for nuclei-generated +// issues. +// +// The layer can be persisted to leveldb based storage for further use. +package dedupe + +import ( + "crypto/sha1" + "path" + "unsafe" + + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/types" + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/errors" +) + +// Storage is a duplicate detecting storage for nuclei scan events. +type Storage struct { + storage *leveldb.DB +} + +const storageFilename = "nuclei-events.db" + +// New creates a new duplicate detecting storage for nuclei scan events. +func New(folder string) (*Storage, error) { + path := path.Join(folder, storageFilename) + + db, err := leveldb.OpenFile(path, nil) + if err != nil { + if !errors.IsCorrupted(err) { + return nil, err + } + + // If the metadata is corrupted, try to recover + db, err = leveldb.RecoverFile(path, nil) + if err != nil { + return nil, err + } + } + return &Storage{storage: db}, nil +} + +// Close closes the storage for further operations +func (s *Storage) Close() { + s.storage.Close() +} + +// Index indexes an item in storage and returns true if the item +// was unique. +func (s *Storage) Index(result *output.ResultEvent) (bool, error) { + hasher := sha1.New() + if result.TemplateID != "" { + hasher.Write(unsafeToBytes(result.TemplateID)) + } + if result.MatcherName != "" { + hasher.Write(unsafeToBytes(result.MatcherName)) + } + if result.ExtractorName != "" { + hasher.Write(unsafeToBytes(result.ExtractorName)) + } + if result.Type != "" { + hasher.Write(unsafeToBytes(result.Type)) + } + if result.Host != "" { + hasher.Write(unsafeToBytes(result.Host)) + } + if result.Matched != "" { + hasher.Write(unsafeToBytes(result.Matched)) + } + for _, v := range result.ExtractedResults { + hasher.Write(unsafeToBytes(v)) + } + for k, v := range result.Metadata { + hasher.Write(unsafeToBytes(k)) + hasher.Write(unsafeToBytes(types.ToString(v))) + } + if result.Request != "" { + hasher.Write(unsafeToBytes(result.Request)) // Very dumb, change later. + } + hash := hasher.Sum(nil) + + exists, err := s.storage.Has(hash, nil) + if err != nil { + // if we have an error, return with it but mark it as true + // since we don't want to loose an issue considering it a dupe. + return true, err + } + if !exists { + return true, s.storage.Put(hash, nil, nil) + } + return false, err +} + +// unsafeToBytes converts a string to byte slice and does it with +// zero allocations. +// +// Reference - https://stackoverflow.com/questions/59209493/how-to-use-unsafe-get-a-byte-slice-from-a-string-without-memory-copy +func unsafeToBytes(data string) []byte { + return *(*[]byte)(unsafe.Pointer(&data)) +} diff --git a/v2/pkg/reporting/issues/dedupe/dedupe_test.go b/v2/pkg/reporting/issues/dedupe/dedupe_test.go new file mode 100644 index 00000000..5d77f35c --- /dev/null +++ b/v2/pkg/reporting/issues/dedupe/dedupe_test.go @@ -0,0 +1,31 @@ +package dedupe + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/stretchr/testify/require" +) + +func TestDedupeDuplicates(t *testing.T) { + tempDir, err := ioutil.TempDir("", "nuclei") + require.Nil(t, err, "could not create temporary storage") + defer os.RemoveAll(tempDir) + + storage, err := New(tempDir) + require.Nil(t, err, "could not create duplicate storage") + + tests := []*output.ResultEvent{ + {TemplateID: "test", Host: "https://example.com"}, + {TemplateID: "test", Host: "https://example.com"}, + } + first, err := storage.Index(tests[0]) + require.Nil(t, err, "could not index item") + require.True(t, first, "could not index valid item") + + second, err := storage.Index(tests[1]) + require.Nil(t, err, "could not index item") + require.False(t, second, "could index duplicate item") +} diff --git a/v2/pkg/reporting/issues/format/format.go b/v2/pkg/reporting/issues/format/format.go new file mode 100644 index 00000000..51955434 --- /dev/null +++ b/v2/pkg/reporting/issues/format/format.go @@ -0,0 +1,96 @@ +package format + +import ( + "bytes" + "fmt" + "strings" + + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) + +// Summary returns a formatted built one line summary of the event +func Summary(output *output.ResultEvent) string { + template := GetMatchedTemplate(output) + + builder := &strings.Builder{} + builder.WriteString("[") + builder.WriteString(template) + builder.WriteString("] [") + builder.WriteString(output.Info["severity"]) + builder.WriteString("] ") + builder.WriteString(output.Info["name"]) + builder.WriteString(" found on ") + builder.WriteString(output.Host) + data := builder.String() + return data +} + +// MarkdownDescription formats a short description of the generated +// event by the nuclei scanner in Markdown format. +func MarkdownDescription(output *output.ResultEvent) string { + template := GetMatchedTemplate(output) + builder := &bytes.Buffer{} + builder.WriteString("**Details**: **") + builder.WriteString(template) + builder.WriteString("** ") + builder.WriteString(" matched at ") + builder.WriteString(output.Host) + builder.WriteString("\n\n**Protocol**: ") + builder.WriteString(strings.ToUpper(output.Type)) + builder.WriteString("\n\n**Full URL**: ") + builder.WriteString(output.Matched) + builder.WriteString("\n\n**Timestamp**: ") + builder.WriteString(output.Timestamp.Format("Mon Jan 2 15:04:05 -0700 MST 2006")) + builder.WriteString("\n\n**Template Information**\n\n| Key | Value |\n|---|---|\n") + for k, v := range output.Info { + builder.WriteString(fmt.Sprintf("| %s | %s |\n", k, v)) + } + builder.WriteString("\n**Request**\n\n```\n") + builder.WriteString(output.Request) + builder.WriteString("\n```\n\n**Response**\n\n```\n") + builder.WriteString(output.Response) + builder.WriteString("\n```\n\n") + + if len(output.ExtractedResults) > 0 || len(output.Metadata) > 0 { + builder.WriteString("**Extra Information**\n\n") + if len(output.ExtractedResults) > 0 { + builder.WriteString("**Extracted results**:\n\n") + for _, v := range output.ExtractedResults { + builder.WriteString("- ") + builder.WriteString(v) + builder.WriteString("\n") + } + builder.WriteString("\n") + } + if len(output.Metadata) > 0 { + builder.WriteString("**Metadata**:\n\n") + for k, v := range output.Metadata { + builder.WriteString("- ") + builder.WriteString(k) + builder.WriteString(": ") + builder.WriteString(types.ToString(v)) + builder.WriteString("\n") + } + builder.WriteString("\n") + } + } + data := builder.String() + return data +} + +// GetMatchedTemplate returns the matched template from a result event +func GetMatchedTemplate(output *output.ResultEvent) string { + builder := &strings.Builder{} + builder.WriteString(output.TemplateID) + if output.MatcherName != "" { + builder.WriteString(":") + builder.WriteString(output.MatcherName) + } + if output.ExtractorName != "" { + builder.WriteString(":") + builder.WriteString(output.ExtractorName) + } + template := builder.String() + return template +} diff --git a/v2/pkg/reporting/issues/github/github.go b/v2/pkg/reporting/issues/github/github.go new file mode 100644 index 00000000..a7443634 --- /dev/null +++ b/v2/pkg/reporting/issues/github/github.go @@ -0,0 +1,69 @@ +package github + +import ( + "context" + "net/url" + + "golang.org/x/oauth2" + + "github.com/google/go-github/github" + "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues/format" +) + +// Integration is a client for a issue tracker integration +type Integration struct { + client *github.Client + options *Options +} + +// Options contains the configuration options for github issue tracker client +type Options struct { + // BaseURL is the optional self-hosted github application url + BaseURL string `yaml:"base-url"` + // Username is the username of the github user + Username string `yaml:"username"` + // Owner is the owner name of the repository for issues. + Owner string `yaml:"owner"` + // Token is the token for github account. + Token string `yaml:"token"` + // ProjectName is the name of the repository. + ProjectName string `yaml:"project-name"` + // IssueLabel is the label of the created issue type + IssueLabel string `yaml:"issue-label"` +} + +// New creates a new issue tracker integration client based on options. +func New(options *Options) (*Integration, error) { + ctx := context.Background() + ts := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: options.Token}, + ) + tc := oauth2.NewClient(ctx, ts) + + client := github.NewClient(tc) + if options.BaseURL != "" { + parsed, err := url.Parse(options.BaseURL) + if err != nil { + return nil, errors.Wrap(err, "could not parse custom baseurl") + } + client.BaseURL = parsed + } + return &Integration{client: client, options: options}, nil +} + +// CreateIssue creates an issue in the tracker +func (i *Integration) CreateIssue(event *output.ResultEvent) error { + summary := format.Summary(event) + description := format.MarkdownDescription(event) + + req := &github.IssueRequest{ + Title: &summary, + Body: &description, + Labels: &[]string{i.options.IssueLabel}, + Assignees: &[]string{i.options.Username}, + } + _, _, err := i.client.Issues.Create(context.Background(), i.options.Owner, i.options.ProjectName, req) + return err +} diff --git a/v2/pkg/reporting/issues/gitlab/gitlab.go b/v2/pkg/reporting/issues/gitlab/gitlab.go new file mode 100644 index 00000000..895507c8 --- /dev/null +++ b/v2/pkg/reporting/issues/gitlab/gitlab.go @@ -0,0 +1,59 @@ +package gitlab + +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues/format" + "github.com/xanzy/go-gitlab" +) + +// Integration is a client for a issue tracker integration +type Integration struct { + client *gitlab.Client + userID int + options *Options +} + +// Options contains the configuration options for gitlab issue tracker client +type Options struct { + // BaseURL is the optional self-hosted gitlab application url + BaseURL string `yaml:"base-url"` + // Username is the username of the gitlab user + Username string `yaml:"username"` + // Token is the token for gitlab account. + Token string `yaml:"token"` + // ProjectName is the name of the repository. + ProjectName string `yaml:"project-name"` + // IssueLabel is the label of the created issue type + IssueLabel string `yaml:"issue-label"` +} + +// New creates a new issue tracker integration client based on options. +func New(options *Options) (*Integration, error) { + gitlabOpts := []gitlab.ClientOptionFunc{} + if options.BaseURL != "" { + gitlabOpts = append(gitlabOpts, gitlab.WithBaseURL(options.BaseURL)) + } + git, err := gitlab.NewClient(options.Token, gitlabOpts...) + if err != nil { + return nil, err + } + user, _, err := git.Users.CurrentUser() + if err != nil { + return nil, err + } + return &Integration{client: git, userID: user.ID, options: options}, nil +} + +// CreateIssue creates an issue in the tracker +func (i *Integration) CreateIssue(event *output.ResultEvent) error { + summary := format.Summary(event) + description := format.MarkdownDescription(event) + + _, _, err := i.client.Issues.CreateIssue(i.options.ProjectName, &gitlab.CreateIssueOptions{ + Title: &summary, + Description: &description, + Labels: gitlab.Labels{i.options.IssueLabel}, + AssigneeIDs: []int{i.userID}, + }) + return err +} diff --git a/v2/pkg/reporting/issues/issues.go b/v2/pkg/reporting/issues/issues.go new file mode 100644 index 00000000..554e1192 --- /dev/null +++ b/v2/pkg/reporting/issues/issues.go @@ -0,0 +1,88 @@ +package issues + +import ( + "os" + + "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues/dedupe" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues/github" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues/gitlab" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues/jira" + "gopkg.in/yaml.v2" +) + +// Options is a configuration file for nuclei reporting module +type Options struct { + // Github contains configuration options for Github Issue Tracker + Github *github.Options `yaml:"github"` + // Gitlab contains configuration options for Gitlab Issue Tracker + Gitlab *gitlab.Options `yaml:"gitlab"` + // Jira contains configuration options for Jira Issue Tracker + Jira *jira.Options `yaml:"jira"` +} + +// Tracker is an interface implemented by an issue tracker +type Tracker interface { + // CreateIssue creates an issue in the tracker + CreateIssue(event *output.ResultEvent) error +} + +// Client is a client for nuclei issue tracking module +type Client struct { + tracker Tracker + dedupe *dedupe.Storage +} + +// New creates a new nuclei issue tracker reporting client +func New(config, directory string) (*Client, error) { + file, err := os.Open(config) + if err != nil { + return nil, errors.Wrap(err, "could not open reporting config file") + } + defer file.Close() + + options := &Options{} + if err := yaml.NewDecoder(file).Decode(options); err != nil { + return nil, err + } + var tracker Tracker + if options.Github != nil { + tracker, err = github.New(options.Github) + } + if options.Gitlab != nil { + tracker, err = gitlab.New(options.Gitlab) + } + if options.Jira != nil { + tracker, err = jira.New(options.Jira) + } + if err != nil { + return nil, errors.Wrap(err, "could not create reporting client") + } + if tracker == nil { + return nil, errors.New("no issue tracker configuration found") + } + storage, err := dedupe.New(directory) + if err != nil { + return nil, err + } + return &Client{tracker: tracker, dedupe: storage}, nil +} + +// Close closes the issue tracker reporting client +func (c *Client) Close() { + c.dedupe.Close() +} + +// CreateIssue creates an issue in the tracker +func (c *Client) CreateIssue(event *output.ResultEvent) error { + found, err := c.dedupe.Index(event) + if err != nil { + c.tracker.CreateIssue(event) + return err + } + if found { + return c.tracker.CreateIssue(event) + } + return nil +} diff --git a/v2/pkg/reporting/issues/jira/jira.go b/v2/pkg/reporting/issues/jira/jira.go new file mode 100644 index 00000000..735da2cd --- /dev/null +++ b/v2/pkg/reporting/issues/jira/jira.go @@ -0,0 +1,128 @@ +package jira + +import ( + "bytes" + "fmt" + "io/ioutil" + "strings" + + jira "github.com/andygrunwald/go-jira" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues/format" + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) + +// Integration is a client for a issue tracker integration +type Integration struct { + jira *jira.Client + options *Options +} + +// Options contains the configuration options for jira client +type Options struct { + // URL is the URL of the jira server + URL string `yaml:"url"` + // AccountID is the accountID of the jira user. + AccountID string `yaml:"account-id"` + // Email is the email of the user for jira instance + Email string `yaml:"email"` + // Token is the token for jira instance. + Token string `yaml:"token"` + // ProjectName is the name of the project. + ProjectName string `yaml:"project-name"` + // IssueType is the name of the created issue type + IssueType string `yaml:"issue-type"` +} + +// New creates a new issue tracker integration client based on options. +func New(options *Options) (*Integration, error) { + tp := jira.BasicAuthTransport{ + Username: options.Email, + Password: options.Token, + } + jiraClient, err := jira.NewClient(tp.Client(), options.URL) + if err != nil { + return nil, err + } + return &Integration{jira: jiraClient, options: options}, nil +} + +// CreateIssue creates an issue in the tracker +func (i *Integration) CreateIssue(event *output.ResultEvent) error { + summary := format.Summary(event) + + issueData := &jira.Issue{ + Fields: &jira.IssueFields{ + Assignee: &jira.User{AccountID: i.options.AccountID}, + Reporter: &jira.User{AccountID: i.options.AccountID}, + Description: jiraFormatDescription(event), + Type: jira.IssueType{Name: i.options.IssueType}, + Project: jira.Project{Key: i.options.ProjectName}, + Summary: summary, + }, + } + _, resp, err := i.jira.Issue.Create(issueData) + if err != nil { + var data string + if resp != nil && resp.Body != nil { + d, _ := ioutil.ReadAll(resp.Body) + data = string(d) + } + return fmt.Errorf("%s => %s", err, data) + } + return nil +} + +// jiraFormatDescription formats a short description of the generated +// event by the nuclei scanner in Jira format. +func jiraFormatDescription(output *output.ResultEvent) string { + template := format.GetMatchedTemplate(output) + + builder := &bytes.Buffer{} + builder.WriteString("*Details*: *") + builder.WriteString(template) + builder.WriteString("* ") + builder.WriteString(" matched at ") + builder.WriteString(output.Host) + builder.WriteString("\n\n*Protocol*: ") + builder.WriteString(strings.ToUpper(output.Type)) + builder.WriteString("\n\n*Full URL*: ") + builder.WriteString(output.Matched) + builder.WriteString("\n\n*Timestamp*: ") + builder.WriteString(output.Timestamp.Format("Mon Jan 2 15:04:05 -0700 MST 2006")) + builder.WriteString("\n\n*Template Information*\n\n| Key | Value |\n") + for k, v := range output.Info { + builder.WriteString(fmt.Sprintf("| %s | %s |\n", k, v)) + } + builder.WriteString("\n*Request*\n\n{code}\n") + builder.WriteString(output.Request) + builder.WriteString("\n{code}\n\n*Response*\n\n{code}\n") + builder.WriteString(output.Response) + builder.WriteString("\n{code}\n\n") + + if len(output.ExtractedResults) > 0 || len(output.Metadata) > 0 { + builder.WriteString("*Extra Information*\n\n") + if len(output.ExtractedResults) > 0 { + builder.WriteString("*Extracted results*:\n\n") + for _, v := range output.ExtractedResults { + builder.WriteString("- ") + builder.WriteString(v) + builder.WriteString("\n") + } + builder.WriteString("\n") + } + if len(output.Metadata) > 0 { + builder.WriteString("*Metadata*:\n\n") + for k, v := range output.Metadata { + builder.WriteString("- ") + builder.WriteString(k) + builder.WriteString(": ") + builder.WriteString(types.ToString(v)) + builder.WriteString("\n") + } + builder.WriteString("\n") + } + } + data := builder.String() + return data +} diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index aa1c87cd..6ccb3b69 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -80,4 +80,9 @@ type Options struct { ExcludedTemplates goflags.StringSlice // CustomHeaders is the list of custom global headers to send with each request. CustomHeaders goflags.StringSlice + // ReportingConfig is the config file for nuclei reporting module + ReportingConfig string + // ReportingDirectory is the directory to store nuclei issue deduplication data + // for reporting in. + ReportingDirectory string } From 41e9aa21e77fecb12f41a04f9bfeed44d7d90576 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Tue, 2 Feb 2021 19:24:55 +0530 Subject: [PATCH 2/4] Misc --- v2/pkg/protocols/http/request.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 1f154460..095c2fa2 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -1,7 +1,6 @@ package http import ( - "bytes" "io" "io/ioutil" "net/http" @@ -219,7 +218,7 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynam dumpedRequest []byte fromcache bool ) - if r.options.Options.Debug || r.options.ProjectFile != nil || r.options.Options.DebugRequests { + if r.options.Options.Debug || r.options.ProjectFile != nil || r.options.Options.DebugRequests || r.options.Options.JSONRequests { dumpedRequest, err = dump(request, reqURL) if err != nil { return err @@ -283,7 +282,7 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynam duration := time.Since(timeStart) // Dump response - Step 1 - Decompression not yet handled var dumpedResponse []byte - if r.options.Options.Debug || r.options.Options.DebugResponse { + if r.options.Options.Debug || r.options.Options.DebugResponse || r.options.Options.JSONRequests { var dumpErr error dumpedResponse, dumpErr = httputil.DumpResponse(resp, true) if dumpErr != nil { @@ -302,7 +301,6 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynam // net/http doesn't automatically decompress the response body if an // encoding has been specified by the user in the request so in case we have to // manually do it. - dataOrig := data data, err = handleDecompression(request, data) if err != nil { return errors.Wrap(err, "could not decompress http body") @@ -310,7 +308,7 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynam // Dump response - step 2 - replace gzip body with deflated one or with itself (NOP operation) if r.options.Options.Debug || r.options.Options.DebugResponse { - dumpedResponse = bytes.ReplaceAll(dumpedResponse, dataOrig, data) + dumpedResponse = data gologger.Info().Msgf("[%s] Dumped HTTP response for %s\n\n", r.options.TemplateID, formedURL) gologger.Print().Msgf("%s", string(dumpedResponse)) } From 3fe6290eed6ff807d2dcc85fd480f9c11ab68d2d Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Sun, 7 Feb 2021 23:41:33 +0530 Subject: [PATCH 3/4] Misc fixes --- v2/cmd/nuclei/main.go | 2 +- v2/internal/runner/runner.go | 2 +- v2/pkg/reporting/issues/dedupe/dedupe.go | 31 ++++++++++++++++-------- v2/pkg/reporting/issues/format/format.go | 2 +- v2/pkg/reporting/issues/issues.go | 4 +-- v2/pkg/types/types.go | 5 ++-- 6 files changed, 28 insertions(+), 18 deletions(-) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 354f3030..5a24b8f1 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -75,7 +75,7 @@ based on templates offering massive extensibility and ease of use.`) set.BoolVarP(&options.TemplatesVersion, "templates-version", "tv", false, "Shows the installed nuclei-templates version") set.StringVarP(&options.BurpCollaboratorBiid, "burp-collaborator-biid", "biid", "", "Burp Collaborator BIID") set.StringVarP(&options.ReportingConfig, "reporting-config", "rc", "", "Nuclei Reporting Module configuration file") - set.StringVarP(&options.ReportingDirectory, "reporting-directory", "rd", "", "Nuclei Reporting Module cache directory for issue deduplication") + set.StringVarP(&options.ReportingDB, "report-db", "rdb", "", "Local Nuclei Reporting Database") _ = set.Parse() if cfgFile != "" { diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 0815a479..e5609d20 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -57,7 +57,7 @@ func New(options *types.Options) (*Runner, error) { runner.catalogue = catalogue.New(runner.options.TemplatesDirectory) if options.ReportingConfig != "" { - if client, err := issues.New(options.ReportingConfig, options.ReportingDirectory); err != nil { + if client, err := issues.New(options.ReportingConfig, options.ReportingDB); err != nil { gologger.Fatal().Msgf("Could not create issue reporting client: %s\n", err) } else { runner.issuesClient = client diff --git a/v2/pkg/reporting/issues/dedupe/dedupe.go b/v2/pkg/reporting/issues/dedupe/dedupe.go index 7df95c2c..967c06a5 100644 --- a/v2/pkg/reporting/issues/dedupe/dedupe.go +++ b/v2/pkg/reporting/issues/dedupe/dedupe.go @@ -6,7 +6,8 @@ package dedupe import ( "crypto/sha1" - "path" + "io/ioutil" + "os" "unsafe" "github.com/projectdiscovery/nuclei/v2/pkg/output" @@ -17,33 +18,46 @@ import ( // Storage is a duplicate detecting storage for nuclei scan events. type Storage struct { - storage *leveldb.DB + temporary string + storage *leveldb.DB } const storageFilename = "nuclei-events.db" // New creates a new duplicate detecting storage for nuclei scan events. -func New(folder string) (*Storage, error) { - path := path.Join(folder, storageFilename) +func New(dbPath string) (*Storage, error) { + storage := &Storage{} - db, err := leveldb.OpenFile(path, nil) + var err error + if dbPath == "" { + dbPath, err = ioutil.TempDir("", "nuclei-report-*") + storage.temporary = dbPath + } + if err != nil { + return nil, err + } + + storage.storage, err = leveldb.OpenFile(dbPath, nil) if err != nil { if !errors.IsCorrupted(err) { return nil, err } // If the metadata is corrupted, try to recover - db, err = leveldb.RecoverFile(path, nil) + storage.storage, err = leveldb.RecoverFile(dbPath, nil) if err != nil { return nil, err } } - return &Storage{storage: db}, nil + return storage, nil } // Close closes the storage for further operations func (s *Storage) Close() { s.storage.Close() + if s.temporary != "" { + os.RemoveAll(s.temporary) + } } // Index indexes an item in storage and returns true if the item @@ -75,9 +89,6 @@ func (s *Storage) Index(result *output.ResultEvent) (bool, error) { hasher.Write(unsafeToBytes(k)) hasher.Write(unsafeToBytes(types.ToString(v))) } - if result.Request != "" { - hasher.Write(unsafeToBytes(result.Request)) // Very dumb, change later. - } hash := hasher.Sum(nil) exists, err := s.storage.Has(hash, nil) diff --git a/v2/pkg/reporting/issues/format/format.go b/v2/pkg/reporting/issues/format/format.go index 51955434..ff19cc80 100644 --- a/v2/pkg/reporting/issues/format/format.go +++ b/v2/pkg/reporting/issues/format/format.go @@ -48,7 +48,7 @@ func MarkdownDescription(output *output.ResultEvent) string { } builder.WriteString("\n**Request**\n\n```\n") builder.WriteString(output.Request) - builder.WriteString("\n```\n\n**Response**\n\n```\n") + builder.WriteString("\n```\n\n
**Response**\n\n```\n") builder.WriteString(output.Response) builder.WriteString("\n```\n\n") diff --git a/v2/pkg/reporting/issues/issues.go b/v2/pkg/reporting/issues/issues.go index 554e1192..ce1af3f4 100644 --- a/v2/pkg/reporting/issues/issues.go +++ b/v2/pkg/reporting/issues/issues.go @@ -35,7 +35,7 @@ type Client struct { } // New creates a new nuclei issue tracker reporting client -func New(config, directory string) (*Client, error) { +func New(config, db string) (*Client, error) { file, err := os.Open(config) if err != nil { return nil, errors.Wrap(err, "could not open reporting config file") @@ -62,7 +62,7 @@ func New(config, directory string) (*Client, error) { if tracker == nil { return nil, errors.New("no issue tracker configuration found") } - storage, err := dedupe.New(directory) + storage, err := dedupe.New(db) if err != nil { return nil, err } diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 6ccb3b69..fac074ef 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -80,9 +80,8 @@ type Options struct { ExcludedTemplates goflags.StringSlice // CustomHeaders is the list of custom global headers to send with each request. CustomHeaders goflags.StringSlice + // ReportingDB is the db for report storage as well as deduplication + ReportingDB string // ReportingConfig is the config file for nuclei reporting module ReportingConfig string - // ReportingDirectory is the directory to store nuclei issue deduplication data - // for reporting in. - ReportingDirectory string } From 4c7f4405b138085fd4971b089120cf3ebe8c87bd Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Mon, 8 Feb 2021 01:43:51 +0530 Subject: [PATCH 4/4] Bug fixes, added filters --- v2/pkg/protocols/http/request.go | 15 +++---- v2/pkg/reporting/issues/issues.go | 75 ++++++++++++++++++++++++++++++- 2 files changed, 80 insertions(+), 10 deletions(-) diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 095c2fa2..005a066c 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -213,17 +213,14 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynam r.setCustomHeaders(request) var ( - resp *http.Response - err error - dumpedRequest []byte - fromcache bool + resp *http.Response + fromcache bool ) - if r.options.Options.Debug || r.options.ProjectFile != nil || r.options.Options.DebugRequests || r.options.Options.JSONRequests { - dumpedRequest, err = dump(request, reqURL) - if err != nil { - return err - } + dumpedRequest, err := dump(request, reqURL) + if err != nil { + return err } + if r.options.Options.Debug || r.options.Options.DebugRequests { gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", r.options.TemplateID, reqURL) gologger.Print().Msgf("%s", string(dumpedRequest)) diff --git a/v2/pkg/reporting/issues/issues.go b/v2/pkg/reporting/issues/issues.go index ce1af3f4..6ed66fec 100644 --- a/v2/pkg/reporting/issues/issues.go +++ b/v2/pkg/reporting/issues/issues.go @@ -2,6 +2,7 @@ package issues import ( "os" + "strings" "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/output" @@ -14,6 +15,10 @@ import ( // Options is a configuration file for nuclei reporting module type Options struct { + // AllowList contains a list of allowed events for reporting module + AllowList *Filter `yaml:"allow-list"` + // DenyList contains a list of denied events for reporting module + DenyList *Filter `yaml:"deny-list"` // Github contains configuration options for Github Issue Tracker Github *github.Options `yaml:"github"` // Gitlab contains configuration options for Gitlab Issue Tracker @@ -22,6 +27,50 @@ type Options struct { Jira *jira.Options `yaml:"jira"` } +// Filter filters the received event and decides whether to perform +// reporting for it or not. +type Filter struct { + Severity string `yaml:"severity"` + severity []string + Tags string `yaml:"tags"` + tags []string +} + +// Compile compiles the filter creating match structures. +func (f *Filter) Compile() { + parts := strings.Split(f.Severity, ",") + for _, part := range parts { + f.severity = append(f.severity, strings.TrimSpace(part)) + } + parts = strings.Split(f.Tags, ",") + for _, part := range parts { + f.tags = append(f.tags, strings.TrimSpace(part)) + } +} + +// GetMatch returns true if a filter matches result event +func (f *Filter) GetMatch(event *output.ResultEvent) bool { + severity := event.Info["severity"] + if len(f.severity) > 0 { + if stringSliceContains(f.severity, severity) { + return true + } + return false + } + + tags := event.Info["tags"] + tagParts := strings.Split(tags, ",") + for i, tag := range tagParts { + tagParts[i] = strings.TrimSpace(tag) + } + for _, tag := range f.tags { + if stringSliceContains(tagParts, tag) { + return true + } + } + return false +} + // Tracker is an interface implemented by an issue tracker type Tracker interface { // CreateIssue creates an issue in the tracker @@ -31,6 +80,7 @@ type Tracker interface { // Client is a client for nuclei issue tracking module type Client struct { tracker Tracker + options *Options dedupe *dedupe.Storage } @@ -46,6 +96,13 @@ func New(config, db string) (*Client, error) { if err := yaml.NewDecoder(file).Decode(options); err != nil { return nil, err } + if options.AllowList != nil { + options.AllowList.Compile() + } + if options.DenyList != nil { + options.DenyList.Compile() + } + var tracker Tracker if options.Github != nil { tracker, err = github.New(options.Github) @@ -66,7 +123,7 @@ func New(config, db string) (*Client, error) { if err != nil { return nil, err } - return &Client{tracker: tracker, dedupe: storage}, nil + return &Client{tracker: tracker, dedupe: storage, options: options}, nil } // Close closes the issue tracker reporting client @@ -76,6 +133,13 @@ func (c *Client) Close() { // CreateIssue creates an issue in the tracker func (c *Client) CreateIssue(event *output.ResultEvent) error { + if c.options.AllowList != nil && !c.options.AllowList.GetMatch(event) { + return nil + } + if c.options.DenyList != nil && c.options.DenyList.GetMatch(event) { + return nil + } + found, err := c.dedupe.Index(event) if err != nil { c.tracker.CreateIssue(event) @@ -86,3 +150,12 @@ func (c *Client) CreateIssue(event *output.ResultEvent) error { } return nil } + +func stringSliceContains(slice []string, item string) bool { + for _, i := range slice { + if strings.EqualFold(i, item) { + return true + } + } + return false +}