Added jira+github+gitlab issue tracker integration to nuclei

dev
Ice3man543 2021-02-02 12:10:47 +05:30
parent 261b44b036
commit 038b4aa5c2
21 changed files with 662 additions and 25 deletions

View File

@ -74,6 +74,8 @@ based on templates offering massive extensibility and ease of use.`)
set.BoolVarP(&options.NoMeta, "no-meta", "nm", false, "Don't display metadata for the matches")
set.BoolVarP(&options.TemplatesVersion, "templates-version", "tv", false, "Shows the installed nuclei-templates version")
set.StringVarP(&options.BurpCollaboratorBiid, "burp-collaborator-biid", "biid", "", "Burp Collaborator BIID")
set.StringVarP(&options.ReportingConfig, "reporting-config", "rc", "", "Nuclei Reporting Module configuration file")
set.StringVarP(&options.ReportingDirectory, "reporting-directory", "rd", "", "Nuclei Reporting Module cache directory for issue deduplication")
_ = set.Parse()
if cfgFile != "" {

View File

@ -4,11 +4,13 @@ go 1.15
require (
github.com/Knetic/govaluate v3.0.0+incompatible
github.com/andygrunwald/go-jira v1.13.0
github.com/blang/semver v3.5.1+incompatible
github.com/corpix/uarand v0.1.1
github.com/golang/protobuf v1.4.3 // indirect
github.com/golang/snappy v0.0.2 // indirect
github.com/google/go-cmp v0.5.2 // indirect
github.com/google/go-github v17.0.0+incompatible
github.com/google/go-github/v32 v32.1.0
github.com/gorilla/mux v1.8.0
github.com/json-iterator/go v1.1.10
@ -36,11 +38,14 @@ require (
github.com/spaolacci/murmur3 v1.1.0
github.com/spf13/cast v1.3.1
github.com/stretchr/testify v1.7.0
github.com/syndtr/goleveldb v1.0.0
github.com/xanzy/go-gitlab v0.42.0
go.uber.org/atomic v1.7.0
go.uber.org/multierr v1.6.0
go.uber.org/ratelimit v0.1.0
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad // indirect
golang.org/x/net v0.0.0-20210119194325-5f4716e94777
golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 // indirect
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect

View File

@ -4,6 +4,8 @@ github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8L
github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic=
github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA=
github.com/andygrunwald/go-jira v1.13.0 h1:vvIImGgX32bHfoiyUwkNo+/YrPnRczNarvhLOncP6dE=
github.com/andygrunwald/go-jira v1.13.0/go.mod h1:jYi4kFDbRPZTJdJOVJO4mpMMIwdB+rcZwSO58DzPd2I=
github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
@ -14,9 +16,12 @@ github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/fatih/structs v1.0.0 h1:BrX964Rv5uQ3wwS+KRUAJCBBw5PQmgJfJ6v4yly5QwU=
github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
@ -38,12 +43,18 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI=
github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
github.com/hashicorp/go-retryablehttp v0.6.4 h1:BbgctKO892xEyOXnGiaAwIoSq1QZ/SS4AhjoAh9DnfY=
github.com/hashicorp/go-retryablehttp v0.6.4/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw=
@ -71,6 +82,7 @@ github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@ -94,9 +106,14 @@ github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
github.com/trivago/tgo v1.0.1 h1:bxatjJIXNIpV18bucU4Uk/LaoxvxuOlp/oowRHyncLQ=
github.com/trivago/tgo v1.0.1/go.mod h1:w4dpD+3tzNIIiIfkWWa85w5/B77tlvdZckQ+6PkFnhc=
github.com/xanzy/go-gitlab v0.42.0 h1:daNdMFnw2FG+lDRBcX+YLnKbqIKMdefVyVztMHwsFhk=
github.com/xanzy/go-gitlab v0.42.0/go.mod h1:sPLojNBn68fMUWSxIJtdVVIP8uSBYqesTfDUseX11Ug=
go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
@ -104,6 +121,7 @@ go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9i
go.uber.org/ratelimit v0.1.0 h1:U2AruXqeTb4Eh9sYQSTrMhH8Cb7M0Ian2ibBOnBcnAw=
go.uber.org/ratelimit v0.1.0/go.mod h1:2X8KaoNd1J0lZV+PxJk/5+DGbO/tpwLR1m++a7FnB/Y=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY=
@ -117,6 +135,7 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181108082009-03003ca0c849/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@ -124,7 +143,10 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew=
golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288 h1:JIqe8uIcRBHXDQVvZtHwp80ai3Lw3IJAeJEs55Dc1W0=
golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@ -145,6 +167,8 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
@ -157,6 +181,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=

View File

@ -17,6 +17,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/projectfile"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/clusterer"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
"github.com/remeh/sizedwaitgroup"
@ -36,6 +37,7 @@ type Runner struct {
catalogue *catalogue.Catalogue
progress *progress.Progress
colorizer aurora.Aurora
issuesClient *issues.Client
severityColors *colorizer.Colorizer
ratelimiter ratelimit.Limiter
}
@ -54,6 +56,13 @@ func New(options *types.Options) (*Runner, error) {
}
runner.catalogue = catalogue.New(runner.options.TemplatesDirectory)
if options.ReportingConfig != "" {
if client, err := issues.New(options.ReportingConfig, options.ReportingDirectory); err != nil {
gologger.Fatal().Msgf("Could not create issue reporting client: %s\n", err)
} else {
runner.issuesClient = client
}
}
// output coloring
useColor := !options.NoColor
runner.colorizer = aurora.NewAurora(useColor)
@ -202,12 +211,13 @@ func (r *Runner) RunEnumeration() {
}
executerOpts := &protocols.ExecuterOptions{
Output: r.output,
Options: r.options,
Progress: r.progress,
Catalogue: r.catalogue,
RateLimiter: r.ratelimiter,
ProjectFile: r.projectFile,
Output: r.output,
Options: r.options,
Progress: r.progress,
Catalogue: r.catalogue,
IssuesClient: r.issuesClient,
RateLimiter: r.ratelimiter,
ProjectFile: r.projectFile,
}
// pre-parse all the templates, apply filters
finalTemplates := []*templates.Template{}
@ -292,6 +302,9 @@ func (r *Runner) RunEnumeration() {
r.progress.Stop()
}
if r.issuesClient != nil {
r.issuesClient.Close()
}
if !results.Load() {
if r.output != nil {
r.output.Close()

View File

@ -43,12 +43,13 @@ func (r *Runner) getParsedTemplatesFor(templatePaths []string, severities []stri
// parseTemplateFile returns the parsed template file
func (r *Runner) parseTemplateFile(file string) (*templates.Template, error) {
executerOpts := &protocols.ExecuterOptions{
Output: r.output,
Options: r.options,
Progress: r.progress,
Catalogue: r.catalogue,
RateLimiter: r.ratelimiter,
ProjectFile: r.projectFile,
Output: r.output,
Options: r.options,
Progress: r.progress,
Catalogue: r.catalogue,
RateLimiter: r.ratelimiter,
IssuesClient: r.issuesClient,
ProjectFile: r.projectFile,
}
template, err := templates.Parse(file, executerOpts)
if err != nil {

View File

@ -1,13 +1,10 @@
package output
import (
"time"
jsoniter "github.com/json-iterator/go"
)
// formatJSON formats the output for json based formatting
func (w *StandardWriter) formatJSON(output *ResultEvent) ([]byte, error) {
output.Timestamp = time.Now()
return jsoniter.Marshal(output)
}

View File

@ -1,6 +1,7 @@
package clusterer
import (
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
@ -70,6 +71,11 @@ func (e *Executer) Execute(input string) (bool, error) {
event.Results = e.requests.MakeResultEvent(event)
results = true
for _, r := range event.Results {
if e.options.IssuesClient != nil {
if err := e.options.IssuesClient.CreateIssue(r); err != nil {
gologger.Warning().Msgf("Could not create issue on tracker: %s", err)
}
}
e.options.Output.Write(r)
e.options.Progress.IncrementMatched()
}

View File

@ -64,6 +64,11 @@ func (e *Executer) Execute(input string) (bool, error) {
return
}
for _, result := range event.Results {
if e.options.IssuesClient != nil {
if err := e.options.IssuesClient.CreateIssue(result); err != nil {
gologger.Warning().Msgf("Could not create issue on tracker: %s", err)
}
}
results = true
e.options.Output.Write(result)
e.options.Progress.IncrementMatched()

View File

@ -2,6 +2,7 @@ package dns
import (
"bytes"
"time"
"github.com/miekg/dns"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
@ -154,6 +155,7 @@ func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *out
Host: wrapped.InternalEvent["host"].(string),
Matched: wrapped.InternalEvent["matched"].(string),
ExtractedResults: wrapped.OperatorsResult.OutputExtracts,
Timestamp: time.Now(),
}
if r.options.Options.JSONRequests {
data.Request = wrapped.InternalEvent["request"].(string)

View File

@ -1,6 +1,8 @@
package file
import (
"time"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
@ -113,6 +115,7 @@ func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *out
Host: wrapped.InternalEvent["host"].(string),
Matched: wrapped.InternalEvent["matched"].(string),
ExtractedResults: wrapped.OperatorsResult.OutputExtracts,
Timestamp: time.Now(),
}
if r.options.Options.JSONRequests {
data.Response = wrapped.InternalEvent["raw"].(string)

View File

@ -2,7 +2,6 @@ package http
import (
"net/http"
"net/http/httputil"
"strings"
"time"
@ -86,10 +85,9 @@ func (r *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, r
data["host"] = host
data["matched"] = matched
if r.options.Options.JSONRequests {
data["request"] = rawReq
data["response"] = rawResp
}
data["request"] = rawReq
data["raw"] = rawResp
data["response"] = rawResp
data["content_length"] = resp.ContentLength
data["status_code"] = resp.StatusCode
@ -103,11 +101,6 @@ func (r *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, r
data[k] = strings.Join(v, " ")
}
data["all_headers"] = headers
if r, err := httputil.DumpResponse(resp, true); err == nil {
rawString := string(r)
data["raw"] = rawString
}
data["duration"] = duration.Seconds()
data["template-id"] = r.options.TemplateID
data["template-info"] = r.options.TemplateInfo
@ -152,6 +145,7 @@ func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *out
Metadata: wrapped.OperatorsResult.PayloadValues,
ExtractedResults: wrapped.OperatorsResult.OutputExtracts,
IP: wrapped.InternalEvent["ip"].(string),
Timestamp: time.Now(),
}
if r.options.Options.JSONRequests {
data.Request = wrapped.InternalEvent["request"].(string)

View File

@ -1,6 +1,8 @@
package network
import (
"time"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
@ -117,6 +119,7 @@ func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *out
Matched: wrapped.InternalEvent["matched"].(string),
ExtractedResults: wrapped.OperatorsResult.OutputExtracts,
IP: wrapped.InternalEvent["ip"].(string),
Timestamp: time.Now(),
}
if r.options.Options.JSONRequests {
data.Request = wrapped.InternalEvent["request"].(string)

View File

@ -7,6 +7,7 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/projectfile"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
"go.uber.org/ratelimit"
)
@ -35,6 +36,8 @@ type ExecuterOptions struct {
Output output.Writer
// Options contains configuration options for the executer.
Options *types.Options
// IssuesClient is a client for nuclei issue tracker reporting
IssuesClient *issues.Client
// Progress is a progress client for scan reporting
Progress *progress.Progress
// RateLimiter is a rate-limiter for limiting sent number of requests.

View File

@ -0,0 +1,101 @@
// Package dedupe implements deduplication layer for nuclei-generated
// issues.
//
// The layer can be persisted to leveldb based storage for further use.
package dedupe
import (
"crypto/sha1"
"path"
"unsafe"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/errors"
)
// Storage is a duplicate detecting storage for nuclei scan events.
type Storage struct {
storage *leveldb.DB
}
const storageFilename = "nuclei-events.db"
// New creates a new duplicate detecting storage for nuclei scan events.
func New(folder string) (*Storage, error) {
path := path.Join(folder, storageFilename)
db, err := leveldb.OpenFile(path, nil)
if err != nil {
if !errors.IsCorrupted(err) {
return nil, err
}
// If the metadata is corrupted, try to recover
db, err = leveldb.RecoverFile(path, nil)
if err != nil {
return nil, err
}
}
return &Storage{storage: db}, nil
}
// Close closes the storage for further operations
func (s *Storage) Close() {
s.storage.Close()
}
// Index indexes an item in storage and returns true if the item
// was unique.
func (s *Storage) Index(result *output.ResultEvent) (bool, error) {
hasher := sha1.New()
if result.TemplateID != "" {
hasher.Write(unsafeToBytes(result.TemplateID))
}
if result.MatcherName != "" {
hasher.Write(unsafeToBytes(result.MatcherName))
}
if result.ExtractorName != "" {
hasher.Write(unsafeToBytes(result.ExtractorName))
}
if result.Type != "" {
hasher.Write(unsafeToBytes(result.Type))
}
if result.Host != "" {
hasher.Write(unsafeToBytes(result.Host))
}
if result.Matched != "" {
hasher.Write(unsafeToBytes(result.Matched))
}
for _, v := range result.ExtractedResults {
hasher.Write(unsafeToBytes(v))
}
for k, v := range result.Metadata {
hasher.Write(unsafeToBytes(k))
hasher.Write(unsafeToBytes(types.ToString(v)))
}
if result.Request != "" {
hasher.Write(unsafeToBytes(result.Request)) // Very dumb, change later.
}
hash := hasher.Sum(nil)
exists, err := s.storage.Has(hash, nil)
if err != nil {
// if we have an error, return with it but mark it as true
// since we don't want to loose an issue considering it a dupe.
return true, err
}
if !exists {
return true, s.storage.Put(hash, nil, nil)
}
return false, err
}
// unsafeToBytes converts a string to byte slice and does it with
// zero allocations.
//
// Reference - https://stackoverflow.com/questions/59209493/how-to-use-unsafe-get-a-byte-slice-from-a-string-without-memory-copy
func unsafeToBytes(data string) []byte {
return *(*[]byte)(unsafe.Pointer(&data))
}

View File

@ -0,0 +1,31 @@
package dedupe
import (
"io/ioutil"
"os"
"testing"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/stretchr/testify/require"
)
func TestDedupeDuplicates(t *testing.T) {
tempDir, err := ioutil.TempDir("", "nuclei")
require.Nil(t, err, "could not create temporary storage")
defer os.RemoveAll(tempDir)
storage, err := New(tempDir)
require.Nil(t, err, "could not create duplicate storage")
tests := []*output.ResultEvent{
{TemplateID: "test", Host: "https://example.com"},
{TemplateID: "test", Host: "https://example.com"},
}
first, err := storage.Index(tests[0])
require.Nil(t, err, "could not index item")
require.True(t, first, "could not index valid item")
second, err := storage.Index(tests[1])
require.Nil(t, err, "could not index item")
require.False(t, second, "could index duplicate item")
}

View File

@ -0,0 +1,96 @@
package format
import (
"bytes"
"fmt"
"strings"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
)
// Summary returns a formatted built one line summary of the event
func Summary(output *output.ResultEvent) string {
template := GetMatchedTemplate(output)
builder := &strings.Builder{}
builder.WriteString("[")
builder.WriteString(template)
builder.WriteString("] [")
builder.WriteString(output.Info["severity"])
builder.WriteString("] ")
builder.WriteString(output.Info["name"])
builder.WriteString(" found on ")
builder.WriteString(output.Host)
data := builder.String()
return data
}
// MarkdownDescription formats a short description of the generated
// event by the nuclei scanner in Markdown format.
func MarkdownDescription(output *output.ResultEvent) string {
template := GetMatchedTemplate(output)
builder := &bytes.Buffer{}
builder.WriteString("**Details**: **")
builder.WriteString(template)
builder.WriteString("** ")
builder.WriteString(" matched at ")
builder.WriteString(output.Host)
builder.WriteString("\n\n**Protocol**: ")
builder.WriteString(strings.ToUpper(output.Type))
builder.WriteString("\n\n**Full URL**: ")
builder.WriteString(output.Matched)
builder.WriteString("\n\n**Timestamp**: ")
builder.WriteString(output.Timestamp.Format("Mon Jan 2 15:04:05 -0700 MST 2006"))
builder.WriteString("\n\n**Template Information**\n\n| Key | Value |\n|---|---|\n")
for k, v := range output.Info {
builder.WriteString(fmt.Sprintf("| %s | %s |\n", k, v))
}
builder.WriteString("\n**Request**\n\n```\n")
builder.WriteString(output.Request)
builder.WriteString("\n```\n\n**Response**\n\n```\n")
builder.WriteString(output.Response)
builder.WriteString("\n```\n\n")
if len(output.ExtractedResults) > 0 || len(output.Metadata) > 0 {
builder.WriteString("**Extra Information**\n\n")
if len(output.ExtractedResults) > 0 {
builder.WriteString("**Extracted results**:\n\n")
for _, v := range output.ExtractedResults {
builder.WriteString("- ")
builder.WriteString(v)
builder.WriteString("\n")
}
builder.WriteString("\n")
}
if len(output.Metadata) > 0 {
builder.WriteString("**Metadata**:\n\n")
for k, v := range output.Metadata {
builder.WriteString("- ")
builder.WriteString(k)
builder.WriteString(": ")
builder.WriteString(types.ToString(v))
builder.WriteString("\n")
}
builder.WriteString("\n")
}
}
data := builder.String()
return data
}
// GetMatchedTemplate returns the matched template from a result event
func GetMatchedTemplate(output *output.ResultEvent) string {
builder := &strings.Builder{}
builder.WriteString(output.TemplateID)
if output.MatcherName != "" {
builder.WriteString(":")
builder.WriteString(output.MatcherName)
}
if output.ExtractorName != "" {
builder.WriteString(":")
builder.WriteString(output.ExtractorName)
}
template := builder.String()
return template
}

View File

@ -0,0 +1,69 @@
package github
import (
"context"
"net/url"
"golang.org/x/oauth2"
"github.com/google/go-github/github"
"github.com/pkg/errors"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues/format"
)
// Integration is a client for a issue tracker integration
type Integration struct {
client *github.Client
options *Options
}
// Options contains the configuration options for github issue tracker client
type Options struct {
// BaseURL is the optional self-hosted github application url
BaseURL string `yaml:"base-url"`
// Username is the username of the github user
Username string `yaml:"username"`
// Owner is the owner name of the repository for issues.
Owner string `yaml:"owner"`
// Token is the token for github account.
Token string `yaml:"token"`
// ProjectName is the name of the repository.
ProjectName string `yaml:"project-name"`
// IssueLabel is the label of the created issue type
IssueLabel string `yaml:"issue-label"`
}
// New creates a new issue tracker integration client based on options.
func New(options *Options) (*Integration, error) {
ctx := context.Background()
ts := oauth2.StaticTokenSource(
&oauth2.Token{AccessToken: options.Token},
)
tc := oauth2.NewClient(ctx, ts)
client := github.NewClient(tc)
if options.BaseURL != "" {
parsed, err := url.Parse(options.BaseURL)
if err != nil {
return nil, errors.Wrap(err, "could not parse custom baseurl")
}
client.BaseURL = parsed
}
return &Integration{client: client, options: options}, nil
}
// CreateIssue creates an issue in the tracker
func (i *Integration) CreateIssue(event *output.ResultEvent) error {
summary := format.Summary(event)
description := format.MarkdownDescription(event)
req := &github.IssueRequest{
Title: &summary,
Body: &description,
Labels: &[]string{i.options.IssueLabel},
Assignees: &[]string{i.options.Username},
}
_, _, err := i.client.Issues.Create(context.Background(), i.options.Owner, i.options.ProjectName, req)
return err
}

View File

@ -0,0 +1,59 @@
package gitlab
import (
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues/format"
"github.com/xanzy/go-gitlab"
)
// Integration is a client for a issue tracker integration
type Integration struct {
client *gitlab.Client
userID int
options *Options
}
// Options contains the configuration options for gitlab issue tracker client
type Options struct {
// BaseURL is the optional self-hosted gitlab application url
BaseURL string `yaml:"base-url"`
// Username is the username of the gitlab user
Username string `yaml:"username"`
// Token is the token for gitlab account.
Token string `yaml:"token"`
// ProjectName is the name of the repository.
ProjectName string `yaml:"project-name"`
// IssueLabel is the label of the created issue type
IssueLabel string `yaml:"issue-label"`
}
// New creates a new issue tracker integration client based on options.
func New(options *Options) (*Integration, error) {
gitlabOpts := []gitlab.ClientOptionFunc{}
if options.BaseURL != "" {
gitlabOpts = append(gitlabOpts, gitlab.WithBaseURL(options.BaseURL))
}
git, err := gitlab.NewClient(options.Token, gitlabOpts...)
if err != nil {
return nil, err
}
user, _, err := git.Users.CurrentUser()
if err != nil {
return nil, err
}
return &Integration{client: git, userID: user.ID, options: options}, nil
}
// CreateIssue creates an issue in the tracker
func (i *Integration) CreateIssue(event *output.ResultEvent) error {
summary := format.Summary(event)
description := format.MarkdownDescription(event)
_, _, err := i.client.Issues.CreateIssue(i.options.ProjectName, &gitlab.CreateIssueOptions{
Title: &summary,
Description: &description,
Labels: gitlab.Labels{i.options.IssueLabel},
AssigneeIDs: []int{i.userID},
})
return err
}

View File

@ -0,0 +1,88 @@
package issues
import (
"os"
"github.com/pkg/errors"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues/dedupe"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues/github"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues/gitlab"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues/jira"
"gopkg.in/yaml.v2"
)
// Options is a configuration file for nuclei reporting module
type Options struct {
// Github contains configuration options for Github Issue Tracker
Github *github.Options `yaml:"github"`
// Gitlab contains configuration options for Gitlab Issue Tracker
Gitlab *gitlab.Options `yaml:"gitlab"`
// Jira contains configuration options for Jira Issue Tracker
Jira *jira.Options `yaml:"jira"`
}
// Tracker is an interface implemented by an issue tracker
type Tracker interface {
// CreateIssue creates an issue in the tracker
CreateIssue(event *output.ResultEvent) error
}
// Client is a client for nuclei issue tracking module
type Client struct {
tracker Tracker
dedupe *dedupe.Storage
}
// New creates a new nuclei issue tracker reporting client
func New(config, directory string) (*Client, error) {
file, err := os.Open(config)
if err != nil {
return nil, errors.Wrap(err, "could not open reporting config file")
}
defer file.Close()
options := &Options{}
if err := yaml.NewDecoder(file).Decode(options); err != nil {
return nil, err
}
var tracker Tracker
if options.Github != nil {
tracker, err = github.New(options.Github)
}
if options.Gitlab != nil {
tracker, err = gitlab.New(options.Gitlab)
}
if options.Jira != nil {
tracker, err = jira.New(options.Jira)
}
if err != nil {
return nil, errors.Wrap(err, "could not create reporting client")
}
if tracker == nil {
return nil, errors.New("no issue tracker configuration found")
}
storage, err := dedupe.New(directory)
if err != nil {
return nil, err
}
return &Client{tracker: tracker, dedupe: storage}, nil
}
// Close closes the issue tracker reporting client
func (c *Client) Close() {
c.dedupe.Close()
}
// CreateIssue creates an issue in the tracker
func (c *Client) CreateIssue(event *output.ResultEvent) error {
found, err := c.dedupe.Index(event)
if err != nil {
c.tracker.CreateIssue(event)
return err
}
if found {
return c.tracker.CreateIssue(event)
}
return nil
}

View File

@ -0,0 +1,128 @@
package jira
import (
"bytes"
"fmt"
"io/ioutil"
"strings"
jira "github.com/andygrunwald/go-jira"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues/format"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
)
// Integration is a client for a issue tracker integration
type Integration struct {
jira *jira.Client
options *Options
}
// Options contains the configuration options for jira client
type Options struct {
// URL is the URL of the jira server
URL string `yaml:"url"`
// AccountID is the accountID of the jira user.
AccountID string `yaml:"account-id"`
// Email is the email of the user for jira instance
Email string `yaml:"email"`
// Token is the token for jira instance.
Token string `yaml:"token"`
// ProjectName is the name of the project.
ProjectName string `yaml:"project-name"`
// IssueType is the name of the created issue type
IssueType string `yaml:"issue-type"`
}
// New creates a new issue tracker integration client based on options.
func New(options *Options) (*Integration, error) {
tp := jira.BasicAuthTransport{
Username: options.Email,
Password: options.Token,
}
jiraClient, err := jira.NewClient(tp.Client(), options.URL)
if err != nil {
return nil, err
}
return &Integration{jira: jiraClient, options: options}, nil
}
// CreateIssue creates an issue in the tracker
func (i *Integration) CreateIssue(event *output.ResultEvent) error {
summary := format.Summary(event)
issueData := &jira.Issue{
Fields: &jira.IssueFields{
Assignee: &jira.User{AccountID: i.options.AccountID},
Reporter: &jira.User{AccountID: i.options.AccountID},
Description: jiraFormatDescription(event),
Type: jira.IssueType{Name: i.options.IssueType},
Project: jira.Project{Key: i.options.ProjectName},
Summary: summary,
},
}
_, resp, err := i.jira.Issue.Create(issueData)
if err != nil {
var data string
if resp != nil && resp.Body != nil {
d, _ := ioutil.ReadAll(resp.Body)
data = string(d)
}
return fmt.Errorf("%s => %s", err, data)
}
return nil
}
// jiraFormatDescription formats a short description of the generated
// event by the nuclei scanner in Jira format.
func jiraFormatDescription(output *output.ResultEvent) string {
template := format.GetMatchedTemplate(output)
builder := &bytes.Buffer{}
builder.WriteString("*Details*: *")
builder.WriteString(template)
builder.WriteString("* ")
builder.WriteString(" matched at ")
builder.WriteString(output.Host)
builder.WriteString("\n\n*Protocol*: ")
builder.WriteString(strings.ToUpper(output.Type))
builder.WriteString("\n\n*Full URL*: ")
builder.WriteString(output.Matched)
builder.WriteString("\n\n*Timestamp*: ")
builder.WriteString(output.Timestamp.Format("Mon Jan 2 15:04:05 -0700 MST 2006"))
builder.WriteString("\n\n*Template Information*\n\n| Key | Value |\n")
for k, v := range output.Info {
builder.WriteString(fmt.Sprintf("| %s | %s |\n", k, v))
}
builder.WriteString("\n*Request*\n\n{code}\n")
builder.WriteString(output.Request)
builder.WriteString("\n{code}\n\n*Response*\n\n{code}\n")
builder.WriteString(output.Response)
builder.WriteString("\n{code}\n\n")
if len(output.ExtractedResults) > 0 || len(output.Metadata) > 0 {
builder.WriteString("*Extra Information*\n\n")
if len(output.ExtractedResults) > 0 {
builder.WriteString("*Extracted results*:\n\n")
for _, v := range output.ExtractedResults {
builder.WriteString("- ")
builder.WriteString(v)
builder.WriteString("\n")
}
builder.WriteString("\n")
}
if len(output.Metadata) > 0 {
builder.WriteString("*Metadata*:\n\n")
for k, v := range output.Metadata {
builder.WriteString("- ")
builder.WriteString(k)
builder.WriteString(": ")
builder.WriteString(types.ToString(v))
builder.WriteString("\n")
}
builder.WriteString("\n")
}
}
data := builder.String()
return data
}

View File

@ -80,4 +80,9 @@ type Options struct {
ExcludedTemplates goflags.StringSlice
// CustomHeaders is the list of custom global headers to send with each request.
CustomHeaders goflags.StringSlice
// ReportingConfig is the config file for nuclei reporting module
ReportingConfig string
// ReportingDirectory is the directory to store nuclei issue deduplication data
// for reporting in.
ReportingDirectory string
}