From 58237f103ae972d747848133e69483977398a73f Mon Sep 17 00:00:00 2001 From: Ice3man Date: Sun, 21 Feb 2021 16:31:34 +0530 Subject: [PATCH] Add headless chrome based templates support (#562) --- v2/cmd/nuclei/main.go | 7 +- v2/go.mod | 21 +- v2/go.sum | 341 +++++++++++- v2/internal/runner/options.go | 7 + v2/internal/runner/runner.go | 15 + v2/internal/runner/templates.go | 1 + v2/internal/testutils/testutils.go | 1 + v2/pkg/operators/operators.go | 17 +- v2/pkg/protocols/headless/engine/action.go | 135 +++++ v2/pkg/protocols/headless/engine/engine.go | 124 +++++ .../protocols/headless/engine/engine_unix.go | 11 + .../headless/engine/engine_windows.go | 12 + .../protocols/headless/engine/http_client.go | 39 ++ v2/pkg/protocols/headless/engine/instance.go | 57 ++ v2/pkg/protocols/headless/engine/page.go | 81 +++ .../protocols/headless/engine/page_actions.go | 505 ++++++++++++++++++ v2/pkg/protocols/headless/engine/rules.go | 48 ++ v2/pkg/protocols/headless/headless.go | 52 ++ v2/pkg/protocols/headless/operators.go | 122 +++++ v2/pkg/protocols/headless/request.go | 79 +++ v2/pkg/protocols/headless/request_test.go | 55 ++ v2/pkg/protocols/http/build_request.go | 4 +- v2/pkg/protocols/network/network.go | 2 +- v2/pkg/protocols/protocols.go | 3 + v2/pkg/templates/compile.go | 8 +- v2/pkg/templates/preprocessors.go | 3 + v2/pkg/templates/templates.go | 11 +- v2/pkg/types/types.go | 4 +- v2/pkg/workflows/workflows.go | 2 +- 29 files changed, 1741 insertions(+), 26 deletions(-) create mode 100644 v2/pkg/protocols/headless/engine/action.go create mode 100644 v2/pkg/protocols/headless/engine/engine.go create mode 100644 v2/pkg/protocols/headless/engine/engine_unix.go create mode 100644 v2/pkg/protocols/headless/engine/engine_windows.go create mode 100644 v2/pkg/protocols/headless/engine/http_client.go create mode 100644 v2/pkg/protocols/headless/engine/instance.go create mode 100644 v2/pkg/protocols/headless/engine/page.go create mode 100644 v2/pkg/protocols/headless/engine/page_actions.go create mode 100644 v2/pkg/protocols/headless/engine/rules.go create mode 100644 v2/pkg/protocols/headless/headless.go create mode 100644 v2/pkg/protocols/headless/operators.go create mode 100644 v2/pkg/protocols/headless/request.go create mode 100644 v2/pkg/protocols/headless/request_test.go diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index cc34bb34..634afce0 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -4,6 +4,9 @@ import ( "os" "path" + "net/http" + _ "net/http/pprof" + "github.com/projectdiscovery/goflags" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/internal/runner" @@ -17,6 +20,7 @@ var ( func main() { readConfig() + go http.ListenAndServe(":6060", http.DefaultServeMux) runner.ParseOptions(options) @@ -79,7 +83,8 @@ based on templates offering massive extensibility and ease of use.`) set.StringVarP(&options.ReportingDB, "report-db", "rdb", "", "Local Nuclei Reporting Database") set.StringSliceVar(&options.Tags, "tags", []string{}, "Tags to execute templates for") set.StringVarP(&options.ResolversFile, "resolvers", "r", "", "File containing resolver list for nuclei") - + set.BoolVar(&options.Headless, "headless", false, "Enable headless browser based templates support") + set.BoolVar(&options.ShowBrowser, "show-browser", false, "Show the browser on the screen") _ = set.Parse() if cfgFile != "" { diff --git a/v2/go.mod b/v2/go.mod index 5e1a470e..b889417e 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -7,24 +7,28 @@ require ( github.com/andygrunwald/go-jira v1.13.0 github.com/blang/semver v3.5.1+incompatible github.com/corpix/uarand v0.1.1 + github.com/fatih/structs v1.1.0 // indirect + github.com/go-rod/rod v0.91.1 github.com/golang/protobuf v1.4.3 // indirect github.com/google/go-cmp v0.5.2 // indirect github.com/google/go-github v17.0.0+incompatible github.com/google/go-github/v32 v32.1.0 github.com/gorilla/mux v1.8.0 + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-retryablehttp v0.6.8 // indirect github.com/json-iterator/go v1.1.10 github.com/karrick/godirwalk v1.16.1 - github.com/kr/pretty v0.1.0 // indirect github.com/logrusorgru/aurora v2.0.3+incompatible github.com/mattn/go-runewidth v0.0.10 // indirect github.com/miekg/dns v1.1.38 + github.com/mitchellh/go-ps v1.0.0 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.1 // indirect github.com/olekukonko/tablewriter v0.0.4 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.7 github.com/projectdiscovery/collaborator v0.0.2 - github.com/projectdiscovery/fastdialer v0.0.3 + github.com/projectdiscovery/fastdialer v0.0.5 github.com/projectdiscovery/goflags v0.0.2 github.com/projectdiscovery/gologger v1.1.3 github.com/projectdiscovery/hmap v0.0.1 @@ -39,16 +43,19 @@ require ( github.com/spf13/cast v1.3.1 github.com/stretchr/testify v1.7.0 github.com/syndtr/goleveldb v1.0.0 + github.com/trivago/tgo v1.0.7 // indirect github.com/valyala/fasttemplate v1.2.1 - github.com/xanzy/go-gitlab v0.42.0 + github.com/xanzy/go-gitlab v0.44.0 go.uber.org/atomic v1.7.0 go.uber.org/multierr v1.6.0 go.uber.org/ratelimit v0.1.0 + golang.org/x/crypto v0.0.0-20210218145215-b8e89b74b9df // indirect golang.org/x/net v0.0.0-20210119194325-5f4716e94777 - golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288 + golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 // indirect - golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect - google.golang.org/protobuf v1.25.0 // indirect - gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect + golang.org/x/sys v0.0.0-20210218155724-8ebf48af031b // indirect + golang.org/x/text v0.3.4 // indirect + golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect + google.golang.org/appengine v1.6.7 // indirect gopkg.in/yaml.v2 v2.4.0 ) diff --git a/v2/go.sum b/v2/go.sum index 20ec3cd3..b355e5bb 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -1,5 +1,39 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f6s9Tn1Tt7/WTEg= github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic= github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= @@ -9,71 +43,139 @@ github.com/andygrunwald/go-jira v1.13.0/go.mod h1:jYi4kFDbRPZTJdJOVJO4mpMMIwdB+r github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4= github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA= +github.com/corpix/uarand v0.1.1 h1:RMr1TWc9F4n5jiPDzFHtmaUXLKLNUFK0SgCLo4BhX/U= github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/structs v1.0.0 h1:BrX964Rv5uQ3wwS+KRUAJCBBw5PQmgJfJ6v4yly5QwU= github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= +github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-rod/rod v0.91.1 h1:7xIlC/bXCXosZqZUl2x6GVB8tv4yMQ4W/ZVdGVa1qYI= +github.com/go-rod/rod v0.91.1/go.mod h1:/W4lcZiCALPD603MnJGIvhtywP3R6yRB9EDfFfsHiiI= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.2 h1:aeE13tS0IiQgFjYdoL8qN3K1N2bXXtI6Vi51/y7BpMw= github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= +github.com/google/go-github/v32 v32.1.0 h1:GWkQOdXqviCPx7Q7Fj+KyPoGm4SwHRh8rheoPhd27II= github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI= github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-retryablehttp v0.6.4 h1:BbgctKO892xEyOXnGiaAwIoSq1QZ/SS4AhjoAh9DnfY= github.com/hashicorp/go-retryablehttp v0.6.4/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= +github.com/hashicorp/go-retryablehttp v0.6.8 h1:92lWxgpa+fF3FozM4B3UZtHZMJX8T5XT+TFdCxsPyWs= +github.com/hashicorp/go-retryablehttp v0.6.8/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.10 h1:CoZ3S2P7pvtP45xOtBw+/mDL2z0RKI576gSkzRRpdGg= github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.38 h1:MtIY+fmHUVVgv1AXzmKMWcwdCYxTRPG1EDjpqF4RCEw= github.com/miekg/dns v1.1.38/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc= +github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM= github.com/olekukonko/tablewriter v0.0.4 h1:vHD/YYe1Wolo78koG299f7V/VAS08c6IpCLn+Ejf/w8= @@ -84,44 +186,80 @@ github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/projectdiscovery/clistats v0.0.7 h1:Q/erjrk2p3BIQq1RaHVtBpgboghNz0u1/lyQ2fr8Cn0= github.com/projectdiscovery/clistats v0.0.7/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg= +github.com/projectdiscovery/collaborator v0.0.2 h1:BSiMlWM3NvuKbpedn6fIjjEo5b7q5zmiJ6tI7+6mB3s= github.com/projectdiscovery/collaborator v0.0.2/go.mod h1:J1z0fC7Svutz3LJqoRyTHA3F0Suh4livmkYv8MnKw20= -github.com/projectdiscovery/fastdialer v0.0.3 h1:KYj1DmOEl5ogDuRu0ny8DeDFhGS+W0+sQw1qc7otVSw= -github.com/projectdiscovery/fastdialer v0.0.3/go.mod h1:RtocEuqSbT74NRi+wJI1/lz7da3ncayzbzetrgd3QaQ= +github.com/projectdiscovery/fastdialer v0.0.5 h1:wTeZRPFED1VQRcw5aGoeZ6UlhJ6Cra/Jqw2Wi5kPuis= +github.com/projectdiscovery/fastdialer v0.0.5/go.mod h1:m20Ls/JratRO8wSSpvOMIKu4aFDh9c5zwCH8+5JO0nA= github.com/projectdiscovery/goflags v0.0.2 h1:4vB5+mA41xgW6V1y4YD1A+iI8Kq68iTTny50XuSYKdo= github.com/projectdiscovery/goflags v0.0.2/go.mod h1:Ae1mJ5MIIqjys0lFe3GiMZ10Z8VLaxkYJ1ySA4Zv8HA= +github.com/projectdiscovery/gologger v1.1.3 h1:rKWZW2QUigRV1jnlWwWJbJRvz8b+T/+bB5qemDGGBJU= github.com/projectdiscovery/gologger v1.1.3/go.mod h1:jdXflz3TLB8bcVNzb0v26TztI9KPz8Lr4BVdUhNUs6E= +github.com/projectdiscovery/hmap v0.0.1 h1:VAONbJw5jP+syI5smhsfkrq9XPGn4aiYy5pR6KR1wog= github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0= +github.com/projectdiscovery/rawhttp v0.0.4 h1:O5IreNGk83d4xTD9e6SpkKbX0sHTs8K1Q33Bz4eYl2E= github.com/projectdiscovery/rawhttp v0.0.4/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= github.com/projectdiscovery/retryabledns v1.0.6 h1:fz33puVeUKJJ5s2POSlxO4WA4iodW6Yzm/EVNuO/93w= github.com/projectdiscovery/retryabledns v1.0.6/go.mod h1:/UzJn4I+cPdQl6pKiiQfvVAT636YZvJQYZhYhGB0dUQ= +github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNwshuwptuBVYWw9lx8RE= github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/segmentio/ksuid v1.0.3 h1:FoResxvleQwYiPAVKe1tMUlEirodZqlqglIuFsdDntY= github.com/segmentio/ksuid v1.0.3/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/trivago/tgo v1.0.1 h1:bxatjJIXNIpV18bucU4Uk/LaoxvxuOlp/oowRHyncLQ= github.com/trivago/tgo v1.0.1/go.mod h1:w4dpD+3tzNIIiIfkWWa85w5/B77tlvdZckQ+6PkFnhc= +github.com/trivago/tgo v1.0.7 h1:uaWH/XIy9aWYWpjm2CU3RpcqZXmX2ysQ9/Go+d9gyrM= +github.com/trivago/tgo v1.0.7/go.mod h1:w4dpD+3tzNIIiIfkWWa85w5/B77tlvdZckQ+6PkFnhc= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= -github.com/xanzy/go-gitlab v0.42.0 h1:daNdMFnw2FG+lDRBcX+YLnKbqIKMdefVyVztMHwsFhk= -github.com/xanzy/go-gitlab v0.42.0/go.mod h1:sPLojNBn68fMUWSxIJtdVVIP8uSBYqesTfDUseX11Ug= +github.com/xanzy/go-gitlab v0.44.0 h1:cEiGhqu7EpFGuei2a2etAwB+x6403E5CvpLn35y+GPs= +github.com/xanzy/go-gitlab v0.44.0/go.mod h1:sPLojNBn68fMUWSxIJtdVVIP8uSBYqesTfDUseX11Ug= +github.com/ysmood/goob v0.3.0 h1:XZ51cZJ4W3WCoCiUktixzMIQF86W7G5VFL4QQ/Q2uS0= +github.com/ysmood/goob v0.3.0/go.mod h1:S3lq113Y91y1UBf1wj1pFOxeahvfKkCk6mTWTWbDdWs= +github.com/ysmood/got v0.9.3 h1:qx51X49jL/WAiqZzPTkPZ0zp5pTmrWJa4zYFTYo0gHI= +github.com/ysmood/got v0.9.3/go.mod h1:pE1l4LOwOBhQg6A/8IAatkGp7uZjnalzrZolnlhhMgY= +github.com/ysmood/gotrace v0.2.0 h1:IkTC6rJREwXSaG8yWK+NFwIJGIsxA1DjC6/gxYyQttE= +github.com/ysmood/gotrace v0.2.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM= +github.com/ysmood/gson v0.6.3 h1:4cU+5oOdsyundXHy00t99H0rLXLthuseD3x6W+xmCiU= +github.com/ysmood/gson v0.6.3/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg= +github.com/ysmood/leakless v0.6.12 h1:XxtRYl97bJklfv4BZVdyGnd/y42p6w8lu1hUzfCkT4M= +github.com/ysmood/leakless v0.6.12/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= @@ -130,74 +268,259 @@ go.uber.org/ratelimit v0.1.0 h1:U2AruXqeTb4Eh9sYQSTrMhH8Cb7M0Ian2ibBOnBcnAw= go.uber.org/ratelimit v0.1.0/go.mod h1:2X8KaoNd1J0lZV+PxJk/5+DGbO/tpwLR1m++a7FnB/Y= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210218145215-b8e89b74b9df h1:y7QZzfUiTwWam+xBn29Ulb8CBwVN5UdzmMDavl9Whlw= +golang.org/x/crypto v0.0.0-20210218145215-b8e89b74b9df/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181108082009-03003ca0c849/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288 h1:JIqe8uIcRBHXDQVvZtHwp80ai3Lw3IJAeJEs55Dc1W0= golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99 h1:5vD4XjIc0X5+kHZjx4UecYdjA6mJo+XXNoaW0EjU5Os= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210218155724-8ebf48af031b h1:lAZ0/chPUDWwjqosYR0X4M490zQhMsiJ4K3DbA7o+3g= +golang.org/x/sys v0.0.0-20210218155724-8ebf48af031b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 h1:Hir2P/De0WpUhtrKGGjvSb2YxUgyZ7EFOSLIcSSpiwE= +golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -206,11 +529,13 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= @@ -222,4 +547,12 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 85d33492..368c63fd 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -44,6 +44,13 @@ func ParseOptions(options *types.Options) { gologger.Fatal().Msgf("Program exiting: %s\n", err) } + // Auto adjust rate limits when using headless mode if the user + // hasn't specified any custom limits. + if options.Headless && options.BulkSize == 25 && options.TemplateThreads == 10 { + options.BulkSize = 2 + options.TemplateThreads = 2 + } + // Load the resolvers if user asked for them loadResolvers(options) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 4cfe171c..1993d81b 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -17,6 +17,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/projectfile" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/clusterer" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues" "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/types" @@ -39,6 +40,7 @@ type Runner struct { colorizer aurora.Aurora issuesClient *issues.Client severityColors *colorizer.Colorizer + browser *engine.Browser ratelimiter ratelimit.Limiter } @@ -47,6 +49,13 @@ func New(options *types.Options) (*Runner, error) { runner := &Runner{ options: options, } + if options.Headless { + browser, err := engine.New(options) + if err != nil { + return nil, err + } + runner.browser = browser + } if err := runner.updateTemplates(); err != nil { gologger.Warning().Msgf("Could not update templates: %s\n", err) } @@ -239,6 +248,7 @@ func (r *Runner) RunEnumeration() { Catalogue: r.catalogue, RateLimiter: r.ratelimiter, IssuesClient: r.issuesClient, + Browser: r.browser, ProjectFile: r.projectFile, } clusterID := fmt.Sprintf("cluster-%s", xid.New().String()) @@ -312,4 +322,9 @@ func (r *Runner) RunEnumeration() { } gologger.Info().Msgf("No results found. Better luck next time!") } + + if r.browser != nil { + r.browser.Close() + // kill chrome process here + } } diff --git a/v2/internal/runner/templates.go b/v2/internal/runner/templates.go index 22163857..c585ba39 100644 --- a/v2/internal/runner/templates.go +++ b/v2/internal/runner/templates.go @@ -51,6 +51,7 @@ func (r *Runner) parseTemplateFile(file string) (*templates.Template, error) { IssuesClient: r.issuesClient, RateLimiter: r.ratelimiter, ProjectFile: r.projectFile, + Browser: r.browser, } template, err := templates.Parse(file, executerOpts) if err != nil { diff --git a/v2/internal/testutils/testutils.go b/v2/internal/testutils/testutils.go index b62452c9..8098b2ab 100644 --- a/v2/internal/testutils/testutils.go +++ b/v2/internal/testutils/testutils.go @@ -112,6 +112,7 @@ func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) *protoco Progress: progress, ProjectFile: nil, IssuesClient: nil, + Browser: nil, Catalogue: catalogue.New(options.TemplatesDirectory), RateLimiter: ratelimit.New(options.RateLimit), } diff --git a/v2/pkg/operators/operators.go b/v2/pkg/operators/operators.go index aabc2248..41378fa2 100644 --- a/v2/pkg/operators/operators.go +++ b/v2/pkg/operators/operators.go @@ -10,13 +10,13 @@ import ( type Operators struct { // Matchers contains the detection mechanism for the request to identify // whether the request was successful - Matchers []*matchers.Matcher `yaml:"matchers"` + Matchers []*matchers.Matcher `yaml:"matchers,omitempty"` // Extractors contains the extraction mechanism for the request to identify // and extract parts of the response. - Extractors []*extractors.Extractor `yaml:"extractors"` + Extractors []*extractors.Extractor `yaml:"extractors,omitempty"` // MatchersCondition is the condition of the matchers // whether to use AND or OR. Default is OR. - MatchersCondition string `yaml:"matchers-condition"` + MatchersCondition string `yaml:"matchers-condition,omitempty"` // cached variables that may be used along with request. matchersCondition matchers.ConditionType } @@ -49,6 +49,10 @@ func (r *Operators) GetMatchersCondition() matchers.ConditionType { // Result is a result structure created from operators running on data. type Result struct { + // Matched is true if any matchers matched + Matched bool + // Extracted is true if any result type values were extracted + Extracted bool // Matches is a map of matcher names that we matched Matches map[string]struct{} // Extracts contains all the data extracted from inputs @@ -115,13 +119,18 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac } } + result.Matched = matches + result.Extracted = len(result.OutputExtracts) > 0 + if len(result.DynamicValues) > 0 { + return result, true + } // Don't print if we have matchers and they have not matched, irregardless of extractor if len(r.Matchers) > 0 && !matches { return nil, false } // Write a final string of output if matcher type is // AND or if we have extractors for the mechanism too. - if len(result.Extracts) > 0 || len(result.DynamicValues) > 0 || len(result.OutputExtracts) > 0 || matches { + if len(result.Extracts) > 0 || len(result.OutputExtracts) > 0 || matches { return result, true } return nil, false diff --git a/v2/pkg/protocols/headless/engine/action.go b/v2/pkg/protocols/headless/engine/action.go new file mode 100644 index 00000000..6d78f3a1 --- /dev/null +++ b/v2/pkg/protocols/headless/engine/action.go @@ -0,0 +1,135 @@ +package engine + +import "strings" + +// ActionType defines the action type for a browser action +type ActionType int8 + +// Types to be executed by the user. +const ( + // ActionNavigate performs a navigation to the specified URL + // URL can include nuclei payload data such as URL, Hostname, etc. + ActionNavigate ActionType = iota + 1 + // ActionScript executes a JS snippet on the page. + ActionScript + // ActionClick performs the left-click action on an Element. + ActionClick + // ActionRightClick performs the right-click action on an Element. + ActionRightClick + // ActionTextInput performs an action for a text input + ActionTextInput + // ActionScreenshot performs the screenshot action writing to a file. + ActionScreenshot + // ActionTimeInput performs an action on a time input. + ActionTimeInput + // ActionSelectInput performs an action on a select input. + ActionSelectInput + // ActionFilesInput performs an action on a file input. + ActionFilesInput + // ActionWaitLoad waits for the page to stop loading. + ActionWaitLoad + // ActionGetResource performs a get resource action on an element + ActionGetResource + // ActionExtract performs an extraction on an element + ActionExtract + // ActionSetMethod sets the request method + ActionSetMethod + // ActionAddHeader adds a header to the request + ActionAddHeader + // ActionSetHeader sets a header in the request + ActionSetHeader + // ActionDeleteHeader deletes a header from the request + ActionDeleteHeader + // ActionSetBody sets the value of the request body + ActionSetBody + // ActionWaitEvent waits for a specific event. + ActionWaitEvent + // ActionKeyboard performs a keyboard action event on a page. + ActionKeyboard +) + +// ActionStringToAction converts an action from string to internal representation +var ActionStringToAction = map[string]ActionType{ + "navigate": ActionNavigate, + "script": ActionScript, + "click": ActionClick, + "rightclick": ActionRightClick, + "text": ActionTextInput, + "screenshot": ActionScreenshot, + "time": ActionTimeInput, + "select": ActionSelectInput, + "files": ActionFilesInput, + "waitload": ActionWaitLoad, + "getresource": ActionGetResource, + "extract": ActionExtract, + "setmethod": ActionSetMethod, + "addheader": ActionAddHeader, + "setheader": ActionSetHeader, + "deleteheader": ActionDeleteHeader, + "setbody": ActionSetBody, + "waitevent": ActionWaitEvent, + "keyboard": ActionKeyboard, +} + +// ActionToActionString converts an action from internal representation to string +var ActionToActionString = map[ActionType]string{ + ActionNavigate: "navigate", + ActionScript: "script", + ActionClick: "click", + ActionRightClick: "rightclick", + ActionTextInput: "text", + ActionScreenshot: "screenshot", + ActionTimeInput: "time", + ActionSelectInput: "select", + ActionFilesInput: "files", + ActionWaitLoad: "waitload", + ActionGetResource: "getresource", + ActionExtract: "extract", + ActionSetMethod: "set-method", + ActionAddHeader: "addheader", + ActionSetHeader: "setheader", + ActionDeleteHeader: "deleteheader", + ActionSetBody: "setbody", + ActionWaitEvent: "waitevent", + ActionKeyboard: "keyboard", +} + +// Action is an action taken by the browser to reach a navigation +// +// Each step that the browser executes is an action. Most navigations +// usually start from the ActionLoadURL event, and further navigations +// are discovered on the found page. We also keep track and only +// scrape new navigation from pages we haven't crawled yet. +type Action struct { + Data map[string]string `yaml:"args,omitempty"` + Name string `yaml:"name,omitempty"` + Description string `yaml:"description,omitempty"` + ActionType string `yaml:"action"` +} + +// String returns the string representation of an action +func (a *Action) String() string { + builder := &strings.Builder{} + builder.WriteString(a.ActionType) + if a.Name != "" { + builder.WriteString(" Name:") + builder.WriteString(a.Name) + } + builder.WriteString(" ") + for k, v := range a.Data { + builder.WriteString(k) + builder.WriteString(":") + builder.WriteString(v) + builder.WriteString(",") + } + return strings.TrimSuffix(builder.String(), ",") +} + +// GetArg returns an arg for a name +func (a *Action) GetArg(name string) string { + v, ok := a.Data[name] + if !ok { + return "" + } + return v +} diff --git a/v2/pkg/protocols/headless/engine/engine.go b/v2/pkg/protocols/headless/engine/engine.go new file mode 100644 index 00000000..926bda45 --- /dev/null +++ b/v2/pkg/protocols/headless/engine/engine.go @@ -0,0 +1,124 @@ +package engine + +import ( + "fmt" + "io/ioutil" + "net/http" + "os" + "strings" + + "github.com/corpix/uarand" + "github.com/go-rod/rod" + "github.com/go-rod/rod/lib/launcher" + ps "github.com/mitchellh/go-ps" + "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) + +// Browser is a browser structure for nuclei headless module +type Browser struct { + customAgent string + tempDir string + previouspids map[int]struct{} // track already running pids + engine *rod.Browser + httpclient *http.Client + options *types.Options +} + +// New creates a new nuclei headless browser module +func New(options *types.Options) (*Browser, error) { + dataStore, err := ioutil.TempDir("", "nuclei-*") + if err != nil { + return nil, errors.Wrap(err, "could not create temporary directory") + } + launcher := launcher.New(). + Leakless(false). + Set("disable-gpu", "true"). + Set("ignore-certificate-errors", "true"). + Set("ignore-certificate-errors", "1"). + Set("disable-crash-reporter", "true"). + Set("disable-notifications", "true"). + Set("hide-scrollbars", "true"). + Set("window-size", fmt.Sprintf("%d,%d", 1080, 1920)). + Set("no-sandbox", "true"). + Set("mute-audio", "true"). + Set("incognito", "true"). + Delete("use-mock-keychain"). + UserDataDir(dataStore) + + if options.ShowBrowser { + launcher = launcher.Headless(false) + } else { + launcher = launcher.Headless(true) + } + if options.ProxyURL != "" { + launcher = launcher.Proxy(options.ProxyURL) + } + launcherURL, err := launcher.Launch() + if err != nil { + return nil, err + } + + browser := rod.New().ControlURL(launcherURL) + if err := browser.Connect(); err != nil { + return nil, err + } + customAgent := "" + for _, option := range options.CustomHeaders { + parts := strings.SplitN(option, ":", 2) + if len(parts) != 2 { + continue + } + if strings.EqualFold(parts[0], "User-Agent") { + customAgent = parts[1] + } + } + if options.RandomAgent { + customAgent = uarand.GetRandom() + } + httpclient, err := newhttpClient(options) + if err != nil { + return nil, err + } + engine := &Browser{ + tempDir: dataStore, + customAgent: customAgent, + engine: browser, + httpclient: httpclient, + options: options, + } + engine.previouspids = engine.findChromeProcesses() + return engine, nil +} + +// Close closes the browser engine +func (b *Browser) Close() { + b.engine.Close() + os.RemoveAll(b.tempDir) + b.killChromeProcesses() +} + +// killChromeProcesses any and all new chrome processes started after +// headless process launch. +func (b *Browser) killChromeProcesses() { + new := b.findChromeProcesses() + for id := range new { + if _, ok := b.previouspids[id]; ok { + continue + } + kill(id) + } +} + +// findChromeProcesses finds chrome process running on host +func (b *Browser) findChromeProcesses() map[int]struct{} { + processes, _ := ps.Processes() + list := make(map[int]struct{}) + for _, process := range processes { + if strings.Contains(process.Executable(), "chrome") || strings.Contains(process.Executable(), "chromium") { + list[process.PPid()] = struct{}{} + list[process.Pid()] = struct{}{} + } + } + return list +} diff --git a/v2/pkg/protocols/headless/engine/engine_unix.go b/v2/pkg/protocols/headless/engine/engine_unix.go new file mode 100644 index 00000000..73a3e55f --- /dev/null +++ b/v2/pkg/protocols/headless/engine/engine_unix.go @@ -0,0 +1,11 @@ +// +build !windows + +package engine + +import ( + "syscall" +) + +func kill(pid int) { + _ = syscall.Kill(-pid, syscall.SIGKILL) +} diff --git a/v2/pkg/protocols/headless/engine/engine_windows.go b/v2/pkg/protocols/headless/engine/engine_windows.go new file mode 100644 index 00000000..feb97ff6 --- /dev/null +++ b/v2/pkg/protocols/headless/engine/engine_windows.go @@ -0,0 +1,12 @@ +// +build windows + +package engine + +import ( + "os/exec" + "strconv" +) + +func kill(pid int) { + _ = exec.Command("taskkill", "/t", "/f", "/pid", strconv.Itoa(pid)).Run() +} diff --git a/v2/pkg/protocols/headless/engine/http_client.go b/v2/pkg/protocols/headless/engine/http_client.go new file mode 100644 index 00000000..fb507b31 --- /dev/null +++ b/v2/pkg/protocols/headless/engine/http_client.go @@ -0,0 +1,39 @@ +package engine + +import ( + "crypto/tls" + "net/http" + "time" + + "github.com/pkg/errors" + "github.com/projectdiscovery/fastdialer/fastdialer" + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) + +// newhttpClient creates a new http client for headless communication with a timeout +func newhttpClient(options *types.Options) (*http.Client, error) { + opts := fastdialer.DefaultOptions + if options.ResolversFile != "" { + opts.BaseResolvers = options.InternalResolversList + } + dialer, err := fastdialer.NewDialer(opts) + if err != nil { + return nil, errors.Wrap(err, "could not create dialer") + } + + transport := &http.Transport{ + DialContext: dialer.Dial, + MaxIdleConns: 500, + MaxIdleConnsPerHost: 500, + MaxConnsPerHost: 500, + TLSClientConfig: &tls.Config{ + Renegotiation: tls.RenegotiateOnceAsClient, + InsecureSkipVerify: true, + }, + } + + return &http.Client{ + Transport: transport, + Timeout: time.Duration(options.Timeout*3) * time.Second, + }, nil +} diff --git a/v2/pkg/protocols/headless/engine/instance.go b/v2/pkg/protocols/headless/engine/instance.go new file mode 100644 index 00000000..f16ceaeb --- /dev/null +++ b/v2/pkg/protocols/headless/engine/instance.go @@ -0,0 +1,57 @@ +package engine + +import ( + "context" + "errors" + "time" + + "github.com/go-rod/rod" + "github.com/go-rod/rod/lib/utils" +) + +// Instance is an isolated browser instance opened for doing operations with it. +type Instance struct { + browser *Browser + engine *rod.Browser +} + +// NewInstance creates a new instance for the current browser. +// +// The login process is repeated only once for a browser, and the created +// isolated browser instance is used for entire navigation ony be one. +// +// Users can also choose to run the login->actions process again +// which uses a new incognito browser instance to run actions. +func (b *Browser) NewInstance() (*Instance, error) { + browser, err := b.engine.Incognito() + if err != nil { + return nil, err + } + + // We use a custom sleeper that sleeps from 100ms to 500 ms waiting + // for an interaction. Used throughout rod for clicking, etc. + browser = browser.Sleeper(func() utils.Sleeper { return maxBackoffSleeper(10) }) + return &Instance{browser: b, engine: browser}, nil +} + +// Close closes all the tabs and pages for a browser instance +func (i *Instance) Close() error { + return i.engine.Close() +} + +// maxBackoffSleeper is a backoff sleeper respecting max backoff values +func maxBackoffSleeper(max int) utils.Sleeper { + count := 0 + backoffSleeper := utils.BackoffSleeper(100*time.Millisecond, 500*time.Millisecond, nil) + + return func(ctx context.Context) error { + if ctx.Err() != nil { + return ctx.Err() + } + if count == max { + return errors.New("max sleep count") + } + count++ + return backoffSleeper(ctx) + } +} diff --git a/v2/pkg/protocols/headless/engine/page.go b/v2/pkg/protocols/headless/engine/page.go new file mode 100644 index 00000000..381420ef --- /dev/null +++ b/v2/pkg/protocols/headless/engine/page.go @@ -0,0 +1,81 @@ +package engine + +import ( + "net/url" + + "github.com/go-rod/rod" + "github.com/go-rod/rod/lib/proto" +) + +// Page is a single page in an isolated browser instanace +type Page struct { + page *rod.Page + rules []requestRule + instance *Instance + router *rod.HijackRouter +} + +// Run runs a list of actions by creating a new page in the browser. +func (i *Instance) Run(baseURL *url.URL, actions []*Action) (map[string]string, *Page, error) { + page, err := i.engine.Page(proto.TargetCreateTarget{}) + if err != nil { + return nil, nil, err + } + if i.browser.customAgent != "" { + if err := page.SetUserAgent(&proto.NetworkSetUserAgentOverride{UserAgent: i.browser.customAgent}); err != nil { + return nil, nil, err + } + } + + createdPage := &Page{page: page, instance: i} + router := page.HijackRequests() + if err := router.Add("*", "", createdPage.routingRuleHandler); err != nil { + return nil, nil, err + } + createdPage.router = router + + err = page.SetViewport(&proto.EmulationSetDeviceMetricsOverride{Viewport: &proto.PageViewport{ + Scale: 1, + Width: float64(1920), + Height: float64(1080), + }}) + if err != nil { + return nil, nil, err + } + _, err = page.SetExtraHeaders([]string{"Accept-Language", "en, en-GB, en-us;"}) + if err != nil { + return nil, nil, err + } + + go router.Run() + data, err := createdPage.ExecuteActions(baseURL, actions) + if err != nil { + return nil, nil, err + } + return data, createdPage, nil +} + +// Close closes a browser page +func (p *Page) Close() { + p.router.Stop() + p.page.Close() +} + +// Page returns the current page for the actions +func (p *Page) Page() *rod.Page { + return p.page +} + +// Browser returns the browser that created the current page +func (p *Page) Browser() *rod.Browser { + return p.instance.engine +} + +// URL returns the URL for the current page. +func (p *Page) URL() string { + info, err := p.page.Info() + if err != nil { + return "" + } + return info.URL +} diff --git a/v2/pkg/protocols/headless/engine/page_actions.go b/v2/pkg/protocols/headless/engine/page_actions.go new file mode 100644 index 00000000..ed6ce124 --- /dev/null +++ b/v2/pkg/protocols/headless/engine/page_actions.go @@ -0,0 +1,505 @@ +package engine + +import ( + "io/ioutil" + "net" + "net/url" + "regexp" + "strconv" + "strings" + "time" + + "github.com/go-rod/rod" + "github.com/go-rod/rod/lib/proto" + "github.com/pkg/errors" + "github.com/segmentio/ksuid" + "github.com/valyala/fasttemplate" +) + +// ExecuteActions executes a list of actions on a page. +func (p *Page) ExecuteActions(baseURL *url.URL, actions []*Action) (map[string]string, error) { + var err error + + outData := make(map[string]string) + for _, act := range actions { + actionType := ActionStringToAction[act.ActionType] + + switch actionType { + case ActionNavigate: + err = p.NavigateURL(act, outData, baseURL) + case ActionScript: + err = p.RunScript(act, outData) + case ActionClick: + err = p.ClickElement(act, outData) + case ActionRightClick: + err = p.RightClickElement(act, outData) + case ActionTextInput: + err = p.InputElement(act, outData) + case ActionScreenshot: + err = p.Screenshot(act, outData) + case ActionTimeInput: + err = p.TimeInputElement(act, outData) + case ActionSelectInput: + err = p.SelectInputElement(act, outData) + case ActionWaitLoad: + err = p.WaitLoad(act, outData) + case ActionGetResource: + err = p.GetResource(act, outData) + case ActionExtract: + err = p.SelectInputElement(act, outData) + case ActionWaitEvent: + err = p.WaitEvent(act, outData) + case ActionFilesInput: + err = p.FilesInput(act, outData) + case ActionAddHeader: + err = p.ActionAddHeader(act, outData) + case ActionSetHeader: + err = p.ActionSetHeader(act, outData) + case ActionDeleteHeader: + err = p.ActionDeleteHeader(act, outData) + case ActionSetBody: + err = p.ActionSetBody(act, outData) + case ActionSetMethod: + err = p.ActionSetMethod(act, outData) + case ActionKeyboard: + err = p.KeyboardAction(act, outData) + default: + continue + } + if err != nil { + return nil, errors.Wrap(err, "error occured executing action") + } + } + return outData, nil +} + +type requestRule struct { + Action ActionType + Part string + Args map[string]string +} + +// ActionAddHeader executes a AddHeader action. +func (p *Page) ActionAddHeader(act *Action, out map[string]string) error { + in := act.GetArg("part") + + args := make(map[string]string) + args["key"] = act.GetArg("key") + args["value"] = act.GetArg("value") + rule := requestRule{ + Action: ActionAddHeader, + Part: in, + Args: args, + } + p.rules = append(p.rules, rule) + return nil +} + +// ActionSetHeader executes a SetHeader action. +func (p *Page) ActionSetHeader(act *Action, out map[string]string) error { + in := act.GetArg("part") + + args := make(map[string]string) + args["key"] = act.GetArg("key") + args["value"] = act.GetArg("value") + rule := requestRule{ + Action: ActionSetHeader, + Part: in, + Args: args, + } + p.rules = append(p.rules, rule) + return nil +} + +// ActionDeleteHeader executes a DeleteHeader action. +func (p *Page) ActionDeleteHeader(act *Action, out map[string]string) error { + in := act.GetArg("part") + + args := make(map[string]string) + args["key"] = act.GetArg("key") + rule := requestRule{ + Action: ActionDeleteHeader, + Part: in, + Args: args, + } + p.rules = append(p.rules, rule) + return nil +} + +// ActionSetBody executes a SetBody action. +func (p *Page) ActionSetBody(act *Action, out map[string]string) error { + in := act.GetArg("part") + + args := make(map[string]string) + args["body"] = act.GetArg("body") + rule := requestRule{ + Action: ActionSetBody, + Part: in, + Args: args, + } + p.rules = append(p.rules, rule) + return nil +} + +// ActionSetMethod executes an SetMethod action. +func (p *Page) ActionSetMethod(act *Action, out map[string]string) error { + in := act.GetArg("part") + + args := make(map[string]string) + args["method"] = act.GetArg("method") + rule := requestRule{ + Action: ActionSetMethod, + Part: in, + Args: args, + } + p.rules = append(p.rules, rule) + return nil +} + +// NavigateURL executes an ActionLoadURL actions loading a URL for the page. +func (p *Page) NavigateURL(action *Action, out map[string]string, parsed *url.URL) error { + url := action.GetArg("url") + if url == "" { + return errors.New("invalid arguments provided") + } + // Handle the dynamic value substituion here. + url, parsed = baseURLWithTemplatePrefs(url, parsed) + values := map[string]interface{}{"Hostname": parsed.Hostname()} + if strings.HasSuffix(parsed.Path, "/") && strings.Contains(url, "{{BaseURL}}/") { + parsed.Path = strings.TrimSuffix(parsed.Path, "/") + } + parsedString := parsed.String() + values["BaseURL"] = parsedString + + final := fasttemplate.ExecuteStringStd(url, "{{", "}}", values) + err := p.page.Navigate(final) + if err != nil { + return errors.Wrap(err, "could not navigate") + } + return nil +} + +// RunScript runs a script on the loaded page +func (p *Page) RunScript(action *Action, out map[string]string) error { + code := action.GetArg("code") + if code == "" { + return errors.New("invalid arguments provided") + } + if action.GetArg("hook") == "true" { + if _, err := p.page.EvalOnNewDocument(code); err != nil { + return err + } + } + data, err := p.page.Eval(code) + if err != nil { + return err + } + if data != nil { + out[action.Name] = data.Value.String() + } + return nil +} + +// ClickElement executes click actions for an element. +func (p *Page) ClickElement(act *Action, out map[string]string) error { + element, err := p.pageElementBy(act.Data) + if err != nil { + return errors.Wrap(err, "could not get element") + } + if err = element.ScrollIntoView(); err != nil { + return errors.Wrap(err, "could not scroll into view") + } + if err = element.Click(proto.InputMouseButtonLeft); err != nil { + return errors.Wrap(err, "could not click element") + } + return nil +} + +// KeyboardAction executes a keyboard action on the page. +func (p *Page) KeyboardAction(act *Action, out map[string]string) error { + return p.page.Keyboard.Press([]rune(act.GetArg("keys"))...) +} + +// RightClickElement executes right click actions for an element. +func (p *Page) RightClickElement(act *Action, out map[string]string) error { + element, err := p.pageElementBy(act.Data) + if err != nil { + return errors.Wrap(err, "could not get element") + } + if err = element.ScrollIntoView(); err != nil { + return errors.Wrap(err, "could not scroll into view") + } + if err = element.Click(proto.InputMouseButtonRight); err != nil { + return errors.Wrap(err, "could not right click element") + } + return nil +} + +// Screenshot executes screenshot action on a page +func (p *Page) Screenshot(act *Action, out map[string]string) error { + to := act.GetArg("to") + if to == "" { + to = ksuid.New().String() + out[act.Name] = to + } + var data []byte + var err error + if act.GetArg("fullpage") == "true" { + data, err = p.page.Screenshot(true, &proto.PageCaptureScreenshot{}) + } else { + data, err = p.page.Screenshot(false, &proto.PageCaptureScreenshot{}) + } + if err != nil { + return errors.Wrap(err, "could not take screenshot") + } + err = ioutil.WriteFile(to+".png", data, 0540) + if err != nil { + return errors.Wrap(err, "could not write screenshot") + } + return nil +} + +// InputElement executes input element actions for an element. +func (p *Page) InputElement(act *Action, out map[string]string) error { + value := act.GetArg("value") + if value == "" { + return errors.New("invalid arguments provided") + } + element, err := p.pageElementBy(act.Data) + if err != nil { + return errors.Wrap(err, "could not get element") + } + if err = element.ScrollIntoView(); err != nil { + return errors.Wrap(err, "could not scroll into view") + } + if err = element.Input(value); err != nil { + return errors.Wrap(err, "could not input element") + } + return nil +} + +// TimeInputElement executes time input on an element +func (p *Page) TimeInputElement(act *Action, out map[string]string) error { + value := act.GetArg("value") + if value == "" { + return errors.New("invalid arguments provided") + } + element, err := p.pageElementBy(act.Data) + if err != nil { + return errors.Wrap(err, "could not get element") + } + if err = element.ScrollIntoView(); err != nil { + return errors.Wrap(err, "could not scroll into view") + } + t, err := time.Parse(time.RFC3339, value) + if err != nil { + return errors.Wrap(err, "could not parse time") + } + if err := element.InputTime(t); err != nil { + return errors.Wrap(err, "could not input element") + } + return nil +} + +// SelectInputElement executes select input statement action on a element +func (p *Page) SelectInputElement(act *Action, out map[string]string) error { + value := act.GetArg("value") + if value == "" { + return errors.New("invalid arguments provided") + } + element, err := p.pageElementBy(act.Data) + if err != nil { + return errors.Wrap(err, "could not get element") + } + if err = element.ScrollIntoView(); err != nil { + return errors.Wrap(err, "could not scroll into view") + } + + selectedbool := false + if act.GetArg("selected") == "true" { + selectedbool = true + } + by := act.GetArg("selector") + if err := element.Select([]string{value}, selectedbool, selectorBy(by)); err != nil { + return errors.Wrap(err, "could not select input") + } + return nil +} + +// WaitLoad waits for the page to load +func (p *Page) WaitLoad(act *Action, out map[string]string) error { + p.page.Timeout(1 * time.Second).WaitNavigation(proto.PageLifecycleEventNameDOMContentLoaded)() + + // Wait for the window.onload event and also wait for the network requests + // to become idle for a maximum duration of 2 seconds. If the requests + // do not finish, + if err := p.page.WaitLoad(); err != nil { + return errors.Wrap(err, "could not reset mouse") + } + _ = p.page.WaitIdle(1 * time.Second) + return nil +} + +// GetResource gets a resource from an element from page. +func (p *Page) GetResource(act *Action, out map[string]string) error { + element, err := p.pageElementBy(act.Data) + if err != nil { + return errors.Wrap(err, "could not get element") + } + resource, err := element.Resource() + if err != nil { + return errors.Wrap(err, "could not get src for element") + } + out[act.Name] = string(resource) + return nil +} + +// FilesInput acts with a file input element on page +func (p *Page) FilesInput(act *Action, out map[string]string) error { + element, err := p.pageElementBy(act.Data) + if err != nil { + return errors.Wrap(err, "could not get element") + } + if err = element.ScrollIntoView(); err != nil { + return errors.Wrap(err, "could not scroll into view") + } + value := act.GetArg("value") + filesPaths := strings.Split(value, ",") + if err := element.SetFiles(filesPaths); err != nil { + return errors.Wrap(err, "could not set files") + } + return nil +} + +// ExtractElement extracts from an element on the page. +func (p *Page) ExtractElement(act *Action, out map[string]string) error { + element, err := p.pageElementBy(act.Data) + if err != nil { + return errors.Wrap(err, "could not get element") + } + if err = element.ScrollIntoView(); err != nil { + return errors.Wrap(err, "could not scroll into view") + } + switch act.GetArg("target") { + case "attribute": + attrName := act.GetArg("attribute") + if attrName == "" { + return errors.New("attribute can't be empty") + } + attrValue, err := element.Attribute(attrName) + if err != nil { + return errors.Wrap(err, "could not get attribute") + } + out[act.Name] = *attrValue + default: + text, err := element.Text() + if err != nil { + return errors.Wrap(err, "could not get element text node") + } + out[act.Name] = text + } + return nil +} + +type protoEvent struct { + event string +} + +// ProtoEvent returns the cdp.Event.Method +func (p *protoEvent) ProtoEvent() string { + return p.event +} + +// WaitEvent waits for an event to happen on the page. +func (p *Page) WaitEvent(act *Action, out map[string]string) error { + event := act.GetArg("event") + if event == "" { + return errors.New("event not recognized") + } + protoEvent := &protoEvent{event: event} + + // Uses another instance in order to be able to chain the timeout only to the wait operation + pagec := p.page + timeout := act.GetArg("timeout") + if timeout != "" { + ts, err := strconv.Atoi(timeout) + if err != nil { + return errors.Wrap(err, "could not get timeout") + } + if ts > 0 { + pagec = p.page.Timeout(time.Duration(ts) * time.Second) + } + } + // Just wait the event to happen + pagec.WaitEvent(protoEvent)() + return nil +} + +// pageElementBy returns a page element from a variety of inputs. +// +// Supported values for by: r -> selector & regex, x -> xpath, js -> eval js, +// search => query, default ("") => selector. +func (p *Page) pageElementBy(data map[string]string) (*rod.Element, error) { + by, ok := data["by"] + if !ok { + by = "" + } + page := p.page + + switch by { + case "r": + return page.ElementR(data["selector"], data["regex"]) + case "x", "xpath": + return page.ElementX(data["xpath"]) + case "js": + return page.ElementByJS(&rod.EvalOptions{JS: data["js"]}) + case "search": + elms, err := page.Search(0, 1, data["query"]) + if err != nil { + return nil, err + } + if len(elms) > 0 { + return elms[0], nil + } + return nil, errors.New("no such element") + default: + return page.Element(data["selector"]) + } +} + +// selectorBy returns a selector from a representation. +func selectorBy(selector string) rod.SelectorType { + switch selector { + case "r": + return rod.SelectorTypeRegex + case "css": + return rod.SelectorTypeCSSSector + case "regex": + return rod.SelectorTypeRegex + case "text": + fallthrough + default: + return rod.SelectorTypeText + } +} + +var ( + urlWithPortRegex = regexp.MustCompile(`{{BaseURL}}:(\d+)`) +) + +// baseURLWithTemplatePrefs returns the url for BaseURL keeping +// the template port and path preference over the user provided one. +func baseURLWithTemplatePrefs(data string, parsed *url.URL) (string, *url.URL) { + // template port preference over input URL port if template has a port + matches := urlWithPortRegex.FindAllStringSubmatch(data, -1) + if len(matches) == 0 { + return data, parsed + } + port := matches[0][1] + parsed.Host = net.JoinHostPort(parsed.Hostname(), port) + data = strings.ReplaceAll(data, ":"+port, "") + if parsed.Path == "" { + parsed.Path = "/" + } + return data, parsed +} diff --git a/v2/pkg/protocols/headless/engine/rules.go b/v2/pkg/protocols/headless/engine/rules.go new file mode 100644 index 00000000..5504fa18 --- /dev/null +++ b/v2/pkg/protocols/headless/engine/rules.go @@ -0,0 +1,48 @@ +package engine + +import ( + "fmt" + + "github.com/go-rod/rod" +) + +// routingRuleHandler handles proxy rule for actions related to request/response modification +func (p *Page) routingRuleHandler(ctx *rod.Hijack) { + for _, rule := range p.rules { + if rule.Part != "request" { + continue + } + + if rule.Action == ActionSetMethod { + ctx.Request.Req().Method = rule.Args["method"] + } else if rule.Action == ActionAddHeader { + ctx.Request.Req().Header.Add(rule.Args["key"], rule.Args["value"]) + } else if rule.Action == ActionSetHeader { + ctx.Request.Req().Header.Set(rule.Args["key"], rule.Args["value"]) + } else if rule.Action == ActionDeleteHeader { + ctx.Request.Req().Header.Del(rule.Args["key"]) + } else if rule.Action == ActionSetBody { + body := rule.Args["body"] + ctx.Request.Req().ContentLength = int64(len(body)) + ctx.Request.SetBody(body) + } + } + ctx.LoadResponse(p.instance.browser.httpclient, true) + + for _, rule := range p.rules { + if rule.Part != "response" { + continue + } + if rule.Action == ActionAddHeader { + ctx.Response.Headers().Add(rule.Args["key"], rule.Args["value"]) + } else if rule.Action == ActionSetHeader { + ctx.Response.Headers().Set(rule.Args["key"], rule.Args["value"]) + } else if rule.Action == ActionDeleteHeader { + ctx.Response.Headers().Del(rule.Args["key"]) + } else if rule.Action == ActionSetBody { + body := rule.Args["body"] + ctx.Response.Headers().Set("Content-Length", fmt.Sprintf("%d", len(body))) + ctx.Response.SetBody(rule.Args["body"]) + } + } +} diff --git a/v2/pkg/protocols/headless/headless.go b/v2/pkg/protocols/headless/headless.go new file mode 100644 index 00000000..77a36931 --- /dev/null +++ b/v2/pkg/protocols/headless/headless.go @@ -0,0 +1,52 @@ +package headless + +import ( + "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine" +) + +// Request contains a Headless protocol request to be made from a template +type Request struct { + ID string `yaml:"id"` + + // Steps is the list of actions to run for headless request + Steps []*engine.Action `yaml:"steps"` + + // Operators for the current request go here. + operators.Operators `yaml:",inline,omitempty"` + CompiledOperators *operators.Operators `yaml:"-"` + + // cache any variables that may be needed for operation. + options *protocols.ExecuterOptions +} + +// Step is a headless protocol request step. +type Step struct { + // Action is the headless action to execute for the script + Action string `yaml:"action"` +} + +// GetID returns the unique ID of the request if any. +func (r *Request) GetID() string { + return r.ID +} + +// Compile compiles the protocol request for further execution. +func (r *Request) Compile(options *protocols.ExecuterOptions) error { + if len(r.Matchers) > 0 || len(r.Extractors) > 0 { + compiled := &r.Operators + if err := compiled.Compile(); err != nil { + return errors.Wrap(err, "could not compile operators") + } + r.CompiledOperators = compiled + } + r.options = options + return nil +} + +// Requests returns the total number of requests the YAML rule will perform +func (r *Request) Requests() int { + return 1 +} diff --git a/v2/pkg/protocols/headless/operators.go b/v2/pkg/protocols/headless/operators.go new file mode 100644 index 00000000..1d7ebb04 --- /dev/null +++ b/v2/pkg/protocols/headless/operators.go @@ -0,0 +1,122 @@ +package headless + +import ( + "time" + + "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) + +// Match matches a generic data response again a given matcher +func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool { + partString := matcher.Part + switch partString { + case "body", "resp", "": + partString = "data" + } + + item, ok := data[partString] + if !ok { + return false + } + itemStr := types.ToString(item) + + switch matcher.GetType() { + case matchers.SizeMatcher: + return matcher.Result(matcher.MatchSize(len(itemStr))) + case matchers.WordsMatcher: + return matcher.Result(matcher.MatchWords(itemStr)) + case matchers.RegexMatcher: + return matcher.Result(matcher.MatchRegex(itemStr)) + case matchers.BinaryMatcher: + return matcher.Result(matcher.MatchBinary(itemStr)) + case matchers.DSLMatcher: + return matcher.Result(matcher.MatchDSL(data)) + } + return false +} + +// Extract performs extracting operation for a extractor on model and returns true or false. +func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { + partString := extractor.Part + switch partString { + case "body", "resp", "": + partString = "data" + } + + item, ok := data[partString] + if !ok { + return nil + } + itemStr := types.ToString(item) + + switch extractor.GetType() { + case extractors.RegexExtractor: + return extractor.ExtractRegex(itemStr) + case extractors.KValExtractor: + return extractor.ExtractKval(data) + } + return nil +} + +// responseToDSLMap converts a DNS response to a map for use in DSL matching +func (r *Request) responseToDSLMap(resp, req string, host, matched string) output.InternalEvent { + data := make(output.InternalEvent, 5) + + // Some data regarding the request metadata + data["host"] = host + data["matched"] = matched + data["req"] = req + data["data"] = resp + data["template-id"] = r.options.TemplateID + data["template-info"] = r.options.TemplateInfo + return data +} + +// MakeResultEvent creates a result event from internal wrapped event +func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { + if len(wrapped.OperatorsResult.DynamicValues) > 0 { + return nil + } + results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1) + + // If we have multiple matchers with names, write each of them separately. + if len(wrapped.OperatorsResult.Matches) > 0 { + for k := range wrapped.OperatorsResult.Matches { + data := r.makeResultEventItem(wrapped) + data.MatcherName = k + results = append(results, data) + } + } else if len(wrapped.OperatorsResult.Extracts) > 0 { + for k, v := range wrapped.OperatorsResult.Extracts { + data := r.makeResultEventItem(wrapped) + data.ExtractedResults = v + data.ExtractorName = k + results = append(results, data) + } + } else { + data := r.makeResultEventItem(wrapped) + results = append(results, data) + } + return results +} + +func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { + data := &output.ResultEvent{ + TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), + Info: wrapped.InternalEvent["template-info"].(map[string]interface{}), + Type: "headless", + Host: types.ToString(wrapped.InternalEvent["host"]), + Matched: types.ToString(wrapped.InternalEvent["matched"]), + ExtractedResults: wrapped.OperatorsResult.OutputExtracts, + Timestamp: time.Now(), + IP: types.ToString(wrapped.InternalEvent["ip"]), + } + if r.options.Options.JSONRequests { + data.Request = types.ToString(wrapped.InternalEvent["request"]) + data.Response = types.ToString(wrapped.InternalEvent["data"]) + } + return data +} diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go new file mode 100644 index 00000000..5a00962a --- /dev/null +++ b/v2/pkg/protocols/headless/request.go @@ -0,0 +1,79 @@ +package headless + +import ( + "net/url" + "strings" + + "github.com/pkg/errors" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" +) + +var _ protocols.Request = &Request{} + +// ExecuteWithResults executes the protocol requests and returns results instead of writing them. +func (r *Request) ExecuteWithResults(input string, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error { + instance, err := r.options.Browser.NewInstance() + if err != nil { + r.options.Output.Request(r.options.TemplateID, input, "headless", err) + r.options.Progress.DecrementRequests(1) + return errors.Wrap(err, "could get html element") + } + defer instance.Close() + + parsed, err := url.Parse(input) + if err != nil { + r.options.Output.Request(r.options.TemplateID, input, "headless", err) + r.options.Progress.DecrementRequests(1) + return errors.Wrap(err, "could get html element") + } + out, page, err := instance.Run(parsed, r.Steps) + if err != nil { + r.options.Output.Request(r.options.TemplateID, input, "headless", err) + r.options.Progress.DecrementRequests(1) + return errors.Wrap(err, "could get html element") + } + defer page.Close() + + r.options.Output.Request(r.options.TemplateID, input, "headless", nil) + r.options.Progress.IncrementRequests() + gologger.Verbose().Msgf("Sent Headless request to %s", input) + + reqBuilder := &strings.Builder{} + if r.options.Options.Debug || r.options.Options.DebugRequests { + gologger.Info().Msgf("[%s] Dumped Headless request for %s", r.options.TemplateID, input) + + for _, act := range r.Steps { + reqBuilder.WriteString(act.String()) + reqBuilder.WriteString("\n") + } + gologger.Print().Msgf("%s", reqBuilder.String()) + } + + var respBody string + html, err := page.Page().Element("html") + if err == nil { + respBody, _ = html.HTML() + } + outputEvent := r.responseToDSLMap(respBody, reqBuilder.String(), input, input) + for k, v := range out { + outputEvent[k] = v + } + + if r.options.Options.Debug || r.options.Options.DebugResponse { + gologger.Debug().Msgf("[%s] Dumped Headless response for %s", r.options.TemplateID, input) + gologger.Print().Msgf("%s", respBody) + } + + event := &output.InternalWrappedEvent{InternalEvent: outputEvent} + if r.CompiledOperators != nil { + result, ok := r.CompiledOperators.Execute(outputEvent, r.Match, r.Extract) + if ok && result != nil { + event.OperatorsResult = result + event.Results = r.MakeResultEvent(event) + } + } + callback(event) + return nil +} diff --git a/v2/pkg/protocols/headless/request_test.go b/v2/pkg/protocols/headless/request_test.go new file mode 100644 index 00000000..b75ee392 --- /dev/null +++ b/v2/pkg/protocols/headless/request_test.go @@ -0,0 +1,55 @@ +package headless + +import ( + "fmt" + "testing" + + "github.com/projectdiscovery/nuclei/v2/internal/testutils" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine" + "github.com/stretchr/testify/require" +) + +func TestHeadlessExecuteWithResults(t *testing.T) { + options := testutils.DefaultOptions + + testutils.Init(options) + templateID := "testing-headless" + request := &Request{ + ID: templateID, + Steps: []*engine.Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + }, + Operators: operators.Operators{ + Matchers: []*matchers.Matcher{{ + Name: "test", + Part: "data", + Type: "word", + Words: []string{"Example Domain"}, + }}, + }, + } + executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + ID: templateID, + Info: map[string]interface{}{"severity": "low", "name": "test"}, + }) + options.Headless = true + browser, err := engine.New(options) + require.Nil(t, err, "could not create browser") + executerOpts.Browser = browser + + err = request.Compile(executerOpts) + require.Nil(t, err, "could not compile headless request") + + metadata := make(output.InternalEvent) + previous := make(output.InternalEvent) + err = request.ExecuteWithResults("https://example.com", metadata, previous, func(event *output.InternalWrappedEvent) { + for _, result := range event.Results { + fmt.Printf("Result: %+v\n", result) + } + }) + require.Nil(t, err, "could not execute headless request") +} diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 040cc797..5f17e9e1 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -55,7 +55,7 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa "Hostname": parsed.Hostname(), }) - isRawRequest := strings.Contains(data, "\n") + isRawRequest := len(r.request.Raw) > 0 if !isRawRequest && strings.HasSuffix(parsed.Path, "/") && strings.Contains(data, "{{BaseURL}}/") { parsed.Path = strings.TrimSuffix(parsed.Path, "/") } @@ -114,7 +114,7 @@ func (r *requestGenerator) makeHTTPRequestFromModel(ctx context.Context, data st // makeHTTPRequestFromRaw creates a *http.Request from a raw request func (r *requestGenerator) makeHTTPRequestFromRaw(ctx context.Context, baseURL, data string, values, payloads map[string]interface{}) (*generatedRequest, error) { - data += "\r\n" + data = strings.TrimSuffix(data, "\r\n") return r.handleRawWithPaylods(ctx, data, baseURL, values, payloads) } diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index 9e254051..cbf9e38e 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -25,7 +25,7 @@ type Request struct { ReadSize int `yaml:"read-size"` // Operators for the current request go here. - operators.Operators `yaml:",inline"` + operators.Operators `yaml:",inline,omitempty"` CompiledOperators *operators.Operators // cache any variables that may be needed for operation. diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index cc279af2..b14d3aeb 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -8,6 +8,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/projectfile" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/issues" "github.com/projectdiscovery/nuclei/v2/pkg/types" "go.uber.org/ratelimit" @@ -47,6 +48,8 @@ type ExecuterOptions struct { Catalogue *catalogue.Catalogue // ProjectFile is the project file for nuclei ProjectFile *projectfile.ProjectFile + // Browser is a browser engine for running headless templates + Browser *engine.Browser Operators []*operators.Operators // only used by offlinehttp module } diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index a0b5cb96..21d50ffd 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -60,7 +60,7 @@ func Parse(filePath string, options protocols.ExecuterOptions) (*Template, error options.TemplatePath = filePath // If no requests, and it is also not a workflow, return error. - if len(template.RequestsDNS)+len(template.RequestsHTTP)+len(template.RequestsFile)+len(template.RequestsNetwork)+len(template.Workflows) == 0 { + if len(template.RequestsDNS)+len(template.RequestsHTTP)+len(template.RequestsFile)+len(template.RequestsNetwork)+len(template.RequestsHeadless)+len(template.Workflows) == 0 { return nil, fmt.Errorf("no requests defined for %s", template.ID) } @@ -109,6 +109,12 @@ func Parse(filePath string, options protocols.ExecuterOptions) (*Template, error } template.Executer = executer.NewExecuter(requests, &options) } + if len(template.RequestsHeadless) > 0 && !options.Options.OfflineHTTP && options.Options.Headless { + for _, req := range template.RequestsHeadless { + requests = append(requests, req) + } + template.Executer = executer.NewExecuter(requests, &options) + } if template.Executer != nil { err := template.Executer.Compile() if err != nil { diff --git a/v2/pkg/templates/preprocessors.go b/v2/pkg/templates/preprocessors.go index d8d4de83..83241a0a 100644 --- a/v2/pkg/templates/preprocessors.go +++ b/v2/pkg/templates/preprocessors.go @@ -19,6 +19,9 @@ func (t *Template) expandPreprocessors(data []byte) []byte { continue } value := expression[1] + if strings.Contains(value, "(") || strings.Contains(value, ")") { + continue + } if _, ok := foundMap[value]; ok { continue diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index 2e55bfc1..f2e84150 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -4,6 +4,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/file" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network" "github.com/projectdiscovery/nuclei/v2/pkg/workflows" @@ -23,13 +24,15 @@ type Template struct { RequestsFile []*file.Request `yaml:"file,omitempty"` // RequestsNetwork contains the network request to make in the template RequestsNetwork []*network.Request `yaml:"network,omitempty"` + // RequestsHeadless contains the headless request to make in the template. + RequestsHeadless []*headless.Request `yaml:"headless,omitempty"` // Workflows is a yaml based workflow declaration code. - workflows.Workflow `yaml:",inline"` - CompiledWorkflow *workflows.Workflow + workflows.Workflow `yaml:",inline,omitempty"` + CompiledWorkflow *workflows.Workflow `yaml:"-"` // TotalRequests is the total number of requests for the template. - TotalRequests int + TotalRequests int `yaml:"-"` // Executer is the actual template executor for running template requests - Executer protocols.Executer + Executer protocols.Executer `yaml:"-"` } diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index f7ab7944..3aa7d627 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -94,6 +94,8 @@ type Options struct { OfflineHTTP bool // ResolversFile is a file containing resolvers for nuclei. ResolversFile string - + // Headless specifies whether to allow headless mode templates + Headless bool + ShowBrowser bool // show the browser too InternalResolversList []string // normalized from resolvers flag as well as file provided. } diff --git a/v2/pkg/workflows/workflows.go b/v2/pkg/workflows/workflows.go index 0b520043..860b1fdc 100644 --- a/v2/pkg/workflows/workflows.go +++ b/v2/pkg/workflows/workflows.go @@ -5,7 +5,7 @@ import "github.com/projectdiscovery/nuclei/v2/pkg/protocols" // Workflow is a workflow to execute with chained requests, etc. type Workflow struct { // Workflows is a yaml based workflow declaration code. - Workflows []*WorkflowTemplate `yaml:"workflows"` + Workflows []*WorkflowTemplate `yaml:"workflows,omitempty"` } // WorkflowTemplate is a template to be ran as part of a workflow