daffainfo
/
nuclei
mirror of https://github.com/daffainfo/nuclei.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

poc working implementation

dev
Mzack9999 2020-10-18 03:09:24 +02:00
parent d5cd01d43b
commit 5623fd4b36
7 changed files with 232 additions and 176 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
v2/internal/runner/processor.go
View File

@ -48,7 +48,6 @@ func (r *Runner) processTemplateWithList(p progress.IProgress, template *templat
ColoredOutput: !r.options.NoColor,
Colorizer: r.colorizer,
Decolorizer: r.decolorizer,
HM: r.hm,
})
case *requests.BulkHTTPRequest:
httpExecuter, err = executer.NewHTTPExecuter(&executer.HTTPOptions{
@ -68,7 +67,7 @@ func (r *Runner) processTemplateWithList(p progress.IProgress, template *templat
Colorizer: &r.colorizer,
Decolorizer: r.decolorizer,
StopAtFirstMatch: r.options.StopAtFirstMatch,
HM: r.hm,
PF: r.pf,
})
}
@ -228,6 +227,7 @@ func (r *Runner) preloadWorkflowTemplates(p progress.IProgress, workflow *workfl
ColoredOutput: !r.options.NoColor,
Colorizer: &r.colorizer,
Decolorizer: r.decolorizer,
PF: r.pf,
}
} else if len(t.RequestsDNS) > 0 {
template.DNSOptions = &executer.DNSOptions{

17
v2/internal/runner/runner.go
View File

@ -10,16 +10,17 @@ import (
"strings"
"github.com/logrusorgru/aurora"
"github.com/remeh/sizedwaitgroup"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/hmap/store/hybrid"
"github.com/projectdiscovery/nuclei/v2/internal/bufwriter"
"github.com/projectdiscovery/nuclei/v2/internal/progress"
"github.com/projectdiscovery/nuclei/v2/pkg/atomicboolean"
"github.com/projectdiscovery/nuclei/v2/pkg/colorizer"
"github.com/projectdiscovery/nuclei/v2/pkg/globalratelimiter"
"github.com/projectdiscovery/nuclei/v2/pkg/projectfile"
projetctfile "github.com/projectdiscovery/nuclei/v2/pkg/projectfile"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
"github.com/projectdiscovery/nuclei/v2/pkg/workflows"
"github.com/remeh/sizedwaitgroup"
)
// Runner is a client for running the enumeration process.
@ -35,7 +36,7 @@ type Runner struct {
// options contains configuration options for runner
options *Options
hm *hybrid.HybridMap
pf *projectfile.ProjectFile
// progress tracking
progress progress.IProgress
@ -169,12 +170,10 @@ func New(options *Options) (*Runner, error) {
// create project file if requested or load existing one
if options.Project {
hOptions := hybrid.DefaultDiskOptions
hOptions.Path = options.ProjectPath
var err error
runner.hm, err = hybrid.New(hOptions)
runner.pf, err = projetctfile.New(&projetctfile.Options{Path: options.ProjectPath, Cleanup: options.ProjectPath == ""})
if err != nil {
return runner, err
return nil, err
}
}
@ -187,8 +186,8 @@ func (r *Runner) Close() {
r.output.Close()
}
os.Remove(r.tempFile)
if r.hm != nil {
r.hm.Close()
if r.pf != nil {
r.pf.Close()
}
}

2
v2/pkg/executer/executer_dns.go
View File

@ -52,7 +52,6 @@ type DNSOptions struct {
Template *templates.Template
DNSRequest *requests.DNSRequest
Writer *bufwriter.Writer
HM *hybrid.HybridMap
Colorizer colorizer.NucleiColorizer
Decolorizer *regexp.Regexp
@ -74,7 +73,6 @@ func NewDNSExecuter(options *DNSOptions) *DNSExecuter {
coloredOutput: options.ColoredOutput,
colorizer: options.Colorizer,
decolorizer: options.Decolorizer,
hm: options.HM,
}
return executer

45
v2/pkg/executer/executer_http.go
View File

@ -19,13 +19,13 @@ import (
"github.com/pkg/errors"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/hmap/store/hybrid"
"github.com/projectdiscovery/httpx/common/cache"
"github.com/projectdiscovery/nuclei/v2/internal/bufwriter"
"github.com/projectdiscovery/nuclei/v2/internal/progress"
"github.com/projectdiscovery/nuclei/v2/pkg/colorizer"
"github.com/projectdiscovery/nuclei/v2/pkg/globalratelimiter"
"github.com/projectdiscovery/nuclei/v2/pkg/matchers"
projetctfile "github.com/projectdiscovery/nuclei/v2/pkg/projectfile"
"github.com/projectdiscovery/nuclei/v2/pkg/requests"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
"github.com/projectdiscovery/rawhttp"
@ -42,7 +42,7 @@ const (
// HTTPExecuter is client for performing HTTP requests
// for a template.
type HTTPExecuter struct {
hm *hybrid.HybridMap
pf *projetctfile.ProjectFile
customHeaders requests.CustomHeaders
colorizer colorizer.NucleiColorizer
httpClient *retryablehttp.Client
@ -79,7 +79,7 @@ type HTTPOptions struct {
CookieReuse bool
ColoredOutput bool
StopAtFirstMatch bool
HM *hybrid.HybridMap
PF *projetctfile.ProjectFile
}
// NewHTTPExecuter creates a new HTTP executer from a template
@ -133,7 +133,7 @@ func NewHTTPExecuter(options *HTTPOptions) (*HTTPExecuter, error) {
colorizer: *options.Colorizer,
decolorizer: options.Decolorizer,
stopAtFirstMatch: options.StopAtFirstMatch,
hm: options.HM,
pf: options.PF,
}
return executer, nil
@ -326,7 +326,7 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest,
fromcache bool
)
if e.debug || e.hm != nil {
if e.debug || e.pf != nil {
dumpedRequest, err = requests.Dump(request, reqURL)
if err != nil {
return err
@ -364,19 +364,12 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest,
}
} else {
// if nuclei-project is available check if the request was already sent previously
if e.hm != nil {
reqHash, err := hash(dumpedRequest)
// if the computation was successful check within the cache
if err == nil {
data, ok := e.hm.Get(reqHash)
// if found reuse the item
if ok {
var httprecord HTTPRecord
httprecord.Response = newInternalResponse()
unmarshal(data, &httprecord)
resp = fromInternalResponse(httprecord.Response)
fromcache = true
}
if e.pf != nil {
// if unavailable fail silently
fromcache = true
resp, err = e.pf.Get(dumpedRequest)
if err != nil {
fromcache = false
}
}
@ -427,20 +420,8 @@ func (e *HTTPExecuter) handleHTTP(reqURL string, request *requests.HTTPRequest,
}
// if nuclei-project is enabled store the response if not previously done
if e.hm != nil && !fromcache {
reqHash, err := hash(dumpedRequest)
// if the computation was successful store within the cache
if err == nil {
var httprecord HTTPRecord
intResp := toInternalResponse(resp, data)
httprecord.Request = dumpedRequest
httprecord.Response = intResp
data, err := marshal(httprecord)
// once marshaled without errors store in the cache
if err == nil {
e.hm.Set(reqHash, data)
}
}
if e.pf != nil && !fromcache {
e.pf.Set(dumpedRequest, resp, data)
}
// Convert response body from []byte to string with zero copy

131
v2/pkg/executer/http_utils.go
View File

@ -1,12 +1,6 @@
package executer
import (
"bytes"
"crypto/sha256"
"encoding/gob"
"encoding/hex"
"io"
"io/ioutil"
"net/http"
"strings"
"unsafe"
@ -55,128 +49,3 @@ func headersToString(headers http.Header) string {
return builder.String()
}
func hash(v interface{}) (string, error) {
data, err := marshal(v)
if err != nil {
return "", err
}
sh := sha256.New()
io.WriteString(sh, string(data))
return hex.EncodeToString(sh.Sum(nil)), nil
}
func marshal(data interface{}) ([]byte, error) {
var b bytes.Buffer
enc := gob.NewEncoder(&b)
err := enc.Encode(data)
if err != nil {
return nil, err
}
return b.Bytes(), nil
}
func unmarshal(data []byte, obj interface{}) error {
var b bytes.Buffer
dec := gob.NewDecoder(&b)
err := dec.Decode(obj)
if err != nil {
return err
}
return nil
}
type HTTPRecord struct {
Request []byte
Response *InternalResponse
}
type InternalRequest struct {
Target string
HTTPMajor int
HTTPMinor int
Method string
Headers map[string][]string
Body []byte
}
type InternalResponse struct {
HTTPMajor int
HTTPMinor int
StatusCode int
StatusReason string
Headers map[string][]string
Body []byte
}
func newInternalRquest() *InternalRequest {
return &InternalRequest{
Headers: make(map[string][]string),
}
}
func newInternalResponse() *InternalResponse {
return &InternalResponse{
Headers: make(map[string][]string),
}
}
func toInternalRequest(req *http.Request, target string, body []byte) *InternalRequest {
intReq := newInternalRquest()
intReq.Target = target
intReq.HTTPMajor = req.ProtoMajor
intReq.HTTPMinor = req.ProtoMinor
for k, v := range req.Header {
intReq.Headers[k] = v
}
intReq.Headers = req.Header
intReq.Method = req.Method
intReq.Body = body
return intReq
}
func toInternalResponse(resp *http.Response, body []byte) *InternalResponse {
intResp := newInternalResponse()
intResp.HTTPMajor = resp.ProtoMajor
intResp.HTTPMinor = resp.ProtoMinor
intResp.StatusCode = resp.StatusCode
intResp.StatusReason = resp.Status
for k, v := range resp.Header {
intResp.Headers[k] = v
}
intResp.Body = body
return intResp
}
func fromInternalResponse(intResp *InternalResponse) *http.Response {
var contentLength int64
if intResp.Body != nil {
contentLength = int64(len(intResp.Body))
}
return &http.Response{
ProtoMinor: intResp.HTTPMinor,
ProtoMajor: intResp.HTTPMajor,
Status: intResp.StatusReason,
StatusCode: intResp.StatusCode,
Header: intResp.Headers,
ContentLength: contentLength,
Body: ioutil.NopCloser(bytes.NewReader(intResp.Body)),
}
}
func fromInternalRequest(intReq *InternalRequest) *http.Request {
return &http.Request{
ProtoMinor: intReq.HTTPMinor,
ProtoMajor: intReq.HTTPMajor,
Header: intReq.Headers,
ContentLength: int64(len(intReq.Body)),
Body: ioutil.NopCloser(bytes.NewReader(intReq.Body)),
}
}

135
v2/pkg/projectfile/httputil.go Normal file
View File

@ -0,0 +1,135 @@
package projectfile
import (
"bytes"
"crypto/sha256"
"encoding/gob"
"encoding/hex"
"io"
"io/ioutil"
"net/http"
)
func hash(v interface{}) (string, error) {
data, err := marshal(v)
if err != nil {
return "", err
}
sh := sha256.New()
io.WriteString(sh, string(data))
return hex.EncodeToString(sh.Sum(nil)), nil
}
func marshal(data interface{}) ([]byte, error) {
var b bytes.Buffer
enc := gob.NewEncoder(&b)
err := enc.Encode(data)
if err != nil {
return nil, err
}
return b.Bytes(), nil
}
func unmarshal(data []byte, obj interface{}) error {
dec := gob.NewDecoder(bytes.NewBuffer(data))
err := dec.Decode(obj)
if err != nil {
return err
}
return nil
}
type HTTPRecord struct {
Request []byte
Response *InternalResponse
}
type InternalRequest struct {
Target string
HTTPMajor int
HTTPMinor int
Method string
Headers map[string][]string
Body []byte
}
type InternalResponse struct {
HTTPMajor int
HTTPMinor int
StatusCode int
StatusReason string
Headers map[string][]string
Body []byte
}
func newInternalRquest() *InternalRequest {
return &InternalRequest{
Headers: make(map[string][]string),
}
}
func newInternalResponse() *InternalResponse {
return &InternalResponse{
Headers: make(map[string][]string),
}
}
func toInternalRequest(req *http.Request, target string, body []byte) *InternalRequest {
intReq := newInternalRquest()
intReq.Target = target
intReq.HTTPMajor = req.ProtoMajor
intReq.HTTPMinor = req.ProtoMinor
for k, v := range req.Header {
intReq.Headers[k] = v
}
intReq.Headers = req.Header
intReq.Method = req.Method
intReq.Body = body
return intReq
}
func toInternalResponse(resp *http.Response, body []byte) *InternalResponse {
intResp := newInternalResponse()
intResp.HTTPMajor = resp.ProtoMajor
intResp.HTTPMinor = resp.ProtoMinor
intResp.StatusCode = resp.StatusCode
intResp.StatusReason = resp.Status
for k, v := range resp.Header {
intResp.Headers[k] = v
}
intResp.Body = body
return intResp
}
func fromInternalResponse(intResp *InternalResponse) *http.Response {
var contentLength int64
if intResp.Body != nil {
contentLength = int64(len(intResp.Body))
}
return &http.Response{
ProtoMinor: intResp.HTTPMinor,
ProtoMajor: intResp.HTTPMajor,
Status: intResp.StatusReason,
StatusCode: intResp.StatusCode,
Header: intResp.Headers,
ContentLength: contentLength,
Body: ioutil.NopCloser(bytes.NewReader(intResp.Body)),
}
}
func fromInternalRequest(intReq *InternalRequest) *http.Request {
return &http.Request{
ProtoMinor: intReq.HTTPMinor,
ProtoMajor: intReq.HTTPMajor,
Header: intReq.Headers,
ContentLength: int64(len(intReq.Body)),
Body: ioutil.NopCloser(bytes.NewReader(intReq.Body)),
}
}

74
v2/pkg/projectfile/project.go Normal file
View File

@ -0,0 +1,74 @@
package projectfile
import (
"fmt"
"net/http"
"github.com/projectdiscovery/hmap/store/hybrid"
)
type Options struct {
Path string
Cleanup bool
}
type ProjectFile struct {
Path string
hm *hybrid.HybridMap
}
func New(options *Options) (*ProjectFile, error) {
var p ProjectFile
hOptions := hybrid.DefaultDiskOptions
hOptions.Path = options.Path
hOptions.Cleanup = options.Cleanup
var err error
p.hm, err = hybrid.New(hOptions)
if err != nil {
return nil, err
}
return &p, nil
}
func (pf *ProjectFile) Get(req []byte) (*http.Response, error) {
reqHash, err := hash(req)
if err != nil {
return nil, err
}
data, ok := pf.hm.Get(reqHash)
if !ok {
return nil, fmt.Errorf("Not found")
}
var httprecord HTTPRecord
httprecord.Response = newInternalResponse()
err = unmarshal(data, &httprecord)
if err != nil {
return nil, err
}
return fromInternalResponse(httprecord.Response), nil
}
func (pf *ProjectFile) Set(req []byte, resp *http.Response, data []byte) error {
reqHash, err := hash(req)
if err != nil {
return err
}
var httprecord HTTPRecord
httprecord.Request = req
httprecord.Response = toInternalResponse(resp, data)
data, err = marshal(httprecord)
if err != nil {
return err
}
return pf.hm.Set(reqHash, data)
}
func (pf *ProjectFile) Close() {
pf.hm.Close()
}