Removed es exporter cli args + misc changes

v2/cmd/nuclei/issue-tracker-config.yaml

@@ -47,3 +47,20 @@
 #  project-name: ""
 #  # issue-type is the name of the created issue type
 #  issue-type: ""
+
+# elasticsearch contains configuration options for elasticsearch exporter
+#elasticsearch:
+#  # IP for elasticsearch instance
+#  ip: 127.0.0.1
+#  # Port is the port of elasticsearch instance
+#  port: 9200
+#  # IndexName is the name of the elasticsearch index
+#  index-name: nuclei
+#  # SSL enables ssl for elasticsearch connection
+#  # ssl: false
+#  # SSLVerification disables SSL verification for elasticsearch
+#  # ssl-verification: false
+#  # Username for the elasticsearch instance
+#  # username: test
+#  # Pasword is the password for elasticsearch instance
+#  # password: test

v2/cmd/nuclei/main.go

@@ -90,13 +90,6 @@ on extensive configurability, massive extensibility and ease of use.`)
 
 		flagSet.StringVarP(&options.DiskExportDirectory, "markdown-export", "me", "", "directory to export results in markdown format"),
 		flagSet.StringVarP(&options.SarifExport, "sarif-export", "se", "", "file to export results in SARIF format"),
-
-		flagSet.StringVar(&options.ESExport, "es-ip", "", "the ip address for elasticsearch . Ex: 127.0.0.1"),
-		flagSet.IntVar(&options.ESPort, "es-port", 9200, "the port number on which elasticsearch is running"),
-		flagSet.BoolVar(&options.ESSSL, "es-ssl", true, "false if http is to be used"),
-		flagSet.BoolVar(&options.ESSSLVerificaiton, "es-ssl-verify", true, "set to false if ssl verificaiton needs to be disabled"),
-		flagSet.StringVar(&options.ESUsername, "es-user", "", "the username for elasticsearch"),
-		flagSet.StringVar(&options.ESPassword, "es-password", "", "the password for elasticsearch"),
 	)
 
 	createGroup(flagSet, "configs", "Configurations",

v2/internal/runner/runner.go

@@ -35,7 +35,6 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine"
 	"github.com/projectdiscovery/nuclei/v2/pkg/reporting"
 	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/disk"
-	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/es"
 	"github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/sarif"
 	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
@@ -106,22 +105,6 @@ func New(options *types.Options) (*Runner, error) {
 			reportingOptions.DiskExporter = &disk.Options{Directory: options.DiskExportDirectory}
 		}
 	}
-	if options.ESExport != "" {
-		es_temp := &es.Options{
-			ESIP:              options.ESExport,
-			ESPort:            options.ESPort,
-			ESSSL:             options.ESSSL,
-			ESSSLVerificaiton: options.ESSSLVerificaiton,
-			ESUsername:        options.ESUsername,
-			ESPassword:        options.ESPassword,
-		}
-		if reportingOptions != nil {
-			reportingOptions.ElasticsearchExporter = es_temp
-		} else {
-			reportingOptions = &reporting.Options{}
-			reportingOptions.ElasticsearchExporter = es_temp
-		}
-	}
 	if options.SarifExport != "" {
 		if reportingOptions != nil {
 			reportingOptions.SarifExporter = &sarif.Options{File: options.SarifExport}

v2/pkg/reporting/exporters/es/elasticsearch.go

@@ -1,35 +1,37 @@
 package es
 
 import (
+	"bytes"
 	"crypto/tls"
-	"errors"
 	"fmt"
 	"io"
 	"net/http"
-	"strings"
-	"sync"
 	"time"
 
-	b64 "encoding/base64"
+	"encoding/base64"
 	"encoding/json"
 
+	"github.com/pkg/errors"
 	"github.com/projectdiscovery/nuclei/v2/pkg/output"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate"
 )
 
 // Options contains necessary options required for elasticsearch communicaiton
 type Options struct {
-	// Full url for elasticsearch
-	ESIP string
-	// Full url for elasticsearch
-	ESPort int
-	// Enable/Disable SSL
-	ESSSL bool
-	// Enable/DIsable SSL verificaiton
-	ESSSLVerificaiton bool
-	// Elasticsearch username
-	ESUsername string
-	// Elasticsearch password
-	ESPassword string
+	// IP for elasticsearch instance
+	IP string `yaml:"ip"`
+	// Port is the port of elasticsearch instance
+	Port int `yaml:"port"`
+	// SSL enables ssl for elasticsearch connection
+	SSL bool `yaml:"ssl"`
+	// SSLVerification disables SSL verification for elasticsearch
+	SSLVerification bool `yaml:"ssl-verification"`
+	// Username for the elasticsearch instance
+	Username string `yaml:"username"`
+	// Pasword is the password for elasticsearch instance
+	Password string `yaml:"password"`
+	// IndexName is the name of the elasticsearch index
+	IndexName string `yaml:"index-name"`
 }
 
 type data struct {
@@ -39,59 +41,57 @@ type data struct {
 
 // Exporter type for elasticsearch
 type Exporter struct {
-	elasticsearch *http.Client
-	req           *http.Request
-	wg            *sync.Mutex
+	url            string
+	authentication string
+	elasticsearch  *http.Client
 }
 
 // New creates and returns a new exporter for elasticsearch
 func New(option *Options) (*Exporter, error) {
-
 	var ei *Exporter
 
-	tr := &http.Transport{
-		TLSClientConfig: &tls.Config{InsecureSkipVerify: option.ESSSLVerificaiton},
+	client := &http.Client{
+		Timeout: 5 * time.Second,
+		Transport: &http.Transport{
+			MaxIdleConns:        10,
+			MaxIdleConnsPerHost: 10,
+			DialContext:         protocolstate.Dialer.Dial,
+			TLSClientConfig:     &tls.Config{InsecureSkipVerify: option.SSLVerification},
+		},
 	}
-	c := &http.Client{
-		Timeout:   5 * time.Second,
-		Transport: tr,
-	}
-
 	// preparing url for elasticsearch
-	url := `http://`
-	if option.ESSSL {
-		url = `https://`
+	scheme := "http://"
+	if option.SSL {
+		scheme = "https://"
 	}
-	url = fmt.Sprintf(url+"%s:%d/nuclei-export/_doc", option.ESIP, option.ESPort)
-
-	// creafting a request
-	req2, err := http.NewRequest(http.MethodPost, url, nil)
-	if err != nil {
-		return ei, nil
-	}
-
 	// if authentication is required
-	if len(option.ESUsername) != 0 && len(option.ESPassword) != 0 {
-		auth := b64.StdEncoding.EncodeToString([]byte(option.ESUsername + ":" + option.ESPassword))
+	var authentication string
+	if len(option.Username) > 0 && len(option.Password) > 0 {
+		auth := base64.StdEncoding.EncodeToString([]byte(option.Username + ":" + option.Password))
 		auth = "Basic " + auth
-		req2.Header.Add("Authorization", auth)
+		authentication = auth
 	}
-	req2.Header.Add("Content-Type", "application/json")
+	url := fmt.Sprintf("%s%s:%d/%s/_doc", scheme, option.IP, option.Port, option.IndexName)
 
 	ei = &Exporter{
-		elasticsearch: c,
-		req:           req2,
-		wg:            &sync.Mutex{},
+		url:            url,
+		authentication: authentication,
+		elasticsearch:  client,
 	}
-
 	return ei, nil
 }
 
 // Export exports a passed result event to disk
 func (i *Exporter) Export(event *output.ResultEvent) error {
-	i.wg.Lock()
-	defer i.wg.Unlock()
-	defer func() { i.req.Body = nil }()
+	// creating a request
+	req, err := http.NewRequest(http.MethodPost, i.url, nil)
+	if err != nil {
+		return errors.Wrap(err, "could not make request")
+	}
+	if len(i.authentication) > 0 {
+		req.Header.Add("Authorization", i.authentication)
+	}
+	req.Header.Add("Content-Type", "application/json")
 
 	d := data{
 		Event:     event,
@@ -101,9 +101,9 @@ func (i *Exporter) Export(event *output.ResultEvent) error {
 	if err != nil {
 		return err
 	}
-	i.req.Body = io.NopCloser(strings.NewReader(string(b)))
+	req.Body = io.NopCloser(bytes.NewReader(b))
 
-	res, err := i.elasticsearch.Do(i.req)
+	res, err := i.elasticsearch.Do(req)
 	b, _ = io.ReadAll(res.Body)
 	if err != nil {
 		return errors.New(err.Error() + "error thrown by elasticsearch " + string(b))
@@ -112,7 +112,6 @@ func (i *Exporter) Export(event *output.ResultEvent) error {
 	if res.StatusCode >= 300 {
 		return errors.New("elasticsearch responded with an error: " + string(b))
 	}
-
 	return nil
 }
 

v2/pkg/reporting/reporting.go

@@ -33,8 +33,9 @@ type Options struct {
 	// DiskExporter contains configuration options for Disk Exporter Module
 	DiskExporter *disk.Options `yaml:"disk"`
 	// SarifExporter contains configuration options for Sarif Exporter Module
-	SarifExporter         *sarif.Options `yaml:"sarif"`
-	ElasticsearchExporter *es.Options    `yaml:"elasticsearch"`
+	SarifExporter *sarif.Options `yaml:"sarif"`
+	// ElasticsearchExporter contains configuration options for Elasticsearch Exporter Module
+	ElasticsearchExporter *es.Options `yaml:"elasticsearch"`
 }
 
 // Filter filters the received event and decides whether to perform

v2/pkg/types/types.go

@@ -152,16 +152,4 @@ type Options struct {
 	NoUpdateTemplates bool
 	// EnvironmentVariables enables support for environment variables
 	EnvironmentVariables bool
-	// Full url for elasticsearch
-	ESExport string
-	// Full url for elasticsearch
-	ESPort int
-	// Enable/Disable SSL
-	ESSSL bool
-	// Enable/DIsable SSL verificaiton
-	ESSSLVerificaiton bool
-	// Elasticsearch username
-	ESUsername string
-	// Elasticsearch password
-	ESPassword string
 }