misc sdk enhancements (#4301)

* add template sign/parse  methods

* export installer package

* add readme

* consistent implementation of writefailure

* fix lint error

cmd/nuclei/main.go

@@ -16,9 +16,9 @@ import (
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/gologger/levels"
 	"github.com/projectdiscovery/interactsh/pkg/client"
-	"github.com/projectdiscovery/nuclei/v3/internal/installer"
 	"github.com/projectdiscovery/nuclei/v3/internal/runner"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
+	"github.com/projectdiscovery/nuclei/v3/pkg/installer"
 	"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
 	"github.com/projectdiscovery/nuclei/v3/pkg/operators/common/dsl"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/uncover"

integration_tests/protocols/keys/README.md

@@ -0,0 +1,3 @@
+## keys
+
+the keys stored here especially `ci-private-key.pem` and `ci.crt` are used in integration tests to test template signing and verfication functionality introduced in nuclei v3

internal/runner/runner.go

@@ -12,8 +12,8 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/projectdiscovery/nuclei/v3/internal/installer"
 	"github.com/projectdiscovery/nuclei/v3/internal/runner/nucleicloud"
+	"github.com/projectdiscovery/nuclei/v3/pkg/installer"
 	uncoverlib "github.com/projectdiscovery/uncover"
 	permissionutil "github.com/projectdiscovery/utils/permission"
 	updateutils "github.com/projectdiscovery/utils/update"

lib/sdk.go

@@ -2,6 +2,7 @@ package nuclei
 
 import (
 	"bufio"
+	"bytes"
 	"io"
 
 	"github.com/projectdiscovery/httpx/common/httpx"
@@ -18,6 +19,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting"
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
+	"github.com/projectdiscovery/nuclei/v3/pkg/templates/signer"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 	"github.com/projectdiscovery/ratelimit"
 	"github.com/projectdiscovery/retryablehttp-go"
@@ -127,6 +129,39 @@ func (e *NucleiEngine) LoadTargetsFromReader(reader io.Reader, probeNonHttp bool
 	}
 }
 
+// GetExecuterOptions returns the nuclei executor options
+func (e *NucleiEngine) GetExecuterOptions() *protocols.ExecutorOptions {
+	return &e.executerOpts
+}
+
+// ParseTemplate parses a template from given data
+// template verification status can be accessed from template.Verified
+func (e *NucleiEngine) ParseTemplate(data []byte) (*templates.Template, error) {
+	return templates.ParseTemplateFromReader(bytes.NewReader(data), nil, e.executerOpts)
+}
+
+// SignTemplate signs the tempalate using given signer
+func (e *NucleiEngine) SignTemplate(tmplSigner *signer.TemplateSigner, data []byte) ([]byte, error) {
+	tmpl, err := e.ParseTemplate(data)
+	if err != nil {
+		return data, err
+	}
+	if tmpl.Verified {
+		// already signed
+		return data, nil
+	}
+	if len(tmpl.Workflows) > 0 {
+		return data, templates.ErrNotATemplate
+	}
+	signatureData, err := tmplSigner.Sign(data, tmpl)
+	if err != nil {
+		return data, err
+	}
+	buff := bytes.NewBuffer(signer.RemoveSignatureFromData(data))
+	buff.WriteString("\n" + signatureData)
+	return buff.Bytes(), err
+}
+
 // Close all resources used by nuclei engine
 func (e *NucleiEngine) Close() {
 	e.interactshClient.Close()

lib/sdk_private.go

@@ -11,12 +11,12 @@ import (
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/gologger/levels"
 	"github.com/projectdiscovery/httpx/common/httpx"
-	"github.com/projectdiscovery/nuclei/v3/internal/installer"
 	"github.com/projectdiscovery/nuclei/v3/internal/runner"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/disk"
 	"github.com/projectdiscovery/nuclei/v3/pkg/core"
 	"github.com/projectdiscovery/nuclei/v3/pkg/core/inputs"
+	"github.com/projectdiscovery/nuclei/v3/pkg/installer"
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
 	"github.com/projectdiscovery/nuclei/v3/pkg/progress"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"

pkg/testutils/testutils.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/projectdiscovery/ratelimit"
+	"go.uber.org/multierr"
 
 	"github.com/logrusorgru/aurora"
 
@@ -140,35 +141,46 @@ func (m *MockOutputWriter) Request(templateID, url, requestType string, err erro
 
 // WriteFailure writes the event to file and/or screen.
 func (m *MockOutputWriter) WriteFailure(wrappedEvent *output.InternalWrappedEvent) error {
-	if m.WriteCallback != nil {
+	// if failure event has more than one result, write them all
-		// create event
+	if len(wrappedEvent.Results) > 0 {
-		event := wrappedEvent.InternalEvent
+		errs := []error{}
-		templatePath, templateURL := utils.TemplatePathURL(types.ToString(event["template-path"]), types.ToString(event["template-id"]))
+		for _, result := range wrappedEvent.Results {
-		var templateInfo model.Info
+			result.MatcherStatus = false // just in case
-		if ti, ok := event["template-info"].(model.Info); ok {
+			if err := m.Write(result); err != nil {
-			templateInfo = ti
+				errs = append(errs, err)
+			}
 		}
-		data := &output.ResultEvent{
+		if len(errs) > 0 {
-			Template:      templatePath,
+			return multierr.Combine(errs...)
-			TemplateURL:   templateURL,
-			TemplateID:    types.ToString(event["template-id"]),
-			TemplatePath:  types.ToString(event["template-path"]),
-			Info:          templateInfo,
-			Type:          types.ToString(event["type"]),
-			Host:          types.ToString(event["host"]),
-			Request:       types.ToString(event["request"]),
-			Response:      types.ToString(event["response"]),
-			MatcherStatus: false,
-			Timestamp:     time.Now(),
 		}
-		m.WriteCallback(data)
+		return nil
 	}
-	return nil
-}
-func (m *MockOutputWriter) WriteStoreDebugData(host, templateID, eventType string, data string) {
 
+	// create event
+	event := wrappedEvent.InternalEvent
+	templatePath, templateURL := utils.TemplatePathURL(types.ToString(event["template-path"]), types.ToString(event["template-id"]))
+	var templateInfo model.Info
+	if ti, ok := event["template-info"].(model.Info); ok {
+		templateInfo = ti
+	}
+	data := &output.ResultEvent{
+		Template:      templatePath,
+		TemplateURL:   templateURL,
+		TemplateID:    types.ToString(event["template-id"]),
+		TemplatePath:  types.ToString(event["template-path"]),
+		Info:          templateInfo,
+		Type:          types.ToString(event["type"]),
+		Host:          types.ToString(event["host"]),
+		Request:       types.ToString(event["request"]),
+		Response:      types.ToString(event["response"]),
+		MatcherStatus: false,
+		Timestamp:     time.Now(),
+	}
+	return m.Write(data)
 }
 
+func (m *MockOutputWriter) WriteStoreDebugData(host, templateID, eventType string, data string) {}
+
 type MockProgressClient struct{}
 
 // Stop stops the progress recorder.