Merge branch 'dev' into dependabot/go_modules/dev/github.com/projectdiscovery/hmap-0.0.23

cmd/nuclei/main.go

@@ -16,9 +16,9 @@ import (
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/gologger/levels"
 	"github.com/projectdiscovery/interactsh/pkg/client"
-	"github.com/projectdiscovery/nuclei/v3/internal/installer"
 	"github.com/projectdiscovery/nuclei/v3/internal/runner"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
+	"github.com/projectdiscovery/nuclei/v3/pkg/installer"
 	"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
 	"github.com/projectdiscovery/nuclei/v3/pkg/operators/common/dsl"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/uncover"

integration_tests/protocols/keys/README.md

@@ -0,0 +1,3 @@
+## keys
+
+the keys stored here especially `ci-private-key.pem` and `ci.crt` are used in integration tests to test template signing and verfication functionality introduced in nuclei v3

internal/runner/runner.go

@@ -12,8 +12,8 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/projectdiscovery/nuclei/v3/internal/installer"
 	"github.com/projectdiscovery/nuclei/v3/internal/runner/nucleicloud"
+	"github.com/projectdiscovery/nuclei/v3/pkg/installer"
 	uncoverlib "github.com/projectdiscovery/uncover"
 	permissionutil "github.com/projectdiscovery/utils/permission"
 	updateutils "github.com/projectdiscovery/utils/update"

lib/example_test.go

@@ -77,7 +77,7 @@ func ExampleThreadSafeNucleiEngine() {
 func TestMain(m *testing.M) {
 	// this file only contains testtables examples https://go.dev/blog/examples
 	// and actual functionality test are in sdk_test.go
-	if os.Getenv("GH_ACTION") != "" {
+	if os.Getenv("GH_ACTION") != "" || os.Getenv("CI") != "" {
 		// no need to run this test on github actions
 		return
 	}

lib/sdk.go

@@ -2,6 +2,7 @@ package nuclei
 
 import (
 	"bufio"
+	"bytes"
 	"io"
 
 	"github.com/projectdiscovery/httpx/common/httpx"
@@ -18,6 +19,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting"
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
+	"github.com/projectdiscovery/nuclei/v3/pkg/templates/signer"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 	"github.com/projectdiscovery/ratelimit"
 	"github.com/projectdiscovery/retryablehttp-go"
@@ -127,6 +129,39 @@ func (e *NucleiEngine) LoadTargetsFromReader(reader io.Reader, probeNonHttp bool
 	}
 }
 
+// GetExecuterOptions returns the nuclei executor options
+func (e *NucleiEngine) GetExecuterOptions() *protocols.ExecutorOptions {
+	return &e.executerOpts
+}
+
+// ParseTemplate parses a template from given data
+// template verification status can be accessed from template.Verified
+func (e *NucleiEngine) ParseTemplate(data []byte) (*templates.Template, error) {
+	return templates.ParseTemplateFromReader(bytes.NewReader(data), nil, e.executerOpts)
+}
+
+// SignTemplate signs the tempalate using given signer
+func (e *NucleiEngine) SignTemplate(tmplSigner *signer.TemplateSigner, data []byte) ([]byte, error) {
+	tmpl, err := e.ParseTemplate(data)
+	if err != nil {
+		return data, err
+	}
+	if tmpl.Verified {
+		// already signed
+		return data, nil
+	}
+	if len(tmpl.Workflows) > 0 {
+		return data, templates.ErrNotATemplate
+	}
+	signatureData, err := tmplSigner.Sign(data, tmpl)
+	if err != nil {
+		return data, err
+	}
+	buff := bytes.NewBuffer(signer.RemoveSignatureFromData(data))
+	buff.WriteString("\n" + signatureData)
+	return buff.Bytes(), err
+}
+
 // Close all resources used by nuclei engine
 func (e *NucleiEngine) Close() {
 	e.interactshClient.Close()

lib/sdk_private.go

@@ -11,12 +11,12 @@ import (
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/gologger/levels"
 	"github.com/projectdiscovery/httpx/common/httpx"
-	"github.com/projectdiscovery/nuclei/v3/internal/installer"
 	"github.com/projectdiscovery/nuclei/v3/internal/runner"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/disk"
 	"github.com/projectdiscovery/nuclei/v3/pkg/core"
 	"github.com/projectdiscovery/nuclei/v3/pkg/core/inputs"
+	"github.com/projectdiscovery/nuclei/v3/pkg/installer"
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
 	"github.com/projectdiscovery/nuclei/v3/pkg/progress"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"

pkg/external/customtemplates/github.go

@@ -13,6 +13,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 	fileutil "github.com/projectdiscovery/utils/file"
+	folderutil "github.com/projectdiscovery/utils/folder"
 	"golang.org/x/oauth2"
 	"gopkg.in/src-d/go-git.v4/plumbing/transport/http"
 )
@@ -85,10 +86,22 @@ func NewGitHubProviders(options *types.Options) ([]*customTemplateGitHubRepo, er
 			githubToken: options.GitHubToken,
 		}
 		providers = append(providers, customTemplateRepo)
+
+		customTemplateRepo.restructureRepoDir()
 	}
 	return providers, nil
 }
 
+func (customTemplateRepo *customTemplateGitHubRepo) restructureRepoDir() {
+	customGitHubTemplatesDirectory := config.DefaultConfig.CustomGitHubTemplatesDirectory
+	oldRepoClonePath := filepath.Join(customGitHubTemplatesDirectory, customTemplateRepo.reponame+"-"+customTemplateRepo.owner)
+	newRepoClonePath := customTemplateRepo.getLocalRepoClonePath(customGitHubTemplatesDirectory)
+
+	if fileutil.FolderExists(oldRepoClonePath) && !fileutil.FolderExists(newRepoClonePath) {
+		_ = folderutil.SyncDirectory(oldRepoClonePath, newRepoClonePath)
+	}
+}
+
 // getOwnerAndRepo returns the owner, repo, err from the given string
 // e.g., it takes input projectdiscovery/nuclei-templates and
 // returns owner => projectdiscovery, repo => nuclei-templates
@@ -154,9 +167,9 @@ func (ctr *customTemplateGitHubRepo) pullChanges(repoPath, githubToken string) e
 	return nil
 }
 
-// All Custom github repos are cloned in the format of 'reponame-owner' for uniqueness
+// All Custom github repos are cloned in the format of 'owner/reponame' for uniqueness
 func (ctr *customTemplateGitHubRepo) getLocalRepoClonePath(downloadPath string) string {
-	return filepath.Join(downloadPath, ctr.reponame+"-"+ctr.owner)
+	return filepath.Join(downloadPath, ctr.owner, ctr.reponame)
 }
 
 // returns the auth object with username and github token as password

pkg/external/customtemplates/github_test.go

@@ -30,6 +30,6 @@ func TestDownloadCustomTemplatesFromGitHub(t *testing.T) {
 
 	ctm.Download(context.Background())
 
-	require.DirExists(t, filepath.Join(templatesDirectory, "github", "nuclei-templates-projectdiscovery"), "cloned directory does not exists")
-	require.DirExists(t, filepath.Join(templatesDirectory, "github", "nuclei-templates-ehsandeep"), "cloned directory does not exists")
+	require.DirExists(t, filepath.Join(templatesDirectory, "github", "projectdiscovery", "nuclei-templates"), "cloned directory does not exists")
+	require.DirExists(t, filepath.Join(templatesDirectory, "github", "ehsandeep", "nuclei-templates"), "cloned directory does not exists")
 }

pkg/testutils/testutils.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/projectdiscovery/ratelimit"
+	"go.uber.org/multierr"
 
 	"github.com/logrusorgru/aurora"
 
@@ -140,35 +141,46 @@ func (m *MockOutputWriter) Request(templateID, url, requestType string, err erro
 
 // WriteFailure writes the event to file and/or screen.
 func (m *MockOutputWriter) WriteFailure(wrappedEvent *output.InternalWrappedEvent) error {
-	if m.WriteCallback != nil {
-		// create event
-		event := wrappedEvent.InternalEvent
-		templatePath, templateURL := utils.TemplatePathURL(types.ToString(event["template-path"]), types.ToString(event["template-id"]))
-		var templateInfo model.Info
-		if ti, ok := event["template-info"].(model.Info); ok {
-			templateInfo = ti
+	// if failure event has more than one result, write them all
+	if len(wrappedEvent.Results) > 0 {
+		errs := []error{}
+		for _, result := range wrappedEvent.Results {
+			result.MatcherStatus = false // just in case
+			if err := m.Write(result); err != nil {
+				errs = append(errs, err)
+			}
 		}
-		data := &output.ResultEvent{
-			Template:      templatePath,
-			TemplateURL:   templateURL,
-			TemplateID:    types.ToString(event["template-id"]),
-			TemplatePath:  types.ToString(event["template-path"]),
-			Info:          templateInfo,
-			Type:          types.ToString(event["type"]),
-			Host:          types.ToString(event["host"]),
-			Request:       types.ToString(event["request"]),
-			Response:      types.ToString(event["response"]),
-			MatcherStatus: false,
-			Timestamp:     time.Now(),
+		if len(errs) > 0 {
+			return multierr.Combine(errs...)
 		}
-		m.WriteCallback(data)
+		return nil
 	}
-	return nil
-}
-func (m *MockOutputWriter) WriteStoreDebugData(host, templateID, eventType string, data string) {
 
+	// create event
+	event := wrappedEvent.InternalEvent
+	templatePath, templateURL := utils.TemplatePathURL(types.ToString(event["template-path"]), types.ToString(event["template-id"]))
+	var templateInfo model.Info
+	if ti, ok := event["template-info"].(model.Info); ok {
+		templateInfo = ti
+	}
+	data := &output.ResultEvent{
+		Template:      templatePath,
+		TemplateURL:   templateURL,
+		TemplateID:    types.ToString(event["template-id"]),
+		TemplatePath:  types.ToString(event["template-path"]),
+		Info:          templateInfo,
+		Type:          types.ToString(event["type"]),
+		Host:          types.ToString(event["host"]),
+		Request:       types.ToString(event["request"]),
+		Response:      types.ToString(event["response"]),
+		MatcherStatus: false,
+		Timestamp:     time.Now(),
+	}
+	return m.Write(data)
 }
 
+func (m *MockOutputWriter) WriteStoreDebugData(host, templateID, eventType string, data string) {}
+
 type MockProgressClient struct{}
 
 // Stop stops the progress recorder.