Merge pull request #3031 from projectdiscovery/parse_from_reader

Add ParseTemplateFromReader function

v2/pkg/templates/compile.go

@@ -2,6 +2,8 @@ package templates
 
 import (
 	"fmt"
+	"io"
+	"net/http"
 	"reflect"
 
 	"github.com/pkg/errors"
@@ -35,44 +37,25 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute
 		return value.(*Template), err
 	}
 
-	template := &Template{}
-
-	data, err := utils.ReadFromPathOrURL(filePath, options.Catalog)
+	var reader io.ReadCloser
+	if utils.IsURL(filePath) {
+		resp, err := http.Get(filePath)
+		if err != nil {
+			return nil, err
+		}
+		reader = resp.Body
+	} else {
+		var err error
+		reader, err = options.Catalog.OpenFile(filePath)
+		if err != nil {
+			return nil, err
+		}
+	}
+	defer reader.Close()
+	template, err := ParseTemplateFromReader(reader, preprocessor, options)
 	if err != nil {
 		return nil, err
 	}
-
-	data = template.expandPreprocessors(data)
-	if preprocessor != nil {
-		data = preprocessor.Process(data)
-	}
-
-	if err := yaml.Unmarshal(data, template); err != nil {
-		return nil, err
-	}
-
-	if utils.IsBlank(template.Info.Name) {
-		return nil, errors.New("no template name field provided")
-	}
-	if template.Info.Authors.IsEmpty() {
-		return nil, errors.New("no template author field provided")
-	}
-
-	// Setting up variables regarding template metadata
-	options.TemplateID = template.ID
-	options.TemplateInfo = template.Info
-	options.TemplatePath = filePath
-	options.StopAtFirstMatch = template.StopAtFirstMatch
-
-	if template.Variables.Len() > 0 {
-		options.Variables = template.Variables
-	}
-
-	// If no requests, and it is also not a workflow, return error.
-	if template.Requests() == 0 {
-		return nil, fmt.Errorf("no requests defined for %s", template.ID)
-	}
-
 	// Compile the workflow request
 	if len(template.Workflows) > 0 {
 		compiled := &template.Workflow
@@ -81,24 +64,7 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute
 		template.CompiledWorkflow = compiled
 		template.CompiledWorkflow.Options = &options
 	}
-
-	if err := template.compileProtocolRequests(options); err != nil {
-		return nil, err
-	}
-
-	if template.Executer != nil {
-		if err := template.Executer.Compile(); err != nil {
-			return nil, errors.Wrap(err, "could not compile request")
-		}
-		template.TotalRequests = template.Executer.Requests()
-	}
-	if template.Executer == nil && template.CompiledWorkflow == nil {
-		return nil, ErrCreateTemplateExecutor
-	}
 	template.Path = filePath
-
-	template.parseSelfContainedRequests()
-
 	parsedTemplatesCache.Store(filePath, template, err)
 	return template, nil
 }
@@ -221,3 +187,60 @@ mainLoop:
 		template.Executer = executer.NewExecuter([]protocols.Request{&offlinehttp.Request{}}, &options)
 	}
 }
+
+// ParseTemplateFromReader reads the template from reader
+// returns the parsed template
+func ParseTemplateFromReader(reader io.Reader, preprocessor Preprocessor, options protocols.ExecuterOptions) (*Template, error) {
+	template := &Template{}
+	data, err := io.ReadAll(reader)
+	if err != nil {
+		return nil, err
+	}
+
+	data = template.expandPreprocessors(data)
+	if preprocessor != nil {
+		data = preprocessor.Process(data)
+	}
+
+	if err := yaml.Unmarshal(data, template); err != nil {
+		return nil, err
+	}
+
+	if utils.IsBlank(template.Info.Name) {
+		return nil, errors.New("no template name field provided")
+	}
+	if template.Info.Authors.IsEmpty() {
+		return nil, errors.New("no template author field provided")
+	}
+
+	// Setting up variables regarding template metadata
+	options.TemplateID = template.ID
+	options.TemplateInfo = template.Info
+	options.StopAtFirstMatch = template.StopAtFirstMatch
+
+	if template.Variables.Len() > 0 {
+		options.Variables = template.Variables
+	}
+
+	// If no requests, and it is also not a workflow, return error.
+	if template.Requests() == 0 {
+		return nil, fmt.Errorf("no requests defined for %s", template.ID)
+	}
+
+	if err := template.compileProtocolRequests(options); err != nil {
+		return nil, err
+	}
+
+	if template.Executer != nil {
+		if err := template.Executer.Compile(); err != nil {
+			return nil, errors.Wrap(err, "could not compile request")
+		}
+		template.TotalRequests = template.Executer.Requests()
+	}
+	if template.Executer == nil && template.CompiledWorkflow == nil {
+		return nil, ErrCreateTemplateExecutor
+	}
+	template.parseSelfContainedRequests()
+
+	return template, nil
+}

v2/pkg/templates/compile_test.go

@@ -0,0 +1,184 @@
+package templates_test
+
+import (
+	"context"
+	"log"
+	"testing"
+	"time"
+
+	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity"
+	"github.com/projectdiscovery/nuclei/v2/pkg/model/types/stringslice"
+	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
+	"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
+	"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
+	"github.com/projectdiscovery/nuclei/v2/pkg/parsers"
+	"github.com/projectdiscovery/nuclei/v2/pkg/progress"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/variables"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http"
+	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/ssl"
+	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
+	"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
+	"github.com/projectdiscovery/nuclei/v2/pkg/workflows"
+	"github.com/projectdiscovery/ratelimit"
+	"github.com/stretchr/testify/require"
+)
+
+var executerOpts protocols.ExecuterOptions
+
+func setup() {
+	options := testutils.DefaultOptions
+	testutils.Init(options)
+	progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0)
+
+	executerOpts = protocols.ExecuterOptions{
+		Output:       testutils.NewMockOutputWriter(),
+		Options:      options,
+		Progress:     progressImpl,
+		ProjectFile:  nil,
+		IssuesClient: nil,
+		Browser:      nil,
+		Catalog:      disk.NewCatalog(options.TemplatesDirectory),
+		RateLimiter:  ratelimit.New(context.Background(), uint(options.RateLimit), time.Second),
+	}
+	workflowLoader, err := parsers.NewLoader(&executerOpts)
+	if err != nil {
+		log.Fatalf("Could not create workflow loader: %s\n", err)
+	}
+	executerOpts.WorkflowLoader = workflowLoader
+
+}
+
+func Test_ParseFromURL(t *testing.T) {
+	filePath := "https://api.nuclei.sh/api/v1/templates/raw/ssl/tls-version.yaml"
+	expectedTemplate := &templates.Template{
+		ID: "tls-version",
+		Info: model.Info{
+			Authors:        stringslice.StringSlice{Value: []string{"pdteam"}},
+			SeverityHolder: severity.Holder{Severity: severity.Info},
+			Name:           "TLS Version",
+			Tags:           stringslice.StringSlice{Value: []string{"ssl"}},
+		},
+		RequestsSSL: []*ssl.Request{
+			{
+				Address: "{{Host}}:{{Port}}",
+				Operators: operators.Operators{
+					Extractors: []*extractors.Extractor{
+						{
+							Type: extractors.ExtractorTypeHolder{ExtractorType: extractors.JSONExtractor},
+							JSON: []string{".tls_version"},
+						},
+					},
+				},
+			}},
+		TotalRequests: 1,
+		Executer:      nil,
+		Path:          "https://api.nuclei.sh/api/v1/templates/raw/ssl/tls-version.yaml",
+	}
+	setup()
+	got, err := templates.Parse(filePath, nil, executerOpts)
+	require.Nil(t, err, "could not parse template")
+	require.Equal(t, expectedTemplate.ID, got.ID)
+	require.Equal(t, expectedTemplate.Info, got.Info)
+	require.Equal(t, expectedTemplate.TotalRequests, got.TotalRequests)
+	require.Equal(t, expectedTemplate.Path, got.Path)
+	require.Equal(t, expectedTemplate.RequestsSSL[0].Address, got.RequestsSSL[0].Address)
+	require.Equal(t, expectedTemplate.RequestsSSL[0].Extractors[0].Type, got.RequestsSSL[0].Extractors[0].Type)
+	require.Equal(t, expectedTemplate.RequestsSSL[0].Extractors[0].JSON, got.RequestsSSL[0].Extractors[0].JSON)
+	require.Equal(t, len(expectedTemplate.RequestsSSL), len(got.RequestsSSL))
+}
+
+func Test_ParseFromFile(t *testing.T) {
+	filePath := "tests/match-1.yaml"
+	expectedTemplate := &templates.Template{
+		ID: "basic-get",
+		Info: model.Info{
+			Name:           "Basic GET Request",
+			Authors:        stringslice.StringSlice{Value: []string{"pdteam"}},
+			SeverityHolder: severity.Holder{Severity: severity.Info},
+		},
+		RequestsHTTP: []*http.Request{{
+			Operators: operators.Operators{
+				Matchers: []*matchers.Matcher{{
+					Type: matchers.MatcherTypeHolder{
+						MatcherType: matchers.WordsMatcher,
+					},
+					Words: []string{"This is test matcher text"},
+				}},
+			},
+			Path:       []string{"{{BaseURL}}"},
+			AttackType: generators.AttackTypeHolder{},
+			Method: http.HTTPMethodTypeHolder{
+				MethodType: http.HTTPGet,
+			},
+		}},
+		TotalRequests: 1,
+		Executer:      nil,
+		Path:          "tests/match-1.yaml",
+	}
+	setup()
+	got, err := templates.Parse(filePath, nil, executerOpts)
+	require.Nil(t, err, "could not parse template")
+	require.Equal(t, expectedTemplate.ID, got.ID)
+	require.Equal(t, expectedTemplate.Info, got.Info)
+	require.Equal(t, expectedTemplate.TotalRequests, got.TotalRequests)
+	require.Equal(t, expectedTemplate.Path, got.Path)
+	require.Equal(t, expectedTemplate.RequestsHTTP[0].Path, got.RequestsHTTP[0].Path)
+	require.Equal(t, expectedTemplate.RequestsHTTP[0].Operators.Matchers[0].Words, got.RequestsHTTP[0].Operators.Matchers[0].Words)
+	require.Equal(t, len(expectedTemplate.RequestsHTTP), len(got.RequestsHTTP))
+
+	// Test cache
+	got, err = templates.Parse(filePath, nil, executerOpts)
+	require.Nil(t, err, "could not parse template")
+	require.Equal(t, expectedTemplate.ID, got.ID)
+}
+
+func Test_ParseWorkflow(t *testing.T) {
+	filePath := "tests/workflow.yaml"
+	expectedTemplate := &templates.Template{
+		ID: "workflow-example",
+		Info: model.Info{
+			Name:           "Test Workflow Template",
+			Authors:        stringslice.StringSlice{Value: []string{"pdteam"}},
+			SeverityHolder: severity.Holder{Severity: severity.Info},
+		},
+		Workflow: workflows.Workflow{
+			Workflows: []*workflows.WorkflowTemplate{{Template: "tests/match-1.yaml"}, {Template: "tests/match-1.yaml"}},
+			Options:   &protocols.ExecuterOptions{},
+		},
+		CompiledWorkflow: &workflows.Workflow{},
+		SelfContained:    false,
+		StopAtFirstMatch: false,
+		Signature:        http.SignatureTypeHolder{},
+		Variables:        variables.Variable{},
+		TotalRequests:    0,
+		Executer:         nil,
+		Path:             "tests/workflow.yaml",
+	}
+	setup()
+	got, err := templates.Parse(filePath, nil, executerOpts)
+	require.Nil(t, err, "could not parse template")
+	require.Equal(t, expectedTemplate.ID, got.ID)
+	require.Equal(t, expectedTemplate.Info, got.Info)
+	require.Equal(t, expectedTemplate.TotalRequests, got.TotalRequests)
+	require.Equal(t, expectedTemplate.Path, got.Path)
+	require.Equal(t, expectedTemplate.Workflow.Workflows[0].Template, got.Workflow.Workflows[0].Template)
+	require.Equal(t, len(expectedTemplate.Workflows), len(got.Workflows))
+}
+
+func Test_WrongTemplate(t *testing.T) {
+	setup()
+
+	filePath := "tests/no-author.yaml"
+	got, err := templates.Parse(filePath, nil, executerOpts)
+	require.Nil(t, got, "could not parse template")
+	require.ErrorContains(t, err, "no template author field provided")
+
+	filePath = "tests/no-req.yaml"
+	got, err = templates.Parse(filePath, nil, executerOpts)
+	require.Nil(t, got, "could not parse template")
+	require.ErrorContains(t, err, "no requests defined ")
+}

v2/pkg/templates/tests/match-1.yaml

@@ -0,0 +1,15 @@
+id: basic-get
+
+info:
+  name: Basic GET Request
+  author: pdteam
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+    matchers:
+      - type: word
+        words:
+          - "This is test matcher text"

v2/pkg/templates/tests/no-author.yaml

@@ -0,0 +1,14 @@
+id: basic-get
+
+info:
+  name: Basic GET Request
+  severity: info
+
+requests:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+    matchers:
+      - type: word
+        words:
+          - "This is test matcher text"

v2/pkg/templates/tests/no-req.yaml

@@ -0,0 +1,9 @@
+id: basic-get
+
+info:
+  name: Basic GET Request
+  author: pdteam
+  severity: info
+
+requests:
+

v2/pkg/templates/tests/workflow.yaml

@@ -0,0 +1,10 @@
+id: workflow-example
+
+info:
+  name: Test Workflow Template
+  author: pdteam
+  severity: info
+
+workflows:
+  - template: tests/match-1.yaml
+  - template: tests/match-1.yaml