Misc

v2/internal/runner/templates.go

@@ -31,8 +31,6 @@ func (r *Runner) getParsedTemplatesFor(templatePaths []string, severities []stri
 			workflowCount++
 		}
 		sev := strings.ToLower(types.ToString(t.Info["severity"]))
-
-		fmt.Printf("info: %+v\n", t.Info)
 		if !filterBySeverity || hasMatchingSeverity(sev, severities) {
 			parsedTemplates[t.ID] = t
 			gologger.Info().Msgf("%s\n", r.templateLogMsg(t.ID, types.ToString(t.Info["name"]), types.ToString(t.Info["author"]), sev))

v2/pkg/protocols/http/request.go

@@ -27,10 +27,10 @@ import (
 const defaultMaxWorkers = 150
 
 // executeRaceRequest executes race condition request for a URL
-func (e *Request) executeRaceRequest(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
-	generator := e.newGenerator()
+func (r *Request) executeRaceRequest(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
+	generator := r.newGenerator()
 
-	maxWorkers := e.RaceNumberRequests
+	maxWorkers := r.RaceNumberRequests
 	swg := sizedwaitgroup.New(maxWorkers)
 
 	var requestErr error
@@ -40,10 +40,10 @@ func (e *Request) executeRaceRequest(reqURL string, dynamicValues, previous outp
 	if err != nil {
 		return err
 	}
-	for i := 0; i < e.RaceNumberRequests; i++ {
+	for i := 0; i < r.RaceNumberRequests; i++ {
 		swg.Add()
 		go func(httpRequest *generatedRequest) {
-			err := e.executeRequest(reqURL, httpRequest, dynamicValues, previous, callback)
+			err := r.executeRequest(reqURL, httpRequest, dynamicValues, previous, callback)
 			mutex.Lock()
 			if err != nil {
 				requestErr = multierr.Append(requestErr, err)
@@ -56,12 +56,12 @@ func (e *Request) executeRaceRequest(reqURL string, dynamicValues, previous outp
 	return requestErr
 }
 
-// executeRaceRequest executes race condition request for a URL
-func (e *Request) executeParallelHTTP(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
-	generator := e.newGenerator()
+// executeRaceRequest executes parallel requests for a template
+func (r *Request) executeParallelHTTP(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
+	generator := r.newGenerator()
 
 	// Workers that keeps enqueuing new requests
-	maxWorkers := e.Threads
+	maxWorkers := r.Threads
 	swg := sizedwaitgroup.New(maxWorkers)
 
 	var requestErr error
@@ -72,30 +72,30 @@ func (e *Request) executeParallelHTTP(reqURL string, dynamicValues, previous out
 			break
 		}
 		if err != nil {
-			e.options.Progress.DecrementRequests(int64(generator.Total()))
+			r.options.Progress.DecrementRequests(int64(generator.Total()))
 			return err
 		}
 		swg.Add()
 		go func(httpRequest *generatedRequest) {
 			defer swg.Done()
 
-			e.options.RateLimiter.Take()
-			err := e.executeRequest(reqURL, httpRequest, dynamicValues, previous, callback)
+			r.options.RateLimiter.Take()
+			err := r.executeRequest(reqURL, httpRequest, dynamicValues, previous, callback)
 			mutex.Lock()
 			if err != nil {
 				requestErr = multierr.Append(requestErr, err)
 			}
 			mutex.Unlock()
 		}(request)
-		e.options.Progress.IncrementRequests()
+		r.options.Progress.IncrementRequests()
 	}
 	swg.Wait()
 	return requestErr
 }
 
-// executeRaceRequest executes race condition request for a URL
-func (e *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
-	generator := e.newGenerator()
+// executeRaceRequest executes turbo http request for a URL
+func (r *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
+	generator := r.newGenerator()
 
 	// need to extract the target from the url
 	URL, err := url.Parse(reqURL)
@@ -106,11 +106,11 @@ func (e *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output
 	pipeOptions := rawhttp.DefaultPipelineOptions
 	pipeOptions.Host = URL.Host
 	pipeOptions.MaxConnections = 1
-	if e.PipelineConcurrentConnections > 0 {
-		pipeOptions.MaxConnections = e.PipelineConcurrentConnections
+	if r.PipelineConcurrentConnections > 0 {
+		pipeOptions.MaxConnections = r.PipelineConcurrentConnections
 	}
-	if e.PipelineRequestsPerConnection > 0 {
-		pipeOptions.MaxPendingRequests = e.PipelineRequestsPerConnection
+	if r.PipelineRequestsPerConnection > 0 {
+		pipeOptions.MaxPendingRequests = r.PipelineRequestsPerConnection
 	}
 	pipeclient := rawhttp.NewPipelineClient(pipeOptions)
 
@@ -130,7 +130,7 @@ func (e *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output
 			break
 		}
 		if err != nil {
-			e.options.Progress.DecrementRequests(int64(generator.Total()))
+			r.options.Progress.DecrementRequests(int64(generator.Total()))
 			return err
 		}
 		request.pipelinedClient = pipeclient
@@ -139,14 +139,14 @@ func (e *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output
 		go func(httpRequest *generatedRequest) {
 			defer swg.Done()
 
-			err := e.executeRequest(reqURL, httpRequest, dynamicValues, previous, callback)
+			err := r.executeRequest(reqURL, httpRequest, dynamicValues, previous, callback)
 			mutex.Lock()
 			if err != nil {
 				requestErr = multierr.Append(requestErr, err)
 			}
 			mutex.Unlock()
 		}(request)
-		e.options.Progress.IncrementRequests()
+		r.options.Progress.IncrementRequests()
 	}
 	swg.Wait()
 	return requestErr
@@ -352,12 +352,12 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, dynam
 const two = 2
 
 // setCustomHeaders sets the custom headers for generated request
-func (e *Request) setCustomHeaders(r *generatedRequest) {
-	for k, v := range e.customHeaders {
-		if r.rawRequest != nil {
-			r.rawRequest.Headers[k] = v
+func (r *Request) setCustomHeaders(req *generatedRequest) {
+	for k, v := range r.customHeaders {
+		if req.rawRequest != nil {
+			req.rawRequest.Headers[k] = v
 		} else {
-			r.request.Header.Set(strings.TrimSpace(k), strings.TrimSpace(v))
+			req.request.Header.Set(strings.TrimSpace(k), strings.TrimSpace(v))
 		}
 	}
 }

v2/pkg/protocols/http/request_test.go

@@ -0,0 +1 @@
+package http

v2/pkg/workflows/execute.go

@@ -34,7 +34,7 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res
 				if len(template.Executers) == 1 {
 					mainErr = err
 				} else {
-					gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", err)
+					gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", template.Template, err)
 				}
 				continue
 			}
@@ -76,7 +76,7 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res
 				if len(template.Executers) == 1 {
 					mainErr = err
 				} else {
-					gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", err)
+					gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", template.Template, err)
 				}
 				continue
 			}
@@ -84,7 +84,7 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res
 				if len(template.Executers) == 1 {
 					mainErr = executionErr
 				} else {
-					gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", executionErr)
+					gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", template.Template, executionErr)
 				}
 			}
 		}

v2/pkg/workflows/execute_test.go

@@ -14,11 +14,10 @@ func TestWorkflowsSimple(t *testing.T) {
 	progress, _ := progress.NewProgress(false, false, 0)
 
 	workflow := &Workflow{Workflows: []*WorkflowTemplate{
-		{Executers: []protocols.Executer{&mockExecuter{result: true}}},
-	},
-		options: &protocols.ExecuterOptions{
-			Progress: progress,
-		}}
+		{Executers: []*ProtocolExecuterPair{{
+			Executer: &mockExecuter{result: true}, Options: &protocols.ExecuterOptions{Progress: progress}},
+		}},
+	}}
 
 	matched, err := workflow.RunWorkflow("https://test.com")
 	require.Nil(t, err, "could not run workflow")
@@ -30,14 +29,17 @@ func TestWorkflowsSimpleMultiple(t *testing.T) {
 
 	var firstInput, secondInput string
 	workflow := &Workflow{Workflows: []*WorkflowTemplate{
-		{Executers: []protocols.Executer{&mockExecuter{result: true, executeHook: func(input string) {
-			firstInput = input
-		}}}},
-		{Executers: []protocols.Executer{&mockExecuter{result: true, executeHook: func(input string) {
-			secondInput = input
-		}}}},
-	},
-		options: &protocols.ExecuterOptions{Progress: progress}}
+		{Executers: []*ProtocolExecuterPair{{
+			Executer: &mockExecuter{result: true, executeHook: func(input string) {
+				firstInput = input
+			}}, Options: &protocols.ExecuterOptions{Progress: progress}},
+		}},
+		{Executers: []*ProtocolExecuterPair{{
+			Executer: &mockExecuter{result: true, executeHook: func(input string) {
+				secondInput = input
+			}}, Options: &protocols.ExecuterOptions{Progress: progress}},
+		}},
+	}}
 
 	matched, err := workflow.RunWorkflow("https://test.com")
 	require.Nil(t, err, "could not run workflow")
@@ -52,16 +54,16 @@ func TestWorkflowsSubtemplates(t *testing.T) {
 
 	var firstInput, secondInput string
 	workflow := &Workflow{Workflows: []*WorkflowTemplate{
-		{Executers: []protocols.Executer{&mockExecuter{result: true, executeHook: func(input string) {
-			firstInput = input
-		}}},
-			Subtemplates: []*WorkflowTemplate{
-				{Executers: []protocols.Executer{&mockExecuter{result: true, executeHook: func(input string) {
-					secondInput = input
-				}}},
-				}}},
-	},
-		options: &protocols.ExecuterOptions{Progress: progress}}
+		{Executers: []*ProtocolExecuterPair{{
+			Executer: &mockExecuter{result: true, executeHook: func(input string) {
+				firstInput = input
+			}}, Options: &protocols.ExecuterOptions{Progress: progress}},
+		}, Subtemplates: []*WorkflowTemplate{{Executers: []*ProtocolExecuterPair{{
+			Executer: &mockExecuter{result: true, executeHook: func(input string) {
+				secondInput = input
+			}}, Options: &protocols.ExecuterOptions{Progress: progress}},
+		}}}},
+	}}
 
 	matched, err := workflow.RunWorkflow("https://test.com")
 	require.Nil(t, err, "could not run workflow")
@@ -76,16 +78,16 @@ func TestWorkflowsSubtemplatesNoMatch(t *testing.T) {
 
 	var firstInput, secondInput string
 	workflow := &Workflow{Workflows: []*WorkflowTemplate{
-		{Executers: []protocols.Executer{&mockExecuter{result: false, executeHook: func(input string) {
-			firstInput = input
-		}}},
-			Subtemplates: []*WorkflowTemplate{
-				{Executers: []protocols.Executer{&mockExecuter{result: true, executeHook: func(input string) {
-					secondInput = input
-				}}},
-				}}},
-	},
-		options: &protocols.ExecuterOptions{Progress: progress}}
+		{Executers: []*ProtocolExecuterPair{{
+			Executer: &mockExecuter{result: false, executeHook: func(input string) {
+				firstInput = input
+			}}, Options: &protocols.ExecuterOptions{Progress: progress}},
+		}, Subtemplates: []*WorkflowTemplate{{Executers: []*ProtocolExecuterPair{{
+			Executer: &mockExecuter{result: true, executeHook: func(input string) {
+				secondInput = input
+			}}, Options: &protocols.ExecuterOptions{Progress: progress}},
+		}}}},
+	}}
 
 	matched, err := workflow.RunWorkflow("https://test.com")
 	require.Nil(t, err, "could not run workflow")
@@ -100,24 +102,21 @@ func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) {
 
 	var firstInput, secondInput string
 	workflow := &Workflow{Workflows: []*WorkflowTemplate{
-		{Executers: []protocols.Executer{&mockExecuter{result: true, executeHook: func(input string) {
-			firstInput = input
-		}, outputs: []*output.InternalWrappedEvent{
-			{OperatorsResult: &operators.Result{
-				Matches:  map[string]struct{}{"tomcat": {}},
-				Extracts: map[string][]string{},
-			}},
-		}}},
-			Matchers: []*Matcher{
-				{Name: "tomcat", Subtemplates: []*WorkflowTemplate{
-					{Executers: []protocols.Executer{&mockExecuter{result: true, executeHook: func(input string) {
-						secondInput = input
-					}}},
-					}}},
-			},
-		},
-	},
-		options: &protocols.ExecuterOptions{Progress: progress}}
+		{Executers: []*ProtocolExecuterPair{{
+			Executer: &mockExecuter{result: true, executeHook: func(input string) {
+				firstInput = input
+			}, outputs: []*output.InternalWrappedEvent{
+				{OperatorsResult: &operators.Result{
+					Matches:  map[string]struct{}{"tomcat": {}},
+					Extracts: map[string][]string{},
+				}},
+			}}, Options: &protocols.ExecuterOptions{Progress: progress}},
+		}, Matchers: []*Matcher{{Name: "tomcat", Subtemplates: []*WorkflowTemplate{{Executers: []*ProtocolExecuterPair{{
+			Executer: &mockExecuter{result: true, executeHook: func(input string) {
+				secondInput = input
+			}}, Options: &protocols.ExecuterOptions{Progress: progress}},
+		}}}}}},
+	}}
 
 	matched, err := workflow.RunWorkflow("https://test.com")
 	require.Nil(t, err, "could not run workflow")
@@ -132,24 +131,21 @@ func TestWorkflowsSubtemplatesWithMatcherNoMatch(t *testing.T) {
 
 	var firstInput, secondInput string
 	workflow := &Workflow{Workflows: []*WorkflowTemplate{
-		{Executers: []protocols.Executer{&mockExecuter{result: true, executeHook: func(input string) {
-			firstInput = input
-		}, outputs: []*output.InternalWrappedEvent{
-			{OperatorsResult: &operators.Result{
-				Matches:  map[string]struct{}{"tomcat": {}},
-				Extracts: map[string][]string{},
-			}},
-		}}},
-			Matchers: []*Matcher{
-				{Name: "apache", Subtemplates: []*WorkflowTemplate{
-					{Executers: []protocols.Executer{&mockExecuter{result: true, executeHook: func(input string) {
-						secondInput = input
-					}}}},
+		{Executers: []*ProtocolExecuterPair{{
+			Executer: &mockExecuter{result: true, executeHook: func(input string) {
+				firstInput = input
+			}, outputs: []*output.InternalWrappedEvent{
+				{OperatorsResult: &operators.Result{
+					Matches:  map[string]struct{}{"tomcat": {}},
+					Extracts: map[string][]string{},
 				}},
-			},
-		},
-	},
-		options: &protocols.ExecuterOptions{Progress: progress}}
+			}}, Options: &protocols.ExecuterOptions{Progress: progress}},
+		}, Matchers: []*Matcher{{Name: "apache", Subtemplates: []*WorkflowTemplate{{Executers: []*ProtocolExecuterPair{{
+			Executer: &mockExecuter{result: true, executeHook: func(input string) {
+				secondInput = input
+			}}, Options: &protocols.ExecuterOptions{Progress: progress}},
+		}}}}}},
+	}}
 
 	matched, err := workflow.RunWorkflow("https://test.com")
 	require.Nil(t, err, "could not run workflow")