Merge pull request #403 from projectdiscovery/add-yaml-based-workflow

Add yaml based workflow generator

v2/internal/runner/processor.go

@@ -292,12 +292,14 @@ func (r *Runner) preloadWorkflowTemplates(p *progress.Progress, workflow *workfl
 						ProxySocksURL: r.options.ProxySocksURL,
 						CustomHeaders: r.options.CustomHeaders,
 						CookieJar:     jar,
+						TraceLog:      r.traceLog,
 					}
 				} else if len(t.RequestsDNS) > 0 {
 					template.DNSOptions = &executer.DNSOptions{
 						Debug:    r.options.Debug,
 						Template: t,
 						Writer:   r.output,
+						TraceLog: r.traceLog,
 					}
 				}
 				if template.DNSOptions != nil || template.HTTPOptions != nil {

v2/pkg/workflows/compile.go

@@ -1,9 +1,9 @@
 package workflows
 
 import (
-	"errors"
 	"os"
 
+	"github.com/pkg/errors"
 	"gopkg.in/yaml.v2"
 )
 
@@ -22,11 +22,14 @@ func Parse(file string) (*Workflow, error) {
 		return nil, err
 	}
 
+	if len(workflow.Workflows) > 0 {
+		if err := workflow.generateLogicFromWorkflows(); err != nil {
+			return nil, errors.Wrap(err, "could not generate workflow")
+		}
+	}
 	if workflow.Logic == "" {
 		return nil, errors.New("no logic provided")
 	}
-
 	workflow.path = file
-
 	return workflow, nil
 }

v2/pkg/workflows/generator.go

@@ -0,0 +1,78 @@
+package workflows
+
+import (
+	"errors"
+	"strings"
+
+	"github.com/segmentio/ksuid"
+)
+
+// generateLogicFromWorkflows generates a workflow logic using the
+// yaml based workflow declaration.
+//
+// The implementation is very basic and contains a simple yaml->tengo
+// convertor that implements basic required features.
+func (w *Workflow) generateLogicFromWorkflows() error {
+	w.Variables = make(map[string]string)
+
+	workflowBuilder := &strings.Builder{}
+	for _, template := range w.Workflows {
+		if err := w.generateTemplateFunc(template, workflowBuilder); err != nil {
+			return err
+		}
+	}
+	w.Logic = workflowBuilder.String()
+	return nil
+}
+
+func (w *Workflow) generateTemplateFunc(template *WorkflowTemplate, workflowBuilder *strings.Builder) error {
+	builder := &strings.Builder{}
+	builder.WriteString("var_")
+	builder.WriteString(ksuid.New().String())
+	ID := builder.String()
+	w.Variables[ID] = template.Template
+
+	if len(template.Subtemplates) > 0 && len(template.Matchers) > 0 {
+		return errors.New("subtemplates and matchers cannot be present together")
+	}
+	workflowBuilder.WriteRune('\n')
+	if len(template.Matchers) > 0 {
+		workflowBuilder.WriteString(ID)
+		workflowBuilder.WriteString("()\n")
+
+		for _, matcher := range template.Matchers {
+			if len(matcher.Subtemplates) == 0 {
+				return errors.New("no subtemplates present for matcher")
+			}
+			workflowBuilder.WriteString("\nif ")
+			workflowBuilder.WriteString(ID)
+			workflowBuilder.WriteString("[\"")
+			workflowBuilder.WriteString(matcher.Name)
+			workflowBuilder.WriteString("\"] {")
+
+			for _, subtemplate := range matcher.Subtemplates {
+				if err := w.generateTemplateFunc(subtemplate, workflowBuilder); err != nil {
+					return err
+				}
+			}
+			workflowBuilder.WriteString("\n}")
+		}
+	}
+	if len(template.Subtemplates) > 0 {
+		workflowBuilder.WriteString("if ")
+		workflowBuilder.WriteString(ID)
+		workflowBuilder.WriteString("() {")
+
+		for _, subtemplate := range template.Subtemplates {
+			if err := w.generateTemplateFunc(subtemplate, workflowBuilder); err != nil {
+				return err
+			}
+		}
+		workflowBuilder.WriteString("\n}")
+	}
+	if len(template.Matchers) == 0 && len(template.Subtemplates) == 0 {
+		workflowBuilder.WriteString(ID)
+		workflowBuilder.WriteString("();")
+	}
+	return nil
+}

v2/pkg/workflows/workflows.go

@@ -12,7 +12,22 @@ type Workflow struct {
 	Variables map[string]string `yaml:"variables"`
 	// Logic contains the workflow pseudo-code
 	Logic string `yaml:"logic"`
-	path  string
+	// Workflows is a yaml based workflow declaration code.
+	Workflows []*WorkflowTemplate `yaml:"workflows"`
+	path      string
+}
+
+// WorkflowTemplate is a template to be ran as part of a workflow
+type WorkflowTemplate struct {
+	Template     string              `yaml:"template"`
+	Matchers     []*Matcher          `yaml:"matchers"`
+	Subtemplates []*WorkflowTemplate `yaml:"subtemplates"`
+}
+
+// Matcher performs conditional matching on the workflow template results.
+type Matcher struct {
+	Name         string              `yaml:"name"`
+	Subtemplates []*WorkflowTemplate `yaml:"subtemplates"`
 }
 
 // GetPath of the workflow