support env var from report yaml (#3188)

* added support yaml for report options

* better to use .HasPrefix()

* wip: working on unmarshal YAML optimization

* managed yaml tag + nil pointers + unit test

* implemented tests

* removed unused code from reporting + code refactoring

* WIP: code refactoring and tests

* check on env var

* more test coverage and added callback func

* docs + renaming func

* moved callback logic + removed yaml validation

* used yaml decoder

* struct typo

* refactoring walk method with generic signature

* removed yamlwrapper refs, used yaml2 + docs

implemented test to check also fields without yaml tag

* used DecodeAndValidate()

* removed double import reference

---------

Co-authored-by: mzack <marco.rivoli.nvh@gmail.com>
Co-authored-by: Mzack9999 <mzack9999@protonmail.com>

integration_tests/test-issue-tracker-config2.yaml

@@ -6,7 +6,7 @@ deny-list:
   severity: low
 
 # GitHub contains configuration options for GitHub issue tracker
-GitHub:
+gitHub:
   # base-url is the optional self-hosted GitHub application url
   base-url: https://localhost:8443/GitHub
   # username is the username of the GitHub user
@@ -21,7 +21,7 @@ GitHub:
   issue-label: bug
 
 # GitLab contains configuration options for GitLab issue tracker
-GitLab:
+gitLab:
   # base-url is the optional self-hosted GitLab application url
   base-url: https://localhost:8443/GitLab
   # username is the username of the GitLab user
@@ -34,7 +34,7 @@ GitLab:
   issue-label: bug
 
 # Jira contains configuration options for Jira issue tracker
-Jira:
+jira:
   # cloud is the boolean which tells if Jira instance is running in the cloud or on-prem version is used
   cloud: true
   # update-existing is the boolean which tells if the existing, opened issue should be updated or new one should be created

v2/internal/runner/runner.go

@@ -11,6 +11,7 @@ import (
 	_ "net/http/pprof"
 	"os"
 	"path/filepath"
+	"reflect"
 	"strconv"
 	"strings"
 	"sync/atomic"
@@ -55,7 +56,6 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/utils"
 	"github.com/projectdiscovery/nuclei/v2/pkg/utils/stats"
 	"github.com/projectdiscovery/nuclei/v2/pkg/utils/yaml"
-	yamlwrapper "github.com/projectdiscovery/nuclei/v2/pkg/utils/yaml"
 	"github.com/projectdiscovery/retryablehttp-go"
 	stringsutil "github.com/projectdiscovery/utils/strings"
 )
@@ -114,7 +114,6 @@ func New(options *types.Options) (*Runner, error) {
 	// TODO: refactor to pass options reference globally without cycles
 	parsers.NoStrictSyntax = options.NoStrictSyntax
 	yaml.StrictSyntax = !options.NoStrictSyntax
-
 	// parse the runner.options.GithubTemplateRepo and store the valid repos in runner.customTemplateRepos
 	runner.customTemplates = customtemplates.ParseCustomTemplates(runner.options)
 
@@ -306,11 +305,13 @@ func createReportingOptions(options *types.Options) (*reporting.Options, error)
 		}
 
 		reportingOptions = &reporting.Options{}
-		if err := yamlwrapper.DecodeAndValidate(file, reportingOptions); err != nil {
+		if err := yaml.DecodeAndValidate(file, reportingOptions); err != nil {
 			file.Close()
 			return nil, errors.Wrap(err, "could not parse reporting config file")
 		}
 		file.Close()
+
+		Walk(reportingOptions, expandEndVars)
 	}
 	if options.MarkdownExportDirectory != "" {
 		if reportingOptions != nil {
@@ -798,3 +799,52 @@ func (r *Runner) SaveResumeConfig(path string) error {
 
 	return os.WriteFile(path, data, os.ModePerm)
 }
+
+type WalkFunc func(reflect.Value, reflect.StructField)
+
+// Walk traverses a struct and executes a callback function on each value in the struct.
+// The interface{} passed to the function should be a pointer to a struct or a struct.
+// WalkFunc is the callback function used for each value in the struct. It is passed the
+// reflect.Value and reflect.Type of the value in the struct.
+func Walk(s interface{}, callback WalkFunc) {
+	structValue := reflect.ValueOf(s)
+	if structValue.Kind() == reflect.Ptr {
+		structValue = structValue.Elem()
+	}
+	if structValue.Kind() != reflect.Struct {
+		return
+	}
+	for i := 0; i < structValue.NumField(); i++ {
+		field := structValue.Field(i)
+		fieldType := structValue.Type().Field(i)
+		if !fieldType.IsExported() {
+			continue
+		}
+		if field.Kind() == reflect.Struct {
+			Walk(field.Addr().Interface(), callback)
+		} else if field.Kind() == reflect.Ptr && field.Elem().Kind() == reflect.Struct {
+			Walk(field.Interface(), callback)
+		} else {
+			callback(field, fieldType)
+		}
+	}
+}
+
+// expandEndVars looks for values in a struct tagged with "yaml" and checks if they are prefixed with '$'.
+// If they are, it will try to retrieve the value from the environment and if it exists, it will set the
+// value of the field to that of the environment variable.
+func expandEndVars(f reflect.Value, fieldType reflect.StructField) {
+	if _, ok := fieldType.Tag.Lookup("yaml"); !ok {
+		return
+	}
+	if f.Kind() == reflect.String {
+		str := f.String()
+		if strings.HasPrefix(str, "$") {
+			env := strings.TrimPrefix(str, "$")
+			retrievedEnv := os.Getenv(env)
+			if retrievedEnv != "" {
+				f.SetString(os.Getenv(env))
+			}
+		}
+	}
+}

v2/internal/runner/runner_test.go

@@ -1,6 +1,7 @@
 package runner
 
 import (
+	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -24,3 +25,120 @@ func Test_createReportingOptions(t *testing.T) {
 	assert.Equal(t, resultOptions2.AllowList.Severities, resultOptions.AllowList.Severities)
 	assert.Equal(t, resultOptions2.DenyList.Severities, resultOptions.DenyList.Severities)
 }
+
+type TestStruct1 struct {
+	A      string       `yaml:"a"`
+	Struct *TestStruct2 `yaml:"b"`
+}
+
+type TestStruct2 struct {
+	B string `yaml:"b"`
+}
+
+type TestStruct3 struct {
+	A string `yaml:"a"`
+	B string `yaml:"b"`
+	C string `yaml:"c"`
+}
+
+type TestStruct4 struct {
+	A      string       `yaml:"a"`
+	Struct *TestStruct3 `yaml:"b"`
+}
+
+type TestStruct5 struct {
+	A []string  `yaml:"a"`
+	B [2]string `yaml:"b"`
+}
+
+type TestStruct6 struct {
+	A string       `yaml:"a"`
+	B *TestStruct2 `yaml:"b"`
+	C string
+}
+
+func TestWalkReflectStructAssignsEnvVars(t *testing.T) {
+	testStruct := &TestStruct1{
+		A: "$VAR_EXAMPLE",
+		Struct: &TestStruct2{
+			B: "$VAR_TWO",
+		},
+	}
+	os.Setenv("VAR_EXAMPLE", "value")
+	os.Setenv("VAR_TWO", "value2")
+
+	Walk(testStruct, expandEndVars)
+
+	assert.Equal(t, "value", testStruct.A)
+	assert.Equal(t, "value2", testStruct.Struct.B)
+}
+
+func TestWalkReflectStructHandlesDifferentTypes(t *testing.T) {
+	testStruct := &TestStruct3{
+		A: "$VAR_EXAMPLE",
+		B: "$VAR_TWO",
+		C: "$VAR_THREE",
+	}
+	os.Setenv("VAR_EXAMPLE", "value")
+	os.Setenv("VAR_TWO", "2")
+	os.Setenv("VAR_THREE", "true")
+
+	Walk(testStruct, expandEndVars)
+
+	assert.Equal(t, "value", testStruct.A)
+	assert.Equal(t, "2", testStruct.B)
+	assert.Equal(t, "true", testStruct.C)
+}
+
+func TestWalkReflectStructEmpty(t *testing.T) {
+	testStruct := &TestStruct3{
+		A: "$VAR_EXAMPLE",
+		B: "",
+		C: "$VAR_THREE",
+	}
+	os.Setenv("VAR_EXAMPLE", "value")
+	os.Setenv("VAR_TWO", "2")
+	os.Setenv("VAR_THREE", "true")
+
+	Walk(testStruct, expandEndVars)
+
+	assert.Equal(t, "value", testStruct.A)
+	assert.Equal(t, "", testStruct.B)
+	assert.Equal(t, "true", testStruct.C)
+}
+
+func TestWalkReflectStructWithNoYamlTag(t *testing.T) {
+	test := &TestStruct6{
+		A: "$GITHUB_USER",
+		B: &TestStruct2{
+			B: "$GITHUB_USER",
+		},
+		C: "$GITHUB_USER",
+	}
+
+	os.Setenv("GITHUB_USER", "testuser")
+
+	Walk(test, expandEndVars)
+	assert.Equal(t, "testuser", test.A)
+	assert.Equal(t, "testuser", test.B.B, test.B)
+	assert.Equal(t, "$GITHUB_USER", test.C)
+}
+
+func TestWalkReflectStructHandlesNestedStructs(t *testing.T) {
+	testStruct := &TestStruct4{
+		A: "$VAR_EXAMPLE",
+		Struct: &TestStruct3{
+			B: "$VAR_TWO",
+			C: "$VAR_THREE",
+		},
+	}
+	os.Setenv("VAR_EXAMPLE", "value")
+	os.Setenv("VAR_TWO", "2")
+	os.Setenv("VAR_THREE", "true")
+
+	Walk(testStruct, expandEndVars)
+
+	assert.Equal(t, "value", testStruct.A)
+	assert.Equal(t, "2", testStruct.Struct.B)
+	assert.Equal(t, "true", testStruct.Struct.C)
+}