From 83681fb308e6df10612d658b456bbd9e8af9b13c Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com> Date: Mon, 30 Oct 2023 19:02:06 +0530 Subject: [PATCH] misc sdk enhancements (#4301) * add template sign/parse methods * export installer package * add readme * consistent implementation of writefailure * fix lint error --- cmd/nuclei/main.go | 2 +- integration_tests/protocols/keys/README.md | 3 + internal/runner/runner.go | 2 +- lib/sdk.go | 35 +++++++++++ lib/sdk_private.go | 2 +- {internal => pkg}/installer/doc.go | 0 {internal => pkg}/installer/template.go | 0 {internal => pkg}/installer/template_test.go | 0 {internal => pkg}/installer/util.go | 0 {internal => pkg}/installer/versioncheck.go | 0 .../installer/versioncheck_test.go | 0 .../installer/zipslip_unix_test.go | 0 pkg/testutils/testutils.go | 58 +++++++++++-------- 13 files changed, 76 insertions(+), 26 deletions(-) create mode 100644 integration_tests/protocols/keys/README.md rename {internal => pkg}/installer/doc.go (100%) rename {internal => pkg}/installer/template.go (100%) rename {internal => pkg}/installer/template_test.go (100%) rename {internal => pkg}/installer/util.go (100%) rename {internal => pkg}/installer/versioncheck.go (100%) rename {internal => pkg}/installer/versioncheck_test.go (100%) rename {internal => pkg}/installer/zipslip_unix_test.go (100%) diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go index 3f8ff504..539a5047 100644 --- a/cmd/nuclei/main.go +++ b/cmd/nuclei/main.go @@ -16,9 +16,9 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/levels" "github.com/projectdiscovery/interactsh/pkg/client" - "github.com/projectdiscovery/nuclei/v3/internal/installer" "github.com/projectdiscovery/nuclei/v3/internal/runner" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" + "github.com/projectdiscovery/nuclei/v3/pkg/installer" "github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v3/pkg/operators/common/dsl" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/uncover" diff --git a/integration_tests/protocols/keys/README.md b/integration_tests/protocols/keys/README.md new file mode 100644 index 00000000..7ba87b6f --- /dev/null +++ b/integration_tests/protocols/keys/README.md @@ -0,0 +1,3 @@ +## keys + +the keys stored here especially `ci-private-key.pem` and `ci.crt` are used in integration tests to test template signing and verfication functionality introduced in nuclei v3 \ No newline at end of file diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 7ae8079e..4d320b4c 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -12,8 +12,8 @@ import ( "sync/atomic" "time" - "github.com/projectdiscovery/nuclei/v3/internal/installer" "github.com/projectdiscovery/nuclei/v3/internal/runner/nucleicloud" + "github.com/projectdiscovery/nuclei/v3/pkg/installer" uncoverlib "github.com/projectdiscovery/uncover" permissionutil "github.com/projectdiscovery/utils/permission" updateutils "github.com/projectdiscovery/utils/update" diff --git a/lib/sdk.go b/lib/sdk.go index ddfd583c..70a28299 100644 --- a/lib/sdk.go +++ b/lib/sdk.go @@ -2,6 +2,7 @@ package nuclei import ( "bufio" + "bytes" "io" "github.com/projectdiscovery/httpx/common/httpx" @@ -18,6 +19,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine" "github.com/projectdiscovery/nuclei/v3/pkg/reporting" "github.com/projectdiscovery/nuclei/v3/pkg/templates" + "github.com/projectdiscovery/nuclei/v3/pkg/templates/signer" "github.com/projectdiscovery/nuclei/v3/pkg/types" "github.com/projectdiscovery/ratelimit" "github.com/projectdiscovery/retryablehttp-go" @@ -127,6 +129,39 @@ func (e *NucleiEngine) LoadTargetsFromReader(reader io.Reader, probeNonHttp bool } } +// GetExecuterOptions returns the nuclei executor options +func (e *NucleiEngine) GetExecuterOptions() *protocols.ExecutorOptions { + return &e.executerOpts +} + +// ParseTemplate parses a template from given data +// template verification status can be accessed from template.Verified +func (e *NucleiEngine) ParseTemplate(data []byte) (*templates.Template, error) { + return templates.ParseTemplateFromReader(bytes.NewReader(data), nil, e.executerOpts) +} + +// SignTemplate signs the tempalate using given signer +func (e *NucleiEngine) SignTemplate(tmplSigner *signer.TemplateSigner, data []byte) ([]byte, error) { + tmpl, err := e.ParseTemplate(data) + if err != nil { + return data, err + } + if tmpl.Verified { + // already signed + return data, nil + } + if len(tmpl.Workflows) > 0 { + return data, templates.ErrNotATemplate + } + signatureData, err := tmplSigner.Sign(data, tmpl) + if err != nil { + return data, err + } + buff := bytes.NewBuffer(signer.RemoveSignatureFromData(data)) + buff.WriteString("\n" + signatureData) + return buff.Bytes(), err +} + // Close all resources used by nuclei engine func (e *NucleiEngine) Close() { e.interactshClient.Close() diff --git a/lib/sdk_private.go b/lib/sdk_private.go index a0e6292e..cd84b0b6 100644 --- a/lib/sdk_private.go +++ b/lib/sdk_private.go @@ -11,12 +11,12 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/levels" "github.com/projectdiscovery/httpx/common/httpx" - "github.com/projectdiscovery/nuclei/v3/internal/installer" "github.com/projectdiscovery/nuclei/v3/internal/runner" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/disk" "github.com/projectdiscovery/nuclei/v3/pkg/core" "github.com/projectdiscovery/nuclei/v3/pkg/core/inputs" + "github.com/projectdiscovery/nuclei/v3/pkg/installer" "github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/nuclei/v3/pkg/progress" "github.com/projectdiscovery/nuclei/v3/pkg/protocols" diff --git a/internal/installer/doc.go b/pkg/installer/doc.go similarity index 100% rename from internal/installer/doc.go rename to pkg/installer/doc.go diff --git a/internal/installer/template.go b/pkg/installer/template.go similarity index 100% rename from internal/installer/template.go rename to pkg/installer/template.go diff --git a/internal/installer/template_test.go b/pkg/installer/template_test.go similarity index 100% rename from internal/installer/template_test.go rename to pkg/installer/template_test.go diff --git a/internal/installer/util.go b/pkg/installer/util.go similarity index 100% rename from internal/installer/util.go rename to pkg/installer/util.go diff --git a/internal/installer/versioncheck.go b/pkg/installer/versioncheck.go similarity index 100% rename from internal/installer/versioncheck.go rename to pkg/installer/versioncheck.go diff --git a/internal/installer/versioncheck_test.go b/pkg/installer/versioncheck_test.go similarity index 100% rename from internal/installer/versioncheck_test.go rename to pkg/installer/versioncheck_test.go diff --git a/internal/installer/zipslip_unix_test.go b/pkg/installer/zipslip_unix_test.go similarity index 100% rename from internal/installer/zipslip_unix_test.go rename to pkg/installer/zipslip_unix_test.go diff --git a/pkg/testutils/testutils.go b/pkg/testutils/testutils.go index 12485c5b..da2d2030 100644 --- a/pkg/testutils/testutils.go +++ b/pkg/testutils/testutils.go @@ -5,6 +5,7 @@ import ( "time" "github.com/projectdiscovery/ratelimit" + "go.uber.org/multierr" "github.com/logrusorgru/aurora" @@ -140,35 +141,46 @@ func (m *MockOutputWriter) Request(templateID, url, requestType string, err erro // WriteFailure writes the event to file and/or screen. func (m *MockOutputWriter) WriteFailure(wrappedEvent *output.InternalWrappedEvent) error { - if m.WriteCallback != nil { - // create event - event := wrappedEvent.InternalEvent - templatePath, templateURL := utils.TemplatePathURL(types.ToString(event["template-path"]), types.ToString(event["template-id"])) - var templateInfo model.Info - if ti, ok := event["template-info"].(model.Info); ok { - templateInfo = ti + // if failure event has more than one result, write them all + if len(wrappedEvent.Results) > 0 { + errs := []error{} + for _, result := range wrappedEvent.Results { + result.MatcherStatus = false // just in case + if err := m.Write(result); err != nil { + errs = append(errs, err) + } } - data := &output.ResultEvent{ - Template: templatePath, - TemplateURL: templateURL, - TemplateID: types.ToString(event["template-id"]), - TemplatePath: types.ToString(event["template-path"]), - Info: templateInfo, - Type: types.ToString(event["type"]), - Host: types.ToString(event["host"]), - Request: types.ToString(event["request"]), - Response: types.ToString(event["response"]), - MatcherStatus: false, - Timestamp: time.Now(), + if len(errs) > 0 { + return multierr.Combine(errs...) } - m.WriteCallback(data) + return nil } - return nil -} -func (m *MockOutputWriter) WriteStoreDebugData(host, templateID, eventType string, data string) { + // create event + event := wrappedEvent.InternalEvent + templatePath, templateURL := utils.TemplatePathURL(types.ToString(event["template-path"]), types.ToString(event["template-id"])) + var templateInfo model.Info + if ti, ok := event["template-info"].(model.Info); ok { + templateInfo = ti + } + data := &output.ResultEvent{ + Template: templatePath, + TemplateURL: templateURL, + TemplateID: types.ToString(event["template-id"]), + TemplatePath: types.ToString(event["template-path"]), + Info: templateInfo, + Type: types.ToString(event["type"]), + Host: types.ToString(event["host"]), + Request: types.ToString(event["request"]), + Response: types.ToString(event["response"]), + MatcherStatus: false, + Timestamp: time.Now(), + } + return m.Write(data) } +func (m *MockOutputWriter) WriteStoreDebugData(host, templateID, eventType string, data string) {} + type MockProgressClient struct{} // Stop stops the progress recorder.