daffainfo
/
nuclei
mirror of https://github.com/daffainfo/nuclei.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #955 from projectdiscovery/structure-overhaul 

Nuclei Structure + Template Loading Overhaul
dev
Ice3man 2021-08-19 17:53:07 +05:30 committed by GitHub
parent b9d4f3021c 6bfd6bd28f
commit 6bea033d53
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 185 additions and 219 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
v2/internal/runner/templates.go
View File

@ -1,42 +1,18 @@
package runner
import (
"bytes"
"fmt"
"io/ioutil"
"os"
"strings"
"github.com/karrick/godirwalk"
"gopkg.in/yaml.v2"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/internal/severity"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
"github.com/projectdiscovery/nuclei/v2/pkg/parsers"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
)
// parseTemplateFile returns the parsed template file
func (r *Runner) parseTemplateFile(file string) (*templates.Template, error) {
f, err := os.Open(file)
if err != nil {
return nil, err
}
defer f.Close()
data, err := ioutil.ReadAll(f)
if err != nil {
return nil, err
}
template := &templates.Template{}
err = yaml.NewDecoder(bytes.NewReader(data)).Decode(template)
if err != nil {
return nil, err
}
return template, nil
}
func (r *Runner) templateLogMsg(id, name, author string, templateSeverity severity.Severity) string {
// Display the message for the template
return fmt.Sprintf("[%s] %s (%s) [%s]",
@ -70,7 +46,7 @@ func appendAtSignToAuthors(author string) string {
}
func (r *Runner) logAvailableTemplate(tplPath string) {
t, err := r.parseTemplateFile(tplPath)
t, err := parsers.ParseTemplate(tplPath)
if err != nil {
gologger.Error().Msgf("Could not parse file '%s': %s\n", tplPath, err)
} else {

103
v2/pkg/catalog/loader/filter/tag_filter.go
View File

@ -19,15 +19,12 @@ type TagFilter struct {
// ErrExcluded is returned for excluded templates
var ErrExcluded = errors.New("the template was excluded")
// Match takes a tag and whether the template was matched from user
// input and returns true or false using a tag filter.
//
// If the tag was specified in deny list, it will not return true
// unless it is explicitly specified by user in includeTags which is the
// matchAllows section.
//
// It returns true if the tag is specified, or false.
func (tagFilter *TagFilter) Match(templateTags, templateAuthors []string, templateSeverity severity.Severity) (bool, error) {
// Match filters templates based on user provided tags, authors, extraTags and severity.
// If the template contains tags specified in the deny list, it will not be matched
// unless it is explicitly specified by user using the includeTags (matchAllows field).
// Matching rule: (tag1 OR tag2...) AND (author1 OR author2...) AND (severity1 OR severity2...) AND (extraTags1 OR extraTags2...)
// Returns true if the template matches the filter criteria, false otherwise.
func (tagFilter *TagFilter) Match(templateTags, templateAuthors []string, templateSeverity severity.Severity, extraTags []string) (bool, error) {
for _, templateTag := range templateTags {
_, blocked := tagFilter.block[templateTag]
_, allowed := tagFilter.matchAllows[templateTag]
@ -37,30 +34,45 @@ func (tagFilter *TagFilter) Match(templateTags, templateAuthors []string, templa
}
}
if !isTagMatch(templateTags, tagFilter) {
if !isExtraTagMatch(extraTags, templateTags) {
return false, nil
}
if !isAuthorMatch(templateAuthors, tagFilter) {
if !isTagMatch(tagFilter, templateTags) {
return false, nil
}
if len(tagFilter.severities) > 0 {
if _, ok := tagFilter.severities[templateSeverity]; !ok {
return false, nil
}
if !isAuthorMatch(tagFilter, templateAuthors) {
return false, nil
}
if !isSeverityMatch(tagFilter, templateSeverity) {
return false, nil
}
return true, nil
}
func isAuthorMatch(templateAuthors []string, tagFilter *TagFilter) bool {
func isSeverityMatch(tagFilter *TagFilter, templateSeverity severity.Severity) bool {
if len(tagFilter.severities) == 0 {
return true
}
if _, ok := tagFilter.severities[templateSeverity]; ok {
return true
}
return false
}
func isAuthorMatch(tagFilter *TagFilter, templateAuthors []string) bool {
if len(tagFilter.authors) == 0 {
return true
}
for _, templateAuthor := range templateAuthors {
if _, ok := tagFilter.authors[templateAuthor]; ok {
templateAuthorMap := toMap(templateAuthors)
for requiredAuthor := range tagFilter.authors {
if _, ok := templateAuthorMap[requiredAuthor]; ok {
return true
}
}
@ -68,7 +80,22 @@ func isAuthorMatch(templateAuthors []string, tagFilter *TagFilter) bool {
return false
}
func isTagMatch(templateTags []string, tagFilter *TagFilter) bool {
func isExtraTagMatch(extraTags []string, templateTags []string) bool {
if len(extraTags) == 0 {
return true
}
templatesTagMap := toMap(templateTags)
for _, extraTag := range extraTags {
if _, ok := templatesTagMap[extraTag]; ok {
return true
}
}
return false
}
func isTagMatch(tagFilter *TagFilter, templateTags []string) bool {
if len(tagFilter.allowedTags) == 0 {
return true
}
@ -82,42 +109,6 @@ func isTagMatch(templateTags []string, tagFilter *TagFilter) bool {
return false
}
// MatchWithWorkflowTags takes an addition list of allowed tags and returns true if the match was successful.
func (tagFilter *TagFilter) MatchWithWorkflowTags(templateTags, templateAuthors []string, templateSeverity severity.Severity, workflowTags []string) (bool, error) {
for _, templateTag := range templateTags {
_, blocked := tagFilter.block[templateTag]
_, allowed := tagFilter.matchAllows[templateTag]
if blocked && !allowed { // the whitelist has precedence over the blacklist
return false, ErrExcluded
}
}
templatesTagMap := toMap(templateTags)
for _, workflowTag := range workflowTags {
if _, ok := templatesTagMap[workflowTag]; !ok {
return false, nil
}
}
if len(tagFilter.authors) > 0 {
templateAuthorTagMap := toMap(templateAuthors)
for requiredAuthor := range tagFilter.authors {
if _, ok := templateAuthorTagMap[requiredAuthor]; !ok {
return false, nil
}
}
}
if len(tagFilter.severities) > 0 {
if _, ok := tagFilter.severities[templateSeverity]; !ok {
return false, nil
}
}
return true, nil
}
type Config struct {
Tags []string
ExcludeTags []string
@ -193,7 +184,7 @@ func splitCommaTrim(value string) []string {
}
func toMap(slice []string) map[string]struct{} {
result := make(map[string]struct{})
result := make(map[string]struct{}, len(slice))
for _, value := range slice {
if _, ok := result[value]; !ok {
result[value] = struct{}{}

24
v2/pkg/catalog/loader/filter/tag_filter_test.go
View File

@ -15,11 +15,11 @@ func TestTagBasedFilter(t *testing.T) {
filter := New(config)
t.Run("true", func(t *testing.T) {
matched, _ := filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low)
matched, _ := filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil)
require.True(t, matched, "could not get correct match")
})
t.Run("false", func(t *testing.T) {
matched, _ := filter.Match([]string{"consul"}, []string{"pdteam"}, severity.Low)
matched, _ := filter.Match([]string{"consul"}, []string{"pdteam"}, severity.Low, nil)
require.False(t, matched, "could not get correct match")
})
t.Run("not-match-excludes", func(t *testing.T) {
@ -27,7 +27,7 @@ func TestTagBasedFilter(t *testing.T) {
ExcludeTags: []string{"dos"},
}
filter := New(config)
matched, err := filter.Match([]string{"dos"}, []string{"pdteam"}, severity.Low)
matched, err := filter.Match([]string{"dos"}, []string{"pdteam"}, severity.Low, nil)
require.False(t, matched, "could not get correct match")
require.Equal(t, ErrExcluded, err, "could not get correct error")
})
@ -38,7 +38,7 @@ func TestTagBasedFilter(t *testing.T) {
IncludeTags: []string{"fuzz"},
}
filter := New(config)
matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low)
matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil)
require.Nil(t, err, "could not get match")
require.True(t, matched, "could not get correct match")
})
@ -48,7 +48,7 @@ func TestTagBasedFilter(t *testing.T) {
ExcludeTags: []string{"fuzz"},
}
filter := New(config)
matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low)
matched, err := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil)
require.Nil(t, err, "could not get match")
require.True(t, matched, "could not get correct match")
})
@ -57,7 +57,7 @@ func TestTagBasedFilter(t *testing.T) {
Authors: []string{"pdteam"},
}
filter := New(config)
matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low)
matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil)
require.True(t, matched, "could not get correct match")
})
t.Run("match-severity", func(t *testing.T) {
@ -65,7 +65,7 @@ func TestTagBasedFilter(t *testing.T) {
Severities: severity.Severities{severity.High},
}
filter := New(config)
matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High)
matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil)
require.True(t, matched, "could not get correct match")
})
t.Run("match-exclude-with-tags", func(t *testing.T) {
@ -74,7 +74,7 @@ func TestTagBasedFilter(t *testing.T) {
ExcludeTags: []string{"another"},
}
filter := New(config)
matched, _ := filter.Match([]string{"another"}, []string{"pdteam"}, severity.High)
matched, _ := filter.Match([]string{"another"}, []string{"pdteam"}, severity.High, nil)
require.False(t, matched, "could not get correct match")
})
t.Run("match-conditions", func(t *testing.T) {
@ -84,13 +84,13 @@ func TestTagBasedFilter(t *testing.T) {
Severities: severity.Severities{severity.High},
}
filter := New(config)
matched, _ := filter.Match([]string{"jira"}, []string{"pdteam"}, severity.High)
matched, _ := filter.Match([]string{"jira"}, []string{"pdteam"}, severity.High, nil)
require.True(t, matched, "could not get correct match")
matched, _ = filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low)
matched, _ = filter.Match([]string{"jira"}, []string{"pdteam"}, severity.Low, nil)
require.False(t, matched, "could not get correct match")
matched, _ = filter.Match([]string{"jira"}, []string{"random"}, severity.Low)
matched, _ = filter.Match([]string{"jira"}, []string{"random"}, severity.Low, nil)
require.False(t, matched, "could not get correct match")
matched, _ = filter.Match([]string{"consul"}, []string{"random"}, severity.Low)
matched, _ = filter.Match([]string{"consul"}, []string{"random"}, severity.Low, nil)
require.False(t, matched, "could not get correct match")
})
}

130
v2/pkg/catalog/loader/loader.go
View File

@ -70,103 +70,89 @@ func New(config *Config) (*Store, error) {
}
// Templates returns all the templates in the store
func (s *Store) Templates() []*templates.Template {
return s.templates
func (store *Store) Templates() []*templates.Template {
return store.templates
}
// Workflows returns all the workflows in the store
func (s *Store) Workflows() []*templates.Template {
return s.workflows
func (store *Store) Workflows() []*templates.Template {
return store.workflows
}
// RegisterPreprocessor allows a custom preprocessor to be passed to the store to run against templates
func (s *Store) RegisterPreprocessor(preprocessor templates.Preprocessor) {
s.preprocessor = preprocessor
func (store *Store) RegisterPreprocessor(preprocessor templates.Preprocessor) {
store.preprocessor = preprocessor
}
// Load loads all the templates from a store, performs filtering and returns
// the complete compiled templates for a nuclei execution configuration.
func (s *Store) Load() {
s.templates = s.LoadTemplates(s.finalTemplates)
s.workflows = s.LoadWorkflows(s.config.Workflows)
func (store *Store) Load() {
store.templates = store.LoadTemplates(store.finalTemplates)
store.workflows = store.LoadWorkflows(store.config.Workflows)
}
// ValidateTemplates takes a list of templates and validates them
// erroring out on discovering any faulty templates.
func (s *Store) ValidateTemplates(templatesList, workflowsList []string) bool {
includedTemplates := s.config.Catalog.GetTemplatesPath(templatesList)
includedWorkflows := s.config.Catalog.GetTemplatesPath(workflowsList)
templatesMap := s.pathFilter.Match(includedTemplates)
workflowsMap := s.pathFilter.Match(includedWorkflows)
func (store *Store) ValidateTemplates(templatesList, workflowsList []string) bool {
templatePaths := store.config.Catalog.GetTemplatesPath(templatesList)
workflowPaths := store.config.Catalog.GetTemplatesPath(workflowsList)
filteredTemplatePaths := store.pathFilter.Match(templatePaths)
filteredWorkflowPaths := store.pathFilter.Match(workflowPaths)
notErrored := true
for k := range templatesMap {
_, err := s.loadTemplate(k, false)
errorValidationFunc := func(message string, template string, err error) {
if strings.Contains(err.Error(), "cannot create template executer") {
return
}
if err == filter.ErrExcluded {
return
}
notErrored = false
gologger.Error().Msgf(message, template, err)
}
for templatePath := range filteredTemplatePaths {
_, err := parsers.LoadTemplate(templatePath, store.tagFilter, nil)
if err != nil {
if strings.Contains(err.Error(), "cannot create template executer") {
continue
}
if err == filter.ErrExcluded {
continue
}
notErrored = false
gologger.Error().Msgf("Error occurred loading template %s: %s\n", k, err)
errorValidationFunc("Error occurred loading template %s: %s\n", templatePath, err)
continue
}
_, err = templates.Parse(k, s.preprocessor, s.config.ExecutorOptions)
_, err = templates.Parse(templatePath, store.preprocessor, store.config.ExecutorOptions)
if err != nil {
if strings.Contains(err.Error(), "cannot create template executer") {
continue
}
if err == filter.ErrExcluded {
continue
}
notErrored = false
gologger.Error().Msgf("Error occurred parsing template %s: %s\n", k, err)
errorValidationFunc("Error occurred parsing template %s: %s\n", templatePath, err)
continue
}
}
for k := range workflowsMap {
_, err := s.loadTemplate(k, true)
for workflowPath := range filteredWorkflowPaths {
_, err := parsers.LoadWorkflow(workflowPath, store.tagFilter)
if err != nil {
if strings.Contains(err.Error(), "cannot create template executer") {
continue
}
if err == filter.ErrExcluded {
continue
}
notErrored = false
gologger.Error().Msgf("Error occurred loading workflow %s: %s\n", k, err)
errorValidationFunc("Error occurred loading workflow %s: %s\n", workflowPath, err)
continue
}
_, err = templates.Parse(k, s.preprocessor, s.config.ExecutorOptions)
_, err = templates.Parse(workflowPath, store.preprocessor, store.config.ExecutorOptions)
if err != nil {
if strings.Contains(err.Error(), "cannot create template executer") {
continue
}
if err == filter.ErrExcluded {
continue
}
notErrored = false
gologger.Error().Msgf("Error occurred parsing workflow %s: %s\n", k, err)
errorValidationFunc("Error occurred parsing workflow %s: %s\n", workflowPath, err)
continue
}
}
return notErrored
}
// LoadTemplates takes a list of templates and returns paths for them
func (s *Store) LoadTemplates(templatesList []string) []*templates.Template {
includedTemplates := s.config.Catalog.GetTemplatesPath(templatesList)
templatesMap := s.pathFilter.Match(includedTemplates)
func (store *Store) LoadTemplates(templatesList []string) []*templates.Template {
includedTemplates := store.config.Catalog.GetTemplatesPath(templatesList)
templatePathMap := store.pathFilter.Match(includedTemplates)
loadedTemplates := make([]*templates.Template, 0, len(templatesMap))
for k := range templatesMap {
loaded, err := s.loadTemplate(k, false)
loadedTemplates := make([]*templates.Template, 0, len(templatePathMap))
for templatePath := range templatePathMap {
loaded, err := parsers.LoadTemplate(templatePath, store.tagFilter, nil)
if err != nil {
gologger.Warning().Msgf("Could not load template %s: %s\n", k, err)
gologger.Warning().Msgf("Could not load template %s: %s\n", templatePath, err)
}
if loaded {
parsed, err := templates.Parse(k, s.preprocessor, s.config.ExecutorOptions)
parsed, err := templates.Parse(templatePath, store.preprocessor, store.config.ExecutorOptions)
if err != nil {
gologger.Warning().Msgf("Could not parse template %s: %s\n", k, err)
gologger.Warning().Msgf("Could not parse template %s: %s\n", templatePath, err)
} else if parsed != nil {
loadedTemplates = append(loadedTemplates, parsed)
}
@ -176,20 +162,20 @@ func (s *Store) LoadTemplates(templatesList []string) []*templates.Template {
}
// LoadWorkflows takes a list of workflows and returns paths for them
func (s *Store) LoadWorkflows(workflowsList []string) []*templates.Template {
includedWorkflows := s.config.Catalog.GetTemplatesPath(workflowsList)
workflowsMap := s.pathFilter.Match(includedWorkflows)
func (store *Store) LoadWorkflows(workflowsList []string) []*templates.Template {
includedWorkflows := store.config.Catalog.GetTemplatesPath(workflowsList)
workflowPathMap := store.pathFilter.Match(includedWorkflows)
loadedWorkflows := make([]*templates.Template, 0, len(workflowsMap))
for k := range workflowsMap {
loaded, err := s.loadTemplate(k, true)
loadedWorkflows := make([]*templates.Template, 0, len(workflowPathMap))
for workflowPath := range workflowPathMap {
loaded, err := parsers.LoadWorkflow(workflowPath, store.tagFilter)
if err != nil {
gologger.Warning().Msgf("Could not load workflow %s: %s\n", k, err)
gologger.Warning().Msgf("Could not load workflow %s: %s\n", workflowPath, err)
}
if loaded {
parsed, err := templates.Parse(k, s.preprocessor, s.config.ExecutorOptions)
parsed, err := templates.Parse(workflowPath, store.preprocessor, store.config.ExecutorOptions)
if err != nil {
gologger.Warning().Msgf("Could not parse workflow %s: %s\n", k, err)
gologger.Warning().Msgf("Could not parse workflow %s: %s\n", workflowPath, err)
} else if parsed != nil {
loadedWorkflows = append(loadedWorkflows, parsed)
}
@ -197,7 +183,3 @@ func (s *Store) LoadWorkflows(workflowsList []string) []*templates.Template {
}
return loadedWorkflows
}
func (s *Store) loadTemplate(templatePath string, isWorkflow bool) (bool, error) {
return parsers.Load(templatePath, isWorkflow, nil, s.tagFilter) // TODO consider separating template and workflow loading logic
}

3
v2/pkg/model/model.go
View File

@ -3,9 +3,10 @@ package model
import (
"encoding/json"
"fmt"
"gopkg.in/yaml.v2"
"strings"
"gopkg.in/yaml.v2"
"github.com/projectdiscovery/nuclei/v2/internal/severity"
"github.com/projectdiscovery/nuclei/v2/pkg/utils"
)

10
v2/pkg/model/worflow_loader.go
View File

@ -1,10 +1,12 @@
package model
// TODO shouldn't this rather be TemplateLoader?
// WorkflowLoader is a loader interface required for workflow initialization.
type WorkflowLoader interface {
// ListTags lists a list of templates for tags from the provided templates directory
ListTags(workflowTags []string) []string
// GetTemplatePathsByTags returns a list of template paths based on the provided tags from the templates directory
GetTemplatePathsByTags(tags []string) []string
// ListTemplates takes a list of templates and returns paths for them
ListTemplates(templatesList []string, noValidate bool) []string
// GetTemplatePaths takes a list of templates and returns paths for them
GetTemplatePaths(templatesList []string, noValidate bool) []string
}

68
v2/pkg/parsers/parser.go
View File

@ -1,13 +1,14 @@
package parsers
import (
"bytes"
"fmt"
"io/ioutil"
"os"
"regexp"
"gopkg.in/yaml.v2"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader/filter"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/templates"
@ -16,49 +17,57 @@ import (
const mandatoryFieldMissingTemplate = "mandatory '%s' field is missing"
// Load loads a template by parsing metadata and running all tag and path based filters on the template.
func Load(templatePath string, isWorkflow bool, workflowTags []string, tagFilter *filter.TagFilter) (bool, error) {
template, templateParseError := parseTemplate(templatePath)
// LoadTemplate returns true if the template is valid and matches the filtering criteria.
func LoadTemplate(templatePath string, tagFilter *filter.TagFilter, extraTags []string) (bool, error) {
template, templateParseError := ParseTemplate(templatePath)
if templateParseError != nil {
return false, templateParseError
}
if len(template.Workflows) > 0 {
return false, nil
}
templateInfo := template.Info
if validationError := validateMandatoryInfoFields(&templateInfo); validationError != nil {
return false, validationError
}
if len(template.Workflows) > 0 {
if isWorkflow {
return true, nil // if a workflow is declared and this template is a workflow, then load
} else { //nolint:indent-error-flow,revive // preferred: readability and extensibility
return false, nil // if a workflow is declared and this template is not a workflow then do not load
}
} else if isWorkflow {
return false, nil // if no workflows are declared and this template is a workflow then do not load
} else { // if workflows are not declared and the template is not a workflow then parse it
return isInfoMetadataMatch(tagFilter, &templateInfo, workflowTags)
}
return isTemplateInfoMetadataMatch(tagFilter, &templateInfo, extraTags)
}
func isInfoMetadataMatch(tagFilter *filter.TagFilter, templateInfo *model.Info, workflowTags []string) (bool, error) {
// LoadWorkflow returns true if the workflow is valid and matches the filtering criteria.
func LoadWorkflow(templatePath string, tagFilter *filter.TagFilter) (bool, error) {
template, templateParseError := ParseTemplate(templatePath)
if templateParseError != nil {
return false, templateParseError
}
templateInfo := template.Info
if len(template.Workflows) > 0 {
if validationError := validateMandatoryInfoFields(&templateInfo); validationError != nil {
return false, validationError
}
return isTemplateInfoMetadataMatch(tagFilter, &templateInfo, nil) // we don't want workflows to be loaded by tags
}
return false, nil
}
func isTemplateInfoMetadataMatch(tagFilter *filter.TagFilter, templateInfo *model.Info, extraTags []string) (bool, error) {
templateTags := templateInfo.Tags.ToSlice()
templateAuthors := templateInfo.Authors.ToSlice()
templateSeverity := templateInfo.SeverityHolder.Severity
var match bool
var err error
if len(workflowTags) == 0 {
match, err = tagFilter.Match(templateTags, templateAuthors, templateSeverity)
} else {
match, err = tagFilter.MatchWithWorkflowTags(templateTags, templateAuthors, templateSeverity, workflowTags)
}
match, err := tagFilter.Match(templateTags, templateAuthors, templateSeverity, extraTags)
if err == filter.ErrExcluded {
return false, filter.ErrExcluded
}
return match, nil
return match, err
}
func validateMandatoryInfoFields(info *model.Info) error {
@ -76,7 +85,10 @@ func validateMandatoryInfoFields(info *model.Info) error {
return nil
}
func parseTemplate(templatePath string) (*templates.Template, error) {
var fieldErrorRegexp = regexp.MustCompile(`not found in`)
// ParseTemplate parses a template and returns a *templates.Template structure
func ParseTemplate(templatePath string) (*templates.Template, error) {
f, err := os.Open(templatePath)
if err != nil {
return nil, err
@ -89,8 +101,12 @@ func parseTemplate(templatePath string) (*templates.Template, error) {
}
template := &templates.Template{}
err = yaml.NewDecoder(bytes.NewReader(data)).Decode(template)
err = yaml.UnmarshalStrict(data, template)
if err != nil {
if fieldErrorRegexp.MatchString(err.Error()) {
gologger.Warning().Msgf("Unrecognized fields in template %s: %s", templatePath, err)
return template, nil
}
return nil, err
}
return template, nil

30
v2/pkg/parsers/workflow_loader.go
View File

@ -29,35 +29,33 @@ func NewLoader(options *protocols.ExecuterOptions) (model.WorkflowLoader, error)
return &workflowLoader{pathFilter: pathFilter, tagFilter: tagFilter, options: options}, nil
}
// ListTags lists a list of templates for tags from the provided templates directory
func (w *workflowLoader) ListTags(workflowTags []string) []string {
func (w *workflowLoader) GetTemplatePathsByTags(templateTags []string) []string {
includedTemplates := w.options.Catalog.GetTemplatesPath([]string{w.options.Options.TemplatesDirectory})
templatesMap := w.pathFilter.Match(includedTemplates)
templatePathMap := w.pathFilter.Match(includedTemplates)
loadedTemplates := make([]string, 0, len(templatesMap))
for k := range templatesMap {
loaded, err := Load(k, false, workflowTags, w.tagFilter)
loadedTemplates := make([]string, 0, len(templatePathMap))
for templatePath := range templatePathMap {
loaded, err := LoadTemplate(templatePath, w.tagFilter, templateTags)
if err != nil {
gologger.Warning().Msgf("Could not load template %s: %s\n", k, err)
gologger.Warning().Msgf("Could not load template %s: %s\n", templatePath, err)
} else if loaded {
loadedTemplates = append(loadedTemplates, k)
loadedTemplates = append(loadedTemplates, templatePath)
}
}
return loadedTemplates
}
// ListTemplates takes a list of templates and returns paths for them
func (w *workflowLoader) ListTemplates(templatesList []string, noValidate bool) []string {
func (w *workflowLoader) GetTemplatePaths(templatesList []string, noValidate bool) []string {
includedTemplates := w.options.Catalog.GetTemplatesPath(templatesList)
templatesMap := w.pathFilter.Match(includedTemplates)
templatesPathMap := w.pathFilter.Match(includedTemplates)
loadedTemplates := make([]string, 0, len(templatesMap))
for k := range templatesMap {
matched, err := Load(k, false, nil, w.tagFilter)
loadedTemplates := make([]string, 0, len(templatesPathMap))
for templatePath := range templatesPathMap {
matched, err := LoadTemplate(templatePath, w.tagFilter, nil)
if err != nil {
gologger.Warning().Msgf("Could not load template %s: %s\n", k, err)
gologger.Warning().Msgf("Could not load template %s: %s\n", templatePath, err)
} else if matched || noValidate {
loadedTemplates = append(loadedTemplates, k)
loadedTemplates = append(loadedTemplates, templatePath)
}
}
return loadedTemplates

8
v2/pkg/templates/workflows.go
View File

@ -48,11 +48,11 @@ func parseWorkflow(preprocessor Preprocessor, workflow *workflows.WorkflowTempla
func parseWorkflowTemplate(workflow *workflows.WorkflowTemplate, preprocessor Preprocessor, options *protocols.ExecuterOptions, loader model.WorkflowLoader, noValidate bool) error {
var paths []string
workflowTags := workflow.Tags
if !workflowTags.IsEmpty() {
paths = loader.ListTags(workflowTags.ToSlice())
subTemplateTags := workflow.Tags
if !subTemplateTags.IsEmpty() {
paths = loader.GetTemplatePathsByTags(subTemplateTags.ToSlice())
} else {
paths = loader.ListTemplates([]string{workflow.Template}, noValidate)
paths = loader.GetTemplatePaths([]string{workflow.Template}, noValidate)
}
if len(paths) == 0 {
return nil