daffainfo
/
nuclei
mirror of https://github.com/daffainfo/nuclei.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Extractor + Headless Actions structures to enums (#1217) 

* enum support for extractor
dev
LuitelSamikshya 2021-11-18 14:11:10 -06:00 committed by GitHub
parent ccb588f383
commit 2856e7e247
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 382 additions and 226 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
v2/pkg/operators/extractors/compile.go
View File

@ -10,13 +10,12 @@ import (
// CompileExtractors performs the initial setup operation on an extractor
func (e *Extractor) CompileExtractors() error {
var ok bool
// Set up the extractor type
e.extractorType, ok = ExtractorTypes[e.Type]
if !ok {
computedType, err := toExtractorTypes(e.GetType().String())
if err != nil {
return fmt.Errorf("unknown extractor type specified: %s", e.Type)
}
e.extractorType = computedType
// Compile the regexes
for _, regex := range e.Regex {
compiled, err := regexp.Compile(regex)
@ -25,7 +24,6 @@ func (e *Extractor) CompileExtractors() error {
}
e.regexCompiled = append(e.regexCompiled, compiled)
}
for i, kval := range e.KVal {
e.KVal[i] = strings.ToLower(kval)
}
@ -43,7 +41,7 @@ func (e *Extractor) CompileExtractors() error {
}
if e.CaseInsensitive {
if e.Type != "kval" {
if e.GetType() != KValExtractor {
return fmt.Errorf("case-insensitive flag is supported only for 'kval' extractors (not '%s')", e.Type)
}
for i := range e.KVal {

105
v2/pkg/operators/extractors/extractor_types.go Normal file
View File

@ -0,0 +1,105 @@
package extractors
import (
"encoding/json"
"errors"
"strings"
"github.com/alecthomas/jsonschema"
)
// ExtractorType is the type of the extractor specified
type ExtractorType int
const (
// RegexExtractor extracts responses with regexes
RegexExtractor ExtractorType = iota + 1
// KValExtractor extracts responses with key:value
KValExtractor
// XPathExtractor extracts responses with Xpath selectors
XPathExtractor
// JSONExtractor extracts responses with json
JSONExtractor
//limit
limit
)
// extractorMappings is a table for conversion of extractor type from string.
var extractorMappings = map[ExtractorType]string{
RegexExtractor: "regex",
KValExtractor: "kval",
XPathExtractor: "xpath",
JSONExtractor: "json",
}
// GetType returns the type of the matcher
func (e *Extractor) GetType() ExtractorType {
return e.Type.ExtractorType
}
// GetSupportedExtractorTypes returns list of supported types
func GetSupportedExtractorTypes() []ExtractorType {
var result []ExtractorType
for index := ExtractorType(1); index < limit; index++ {
result = append(result, index)
}
return result
}
func toExtractorTypes(valueToMap string) (ExtractorType, error) {
normalizedValue := normalizeValue(valueToMap)
for key, currentValue := range extractorMappings {
if normalizedValue == currentValue {
return key, nil
}
}
return -1, errors.New("Invalid extractor type: " + valueToMap)
}
func normalizeValue(value string) string {
return strings.TrimSpace(strings.ToLower(value))
}
func (t ExtractorType) String() string {
return extractorMappings[t]
}
// TypeHolder is used to hold internal type of the extractor
type TypeHolder struct {
ExtractorType ExtractorType
}
func (holder TypeHolder) JSONSchemaType() *jsonschema.Type {
gotType := &jsonschema.Type{
Type: "string",
Title: "type of the extractor",
Description: "Type of the extractor",
}
for _, types := range GetSupportedExtractorTypes() {
gotType.Enum = append(gotType.Enum, types.String())
}
return gotType
}
func (holder *TypeHolder) UnmarshalYAML(unmarshal func(interface{}) error) error {
var marshalledTypes string
if err := unmarshal(&marshalledTypes); err != nil {
return err
}
computedType, err := toExtractorTypes(marshalledTypes)
if err != nil {
return err
}
holder.ExtractorType = computedType
return nil
}
func (holder *TypeHolder) MarshalJSON() ([]byte, error) {
return json.Marshal(holder.ExtractorType.String())
}
func (holder TypeHolder) MarshalYAML() (interface{}, error) {
return holder.ExtractorType.String(), nil
}

29
v2/pkg/operators/extractors/extractors.go
View File

@ -21,7 +21,7 @@ type Extractor struct {
// - "kval"
// - "json"
// - "xpath"
Type string `yaml:"type" jsonschema:"title=type of the extractor,description=Type of the extractor,enum=regex,enum=kval,enum=json,enum=xpath"`
Type TypeHolder `json:"name,omitempty" yaml:"type"`
// extractorType is the internal type of the extractor
extractorType ExtractorType
@ -113,30 +113,3 @@ type Extractor struct {
// - true
CaseInsensitive bool `yaml:"case-insensitive,omitempty" jsonschema:"title=use case insensitive extract,description=use case insensitive extract"`
}
// ExtractorType is the type of the extractor specified
type ExtractorType = int
const (
// RegexExtractor extracts responses with regexes
RegexExtractor ExtractorType = iota + 1
// KValExtractor extracts responses with key:value
KValExtractor
// XPathExtractor extracts responses with Xpath selectors
XPathExtractor
// JSONExtractor extracts responses with json
JSONExtractor
)
// ExtractorTypes is a table for conversion of extractor type from string.
var ExtractorTypes = map[string]ExtractorType{
"regex": RegexExtractor,
"kval": KValExtractor,
"xpath": XPathExtractor,
"json": JSONExtractor,
}
// GetType returns the type of the matcher
func (e *Extractor) GetType() ExtractorType {
return e.extractorType
}

6
v2/pkg/protocols/dns/operators_test.go
View File

@ -192,7 +192,7 @@ func TestDNSOperatorExtract(t *testing.T) {
t.Run("extract", func(t *testing.T) {
extractor := &extractors.Extractor{
Part: "raw",
Type: "regex",
Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor},
Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
}
err = extractor.CompileExtractors()
@ -205,7 +205,7 @@ func TestDNSOperatorExtract(t *testing.T) {
t.Run("kval", func(t *testing.T) {
extractor := &extractors.Extractor{
Type: "kval",
Type: extractors.TypeHolder{ExtractorType: extractors.KValExtractor},
KVal: []string{"rcode"},
}
err = extractor.CompileExtractors()
@ -238,7 +238,7 @@ func TestDNSMakeResult(t *testing.T) {
}},
Extractors: []*extractors.Extractor{{
Part: "raw",
Type: "regex",
Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor},
Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
}},
},

2
v2/pkg/protocols/dns/request_test.go
View File

@ -35,7 +35,7 @@ func TestDNSExecuteWithResults(t *testing.T) {
}},
Extractors: []*extractors.Extractor{{
Part: "raw",
Type: "regex",
Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor},
Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
}},
},

6
v2/pkg/protocols/file/operators_test.go
View File

@ -154,7 +154,7 @@ func TestFileOperatorExtract(t *testing.T) {
t.Run("extract", func(t *testing.T) {
extractor := &extractors.Extractor{
Part: "raw",
Type: "regex",
Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor},
Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
}
err = extractor.CompileExtractors()
@ -167,7 +167,7 @@ func TestFileOperatorExtract(t *testing.T) {
t.Run("kval", func(t *testing.T) {
extractor := &extractors.Extractor{
Type: "kval",
Type: extractors.TypeHolder{ExtractorType: extractors.KValExtractor},
KVal: []string{"raw"},
}
err = extractor.CompileExtractors()
@ -250,7 +250,7 @@ func testFileMakeResult(t *testing.T, matchers []*matchers.Matcher, matcherCondi
Matchers: matchers,
Extractors: []*extractors.Extractor{{
Part: "raw",
Type: "regex",
Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor},
Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
}},
},

2
v2/pkg/protocols/file/request_test.go
View File

@ -37,7 +37,7 @@ func TestFileExecuteWithResults(t *testing.T) {
}},
Extractors: []*extractors.Extractor{{
Part: "raw",
Type: "regex",
Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor},
Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
}},
},

108
v2/pkg/protocols/headless/engine/action.go
View File

@ -2,110 +2,6 @@ package engine
import "strings"
// ActionType defines the action type for a browser action
type ActionType int8
// Types to be executed by the user.
const (
// ActionNavigate performs a navigation to the specified URL
// URL can include nuclei payload data such as URL, Hostname, etc.
ActionNavigate ActionType = iota + 1
// ActionScript executes a JS snippet on the page.
ActionScript
// ActionClick performs the left-click action on an Element.
ActionClick
// ActionRightClick performs the right-click action on an Element.
ActionRightClick
// ActionTextInput performs an action for a text input
ActionTextInput
// ActionScreenshot performs the screenshot action writing to a file.
ActionScreenshot
// ActionTimeInput performs an action on a time input.
ActionTimeInput
// ActionSelectInput performs an action on a select input.
ActionSelectInput
// ActionFilesInput performs an action on a file input.
ActionFilesInput
// ActionWaitLoad waits for the page to stop loading.
ActionWaitLoad
// ActionGetResource performs a get resource action on an element
ActionGetResource
// ActionExtract performs an extraction on an element
ActionExtract
// ActionSetMethod sets the request method
ActionSetMethod
// ActionAddHeader adds a header to the request
ActionAddHeader
// ActionSetHeader sets a header in the request
ActionSetHeader
// ActionDeleteHeader deletes a header from the request
ActionDeleteHeader
// ActionSetBody sets the value of the request body
ActionSetBody
// ActionWaitEvent waits for a specific event.
ActionWaitEvent
// ActionKeyboard performs a keyboard action event on a page.
ActionKeyboard
// ActionDebug debug slows down headless and adds a sleep to each page.
ActionDebug
// ActionSleep executes a sleep for a specified duration
ActionSleep
// ActionWaitVisible waits until an element appears.
ActionWaitVisible
)
// ActionStringToAction converts an action from string to internal representation
var ActionStringToAction = map[string]ActionType{
"navigate": ActionNavigate,
"script": ActionScript,
"click": ActionClick,
"rightclick": ActionRightClick,
"text": ActionTextInput,
"screenshot": ActionScreenshot,
"time": ActionTimeInput,
"select": ActionSelectInput,
"files": ActionFilesInput,
"waitload": ActionWaitLoad,
"getresource": ActionGetResource,
"extract": ActionExtract,
"setmethod": ActionSetMethod,
"addheader": ActionAddHeader,
"setheader": ActionSetHeader,
"deleteheader": ActionDeleteHeader,
"setbody": ActionSetBody,
"waitevent": ActionWaitEvent,
"keyboard": ActionKeyboard,
"debug": ActionDebug,
"sleep": ActionSleep,
"waitvisible": ActionWaitVisible,
}
// ActionToActionString converts an action from internal representation to string
var ActionToActionString = map[ActionType]string{
ActionNavigate: "navigate",
ActionScript: "script",
ActionClick: "click",
ActionRightClick: "rightclick",
ActionTextInput: "text",
ActionScreenshot: "screenshot",
ActionTimeInput: "time",
ActionSelectInput: "select",
ActionFilesInput: "files",
ActionWaitLoad: "waitload",
ActionGetResource: "getresource",
ActionExtract: "extract",
ActionSetMethod: "set-method",
ActionAddHeader: "addheader",
ActionSetHeader: "setheader",
ActionDeleteHeader: "deleteheader",
ActionSetBody: "setbody",
ActionWaitEvent: "waitevent",
ActionKeyboard: "keyboard",
ActionDebug: "debug",
ActionSleep: "sleep",
ActionWaitVisible: "waitvisible",
}
// Action is an action taken by the browser to reach a navigation
//
// Each step that the browser executes is an action. Most navigations
@ -152,13 +48,13 @@ type Action struct {
// - "keyboard"
// - "debug"
// - "sleep"
ActionType string `yaml:"action" jsonschema:"title=action to perform,description=Type of actions to perform,enum=navigate,enum=script,enum=click,enum=rightclick,enum=text,enum=screenshot,enum=time,enum=select,enum=files,enum=waitload,enum=getresource,enum=extract,enum=setmethod,enum=addheader,enum=setheader,enum=deleteheader,enum=setbody,enum=waitevent,enum=keyboard,enum=debug,enum=sleep"`
ActionType ActionTypeHolder `yaml:"action" jsonschema:"title=action to perform,description=Type of actions to perform,enum=navigate,enum=script,enum=click,enum=rightclick,enum=text,enum=screenshot,enum=time,enum=select,enum=files,enum=waitload,enum=getresource,enum=extract,enum=setmethod,enum=addheader,enum=setheader,enum=deleteheader,enum=setbody,enum=waitevent,enum=keyboard,enum=debug,enum=sleep"`
}
// String returns the string representation of an action
func (a *Action) String() string {
builder := &strings.Builder{}
builder.WriteString(a.ActionType)
builder.WriteString(a.ActionType.String())
if a.Name != "" {
builder.WriteString(" Name:")
builder.WriteString(a.Name)

185
v2/pkg/protocols/headless/engine/action_types.go Normal file
View File

@ -0,0 +1,185 @@
package engine
import (
"encoding/json"
"errors"
"strings"
"github.com/alecthomas/jsonschema"
)
// ActionType defines the action type for a browser action
type ActionType int8
// Types to be executed by the user.
const (
// ActionNavigate performs a navigation to the specified URL
// URL can include nuclei payload data such as URL, Hostname, etc.
ActionNavigate ActionType = iota + 1
// ActionScript executes a JS snippet on the page.
ActionScript
// ActionClick performs the left-click action on an Element.
ActionClick
// ActionRightClick performs the right-click action on an Element.
ActionRightClick
// ActionTextInput performs an action for a text input
ActionTextInput
// ActionScreenshot performs the screenshot action writing to a file.
ActionScreenshot
// ActionTimeInput performs an action on a time input.
ActionTimeInput
// ActionSelectInput performs an action on a select input.
ActionSelectInput
// ActionFilesInput performs an action on a file input.
ActionFilesInput
// ActionWaitLoad waits for the page to stop loading.
ActionWaitLoad
// ActionGetResource performs a get resource action on an element
ActionGetResource
// ActionExtract performs an extraction on an element
ActionExtract
// ActionSetMethod sets the request method
ActionSetMethod
// ActionAddHeader adds a header to the request
ActionAddHeader
// ActionSetHeader sets a header in the request
ActionSetHeader
// ActionDeleteHeader deletes a header from the request
ActionDeleteHeader
// ActionSetBody sets the value of the request body
ActionSetBody
// ActionWaitEvent waits for a specific event.
ActionWaitEvent
// ActionKeyboard performs a keyboard action event on a page.
ActionKeyboard
// ActionDebug debug slows down headless and adds a sleep to each page.
ActionDebug
// ActionSleep executes a sleep for a specified duration
ActionSleep
// ActionWaitVisible waits until an element appears.
ActionWaitVisible
// limit
limit
)
// ActionStringToAction converts an action from string to internal representation
var ActionStringToAction = map[string]ActionType{
"navigate": ActionNavigate,
"script": ActionScript,
"click": ActionClick,
"rightclick": ActionRightClick,
"text": ActionTextInput,
"screenshot": ActionScreenshot,
"time": ActionTimeInput,
"select": ActionSelectInput,
"files": ActionFilesInput,
"waitload": ActionWaitLoad,
"getresource": ActionGetResource,
"extract": ActionExtract,
"setmethod": ActionSetMethod,
"addheader": ActionAddHeader,
"setheader": ActionSetHeader,
"deleteheader": ActionDeleteHeader,
"setbody": ActionSetBody,
"waitevent": ActionWaitEvent,
"keyboard": ActionKeyboard,
"debug": ActionDebug,
"sleep": ActionSleep,
"waitvisible": ActionWaitVisible,
}
// ActionToActionString converts an action from internal representation to string
var ActionToActionString = map[ActionType]string{
ActionNavigate: "navigate",
ActionScript: "script",
ActionClick: "click",
ActionRightClick: "rightclick",
ActionTextInput: "text",
ActionScreenshot: "screenshot",
ActionTimeInput: "time",
ActionSelectInput: "select",
ActionFilesInput: "files",
ActionWaitLoad: "waitload",
ActionGetResource: "getresource",
ActionExtract: "extract",
ActionSetMethod: "set-method",
ActionAddHeader: "addheader",
ActionSetHeader: "setheader",
ActionDeleteHeader: "deleteheader",
ActionSetBody: "setbody",
ActionWaitEvent: "waitevent",
ActionKeyboard: "keyboard",
ActionDebug: "debug",
ActionSleep: "sleep",
ActionWaitVisible: "waitvisible",
}
// GetSupportedActionTypes returns list of supported types
func GetSupportedActionTypes() []ActionType {
var result []ActionType
for index := ActionType(1); index < limit; index++ {
result = append(result, index)
}
return result
}
func toActionTypes(valueToMap string) (ActionType, error) {
normalizedValue := normalizeValue(valueToMap)
for key, currentValue := range ActionToActionString {
if normalizedValue == currentValue {
return key, nil
}
}
return -1, errors.New("Invalid action type: " + valueToMap)
}
func normalizeValue(value string) string {
return strings.TrimSpace(strings.ToLower(value))
}
func (t ActionType) String() string {
return ActionToActionString[t]
}
// ActionTypeHolder is used to hold internal type of the action
type ActionTypeHolder struct {
ActionType ActionType
}
func (holder ActionTypeHolder) String() string {
return holder.ActionType.String()
}
func (holder ActionTypeHolder) JSONSchemaType() *jsonschema.Type {
gotType := &jsonschema.Type{
Type: "string",
Title: "action to perform",
Description: "Type of actions to perform,enum=navigate,enum=script,enum=click,enum=rightclick,enum=text,enum=screenshot,enum=time,enum=select,enum=files,enum=waitload,enum=getresource,enum=extract,enum=setmethod,enum=addheader,enum=setheader,enum=deleteheader,enum=setbody,enum=waitevent,enum=keyboard,enum=debug,enum=sleep",
}
for _, types := range GetSupportedActionTypes() {
gotType.Enum = append(gotType.Enum, types.String())
}
return gotType
}
func (holder *ActionTypeHolder) UnmarshalYAML(unmarshal func(interface{}) error) error {
var marshalledTypes string
if err := unmarshal(&marshalledTypes); err != nil {
return err
}
computedType, err := toActionTypes(marshalledTypes)
if err != nil {
return err
}
holder.ActionType = computedType
return nil
}
func (holder *ActionTypeHolder) MarshalJSON() ([]byte, error) {
return json.Marshal(holder.ActionType.String())
}
func (holder ActionTypeHolder) MarshalYAML() (interface{}, error) {
return holder.ActionType.String(), nil
}

4
v2/pkg/protocols/headless/engine/page_actions.go
View File

@ -24,9 +24,7 @@ func (p *Page) ExecuteActions(baseURL *url.URL, actions []*Action) (map[string]s
outData := make(map[string]string)
for _, act := range actions {
actionType := ActionStringToAction[act.ActionType]
switch actionType {
switch act.ActionType.ActionType {
case ActionNavigate:
err = p.NavigateURL(act, outData, baseURL)
case ActionScript:

119
v2/pkg/protocols/headless/engine/page_actions_test.go
View File

@ -28,7 +28,7 @@ func TestActionNavigate(t *testing.T) {
</body>
</html>`
actions := []*Action{{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}}
actions := []*Action{{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}}}
testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
require.Nil(t, err, "could not run page actions")
@ -50,10 +50,11 @@ func TestActionScript(t *testing.T) {
t.Run("run-and-results", func(t *testing.T) {
actions := []*Action{
{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: "waitload"},
{ActionType: "script", Name: "test", Data: map[string]string{"code": "window.test"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionScript)}, Name: "test", Data: map[string]string{"code": "window.test"}},
}
testHeadlessSimpleResponse(t, response, actions, timeout, func(page *Page, err error, out map[string]string) {
require.Nil(t, err, "could not run page actions")
require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly")
@ -63,10 +64,10 @@ func TestActionScript(t *testing.T) {
t.Run("hook", func(t *testing.T) {
actions := []*Action{
{ActionType: "script", Data: map[string]string{"code": "window.test = 'some-data';", "hook": "true"}},
{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: "waitload"},
{ActionType: "script", Name: "test", Data: map[string]string{"code": "window.test"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionScript)}, Data: map[string]string{"code": "window.test = 'some-data';", "hook": "true"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionScript)}, Name: "test", Data: map[string]string{"code": "window.test"}},
}
testHeadlessSimpleResponse(t, response, actions, timeout, func(page *Page, err error, out map[string]string) {
require.Nil(t, err, "could not run page actions")
@ -87,9 +88,9 @@ func TestActionClick(t *testing.T) {
</html>`
actions := []*Action{
{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: "waitload"},
{ActionType: "click", Data: map[string]string{"selector": "button"}}, // Use css selector for clicking
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionClick)}, Data: map[string]string{"selector": "button"}}, // Use css selector for clicking
}
testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
@ -120,9 +121,9 @@ func TestActionRightClick(t *testing.T) {
</html>`
actions := []*Action{
{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: "waitload"},
{ActionType: "rightclick", Data: map[string]string{"selector": "button"}}, // Use css selector for clicking
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionRightClick)}, Data: map[string]string{"selector": "button"}}, // Use css selector for clicking
}
testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
@ -145,9 +146,9 @@ func TestActionTextInput(t *testing.T) {
</html>`
actions := []*Action{
{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: "waitload"},
{ActionType: "text", Data: map[string]string{"selector": "input", "value": "test"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionTextInput)}, Data: map[string]string{"selector": "input", "value": "test"}},
}
testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
@ -162,9 +163,9 @@ func TestActionTextInput(t *testing.T) {
func TestActionHeadersChange(t *testing.T) {
actions := []*Action{
{ActionType: "setheader", Data: map[string]string{"part": "request", "key": "Test", "value": "Hello"}},
{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: "waitload"},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionSetHeader)}, Data: map[string]string{"part": "request", "key": "Test", "value": "Hello"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}},
}
handler := func(w http.ResponseWriter, r *http.Request) {
@ -189,9 +190,9 @@ func TestActionScreenshot(t *testing.T) {
</html>`
actions := []*Action{
{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: "waitload"},
{ActionType: "screenshot", Data: map[string]string{"to": "test"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionScreenshot)}, Data: map[string]string{"to": "test"}},
}
testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
@ -214,9 +215,9 @@ func TestActionTimeInput(t *testing.T) {
</html>`
actions := []*Action{
{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: "waitload"},
{ActionType: "time", Data: map[string]string{"selector": "input", "value": "2006-01-02T15:04:05Z"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionTimeInput)}, Data: map[string]string{"selector": "input", "value": "2006-01-02T15:04:05Z"}},
}
testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
@ -242,9 +243,9 @@ func TestActionSelectInput(t *testing.T) {
</html>`
actions := []*Action{
{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: "waitload"},
{ActionType: "select", Data: map[string]string{"by": "x", "xpath": "//select[@id='test']", "value": "Test2", "selected": "true"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionSelectInput)}, Data: map[string]string{"by": "x", "xpath": "//select[@id='test']", "value": "Test2", "selected": "true"}},
}
testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
@ -265,9 +266,9 @@ func TestActionFilesInput(t *testing.T) {
</html>`
actions := []*Action{
{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: "waitload"},
{ActionType: "files", Data: map[string]string{"selector": "input", "value": "test1.pdf"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionFilesInput)}, Data: map[string]string{"selector": "input", "value": "test1.pdf"}},
}
testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
@ -291,8 +292,8 @@ func TestActionWaitLoad(t *testing.T) {
</html>`
actions := []*Action{
{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: "waitload"},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}},
}
testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
@ -316,8 +317,8 @@ func TestActionGetResource(t *testing.T) {
</html>`
actions := []*Action{
{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: "getresource", Data: map[string]string{"by": "x", "xpath": "//img[@id='test']"}, Name: "src"},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionGetResource)}, Data: map[string]string{"by": "x", "xpath": "//img[@id='test']"}, Name: "src"},
}
testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
@ -336,8 +337,8 @@ func TestActionExtract(t *testing.T) {
</html>`
actions := []*Action{
{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: "extract", Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}, Name: "extract"},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionExtract)}, Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}, Name: "extract"},
}
testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
@ -355,8 +356,8 @@ func TestActionSetMethod(t *testing.T) {
</html>`
actions := []*Action{
{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: "setmethod", Data: map[string]string{"part": "x", "method": "SET"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionSetMethod)}, Data: map[string]string{"part": "x", "method": "SET"}},
}
testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
@ -367,9 +368,9 @@ func TestActionSetMethod(t *testing.T) {
func TestActionAddHeader(t *testing.T) {
actions := []*Action{
{ActionType: "addheader", Data: map[string]string{"part": "request", "key": "Test", "value": "Hello"}},
{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: "waitload"},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionAddHeader)}, Data: map[string]string{"part": "request", "key": "Test", "value": "Hello"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}},
}
handler := func(w http.ResponseWriter, r *http.Request) {
@ -386,11 +387,11 @@ func TestActionAddHeader(t *testing.T) {
func TestActionDeleteHeader(t *testing.T) {
actions := []*Action{
{ActionType: "addheader", Data: map[string]string{"part": "request", "key": "Test1", "value": "Hello"}},
{ActionType: "addheader", Data: map[string]string{"part": "request", "key": "Test2", "value": "World"}},
{ActionType: "deleteheader", Data: map[string]string{"part": "request", "key": "Test2"}},
{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: "waitload"},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionAddHeader)}, Data: map[string]string{"part": "request", "key": "Test1", "value": "Hello"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionAddHeader)}, Data: map[string]string{"part": "request", "key": "Test2", "value": "World"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionDeleteHeader)}, Data: map[string]string{"part": "request", "key": "Test2"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}},
}
handler := func(w http.ResponseWriter, r *http.Request) {
@ -407,9 +408,9 @@ func TestActionDeleteHeader(t *testing.T) {
func TestActionSetBody(t *testing.T) {
actions := []*Action{
{ActionType: "setbody", Data: map[string]string{"part": "request", "body": "hello"}},
{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: "waitload"},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionSetBody)}, Data: map[string]string{"part": "request", "body": "hello"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}},
}
handler := func(w http.ResponseWriter, r *http.Request) {
@ -435,10 +436,10 @@ func TestActionKeyboard(t *testing.T) {
</html>`
actions := []*Action{
{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: "waitload"},
{ActionType: "click", Data: map[string]string{"selector": "input"}},
{ActionType: "keyboard", Data: map[string]string{"keys": "Test2"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitLoad)}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionClick)}, Data: map[string]string{"selector": "input"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionKeyboard)}, Data: map[string]string{"keys": "Test2"}},
}
testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
@ -461,8 +462,8 @@ func TestActionSleep(t *testing.T) {
</html>`
actions := []*Action{
{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: "sleep", Data: map[string]string{"duration": "2"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionSleep)}, Data: map[string]string{"duration": "2"}},
}
testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) {
@ -484,8 +485,8 @@ func TestActionWaitVisible(t *testing.T) {
</html>`
actions := []*Action{
{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: "waitvisible", Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionNavigate)}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionType(ActionWaitVisible)}, Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}},
}
t.Run("wait for an element being visible", func(t *testing.T) {

14
v2/pkg/protocols/http/operators_test.go
View File

@ -166,7 +166,7 @@ func TestHTTPOperatorExtract(t *testing.T) {
t.Run("extract", func(t *testing.T) {
extractor := &extractors.Extractor{
Part: "body",
Type: "regex",
Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor},
Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
}
err = extractor.CompileExtractors()
@ -179,7 +179,7 @@ func TestHTTPOperatorExtract(t *testing.T) {
t.Run("kval", func(t *testing.T) {
extractor := &extractors.Extractor{
Type: "kval",
Type: extractors.TypeHolder{ExtractorType: extractors.KValExtractor},
KVal: []string{"test_header"},
}
err = extractor.CompileExtractors()
@ -195,7 +195,7 @@ func TestHTTPOperatorExtract(t *testing.T) {
t.Run("jq-simple", func(t *testing.T) {
extractor := &extractors.Extractor{
Type: "json",
Type: extractors.TypeHolder{ExtractorType: extractors.JSONExtractor},
JSON: []string{".batters | .batter | .[] | .id"},
}
err = extractor.CompileExtractors()
@ -207,7 +207,7 @@ func TestHTTPOperatorExtract(t *testing.T) {
})
t.Run("jq-array", func(t *testing.T) {
extractor := &extractors.Extractor{
Type: "json",
Type: extractors.TypeHolder{ExtractorType: extractors.JSONExtractor},
JSON: []string{".array"},
}
err = extractor.CompileExtractors()
@ -219,7 +219,7 @@ func TestHTTPOperatorExtract(t *testing.T) {
})
t.Run("jq-object", func(t *testing.T) {
extractor := &extractors.Extractor{
Type: "json",
Type: extractors.TypeHolder{ExtractorType: extractors.JSONExtractor},
JSON: []string{".batters"},
}
err = extractor.CompileExtractors()
@ -235,7 +235,7 @@ func TestHTTPOperatorExtract(t *testing.T) {
event["body"] = exampleResponseBody
extractor := &extractors.Extractor{
Type: "kval",
Type: extractors.TypeHolder{ExtractorType: extractors.KValExtractor},
KVal: []string{"TEST_HEADER"}, // only applies to KVal
CaseInsensitive: true,
}
@ -267,7 +267,7 @@ func TestHTTPMakeResult(t *testing.T) {
}},
Extractors: []*extractors.Extractor{{
Part: "body",
Type: "regex",
Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor},
Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
}},
},

6
v2/pkg/protocols/network/operators_test.go
View File

@ -149,7 +149,7 @@ func TestNetworkOperatorExtract(t *testing.T) {
t.Run("extract", func(t *testing.T) {
extractor := &extractors.Extractor{
Part: "data",
Type: "regex",
Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor},
Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
}
err = extractor.CompileExtractors()
@ -162,7 +162,7 @@ func TestNetworkOperatorExtract(t *testing.T) {
t.Run("kval", func(t *testing.T) {
extractor := &extractors.Extractor{
Type: "kval",
Type: extractors.TypeHolder{ExtractorType: extractors.KValExtractor},
KVal: []string{"request"},
}
err = extractor.CompileExtractors()
@ -193,7 +193,7 @@ func TestNetworkMakeResult(t *testing.T) {
}},
Extractors: []*extractors.Extractor{{
Part: "data",
Type: "regex",
Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor},
Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
}},
},

2
v2/pkg/protocols/network/request_test.go
View File

@ -38,7 +38,7 @@ func TestNetworkExecuteWithResults(t *testing.T) {
}},
Extractors: []*extractors.Extractor{{
Part: "data",
Type: "regex",
Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor},
Regex: []string{"<h1>.*</h1>"},
}},
},

6
v2/pkg/protocols/offlinehttp/operators_test.go
View File

@ -139,7 +139,7 @@ func TestHTTPOperatorExtract(t *testing.T) {
t.Run("extract", func(t *testing.T) {
extractor := &extractors.Extractor{
Part: "body",
Type: "regex",
Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor},
Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
}
err = extractor.CompileExtractors()
@ -152,7 +152,7 @@ func TestHTTPOperatorExtract(t *testing.T) {
t.Run("kval", func(t *testing.T) {
extractor := &extractors.Extractor{
Type: "kval",
Type: extractors.TypeHolder{ExtractorType: extractors.KValExtractor},
KVal: []string{"test-header"},
Part: "header",
}
@ -184,7 +184,7 @@ func TestHTTPMakeResult(t *testing.T) {
}},
Extractors: []*extractors.Extractor{{
Part: "body",
Type: "regex",
Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor},
Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
}},
}}

4
v2/pkg/templates/templates_doc_examples.go
View File

@ -44,7 +44,7 @@ var (
Recursion: true,
Operators: operators.Operators{
Extractors: []*extractors.Extractor{
{Type: "regex", Regex: []string{"ec2-[-\\d]+\\.compute[-\\d]*\\.amazonaws\\.com", "ec2-[-\\d]+\\.[\\w\\d\\-]+\\.compute[-\\d]*\\.amazonaws\\.com"}},
{Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"ec2-[-\\d]+\\.compute[-\\d]*\\.amazonaws\\.com", "ec2-[-\\d]+\\.[\\w\\d\\-]+\\.compute[-\\d]*\\.amazonaws\\.com"}},
},
},
}
@ -54,7 +54,7 @@ var (
Extensions: []string{"all"},
Operators: operators.Operators{
Extractors: []*extractors.Extractor{
{Type: "regex", Regex: []string{"amzn\\.mws\\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"}},
{Type: extractors.TypeHolder{ExtractorType: extractors.RegexExtractor}, Regex: []string{"amzn\\.mws\\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"}},
},
},
}