id: robots-txt-endpoint

info:
  name: robots.txt endpoint prober
  author: CasperGN,pdteam
  severity: info

requests:
  - method: GET
    path:
      - "{{RootURL}}/robots.txt"
      - "{{RootURL}}{{endpoint}}"

    extractors:
      - type: regex
        name: endpoint
        group: 1
        regex:
          - '(?m:\s(/[[:alpha:]]+[[:graph:]]+))'
        internal: true

    iterate-all: true
    matchers:
      - type: status
        status:
          - 200