id: robots-txt-endpoint

info:
  name: robots.txt endpoint prober
  author: CasperGN,pdteam
  severity: info
  tags: misc,generic
  metadata:
    max-request: 2

http:
  - method: GET
    path:
      - "{{RootURL}}/robots.txt"
      - "{{RootURL}}{{endpoint}}"

    extractors:
      - type: regex
        name: endpoint
        group: 1
        regex:
          - '(?m:\s(/[[:alpha:]]+[[:graph:]]+))'
        internal: true

    iterate-all: true
    matchers-condition: and
    matchers:
      - type: word
        part: body
        words:
          - 'User-agent:'
          - 'Disallow:'
          - 'Allow:'

      - type: word
        part: header
        words:
          - "text/plain"

      - type: status
        status:
          - 200