id: robots-txt-endpoint

info:
  name: robots.txt endpoint prober
  author: CasperGN,pdteam
  severity: info
  metadata:
    max-request: 2
  tags: miscellaneous,misc,generic

http:
  - method: GET
    path:
      - "{{RootURL}}/robots.txt"
      - "{{RootURL}}{{endpoint}}"

    extractors:
      - type: regex
        name: endpoint
        group: 1
        regex:
          - '(?m:\s(/[[:alpha:]]+[[:graph:]]+))'
        internal: true
    iterate-all: true

    matchers-condition: and
    matchers:
      - type: word
        part: body
        words:
          - 'User-agent:'
          - 'Disallow:'
          - 'Allow:'

      - type: word
        part: header
        words:
          - "text/plain"

      - type: status
        status:
          - 200
# digest: 4a0a004730450220759dd8e01643b5e835327cbcbf62aba25a4d413d9c6fd4702c4b3ab35934b530022100e7932bb06675ad9e55c4020022620ae51d6b5f8b3a8fedb00ea6e0b9ac3fe86d:922c64590222798bb761d5b6d8e72950