2022-05-20 09:43:00 +00:00
|
|
|
id: robots-txt-endpoint
|
|
|
|
|
|
|
|
info:
|
|
|
|
name: robots.txt endpoint prober
|
|
|
|
author: CasperGN,pdteam
|
|
|
|
severity: info
|
2023-04-28 08:11:21 +00:00
|
|
|
metadata:
|
|
|
|
max-request: 2
|
2023-10-14 11:27:55 +00:00
|
|
|
tags: misc,generic
|
2022-05-20 09:43:00 +00:00
|
|
|
|
2023-04-27 04:28:59 +00:00
|
|
|
http:
|
2022-05-20 09:43:00 +00:00
|
|
|
- method: GET
|
|
|
|
path:
|
|
|
|
- "{{RootURL}}/robots.txt"
|
|
|
|
- "{{RootURL}}{{endpoint}}"
|
|
|
|
|
|
|
|
extractors:
|
|
|
|
- type: regex
|
|
|
|
name: endpoint
|
|
|
|
group: 1
|
|
|
|
regex:
|
|
|
|
- '(?m:\s(/[[:alpha:]]+[[:graph:]]+))'
|
|
|
|
internal: true
|
|
|
|
iterate-all: true
|
2023-10-14 11:27:55 +00:00
|
|
|
|
2023-06-15 07:29:10 +00:00
|
|
|
matchers-condition: and
|
2022-05-20 09:43:00 +00:00
|
|
|
matchers:
|
2023-06-15 07:29:10 +00:00
|
|
|
- type: word
|
|
|
|
part: body
|
|
|
|
words:
|
|
|
|
- 'User-agent:'
|
|
|
|
- 'Disallow:'
|
|
|
|
- 'Allow:'
|
|
|
|
|
|
|
|
- type: word
|
|
|
|
part: header
|
|
|
|
words:
|
|
|
|
- "text/plain"
|
|
|
|
|
2022-05-20 09:43:00 +00:00
|
|
|
- type: status
|
|
|
|
status:
|
2023-05-22 19:10:19 +00:00
|
|
|
- 200
|
2023-10-20 11:41:13 +00:00
|
|
|
|
|
|
|
# digest: 490a0046304402207a390f453c435c533b20adfa0b547d200d75c16ea6fec67753cfd1bcf4c798c2022005a82d5a928b8f08ba54644f69befb25a9f69d8b919539c67136046664a4c76d:922c64590222798bb761d5b6d8e72950
|