Adding a Nokogiri stream parser for Nexpose raw XML files.

git-svn-id: file:///home/svn/framework3/trunk@12740 4d416f70-5f16-0410-b530-b9f4589650da
unstable
Tod Beardsley 2011-05-27 17:30:11 +00:00
parent 33135af296
commit a891d53be4
5 changed files with 431 additions and 20 deletions

View File

@ -2996,9 +2996,6 @@ class DBManager
import_nexpose_noko_stream(noko_args)
end
return true
else
# parser = ""
# yield(:parser, parser)
end
data = args[:data]
@ -3117,9 +3114,22 @@ class DBManager
end
def import_nexpose_rawxml(args={}, &block)
data = args[:data]
wspace = args[:wspace] || workspace
bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : []
wspace = args[:wspace] || workspace
if Rex::Parser.nokogiri_loaded
parser = "Nokogiri v#{::Nokogiri::VERSION}"
noko_args = args.dup
noko_args[:blacklist] = bl
noko_args[:wspace] = wspace
if block
yield(:parser, parser)
import_nexpose_raw_noko_stream(noko_args) {|type, data| yield type,data}
else
import_nexpose_raw_noko_stream(noko_args)
end
return true
end
data = args[:data]
# Use a stream parser instead of a tree parser so we can deal with
# huge results files without running out of memory.
@ -3793,6 +3803,16 @@ class DBManager
import_nmap_xml(args.merge(:data => data))
end
def import_nexpose_raw_noko_stream(args, &block)
if block
doc = Rex::Parser::NexposeRawDocument.new(args,framework.db) {|type, data| yield type,data }
else
doc = Rex::Parser::NexposeRawDocument.new(args,self)
end
parser = ::Nokogiri::XML::SAX::Parser.new(doc)
parser.parse(args[:data])
end
def import_nexpose_noko_stream(args, &block)
if block
doc = Rex::Parser::NexposeSimpleDocument.new(args,framework.db) {|type, data| yield type,data }

View File

@ -1264,6 +1264,9 @@ class Db
case type
when :debug
print_error("DEBUG: #{data.inspect}")
when :vuln
inst = data[1] == 1 ? "instance" : "instances"
print_status("Importing vulnerability '#{data[0]}' (#{data[1]} #{inst})")
when :filetype
print_status("Importing '#{data}' data")
when :parser

View File

@ -0,0 +1,363 @@
require File.join(File.expand_path(File.dirname(__FILE__)),"nokogiri_doc_mixin")
module Rex
module Parser
# If Nokogiri is available, define Template document class.
load_nokogiri && class NexposeRawDocument < Nokogiri::XML::SAX::Document
include NokogiriDocMixin
attr_reader :tests
# Triggered every time a new element is encountered. We keep state
# ourselves with the @state variable, turning things on when we
# get here (and turning things off when we exit in end_element()).
def start_element(name=nil,attrs=[])
attrs = normalize_attrs(attrs)
block = @block
@state[:current_tag][name] = true
case name
when "nodes" # There are two main sections, nodes and VulnerabilityDefinitions
@tests = []
when "node"
record_host(attrs)
when "name"
@state[:has_text] = true
when "endpoint"
record_service(attrs)
when "service"
record_service_info(attrs)
when "fingerprint"
record_service_fingerprint(attrs)
when "os"
record_os_fingerprint(attrs)
when "test" # All the vulns tested for
record_host_test(attrs)
record_service_test(attrs)
when "vulnerability"
record_vuln(attrs)
when "reference"
@state[:has_text] = true
record_reference(attrs)
end
end
# This breaks on xml-encoded characters, so need to append
def characters(text)
return unless @state[:has_text]
@text ||= ""
@text << text
end
# When we exit a tag, this is triggered.
def end_element(name=nil)
block = @block
case name
when "node" # Wrap it up
collect_host_data
host_object = report_host &block
report_services(host_object)
report_fingerprint(host_object)
# Reset the state once we close a host
@state.delete_if {|k| k.to_s !~ /^(current_tag|in_nodes)$/}
@report_data = {:wspace => @args[:wspace]}
when "name"
collect_hostname
@state[:has_text] = false
when "endpoint"
collect_service_data
when "os"
collect_os_fingerprints
when "test"
save_test
when "vulnerability"
collect_vuln_info
report_vuln(&block)
@state.delete_if {|k| k.to_s !~ /^(current_tag|in_vulndefs)$/}
when "reference"
@state[:has_text] = false
collect_reference
@text = nil
end
@state[:current_tag].delete name
end
def collect_reference
return unless in_tag("references")
return unless in_tag("vulnerability")
@state[:ref][:value] = @text.to_s.strip
@report_data[:refs] ||= []
@report_data[:refs] << @state[:ref]
@state[:ref] = nil
end
def collect_vuln_info
return unless in_tag("VulnerabilityDefinitions")
return unless in_tag("vulnerability")
return unless @state[:vuln]
vuln = @state[:vuln]
vuln[:refs] = @report_data[:refs]
@report_data[:vuln] = vuln
@state[:vuln] = nil
@report_data[:refs] = nil
end
def report_vuln(&block)
return unless in_tag("VulnerabilityDefinitions")
return unless @report_data[:vuln]
return unless @report_data[:vuln][:matches].kind_of? Array
refs = normalize_references(@report_data[:vuln][:refs])
refs << "NEXPOSE-#{report_data[:vuln]["id"]}"
db.emit(:vuln, [refs.last,@report_data[:vuln][:matches].size], &block)
data = {
:workspace => @args[:wspace],
:name => refs.last,
:info => @report_data[:vuln]["title"],
:refs => refs.uniq
}
hosts_keys = {}
@report_data[:vuln][:matches].each do |match|
host_data = data.dup
host_data[:host] = match[:host]
host_data[:port] = match[:port] if match[:port]
host_data[:proto] = match[:protocol] if match[:protocol]
db.report_vuln(host_data)
if match[:key]
hosts_keys[host_data[:host]] ||= []
hosts_keys[host_data[:host]] << match[:key]
end
end
report_key_note(hosts_keys,data)
@report_data[:vuln] = nil
end
def report_key_note(hosts_keys,data)
return if hosts_keys.empty?
hosts_keys.each do |key_host,key_values|
key_note = {
:workspace => @args[:wspace],
:host => key_host,
:type => "host.vuln.nexpose_keys",
:data => {},
:update => :unique_data
}
key_values.each do |key_value|
key_note[:data][data[:name]] ||= []
next if key_note[:data][data[:name]].include? key_value
key_note[:data][data[:name]] << key_value
end
db.report_note(key_note)
end
end
def record_reference(attrs)
return unless in_tag("VulnerabilityDefinitions")
return unless in_tag("vulnerability")
@state[:ref] = attr_hash(attrs)
end
def record_vuln(attrs)
return unless in_tag("VulnerabilityDefinitions")
vuln = attr_hash(attrs)
matching_tests = @tests.select {|x| x[:id] == vuln["id"].downcase}
return if matching_tests.empty?
@state[:vuln] = vuln
@state[:vuln][:matches] = matching_tests
end
def save_test
return unless in_tag("nodes")
return unless in_tag("node")
return unless @state[:test]
test = { :id => @state[:test][:id]}
test[:host] = @state[:address]
test[:port] = @state[:test][:port] if @state[:test][:port]
test[:protocol] = @state[:test][:protocol] if @state[:test][:protocol]
test[:key] = @state[:test][:key] if @state[:test][:key]
@tests << test
@state[:test] = nil
end
def record_os_fingerprint(attrs)
return unless in_tag("nodes")
return unless in_tag("fingerprints")
return unless in_tag("node")
return if in_tag("service")
@state[:os] = attr_hash(attrs)
end
# Just keep the highest scoring, which is usually the most vague. :(
def collect_os_fingerprints
@report_data[:os] ||= {}
return unless @state[:os]["certainty"].to_f > 0
return if @report_data[:os]["os_certainty"].to_f > @state[:os]["certainty"].to_f
@report_data[:os] = {} # Zero it out if we're replacing it.
@report_data[:os]["os_certainty"] = @state[:os]["certainty"]
@report_data[:os]["os_vendor"] = @state[:os]["vendor"]
@report_data[:os]["os_family"] = @state[:os]["family"]
@report_data[:os]["os_product"] = @state[:os]["product"]
@report_data[:os]["os_version"] = @state[:os]["version"]
@report_data[:os]["os_arch"] = @state[:os]["arch"]
end
# Just taking the first one.
def collect_hostname
if in_tag("node")
@state[:hostname] ||= @text.to_s.strip if @text
@text = nil
end
end
def record_service_fingerprint(attrs)
return unless in_tag("nodes")
return unless in_tag("node")
return unless in_tag("service")
return unless in_tag("fingerprint")
@state[:service_fingerprint] = attr_hash(attrs)
end
def record_service_info(attrs)
return unless in_tag("nodes")
return unless in_tag("node")
return unless in_tag("service")
@state[:service].merge! attr_hash(attrs)
end
def report_fingerprint(host_object)
return unless host_object.kind_of? ::Msf::DBManager::Host
return unless @report_data[:os].kind_of? Hash
note = {
:workspace => host_object.workspace,
:host => host_object,
:type => "host.os.nexpose_fingerprint",
:data => {
:family => @report_data[:os]["os_family"],
:certainty => @report_data[:os]["os_certainty"]
}
}
note[:data][:vendor] = @report_data[:os]["os_vendor"] if @report_data[:os]["os_vendor"]
note[:data][:product] = @report_data[:os]["os_product"] if @report_data[:os]["os_prduct"]
note[:data][:version] = @report_data[:os]["os_version"] if @report_data[:os]["os_version"]
note[:data][:arch] = @report_data[:os]["os_arch"] if @report_data[:os]["os_arch"]
db.report_note(note)
end
def report_services(host_object)
return unless host_object.kind_of? ::Msf::DBManager::Host
return unless @report_data[:ports]
return if @report_data[:ports].empty?
reported = []
@report_data[:ports].each do |svc|
reported << db.report_service(svc.merge(:host => host_object))
end
reported
end
def record_service(attrs)
return unless in_tag("nodes")
return unless in_tag("node")
return unless in_tag("endpoint")
@state[:service] = attr_hash(attrs)
end
def collect_service_data
return unless in_tag("node")
return unless in_tag("endpoint")
port_hash = {}
@report_data[:ports] ||= []
@state[:service].each do |k,v|
case k
when "protocol"
port_hash[:protocol] = v
when "port"
port_hash[:port] = v
when "status"
port_hash[:status] = (v == "open" ? Msf::ServiceState::Open : Msf::ServiceState::Closed)
end
end
if @state[:service]
port_hash[:name] = @state[:service]["name"] if @state[:service]["name"] != "<unknown>"
end
if @state[:service_fingerprint]
info = []
info << @state[:service_fingerprint]["product"] if @state[:service_fingerprint]["product"]
info << @state[:service_fingerprint]["version"] if @state[:service_fingerprint]["version"]
port_hash[:info] = info.join(" ") if info[0]
end
@report_data[:ports] << port_hash
end
def actually_vulnerable(test)
return false unless test.has_key? "status"
return false unless test.has_key? "id"
['vulnerable-exploited', 'vulnerable-version', 'potential'].include? test["status"]
end
def record_host_test(attrs)
return unless in_tag("nodes")
return unless in_tag("node")
return if in_tag("service")
return unless in_tag("tests")
test = attr_hash(attrs)
return unless actually_vulnerable(test)
@state[:test] = {:id => test["id"].downcase}
@state[:test][:key] = test["key"] if test["key"]
end
def record_service_test(attrs)
return unless in_tag("nodes")
return unless in_tag("node")
return unless in_tag("service")
return unless in_tag("tests")
test = attr_hash(attrs)
return unless actually_vulnerable(test)
@state[:test] = {
:id => test["id"].downcase,
:port => @state[:service]["port"],
:protocol => @state[:service]["protocol"],
}
@state[:test][:key] = test["key"] if test["key"]
end
def record_host(attrs)
return unless in_tag("nodes")
host_attrs = attr_hash(attrs)
if host_attrs["status"] == "alive"
@state[:host_is_alive] = true
@state[:address] = host_attrs["address"]
@state[:mac] = host_attrs["hardware-address"] if host_attrs["hardware-address"]
end
end
def collect_host_data
return unless in_tag("node")
@report_data[:host] = @state[:address]
@report_data[:state] = Msf::HostState::Alive
@report_data[:name] = @state[:hostname] if @state[:hostname]
if @state[:mac]
if @state[:mac] =~ /[0-9a-fA-f]{12}/
@report_data[:mac] = @state[:mac].scan(/.{2}/).join(":")
else
@report_data[:mac] = @state[:mac]
end
end
end
def report_host(&block)
if host_is_okay
db.emit(:address,@report_data[:host],&block) if block
host_object = db.report_host( @report_data.merge(
:workspace => @args[:wspace] ) )
if host_object
db.report_import_note(host_object.workspace, host_object)
end
host_object
end
end
end
end
end

View File

@ -205,21 +205,6 @@ module Rex
vuln_result =~ /^V[VE]$/
end
def normalize_ref(ref_type, ref_value)
return if ref_type.nil? || ref_type.empty? || ref_value.nil? || ref_value.empty?
ref_value = ref_value.strip
ref_type = ref_type.strip.upcase
ret = case ref_type
when "CVE"
ref_value.gsub("CAN", "CVE")
when "MS"
"MSB-MS-#{ref_value}"
else
"#{ref_type}-#{ref_value}"
end
return ret
end
def record_device(attrs)
@state[:in_device] = true
attrs.each do |k,v|

View File

@ -24,6 +24,13 @@ module Rex
!!@nokogiri_loaded
end
# Useful during development, shouldn't be used in normal operation.
def self.reload(fname)
$stdout.puts "Reloading #{fname}..."
load __FILE__
load File.join(File.expand_path(File.dirname(__FILE__)),fname)
end
end
end
@ -59,6 +66,39 @@ module Parser
!!valid
end
def normalize_ref(ref_type, ref_value)
return if ref_type.nil? || ref_type.empty? || ref_value.nil? || ref_value.empty?
ref_value = ref_value.strip
ref_type = ref_type.strip.upcase
ret = case ref_type
when "CVE"
ref_value.gsub("CAN", "CVE")
when "MS"
"MSB-MS-#{ref_value}"
when "URL", "BID"
"#{ref_type}-#{ref_value}"
else # Handle others?
"#{ref_type}-#{ref_value}"
end
return ret
end
def normalize_references(orig_refs)
return [] unless orig_refs
refs = []
orig_refs.each do |ref_hash|
ref_hash_sym = Hash[ref_hash.map {|k, v| [k.to_sym, v] }]
ref_type = ref_hash_sym[:source].to_s.strip.upcase
ref_value = ref_hash_sym[:value].to_s.strip
refs << normalize_ref(ref_type, ref_value)
end
return refs.compact.uniq
end
def in_tag(tagname)
@state[:current_tag].keys.include? tagname
end
# If there's an address, it's not on the blacklist,
# it has ports, and the port list isn't
# empty... it's okay.