infosecn1nja
/
metasploit-framework
mirror of https://github.com/infosecn1nja/metasploit-framework.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Converts XML importer to use Nokogiri Reader 

MS-255
bug/bundler_fix
dmohanty-r7 2016-04-16 23:16:26 -05:00
parent 93ce0fe912
commit 802dfabbe3
1 changed files with 282 additions and 264 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

526
lib/msf/core/db_manager/import/metasploit_framework/xml.rb
View File

@ -71,18 +71,18 @@ module Msf::DBManager::Import::MetasploitFramework::XML
# @param note_data [Hash] hash containing note attributes to be passed along
# @return [void]
def import_msf_note_element(note, allow_yaml, note_data={})
note_data[:type] = nils_for_nulls(note.elements["ntype"].text.to_s.strip)
note_data[:data] = nils_for_nulls(unserialize_object(note.elements["data"], allow_yaml))
note_data[:type] = nils_for_nulls(note.at("ntype").text.to_s.strip)
note_data[:data] = nils_for_nulls(unserialize_object(note.at("data"), allow_yaml))
if note.elements["critical"].text
note_data[:critical] = true unless note.elements["critical"].text.to_s.strip == "NULL"
if note.at("critical").text
note_data[:critical] = true unless note.at("critical").text.to_s.strip == "NULL"
end
if note.elements["seen"].text
note_data[:seen] = true unless note.elements["critical"].text.to_s.strip == "NULL"
if note.at("seen").text
note_data[:seen] = true unless note.at("critical").text.to_s.strip == "NULL"
end
%W{created-at updated-at}.each { |datum|
if note.elements[datum].text
note_data[datum.gsub("-","_")] = nils_for_nulls(note.elements[datum].text.to_s.strip)
if note.at(datum).text
note_data[datum.gsub("-","_")] = nils_for_nulls(note.at(datum).text.to_s.strip)
end
}
report_note(note_data)
@ -115,7 +115,7 @@ module Msf::DBManager::Import::MetasploitFramework::XML
# FIXME https://www.pivotaltracker.com/story/show/46578647
# FIXME https://www.pivotaltracker.com/story/show/47128407
unserialized_params = unserialize_object(
element.elements['params'],
element.at('params'),
options[:allow_yaml]
)
info[:params] = nils_for_nulls(unserialized_params)
@ -162,7 +162,7 @@ module Msf::DBManager::Import::MetasploitFramework::XML
# FIXME https://www.pivotaltracker.com/story/show/46578647
# FIXME https://www.pivotaltracker.com/story/show/47128407
unserialized_headers = unserialize_object(
element.elements['headers'],
element.at('headers'),
options[:allow_yaml]
)
info[:headers] = nils_for_nulls(unserialized_headers)
@ -209,7 +209,7 @@ module Msf::DBManager::Import::MetasploitFramework::XML
# FIXME https://www.pivotaltracker.com/story/show/46578647
# FIXME https://www.pivotaltracker.com/story/show/47128407
unserialized_params = unserialize_object(
element.elements['params'],
element.at('params'),
options[:allow_yaml]
)
info[:params] = nils_for_nulls(unserialized_params)
@ -228,246 +228,27 @@ module Msf::DBManager::Import::MetasploitFramework::XML
# them.
# TODO: loot, tasks, and reports
def import_msf_xml(args={}, &block)
data = args[:data]
wspace = args[:wspace] || workspace
bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : []
doc = rexmlify(data)
metadata = check_msf_xml_version!(doc)
#doc = rexmlify(data)
doc = Nokogiri::XML::Reader.from_memory(data)
metadata = check_msf_xml_version!(doc.first.name)
allow_yaml = metadata[:allow_yaml]
btag = metadata[:root_tag]
doc.elements.each("/#{btag}/hosts/host") do |host|
host_data = {}
host_data[:task] = args[:task]
host_data[:workspace] = wspace
# A regression resulted in the address field being serialized in some cases.
# Lets handle both instances to keep things happy. See #5837 & #5985
addr = nils_for_nulls(host.elements["address"])
next unless addr
# No period or colon means this must be in base64-encoded serialized form
if addr !~ /[\.\:]/
addr = unserialize_object(addr)
end
host_data[:host] = addr
if bl.include? host_data[:host]
next
else
yield(:address,host_data[:host]) if block
end
host_data[:mac] = nils_for_nulls(host.elements["mac"].text.to_s.strip)
if host.elements["comm"].text
host_data[:comm] = nils_for_nulls(host.elements["comm"].text.to_s.strip)
end
%W{created-at updated-at name state os-flavor os-lang os-name os-sp purpose}.each { |datum|
if host.elements[datum].text
host_data[datum.gsub('-','_')] = nils_for_nulls(host.elements[datum].text.to_s.strip)
end
}
host_address = host_data[:host].dup # Preserve after report_host() deletes
hobj = report_host(host_data)
host.elements.each("host_details/host_detail") do |hdet|
hdet_data = {}
hdet.elements.each do |det|
next if ["id", "host-id"].include?(det.name)
if det.text
hdet_data[det.name.gsub('-','_')] = nils_for_nulls(det.text.to_s.strip)
end
end
report_host_details(hobj, hdet_data)
end
host.elements.each("exploit_attempts/exploit_attempt") do |hdet|
hdet_data = {}
hdet.elements.each do |det|
next if ["id", "host-id", "session-id", "vuln-id", "service-id", "loot-id"].include?(det.name)
if det.text
hdet_data[det.name.gsub('-','_')] = nils_for_nulls(det.text.to_s.strip)
end
end
report_exploit_attempt(hobj, hdet_data)
end
host.elements.each('services/service') do |service|
service_data = {}
service_data[:task] = args[:task]
service_data[:workspace] = wspace
service_data[:host] = hobj
service_data[:port] = nils_for_nulls(service.elements["port"].text.to_s.strip).to_i
service_data[:proto] = nils_for_nulls(service.elements["proto"].text.to_s.strip)
%W{created-at updated-at name state info}.each { |datum|
if service.elements[datum].text
if datum == "info"
service_data["info"] = nils_for_nulls(unserialize_object(service.elements[datum], false))
else
service_data[datum.gsub("-","_")] = nils_for_nulls(service.elements[datum].text.to_s.strip)
end
end
}
report_service(service_data)
end
host.elements.each('notes/note') do |note|
note_data = {}
note_data[:workspace] = wspace
note_data[:host] = hobj
import_msf_note_element(note,allow_yaml,note_data)
end
host.elements.each('tags/tag') do |tag|
tag_data = {}
tag_data[:addr] = host_address
tag_data[:wspace] = wspace
tag_data[:name] = tag.elements["name"].text.to_s.strip
tag_data[:desc] = tag.elements["desc"].text.to_s.strip
if tag.elements["report-summary"].text
tag_data[:summary] = tag.elements["report-summary"].text.to_s.strip
end
if tag.elements["report-detail"].text
tag_data[:detail] = tag.elements["report-detail"].text.to_s.strip
end
if tag.elements["critical"].text
tag_data[:crit] = true unless tag.elements["critical"].text.to_s.strip == "NULL"
end
report_host_tag(tag_data)
end
host.elements.each('vulns/vuln') do |vuln|
vuln_data = {}
vuln_data[:workspace] = wspace
vuln_data[:host] = hobj
vuln_data[:info] = nils_for_nulls(unserialize_object(vuln.elements["info"], allow_yaml))
vuln_data[:name] = nils_for_nulls(vuln.elements["name"].text.to_s.strip)
%W{created-at updated-at exploited-at}.each { |datum|
if vuln.elements[datum] and vuln.elements[datum].text
vuln_data[datum.gsub("-","_")] = nils_for_nulls(vuln.elements[datum].text.to_s.strip)
end
}
if vuln.elements["refs"]
vuln_data[:refs] = []
vuln.elements.each("refs/ref") do |ref|
vuln_data[:refs] << nils_for_nulls(ref.text.to_s.strip)
end
end
vobj = report_vuln(vuln_data)
vuln.elements.each("notes/note") do |note|
note_data = {}
note_data[:workspace] = wspace
note_data[:vuln_id] = vobj.id
import_msf_note_element(note,allow_yaml,note_data)
end
vuln.elements.each("vuln_details/vuln_detail") do |vdet|
vdet_data = {}
vdet.elements.each do |det|
next if ["id", "vuln-id"].include?(det.name)
if det.text
vdet_data[det.name.gsub('-','_')] = nils_for_nulls(det.text.to_s.strip)
end
end
report_vuln_details(vobj, vdet_data)
end
vuln.elements.each("vuln_attempts/vuln_attempt") do |vdet|
vdet_data = {}
vdet.elements.each do |det|
next if ["id", "vuln-id", "loot-id", "session-id"].include?(det.name)
if det.text
vdet_data[det.name.gsub('-','_')] = nils_for_nulls(det.text.to_s.strip)
end
end
report_vuln_attempt(vobj, vdet_data)
end
end
## Handle old-style (pre 4.10) XML files
if btag == "MetasploitV4"
if host.elements['creds'].present?
unless host.elements['creds'].elements.empty?
origin = Metasploit::Credential::Origin::Import.create(filename: "console-import-#{Time.now.to_i}")
host.elements.each('creds/cred') do |cred|
username = cred.elements['user'].try(:text)
proto = cred.elements['proto'].try(:text)
sname = cred.elements['sname'].try(:text)
port = cred.elements['port'].try(:text)
# Handle blanks by resetting to sane default values
proto = "tcp" if proto.blank?
pass = cred.elements['pass'].try(:text)
pass = "" if pass == "*MASKED*"
private = create_credential_private(private_data: pass, private_type: :password)
public = create_credential_public(username: username)
core = create_credential_core(private: private, public: public, origin: origin, workspace_id: wspace.id)
create_credential_login(core: core,
workspace_id: wspace.id,
address: hobj.address,
port: port,
protocol: proto,
service_name: sname,
status: Metasploit::Model::Login::Status::UNTRIED)
end
end
end
end
host.elements.each('sessions/session') do |sess|
sess_id = nils_for_nulls(sess.elements["id"].text.to_s.strip.to_i)
sess_data = {}
sess_data[:host] = hobj
%W{desc platform port stype}.each {|datum|
if sess.elements[datum].respond_to? :text
sess_data[datum.intern] = nils_for_nulls(sess.elements[datum].text.to_s.strip)
end
}
%W{opened-at close-reason closed-at via-exploit via-payload}.each {|datum|
if sess.elements[datum].respond_to? :text
sess_data[datum.gsub("-","_").intern] = nils_for_nulls(sess.elements[datum].text.to_s.strip)
end
}
sess_data[:datastore] = nils_for_nulls(unserialize_object(sess.elements["datastore"], allow_yaml))
if sess.elements["routes"]
sess_data[:routes] = nils_for_nulls(unserialize_object(sess.elements["routes"], allow_yaml)) || []
end
if not sess_data[:closed_at] # Fake a close if we don't already have one
sess_data[:closed_at] = Time.now.utc
sess_data[:close_reason] = "Imported at #{Time.now.utc}"
end
existing_session = get_session(
:workspace => sess_data[:host].workspace,
:addr => sess_data[:host].address,
:time => sess_data[:opened_at]
)
this_session = existing_session || report_session(sess_data)
next if existing_session
sess.elements.each('events/event') do |sess_event|
sess_event_data = {}
sess_event_data[:session] = this_session
%W{created-at etype local-path remote-path}.each {|datum|
if sess_event.elements[datum].respond_to? :text
sess_event_data[datum.gsub("-","_").intern] = nils_for_nulls(sess_event.elements[datum].text.to_s.strip)
end
}
%W{command output}.each {|datum|
if sess_event.elements[datum].respond_to? :text
sess_event_data[datum.gsub("-","_").intern] = nils_for_nulls(unserialize_object(sess_event.elements[datum], allow_yaml))
end
}
report_session_event(sess_event_data)
end
doc.each do |node|
case node.name
when 'host'
parse_host(Nokogiri::XML(node.outer_xml).at('./host'), wspace, bl, allow_yaml, btag, args, &block)
end
end
# require 'pry'
# binding.pry
# Import web sites
doc.elements.each("/#{btag}/web_sites/web_site") do |web|
@ -475,17 +256,17 @@ module Msf::DBManager::Import::MetasploitFramework::XML
info[:workspace] = wspace
%W{host port vhost ssl comments}.each do |datum|
if web.elements[datum].respond_to? :text
info[datum.intern] = nils_for_nulls(web.elements[datum].text.to_s.strip)
if web.at(datum).respond_to? :text
info[datum.intern] = nils_for_nulls(web.at(datum).text.to_s.strip)
end
end
info[:options] = nils_for_nulls(unserialize_object(web.elements["options"], allow_yaml)) if web.elements["options"].respond_to?(:text)
info[:options] = nils_for_nulls(unserialize_object(web.at("options"), allow_yaml)) if web.at("options").respond_to?(:text)
info[:ssl] = (info[:ssl] and info[:ssl].to_s.strip.downcase == "true") ? true : false
%W{created-at updated-at}.each { |datum|
if web.elements[datum].text
info[datum.gsub("-","_")] = nils_for_nulls(web.elements[datum].text.to_s.strip)
if web.at(datum).text
info[datum.gsub("-","_")] = nils_for_nulls(web.at(datum).text.to_s.strip)
end
}
@ -508,6 +289,241 @@ module Msf::DBManager::Import::MetasploitFramework::XML
private
def node_elements_to_hash()
end
def parse_host(host, wspace, bl, allow_yaml, btag, args, &block)
host_data = {}
host_data[:task] = args[:task]
host_data[:workspace] = wspace
# A regression resulted in the address field being serialized in some cases.
# Lets handle both instances to keep things happy. See #5837 & #5985
addr = nils_for_nulls(host.at('address'))
return 0 unless addr
# No period or colon means this must be in base64-encoded serialized form
if addr !~ /[\.\:]/
addr = unserialize_object(addr)
end
host_data[:host] = addr
if bl.include? host_data[:host]
return 0
else
yield(:address,host_data[:host]) if block
end
host_data[:mac] = nils_for_nulls(host.at("mac").text.to_s.strip)
if host.at("comm").text
host_data[:comm] = nils_for_nulls(host.at("comm").text.to_s.strip)
end
%W{created-at updated-at name state os-flavor os-lang os-name os-sp purpose}.each { |datum|
if host.at(datum).text
host_data[datum.gsub('-','_')] = nils_for_nulls(host.at(datum).text.to_s.strip)
end
}
host_address = host_data[:host].dup # Preserve after report_host() deletes
hobj = report_host(host_data)
host.xpath("host_details/host_detail").each do |hdet|
hdet_data = {}
hdet.elements.each do |det|
return 0 if ["id", "host-id"].include?(det.name)
if det.text
hdet_data[det.name.gsub('-','_')] = nils_for_nulls(det.text.to_s.strip)
end
end
report_host_details(hobj, hdet_data)
end
host.xpath("exploit_attempts/exploit_attempt").each do |hdet|
hdet_data = {}
hdet.elements.each do |det|
return 0 if ["id", "host-id", "session-id", "vuln-id", "service-id", "loot-id"].include?(det.name)
if det.text
hdet_data[det.name.gsub('-','_')] = nils_for_nulls(det.text.to_s.strip)
end
end
report_exploit_attempt(hobj, hdet_data)
end
host.xpath('services/service').each do |service|
service_data = {}
service_data[:task] = args[:task]
service_data[:workspace] = wspace
service_data[:host] = hobj
service_data[:port] = nils_for_nulls(service.at("port").text.to_s.strip).to_i
service_data[:proto] = nils_for_nulls(service.at("proto").text.to_s.strip)
%W{created-at updated-at name state info}.each { |datum|
if service.at(datum).text
if datum == "info"
service_data["info"] = nils_for_nulls(unserialize_object(service.at(datum), false))
else
service_data[datum.gsub("-","_")] = nils_for_nulls(service.at(datum).text.to_s.strip)
end
end
}
report_service(service_data)
end
host.xpath('notes/note').each do |note|
note_data = {}
note_data[:workspace] = wspace
note_data[:host] = hobj
import_msf_note_element(note,allow_yaml,note_data)
end
host.xpath('tags/tag').each do |tag|
tag_data = {}
tag_data[:addr] = host_address
tag_data[:wspace] = wspace
tag_data[:name] = tag.at("name").text.to_s.strip
tag_data[:desc] = tag.at("desc").text.to_s.strip
if tag.at("report-summary").text
tag_data[:summary] = tag.at("report-summary").text.to_s.strip
end
if tag.at("report-detail").text
tag_data[:detail] = tag.at("report-detail").text.to_s.strip
end
if tag.at("critical").text
tag_data[:crit] = true unless tag.at("critical").text.to_s.strip == "NULL"
end
report_host_tag(tag_data)
end
host.xpath('vulns/vuln').each do |vuln|
vuln_data = {}
vuln_data[:workspace] = wspace
vuln_data[:host] = hobj
vuln_data[:info] = nils_for_nulls(unserialize_object(vuln.at("info"), allow_yaml))
vuln_data[:name] = nils_for_nulls(vuln.at("name").text.to_s.strip)
%W{created-at updated-at exploited-at}.each { |datum|
if vuln.at(datum) and vuln.at(datum).text
vuln_data[datum.gsub("-","_")] = nils_for_nulls(vuln.at(datum).text.to_s.strip)
end
}
if vuln.at("refs")
vuln_data[:refs] = []
vuln.xpath("refs/ref").each do |ref|
vuln_data[:refs] << nils_for_nulls(ref.text.to_s.strip)
end
end
vobj = report_vuln(vuln_data)
vuln.xpath("notes/note").each do |note|
note_data = {}
note_data[:workspace] = wspace
note_data[:vuln_id] = vobj.id
import_msf_note_element(note,allow_yaml,note_data)
end
vuln.xpath("vuln_details/vuln_detail").each do |vdet|
vdet_data = {}
vdet.elements.each do |det|
return 0 if ["id", "vuln-id"].include?(det.name)
if det.text
vdet_data[det.name.gsub('-','_')] = nils_for_nulls(det.text.to_s.strip)
end
end
report_vuln_details(vobj, vdet_data)
end
vuln.xpath("vuln_attempts/vuln_attempt").each do |vdet|
vdet_data = {}
vdet.elements.each do |det|
return 0 if ["id", "vuln-id", "loot-id", "session-id"].include?(det.name)
if det.text
vdet_data[det.name.gsub('-','_')] = nils_for_nulls(det.text.to_s.strip)
end
end
report_vuln_attempt(vobj, vdet_data)
end
end
## Handle old-style (pre 4.10) XML files
if btag == "MetasploitV4"
if host.at('creds').present?
unless host.at('creds').elements.empty?
origin = Metasploit::Credential::Origin::Import.create(filename: "console-import-#{Time.now.to_i}")
host.xpath('creds/cred').each do |cred|
username = cred.at('user').try(:text)
proto = cred.at('proto').try(:text)
sname = cred.at('sname').try(:text)
port = cred.at('port').try(:text)
# Handle blanks by resetting to sane default values
proto = "tcp" if proto.blank?
pass = cred.at('pass').try(:text)
pass = "" if pass == "*MASKED*"
private = create_credential_private(private_data: pass, private_type: :password)
public = create_credential_public(username: username)
core = create_credential_core(private: private, public: public, origin: origin, workspace_id: wspace.id)
create_credential_login(core: core,
workspace_id: wspace.id,
address: hobj.address,
port: port,
protocol: proto,
service_name: sname,
status: Metasploit::Model::Login::Status::UNTRIED)
end
end
end
end
host.xpath('sessions/session').each do |sess|
sess_id = nils_for_nulls(sess.at("id").text.to_s.strip.to_i)
sess_data = {}
sess_data[:host] = hobj
%W{desc platform port stype}.each {|datum|
if sess.at(datum).respond_to? :text
sess_data[datum.intern] = nils_for_nulls(sess.at(datum).text.to_s.strip)
end
}
%W{opened-at close-reason closed-at via-exploit via-payload}.each {|datum|
if sess.at(datum).respond_to? :text
sess_data[datum.gsub("-","_").intern] = nils_for_nulls(sess.at(datum).text.to_s.strip)
end
}
sess_data[:datastore] = nils_for_nulls(unserialize_object(sess.at("datastore"), allow_yaml))
if sess.at("routes")
sess_data[:routes] = nils_for_nulls(unserialize_object(sess.at("routes"), allow_yaml)) || []
end
if not sess_data[:closed_at] # Fake a close if we don't already have one
sess_data[:closed_at] = Time.now.utc
sess_data[:close_reason] = "Imported at #{Time.now.utc}"
end
existing_session = get_session(
:workspace => sess_data[:host].workspace,
:addr => sess_data[:host].address,
:time => sess_data[:opened_at]
)
this_session = existing_session || report_session(sess_data)
return 0 if existing_session
sess.xpath('events/event').each do |sess_event|
sess_event_data = {}
sess_event_data[:session] = this_session
%W{created-at etype local-path remote-path}.each {|datum|
if sess_event.at(datum).respond_to? :text
sess_event_data[datum.gsub("-","_").intern] = nils_for_nulls(sess_event.at(datum).text.to_s.strip)
end
}
%W{command output}.each {|datum|
if sess_event.at(datum).respond_to? :text
sess_event_data[datum.gsub("-","_").intern] = nils_for_nulls(unserialize_object(sess_event.at(datum), allow_yaml))
end
}
report_session_event(sess_event_data)
end
end
end
# Checks if the XML document has a format version that the importer
# understands.
#
@ -518,28 +534,30 @@ module Msf::DBManager::Import::MetasploitFramework::XML
# {Msf::DBManager#unserialize_object}. `:root_tag` the tag name of the
# root element for MSF XML.
# @raise [Msf::DBImportError] if unsupported format
def check_msf_xml_version!(document)
def check_msf_xml_version!(name)
metadata = {
# FIXME https://www.pivotaltracker.com/story/show/47128407
:allow_yaml => false,
:root_tag => nil
}
if document.elements['MetasploitExpressV1']
case name
when 'MetasploitExpressV1'
# FIXME https://www.pivotaltracker.com/story/show/47128407
metadata[:allow_yaml] = true
metadata[:root_tag] = 'MetasploitExpressV1'
elsif document.elements['MetasploitExpressV2']
when 'MetasploitExpressV2'
# FIXME https://www.pivotaltracker.com/story/show/47128407
metadata[:allow_yaml] = true
metadata[:root_tag] = 'MetasploitExpressV2'
elsif document.elements['MetasploitExpressV3']
when 'MetasploitExpressV3'
metadata[:root_tag] = 'MetasploitExpressV3'
elsif document.elements['MetasploitExpressV4']
when 'MetasploitExpressV4'
metadata[:root_tag] = 'MetasploitExpressV4'
elsif document.elements['MetasploitV4']
when 'MetasploitV4'
metadata[:root_tag] = 'MetasploitV4'
elsif document.elements['MetasploitV5']
when 'MetasploitV5'
metadata[:root_tag] = 'MetasploitV5'
end
@ -564,7 +582,7 @@ module Msf::DBManager::Import::MetasploitFramework::XML
# @return [nil] if element with `child_name` does not exist under
# `parent_element`.
def import_msf_text_element(parent_element, child_name)
child_element = parent_element.elements[child_name]
child_element = parent_element.at(child.name)
info = {}
if child_element