From 802dfabbe331badae710240201c433c2bf436bbd Mon Sep 17 00:00:00 2001 From: dmohanty-r7 Date: Sat, 16 Apr 2016 23:16:26 -0500 Subject: [PATCH 01/12] Converts XML importer to use Nokogiri Reader MS-255 --- .../import/metasploit_framework/xml.rb | 546 +++++++++--------- 1 file changed, 282 insertions(+), 264 deletions(-) diff --git a/lib/msf/core/db_manager/import/metasploit_framework/xml.rb b/lib/msf/core/db_manager/import/metasploit_framework/xml.rb index 15c961f55a..ce68d03858 100644 --- a/lib/msf/core/db_manager/import/metasploit_framework/xml.rb +++ b/lib/msf/core/db_manager/import/metasploit_framework/xml.rb @@ -71,18 +71,18 @@ module Msf::DBManager::Import::MetasploitFramework::XML # @param note_data [Hash] hash containing note attributes to be passed along # @return [void] def import_msf_note_element(note, allow_yaml, note_data={}) - note_data[:type] = nils_for_nulls(note.elements["ntype"].text.to_s.strip) - note_data[:data] = nils_for_nulls(unserialize_object(note.elements["data"], allow_yaml)) + note_data[:type] = nils_for_nulls(note.at("ntype").text.to_s.strip) + note_data[:data] = nils_for_nulls(unserialize_object(note.at("data"), allow_yaml)) - if note.elements["critical"].text - note_data[:critical] = true unless note.elements["critical"].text.to_s.strip == "NULL" + if note.at("critical").text + note_data[:critical] = true unless note.at("critical").text.to_s.strip == "NULL" end - if note.elements["seen"].text - note_data[:seen] = true unless note.elements["critical"].text.to_s.strip == "NULL" + if note.at("seen").text + note_data[:seen] = true unless note.at("critical").text.to_s.strip == "NULL" end %W{created-at updated-at}.each { |datum| - if note.elements[datum].text - note_data[datum.gsub("-","_")] = nils_for_nulls(note.elements[datum].text.to_s.strip) + if note.at(datum).text + note_data[datum.gsub("-","_")] = nils_for_nulls(note.at(datum).text.to_s.strip) end } report_note(note_data) @@ -115,7 +115,7 @@ module Msf::DBManager::Import::MetasploitFramework::XML # FIXME https://www.pivotaltracker.com/story/show/46578647 # FIXME https://www.pivotaltracker.com/story/show/47128407 unserialized_params = unserialize_object( - element.elements['params'], + element.at('params'), options[:allow_yaml] ) info[:params] = nils_for_nulls(unserialized_params) @@ -162,7 +162,7 @@ module Msf::DBManager::Import::MetasploitFramework::XML # FIXME https://www.pivotaltracker.com/story/show/46578647 # FIXME https://www.pivotaltracker.com/story/show/47128407 unserialized_headers = unserialize_object( - element.elements['headers'], + element.at('headers'), options[:allow_yaml] ) info[:headers] = nils_for_nulls(unserialized_headers) @@ -209,7 +209,7 @@ module Msf::DBManager::Import::MetasploitFramework::XML # FIXME https://www.pivotaltracker.com/story/show/46578647 # FIXME https://www.pivotaltracker.com/story/show/47128407 unserialized_params = unserialize_object( - element.elements['params'], + element.at('params'), options[:allow_yaml] ) info[:params] = nils_for_nulls(unserialized_params) @@ -228,246 +228,27 @@ module Msf::DBManager::Import::MetasploitFramework::XML # them. # TODO: loot, tasks, and reports def import_msf_xml(args={}, &block) + data = args[:data] wspace = args[:wspace] || workspace bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - doc = rexmlify(data) - metadata = check_msf_xml_version!(doc) + #doc = rexmlify(data) + doc = Nokogiri::XML::Reader.from_memory(data) + + metadata = check_msf_xml_version!(doc.first.name) allow_yaml = metadata[:allow_yaml] btag = metadata[:root_tag] - doc.elements.each("/#{btag}/hosts/host") do |host| - host_data = {} - host_data[:task] = args[:task] - host_data[:workspace] = wspace - - # A regression resulted in the address field being serialized in some cases. - # Lets handle both instances to keep things happy. See #5837 & #5985 - addr = nils_for_nulls(host.elements["address"]) - next unless addr - - # No period or colon means this must be in base64-encoded serialized form - if addr !~ /[\.\:]/ - addr = unserialize_object(addr) - end - - host_data[:host] = addr - if bl.include? host_data[:host] - next - else - yield(:address,host_data[:host]) if block - end - host_data[:mac] = nils_for_nulls(host.elements["mac"].text.to_s.strip) - if host.elements["comm"].text - host_data[:comm] = nils_for_nulls(host.elements["comm"].text.to_s.strip) - end - %W{created-at updated-at name state os-flavor os-lang os-name os-sp purpose}.each { |datum| - if host.elements[datum].text - host_data[datum.gsub('-','_')] = nils_for_nulls(host.elements[datum].text.to_s.strip) - end - } - host_address = host_data[:host].dup # Preserve after report_host() deletes - hobj = report_host(host_data) - - host.elements.each("host_details/host_detail") do |hdet| - hdet_data = {} - hdet.elements.each do |det| - next if ["id", "host-id"].include?(det.name) - if det.text - hdet_data[det.name.gsub('-','_')] = nils_for_nulls(det.text.to_s.strip) - end - end - report_host_details(hobj, hdet_data) - end - - host.elements.each("exploit_attempts/exploit_attempt") do |hdet| - hdet_data = {} - hdet.elements.each do |det| - next if ["id", "host-id", "session-id", "vuln-id", "service-id", "loot-id"].include?(det.name) - if det.text - hdet_data[det.name.gsub('-','_')] = nils_for_nulls(det.text.to_s.strip) - end - end - report_exploit_attempt(hobj, hdet_data) - end - - host.elements.each('services/service') do |service| - service_data = {} - service_data[:task] = args[:task] - service_data[:workspace] = wspace - service_data[:host] = hobj - service_data[:port] = nils_for_nulls(service.elements["port"].text.to_s.strip).to_i - service_data[:proto] = nils_for_nulls(service.elements["proto"].text.to_s.strip) - %W{created-at updated-at name state info}.each { |datum| - if service.elements[datum].text - if datum == "info" - service_data["info"] = nils_for_nulls(unserialize_object(service.elements[datum], false)) - else - service_data[datum.gsub("-","_")] = nils_for_nulls(service.elements[datum].text.to_s.strip) - end - end - } - report_service(service_data) - end - - host.elements.each('notes/note') do |note| - note_data = {} - note_data[:workspace] = wspace - note_data[:host] = hobj - import_msf_note_element(note,allow_yaml,note_data) - end - - host.elements.each('tags/tag') do |tag| - tag_data = {} - tag_data[:addr] = host_address - tag_data[:wspace] = wspace - tag_data[:name] = tag.elements["name"].text.to_s.strip - tag_data[:desc] = tag.elements["desc"].text.to_s.strip - if tag.elements["report-summary"].text - tag_data[:summary] = tag.elements["report-summary"].text.to_s.strip - end - if tag.elements["report-detail"].text - tag_data[:detail] = tag.elements["report-detail"].text.to_s.strip - end - if tag.elements["critical"].text - tag_data[:crit] = true unless tag.elements["critical"].text.to_s.strip == "NULL" - end - report_host_tag(tag_data) - end - - host.elements.each('vulns/vuln') do |vuln| - vuln_data = {} - vuln_data[:workspace] = wspace - vuln_data[:host] = hobj - vuln_data[:info] = nils_for_nulls(unserialize_object(vuln.elements["info"], allow_yaml)) - vuln_data[:name] = nils_for_nulls(vuln.elements["name"].text.to_s.strip) - %W{created-at updated-at exploited-at}.each { |datum| - if vuln.elements[datum] and vuln.elements[datum].text - vuln_data[datum.gsub("-","_")] = nils_for_nulls(vuln.elements[datum].text.to_s.strip) - end - } - if vuln.elements["refs"] - vuln_data[:refs] = [] - vuln.elements.each("refs/ref") do |ref| - vuln_data[:refs] << nils_for_nulls(ref.text.to_s.strip) - end - end - - vobj = report_vuln(vuln_data) - - vuln.elements.each("notes/note") do |note| - note_data = {} - note_data[:workspace] = wspace - note_data[:vuln_id] = vobj.id - import_msf_note_element(note,allow_yaml,note_data) - end - - vuln.elements.each("vuln_details/vuln_detail") do |vdet| - vdet_data = {} - vdet.elements.each do |det| - next if ["id", "vuln-id"].include?(det.name) - if det.text - vdet_data[det.name.gsub('-','_')] = nils_for_nulls(det.text.to_s.strip) - end - end - report_vuln_details(vobj, vdet_data) - end - - vuln.elements.each("vuln_attempts/vuln_attempt") do |vdet| - vdet_data = {} - vdet.elements.each do |det| - next if ["id", "vuln-id", "loot-id", "session-id"].include?(det.name) - if det.text - vdet_data[det.name.gsub('-','_')] = nils_for_nulls(det.text.to_s.strip) - end - end - report_vuln_attempt(vobj, vdet_data) - end - end - - ## Handle old-style (pre 4.10) XML files - if btag == "MetasploitV4" - if host.elements['creds'].present? - unless host.elements['creds'].elements.empty? - origin = Metasploit::Credential::Origin::Import.create(filename: "console-import-#{Time.now.to_i}") - - host.elements.each('creds/cred') do |cred| - username = cred.elements['user'].try(:text) - proto = cred.elements['proto'].try(:text) - sname = cred.elements['sname'].try(:text) - port = cred.elements['port'].try(:text) - - # Handle blanks by resetting to sane default values - proto = "tcp" if proto.blank? - pass = cred.elements['pass'].try(:text) - pass = "" if pass == "*MASKED*" - - private = create_credential_private(private_data: pass, private_type: :password) - public = create_credential_public(username: username) - core = create_credential_core(private: private, public: public, origin: origin, workspace_id: wspace.id) - - create_credential_login(core: core, - workspace_id: wspace.id, - address: hobj.address, - port: port, - protocol: proto, - service_name: sname, - status: Metasploit::Model::Login::Status::UNTRIED) - end - end - end - end - - - host.elements.each('sessions/session') do |sess| - sess_id = nils_for_nulls(sess.elements["id"].text.to_s.strip.to_i) - sess_data = {} - sess_data[:host] = hobj - %W{desc platform port stype}.each {|datum| - if sess.elements[datum].respond_to? :text - sess_data[datum.intern] = nils_for_nulls(sess.elements[datum].text.to_s.strip) - end - } - %W{opened-at close-reason closed-at via-exploit via-payload}.each {|datum| - if sess.elements[datum].respond_to? :text - sess_data[datum.gsub("-","_").intern] = nils_for_nulls(sess.elements[datum].text.to_s.strip) - end - } - sess_data[:datastore] = nils_for_nulls(unserialize_object(sess.elements["datastore"], allow_yaml)) - if sess.elements["routes"] - sess_data[:routes] = nils_for_nulls(unserialize_object(sess.elements["routes"], allow_yaml)) || [] - end - if not sess_data[:closed_at] # Fake a close if we don't already have one - sess_data[:closed_at] = Time.now.utc - sess_data[:close_reason] = "Imported at #{Time.now.utc}" - end - - existing_session = get_session( - :workspace => sess_data[:host].workspace, - :addr => sess_data[:host].address, - :time => sess_data[:opened_at] - ) - this_session = existing_session || report_session(sess_data) - next if existing_session - sess.elements.each('events/event') do |sess_event| - sess_event_data = {} - sess_event_data[:session] = this_session - %W{created-at etype local-path remote-path}.each {|datum| - if sess_event.elements[datum].respond_to? :text - sess_event_data[datum.gsub("-","_").intern] = nils_for_nulls(sess_event.elements[datum].text.to_s.strip) - end - } - %W{command output}.each {|datum| - if sess_event.elements[datum].respond_to? :text - sess_event_data[datum.gsub("-","_").intern] = nils_for_nulls(unserialize_object(sess_event.elements[datum], allow_yaml)) - end - } - report_session_event(sess_event_data) - end + doc.each do |node| + case node.name + when 'host' + parse_host(Nokogiri::XML(node.outer_xml).at('./host'), wspace, bl, allow_yaml, btag, args, &block) end end + # require 'pry' + # binding.pry # Import web sites doc.elements.each("/#{btag}/web_sites/web_site") do |web| @@ -475,17 +256,17 @@ module Msf::DBManager::Import::MetasploitFramework::XML info[:workspace] = wspace %W{host port vhost ssl comments}.each do |datum| - if web.elements[datum].respond_to? :text - info[datum.intern] = nils_for_nulls(web.elements[datum].text.to_s.strip) + if web.at(datum).respond_to? :text + info[datum.intern] = nils_for_nulls(web.at(datum).text.to_s.strip) end end - info[:options] = nils_for_nulls(unserialize_object(web.elements["options"], allow_yaml)) if web.elements["options"].respond_to?(:text) + info[:options] = nils_for_nulls(unserialize_object(web.at("options"), allow_yaml)) if web.at("options").respond_to?(:text) info[:ssl] = (info[:ssl] and info[:ssl].to_s.strip.downcase == "true") ? true : false %W{created-at updated-at}.each { |datum| - if web.elements[datum].text - info[datum.gsub("-","_")] = nils_for_nulls(web.elements[datum].text.to_s.strip) + if web.at(datum).text + info[datum.gsub("-","_")] = nils_for_nulls(web.at(datum).text.to_s.strip) end } @@ -508,6 +289,241 @@ module Msf::DBManager::Import::MetasploitFramework::XML private + def node_elements_to_hash() + end + + def parse_host(host, wspace, bl, allow_yaml, btag, args, &block) + + host_data = {} + host_data[:task] = args[:task] + host_data[:workspace] = wspace + + # A regression resulted in the address field being serialized in some cases. + # Lets handle both instances to keep things happy. See #5837 & #5985 + addr = nils_for_nulls(host.at('address')) + return 0 unless addr + + # No period or colon means this must be in base64-encoded serialized form + if addr !~ /[\.\:]/ + addr = unserialize_object(addr) + end + + host_data[:host] = addr + if bl.include? host_data[:host] + return 0 + else + yield(:address,host_data[:host]) if block + end + host_data[:mac] = nils_for_nulls(host.at("mac").text.to_s.strip) + if host.at("comm").text + host_data[:comm] = nils_for_nulls(host.at("comm").text.to_s.strip) + end + %W{created-at updated-at name state os-flavor os-lang os-name os-sp purpose}.each { |datum| + if host.at(datum).text + host_data[datum.gsub('-','_')] = nils_for_nulls(host.at(datum).text.to_s.strip) + end + } + host_address = host_data[:host].dup # Preserve after report_host() deletes + hobj = report_host(host_data) + + host.xpath("host_details/host_detail").each do |hdet| + hdet_data = {} + hdet.elements.each do |det| + return 0 if ["id", "host-id"].include?(det.name) + if det.text + hdet_data[det.name.gsub('-','_')] = nils_for_nulls(det.text.to_s.strip) + end + end + report_host_details(hobj, hdet_data) + end + + host.xpath("exploit_attempts/exploit_attempt").each do |hdet| + hdet_data = {} + hdet.elements.each do |det| + return 0 if ["id", "host-id", "session-id", "vuln-id", "service-id", "loot-id"].include?(det.name) + if det.text + hdet_data[det.name.gsub('-','_')] = nils_for_nulls(det.text.to_s.strip) + end + end + report_exploit_attempt(hobj, hdet_data) + end + + host.xpath('services/service').each do |service| + service_data = {} + service_data[:task] = args[:task] + service_data[:workspace] = wspace + service_data[:host] = hobj + service_data[:port] = nils_for_nulls(service.at("port").text.to_s.strip).to_i + service_data[:proto] = nils_for_nulls(service.at("proto").text.to_s.strip) + %W{created-at updated-at name state info}.each { |datum| + if service.at(datum).text + if datum == "info" + service_data["info"] = nils_for_nulls(unserialize_object(service.at(datum), false)) + else + service_data[datum.gsub("-","_")] = nils_for_nulls(service.at(datum).text.to_s.strip) + end + end + } + report_service(service_data) + end + + host.xpath('notes/note').each do |note| + note_data = {} + note_data[:workspace] = wspace + note_data[:host] = hobj + import_msf_note_element(note,allow_yaml,note_data) + end + + host.xpath('tags/tag').each do |tag| + tag_data = {} + tag_data[:addr] = host_address + tag_data[:wspace] = wspace + tag_data[:name] = tag.at("name").text.to_s.strip + tag_data[:desc] = tag.at("desc").text.to_s.strip + if tag.at("report-summary").text + tag_data[:summary] = tag.at("report-summary").text.to_s.strip + end + if tag.at("report-detail").text + tag_data[:detail] = tag.at("report-detail").text.to_s.strip + end + if tag.at("critical").text + tag_data[:crit] = true unless tag.at("critical").text.to_s.strip == "NULL" + end + report_host_tag(tag_data) + end + + host.xpath('vulns/vuln').each do |vuln| + vuln_data = {} + vuln_data[:workspace] = wspace + vuln_data[:host] = hobj + vuln_data[:info] = nils_for_nulls(unserialize_object(vuln.at("info"), allow_yaml)) + vuln_data[:name] = nils_for_nulls(vuln.at("name").text.to_s.strip) + %W{created-at updated-at exploited-at}.each { |datum| + if vuln.at(datum) and vuln.at(datum).text + vuln_data[datum.gsub("-","_")] = nils_for_nulls(vuln.at(datum).text.to_s.strip) + end + } + if vuln.at("refs") + vuln_data[:refs] = [] + vuln.xpath("refs/ref").each do |ref| + vuln_data[:refs] << nils_for_nulls(ref.text.to_s.strip) + end + end + + vobj = report_vuln(vuln_data) + + vuln.xpath("notes/note").each do |note| + note_data = {} + note_data[:workspace] = wspace + note_data[:vuln_id] = vobj.id + import_msf_note_element(note,allow_yaml,note_data) + end + + vuln.xpath("vuln_details/vuln_detail").each do |vdet| + vdet_data = {} + vdet.elements.each do |det| + return 0 if ["id", "vuln-id"].include?(det.name) + if det.text + vdet_data[det.name.gsub('-','_')] = nils_for_nulls(det.text.to_s.strip) + end + end + report_vuln_details(vobj, vdet_data) + end + + vuln.xpath("vuln_attempts/vuln_attempt").each do |vdet| + vdet_data = {} + vdet.elements.each do |det| + return 0 if ["id", "vuln-id", "loot-id", "session-id"].include?(det.name) + if det.text + vdet_data[det.name.gsub('-','_')] = nils_for_nulls(det.text.to_s.strip) + end + end + report_vuln_attempt(vobj, vdet_data) + end + end + + ## Handle old-style (pre 4.10) XML files + if btag == "MetasploitV4" + if host.at('creds').present? + unless host.at('creds').elements.empty? + origin = Metasploit::Credential::Origin::Import.create(filename: "console-import-#{Time.now.to_i}") + + host.xpath('creds/cred').each do |cred| + username = cred.at('user').try(:text) + proto = cred.at('proto').try(:text) + sname = cred.at('sname').try(:text) + port = cred.at('port').try(:text) + + # Handle blanks by resetting to sane default values + proto = "tcp" if proto.blank? + pass = cred.at('pass').try(:text) + pass = "" if pass == "*MASKED*" + + private = create_credential_private(private_data: pass, private_type: :password) + public = create_credential_public(username: username) + core = create_credential_core(private: private, public: public, origin: origin, workspace_id: wspace.id) + + create_credential_login(core: core, + workspace_id: wspace.id, + address: hobj.address, + port: port, + protocol: proto, + service_name: sname, + status: Metasploit::Model::Login::Status::UNTRIED) + end + end + end + end + + + host.xpath('sessions/session').each do |sess| + sess_id = nils_for_nulls(sess.at("id").text.to_s.strip.to_i) + sess_data = {} + sess_data[:host] = hobj + %W{desc platform port stype}.each {|datum| + if sess.at(datum).respond_to? :text + sess_data[datum.intern] = nils_for_nulls(sess.at(datum).text.to_s.strip) + end + } + %W{opened-at close-reason closed-at via-exploit via-payload}.each {|datum| + if sess.at(datum).respond_to? :text + sess_data[datum.gsub("-","_").intern] = nils_for_nulls(sess.at(datum).text.to_s.strip) + end + } + sess_data[:datastore] = nils_for_nulls(unserialize_object(sess.at("datastore"), allow_yaml)) + if sess.at("routes") + sess_data[:routes] = nils_for_nulls(unserialize_object(sess.at("routes"), allow_yaml)) || [] + end + if not sess_data[:closed_at] # Fake a close if we don't already have one + sess_data[:closed_at] = Time.now.utc + sess_data[:close_reason] = "Imported at #{Time.now.utc}" + end + + existing_session = get_session( + :workspace => sess_data[:host].workspace, + :addr => sess_data[:host].address, + :time => sess_data[:opened_at] + ) + this_session = existing_session || report_session(sess_data) + return 0 if existing_session + sess.xpath('events/event').each do |sess_event| + sess_event_data = {} + sess_event_data[:session] = this_session + %W{created-at etype local-path remote-path}.each {|datum| + if sess_event.at(datum).respond_to? :text + sess_event_data[datum.gsub("-","_").intern] = nils_for_nulls(sess_event.at(datum).text.to_s.strip) + end + } + %W{command output}.each {|datum| + if sess_event.at(datum).respond_to? :text + sess_event_data[datum.gsub("-","_").intern] = nils_for_nulls(unserialize_object(sess_event.at(datum), allow_yaml)) + end + } + report_session_event(sess_event_data) + end + end + end + # Checks if the XML document has a format version that the importer # understands. # @@ -518,29 +534,31 @@ module Msf::DBManager::Import::MetasploitFramework::XML # {Msf::DBManager#unserialize_object}. `:root_tag` the tag name of the # root element for MSF XML. # @raise [Msf::DBImportError] if unsupported format - def check_msf_xml_version!(document) + def check_msf_xml_version!(name) + metadata = { # FIXME https://www.pivotaltracker.com/story/show/47128407 :allow_yaml => false, :root_tag => nil } - if document.elements['MetasploitExpressV1'] - # FIXME https://www.pivotaltracker.com/story/show/47128407 - metadata[:allow_yaml] = true - metadata[:root_tag] = 'MetasploitExpressV1' - elsif document.elements['MetasploitExpressV2'] - # FIXME https://www.pivotaltracker.com/story/show/47128407 - metadata[:allow_yaml] = true - metadata[:root_tag] = 'MetasploitExpressV2' - elsif document.elements['MetasploitExpressV3'] - metadata[:root_tag] = 'MetasploitExpressV3' - elsif document.elements['MetasploitExpressV4'] - metadata[:root_tag] = 'MetasploitExpressV4' - elsif document.elements['MetasploitV4'] - metadata[:root_tag] = 'MetasploitV4' - elsif document.elements['MetasploitV5'] - metadata[:root_tag] = 'MetasploitV5' + case name + when 'MetasploitExpressV1' + # FIXME https://www.pivotaltracker.com/story/show/47128407 + metadata[:allow_yaml] = true + metadata[:root_tag] = 'MetasploitExpressV1' + when 'MetasploitExpressV2' + # FIXME https://www.pivotaltracker.com/story/show/47128407 + metadata[:allow_yaml] = true + metadata[:root_tag] = 'MetasploitExpressV2' + when 'MetasploitExpressV3' + metadata[:root_tag] = 'MetasploitExpressV3' + when 'MetasploitExpressV4' + metadata[:root_tag] = 'MetasploitExpressV4' + when 'MetasploitV4' + metadata[:root_tag] = 'MetasploitV4' + when 'MetasploitV5' + metadata[:root_tag] = 'MetasploitV5' end unless metadata[:root_tag] @@ -564,7 +582,7 @@ module Msf::DBManager::Import::MetasploitFramework::XML # @return [nil] if element with `child_name` does not exist under # `parent_element`. def import_msf_text_element(parent_element, child_name) - child_element = parent_element.elements[child_name] + child_element = parent_element.at(child.name) info = {} if child_element From f1d8e1d6936aee0f39d7786c7ccbd9d129ad4ae2 Mon Sep 17 00:00:00 2001 From: dmohanty-r7 Date: Sun, 17 Apr 2016 23:43:34 -0500 Subject: [PATCH 02/12] Parse web_data in xml import MS-255 --- .../import/metasploit_framework/xml.rb | 76 +++++++++---------- 1 file changed, 34 insertions(+), 42 deletions(-) diff --git a/lib/msf/core/db_manager/import/metasploit_framework/xml.rb b/lib/msf/core/db_manager/import/metasploit_framework/xml.rb index ce68d03858..f97ccfdb38 100644 --- a/lib/msf/core/db_manager/import/metasploit_framework/xml.rb +++ b/lib/msf/core/db_manager/import/metasploit_framework/xml.rb @@ -243,53 +243,45 @@ module Msf::DBManager::Import::MetasploitFramework::XML doc.each do |node| case node.name when 'host' - parse_host(Nokogiri::XML(node.outer_xml).at('./host'), wspace, bl, allow_yaml, btag, args, &block) - end - end - - # require 'pry' - # binding.pry - - # Import web sites - doc.elements.each("/#{btag}/web_sites/web_site") do |web| - info = {} - info[:workspace] = wspace - - %W{host port vhost ssl comments}.each do |datum| - if web.at(datum).respond_to? :text - info[datum.intern] = nils_for_nulls(web.at(datum).text.to_s.strip) - end - end - - info[:options] = nils_for_nulls(unserialize_object(web.at("options"), allow_yaml)) if web.at("options").respond_to?(:text) - info[:ssl] = (info[:ssl] and info[:ssl].to_s.strip.downcase == "true") ? true : false - - %W{created-at updated-at}.each { |datum| - if web.at(datum).text - info[datum.gsub("-","_")] = nils_for_nulls(web.at(datum).text.to_s.strip) - end - } - - report_web_site(info) - yield(:web_site, "#{info[:host]}:#{info[:port]} (#{info[:vhost]})") if block - end - - %W{page form vuln}.each do |wtype| - doc.elements.each("/#{btag}/web_#{wtype}s/web_#{wtype}") do |element| - send( - "import_msf_web_#{wtype}_element", - element, - :allow_yaml => allow_yaml, - :workspace => wspace, - &block - ) + parse_host(Nokogiri::XML(node.outer_xml).at("./#{node.name}"), wspace, bl, allow_yaml, btag, args, &block) + when 'web_site' + parse_web_site(Nokogiri::XML(node.outer_xml).at("./#{node.name}"), wspace, bl, allow_yaml, btag, args, &block) + when 'web_page', 'web_form', 'web_vuln' + send( + "import_msf_#{node.name}_element", + Nokogiri::XML(node.outer_xml).at("./#{node.name}"), + :allow_yaml => allow_yaml, + :workspace => wspace, + &block + ) end end end private - def node_elements_to_hash() + def parse_web_site(web, wspace, bl, allow_yaml, btag, args, &block) + # Import web sites + info = {} + info[:workspace] = wspace + + %W{host port vhost ssl comments}.each do |datum| + if web.at(datum).respond_to? :text + info[datum.intern] = nils_for_nulls(web.at(datum).text.to_s.strip) + end + end + + info[:options] = nils_for_nulls(unserialize_object(web.at("options"), allow_yaml)) if web.at("options").respond_to?(:text) + info[:ssl] = (info[:ssl] and info[:ssl].to_s.strip.downcase == "true") ? true : false + + %W{created-at updated-at}.each { |datum| + if web.at(datum).text + info[datum.gsub("-","_")] = nils_for_nulls(web.at(datum).text.to_s.strip) + end + } + + report_web_site(info) + yield(:web_site, "#{info[:host]}:#{info[:port]} (#{info[:vhost]})") if block end def parse_host(host, wspace, bl, allow_yaml, btag, args, &block) @@ -582,7 +574,7 @@ module Msf::DBManager::Import::MetasploitFramework::XML # @return [nil] if element with `child_name` does not exist under # `parent_element`. def import_msf_text_element(parent_element, child_name) - child_element = parent_element.at(child.name) + child_element = parent_element.at(child_name) info = {} if child_element From e6a8d69b0b7b817e789a6afb4052c8afd0508975 Mon Sep 17 00:00:00 2001 From: dmohanty-r7 Date: Mon, 18 Apr 2016 01:02:41 -0500 Subject: [PATCH 03/12] Force encoding of XML import MS-255 --- lib/msf/core/db_manager/import.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index 3a243fdfb1..8b994583b6 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -229,7 +229,7 @@ module Msf::DBManager::Import end # This is a text string, lets make sure its treated as binary - data = data.unpack("C*").pack("C*") + data.force_encoding(Encoding::ASCII_8BIT) if data and data.to_s.strip.length == 0 raise Msf::DBImportError.new("The data provided to the import function was empty") end From e4fcaefc8caa7efe061d340be9e272d9a22d1eae Mon Sep 17 00:00:00 2001 From: dmohanty-r7 Date: Mon, 18 Apr 2016 08:39:55 -0500 Subject: [PATCH 04/12] Unpack and pack an unsigned integer per 8 bytes MS-255 --- lib/msf/core/db_manager/import.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index 8b994583b6..641cfbe1b3 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -229,7 +229,7 @@ module Msf::DBManager::Import end # This is a text string, lets make sure its treated as binary - data.force_encoding(Encoding::ASCII_8BIT) + data = data.unpack("Q*").pack("Q*") if data and data.to_s.strip.length == 0 raise Msf::DBImportError.new("The data provided to the import function was empty") end From 83ff60c111890c369b97edf53d5d24ee12879beb Mon Sep 17 00:00:00 2001 From: dmohanty-r7 Date: Mon, 18 Apr 2016 10:29:35 -0500 Subject: [PATCH 05/12] Force encoding on import xml MS-255 --- lib/msf/core/db_manager/import.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index 641cfbe1b3..8b994583b6 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -229,7 +229,7 @@ module Msf::DBManager::Import end # This is a text string, lets make sure its treated as binary - data = data.unpack("Q*").pack("Q*") + data.force_encoding(Encoding::ASCII_8BIT) if data and data.to_s.strip.length == 0 raise Msf::DBImportError.new("The data provided to the import function was empty") end From 0759848ad5a30b709b4a56b05e47858091fb7d32 Mon Sep 17 00:00:00 2001 From: dmohanty-r7 Date: Mon, 18 Apr 2016 15:47:36 -0500 Subject: [PATCH 06/12] Use Nokogiri Reader in zip import MS-255 --- .../import/metasploit_framework/zip.rb | 238 +++++++++--------- 1 file changed, 124 insertions(+), 114 deletions(-) diff --git a/lib/msf/core/db_manager/import/metasploit_framework/zip.rb b/lib/msf/core/db_manager/import/metasploit_framework/zip.rb index 791842de85..c121a07ca2 100644 --- a/lib/msf/core/db_manager/import/metasploit_framework/zip.rb +++ b/lib/msf/core/db_manager/import/metasploit_framework/zip.rb @@ -10,137 +10,147 @@ module Msf::DBManager::Import::MetasploitFramework::Zip allow_yaml = false btag = nil - doc = rexmlify(data) - if doc.elements["MetasploitExpressV1"] - m_ver = 1 - allow_yaml = true - btag = "MetasploitExpressV1" - elsif doc.elements["MetasploitExpressV2"] - m_ver = 2 - allow_yaml = true - btag = "MetasploitExpressV2" - elsif doc.elements["MetasploitExpressV3"] - m_ver = 3 - btag = "MetasploitExpressV3" - elsif doc.elements["MetasploitExpressV4"] - m_ver = 4 - btag = "MetasploitExpressV4" - elsif doc.elements["MetasploitV4"] - m_ver = 4 - btag = "MetasploitV4" - else - m_ver = nil + doc = Nokogiri::XML::Reader.from_memory(data) + + case doc.first.name + when "MetasploitExpressV1" + m_ver = 1 + allow_yaml = true + btag = "MetasploitExpressV1" + when "MetasploitExpressV2" + m_ver = 2 + allow_yaml = true + btag = "MetasploitExpressV2" + when "MetasploitExpressV3" + m_ver = 3 + btag = "MetasploitExpressV3" + when "MetasploitExpressV4" + m_ver = 4 + btag = "MetasploitExpressV4" + when "MetasploitV4" + m_ver = 4 + btag = "MetasploitV4" + else + m_ver = nil end unless m_ver and btag raise Msf::DBImportError.new("Unsupported Metasploit XML document format") end host_info = {} - doc.elements.each("/#{btag}/hosts/host") do |host| - host_info[host.elements["id"].text.to_s.strip] = nils_for_nulls(host.elements["address"].text.to_s.strip) - end - # Import Loot - doc.elements.each("/#{btag}/loots/loot") do |loot| - next if bl.include? host_info[loot.elements["host-id"].text.to_s.strip] - loot_info = {} - loot_info[:host] = host_info[loot.elements["host-id"].text.to_s.strip] - loot_info[:workspace] = args[:wspace] - loot_info[:ctype] = nils_for_nulls(loot.elements["content-type"].text.to_s.strip) - loot_info[:info] = nils_for_nulls(unserialize_object(loot.elements["info"], allow_yaml)) - loot_info[:ltype] = nils_for_nulls(loot.elements["ltype"].text.to_s.strip) - loot_info[:name] = nils_for_nulls(loot.elements["name"].text.to_s.strip) - loot_info[:created_at] = nils_for_nulls(loot.elements["created-at"].text.to_s.strip) - loot_info[:updated_at] = nils_for_nulls(loot.elements["updated-at"].text.to_s.strip) - loot_info[:name] = nils_for_nulls(loot.elements["name"].text.to_s.strip) - loot_info[:orig_path] = nils_for_nulls(loot.elements["path"].text.to_s.strip) - loot_info[:task] = args[:task] - tmp = args[:ifd][:zip_tmp] - loot_info[:orig_path].gsub!(/^\./,tmp) if loot_info[:orig_path] - if !loot.elements["service-id"].text.to_s.strip.empty? - unless loot.elements["service-id"].text.to_s.strip == "NULL" - loot_info[:service] = loot.elements["service-id"].text.to_s.strip - end + doc.each do |node| + case node.name + when 'host', 'loot', 'task', 'report' + send("parse_zip_#{node.name}", Nokogiri::XML(node.outer_xml).at("./#{node.name}"), wspace, bl, allow_yaml, btag, args, basedir, host_info, &block) end + end + end - # Only report loot if we actually have it. - # TODO: Copypasta. Separate this out. - if ::File.exist? loot_info[:orig_path] - loot_dir = ::File.join(basedir,"loot") - loot_file = ::File.split(loot_info[:orig_path]).last - if ::File.exist? loot_dir - unless (::File.directory?(loot_dir) && ::File.writable?(loot_dir)) - raise Msf::DBImportError.new("Could not move files to #{loot_dir}") - end - else - ::FileUtils.mkdir_p(loot_dir) - end - new_loot = ::File.join(loot_dir,loot_file) - loot_info[:path] = new_loot - if ::File.exist?(new_loot) - ::File.unlink new_loot # Delete it, and don't report it. - else - report_loot(loot_info) # It's new, so report it. - end - ::FileUtils.copy(loot_info[:orig_path], new_loot) - yield(:msf_loot, new_loot) if block + def parse_zip_host(host, wspace, bl, allow_yaml, btag, args, basedir, host_info, &block) + if host.at("id") + host_info[host.at("id").text.to_s.strip] = nils_for_nulls(host.at("address").text.to_s.strip) + end + end + + def parse_zip_loot(loot, wspace, bl, allow_yaml, btag, args, basedir, host_info, &block) + return 0 if loot.at("host-id").nil? || bl.include?(host_info[loot.at("host-id").text.to_s.strip]) + loot_info = {} + loot_info[:host] = host_info[loot.at("host-id").text.to_s.strip] + loot_info[:workspace] = args[:wspace] + loot_info[:ctype] = nils_for_nulls(loot.at("content-type").text.to_s.strip) + loot_info[:info] = nils_for_nulls(unserialize_object(loot.at("info"), allow_yaml)) + loot_info[:ltype] = nils_for_nulls(loot.at("ltype").text.to_s.strip) + loot_info[:name] = nils_for_nulls(loot.at("name").text.to_s.strip) + loot_info[:created_at] = nils_for_nulls(loot.at("created-at").text.to_s.strip) + loot_info[:updated_at] = nils_for_nulls(loot.at("updated-at").text.to_s.strip) + loot_info[:name] = nils_for_nulls(loot.at("name").text.to_s.strip) + loot_info[:orig_path] = nils_for_nulls(loot.at("path").text.to_s.strip) + loot_info[:task] = args[:task] + tmp = args[:ifd][:zip_tmp] + loot_info[:orig_path].gsub!(/^\./,tmp) if loot_info[:orig_path] + if !loot.at("service-id").text.to_s.strip.empty? + unless loot.at("service-id").text.to_s.strip == "NULL" + loot_info[:service] = loot.at("service-id").text.to_s.strip end end - # Import Tasks - doc.elements.each("/#{btag}/tasks/task") do |task| - task_info = {} - task_info[:workspace] = args[:wspace] - # Should user be imported (original) or declared (the importing user)? - task_info[:user] = nils_for_nulls(task.elements["created-by"].text.to_s.strip) - task_info[:desc] = nils_for_nulls(task.elements["description"].text.to_s.strip) - task_info[:info] = nils_for_nulls(unserialize_object(task.elements["info"], allow_yaml)) - task_info[:mod] = nils_for_nulls(task.elements["module"].text.to_s.strip) - task_info[:options] = nils_for_nulls(task.elements["options"].text.to_s.strip) - task_info[:prog] = nils_for_nulls(task.elements["progress"].text.to_s.strip).to_i - task_info[:created_at] = nils_for_nulls(task.elements["created-at"].text.to_s.strip) - task_info[:updated_at] = nils_for_nulls(task.elements["updated-at"].text.to_s.strip) - if !task.elements["completed-at"].text.to_s.empty? - task_info[:completed_at] = nils_for_nulls(task.elements["completed-at"].text.to_s.strip) - end - if !task.elements["error"].text.to_s.empty? - task_info[:error] = nils_for_nulls(task.elements["error"].text.to_s.strip) - end - if !task.elements["result"].text.to_s.empty? - task_info[:result] = nils_for_nulls(task.elements["result"].text.to_s.strip) - end - task_info[:orig_path] = nils_for_nulls(task.elements["path"].text.to_s.strip) - tmp = args[:ifd][:zip_tmp] - task_info[:orig_path].gsub!(/^\./,tmp) if task_info[:orig_path] - - # Only report a task if we actually have it. - # TODO: Copypasta. Separate this out. - if ::File.exist? task_info[:orig_path] - tasks_dir = ::File.join(basedir,"tasks") - task_file = ::File.split(task_info[:orig_path]).last - if ::File.exist? tasks_dir - unless (::File.directory?(tasks_dir) && ::File.writable?(tasks_dir)) - raise Msf::DBImportError.new("Could not move files to #{tasks_dir}") - end - else - ::FileUtils.mkdir_p(tasks_dir) + # Only report loot if we actually have it. + # TODO: Copypasta. Separate this out. + if ::File.exists? loot_info[:orig_path] + loot_dir = ::File.join(basedir,"loot") + loot_file = ::File.split(loot_info[:orig_path]).last + if ::File.exists? loot_dir + unless (::File.directory?(loot_dir) && ::File.writable?(loot_dir)) + raise Msf::DBImportError.new("Could not move files to #{loot_dir}") end - new_task = ::File.join(tasks_dir,task_file) - task_info[:path] = new_task - if ::File.exist?(new_task) - ::File.unlink new_task # Delete it, and don't report it. - else - report_task(task_info) # It's new, so report it. - end - ::FileUtils.copy(task_info[:orig_path], new_task) - yield(:msf_task, new_task) if block + else + ::FileUtils.mkdir_p(loot_dir) end + new_loot = ::File.join(loot_dir,loot_file) + loot_info[:path] = new_loot + if ::File.exists?(new_loot) + ::File.unlink new_loot # Delete it, and don't report it. + else + report_loot(loot_info) # It's new, so report it. + end + ::FileUtils.copy(loot_info[:orig_path], new_loot) + yield(:msf_loot, new_loot) if block end + end - # Import Reports - doc.elements.each("/#{btag}/reports/report") do |report| - import_report(report, args, basedir) + def parse_zip_task(task, wspace, bl, allow_yaml, btag, args, basedir, host_info, &block) + task_info = {} + task_info[:workspace] = args[:wspace] + return 0 unless task.at("path") + # Should user be imported (original) or declared (the importing user)? + task_info[:user] = nils_for_nulls(task.at("created-by").text.to_s.strip) + task_info[:desc] = nils_for_nulls(task.at("description").text.to_s.strip) + task_info[:info] = nils_for_nulls(unserialize_object(task.at("info"), allow_yaml)) + task_info[:mod] = nils_for_nulls(task.at("module").text.to_s.strip) + task_info[:options] = nils_for_nulls(task.at("options").text.to_s.strip) + task_info[:prog] = nils_for_nulls(task.at("progress").text.to_s.strip).to_i + task_info[:created_at] = nils_for_nulls(task.at("created-at").text.to_s.strip) + task_info[:updated_at] = nils_for_nulls(task.at("updated-at").text.to_s.strip) + if !task.at("completed-at").text.to_s.empty? + task_info[:completed_at] = nils_for_nulls(task.at("completed-at").text.to_s.strip) end + if !task.at("error").text.to_s.empty? + task_info[:error] = nils_for_nulls(task.at("error").text.to_s.strip) + end + if !task.at("result").text.to_s.empty? + task_info[:result] = nils_for_nulls(task.at("result").text.to_s.strip) + end + task_info[:orig_path] = nils_for_nulls(task.at("path").text.to_s.strip) + tmp = args[:ifd][:zip_tmp] + task_info[:orig_path].gsub!(/^\./,tmp) if task_info[:orig_path] + + # Only report a task if we actually have it. + # TODO: Copypasta. Separate this out. + if ::File.exists? task_info[:orig_path] + tasks_dir = ::File.join(basedir,"tasks") + task_file = ::File.split(task_info[:orig_path]).last + if ::File.exists? tasks_dir + unless (::File.directory?(tasks_dir) && ::File.writable?(tasks_dir)) + raise Msf::DBImportError.new("Could not move files to #{tasks_dir}") + end + else + ::FileUtils.mkdir_p(tasks_dir) + end + new_task = ::File.join(tasks_dir,task_file) + task_info[:path] = new_task + if ::File.exists?(new_task) + ::File.unlink new_task # Delete it, and don't report it. + else + report_task(task_info) # It's new, so report it. + end + ::FileUtils.copy(task_info[:orig_path], new_task) + yield(:msf_task, new_task) if block + end + end + + def parse_zip_report(report, wspace, bl, allow_yaml, btag, args, basedir, host_info, &block) + import_report(report, args, basedir) end # Import a Metasploit Express ZIP file. Note that this requires From 0e568674d73f5d1fbf0d3de73d44b400dc1f88fa Mon Sep 17 00:00:00 2001 From: dmohanty-r7 Date: Wed, 20 Apr 2016 16:12:04 -0500 Subject: [PATCH 07/12] Add comments on parse functions MS-255 --- lib/msf/core/db_manager/import/metasploit_framework/xml.rb | 3 ++- lib/msf/core/db_manager/import/metasploit_framework/zip.rb | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/msf/core/db_manager/import/metasploit_framework/xml.rb b/lib/msf/core/db_manager/import/metasploit_framework/xml.rb index f97ccfdb38..24fe1f9173 100644 --- a/lib/msf/core/db_manager/import/metasploit_framework/xml.rb +++ b/lib/msf/core/db_manager/import/metasploit_framework/xml.rb @@ -233,7 +233,6 @@ module Msf::DBManager::Import::MetasploitFramework::XML wspace = args[:wspace] || workspace bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - #doc = rexmlify(data) doc = Nokogiri::XML::Reader.from_memory(data) metadata = check_msf_xml_version!(doc.first.name) @@ -260,6 +259,7 @@ module Msf::DBManager::Import::MetasploitFramework::XML private + # Parses website Nokogiri::XML::Element def parse_web_site(web, wspace, bl, allow_yaml, btag, args, &block) # Import web sites info = {} @@ -284,6 +284,7 @@ module Msf::DBManager::Import::MetasploitFramework::XML yield(:web_site, "#{info[:host]}:#{info[:port]} (#{info[:vhost]})") if block end + # Parses host Nokogiri::XML::Element def parse_host(host, wspace, bl, allow_yaml, btag, args, &block) host_data = {} diff --git a/lib/msf/core/db_manager/import/metasploit_framework/zip.rb b/lib/msf/core/db_manager/import/metasploit_framework/zip.rb index c121a07ca2..8eb11b7293 100644 --- a/lib/msf/core/db_manager/import/metasploit_framework/zip.rb +++ b/lib/msf/core/db_manager/import/metasploit_framework/zip.rb @@ -47,12 +47,14 @@ module Msf::DBManager::Import::MetasploitFramework::Zip end end + # Parses host Nokogiri::XML::Element def parse_zip_host(host, wspace, bl, allow_yaml, btag, args, basedir, host_info, &block) if host.at("id") host_info[host.at("id").text.to_s.strip] = nils_for_nulls(host.at("address").text.to_s.strip) end end + # Parses loot Nokogiri::XML::Element def parse_zip_loot(loot, wspace, bl, allow_yaml, btag, args, basedir, host_info, &block) return 0 if loot.at("host-id").nil? || bl.include?(host_info[loot.at("host-id").text.to_s.strip]) loot_info = {} @@ -99,6 +101,7 @@ module Msf::DBManager::Import::MetasploitFramework::Zip end end + # Parses task Nokogiri::XML::Element def parse_zip_task(task, wspace, bl, allow_yaml, btag, args, basedir, host_info, &block) task_info = {} task_info[:workspace] = args[:wspace] @@ -149,6 +152,7 @@ module Msf::DBManager::Import::MetasploitFramework::Zip end end + # Parses report Nokogiri::XML::Element def parse_zip_report(report, wspace, bl, allow_yaml, btag, args, basedir, host_info, &block) import_report(report, args, basedir) end From 050061762b71a7f0554fa1eb23e979a1a2fd207c Mon Sep 17 00:00:00 2001 From: dmohanty-r7 Date: Mon, 25 Apr 2016 23:37:03 -0500 Subject: [PATCH 08/12] Fix db_manager rspec tests MS-255 --- .../import/metasploit_framework/xml.rb | 13 +++++++----- .../import/metasploit_framework/xml.rb | 21 ++++++++++--------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/lib/msf/core/db_manager/import/metasploit_framework/xml.rb b/lib/msf/core/db_manager/import/metasploit_framework/xml.rb index 24fe1f9173..b5dbef4a62 100644 --- a/lib/msf/core/db_manager/import/metasploit_framework/xml.rb +++ b/lib/msf/core/db_manager/import/metasploit_framework/xml.rb @@ -241,11 +241,12 @@ module Msf::DBManager::Import::MetasploitFramework::XML doc.each do |node| case node.name - when 'host' - parse_host(Nokogiri::XML(node.outer_xml).at("./#{node.name}"), wspace, bl, allow_yaml, btag, args, &block) - when 'web_site' - parse_web_site(Nokogiri::XML(node.outer_xml).at("./#{node.name}"), wspace, bl, allow_yaml, btag, args, &block) - when 'web_page', 'web_form', 'web_vuln' + when 'host' + parse_host(Nokogiri::XML(node.outer_xml).at("./#{node.name}"), wspace, bl, allow_yaml, btag, args, &block) + when 'web_site' + parse_web_site(Nokogiri::XML(node.outer_xml).at("./#{node.name}"), wspace, bl, allow_yaml, btag, args, &block) + when 'web_page', 'web_form', 'web_vuln' + unless node.inner_xml.empty? send( "import_msf_#{node.name}_element", Nokogiri::XML(node.outer_xml).at("./#{node.name}"), @@ -253,6 +254,7 @@ module Msf::DBManager::Import::MetasploitFramework::XML :workspace => wspace, &block ) + end end end end @@ -625,6 +627,7 @@ module Msf::DBManager::Import::MetasploitFramework::XML info[:ssl] = (info[:ssl] and info[:ssl].to_s.strip.downcase == "true") ? true : false specialized_info = specialization.call(element, options) + info.merge!(specialized_info) self.send("report_web_#{type}", info) diff --git a/spec/support/shared/examples/msf/db_manager/import/metasploit_framework/xml.rb b/spec/support/shared/examples/msf/db_manager/import/metasploit_framework/xml.rb index 7c320dcfd8..ad5236f5f8 100644 --- a/spec/support/shared/examples/msf/db_manager/import/metasploit_framework/xml.rb +++ b/spec/support/shared/examples/msf/db_manager/import/metasploit_framework/xml.rb @@ -27,7 +27,7 @@ RSpec.shared_examples_for 'Msf::DBManager::Import::MetasploitFramework::XML' do end let(:document) do - REXML::Document.new(source) + Nokogiri::XML::Reader.from_memory(source) end let(:element) do @@ -132,7 +132,7 @@ RSpec.shared_examples_for 'Msf::DBManager::Import::MetasploitFramework::XML' do end subject(:metadata) do - db_manager.send(:check_msf_xml_version!, document) + db_manager.send(:check_msf_xml_version!, Nokogiri::XML(document.source).elements.first.name) end it_should_behave_like( @@ -175,7 +175,7 @@ RSpec.shared_examples_for 'Msf::DBManager::Import::MetasploitFramework::XML' do context '#import_msf_text_element' do let(:parent_element) do - document.root + Nokogiri::XML(document.source).elements.first end let(:child_name) do @@ -252,7 +252,7 @@ RSpec.shared_examples_for 'Msf::DBManager::Import::MetasploitFramework::XML' do context 'import_msf_web_element' do let(:element) do - document.root + Nokogiri::XML(document.source).elements.first end let(:options) do @@ -575,11 +575,12 @@ RSpec.shared_examples_for 'Msf::DBManager::Import::MetasploitFramework::XML' do end context 'call to #import_msf_web_element' do + it_should_behave_like 'Msf::DBManager::Import::MetasploitFramework::XML#import_msf_web_element specialization' context 'specialization return' do let(:element) do - document.root + Nokogiri::XML(document.source).elements.first end let(:source) do @@ -619,7 +620,7 @@ RSpec.shared_examples_for 'Msf::DBManager::Import::MetasploitFramework::XML' do context 'with required attributes' do let(:element) do - document.root + Nokogiri::XML(document.source).elements.first end let(:source) do @@ -675,7 +676,7 @@ RSpec.shared_examples_for 'Msf::DBManager::Import::MetasploitFramework::XML' do context 'specialization return' do let(:element) do - document.root + Nokogiri::XML(document.source).elements.first end let(:source) do @@ -774,7 +775,7 @@ RSpec.shared_examples_for 'Msf::DBManager::Import::MetasploitFramework::XML' do context 'with required attributes' do let(:element) do - document.root + Nokogiri::XML(document.source).elements.first end let(:source) do @@ -846,7 +847,7 @@ RSpec.shared_examples_for 'Msf::DBManager::Import::MetasploitFramework::XML' do context 'specialization return' do let(:element) do - document.root + Nokogiri::XML(document.source).elements.first end let(:source) do @@ -949,7 +950,7 @@ RSpec.shared_examples_for 'Msf::DBManager::Import::MetasploitFramework::XML' do context 'with required attributes' do let(:element) do - document.root + Nokogiri::XML(document.source).elements.first end let(:source) do From 56fd5a745efe88e4449f79fc8fc3878822ed7cb4 Mon Sep 17 00:00:00 2001 From: dmohanty-r7 Date: Tue, 26 Apr 2016 11:05:27 -0500 Subject: [PATCH 09/12] Do not parse element if empty MS-255 --- .../import/metasploit_framework/xml.rb | 3 - .../import/metasploit_framework/zip.rb | 63 +++++++++---------- 2 files changed, 30 insertions(+), 36 deletions(-) diff --git a/lib/msf/core/db_manager/import/metasploit_framework/xml.rb b/lib/msf/core/db_manager/import/metasploit_framework/xml.rb index b5dbef4a62..f692ecfb7a 100644 --- a/lib/msf/core/db_manager/import/metasploit_framework/xml.rb +++ b/lib/msf/core/db_manager/import/metasploit_framework/xml.rb @@ -228,13 +228,11 @@ module Msf::DBManager::Import::MetasploitFramework::XML # them. # TODO: loot, tasks, and reports def import_msf_xml(args={}, &block) - data = args[:data] wspace = args[:wspace] || workspace bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] doc = Nokogiri::XML::Reader.from_memory(data) - metadata = check_msf_xml_version!(doc.first.name) allow_yaml = metadata[:allow_yaml] btag = metadata[:root_tag] @@ -627,7 +625,6 @@ module Msf::DBManager::Import::MetasploitFramework::XML info[:ssl] = (info[:ssl] and info[:ssl].to_s.strip.downcase == "true") ? true : false specialized_info = specialization.call(element, options) - info.merge!(specialized_info) self.send("report_web_#{type}", info) diff --git a/lib/msf/core/db_manager/import/metasploit_framework/zip.rb b/lib/msf/core/db_manager/import/metasploit_framework/zip.rb index 8eb11b7293..09167eb0bd 100644 --- a/lib/msf/core/db_manager/import/metasploit_framework/zip.rb +++ b/lib/msf/core/db_manager/import/metasploit_framework/zip.rb @@ -11,27 +11,26 @@ module Msf::DBManager::Import::MetasploitFramework::Zip btag = nil doc = Nokogiri::XML::Reader.from_memory(data) - case doc.first.name - when "MetasploitExpressV1" - m_ver = 1 - allow_yaml = true - btag = "MetasploitExpressV1" - when "MetasploitExpressV2" - m_ver = 2 - allow_yaml = true - btag = "MetasploitExpressV2" - when "MetasploitExpressV3" - m_ver = 3 - btag = "MetasploitExpressV3" - when "MetasploitExpressV4" - m_ver = 4 - btag = "MetasploitExpressV4" - when "MetasploitV4" - m_ver = 4 - btag = "MetasploitV4" - else - m_ver = nil + when "MetasploitExpressV1" + m_ver = 1 + allow_yaml = true + btag = "MetasploitExpressV1" + when "MetasploitExpressV2" + m_ver = 2 + allow_yaml = true + btag = "MetasploitExpressV2" + when "MetasploitExpressV3" + m_ver = 3 + btag = "MetasploitExpressV3" + when "MetasploitExpressV4" + m_ver = 4 + btag = "MetasploitExpressV4" + when "MetasploitV4" + m_ver = 4 + btag = "MetasploitV4" + else + m_ver = nil end unless m_ver and btag raise Msf::DBImportError.new("Unsupported Metasploit XML document format") @@ -40,23 +39,22 @@ module Msf::DBManager::Import::MetasploitFramework::Zip host_info = {} doc.each do |node| - case node.name - when 'host', 'loot', 'task', 'report' + if ['host', 'loot', 'task', 'report'].include?(node.name) + unless node.inner_xml.empty? send("parse_zip_#{node.name}", Nokogiri::XML(node.outer_xml).at("./#{node.name}"), wspace, bl, allow_yaml, btag, args, basedir, host_info, &block) + end end end end # Parses host Nokogiri::XML::Element def parse_zip_host(host, wspace, bl, allow_yaml, btag, args, basedir, host_info, &block) - if host.at("id") - host_info[host.at("id").text.to_s.strip] = nils_for_nulls(host.at("address").text.to_s.strip) - end + host_info[host.at("id").text.to_s.strip] = nils_for_nulls(host.at("address").text.to_s.strip) end # Parses loot Nokogiri::XML::Element def parse_zip_loot(loot, wspace, bl, allow_yaml, btag, args, basedir, host_info, &block) - return 0 if loot.at("host-id").nil? || bl.include?(host_info[loot.at("host-id").text.to_s.strip]) + return 0 if bl.include? host_info[loot.at("host-id").text.to_s.strip] loot_info = {} loot_info[:host] = host_info[loot.at("host-id").text.to_s.strip] loot_info[:workspace] = args[:wspace] @@ -79,10 +77,10 @@ module Msf::DBManager::Import::MetasploitFramework::Zip # Only report loot if we actually have it. # TODO: Copypasta. Separate this out. - if ::File.exists? loot_info[:orig_path] + if ::File.exist? loot_info[:orig_path] loot_dir = ::File.join(basedir,"loot") loot_file = ::File.split(loot_info[:orig_path]).last - if ::File.exists? loot_dir + if ::File.exist? loot_dir unless (::File.directory?(loot_dir) && ::File.writable?(loot_dir)) raise Msf::DBImportError.new("Could not move files to #{loot_dir}") end @@ -91,7 +89,7 @@ module Msf::DBManager::Import::MetasploitFramework::Zip end new_loot = ::File.join(loot_dir,loot_file) loot_info[:path] = new_loot - if ::File.exists?(new_loot) + if ::File.exist?(new_loot) ::File.unlink new_loot # Delete it, and don't report it. else report_loot(loot_info) # It's new, so report it. @@ -105,7 +103,6 @@ module Msf::DBManager::Import::MetasploitFramework::Zip def parse_zip_task(task, wspace, bl, allow_yaml, btag, args, basedir, host_info, &block) task_info = {} task_info[:workspace] = args[:wspace] - return 0 unless task.at("path") # Should user be imported (original) or declared (the importing user)? task_info[:user] = nils_for_nulls(task.at("created-by").text.to_s.strip) task_info[:desc] = nils_for_nulls(task.at("description").text.to_s.strip) @@ -130,10 +127,10 @@ module Msf::DBManager::Import::MetasploitFramework::Zip # Only report a task if we actually have it. # TODO: Copypasta. Separate this out. - if ::File.exists? task_info[:orig_path] + if ::File.exist? task_info[:orig_path] tasks_dir = ::File.join(basedir,"tasks") task_file = ::File.split(task_info[:orig_path]).last - if ::File.exists? tasks_dir + if ::File.exist? tasks_dir unless (::File.directory?(tasks_dir) && ::File.writable?(tasks_dir)) raise Msf::DBImportError.new("Could not move files to #{tasks_dir}") end @@ -142,7 +139,7 @@ module Msf::DBManager::Import::MetasploitFramework::Zip end new_task = ::File.join(tasks_dir,task_file) task_info[:path] = new_task - if ::File.exists?(new_task) + if ::File.exist?(new_task) ::File.unlink new_task # Delete it, and don't report it. else report_task(task_info) # It's new, so report it. From f4f607d81589cd2b3d2578900576d0e0579c3f28 Mon Sep 17 00:00:00 2001 From: dmohanty-r7 Date: Tue, 26 Apr 2016 11:16:22 -0500 Subject: [PATCH 10/12] Correct comments to use Nokogiri::XML::Element MS-255 --- .../import/metasploit_framework/xml.rb | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/msf/core/db_manager/import/metasploit_framework/xml.rb b/lib/msf/core/db_manager/import/metasploit_framework/xml.rb index f692ecfb7a..20255bc867 100644 --- a/lib/msf/core/db_manager/import/metasploit_framework/xml.rb +++ b/lib/msf/core/db_manager/import/metasploit_framework/xml.rb @@ -66,7 +66,7 @@ module Msf::DBManager::Import::MetasploitFramework::XML # Imports `Mdm::Note` objects from the XML element. # - # @param note [REXML::Element] The Note element + # @param note [Nokogiri::XML::Element] The Note element # @param allow_yaml [Boolean] whether to allow yaml # @param note_data [Hash] hash containing note attributes to be passed along # @return [void] @@ -90,7 +90,7 @@ module Msf::DBManager::Import::MetasploitFramework::XML # Imports web_form element using Msf::DBManager#report_web_form. # - # @param element [REXML::Element] web_form element. + # @param element [Nokogiri::XML::Element] web_form element. # @param options [Hash{Symbol => Object}] options # @option options [Boolean] :allow_yaml (false) Whether to allow YAML when # deserializing params. @@ -126,7 +126,7 @@ module Msf::DBManager::Import::MetasploitFramework::XML # Imports web_page element using Msf::DBManager#report_web_page. # - # @param element [REXML::Element] web_page element. + # @param element [Nokogiri::XML::Element] web_page element. # @param options [Hash{Symbol => Object}] options # @option options [Boolean] :allow_yaml (false) Whether to allow YAML when # deserializing headers. @@ -173,7 +173,7 @@ module Msf::DBManager::Import::MetasploitFramework::XML # Imports web_vuln element using Msf::DBManager#report_web_vuln. # - # @param element [REXML::Element] web_vuln element. + # @param element [Nokogiri::XML::Element] web_vuln element. # @param options [Hash{Symbol => Object}] options # @option options [Boolean] :allow_yaml (false) Whether to allow YAML when # deserializing headers. @@ -520,8 +520,8 @@ module Msf::DBManager::Import::MetasploitFramework::XML # Checks if the XML document has a format version that the importer # understands. # - # @param document [REXML::Document] a REXML::Document produced by - # {Msf::DBManager#rexmlify}. + # @param name [String] the root node name produced by + # {Nokogiri::XML::Reader#from_memory}. # @return [Hash{Symbol => Object}] `:allow_yaml` is true if the format # requires YAML loading when calling # {Msf::DBManager#unserialize_object}. `:root_tag` the tag name of the @@ -564,7 +564,7 @@ module Msf::DBManager::Import::MetasploitFramework::XML # Retrieves text of element if it exists. # - # @param parent_element [REXML::Element] element under which element with + # @param parent_element [Nokogiri::XML::Element] element under which element with # `child_name` exists. # @param child_name [String] the name of the element under # `parent_element` whose text should be returned @@ -591,7 +591,7 @@ module Msf::DBManager::Import::MetasploitFramework::XML # Msf::DBManager#report_web_form, Msf::DBManager#report_web_page, and # Msf::DBManager#report_web_vuln, respectively. # - # @param element [REXML::Element] the web_form, web_page, or web_vuln + # @param element [Nokogiri::XML::Element] the web_form, web_page, or web_vuln # element. # @param options [Hash{Symbol => Object}] options # @option options [Boolean] :allow_yaml (false) Whether to allow YAML when @@ -604,7 +604,7 @@ module Msf::DBManager::Import::MetasploitFramework::XML # (Msf::DBManager#workspace) workspace under which to report the # imported record. # @yield [element, options] - # @yieldparam element [REXML::Element] the web_form, web_page, or + # @yieldparam element [Nokogiri::XML::Element] the web_form, web_page, or # web_vuln element passed to {#import_msf_web_element}. # @yieldparam options [Hash{Symbol => Object}] options for parsing # @yieldreturn [Hash{Symbol => Object}] info From 5a4e70fdf0c4a7f468e627e9f5035ca5401f84a1 Mon Sep 17 00:00:00 2001 From: dmohanty-r7 Date: Tue, 26 Apr 2016 11:20:51 -0500 Subject: [PATCH 11/12] Fixes indentation in check_msf_xml_version! MS-255 --- .../import/metasploit_framework/xml.rb | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/lib/msf/core/db_manager/import/metasploit_framework/xml.rb b/lib/msf/core/db_manager/import/metasploit_framework/xml.rb index 20255bc867..ae6b5ae0f6 100644 --- a/lib/msf/core/db_manager/import/metasploit_framework/xml.rb +++ b/lib/msf/core/db_manager/import/metasploit_framework/xml.rb @@ -536,22 +536,22 @@ module Msf::DBManager::Import::MetasploitFramework::XML } case name - when 'MetasploitExpressV1' - # FIXME https://www.pivotaltracker.com/story/show/47128407 - metadata[:allow_yaml] = true - metadata[:root_tag] = 'MetasploitExpressV1' - when 'MetasploitExpressV2' - # FIXME https://www.pivotaltracker.com/story/show/47128407 - metadata[:allow_yaml] = true - metadata[:root_tag] = 'MetasploitExpressV2' - when 'MetasploitExpressV3' - metadata[:root_tag] = 'MetasploitExpressV3' - when 'MetasploitExpressV4' - metadata[:root_tag] = 'MetasploitExpressV4' - when 'MetasploitV4' - metadata[:root_tag] = 'MetasploitV4' - when 'MetasploitV5' - metadata[:root_tag] = 'MetasploitV5' + when 'MetasploitExpressV1' + # FIXME https://www.pivotaltracker.com/story/show/47128407 + metadata[:allow_yaml] = true + metadata[:root_tag] = 'MetasploitExpressV1' + when 'MetasploitExpressV2' + # FIXME https://www.pivotaltracker.com/story/show/47128407 + metadata[:allow_yaml] = true + metadata[:root_tag] = 'MetasploitExpressV2' + when 'MetasploitExpressV3' + metadata[:root_tag] = 'MetasploitExpressV3' + when 'MetasploitExpressV4' + metadata[:root_tag] = 'MetasploitExpressV4' + when 'MetasploitV4' + metadata[:root_tag] = 'MetasploitV4' + when 'MetasploitV5' + metadata[:root_tag] = 'MetasploitV5' end unless metadata[:root_tag] From 20ec56d06ad218b90c15b0ec0949114986d02c34 Mon Sep 17 00:00:00 2001 From: dmohanty-r7 Date: Thu, 28 Apr 2016 13:16:10 -0500 Subject: [PATCH 12/12] Do not parse empty web_sites MS-255 --- .../db_manager/import/metasploit_framework/xml.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/msf/core/db_manager/import/metasploit_framework/xml.rb b/lib/msf/core/db_manager/import/metasploit_framework/xml.rb index ae6b5ae0f6..428cc77c0e 100644 --- a/lib/msf/core/db_manager/import/metasploit_framework/xml.rb +++ b/lib/msf/core/db_manager/import/metasploit_framework/xml.rb @@ -238,13 +238,13 @@ module Msf::DBManager::Import::MetasploitFramework::XML btag = metadata[:root_tag] doc.each do |node| - case node.name - when 'host' - parse_host(Nokogiri::XML(node.outer_xml).at("./#{node.name}"), wspace, bl, allow_yaml, btag, args, &block) - when 'web_site' - parse_web_site(Nokogiri::XML(node.outer_xml).at("./#{node.name}"), wspace, bl, allow_yaml, btag, args, &block) - when 'web_page', 'web_form', 'web_vuln' - unless node.inner_xml.empty? + unless node.inner_xml.empty? + case node.name + when 'host' + parse_host(Nokogiri::XML(node.outer_xml).at("./#{node.name}"), wspace, bl, allow_yaml, btag, args, &block) + when 'web_site' + parse_web_site(Nokogiri::XML(node.outer_xml).at("./#{node.name}"), wspace, bl, allow_yaml, btag, args, &block) + when 'web_page', 'web_form', 'web_vuln' send( "import_msf_#{node.name}_element", Nokogiri::XML(node.outer_xml).at("./#{node.name}"),