From 6e6780da86dae2137b9bed2a55fa50c7a8e4f715 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 8 Oct 2014 11:37:59 -0500 Subject: [PATCH 001/159] Split Msf::HostState into own file MSP-11124 --- lib/msf/core/db.rb | 21 +-------------------- lib/msf/core/host_state.rb | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 20 deletions(-) create mode 100644 lib/msf/core/host_state.rb diff --git a/lib/msf/core/db.rb b/lib/msf/core/db.rb index 1e27e3f967..c48625ad63 100644 --- a/lib/msf/core/db.rb +++ b/lib/msf/core/db.rb @@ -57,29 +57,10 @@ require 'rex/parser/retina_xml' require 'metasploit/framework/require' require 'msf/core/db_manager/import_msf_xml' +require 'msf/core/host_state' module Msf -### -# -# The states that a host can be in. -# -### -module HostState - # - # The host is alive. - # - Alive = "alive" - # - # The host is dead. - # - Dead = "down" - # - # The host state is unknown. - # - Unknown = "unknown" -end - ### # # The states that a service can be in. diff --git a/lib/msf/core/host_state.rb b/lib/msf/core/host_state.rb new file mode 100644 index 0000000000..2267dfcc03 --- /dev/null +++ b/lib/msf/core/host_state.rb @@ -0,0 +1,21 @@ +module Msf +### +# +# The states that a host can be in. +# +### +module HostState + # + # The host is alive. + # + Alive = "alive" + # + # The host is dead. + # + Dead = "down" + # + # The host state is unknown. + # + Unknown = "unknown" +end +end From 5ecd194a0d6866d44eb018ce914df07855359318 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 8 Oct 2014 11:43:28 -0500 Subject: [PATCH 002/159] Fix indent in Msf::HostState MSP-11124 --- lib/msf/core/host_state.rb | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/msf/core/host_state.rb b/lib/msf/core/host_state.rb index 2267dfcc03..31b3b6eaa4 100644 --- a/lib/msf/core/host_state.rb +++ b/lib/msf/core/host_state.rb @@ -1,10 +1,8 @@ -module Msf ### -# # The states that a host can be in. # ### -module HostState +module Msf::HostState # # The host is alive. # @@ -18,4 +16,3 @@ module HostState # Unknown = "unknown" end -end From 0708ac136179a10e74db33d5201bf1816b7d964b Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 8 Oct 2014 11:47:04 -0500 Subject: [PATCH 003/159] Fix comment style in Msf::HostState MSP-11124 --- lib/msf/core/host_state.rb | 9 --------- 1 file changed, 9 deletions(-) diff --git a/lib/msf/core/host_state.rb b/lib/msf/core/host_state.rb index 31b3b6eaa4..94527dfd9a 100644 --- a/lib/msf/core/host_state.rb +++ b/lib/msf/core/host_state.rb @@ -1,18 +1,9 @@ -### # The states that a host can be in. -# -### module Msf::HostState - # # The host is alive. - # Alive = "alive" - # # The host is dead. - # Dead = "down" - # # The host state is unknown. - # Unknown = "unknown" end From 57d9dc306c5e69bd41c7079410cff83cc9156449 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 8 Oct 2014 13:45:15 -0500 Subject: [PATCH 004/159] Extract Msf::ServiceState MSP-11124 Extract Msf::ServiceState from `lib/msf/core/db.rb` and put it into `lib/msf/core/service_state.rb`. --- lib/msf/core/db.rb | 13 +------------ lib/msf/core/service_state.rb | 13 +++++++++++++ 2 files changed, 14 insertions(+), 12 deletions(-) create mode 100644 lib/msf/core/service_state.rb diff --git a/lib/msf/core/db.rb b/lib/msf/core/db.rb index c48625ad63..daf0212d51 100644 --- a/lib/msf/core/db.rb +++ b/lib/msf/core/db.rb @@ -58,21 +58,10 @@ require 'rex/parser/retina_xml' require 'metasploit/framework/require' require 'msf/core/db_manager/import_msf_xml' require 'msf/core/host_state' +require 'msf/core/service_state' module Msf -### -# -# The states that a service can be in. -# -### -module ServiceState - Open = "open" - Closed = "closed" - Filtered = "filtered" - Unknown = "unknown" -end - ### # # Events that can occur in the host/service database. diff --git a/lib/msf/core/service_state.rb b/lib/msf/core/service_state.rb new file mode 100644 index 0000000000..3f3e0bc0d4 --- /dev/null +++ b/lib/msf/core/service_state.rb @@ -0,0 +1,13 @@ +module Msf +### +# +# The states that a service can be in. +# +### +module ServiceState + Open = "open" + Closed = "closed" + Filtered = "filtered" + Unknown = "unknown" +end +end From 46156fbbc669f5c549f71412ed564fa14db0601c Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 8 Oct 2014 13:50:26 -0500 Subject: [PATCH 005/159] Fix indentation in Msf::ServiceState MSP-11124 --- lib/msf/core/service_state.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/msf/core/service_state.rb b/lib/msf/core/service_state.rb index 3f3e0bc0d4..1cea78cff9 100644 --- a/lib/msf/core/service_state.rb +++ b/lib/msf/core/service_state.rb @@ -1,13 +1,11 @@ -module Msf ### # # The states that a service can be in. # ### -module ServiceState +module Msf::ServiceState Open = "open" Closed = "closed" Filtered = "filtered" Unknown = "unknown" end -end From 6b3d70ce00edb747093bdf739240ab4690da04e8 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 8 Oct 2014 13:52:42 -0500 Subject: [PATCH 006/159] Fix code style in Msf::ServiceState MSP-11124 --- lib/msf/core/service_state.rb | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/msf/core/service_state.rb b/lib/msf/core/service_state.rb index 1cea78cff9..41c398d6db 100644 --- a/lib/msf/core/service_state.rb +++ b/lib/msf/core/service_state.rb @@ -1,11 +1,7 @@ -### -# # The states that a service can be in. -# -### module Msf::ServiceState - Open = "open" Closed = "closed" Filtered = "filtered" + Open = "open" Unknown = "unknown" end From 2206a863879925f2b35150d7747895a65c9ad613 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 8 Oct 2014 14:01:58 -0500 Subject: [PATCH 007/159] Extract Msf::DatabaseEvent MSP-11124 Extract `Msf::DatabaseEvent` from `lib/msf/core/db.rb` into a more conventional `lib/msf/core/database_event.rb`. --- lib/msf/core/database_event.rb | 57 ++++++++++++++++++++++++++++++++++ lib/msf/core/db.rb | 56 +-------------------------------- 2 files changed, 58 insertions(+), 55 deletions(-) create mode 100644 lib/msf/core/database_event.rb diff --git a/lib/msf/core/database_event.rb b/lib/msf/core/database_event.rb new file mode 100644 index 0000000000..81075a0616 --- /dev/null +++ b/lib/msf/core/database_event.rb @@ -0,0 +1,57 @@ +module Msf + +### +# +# Events that can occur in the host/service database. +# +### +module DatabaseEvent + + # + # Called when an existing host's state changes + # + def on_db_host_state(host, ostate) + end + + # + # Called when an existing service's state changes + # + def on_db_service_state(host, port, ostate) + end + + # + # Called when a new host is added to the database. The host parameter is + # of type Host. + # + def on_db_host(host) + end + + # + # Called when a new client is added to the database. The client + # parameter is of type Client. + # + def on_db_client(client) + end + + # + # Called when a new service is added to the database. The service + # parameter is of type Service. + # + def on_db_service(service) + end + + # + # Called when an applicable vulnerability is found for a service. The vuln + # parameter is of type Vuln. + # + def on_db_vuln(vuln) + end + + # + # Called when a new reference is created. + # + def on_db_ref(ref) + end + +end +end diff --git a/lib/msf/core/db.rb b/lib/msf/core/db.rb index daf0212d51..c2ea990575 100644 --- a/lib/msf/core/db.rb +++ b/lib/msf/core/db.rb @@ -56,67 +56,13 @@ require 'rex/parser/retina_xml' # require 'metasploit/framework/require' +require 'msf/core/database_event' require 'msf/core/db_manager/import_msf_xml' require 'msf/core/host_state' require 'msf/core/service_state' module Msf -### -# -# Events that can occur in the host/service database. -# -### -module DatabaseEvent - - # - # Called when an existing host's state changes - # - def on_db_host_state(host, ostate) - end - - # - # Called when an existing service's state changes - # - def on_db_service_state(host, port, ostate) - end - - # - # Called when a new host is added to the database. The host parameter is - # of type Host. - # - def on_db_host(host) - end - - # - # Called when a new client is added to the database. The client - # parameter is of type Client. - # - def on_db_client(client) - end - - # - # Called when a new service is added to the database. The service - # parameter is of type Service. - # - def on_db_service(service) - end - - # - # Called when an applicable vulnerability is found for a service. The vuln - # parameter is of type Vuln. - # - def on_db_vuln(vuln) - end - - # - # Called when a new reference is created. - # - def on_db_ref(ref) - end - -end - class DBImportError < RuntimeError end From 68245159490504faf41f4ad10fa2f3da06620efc Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 8 Oct 2014 14:04:21 -0500 Subject: [PATCH 008/159] Fix indentation and whitespace in Msf::DatabaseEvent MSP-11124 --- lib/msf/core/database_event.rb | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/lib/msf/core/database_event.rb b/lib/msf/core/database_event.rb index 81075a0616..700fcafecc 100644 --- a/lib/msf/core/database_event.rb +++ b/lib/msf/core/database_event.rb @@ -1,12 +1,9 @@ -module Msf - ### # # Events that can occur in the host/service database. # ### -module DatabaseEvent - +module Msf::DatabaseEvent # # Called when an existing host's state changes # @@ -52,6 +49,4 @@ module DatabaseEvent # def on_db_ref(ref) end - -end end From 7a5ce1973592eb846fa0b386d6a0301de2f63143 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 8 Oct 2014 14:07:05 -0500 Subject: [PATCH 009/159] Fix code style MSP-11124 Fix comment style and order methods. --- lib/msf/core/database_event.rb | 54 ++++++++++++---------------------- 1 file changed, 18 insertions(+), 36 deletions(-) diff --git a/lib/msf/core/database_event.rb b/lib/msf/core/database_event.rb index 700fcafecc..148a5dbd9e 100644 --- a/lib/msf/core/database_event.rb +++ b/lib/msf/core/database_event.rb @@ -1,52 +1,34 @@ -### -# # Events that can occur in the host/service database. -# -### module Msf::DatabaseEvent - # - # Called when an existing host's state changes - # - def on_db_host_state(host, ostate) - end - - # - # Called when an existing service's state changes - # - def on_db_service_state(host, port, ostate) - end - - # - # Called when a new host is added to the database. The host parameter is - # of type Host. - # - def on_db_host(host) - end - - # # Called when a new client is added to the database. The client # parameter is of type Client. - # def on_db_client(client) end - # + # Called when a new host is added to the database. The host parameter is + # of type Host. + def on_db_host(host) + end + + # Called when an existing host's state changes + def on_db_host_state(host, ostate) + end + + # Called when a new reference is created. + def on_db_ref(ref) + end + # Called when a new service is added to the database. The service # parameter is of type Service. - # def on_db_service(service) end - # - # Called when an applicable vulnerability is found for a service. The vuln - # parameter is of type Vuln. - # - def on_db_vuln(vuln) + # Called when an existing service's state changes + def on_db_service_state(host, port, ostate) end - # - # Called when a new reference is created. - # - def on_db_ref(ref) + # Called when an applicable vulnerability is found for a service. The vuln + # parameter is of type Vuln. + def on_db_vuln(vuln) end end From cffc74d57183e64f612ed9205ad99bed102ca55f Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 8 Oct 2014 14:13:26 -0500 Subject: [PATCH 010/159] Extract Msf::DBImportError MSP-11124 --- lib/msf/core/db.rb | 4 +--- lib/msf/core/db_import_error.rb | 2 ++ 2 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 lib/msf/core/db_import_error.rb diff --git a/lib/msf/core/db.rb b/lib/msf/core/db.rb index c2ea990575..ed5a1aa2fb 100644 --- a/lib/msf/core/db.rb +++ b/lib/msf/core/db.rb @@ -57,15 +57,13 @@ require 'rex/parser/retina_xml' require 'metasploit/framework/require' require 'msf/core/database_event' +require 'msf/core/db_import_error' require 'msf/core/db_manager/import_msf_xml' require 'msf/core/host_state' require 'msf/core/service_state' module Msf -class DBImportError < RuntimeError -end - ### # # The DB module ActiveRecord definitions for the DBManager diff --git a/lib/msf/core/db_import_error.rb b/lib/msf/core/db_import_error.rb new file mode 100644 index 0000000000..bd65bf902e --- /dev/null +++ b/lib/msf/core/db_import_error.rb @@ -0,0 +1,2 @@ +class Msf::DBImportError < RuntimeError +end From 15f94612794547a999edc79300ad30a38dfcb66b Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 8 Oct 2014 14:27:22 -0500 Subject: [PATCH 011/159] Merge db.rb into db_manager.rb MSP-11124 The class name is DBManager, so the correct file name is db_manager.rb --- lib/msf/core/db.rb | 6313 ------------------------------------ lib/msf/core/db_manager.rb | 6304 ++++++++++++++++++++++++++++++++++- 2 files changed, 6299 insertions(+), 6318 deletions(-) delete mode 100644 lib/msf/core/db.rb diff --git a/lib/msf/core/db.rb b/lib/msf/core/db.rb deleted file mode 100644 index ed5a1aa2fb..0000000000 --- a/lib/msf/core/db.rb +++ /dev/null @@ -1,6313 +0,0 @@ -# -*- coding: binary -*- - -# -# Standard Library -# - -require 'csv' -require 'tmpdir' -require 'uri' - -# -# -# Gems -# -# - -# -# PacketFu -# - -require 'packetfu' - -# -# Rex -# - - -require 'rex/socket' - -# Check Rex::Parser.nokogiri_loaded for status of the Nokogiri parsers -require 'rex/parser/acunetix_nokogiri' -require 'rex/parser/appscan_nokogiri' -require 'rex/parser/burp_session_nokogiri' -require 'rex/parser/ci_nokogiri' -require 'rex/parser/foundstone_nokogiri' -require 'rex/parser/fusionvm_nokogiri' -require 'rex/parser/mbsa_nokogiri' -require 'rex/parser/nexpose_raw_nokogiri' -require 'rex/parser/nexpose_simple_nokogiri' -require 'rex/parser/nmap_nokogiri' -require 'rex/parser/openvas_nokogiri' -require 'rex/parser/wapiti_nokogiri' -require 'rex/parser/outpost24_nokogiri' - -# Legacy XML parsers -- these will be converted some day -require 'rex/parser/ip360_aspl_xml' -require 'rex/parser/ip360_xml' -require 'rex/parser/nessus_xml' -require 'rex/parser/netsparker_xml' -require 'rex/parser/nexpose_xml' -require 'rex/parser/nmap_xml' -require 'rex/parser/retina_xml' - -# -# Project -# - -require 'metasploit/framework/require' -require 'msf/core/database_event' -require 'msf/core/db_import_error' -require 'msf/core/db_manager/import_msf_xml' -require 'msf/core/host_state' -require 'msf/core/service_state' - -module Msf - -### -# -# The DB module ActiveRecord definitions for the DBManager -# -### -class DBManager - extend Metasploit::Framework::Require - - include Msf::DBManager::ImportMsfXml - optionally_include_metasploit_credential_creation - - def rfc3330_reserved(ip) - case ip.class.to_s - when "PacketFu::Octets" - ip_x = ip.to_x - ip_i = ip.to_i - when "String" - if ipv46_validator(ip) - ip_x = ip - ip_i = Rex::Socket.addr_atoi(ip) - else - raise ArgumentError, "Invalid IP address: #{ip.inspect}" - end - when "Fixnum" - if (0..2**32-1).include? ip - ip_x = Rex::Socket.addr_itoa(ip) - ip_i = ip - else - raise ArgumentError, "Invalid IP address: #{ip.inspect}" - end - else - raise ArgumentError, "Invalid IP address: #{ip.inspect}" - end - return true if Rex::Socket::RangeWalker.new("0.0.0.0-0.255.255.255").include? ip_x - return true if Rex::Socket::RangeWalker.new("127.0.0.0-127.255.255.255").include? ip_x - return true if Rex::Socket::RangeWalker.new("169.254.0.0-169.254.255.255").include? ip_x - return true if Rex::Socket::RangeWalker.new("224.0.0.0-239.255.255.255").include? ip_x - return true if Rex::Socket::RangeWalker.new("255.255.255.255-255.255.255.255").include? ip_x - return false - end - - def ipv46_validator(addr) - ipv4_validator(addr) or ipv6_validator(addr) - end - - def ipv4_validator(addr) - return false unless addr.kind_of? String - Rex::Socket.is_ipv4?(addr) - end - - def ipv6_validator(addr) - Rex::Socket.is_ipv6?(addr) - end - - # Takes a space-delimited set of ips and ranges, and subjects - # them to RangeWalker for validation. Returns true or false. - def validate_ips(ips) - ret = true - begin - ips.split(/\s+/).each {|ip| - unless Rex::Socket::RangeWalker.new(ip).ranges - ret = false - break - end - } - rescue - ret = false - end - return ret - end - - - # - # Determines if the database is functional - # - def check - ::ActiveRecord::Base.connection_pool.with_connection { - res = ::Mdm::Host.find(:first) - } - end - - - def default_workspace - ::ActiveRecord::Base.connection_pool.with_connection { - ::Mdm::Workspace.default - } - end - - def find_workspace(name) - ::ActiveRecord::Base.connection_pool.with_connection { - ::Mdm::Workspace.find_by_name(name) - } - end - - # - # Creates a new workspace in the database - # - def add_workspace(name) - ::ActiveRecord::Base.connection_pool.with_connection { - ::Mdm::Workspace.find_or_create_by_name(name) - } - end - - def workspaces - ::ActiveRecord::Base.connection_pool.with_connection { - ::Mdm::Workspace.find(:all) - } - end - - # - # Wait for all pending write to finish - # - def sync - # There is no more queue. - end - - # - # Find a host. Performs no database writes. - # - def get_host(opts) - if opts.kind_of? ::Mdm::Host - return opts - elsif opts.kind_of? String - raise RuntimeError, "This invokation of get_host is no longer supported: #{caller}" - else - address = opts[:addr] || opts[:address] || opts[:host] || return - return address if address.kind_of? ::Mdm::Host - end - ::ActiveRecord::Base.connection_pool.with_connection { - wspace = opts.delete(:workspace) || workspace - if wspace.kind_of? String - wspace = find_workspace(wspace) - end - - address = normalize_host(address) - return wspace.hosts.find_by_address(address) - } - end - - # - # Exactly like report_host but waits for the database to create a host and returns it. - # - def find_or_create_host(opts) - report_host(opts) - end - - # - # Report a host's attributes such as operating system and service pack - # - # The opts parameter MUST contain - # +:host+:: -- the host's ip address - # - # The opts parameter can contain: - # +:state+:: -- one of the Msf::HostState constants - # +:os_name+:: -- something like "Windows", "Linux", or "Mac OS X" - # +:os_flavor+:: -- something like "Enterprise", "Pro", or "Home" - # +:os_sp+:: -- something like "SP2" - # +:os_lang+:: -- something like "English", "French", or "en-US" - # +:arch+:: -- one of the ARCH_* constants - # +:mac+:: -- the host's MAC address - # +:scope+:: -- interface identifier for link-local IPv6 - # +:virtual_host+:: -- the name of the VM host software, eg "VMWare", "QEMU", "Xen", etc. - # - def report_host(opts) - - return if not active - addr = opts.delete(:host) || return - - # Sometimes a host setup through a pivot will see the address as "Remote Pipe" - if addr.eql? "Remote Pipe" - return - end - - ::ActiveRecord::Base.connection_pool.with_connection { - wspace = opts.delete(:workspace) || workspace - if wspace.kind_of? String - wspace = find_workspace(wspace) - end - - ret = { } - - if not addr.kind_of? ::Mdm::Host - addr = normalize_host(addr) - addr, scope = addr.split('%', 2) - opts[:scope] = scope if scope - - unless ipv46_validator(addr) - raise ::ArgumentError, "Invalid IP address in report_host(): #{addr}" - end - - if opts[:comm] and opts[:comm].length > 0 - host = wspace.hosts.find_or_initialize_by_address_and_comm(addr, opts[:comm]) - else - host = wspace.hosts.find_or_initialize_by_address(addr) - end - else - host = addr - end - - # Truncate the info field at the maximum field length - if opts[:info] - opts[:info] = opts[:info][0,65535] - end - - # Truncate the name field at the maximum field length - if opts[:name] - opts[:name] = opts[:name][0,255] - end - - opts.each { |k,v| - if (host.attribute_names.include?(k.to_s)) - unless host.attribute_locked?(k.to_s) - host[k] = v.to_s.gsub(/[\x00-\x1f]/n, '') - end - else - dlog("Unknown attribute for ::Mdm::Host: #{k}") - end - } - host.info = host.info[0,::Mdm::Host.columns_hash["info"].limit] if host.info - - # Set default fields if needed - host.state = HostState::Alive if not host.state - host.comm = '' if not host.comm - host.workspace = wspace if not host.workspace - - if host.changed? - msf_import_timestamps(opts,host) - host.save! - end - - if opts[:task] - Mdm::TaskHost.create( - :task => opts[:task], - :host => host - ) - end - - host - } - end - - - # - # Update a host's attributes via semi-standardized sysinfo hash (Meterpreter) - # - # The opts parameter MUST contain the following entries - # +:host+:: -- the host's ip address - # +:info+:: -- the information hash - # * 'Computer' -- the host name - # * 'OS' -- the operating system string - # * 'Architecture' -- the hardware architecture - # * 'System Language' -- the system language - # - # The opts parameter can contain: - # +:workspace+:: -- the workspace for this host - # - def update_host_via_sysinfo(opts) - - return if not active - addr = opts.delete(:host) || return - info = opts.delete(:info) || return - - # Sometimes a host setup through a pivot will see the address as "Remote Pipe" - if addr.eql? "Remote Pipe" - return - end - - ::ActiveRecord::Base.connection_pool.with_connection { - wspace = opts.delete(:workspace) || workspace - if wspace.kind_of? String - wspace = find_workspace(wspace) - end - - if not addr.kind_of? ::Mdm::Host - addr = normalize_host(addr) - addr, scope = addr.split('%', 2) - opts[:scope] = scope if scope - - unless ipv46_validator(addr) - raise ::ArgumentError, "Invalid IP address in report_host(): #{addr}" - end - - if opts[:comm] and opts[:comm].length > 0 - host = wspace.hosts.find_or_initialize_by_address_and_comm(addr, opts[:comm]) - else - host = wspace.hosts.find_or_initialize_by_address(addr) - end - else - host = addr - end - - res = {} - - if info['Computer'] - res[:name] = info['Computer'] - end - - if info['Architecture'] - res[:arch] = info['Architecture'].split(/\s+/).first - end - - if info['OS'] =~ /^Windows\s*([^\(]+)\(([^\)]+)\)/i - res[:os_name] = "Windows #{$1.strip}" - build = $2.strip - - if build =~ /Service Pack (\d+)/ - res[:os_sp] = "SP" + $1 - end - end - - if info["System Language"] - case info["System Language"] - when /^en_/ - res[:os_lang] = "English" - end - end - - - # Truncate the info field at the maximum field length - if res[:info] - res[:info] = res[:info][0,65535] - end - - # Truncate the name field at the maximum field length - if res[:name] - res[:name] = res[:name][0,255] - end - - res.each { |k,v| - - if (host.attribute_names.include?(k.to_s)) - unless host.attribute_locked?(k.to_s) - host[k] = v.to_s.gsub(/[\x00-\x1f]/n, '') - end - else - dlog("Unknown attribute for Host: #{k}") - end - } - - # Set default fields if needed - host.state = HostState::Alive if not host.state - host.comm = '' if not host.comm - host.workspace = wspace if not host.workspace - - if host.changed? - host.save! - end - - host - } - end - # - # Iterates over the hosts table calling the supplied block with the host - # instance of each entry. - # - def each_host(wspace=workspace, &block) - ::ActiveRecord::Base.connection_pool.with_connection { - wspace.hosts.each do |host| - block.call(host) - end - } - end - - # - # Returns a list of all hosts in the database - # - def hosts(wspace = workspace, only_up = false, addresses = nil) - ::ActiveRecord::Base.connection_pool.with_connection { - conditions = {} - conditions[:state] = [Msf::HostState::Alive, Msf::HostState::Unknown] if only_up - conditions[:address] = addresses if addresses - wspace.hosts.where(conditions).order(:address) - } - end - - - - def find_or_create_service(opts) - report_service(opts) - end - - # - # Record a service in the database. - # - # opts MUST contain - # +:host+:: the host where this service is running - # +:port+:: the port where this service listens - # +:proto+:: the transport layer protocol (e.g. tcp, udp) - # - # opts may contain - # +:name+:: the application layer protocol (e.g. ssh, mssql, smb) - # +:sname+:: an alias for the above - # - def report_service(opts) - return if not active - ::ActiveRecord::Base.connection_pool.with_connection { |conn| - addr = opts.delete(:host) || return - hname = opts.delete(:host_name) - hmac = opts.delete(:mac) - host = nil - wspace = opts.delete(:workspace) || workspace - hopts = {:workspace => wspace, :host => addr} - hopts[:name] = hname if hname - hopts[:mac] = hmac if hmac - - # Other report_* methods take :sname to mean the service name, so we - # map it here to ensure it ends up in the right place despite not being - # a real column. - if opts[:sname] - opts[:name] = opts.delete(:sname) - end - - if addr.kind_of? ::Mdm::Host - host = addr - addr = host.address - else - host = report_host(hopts) - end - - if opts[:port].to_i.zero? - dlog("Skipping port zero for service '%s' on host '%s'" % [opts[:name],host.address]) - return nil - end - - ret = {} -=begin - host = get_host(:workspace => wspace, :address => addr) - if host - host.updated_at = host.created_at - host.state = HostState::Alive - host.save! - end -=end - - proto = opts[:proto] || 'tcp' - - service = host.services.find_or_initialize_by_port_and_proto(opts[:port].to_i, proto) - opts.each { |k,v| - if (service.attribute_names.include?(k.to_s)) - service[k] = ((v and k == :name) ? v.to_s.downcase : v) - else - dlog("Unknown attribute for Service: #{k}") - end - } - service.state ||= ServiceState::Open - service.info ||= "" - - if (service and service.changed?) - msf_import_timestamps(opts,service) - service.save! - end - - if opts[:task] - Mdm::TaskService.create( - :task => opts[:task], - :service => service - ) - end - - ret[:service] = service - } - end - - def get_service(wspace, host, proto, port) - ::ActiveRecord::Base.connection_pool.with_connection { - host = get_host(:workspace => wspace, :address => host) - return if not host - return host.services.find_by_proto_and_port(proto, port) - } - end - - # - # Iterates over the services table calling the supplied block with the - # service instance of each entry. - # - def each_service(wspace=workspace, &block) - ::ActiveRecord::Base.connection_pool.with_connection { - services(wspace).each do |service| - block.call(service) - end - } - end - - # - # Returns a list of all services in the database - # - def services(wspace = workspace, only_up = false, proto = nil, addresses = nil, ports = nil, names = nil) - ::ActiveRecord::Base.connection_pool.with_connection { - conditions = {} - conditions[:state] = [ServiceState::Open] if only_up - conditions[:proto] = proto if proto - conditions["hosts.address"] = addresses if addresses - conditions[:port] = ports if ports - conditions[:name] = names if names - wspace.services.includes(:host).where(conditions).order("hosts.address, port") - } - end - - # Returns a session based on opened_time, host address, and workspace - # (or returns nil) - def get_session(opts) - return if not active - ::ActiveRecord::Base.connection_pool.with_connection { - wspace = opts[:workspace] || opts[:wspace] || workspace - addr = opts[:addr] || opts[:address] || opts[:host] || return - host = get_host(:workspace => wspace, :host => addr) - time = opts[:opened_at] || opts[:created_at] || opts[:time] || return - ::Mdm::Session.find_by_host_id_and_opened_at(host.id, time) - } - end - - # @note The Mdm::Session#desc will be truncated to 255 characters. - # @todo https://www.pivotaltracker.com/story/show/48249739 - # - # @overload report_session(opts) - # Creates an Mdm::Session from Msf::Session. If +via_exploit+ is set on the - # +session+, then an Mdm::Vuln and Mdm::ExploitAttempt is created for the - # session's host. The Mdm::Host for the +session_host+ is created using - # The session.session_host, +session.arch+ (if +session+ responds to arch), - # and the workspace derived from opts or the +session+. The Mdm::Session is - # assumed to be +last_seen+ and +opened_at+ at the time report_session is - # called. +session.exploit_datastore['ParentModule']+ is used for the - # Mdm::Session#via_exploit if +session.via_exploit+ is - # 'exploit/multi/handler'. - # - # @param opts [Hash{Symbol => Object}] options - # @option opt [Msf::Session, #datastore, #platform, #type, #via_exploit, #via_payload] :session - # The in-memory session to persist to the database. - # @option opts [Mdm::Workspace] :workspace The workspace for in which the - # :session host is contained. Also used as the workspace for the - # Mdm::ExploitAttempt and Mdm::Vuln. Defaults to Mdm::Worksapce with - # Mdm::Workspace#name equal to +session.workspace+. - # @return [nil] if {Msf::DBManager#active} is +false+. - # @return [Mdm::Session] if session is saved - # @raise [ArgumentError] if :session is not an {Msf::Session}. - # @raise [ActiveRecord::RecordInvalid] if session is invalid and cannot be - # saved, in which case, the Mdm::ExploitAttempt and Mdm::Vuln will not be - # created, but the Mdm::Host will have been. (There is no transaction - # to rollback the Mdm::Host creation.) - # @see #find_or_create_host - # @see #normalize_host - # @see #report_exploit_success - # @see #report_vuln - # - # @overload report_session(opts) - # Creates an Mdm::Session from Mdm::Host. - # - # @param opts [Hash{Symbol => Object}] options - # @option opts [DateTime, Time] :closed_at The date and time the sesion was - # closed. - # @option opts [String] :close_reason Reason the session was closed. - # @option opts [Hash] :datastore {Msf::DataStore#to_h}. - # @option opts [String] :desc Session description. Will be truncated to 255 - # characters. - # @option opts [Mdm::Host] :host The host on which the session was opened. - # @option opts [DateTime, Time] :last_seen The last date and time the - # session was seen to be open. Defaults to :closed_at's value. - # @option opts [DateTime, Time] :opened_at The date and time that the - # session was opened. - # @option opts [String] :platform The platform of the host. - # @option opts [Array] :routes ([]) The routes through the session for - # pivoting. - # @option opts [String] :stype Session type. - # @option opts [String] :via_exploit The {Msf::Module#fullname} of the - # exploit that was used to open the session. - # @option option [String] :via_payload the {MSf::Module#fullname} of the - # payload sent to the host when the exploit was successful. - # @return [nil] if {Msf::DBManager#active} is +false+. - # @return [Mdm::Session] if session is saved. - # @raise [ArgumentError] if :host is not an Mdm::Host. - # @raise [ActiveRecord::RecordInvalid] if session is invalid and cannot be - # saved. - # - # @raise ArgumentError if :host and :session is +nil+ - def report_session(opts) - return if not active - ::ActiveRecord::Base.connection_pool.with_connection { - if opts[:session] - raise ArgumentError.new("Invalid :session, expected Msf::Session") unless opts[:session].kind_of? Msf::Session - session = opts[:session] - wspace = opts[:workspace] || find_workspace(session.workspace) - h_opts = { } - h_opts[:host] = normalize_host(session) - h_opts[:arch] = session.arch if session.respond_to?(:arch) and session.arch - h_opts[:workspace] = wspace - host = find_or_create_host(h_opts) - sess_data = { - :host_id => host.id, - :stype => session.type, - :desc => session.info, - :platform => session.platform, - :via_payload => session.via_payload, - :via_exploit => session.via_exploit, - :routes => [], - :datastore => session.exploit_datastore.to_h, - :port => session.session_port, - :opened_at => Time.now.utc, - :last_seen => Time.now.utc, - :local_id => session.sid - } - elsif opts[:host] - raise ArgumentError.new("Invalid :host, expected Host object") unless opts[:host].kind_of? ::Mdm::Host - host = opts[:host] - sess_data = { - :host_id => host.id, - :stype => opts[:stype], - :desc => opts[:desc], - :platform => opts[:platform], - :via_payload => opts[:via_payload], - :via_exploit => opts[:via_exploit], - :routes => opts[:routes] || [], - :datastore => opts[:datastore], - :opened_at => opts[:opened_at], - :closed_at => opts[:closed_at], - :last_seen => opts[:last_seen] || opts[:closed_at], - :close_reason => opts[:close_reason], - } - else - raise ArgumentError.new("Missing option :session or :host") - end - ret = {} - - # Truncate the session data if necessary - if sess_data[:desc] - sess_data[:desc] = sess_data[:desc][0,255] - end - - # In the case of multi handler we cannot yet determine the true - # exploit responsible. But we can at least show the parent versus - # just the generic handler: - if session and session.via_exploit == "exploit/multi/handler" and sess_data[:datastore]['ParentModule'] - sess_data[:via_exploit] = sess_data[:datastore]['ParentModule'] - end - - s = ::Mdm::Session.new(sess_data) - s.save! - - if session and session.exploit_task and session.exploit_task.record - session_task = session.exploit_task.record - if session_task.class == Mdm::Task - Mdm::TaskSession.create(:task => session_task, :session => s ) - end - end - - - if opts[:session] - session.db_record = s - end - - # If this is a live session, we know the host is vulnerable to something. - if opts[:session] and session.via_exploit - mod = framework.modules.create(session.via_exploit) - - if session.via_exploit == "exploit/multi/handler" and sess_data[:datastore]['ParentModule'] - mod_fullname = sess_data[:datastore]['ParentModule'] - mod_name = ::Mdm::Module::Detail.find_by_fullname(mod_fullname).name - else - mod_name = mod.name - mod_fullname = mod.fullname - end - - vuln_info = { - :host => host.address, - :name => mod_name, - :refs => mod.references, - :workspace => wspace, - :exploited_at => Time.now.utc, - :info => "Exploited by #{mod_fullname} to create Session #{s.id}" - } - - port = session.exploit_datastore["RPORT"] - service = (port ? host.services.find_by_port(port.to_i) : nil) - - vuln_info[:service] = service if service - - vuln = framework.db.report_vuln(vuln_info) - - if session.via_exploit == "exploit/multi/handler" and sess_data[:datastore]['ParentModule'] - via_exploit = sess_data[:datastore]['ParentModule'] - else - via_exploit = session.via_exploit - end - attempt_info = { - :timestamp => Time.now.utc, - :workspace => wspace, - :module => via_exploit, - :username => session.username, - :refs => mod.references, - :session_id => s.id, - :host => host, - :service => service, - :vuln => vuln - } - - framework.db.report_exploit_success(attempt_info) - - end - - s - } - end - - # - # Record a session event in the database - # - # opts MUST contain one of: - # +:session+:: the Msf::Session OR the ::Mdm::Session we are reporting - # +:etype+:: event type, enum: command, output, upload, download, filedelete - # - # opts may contain - # +:output+:: the data for an output event - # +:command+:: the data for an command event - # +:remote_path+:: path to the associated file for upload, download, and filedelete events - # +:local_path+:: path to the associated file for upload, and download - # - def report_session_event(opts) - return if not active - raise ArgumentError.new("Missing required option :session") if opts[:session].nil? - raise ArgumentError.new("Expected an :etype") unless opts[:etype] - session = nil - - ::ActiveRecord::Base.connection_pool.with_connection { - if opts[:session].respond_to? :db_record - session = opts[:session].db_record - if session.nil? - # The session doesn't have a db_record which means - # a) the database wasn't connected at session registration time - # or - # b) something awful happened and the report_session call failed - # - # Either way, we can't do anything with this session as is, so - # log a warning and punt. - wlog("Warning: trying to report a session_event for a session with no db_record (#{opts[:session].sid})") - return - end - event_data = { :created_at => Time.now } - else - session = opts[:session] - event_data = { :created_at => opts[:created_at] } - end - - event_data[:session_id] = session.id - [:remote_path, :local_path, :output, :command, :etype].each do |attr| - event_data[attr] = opts[attr] if opts[attr] - end - - s = ::Mdm::SessionEvent.create(event_data) - } - end - - def report_session_route(session, route) - return if not active - if session.respond_to? :db_record - s = session.db_record - else - s = session - end - unless s.respond_to?(:routes) - raise ArgumentError.new("Invalid :session, expected Session object got #{session.class}") - end - - ::ActiveRecord::Base.connection_pool.with_connection { - - subnet, netmask = route.split("/") - s.routes.create(:subnet => subnet, :netmask => netmask) - } - end - - def report_session_route_remove(session, route) - return if not active - if session.respond_to? :db_record - s = session.db_record - else - s = session - end - unless s.respond_to?(:routes) - raise ArgumentError.new("Invalid :session, expected Session object got #{session.class}") - end - - ::ActiveRecord::Base.connection_pool.with_connection { - subnet, netmask = route.split("/") - r = s.routes.find_by_subnet_and_netmask(subnet, netmask) - r.destroy if r - } - end - - - def report_exploit_success(opts) - ::ActiveRecord::Base.connection_pool.with_connection { - - wspace = opts.delete(:workspace) || workspace - mrefs = opts.delete(:refs) || return - host = opts.delete(:host) - port = opts.delete(:port) - prot = opts.delete(:proto) - svc = opts.delete(:service) - vuln = opts.delete(:vuln) - - timestamp = opts.delete(:timestamp) - username = opts.delete(:username) - mname = opts.delete(:module) - - # Look up or generate the host as appropriate - if not (host and host.kind_of? ::Mdm::Host) - if svc.kind_of? ::Mdm::Service - host = svc.host - else - host = report_host(:workspace => wspace, :address => host ) - end - end - - # Bail if we dont have a host object - return if not host - - # Look up or generate the service as appropriate - if port and svc.nil? - svc = report_service(:workspace => wspace, :host => host, :port => port, :proto => prot ) if port - end - - if not vuln - # Create a references map from the module list - ref_objs = ::Mdm::Ref.where(:name => mrefs.map { |ref| - if ref.respond_to?(:ctx_id) and ref.respond_to?(:ctx_val) - "#{ref.ctx_id}-#{ref.ctx_val}" - else - ref.to_s - end - }) - - # Try find a matching vulnerability - vuln = find_vuln_by_refs(ref_objs, host, svc) - end - - # We have match, lets create a vuln_attempt record - if vuln - attempt_info = { - :vuln_id => vuln.id, - :attempted_at => timestamp || Time.now.utc, - :exploited => true, - :username => username || "unknown", - :module => mname - } - - attempt_info[:session_id] = opts[:session_id] if opts[:session_id] - attempt_info[:loot_id] = opts[:loot_id] if opts[:loot_id] - - vuln.vuln_attempts.create(attempt_info) - - # Correct the vuln's associated service if necessary - if svc and vuln.service_id.nil? - vuln.service = svc - vuln.save - end - end - - # Report an exploit attempt all the same - attempt_info = { - :attempted_at => timestamp || Time.now.utc, - :exploited => true, - :username => username || "unknown", - :module => mname - } - - attempt_info[:vuln_id] = vuln.id if vuln - attempt_info[:session_id] = opts[:session_id] if opts[:session_id] - attempt_info[:loot_id] = opts[:loot_id] if opts[:loot_id] - - if svc - attempt_info[:port] = svc.port - attempt_info[:proto] = svc.proto - end - - if port and svc.nil? - attempt_info[:port] = port - attempt_info[:proto] = prot || "tcp" - end - - host.exploit_attempts.create(attempt_info) - } - end - - def report_exploit_failure(opts) - - ::ActiveRecord::Base.connection_pool.with_connection { - wspace = opts.delete(:workspace) || workspace - mrefs = opts.delete(:refs) || return - host = opts.delete(:host) - port = opts.delete(:port) - prot = opts.delete(:proto) - svc = opts.delete(:service) - vuln = opts.delete(:vuln) - - timestamp = opts.delete(:timestamp) - freason = opts.delete(:fail_reason) - fdetail = opts.delete(:fail_detail) - username = opts.delete(:username) - mname = opts.delete(:module) - - # Look up the host as appropriate - if not (host and host.kind_of? ::Mdm::Host) - if svc.kind_of? ::Mdm::Service - host = svc.host - else - host = get_host( :workspace => wspace, :address => host ) - end - end - - # Bail if we dont have a host object - return if not host - - # Look up the service as appropriate - if port and svc.nil? - prot ||= "tcp" - svc = get_service(wspace, host, prot, port) if port - end - - if not vuln - # Create a references map from the module list - ref_objs = ::Mdm::Ref.where(:name => mrefs.map { |ref| - if ref.respond_to?(:ctx_id) and ref.respond_to?(:ctx_val) - "#{ref.ctx_id}-#{ref.ctx_val}" - else - ref.to_s - end - }) - - # Try find a matching vulnerability - vuln = find_vuln_by_refs(ref_objs, host, svc) - end - - # Report a vuln_attempt if we found a match - if vuln - attempt_info = { - :attempted_at => timestamp || Time.now.utc, - :exploited => false, - :fail_reason => freason, - :fail_detail => fdetail, - :username => username || "unknown", - :module => mname - } - - vuln.vuln_attempts.create(attempt_info) - end - - # Report an exploit attempt all the same - attempt_info = { - :attempted_at => timestamp || Time.now.utc, - :exploited => false, - :username => username || "unknown", - :module => mname, - :fail_reason => freason, - :fail_detail => fdetail - } - - attempt_info[:vuln_id] = vuln.id if vuln - - if svc - attempt_info[:port] = svc.port - attempt_info[:proto] = svc.proto - end - - if port and svc.nil? - attempt_info[:port] = port - attempt_info[:proto] = prot || "tcp" - end - - host.exploit_attempts.create(attempt_info) - } - end - - - def report_vuln_attempt(vuln, opts) - ::ActiveRecord::Base.connection_pool.with_connection { - return if not vuln - info = {} - - # Opts can be keyed by strings or symbols - ::Mdm::VulnAttempt.column_names.each do |kn| - k = kn.to_sym - next if ['id', 'vuln_id'].include?(kn) - info[k] = opts[kn] if opts[kn] - info[k] = opts[k] if opts[k] - end - - return unless info[:attempted_at] - - vuln.vuln_attempts.create(info) - } - end - - def report_exploit_attempt(host, opts) - ::ActiveRecord::Base.connection_pool.with_connection { - return if not host - info = {} - - # Opts can be keyed by strings or symbols - ::Mdm::VulnAttempt.column_names.each do |kn| - k = kn.to_sym - next if ['id', 'host_id'].include?(kn) - info[k] = opts[kn] if opts[kn] - info[k] = opts[k] if opts[k] - end - - host.exploit_attempts.create(info) - } - end - - def get_client(opts) - ::ActiveRecord::Base.connection_pool.with_connection { - wspace = opts.delete(:workspace) || workspace - host = get_host(:workspace => wspace, :host => opts[:host]) || return - client = host.clients.where({:ua_string => opts[:ua_string]}).first() - return client - } - end - - def find_or_create_client(opts) - report_client(opts) - end - - # - # Report a client running on a host. - # - # opts MUST contain - # +:ua_string+:: the value of the User-Agent header - # +:host+:: the host where this client connected from, can be an ip address or a Host object - # - # opts can contain - # +:ua_name+:: one of the Msf::HttpClients constants - # +:ua_ver+:: detected version of the given client - # +:campaign+:: an id or Campaign object - # - # Returns a Client. - # - def report_client(opts) - return if not active - ::ActiveRecord::Base.connection_pool.with_connection { - addr = opts.delete(:host) || return - wspace = opts.delete(:workspace) || workspace - report_host(:workspace => wspace, :host => addr) - - ret = {} - - host = get_host(:workspace => wspace, :host => addr) - client = host.clients.find_or_initialize_by_ua_string(opts[:ua_string]) - - opts[:ua_string] = opts[:ua_string].to_s - - campaign = opts.delete(:campaign) - if campaign - case campaign - when Campaign - opts[:campaign_id] = campaign.id - else - opts[:campaign_id] = campaign - end - end - - opts.each { |k,v| - if (client.attribute_names.include?(k.to_s)) - client[k] = v - else - dlog("Unknown attribute for Client: #{k}") - end - } - if (client and client.changed?) - client.save! - end - ret[:client] = client - } - end - - # - # This method iterates the vulns table calling the supplied block with the - # vuln instance of each entry. - # - def each_vuln(wspace=workspace,&block) - ::ActiveRecord::Base.connection_pool.with_connection { - wspace.vulns.each do |vulns| - block.call(vulns) - end - } - end - - # - # This methods returns a list of all vulnerabilities in the database - # - def vulns(wspace=workspace) - ::ActiveRecord::Base.connection_pool.with_connection { - wspace.vulns - } - end - - # - # This methods returns a list of all credentials in the database - # - def creds(wspace=workspace) - ::ActiveRecord::Base.connection_pool.with_connection { - Mdm::Cred.includes({:service => :host}).where("hosts.workspace_id = ?", wspace.id) - } - end - - # - # This method returns a list of all exploited hosts in the database. - # - def exploited_hosts(wspace=workspace) - ::ActiveRecord::Base.connection_pool.with_connection { - wspace.exploited_hosts - } - end - - # - # This method iterates the notes table calling the supplied block with the - # note instance of each entry. - # - def each_note(wspace=workspace, &block) - ::ActiveRecord::Base.connection_pool.with_connection { - wspace.notes.each do |note| - block.call(note) - end - } - end - - # - # Find or create a note matching this type/data - # - def find_or_create_note(opts) - report_note(opts) - end - - # - # Report a Note to the database. Notes can be tied to a ::Mdm::Workspace, Host, or Service. - # - # opts MUST contain - # +:type+:: The type of note, e.g. smb_peer_os - # - # opts can contain - # +:workspace+:: the workspace to associate with this Note - # +:host+:: an IP address or a Host object to associate with this Note - # +:service+:: a Service object to associate with this Note - # +:data+:: whatever it is you're making a note of - # +:port+:: along with +:host+ and +:proto+, a service to associate with this Note - # +:proto+:: along with +:host+ and +:port+, a service to associate with this Note - # +:update+:: what to do in case a similar Note exists, see below - # - # The +:update+ option can have the following values: - # +:unique+:: allow only a single Note per +:host+/+:type+ pair - # +:unique_data+:: like +:uniqe+, but also compare +:data+ - # +:insert+:: always insert a new Note even if one with identical values exists - # - # If the provided +:host+ is an IP address and does not exist in the - # database, it will be created. If +:workspace+, +:host+ and +:service+ - # are all omitted, the new Note will be associated with the current - # workspace. - # - def report_note(opts) - return if not active - ::ActiveRecord::Base.connection_pool.with_connection { - wspace = opts.delete(:workspace) || workspace - if wspace.kind_of? String - wspace = find_workspace(wspace) - end - seen = opts.delete(:seen) || false - crit = opts.delete(:critical) || false - host = nil - addr = nil - # Report the host so it's there for the Proc to use below - if opts[:host] - if opts[:host].kind_of? ::Mdm::Host - host = opts[:host] - else - addr = normalize_host(opts[:host]) - host = report_host({:workspace => wspace, :host => addr}) - end - # Do the same for a service if that's also included. - if (opts[:port]) - proto = nil - sname = nil - case opts[:proto].to_s.downcase # Catch incorrect usages - when 'tcp','udp' - proto = opts[:proto] - sname = opts[:sname] if opts[:sname] - when 'dns','snmp','dhcp' - proto = 'udp' - sname = opts[:proto] - else - proto = 'tcp' - sname = opts[:proto] - end - sopts = { - :workspace => wspace, - :host => host, - :port => opts[:port], - :proto => proto - } - sopts[:name] = sname if sname - report_service(sopts) - end - end - # Update Modes can be :unique, :unique_data, :insert - mode = opts[:update] || :unique - - ret = {} - - if addr and not host - host = get_host(:workspace => wspace, :host => addr) - end - if host and (opts[:port] and opts[:proto]) - service = get_service(wspace, host, opts[:proto], opts[:port]) - elsif opts[:service] and opts[:service].kind_of? ::Mdm::Service - service = opts[:service] - end -=begin - if host - host.updated_at = host.created_at - host.state = HostState::Alive - host.save! - end -=end - ntype = opts.delete(:type) || opts.delete(:ntype) || (raise RuntimeError, "A note :type or :ntype is required") - data = opts[:data] - note = nil - - conditions = { :ntype => ntype } - conditions[:host_id] = host[:id] if host - conditions[:service_id] = service[:id] if service - - case mode - when :unique - note = wspace.notes.where(conditions).first_or_initialize - note.data = data - when :unique_data - notes = wspace.notes.where(conditions) - - # Don't make a new Note with the same data as one that already - # exists for the given: type and (host or service) - notes.each do |n| - # Compare the deserialized data from the table to the raw - # data we're looking for. Because of the serialization we - # can't do this easily or reliably in SQL. - if n.data == data - note = n - break - end - end - if not note - # We didn't find one with the data we're looking for, make - # a new one. - note = wspace.notes.new(conditions.merge(:data => data)) - end - else - # Otherwise, assume :insert, which means always make a new one - note = wspace.notes.new - if host - note.host_id = host[:id] - end - if opts[:service] and opts[:service].kind_of? ::Mdm::Service - note.service_id = opts[:service][:id] - end - note.seen = seen - note.critical = crit - note.ntype = ntype - note.data = data - end - msf_import_timestamps(opts,note) - note.save! - ret[:note] = note - } - end - - # - # This methods returns a list of all notes in the database - # - def notes(wspace=workspace) - ::ActiveRecord::Base.connection_pool.with_connection { - wspace.notes - } - end - - # This is only exercised by MSF3 XML importing for now. Needs the wait - # conditions and return hash as well. - def report_host_tag(opts) - name = opts.delete(:name) - raise DBImportError.new("Missing required option :name") unless name - addr = opts.delete(:addr) - raise DBImportError.new("Missing required option :addr") unless addr - wspace = opts.delete(:wspace) - raise DBImportError.new("Missing required option :wspace") unless wspace - ::ActiveRecord::Base.connection_pool.with_connection { - if wspace.kind_of? String - wspace = find_workspace(wspace) - end - - host = nil - report_host(:workspace => wspace, :address => addr) - - - host = get_host(:workspace => wspace, :address => addr) - desc = opts.delete(:desc) - summary = opts.delete(:summary) - detail = opts.delete(:detail) - crit = opts.delete(:crit) - possible_tags = Mdm::Tag.includes(:hosts).where("hosts.workspace_id = ? and tags.name = ?", wspace.id, name).order("tags.id DESC").limit(1) - tag = (possible_tags.blank? ? Mdm::Tag.new : possible_tags.first) - tag.name = name - tag.desc = desc - tag.report_summary = !!summary - tag.report_detail = !!detail - tag.critical = !!crit - tag.hosts = tag.hosts | [host] - tag.save! if tag.changed? - } - end - - # - # Store a set of credentials in the database. - # - # report_auth_info used to create a note, now it creates - # an entry in the creds table. It's much more akin to - # report_vuln() now. - # - # opts MUST contain - # +:host+:: an IP address or Host object reference - # +:port+:: a port number - # - # opts can contain - # +:user+:: the username - # +:pass+:: the password, or path to ssh_key - # +:ptype+:: the type of password (password(ish), hash, or ssh_key) - # +:proto+:: a transport name for the port - # +:sname+:: service name - # +:active+:: by default, a cred is active, unless explicitly false - # +:proof+:: data used to prove the account is actually active. - # - # Sources: Credentials can be sourced from another credential, or from - # a vulnerability. For example, if an exploit was used to dump the - # smb_hashes, and this credential comes from there, the source_id would - # be the Vuln id (as reported by report_vuln) and the type would be "Vuln". - # - # +:source_id+:: The Vuln or Cred id of the source of this cred. - # +:source_type+:: Either Vuln or Cred - # - # TODO: This is written somewhat host-centric, when really the - # Service is the thing. Need to revisit someday. - def report_auth_info(opts={}) - return if not active - raise ArgumentError.new("Missing required option :host") if opts[:host].nil? - raise ArgumentError.new("Missing required option :port") if (opts[:port].nil? and opts[:service].nil?) - - if (not opts[:host].kind_of?(::Mdm::Host)) and (not validate_ips(opts[:host])) - raise ArgumentError.new("Invalid address or object for :host (#{opts[:host].inspect})") - end - - ::ActiveRecord::Base.connection_pool.with_connection { - host = opts.delete(:host) - ptype = opts.delete(:type) || "password" - token = [opts.delete(:user), opts.delete(:pass)] - sname = opts.delete(:sname) - port = opts.delete(:port) - proto = opts.delete(:proto) || "tcp" - proof = opts.delete(:proof) - source_id = opts.delete(:source_id) - source_type = opts.delete(:source_type) - duplicate_ok = opts.delete(:duplicate_ok) - # Nil is true for active. - active = (opts[:active] || opts[:active].nil?) ? true : false - - wspace = opts.delete(:workspace) || workspace - - # Service management; assume the user knows what - # he's talking about. - service = opts.delete(:service) || report_service(:host => host, :port => port, :proto => proto, :name => sname, :workspace => wspace) - - # Non-US-ASCII usernames are tripping up the database at the moment, this is a temporary fix until we update the tables - if (token[0]) - # convert the token to US-ASCII from UTF-8 to prevent an error - token[0] = token[0].unpack("C*").pack("C*") - token[0] = token[0].gsub(/[\x00-\x1f\x7f-\xff]/n){|m| "\\x%.2x" % m.unpack("C")[0] } - end - - if (token[1]) - token[1] = token[1].unpack("C*").pack("C*") - token[1] = token[1].gsub(/[\x00-\x1f\x7f-\xff]/n){|m| "\\x%.2x" % m.unpack("C")[0] } - end - - ret = {} - - # Check to see if the creds already exist. We look also for a downcased username with the - # same password because we can fairly safely assume they are not in fact two seperate creds. - # this allows us to hedge against duplication of creds in the DB. - - if duplicate_ok - # If duplicate usernames are okay, find by both user and password (allows - # for actual duplicates to get modified updated_at, sources, etc) - if token[0].nil? or token[0].empty? - cred = service.creds.find_or_initialize_by_user_and_ptype_and_pass(token[0] || "", ptype, token[1] || "") - else - cred = service.creds.find_by_user_and_ptype_and_pass(token[0] || "", ptype, token[1] || "") - unless cred - dcu = token[0].downcase - cred = service.creds.find_by_user_and_ptype_and_pass( dcu || "", ptype, token[1] || "") - unless cred - cred = service.creds.find_or_initialize_by_user_and_ptype_and_pass(token[0] || "", ptype, token[1] || "") - end - end - end - else - # Create the cred by username only (so we can change passwords) - if token[0].nil? or token[0].empty? - cred = service.creds.find_or_initialize_by_user_and_ptype(token[0] || "", ptype) - else - cred = service.creds.find_by_user_and_ptype(token[0] || "", ptype) - unless cred - dcu = token[0].downcase - cred = service.creds.find_by_user_and_ptype_and_pass( dcu || "", ptype, token[1] || "") - unless cred - cred = service.creds.find_or_initialize_by_user_and_ptype(token[0] || "", ptype) - end - end - end - end - - # Update with the password - cred.pass = (token[1] || "") - - # Annotate the credential - cred.ptype = ptype - cred.active = active - - # Update the source ID only if there wasn't already one. - if source_id and !cred.source_id - cred.source_id = source_id - cred.source_type = source_type if source_type - end - - # Safe proof (lazy way) -- doesn't chop expanded - # characters correctly, but shouldn't ever be a problem. - unless proof.nil? - proof = Rex::Text.to_hex_ascii(proof) - proof = proof[0,4096] - end - cred.proof = proof - - # Update the timestamp - if cred.changed? - msf_import_timestamps(opts,cred) - cred.save! - end - - # Ensure the updated_at is touched any time report_auth_info is called - # except when it's set explicitly (as it is for imports) - unless opts[:updated_at] || opts["updated_at"] - cred.updated_at = Time.now.utc - cred.save! - end - - - if opts[:task] - Mdm::TaskCred.create( - :task => opts[:task], - :cred => cred - ) - end - - ret[:cred] = cred - } - end - - alias :report_cred :report_auth_info - alias :report_auth :report_auth_info - - # - # Find or create a credential matching this type/data - # - def find_or_create_cred(opts) - report_auth_info(opts) - end - - # - # This method iterates the creds table calling the supplied block with the - # cred instance of each entry. - # - def each_cred(wspace=workspace,&block) - ::ActiveRecord::Base.connection_pool.with_connection { - wspace.creds.each do |cred| - block.call(cred) - end - } - end - - def each_exploited_host(wspace=workspace,&block) - ::ActiveRecord::Base.connection_pool.with_connection { - wspace.exploited_hosts.each do |eh| - block.call(eh) - end - } - end - - # - # Find or create a vuln matching this service/name - # - def find_or_create_vuln(opts) - report_vuln(opts) - end - - # - # opts MUST contain - # +:host+:: the host where this vulnerability resides - # +:name+:: the friendly name for this vulnerability (title) - # - # opts can contain - # +:info+:: a human readable description of the vuln, free-form text - # +:refs+:: an array of Ref objects or string names of references - # +:details:: a hash with :key pointed to a find criteria hash and the rest containing VulnDetail fields - # - def report_vuln(opts) - return if not active - raise ArgumentError.new("Missing required option :host") if opts[:host].nil? - raise ArgumentError.new("Deprecated data column for vuln, use .info instead") if opts[:data] - name = opts[:name] || return - info = opts[:info] - - ::ActiveRecord::Base.connection_pool.with_connection { - - wspace = opts.delete(:workspace) || workspace - exploited_at = opts[:exploited_at] || opts["exploited_at"] - details = opts.delete(:details) - rids = opts.delete(:ref_ids) - - if opts[:refs] - rids ||= [] - opts[:refs].each do |r| - if (r.respond_to?(:ctx_id)) and (r.respond_to?(:ctx_val)) - r = "#{r.ctx_id}-#{r.ctx_val}" - end - rids << find_or_create_ref(:name => r) - end - end - - host = nil - addr = nil - if opts[:host].kind_of? ::Mdm::Host - host = opts[:host] - else - host = report_host({:workspace => wspace, :host => opts[:host]}) - addr = normalize_host(opts[:host]) - end - - ret = {} - - # Truncate the info field at the maximum field length - if info - info = info[0,65535] - end - - # Truncate the name field at the maximum field length - name = name[0,255] - - # Placeholder for the vuln object - vuln = nil - - # Identify the associated service - service = opts.delete(:service) - - # Treat port zero as no service - if service or opts[:port].to_i > 0 - - if not service - proto = nil - case opts[:proto].to_s.downcase # Catch incorrect usages, as in report_note - when 'tcp','udp' - proto = opts[:proto] - when 'dns','snmp','dhcp' - proto = 'udp' - sname = opts[:proto] - else - proto = 'tcp' - sname = opts[:proto] - end - - service = host.services.find_or_create_by_port_and_proto(opts[:port].to_i, proto) - end - - # Try to find an existing vulnerability with the same service & references - # If there are multiple matches, choose the one with the most matches - # If a match is found on a vulnerability with no associated service, - # update that vulnerability with our service information. This helps - # prevent dupes of the same vuln found by both local patch and - # service detection. - if rids and rids.length > 0 - vuln = find_vuln_by_refs(rids, host, service) - vuln.service = service if vuln - end - else - # Try to find an existing vulnerability with the same host & references - # If there are multiple matches, choose the one with the most matches - if rids and rids.length > 0 - vuln = find_vuln_by_refs(rids, host) - end - end - - # Try to match based on vuln_details records - if not vuln and opts[:details_match] - vuln = find_vuln_by_details(opts[:details_match], host, service) - if vuln and service and not vuln.service - vuln.service = service - end - end - - # No matches, so create a new vuln record - unless vuln - if service - vuln = service.vulns.find_by_name(name) - else - vuln = host.vulns.find_by_name(name) - end - - unless vuln - - vinf = { - :host_id => host.id, - :name => name, - :info => info - } - - vinf[:service_id] = service.id if service - vuln = Mdm::Vuln.create(vinf) - end - end - - # Set the exploited_at value if provided - vuln.exploited_at = exploited_at if exploited_at - - # Merge the references - if rids - vuln.refs << (rids - vuln.refs) - end - - # Finalize - if vuln.changed? - msf_import_timestamps(opts,vuln) - vuln.save! - end - - # Handle vuln_details parameters - report_vuln_details(vuln, details) if details - - vuln - } - end - - def find_vuln_by_refs(refs, host, service=nil) - - vuln = nil - - # Try to find an existing vulnerability with the same service & references - # If there are multiple matches, choose the one with the most matches - if service - refs_ids = refs.map{|x| x.id } - vuln = service.vulns.find(:all, :include => [:refs], :conditions => { 'refs.id' => refs_ids }).sort { |a,b| - ( refs_ids - a.refs.map{|x| x.id } ).length <=> ( refs_ids - b.refs.map{|x| x.id } ).length - }.first - end - - # Return if we matched based on service - return vuln if vuln - - # Try to find an existing vulnerability with the same host & references - # If there are multiple matches, choose the one with the most matches - refs_ids = refs.map{|x| x.id } - vuln = host.vulns.find(:all, :include => [:refs], :conditions => { 'service_id' => nil, 'refs.id' => refs_ids }).sort { |a,b| - ( refs_ids - a.refs.map{|x| x.id } ).length <=> ( refs_ids - b.refs.map{|x| x.id } ).length - }.first - - return vuln - end - - - def find_vuln_by_details(details_map, host, service=nil) - - # Create a modified version of the criteria in order to match against - # the joined version of the fields - - crit = {} - details_map.each_pair do |k,v| - crit[ "vuln_details.#{k}" ] = v - end - - vuln = nil - - if service - vuln = service.vulns.find(:first, :include => [:vuln_details], :conditions => crit) - end - - # Return if we matched based on service - return vuln if vuln - - # Prevent matches against other services - crit["vulns.service_id"] = nil if service - vuln = host.vulns.find(:first, :include => [:vuln_details], :conditions => crit) - - return vuln - end - - def get_vuln(wspace, host, service, name, data='') - raise RuntimeError, "Not workspace safe: #{caller.inspect}" - ::ActiveRecord::Base.connection_pool.with_connection { - vuln = nil - if (service) - vuln = ::Mdm::Vuln.find.where("name = ? and service_id = ? and host_id = ?", name, service.id, host.id).order("vulns.id DESC").first() - else - vuln = ::Mdm::Vuln.find.where("name = ? and host_id = ?", name, host.id).first() - end - - return vuln - } - end - - # - # Find or create a reference matching this name - # - def find_or_create_ref(opts) - ret = {} - ret[:ref] = get_ref(opts[:name]) - return ret[:ref] if ret[:ref] - - ::ActiveRecord::Base.connection_pool.with_connection { - ref = ::Mdm::Ref.find_or_initialize_by_name(opts[:name]) - if ref and ref.changed? - ref.save! - end - ret[:ref] = ref - } - end - - def get_ref(name) - ::ActiveRecord::Base.connection_pool.with_connection { - ::Mdm::Ref.find_by_name(name) - } - end - - # - # Populate the vuln_details table with additional - # information, matched by a specific criteria - # - def report_vuln_details(vuln, details) - ::ActiveRecord::Base.connection_pool.with_connection { - detail = ::Mdm::VulnDetail.where(( details.delete(:key) || {} ).merge(:vuln_id => vuln.id)).first - if detail - details.each_pair do |k,v| - detail[k] = v - end - detail.save! if detail.changed? - detail - else - detail = ::Mdm::VulnDetail.create(details.merge(:vuln_id => vuln.id)) - end - } - end - - # - # Update vuln_details records en-masse based on specific criteria - # Note that this *can* update data across workspaces - # - def update_vuln_details(details) - ::ActiveRecord::Base.connection_pool.with_connection { - criteria = details.delete(:key) || {} - ::Mdm::VulnDetail.update(key, details) - } - end - - # - # Populate the host_details table with additional - # information, matched by a specific criteria - # - def report_host_details(host, details) - ::ActiveRecord::Base.connection_pool.with_connection { - - detail = ::Mdm::HostDetail.where(( details.delete(:key) || {} ).merge(:host_id => host.id)).first - if detail - details.each_pair do |k,v| - detail[k] = v - end - detail.save! if detail.changed? - detail - else - detail = ::Mdm::HostDetail.create(details.merge(:host_id => host.id)) - end - } - end - - # report_exploit() used to be used to track sessions and which modules - # opened them. That information is now available with the session table - # directly. TODO: kill this completely some day -- for now just warn if - # some other UI is actually using it. - def report_exploit(opts={}) - wlog("Deprecated method call: report_exploit()\n" + - "report_exploit() options: #{opts.inspect}\n" + - "report_exploit() call stack:\n\t#{caller.join("\n\t")}" - ) - end - - # - # Deletes a host and associated data matching this address/comm - # - def del_host(wspace, address, comm='') - ::ActiveRecord::Base.connection_pool.with_connection { - address, scope = address.split('%', 2) - host = wspace.hosts.find_by_address_and_comm(address, comm) - host.destroy if host - } - end - - # - # Deletes a port and associated vulns matching this port - # - def del_service(wspace, address, proto, port, comm='') - - host = get_host(:workspace => wspace, :address => address) - return unless host - - ::ActiveRecord::Base.connection_pool.with_connection { - host.services.where({:proto => proto, :port => port}).each { |s| s.destroy } - } - end - - # - # Find a reference matching this name - # - def has_ref?(name) - ::ActiveRecord::Base.connection_pool.with_connection { - Mdm::Ref.find_by_name(name) - } - end - - # - # Find a vulnerability matching this name - # - def has_vuln?(name) - ::ActiveRecord::Base.connection_pool.with_connection { - Mdm::Vuln.find_by_name(name) - } - end - - # - # Look for an address across all comms - # - def has_host?(wspace,addr) - ::ActiveRecord::Base.connection_pool.with_connection { - address, scope = addr.split('%', 2) - wspace.hosts.find_by_address(addr) - } - end - - def events(wspace=workspace) - ::ActiveRecord::Base.connection_pool.with_connection { - wspace.events.find :all, :order => 'created_at ASC' - } - end - - def report_event(opts = {}) - return if not active - ::ActiveRecord::Base.connection_pool.with_connection { - wspace = opts.delete(:workspace) || workspace - return if not wspace # Temp fix? - uname = opts.delete(:username) - - if ! opts[:host].kind_of? ::Mdm::Host and opts[:host] - opts[:host] = report_host(:workspace => wspace, :host => opts[:host]) - end - - ::Mdm::Event.create(opts.merge(:workspace_id => wspace[:id], :username => uname)) - } - end - - # - # Loot collection - # - # - # This method iterates the loot table calling the supplied block with the - # instance of each entry. - # - def each_loot(wspace=workspace, &block) - ::ActiveRecord::Base.connection_pool.with_connection { - wspace.loots.each do |note| - block.call(note) - end - } - end - - # - # Find or create a loot matching this type/data - # - def find_or_create_loot(opts) - report_loot(opts) - end - - def report_loot(opts) - return if not active - ::ActiveRecord::Base.connection_pool.with_connection { - wspace = opts.delete(:workspace) || workspace - path = opts.delete(:path) || (raise RuntimeError, "A loot :path is required") - - host = nil - addr = nil - - # Report the host so it's there for the Proc to use below - if opts[:host] - if opts[:host].kind_of? ::Mdm::Host - host = opts[:host] - else - host = report_host({:workspace => wspace, :host => opts[:host]}) - addr = normalize_host(opts[:host]) - end - end - - ret = {} - - ltype = opts.delete(:type) || opts.delete(:ltype) || (raise RuntimeError, "A loot :type or :ltype is required") - ctype = opts.delete(:ctype) || opts.delete(:content_type) || 'text/plain' - name = opts.delete(:name) - info = opts.delete(:info) - data = opts[:data] - loot = wspace.loots.new - - if host - loot.host_id = host[:id] - end - if opts[:service] and opts[:service].kind_of? ::Mdm::Service - loot.service_id = opts[:service][:id] - end - - loot.path = path - loot.ltype = ltype - loot.content_type = ctype - loot.data = data - loot.name = name if name - loot.info = info if info - loot.workspace = wspace - msf_import_timestamps(opts,loot) - loot.save! - - ret[:loot] = loot - } - end - - # - # This methods returns a list of all loot in the database - # - def loots(wspace=workspace) - ::ActiveRecord::Base.connection_pool.with_connection { - wspace.loots - } - end - - # - # Find or create a task matching this type/data - # - def find_or_create_task(opts) - report_task(opts) - end - - def report_task(opts) - return if not active - ::ActiveRecord::Base.connection_pool.with_connection { - wspace = opts.delete(:workspace) || workspace - path = opts.delete(:path) || (raise RuntimeError, "A task :path is required") - - ret = {} - - user = opts.delete(:user) - desc = opts.delete(:desc) - error = opts.delete(:error) - info = opts.delete(:info) - mod = opts.delete(:mod) - options = opts.delete(:options) - prog = opts.delete(:prog) - result = opts.delete(:result) - completed_at = opts.delete(:completed_at) - task = wspace.tasks.new - - task.created_by = user - task.description = desc - task.error = error if error - task.info = info - task.module = mod - task.options = options - task.path = path - task.progress = prog - task.result = result if result - msf_import_timestamps(opts,task) - # Having blank completed_ats, while accurate, will cause unstoppable tasks. - if completed_at.nil? || completed_at.empty? - task.completed_at = opts[:updated_at] - else - task.completed_at = completed_at - end - task.save! - ret[:task] = task - } - end - - # - # This methods returns a list of all tasks in the database - # - def tasks(wspace=workspace) - ::ActiveRecord::Base.connection_pool.with_connection { - wspace.tasks - } - end - - - # TODO This method does not attempt to find. It just creates - # a report based on the passed params. - def find_or_create_report(opts) - report_report(opts) - end - - # Creates a Report based on passed parameters. Does not handle - # child artifacts. - # @param opts [Hash] - # @return [Integer] ID of created report - def report_report(opts) - return if not active - created = opts.delete(:created_at) - updated = opts.delete(:updated_at) - state = opts.delete(:state) - - ::ActiveRecord::Base.connection_pool.with_connection { - report = Report.new(opts) - report.created_at = created - report.updated_at = updated - - unless report.valid? - errors = report.errors.full_messages.join('; ') - raise RuntimeError "Report to be imported is not valid: #{errors}" - end - report.state = :complete # Presume complete since it was exported - report.save - - report.id - } - end - - # Creates a ReportArtifact based on passed parameters. - # @param opts [Hash] of ReportArtifact attributes - def report_artifact(opts) - return if not active - - artifacts_dir = Report::ARTIFACT_DIR - tmp_path = opts[:file_path] - artifact_name = File.basename tmp_path - new_path = File.join(artifacts_dir, artifact_name) - created = opts.delete(:created_at) - updated = opts.delete(:updated_at) - - unless File.exists? tmp_path - raise DBImportError 'Report artifact file to be imported does not exist.' - end - - unless (File.directory?(artifacts_dir) && File.writable?(artifacts_dir)) - raise DBImportError "Could not move report artifact file to #{artifacts_dir}." - end - - if File.exists? new_path - unique_basename = "#{(Time.now.to_f*1000).to_i}_#{artifact_name}" - new_path = File.join(artifacts_dir, unique_basename) - end - - FileUtils.copy(tmp_path, new_path) - opts[:file_path] = new_path - artifact = ReportArtifact.new(opts) - artifact.created_at = created - artifact.updated_at = updated - - unless artifact.valid? - errors = artifact.errors.full_messages.join('; ') - raise RuntimeError "Artifact to be imported is not valid: #{errors}" - end - artifact.save - end - - # - # This methods returns a list of all reports in the database - # - def reports(wspace=workspace) - ::ActiveRecord::Base.connection_pool.with_connection { - wspace.reports - } - end - - # - # WMAP - # Support methods - # - - # - # Report a Web Site to the database. WebSites must be tied to an existing Service - # - # opts MUST contain - # +:service+:: the service object this site should be associated with - # +:vhost+:: the virtual host name for this particular web site` - # - # If +:service+ is NOT specified, the following values are mandatory - # +:host+:: the ip address of the server hosting the web site - # +:port+:: the port number of the associated web site - # +:ssl+:: whether or not SSL is in use on this port - # - # These values will be used to create new host and service records - # - # opts can contain - # +:options+:: a hash of options for accessing this particular web site - # +:info+:: if present, report the service with this info - # - # Duplicate records for a given host, port, vhost combination will be overwritten - # - def report_web_site(opts) - return if not active - ::ActiveRecord::Base.connection_pool.with_connection { |conn| - wspace = opts.delete(:workspace) || workspace - vhost = opts.delete(:vhost) - - addr = nil - port = nil - name = nil - serv = nil - info = nil - - if opts[:service] and opts[:service].kind_of?(::Mdm::Service) - serv = opts[:service] - else - addr = opts[:host] - port = opts[:port] - name = opts[:ssl] ? 'https' : 'http' - info = opts[:info] - if not (addr and port) - raise ArgumentError, "report_web_site requires service OR host/port/ssl" - end - - # Force addr to be the address and not hostname - addr = Rex::Socket.getaddress(addr, true) - end - - ret = {} - - host = serv ? serv.host : find_or_create_host( - :workspace => wspace, - :host => addr, - :state => Msf::HostState::Alive - ) - - if host.name.to_s.empty? - host.name = vhost - host.save! - end - - serv = serv ? serv : find_or_create_service( - :workspace => wspace, - :host => host, - :port => port, - :proto => 'tcp', - :state => 'open' - ) - - # Change the service name if it is blank or it has - # been explicitly specified. - if opts.keys.include?(:ssl) or serv.name.to_s.empty? - name = opts[:ssl] ? 'https' : 'http' - serv.name = name - end - # Add the info if it's there. - unless info.to_s.empty? - serv.info = info - end - serv.save! if serv.changed? -=begin - host.updated_at = host.created_at - host.state = HostState::Alive - host.save! -=end - - vhost ||= host.address - site = ::Mdm::WebSite.find_or_initialize_by_vhost_and_service_id(vhost, serv[:id]) - site.options = opts[:options] if opts[:options] - - # XXX: - msf_import_timestamps(opts, site) - site.save! - - ret[:web_site] = site - } - end - - # - # Report a Web Page to the database. WebPage must be tied to an existing Web Site - # - # opts MUST contain - # +:web_site+:: the web site object that this page should be associated with - # +:path+:: the virtual host name for this particular web site - # +:code+:: the http status code from requesting this page - # +:headers+:: this is a HASH of headers (lowercase name as key) of ARRAYs of values - # +:body+:: the document body of the server response - # +:query+:: the query string after the path - # - # If web_site is NOT specified, the following values are mandatory - # +:host+:: the ip address of the server hosting the web site - # +:port+:: the port number of the associated web site - # +:vhost+:: the virtual host for this particular web site - # +:ssl+:: whether or not SSL is in use on this port - # - # These values will be used to create new host, service, and web_site records - # - # opts can contain - # +:cookie+:: the Set-Cookie headers, merged into a string - # +:auth+:: the Authorization headers, merged into a string - # +:ctype+:: the Content-Type headers, merged into a string - # +:mtime+:: the timestamp returned from the server of the last modification time - # +:location+:: the URL that a redirect points to - # - # Duplicate records for a given web_site, path, and query combination will be overwritten - # - - def report_web_page(opts) - return if not active - ::ActiveRecord::Base.connection_pool.with_connection { - wspace = opts.delete(:workspace) || workspace - - path = opts[:path] - code = opts[:code].to_i - body = opts[:body].to_s - query = opts[:query].to_s - headers = opts[:headers] - site = nil - - if not (path and code and body and headers) - raise ArgumentError, "report_web_page requires the path, query, code, body, and headers parameters" - end - - if opts[:web_site] and opts[:web_site].kind_of?(::Mdm::WebSite) - site = opts.delete(:web_site) - else - site = report_web_site( - :workspace => wspace, - :host => opts[:host], :port => opts[:port], - :vhost => opts[:host], :ssl => opts[:ssl] - ) - if not site - raise ArgumentError, "report_web_page was unable to create the associated web site" - end - end - - ret = {} - - page = ::Mdm::WebPage.find_or_initialize_by_web_site_id_and_path_and_query(site[:id], path, query) - page.code = code - page.body = body - page.headers = headers - page.cookie = opts[:cookie] if opts[:cookie] - page.auth = opts[:auth] if opts[:auth] - page.mtime = opts[:mtime] if opts[:mtime] - page.ctype = opts[:ctype] if opts[:ctype] - page.location = opts[:location] if opts[:location] - msf_import_timestamps(opts, page) - page.save! - - ret[:web_page] = page - } - - end - - - # - # Report a Web Form to the database. WebForm must be tied to an existing Web Site - # - # opts MUST contain - # +:web_site+:: the web site object that this page should be associated with - # +:path+:: the virtual host name for this particular web site - # +:query+:: the query string that is appended to the path (not valid for GET) - # +:method+:: the form method, one of GET, POST, or PATH - # +:params+:: an ARRAY of all parameters and values specified in the form - # - # If web_site is NOT specified, the following values are mandatory - # +:host+:: the ip address of the server hosting the web site - # +:port+:: the port number of the associated web site - # +:vhost+:: the virtual host for this particular web site - # +:ssl+:: whether or not SSL is in use on this port - # - # Duplicate records for a given web_site, path, method, and params combination will be overwritten - # - - def report_web_form(opts) - return if not active - ::ActiveRecord::Base.connection_pool.with_connection { - wspace = opts.delete(:workspace) || workspace - - path = opts[:path] - meth = opts[:method].to_s.upcase - para = opts[:params] - quer = opts[:query].to_s - site = nil - - if not (path and meth) - raise ArgumentError, "report_web_form requires the path and method parameters" - end - - if not %W{GET POST PATH}.include?(meth) - raise ArgumentError, "report_web_form requires the method to be one of GET, POST, PATH" - end - - if opts[:web_site] and opts[:web_site].kind_of?(::Mdm::WebSite) - site = opts.delete(:web_site) - else - site = report_web_site( - :workspace => wspace, - :host => opts[:host], :port => opts[:port], - :vhost => opts[:host], :ssl => opts[:ssl] - ) - if not site - raise ArgumentError, "report_web_form was unable to create the associated web site" - end - end - - ret = {} - - # Since one of our serialized fields is used as a unique parameter, we must do the final - # comparisons through ruby and not SQL. - - form = nil - ::Mdm::WebForm.find_all_by_web_site_id_and_path_and_method_and_query(site[:id], path, meth, quer).each do |xform| - if xform.params == para - form = xform - break - end - end - if not form - form = ::Mdm::WebForm.new - form.web_site_id = site[:id] - form.path = path - form.method = meth - form.params = para - form.query = quer - end - - msf_import_timestamps(opts, form) - form.save! - ret[:web_form] = form - } - end - - - # - # Report a Web Vuln to the database. WebVuln must be tied to an existing Web Site - # - # opts MUST contain - # +:web_site+:: the web site object that this page should be associated with - # +:path+:: the virtual host name for this particular web site - # +:query+:: the query string appended to the path (not valid for GET method flaws) - # +:method+:: the form method, one of GET, POST, or PATH - # +:params+:: an ARRAY of all parameters and values specified in the form - # +:pname+:: the specific field where the vulnerability occurs - # +:proof+:: the string showing proof of the vulnerability - # +:risk+:: an INTEGER value from 0 to 5 indicating the risk (5 is highest) - # +:name+:: the string indicating the type of vulnerability - # - # If web_site is NOT specified, the following values are mandatory - # +:host+:: the ip address of the server hosting the web site - # +:port+:: the port number of the associated web site - # +:vhost+:: the virtual host for this particular web site - # +:ssl+:: whether or not SSL is in use on this port - # - # - # Duplicate records for a given web_site, path, method, pname, and name - # combination will be overwritten - # - - def report_web_vuln(opts) - return if not active - ::ActiveRecord::Base.connection_pool.with_connection { - wspace = opts.delete(:workspace) || workspace - - path = opts[:path] - meth = opts[:method] - para = opts[:params] || [] - quer = opts[:query].to_s - pname = opts[:pname] - proof = opts[:proof] - risk = opts[:risk].to_i - name = opts[:name].to_s.strip - blame = opts[:blame].to_s.strip - desc = opts[:description].to_s.strip - conf = opts[:confidence].to_i - cat = opts[:category].to_s.strip - payload = opts[:payload].to_s - owner = opts[:owner] ? opts[:owner].shortname : nil - - - site = nil - - if not (path and meth and proof and pname) - raise ArgumentError, "report_web_vuln requires the path, method, proof, risk, name, params, and pname parameters. Received #{opts.inspect}" - end - - if not %W{GET POST PATH}.include?(meth) - raise ArgumentError, "report_web_vuln requires the method to be one of GET, POST, PATH. Received '#{meth}'" - end - - if risk < 0 or risk > 5 - raise ArgumentError, "report_web_vuln requires the risk to be between 0 and 5 (inclusive). Received '#{risk}'" - end - - if conf < 0 or conf > 100 - raise ArgumentError, "report_web_vuln requires the confidence to be between 1 and 100 (inclusive). Received '#{conf}'" - end - - if cat.empty? - raise ArgumentError, "report_web_vuln requires the category to be a valid string" - end - - if name.empty? - raise ArgumentError, "report_web_vuln requires the name to be a valid string" - end - - if opts[:web_site] and opts[:web_site].kind_of?(::Mdm::WebSite) - site = opts.delete(:web_site) - else - site = report_web_site( - :workspace => wspace, - :host => opts[:host], :port => opts[:port], - :vhost => opts[:host], :ssl => opts[:ssl] - ) - if not site - raise ArgumentError, "report_web_form was unable to create the associated web site" - end - end - - ret = {} - - meth = meth.to_s.upcase - - vuln = ::Mdm::WebVuln.find_or_initialize_by_web_site_id_and_path_and_method_and_pname_and_name_and_category_and_query(site[:id], path, meth, pname, name, cat, quer) - vuln.name = name - vuln.risk = risk - vuln.params = para - vuln.proof = proof.to_s - vuln.category = cat - vuln.blame = blame - vuln.description = desc - vuln.confidence = conf - vuln.payload = payload - vuln.owner = owner - - msf_import_timestamps(opts, vuln) - vuln.save! - - ret[:web_vuln] = vuln - } - end - - # - # WMAP - # Selected host - # - def selected_host - ::ActiveRecord::Base.connection_pool.with_connection { - selhost = ::Mdm::WmapTarget.where("selected != 0").first() - if selhost - return selhost.host - else - return - end - } - end - - # - # WMAP - # Selected target - # - def selected_wmap_target - ::ActiveRecord::Base.connection_pool.with_connection { - ::Mdm::WmapTarget.find.where("selected != 0") - } - end - - # - # WMAP - # Selected port - # - def selected_port - selected_wmap_target.port - end - - # - # WMAP - # Selected ssl - # - def selected_ssl - selected_wmap_target.ssl - end - - # - # WMAP - # Selected id - # - def selected_id - selected_wmap_target.object_id - end - - # - # WMAP - # This method iterates the requests table identifiying possible targets - # This method wiil be remove on second phase of db merging. - # - def each_distinct_target(&block) - request_distinct_targets.each do |target| - block.call(target) - end - end - - # - # WMAP - # This method returns a list of all possible targets available in requests - # This method wiil be remove on second phase of db merging. - # - def request_distinct_targets - ::ActiveRecord::Base.connection_pool.with_connection { - ::Mdm::WmapRequest.select('DISTINCT host,address,port,ssl') - } - end - - # - # WMAP - # This method iterates the requests table returning a list of all requests of a specific target - # - def each_request_target_with_path(&block) - target_requests('AND wmap_requests.path IS NOT NULL').each do |req| - block.call(req) - end - end - - # - # WMAP - # This method iterates the requests table returning a list of all requests of a specific target - # - def each_request_target_with_query(&block) - target_requests('AND wmap_requests.query IS NOT NULL').each do |req| - block.call(req) - end - end - - # - # WMAP - # This method iterates the requests table returning a list of all requests of a specific target - # - def each_request_target_with_body(&block) - target_requests('AND wmap_requests.body IS NOT NULL').each do |req| - block.call(req) - end - end - - # - # WMAP - # This method iterates the requests table returning a list of all requests of a specific target - # - def each_request_target_with_headers(&block) - target_requests('AND wmap_requests.headers IS NOT NULL').each do |req| - block.call(req) - end - end - - # - # WMAP - # This method iterates the requests table returning a list of all requests of a specific target - # - def each_request_target(&block) - target_requests('').each do |req| - block.call(req) - end - end - - # - # WMAP - # This method returns a list of all requests from target - # - def target_requests(extra_condition) - ::ActiveRecord::Base.connection_pool.with_connection { - ::Mdm::WmapRequest.where("wmap_requests.host = ? AND wmap_requests.port = ? #{extra_condition}",selected_host,selected_port) - } - end - - # - # WMAP - # This method iterates the requests table calling the supplied block with the - # request instance of each entry. - # - def each_request(&block) - requests.each do |request| - block.call(request) - end - end - - # - # WMAP - # This method allows to query directly the requests table. To be used mainly by modules - # - def request_sql(host,port,extra_condition) - ::ActiveRecord::Base.connection_pool.with_connection { - ::Mdm::WmapRequest.where("wmap_requests.host = ? AND wmap_requests.port = ? #{extra_condition}", host , port) - } - end - - # - # WMAP - # This methods returns a list of all targets in the database - # - def requests - ::ActiveRecord::Base.connection_pool.with_connection { - ::Mdm::WmapRequest.find(:all) - } - end - - # - # WMAP - # This method iterates the targets table calling the supplied block with the - # target instance of each entry. - # - def each_target(&block) - targets.each do |target| - block.call(target) - end - end - - # - # WMAP - # This methods returns a list of all targets in the database - # - def targets - ::ActiveRecord::Base.connection_pool.with_connection { - ::Mdm::WmapTarget.find(:all) - } - end - - # - # WMAP - # This methods deletes all targets from targets table in the database - # - def delete_all_targets - ::ActiveRecord::Base.connection_pool.with_connection { - ::Mdm::WmapTarget.delete_all - } - end - - # - # WMAP - # Find a target matching this id - # - def get_target(id) - ::ActiveRecord::Base.connection_pool.with_connection { - target = ::Mdm::WmapTarget.where("id = ?", id).first() - return target - } - end - - # - # WMAP - # Create a target - # - def create_target(host,port,ssl,sel) - ::ActiveRecord::Base.connection_pool.with_connection { - tar = ::Mdm::WmapTarget.create( - :host => host, - :address => host, - :port => port, - :ssl => ssl, - :selected => sel - ) - #framework.events.on_db_target(rec) - } - end - - - # - # WMAP - # Create a request (by hand) - # - def create_request(host,port,ssl,meth,path,headers,query,body,respcode,resphead,response) - ::ActiveRecord::Base.connection_pool.with_connection { - req = ::Mdm::WmapRequest.create( - :host => host, - :address => host, - :port => port, - :ssl => ssl, - :meth => meth, - :path => path, - :headers => headers, - :query => query, - :body => body, - :respcode => respcode, - :resphead => resphead, - :response => response - ) - #framework.events.on_db_request(rec) - } - end - - # - # WMAP - # Quick way to query the database (used by wmap_sql) - # - def sql_query(sqlquery) - ::ActiveRecord::Base.connection_pool.with_connection { - ActiveRecord::Base.connection.select_all(sqlquery) - } - end - - - # Returns a REXML::Document from the given data. - def rexmlify(data) - if data.kind_of?(REXML::Document) - return data - else - # Make an attempt to recover from a REXML import fail, since - # it's better than dying outright. - begin - return REXML::Document.new(data) - rescue REXML::ParseException => e - dlog("REXML error: Badly formatted XML, attempting to recover. Error was: #{e.inspect}") - return REXML::Document.new(data.gsub(/([\x00-\x08\x0b\x0c\x0e-\x1f\x80-\xff])/n){ |x| "\\x%.2x" % x.unpack("C*")[0] }) - end - end - end - - # Handles timestamps from Metasploit Express/Pro imports. - def msf_import_timestamps(opts,obj) - obj.created_at = opts["created_at"] if opts["created_at"] - obj.created_at = opts[:created_at] if opts[:created_at] - obj.updated_at = opts["updated_at"] ? opts["updated_at"] : obj.created_at - obj.updated_at = opts[:updated_at] ? opts[:updated_at] : obj.created_at - return obj - end - - ## - # - # Import methods - # - ## - - # - # Generic importer that automatically determines the file type being - # imported. Since this looks for vendor-specific strings in the given - # file, there shouldn't be any false detections, but no guarantees. - # - def import_file(args={}, &block) - filename = args[:filename] || args['filename'] - wspace = args[:wspace] || args['wspace'] || workspace - @import_filedata = {} - @import_filedata[:filename] = filename - - data = "" - ::File.open(filename, 'rb') do |f| - # This check is the largest (byte-wise) that we need to do - # since the other 4-byte checks will be subsets of this larger one. - data = f.read(Metasploit::Credential::Exporter::Pwdump::FILE_ID_STRING.size) - end - if data.nil? - raise DBImportError.new("Zero-length file") - end - - if data.index(Metasploit::Credential::Exporter::Pwdump::FILE_ID_STRING) - data = ::File.open(filename, 'rb') - else - case data[0,4] - when "PK\x03\x04" - data = Zip::File.open(filename) - when "\xd4\xc3\xb2\xa1", "\xa1\xb2\xc3\xd4" - data = PacketFu::PcapFile.new(:filename => filename) - else - ::File.open(filename, 'rb') do |f| - sz = f.stat.size - data = f.read(sz) - end - end - end - - - if block - import(args.merge(:data => data)) { |type,data| yield type,data } - else - import(args.merge(:data => data)) - end - end - - # A dispatcher method that figures out the data's file type, - # and sends it off to the appropriate importer. Note that - # import_file_detect will raise an error if the filetype - # is unknown. - def import(args={}, &block) - data = args[:data] || args['data'] - ftype = import_filetype_detect(data) - yield(:filetype, @import_filedata[:type]) if block - self.send "import_#{ftype}".to_sym, args, &block - end - - # Returns one of the following: - # - # :acunetix_xml - # :amap_log - # :amap_mlog - # :appscan_xml - # :burp_session_xml - # :ci_xml - # :foundstone_xml - # :fusionvm_xml - # :ip360_aspl_xml - # :ip360_xml_v3 - # :ip_list - # :libpcap - # :mbsa_xml - # :msf_cred_dump_zip - # :msf_pwdump - # :msf_xml - # :msf_zip - # :nessus_nbe - # :nessus_xml - # :nessus_xml_v2 - # :netsparker_xml - # :nexpose_rawxml - # :nexpose_simplexml - # :nikto_xml - # :nmap_xml - # :openvas_new_xml - # :openvas_xml - # :outpost24_xml - # :qualys_asset_xml - # :qualys_scan_xml - # :retina_xml - # :spiceworks_csv - # :wapiti_xml - # - # If there is no match, an error is raised instead. - # - # @raise DBImportError if the type can't be detected - def import_filetype_detect(data) - - if data and data.kind_of? Zip::File - if data.entries.empty? - raise DBImportError.new("The zip file provided is empty.") - end - - @import_filedata ||= {} - @import_filedata[:zip_filename] = File.split(data.to_s).last - @import_filedata[:zip_basename] = @import_filedata[:zip_filename].gsub(/\.zip$/,"") - @import_filedata[:zip_entry_names] = data.entries.map {|x| x.name} - - if @import_filedata[:zip_entry_names].include?(Metasploit::Credential::Importer::Zip::MANIFEST_FILE_NAME) - @import_filedata[:type] = "Metasploit Credential Dump" - return :msf_cred_dump_zip - end - - xml_files = @import_filedata[:zip_entry_names].grep(/^(.*)\.xml$/) - - # TODO This check for our zip export should be more extensive - if xml_files.empty? - raise DBImportError.new("The zip file provided is not a Metasploit Zip Export") - end - - @import_filedata[:zip_xml] = xml_files.first - @import_filedata[:type] = "Metasploit Zip Export" - - return :msf_zip - end - - if data and data.kind_of? PacketFu::PcapFile - # Don't check for emptiness here because unlike other formats, we - # haven't read any actual data in yet, only magic bytes to discover - # that this is indeed a pcap file. - #raise DBImportError.new("The pcap file provided is empty.") if data.body.empty? - @import_filedata ||= {} - @import_filedata[:type] = "Libpcap Packet Capture" - return :libpcap - end - - # msfpwdump - if data.present? && data.kind_of?(::File) - @import_filedata[:type] = "Metasploit PWDump Export" - return :msf_pwdump - end - - # This is a text string, lets make sure its treated as binary - data = data.unpack("C*").pack("C*") - if data and data.to_s.strip.length == 0 - raise DBImportError.new("The data provided to the import function was empty") - end - - # Parse the first line or 4k of data from the file - di = data.index("\n") || 4096 - - firstline = data[0, di] - @import_filedata ||= {} - if (firstline.index("")) - @import_filedata[:type] = "Retina XML" - return :retina_xml - elsif (firstline.index(//)) - @import_filedata[:type] = "OpenVAS XML" - return :openvas_new_xml - elsif (firstline.index(/")) - @import_filedata[:type] = "Nessus XML (v1)" - return :nessus_xml - elsif (firstline.index("]/ - - case $1 - when "niktoscan" - @import_filedata[:type] = "Nikto XML" - return :nikto_xml - when "nmaprun" - @import_filedata[:type] = "Nmap XML" - return :nmap_xml - when "openvas-report" - @import_filedata[:type] = "OpenVAS Report" - return :openvas_xml - when "NessusClientData" - @import_filedata[:type] = "Nessus XML (v1)" - return :nessus_xml - when "NessusClientData_v2" - @import_filedata[:type] = "Nessus XML (v2)" - return :nessus_xml_v2 - when "SCAN" - @import_filedata[:type] = "Qualys Scan XML" - return :qualys_scan_xml - when "report" - @import_filedata[:type] = "Wapiti XML" - return :wapiti_xml - when "ASSET_DATA_REPORT" - @import_filedata[:type] = "Qualys Asset XML" - return :qualys_asset_xml - when /MetasploitExpressV[1234]/ - @import_filedata[:type] = "Metasploit XML" - return :msf_xml - when /MetasploitV4/ - @import_filedata[:type] = "Metasploit XML" - return :msf_xml - when /netsparker/ - @import_filedata[:type] = "NetSparker XML" - return :netsparker_xml - when /audits?/ # and are both valid for nCircle. wtfmate. - @import_filedata[:type] = "IP360 XML v3" - return :ip360_xml_v3 - when /ontology/ - @import_filedata[:type] = "IP360 ASPL" - return :ip360_aspl_xml - when /ReportInfo/ - @import_filedata[:type] = "Foundstone" - return :foundstone_xml - when /ScanGroup/ - @import_filedata[:type] = "Acunetix" - return :acunetix_xml - when /AppScanInfo/ # Actually the second line - @import_filedata[:type] = "Appscan" - return :appscan_xml - when "entities" - if line =~ /creator.*\x43\x4f\x52\x45\x20\x49\x4d\x50\x41\x43\x54/ni - @import_filedata[:type] = "CI" - return :ci_xml - end - when "main" - @import_filedata[:type] = "Outpost24 XML" - return :outpost24_xml - else - # Give up if we haven't hit the root tag in the first few lines - break if line_count > 10 - end - line_count += 1 - } - elsif (firstline.index("timestamps|||scan_start")) - @import_filedata[:type] = "Nessus NBE Report" - # then it's a nessus nbe - return :nessus_nbe - elsif (firstline.index("# amap v")) - # then it's an amap mlog - @import_filedata[:type] = "Amap Log -m" - return :amap_mlog - elsif (firstline.index("amap v")) - # then it's an amap log - @import_filedata[:type] = "Amap Log" - return :amap_log - elsif ipv46_validator(firstline) - # then its an IP list - @import_filedata[:type] = "IP Address List" - return :ip_list - elsif (data[0,1024].index(" wspace, - :host => addr, - :type => "service.nikto.scan.description", - :data => desc_text, - :proto => "tcp", - :port => port.to_i, - :sname => uri.scheme, - :update => :unique_data, - :task => args[:task] - } - # Always report it as a note. - report_note(desc_data) - # Sometimes report it as a vuln, too. - # XXX: There's a Vuln.info field but nothing reads from it? See Bug #5837 - if item.attributes['osvdbid'].to_i != 0 - desc_data[:refs] = ["OSVDB-#{item.attributes['osvdbid']}"] - desc_data[:name] = "NIKTO-#{item.attributes['id']}" - desc_data.delete(:data) - desc_data.delete(:type) - desc_data.delete(:update) - report_vuln(desc_data) - end - end - end - end - end - end - - def import_wapiti_xml_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - - data = "" - ::File.open(filename, 'rb') do |f| - data = f.read(f.stat.size) - end - import_wapiti_xml(args.merge(:data => data)) - end - - def import_wapiti_xml(args={}, &block) - if block - doc = Rex::Parser::WapitiDocument.new(args,framework.db) {|type, data| yield type,data } - else - doc = Rex::Parser::WapitiDocument.new(args,self) - end - parser = ::Nokogiri::XML::SAX::Parser.new(doc) - parser.parse(args[:data]) - end - - def import_openvas_new_xml_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - - data = "" - ::File.open(filename, 'rb') do |f| - data = f.read(f.stat.size) - end - import_wapiti_xml(args.merge(:data => data)) - end - - def import_openvas_new_xml(args={}, &block) - if block - doc = Rex::Parser::OpenVASDocument.new(args,framework.db) {|type, data| yield type,data } - else - doc = Rex::Parser::OpenVASDocument.new(args,self) - end - parser = ::Nokogiri::XML::SAX::Parser.new(doc) - parser.parse(args[:data]) - end - - def import_libpcap_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - - data = PacketFu::PcapFile.new(:filename => filename) - import_libpcap(args.merge(:data => data)) - end - - # The libpcap file format is handled by PacketFu for data - # extraction. TODO: Make this its own mixin, and possibly - # extend PacketFu to do better stream analysis on the fly. - def import_libpcap(args={}, &block) - data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - # seen_hosts is only used for determining when to yield an address. Once we get - # some packet analysis going, the values will have all sorts of info. The plan - # is to ru through all the packets as a first pass and report host and service, - # then, once we have everything parsed, we can reconstruct sessions and ngrep - # out things like authentication sequences, examine ttl's and window sizes, all - # kinds of crazy awesome stuff like that. - seen_hosts = {} - decoded_packets = 0 - last_count = 0 - data.read_packet_bytes do |p| - if (decoded_packets >= last_count + 1000) and block - yield(:pcap_count, decoded_packets) - last_count = decoded_packets - end - decoded_packets += 1 - - pkt = PacketFu::Packet.parse(p) rescue next # Just silently skip bad packets - - next unless pkt.is_ip? # Skip anything that's not IP. Technically, not Ethernet::Ip - next if pkt.is_tcp? && (pkt.tcp_src == 0 || pkt.tcp_dst == 0) # Skip port 0 - next if pkt.is_udp? && (pkt.udp_src == 0 || pkt.udp_dst == 0) # Skip port 0 - saddr = pkt.ip_saddr - daddr = pkt.ip_daddr - - # Handle blacklists and obviously useless IP addresses, and report the host. - next if (bl | [saddr,daddr]).size == bl.size # Both hosts are blacklisted, skip everything. - unless( bl.include?(saddr) || rfc3330_reserved(saddr)) - yield(:address,saddr) if block and !seen_hosts.keys.include?(saddr) - unless seen_hosts[saddr] - report_host( - :workspace => wspace, - :host => saddr, - :state => Msf::HostState::Alive, - :task => args[:task] - ) - end - seen_hosts[saddr] ||= [] - - end - unless( bl.include?(daddr) || rfc3330_reserved(daddr)) - yield(:address,daddr) if block and !seen_hosts.keys.include?(daddr) - unless seen_hosts[daddr] - report_host( - :workspace => wspace, - :host => daddr, - :state => Msf::HostState::Alive, - :task => args[:task] - ) - end - seen_hosts[daddr] ||= [] - end - - if pkt.is_tcp? # First pass on TCP packets - if (pkt.tcp_flags.syn == 1 and pkt.tcp_flags.ack == 1) or # Oh, this kills me - pkt.tcp_src < 1024 # If it's a low port, assume it's a proper service. - if seen_hosts[saddr] - unless seen_hosts[saddr].include? [pkt.tcp_src,"tcp"] - report_service( - :workspace => wspace, :host => saddr, - :proto => "tcp", :port => pkt.tcp_src, - :state => Msf::ServiceState::Open, - :task => args[:task] - ) - seen_hosts[saddr] << [pkt.tcp_src,"tcp"] - yield(:service,"%s:%d/%s" % [saddr,pkt.tcp_src,"tcp"]) - end - end - end - elsif pkt.is_udp? # First pass on UDP packets - if pkt.udp_src == pkt.udp_dst # Very basic p2p detection. - [saddr,daddr].each do |xaddr| - if seen_hosts[xaddr] - unless seen_hosts[xaddr].include? [pkt.udp_src,"udp"] - report_service( - :workspace => wspace, :host => xaddr, - :proto => "udp", :port => pkt.udp_src, - :state => Msf::ServiceState::Open, - :task => args[:task] - ) - seen_hosts[xaddr] << [pkt.udp_src,"udp"] - yield(:service,"%s:%d/%s" % [xaddr,pkt.udp_src,"udp"]) - end - end - end - elsif pkt.udp_src < 1024 # Probably a service - if seen_hosts[saddr] - unless seen_hosts[saddr].include? [pkt.udp_src,"udp"] - report_service( - :workspace => wspace, :host => saddr, - :proto => "udp", :port => pkt.udp_src, - :state => Msf::ServiceState::Open, - :task => args[:task] - ) - seen_hosts[saddr] << [pkt.udp_src,"udp"] - yield(:service,"%s:%d/%s" % [saddr,pkt.udp_src,"udp"]) - end - end - end - end # tcp or udp - - inspect_single_packet(pkt,wspace,args) - - end # data.body.map - - # Right about here, we should have built up some streams for some stream analysis. - # Not sure what form that will take, but people like shoving many hundreds of - # thousands of packets through this thing, so it'll need to be memory efficient. - - end - - # Do all the single packet analysis we can while churning through the pcap - # the first time. Multiple packet inspection will come later, where we can - # do stream analysis, compare requests and responses, etc. - def inspect_single_packet(pkt,wspace,args) - if pkt.is_tcp? or pkt.is_udp? - inspect_single_packet_http(pkt,wspace,args) - end - end - - # Checks for packets that are headed towards port 80, are tcp, contain an HTTP/1.0 - # line, contains an Authorization line, contains a b64-encoded credential, and - # extracts it. Reports this credential and solidifies the service as HTTP. - def inspect_single_packet_http(pkt,wspace,args) - task = args.fetch(:task, nil) - # First, check the server side (data from port 80). - if pkt.is_tcp? and pkt.tcp_src == 80 and !pkt.payload.nil? and !pkt.payload.empty? - if pkt.payload =~ /^HTTP\x2f1\x2e[01]/n - http_server_match = pkt.payload.match(/\nServer:\s+([^\r\n]+)[\r\n]/n) - if http_server_match.kind_of?(MatchData) and http_server_match[1] - report_service( - :workspace => wspace, - :host => pkt.ip_saddr, - :port => pkt.tcp_src, - :proto => "tcp", - :name => "http", - :info => http_server_match[1], - :state => Msf::ServiceState::Open, - :task => task - ) - # That's all we want to know from this service. - return :something_significant - end - end - end - - # Next, check the client side (data to port 80) - if pkt.is_tcp? and pkt.tcp_dst == 80 and !pkt.payload.nil? and !pkt.payload.empty? - if pkt.payload.match(/[\x00-\x20]HTTP\x2f1\x2e[10]/n) - auth_match = pkt.payload.match(/\nAuthorization:\s+Basic\s+([A-Za-z0-9=\x2b]+)/n) - if auth_match.kind_of?(MatchData) and auth_match[1] - b64_cred = auth_match[1] - else - return false - end - # If we're this far, we can surmise that at least the client is a web browser, - # he thinks the server is HTTP and he just made an authentication attempt. At - # this point, we'll just believe everything the packet says -- validation ought - # to come later. - user,pass = b64_cred.unpack("m*").first.split(/:/,2) - report_service( - :workspace => wspace, - :host => pkt.ip_daddr, - :port => pkt.tcp_dst, - :proto => "tcp", - :name => "http", - :task => task - ) - - service_data = { - address: pkt.ip_daddr, - port: pkt.tcp_dst, - service_name: 'http', - protocol: 'tcp', - workspace_id: wspace.id - } - service_data[:task_id] = task.id if task - - filename = args[:filename] - - credential_data = { - origin_type: :import, - private_data: pass, - private_type: :password, - username: user, - filename: filename - } - credential_data.merge!(service_data) - credential_core = create_credential(credential_data) - - login_data = { - core: credential_core, - status: Metasploit::Model::Login::Status::UNTRIED - } - - login_data.merge!(service_data) - - create_credential_login(login_data) - - # That's all we want to know from this service. - return :something_significant - end - end - end - - def import_spiceworks_csv(args={}, &block) - data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - CSV.parse(data) do |row| - next unless (["Name", "Manufacturer", "Device Type"] & row).empty? #header - name = row[0] - manufacturer = row[1] - device = row[2] - model = row[3] - ip = row[4] - serialno = row[5] - location = row[6] - os = row[7] - - next unless ip - next if bl.include? ip - - conf = { - :workspace => wspace, - :host => ip, - :name => name, - :task => args[:task] - } - - - if os - report_note( - :workspace => wspace, - :task => args[:task], - :host => ip, - :type => 'host.os.spiceworks_fingerprint', - :data => { - :os => os.to_s.strip - } - ) - end - - info = [] - info << "Serial Number: #{serialno}" unless (serialno.blank? or serialno == name) - info << "Location: #{location}" unless location.blank? - conf[:info] = info.join(", ") unless info.empty? - - host = report_host(conf) - report_import_note(wspace, host) - end - end - - - # Perform in an import of an msfpwdump file - def import_msf_pwdump(args={}, &block) - filename = File.basename(args[:data].path) - wspace = args[:wspace] || workspace - origin = Metasploit::Credential::Origin::Import.create!(filename: filename) - importer = Metasploit::Credential::Importer::Pwdump.new(input: args[:data], workspace: wspace, filename: filename, origin:origin) - importer.import! - importer.input.close unless importer.input.closed? - end - - # If hex notation is present, turn them into a character. - def dehex(str) - hexen = str.scan(/\x5cx[0-9a-fA-F]{2}/n) - hexen.each { |h| - str.gsub!(h,h[2,2].to_i(16).chr) - } - return str - end - - - # - # Nexpose Simple XML - # - # XXX At some point we'll want to make this a stream parser for dealing - # with large results files - # - def import_nexpose_simplexml_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - - data = "" - ::File.open(filename, 'rb') do |f| - data = f.read(f.stat.size) - end - import_nexpose_simplexml(args.merge(:data => data)) - end - - # Import a Metasploit XML file. - def import_msf_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - - data = "" - ::File.open(filename, 'rb') do |f| - data = f.read(f.stat.size) - end - import_msf_xml(args.merge(:data => data)) - end - - # Import a Metasploit Express ZIP file. Note that this requires - # a fair bit of filesystem manipulation, and is very much tied - # up with the Metasploit Express ZIP file format export (for - # obvious reasons). In the event directories exist, they will - # be reused. If target files exist, they will be overwritten. - # - # XXX: Refactor so it's not quite as sanity-blasting. - def import_msf_zip(args={}, &block) - data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - - new_tmp = ::File.join(Dir::tmpdir,"msf","imp_#{Rex::Text::rand_text_alphanumeric(4)}",@import_filedata[:zip_basename]) - if ::File.exists? new_tmp - unless (::File.directory?(new_tmp) && ::File.writable?(new_tmp)) - raise DBImportError.new("Could not extract zip file to #{new_tmp}") - end - else - FileUtils.mkdir_p(new_tmp) - end - @import_filedata[:zip_tmp] = new_tmp - - # Grab the list of unique basedirs over all entries. - @import_filedata[:zip_tmp_subdirs] = @import_filedata[:zip_entry_names].map {|x| ::File.split(x)}.map {|x| x[0]}.uniq.reject {|x| x == "."} - - # mkdir all of the base directores we just pulled out, if they don't - # already exist - @import_filedata[:zip_tmp_subdirs].each {|sub| - tmp_subdirs = ::File.join(@import_filedata[:zip_tmp],sub) - if File.exists? tmp_subdirs - unless (::File.directory?(tmp_subdirs) && File.writable?(tmp_subdirs)) - # if it exists but we can't write to it, give up - raise DBImportError.new("Could not extract zip file to #{tmp_subdirs}") - end - else - ::FileUtils.mkdir(tmp_subdirs) - end - } - - data.entries.each do |e| - target = ::File.join(@import_filedata[:zip_tmp], e.name) - data.extract(e,target) - - if target =~ /\.xml\z/ - target_data = ::File.open(target, "rb") {|f| f.read 1024} - if import_filetype_detect(target_data) == :msf_xml - @import_filedata[:zip_extracted_xml] = target - end - end - end - - # Import any creds if there are some in the import file - Dir.entries(@import_filedata[:zip_tmp]).each do |entry| - if entry =~ /^.*#{Regexp.quote(Metasploit::Credential::Exporter::Core::CREDS_DUMP_FILE_IDENTIFIER)}.*/ - manifest_file_path = File.join(@import_filedata[:zip_tmp], entry, Metasploit::Credential::Importer::Zip::MANIFEST_FILE_NAME) - if File.exists? manifest_file_path - import_msf_cred_dump(manifest_file_path, wspace) - end - end - end - - # This will kick the newly-extracted XML file through - # the import_file process all over again. - if @import_filedata[:zip_extracted_xml] - new_args = args.dup - new_args[:filename] = @import_filedata[:zip_extracted_xml] - new_args[:data] = nil - new_args[:ifd] = @import_filedata.dup - if block - import_file(new_args, &block) - else - import_file(new_args) - end - end - - # Kick down to all the MSFX ZIP specific items - if block - import_msf_collateral(new_args, &block) - else - import_msf_collateral(new_args) - end - end - - # Imports loot, tasks, and reports from an MSF ZIP report. - # XXX: This function is stupidly long. It needs to be refactored. - def import_msf_collateral(args={}, &block) - data = ::File.open(args[:filename], "rb") {|f| f.read(f.stat.size)} - wspace = args[:wspace] || args['wspace'] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - basedir = args[:basedir] || args['basedir'] || ::File.join(Msf::Config.data_directory, "msf") - - allow_yaml = false - btag = nil - - doc = rexmlify(data) - if doc.elements["MetasploitExpressV1"] - m_ver = 1 - allow_yaml = true - btag = "MetasploitExpressV1" - elsif doc.elements["MetasploitExpressV2"] - m_ver = 2 - allow_yaml = true - btag = "MetasploitExpressV2" - elsif doc.elements["MetasploitExpressV3"] - m_ver = 3 - btag = "MetasploitExpressV3" - elsif doc.elements["MetasploitExpressV4"] - m_ver = 4 - btag = "MetasploitExpressV4" - elsif doc.elements["MetasploitV4"] - m_ver = 4 - btag = "MetasploitV4" - else - m_ver = nil - end - unless m_ver and btag - raise DBImportError.new("Unsupported Metasploit XML document format") - end - - host_info = {} - doc.elements.each("/#{btag}/hosts/host") do |host| - host_info[host.elements["id"].text.to_s.strip] = nils_for_nulls(host.elements["address"].text.to_s.strip) - end - - # Import Loot - doc.elements.each("/#{btag}/loots/loot") do |loot| - next if bl.include? host_info[loot.elements["host-id"].text.to_s.strip] - loot_info = {} - loot_info[:host] = host_info[loot.elements["host-id"].text.to_s.strip] - loot_info[:workspace] = args[:wspace] - loot_info[:ctype] = nils_for_nulls(loot.elements["content-type"].text.to_s.strip) - loot_info[:info] = nils_for_nulls(unserialize_object(loot.elements["info"], allow_yaml)) - loot_info[:ltype] = nils_for_nulls(loot.elements["ltype"].text.to_s.strip) - loot_info[:name] = nils_for_nulls(loot.elements["name"].text.to_s.strip) - loot_info[:created_at] = nils_for_nulls(loot.elements["created-at"].text.to_s.strip) - loot_info[:updated_at] = nils_for_nulls(loot.elements["updated-at"].text.to_s.strip) - loot_info[:name] = nils_for_nulls(loot.elements["name"].text.to_s.strip) - loot_info[:orig_path] = nils_for_nulls(loot.elements["path"].text.to_s.strip) - loot_info[:task] = args[:task] - tmp = args[:ifd][:zip_tmp] - loot_info[:orig_path].gsub!(/^\./,tmp) if loot_info[:orig_path] - if !loot.elements["service-id"].text.to_s.strip.empty? - unless loot.elements["service-id"].text.to_s.strip == "NULL" - loot_info[:service] = loot.elements["service-id"].text.to_s.strip - end - end - - # Only report loot if we actually have it. - # TODO: Copypasta. Seperate this out. - if ::File.exists? loot_info[:orig_path] - loot_dir = ::File.join(basedir,"loot") - loot_file = ::File.split(loot_info[:orig_path]).last - if ::File.exists? loot_dir - unless (::File.directory?(loot_dir) && ::File.writable?(loot_dir)) - raise DBImportError.new("Could not move files to #{loot_dir}") - end - else - ::FileUtils.mkdir_p(loot_dir) - end - new_loot = ::File.join(loot_dir,loot_file) - loot_info[:path] = new_loot - if ::File.exists?(new_loot) - ::File.unlink new_loot # Delete it, and don't report it. - else - report_loot(loot_info) # It's new, so report it. - end - ::FileUtils.copy(loot_info[:orig_path], new_loot) - yield(:msf_loot, new_loot) if block - end - end - - # Import Tasks - doc.elements.each("/#{btag}/tasks/task") do |task| - task_info = {} - task_info[:workspace] = args[:wspace] - # Should user be imported (original) or declared (the importing user)? - task_info[:user] = nils_for_nulls(task.elements["created-by"].text.to_s.strip) - task_info[:desc] = nils_for_nulls(task.elements["description"].text.to_s.strip) - task_info[:info] = nils_for_nulls(unserialize_object(task.elements["info"], allow_yaml)) - task_info[:mod] = nils_for_nulls(task.elements["module"].text.to_s.strip) - task_info[:options] = nils_for_nulls(task.elements["options"].text.to_s.strip) - task_info[:prog] = nils_for_nulls(task.elements["progress"].text.to_s.strip).to_i - task_info[:created_at] = nils_for_nulls(task.elements["created-at"].text.to_s.strip) - task_info[:updated_at] = nils_for_nulls(task.elements["updated-at"].text.to_s.strip) - if !task.elements["completed-at"].text.to_s.empty? - task_info[:completed_at] = nils_for_nulls(task.elements["completed-at"].text.to_s.strip) - end - if !task.elements["error"].text.to_s.empty? - task_info[:error] = nils_for_nulls(task.elements["error"].text.to_s.strip) - end - if !task.elements["result"].text.to_s.empty? - task_info[:result] = nils_for_nulls(task.elements["result"].text.to_s.strip) - end - task_info[:orig_path] = nils_for_nulls(task.elements["path"].text.to_s.strip) - tmp = args[:ifd][:zip_tmp] - task_info[:orig_path].gsub!(/^\./,tmp) if task_info[:orig_path] - - # Only report a task if we actually have it. - # TODO: Copypasta. Seperate this out. - if ::File.exists? task_info[:orig_path] - tasks_dir = ::File.join(basedir,"tasks") - task_file = ::File.split(task_info[:orig_path]).last - if ::File.exists? tasks_dir - unless (::File.directory?(tasks_dir) && ::File.writable?(tasks_dir)) - raise DBImportError.new("Could not move files to #{tasks_dir}") - end - else - ::FileUtils.mkdir_p(tasks_dir) - end - new_task = ::File.join(tasks_dir,task_file) - task_info[:path] = new_task - if ::File.exists?(new_task) - ::File.unlink new_task # Delete it, and don't report it. - else - report_task(task_info) # It's new, so report it. - end - ::FileUtils.copy(task_info[:orig_path], new_task) - yield(:msf_task, new_task) if block - end - end - - # Import Reports - doc.elements.each("/#{btag}/reports/report") do |report| - import_report(report, args, basedir) - end - end - - # Import credentials given a path to a valid manifest file - # - # @param creds_dump_manifest_path [String] - # @param workspace [Mdm::Workspace] Default: {#workspace} - # @return [void] - def import_msf_cred_dump(creds_dump_manifest_path, workspace) - manifest_file = File.open(creds_dump_manifest_path) - origin = Metasploit::Credential::Origin::Import.create!(filename: File.basename(creds_dump_manifest_path)) - importer = Metasploit::Credential::Importer::Core.new(workspace: workspace, input: manifest_file, origin: origin) - importer.import! - end - - # Import credentials given a path to a valid manifest file - # - # @option args [String] :filename - # @option args [Mdm::Workspace] :wspace Default: {#workspace} - # @return [void] - def import_msf_cred_dump_zip(args = {}) - wspace = args[:wspace] || workspace - origin = Metasploit::Credential::Origin::Import.create!(filename: File.basename(args[:filename])) - importer = Metasploit::Credential::Importer::Zip.new(workspace: wspace, input: File.open(args[:filename]), origin: origin) - importer.import! - nil - end - - # @param report [REXML::Element] to be imported - # @param args [Hash] - # @param base_dir [String] - def import_report(report, args, base_dir) - tmp = args[:ifd][:zip_tmp] - report_info = {} - - report.elements.each do |e| - node_name = e.name - node_value = e.text - - # These need to be converted back to arrays: - array_attrs = %w|addresses file-formats options sections| - if array_attrs.member? node_name - node_value = JSON.parse(node_value) - end - # Don't restore these values: - skip_nodes = %w|id workspace-id artifacts| - next if skip_nodes.member? node_name - - report_info[node_name.parameterize.underscore.to_sym] = node_value - end - # Use current workspace - report_info[:workspace_id] = args[:wspace].id - - # Create report, need new ID to record artifacts - report_id = report_report(report_info) - - # Handle artifacts - report.elements['artifacts'].elements.each do |artifact| - artifact_opts = {} - artifact.elements.each do |attr| - skip_nodes = %w|id accessed-at| - next if skip_nodes.member? attr.name - - symboled_attr = attr.name.parameterize.underscore.to_sym - artifact_opts[symboled_attr] = attr.text - end - # Use new Report as parent - artifact_opts[:report_id] = report_id - # Update to full path - artifact_opts[:file_path].gsub!(/^\./, tmp) - - report_artifact(artifact_opts) - end - end - - # Convert the string "NULL" to actual nil - def nils_for_nulls(str) - str == "NULL" ? nil : str - end - - def import_nexpose_simplexml(args={}, &block) - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - wspace = args[:wspace] || workspace - if Rex::Parser.nokogiri_loaded - parser = "Nokogiri v#{::Nokogiri::VERSION}" - noko_args = args.dup - noko_args[:blacklist] = bl - noko_args[:wspace] = wspace - if block - yield(:parser, parser) - import_nexpose_noko_stream(noko_args) {|type, data| yield type,data} - else - import_nexpose_noko_stream(noko_args) - end - return true - end - data = args[:data] - - doc = rexmlify(data) - doc.elements.each('/NeXposeSimpleXML/devices/device') do |dev| - addr = dev.attributes['address'].to_s - if bl.include? addr - next - else - yield(:address,addr) if block - end - - fprint = {} - - dev.elements.each('fingerprint/description') do |str| - fprint[:desc] = str.text.to_s.strip - end - dev.elements.each('fingerprint/vendor') do |str| - fprint[:vendor] = str.text.to_s.strip - end - dev.elements.each('fingerprint/family') do |str| - fprint[:family] = str.text.to_s.strip - end - dev.elements.each('fingerprint/product') do |str| - fprint[:product] = str.text.to_s.strip - end - dev.elements.each('fingerprint/version') do |str| - fprint[:version] = str.text.to_s.strip - end - dev.elements.each('fingerprint/architecture') do |str| - fprint[:arch] = str.text.to_s.upcase.strip - end - - conf = { - :workspace => wspace, - :host => addr, - :state => Msf::HostState::Alive, - :task => args[:task] - } - - host = report_host(conf) - report_import_note(wspace, host) - - report_note( - :workspace => wspace, - :host => host, - :type => 'host.os.nexpose_fingerprint', - :data => fprint, - :task => args[:task] - ) - - # Load vulnerabilities not associated with a service - dev.elements.each('vulnerabilities/vulnerability') do |vuln| - vid = vuln.attributes['id'].to_s.downcase - refs = process_nexpose_data_sxml_refs(vuln) - next if not refs - report_vuln( - :workspace => wspace, - :host => host, - :name => 'NEXPOSE-' + vid, - :info => vid, - :refs => refs, - :task => args[:task] - ) - end - - # Load the services - dev.elements.each('services/service') do |svc| - sname = svc.attributes['name'].to_s - sprot = svc.attributes['protocol'].to_s.downcase - sport = svc.attributes['port'].to_s.to_i - next if sport == 0 - - name = sname.split('(')[0].strip - info = '' - - svc.elements.each('fingerprint/description') do |str| - info = str.text.to_s.strip - end - - if(sname.downcase != '') - report_service( - :workspace => wspace, - :host => host, - :proto => sprot, - :port => sport, - :name => name, - :info => info, - :task => args[:task] - ) - else - report_service( - :workspace => wspace, - :host => host, - :proto => sprot, - :port => sport, - :info => info, - :task => args[:task] - ) - end - - # Load vulnerabilities associated with this service - svc.elements.each('vulnerabilities/vulnerability') do |vuln| - vid = vuln.attributes['id'].to_s.downcase - refs = process_nexpose_data_sxml_refs(vuln) - next if not refs - report_vuln( - :workspace => wspace, - :host => host, - :port => sport, - :proto => sprot, - :name => 'NEXPOSE-' + vid, - :info => vid, - :refs => refs, - :task => args[:task] - ) - end - end - end - end - - - # - # Nexpose Raw XML - # - def import_nexpose_rawxml_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - - data = "" - ::File.open(filename, 'rb') do |f| - data = f.read(f.stat.size) - end - import_nexpose_rawxml(args.merge(:data => data)) - end - - def import_nexpose_rawxml(args={}, &block) - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - wspace = args[:wspace] || workspace - if Rex::Parser.nokogiri_loaded - parser = "Nokogiri v#{::Nokogiri::VERSION}" - noko_args = args.dup - noko_args[:blacklist] = bl - noko_args[:wspace] = wspace - if block - yield(:parser, parser) - import_nexpose_raw_noko_stream(noko_args) {|type, data| yield type,data} - else - import_nexpose_raw_noko_stream(noko_args) - end - return true - end - data = args[:data] - - # Use a stream parser instead of a tree parser so we can deal with - # huge results files without running out of memory. - parser = Rex::Parser::NexposeXMLStreamParser.new - - # Since all the Refs have to be in the database before we can use them - # in a Vuln, we store all the hosts until we finish parsing and only - # then put everything in the database. This is memory-intensive for - # large files, but should be much less so than a tree parser. - # - # This method is also considerably faster than parsing through the tree - # looking for references every time we hit a vuln. - hosts = [] - vulns = [] - - # The callback merely populates our in-memory table of hosts and vulns - parser.callback = Proc.new { |type, value| - case type - when :host - # XXX: Blacklist should be checked here instead of saving a - # host we're just going to throw away later - hosts.push(value) - when :vuln - value["id"] = value["id"].downcase if value["id"] - vulns.push(value) - end - } - - REXML::Document.parse_stream(data, parser) - - vuln_refs = nexpose_refs_to_struct(vulns) - hosts.each do |host| - if bl.include? host["addr"] - next - else - yield(:address,host["addr"]) if block - end - nexpose_host_from_rawxml(host, vuln_refs, wspace) - end - end - - # - # Takes an array of vuln hashes, as returned by the NeXpose rawxml stream - # parser, like: - # [ - # {"id"=>"winreg-notes-protocol-handler", severity="8", "refs"=>[{"source"=>"BID", "value"=>"10600"}, ...]} - # {"id"=>"windows-zotob-c", severity="8", "refs"=>[{"source"=>"BID", "value"=>"14513"}, ...]} - # ] - # and transforms it into a struct, containing :id, :refs, :title, and :severity - # - # Other attributes can be added later, as needed. - def nexpose_refs_to_struct(vulns) - ret = [] - vulns.each do |vuln| - next if ret.map {|v| v.id}.include? vuln["id"] - vstruct = Struct.new(:id, :refs, :title, :severity).new - vstruct.id = vuln["id"] - vstruct.title = vuln["title"] - vstruct.severity = vuln["severity"] - vstruct.refs = [] - vuln["refs"].each do |ref| - if ref['source'] == 'BID' - vstruct.refs.push('BID-' + ref["value"]) - elsif ref['source'] == 'CVE' - # value is CVE-$ID - vstruct.refs.push(ref["value"]) - elsif ref['source'] == 'MS' - vstruct.refs.push('MSB-' + ref["value"]) - elsif ref['source'] == 'URL' - vstruct.refs.push('URL-' + ref["value"]) - end - end - ret.push vstruct - end - return ret - end - - # Takes a Host object, an array of vuln structs (generated by nexpose_refs_to_struct()), - # and a workspace, and reports the vulns on that host. - def nexpose_host_from_rawxml(h, vstructs, wspace,task=nil) - hobj = nil - data = {:workspace => wspace} - if h["addr"] - addr = h["addr"] - else - # Can't report it if it doesn't have an IP - return - end - data[:host] = addr - if (h["hardware-address"]) - # Put colons between each octet of the MAC address - data[:mac] = h["hardware-address"].gsub(':', '').scan(/../).join(':') - end - data[:state] = (h["status"] == "alive") ? Msf::HostState::Alive : Msf::HostState::Dead - - # Since we only have one name field per host in the database, just - # take the first one. - if (h["names"] and h["names"].first) - data[:name] = h["names"].first - end - - if (data[:state] != Msf::HostState::Dead) - hobj = report_host(data) - report_import_note(wspace, hobj) - end - - if h["notes"] - note = { - :workspace => wspace, - :host => (hobj || addr), - :type => "host.vuln.nexpose_keys", - :data => {}, - :mode => :unique_data, - :task => task - } - h["notes"].each do |v,k| - note[:data][v] ||= [] - next if note[:data][v].include? k - note[:data][v] << k - end - report_note(note) - end - - if h["os_family"] - note = { - :workspace => wspace, - :host => hobj || addr, - :type => 'host.os.nexpose_fingerprint', - :task => task, - :data => { - :family => h["os_family"], - :certainty => h["os_certainty"] - } - } - note[:data][:vendor] = h["os_vendor"] if h["os_vendor"] - note[:data][:product] = h["os_product"] if h["os_product"] - note[:data][:version] = h["os_version"] if h["os_version"] - note[:data][:arch] = h["arch"] if h["arch"] - - report_note(note) - end - - h["endpoints"].each { |p| - extra = "" - extra << p["product"] + " " if p["product"] - extra << p["version"] + " " if p["version"] - - # Skip port-0 endpoints - next if p["port"].to_i == 0 - - # XXX This should probably be handled in a more standard way - # extra << "(" + p["certainty"] + " certainty) " if p["certainty"] - - data = {} - data[:workspace] = wspace - data[:proto] = p["protocol"].downcase - data[:port] = p["port"].to_i - data[:state] = p["status"] - data[:host] = hobj || addr - data[:info] = extra if not extra.empty? - data[:task] = task - if p["name"] != "" - data[:name] = p["name"] - end - report_service(data) - } - - h["vulns"].each_pair { |k,v| - - next if v["status"] !~ /^vulnerable/ - vstruct = vstructs.select {|vs| vs.id.to_s.downcase == v["id"].to_s.downcase}.first - next unless vstruct - data = {} - data[:workspace] = wspace - data[:host] = hobj || addr - data[:proto] = v["protocol"].downcase if v["protocol"] - data[:port] = v["port"].to_i if v["port"] - data[:name] = "NEXPOSE-" + v["id"] - data[:info] = vstruct.title - data[:refs] = vstruct.refs - data[:task] = task - report_vuln(data) - } - end - - - # - # Retina XML - # - - # Process a Retina XML file - def import_retina_xml_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - - data = "" - ::File.open(filename, 'rb') do |f| - data = f.read(f.stat.size) - end - import_retina_xml(args.merge(:data => data)) - end - - # Process Retina XML - def import_retina_xml(args={}, &block) - data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - msg = "Warning: The Retina XML format does not associate vulnerabilities with the\n" - msg << "specific service on which they were found.\n" - msg << "This makes it impossible to correlate exploits to discovered vulnerabilities\n" - msg << "in a reliable fashion." - - yield(:warning,msg) if block - - parser = Rex::Parser::RetinaXMLStreamParser.new - parser.on_found_host = Proc.new do |host| - hobj = nil - data = { - :workspace => wspace, - :task => args[:task] - } - addr = host['address'] - next if not addr - - next if bl.include? addr - data[:host] = addr - - if host['mac'] - data[:mac] = host['mac'] - end - - data[:state] = Msf::HostState::Alive - - if host['hostname'] - data[:name] = host['hostname'] - end - - if host['netbios'] - data[:name] = host['netbios'] - end - - yield(:address, data[:host]) if block - - # Import Host - hobj = report_host(data) - report_import_note(wspace, hobj) - - # Import OS fingerprint - if host["os"] - note = { - :workspace => wspace, - :host => addr, - :type => 'host.os.retina_fingerprint', - :task => args[:task], - :data => { - :os => host["os"] - } - } - report_note(note) - end - - # Import vulnerabilities - host['vulns'].each do |vuln| - refs = vuln['refs'].map{|v| v.join("-")} - refs << "RETINA-#{vuln['rthid']}" if vuln['rthid'] - - vuln_info = { - :workspace => wspace, - :host => addr, - :name => vuln['name'], - :info => vuln['description'], - :refs => refs, - :task => args[:task] - } - - report_vuln(vuln_info) - end - end - - REXML::Document.parse_stream(data, parser) - end - - # - # NetSparker XML - # - - # Process a NetSparker XML file - def import_netsparker_xml_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - - data = "" - ::File.open(filename, 'rb') do |f| - data = f.read(f.stat.size) - end - import_netsparker_xml(args.merge(:data => data)) - end - - # Process NetSparker XML - def import_netsparker_xml(args={}, &block) - data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - addr = nil - parser = Rex::Parser::NetSparkerXMLStreamParser.new - parser.on_found_vuln = Proc.new do |vuln| - data = {:workspace => wspace} - - # Parse the URL - url = vuln['url'] - return if not url - - # Crack the URL into a URI - uri = URI(url) rescue nil - return if not uri - - # Resolve the host and cache the IP - if not addr - baddr = Rex::Socket.addr_aton(uri.host) rescue nil - if baddr - addr = Rex::Socket.addr_ntoa(baddr) - yield(:address, addr) if block - end - end - - # Bail early if we have no IP address - if not addr - raise Interrupt, "Not a valid IP address" - end - - if bl.include?(addr) - raise Interrupt, "IP address is on the blacklist" - end - - data[:host] = addr - data[:vhost] = uri.host - data[:port] = uri.port - data[:ssl] = (uri.scheme == "ssl") - - body = nil - # First report a web page - if vuln['response'] - headers = {} - code = 200 - head,body = vuln['response'].to_s.split(/\r?\n\r?\n/, 2) - if body - - if head =~ /^HTTP\d+\.\d+\s+(\d+)\s*/ - code = $1.to_i - end - - headers = {} - head.split(/\r?\n/).each do |line| - hname,hval = line.strip.split(/\s*:\s*/, 2) - next if hval.to_s.strip.empty? - headers[hname.downcase] ||= [] - headers[hname.downcase] << hval - end - - info = { - :path => uri.path, - :query => uri.query, - :code => code, - :body => body, - :headers => headers, - :task => args[:task] - } - info.merge!(data) - - if headers['content-type'] - info[:ctype] = headers['content-type'][0] - end - - if headers['set-cookie'] - info[:cookie] = headers['set-cookie'].join("\n") - end - - if headers['authorization'] - info[:auth] = headers['authorization'].join("\n") - end - - if headers['location'] - info[:location] = headers['location'][0] - end - - if headers['last-modified'] - info[:mtime] = headers['last-modified'][0] - end - - # Report the web page to the database - report_web_page(info) - - yield(:web_page, url) if block - end - end # End web_page reporting - - - details = netsparker_vulnerability_map(vuln) - - method = netsparker_method_map(vuln) - pname = netsparker_pname_map(vuln) - params = netsparker_params_map(vuln) - - proof = '' - - if vuln['info'] and vuln['info'].length > 0 - proof << vuln['info'].map{|x| "#{x[0]}: #{x[1]}\n" }.join + "\n" - end - - if proof.empty? - if body - proof << body + "\n" - else - proof << vuln['response'].to_s + "\n" - end - end - - if params.empty? and pname - params = [[pname, vuln['vparam_name'].to_s]] - end - - info = { - # XXX: There is a :request attr in the model, but report_web_vuln - # doesn't seem to know about it, so this gets ignored. - #:request => vuln['request'], - :path => uri.path, - :query => uri.query, - :method => method, - :params => params, - :pname => pname.to_s, - :proof => proof, - :risk => details[:risk], - :name => details[:name], - :blame => details[:blame], - :category => details[:category], - :description => details[:description], - :confidence => details[:confidence], - :task => args[:task] - } - info.merge!(data) - - next if vuln['type'].to_s.empty? - - report_web_vuln(info) - yield(:web_vuln, url) if block - end - - # We throw interrupts in our parser when the job is hopeless - begin - REXML::Document.parse_stream(data, parser) - rescue ::Interrupt => e - wlog("The netsparker_xml_import() job was interrupted: #{e}") - end - end - - def netsparker_method_map(vuln) - case vuln['vparam_type'] - when "FullQueryString" - "GET" - when "Querystring" - "GET" - when "Post" - "POST" - when "RawUrlInjection" - "GET" - else - "GET" - end - end - - def netsparker_pname_map(vuln) - case vuln['vparam_name'] - when "URI-BASED", "Query Based" - "PATH" - else - vuln['vparam_name'] - end - end - - def netsparker_params_map(vuln) - [] - end - - def netsparker_vulnerability_map(vuln) - res = { - :risk => 1, - :name => 'Information Disclosure', - :blame => 'System Administrator', - :category => 'info', - :description => "This is an information leak", - :confidence => 100 - } - - # Risk is a value from 1-5 indicating the severity of the issue - # Examples: 1, 4, 5 - - # Name is a descriptive name for this vulnerability. - # Examples: XSS, ReflectiveXSS, PersistentXSS - - # Blame indicates who is at fault for the vulnerability - # Examples: App Developer, Server Developer, System Administrator - - # Category indicates the general class of vulnerability - # Examples: info, xss, sql, rfi, lfi, cmd - - # Description is a textual summary of the vulnerability - # Examples: "A reflective cross-site scripting attack" - # "The web server leaks the internal IP address" - # "The cookie is not set to HTTP-only" - - # - # Confidence is a value from 1 to 100 indicating how confident the - # software is that the results are valid. - # Examples: 100, 90, 75, 15, 10, 0 - - case vuln['type'].to_s - when "ApacheDirectoryListing" - res = { - :risk => 1, - :name => 'Directory Listing', - :blame => 'System Administrator', - :category => 'info', - :description => "", - :confidence => 100 - } - when "ApacheMultiViewsEnabled" - res = { - :risk => 1, - :name => 'Apache MultiViews Enabled', - :blame => 'System Administrator', - :category => 'info', - :description => "", - :confidence => 100 - } - when "ApacheVersion" - res = { - :risk => 1, - :name => 'Web Server Version', - :blame => 'System Administrator', - :category => 'info', - :description => "", - :confidence => 100 - } - when "PHPVersion" - res = { - :risk => 1, - :name => 'PHP Module Version', - :blame => 'System Administrator', - :category => 'info', - :description => "", - :confidence => 100 - } - when "AutoCompleteEnabled" - res = { - :risk => 1, - :name => 'Form AutoComplete Enabled', - :blame => 'App Developer', - :category => 'info', - :description => "", - :confidence => 100 - } - when "CookieNotMarkedAsHttpOnly" - res = { - :risk => 1, - :name => 'Cookie Not HttpOnly', - :blame => 'App Developer', - :category => 'info', - :description => "", - :confidence => 100 - } - when "EmailDisclosure" - res = { - :risk => 1, - :name => 'Email Address Disclosure', - :blame => 'App Developer', - :category => 'info', - :description => "", - :confidence => 100 - } - when "ForbiddenResource" - res = { - :risk => 1, - :name => 'Forbidden Resource', - :blame => 'App Developer', - :category => 'info', - :description => "", - :confidence => 100 - } - when "FileUploadFound" - res = { - :risk => 1, - :name => 'File Upload Form', - :blame => 'App Developer', - :category => 'info', - :description => "", - :confidence => 100 - } - when "PasswordOverHTTP" - res = { - :risk => 2, - :name => 'Password Over HTTP', - :blame => 'App Developer', - :category => 'info', - :description => "", - :confidence => 100 - } - when "MySQL5Identified" - res = { - :risk => 1, - :name => 'MySQL 5 Identified', - :blame => 'App Developer', - :category => 'info', - :description => "", - :confidence => 100 - } - when "PossibleInternalWindowsPathLeakage" - res = { - :risk => 1, - :name => 'Path Leakage - Windows', - :blame => 'App Developer', - :category => 'info', - :description => "", - :confidence => 100 - } - when "PossibleInternalUnixPathLeakage" - res = { - :risk => 1, - :name => 'Path Leakage - Unix', - :blame => 'App Developer', - :category => 'info', - :description => "", - :confidence => 100 - } - when "PossibleXSS", "LowPossibilityPermanentXSS", "XSS", "PermanentXSS" - conf = 100 - conf = 25 if vuln['type'].to_s == "LowPossibilityPermanentXSS" - conf = 50 if vuln['type'].to_s == "PossibleXSS" - res = { - :risk => 3, - :name => 'Cross-Site Scripting', - :blame => 'App Developer', - :category => 'xss', - :description => "", - :confidence => conf - } - - when "ConfirmedBlindSQLInjection", "ConfirmedSQLInjection", "HighlyPossibleSqlInjection", "DatabaseErrorMessages" - conf = 100 - conf = 90 if vuln['type'].to_s == "HighlyPossibleSqlInjection" - conf = 25 if vuln['type'].to_s == "DatabaseErrorMessages" - res = { - :risk => 5, - :name => 'SQL Injection', - :blame => 'App Developer', - :category => 'sql', - :description => "", - :confidence => conf - } - else - conf = 100 - res = { - :risk => 1, - :name => vuln['type'].to_s, - :blame => 'App Developer', - :category => 'info', - :description => "", - :confidence => conf - } - end - - res - end - - def import_fusionvm_xml(args={}) - args[:wspace] ||= workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - doc = Rex::Parser::FusionVMDocument.new(args,self) - parser = ::Nokogiri::XML::SAX::Parser.new(doc) - parser.parse(args[:data]) - end - - - # - # Import Nmap's -oX xml output - # - def import_nmap_xml_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - - data = "" - ::File.open(filename, 'rb') do |f| - data = f.read(f.stat.size) - end - import_nmap_xml(args.merge(:data => data)) - end - - def import_nexpose_raw_noko_stream(args, &block) - if block - doc = Rex::Parser::NexposeRawDocument.new(args,framework.db) {|type, data| yield type,data } - else - doc = Rex::Parser::NexposeRawDocument.new(args,self) - end - parser = ::Nokogiri::XML::SAX::Parser.new(doc) - parser.parse(args[:data]) - end - - def import_nexpose_noko_stream(args, &block) - if block - doc = Rex::Parser::NexposeSimpleDocument.new(args,framework.db) {|type, data| yield type,data } - else - doc = Rex::Parser::NexposeSimpleDocument.new(args,self) - end - parser = ::Nokogiri::XML::SAX::Parser.new(doc) - parser.parse(args[:data]) - end - - def import_nmap_noko_stream(args, &block) - if block - doc = Rex::Parser::NmapDocument.new(args,framework.db) {|type, data| yield type,data } - else - doc = Rex::Parser::NmapDocument.new(args,self) - end - parser = ::Nokogiri::XML::SAX::Parser.new(doc) - parser.parse(args[:data]) - end - - # If you have Nokogiri installed, you'll be shunted over to - # that. Otherwise, you'll hit the old NmapXMLStreamParser. - def import_nmap_xml(args={}, &block) - return nil if args[:data].nil? or args[:data].empty? - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - - if Rex::Parser.nokogiri_loaded - noko_args = args.dup - noko_args[:blacklist] = bl - noko_args[:wspace] = wspace - if block - yield(:parser, "Nokogiri v#{::Nokogiri::VERSION}") - import_nmap_noko_stream(noko_args) {|type, data| yield type,data } - else - import_nmap_noko_stream(noko_args) - end - return true - end - - # XXX: Legacy nmap xml parser starts here. - - fix_services = args[:fix_services] - data = args[:data] - - # Use a stream parser instead of a tree parser so we can deal with - # huge results files without running out of memory. - parser = Rex::Parser::NmapXMLStreamParser.new - yield(:parser, parser.class.name) if block - - # Whenever the parser pulls a host out of the nmap results, store - # it, along with any associated services, in the database. - parser.on_found_host = Proc.new { |h| - hobj = nil - data = {:workspace => wspace} - if (h["addrs"].has_key?("ipv4")) - addr = h["addrs"]["ipv4"] - elsif (h["addrs"].has_key?("ipv6")) - addr = h["addrs"]["ipv6"] - else - # Can't report it if it doesn't have an IP - raise RuntimeError, "At least one IPv4 or IPv6 address is required" - end - next if bl.include? addr - data[:host] = addr - if (h["addrs"].has_key?("mac")) - data[:mac] = h["addrs"]["mac"] - end - data[:state] = (h["status"] == "up") ? Msf::HostState::Alive : Msf::HostState::Dead - data[:task] = args[:task] - - if ( h["reverse_dns"] ) - data[:name] = h["reverse_dns"] - end - - # Only report alive hosts with ports to speak of. - if(data[:state] != Msf::HostState::Dead) - if h["ports"].size > 0 - if fix_services - port_states = h["ports"].map {|p| p["state"]}.reject {|p| p == "filtered"} - next if port_states.compact.empty? - end - yield(:address,data[:host]) if block - hobj = report_host(data) - report_import_note(wspace,hobj) - end - end - - if( h["os_vendor"] ) - note = { - :workspace => wspace, - :host => hobj || addr, - :type => 'host.os.nmap_fingerprint', - :task => args[:task], - :data => { - :os_vendor => h["os_vendor"], - :os_family => h["os_family"], - :os_version => h["os_version"], - :os_accuracy => h["os_accuracy"] - } - } - - if(h["os_match"]) - note[:data][:os_match] = h['os_match'] - end - - report_note(note) - end - - if (h["last_boot"]) - report_note( - :workspace => wspace, - :host => hobj || addr, - :type => 'host.last_boot', - :task => args[:task], - :data => { - :time => h["last_boot"] - } - ) - end - - if (h["trace"]) - hops = [] - h["trace"]["hops"].each do |hop| - hops << { - "ttl" => hop["ttl"].to_i, - "address" => hop["ipaddr"].to_s, - "rtt" => hop["rtt"].to_f, - "name" => hop["host"].to_s - } - end - report_note( - :workspace => wspace, - :host => hobj || addr, - :type => 'host.nmap.traceroute', - :task => args[:task], - :data => { - 'port' => h["trace"]["port"].to_i, - 'proto' => h["trace"]["proto"].to_s, - 'hops' => hops - } - ) - end - - - # Put all the ports, regardless of state, into the db. - h["ports"].each { |p| - # Localhost port results are pretty unreliable -- if it's - # unknown, it's no good (possibly Windows-only) - if ( - p["state"] == "unknown" && - h["status_reason"] == "localhost-response" - ) - next - end - extra = "" - extra << p["product"] + " " if p["product"] - extra << p["version"] + " " if p["version"] - extra << p["extrainfo"] + " " if p["extrainfo"] - - data = {} - data[:workspace] = wspace - if fix_services - data[:proto] = nmap_msf_service_map(p["protocol"]) - else - data[:proto] = p["protocol"].downcase - end - data[:port] = p["portid"].to_i - data[:state] = p["state"] - data[:host] = hobj || addr - data[:info] = extra if not extra.empty? - data[:task] = args[:task] - if p["name"] != "unknown" - data[:name] = p["name"] - end - report_service(data) - } - #Parse the scripts output - if h["scripts"] - h["scripts"].each do |key,val| - if key == "smb-check-vulns" - if val =~ /MS08-067: VULNERABLE/ - vuln_info = { - :workspace => wspace, - :task => args[:task], - :host => hobj || addr, - :port => 445, - :proto => 'tcp', - :name => 'MS08-067', - :info => 'Microsoft Windows Server Service Crafted RPC Request Handling Unspecified Remote Code Execution', - :refs =>['CVE-2008-4250', - 'BID-31874', - 'OSVDB-49243', - 'CWE-94', - 'MSFT-MS08-067', - 'MSF-Microsoft Server Service Relative Path Stack Corruption', - 'NSS-34476'] - } - report_vuln(vuln_info) - end - if val =~ /MS06-025: VULNERABLE/ - vuln_info = { - :workspace => wspace, - :task => args[:task], - :host => hobj || addr, - :port => 445, - :proto => 'tcp', - :name => 'MS06-025', - :info => 'Vulnerability in Routing and Remote Access Could Allow Remote Code Execution', - :refs =>['CVE-2006-2370', - 'CVE-2006-2371', - 'BID-18325', - 'BID-18358', - 'BID-18424', - 'OSVDB-26436', - 'OSVDB-26437', - 'MSFT-MS06-025', - 'MSF-Microsoft RRAS Service RASMAN Registry Overflow', - 'NSS-21689'] - } - report_vuln(vuln_info) - end - # This one has NOT been Tested , remove this comment if confirmed working - if val =~ /MS07-029: VULNERABLE/ - vuln_info = { - :workspace => wspace, - :task => args[:task], - :host => hobj || addr, - :port => 445, - :proto => 'tcp', - :name => 'MS07-029', - :info => 'Vulnerability in Windows DNS RPC Interface Could Allow Remote Code Execution', - # Add more refs based on nessus/nexpose .. results - :refs =>['CVE-2007-1748', - 'OSVDB-34100', - 'MSF-Microsoft DNS RPC Service extractQuotedChar()', - 'NSS-25168'] - } - report_vuln(vuln_info) - end - end - end - end - } - - # XXX: Legacy nmap xml parser ends here. - - REXML::Document.parse_stream(data, parser) - end - - def nmap_msf_service_map(proto) - service_name_map(proto) - end - - # - # This method normalizes an incoming service name to one of the - # the standard ones recognized by metasploit - # - def service_name_map(proto) - return proto unless proto.kind_of? String - case proto.downcase - when "msrpc", "nfs-or-iis", "dce endpoint resolution" - "dcerpc" - when "ms-sql-s", "tds" - "mssql" - when "ms-sql-m","microsoft sql monitor" - "mssql-m" - when "postgresql"; "postgres" - when "http-proxy"; "http" - when "iiimsf"; "db2" - when "oracle-tns"; "oracle" - when "quickbooksrds"; "metasploit" - when "microsoft remote display protocol" - "rdp" - when "vmware authentication daemon" - "vmauthd" - when "netbios-ns", "cifs name service" - "netbios" - when "netbios-ssn", "microsoft-ds", "cifs" - "smb" - when "remote shell" - "shell" - when "remote login" - "login" - when "nfs lockd" - "lockd" - when "hp jetdirect" - "jetdirect" - when "dhcp server" - "dhcp" - when /^dns-(udp|tcp)$/; "dns" - when /^dce[\s+]rpc$/; "dcerpc" - else - proto.downcase.gsub(/\s*\(.*/, '') # "service (some service)" - end - end - - def report_import_note(wspace,addr) - if @import_filedata.kind_of?(Hash) && @import_filedata[:filename] && @import_filedata[:filename] !~ /msfe-nmap[0-9]{8}/ - report_note( - :workspace => wspace, - :host => addr, - :type => 'host.imported', - :data => @import_filedata.merge(:time=> Time.now.utc) - ) - end - end - - # - # Import Nessus NBE files - # - def import_nessus_nbe_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - - data = "" - ::File.open(filename, 'rb') do |f| - data = f.read(f.stat.size) - end - import_nessus_nbe(args.merge(:data => data)) - end - - # There is no place the NBE actually stores the plugin name used to - # scan. You get "Security Note" or "Security Warning," and that's it. - def import_nessus_nbe(args={}, &block) - nbe_data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - - nbe_copy = nbe_data.dup - # First pass, just to build the address map. - addr_map = {} - - # Cache host objects before passing into handle_nessus() - hobj_map = {} - - nbe_copy.each_line do |line| - r = line.split('|') - next if r[0] != 'results' - next if r[4] != "12053" - data = r[6] - addr,hname = data.match(/([0-9\x2e]+) resolves as (.+)\x2e\\n/n)[1,2] - addr_map[hname] = addr - end - - nbe_data.each_line do |line| - r = line.split('|') - next if r[0] != 'results' - hname = r[2] - if addr_map[hname] - addr = addr_map[hname] - else - addr = hname # Must be unresolved, probably an IP address. - end - port = r[3] - nasl = r[4] - type = r[5] - data = r[6] - - # If there's no resolution, or if it's malformed, skip it. - next unless ipv46_validator(addr) - - if bl.include? addr - next - else - yield(:address,addr) if block - end - - hobj_map[ addr ] ||= report_host(:host => addr, :workspace => wspace, :task => args[:task]) - - # Match the NBE types with the XML severity ratings - case type - # log messages don't actually have any data, they are just - # complaints about not being able to perform this or that test - # because such-and-such was missing - when "Log Message"; next - when "Security Hole"; severity = 3 - when "Security Warning"; severity = 2 - when "Security Note"; severity = 1 - # a severity 0 means there's no extra data, it's just an open port - else; severity = 0 - end - if nasl == "11936" - os = data.match(/The remote host is running (.*)\\n/)[1] - report_note( - :workspace => wspace, - :task => args[:task], - :host => hobj_map[ addr ], - :type => 'host.os.nessus_fingerprint', - :data => { - :os => os.to_s.strip - } - ) - end - - next if nasl.to_s.strip.empty? - plugin_name = nil # NBE doesn't ever populate this - handle_nessus(wspace, hobj_map[ addr ], port, nasl, plugin_name, severity, data) - end - end - - # - # Of course they had to change the nessus format. - # - def import_openvas_xml(args={}, &block) - filename = args[:filename] - wspace = args[:wspace] || workspace - - raise DBImportError.new("No OpenVAS XML support. Please submit a patch to msfdev[at]metasploit.com") - end - - # - # Import IP360 XML v3 output - # - def import_ip360_xml_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - - data = "" - ::File.open(filename, 'rb') do |f| - data = f.read(f.stat.size) - end - import_ip360_xml_v3(args.merge(:data => data)) - end - - # - # Import Nessus XML v1 and v2 output - # - # Old versions of openvas exported this as well - # - def import_nessus_xml_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - - data = "" - ::File.open(filename, 'rb') do |f| - data = f.read(f.stat.size) - end - - if data.index("NessusClientData_v2") - import_nessus_xml_v2(args.merge(:data => data)) - else - import_nessus_xml(args.merge(:data => data)) - end - end - - def import_nessus_xml(args={}, &block) - data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - - doc = rexmlify(data) - doc.elements.each('/NessusClientData/Report/ReportHost') do |host| - hobj = nil - addr = nil - hname = nil - os = nil - # If the name is resolved, the Nessus plugin for DNS - # resolution should be there. If not, fall back to the - # HostName - host.elements.each('ReportItem') do |item| - next unless item.elements['pluginID'].text == "12053" - addr = item.elements['data'].text.match(/([0-9\x2e]+) resolves as/n)[1] - hname = host.elements['HostName'].text - end - addr ||= host.elements['HostName'].text - next unless ipv46_validator(addr) # Skip resolved names and SCAN-ERROR. - if bl.include? addr - next - else - yield(:address,addr) if block - end - - hinfo = { - :workspace => wspace, - :host => addr, - :task => args[:task] - } - - # Record the hostname - hinfo.merge!(:name => hname.to_s.strip) if hname - hobj = report_host(hinfo) - report_import_note(wspace,hobj) - - # Record the OS - os ||= host.elements["os_name"] - if os - report_note( - :workspace => wspace, - :task => args[:task], - :host => hobj, - :type => 'host.os.nessus_fingerprint', - :data => { - :os => os.text.to_s.strip - } - ) - end - - host.elements.each('ReportItem') do |item| - nasl = item.elements['pluginID'].text - plugin_name = item.elements['pluginName'].text - port = item.elements['port'].text - data = item.elements['data'].text - severity = item.elements['severity'].text - - handle_nessus(wspace, hobj, port, nasl, plugin_name, severity, data, args[:task]) - end - end - end - - def import_nessus_xml_v2(args={}, &block) - data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - - #@host = { - #'hname' => nil, - #'addr' => nil, - #'mac' => nil, - #'os' => nil, - #'ports' => [ 'port' => { 'port' => nil, - # 'svc_name' => nil, - # 'proto' => nil, - # 'severity' => nil, - # 'nasl' => nil, - # 'description' => nil, - # 'cve' => [], - # 'bid' => [], - # 'xref' => [] - # } - # ] - #} - parser = Rex::Parser::NessusXMLStreamParser.new - parser.on_found_host = Proc.new { |host| - - hobj = nil - addr = host['addr'] || host['hname'] - - next unless ipv46_validator(addr) # Catches SCAN-ERROR, among others. - - if bl.include? addr - next - else - yield(:address,addr) if block - end - - os = host['os'] - hname = host['hname'] - mac = host['mac'] - - host_info = { - :workspace => wspace, - :host => addr, - :task => args[:task] - } - host_info[:name] = hname.to_s.strip if hname - # Short mac, protect against Nessus's habit of saving multiple macs - # We can't use them anyway, so take just the first. - host_info[:mac] = mac.to_s.strip.upcase.split(/\s+/).first if mac - - hobj = report_host(host_info) - report_import_note(wspace,hobj) - - os = host['os'] - yield(:os,os) if block - if os - report_note( - :workspace => wspace, - :task => args[:task], - :host => hobj, - :type => 'host.os.nessus_fingerprint', - :data => { - :os => os.to_s.strip - } - ) - end - - host['ports'].each do |item| - next if item['port'] == 0 - msf = nil - nasl = item['nasl'].to_s - nasl_name = item['nasl_name'].to_s - port = item['port'].to_s - proto = item['proto'] || "tcp" - sname = item['svc_name'] - severity = item['severity'] - description = item['description'] - cve = item['cve'] - bid = item['bid'] - xref = item['xref'] - msf = item['msf'] - - yield(:port,port) if block - - handle_nessus_v2(wspace, hobj, port, proto, sname, nasl, nasl_name, severity, description, cve, bid, xref, msf, args[:task]) - - end - yield(:end,hname) if block - } - - REXML::Document.parse_stream(data, parser) - - end - - def import_mbsa_xml(args={}, &block) - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - wspace = args[:wspace] || workspace - if Rex::Parser.nokogiri_loaded - parser = "Nokogiri v#{::Nokogiri::VERSION}" - noko_args = args.dup - noko_args[:blacklist] = bl - noko_args[:wspace] = wspace - if block - yield(:parser, parser) - import_mbsa_noko_stream(noko_args) {|type, data| yield type,data} - else - import_mbsa_noko_stream(noko_args) - end - return true - else # Sorry - raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.") - end - end - - def import_mbsa_noko_stream(args={},&block) - if block - doc = Rex::Parser::MbsaDocument.new(args,framework.db) {|type, data| yield type,data } - else - doc = Rex::Parser::MbsaDocument.new(args,self) - end - parser = ::Nokogiri::XML::SAX::Parser.new(doc) - parser.parse(args[:data]) - end - - def import_foundstone_xml(args={}, &block) - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - wspace = args[:wspace] || workspace - if Rex::Parser.nokogiri_loaded - parser = "Nokogiri v#{::Nokogiri::VERSION}" - noko_args = args.dup - noko_args[:blacklist] = bl - noko_args[:wspace] = wspace - if block - yield(:parser, parser) - import_foundstone_noko_stream(noko_args) {|type, data| yield type,data} - else - import_foundstone_noko_stream(noko_args) - end - return true - else # Sorry - raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.") - end - end - - def import_foundstone_noko_stream(args={},&block) - if block - doc = Rex::Parser::FoundstoneDocument.new(args,framework.db) {|type, data| yield type,data } - else - doc = Rex::Parser::FoundstoneDocument.new(args,self) - end - parser = ::Nokogiri::XML::SAX::Parser.new(doc) - parser.parse(args[:data]) - end - - def import_acunetix_xml(args={}, &block) - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - wspace = args[:wspace] || workspace - if Rex::Parser.nokogiri_loaded - parser = "Nokogiri v#{::Nokogiri::VERSION}" - noko_args = args.dup - noko_args[:blacklist] = bl - noko_args[:wspace] = wspace - if block - yield(:parser, parser) - import_acunetix_noko_stream(noko_args) {|type, data| yield type,data} - else - import_acunetix_noko_stream(noko_args) - end - return true - else # Sorry - raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.") - end - end - - def import_ci_xml(args={}, &block) - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - wspace = args[:wspace] || workspace - if Rex::Parser.nokogiri_loaded - parser = "Nokogiri v#{::Nokogiri::VERSION}" - noko_args = args.dup - noko_args[:blacklist] = bl - noko_args[:wspace] = wspace - if block - yield(:parser, parser) - import_ci_noko_stream(noko_args) {|type, data| yield type,data} - else - import_ci_noko_stream(noko_args) - end - return true - else # Sorry - raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.") - end - end - - def import_acunetix_noko_stream(args={},&block) - if block - doc = Rex::Parser::AcunetixDocument.new(args,framework.db) {|type, data| yield type,data } - else - doc = Rex::Parser::AcunetixFoundstoneDocument.new(args,self) - end - parser = ::Nokogiri::XML::SAX::Parser.new(doc) - parser.parse(args[:data]) - end - - - def import_appscan_xml(args={}, &block) - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - wspace = args[:wspace] || workspace - if Rex::Parser.nokogiri_loaded - parser = "Nokogiri v#{::Nokogiri::VERSION}" - noko_args = args.dup - noko_args[:blacklist] = bl - noko_args[:wspace] = wspace - if block - yield(:parser, parser) - import_appscan_noko_stream(noko_args) {|type, data| yield type,data} - else - import_appscan_noko_stream(noko_args) - end - return true - else # Sorry - raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.") - end - end - - def import_appscan_noko_stream(args={},&block) - if block - doc = Rex::Parser::AppscanDocument.new(args,framework.db) {|type, data| yield type,data } - else - doc = Rex::Parser::AppscanDocument.new(args,self) - end - parser = ::Nokogiri::XML::SAX::Parser.new(doc) - parser.parse(args[:data]) - end - - def import_burp_session_xml(args={}, &block) - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - wspace = args[:wspace] || workspace - if Rex::Parser.nokogiri_loaded - # Rex::Parser.reload("burp_session_nokogiri.rb") - parser = "Nokogiri v#{::Nokogiri::VERSION}" - noko_args = args.dup - noko_args[:blacklist] = bl - noko_args[:wspace] = wspace - if block - yield(:parser, parser) - import_burp_session_noko_stream(noko_args) {|type, data| yield type,data} - else - import_burp_session_noko_stream(noko_args) - end - return true - else # Sorry - raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.") - end - end - - def import_burp_session_noko_stream(args={},&block) - if block - doc = Rex::Parser::BurpSessionDocument.new(args,framework.db) {|type, data| yield type,data } - else - doc = Rex::Parser::BurpSessionDocument.new(args,self) - end - parser = ::Nokogiri::XML::SAX::Parser.new(doc) - parser.parse(args[:data]) - end - - - # - # Import IP360's ASPL database - # - def import_ip360_aspl_xml(args={}, &block) - data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - - if not data.index(" {'name' => { }, 'cve' => { }, 'bid' => { } } - # 'oses' => {'name' } } - - aspl_path = nil - aspl_paths = [ - ::File.join(Msf::Config.config_directory, "data", "ncircle", "ip360.aspl"), - ::File.join(Msf::Config.data_directory, "ncircle", "ip360.aspl") - ] - - aspl_paths.each do |tpath| - next if not (::File.exist?(tpath) and ::File.readable?(tpath)) - aspl_path = tpath - break - end - - if not aspl_path - raise DBImportError.new("The nCircle IP360 ASPL file is not present.\n Download ASPL from nCircle VNE | Administer | Support | Resources, unzip it, and import it first") - end - - # parse nCircle ASPL file - aspl = "" - ::File.open(aspl_path, "rb") do |f| - aspl = f.read(f.stat.size) - end - - @asplhash = nil - parser = Rex::Parser::IP360ASPLXMLStreamParser.new - parser.on_found_aspl = Proc.new { |asplh| - @asplhash = asplh - } - REXML::Document.parse_stream(aspl, parser) - - # nCircle has some quotes escaped which causes the parser to break - # we don't need these lines so just replace \" with " - data.gsub!(/\\"/,'"') - - # parse nCircle Scan Output - parser = Rex::Parser::IP360XMLStreamParser.new - parser.on_found_host = Proc.new { |host| - hobj = nil - addr = host['addr'] || host['hname'] - - next unless ipv46_validator(addr) # Catches SCAN-ERROR, among others. - - if bl.include? addr - next - else - yield(:address,addr) if block - end - - os = host['os'] - hname = host['hname'] - mac = host['mac'] - - host_hash = { - :workspace => wspace, - :host => addr, - :task => args[:task] - } - host_hash[:name] = hname.to_s.strip if hname - host_hash[:mac] = mac.to_s.strip.upcase if mac - - hobj = report_host(host_hash) - - yield(:os, os) if block - if os - report_note( - :workspace => wspace, - :task => args[:task], - :host => hobj, - :type => 'host.os.ip360_fingerprint', - :data => { - :os => @asplhash['oses'][os].to_s.strip - } - ) - end - - host['apps'].each do |item| - port = item['port'].to_s - proto = item['proto'].to_s - - handle_ip360_v3_svc(wspace, hobj, port, proto, hname, args[:task]) - end - - - host['vulns'].each do |item| - vulnid = item['vulnid'].to_s - port = item['port'].to_s - proto = item['proto'] || "tcp" - vulnname = @asplhash['vulns']['name'][vulnid] - cves = @asplhash['vulns']['cve'][vulnid] - bids = @asplhash['vulns']['bid'][vulnid] - - yield(:port, port) if block - - handle_ip360_v3_vuln(wspace, hobj, port, proto, hname, vulnid, vulnname, cves, bids, args[:task]) - - end - - yield(:end, hname) if block - } - - REXML::Document.parse_stream(data, parser) - end - - def find_qualys_asset_vuln_refs(doc) - vuln_refs = {} - doc.elements.each("/ASSET_DATA_REPORT/GLOSSARY/VULN_DETAILS_LIST/VULN_DETAILS") do |vuln| - next unless vuln.elements['QID'] && vuln.elements['QID'].first - qid = vuln.elements['QID'].first.to_s - vuln_refs[qid] ||= [] - vuln.elements.each('CVE_ID_LIST/CVE_ID') do |ref| - vuln_refs[qid].push('CVE-' + /C..-([0-9\-]{9})/.match(ref.elements['ID'].text.to_s)[1]) - end - vuln.elements.each('BUGTRAQ_ID_LIST/BUGTRAQ_ID') do |ref| - vuln_refs[qid].push('BID-' + ref.elements['ID'].text.to_s) - end - end - return vuln_refs - end - - # Pull out vulnerabilities that have at least one matching - # ref -- many "vulns" are not vulns, just audit information. - def find_qualys_asset_vulns(host,wspace,hobj,vuln_refs,task_id,&block) - host.elements.each("VULN_INFO_LIST/VULN_INFO") do |vi| - next unless vi.elements["QID"] - vi.elements.each("QID") do |qid| - next if vuln_refs[qid.text].nil? || vuln_refs[qid.text].empty? - handle_qualys(wspace, hobj, nil, nil, qid.text, nil, vuln_refs[qid.text], nil, nil, task_id) - end - end - end - - # Takes QID numbers and finds the discovered services in - # a qualys_asset_xml. - def find_qualys_asset_ports(i,host,wspace,hobj,task_id) - return unless (i == 82023 || i == 82004) - proto = i == 82023 ? 'tcp' : 'udp' - qid = host.elements["VULN_INFO_LIST/VULN_INFO/QID[@id='qid_#{i}']"] - qid_result = qid.parent.elements["RESULT[@format='table']"] if qid - hports = qid_result.first.to_s if qid_result - if hports - hports.scan(/([0-9]+)\t(.*?)\t.*?\t([^\t\n]*)/) do |match| - if match[2] == nil or match[2].strip == 'unknown' - name = match[1].strip - else - name = match[2].strip - end - handle_qualys(wspace, hobj, match[0].to_s, proto, 0, nil, nil, name, nil, task_id) - end - end - end - - # - # Import Qualys's Asset Data Report format - # - def import_qualys_asset_xml(args={}, &block) - data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - doc = rexmlify(data) - vuln_refs = find_qualys_asset_vuln_refs(doc) - - # 2nd pass, actually grab the hosts. - doc.elements.each("/ASSET_DATA_REPORT/HOST_LIST/HOST") do |host| - hobj = nil - addr = host.elements["IP"].text if host.elements["IP"] - next unless validate_ips(addr) - if bl.include? addr - next - else - yield(:address,addr) if block - end - hname = ( # Prefer NetBIOS over DNS - (host.elements["NETBIOS"].text if host.elements["NETBIOS"]) || - (host.elements["DNS"].text if host.elements["DNS"]) || - "" ) - hobj = report_host(:workspace => wspace, :host => addr, :name => hname, :state => Msf::HostState::Alive, :task => args[:task]) - report_import_note(wspace,hobj) - - if host.elements["OPERATING_SYSTEM"] - hos = host.elements["OPERATING_SYSTEM"].text - report_note( - :workspace => wspace, - :task => args[:task], - :host => hobj, - :type => 'host.os.qualys_fingerprint', - :data => { :os => hos } - ) - end - - # Report open ports. - find_qualys_asset_ports(82023,host,wspace,hobj, args[:task]) # TCP - find_qualys_asset_ports(82004,host,wspace,hobj, args[:task]) # UDP - - # Report vulns - find_qualys_asset_vulns(host,wspace,hobj,vuln_refs, args[:task],&block) - - end # host - - end - - # - # Import Qualys' Scan xml output - # - def import_qualys_scan_xml_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - - data = "" - ::File.open(filename, 'rb') do |f| - data = f.read(f.stat.size) - end - import_qualys_scan_xml(args.merge(:data => data)) - end - - def import_qualys_scan_xml(args={}, &block) - data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - - - doc = rexmlify(data) - doc.elements.each('/SCAN/IP') do |host| - hobj = nil - addr = host.attributes['value'] - if bl.include? addr - next - else - yield(:address,addr) if block - end - hname = host.attributes['name'] || '' - - hobj = report_host(:workspace => wspace, :host => addr, :name => hname, :state => Msf::HostState::Alive, :task => args[:task]) - report_import_note(wspace,hobj) - - if host.elements["OS"] - hos = host.elements["OS"].text - report_note( - :workspace => wspace, - :task => args[:task], - :host => hobj, - :type => 'host.os.qualys_fingerprint', - :data => { - :os => hos - } - ) - end - - # Open TCP Services List (Qualys ID 82023) - services_tcp = host.elements["SERVICES/CAT/SERVICE[@number='82023']/RESULT"] - if services_tcp - services_tcp.text.scan(/([0-9]+)\t(.*?)\t.*?\t([^\t\n]*)/) do |match| - if match[2] == nil or match[2].strip == 'unknown' - name = match[1].strip - else - name = match[2].strip - end - handle_qualys(wspace, hobj, match[0].to_s, 'tcp', 0, nil, nil, name, nil, args[:task]) - end - end - # Open UDP Services List (Qualys ID 82004) - services_udp = host.elements["SERVICES/CAT/SERVICE[@number='82004']/RESULT"] - if services_udp - services_udp.text.scan(/([0-9]+)\t(.*?)\t.*?\t([^\t\n]*)/) do |match| - if match[2] == nil or match[2].strip == 'unknown' - name = match[1].strip - else - name = match[2].strip - end - handle_qualys(wspace, hobj, match[0].to_s, 'udp', 0, nil, nil, name, nil, args[:task]) - end - end - - # VULNS are confirmed, PRACTICES are unconfirmed vulnerabilities - host.elements.each('VULNS/CAT | PRACTICES/CAT') do |cat| - port = cat.attributes['port'] - protocol = cat.attributes['protocol'] - cat.elements.each('VULN | PRACTICE') do |vuln| - refs = [] - qid = vuln.attributes['number'] - severity = vuln.attributes['severity'] - title = vuln.elements['TITLE'].text.to_s - vuln.elements.each('VENDOR_REFERENCE_LIST/VENDOR_REFERENCE') do |ref| - refs.push(ref.elements['ID'].text.to_s) - end - vuln.elements.each('CVE_ID_LIST/CVE_ID') do |ref| - refs.push('CVE-' + /C..-([0-9\-]{9})/.match(ref.elements['ID'].text.to_s)[1]) - end - vuln.elements.each('BUGTRAQ_ID_LIST/BUGTRAQ_ID') do |ref| - refs.push('BID-' + ref.elements['ID'].text.to_s) - end - - handle_qualys(wspace, hobj, port, protocol, qid, severity, refs, nil,title, args[:task]) - end - end - end - end - - def import_ip_list_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - - data = "" - ::File.open(filename, 'rb') do |f| - data = f.read(f.stat.size) - end - import_ip_list(args.merge(:data => data)) - end - - def import_ip_list(args={}, &block) - data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - - data.each_line do |ip| - ip.strip! - if bl.include? ip - next - else - yield(:address,ip) if block - end - host = find_or_create_host(:workspace => wspace, :host=> ip, :state => Msf::HostState::Alive, :task => args[:task]) - end - end - - def import_amap_log_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - data = "" - ::File.open(filename, 'rb') do |f| - data = f.read(f.stat.size) - end - - case import_filetype_detect(data) - when :amap_log - import_amap_log(args.merge(:data => data)) - when :amap_mlog - import_amap_mlog(args.merge(:data => data)) - else - raise DBImportError.new("Could not determine file type") - end - end - - def import_amap_log(args={}, &block) - data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - - data.each_line do |line| - next if line =~ /^#/ - next if line !~ /^Protocol on ([^:]+):([^\x5c\x2f]+)[\x5c\x2f](tcp|udp) matches (.*)$/n - addr = $1 - next if bl.include? addr - port = $2.to_i - proto = $3.downcase - name = $4 - host = find_or_create_host(:workspace => wspace, :host => addr, :state => Msf::HostState::Alive, :task => args[:task]) - next if not host - yield(:address,addr) if block - info = { - :workspace => wspace, - :task => args[:task], - :host => host, - :proto => proto, - :port => port - } - if name != "unidentified" - info[:name] = name - end - service = find_or_create_service(info) - end - end - - def import_amap_mlog(args={}, &block) - data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - - data.each_line do |line| - next if line =~ /^#/ - r = line.split(':') - next if r.length < 6 - - addr = r[0] - next if bl.include? addr - port = r[1].to_i - proto = r[2].downcase - status = r[3] - name = r[5] - next if status != "open" - - host = find_or_create_host(:workspace => wspace, :host => addr, :state => Msf::HostState::Alive, :task => args[:task]) - next if not host - yield(:address,addr) if block - info = { - :workspace => wspace, - :task => args[:task], - :host => host, - :proto => proto, - :port => port - } - if name != "unidentified" - info[:name] = name - end - service = find_or_create_service(info) - end - end - - def import_ci_noko_stream(args, &block) - if block - doc = Rex::Parser::CIDocument.new(args,framework.db) {|type, data| yield type,data } - else - doc = Rex::Parser::CI.new(args,self) - end - parser = ::Nokogiri::XML::SAX::Parser.new(doc) - parser.parse(args[:data]) - end - - def import_outpost24_xml(args={}, &block) - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - wspace = args[:wspace] || workspace - if Rex::Parser.nokogiri_loaded - parser = "Nokogiri v#{::Nokogiri::VERSION}" - noko_args = args.dup - noko_args[:blacklist] = bl - noko_args[:wspace] = wspace - if block - yield(:parser, parser) - import_outpost24_noko_stream(noko_args) {|type, data| yield type,data} - else - import_outpost24_noko_stream(noko_args) - end - return true - else # Sorry - raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.") - end - end - - def import_outpost24_noko_stream(args={},&block) - if block - doc = Rex::Parser::Outpost24Document.new(args,framework.db) {|type, data| yield type,data } - else - doc = Rex::Parser::Outpost24Document.new(args,self) - end - parser = ::Nokogiri::XML::SAX::Parser.new(doc) - parser.parse(args[:data]) - end - - - def unserialize_object(xml_elem, allow_yaml = false) - return nil unless xml_elem - string = xml_elem.text.to_s.strip - return string unless string.is_a?(String) - return nil if (string.empty? || string.nil?) - - begin - # Validate that it is properly formed base64 first - if string.gsub(/\s+/, '') =~ /^([a-z0-9A-Z\+\/=]+)$/ - Marshal.load($1.unpack("m")[0]) - else - if allow_yaml - begin - YAML.load(string) - rescue - dlog("Badly formatted YAML: '#{string}'") - string - end - else - string - end - end - rescue ::Exception => e - if allow_yaml - YAML.load(string) rescue string - else - string - end - end - end - - # - # Returns something suitable for the +:host+ parameter to the various report_* methods - # - # Takes a Host object, a Session object, an Msf::Session object or a String - # address - # - def normalize_host(host) - return host if host.kind_of? ::Mdm::Host - norm_host = nil - - if (host.kind_of? String) - - if Rex::Socket.is_ipv4?(host) - # If it's an IPv4 addr with a port on the end, strip the port - if host =~ /((\d{1,3}\.){3}\d{1,3}):\d+/ - norm_host = $1 - else - norm_host = host - end - elsif Rex::Socket.is_ipv6?(host) - # If it's an IPv6 addr, drop the scope - address, scope = host.split('%', 2) - norm_host = address - else - norm_host = Rex::Socket.getaddress(host, true) - end - elsif host.kind_of? ::Mdm::Session - norm_host = host.host - elsif host.respond_to?(:session_host) - # Then it's an Msf::Session object - thost = host.session_host - norm_host = thost - end - - # If we got here and don't have a norm_host yet, it could be a - # Msf::Session object with an empty or nil tunnel_host and tunnel_peer; - # see if it has a socket and use its peerhost if so. - if ( - norm_host.nil? and - host.respond_to?(:sock) and - host.sock.respond_to?(:peerhost) and - host.sock.peerhost.to_s.length > 0 - ) - norm_host = session.sock.peerhost - end - # If We got here and still don't have a real host, there's nothing left - # to try, just log it and return what we were given - if not norm_host - dlog("Host could not be normalized: #{host.inspect}") - norm_host = host - end - - norm_host - end - - # A way to sneak the yield back into the db importer. - # Used by the SAX parsers. - def emit(sym,data,&block) - yield(sym,data) - end - -protected - - # - # This holds all of the shared parsing/handling used by the - # Nessus NBE and NESSUS v1 methods - # - def handle_nessus(wspace, hobj, port, nasl, plugin_name, severity, data,task=nil) - addr = hobj.address - # The port section looks like: - # http (80/tcp) - p = port.match(/^([^\(]+)\((\d+)\/([^\)]+)\)/) - return if not p - - # Unnecessary as the caller should already have reported this host - #report_host(:workspace => wspace, :host => addr, :state => Msf::HostState::Alive) - name = p[1].strip - port = p[2].to_i - proto = p[3].downcase - - info = { :workspace => wspace, :host => hobj, :port => port, :proto => proto, :task => task } - if name != "unknown" and name[-1,1] != "?" - info[:name] = name - end - report_service(info) - - if nasl.nil? || nasl.empty? || nasl == 0 || nasl == "0" - return - end - - data.gsub!("\\n", "\n") - - refs = [] - - if (data =~ /^CVE : (.*)$/) - $1.gsub(/C(VE|AN)\-/, '').split(',').map { |r| r.strip }.each do |r| - refs.push('CVE-' + r) - end - end - - if (data =~ /^BID : (.*)$/) - $1.split(',').map { |r| r.strip }.each do |r| - refs.push('BID-' + r) - end - end - - if (data =~ /^Other references : (.*)$/) - $1.split(',').map { |r| r.strip }.each do |r| - ref_id, ref_val = r.split(':') - ref_val ? refs.push(ref_id + '-' + ref_val) : refs.push(ref_id) - end - end - - nss = 'NSS-' + nasl.to_s.strip - refs << nss - - unless plugin_name.to_s.strip.empty? - vuln_name = plugin_name - else - vuln_name = nss - end - - vuln_info = { - :workspace => wspace, - :host => hobj, - :port => port, - :proto => proto, - :name => vuln_name, - :info => data, - :refs => refs, - :task => task, - } - report_vuln(vuln_info) - end - - # - # NESSUS v2 file format has a dramatically different layout - # for ReportItem data - # - def handle_nessus_v2(wspace,hobj,port,proto,name,nasl,nasl_name,severity,description,cve,bid,xref,msf,task=nil) - addr = hobj.address - - info = { :workspace => wspace, :host => hobj, :port => port, :proto => proto, :task => task } - - unless name =~ /^unknown$|\?$/ - info[:name] = name - end - - if port.to_i != 0 - report_service(info) - end - - if nasl.nil? || nasl.empty? || nasl == 0 || nasl == "0" - return - end - - refs = [] - - cve.each do |r| - r.to_s.gsub!(/C(VE|AN)\-/, '') - refs.push('CVE-' + r.to_s) - end if cve - - bid.each do |r| - refs.push('BID-' + r.to_s) - end if bid - - xref.each do |r| - ref_id, ref_val = r.to_s.split(':') - ref_val ? refs.push(ref_id + '-' + ref_val) : refs.push(ref_id) - end if xref - - msfref = "MSF-" << msf if msf - refs.push msfref if msfref - - nss = 'NSS-' + nasl - if nasl_name.nil? || nasl_name.empty? - vuln_name = nss - else - vuln_name = nasl_name - end - - refs << nss.strip - - vuln = { - :workspace => wspace, - :host => hobj, - :name => vuln_name, - :info => description ? description : "", - :refs => refs, - :task => task, - } - - if port.to_i != 0 - vuln[:port] = port - vuln[:proto] = proto - end - - report_vuln(vuln) - end - - # - # IP360 v3 vuln - # - def handle_ip360_v3_svc(wspace,hobj,port,proto,hname,task=nil) - addr = hobj.address - report_host(:workspace => wspace, :host => hobj, :state => Msf::HostState::Alive, :task => task) - - info = { :workspace => wspace, :host => hobj, :port => port, :proto => proto, :task => task } - if hname != "unknown" and hname[-1,1] != "?" - info[:name] = hname - end - - if port.to_i != 0 - report_service(info) - end - end #handle_ip360_v3_svc - - # - # IP360 v3 vuln - # - def handle_ip360_v3_vuln(wspace,hobj,port,proto,hname,vulnid,vulnname,cves,bids,task=nil) - info = { :workspace => wspace, :host => hobj, :port => port, :proto => proto, :task => task } - if hname != "unknown" and hname[-1,1] != "?" - info[:name] = hname - end - - if port.to_i != 0 - report_service(info) - end - - refs = [] - - cves.split(/,/).each do |cve| - refs.push(cve.to_s) - end if cves - - bids.split(/,/).each do |bid| - refs.push('BID-' + bid.to_s) - end if bids - - description = nil # not working yet - vuln = { - :workspace => wspace, - :host => hobj, - :name => vulnname, - :info => description ? description : "", - :refs => refs, - :task => task - } - - if port.to_i != 0 - vuln[:port] = port - vuln[:proto] = proto - end - - report_vuln(vuln) - end #handle_ip360_v3_vuln - - # - # Qualys report parsing/handling - # - def handle_qualys(wspace, hobj, port, protocol, qid, severity, refs, name=nil, title=nil, task=nil) - addr = hobj.address - port = port.to_i if port - - info = { :workspace => wspace, :host => hobj, :port => port, :proto => protocol, :task => task } - if name and name != 'unknown' and name != 'No registered hostname' - info[:name] = name - end - - if info[:host] && info[:port] && info[:proto] - report_service(info) - end - - fixed_refs = [] - if refs - refs.each do |ref| - case ref - when /^MS[0-9]{2}-[0-9]{3}/ - fixed_refs << "MSB-#{ref}" - else - fixed_refs << ref - end - end - end - - return if qid == 0 - title = 'QUALYS-' + qid if title.nil? or title.empty? - if addr - report_vuln( - :workspace => wspace, - :task => task, - :host => hobj, - :port => port, - :proto => protocol, - :name => title, - :refs => fixed_refs - ) - end - end - - def process_nexpose_data_sxml_refs(vuln) - refs = [] - vid = vuln.attributes['id'].to_s.downcase - vry = vuln.attributes['resultCode'].to_s.upcase - - # Only process vuln-exploitable and vuln-version statuses - return if vry !~ /^V[VE]$/ - - refs = [] - vuln.elements.each('id') do |ref| - rtyp = ref.attributes['type'].to_s.upcase - rval = ref.text.to_s.strip - case rtyp - when 'CVE' - refs << rval.gsub('CAN', 'CVE') - when 'MS' # obsolete? - refs << "MSB-MS-#{rval}" - else - refs << "#{rtyp}-#{rval}" - end - end - - refs << "NEXPOSE-#{vid}" - refs - end - -end - -end diff --git a/lib/msf/core/db_manager.rb b/lib/msf/core/db_manager.rb index 6f8ee75ffc..2301f339a9 100644 --- a/lib/msf/core/db_manager.rb +++ b/lib/msf/core/db_manager.rb @@ -1,12 +1,67 @@ # -*- coding: binary -*- -require 'msf/base/config' -require 'msf/core' -require 'msf/core/db' -require 'msf/core/db_manager/migration' -require 'msf/core/task_manager' +# +# Standard Library +# + +require 'csv' require 'fileutils' require 'shellwords' +require 'tmpdir' +require 'uri' + +# +# +# Gems +# +# + +# +# PacketFu +# + +require 'packetfu' + +# +# Rex +# + +require 'rex/parser/acunetix_nokogiri' +require 'rex/parser/appscan_nokogiri' +require 'rex/parser/burp_session_nokogiri' +require 'rex/parser/ci_nokogiri' +require 'rex/parser/foundstone_nokogiri' +require 'rex/parser/fusionvm_nokogiri' +require 'rex/parser/ip360_aspl_xml' +require 'rex/parser/ip360_xml' +require 'rex/parser/mbsa_nokogiri' +require 'rex/parser/nessus_xml' +require 'rex/parser/netsparker_xml' +require 'rex/parser/nexpose_raw_nokogiri' +require 'rex/parser/nexpose_simple_nokogiri' +require 'rex/parser/nexpose_xml' +require 'rex/parser/nmap_nokogiri' +require 'rex/parser/nmap_xml' +require 'rex/parser/openvas_nokogiri' +require 'rex/parser/outpost24_nokogiri' +require 'rex/parser/retina_xml' +require 'rex/parser/wapiti_nokogiri' +require 'rex/socket' + +# +# Project +# + +require 'metasploit/framework/require' +require 'msf/base/config' +require 'msf/core' +require 'msf/core/database_event' +require 'msf/core/db_import_error' +require 'msf/core/db_manager/import_msf_xml' +require 'msf/core/db_manager/migration' +require 'msf/core/host_state' +require 'msf/core/service_state' +require 'msf/core/task_manager' module Msf @@ -18,6 +73,11 @@ module Msf ### class DBManager + extend Metasploit::Framework::Require + + include Msf::DBManager::ImportMsfXml + optionally_include_metasploit_credential_creation + # Provides :framework and other accessors include Msf::DBManager::Migration include Msf::Framework::Offspring @@ -727,5 +787,6239 @@ class DBManager query end + + def rfc3330_reserved(ip) + case ip.class.to_s + when "PacketFu::Octets" + ip_x = ip.to_x + ip_i = ip.to_i + when "String" + if ipv46_validator(ip) + ip_x = ip + ip_i = Rex::Socket.addr_atoi(ip) + else + raise ArgumentError, "Invalid IP address: #{ip.inspect}" + end + when "Fixnum" + if (0..2**32-1).include? ip + ip_x = Rex::Socket.addr_itoa(ip) + ip_i = ip + else + raise ArgumentError, "Invalid IP address: #{ip.inspect}" + end + else + raise ArgumentError, "Invalid IP address: #{ip.inspect}" + end + return true if Rex::Socket::RangeWalker.new("0.0.0.0-0.255.255.255").include? ip_x + return true if Rex::Socket::RangeWalker.new("127.0.0.0-127.255.255.255").include? ip_x + return true if Rex::Socket::RangeWalker.new("169.254.0.0-169.254.255.255").include? ip_x + return true if Rex::Socket::RangeWalker.new("224.0.0.0-239.255.255.255").include? ip_x + return true if Rex::Socket::RangeWalker.new("255.255.255.255-255.255.255.255").include? ip_x + return false + end + + def ipv46_validator(addr) + ipv4_validator(addr) or ipv6_validator(addr) + end + + def ipv4_validator(addr) + return false unless addr.kind_of? String + Rex::Socket.is_ipv4?(addr) + end + + def ipv6_validator(addr) + Rex::Socket.is_ipv6?(addr) + end + + # Takes a space-delimited set of ips and ranges, and subjects + # them to RangeWalker for validation. Returns true or false. + def validate_ips(ips) + ret = true + begin + ips.split(/\s+/).each {|ip| + unless Rex::Socket::RangeWalker.new(ip).ranges + ret = false + break + end + } + rescue + ret = false + end + return ret + end + + + # + # Determines if the database is functional + # + def check + ::ActiveRecord::Base.connection_pool.with_connection { + res = ::Mdm::Host.find(:first) + } + end + + + def default_workspace + ::ActiveRecord::Base.connection_pool.with_connection { + ::Mdm::Workspace.default + } + end + + def find_workspace(name) + ::ActiveRecord::Base.connection_pool.with_connection { + ::Mdm::Workspace.find_by_name(name) + } + end + + # + # Creates a new workspace in the database + # + def add_workspace(name) + ::ActiveRecord::Base.connection_pool.with_connection { + ::Mdm::Workspace.find_or_create_by_name(name) + } + end + + def workspaces + ::ActiveRecord::Base.connection_pool.with_connection { + ::Mdm::Workspace.find(:all) + } + end + + # + # Wait for all pending write to finish + # + def sync + # There is no more queue. + end + + # + # Find a host. Performs no database writes. + # + def get_host(opts) + if opts.kind_of? ::Mdm::Host + return opts + elsif opts.kind_of? String + raise RuntimeError, "This invokation of get_host is no longer supported: #{caller}" + else + address = opts[:addr] || opts[:address] || opts[:host] || return + return address if address.kind_of? ::Mdm::Host + end + ::ActiveRecord::Base.connection_pool.with_connection { + wspace = opts.delete(:workspace) || workspace + if wspace.kind_of? String + wspace = find_workspace(wspace) + end + + address = normalize_host(address) + return wspace.hosts.find_by_address(address) + } + end + + # + # Exactly like report_host but waits for the database to create a host and returns it. + # + def find_or_create_host(opts) + report_host(opts) + end + + # + # Report a host's attributes such as operating system and service pack + # + # The opts parameter MUST contain + # +:host+:: -- the host's ip address + # + # The opts parameter can contain: + # +:state+:: -- one of the Msf::HostState constants + # +:os_name+:: -- something like "Windows", "Linux", or "Mac OS X" + # +:os_flavor+:: -- something like "Enterprise", "Pro", or "Home" + # +:os_sp+:: -- something like "SP2" + # +:os_lang+:: -- something like "English", "French", or "en-US" + # +:arch+:: -- one of the ARCH_* constants + # +:mac+:: -- the host's MAC address + # +:scope+:: -- interface identifier for link-local IPv6 + # +:virtual_host+:: -- the name of the VM host software, eg "VMWare", "QEMU", "Xen", etc. + # + def report_host(opts) + + return if not active + addr = opts.delete(:host) || return + + # Sometimes a host setup through a pivot will see the address as "Remote Pipe" + if addr.eql? "Remote Pipe" + return + end + + ::ActiveRecord::Base.connection_pool.with_connection { + wspace = opts.delete(:workspace) || workspace + if wspace.kind_of? String + wspace = find_workspace(wspace) + end + + ret = { } + + if not addr.kind_of? ::Mdm::Host + addr = normalize_host(addr) + addr, scope = addr.split('%', 2) + opts[:scope] = scope if scope + + unless ipv46_validator(addr) + raise ::ArgumentError, "Invalid IP address in report_host(): #{addr}" + end + + if opts[:comm] and opts[:comm].length > 0 + host = wspace.hosts.find_or_initialize_by_address_and_comm(addr, opts[:comm]) + else + host = wspace.hosts.find_or_initialize_by_address(addr) + end + else + host = addr + end + + # Truncate the info field at the maximum field length + if opts[:info] + opts[:info] = opts[:info][0,65535] + end + + # Truncate the name field at the maximum field length + if opts[:name] + opts[:name] = opts[:name][0,255] + end + + opts.each { |k,v| + if (host.attribute_names.include?(k.to_s)) + unless host.attribute_locked?(k.to_s) + host[k] = v.to_s.gsub(/[\x00-\x1f]/n, '') + end + else + dlog("Unknown attribute for ::Mdm::Host: #{k}") + end + } + host.info = host.info[0,::Mdm::Host.columns_hash["info"].limit] if host.info + + # Set default fields if needed + host.state = HostState::Alive if not host.state + host.comm = '' if not host.comm + host.workspace = wspace if not host.workspace + + if host.changed? + msf_import_timestamps(opts,host) + host.save! + end + + if opts[:task] + Mdm::TaskHost.create( + :task => opts[:task], + :host => host + ) + end + + host + } + end + + + # + # Update a host's attributes via semi-standardized sysinfo hash (Meterpreter) + # + # The opts parameter MUST contain the following entries + # +:host+:: -- the host's ip address + # +:info+:: -- the information hash + # * 'Computer' -- the host name + # * 'OS' -- the operating system string + # * 'Architecture' -- the hardware architecture + # * 'System Language' -- the system language + # + # The opts parameter can contain: + # +:workspace+:: -- the workspace for this host + # + def update_host_via_sysinfo(opts) + + return if not active + addr = opts.delete(:host) || return + info = opts.delete(:info) || return + + # Sometimes a host setup through a pivot will see the address as "Remote Pipe" + if addr.eql? "Remote Pipe" + return + end + + ::ActiveRecord::Base.connection_pool.with_connection { + wspace = opts.delete(:workspace) || workspace + if wspace.kind_of? String + wspace = find_workspace(wspace) + end + + if not addr.kind_of? ::Mdm::Host + addr = normalize_host(addr) + addr, scope = addr.split('%', 2) + opts[:scope] = scope if scope + + unless ipv46_validator(addr) + raise ::ArgumentError, "Invalid IP address in report_host(): #{addr}" + end + + if opts[:comm] and opts[:comm].length > 0 + host = wspace.hosts.find_or_initialize_by_address_and_comm(addr, opts[:comm]) + else + host = wspace.hosts.find_or_initialize_by_address(addr) + end + else + host = addr + end + + res = {} + + if info['Computer'] + res[:name] = info['Computer'] + end + + if info['Architecture'] + res[:arch] = info['Architecture'].split(/\s+/).first + end + + if info['OS'] =~ /^Windows\s*([^\(]+)\(([^\)]+)\)/i + res[:os_name] = "Windows #{$1.strip}" + build = $2.strip + + if build =~ /Service Pack (\d+)/ + res[:os_sp] = "SP" + $1 + end + end + + if info["System Language"] + case info["System Language"] + when /^en_/ + res[:os_lang] = "English" + end + end + + + # Truncate the info field at the maximum field length + if res[:info] + res[:info] = res[:info][0,65535] + end + + # Truncate the name field at the maximum field length + if res[:name] + res[:name] = res[:name][0,255] + end + + res.each { |k,v| + + if (host.attribute_names.include?(k.to_s)) + unless host.attribute_locked?(k.to_s) + host[k] = v.to_s.gsub(/[\x00-\x1f]/n, '') + end + else + dlog("Unknown attribute for Host: #{k}") + end + } + + # Set default fields if needed + host.state = HostState::Alive if not host.state + host.comm = '' if not host.comm + host.workspace = wspace if not host.workspace + + if host.changed? + host.save! + end + + host + } + end + # + # Iterates over the hosts table calling the supplied block with the host + # instance of each entry. + # + def each_host(wspace=workspace, &block) + ::ActiveRecord::Base.connection_pool.with_connection { + wspace.hosts.each do |host| + block.call(host) + end + } + end + + # + # Returns a list of all hosts in the database + # + def hosts(wspace = workspace, only_up = false, addresses = nil) + ::ActiveRecord::Base.connection_pool.with_connection { + conditions = {} + conditions[:state] = [Msf::HostState::Alive, Msf::HostState::Unknown] if only_up + conditions[:address] = addresses if addresses + wspace.hosts.where(conditions).order(:address) + } + end + + + + def find_or_create_service(opts) + report_service(opts) + end + + # + # Record a service in the database. + # + # opts MUST contain + # +:host+:: the host where this service is running + # +:port+:: the port where this service listens + # +:proto+:: the transport layer protocol (e.g. tcp, udp) + # + # opts may contain + # +:name+:: the application layer protocol (e.g. ssh, mssql, smb) + # +:sname+:: an alias for the above + # + def report_service(opts) + return if not active + ::ActiveRecord::Base.connection_pool.with_connection { |conn| + addr = opts.delete(:host) || return + hname = opts.delete(:host_name) + hmac = opts.delete(:mac) + host = nil + wspace = opts.delete(:workspace) || workspace + hopts = {:workspace => wspace, :host => addr} + hopts[:name] = hname if hname + hopts[:mac] = hmac if hmac + + # Other report_* methods take :sname to mean the service name, so we + # map it here to ensure it ends up in the right place despite not being + # a real column. + if opts[:sname] + opts[:name] = opts.delete(:sname) + end + + if addr.kind_of? ::Mdm::Host + host = addr + addr = host.address + else + host = report_host(hopts) + end + + if opts[:port].to_i.zero? + dlog("Skipping port zero for service '%s' on host '%s'" % [opts[:name],host.address]) + return nil + end + + ret = {} +=begin + host = get_host(:workspace => wspace, :address => addr) + if host + host.updated_at = host.created_at + host.state = HostState::Alive + host.save! + end +=end + + proto = opts[:proto] || 'tcp' + + service = host.services.find_or_initialize_by_port_and_proto(opts[:port].to_i, proto) + opts.each { |k,v| + if (service.attribute_names.include?(k.to_s)) + service[k] = ((v and k == :name) ? v.to_s.downcase : v) + else + dlog("Unknown attribute for Service: #{k}") + end + } + service.state ||= ServiceState::Open + service.info ||= "" + + if (service and service.changed?) + msf_import_timestamps(opts,service) + service.save! + end + + if opts[:task] + Mdm::TaskService.create( + :task => opts[:task], + :service => service + ) + end + + ret[:service] = service + } + end + + def get_service(wspace, host, proto, port) + ::ActiveRecord::Base.connection_pool.with_connection { + host = get_host(:workspace => wspace, :address => host) + return if not host + return host.services.find_by_proto_and_port(proto, port) + } + end + + # + # Iterates over the services table calling the supplied block with the + # service instance of each entry. + # + def each_service(wspace=workspace, &block) + ::ActiveRecord::Base.connection_pool.with_connection { + services(wspace).each do |service| + block.call(service) + end + } + end + + # + # Returns a list of all services in the database + # + def services(wspace = workspace, only_up = false, proto = nil, addresses = nil, ports = nil, names = nil) + ::ActiveRecord::Base.connection_pool.with_connection { + conditions = {} + conditions[:state] = [ServiceState::Open] if only_up + conditions[:proto] = proto if proto + conditions["hosts.address"] = addresses if addresses + conditions[:port] = ports if ports + conditions[:name] = names if names + wspace.services.includes(:host).where(conditions).order("hosts.address, port") + } + end + + # Returns a session based on opened_time, host address, and workspace + # (or returns nil) + def get_session(opts) + return if not active + ::ActiveRecord::Base.connection_pool.with_connection { + wspace = opts[:workspace] || opts[:wspace] || workspace + addr = opts[:addr] || opts[:address] || opts[:host] || return + host = get_host(:workspace => wspace, :host => addr) + time = opts[:opened_at] || opts[:created_at] || opts[:time] || return + ::Mdm::Session.find_by_host_id_and_opened_at(host.id, time) + } + end + + # @note The Mdm::Session#desc will be truncated to 255 characters. + # @todo https://www.pivotaltracker.com/story/show/48249739 + # + # @overload report_session(opts) + # Creates an Mdm::Session from Msf::Session. If +via_exploit+ is set on the + # +session+, then an Mdm::Vuln and Mdm::ExploitAttempt is created for the + # session's host. The Mdm::Host for the +session_host+ is created using + # The session.session_host, +session.arch+ (if +session+ responds to arch), + # and the workspace derived from opts or the +session+. The Mdm::Session is + # assumed to be +last_seen+ and +opened_at+ at the time report_session is + # called. +session.exploit_datastore['ParentModule']+ is used for the + # Mdm::Session#via_exploit if +session.via_exploit+ is + # 'exploit/multi/handler'. + # + # @param opts [Hash{Symbol => Object}] options + # @option opt [Msf::Session, #datastore, #platform, #type, #via_exploit, #via_payload] :session + # The in-memory session to persist to the database. + # @option opts [Mdm::Workspace] :workspace The workspace for in which the + # :session host is contained. Also used as the workspace for the + # Mdm::ExploitAttempt and Mdm::Vuln. Defaults to Mdm::Worksapce with + # Mdm::Workspace#name equal to +session.workspace+. + # @return [nil] if {Msf::DBManager#active} is +false+. + # @return [Mdm::Session] if session is saved + # @raise [ArgumentError] if :session is not an {Msf::Session}. + # @raise [ActiveRecord::RecordInvalid] if session is invalid and cannot be + # saved, in which case, the Mdm::ExploitAttempt and Mdm::Vuln will not be + # created, but the Mdm::Host will have been. (There is no transaction + # to rollback the Mdm::Host creation.) + # @see #find_or_create_host + # @see #normalize_host + # @see #report_exploit_success + # @see #report_vuln + # + # @overload report_session(opts) + # Creates an Mdm::Session from Mdm::Host. + # + # @param opts [Hash{Symbol => Object}] options + # @option opts [DateTime, Time] :closed_at The date and time the sesion was + # closed. + # @option opts [String] :close_reason Reason the session was closed. + # @option opts [Hash] :datastore {Msf::DataStore#to_h}. + # @option opts [String] :desc Session description. Will be truncated to 255 + # characters. + # @option opts [Mdm::Host] :host The host on which the session was opened. + # @option opts [DateTime, Time] :last_seen The last date and time the + # session was seen to be open. Defaults to :closed_at's value. + # @option opts [DateTime, Time] :opened_at The date and time that the + # session was opened. + # @option opts [String] :platform The platform of the host. + # @option opts [Array] :routes ([]) The routes through the session for + # pivoting. + # @option opts [String] :stype Session type. + # @option opts [String] :via_exploit The {Msf::Module#fullname} of the + # exploit that was used to open the session. + # @option option [String] :via_payload the {MSf::Module#fullname} of the + # payload sent to the host when the exploit was successful. + # @return [nil] if {Msf::DBManager#active} is +false+. + # @return [Mdm::Session] if session is saved. + # @raise [ArgumentError] if :host is not an Mdm::Host. + # @raise [ActiveRecord::RecordInvalid] if session is invalid and cannot be + # saved. + # + # @raise ArgumentError if :host and :session is +nil+ + def report_session(opts) + return if not active + ::ActiveRecord::Base.connection_pool.with_connection { + if opts[:session] + raise ArgumentError.new("Invalid :session, expected Msf::Session") unless opts[:session].kind_of? Msf::Session + session = opts[:session] + wspace = opts[:workspace] || find_workspace(session.workspace) + h_opts = { } + h_opts[:host] = normalize_host(session) + h_opts[:arch] = session.arch if session.respond_to?(:arch) and session.arch + h_opts[:workspace] = wspace + host = find_or_create_host(h_opts) + sess_data = { + :host_id => host.id, + :stype => session.type, + :desc => session.info, + :platform => session.platform, + :via_payload => session.via_payload, + :via_exploit => session.via_exploit, + :routes => [], + :datastore => session.exploit_datastore.to_h, + :port => session.session_port, + :opened_at => Time.now.utc, + :last_seen => Time.now.utc, + :local_id => session.sid + } + elsif opts[:host] + raise ArgumentError.new("Invalid :host, expected Host object") unless opts[:host].kind_of? ::Mdm::Host + host = opts[:host] + sess_data = { + :host_id => host.id, + :stype => opts[:stype], + :desc => opts[:desc], + :platform => opts[:platform], + :via_payload => opts[:via_payload], + :via_exploit => opts[:via_exploit], + :routes => opts[:routes] || [], + :datastore => opts[:datastore], + :opened_at => opts[:opened_at], + :closed_at => opts[:closed_at], + :last_seen => opts[:last_seen] || opts[:closed_at], + :close_reason => opts[:close_reason], + } + else + raise ArgumentError.new("Missing option :session or :host") + end + ret = {} + + # Truncate the session data if necessary + if sess_data[:desc] + sess_data[:desc] = sess_data[:desc][0,255] + end + + # In the case of multi handler we cannot yet determine the true + # exploit responsible. But we can at least show the parent versus + # just the generic handler: + if session and session.via_exploit == "exploit/multi/handler" and sess_data[:datastore]['ParentModule'] + sess_data[:via_exploit] = sess_data[:datastore]['ParentModule'] + end + + s = ::Mdm::Session.new(sess_data) + s.save! + + if session and session.exploit_task and session.exploit_task.record + session_task = session.exploit_task.record + if session_task.class == Mdm::Task + Mdm::TaskSession.create(:task => session_task, :session => s ) + end + end + + + if opts[:session] + session.db_record = s + end + + # If this is a live session, we know the host is vulnerable to something. + if opts[:session] and session.via_exploit + mod = framework.modules.create(session.via_exploit) + + if session.via_exploit == "exploit/multi/handler" and sess_data[:datastore]['ParentModule'] + mod_fullname = sess_data[:datastore]['ParentModule'] + mod_name = ::Mdm::Module::Detail.find_by_fullname(mod_fullname).name + else + mod_name = mod.name + mod_fullname = mod.fullname + end + + vuln_info = { + :host => host.address, + :name => mod_name, + :refs => mod.references, + :workspace => wspace, + :exploited_at => Time.now.utc, + :info => "Exploited by #{mod_fullname} to create Session #{s.id}" + } + + port = session.exploit_datastore["RPORT"] + service = (port ? host.services.find_by_port(port.to_i) : nil) + + vuln_info[:service] = service if service + + vuln = framework.db.report_vuln(vuln_info) + + if session.via_exploit == "exploit/multi/handler" and sess_data[:datastore]['ParentModule'] + via_exploit = sess_data[:datastore]['ParentModule'] + else + via_exploit = session.via_exploit + end + attempt_info = { + :timestamp => Time.now.utc, + :workspace => wspace, + :module => via_exploit, + :username => session.username, + :refs => mod.references, + :session_id => s.id, + :host => host, + :service => service, + :vuln => vuln + } + + framework.db.report_exploit_success(attempt_info) + + end + + s + } + end + + # + # Record a session event in the database + # + # opts MUST contain one of: + # +:session+:: the Msf::Session OR the ::Mdm::Session we are reporting + # +:etype+:: event type, enum: command, output, upload, download, filedelete + # + # opts may contain + # +:output+:: the data for an output event + # +:command+:: the data for an command event + # +:remote_path+:: path to the associated file for upload, download, and filedelete events + # +:local_path+:: path to the associated file for upload, and download + # + def report_session_event(opts) + return if not active + raise ArgumentError.new("Missing required option :session") if opts[:session].nil? + raise ArgumentError.new("Expected an :etype") unless opts[:etype] + session = nil + + ::ActiveRecord::Base.connection_pool.with_connection { + if opts[:session].respond_to? :db_record + session = opts[:session].db_record + if session.nil? + # The session doesn't have a db_record which means + # a) the database wasn't connected at session registration time + # or + # b) something awful happened and the report_session call failed + # + # Either way, we can't do anything with this session as is, so + # log a warning and punt. + wlog("Warning: trying to report a session_event for a session with no db_record (#{opts[:session].sid})") + return + end + event_data = { :created_at => Time.now } + else + session = opts[:session] + event_data = { :created_at => opts[:created_at] } + end + + event_data[:session_id] = session.id + [:remote_path, :local_path, :output, :command, :etype].each do |attr| + event_data[attr] = opts[attr] if opts[attr] + end + + s = ::Mdm::SessionEvent.create(event_data) + } + end + + def report_session_route(session, route) + return if not active + if session.respond_to? :db_record + s = session.db_record + else + s = session + end + unless s.respond_to?(:routes) + raise ArgumentError.new("Invalid :session, expected Session object got #{session.class}") + end + + ::ActiveRecord::Base.connection_pool.with_connection { + + subnet, netmask = route.split("/") + s.routes.create(:subnet => subnet, :netmask => netmask) + } + end + + def report_session_route_remove(session, route) + return if not active + if session.respond_to? :db_record + s = session.db_record + else + s = session + end + unless s.respond_to?(:routes) + raise ArgumentError.new("Invalid :session, expected Session object got #{session.class}") + end + + ::ActiveRecord::Base.connection_pool.with_connection { + subnet, netmask = route.split("/") + r = s.routes.find_by_subnet_and_netmask(subnet, netmask) + r.destroy if r + } + end + + + def report_exploit_success(opts) + ::ActiveRecord::Base.connection_pool.with_connection { + + wspace = opts.delete(:workspace) || workspace + mrefs = opts.delete(:refs) || return + host = opts.delete(:host) + port = opts.delete(:port) + prot = opts.delete(:proto) + svc = opts.delete(:service) + vuln = opts.delete(:vuln) + + timestamp = opts.delete(:timestamp) + username = opts.delete(:username) + mname = opts.delete(:module) + + # Look up or generate the host as appropriate + if not (host and host.kind_of? ::Mdm::Host) + if svc.kind_of? ::Mdm::Service + host = svc.host + else + host = report_host(:workspace => wspace, :address => host ) + end + end + + # Bail if we dont have a host object + return if not host + + # Look up or generate the service as appropriate + if port and svc.nil? + svc = report_service(:workspace => wspace, :host => host, :port => port, :proto => prot ) if port + end + + if not vuln + # Create a references map from the module list + ref_objs = ::Mdm::Ref.where(:name => mrefs.map { |ref| + if ref.respond_to?(:ctx_id) and ref.respond_to?(:ctx_val) + "#{ref.ctx_id}-#{ref.ctx_val}" + else + ref.to_s + end + }) + + # Try find a matching vulnerability + vuln = find_vuln_by_refs(ref_objs, host, svc) + end + + # We have match, lets create a vuln_attempt record + if vuln + attempt_info = { + :vuln_id => vuln.id, + :attempted_at => timestamp || Time.now.utc, + :exploited => true, + :username => username || "unknown", + :module => mname + } + + attempt_info[:session_id] = opts[:session_id] if opts[:session_id] + attempt_info[:loot_id] = opts[:loot_id] if opts[:loot_id] + + vuln.vuln_attempts.create(attempt_info) + + # Correct the vuln's associated service if necessary + if svc and vuln.service_id.nil? + vuln.service = svc + vuln.save + end + end + + # Report an exploit attempt all the same + attempt_info = { + :attempted_at => timestamp || Time.now.utc, + :exploited => true, + :username => username || "unknown", + :module => mname + } + + attempt_info[:vuln_id] = vuln.id if vuln + attempt_info[:session_id] = opts[:session_id] if opts[:session_id] + attempt_info[:loot_id] = opts[:loot_id] if opts[:loot_id] + + if svc + attempt_info[:port] = svc.port + attempt_info[:proto] = svc.proto + end + + if port and svc.nil? + attempt_info[:port] = port + attempt_info[:proto] = prot || "tcp" + end + + host.exploit_attempts.create(attempt_info) + } + end + + def report_exploit_failure(opts) + + ::ActiveRecord::Base.connection_pool.with_connection { + wspace = opts.delete(:workspace) || workspace + mrefs = opts.delete(:refs) || return + host = opts.delete(:host) + port = opts.delete(:port) + prot = opts.delete(:proto) + svc = opts.delete(:service) + vuln = opts.delete(:vuln) + + timestamp = opts.delete(:timestamp) + freason = opts.delete(:fail_reason) + fdetail = opts.delete(:fail_detail) + username = opts.delete(:username) + mname = opts.delete(:module) + + # Look up the host as appropriate + if not (host and host.kind_of? ::Mdm::Host) + if svc.kind_of? ::Mdm::Service + host = svc.host + else + host = get_host( :workspace => wspace, :address => host ) + end + end + + # Bail if we dont have a host object + return if not host + + # Look up the service as appropriate + if port and svc.nil? + prot ||= "tcp" + svc = get_service(wspace, host, prot, port) if port + end + + if not vuln + # Create a references map from the module list + ref_objs = ::Mdm::Ref.where(:name => mrefs.map { |ref| + if ref.respond_to?(:ctx_id) and ref.respond_to?(:ctx_val) + "#{ref.ctx_id}-#{ref.ctx_val}" + else + ref.to_s + end + }) + + # Try find a matching vulnerability + vuln = find_vuln_by_refs(ref_objs, host, svc) + end + + # Report a vuln_attempt if we found a match + if vuln + attempt_info = { + :attempted_at => timestamp || Time.now.utc, + :exploited => false, + :fail_reason => freason, + :fail_detail => fdetail, + :username => username || "unknown", + :module => mname + } + + vuln.vuln_attempts.create(attempt_info) + end + + # Report an exploit attempt all the same + attempt_info = { + :attempted_at => timestamp || Time.now.utc, + :exploited => false, + :username => username || "unknown", + :module => mname, + :fail_reason => freason, + :fail_detail => fdetail + } + + attempt_info[:vuln_id] = vuln.id if vuln + + if svc + attempt_info[:port] = svc.port + attempt_info[:proto] = svc.proto + end + + if port and svc.nil? + attempt_info[:port] = port + attempt_info[:proto] = prot || "tcp" + end + + host.exploit_attempts.create(attempt_info) + } + end + + + def report_vuln_attempt(vuln, opts) + ::ActiveRecord::Base.connection_pool.with_connection { + return if not vuln + info = {} + + # Opts can be keyed by strings or symbols + ::Mdm::VulnAttempt.column_names.each do |kn| + k = kn.to_sym + next if ['id', 'vuln_id'].include?(kn) + info[k] = opts[kn] if opts[kn] + info[k] = opts[k] if opts[k] + end + + return unless info[:attempted_at] + + vuln.vuln_attempts.create(info) + } + end + + def report_exploit_attempt(host, opts) + ::ActiveRecord::Base.connection_pool.with_connection { + return if not host + info = {} + + # Opts can be keyed by strings or symbols + ::Mdm::VulnAttempt.column_names.each do |kn| + k = kn.to_sym + next if ['id', 'host_id'].include?(kn) + info[k] = opts[kn] if opts[kn] + info[k] = opts[k] if opts[k] + end + + host.exploit_attempts.create(info) + } + end + + def get_client(opts) + ::ActiveRecord::Base.connection_pool.with_connection { + wspace = opts.delete(:workspace) || workspace + host = get_host(:workspace => wspace, :host => opts[:host]) || return + client = host.clients.where({:ua_string => opts[:ua_string]}).first() + return client + } + end + + def find_or_create_client(opts) + report_client(opts) + end + + # + # Report a client running on a host. + # + # opts MUST contain + # +:ua_string+:: the value of the User-Agent header + # +:host+:: the host where this client connected from, can be an ip address or a Host object + # + # opts can contain + # +:ua_name+:: one of the Msf::HttpClients constants + # +:ua_ver+:: detected version of the given client + # +:campaign+:: an id or Campaign object + # + # Returns a Client. + # + def report_client(opts) + return if not active + ::ActiveRecord::Base.connection_pool.with_connection { + addr = opts.delete(:host) || return + wspace = opts.delete(:workspace) || workspace + report_host(:workspace => wspace, :host => addr) + + ret = {} + + host = get_host(:workspace => wspace, :host => addr) + client = host.clients.find_or_initialize_by_ua_string(opts[:ua_string]) + + opts[:ua_string] = opts[:ua_string].to_s + + campaign = opts.delete(:campaign) + if campaign + case campaign + when Campaign + opts[:campaign_id] = campaign.id + else + opts[:campaign_id] = campaign + end + end + + opts.each { |k,v| + if (client.attribute_names.include?(k.to_s)) + client[k] = v + else + dlog("Unknown attribute for Client: #{k}") + end + } + if (client and client.changed?) + client.save! + end + ret[:client] = client + } + end + + # + # This method iterates the vulns table calling the supplied block with the + # vuln instance of each entry. + # + def each_vuln(wspace=workspace,&block) + ::ActiveRecord::Base.connection_pool.with_connection { + wspace.vulns.each do |vulns| + block.call(vulns) + end + } + end + + # + # This methods returns a list of all vulnerabilities in the database + # + def vulns(wspace=workspace) + ::ActiveRecord::Base.connection_pool.with_connection { + wspace.vulns + } + end + + # + # This methods returns a list of all credentials in the database + # + def creds(wspace=workspace) + ::ActiveRecord::Base.connection_pool.with_connection { + Mdm::Cred.includes({:service => :host}).where("hosts.workspace_id = ?", wspace.id) + } + end + + # + # This method returns a list of all exploited hosts in the database. + # + def exploited_hosts(wspace=workspace) + ::ActiveRecord::Base.connection_pool.with_connection { + wspace.exploited_hosts + } + end + + # + # This method iterates the notes table calling the supplied block with the + # note instance of each entry. + # + def each_note(wspace=workspace, &block) + ::ActiveRecord::Base.connection_pool.with_connection { + wspace.notes.each do |note| + block.call(note) + end + } + end + + # + # Find or create a note matching this type/data + # + def find_or_create_note(opts) + report_note(opts) + end + + # + # Report a Note to the database. Notes can be tied to a ::Mdm::Workspace, Host, or Service. + # + # opts MUST contain + # +:type+:: The type of note, e.g. smb_peer_os + # + # opts can contain + # +:workspace+:: the workspace to associate with this Note + # +:host+:: an IP address or a Host object to associate with this Note + # +:service+:: a Service object to associate with this Note + # +:data+:: whatever it is you're making a note of + # +:port+:: along with +:host+ and +:proto+, a service to associate with this Note + # +:proto+:: along with +:host+ and +:port+, a service to associate with this Note + # +:update+:: what to do in case a similar Note exists, see below + # + # The +:update+ option can have the following values: + # +:unique+:: allow only a single Note per +:host+/+:type+ pair + # +:unique_data+:: like +:uniqe+, but also compare +:data+ + # +:insert+:: always insert a new Note even if one with identical values exists + # + # If the provided +:host+ is an IP address and does not exist in the + # database, it will be created. If +:workspace+, +:host+ and +:service+ + # are all omitted, the new Note will be associated with the current + # workspace. + # + def report_note(opts) + return if not active + ::ActiveRecord::Base.connection_pool.with_connection { + wspace = opts.delete(:workspace) || workspace + if wspace.kind_of? String + wspace = find_workspace(wspace) + end + seen = opts.delete(:seen) || false + crit = opts.delete(:critical) || false + host = nil + addr = nil + # Report the host so it's there for the Proc to use below + if opts[:host] + if opts[:host].kind_of? ::Mdm::Host + host = opts[:host] + else + addr = normalize_host(opts[:host]) + host = report_host({:workspace => wspace, :host => addr}) + end + # Do the same for a service if that's also included. + if (opts[:port]) + proto = nil + sname = nil + case opts[:proto].to_s.downcase # Catch incorrect usages + when 'tcp','udp' + proto = opts[:proto] + sname = opts[:sname] if opts[:sname] + when 'dns','snmp','dhcp' + proto = 'udp' + sname = opts[:proto] + else + proto = 'tcp' + sname = opts[:proto] + end + sopts = { + :workspace => wspace, + :host => host, + :port => opts[:port], + :proto => proto + } + sopts[:name] = sname if sname + report_service(sopts) + end + end + # Update Modes can be :unique, :unique_data, :insert + mode = opts[:update] || :unique + + ret = {} + + if addr and not host + host = get_host(:workspace => wspace, :host => addr) + end + if host and (opts[:port] and opts[:proto]) + service = get_service(wspace, host, opts[:proto], opts[:port]) + elsif opts[:service] and opts[:service].kind_of? ::Mdm::Service + service = opts[:service] + end +=begin + if host + host.updated_at = host.created_at + host.state = HostState::Alive + host.save! + end +=end + ntype = opts.delete(:type) || opts.delete(:ntype) || (raise RuntimeError, "A note :type or :ntype is required") + data = opts[:data] + note = nil + + conditions = { :ntype => ntype } + conditions[:host_id] = host[:id] if host + conditions[:service_id] = service[:id] if service + + case mode + when :unique + note = wspace.notes.where(conditions).first_or_initialize + note.data = data + when :unique_data + notes = wspace.notes.where(conditions) + + # Don't make a new Note with the same data as one that already + # exists for the given: type and (host or service) + notes.each do |n| + # Compare the deserialized data from the table to the raw + # data we're looking for. Because of the serialization we + # can't do this easily or reliably in SQL. + if n.data == data + note = n + break + end + end + if not note + # We didn't find one with the data we're looking for, make + # a new one. + note = wspace.notes.new(conditions.merge(:data => data)) + end + else + # Otherwise, assume :insert, which means always make a new one + note = wspace.notes.new + if host + note.host_id = host[:id] + end + if opts[:service] and opts[:service].kind_of? ::Mdm::Service + note.service_id = opts[:service][:id] + end + note.seen = seen + note.critical = crit + note.ntype = ntype + note.data = data + end + msf_import_timestamps(opts,note) + note.save! + ret[:note] = note + } + end + + # + # This methods returns a list of all notes in the database + # + def notes(wspace=workspace) + ::ActiveRecord::Base.connection_pool.with_connection { + wspace.notes + } + end + + # This is only exercised by MSF3 XML importing for now. Needs the wait + # conditions and return hash as well. + def report_host_tag(opts) + name = opts.delete(:name) + raise DBImportError.new("Missing required option :name") unless name + addr = opts.delete(:addr) + raise DBImportError.new("Missing required option :addr") unless addr + wspace = opts.delete(:wspace) + raise DBImportError.new("Missing required option :wspace") unless wspace + ::ActiveRecord::Base.connection_pool.with_connection { + if wspace.kind_of? String + wspace = find_workspace(wspace) + end + + host = nil + report_host(:workspace => wspace, :address => addr) + + + host = get_host(:workspace => wspace, :address => addr) + desc = opts.delete(:desc) + summary = opts.delete(:summary) + detail = opts.delete(:detail) + crit = opts.delete(:crit) + possible_tags = Mdm::Tag.includes(:hosts).where("hosts.workspace_id = ? and tags.name = ?", wspace.id, name).order("tags.id DESC").limit(1) + tag = (possible_tags.blank? ? Mdm::Tag.new : possible_tags.first) + tag.name = name + tag.desc = desc + tag.report_summary = !!summary + tag.report_detail = !!detail + tag.critical = !!crit + tag.hosts = tag.hosts | [host] + tag.save! if tag.changed? + } + end + + # + # Store a set of credentials in the database. + # + # report_auth_info used to create a note, now it creates + # an entry in the creds table. It's much more akin to + # report_vuln() now. + # + # opts MUST contain + # +:host+:: an IP address or Host object reference + # +:port+:: a port number + # + # opts can contain + # +:user+:: the username + # +:pass+:: the password, or path to ssh_key + # +:ptype+:: the type of password (password(ish), hash, or ssh_key) + # +:proto+:: a transport name for the port + # +:sname+:: service name + # +:active+:: by default, a cred is active, unless explicitly false + # +:proof+:: data used to prove the account is actually active. + # + # Sources: Credentials can be sourced from another credential, or from + # a vulnerability. For example, if an exploit was used to dump the + # smb_hashes, and this credential comes from there, the source_id would + # be the Vuln id (as reported by report_vuln) and the type would be "Vuln". + # + # +:source_id+:: The Vuln or Cred id of the source of this cred. + # +:source_type+:: Either Vuln or Cred + # + # TODO: This is written somewhat host-centric, when really the + # Service is the thing. Need to revisit someday. + def report_auth_info(opts={}) + return if not active + raise ArgumentError.new("Missing required option :host") if opts[:host].nil? + raise ArgumentError.new("Missing required option :port") if (opts[:port].nil? and opts[:service].nil?) + + if (not opts[:host].kind_of?(::Mdm::Host)) and (not validate_ips(opts[:host])) + raise ArgumentError.new("Invalid address or object for :host (#{opts[:host].inspect})") + end + + ::ActiveRecord::Base.connection_pool.with_connection { + host = opts.delete(:host) + ptype = opts.delete(:type) || "password" + token = [opts.delete(:user), opts.delete(:pass)] + sname = opts.delete(:sname) + port = opts.delete(:port) + proto = opts.delete(:proto) || "tcp" + proof = opts.delete(:proof) + source_id = opts.delete(:source_id) + source_type = opts.delete(:source_type) + duplicate_ok = opts.delete(:duplicate_ok) + # Nil is true for active. + active = (opts[:active] || opts[:active].nil?) ? true : false + + wspace = opts.delete(:workspace) || workspace + + # Service management; assume the user knows what + # he's talking about. + service = opts.delete(:service) || report_service(:host => host, :port => port, :proto => proto, :name => sname, :workspace => wspace) + + # Non-US-ASCII usernames are tripping up the database at the moment, this is a temporary fix until we update the tables + if (token[0]) + # convert the token to US-ASCII from UTF-8 to prevent an error + token[0] = token[0].unpack("C*").pack("C*") + token[0] = token[0].gsub(/[\x00-\x1f\x7f-\xff]/n){|m| "\\x%.2x" % m.unpack("C")[0] } + end + + if (token[1]) + token[1] = token[1].unpack("C*").pack("C*") + token[1] = token[1].gsub(/[\x00-\x1f\x7f-\xff]/n){|m| "\\x%.2x" % m.unpack("C")[0] } + end + + ret = {} + + # Check to see if the creds already exist. We look also for a downcased username with the + # same password because we can fairly safely assume they are not in fact two seperate creds. + # this allows us to hedge against duplication of creds in the DB. + + if duplicate_ok + # If duplicate usernames are okay, find by both user and password (allows + # for actual duplicates to get modified updated_at, sources, etc) + if token[0].nil? or token[0].empty? + cred = service.creds.find_or_initialize_by_user_and_ptype_and_pass(token[0] || "", ptype, token[1] || "") + else + cred = service.creds.find_by_user_and_ptype_and_pass(token[0] || "", ptype, token[1] || "") + unless cred + dcu = token[0].downcase + cred = service.creds.find_by_user_and_ptype_and_pass( dcu || "", ptype, token[1] || "") + unless cred + cred = service.creds.find_or_initialize_by_user_and_ptype_and_pass(token[0] || "", ptype, token[1] || "") + end + end + end + else + # Create the cred by username only (so we can change passwords) + if token[0].nil? or token[0].empty? + cred = service.creds.find_or_initialize_by_user_and_ptype(token[0] || "", ptype) + else + cred = service.creds.find_by_user_and_ptype(token[0] || "", ptype) + unless cred + dcu = token[0].downcase + cred = service.creds.find_by_user_and_ptype_and_pass( dcu || "", ptype, token[1] || "") + unless cred + cred = service.creds.find_or_initialize_by_user_and_ptype(token[0] || "", ptype) + end + end + end + end + + # Update with the password + cred.pass = (token[1] || "") + + # Annotate the credential + cred.ptype = ptype + cred.active = active + + # Update the source ID only if there wasn't already one. + if source_id and !cred.source_id + cred.source_id = source_id + cred.source_type = source_type if source_type + end + + # Safe proof (lazy way) -- doesn't chop expanded + # characters correctly, but shouldn't ever be a problem. + unless proof.nil? + proof = Rex::Text.to_hex_ascii(proof) + proof = proof[0,4096] + end + cred.proof = proof + + # Update the timestamp + if cred.changed? + msf_import_timestamps(opts,cred) + cred.save! + end + + # Ensure the updated_at is touched any time report_auth_info is called + # except when it's set explicitly (as it is for imports) + unless opts[:updated_at] || opts["updated_at"] + cred.updated_at = Time.now.utc + cred.save! + end + + + if opts[:task] + Mdm::TaskCred.create( + :task => opts[:task], + :cred => cred + ) + end + + ret[:cred] = cred + } + end + + alias :report_cred :report_auth_info + alias :report_auth :report_auth_info + + # + # Find or create a credential matching this type/data + # + def find_or_create_cred(opts) + report_auth_info(opts) + end + + # + # This method iterates the creds table calling the supplied block with the + # cred instance of each entry. + # + def each_cred(wspace=workspace,&block) + ::ActiveRecord::Base.connection_pool.with_connection { + wspace.creds.each do |cred| + block.call(cred) + end + } + end + + def each_exploited_host(wspace=workspace,&block) + ::ActiveRecord::Base.connection_pool.with_connection { + wspace.exploited_hosts.each do |eh| + block.call(eh) + end + } + end + + # + # Find or create a vuln matching this service/name + # + def find_or_create_vuln(opts) + report_vuln(opts) + end + + # + # opts MUST contain + # +:host+:: the host where this vulnerability resides + # +:name+:: the friendly name for this vulnerability (title) + # + # opts can contain + # +:info+:: a human readable description of the vuln, free-form text + # +:refs+:: an array of Ref objects or string names of references + # +:details:: a hash with :key pointed to a find criteria hash and the rest containing VulnDetail fields + # + def report_vuln(opts) + return if not active + raise ArgumentError.new("Missing required option :host") if opts[:host].nil? + raise ArgumentError.new("Deprecated data column for vuln, use .info instead") if opts[:data] + name = opts[:name] || return + info = opts[:info] + + ::ActiveRecord::Base.connection_pool.with_connection { + + wspace = opts.delete(:workspace) || workspace + exploited_at = opts[:exploited_at] || opts["exploited_at"] + details = opts.delete(:details) + rids = opts.delete(:ref_ids) + + if opts[:refs] + rids ||= [] + opts[:refs].each do |r| + if (r.respond_to?(:ctx_id)) and (r.respond_to?(:ctx_val)) + r = "#{r.ctx_id}-#{r.ctx_val}" + end + rids << find_or_create_ref(:name => r) + end + end + + host = nil + addr = nil + if opts[:host].kind_of? ::Mdm::Host + host = opts[:host] + else + host = report_host({:workspace => wspace, :host => opts[:host]}) + addr = normalize_host(opts[:host]) + end + + ret = {} + + # Truncate the info field at the maximum field length + if info + info = info[0,65535] + end + + # Truncate the name field at the maximum field length + name = name[0,255] + + # Placeholder for the vuln object + vuln = nil + + # Identify the associated service + service = opts.delete(:service) + + # Treat port zero as no service + if service or opts[:port].to_i > 0 + + if not service + proto = nil + case opts[:proto].to_s.downcase # Catch incorrect usages, as in report_note + when 'tcp','udp' + proto = opts[:proto] + when 'dns','snmp','dhcp' + proto = 'udp' + sname = opts[:proto] + else + proto = 'tcp' + sname = opts[:proto] + end + + service = host.services.find_or_create_by_port_and_proto(opts[:port].to_i, proto) + end + + # Try to find an existing vulnerability with the same service & references + # If there are multiple matches, choose the one with the most matches + # If a match is found on a vulnerability with no associated service, + # update that vulnerability with our service information. This helps + # prevent dupes of the same vuln found by both local patch and + # service detection. + if rids and rids.length > 0 + vuln = find_vuln_by_refs(rids, host, service) + vuln.service = service if vuln + end + else + # Try to find an existing vulnerability with the same host & references + # If there are multiple matches, choose the one with the most matches + if rids and rids.length > 0 + vuln = find_vuln_by_refs(rids, host) + end + end + + # Try to match based on vuln_details records + if not vuln and opts[:details_match] + vuln = find_vuln_by_details(opts[:details_match], host, service) + if vuln and service and not vuln.service + vuln.service = service + end + end + + # No matches, so create a new vuln record + unless vuln + if service + vuln = service.vulns.find_by_name(name) + else + vuln = host.vulns.find_by_name(name) + end + + unless vuln + + vinf = { + :host_id => host.id, + :name => name, + :info => info + } + + vinf[:service_id] = service.id if service + vuln = Mdm::Vuln.create(vinf) + end + end + + # Set the exploited_at value if provided + vuln.exploited_at = exploited_at if exploited_at + + # Merge the references + if rids + vuln.refs << (rids - vuln.refs) + end + + # Finalize + if vuln.changed? + msf_import_timestamps(opts,vuln) + vuln.save! + end + + # Handle vuln_details parameters + report_vuln_details(vuln, details) if details + + vuln + } + end + + def find_vuln_by_refs(refs, host, service=nil) + + vuln = nil + + # Try to find an existing vulnerability with the same service & references + # If there are multiple matches, choose the one with the most matches + if service + refs_ids = refs.map{|x| x.id } + vuln = service.vulns.find(:all, :include => [:refs], :conditions => { 'refs.id' => refs_ids }).sort { |a,b| + ( refs_ids - a.refs.map{|x| x.id } ).length <=> ( refs_ids - b.refs.map{|x| x.id } ).length + }.first + end + + # Return if we matched based on service + return vuln if vuln + + # Try to find an existing vulnerability with the same host & references + # If there are multiple matches, choose the one with the most matches + refs_ids = refs.map{|x| x.id } + vuln = host.vulns.find(:all, :include => [:refs], :conditions => { 'service_id' => nil, 'refs.id' => refs_ids }).sort { |a,b| + ( refs_ids - a.refs.map{|x| x.id } ).length <=> ( refs_ids - b.refs.map{|x| x.id } ).length + }.first + + return vuln + end + + + def find_vuln_by_details(details_map, host, service=nil) + + # Create a modified version of the criteria in order to match against + # the joined version of the fields + + crit = {} + details_map.each_pair do |k,v| + crit[ "vuln_details.#{k}" ] = v + end + + vuln = nil + + if service + vuln = service.vulns.find(:first, :include => [:vuln_details], :conditions => crit) + end + + # Return if we matched based on service + return vuln if vuln + + # Prevent matches against other services + crit["vulns.service_id"] = nil if service + vuln = host.vulns.find(:first, :include => [:vuln_details], :conditions => crit) + + return vuln + end + + def get_vuln(wspace, host, service, name, data='') + raise RuntimeError, "Not workspace safe: #{caller.inspect}" + ::ActiveRecord::Base.connection_pool.with_connection { + vuln = nil + if (service) + vuln = ::Mdm::Vuln.find.where("name = ? and service_id = ? and host_id = ?", name, service.id, host.id).order("vulns.id DESC").first() + else + vuln = ::Mdm::Vuln.find.where("name = ? and host_id = ?", name, host.id).first() + end + + return vuln + } + end + + # + # Find or create a reference matching this name + # + def find_or_create_ref(opts) + ret = {} + ret[:ref] = get_ref(opts[:name]) + return ret[:ref] if ret[:ref] + + ::ActiveRecord::Base.connection_pool.with_connection { + ref = ::Mdm::Ref.find_or_initialize_by_name(opts[:name]) + if ref and ref.changed? + ref.save! + end + ret[:ref] = ref + } + end + + def get_ref(name) + ::ActiveRecord::Base.connection_pool.with_connection { + ::Mdm::Ref.find_by_name(name) + } + end + + # + # Populate the vuln_details table with additional + # information, matched by a specific criteria + # + def report_vuln_details(vuln, details) + ::ActiveRecord::Base.connection_pool.with_connection { + detail = ::Mdm::VulnDetail.where(( details.delete(:key) || {} ).merge(:vuln_id => vuln.id)).first + if detail + details.each_pair do |k,v| + detail[k] = v + end + detail.save! if detail.changed? + detail + else + detail = ::Mdm::VulnDetail.create(details.merge(:vuln_id => vuln.id)) + end + } + end + + # + # Update vuln_details records en-masse based on specific criteria + # Note that this *can* update data across workspaces + # + def update_vuln_details(details) + ::ActiveRecord::Base.connection_pool.with_connection { + criteria = details.delete(:key) || {} + ::Mdm::VulnDetail.update(key, details) + } + end + + # + # Populate the host_details table with additional + # information, matched by a specific criteria + # + def report_host_details(host, details) + ::ActiveRecord::Base.connection_pool.with_connection { + + detail = ::Mdm::HostDetail.where(( details.delete(:key) || {} ).merge(:host_id => host.id)).first + if detail + details.each_pair do |k,v| + detail[k] = v + end + detail.save! if detail.changed? + detail + else + detail = ::Mdm::HostDetail.create(details.merge(:host_id => host.id)) + end + } + end + + # report_exploit() used to be used to track sessions and which modules + # opened them. That information is now available with the session table + # directly. TODO: kill this completely some day -- for now just warn if + # some other UI is actually using it. + def report_exploit(opts={}) + wlog("Deprecated method call: report_exploit()\n" + + "report_exploit() options: #{opts.inspect}\n" + + "report_exploit() call stack:\n\t#{caller.join("\n\t")}" + ) + end + + # + # Deletes a host and associated data matching this address/comm + # + def del_host(wspace, address, comm='') + ::ActiveRecord::Base.connection_pool.with_connection { + address, scope = address.split('%', 2) + host = wspace.hosts.find_by_address_and_comm(address, comm) + host.destroy if host + } + end + + # + # Deletes a port and associated vulns matching this port + # + def del_service(wspace, address, proto, port, comm='') + + host = get_host(:workspace => wspace, :address => address) + return unless host + + ::ActiveRecord::Base.connection_pool.with_connection { + host.services.where({:proto => proto, :port => port}).each { |s| s.destroy } + } + end + + # + # Find a reference matching this name + # + def has_ref?(name) + ::ActiveRecord::Base.connection_pool.with_connection { + Mdm::Ref.find_by_name(name) + } + end + + # + # Find a vulnerability matching this name + # + def has_vuln?(name) + ::ActiveRecord::Base.connection_pool.with_connection { + Mdm::Vuln.find_by_name(name) + } + end + + # + # Look for an address across all comms + # + def has_host?(wspace,addr) + ::ActiveRecord::Base.connection_pool.with_connection { + address, scope = addr.split('%', 2) + wspace.hosts.find_by_address(addr) + } + end + + def events(wspace=workspace) + ::ActiveRecord::Base.connection_pool.with_connection { + wspace.events.find :all, :order => 'created_at ASC' + } + end + + def report_event(opts = {}) + return if not active + ::ActiveRecord::Base.connection_pool.with_connection { + wspace = opts.delete(:workspace) || workspace + return if not wspace # Temp fix? + uname = opts.delete(:username) + + if ! opts[:host].kind_of? ::Mdm::Host and opts[:host] + opts[:host] = report_host(:workspace => wspace, :host => opts[:host]) + end + + ::Mdm::Event.create(opts.merge(:workspace_id => wspace[:id], :username => uname)) + } + end + + # + # Loot collection + # + # + # This method iterates the loot table calling the supplied block with the + # instance of each entry. + # + def each_loot(wspace=workspace, &block) + ::ActiveRecord::Base.connection_pool.with_connection { + wspace.loots.each do |note| + block.call(note) + end + } + end + + # + # Find or create a loot matching this type/data + # + def find_or_create_loot(opts) + report_loot(opts) + end + + def report_loot(opts) + return if not active + ::ActiveRecord::Base.connection_pool.with_connection { + wspace = opts.delete(:workspace) || workspace + path = opts.delete(:path) || (raise RuntimeError, "A loot :path is required") + + host = nil + addr = nil + + # Report the host so it's there for the Proc to use below + if opts[:host] + if opts[:host].kind_of? ::Mdm::Host + host = opts[:host] + else + host = report_host({:workspace => wspace, :host => opts[:host]}) + addr = normalize_host(opts[:host]) + end + end + + ret = {} + + ltype = opts.delete(:type) || opts.delete(:ltype) || (raise RuntimeError, "A loot :type or :ltype is required") + ctype = opts.delete(:ctype) || opts.delete(:content_type) || 'text/plain' + name = opts.delete(:name) + info = opts.delete(:info) + data = opts[:data] + loot = wspace.loots.new + + if host + loot.host_id = host[:id] + end + if opts[:service] and opts[:service].kind_of? ::Mdm::Service + loot.service_id = opts[:service][:id] + end + + loot.path = path + loot.ltype = ltype + loot.content_type = ctype + loot.data = data + loot.name = name if name + loot.info = info if info + loot.workspace = wspace + msf_import_timestamps(opts,loot) + loot.save! + + ret[:loot] = loot + } + end + + # + # This methods returns a list of all loot in the database + # + def loots(wspace=workspace) + ::ActiveRecord::Base.connection_pool.with_connection { + wspace.loots + } + end + + # + # Find or create a task matching this type/data + # + def find_or_create_task(opts) + report_task(opts) + end + + def report_task(opts) + return if not active + ::ActiveRecord::Base.connection_pool.with_connection { + wspace = opts.delete(:workspace) || workspace + path = opts.delete(:path) || (raise RuntimeError, "A task :path is required") + + ret = {} + + user = opts.delete(:user) + desc = opts.delete(:desc) + error = opts.delete(:error) + info = opts.delete(:info) + mod = opts.delete(:mod) + options = opts.delete(:options) + prog = opts.delete(:prog) + result = opts.delete(:result) + completed_at = opts.delete(:completed_at) + task = wspace.tasks.new + + task.created_by = user + task.description = desc + task.error = error if error + task.info = info + task.module = mod + task.options = options + task.path = path + task.progress = prog + task.result = result if result + msf_import_timestamps(opts,task) + # Having blank completed_ats, while accurate, will cause unstoppable tasks. + if completed_at.nil? || completed_at.empty? + task.completed_at = opts[:updated_at] + else + task.completed_at = completed_at + end + task.save! + ret[:task] = task + } + end + + # + # This methods returns a list of all tasks in the database + # + def tasks(wspace=workspace) + ::ActiveRecord::Base.connection_pool.with_connection { + wspace.tasks + } + end + + + # TODO This method does not attempt to find. It just creates + # a report based on the passed params. + def find_or_create_report(opts) + report_report(opts) + end + + # Creates a Report based on passed parameters. Does not handle + # child artifacts. + # @param opts [Hash] + # @return [Integer] ID of created report + def report_report(opts) + return if not active + created = opts.delete(:created_at) + updated = opts.delete(:updated_at) + state = opts.delete(:state) + + ::ActiveRecord::Base.connection_pool.with_connection { + report = Report.new(opts) + report.created_at = created + report.updated_at = updated + + unless report.valid? + errors = report.errors.full_messages.join('; ') + raise RuntimeError "Report to be imported is not valid: #{errors}" + end + report.state = :complete # Presume complete since it was exported + report.save + + report.id + } + end + + # Creates a ReportArtifact based on passed parameters. + # @param opts [Hash] of ReportArtifact attributes + def report_artifact(opts) + return if not active + + artifacts_dir = Report::ARTIFACT_DIR + tmp_path = opts[:file_path] + artifact_name = File.basename tmp_path + new_path = File.join(artifacts_dir, artifact_name) + created = opts.delete(:created_at) + updated = opts.delete(:updated_at) + + unless File.exists? tmp_path + raise DBImportError 'Report artifact file to be imported does not exist.' + end + + unless (File.directory?(artifacts_dir) && File.writable?(artifacts_dir)) + raise DBImportError "Could not move report artifact file to #{artifacts_dir}." + end + + if File.exists? new_path + unique_basename = "#{(Time.now.to_f*1000).to_i}_#{artifact_name}" + new_path = File.join(artifacts_dir, unique_basename) + end + + FileUtils.copy(tmp_path, new_path) + opts[:file_path] = new_path + artifact = ReportArtifact.new(opts) + artifact.created_at = created + artifact.updated_at = updated + + unless artifact.valid? + errors = artifact.errors.full_messages.join('; ') + raise RuntimeError "Artifact to be imported is not valid: #{errors}" + end + artifact.save + end + + # + # This methods returns a list of all reports in the database + # + def reports(wspace=workspace) + ::ActiveRecord::Base.connection_pool.with_connection { + wspace.reports + } + end + + # + # WMAP + # Support methods + # + + # + # Report a Web Site to the database. WebSites must be tied to an existing Service + # + # opts MUST contain + # +:service+:: the service object this site should be associated with + # +:vhost+:: the virtual host name for this particular web site` + # + # If +:service+ is NOT specified, the following values are mandatory + # +:host+:: the ip address of the server hosting the web site + # +:port+:: the port number of the associated web site + # +:ssl+:: whether or not SSL is in use on this port + # + # These values will be used to create new host and service records + # + # opts can contain + # +:options+:: a hash of options for accessing this particular web site + # +:info+:: if present, report the service with this info + # + # Duplicate records for a given host, port, vhost combination will be overwritten + # + def report_web_site(opts) + return if not active + ::ActiveRecord::Base.connection_pool.with_connection { |conn| + wspace = opts.delete(:workspace) || workspace + vhost = opts.delete(:vhost) + + addr = nil + port = nil + name = nil + serv = nil + info = nil + + if opts[:service] and opts[:service].kind_of?(::Mdm::Service) + serv = opts[:service] + else + addr = opts[:host] + port = opts[:port] + name = opts[:ssl] ? 'https' : 'http' + info = opts[:info] + if not (addr and port) + raise ArgumentError, "report_web_site requires service OR host/port/ssl" + end + + # Force addr to be the address and not hostname + addr = Rex::Socket.getaddress(addr, true) + end + + ret = {} + + host = serv ? serv.host : find_or_create_host( + :workspace => wspace, + :host => addr, + :state => Msf::HostState::Alive + ) + + if host.name.to_s.empty? + host.name = vhost + host.save! + end + + serv = serv ? serv : find_or_create_service( + :workspace => wspace, + :host => host, + :port => port, + :proto => 'tcp', + :state => 'open' + ) + + # Change the service name if it is blank or it has + # been explicitly specified. + if opts.keys.include?(:ssl) or serv.name.to_s.empty? + name = opts[:ssl] ? 'https' : 'http' + serv.name = name + end + # Add the info if it's there. + unless info.to_s.empty? + serv.info = info + end + serv.save! if serv.changed? +=begin + host.updated_at = host.created_at + host.state = HostState::Alive + host.save! +=end + + vhost ||= host.address + site = ::Mdm::WebSite.find_or_initialize_by_vhost_and_service_id(vhost, serv[:id]) + site.options = opts[:options] if opts[:options] + + # XXX: + msf_import_timestamps(opts, site) + site.save! + + ret[:web_site] = site + } + end + + # + # Report a Web Page to the database. WebPage must be tied to an existing Web Site + # + # opts MUST contain + # +:web_site+:: the web site object that this page should be associated with + # +:path+:: the virtual host name for this particular web site + # +:code+:: the http status code from requesting this page + # +:headers+:: this is a HASH of headers (lowercase name as key) of ARRAYs of values + # +:body+:: the document body of the server response + # +:query+:: the query string after the path + # + # If web_site is NOT specified, the following values are mandatory + # +:host+:: the ip address of the server hosting the web site + # +:port+:: the port number of the associated web site + # +:vhost+:: the virtual host for this particular web site + # +:ssl+:: whether or not SSL is in use on this port + # + # These values will be used to create new host, service, and web_site records + # + # opts can contain + # +:cookie+:: the Set-Cookie headers, merged into a string + # +:auth+:: the Authorization headers, merged into a string + # +:ctype+:: the Content-Type headers, merged into a string + # +:mtime+:: the timestamp returned from the server of the last modification time + # +:location+:: the URL that a redirect points to + # + # Duplicate records for a given web_site, path, and query combination will be overwritten + # + + def report_web_page(opts) + return if not active + ::ActiveRecord::Base.connection_pool.with_connection { + wspace = opts.delete(:workspace) || workspace + + path = opts[:path] + code = opts[:code].to_i + body = opts[:body].to_s + query = opts[:query].to_s + headers = opts[:headers] + site = nil + + if not (path and code and body and headers) + raise ArgumentError, "report_web_page requires the path, query, code, body, and headers parameters" + end + + if opts[:web_site] and opts[:web_site].kind_of?(::Mdm::WebSite) + site = opts.delete(:web_site) + else + site = report_web_site( + :workspace => wspace, + :host => opts[:host], :port => opts[:port], + :vhost => opts[:host], :ssl => opts[:ssl] + ) + if not site + raise ArgumentError, "report_web_page was unable to create the associated web site" + end + end + + ret = {} + + page = ::Mdm::WebPage.find_or_initialize_by_web_site_id_and_path_and_query(site[:id], path, query) + page.code = code + page.body = body + page.headers = headers + page.cookie = opts[:cookie] if opts[:cookie] + page.auth = opts[:auth] if opts[:auth] + page.mtime = opts[:mtime] if opts[:mtime] + page.ctype = opts[:ctype] if opts[:ctype] + page.location = opts[:location] if opts[:location] + msf_import_timestamps(opts, page) + page.save! + + ret[:web_page] = page + } + + end + + + # + # Report a Web Form to the database. WebForm must be tied to an existing Web Site + # + # opts MUST contain + # +:web_site+:: the web site object that this page should be associated with + # +:path+:: the virtual host name for this particular web site + # +:query+:: the query string that is appended to the path (not valid for GET) + # +:method+:: the form method, one of GET, POST, or PATH + # +:params+:: an ARRAY of all parameters and values specified in the form + # + # If web_site is NOT specified, the following values are mandatory + # +:host+:: the ip address of the server hosting the web site + # +:port+:: the port number of the associated web site + # +:vhost+:: the virtual host for this particular web site + # +:ssl+:: whether or not SSL is in use on this port + # + # Duplicate records for a given web_site, path, method, and params combination will be overwritten + # + + def report_web_form(opts) + return if not active + ::ActiveRecord::Base.connection_pool.with_connection { + wspace = opts.delete(:workspace) || workspace + + path = opts[:path] + meth = opts[:method].to_s.upcase + para = opts[:params] + quer = opts[:query].to_s + site = nil + + if not (path and meth) + raise ArgumentError, "report_web_form requires the path and method parameters" + end + + if not %W{GET POST PATH}.include?(meth) + raise ArgumentError, "report_web_form requires the method to be one of GET, POST, PATH" + end + + if opts[:web_site] and opts[:web_site].kind_of?(::Mdm::WebSite) + site = opts.delete(:web_site) + else + site = report_web_site( + :workspace => wspace, + :host => opts[:host], :port => opts[:port], + :vhost => opts[:host], :ssl => opts[:ssl] + ) + if not site + raise ArgumentError, "report_web_form was unable to create the associated web site" + end + end + + ret = {} + + # Since one of our serialized fields is used as a unique parameter, we must do the final + # comparisons through ruby and not SQL. + + form = nil + ::Mdm::WebForm.find_all_by_web_site_id_and_path_and_method_and_query(site[:id], path, meth, quer).each do |xform| + if xform.params == para + form = xform + break + end + end + if not form + form = ::Mdm::WebForm.new + form.web_site_id = site[:id] + form.path = path + form.method = meth + form.params = para + form.query = quer + end + + msf_import_timestamps(opts, form) + form.save! + ret[:web_form] = form + } + end + + + # + # Report a Web Vuln to the database. WebVuln must be tied to an existing Web Site + # + # opts MUST contain + # +:web_site+:: the web site object that this page should be associated with + # +:path+:: the virtual host name for this particular web site + # +:query+:: the query string appended to the path (not valid for GET method flaws) + # +:method+:: the form method, one of GET, POST, or PATH + # +:params+:: an ARRAY of all parameters and values specified in the form + # +:pname+:: the specific field where the vulnerability occurs + # +:proof+:: the string showing proof of the vulnerability + # +:risk+:: an INTEGER value from 0 to 5 indicating the risk (5 is highest) + # +:name+:: the string indicating the type of vulnerability + # + # If web_site is NOT specified, the following values are mandatory + # +:host+:: the ip address of the server hosting the web site + # +:port+:: the port number of the associated web site + # +:vhost+:: the virtual host for this particular web site + # +:ssl+:: whether or not SSL is in use on this port + # + # + # Duplicate records for a given web_site, path, method, pname, and name + # combination will be overwritten + # + + def report_web_vuln(opts) + return if not active + ::ActiveRecord::Base.connection_pool.with_connection { + wspace = opts.delete(:workspace) || workspace + + path = opts[:path] + meth = opts[:method] + para = opts[:params] || [] + quer = opts[:query].to_s + pname = opts[:pname] + proof = opts[:proof] + risk = opts[:risk].to_i + name = opts[:name].to_s.strip + blame = opts[:blame].to_s.strip + desc = opts[:description].to_s.strip + conf = opts[:confidence].to_i + cat = opts[:category].to_s.strip + payload = opts[:payload].to_s + owner = opts[:owner] ? opts[:owner].shortname : nil + + + site = nil + + if not (path and meth and proof and pname) + raise ArgumentError, "report_web_vuln requires the path, method, proof, risk, name, params, and pname parameters. Received #{opts.inspect}" + end + + if not %W{GET POST PATH}.include?(meth) + raise ArgumentError, "report_web_vuln requires the method to be one of GET, POST, PATH. Received '#{meth}'" + end + + if risk < 0 or risk > 5 + raise ArgumentError, "report_web_vuln requires the risk to be between 0 and 5 (inclusive). Received '#{risk}'" + end + + if conf < 0 or conf > 100 + raise ArgumentError, "report_web_vuln requires the confidence to be between 1 and 100 (inclusive). Received '#{conf}'" + end + + if cat.empty? + raise ArgumentError, "report_web_vuln requires the category to be a valid string" + end + + if name.empty? + raise ArgumentError, "report_web_vuln requires the name to be a valid string" + end + + if opts[:web_site] and opts[:web_site].kind_of?(::Mdm::WebSite) + site = opts.delete(:web_site) + else + site = report_web_site( + :workspace => wspace, + :host => opts[:host], :port => opts[:port], + :vhost => opts[:host], :ssl => opts[:ssl] + ) + if not site + raise ArgumentError, "report_web_form was unable to create the associated web site" + end + end + + ret = {} + + meth = meth.to_s.upcase + + vuln = ::Mdm::WebVuln.find_or_initialize_by_web_site_id_and_path_and_method_and_pname_and_name_and_category_and_query(site[:id], path, meth, pname, name, cat, quer) + vuln.name = name + vuln.risk = risk + vuln.params = para + vuln.proof = proof.to_s + vuln.category = cat + vuln.blame = blame + vuln.description = desc + vuln.confidence = conf + vuln.payload = payload + vuln.owner = owner + + msf_import_timestamps(opts, vuln) + vuln.save! + + ret[:web_vuln] = vuln + } + end + + # + # WMAP + # Selected host + # + def selected_host + ::ActiveRecord::Base.connection_pool.with_connection { + selhost = ::Mdm::WmapTarget.where("selected != 0").first() + if selhost + return selhost.host + else + return + end + } + end + + # + # WMAP + # Selected target + # + def selected_wmap_target + ::ActiveRecord::Base.connection_pool.with_connection { + ::Mdm::WmapTarget.find.where("selected != 0") + } + end + + # + # WMAP + # Selected port + # + def selected_port + selected_wmap_target.port + end + + # + # WMAP + # Selected ssl + # + def selected_ssl + selected_wmap_target.ssl + end + + # + # WMAP + # Selected id + # + def selected_id + selected_wmap_target.object_id + end + + # + # WMAP + # This method iterates the requests table identifiying possible targets + # This method wiil be remove on second phase of db merging. + # + def each_distinct_target(&block) + request_distinct_targets.each do |target| + block.call(target) + end + end + + # + # WMAP + # This method returns a list of all possible targets available in requests + # This method wiil be remove on second phase of db merging. + # + def request_distinct_targets + ::ActiveRecord::Base.connection_pool.with_connection { + ::Mdm::WmapRequest.select('DISTINCT host,address,port,ssl') + } + end + + # + # WMAP + # This method iterates the requests table returning a list of all requests of a specific target + # + def each_request_target_with_path(&block) + target_requests('AND wmap_requests.path IS NOT NULL').each do |req| + block.call(req) + end + end + + # + # WMAP + # This method iterates the requests table returning a list of all requests of a specific target + # + def each_request_target_with_query(&block) + target_requests('AND wmap_requests.query IS NOT NULL').each do |req| + block.call(req) + end + end + + # + # WMAP + # This method iterates the requests table returning a list of all requests of a specific target + # + def each_request_target_with_body(&block) + target_requests('AND wmap_requests.body IS NOT NULL').each do |req| + block.call(req) + end + end + + # + # WMAP + # This method iterates the requests table returning a list of all requests of a specific target + # + def each_request_target_with_headers(&block) + target_requests('AND wmap_requests.headers IS NOT NULL').each do |req| + block.call(req) + end + end + + # + # WMAP + # This method iterates the requests table returning a list of all requests of a specific target + # + def each_request_target(&block) + target_requests('').each do |req| + block.call(req) + end + end + + # + # WMAP + # This method returns a list of all requests from target + # + def target_requests(extra_condition) + ::ActiveRecord::Base.connection_pool.with_connection { + ::Mdm::WmapRequest.where("wmap_requests.host = ? AND wmap_requests.port = ? #{extra_condition}",selected_host,selected_port) + } + end + + # + # WMAP + # This method iterates the requests table calling the supplied block with the + # request instance of each entry. + # + def each_request(&block) + requests.each do |request| + block.call(request) + end + end + + # + # WMAP + # This method allows to query directly the requests table. To be used mainly by modules + # + def request_sql(host,port,extra_condition) + ::ActiveRecord::Base.connection_pool.with_connection { + ::Mdm::WmapRequest.where("wmap_requests.host = ? AND wmap_requests.port = ? #{extra_condition}", host , port) + } + end + + # + # WMAP + # This methods returns a list of all targets in the database + # + def requests + ::ActiveRecord::Base.connection_pool.with_connection { + ::Mdm::WmapRequest.find(:all) + } + end + + # + # WMAP + # This method iterates the targets table calling the supplied block with the + # target instance of each entry. + # + def each_target(&block) + targets.each do |target| + block.call(target) + end + end + + # + # WMAP + # This methods returns a list of all targets in the database + # + def targets + ::ActiveRecord::Base.connection_pool.with_connection { + ::Mdm::WmapTarget.find(:all) + } + end + + # + # WMAP + # This methods deletes all targets from targets table in the database + # + def delete_all_targets + ::ActiveRecord::Base.connection_pool.with_connection { + ::Mdm::WmapTarget.delete_all + } + end + + # + # WMAP + # Find a target matching this id + # + def get_target(id) + ::ActiveRecord::Base.connection_pool.with_connection { + target = ::Mdm::WmapTarget.where("id = ?", id).first() + return target + } + end + + # + # WMAP + # Create a target + # + def create_target(host,port,ssl,sel) + ::ActiveRecord::Base.connection_pool.with_connection { + tar = ::Mdm::WmapTarget.create( + :host => host, + :address => host, + :port => port, + :ssl => ssl, + :selected => sel + ) + #framework.events.on_db_target(rec) + } + end + + + # + # WMAP + # Create a request (by hand) + # + def create_request(host,port,ssl,meth,path,headers,query,body,respcode,resphead,response) + ::ActiveRecord::Base.connection_pool.with_connection { + req = ::Mdm::WmapRequest.create( + :host => host, + :address => host, + :port => port, + :ssl => ssl, + :meth => meth, + :path => path, + :headers => headers, + :query => query, + :body => body, + :respcode => respcode, + :resphead => resphead, + :response => response + ) + #framework.events.on_db_request(rec) + } + end + + # + # WMAP + # Quick way to query the database (used by wmap_sql) + # + def sql_query(sqlquery) + ::ActiveRecord::Base.connection_pool.with_connection { + ActiveRecord::Base.connection.select_all(sqlquery) + } + end + + + # Returns a REXML::Document from the given data. + def rexmlify(data) + if data.kind_of?(REXML::Document) + return data + else + # Make an attempt to recover from a REXML import fail, since + # it's better than dying outright. + begin + return REXML::Document.new(data) + rescue REXML::ParseException => e + dlog("REXML error: Badly formatted XML, attempting to recover. Error was: #{e.inspect}") + return REXML::Document.new(data.gsub(/([\x00-\x08\x0b\x0c\x0e-\x1f\x80-\xff])/n){ |x| "\\x%.2x" % x.unpack("C*")[0] }) + end + end + end + + # Handles timestamps from Metasploit Express/Pro imports. + def msf_import_timestamps(opts,obj) + obj.created_at = opts["created_at"] if opts["created_at"] + obj.created_at = opts[:created_at] if opts[:created_at] + obj.updated_at = opts["updated_at"] ? opts["updated_at"] : obj.created_at + obj.updated_at = opts[:updated_at] ? opts[:updated_at] : obj.created_at + return obj + end + + ## + # + # Import methods + # + ## + + # + # Generic importer that automatically determines the file type being + # imported. Since this looks for vendor-specific strings in the given + # file, there shouldn't be any false detections, but no guarantees. + # + def import_file(args={}, &block) + filename = args[:filename] || args['filename'] + wspace = args[:wspace] || args['wspace'] || workspace + @import_filedata = {} + @import_filedata[:filename] = filename + + data = "" + ::File.open(filename, 'rb') do |f| + # This check is the largest (byte-wise) that we need to do + # since the other 4-byte checks will be subsets of this larger one. + data = f.read(Metasploit::Credential::Exporter::Pwdump::FILE_ID_STRING.size) + end + if data.nil? + raise DBImportError.new("Zero-length file") + end + + if data.index(Metasploit::Credential::Exporter::Pwdump::FILE_ID_STRING) + data = ::File.open(filename, 'rb') + else + case data[0,4] + when "PK\x03\x04" + data = Zip::File.open(filename) + when "\xd4\xc3\xb2\xa1", "\xa1\xb2\xc3\xd4" + data = PacketFu::PcapFile.new(:filename => filename) + else + ::File.open(filename, 'rb') do |f| + sz = f.stat.size + data = f.read(sz) + end + end + end + + + if block + import(args.merge(:data => data)) { |type,data| yield type,data } + else + import(args.merge(:data => data)) + end + end + + # A dispatcher method that figures out the data's file type, + # and sends it off to the appropriate importer. Note that + # import_file_detect will raise an error if the filetype + # is unknown. + def import(args={}, &block) + data = args[:data] || args['data'] + ftype = import_filetype_detect(data) + yield(:filetype, @import_filedata[:type]) if block + self.send "import_#{ftype}".to_sym, args, &block + end + + # Returns one of the following: + # + # :acunetix_xml + # :amap_log + # :amap_mlog + # :appscan_xml + # :burp_session_xml + # :ci_xml + # :foundstone_xml + # :fusionvm_xml + # :ip360_aspl_xml + # :ip360_xml_v3 + # :ip_list + # :libpcap + # :mbsa_xml + # :msf_cred_dump_zip + # :msf_pwdump + # :msf_xml + # :msf_zip + # :nessus_nbe + # :nessus_xml + # :nessus_xml_v2 + # :netsparker_xml + # :nexpose_rawxml + # :nexpose_simplexml + # :nikto_xml + # :nmap_xml + # :openvas_new_xml + # :openvas_xml + # :outpost24_xml + # :qualys_asset_xml + # :qualys_scan_xml + # :retina_xml + # :spiceworks_csv + # :wapiti_xml + # + # If there is no match, an error is raised instead. + # + # @raise DBImportError if the type can't be detected + def import_filetype_detect(data) + + if data and data.kind_of? Zip::File + if data.entries.empty? + raise DBImportError.new("The zip file provided is empty.") + end + + @import_filedata ||= {} + @import_filedata[:zip_filename] = File.split(data.to_s).last + @import_filedata[:zip_basename] = @import_filedata[:zip_filename].gsub(/\.zip$/,"") + @import_filedata[:zip_entry_names] = data.entries.map {|x| x.name} + + if @import_filedata[:zip_entry_names].include?(Metasploit::Credential::Importer::Zip::MANIFEST_FILE_NAME) + @import_filedata[:type] = "Metasploit Credential Dump" + return :msf_cred_dump_zip + end + + xml_files = @import_filedata[:zip_entry_names].grep(/^(.*)\.xml$/) + + # TODO This check for our zip export should be more extensive + if xml_files.empty? + raise DBImportError.new("The zip file provided is not a Metasploit Zip Export") + end + + @import_filedata[:zip_xml] = xml_files.first + @import_filedata[:type] = "Metasploit Zip Export" + + return :msf_zip + end + + if data and data.kind_of? PacketFu::PcapFile + # Don't check for emptiness here because unlike other formats, we + # haven't read any actual data in yet, only magic bytes to discover + # that this is indeed a pcap file. + #raise DBImportError.new("The pcap file provided is empty.") if data.body.empty? + @import_filedata ||= {} + @import_filedata[:type] = "Libpcap Packet Capture" + return :libpcap + end + + # msfpwdump + if data.present? && data.kind_of?(::File) + @import_filedata[:type] = "Metasploit PWDump Export" + return :msf_pwdump + end + + # This is a text string, lets make sure its treated as binary + data = data.unpack("C*").pack("C*") + if data and data.to_s.strip.length == 0 + raise DBImportError.new("The data provided to the import function was empty") + end + + # Parse the first line or 4k of data from the file + di = data.index("\n") || 4096 + + firstline = data[0, di] + @import_filedata ||= {} + if (firstline.index("")) + @import_filedata[:type] = "Retina XML" + return :retina_xml + elsif (firstline.index(//)) + @import_filedata[:type] = "OpenVAS XML" + return :openvas_new_xml + elsif (firstline.index(/")) + @import_filedata[:type] = "Nessus XML (v1)" + return :nessus_xml + elsif (firstline.index("]/ + + case $1 + when "niktoscan" + @import_filedata[:type] = "Nikto XML" + return :nikto_xml + when "nmaprun" + @import_filedata[:type] = "Nmap XML" + return :nmap_xml + when "openvas-report" + @import_filedata[:type] = "OpenVAS Report" + return :openvas_xml + when "NessusClientData" + @import_filedata[:type] = "Nessus XML (v1)" + return :nessus_xml + when "NessusClientData_v2" + @import_filedata[:type] = "Nessus XML (v2)" + return :nessus_xml_v2 + when "SCAN" + @import_filedata[:type] = "Qualys Scan XML" + return :qualys_scan_xml + when "report" + @import_filedata[:type] = "Wapiti XML" + return :wapiti_xml + when "ASSET_DATA_REPORT" + @import_filedata[:type] = "Qualys Asset XML" + return :qualys_asset_xml + when /MetasploitExpressV[1234]/ + @import_filedata[:type] = "Metasploit XML" + return :msf_xml + when /MetasploitV4/ + @import_filedata[:type] = "Metasploit XML" + return :msf_xml + when /netsparker/ + @import_filedata[:type] = "NetSparker XML" + return :netsparker_xml + when /audits?/ # and are both valid for nCircle. wtfmate. + @import_filedata[:type] = "IP360 XML v3" + return :ip360_xml_v3 + when /ontology/ + @import_filedata[:type] = "IP360 ASPL" + return :ip360_aspl_xml + when /ReportInfo/ + @import_filedata[:type] = "Foundstone" + return :foundstone_xml + when /ScanGroup/ + @import_filedata[:type] = "Acunetix" + return :acunetix_xml + when /AppScanInfo/ # Actually the second line + @import_filedata[:type] = "Appscan" + return :appscan_xml + when "entities" + if line =~ /creator.*\x43\x4f\x52\x45\x20\x49\x4d\x50\x41\x43\x54/ni + @import_filedata[:type] = "CI" + return :ci_xml + end + when "main" + @import_filedata[:type] = "Outpost24 XML" + return :outpost24_xml + else + # Give up if we haven't hit the root tag in the first few lines + break if line_count > 10 + end + line_count += 1 + } + elsif (firstline.index("timestamps|||scan_start")) + @import_filedata[:type] = "Nessus NBE Report" + # then it's a nessus nbe + return :nessus_nbe + elsif (firstline.index("# amap v")) + # then it's an amap mlog + @import_filedata[:type] = "Amap Log -m" + return :amap_mlog + elsif (firstline.index("amap v")) + # then it's an amap log + @import_filedata[:type] = "Amap Log" + return :amap_log + elsif ipv46_validator(firstline) + # then its an IP list + @import_filedata[:type] = "IP Address List" + return :ip_list + elsif (data[0,1024].index(" wspace, + :host => addr, + :type => "service.nikto.scan.description", + :data => desc_text, + :proto => "tcp", + :port => port.to_i, + :sname => uri.scheme, + :update => :unique_data, + :task => args[:task] + } + # Always report it as a note. + report_note(desc_data) + # Sometimes report it as a vuln, too. + # XXX: There's a Vuln.info field but nothing reads from it? See Bug #5837 + if item.attributes['osvdbid'].to_i != 0 + desc_data[:refs] = ["OSVDB-#{item.attributes['osvdbid']}"] + desc_data[:name] = "NIKTO-#{item.attributes['id']}" + desc_data.delete(:data) + desc_data.delete(:type) + desc_data.delete(:update) + report_vuln(desc_data) + end + end + end + end + end + end + + def import_wapiti_xml_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + + data = "" + ::File.open(filename, 'rb') do |f| + data = f.read(f.stat.size) + end + import_wapiti_xml(args.merge(:data => data)) + end + + def import_wapiti_xml(args={}, &block) + if block + doc = Rex::Parser::WapitiDocument.new(args,framework.db) {|type, data| yield type,data } + else + doc = Rex::Parser::WapitiDocument.new(args,self) + end + parser = ::Nokogiri::XML::SAX::Parser.new(doc) + parser.parse(args[:data]) + end + + def import_openvas_new_xml_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + + data = "" + ::File.open(filename, 'rb') do |f| + data = f.read(f.stat.size) + end + import_wapiti_xml(args.merge(:data => data)) + end + + def import_openvas_new_xml(args={}, &block) + if block + doc = Rex::Parser::OpenVASDocument.new(args,framework.db) {|type, data| yield type,data } + else + doc = Rex::Parser::OpenVASDocument.new(args,self) + end + parser = ::Nokogiri::XML::SAX::Parser.new(doc) + parser.parse(args[:data]) + end + + def import_libpcap_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + + data = PacketFu::PcapFile.new(:filename => filename) + import_libpcap(args.merge(:data => data)) + end + + # The libpcap file format is handled by PacketFu for data + # extraction. TODO: Make this its own mixin, and possibly + # extend PacketFu to do better stream analysis on the fly. + def import_libpcap(args={}, &block) + data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + # seen_hosts is only used for determining when to yield an address. Once we get + # some packet analysis going, the values will have all sorts of info. The plan + # is to ru through all the packets as a first pass and report host and service, + # then, once we have everything parsed, we can reconstruct sessions and ngrep + # out things like authentication sequences, examine ttl's and window sizes, all + # kinds of crazy awesome stuff like that. + seen_hosts = {} + decoded_packets = 0 + last_count = 0 + data.read_packet_bytes do |p| + if (decoded_packets >= last_count + 1000) and block + yield(:pcap_count, decoded_packets) + last_count = decoded_packets + end + decoded_packets += 1 + + pkt = PacketFu::Packet.parse(p) rescue next # Just silently skip bad packets + + next unless pkt.is_ip? # Skip anything that's not IP. Technically, not Ethernet::Ip + next if pkt.is_tcp? && (pkt.tcp_src == 0 || pkt.tcp_dst == 0) # Skip port 0 + next if pkt.is_udp? && (pkt.udp_src == 0 || pkt.udp_dst == 0) # Skip port 0 + saddr = pkt.ip_saddr + daddr = pkt.ip_daddr + + # Handle blacklists and obviously useless IP addresses, and report the host. + next if (bl | [saddr,daddr]).size == bl.size # Both hosts are blacklisted, skip everything. + unless( bl.include?(saddr) || rfc3330_reserved(saddr)) + yield(:address,saddr) if block and !seen_hosts.keys.include?(saddr) + unless seen_hosts[saddr] + report_host( + :workspace => wspace, + :host => saddr, + :state => Msf::HostState::Alive, + :task => args[:task] + ) + end + seen_hosts[saddr] ||= [] + + end + unless( bl.include?(daddr) || rfc3330_reserved(daddr)) + yield(:address,daddr) if block and !seen_hosts.keys.include?(daddr) + unless seen_hosts[daddr] + report_host( + :workspace => wspace, + :host => daddr, + :state => Msf::HostState::Alive, + :task => args[:task] + ) + end + seen_hosts[daddr] ||= [] + end + + if pkt.is_tcp? # First pass on TCP packets + if (pkt.tcp_flags.syn == 1 and pkt.tcp_flags.ack == 1) or # Oh, this kills me + pkt.tcp_src < 1024 # If it's a low port, assume it's a proper service. + if seen_hosts[saddr] + unless seen_hosts[saddr].include? [pkt.tcp_src,"tcp"] + report_service( + :workspace => wspace, :host => saddr, + :proto => "tcp", :port => pkt.tcp_src, + :state => Msf::ServiceState::Open, + :task => args[:task] + ) + seen_hosts[saddr] << [pkt.tcp_src,"tcp"] + yield(:service,"%s:%d/%s" % [saddr,pkt.tcp_src,"tcp"]) + end + end + end + elsif pkt.is_udp? # First pass on UDP packets + if pkt.udp_src == pkt.udp_dst # Very basic p2p detection. + [saddr,daddr].each do |xaddr| + if seen_hosts[xaddr] + unless seen_hosts[xaddr].include? [pkt.udp_src,"udp"] + report_service( + :workspace => wspace, :host => xaddr, + :proto => "udp", :port => pkt.udp_src, + :state => Msf::ServiceState::Open, + :task => args[:task] + ) + seen_hosts[xaddr] << [pkt.udp_src,"udp"] + yield(:service,"%s:%d/%s" % [xaddr,pkt.udp_src,"udp"]) + end + end + end + elsif pkt.udp_src < 1024 # Probably a service + if seen_hosts[saddr] + unless seen_hosts[saddr].include? [pkt.udp_src,"udp"] + report_service( + :workspace => wspace, :host => saddr, + :proto => "udp", :port => pkt.udp_src, + :state => Msf::ServiceState::Open, + :task => args[:task] + ) + seen_hosts[saddr] << [pkt.udp_src,"udp"] + yield(:service,"%s:%d/%s" % [saddr,pkt.udp_src,"udp"]) + end + end + end + end # tcp or udp + + inspect_single_packet(pkt,wspace,args) + + end # data.body.map + + # Right about here, we should have built up some streams for some stream analysis. + # Not sure what form that will take, but people like shoving many hundreds of + # thousands of packets through this thing, so it'll need to be memory efficient. + + end + + # Do all the single packet analysis we can while churning through the pcap + # the first time. Multiple packet inspection will come later, where we can + # do stream analysis, compare requests and responses, etc. + def inspect_single_packet(pkt,wspace,args) + if pkt.is_tcp? or pkt.is_udp? + inspect_single_packet_http(pkt,wspace,args) + end + end + + # Checks for packets that are headed towards port 80, are tcp, contain an HTTP/1.0 + # line, contains an Authorization line, contains a b64-encoded credential, and + # extracts it. Reports this credential and solidifies the service as HTTP. + def inspect_single_packet_http(pkt,wspace,args) + task = args.fetch(:task, nil) + # First, check the server side (data from port 80). + if pkt.is_tcp? and pkt.tcp_src == 80 and !pkt.payload.nil? and !pkt.payload.empty? + if pkt.payload =~ /^HTTP\x2f1\x2e[01]/n + http_server_match = pkt.payload.match(/\nServer:\s+([^\r\n]+)[\r\n]/n) + if http_server_match.kind_of?(MatchData) and http_server_match[1] + report_service( + :workspace => wspace, + :host => pkt.ip_saddr, + :port => pkt.tcp_src, + :proto => "tcp", + :name => "http", + :info => http_server_match[1], + :state => Msf::ServiceState::Open, + :task => task + ) + # That's all we want to know from this service. + return :something_significant + end + end + end + + # Next, check the client side (data to port 80) + if pkt.is_tcp? and pkt.tcp_dst == 80 and !pkt.payload.nil? and !pkt.payload.empty? + if pkt.payload.match(/[\x00-\x20]HTTP\x2f1\x2e[10]/n) + auth_match = pkt.payload.match(/\nAuthorization:\s+Basic\s+([A-Za-z0-9=\x2b]+)/n) + if auth_match.kind_of?(MatchData) and auth_match[1] + b64_cred = auth_match[1] + else + return false + end + # If we're this far, we can surmise that at least the client is a web browser, + # he thinks the server is HTTP and he just made an authentication attempt. At + # this point, we'll just believe everything the packet says -- validation ought + # to come later. + user,pass = b64_cred.unpack("m*").first.split(/:/,2) + report_service( + :workspace => wspace, + :host => pkt.ip_daddr, + :port => pkt.tcp_dst, + :proto => "tcp", + :name => "http", + :task => task + ) + + service_data = { + address: pkt.ip_daddr, + port: pkt.tcp_dst, + service_name: 'http', + protocol: 'tcp', + workspace_id: wspace.id + } + service_data[:task_id] = task.id if task + + filename = args[:filename] + + credential_data = { + origin_type: :import, + private_data: pass, + private_type: :password, + username: user, + filename: filename + } + credential_data.merge!(service_data) + credential_core = create_credential(credential_data) + + login_data = { + core: credential_core, + status: Metasploit::Model::Login::Status::UNTRIED + } + + login_data.merge!(service_data) + + create_credential_login(login_data) + + # That's all we want to know from this service. + return :something_significant + end + end + end + + def import_spiceworks_csv(args={}, &block) + data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + CSV.parse(data) do |row| + next unless (["Name", "Manufacturer", "Device Type"] & row).empty? #header + name = row[0] + manufacturer = row[1] + device = row[2] + model = row[3] + ip = row[4] + serialno = row[5] + location = row[6] + os = row[7] + + next unless ip + next if bl.include? ip + + conf = { + :workspace => wspace, + :host => ip, + :name => name, + :task => args[:task] + } + + + if os + report_note( + :workspace => wspace, + :task => args[:task], + :host => ip, + :type => 'host.os.spiceworks_fingerprint', + :data => { + :os => os.to_s.strip + } + ) + end + + info = [] + info << "Serial Number: #{serialno}" unless (serialno.blank? or serialno == name) + info << "Location: #{location}" unless location.blank? + conf[:info] = info.join(", ") unless info.empty? + + host = report_host(conf) + report_import_note(wspace, host) + end + end + + + # Perform in an import of an msfpwdump file + def import_msf_pwdump(args={}, &block) + filename = File.basename(args[:data].path) + wspace = args[:wspace] || workspace + origin = Metasploit::Credential::Origin::Import.create!(filename: filename) + importer = Metasploit::Credential::Importer::Pwdump.new(input: args[:data], workspace: wspace, filename: filename, origin:origin) + importer.import! + importer.input.close unless importer.input.closed? + end + + # If hex notation is present, turn them into a character. + def dehex(str) + hexen = str.scan(/\x5cx[0-9a-fA-F]{2}/n) + hexen.each { |h| + str.gsub!(h,h[2,2].to_i(16).chr) + } + return str + end + + + # + # Nexpose Simple XML + # + # XXX At some point we'll want to make this a stream parser for dealing + # with large results files + # + def import_nexpose_simplexml_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + + data = "" + ::File.open(filename, 'rb') do |f| + data = f.read(f.stat.size) + end + import_nexpose_simplexml(args.merge(:data => data)) + end + + # Import a Metasploit XML file. + def import_msf_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + + data = "" + ::File.open(filename, 'rb') do |f| + data = f.read(f.stat.size) + end + import_msf_xml(args.merge(:data => data)) + end + + # Import a Metasploit Express ZIP file. Note that this requires + # a fair bit of filesystem manipulation, and is very much tied + # up with the Metasploit Express ZIP file format export (for + # obvious reasons). In the event directories exist, they will + # be reused. If target files exist, they will be overwritten. + # + # XXX: Refactor so it's not quite as sanity-blasting. + def import_msf_zip(args={}, &block) + data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + + new_tmp = ::File.join(Dir::tmpdir,"msf","imp_#{Rex::Text::rand_text_alphanumeric(4)}",@import_filedata[:zip_basename]) + if ::File.exists? new_tmp + unless (::File.directory?(new_tmp) && ::File.writable?(new_tmp)) + raise DBImportError.new("Could not extract zip file to #{new_tmp}") + end + else + FileUtils.mkdir_p(new_tmp) + end + @import_filedata[:zip_tmp] = new_tmp + + # Grab the list of unique basedirs over all entries. + @import_filedata[:zip_tmp_subdirs] = @import_filedata[:zip_entry_names].map {|x| ::File.split(x)}.map {|x| x[0]}.uniq.reject {|x| x == "."} + + # mkdir all of the base directores we just pulled out, if they don't + # already exist + @import_filedata[:zip_tmp_subdirs].each {|sub| + tmp_subdirs = ::File.join(@import_filedata[:zip_tmp],sub) + if File.exists? tmp_subdirs + unless (::File.directory?(tmp_subdirs) && File.writable?(tmp_subdirs)) + # if it exists but we can't write to it, give up + raise DBImportError.new("Could not extract zip file to #{tmp_subdirs}") + end + else + ::FileUtils.mkdir(tmp_subdirs) + end + } + + data.entries.each do |e| + target = ::File.join(@import_filedata[:zip_tmp], e.name) + data.extract(e,target) + + if target =~ /\.xml\z/ + target_data = ::File.open(target, "rb") {|f| f.read 1024} + if import_filetype_detect(target_data) == :msf_xml + @import_filedata[:zip_extracted_xml] = target + end + end + end + + # Import any creds if there are some in the import file + Dir.entries(@import_filedata[:zip_tmp]).each do |entry| + if entry =~ /^.*#{Regexp.quote(Metasploit::Credential::Exporter::Core::CREDS_DUMP_FILE_IDENTIFIER)}.*/ + manifest_file_path = File.join(@import_filedata[:zip_tmp], entry, Metasploit::Credential::Importer::Zip::MANIFEST_FILE_NAME) + if File.exists? manifest_file_path + import_msf_cred_dump(manifest_file_path, wspace) + end + end + end + + # This will kick the newly-extracted XML file through + # the import_file process all over again. + if @import_filedata[:zip_extracted_xml] + new_args = args.dup + new_args[:filename] = @import_filedata[:zip_extracted_xml] + new_args[:data] = nil + new_args[:ifd] = @import_filedata.dup + if block + import_file(new_args, &block) + else + import_file(new_args) + end + end + + # Kick down to all the MSFX ZIP specific items + if block + import_msf_collateral(new_args, &block) + else + import_msf_collateral(new_args) + end + end + + # Imports loot, tasks, and reports from an MSF ZIP report. + # XXX: This function is stupidly long. It needs to be refactored. + def import_msf_collateral(args={}, &block) + data = ::File.open(args[:filename], "rb") {|f| f.read(f.stat.size)} + wspace = args[:wspace] || args['wspace'] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + basedir = args[:basedir] || args['basedir'] || ::File.join(Msf::Config.data_directory, "msf") + + allow_yaml = false + btag = nil + + doc = rexmlify(data) + if doc.elements["MetasploitExpressV1"] + m_ver = 1 + allow_yaml = true + btag = "MetasploitExpressV1" + elsif doc.elements["MetasploitExpressV2"] + m_ver = 2 + allow_yaml = true + btag = "MetasploitExpressV2" + elsif doc.elements["MetasploitExpressV3"] + m_ver = 3 + btag = "MetasploitExpressV3" + elsif doc.elements["MetasploitExpressV4"] + m_ver = 4 + btag = "MetasploitExpressV4" + elsif doc.elements["MetasploitV4"] + m_ver = 4 + btag = "MetasploitV4" + else + m_ver = nil + end + unless m_ver and btag + raise DBImportError.new("Unsupported Metasploit XML document format") + end + + host_info = {} + doc.elements.each("/#{btag}/hosts/host") do |host| + host_info[host.elements["id"].text.to_s.strip] = nils_for_nulls(host.elements["address"].text.to_s.strip) + end + + # Import Loot + doc.elements.each("/#{btag}/loots/loot") do |loot| + next if bl.include? host_info[loot.elements["host-id"].text.to_s.strip] + loot_info = {} + loot_info[:host] = host_info[loot.elements["host-id"].text.to_s.strip] + loot_info[:workspace] = args[:wspace] + loot_info[:ctype] = nils_for_nulls(loot.elements["content-type"].text.to_s.strip) + loot_info[:info] = nils_for_nulls(unserialize_object(loot.elements["info"], allow_yaml)) + loot_info[:ltype] = nils_for_nulls(loot.elements["ltype"].text.to_s.strip) + loot_info[:name] = nils_for_nulls(loot.elements["name"].text.to_s.strip) + loot_info[:created_at] = nils_for_nulls(loot.elements["created-at"].text.to_s.strip) + loot_info[:updated_at] = nils_for_nulls(loot.elements["updated-at"].text.to_s.strip) + loot_info[:name] = nils_for_nulls(loot.elements["name"].text.to_s.strip) + loot_info[:orig_path] = nils_for_nulls(loot.elements["path"].text.to_s.strip) + loot_info[:task] = args[:task] + tmp = args[:ifd][:zip_tmp] + loot_info[:orig_path].gsub!(/^\./,tmp) if loot_info[:orig_path] + if !loot.elements["service-id"].text.to_s.strip.empty? + unless loot.elements["service-id"].text.to_s.strip == "NULL" + loot_info[:service] = loot.elements["service-id"].text.to_s.strip + end + end + + # Only report loot if we actually have it. + # TODO: Copypasta. Seperate this out. + if ::File.exists? loot_info[:orig_path] + loot_dir = ::File.join(basedir,"loot") + loot_file = ::File.split(loot_info[:orig_path]).last + if ::File.exists? loot_dir + unless (::File.directory?(loot_dir) && ::File.writable?(loot_dir)) + raise DBImportError.new("Could not move files to #{loot_dir}") + end + else + ::FileUtils.mkdir_p(loot_dir) + end + new_loot = ::File.join(loot_dir,loot_file) + loot_info[:path] = new_loot + if ::File.exists?(new_loot) + ::File.unlink new_loot # Delete it, and don't report it. + else + report_loot(loot_info) # It's new, so report it. + end + ::FileUtils.copy(loot_info[:orig_path], new_loot) + yield(:msf_loot, new_loot) if block + end + end + + # Import Tasks + doc.elements.each("/#{btag}/tasks/task") do |task| + task_info = {} + task_info[:workspace] = args[:wspace] + # Should user be imported (original) or declared (the importing user)? + task_info[:user] = nils_for_nulls(task.elements["created-by"].text.to_s.strip) + task_info[:desc] = nils_for_nulls(task.elements["description"].text.to_s.strip) + task_info[:info] = nils_for_nulls(unserialize_object(task.elements["info"], allow_yaml)) + task_info[:mod] = nils_for_nulls(task.elements["module"].text.to_s.strip) + task_info[:options] = nils_for_nulls(task.elements["options"].text.to_s.strip) + task_info[:prog] = nils_for_nulls(task.elements["progress"].text.to_s.strip).to_i + task_info[:created_at] = nils_for_nulls(task.elements["created-at"].text.to_s.strip) + task_info[:updated_at] = nils_for_nulls(task.elements["updated-at"].text.to_s.strip) + if !task.elements["completed-at"].text.to_s.empty? + task_info[:completed_at] = nils_for_nulls(task.elements["completed-at"].text.to_s.strip) + end + if !task.elements["error"].text.to_s.empty? + task_info[:error] = nils_for_nulls(task.elements["error"].text.to_s.strip) + end + if !task.elements["result"].text.to_s.empty? + task_info[:result] = nils_for_nulls(task.elements["result"].text.to_s.strip) + end + task_info[:orig_path] = nils_for_nulls(task.elements["path"].text.to_s.strip) + tmp = args[:ifd][:zip_tmp] + task_info[:orig_path].gsub!(/^\./,tmp) if task_info[:orig_path] + + # Only report a task if we actually have it. + # TODO: Copypasta. Seperate this out. + if ::File.exists? task_info[:orig_path] + tasks_dir = ::File.join(basedir,"tasks") + task_file = ::File.split(task_info[:orig_path]).last + if ::File.exists? tasks_dir + unless (::File.directory?(tasks_dir) && ::File.writable?(tasks_dir)) + raise DBImportError.new("Could not move files to #{tasks_dir}") + end + else + ::FileUtils.mkdir_p(tasks_dir) + end + new_task = ::File.join(tasks_dir,task_file) + task_info[:path] = new_task + if ::File.exists?(new_task) + ::File.unlink new_task # Delete it, and don't report it. + else + report_task(task_info) # It's new, so report it. + end + ::FileUtils.copy(task_info[:orig_path], new_task) + yield(:msf_task, new_task) if block + end + end + + # Import Reports + doc.elements.each("/#{btag}/reports/report") do |report| + import_report(report, args, basedir) + end + end + + # Import credentials given a path to a valid manifest file + # + # @param creds_dump_manifest_path [String] + # @param workspace [Mdm::Workspace] Default: {#workspace} + # @return [void] + def import_msf_cred_dump(creds_dump_manifest_path, workspace) + manifest_file = File.open(creds_dump_manifest_path) + origin = Metasploit::Credential::Origin::Import.create!(filename: File.basename(creds_dump_manifest_path)) + importer = Metasploit::Credential::Importer::Core.new(workspace: workspace, input: manifest_file, origin: origin) + importer.import! + end + + # Import credentials given a path to a valid manifest file + # + # @option args [String] :filename + # @option args [Mdm::Workspace] :wspace Default: {#workspace} + # @return [void] + def import_msf_cred_dump_zip(args = {}) + wspace = args[:wspace] || workspace + origin = Metasploit::Credential::Origin::Import.create!(filename: File.basename(args[:filename])) + importer = Metasploit::Credential::Importer::Zip.new(workspace: wspace, input: File.open(args[:filename]), origin: origin) + importer.import! + nil + end + + # @param report [REXML::Element] to be imported + # @param args [Hash] + # @param base_dir [String] + def import_report(report, args, base_dir) + tmp = args[:ifd][:zip_tmp] + report_info = {} + + report.elements.each do |e| + node_name = e.name + node_value = e.text + + # These need to be converted back to arrays: + array_attrs = %w|addresses file-formats options sections| + if array_attrs.member? node_name + node_value = JSON.parse(node_value) + end + # Don't restore these values: + skip_nodes = %w|id workspace-id artifacts| + next if skip_nodes.member? node_name + + report_info[node_name.parameterize.underscore.to_sym] = node_value + end + # Use current workspace + report_info[:workspace_id] = args[:wspace].id + + # Create report, need new ID to record artifacts + report_id = report_report(report_info) + + # Handle artifacts + report.elements['artifacts'].elements.each do |artifact| + artifact_opts = {} + artifact.elements.each do |attr| + skip_nodes = %w|id accessed-at| + next if skip_nodes.member? attr.name + + symboled_attr = attr.name.parameterize.underscore.to_sym + artifact_opts[symboled_attr] = attr.text + end + # Use new Report as parent + artifact_opts[:report_id] = report_id + # Update to full path + artifact_opts[:file_path].gsub!(/^\./, tmp) + + report_artifact(artifact_opts) + end + end + + # Convert the string "NULL" to actual nil + def nils_for_nulls(str) + str == "NULL" ? nil : str + end + + def import_nexpose_simplexml(args={}, &block) + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + wspace = args[:wspace] || workspace + if Rex::Parser.nokogiri_loaded + parser = "Nokogiri v#{::Nokogiri::VERSION}" + noko_args = args.dup + noko_args[:blacklist] = bl + noko_args[:wspace] = wspace + if block + yield(:parser, parser) + import_nexpose_noko_stream(noko_args) {|type, data| yield type,data} + else + import_nexpose_noko_stream(noko_args) + end + return true + end + data = args[:data] + + doc = rexmlify(data) + doc.elements.each('/NeXposeSimpleXML/devices/device') do |dev| + addr = dev.attributes['address'].to_s + if bl.include? addr + next + else + yield(:address,addr) if block + end + + fprint = {} + + dev.elements.each('fingerprint/description') do |str| + fprint[:desc] = str.text.to_s.strip + end + dev.elements.each('fingerprint/vendor') do |str| + fprint[:vendor] = str.text.to_s.strip + end + dev.elements.each('fingerprint/family') do |str| + fprint[:family] = str.text.to_s.strip + end + dev.elements.each('fingerprint/product') do |str| + fprint[:product] = str.text.to_s.strip + end + dev.elements.each('fingerprint/version') do |str| + fprint[:version] = str.text.to_s.strip + end + dev.elements.each('fingerprint/architecture') do |str| + fprint[:arch] = str.text.to_s.upcase.strip + end + + conf = { + :workspace => wspace, + :host => addr, + :state => Msf::HostState::Alive, + :task => args[:task] + } + + host = report_host(conf) + report_import_note(wspace, host) + + report_note( + :workspace => wspace, + :host => host, + :type => 'host.os.nexpose_fingerprint', + :data => fprint, + :task => args[:task] + ) + + # Load vulnerabilities not associated with a service + dev.elements.each('vulnerabilities/vulnerability') do |vuln| + vid = vuln.attributes['id'].to_s.downcase + refs = process_nexpose_data_sxml_refs(vuln) + next if not refs + report_vuln( + :workspace => wspace, + :host => host, + :name => 'NEXPOSE-' + vid, + :info => vid, + :refs => refs, + :task => args[:task] + ) + end + + # Load the services + dev.elements.each('services/service') do |svc| + sname = svc.attributes['name'].to_s + sprot = svc.attributes['protocol'].to_s.downcase + sport = svc.attributes['port'].to_s.to_i + next if sport == 0 + + name = sname.split('(')[0].strip + info = '' + + svc.elements.each('fingerprint/description') do |str| + info = str.text.to_s.strip + end + + if(sname.downcase != '') + report_service( + :workspace => wspace, + :host => host, + :proto => sprot, + :port => sport, + :name => name, + :info => info, + :task => args[:task] + ) + else + report_service( + :workspace => wspace, + :host => host, + :proto => sprot, + :port => sport, + :info => info, + :task => args[:task] + ) + end + + # Load vulnerabilities associated with this service + svc.elements.each('vulnerabilities/vulnerability') do |vuln| + vid = vuln.attributes['id'].to_s.downcase + refs = process_nexpose_data_sxml_refs(vuln) + next if not refs + report_vuln( + :workspace => wspace, + :host => host, + :port => sport, + :proto => sprot, + :name => 'NEXPOSE-' + vid, + :info => vid, + :refs => refs, + :task => args[:task] + ) + end + end + end + end + + + # + # Nexpose Raw XML + # + def import_nexpose_rawxml_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + + data = "" + ::File.open(filename, 'rb') do |f| + data = f.read(f.stat.size) + end + import_nexpose_rawxml(args.merge(:data => data)) + end + + def import_nexpose_rawxml(args={}, &block) + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + wspace = args[:wspace] || workspace + if Rex::Parser.nokogiri_loaded + parser = "Nokogiri v#{::Nokogiri::VERSION}" + noko_args = args.dup + noko_args[:blacklist] = bl + noko_args[:wspace] = wspace + if block + yield(:parser, parser) + import_nexpose_raw_noko_stream(noko_args) {|type, data| yield type,data} + else + import_nexpose_raw_noko_stream(noko_args) + end + return true + end + data = args[:data] + + # Use a stream parser instead of a tree parser so we can deal with + # huge results files without running out of memory. + parser = Rex::Parser::NexposeXMLStreamParser.new + + # Since all the Refs have to be in the database before we can use them + # in a Vuln, we store all the hosts until we finish parsing and only + # then put everything in the database. This is memory-intensive for + # large files, but should be much less so than a tree parser. + # + # This method is also considerably faster than parsing through the tree + # looking for references every time we hit a vuln. + hosts = [] + vulns = [] + + # The callback merely populates our in-memory table of hosts and vulns + parser.callback = Proc.new { |type, value| + case type + when :host + # XXX: Blacklist should be checked here instead of saving a + # host we're just going to throw away later + hosts.push(value) + when :vuln + value["id"] = value["id"].downcase if value["id"] + vulns.push(value) + end + } + + REXML::Document.parse_stream(data, parser) + + vuln_refs = nexpose_refs_to_struct(vulns) + hosts.each do |host| + if bl.include? host["addr"] + next + else + yield(:address,host["addr"]) if block + end + nexpose_host_from_rawxml(host, vuln_refs, wspace) + end + end + + # + # Takes an array of vuln hashes, as returned by the NeXpose rawxml stream + # parser, like: + # [ + # {"id"=>"winreg-notes-protocol-handler", severity="8", "refs"=>[{"source"=>"BID", "value"=>"10600"}, ...]} + # {"id"=>"windows-zotob-c", severity="8", "refs"=>[{"source"=>"BID", "value"=>"14513"}, ...]} + # ] + # and transforms it into a struct, containing :id, :refs, :title, and :severity + # + # Other attributes can be added later, as needed. + def nexpose_refs_to_struct(vulns) + ret = [] + vulns.each do |vuln| + next if ret.map {|v| v.id}.include? vuln["id"] + vstruct = Struct.new(:id, :refs, :title, :severity).new + vstruct.id = vuln["id"] + vstruct.title = vuln["title"] + vstruct.severity = vuln["severity"] + vstruct.refs = [] + vuln["refs"].each do |ref| + if ref['source'] == 'BID' + vstruct.refs.push('BID-' + ref["value"]) + elsif ref['source'] == 'CVE' + # value is CVE-$ID + vstruct.refs.push(ref["value"]) + elsif ref['source'] == 'MS' + vstruct.refs.push('MSB-' + ref["value"]) + elsif ref['source'] == 'URL' + vstruct.refs.push('URL-' + ref["value"]) + end + end + ret.push vstruct + end + return ret + end + + # Takes a Host object, an array of vuln structs (generated by nexpose_refs_to_struct()), + # and a workspace, and reports the vulns on that host. + def nexpose_host_from_rawxml(h, vstructs, wspace,task=nil) + hobj = nil + data = {:workspace => wspace} + if h["addr"] + addr = h["addr"] + else + # Can't report it if it doesn't have an IP + return + end + data[:host] = addr + if (h["hardware-address"]) + # Put colons between each octet of the MAC address + data[:mac] = h["hardware-address"].gsub(':', '').scan(/../).join(':') + end + data[:state] = (h["status"] == "alive") ? Msf::HostState::Alive : Msf::HostState::Dead + + # Since we only have one name field per host in the database, just + # take the first one. + if (h["names"] and h["names"].first) + data[:name] = h["names"].first + end + + if (data[:state] != Msf::HostState::Dead) + hobj = report_host(data) + report_import_note(wspace, hobj) + end + + if h["notes"] + note = { + :workspace => wspace, + :host => (hobj || addr), + :type => "host.vuln.nexpose_keys", + :data => {}, + :mode => :unique_data, + :task => task + } + h["notes"].each do |v,k| + note[:data][v] ||= [] + next if note[:data][v].include? k + note[:data][v] << k + end + report_note(note) + end + + if h["os_family"] + note = { + :workspace => wspace, + :host => hobj || addr, + :type => 'host.os.nexpose_fingerprint', + :task => task, + :data => { + :family => h["os_family"], + :certainty => h["os_certainty"] + } + } + note[:data][:vendor] = h["os_vendor"] if h["os_vendor"] + note[:data][:product] = h["os_product"] if h["os_product"] + note[:data][:version] = h["os_version"] if h["os_version"] + note[:data][:arch] = h["arch"] if h["arch"] + + report_note(note) + end + + h["endpoints"].each { |p| + extra = "" + extra << p["product"] + " " if p["product"] + extra << p["version"] + " " if p["version"] + + # Skip port-0 endpoints + next if p["port"].to_i == 0 + + # XXX This should probably be handled in a more standard way + # extra << "(" + p["certainty"] + " certainty) " if p["certainty"] + + data = {} + data[:workspace] = wspace + data[:proto] = p["protocol"].downcase + data[:port] = p["port"].to_i + data[:state] = p["status"] + data[:host] = hobj || addr + data[:info] = extra if not extra.empty? + data[:task] = task + if p["name"] != "" + data[:name] = p["name"] + end + report_service(data) + } + + h["vulns"].each_pair { |k,v| + + next if v["status"] !~ /^vulnerable/ + vstruct = vstructs.select {|vs| vs.id.to_s.downcase == v["id"].to_s.downcase}.first + next unless vstruct + data = {} + data[:workspace] = wspace + data[:host] = hobj || addr + data[:proto] = v["protocol"].downcase if v["protocol"] + data[:port] = v["port"].to_i if v["port"] + data[:name] = "NEXPOSE-" + v["id"] + data[:info] = vstruct.title + data[:refs] = vstruct.refs + data[:task] = task + report_vuln(data) + } + end + + + # + # Retina XML + # + + # Process a Retina XML file + def import_retina_xml_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + + data = "" + ::File.open(filename, 'rb') do |f| + data = f.read(f.stat.size) + end + import_retina_xml(args.merge(:data => data)) + end + + # Process Retina XML + def import_retina_xml(args={}, &block) + data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + msg = "Warning: The Retina XML format does not associate vulnerabilities with the\n" + msg << "specific service on which they were found.\n" + msg << "This makes it impossible to correlate exploits to discovered vulnerabilities\n" + msg << "in a reliable fashion." + + yield(:warning,msg) if block + + parser = Rex::Parser::RetinaXMLStreamParser.new + parser.on_found_host = Proc.new do |host| + hobj = nil + data = { + :workspace => wspace, + :task => args[:task] + } + addr = host['address'] + next if not addr + + next if bl.include? addr + data[:host] = addr + + if host['mac'] + data[:mac] = host['mac'] + end + + data[:state] = Msf::HostState::Alive + + if host['hostname'] + data[:name] = host['hostname'] + end + + if host['netbios'] + data[:name] = host['netbios'] + end + + yield(:address, data[:host]) if block + + # Import Host + hobj = report_host(data) + report_import_note(wspace, hobj) + + # Import OS fingerprint + if host["os"] + note = { + :workspace => wspace, + :host => addr, + :type => 'host.os.retina_fingerprint', + :task => args[:task], + :data => { + :os => host["os"] + } + } + report_note(note) + end + + # Import vulnerabilities + host['vulns'].each do |vuln| + refs = vuln['refs'].map{|v| v.join("-")} + refs << "RETINA-#{vuln['rthid']}" if vuln['rthid'] + + vuln_info = { + :workspace => wspace, + :host => addr, + :name => vuln['name'], + :info => vuln['description'], + :refs => refs, + :task => args[:task] + } + + report_vuln(vuln_info) + end + end + + REXML::Document.parse_stream(data, parser) + end + + # + # NetSparker XML + # + + # Process a NetSparker XML file + def import_netsparker_xml_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + + data = "" + ::File.open(filename, 'rb') do |f| + data = f.read(f.stat.size) + end + import_netsparker_xml(args.merge(:data => data)) + end + + # Process NetSparker XML + def import_netsparker_xml(args={}, &block) + data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + addr = nil + parser = Rex::Parser::NetSparkerXMLStreamParser.new + parser.on_found_vuln = Proc.new do |vuln| + data = {:workspace => wspace} + + # Parse the URL + url = vuln['url'] + return if not url + + # Crack the URL into a URI + uri = URI(url) rescue nil + return if not uri + + # Resolve the host and cache the IP + if not addr + baddr = Rex::Socket.addr_aton(uri.host) rescue nil + if baddr + addr = Rex::Socket.addr_ntoa(baddr) + yield(:address, addr) if block + end + end + + # Bail early if we have no IP address + if not addr + raise Interrupt, "Not a valid IP address" + end + + if bl.include?(addr) + raise Interrupt, "IP address is on the blacklist" + end + + data[:host] = addr + data[:vhost] = uri.host + data[:port] = uri.port + data[:ssl] = (uri.scheme == "ssl") + + body = nil + # First report a web page + if vuln['response'] + headers = {} + code = 200 + head,body = vuln['response'].to_s.split(/\r?\n\r?\n/, 2) + if body + + if head =~ /^HTTP\d+\.\d+\s+(\d+)\s*/ + code = $1.to_i + end + + headers = {} + head.split(/\r?\n/).each do |line| + hname,hval = line.strip.split(/\s*:\s*/, 2) + next if hval.to_s.strip.empty? + headers[hname.downcase] ||= [] + headers[hname.downcase] << hval + end + + info = { + :path => uri.path, + :query => uri.query, + :code => code, + :body => body, + :headers => headers, + :task => args[:task] + } + info.merge!(data) + + if headers['content-type'] + info[:ctype] = headers['content-type'][0] + end + + if headers['set-cookie'] + info[:cookie] = headers['set-cookie'].join("\n") + end + + if headers['authorization'] + info[:auth] = headers['authorization'].join("\n") + end + + if headers['location'] + info[:location] = headers['location'][0] + end + + if headers['last-modified'] + info[:mtime] = headers['last-modified'][0] + end + + # Report the web page to the database + report_web_page(info) + + yield(:web_page, url) if block + end + end # End web_page reporting + + + details = netsparker_vulnerability_map(vuln) + + method = netsparker_method_map(vuln) + pname = netsparker_pname_map(vuln) + params = netsparker_params_map(vuln) + + proof = '' + + if vuln['info'] and vuln['info'].length > 0 + proof << vuln['info'].map{|x| "#{x[0]}: #{x[1]}\n" }.join + "\n" + end + + if proof.empty? + if body + proof << body + "\n" + else + proof << vuln['response'].to_s + "\n" + end + end + + if params.empty? and pname + params = [[pname, vuln['vparam_name'].to_s]] + end + + info = { + # XXX: There is a :request attr in the model, but report_web_vuln + # doesn't seem to know about it, so this gets ignored. + #:request => vuln['request'], + :path => uri.path, + :query => uri.query, + :method => method, + :params => params, + :pname => pname.to_s, + :proof => proof, + :risk => details[:risk], + :name => details[:name], + :blame => details[:blame], + :category => details[:category], + :description => details[:description], + :confidence => details[:confidence], + :task => args[:task] + } + info.merge!(data) + + next if vuln['type'].to_s.empty? + + report_web_vuln(info) + yield(:web_vuln, url) if block + end + + # We throw interrupts in our parser when the job is hopeless + begin + REXML::Document.parse_stream(data, parser) + rescue ::Interrupt => e + wlog("The netsparker_xml_import() job was interrupted: #{e}") + end + end + + def netsparker_method_map(vuln) + case vuln['vparam_type'] + when "FullQueryString" + "GET" + when "Querystring" + "GET" + when "Post" + "POST" + when "RawUrlInjection" + "GET" + else + "GET" + end + end + + def netsparker_pname_map(vuln) + case vuln['vparam_name'] + when "URI-BASED", "Query Based" + "PATH" + else + vuln['vparam_name'] + end + end + + def netsparker_params_map(vuln) + [] + end + + def netsparker_vulnerability_map(vuln) + res = { + :risk => 1, + :name => 'Information Disclosure', + :blame => 'System Administrator', + :category => 'info', + :description => "This is an information leak", + :confidence => 100 + } + + # Risk is a value from 1-5 indicating the severity of the issue + # Examples: 1, 4, 5 + + # Name is a descriptive name for this vulnerability. + # Examples: XSS, ReflectiveXSS, PersistentXSS + + # Blame indicates who is at fault for the vulnerability + # Examples: App Developer, Server Developer, System Administrator + + # Category indicates the general class of vulnerability + # Examples: info, xss, sql, rfi, lfi, cmd + + # Description is a textual summary of the vulnerability + # Examples: "A reflective cross-site scripting attack" + # "The web server leaks the internal IP address" + # "The cookie is not set to HTTP-only" + + # + # Confidence is a value from 1 to 100 indicating how confident the + # software is that the results are valid. + # Examples: 100, 90, 75, 15, 10, 0 + + case vuln['type'].to_s + when "ApacheDirectoryListing" + res = { + :risk => 1, + :name => 'Directory Listing', + :blame => 'System Administrator', + :category => 'info', + :description => "", + :confidence => 100 + } + when "ApacheMultiViewsEnabled" + res = { + :risk => 1, + :name => 'Apache MultiViews Enabled', + :blame => 'System Administrator', + :category => 'info', + :description => "", + :confidence => 100 + } + when "ApacheVersion" + res = { + :risk => 1, + :name => 'Web Server Version', + :blame => 'System Administrator', + :category => 'info', + :description => "", + :confidence => 100 + } + when "PHPVersion" + res = { + :risk => 1, + :name => 'PHP Module Version', + :blame => 'System Administrator', + :category => 'info', + :description => "", + :confidence => 100 + } + when "AutoCompleteEnabled" + res = { + :risk => 1, + :name => 'Form AutoComplete Enabled', + :blame => 'App Developer', + :category => 'info', + :description => "", + :confidence => 100 + } + when "CookieNotMarkedAsHttpOnly" + res = { + :risk => 1, + :name => 'Cookie Not HttpOnly', + :blame => 'App Developer', + :category => 'info', + :description => "", + :confidence => 100 + } + when "EmailDisclosure" + res = { + :risk => 1, + :name => 'Email Address Disclosure', + :blame => 'App Developer', + :category => 'info', + :description => "", + :confidence => 100 + } + when "ForbiddenResource" + res = { + :risk => 1, + :name => 'Forbidden Resource', + :blame => 'App Developer', + :category => 'info', + :description => "", + :confidence => 100 + } + when "FileUploadFound" + res = { + :risk => 1, + :name => 'File Upload Form', + :blame => 'App Developer', + :category => 'info', + :description => "", + :confidence => 100 + } + when "PasswordOverHTTP" + res = { + :risk => 2, + :name => 'Password Over HTTP', + :blame => 'App Developer', + :category => 'info', + :description => "", + :confidence => 100 + } + when "MySQL5Identified" + res = { + :risk => 1, + :name => 'MySQL 5 Identified', + :blame => 'App Developer', + :category => 'info', + :description => "", + :confidence => 100 + } + when "PossibleInternalWindowsPathLeakage" + res = { + :risk => 1, + :name => 'Path Leakage - Windows', + :blame => 'App Developer', + :category => 'info', + :description => "", + :confidence => 100 + } + when "PossibleInternalUnixPathLeakage" + res = { + :risk => 1, + :name => 'Path Leakage - Unix', + :blame => 'App Developer', + :category => 'info', + :description => "", + :confidence => 100 + } + when "PossibleXSS", "LowPossibilityPermanentXSS", "XSS", "PermanentXSS" + conf = 100 + conf = 25 if vuln['type'].to_s == "LowPossibilityPermanentXSS" + conf = 50 if vuln['type'].to_s == "PossibleXSS" + res = { + :risk => 3, + :name => 'Cross-Site Scripting', + :blame => 'App Developer', + :category => 'xss', + :description => "", + :confidence => conf + } + + when "ConfirmedBlindSQLInjection", "ConfirmedSQLInjection", "HighlyPossibleSqlInjection", "DatabaseErrorMessages" + conf = 100 + conf = 90 if vuln['type'].to_s == "HighlyPossibleSqlInjection" + conf = 25 if vuln['type'].to_s == "DatabaseErrorMessages" + res = { + :risk => 5, + :name => 'SQL Injection', + :blame => 'App Developer', + :category => 'sql', + :description => "", + :confidence => conf + } + else + conf = 100 + res = { + :risk => 1, + :name => vuln['type'].to_s, + :blame => 'App Developer', + :category => 'info', + :description => "", + :confidence => conf + } + end + + res + end + + def import_fusionvm_xml(args={}) + args[:wspace] ||= workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + doc = Rex::Parser::FusionVMDocument.new(args,self) + parser = ::Nokogiri::XML::SAX::Parser.new(doc) + parser.parse(args[:data]) + end + + + # + # Import Nmap's -oX xml output + # + def import_nmap_xml_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + + data = "" + ::File.open(filename, 'rb') do |f| + data = f.read(f.stat.size) + end + import_nmap_xml(args.merge(:data => data)) + end + + def import_nexpose_raw_noko_stream(args, &block) + if block + doc = Rex::Parser::NexposeRawDocument.new(args,framework.db) {|type, data| yield type,data } + else + doc = Rex::Parser::NexposeRawDocument.new(args,self) + end + parser = ::Nokogiri::XML::SAX::Parser.new(doc) + parser.parse(args[:data]) + end + + def import_nexpose_noko_stream(args, &block) + if block + doc = Rex::Parser::NexposeSimpleDocument.new(args,framework.db) {|type, data| yield type,data } + else + doc = Rex::Parser::NexposeSimpleDocument.new(args,self) + end + parser = ::Nokogiri::XML::SAX::Parser.new(doc) + parser.parse(args[:data]) + end + + def import_nmap_noko_stream(args, &block) + if block + doc = Rex::Parser::NmapDocument.new(args,framework.db) {|type, data| yield type,data } + else + doc = Rex::Parser::NmapDocument.new(args,self) + end + parser = ::Nokogiri::XML::SAX::Parser.new(doc) + parser.parse(args[:data]) + end + + # If you have Nokogiri installed, you'll be shunted over to + # that. Otherwise, you'll hit the old NmapXMLStreamParser. + def import_nmap_xml(args={}, &block) + return nil if args[:data].nil? or args[:data].empty? + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + + if Rex::Parser.nokogiri_loaded + noko_args = args.dup + noko_args[:blacklist] = bl + noko_args[:wspace] = wspace + if block + yield(:parser, "Nokogiri v#{::Nokogiri::VERSION}") + import_nmap_noko_stream(noko_args) {|type, data| yield type,data } + else + import_nmap_noko_stream(noko_args) + end + return true + end + + # XXX: Legacy nmap xml parser starts here. + + fix_services = args[:fix_services] + data = args[:data] + + # Use a stream parser instead of a tree parser so we can deal with + # huge results files without running out of memory. + parser = Rex::Parser::NmapXMLStreamParser.new + yield(:parser, parser.class.name) if block + + # Whenever the parser pulls a host out of the nmap results, store + # it, along with any associated services, in the database. + parser.on_found_host = Proc.new { |h| + hobj = nil + data = {:workspace => wspace} + if (h["addrs"].has_key?("ipv4")) + addr = h["addrs"]["ipv4"] + elsif (h["addrs"].has_key?("ipv6")) + addr = h["addrs"]["ipv6"] + else + # Can't report it if it doesn't have an IP + raise RuntimeError, "At least one IPv4 or IPv6 address is required" + end + next if bl.include? addr + data[:host] = addr + if (h["addrs"].has_key?("mac")) + data[:mac] = h["addrs"]["mac"] + end + data[:state] = (h["status"] == "up") ? Msf::HostState::Alive : Msf::HostState::Dead + data[:task] = args[:task] + + if ( h["reverse_dns"] ) + data[:name] = h["reverse_dns"] + end + + # Only report alive hosts with ports to speak of. + if(data[:state] != Msf::HostState::Dead) + if h["ports"].size > 0 + if fix_services + port_states = h["ports"].map {|p| p["state"]}.reject {|p| p == "filtered"} + next if port_states.compact.empty? + end + yield(:address,data[:host]) if block + hobj = report_host(data) + report_import_note(wspace,hobj) + end + end + + if( h["os_vendor"] ) + note = { + :workspace => wspace, + :host => hobj || addr, + :type => 'host.os.nmap_fingerprint', + :task => args[:task], + :data => { + :os_vendor => h["os_vendor"], + :os_family => h["os_family"], + :os_version => h["os_version"], + :os_accuracy => h["os_accuracy"] + } + } + + if(h["os_match"]) + note[:data][:os_match] = h['os_match'] + end + + report_note(note) + end + + if (h["last_boot"]) + report_note( + :workspace => wspace, + :host => hobj || addr, + :type => 'host.last_boot', + :task => args[:task], + :data => { + :time => h["last_boot"] + } + ) + end + + if (h["trace"]) + hops = [] + h["trace"]["hops"].each do |hop| + hops << { + "ttl" => hop["ttl"].to_i, + "address" => hop["ipaddr"].to_s, + "rtt" => hop["rtt"].to_f, + "name" => hop["host"].to_s + } + end + report_note( + :workspace => wspace, + :host => hobj || addr, + :type => 'host.nmap.traceroute', + :task => args[:task], + :data => { + 'port' => h["trace"]["port"].to_i, + 'proto' => h["trace"]["proto"].to_s, + 'hops' => hops + } + ) + end + + + # Put all the ports, regardless of state, into the db. + h["ports"].each { |p| + # Localhost port results are pretty unreliable -- if it's + # unknown, it's no good (possibly Windows-only) + if ( + p["state"] == "unknown" && + h["status_reason"] == "localhost-response" + ) + next + end + extra = "" + extra << p["product"] + " " if p["product"] + extra << p["version"] + " " if p["version"] + extra << p["extrainfo"] + " " if p["extrainfo"] + + data = {} + data[:workspace] = wspace + if fix_services + data[:proto] = nmap_msf_service_map(p["protocol"]) + else + data[:proto] = p["protocol"].downcase + end + data[:port] = p["portid"].to_i + data[:state] = p["state"] + data[:host] = hobj || addr + data[:info] = extra if not extra.empty? + data[:task] = args[:task] + if p["name"] != "unknown" + data[:name] = p["name"] + end + report_service(data) + } + #Parse the scripts output + if h["scripts"] + h["scripts"].each do |key,val| + if key == "smb-check-vulns" + if val =~ /MS08-067: VULNERABLE/ + vuln_info = { + :workspace => wspace, + :task => args[:task], + :host => hobj || addr, + :port => 445, + :proto => 'tcp', + :name => 'MS08-067', + :info => 'Microsoft Windows Server Service Crafted RPC Request Handling Unspecified Remote Code Execution', + :refs =>['CVE-2008-4250', + 'BID-31874', + 'OSVDB-49243', + 'CWE-94', + 'MSFT-MS08-067', + 'MSF-Microsoft Server Service Relative Path Stack Corruption', + 'NSS-34476'] + } + report_vuln(vuln_info) + end + if val =~ /MS06-025: VULNERABLE/ + vuln_info = { + :workspace => wspace, + :task => args[:task], + :host => hobj || addr, + :port => 445, + :proto => 'tcp', + :name => 'MS06-025', + :info => 'Vulnerability in Routing and Remote Access Could Allow Remote Code Execution', + :refs =>['CVE-2006-2370', + 'CVE-2006-2371', + 'BID-18325', + 'BID-18358', + 'BID-18424', + 'OSVDB-26436', + 'OSVDB-26437', + 'MSFT-MS06-025', + 'MSF-Microsoft RRAS Service RASMAN Registry Overflow', + 'NSS-21689'] + } + report_vuln(vuln_info) + end + # This one has NOT been Tested , remove this comment if confirmed working + if val =~ /MS07-029: VULNERABLE/ + vuln_info = { + :workspace => wspace, + :task => args[:task], + :host => hobj || addr, + :port => 445, + :proto => 'tcp', + :name => 'MS07-029', + :info => 'Vulnerability in Windows DNS RPC Interface Could Allow Remote Code Execution', + # Add more refs based on nessus/nexpose .. results + :refs =>['CVE-2007-1748', + 'OSVDB-34100', + 'MSF-Microsoft DNS RPC Service extractQuotedChar()', + 'NSS-25168'] + } + report_vuln(vuln_info) + end + end + end + end + } + + # XXX: Legacy nmap xml parser ends here. + + REXML::Document.parse_stream(data, parser) + end + + def nmap_msf_service_map(proto) + service_name_map(proto) + end + + # + # This method normalizes an incoming service name to one of the + # the standard ones recognized by metasploit + # + def service_name_map(proto) + return proto unless proto.kind_of? String + case proto.downcase + when "msrpc", "nfs-or-iis", "dce endpoint resolution" + "dcerpc" + when "ms-sql-s", "tds" + "mssql" + when "ms-sql-m","microsoft sql monitor" + "mssql-m" + when "postgresql"; "postgres" + when "http-proxy"; "http" + when "iiimsf"; "db2" + when "oracle-tns"; "oracle" + when "quickbooksrds"; "metasploit" + when "microsoft remote display protocol" + "rdp" + when "vmware authentication daemon" + "vmauthd" + when "netbios-ns", "cifs name service" + "netbios" + when "netbios-ssn", "microsoft-ds", "cifs" + "smb" + when "remote shell" + "shell" + when "remote login" + "login" + when "nfs lockd" + "lockd" + when "hp jetdirect" + "jetdirect" + when "dhcp server" + "dhcp" + when /^dns-(udp|tcp)$/; "dns" + when /^dce[\s+]rpc$/; "dcerpc" + else + proto.downcase.gsub(/\s*\(.*/, '') # "service (some service)" + end + end + + def report_import_note(wspace,addr) + if @import_filedata.kind_of?(Hash) && @import_filedata[:filename] && @import_filedata[:filename] !~ /msfe-nmap[0-9]{8}/ + report_note( + :workspace => wspace, + :host => addr, + :type => 'host.imported', + :data => @import_filedata.merge(:time=> Time.now.utc) + ) + end + end + + # + # Import Nessus NBE files + # + def import_nessus_nbe_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + + data = "" + ::File.open(filename, 'rb') do |f| + data = f.read(f.stat.size) + end + import_nessus_nbe(args.merge(:data => data)) + end + + # There is no place the NBE actually stores the plugin name used to + # scan. You get "Security Note" or "Security Warning," and that's it. + def import_nessus_nbe(args={}, &block) + nbe_data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + + nbe_copy = nbe_data.dup + # First pass, just to build the address map. + addr_map = {} + + # Cache host objects before passing into handle_nessus() + hobj_map = {} + + nbe_copy.each_line do |line| + r = line.split('|') + next if r[0] != 'results' + next if r[4] != "12053" + data = r[6] + addr,hname = data.match(/([0-9\x2e]+) resolves as (.+)\x2e\\n/n)[1,2] + addr_map[hname] = addr + end + + nbe_data.each_line do |line| + r = line.split('|') + next if r[0] != 'results' + hname = r[2] + if addr_map[hname] + addr = addr_map[hname] + else + addr = hname # Must be unresolved, probably an IP address. + end + port = r[3] + nasl = r[4] + type = r[5] + data = r[6] + + # If there's no resolution, or if it's malformed, skip it. + next unless ipv46_validator(addr) + + if bl.include? addr + next + else + yield(:address,addr) if block + end + + hobj_map[ addr ] ||= report_host(:host => addr, :workspace => wspace, :task => args[:task]) + + # Match the NBE types with the XML severity ratings + case type + # log messages don't actually have any data, they are just + # complaints about not being able to perform this or that test + # because such-and-such was missing + when "Log Message"; next + when "Security Hole"; severity = 3 + when "Security Warning"; severity = 2 + when "Security Note"; severity = 1 + # a severity 0 means there's no extra data, it's just an open port + else; severity = 0 + end + if nasl == "11936" + os = data.match(/The remote host is running (.*)\\n/)[1] + report_note( + :workspace => wspace, + :task => args[:task], + :host => hobj_map[ addr ], + :type => 'host.os.nessus_fingerprint', + :data => { + :os => os.to_s.strip + } + ) + end + + next if nasl.to_s.strip.empty? + plugin_name = nil # NBE doesn't ever populate this + handle_nessus(wspace, hobj_map[ addr ], port, nasl, plugin_name, severity, data) + end + end + + # + # Of course they had to change the nessus format. + # + def import_openvas_xml(args={}, &block) + filename = args[:filename] + wspace = args[:wspace] || workspace + + raise DBImportError.new("No OpenVAS XML support. Please submit a patch to msfdev[at]metasploit.com") + end + + # + # Import IP360 XML v3 output + # + def import_ip360_xml_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + + data = "" + ::File.open(filename, 'rb') do |f| + data = f.read(f.stat.size) + end + import_ip360_xml_v3(args.merge(:data => data)) + end + + # + # Import Nessus XML v1 and v2 output + # + # Old versions of openvas exported this as well + # + def import_nessus_xml_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + + data = "" + ::File.open(filename, 'rb') do |f| + data = f.read(f.stat.size) + end + + if data.index("NessusClientData_v2") + import_nessus_xml_v2(args.merge(:data => data)) + else + import_nessus_xml(args.merge(:data => data)) + end + end + + def import_nessus_xml(args={}, &block) + data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + + doc = rexmlify(data) + doc.elements.each('/NessusClientData/Report/ReportHost') do |host| + hobj = nil + addr = nil + hname = nil + os = nil + # If the name is resolved, the Nessus plugin for DNS + # resolution should be there. If not, fall back to the + # HostName + host.elements.each('ReportItem') do |item| + next unless item.elements['pluginID'].text == "12053" + addr = item.elements['data'].text.match(/([0-9\x2e]+) resolves as/n)[1] + hname = host.elements['HostName'].text + end + addr ||= host.elements['HostName'].text + next unless ipv46_validator(addr) # Skip resolved names and SCAN-ERROR. + if bl.include? addr + next + else + yield(:address,addr) if block + end + + hinfo = { + :workspace => wspace, + :host => addr, + :task => args[:task] + } + + # Record the hostname + hinfo.merge!(:name => hname.to_s.strip) if hname + hobj = report_host(hinfo) + report_import_note(wspace,hobj) + + # Record the OS + os ||= host.elements["os_name"] + if os + report_note( + :workspace => wspace, + :task => args[:task], + :host => hobj, + :type => 'host.os.nessus_fingerprint', + :data => { + :os => os.text.to_s.strip + } + ) + end + + host.elements.each('ReportItem') do |item| + nasl = item.elements['pluginID'].text + plugin_name = item.elements['pluginName'].text + port = item.elements['port'].text + data = item.elements['data'].text + severity = item.elements['severity'].text + + handle_nessus(wspace, hobj, port, nasl, plugin_name, severity, data, args[:task]) + end + end + end + + def import_nessus_xml_v2(args={}, &block) + data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + + #@host = { + #'hname' => nil, + #'addr' => nil, + #'mac' => nil, + #'os' => nil, + #'ports' => [ 'port' => { 'port' => nil, + # 'svc_name' => nil, + # 'proto' => nil, + # 'severity' => nil, + # 'nasl' => nil, + # 'description' => nil, + # 'cve' => [], + # 'bid' => [], + # 'xref' => [] + # } + # ] + #} + parser = Rex::Parser::NessusXMLStreamParser.new + parser.on_found_host = Proc.new { |host| + + hobj = nil + addr = host['addr'] || host['hname'] + + next unless ipv46_validator(addr) # Catches SCAN-ERROR, among others. + + if bl.include? addr + next + else + yield(:address,addr) if block + end + + os = host['os'] + hname = host['hname'] + mac = host['mac'] + + host_info = { + :workspace => wspace, + :host => addr, + :task => args[:task] + } + host_info[:name] = hname.to_s.strip if hname + # Short mac, protect against Nessus's habit of saving multiple macs + # We can't use them anyway, so take just the first. + host_info[:mac] = mac.to_s.strip.upcase.split(/\s+/).first if mac + + hobj = report_host(host_info) + report_import_note(wspace,hobj) + + os = host['os'] + yield(:os,os) if block + if os + report_note( + :workspace => wspace, + :task => args[:task], + :host => hobj, + :type => 'host.os.nessus_fingerprint', + :data => { + :os => os.to_s.strip + } + ) + end + + host['ports'].each do |item| + next if item['port'] == 0 + msf = nil + nasl = item['nasl'].to_s + nasl_name = item['nasl_name'].to_s + port = item['port'].to_s + proto = item['proto'] || "tcp" + sname = item['svc_name'] + severity = item['severity'] + description = item['description'] + cve = item['cve'] + bid = item['bid'] + xref = item['xref'] + msf = item['msf'] + + yield(:port,port) if block + + handle_nessus_v2(wspace, hobj, port, proto, sname, nasl, nasl_name, severity, description, cve, bid, xref, msf, args[:task]) + + end + yield(:end,hname) if block + } + + REXML::Document.parse_stream(data, parser) + + end + + def import_mbsa_xml(args={}, &block) + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + wspace = args[:wspace] || workspace + if Rex::Parser.nokogiri_loaded + parser = "Nokogiri v#{::Nokogiri::VERSION}" + noko_args = args.dup + noko_args[:blacklist] = bl + noko_args[:wspace] = wspace + if block + yield(:parser, parser) + import_mbsa_noko_stream(noko_args) {|type, data| yield type,data} + else + import_mbsa_noko_stream(noko_args) + end + return true + else # Sorry + raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.") + end + end + + def import_mbsa_noko_stream(args={},&block) + if block + doc = Rex::Parser::MbsaDocument.new(args,framework.db) {|type, data| yield type,data } + else + doc = Rex::Parser::MbsaDocument.new(args,self) + end + parser = ::Nokogiri::XML::SAX::Parser.new(doc) + parser.parse(args[:data]) + end + + def import_foundstone_xml(args={}, &block) + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + wspace = args[:wspace] || workspace + if Rex::Parser.nokogiri_loaded + parser = "Nokogiri v#{::Nokogiri::VERSION}" + noko_args = args.dup + noko_args[:blacklist] = bl + noko_args[:wspace] = wspace + if block + yield(:parser, parser) + import_foundstone_noko_stream(noko_args) {|type, data| yield type,data} + else + import_foundstone_noko_stream(noko_args) + end + return true + else # Sorry + raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.") + end + end + + def import_foundstone_noko_stream(args={},&block) + if block + doc = Rex::Parser::FoundstoneDocument.new(args,framework.db) {|type, data| yield type,data } + else + doc = Rex::Parser::FoundstoneDocument.new(args,self) + end + parser = ::Nokogiri::XML::SAX::Parser.new(doc) + parser.parse(args[:data]) + end + + def import_acunetix_xml(args={}, &block) + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + wspace = args[:wspace] || workspace + if Rex::Parser.nokogiri_loaded + parser = "Nokogiri v#{::Nokogiri::VERSION}" + noko_args = args.dup + noko_args[:blacklist] = bl + noko_args[:wspace] = wspace + if block + yield(:parser, parser) + import_acunetix_noko_stream(noko_args) {|type, data| yield type,data} + else + import_acunetix_noko_stream(noko_args) + end + return true + else # Sorry + raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.") + end + end + + def import_ci_xml(args={}, &block) + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + wspace = args[:wspace] || workspace + if Rex::Parser.nokogiri_loaded + parser = "Nokogiri v#{::Nokogiri::VERSION}" + noko_args = args.dup + noko_args[:blacklist] = bl + noko_args[:wspace] = wspace + if block + yield(:parser, parser) + import_ci_noko_stream(noko_args) {|type, data| yield type,data} + else + import_ci_noko_stream(noko_args) + end + return true + else # Sorry + raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.") + end + end + + def import_acunetix_noko_stream(args={},&block) + if block + doc = Rex::Parser::AcunetixDocument.new(args,framework.db) {|type, data| yield type,data } + else + doc = Rex::Parser::AcunetixFoundstoneDocument.new(args,self) + end + parser = ::Nokogiri::XML::SAX::Parser.new(doc) + parser.parse(args[:data]) + end + + + def import_appscan_xml(args={}, &block) + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + wspace = args[:wspace] || workspace + if Rex::Parser.nokogiri_loaded + parser = "Nokogiri v#{::Nokogiri::VERSION}" + noko_args = args.dup + noko_args[:blacklist] = bl + noko_args[:wspace] = wspace + if block + yield(:parser, parser) + import_appscan_noko_stream(noko_args) {|type, data| yield type,data} + else + import_appscan_noko_stream(noko_args) + end + return true + else # Sorry + raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.") + end + end + + def import_appscan_noko_stream(args={},&block) + if block + doc = Rex::Parser::AppscanDocument.new(args,framework.db) {|type, data| yield type,data } + else + doc = Rex::Parser::AppscanDocument.new(args,self) + end + parser = ::Nokogiri::XML::SAX::Parser.new(doc) + parser.parse(args[:data]) + end + + def import_burp_session_xml(args={}, &block) + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + wspace = args[:wspace] || workspace + if Rex::Parser.nokogiri_loaded + # Rex::Parser.reload("burp_session_nokogiri.rb") + parser = "Nokogiri v#{::Nokogiri::VERSION}" + noko_args = args.dup + noko_args[:blacklist] = bl + noko_args[:wspace] = wspace + if block + yield(:parser, parser) + import_burp_session_noko_stream(noko_args) {|type, data| yield type,data} + else + import_burp_session_noko_stream(noko_args) + end + return true + else # Sorry + raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.") + end + end + + def import_burp_session_noko_stream(args={},&block) + if block + doc = Rex::Parser::BurpSessionDocument.new(args,framework.db) {|type, data| yield type,data } + else + doc = Rex::Parser::BurpSessionDocument.new(args,self) + end + parser = ::Nokogiri::XML::SAX::Parser.new(doc) + parser.parse(args[:data]) + end + + + # + # Import IP360's ASPL database + # + def import_ip360_aspl_xml(args={}, &block) + data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + + if not data.index(" {'name' => { }, 'cve' => { }, 'bid' => { } } + # 'oses' => {'name' } } + + aspl_path = nil + aspl_paths = [ + ::File.join(Msf::Config.config_directory, "data", "ncircle", "ip360.aspl"), + ::File.join(Msf::Config.data_directory, "ncircle", "ip360.aspl") + ] + + aspl_paths.each do |tpath| + next if not (::File.exist?(tpath) and ::File.readable?(tpath)) + aspl_path = tpath + break + end + + if not aspl_path + raise DBImportError.new("The nCircle IP360 ASPL file is not present.\n Download ASPL from nCircle VNE | Administer | Support | Resources, unzip it, and import it first") + end + + # parse nCircle ASPL file + aspl = "" + ::File.open(aspl_path, "rb") do |f| + aspl = f.read(f.stat.size) + end + + @asplhash = nil + parser = Rex::Parser::IP360ASPLXMLStreamParser.new + parser.on_found_aspl = Proc.new { |asplh| + @asplhash = asplh + } + REXML::Document.parse_stream(aspl, parser) + + # nCircle has some quotes escaped which causes the parser to break + # we don't need these lines so just replace \" with " + data.gsub!(/\\"/,'"') + + # parse nCircle Scan Output + parser = Rex::Parser::IP360XMLStreamParser.new + parser.on_found_host = Proc.new { |host| + hobj = nil + addr = host['addr'] || host['hname'] + + next unless ipv46_validator(addr) # Catches SCAN-ERROR, among others. + + if bl.include? addr + next + else + yield(:address,addr) if block + end + + os = host['os'] + hname = host['hname'] + mac = host['mac'] + + host_hash = { + :workspace => wspace, + :host => addr, + :task => args[:task] + } + host_hash[:name] = hname.to_s.strip if hname + host_hash[:mac] = mac.to_s.strip.upcase if mac + + hobj = report_host(host_hash) + + yield(:os, os) if block + if os + report_note( + :workspace => wspace, + :task => args[:task], + :host => hobj, + :type => 'host.os.ip360_fingerprint', + :data => { + :os => @asplhash['oses'][os].to_s.strip + } + ) + end + + host['apps'].each do |item| + port = item['port'].to_s + proto = item['proto'].to_s + + handle_ip360_v3_svc(wspace, hobj, port, proto, hname, args[:task]) + end + + + host['vulns'].each do |item| + vulnid = item['vulnid'].to_s + port = item['port'].to_s + proto = item['proto'] || "tcp" + vulnname = @asplhash['vulns']['name'][vulnid] + cves = @asplhash['vulns']['cve'][vulnid] + bids = @asplhash['vulns']['bid'][vulnid] + + yield(:port, port) if block + + handle_ip360_v3_vuln(wspace, hobj, port, proto, hname, vulnid, vulnname, cves, bids, args[:task]) + + end + + yield(:end, hname) if block + } + + REXML::Document.parse_stream(data, parser) + end + + def find_qualys_asset_vuln_refs(doc) + vuln_refs = {} + doc.elements.each("/ASSET_DATA_REPORT/GLOSSARY/VULN_DETAILS_LIST/VULN_DETAILS") do |vuln| + next unless vuln.elements['QID'] && vuln.elements['QID'].first + qid = vuln.elements['QID'].first.to_s + vuln_refs[qid] ||= [] + vuln.elements.each('CVE_ID_LIST/CVE_ID') do |ref| + vuln_refs[qid].push('CVE-' + /C..-([0-9\-]{9})/.match(ref.elements['ID'].text.to_s)[1]) + end + vuln.elements.each('BUGTRAQ_ID_LIST/BUGTRAQ_ID') do |ref| + vuln_refs[qid].push('BID-' + ref.elements['ID'].text.to_s) + end + end + return vuln_refs + end + + # Pull out vulnerabilities that have at least one matching + # ref -- many "vulns" are not vulns, just audit information. + def find_qualys_asset_vulns(host,wspace,hobj,vuln_refs,task_id,&block) + host.elements.each("VULN_INFO_LIST/VULN_INFO") do |vi| + next unless vi.elements["QID"] + vi.elements.each("QID") do |qid| + next if vuln_refs[qid.text].nil? || vuln_refs[qid.text].empty? + handle_qualys(wspace, hobj, nil, nil, qid.text, nil, vuln_refs[qid.text], nil, nil, task_id) + end + end + end + + # Takes QID numbers and finds the discovered services in + # a qualys_asset_xml. + def find_qualys_asset_ports(i,host,wspace,hobj,task_id) + return unless (i == 82023 || i == 82004) + proto = i == 82023 ? 'tcp' : 'udp' + qid = host.elements["VULN_INFO_LIST/VULN_INFO/QID[@id='qid_#{i}']"] + qid_result = qid.parent.elements["RESULT[@format='table']"] if qid + hports = qid_result.first.to_s if qid_result + if hports + hports.scan(/([0-9]+)\t(.*?)\t.*?\t([^\t\n]*)/) do |match| + if match[2] == nil or match[2].strip == 'unknown' + name = match[1].strip + else + name = match[2].strip + end + handle_qualys(wspace, hobj, match[0].to_s, proto, 0, nil, nil, name, nil, task_id) + end + end + end + + # + # Import Qualys's Asset Data Report format + # + def import_qualys_asset_xml(args={}, &block) + data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + doc = rexmlify(data) + vuln_refs = find_qualys_asset_vuln_refs(doc) + + # 2nd pass, actually grab the hosts. + doc.elements.each("/ASSET_DATA_REPORT/HOST_LIST/HOST") do |host| + hobj = nil + addr = host.elements["IP"].text if host.elements["IP"] + next unless validate_ips(addr) + if bl.include? addr + next + else + yield(:address,addr) if block + end + hname = ( # Prefer NetBIOS over DNS + (host.elements["NETBIOS"].text if host.elements["NETBIOS"]) || + (host.elements["DNS"].text if host.elements["DNS"]) || + "" ) + hobj = report_host(:workspace => wspace, :host => addr, :name => hname, :state => Msf::HostState::Alive, :task => args[:task]) + report_import_note(wspace,hobj) + + if host.elements["OPERATING_SYSTEM"] + hos = host.elements["OPERATING_SYSTEM"].text + report_note( + :workspace => wspace, + :task => args[:task], + :host => hobj, + :type => 'host.os.qualys_fingerprint', + :data => { :os => hos } + ) + end + + # Report open ports. + find_qualys_asset_ports(82023,host,wspace,hobj, args[:task]) # TCP + find_qualys_asset_ports(82004,host,wspace,hobj, args[:task]) # UDP + + # Report vulns + find_qualys_asset_vulns(host,wspace,hobj,vuln_refs, args[:task],&block) + + end # host + + end + + # + # Import Qualys' Scan xml output + # + def import_qualys_scan_xml_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + + data = "" + ::File.open(filename, 'rb') do |f| + data = f.read(f.stat.size) + end + import_qualys_scan_xml(args.merge(:data => data)) + end + + def import_qualys_scan_xml(args={}, &block) + data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + + + doc = rexmlify(data) + doc.elements.each('/SCAN/IP') do |host| + hobj = nil + addr = host.attributes['value'] + if bl.include? addr + next + else + yield(:address,addr) if block + end + hname = host.attributes['name'] || '' + + hobj = report_host(:workspace => wspace, :host => addr, :name => hname, :state => Msf::HostState::Alive, :task => args[:task]) + report_import_note(wspace,hobj) + + if host.elements["OS"] + hos = host.elements["OS"].text + report_note( + :workspace => wspace, + :task => args[:task], + :host => hobj, + :type => 'host.os.qualys_fingerprint', + :data => { + :os => hos + } + ) + end + + # Open TCP Services List (Qualys ID 82023) + services_tcp = host.elements["SERVICES/CAT/SERVICE[@number='82023']/RESULT"] + if services_tcp + services_tcp.text.scan(/([0-9]+)\t(.*?)\t.*?\t([^\t\n]*)/) do |match| + if match[2] == nil or match[2].strip == 'unknown' + name = match[1].strip + else + name = match[2].strip + end + handle_qualys(wspace, hobj, match[0].to_s, 'tcp', 0, nil, nil, name, nil, args[:task]) + end + end + # Open UDP Services List (Qualys ID 82004) + services_udp = host.elements["SERVICES/CAT/SERVICE[@number='82004']/RESULT"] + if services_udp + services_udp.text.scan(/([0-9]+)\t(.*?)\t.*?\t([^\t\n]*)/) do |match| + if match[2] == nil or match[2].strip == 'unknown' + name = match[1].strip + else + name = match[2].strip + end + handle_qualys(wspace, hobj, match[0].to_s, 'udp', 0, nil, nil, name, nil, args[:task]) + end + end + + # VULNS are confirmed, PRACTICES are unconfirmed vulnerabilities + host.elements.each('VULNS/CAT | PRACTICES/CAT') do |cat| + port = cat.attributes['port'] + protocol = cat.attributes['protocol'] + cat.elements.each('VULN | PRACTICE') do |vuln| + refs = [] + qid = vuln.attributes['number'] + severity = vuln.attributes['severity'] + title = vuln.elements['TITLE'].text.to_s + vuln.elements.each('VENDOR_REFERENCE_LIST/VENDOR_REFERENCE') do |ref| + refs.push(ref.elements['ID'].text.to_s) + end + vuln.elements.each('CVE_ID_LIST/CVE_ID') do |ref| + refs.push('CVE-' + /C..-([0-9\-]{9})/.match(ref.elements['ID'].text.to_s)[1]) + end + vuln.elements.each('BUGTRAQ_ID_LIST/BUGTRAQ_ID') do |ref| + refs.push('BID-' + ref.elements['ID'].text.to_s) + end + + handle_qualys(wspace, hobj, port, protocol, qid, severity, refs, nil,title, args[:task]) + end + end + end + end + + def import_ip_list_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + + data = "" + ::File.open(filename, 'rb') do |f| + data = f.read(f.stat.size) + end + import_ip_list(args.merge(:data => data)) + end + + def import_ip_list(args={}, &block) + data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + + data.each_line do |ip| + ip.strip! + if bl.include? ip + next + else + yield(:address,ip) if block + end + host = find_or_create_host(:workspace => wspace, :host=> ip, :state => Msf::HostState::Alive, :task => args[:task]) + end + end + + def import_amap_log_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + data = "" + ::File.open(filename, 'rb') do |f| + data = f.read(f.stat.size) + end + + case import_filetype_detect(data) + when :amap_log + import_amap_log(args.merge(:data => data)) + when :amap_mlog + import_amap_mlog(args.merge(:data => data)) + else + raise DBImportError.new("Could not determine file type") + end + end + + def import_amap_log(args={}, &block) + data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + + data.each_line do |line| + next if line =~ /^#/ + next if line !~ /^Protocol on ([^:]+):([^\x5c\x2f]+)[\x5c\x2f](tcp|udp) matches (.*)$/n + addr = $1 + next if bl.include? addr + port = $2.to_i + proto = $3.downcase + name = $4 + host = find_or_create_host(:workspace => wspace, :host => addr, :state => Msf::HostState::Alive, :task => args[:task]) + next if not host + yield(:address,addr) if block + info = { + :workspace => wspace, + :task => args[:task], + :host => host, + :proto => proto, + :port => port + } + if name != "unidentified" + info[:name] = name + end + service = find_or_create_service(info) + end + end + + def import_amap_mlog(args={}, &block) + data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + + data.each_line do |line| + next if line =~ /^#/ + r = line.split(':') + next if r.length < 6 + + addr = r[0] + next if bl.include? addr + port = r[1].to_i + proto = r[2].downcase + status = r[3] + name = r[5] + next if status != "open" + + host = find_or_create_host(:workspace => wspace, :host => addr, :state => Msf::HostState::Alive, :task => args[:task]) + next if not host + yield(:address,addr) if block + info = { + :workspace => wspace, + :task => args[:task], + :host => host, + :proto => proto, + :port => port + } + if name != "unidentified" + info[:name] = name + end + service = find_or_create_service(info) + end + end + + def import_ci_noko_stream(args, &block) + if block + doc = Rex::Parser::CIDocument.new(args,framework.db) {|type, data| yield type,data } + else + doc = Rex::Parser::CI.new(args,self) + end + parser = ::Nokogiri::XML::SAX::Parser.new(doc) + parser.parse(args[:data]) + end + + def import_outpost24_xml(args={}, &block) + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + wspace = args[:wspace] || workspace + if Rex::Parser.nokogiri_loaded + parser = "Nokogiri v#{::Nokogiri::VERSION}" + noko_args = args.dup + noko_args[:blacklist] = bl + noko_args[:wspace] = wspace + if block + yield(:parser, parser) + import_outpost24_noko_stream(noko_args) {|type, data| yield type,data} + else + import_outpost24_noko_stream(noko_args) + end + return true + else # Sorry + raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.") + end + end + + def import_outpost24_noko_stream(args={},&block) + if block + doc = Rex::Parser::Outpost24Document.new(args,framework.db) {|type, data| yield type,data } + else + doc = Rex::Parser::Outpost24Document.new(args,self) + end + parser = ::Nokogiri::XML::SAX::Parser.new(doc) + parser.parse(args[:data]) + end + + + def unserialize_object(xml_elem, allow_yaml = false) + return nil unless xml_elem + string = xml_elem.text.to_s.strip + return string unless string.is_a?(String) + return nil if (string.empty? || string.nil?) + + begin + # Validate that it is properly formed base64 first + if string.gsub(/\s+/, '') =~ /^([a-z0-9A-Z\+\/=]+)$/ + Marshal.load($1.unpack("m")[0]) + else + if allow_yaml + begin + YAML.load(string) + rescue + dlog("Badly formatted YAML: '#{string}'") + string + end + else + string + end + end + rescue ::Exception => e + if allow_yaml + YAML.load(string) rescue string + else + string + end + end + end + + # + # Returns something suitable for the +:host+ parameter to the various report_* methods + # + # Takes a Host object, a Session object, an Msf::Session object or a String + # address + # + def normalize_host(host) + return host if host.kind_of? ::Mdm::Host + norm_host = nil + + if (host.kind_of? String) + + if Rex::Socket.is_ipv4?(host) + # If it's an IPv4 addr with a port on the end, strip the port + if host =~ /((\d{1,3}\.){3}\d{1,3}):\d+/ + norm_host = $1 + else + norm_host = host + end + elsif Rex::Socket.is_ipv6?(host) + # If it's an IPv6 addr, drop the scope + address, scope = host.split('%', 2) + norm_host = address + else + norm_host = Rex::Socket.getaddress(host, true) + end + elsif host.kind_of? ::Mdm::Session + norm_host = host.host + elsif host.respond_to?(:session_host) + # Then it's an Msf::Session object + thost = host.session_host + norm_host = thost + end + + # If we got here and don't have a norm_host yet, it could be a + # Msf::Session object with an empty or nil tunnel_host and tunnel_peer; + # see if it has a socket and use its peerhost if so. + if ( + norm_host.nil? and + host.respond_to?(:sock) and + host.sock.respond_to?(:peerhost) and + host.sock.peerhost.to_s.length > 0 + ) + norm_host = session.sock.peerhost + end + # If We got here and still don't have a real host, there's nothing left + # to try, just log it and return what we were given + if not norm_host + dlog("Host could not be normalized: #{host.inspect}") + norm_host = host + end + + norm_host + end + + # A way to sneak the yield back into the db importer. + # Used by the SAX parsers. + def emit(sym,data,&block) + yield(sym,data) + end + +protected + + # + # This holds all of the shared parsing/handling used by the + # Nessus NBE and NESSUS v1 methods + # + def handle_nessus(wspace, hobj, port, nasl, plugin_name, severity, data,task=nil) + addr = hobj.address + # The port section looks like: + # http (80/tcp) + p = port.match(/^([^\(]+)\((\d+)\/([^\)]+)\)/) + return if not p + + # Unnecessary as the caller should already have reported this host + #report_host(:workspace => wspace, :host => addr, :state => Msf::HostState::Alive) + name = p[1].strip + port = p[2].to_i + proto = p[3].downcase + + info = { :workspace => wspace, :host => hobj, :port => port, :proto => proto, :task => task } + if name != "unknown" and name[-1,1] != "?" + info[:name] = name + end + report_service(info) + + if nasl.nil? || nasl.empty? || nasl == 0 || nasl == "0" + return + end + + data.gsub!("\\n", "\n") + + refs = [] + + if (data =~ /^CVE : (.*)$/) + $1.gsub(/C(VE|AN)\-/, '').split(',').map { |r| r.strip }.each do |r| + refs.push('CVE-' + r) + end + end + + if (data =~ /^BID : (.*)$/) + $1.split(',').map { |r| r.strip }.each do |r| + refs.push('BID-' + r) + end + end + + if (data =~ /^Other references : (.*)$/) + $1.split(',').map { |r| r.strip }.each do |r| + ref_id, ref_val = r.split(':') + ref_val ? refs.push(ref_id + '-' + ref_val) : refs.push(ref_id) + end + end + + nss = 'NSS-' + nasl.to_s.strip + refs << nss + + unless plugin_name.to_s.strip.empty? + vuln_name = plugin_name + else + vuln_name = nss + end + + vuln_info = { + :workspace => wspace, + :host => hobj, + :port => port, + :proto => proto, + :name => vuln_name, + :info => data, + :refs => refs, + :task => task, + } + report_vuln(vuln_info) + end + + # + # NESSUS v2 file format has a dramatically different layout + # for ReportItem data + # + def handle_nessus_v2(wspace,hobj,port,proto,name,nasl,nasl_name,severity,description,cve,bid,xref,msf,task=nil) + addr = hobj.address + + info = { :workspace => wspace, :host => hobj, :port => port, :proto => proto, :task => task } + + unless name =~ /^unknown$|\?$/ + info[:name] = name + end + + if port.to_i != 0 + report_service(info) + end + + if nasl.nil? || nasl.empty? || nasl == 0 || nasl == "0" + return + end + + refs = [] + + cve.each do |r| + r.to_s.gsub!(/C(VE|AN)\-/, '') + refs.push('CVE-' + r.to_s) + end if cve + + bid.each do |r| + refs.push('BID-' + r.to_s) + end if bid + + xref.each do |r| + ref_id, ref_val = r.to_s.split(':') + ref_val ? refs.push(ref_id + '-' + ref_val) : refs.push(ref_id) + end if xref + + msfref = "MSF-" << msf if msf + refs.push msfref if msfref + + nss = 'NSS-' + nasl + if nasl_name.nil? || nasl_name.empty? + vuln_name = nss + else + vuln_name = nasl_name + end + + refs << nss.strip + + vuln = { + :workspace => wspace, + :host => hobj, + :name => vuln_name, + :info => description ? description : "", + :refs => refs, + :task => task, + } + + if port.to_i != 0 + vuln[:port] = port + vuln[:proto] = proto + end + + report_vuln(vuln) + end + + # + # IP360 v3 vuln + # + def handle_ip360_v3_svc(wspace,hobj,port,proto,hname,task=nil) + addr = hobj.address + report_host(:workspace => wspace, :host => hobj, :state => Msf::HostState::Alive, :task => task) + + info = { :workspace => wspace, :host => hobj, :port => port, :proto => proto, :task => task } + if hname != "unknown" and hname[-1,1] != "?" + info[:name] = hname + end + + if port.to_i != 0 + report_service(info) + end + end #handle_ip360_v3_svc + + # + # IP360 v3 vuln + # + def handle_ip360_v3_vuln(wspace,hobj,port,proto,hname,vulnid,vulnname,cves,bids,task=nil) + info = { :workspace => wspace, :host => hobj, :port => port, :proto => proto, :task => task } + if hname != "unknown" and hname[-1,1] != "?" + info[:name] = hname + end + + if port.to_i != 0 + report_service(info) + end + + refs = [] + + cves.split(/,/).each do |cve| + refs.push(cve.to_s) + end if cves + + bids.split(/,/).each do |bid| + refs.push('BID-' + bid.to_s) + end if bids + + description = nil # not working yet + vuln = { + :workspace => wspace, + :host => hobj, + :name => vulnname, + :info => description ? description : "", + :refs => refs, + :task => task + } + + if port.to_i != 0 + vuln[:port] = port + vuln[:proto] = proto + end + + report_vuln(vuln) + end #handle_ip360_v3_vuln + + # + # Qualys report parsing/handling + # + def handle_qualys(wspace, hobj, port, protocol, qid, severity, refs, name=nil, title=nil, task=nil) + addr = hobj.address + port = port.to_i if port + + info = { :workspace => wspace, :host => hobj, :port => port, :proto => protocol, :task => task } + if name and name != 'unknown' and name != 'No registered hostname' + info[:name] = name + end + + if info[:host] && info[:port] && info[:proto] + report_service(info) + end + + fixed_refs = [] + if refs + refs.each do |ref| + case ref + when /^MS[0-9]{2}-[0-9]{3}/ + fixed_refs << "MSB-#{ref}" + else + fixed_refs << ref + end + end + end + + return if qid == 0 + title = 'QUALYS-' + qid if title.nil? or title.empty? + if addr + report_vuln( + :workspace => wspace, + :task => task, + :host => hobj, + :port => port, + :proto => protocol, + :name => title, + :refs => fixed_refs + ) + end + end + + def process_nexpose_data_sxml_refs(vuln) + refs = [] + vid = vuln.attributes['id'].to_s.downcase + vry = vuln.attributes['resultCode'].to_s.upcase + + # Only process vuln-exploitable and vuln-version statuses + return if vry !~ /^V[VE]$/ + + refs = [] + vuln.elements.each('id') do |ref| + rtyp = ref.attributes['type'].to_s.upcase + rval = ref.text.to_s.strip + case rtyp + when 'CVE' + refs << rval.gsub('CAN', 'CVE') + when 'MS' # obsolete? + refs << "MSB-MS-#{rval}" + else + refs << "#{rtyp}-#{rval}" + end + end + + refs << "NEXPOSE-#{vid}" + refs + end + end end From a054259ee5d103963960b12ec7abe3bb3cdf4b2a Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 8 Oct 2014 15:26:28 -0500 Subject: [PATCH 012/159] Extract Msf::DBManager::Sink MSP-11124 Extract attributes and methods associated with the deprecated sink. --- lib/msf/core/db_manager.rb | 24 +++++------------------- lib/msf/core/db_manager/sink.rb | 27 +++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 19 deletions(-) create mode 100644 lib/msf/core/db_manager/sink.rb diff --git a/lib/msf/core/db_manager.rb b/lib/msf/core/db_manager.rb index 2301f339a9..e138de9779 100644 --- a/lib/msf/core/db_manager.rb +++ b/lib/msf/core/db_manager.rb @@ -75,11 +75,15 @@ module Msf class DBManager extend Metasploit::Framework::Require + autoload :Sink, 'msf/core/db_manager/sink' + include Msf::DBManager::ImportMsfXml optionally_include_metasploit_credential_creation - # Provides :framework and other accessors include Msf::DBManager::Migration + include Msf::DBManager::Sink + + # Provides :framework and other accessors include Msf::Framework::Offspring # @@ -121,9 +125,6 @@ class DBManager # Stores the error message for why the db was not loaded attr_accessor :error - # Stores a TaskManager for serializing database events - attr_accessor :sink - # Flag to indicate that modules are cached attr_accessor :modules_cached @@ -245,21 +246,6 @@ class DBManager end end - # - # Create a new database sink and initialize it - # - def initialize_sink - self.sink = TaskManager.new(framework) - self.sink.start - end - - # - # Add a new task to the sink - # - def queue(proc) - self.sink.queue_proc(proc) - end - # # Connects this instance to a database # diff --git a/lib/msf/core/db_manager/sink.rb b/lib/msf/core/db_manager/sink.rb new file mode 100644 index 0000000000..de14452484 --- /dev/null +++ b/lib/msf/core/db_manager/sink.rb @@ -0,0 +1,27 @@ +module Msf::DBManager::Sink + # + # Attributes + # + + # Stores a TaskManager for serializing database events + attr_accessor :sink + + # + # Instance Methods + # + + # + # Create a new database sink and initialize it + # + def initialize_sink + self.sink = Msf::TaskManager.new(framework) + self.sink.start + end + + # + # Add a new task to the sink + # + def queue(proc) + self.sink.queue_proc(proc) + end +end \ No newline at end of file From 37e0b9ffe0c4fa8b31b6b558f69a83835712b70f Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 8 Oct 2014 15:30:20 -0500 Subject: [PATCH 013/159] Extract Msf::DBManagerSink examples to shared example MSP-11124 --- spec/lib/msf/db_manager_spec.rb | 7 ++----- spec/support/shared/examples/msf/db_manager/sink.rb | 6 ++++++ 2 files changed, 8 insertions(+), 5 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/sink.rb diff --git a/spec/lib/msf/db_manager_spec.rb b/spec/lib/msf/db_manager_spec.rb index 790f1206b3..b5209c2c59 100644 --- a/spec/lib/msf/db_manager_spec.rb +++ b/spec/lib/msf/db_manager_spec.rb @@ -18,8 +18,9 @@ describe Msf::DBManager do db_manager end - it_should_behave_like 'Msf::DBManager::Migration' it_should_behave_like 'Msf::DBManager::ImportMsfXml' + it_should_behave_like 'Msf::DBManager::Migration' + it_should_behave_like 'Msf::DBManager::Sink' context 'CONSTANTS' do context 'ADAPTER' do @@ -180,7 +181,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :import_wapiti_xml_file } it { is_expected.to respond_to :initialize_adapter } it { is_expected.to respond_to :initialize_database_support } - it { is_expected.to respond_to :initialize_sink } it { is_expected.to respond_to :inspect_single_packet } it { is_expected.to respond_to :inspect_single_packet_http } it { is_expected.to respond_to :ipv46_validator } @@ -270,8 +270,6 @@ describe Msf::DBManager do end end - it { is_expected.to respond_to :queue } - context '#remove_module_details' do def remove_module_details db_manager.remove_module_details(mtype, refname) @@ -1427,7 +1425,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :selected_wmap_target } it { is_expected.to respond_to :service_name_map } it { is_expected.to respond_to :services } - it { is_expected.to respond_to :sink } it { is_expected.to respond_to :sql_query } it { is_expected.to respond_to :sync } it { is_expected.to respond_to :target_requests } diff --git a/spec/support/shared/examples/msf/db_manager/sink.rb b/spec/support/shared/examples/msf/db_manager/sink.rb new file mode 100644 index 0000000000..f7e94a0520 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/sink.rb @@ -0,0 +1,6 @@ +shared_examples_for 'Msf::DBManager::Sink' do + it { is_expected.to respond_to :initialize_sink } + it { is_expected.to respond_to :queue } + it { is_expected.to respond_to :sink } + it { is_expected.to respond_to :sink= } +end \ No newline at end of file From a64036f6cfc507b081eefa49516fc8e6ae85b1ef Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 8 Oct 2014 15:38:56 -0500 Subject: [PATCH 014/159] Move Msf::DBManager#sync to Msf::DBManager::Sink MSP-11124 The comment on `#sync` says it's related to `sink`, so move it into its Module. --- lib/msf/core/db_manager.rb | 7 ------- lib/msf/core/db_manager/sink.rb | 7 +++++++ spec/lib/msf/db_manager_spec.rb | 1 - spec/support/shared/examples/msf/db_manager/sink.rb | 1 + 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/msf/core/db_manager.rb b/lib/msf/core/db_manager.rb index e138de9779..12f30ba40d 100644 --- a/lib/msf/core/db_manager.rb +++ b/lib/msf/core/db_manager.rb @@ -872,13 +872,6 @@ class DBManager } end - # - # Wait for all pending write to finish - # - def sync - # There is no more queue. - end - # # Find a host. Performs no database writes. # diff --git a/lib/msf/core/db_manager/sink.rb b/lib/msf/core/db_manager/sink.rb index de14452484..25d05baee5 100644 --- a/lib/msf/core/db_manager/sink.rb +++ b/lib/msf/core/db_manager/sink.rb @@ -24,4 +24,11 @@ module Msf::DBManager::Sink def queue(proc) self.sink.queue_proc(proc) end + + # + # Wait for all pending write to finish + # + def sync + # There is no more queue. + end end \ No newline at end of file diff --git a/spec/lib/msf/db_manager_spec.rb b/spec/lib/msf/db_manager_spec.rb index b5209c2c59..cdac4d3920 100644 --- a/spec/lib/msf/db_manager_spec.rb +++ b/spec/lib/msf/db_manager_spec.rb @@ -1426,7 +1426,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :service_name_map } it { is_expected.to respond_to :services } it { is_expected.to respond_to :sql_query } - it { is_expected.to respond_to :sync } it { is_expected.to respond_to :target_requests } it { is_expected.to respond_to :targets } it { is_expected.to respond_to :tasks } diff --git a/spec/support/shared/examples/msf/db_manager/sink.rb b/spec/support/shared/examples/msf/db_manager/sink.rb index f7e94a0520..af9eb81a46 100644 --- a/spec/support/shared/examples/msf/db_manager/sink.rb +++ b/spec/support/shared/examples/msf/db_manager/sink.rb @@ -3,4 +3,5 @@ shared_examples_for 'Msf::DBManager::Sink' do it { is_expected.to respond_to :queue } it { is_expected.to respond_to :sink } it { is_expected.to respond_to :sink= } + it { is_expected.to respond_to :sync } end \ No newline at end of file From ee0de997d532a8d7cbf4c90b2156e5748d22db58 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 8 Oct 2014 15:46:35 -0500 Subject: [PATCH 015/159] Extract Msf::DBManager::Workspace MSP-11124 Gather together all workspace related methods into `Msf::DBManager::Workspace` and include it in `Msf::DBManager`. --- lib/msf/core/db_manager.rb | 39 ++-------------------------- lib/msf/core/db_manager/workspace.rb | 36 +++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 37 deletions(-) create mode 100644 lib/msf/core/db_manager/workspace.rb diff --git a/lib/msf/core/db_manager.rb b/lib/msf/core/db_manager.rb index 12f30ba40d..deb1655e93 100644 --- a/lib/msf/core/db_manager.rb +++ b/lib/msf/core/db_manager.rb @@ -76,12 +76,14 @@ class DBManager extend Metasploit::Framework::Require autoload :Sink, 'msf/core/db_manager/sink' + autoload :Workspace, 'msf/core/db_manager/workspace' include Msf::DBManager::ImportMsfXml optionally_include_metasploit_credential_creation include Msf::DBManager::Migration include Msf::DBManager::Sink + include Msf::DBManager::Workspace # Provides :framework and other accessors include Msf::Framework::Offspring @@ -369,15 +371,6 @@ class DBManager end end - def workspace=(workspace) - @workspace_name = workspace.name - end - - def workspace - framework.db.find_workspace(@workspace_name) - end - - # @note Does nothing unless {#migrated} is +true+ and {#modules_caching} is # +false+. # @@ -844,34 +837,6 @@ class DBManager } end - - def default_workspace - ::ActiveRecord::Base.connection_pool.with_connection { - ::Mdm::Workspace.default - } - end - - def find_workspace(name) - ::ActiveRecord::Base.connection_pool.with_connection { - ::Mdm::Workspace.find_by_name(name) - } - end - - # - # Creates a new workspace in the database - # - def add_workspace(name) - ::ActiveRecord::Base.connection_pool.with_connection { - ::Mdm::Workspace.find_or_create_by_name(name) - } - end - - def workspaces - ::ActiveRecord::Base.connection_pool.with_connection { - ::Mdm::Workspace.find(:all) - } - end - # # Find a host. Performs no database writes. # diff --git a/lib/msf/core/db_manager/workspace.rb b/lib/msf/core/db_manager/workspace.rb new file mode 100644 index 0000000000..65db179809 --- /dev/null +++ b/lib/msf/core/db_manager/workspace.rb @@ -0,0 +1,36 @@ +module Msf::DBManager::Workspace + # + # Creates a new workspace in the database + # + def add_workspace(name) + ::ActiveRecord::Base.connection_pool.with_connection { + ::Mdm::Workspace.find_or_create_by_name(name) + } + end + + def default_workspace + ::ActiveRecord::Base.connection_pool.with_connection { + ::Mdm::Workspace.default + } + end + + def find_workspace(name) + ::ActiveRecord::Base.connection_pool.with_connection { + ::Mdm::Workspace.find_by_name(name) + } + end + + def workspace + framework.db.find_workspace(@workspace_name) + end + + def workspace=(workspace) + @workspace_name = workspace.name + end + + def workspaces + ::ActiveRecord::Base.connection_pool.with_connection { + ::Mdm::Workspace.find(:all) + } + end +end From 70d4f672e2678ef1cce1b8e69ee353ae1df2f4fe Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 8 Oct 2014 15:54:29 -0500 Subject: [PATCH 016/159] Extract Msf::DBManager::Workspace examples to shared example MSP-11124 --- spec/lib/msf/db_manager_spec.rb | 7 +------ spec/support/shared/examples/msf/db_manager/workspace.rb | 8 ++++++++ 2 files changed, 9 insertions(+), 6 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/workspace.rb diff --git a/spec/lib/msf/db_manager_spec.rb b/spec/lib/msf/db_manager_spec.rb index cdac4d3920..647f85a0f9 100644 --- a/spec/lib/msf/db_manager_spec.rb +++ b/spec/lib/msf/db_manager_spec.rb @@ -21,6 +21,7 @@ describe Msf::DBManager do it_should_behave_like 'Msf::DBManager::ImportMsfXml' it_should_behave_like 'Msf::DBManager::Migration' it_should_behave_like 'Msf::DBManager::Sink' + it_should_behave_like 'Msf::DBManager::Workspace' context 'CONSTANTS' do context 'ADAPTER' do @@ -52,7 +53,6 @@ describe Msf::DBManager do end end - it { is_expected.to respond_to :add_workspace } it { is_expected.to respond_to :after_establish_connection } it { is_expected.to respond_to :check } it { is_expected.to respond_to :connect } @@ -61,7 +61,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :create_request } it { is_expected.to respond_to :create_target } it { is_expected.to respond_to :creds } - it { is_expected.to respond_to :default_workspace } it { is_expected.to respond_to :dehex } it { is_expected.to respond_to :del_host } it { is_expected.to respond_to :del_service } @@ -104,7 +103,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :find_qualys_asset_vulns } it { is_expected.to respond_to :find_vuln_by_details } it { is_expected.to respond_to :find_vuln_by_refs } - it { is_expected.to respond_to :find_workspace } it { is_expected.to respond_to :get_client } it { is_expected.to respond_to :get_host } it { is_expected.to respond_to :get_ref } @@ -2012,7 +2010,4 @@ describe Msf::DBManager do it { is_expected.to respond_to :validate_ips } it { is_expected.to respond_to :vulns } it { is_expected.to respond_to :warn_about_rubies } - it { is_expected.to respond_to :workspace } - it { is_expected.to respond_to :workspace= } - it { is_expected.to respond_to :workspaces } end diff --git a/spec/support/shared/examples/msf/db_manager/workspace.rb b/spec/support/shared/examples/msf/db_manager/workspace.rb new file mode 100644 index 0000000000..873cd3c691 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/workspace.rb @@ -0,0 +1,8 @@ +shared_examples_for 'Msf::DBManager::Workspace' do + it { is_expected.to respond_to :add_workspace } + it { is_expected.to respond_to :default_workspace } + it { is_expected.to respond_to :find_workspace } + it { is_expected.to respond_to :workspace } + it { is_expected.to respond_to :workspace= } + it { is_expected.to respond_to :workspaces } +end \ No newline at end of file From d4a94366a6cc4839bb05c541fb184efb1cc8682a Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Thu, 9 Oct 2014 08:53:41 -0500 Subject: [PATCH 017/159] Extract Msf::DBManager::ModuleCache MSP-11124 Extract methods related to the module cache state and maintenance to `Msf::DBManager::ModuleCache`. --- lib/msf/core/db_manager.rb | 391 +---------------------- lib/msf/core/db_manager/module_cache.rb | 397 ++++++++++++++++++++++++ 2 files changed, 399 insertions(+), 389 deletions(-) create mode 100644 lib/msf/core/db_manager/module_cache.rb diff --git a/lib/msf/core/db_manager.rb b/lib/msf/core/db_manager.rb index deb1655e93..222a1c05c0 100644 --- a/lib/msf/core/db_manager.rb +++ b/lib/msf/core/db_manager.rb @@ -75,6 +75,7 @@ module Msf class DBManager extend Metasploit::Framework::Require + autoload :ModuleCache, 'msf/core/db_manager/module_cache' autoload :Sink, 'msf/core/db_manager/sink' autoload :Workspace, 'msf/core/db_manager/workspace' @@ -82,6 +83,7 @@ class DBManager optionally_include_metasploit_credential_creation include Msf::DBManager::Migration + include Msf::DBManager::ModuleCache include Msf::DBManager::Sink include Msf::DBManager::Workspace @@ -127,12 +129,6 @@ class DBManager # Stores the error message for why the db was not loaded attr_accessor :error - # Flag to indicate that modules are cached - attr_accessor :modules_cached - - # Flag to indicate that the module cacher is running - attr_accessor :modules_caching - def initialize(framework, opts = {}) self.framework = framework @@ -371,237 +367,6 @@ class DBManager end end - # @note Does nothing unless {#migrated} is +true+ and {#modules_caching} is - # +false+. - # - # Destroys all Mdm::Module::Details in the database. - # - # @return [void] - def purge_all_module_details - return if not self.migrated - return if self.modules_caching - - ::ActiveRecord::Base.connection_pool.with_connection do - Mdm::Module::Detail.destroy_all - end - end - - # Destroys the old Mdm::Module::Detail and creates a new Mdm::Module::Detail for - # any module with an Mdm::Module::Detail where the modification time of the - # Mdm::Module::Detail#file differs from the Mdm::Module::Detail#mtime. If the - # Mdm::Module::Detail#file no only exists on disk, then the Mdm::Module::Detail - # is just destroyed without a new one being created. - # - # @return [void] - def update_all_module_details - return if not self.migrated - return if self.modules_caching - - self.framework.cache_thread = Thread.current - - self.modules_cached = false - self.modules_caching = true - - ActiveRecord::Base.connection_pool.with_connection do - - refresh = [] - skip_reference_name_set_by_module_type = Hash.new { |hash, module_type| - hash[module_type] = Set.new - } - - Mdm::Module::Detail.find_each do |md| - - unless md.ready - refresh << md - next - end - - unless md.file and ::File.exists?(md.file) - refresh << md - next - end - - if ::File.mtime(md.file).to_i != md.mtime.to_i - refresh << md - next - end - - skip_reference_name_set = skip_reference_name_set_by_module_type[md.mtype] - skip_reference_name_set.add(md.refname) - end - - refresh.each { |md| md.destroy } - - [ - ['exploit', framework.exploits], - ['auxiliary', framework.auxiliary], - ['post', framework.post], - ['payload', framework.payloads], - ['encoder', framework.encoders], - ['nop', framework.nops] - ].each do |mt| - skip_reference_name_set = skip_reference_name_set_by_module_type[mt[0]] - - mt[1].keys.sort.each do |mn| - next if skip_reference_name_set.include? mn - obj = mt[1].create(mn) - next if not obj - begin - update_module_details(obj) - rescue ::Exception - elog("Error updating module details for #{obj.fullname}: #{$!.class} #{$!}") - end - end - end - - self.framework.cache_initialized = true - end - - # in reverse order of section before with_connection block - self.modules_caching = false - self.modules_cached = true - self.framework.cache_thread = nil - end - - # Creates an Mdm::Module::Detail from a module instance. - # - # @param module_instance [Msf::Module] a metasploit module instance. - # @raise [ActiveRecord::RecordInvalid] if Hash from {#module_to_details_hash} is invalid attributes for - # Mdm::Module::Detail. - # @return [void] - def update_module_details(module_instance) - return if not self.migrated - - ActiveRecord::Base.connection_pool.with_connection do - info = module_to_details_hash(module_instance) - bits = info.delete(:bits) || [] - module_detail = Mdm::Module::Detail.create!(info) - - bits.each do |args| - otype, vals = args - - case otype - when :action - module_detail.add_action(vals[:name]) - when :arch - module_detail.add_arch(vals[:name]) - when :author - module_detail.add_author(vals[:name], vals[:email]) - when :platform - module_detail.add_platform(vals[:name]) - when :ref - module_detail.add_ref(vals[:name]) - when :target - module_detail.add_target(vals[:index], vals[:name]) - end - end - - module_detail.ready = true - module_detail.save! - end - end - - # Destroys Mdm::Module::Detail if one exists for the given - # Mdm::Module::Detail#mtype and Mdm::Module::Detail#refname. - # - # @param mtype [String] module type. - # @param refname [String] module reference name. - # @return [void] - def remove_module_details(mtype, refname) - return if not self.migrated - - ActiveRecord::Base.connection_pool.with_connection do - Mdm::Module::Detail.where(:mtype => mtype, :refname => refname).destroy_all - end - end - - def module_to_details_hash(m) - res = {} - bits = [] - - res[:mtime] = ::File.mtime(m.file_path) rescue Time.now - res[:file] = m.file_path - res[:mtype] = m.type - res[:name] = m.name.to_s - res[:refname] = m.refname - res[:fullname] = m.fullname - res[:rank] = m.rank.to_i - res[:license] = m.license.to_s - - res[:description] = m.description.to_s.strip - - m.arch.map{ |x| - bits << [ :arch, { :name => x.to_s } ] - } - - m.platform.platforms.map{ |x| - bits << [ :platform, { :name => x.to_s.split('::').last.downcase } ] - } - - m.author.map{|x| - bits << [ :author, { :name => x.to_s } ] - } - - m.references.map do |r| - bits << [ :ref, { :name => [r.ctx_id.to_s, r.ctx_val.to_s].join("-") } ] - end - - res[:privileged] = m.privileged? - - - if m.disclosure_date - begin - res[:disclosure_date] = m.disclosure_date.to_datetime.to_time - rescue ::Exception - res.delete(:disclosure_date) - end - end - - if(m.type == "exploit") - - m.targets.each_index do |i| - bits << [ :target, { :index => i, :name => m.targets[i].name.to_s } ] - if m.targets[i].platform - m.targets[i].platform.platforms.each do |name| - bits << [ :platform, { :name => name.to_s.split('::').last.downcase } ] - end - end - if m.targets[i].arch - bits << [ :arch, { :name => m.targets[i].arch.to_s } ] - end - end - - if (m.default_target) - res[:default_target] = m.default_target - end - - # Some modules are a combination, which means they are actually aggressive - res[:stance] = m.stance.to_s.index("aggressive") ? "aggressive" : "passive" - - - m.class.mixins.each do |x| - bits << [ :mixin, { :name => x.to_s } ] - end - end - - if(m.type == "auxiliary") - - m.actions.each_index do |i| - bits << [ :action, { :name => m.actions[i].name.to_s } ] - end - - if (m.default_action) - res[:default_action] = m.default_action.to_s - end - - res[:stance] = m.passive? ? "passive" : "aggressive" - end - - res[:bits] = bits.uniq - - res - end - # Wraps values in +'%'+ for Arel::Prediciation#matches_any and other match* methods that map to SQL +'LIKE'+ or # +'ILIKE'+ # @@ -615,158 +380,6 @@ class DBManager wrapped_values end - # This provides a standard set of search filters for every module. - # - # Supported keywords with the format :: - # +app+:: If +client+ then matches +'passive'+ stance modules, otherwise matches +'active' stance modules. - # +author+:: Matches modules with the given author email or name. - # +bid+:: Matches modules with the given Bugtraq ID. - # +cve+:: Matches modules with the given CVE ID. - # +edb+:: Matches modules with the given Exploit-DB ID. - # +name+:: Matches modules with the given full name or name. - # +os+, +platform+:: Matches modules with the given platform or target name. - # +osvdb+:: Matches modules with the given OSVDB ID. - # +ref+:: Matches modules with the given reference ID. - # +type+:: Matches modules with the given type. - # - # Any text not associated with a keyword is matched against the description, - # the full name, and the name of the module; the name of the module actions; - # the name of the module archs; the name of the module authors; the name of - # module platform; the module refs; or the module target. - # - # @param search_string [String] a string of space separated keyword pairs or - # free form text. - # @return [[]] if search_string is +nil+ - # @return [ActiveRecord::Relation] module details that matched - # +search_string+ - def search_modules(search_string) - search_string ||= '' - search_string += " " - - # Split search terms by space, but allow quoted strings - terms = Shellwords.shellwords(search_string) - terms.delete('') - - # All terms are either included or excluded - value_set_by_keyword = Hash.new { |hash, keyword| - hash[keyword] = Set.new - } - - terms.each do |term| - keyword, value = term.split(':', 2) - - unless value - value = keyword - keyword = 'text' - end - - unless value.empty? - keyword.downcase! - - value_set = value_set_by_keyword[keyword] - value_set.add value - end - end - - query = Mdm::Module::Detail.scoped - - ActiveRecord::Base.connection_pool.with_connection do - # Although AREL supports taking the union or two queries, the ActiveRecord where syntax only supports - # intersection, so creating the where clause has to be delayed until all conditions can be or'd together and - # passed to one call ot where. - union_conditions = [] - - value_set_by_keyword.each do |keyword, value_set| - case keyword - when 'author' - formatted_values = match_values(value_set) - - query = query.includes(:authors) - module_authors = Mdm::Module::Author.arel_table - union_conditions << module_authors[:email].matches_any(formatted_values) - union_conditions << module_authors[:name].matches_any(formatted_values) - when 'name' - formatted_values = match_values(value_set) - - module_details = Mdm::Module::Detail.arel_table - union_conditions << module_details[:fullname].matches_any(formatted_values) - union_conditions << module_details[:name].matches_any(formatted_values) - when 'os', 'platform' - formatted_values = match_values(value_set) - - query = query.includes(:platforms) - union_conditions << Mdm::Module::Platform.arel_table[:name].matches_any(formatted_values) - - query = query.includes(:targets) - union_conditions << Mdm::Module::Target.arel_table[:name].matches_any(formatted_values) - when 'text' - formatted_values = match_values(value_set) - - module_details = Mdm::Module::Detail.arel_table - union_conditions << module_details[:description].matches_any(formatted_values) - union_conditions << module_details[:fullname].matches_any(formatted_values) - union_conditions << module_details[:name].matches_any(formatted_values) - - query = query.includes(:actions) - union_conditions << Mdm::Module::Action.arel_table[:name].matches_any(formatted_values) - - query = query.includes(:archs) - union_conditions << Mdm::Module::Arch.arel_table[:name].matches_any(formatted_values) - - query = query.includes(:authors) - union_conditions << Mdm::Module::Author.arel_table[:name].matches_any(formatted_values) - - query = query.includes(:platforms) - union_conditions << Mdm::Module::Platform.arel_table[:name].matches_any(formatted_values) - - query = query.includes(:refs) - union_conditions << Mdm::Module::Ref.arel_table[:name].matches_any(formatted_values) - - query = query.includes(:targets) - union_conditions << Mdm::Module::Target.arel_table[:name].matches_any(formatted_values) - when 'type' - formatted_values = match_values(value_set) - union_conditions << Mdm::Module::Detail.arel_table[:mtype].matches_any(formatted_values) - when 'app' - formatted_values = value_set.collect { |value| - formatted_value = 'aggressive' - - if value == 'client' - formatted_value = 'passive' - end - - formatted_value - } - - union_conditions << Mdm::Module::Detail.arel_table[:stance].eq_any(formatted_values) - when 'ref' - formatted_values = match_values(value_set) - - query = query.includes(:refs) - union_conditions << Mdm::Module::Ref.arel_table[:name].matches_any(formatted_values) - when 'cve', 'bid', 'osvdb', 'edb' - formatted_values = value_set.collect { |value| - prefix = keyword.upcase - - "#{prefix}-%#{value}%" - } - - query = query.includes(:refs) - union_conditions << Mdm::Module::Ref.arel_table[:name].matches_any(formatted_values) - end - end - - unioned_conditions = union_conditions.inject { |union, condition| - union.or(condition) - } - - query = query.where(unioned_conditions).to_a.uniq { |m| m.fullname } - end - - query - end - - def rfc3330_reserved(ip) case ip.class.to_s when "PacketFu::Octets" diff --git a/lib/msf/core/db_manager/module_cache.rb b/lib/msf/core/db_manager/module_cache.rb new file mode 100644 index 0000000000..73d76aa934 --- /dev/null +++ b/lib/msf/core/db_manager/module_cache.rb @@ -0,0 +1,397 @@ +module Msf::DBManager::ModuleCache + # + # Attributes + # + + # Flag to indicate that modules are cached + attr_accessor :modules_cached + + # Flag to indicate that the module cacher is running + attr_accessor :modules_caching + + # + # Instance Methods + # + + def module_to_details_hash(m) + res = {} + bits = [] + + res[:mtime] = ::File.mtime(m.file_path) rescue Time.now + res[:file] = m.file_path + res[:mtype] = m.type + res[:name] = m.name.to_s + res[:refname] = m.refname + res[:fullname] = m.fullname + res[:rank] = m.rank.to_i + res[:license] = m.license.to_s + + res[:description] = m.description.to_s.strip + + m.arch.map{ |x| + bits << [ :arch, { :name => x.to_s } ] + } + + m.platform.platforms.map{ |x| + bits << [ :platform, { :name => x.to_s.split('::').last.downcase } ] + } + + m.author.map{|x| + bits << [ :author, { :name => x.to_s } ] + } + + m.references.map do |r| + bits << [ :ref, { :name => [r.ctx_id.to_s, r.ctx_val.to_s].join("-") } ] + end + + res[:privileged] = m.privileged? + + + if m.disclosure_date + begin + res[:disclosure_date] = m.disclosure_date.to_datetime.to_time + rescue ::Exception + res.delete(:disclosure_date) + end + end + + if(m.type == "exploit") + + m.targets.each_index do |i| + bits << [ :target, { :index => i, :name => m.targets[i].name.to_s } ] + if m.targets[i].platform + m.targets[i].platform.platforms.each do |name| + bits << [ :platform, { :name => name.to_s.split('::').last.downcase } ] + end + end + if m.targets[i].arch + bits << [ :arch, { :name => m.targets[i].arch.to_s } ] + end + end + + if (m.default_target) + res[:default_target] = m.default_target + end + + # Some modules are a combination, which means they are actually aggressive + res[:stance] = m.stance.to_s.index("aggressive") ? "aggressive" : "passive" + + + m.class.mixins.each do |x| + bits << [ :mixin, { :name => x.to_s } ] + end + end + + if(m.type == "auxiliary") + + m.actions.each_index do |i| + bits << [ :action, { :name => m.actions[i].name.to_s } ] + end + + if (m.default_action) + res[:default_action] = m.default_action.to_s + end + + res[:stance] = m.passive? ? "passive" : "aggressive" + end + + res[:bits] = bits.uniq + + res + end + + # @note Does nothing unless {#migrated} is +true+ and {#modules_caching} is + # +false+. + # + # Destroys all Mdm::Module::Details in the database. + # + # @return [void] + def purge_all_module_details + return if not self.migrated + return if self.modules_caching + + ::ActiveRecord::Base.connection_pool.with_connection do + Mdm::Module::Detail.destroy_all + end + end + + # Destroys Mdm::Module::Detail if one exists for the given + # Mdm::Module::Detail#mtype and Mdm::Module::Detail#refname. + # + # @param mtype [String] module type. + # @param refname [String] module reference name. + # @return [void] + def remove_module_details(mtype, refname) + return if not self.migrated + + ActiveRecord::Base.connection_pool.with_connection do + Mdm::Module::Detail.where(:mtype => mtype, :refname => refname).destroy_all + end + end + + # This provides a standard set of search filters for every module. + # + # Supported keywords with the format :: + # +app+:: If +client+ then matches +'passive'+ stance modules, otherwise matches +'active' stance modules. + # +author+:: Matches modules with the given author email or name. + # +bid+:: Matches modules with the given Bugtraq ID. + # +cve+:: Matches modules with the given CVE ID. + # +edb+:: Matches modules with the given Exploit-DB ID. + # +name+:: Matches modules with the given full name or name. + # +os+, +platform+:: Matches modules with the given platform or target name. + # +osvdb+:: Matches modules with the given OSVDB ID. + # +ref+:: Matches modules with the given reference ID. + # +type+:: Matches modules with the given type. + # + # Any text not associated with a keyword is matched against the description, + # the full name, and the name of the module; the name of the module actions; + # the name of the module archs; the name of the module authors; the name of + # module platform; the module refs; or the module target. + # + # @param search_string [String] a string of space separated keyword pairs or + # free form text. + # @return [[]] if search_string is +nil+ + # @return [ActiveRecord::Relation] module details that matched + # +search_string+ + def search_modules(search_string) + search_string ||= '' + search_string += " " + + # Split search terms by space, but allow quoted strings + terms = Shellwords.shellwords(search_string) + terms.delete('') + + # All terms are either included or excluded + value_set_by_keyword = Hash.new { |hash, keyword| + hash[keyword] = Set.new + } + + terms.each do |term| + keyword, value = term.split(':', 2) + + unless value + value = keyword + keyword = 'text' + end + + unless value.empty? + keyword.downcase! + + value_set = value_set_by_keyword[keyword] + value_set.add value + end + end + + query = Mdm::Module::Detail.scoped + + ActiveRecord::Base.connection_pool.with_connection do + # Although AREL supports taking the union or two queries, the ActiveRecord where syntax only supports + # intersection, so creating the where clause has to be delayed until all conditions can be or'd together and + # passed to one call ot where. + union_conditions = [] + + value_set_by_keyword.each do |keyword, value_set| + case keyword + when 'author' + formatted_values = match_values(value_set) + + query = query.includes(:authors) + module_authors = Mdm::Module::Author.arel_table + union_conditions << module_authors[:email].matches_any(formatted_values) + union_conditions << module_authors[:name].matches_any(formatted_values) + when 'name' + formatted_values = match_values(value_set) + + module_details = Mdm::Module::Detail.arel_table + union_conditions << module_details[:fullname].matches_any(formatted_values) + union_conditions << module_details[:name].matches_any(formatted_values) + when 'os', 'platform' + formatted_values = match_values(value_set) + + query = query.includes(:platforms) + union_conditions << Mdm::Module::Platform.arel_table[:name].matches_any(formatted_values) + + query = query.includes(:targets) + union_conditions << Mdm::Module::Target.arel_table[:name].matches_any(formatted_values) + when 'text' + formatted_values = match_values(value_set) + + module_details = Mdm::Module::Detail.arel_table + union_conditions << module_details[:description].matches_any(formatted_values) + union_conditions << module_details[:fullname].matches_any(formatted_values) + union_conditions << module_details[:name].matches_any(formatted_values) + + query = query.includes(:actions) + union_conditions << Mdm::Module::Action.arel_table[:name].matches_any(formatted_values) + + query = query.includes(:archs) + union_conditions << Mdm::Module::Arch.arel_table[:name].matches_any(formatted_values) + + query = query.includes(:authors) + union_conditions << Mdm::Module::Author.arel_table[:name].matches_any(formatted_values) + + query = query.includes(:platforms) + union_conditions << Mdm::Module::Platform.arel_table[:name].matches_any(formatted_values) + + query = query.includes(:refs) + union_conditions << Mdm::Module::Ref.arel_table[:name].matches_any(formatted_values) + + query = query.includes(:targets) + union_conditions << Mdm::Module::Target.arel_table[:name].matches_any(formatted_values) + when 'type' + formatted_values = match_values(value_set) + union_conditions << Mdm::Module::Detail.arel_table[:mtype].matches_any(formatted_values) + when 'app' + formatted_values = value_set.collect { |value| + formatted_value = 'aggressive' + + if value == 'client' + formatted_value = 'passive' + end + + formatted_value + } + + union_conditions << Mdm::Module::Detail.arel_table[:stance].eq_any(formatted_values) + when 'ref' + formatted_values = match_values(value_set) + + query = query.includes(:refs) + union_conditions << Mdm::Module::Ref.arel_table[:name].matches_any(formatted_values) + when 'cve', 'bid', 'osvdb', 'edb' + formatted_values = value_set.collect { |value| + prefix = keyword.upcase + + "#{prefix}-%#{value}%" + } + + query = query.includes(:refs) + union_conditions << Mdm::Module::Ref.arel_table[:name].matches_any(formatted_values) + end + end + + unioned_conditions = union_conditions.inject { |union, condition| + union.or(condition) + } + + query = query.where(unioned_conditions).to_a.uniq { |m| m.fullname } + end + + query + end + + # Destroys the old Mdm::Module::Detail and creates a new Mdm::Module::Detail for + # any module with an Mdm::Module::Detail where the modification time of the + # Mdm::Module::Detail#file differs from the Mdm::Module::Detail#mtime. If the + # Mdm::Module::Detail#file no only exists on disk, then the Mdm::Module::Detail + # is just destroyed without a new one being created. + # + # @return [void] + def update_all_module_details + return if not self.migrated + return if self.modules_caching + + self.framework.cache_thread = Thread.current + + self.modules_cached = false + self.modules_caching = true + + ActiveRecord::Base.connection_pool.with_connection do + + refresh = [] + skip_reference_name_set_by_module_type = Hash.new { |hash, module_type| + hash[module_type] = Set.new + } + + Mdm::Module::Detail.find_each do |md| + + unless md.ready + refresh << md + next + end + + unless md.file and ::File.exists?(md.file) + refresh << md + next + end + + if ::File.mtime(md.file).to_i != md.mtime.to_i + refresh << md + next + end + + skip_reference_name_set = skip_reference_name_set_by_module_type[md.mtype] + skip_reference_name_set.add(md.refname) + end + + refresh.each { |md| md.destroy } + + [ + ['exploit', framework.exploits], + ['auxiliary', framework.auxiliary], + ['post', framework.post], + ['payload', framework.payloads], + ['encoder', framework.encoders], + ['nop', framework.nops] + ].each do |mt| + skip_reference_name_set = skip_reference_name_set_by_module_type[mt[0]] + + mt[1].keys.sort.each do |mn| + next if skip_reference_name_set.include? mn + obj = mt[1].create(mn) + next if not obj + begin + update_module_details(obj) + rescue ::Exception + elog("Error updating module details for #{obj.fullname}: #{$!.class} #{$!}") + end + end + end + + self.framework.cache_initialized = true + end + + # in reverse order of section before with_connection block + self.modules_caching = false + self.modules_cached = true + self.framework.cache_thread = nil + end + + # Creates an Mdm::Module::Detail from a module instance. + # + # @param module_instance [Msf::Module] a metasploit module instance. + # @raise [ActiveRecord::RecordInvalid] if Hash from {#module_to_details_hash} is invalid attributes for + # Mdm::Module::Detail. + # @return [void] + def update_module_details(module_instance) + return if not self.migrated + + ActiveRecord::Base.connection_pool.with_connection do + info = module_to_details_hash(module_instance) + bits = info.delete(:bits) || [] + module_detail = Mdm::Module::Detail.create!(info) + + bits.each do |args| + otype, vals = args + + case otype + when :action + module_detail.add_action(vals[:name]) + when :arch + module_detail.add_arch(vals[:name]) + when :author + module_detail.add_author(vals[:name], vals[:email]) + when :platform + module_detail.add_platform(vals[:name]) + when :ref + module_detail.add_ref(vals[:name]) + when :target + module_detail.add_target(vals[:index], vals[:name]) + end + end + + module_detail.ready = true + module_detail.save! + end + end +end \ No newline at end of file From 37c38ad27b9bc073e6e224528e8be96dcf93ef77 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Thu, 9 Oct 2014 09:14:45 -0500 Subject: [PATCH 018/159] Extract Msf::DBManager::ModuleCache shared examples MSP-11124 --- spec/lib/msf/db_manager_spec.rb | 1120 +---------------- .../examples/msf/db_manager/module_cache.rb | 1118 ++++++++++++++++ 2 files changed, 1119 insertions(+), 1119 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/module_cache.rb diff --git a/spec/lib/msf/db_manager_spec.rb b/spec/lib/msf/db_manager_spec.rb index 647f85a0f9..762fdd29f1 100644 --- a/spec/lib/msf/db_manager_spec.rb +++ b/spec/lib/msf/db_manager_spec.rb @@ -20,6 +20,7 @@ describe Msf::DBManager do it_should_behave_like 'Msf::DBManager::ImportMsfXml' it_should_behave_like 'Msf::DBManager::Migration' + it_should_behave_like 'Msf::DBManager::ModuleCache' it_should_behave_like 'Msf::DBManager::Sink' it_should_behave_like 'Msf::DBManager::Workspace' @@ -185,10 +186,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :ipv4_validator } it { is_expected.to respond_to :ipv6_validator } it { is_expected.to respond_to :loots } - it { is_expected.to respond_to :modules_cached } - it { is_expected.to respond_to :modules_cached } - it { is_expected.to respond_to :modules_cached= } - it { is_expected.to respond_to :modules_cached= } it { is_expected.to respond_to :msf_import_timestamps } it { is_expected.to respond_to :netsparker_method_map } it { is_expected.to respond_to :netsparker_params_map } @@ -200,144 +197,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :nmap_msf_service_map } it { is_expected.to respond_to :normalize_host } it { is_expected.to respond_to :notes } - - context '#purge_all_module_details' do - def purge_all_module_details - db_manager.purge_all_module_details - end - - let(:migrated) do - false - end - - let(:module_detail_count) do - 2 - end - - let!(:module_details) do - FactoryGirl.create_list( - :mdm_module_detail, - module_detail_count - ) - end - - before(:each) do - db_manager.stub(:migrated => migrated) - end - - context 'with migrated' do - let(:migrated) do - true - end - - let(:modules_caching) do - false - end - - before(:each) do - db_manager.stub(:modules_caching => modules_caching) - end - - context 'with modules_caching' do - let(:modules_caching) do - true - end - - it 'should not destroy Mdm::Module::Details' do - expect { - purge_all_module_details - }.to_not change(Mdm::Module::Detail, :count) - end - end - - context 'without modules_caching' do - it 'should destroy all Mdm::Module::Details' do - expect { - purge_all_module_details - }.to change(Mdm::Module::Detail, :count).by(-module_detail_count) - end - end - end - - context 'without migrated' do - it 'should not destroy Mdm::Module::Details' do - expect { - purge_all_module_details - }.to_not change(Mdm::Module::Detail, :count) - end - end - end - - context '#remove_module_details' do - def remove_module_details - db_manager.remove_module_details(mtype, refname) - end - - let(:migrated) do - false - end - - let(:mtype) do - FactoryGirl.generate :mdm_module_detail_mtype - end - - let(:refname) do - FactoryGirl.generate :mdm_module_detail_refname - end - - let!(:module_detail) do - FactoryGirl.create( - :mdm_module_detail - ) - end - - before(:each) do - db_manager.stub(:migrated => migrated) - end - - context 'with migrated' do - let(:migrated) do - true - end - - let!(:module_detail) do - FactoryGirl.create(:mdm_module_detail) - end - - context 'with matching Mdm::Module::Detail' do - let(:mtype) do - module_detail.mtype - end - - let(:refname) do - module_detail.refname - end - - it 'should destroy Mdm::Module::Detail' do - expect { - remove_module_details - }.to change(Mdm::Module::Detail, :count).by(-1) - end - end - - context 'without matching Mdm::Module::Detail' do - it 'should not destroy Mdm::Module::Detail' do - expect { - remove_module_details - }.to_not change(Mdm::Module::Detail, :count) - end - end - end - - context 'without migrated' do - it 'should not destroy Mdm::Module::Detail' do - expect { - remove_module_details - }.to_not change(Mdm::Module::Detail, :count) - end - end - end - it { is_expected.to respond_to :report_artifact } it { is_expected.to respond_to :report_auth } it { is_expected.to respond_to :report_auth_info } @@ -1013,409 +872,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :requests } it { is_expected.to respond_to :rexmlify } it { is_expected.to respond_to :rfc3330_reserved } - - context '#search_modules' do - subject(:search_modules) do - db_manager.search_modules(search_string) - end - - let(:module_details) do - search_modules.to_a - end - - context 'with app keyword' do - let(:search_string) do - "app:#{app}" - end - - before(:each) do - Mdm::Module::Detail::STANCES.each do |stance| - FactoryGirl.create(:mdm_module_detail, :stance => stance) - end - end - - context 'with client' do - let(:app) do - 'client' - end - - it "should match Mdm::Module::Detail#stance 'passive'" do - module_details.count.should > 0 - - module_details.all? { |module_detail| - module_detail.stance == 'passive' - }.should be_truthy - end - end - - context 'with server' do - let(:app) do - 'server' - end - - it "should match Mdm::Module::Detail#stance 'aggressive'" do - module_details.count.should > 0 - - module_details.all? { |module_detail| - module_detail.stance == 'aggressive' - }.should be_truthy - end - end - end - - context 'with author keyword' do - let(:search_string) do - # us inspect so strings with spaces are quoted correctly - "author:#{author}" - end - - let!(:module_authors) do - FactoryGirl.create_list(:mdm_module_author, 2) - end - - let(:target_module_author) do - module_authors.first - end - - context 'with Mdm::Module::Author#email' do - let(:author) do - target_module_author.email - end - - it 'should match Mdm::Module::Author#email' do - module_details.count.should > 0 - - module_details.all? { |module_detail| - module_detail.authors.any? { |module_author| - module_author.email == target_module_author.email - } - }.should be_truthy - end - end - - context 'with Mdm::Module::Author#name' do - let(:author) do - # use inspect to quote space in name - target_module_author.name.inspect - end - - it 'should match Mdm::Module::Author#name' do - module_details.count.should > 0 - - module_details.all? { |module_detail| - module_detail.authors.any? { |module_author| - module_author.name == target_module_author.name - } - }.should be_truthy - end - end - end - - it_should_behave_like 'Msf::DBManager#search_modules Mdm::Module::Ref#name keyword', :bid - it_should_behave_like 'Msf::DBManager#search_modules Mdm::Module::Ref#name keyword', :cve - it_should_behave_like 'Msf::DBManager#search_modules Mdm::Module::Ref#name keyword', :edb - - context 'with name keyword' do - let(:search_string) do - "name:#{name}" - end - - let!(:existing_module_details) do - FactoryGirl.create_list(:mdm_module_detail, 2) - end - - let(:target_module_detail) do - existing_module_details.first - end - - context 'with Mdm::Module::Detail#fullname' do - let(:name) do - target_module_detail.fullname - end - - it 'should match Mdm::Module::Detail#fullname' do - module_details.count.should > 0 - - module_details.all? { |module_detail| - module_detail.fullname == target_module_detail.fullname - }.should be_truthy - end - end - - context 'with Mdm::Module::Detail#name' do - let(:name) do - # use inspect so spaces are inside quotes - target_module_detail.name.inspect - end - - it 'should match Mdm::Module::Detail#name' do - module_details.count.should > 0 - - module_details.all? { |module_detail| - module_detail.name == target_module_detail.name - }.should be_truthy - end - end - end - - it_should_behave_like 'Msf::DBManager#search_modules Mdm::Module::Platform#name or Mdm::Module::Target#name keyword', :os - - it_should_behave_like 'Msf::DBManager#search_modules Mdm::Module::Ref#name keyword', :osvdb - - it_should_behave_like 'Msf::DBManager#search_modules Mdm::Module::Platform#name or Mdm::Module::Target#name keyword', :platform - - context 'with ref keyword' do - let(:ref) do - FactoryGirl.generate :mdm_module_ref_name - end - - let(:search_string) do - # use inspect to quote spaces in string - "ref:#{ref.inspect}" - end - - let!(:module_ref) do - FactoryGirl.create(:mdm_module_ref) - end - - context 'with Mdm::Module::Ref#name' do - let(:ref) do - module_ref.name - end - - it 'should match Mdm::Module::Ref#name' do - module_details.count.should > 0 - - module_details.all? { |module_detail| - module_detail.refs.any? { |module_ref| - module_ref.name == ref - } - }.should be_truthy - end - end - - context 'without Mdm::Module::Ref#name' do - it 'should not match Mdm::Module::Ref#name' do - module_details.count.should == 0 - end - end - end - - context 'with type keyword' do - let(:type) do - FactoryGirl.generate :mdm_module_detail_mtype - end - - let(:search_string) do - "type:#{type}" - end - - let(:target_module_detail) do - all_module_details.first - end - - let!(:all_module_details) do - FactoryGirl.create_list(:mdm_module_detail, 2) - end - - context 'with Mdm::Module::Ref#name' do - let(:type) do - target_module_detail.mtype - end - - it 'should match Mdm::Module::Detail#mtype' do - module_details.count.should > 0 - - module_details.all? { |module_detail| - module_detail.mtype == type - }.should be_truthy - end - end - - context 'without Mdm::Module::Detail#mtype' do - it 'should not match Mdm::Module::Detail#mtype' do - module_details.count.should == 0 - end - end - end - - context 'without keyword' do - context 'with Mdm::Module::Action#name' do - let(:search_string) do - module_action.name - end - - let!(:module_action) do - FactoryGirl.create(:mdm_module_action) - end - - it 'should match Mdm::Module::Action#name' do - module_details.count.should > 0 - - module_details.all? { |module_detail| - module_detail.actions.any? { |module_action| - module_action.name == search_string - } - }.should be_truthy - end - end - - context 'with Mdm::Module::Arch#name' do - let(:search_string) do - module_arch.name - end - - let!(:module_arch) do - FactoryGirl.create(:mdm_module_arch) - end - - it 'should match Mdm::Module::Arch#name' do - module_details.count.should > 0 - - module_details.all? { |module_detail| - module_detail.archs.any? { |module_arch| - module_arch.name == search_string - } - }.should be_truthy - end - end - - context 'with Mdm::Module::Author#name' do - let(:search_string) do - module_author.name - end - - let!(:module_author) do - FactoryGirl.create(:mdm_module_author) - end - - it 'should match Mdm::Module::Author#name' do - module_details.count.should > 0 - - module_details.all? { |module_detail| - module_detail.authors.any? { |module_author| - module_author.name == search_string - } - }.should be_truthy - end - end - - context 'with Mdm::Module::Detail' do - let(:target_module_detail) do - all_module_details.first - end - - let!(:all_module_details) do - FactoryGirl.create_list(:mdm_module_detail, 3) - end - - context 'with #description' do - let(:search_string) do - # use inspect to quote spaces in string - target_module_detail.description.inspect - end - - it 'should match Mdm::Module::Detail#description' do - module_details.count.should == 1 - - module_details.all? { |module_detail| - module_detail.description == target_module_detail.description - }.should be_truthy - end - end - - context 'with #fullname' do - let(:search_string) do - target_module_detail.fullname - end - - it 'should match Mdm::Module::Detail#fullname' do - module_details.count.should == 1 - - module_details.all? { |module_detail| - module_detail.fullname == search_string - }.should be_truthy - end - end - - context 'with #name' do - let(:search_string) do - # use inspect to quote spaces in string - target_module_detail.name.inspect - end - - it 'should match Mdm::Module::Detail#name' do - module_details.count.should == 1 - - module_details.all? { |module_detail| - module_detail.name == target_module_detail.name - }.should be_truthy - end - end - end - - context 'with Mdm::Module::Platform#name' do - let(:search_string) do - module_platform.name - end - - let!(:module_platform) do - FactoryGirl.create(:mdm_module_platform) - end - - it 'should match Mdm::Module::Platform#name' do - module_details.count.should > 0 - - module_details.all? { |module_detail| - module_detail.platforms.any? { |module_platform| - module_platform.name == search_string - } - }.should be_truthy - end - end - - context 'with Mdm::Module::Ref#name' do - let(:search_string) do - module_ref.name - end - - let!(:module_ref) do - FactoryGirl.create(:mdm_module_ref) - end - - it 'should match Mdm::Module::Ref#name' do - module_details.count.should > 0 - - module_details.all? { |module_detail| - module_detail.refs.any? { |module_ref| - module_ref.name == search_string - } - }.should be_truthy - end - end - - context 'with Mdm::Module::Target#name' do - let(:search_string) do - module_target.name - end - - let!(:module_target) do - FactoryGirl.create(:mdm_module_target) - end - - it 'should match Mdm::Module::Target#name' do - module_details.count.should > 0 - - module_details.all? { |module_detail| - module_detail.targets.any? { |module_target| - module_target.name == search_string - } - }.should be_truthy - end - end - end - end - it { is_expected.to respond_to :selected_host } it { is_expected.to respond_to :selected_id } it { is_expected.to respond_to :selected_port } @@ -1428,581 +884,7 @@ describe Msf::DBManager do it { is_expected.to respond_to :targets } it { is_expected.to respond_to :tasks } it { is_expected.to respond_to :unserialize_object } - - context '#update_all_module_details' do - def update_all_module_details - db_manager.update_all_module_details - end - - let(:migrated) do - false - end - - before(:each) do - db_manager.stub(:migrated => migrated) - end - - context 'with migrated' do - let(:migrated) do - true - end - - let(:modules_caching) do - true - end - - before(:each) do - db_manager.stub(:modules_caching => modules_caching) - end - - context 'with modules_caching' do - it 'should not update module details' do - db_manager.should_not_receive(:update_module_details) - - update_all_module_details - end - end - - context 'without modules_caching' do - let(:modules_caching) do - false - end - - it 'should set framework.cache_thread to current thread and then nil' do - framework.should_receive(:cache_thread=).with(Thread.current).ordered - framework.should_receive(:cache_thread=).with(nil).ordered - - update_all_module_details - end - - it 'should set modules_cached to false and then true' do - db_manager.should_receive(:modules_cached=).with(false).ordered - db_manager.should_receive(:modules_cached=).with(true).ordered - - update_all_module_details - end - - it 'should set modules_caching to true and then false' do - db_manager.should_receive(:modules_caching=).with(true).ordered - db_manager.should_receive(:modules_caching=).with(false).ordered - - update_all_module_details - end - - context 'with Mdm::Module::Details' do - let(:module_pathname) do - parent_pathname.join( - 'exploits', - "#{reference_name}.rb" - ) - end - - let(:modification_time) do - module_pathname.mtime - end - - let(:parent_pathname) do - Metasploit::Framework.root.join('modules') - end - - let(:reference_name) do - 'windows/smb/ms08_067_netapi' - end - - let(:type) do - 'exploit' - end - - let!(:module_detail) do - # needs to reference a real module so that it can be loaded - FactoryGirl.create( - :mdm_module_detail, - :file => module_pathname.to_path, - :mtime => modification_time, - :mtype => type, - :ready => ready, - :refname => reference_name - ) - end - - context '#ready' do - context 'false' do - let(:ready) do - false - end - - it_should_behave_like 'Msf::DBManager#update_all_module_details refresh' - end - - context 'true' do - let(:ready) do - true - end - - context 'with existing Mdm::Module::Detail#file' do - context 'with same Mdm::Module::Detail#mtime and File.mtime' do - it 'should not update module details' do - db_manager.should_not_receive(:update_module_details) - - update_all_module_details - end - end - - context 'without same Mdm::Module::Detail#mtime and File.mtime' do - let(:modification_time) do - # +1 as rand can return 0 and the time must be different for - # this context. - super() - (rand(1.day) + 1) - end - - it_should_behave_like 'Msf::DBManager#update_all_module_details refresh' - end - end - - # Emulates a module being removed or renamed - context 'without existing Mdm::Module::Detail#file' do - # have to compute modification manually since the - # `module_pathname` refers to a non-existent file and - # `module_pathname.mtime` would error. - let(:modification_time) do - Time.now.utc - 1.day - end - - let(:module_pathname) do - parent_pathname.join('exploits', 'deleted.rb') - end - - it 'should not update module details' do - db_manager.should_not_receive(:update_module_details) - - update_all_module_details - end - end - end - end - end - end - end - - context 'without migrated' do - it 'should not update module details' do - db_manager.should_not_receive(:update_module_details) - - update_all_module_details - end - end - end - it { is_expected.to respond_to :update_host_via_sysinfo } - - context '#update_module_details' do - def update_module_details - db_manager.update_module_details(module_instance) - end - - let(:loader) do - loader = framework.modules.send(:loaders).find { |loader| - loader.loadable?(parent_path) - } - - # Override load_error so that rspec will print it instead of going to framework log - def loader.load_error(module_path, error) - raise error - end - - loader - end - - let(:migrated) do - false - end - - let(:module_instance) do - # make sure the module is loaded into the module_set - loaded = loader.load_module(parent_path, module_type, module_reference_name) - - unless loaded - module_path = loader.module_path(parent_path, type, module_reference_name) - - fail "#{description} failed to load: #{module_path}" - end - - module_set.create(module_reference_name) - end - - let(:module_set) do - framework.modules.module_set(module_type) - end - - let(:module_type) do - 'exploit' - end - - let(:module_reference_name) do - 'windows/smb/ms08_067_netapi' - end - - let(:parent_path) do - parent_pathname.to_path - end - - let(:parent_pathname) do - Metasploit::Framework.root.join('modules') - end - - let(:type_directory) do - 'exploits' - end - - before(:each) do - db_manager.stub(:migrated => migrated) - end - - context 'with migrated' do - let(:migrated) do - true - end - - it 'should call module_to_details_hash to get Mdm::Module::Detail attributes and association attributes' do - db_manager.should_receive(:module_to_details_hash).and_call_original - - update_module_details - end - - it 'should create an Mdm::Module::Detail' do - expect { - update_module_details - }.to change(Mdm::Module::Detail, :count).by(1) - end - - - context 'module_to_details_hash' do - let(:module_to_details_hash) do - { - :mtype => module_type, - :privileged => privileged, - :rank => rank, - :refname => module_reference_name, - :stance => stance - } - end - - let(:privileged) do - FactoryGirl.generate :mdm_module_detail_privileged - end - - let(:rank) do - FactoryGirl.generate :mdm_module_detail_rank - end - - let(:stance) do - FactoryGirl.generate :mdm_module_detail_stance - end - - before(:each) do - db_manager.stub( - :module_to_details_hash - ).with( - module_instance - ).and_return( - module_to_details_hash - ) - end - - context 'Mdm::Module::Detail' do - subject(:module_detail) do - Mdm::Module::Detail.last - end - - before(:each) do - update_module_details - end - - it { expect(subject.mtype).to eq(module_type) } - it { expect(subject.privileged).to eq(privileged) } - it { expect(subject.rank).to eq(rank) } - it { expect(subject.ready).to be_truthy } - it { expect(subject.refname).to eq(module_reference_name) } - it { expect(subject.stance).to eq(stance) } - end - - context 'with :bits' do - let(:bits) do - [] - end - - before(:each) do - module_to_details_hash[:bits] = bits - end - - context 'with :action' do - let(:name) do - FactoryGirl.generate :mdm_module_action_name - end - - let(:bits) do - super() << [ - :action, - { - :name => name - } - ] - end - - it 'should create an Mdm::Module::Action' do - expect { - update_module_details - }.to change(Mdm::Module::Action, :count).by(1) - end - - context 'Mdm::Module::Action' do - subject(:module_action) do - module_detail.actions.last - end - - let(:module_detail) do - Mdm::Module::Detail.last - end - - before(:each) do - update_module_details - end - - it { expect(subject.name).to eq(name) } - end - end - - context 'with :arch' do - let(:name) do - FactoryGirl.generate :mdm_module_arch_name - end - - let(:bits) do - super() << [ - :arch, - { - :name => name - } - ] - end - - it 'should create an Mdm::Module::Arch' do - expect { - update_module_details - }.to change(Mdm::Module::Arch, :count).by(1) - end - - context 'Mdm::Module::Arch' do - subject(:module_arch) do - module_detail.archs.last - end - - let(:module_detail) do - Mdm::Module::Detail.last - end - - before(:each) do - update_module_details - end - - it { expect(subject.name).to eq(name) } - end - end - - context 'with :author' do - let(:email) do - FactoryGirl.generate :mdm_module_author_email - end - - let(:name) do - FactoryGirl.generate :mdm_module_author_name - end - - let(:bits) do - super() << [ - :author, - { - :email => email, - :name => name - } - ] - end - - it 'should create an Mdm::Module::Author' do - expect { - update_module_details - }.to change(Mdm::Module::Author, :count).by(1) - end - - context 'Mdm::Module::Author' do - subject(:module_author) do - module_detail.authors.last - end - - let(:module_detail) do - Mdm::Module::Detail.last - end - - before(:each) do - update_module_details - end - - it { expect(subject.name).to eq(name) } - it { expect(subject.email).to eq(email) } - end - end - - context 'with :platform' do - let(:bits) do - super() << [ - :platform, - { - :name => name - } - ] - end - - let(:name) do - FactoryGirl.generate :mdm_module_platform_name - end - - it 'should create an Mdm::Module::Platform' do - expect { - update_module_details - }.to change(Mdm::Module::Platform, :count).by(1) - end - - context 'Mdm::Module::Platform' do - subject(:module_platform) do - module_detail.platforms.last - end - - let(:module_detail) do - Mdm::Module::Detail.last - end - - before(:each) do - update_module_details - end - - it { expect(subject.name).to eq(name) } - end - end - - context 'with :ref' do - let(:bits) do - super() << [ - :ref, - { - :name => name - } - ] - end - - let(:name) do - FactoryGirl.generate :mdm_module_ref_name - end - - it 'should create an Mdm::Module::Ref' do - expect { - update_module_details - }.to change(Mdm::Module::Ref, :count).by(1) - end - - context 'Mdm::Module::Ref' do - subject(:module_ref) do - module_detail.refs.last - end - - let(:module_detail) do - Mdm::Module::Detail.last - end - - before(:each) do - update_module_details - end - - it { expect(subject.name).to eq(name) } - end - end - - context 'with :target' do - let(:bits) do - super() << [ - :target, - { - :index => index, - :name => name - } - ] - end - - let(:index) do - FactoryGirl.generate :mdm_module_target_index - end - - let(:name) do - FactoryGirl.generate :mdm_module_target_name - end - - it 'should create an Mdm::Module::Target' do - expect { - update_module_details - }.to change(Mdm::Module::Target, :count).by(1) - end - - context 'Mdm::Module::Target' do - subject(:module_target) do - module_detail.targets.last - end - - let(:module_detail) do - Mdm::Module::Detail.last - end - - before(:each) do - update_module_details - end - - it { expect(subject.index).to eq(index) } - it { expect(subject.name).to eq(name) } - end - end - end - end - - it_should_behave_like 'Msf::DBManager#update_module_details with module', - :reference_name => 'admin/2wire/xslt_password_reset', - :type => 'auxiliary' - - it_should_behave_like 'Msf::DBManager#update_module_details with module', - :reference_name => 'generic/none', - :type => 'encoder' - - it_should_behave_like 'Msf::DBManager#update_module_details with module', - :reference_name => 'windows/smb/ms08_067_netapi', - :type => 'exploit' - - it_should_behave_like 'Msf::DBManager#update_module_details with module', - :reference_name => 'x64/simple', - :type => 'nop' - - # @todo determine how to load a single payload to test payload type outside of msfconsole - - it_should_behave_like 'Msf::DBManager#update_module_details with module', - :reference_name => 'windows/escalate/screen_unlock', - :type => 'post' - end - - context 'without migrated' do - it 'should not create an Mdm::Module::Detail' do - expect { - update_module_details - }.to_not change(Mdm::Module::Detail, :count) - end - end - end - it { is_expected.to respond_to :update_vuln_details } it { is_expected.to respond_to :usable } it { is_expected.to respond_to :usable= } diff --git a/spec/support/shared/examples/msf/db_manager/module_cache.rb b/spec/support/shared/examples/msf/db_manager/module_cache.rb new file mode 100644 index 0000000000..9d74460817 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/module_cache.rb @@ -0,0 +1,1118 @@ +shared_examples_for 'Msf::DBManager::ModuleCache' do + it { is_expected.to respond_to :module_to_details_hash } + it { is_expected.to respond_to :modules_cached } + it { is_expected.to respond_to :modules_cached= } + it { is_expected.to respond_to :modules_caching } + it { is_expected.to respond_to :modules_caching= } + + context '#purge_all_module_details' do + def purge_all_module_details + db_manager.purge_all_module_details + end + + let(:migrated) do + false + end + + let(:module_detail_count) do + 2 + end + + let!(:module_details) do + FactoryGirl.create_list( + :mdm_module_detail, + module_detail_count + ) + end + + before(:each) do + db_manager.stub(:migrated => migrated) + end + + context 'with migrated' do + let(:migrated) do + true + end + + let(:modules_caching) do + false + end + + before(:each) do + db_manager.stub(:modules_caching => modules_caching) + end + + context 'with modules_caching' do + let(:modules_caching) do + true + end + + it 'should not destroy Mdm::Module::Details' do + expect { + purge_all_module_details + }.to_not change(Mdm::Module::Detail, :count) + end + end + + context 'without modules_caching' do + it 'should destroy all Mdm::Module::Details' do + expect { + purge_all_module_details + }.to change(Mdm::Module::Detail, :count).by(-module_detail_count) + end + end + end + + context 'without migrated' do + it 'should not destroy Mdm::Module::Details' do + expect { + purge_all_module_details + }.to_not change(Mdm::Module::Detail, :count) + end + end + end + + context '#remove_module_details' do + def remove_module_details + db_manager.remove_module_details(mtype, refname) + end + + let(:migrated) do + false + end + + let(:mtype) do + FactoryGirl.generate :mdm_module_detail_mtype + end + + let(:refname) do + FactoryGirl.generate :mdm_module_detail_refname + end + + let!(:module_detail) do + FactoryGirl.create( + :mdm_module_detail + ) + end + + before(:each) do + db_manager.stub(:migrated => migrated) + end + + context 'with migrated' do + let(:migrated) do + true + end + + let!(:module_detail) do + FactoryGirl.create(:mdm_module_detail) + end + + context 'with matching Mdm::Module::Detail' do + let(:mtype) do + module_detail.mtype + end + + let(:refname) do + module_detail.refname + end + + it 'should destroy Mdm::Module::Detail' do + expect { + remove_module_details + }.to change(Mdm::Module::Detail, :count).by(-1) + end + end + + context 'without matching Mdm::Module::Detail' do + it 'should not destroy Mdm::Module::Detail' do + expect { + remove_module_details + }.to_not change(Mdm::Module::Detail, :count) + end + end + end + + context 'without migrated' do + it 'should not destroy Mdm::Module::Detail' do + expect { + remove_module_details + }.to_not change(Mdm::Module::Detail, :count) + end + end + end + + context '#search_modules' do + subject(:search_modules) do + db_manager.search_modules(search_string) + end + + let(:module_details) do + search_modules.to_a + end + + context 'with app keyword' do + let(:search_string) do + "app:#{app}" + end + + before(:each) do + Mdm::Module::Detail::STANCES.each do |stance| + FactoryGirl.create(:mdm_module_detail, :stance => stance) + end + end + + context 'with client' do + let(:app) do + 'client' + end + + it "should match Mdm::Module::Detail#stance 'passive'" do + module_details.count.should > 0 + + module_details.all? { |module_detail| + module_detail.stance == 'passive' + }.should be_truthy + end + end + + context 'with server' do + let(:app) do + 'server' + end + + it "should match Mdm::Module::Detail#stance 'aggressive'" do + module_details.count.should > 0 + + module_details.all? { |module_detail| + module_detail.stance == 'aggressive' + }.should be_truthy + end + end + end + + context 'with author keyword' do + let(:search_string) do + # us inspect so strings with spaces are quoted correctly + "author:#{author}" + end + + let!(:module_authors) do + FactoryGirl.create_list(:mdm_module_author, 2) + end + + let(:target_module_author) do + module_authors.first + end + + context 'with Mdm::Module::Author#email' do + let(:author) do + target_module_author.email + end + + it 'should match Mdm::Module::Author#email' do + module_details.count.should > 0 + + module_details.all? { |module_detail| + module_detail.authors.any? { |module_author| + module_author.email == target_module_author.email + } + }.should be_truthy + end + end + + context 'with Mdm::Module::Author#name' do + let(:author) do + # use inspect to quote space in name + target_module_author.name.inspect + end + + it 'should match Mdm::Module::Author#name' do + module_details.count.should > 0 + + module_details.all? { |module_detail| + module_detail.authors.any? { |module_author| + module_author.name == target_module_author.name + } + }.should be_truthy + end + end + end + + it_should_behave_like 'Msf::DBManager#search_modules Mdm::Module::Ref#name keyword', :bid + it_should_behave_like 'Msf::DBManager#search_modules Mdm::Module::Ref#name keyword', :cve + it_should_behave_like 'Msf::DBManager#search_modules Mdm::Module::Ref#name keyword', :edb + + context 'with name keyword' do + let(:search_string) do + "name:#{name}" + end + + let!(:existing_module_details) do + FactoryGirl.create_list(:mdm_module_detail, 2) + end + + let(:target_module_detail) do + existing_module_details.first + end + + context 'with Mdm::Module::Detail#fullname' do + let(:name) do + target_module_detail.fullname + end + + it 'should match Mdm::Module::Detail#fullname' do + module_details.count.should > 0 + + module_details.all? { |module_detail| + module_detail.fullname == target_module_detail.fullname + }.should be_truthy + end + end + + context 'with Mdm::Module::Detail#name' do + let(:name) do + # use inspect so spaces are inside quotes + target_module_detail.name.inspect + end + + it 'should match Mdm::Module::Detail#name' do + module_details.count.should > 0 + + module_details.all? { |module_detail| + module_detail.name == target_module_detail.name + }.should be_truthy + end + end + end + + it_should_behave_like 'Msf::DBManager#search_modules Mdm::Module::Platform#name or Mdm::Module::Target#name keyword', :os + + it_should_behave_like 'Msf::DBManager#search_modules Mdm::Module::Ref#name keyword', :osvdb + + it_should_behave_like 'Msf::DBManager#search_modules Mdm::Module::Platform#name or Mdm::Module::Target#name keyword', :platform + + context 'with ref keyword' do + let(:ref) do + FactoryGirl.generate :mdm_module_ref_name + end + + let(:search_string) do + # use inspect to quote spaces in string + "ref:#{ref.inspect}" + end + + let!(:module_ref) do + FactoryGirl.create(:mdm_module_ref) + end + + context 'with Mdm::Module::Ref#name' do + let(:ref) do + module_ref.name + end + + it 'should match Mdm::Module::Ref#name' do + module_details.count.should > 0 + + module_details.all? { |module_detail| + module_detail.refs.any? { |module_ref| + module_ref.name == ref + } + }.should be_truthy + end + end + + context 'without Mdm::Module::Ref#name' do + it 'should not match Mdm::Module::Ref#name' do + module_details.count.should == 0 + end + end + end + + context 'with type keyword' do + let(:type) do + FactoryGirl.generate :mdm_module_detail_mtype + end + + let(:search_string) do + "type:#{type}" + end + + let(:target_module_detail) do + all_module_details.first + end + + let!(:all_module_details) do + FactoryGirl.create_list(:mdm_module_detail, 2) + end + + context 'with Mdm::Module::Ref#name' do + let(:type) do + target_module_detail.mtype + end + + it 'should match Mdm::Module::Detail#mtype' do + module_details.count.should > 0 + + module_details.all? { |module_detail| + module_detail.mtype == type + }.should be_truthy + end + end + + context 'without Mdm::Module::Detail#mtype' do + it 'should not match Mdm::Module::Detail#mtype' do + module_details.count.should == 0 + end + end + end + + context 'without keyword' do + context 'with Mdm::Module::Action#name' do + let(:search_string) do + module_action.name + end + + let!(:module_action) do + FactoryGirl.create(:mdm_module_action) + end + + it 'should match Mdm::Module::Action#name' do + module_details.count.should > 0 + + module_details.all? { |module_detail| + module_detail.actions.any? { |module_action| + module_action.name == search_string + } + }.should be_truthy + end + end + + context 'with Mdm::Module::Arch#name' do + let(:search_string) do + module_arch.name + end + + let!(:module_arch) do + FactoryGirl.create(:mdm_module_arch) + end + + it 'should match Mdm::Module::Arch#name' do + module_details.count.should > 0 + + module_details.all? { |module_detail| + module_detail.archs.any? { |module_arch| + module_arch.name == search_string + } + }.should be_truthy + end + end + + context 'with Mdm::Module::Author#name' do + let(:search_string) do + module_author.name + end + + let!(:module_author) do + FactoryGirl.create(:mdm_module_author) + end + + it 'should match Mdm::Module::Author#name' do + module_details.count.should > 0 + + module_details.all? { |module_detail| + module_detail.authors.any? { |module_author| + module_author.name == search_string + } + }.should be_truthy + end + end + + context 'with Mdm::Module::Detail' do + let(:target_module_detail) do + all_module_details.first + end + + let!(:all_module_details) do + FactoryGirl.create_list(:mdm_module_detail, 3) + end + + context 'with #description' do + let(:search_string) do + # use inspect to quote spaces in string + target_module_detail.description.inspect + end + + it 'should match Mdm::Module::Detail#description' do + module_details.count.should == 1 + + module_details.all? { |module_detail| + module_detail.description == target_module_detail.description + }.should be_truthy + end + end + + context 'with #fullname' do + let(:search_string) do + target_module_detail.fullname + end + + it 'should match Mdm::Module::Detail#fullname' do + module_details.count.should == 1 + + module_details.all? { |module_detail| + module_detail.fullname == search_string + }.should be_truthy + end + end + + context 'with #name' do + let(:search_string) do + # use inspect to quote spaces in string + target_module_detail.name.inspect + end + + it 'should match Mdm::Module::Detail#name' do + module_details.count.should == 1 + + module_details.all? { |module_detail| + module_detail.name == target_module_detail.name + }.should be_truthy + end + end + end + + context 'with Mdm::Module::Platform#name' do + let(:search_string) do + module_platform.name + end + + let!(:module_platform) do + FactoryGirl.create(:mdm_module_platform) + end + + it 'should match Mdm::Module::Platform#name' do + module_details.count.should > 0 + + module_details.all? { |module_detail| + module_detail.platforms.any? { |module_platform| + module_platform.name == search_string + } + }.should be_truthy + end + end + + context 'with Mdm::Module::Ref#name' do + let(:search_string) do + module_ref.name + end + + let!(:module_ref) do + FactoryGirl.create(:mdm_module_ref) + end + + it 'should match Mdm::Module::Ref#name' do + module_details.count.should > 0 + + module_details.all? { |module_detail| + module_detail.refs.any? { |module_ref| + module_ref.name == search_string + } + }.should be_truthy + end + end + + context 'with Mdm::Module::Target#name' do + let(:search_string) do + module_target.name + end + + let!(:module_target) do + FactoryGirl.create(:mdm_module_target) + end + + it 'should match Mdm::Module::Target#name' do + module_details.count.should > 0 + + module_details.all? { |module_detail| + module_detail.targets.any? { |module_target| + module_target.name == search_string + } + }.should be_truthy + end + end + end + end + + context '#update_all_module_details' do + def update_all_module_details + db_manager.update_all_module_details + end + + let(:migrated) do + false + end + + before(:each) do + db_manager.stub(:migrated => migrated) + end + + context 'with migrated' do + let(:migrated) do + true + end + + let(:modules_caching) do + true + end + + before(:each) do + db_manager.stub(:modules_caching => modules_caching) + end + + context 'with modules_caching' do + it 'should not update module details' do + db_manager.should_not_receive(:update_module_details) + + update_all_module_details + end + end + + context 'without modules_caching' do + let(:modules_caching) do + false + end + + it 'should set framework.cache_thread to current thread and then nil' do + framework.should_receive(:cache_thread=).with(Thread.current).ordered + framework.should_receive(:cache_thread=).with(nil).ordered + + update_all_module_details + end + + it 'should set modules_cached to false and then true' do + db_manager.should_receive(:modules_cached=).with(false).ordered + db_manager.should_receive(:modules_cached=).with(true).ordered + + update_all_module_details + end + + it 'should set modules_caching to true and then false' do + db_manager.should_receive(:modules_caching=).with(true).ordered + db_manager.should_receive(:modules_caching=).with(false).ordered + + update_all_module_details + end + + context 'with Mdm::Module::Details' do + let(:module_pathname) do + parent_pathname.join( + 'exploits', + "#{reference_name}.rb" + ) + end + + let(:modification_time) do + module_pathname.mtime + end + + let(:parent_pathname) do + Metasploit::Framework.root.join('modules') + end + + let(:reference_name) do + 'windows/smb/ms08_067_netapi' + end + + let(:type) do + 'exploit' + end + + let!(:module_detail) do + # needs to reference a real module so that it can be loaded + FactoryGirl.create( + :mdm_module_detail, + :file => module_pathname.to_path, + :mtime => modification_time, + :mtype => type, + :ready => ready, + :refname => reference_name + ) + end + + context '#ready' do + context 'false' do + let(:ready) do + false + end + + it_should_behave_like 'Msf::DBManager#update_all_module_details refresh' + end + + context 'true' do + let(:ready) do + true + end + + context 'with existing Mdm::Module::Detail#file' do + context 'with same Mdm::Module::Detail#mtime and File.mtime' do + it 'should not update module details' do + db_manager.should_not_receive(:update_module_details) + + update_all_module_details + end + end + + context 'without same Mdm::Module::Detail#mtime and File.mtime' do + let(:modification_time) do + # +1 as rand can return 0 and the time must be different for + # this context. + super() - (rand(1.day) + 1) + end + + it_should_behave_like 'Msf::DBManager#update_all_module_details refresh' + end + end + + # Emulates a module being removed or renamed + context 'without existing Mdm::Module::Detail#file' do + # have to compute modification manually since the + # `module_pathname` refers to a non-existent file and + # `module_pathname.mtime` would error. + let(:modification_time) do + Time.now.utc - 1.day + end + + let(:module_pathname) do + parent_pathname.join('exploits', 'deleted.rb') + end + + it 'should not update module details' do + db_manager.should_not_receive(:update_module_details) + + update_all_module_details + end + end + end + end + end + end + end + + context 'without migrated' do + it 'should not update module details' do + db_manager.should_not_receive(:update_module_details) + + update_all_module_details + end + end + end + + context '#update_module_details' do + def update_module_details + db_manager.update_module_details(module_instance) + end + + let(:loader) do + loader = framework.modules.send(:loaders).find { |loader| + loader.loadable?(parent_path) + } + + # Override load_error so that rspec will print it instead of going to framework log + def loader.load_error(module_path, error) + raise error + end + + loader + end + + let(:migrated) do + false + end + + let(:module_instance) do + # make sure the module is loaded into the module_set + loaded = loader.load_module(parent_path, module_type, module_reference_name) + + unless loaded + module_path = loader.module_path(parent_path, type, module_reference_name) + + fail "#{description} failed to load: #{module_path}" + end + + module_set.create(module_reference_name) + end + + let(:module_set) do + framework.modules.module_set(module_type) + end + + let(:module_type) do + 'exploit' + end + + let(:module_reference_name) do + 'windows/smb/ms08_067_netapi' + end + + let(:parent_path) do + parent_pathname.to_path + end + + let(:parent_pathname) do + Metasploit::Framework.root.join('modules') + end + + let(:type_directory) do + 'exploits' + end + + before(:each) do + db_manager.stub(:migrated => migrated) + end + + context 'with migrated' do + let(:migrated) do + true + end + + it 'should call module_to_details_hash to get Mdm::Module::Detail attributes and association attributes' do + db_manager.should_receive(:module_to_details_hash).and_call_original + + update_module_details + end + + it 'should create an Mdm::Module::Detail' do + expect { + update_module_details + }.to change(Mdm::Module::Detail, :count).by(1) + end + + + context 'module_to_details_hash' do + let(:module_to_details_hash) do + { + :mtype => module_type, + :privileged => privileged, + :rank => rank, + :refname => module_reference_name, + :stance => stance + } + end + + let(:privileged) do + FactoryGirl.generate :mdm_module_detail_privileged + end + + let(:rank) do + FactoryGirl.generate :mdm_module_detail_rank + end + + let(:stance) do + FactoryGirl.generate :mdm_module_detail_stance + end + + before(:each) do + db_manager.stub( + :module_to_details_hash + ).with( + module_instance + ).and_return( + module_to_details_hash + ) + end + + context 'Mdm::Module::Detail' do + subject(:module_detail) do + Mdm::Module::Detail.last + end + + before(:each) do + update_module_details + end + + it { expect(subject.mtype).to eq(module_type) } + it { expect(subject.privileged).to eq(privileged) } + it { expect(subject.rank).to eq(rank) } + it { expect(subject.ready).to be_truthy } + it { expect(subject.refname).to eq(module_reference_name) } + it { expect(subject.stance).to eq(stance) } + end + + context 'with :bits' do + let(:bits) do + [] + end + + before(:each) do + module_to_details_hash[:bits] = bits + end + + context 'with :action' do + let(:name) do + FactoryGirl.generate :mdm_module_action_name + end + + let(:bits) do + super() << [ + :action, + { + :name => name + } + ] + end + + it 'should create an Mdm::Module::Action' do + expect { + update_module_details + }.to change(Mdm::Module::Action, :count).by(1) + end + + context 'Mdm::Module::Action' do + subject(:module_action) do + module_detail.actions.last + end + + let(:module_detail) do + Mdm::Module::Detail.last + end + + before(:each) do + update_module_details + end + + it { expect(subject.name).to eq(name) } + end + end + + context 'with :arch' do + let(:name) do + FactoryGirl.generate :mdm_module_arch_name + end + + let(:bits) do + super() << [ + :arch, + { + :name => name + } + ] + end + + it 'should create an Mdm::Module::Arch' do + expect { + update_module_details + }.to change(Mdm::Module::Arch, :count).by(1) + end + + context 'Mdm::Module::Arch' do + subject(:module_arch) do + module_detail.archs.last + end + + let(:module_detail) do + Mdm::Module::Detail.last + end + + before(:each) do + update_module_details + end + + it { expect(subject.name).to eq(name) } + end + end + + context 'with :author' do + let(:email) do + FactoryGirl.generate :mdm_module_author_email + end + + let(:name) do + FactoryGirl.generate :mdm_module_author_name + end + + let(:bits) do + super() << [ + :author, + { + :email => email, + :name => name + } + ] + end + + it 'should create an Mdm::Module::Author' do + expect { + update_module_details + }.to change(Mdm::Module::Author, :count).by(1) + end + + context 'Mdm::Module::Author' do + subject(:module_author) do + module_detail.authors.last + end + + let(:module_detail) do + Mdm::Module::Detail.last + end + + before(:each) do + update_module_details + end + + it { expect(subject.name).to eq(name) } + it { expect(subject.email).to eq(email) } + end + end + + context 'with :platform' do + let(:bits) do + super() << [ + :platform, + { + :name => name + } + ] + end + + let(:name) do + FactoryGirl.generate :mdm_module_platform_name + end + + it 'should create an Mdm::Module::Platform' do + expect { + update_module_details + }.to change(Mdm::Module::Platform, :count).by(1) + end + + context 'Mdm::Module::Platform' do + subject(:module_platform) do + module_detail.platforms.last + end + + let(:module_detail) do + Mdm::Module::Detail.last + end + + before(:each) do + update_module_details + end + + it { expect(subject.name).to eq(name) } + end + end + + context 'with :ref' do + let(:bits) do + super() << [ + :ref, + { + :name => name + } + ] + end + + let(:name) do + FactoryGirl.generate :mdm_module_ref_name + end + + it 'should create an Mdm::Module::Ref' do + expect { + update_module_details + }.to change(Mdm::Module::Ref, :count).by(1) + end + + context 'Mdm::Module::Ref' do + subject(:module_ref) do + module_detail.refs.last + end + + let(:module_detail) do + Mdm::Module::Detail.last + end + + before(:each) do + update_module_details + end + + it { expect(subject.name).to eq(name) } + end + end + + context 'with :target' do + let(:bits) do + super() << [ + :target, + { + :index => index, + :name => name + } + ] + end + + let(:index) do + FactoryGirl.generate :mdm_module_target_index + end + + let(:name) do + FactoryGirl.generate :mdm_module_target_name + end + + it 'should create an Mdm::Module::Target' do + expect { + update_module_details + }.to change(Mdm::Module::Target, :count).by(1) + end + + context 'Mdm::Module::Target' do + subject(:module_target) do + module_detail.targets.last + end + + let(:module_detail) do + Mdm::Module::Detail.last + end + + before(:each) do + update_module_details + end + + it { expect(subject.index).to eq(index) } + it { expect(subject.name).to eq(name) } + end + end + end + end + + it_should_behave_like 'Msf::DBManager#update_module_details with module', + :reference_name => 'admin/2wire/xslt_password_reset', + :type => 'auxiliary' + + it_should_behave_like 'Msf::DBManager#update_module_details with module', + :reference_name => 'generic/none', + :type => 'encoder' + + it_should_behave_like 'Msf::DBManager#update_module_details with module', + :reference_name => 'windows/smb/ms08_067_netapi', + :type => 'exploit' + + it_should_behave_like 'Msf::DBManager#update_module_details with module', + :reference_name => 'x64/simple', + :type => 'nop' + + # @todo determine how to load a single payload to test payload type outside of msfconsole + + it_should_behave_like 'Msf::DBManager#update_module_details with module', + :reference_name => 'windows/escalate/screen_unlock', + :type => 'post' + end + + context 'without migrated' do + it 'should not create an Mdm::Module::Detail' do + expect { + update_module_details + }.to_not change(Mdm::Module::Detail, :count) + end + end + end +end \ No newline at end of file From 3a96ae9be9312709bc11efa7dbd20a0e99b623d4 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Thu, 9 Oct 2014 09:18:03 -0500 Subject: [PATCH 019/159] Move #match_values to Msf::DBManager::ModuleCache MSP-11124 `#match_values` is only used in `#search_modules`, so `#match_values` should be grouped with `#search_modules` in `Msf::DBManager::ModuleCache`. --- lib/msf/core/db_manager.rb | 12 ------------ lib/msf/core/db_manager/module_cache.rb | 13 +++++++++++++ 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/lib/msf/core/db_manager.rb b/lib/msf/core/db_manager.rb index 222a1c05c0..17f460a8f4 100644 --- a/lib/msf/core/db_manager.rb +++ b/lib/msf/core/db_manager.rb @@ -367,18 +367,6 @@ class DBManager end end - # Wraps values in +'%'+ for Arel::Prediciation#matches_any and other match* methods that map to SQL +'LIKE'+ or - # +'ILIKE'+ - # - # @param values [Set, #each] a list of strings. - # @return [Arrray] strings wrapped like %% - def match_values(values) - wrapped_values = values.collect { |value| - "%#{value}%" - } - - wrapped_values - end def rfc3330_reserved(ip) case ip.class.to_s diff --git a/lib/msf/core/db_manager/module_cache.rb b/lib/msf/core/db_manager/module_cache.rb index 73d76aa934..299faed685 100644 --- a/lib/msf/core/db_manager/module_cache.rb +++ b/lib/msf/core/db_manager/module_cache.rb @@ -13,6 +13,19 @@ module Msf::DBManager::ModuleCache # Instance Methods # + # Wraps values in +'%'+ for Arel::Prediciation#matches_any and other match* methods that map to SQL +'LIKE'+ or + # +'ILIKE'+ + # + # @param values [Set, #each] a list of strings. + # @return [Arrray] strings wrapped like %% + def match_values(values) + wrapped_values = values.collect { |value| + "%#{value}%" + } + + wrapped_values + end + def module_to_details_hash(m) res = {} bits = [] From f5ea81cd976d09ab232b65db03cab0eac9cc5c6f Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Thu, 9 Oct 2014 09:20:27 -0500 Subject: [PATCH 020/159] Test for #match_values MSP-11124 --- spec/support/shared/examples/msf/db_manager/module_cache.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/spec/support/shared/examples/msf/db_manager/module_cache.rb b/spec/support/shared/examples/msf/db_manager/module_cache.rb index 9d74460817..c2839a97de 100644 --- a/spec/support/shared/examples/msf/db_manager/module_cache.rb +++ b/spec/support/shared/examples/msf/db_manager/module_cache.rb @@ -1,4 +1,5 @@ shared_examples_for 'Msf::DBManager::ModuleCache' do + it { is_expected.to respond_to :match_values } it { is_expected.to respond_to :module_to_details_hash } it { is_expected.to respond_to :modules_cached } it { is_expected.to respond_to :modules_cached= } From b0147c994a32ca5d8dfc068e4583fb01a07ae56f Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Thu, 9 Oct 2014 09:35:19 -0500 Subject: [PATCH 021/159] Extract Msf::DBManager::IPAddress MSP-11124 Extract the IP address validation methods to `Msf::DBManager::IPAddress`. --- lib/msf/core/db_manager.rb | 66 ++------------------------- lib/msf/core/db_manager/ip_address.rb | 61 +++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 63 deletions(-) create mode 100644 lib/msf/core/db_manager/ip_address.rb diff --git a/lib/msf/core/db_manager.rb b/lib/msf/core/db_manager.rb index 17f460a8f4..4ea298a859 100644 --- a/lib/msf/core/db_manager.rb +++ b/lib/msf/core/db_manager.rb @@ -75,13 +75,15 @@ module Msf class DBManager extend Metasploit::Framework::Require + autoload :IPAddress, 'msf/core/db_manager/ip_address' autoload :ModuleCache, 'msf/core/db_manager/module_cache' autoload :Sink, 'msf/core/db_manager/sink' autoload :Workspace, 'msf/core/db_manager/workspace' - include Msf::DBManager::ImportMsfXml optionally_include_metasploit_credential_creation + include Msf::DBManager::ImportMsfXml + include Msf::DBManager::IPAddress include Msf::DBManager::Migration include Msf::DBManager::ModuleCache include Msf::DBManager::Sink @@ -367,68 +369,6 @@ class DBManager end end - - def rfc3330_reserved(ip) - case ip.class.to_s - when "PacketFu::Octets" - ip_x = ip.to_x - ip_i = ip.to_i - when "String" - if ipv46_validator(ip) - ip_x = ip - ip_i = Rex::Socket.addr_atoi(ip) - else - raise ArgumentError, "Invalid IP address: #{ip.inspect}" - end - when "Fixnum" - if (0..2**32-1).include? ip - ip_x = Rex::Socket.addr_itoa(ip) - ip_i = ip - else - raise ArgumentError, "Invalid IP address: #{ip.inspect}" - end - else - raise ArgumentError, "Invalid IP address: #{ip.inspect}" - end - return true if Rex::Socket::RangeWalker.new("0.0.0.0-0.255.255.255").include? ip_x - return true if Rex::Socket::RangeWalker.new("127.0.0.0-127.255.255.255").include? ip_x - return true if Rex::Socket::RangeWalker.new("169.254.0.0-169.254.255.255").include? ip_x - return true if Rex::Socket::RangeWalker.new("224.0.0.0-239.255.255.255").include? ip_x - return true if Rex::Socket::RangeWalker.new("255.255.255.255-255.255.255.255").include? ip_x - return false - end - - def ipv46_validator(addr) - ipv4_validator(addr) or ipv6_validator(addr) - end - - def ipv4_validator(addr) - return false unless addr.kind_of? String - Rex::Socket.is_ipv4?(addr) - end - - def ipv6_validator(addr) - Rex::Socket.is_ipv6?(addr) - end - - # Takes a space-delimited set of ips and ranges, and subjects - # them to RangeWalker for validation. Returns true or false. - def validate_ips(ips) - ret = true - begin - ips.split(/\s+/).each {|ip| - unless Rex::Socket::RangeWalker.new(ip).ranges - ret = false - break - end - } - rescue - ret = false - end - return ret - end - - # # Determines if the database is functional # diff --git a/lib/msf/core/db_manager/ip_address.rb b/lib/msf/core/db_manager/ip_address.rb new file mode 100644 index 0000000000..b69dd69bfe --- /dev/null +++ b/lib/msf/core/db_manager/ip_address.rb @@ -0,0 +1,61 @@ +module Msf::DBManager::IPAddress + def ipv46_validator(addr) + ipv4_validator(addr) or ipv6_validator(addr) + end + + def ipv4_validator(addr) + return false unless addr.kind_of? String + Rex::Socket.is_ipv4?(addr) + end + + def ipv6_validator(addr) + Rex::Socket.is_ipv6?(addr) + end + + def rfc3330_reserved(ip) + case ip.class.to_s + when "PacketFu::Octets" + ip_x = ip.to_x + ip_i = ip.to_i + when "String" + if ipv46_validator(ip) + ip_x = ip + ip_i = Rex::Socket.addr_atoi(ip) + else + raise ArgumentError, "Invalid IP address: #{ip.inspect}" + end + when "Fixnum" + if (0..2**32-1).include? ip + ip_x = Rex::Socket.addr_itoa(ip) + ip_i = ip + else + raise ArgumentError, "Invalid IP address: #{ip.inspect}" + end + else + raise ArgumentError, "Invalid IP address: #{ip.inspect}" + end + return true if Rex::Socket::RangeWalker.new("0.0.0.0-0.255.255.255").include? ip_x + return true if Rex::Socket::RangeWalker.new("127.0.0.0-127.255.255.255").include? ip_x + return true if Rex::Socket::RangeWalker.new("169.254.0.0-169.254.255.255").include? ip_x + return true if Rex::Socket::RangeWalker.new("224.0.0.0-239.255.255.255").include? ip_x + return true if Rex::Socket::RangeWalker.new("255.255.255.255-255.255.255.255").include? ip_x + return false + end + + # Takes a space-delimited set of ips and ranges, and subjects + # them to RangeWalker for validation. Returns true or false. + def validate_ips(ips) + ret = true + begin + ips.split(/\s+/).each {|ip| + unless Rex::Socket::RangeWalker.new(ip).ranges + ret = false + break + end + } + rescue + ret = false + end + return ret + end +end \ No newline at end of file From e723aa7eb3d84c99fe57696d058876a7978d458e Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Thu, 9 Oct 2014 09:40:11 -0500 Subject: [PATCH 022/159] Extract Msf::DBManager::IPAddress shared examples MSP-11124 --- spec/lib/msf/db_manager_spec.rb | 6 +----- spec/support/shared/examples/msf/db_manager/ip_address.rb | 7 +++++++ 2 files changed, 8 insertions(+), 5 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/ip_address.rb diff --git a/spec/lib/msf/db_manager_spec.rb b/spec/lib/msf/db_manager_spec.rb index 762fdd29f1..f9c6653999 100644 --- a/spec/lib/msf/db_manager_spec.rb +++ b/spec/lib/msf/db_manager_spec.rb @@ -18,6 +18,7 @@ describe Msf::DBManager do db_manager end + it_should_behave_like 'Msf::DBManager::IPAddress' it_should_behave_like 'Msf::DBManager::ImportMsfXml' it_should_behave_like 'Msf::DBManager::Migration' it_should_behave_like 'Msf::DBManager::ModuleCache' @@ -182,9 +183,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :initialize_database_support } it { is_expected.to respond_to :inspect_single_packet } it { is_expected.to respond_to :inspect_single_packet_http } - it { is_expected.to respond_to :ipv46_validator } - it { is_expected.to respond_to :ipv4_validator } - it { is_expected.to respond_to :ipv6_validator } it { is_expected.to respond_to :loots } it { is_expected.to respond_to :msf_import_timestamps } it { is_expected.to respond_to :netsparker_method_map } @@ -871,7 +869,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :request_sql } it { is_expected.to respond_to :requests } it { is_expected.to respond_to :rexmlify } - it { is_expected.to respond_to :rfc3330_reserved } it { is_expected.to respond_to :selected_host } it { is_expected.to respond_to :selected_id } it { is_expected.to respond_to :selected_port } @@ -889,7 +886,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :usable } it { is_expected.to respond_to :usable= } it { is_expected.to respond_to :validate_import_file } - it { is_expected.to respond_to :validate_ips } it { is_expected.to respond_to :vulns } it { is_expected.to respond_to :warn_about_rubies } end diff --git a/spec/support/shared/examples/msf/db_manager/ip_address.rb b/spec/support/shared/examples/msf/db_manager/ip_address.rb new file mode 100644 index 0000000000..4859b18825 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/ip_address.rb @@ -0,0 +1,7 @@ +shared_examples_for 'Msf::DBManager::IPAddress' do + it { is_expected.to respond_to :ipv46_validator } + it { is_expected.to respond_to :ipv4_validator } + it { is_expected.to respond_to :ipv6_validator } + it { is_expected.to respond_to :rfc3330_reserved } + it { is_expected.to respond_to :validate_ips } +end \ No newline at end of file From bb26f4f3035ddde36f9d46d3cee4e752e6bc35d2 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Thu, 9 Oct 2014 10:11:51 -0500 Subject: [PATCH 023/159] Extract Msf::DBManager::Wmap MSP-11124 Extract methods that are commented as related to WMAP. --- lib/msf/core/db_manager.rb | 259 +------------------------------- lib/msf/core/db_manager/wmap.rb | 189 +++++++++++++++++++++++ 2 files changed, 191 insertions(+), 257 deletions(-) create mode 100644 lib/msf/core/db_manager/wmap.rb diff --git a/lib/msf/core/db_manager.rb b/lib/msf/core/db_manager.rb index 4ea298a859..1dfef479d4 100644 --- a/lib/msf/core/db_manager.rb +++ b/lib/msf/core/db_manager.rb @@ -78,6 +78,7 @@ class DBManager autoload :IPAddress, 'msf/core/db_manager/ip_address' autoload :ModuleCache, 'msf/core/db_manager/module_cache' autoload :Sink, 'msf/core/db_manager/sink' + autoload :WMAP, 'msf/core/db_manager/wmap' autoload :Workspace, 'msf/core/db_manager/workspace' optionally_include_metasploit_credential_creation @@ -87,6 +88,7 @@ class DBManager include Msf::DBManager::Migration include Msf::DBManager::ModuleCache include Msf::DBManager::Sink + include Msf::DBManager::WMAP include Msf::DBManager::Workspace # Provides :framework and other accessors @@ -2720,263 +2722,6 @@ class DBManager } end - # - # WMAP - # Selected host - # - def selected_host - ::ActiveRecord::Base.connection_pool.with_connection { - selhost = ::Mdm::WmapTarget.where("selected != 0").first() - if selhost - return selhost.host - else - return - end - } - end - - # - # WMAP - # Selected target - # - def selected_wmap_target - ::ActiveRecord::Base.connection_pool.with_connection { - ::Mdm::WmapTarget.find.where("selected != 0") - } - end - - # - # WMAP - # Selected port - # - def selected_port - selected_wmap_target.port - end - - # - # WMAP - # Selected ssl - # - def selected_ssl - selected_wmap_target.ssl - end - - # - # WMAP - # Selected id - # - def selected_id - selected_wmap_target.object_id - end - - # - # WMAP - # This method iterates the requests table identifiying possible targets - # This method wiil be remove on second phase of db merging. - # - def each_distinct_target(&block) - request_distinct_targets.each do |target| - block.call(target) - end - end - - # - # WMAP - # This method returns a list of all possible targets available in requests - # This method wiil be remove on second phase of db merging. - # - def request_distinct_targets - ::ActiveRecord::Base.connection_pool.with_connection { - ::Mdm::WmapRequest.select('DISTINCT host,address,port,ssl') - } - end - - # - # WMAP - # This method iterates the requests table returning a list of all requests of a specific target - # - def each_request_target_with_path(&block) - target_requests('AND wmap_requests.path IS NOT NULL').each do |req| - block.call(req) - end - end - - # - # WMAP - # This method iterates the requests table returning a list of all requests of a specific target - # - def each_request_target_with_query(&block) - target_requests('AND wmap_requests.query IS NOT NULL').each do |req| - block.call(req) - end - end - - # - # WMAP - # This method iterates the requests table returning a list of all requests of a specific target - # - def each_request_target_with_body(&block) - target_requests('AND wmap_requests.body IS NOT NULL').each do |req| - block.call(req) - end - end - - # - # WMAP - # This method iterates the requests table returning a list of all requests of a specific target - # - def each_request_target_with_headers(&block) - target_requests('AND wmap_requests.headers IS NOT NULL').each do |req| - block.call(req) - end - end - - # - # WMAP - # This method iterates the requests table returning a list of all requests of a specific target - # - def each_request_target(&block) - target_requests('').each do |req| - block.call(req) - end - end - - # - # WMAP - # This method returns a list of all requests from target - # - def target_requests(extra_condition) - ::ActiveRecord::Base.connection_pool.with_connection { - ::Mdm::WmapRequest.where("wmap_requests.host = ? AND wmap_requests.port = ? #{extra_condition}",selected_host,selected_port) - } - end - - # - # WMAP - # This method iterates the requests table calling the supplied block with the - # request instance of each entry. - # - def each_request(&block) - requests.each do |request| - block.call(request) - end - end - - # - # WMAP - # This method allows to query directly the requests table. To be used mainly by modules - # - def request_sql(host,port,extra_condition) - ::ActiveRecord::Base.connection_pool.with_connection { - ::Mdm::WmapRequest.where("wmap_requests.host = ? AND wmap_requests.port = ? #{extra_condition}", host , port) - } - end - - # - # WMAP - # This methods returns a list of all targets in the database - # - def requests - ::ActiveRecord::Base.connection_pool.with_connection { - ::Mdm::WmapRequest.find(:all) - } - end - - # - # WMAP - # This method iterates the targets table calling the supplied block with the - # target instance of each entry. - # - def each_target(&block) - targets.each do |target| - block.call(target) - end - end - - # - # WMAP - # This methods returns a list of all targets in the database - # - def targets - ::ActiveRecord::Base.connection_pool.with_connection { - ::Mdm::WmapTarget.find(:all) - } - end - - # - # WMAP - # This methods deletes all targets from targets table in the database - # - def delete_all_targets - ::ActiveRecord::Base.connection_pool.with_connection { - ::Mdm::WmapTarget.delete_all - } - end - - # - # WMAP - # Find a target matching this id - # - def get_target(id) - ::ActiveRecord::Base.connection_pool.with_connection { - target = ::Mdm::WmapTarget.where("id = ?", id).first() - return target - } - end - - # - # WMAP - # Create a target - # - def create_target(host,port,ssl,sel) - ::ActiveRecord::Base.connection_pool.with_connection { - tar = ::Mdm::WmapTarget.create( - :host => host, - :address => host, - :port => port, - :ssl => ssl, - :selected => sel - ) - #framework.events.on_db_target(rec) - } - end - - - # - # WMAP - # Create a request (by hand) - # - def create_request(host,port,ssl,meth,path,headers,query,body,respcode,resphead,response) - ::ActiveRecord::Base.connection_pool.with_connection { - req = ::Mdm::WmapRequest.create( - :host => host, - :address => host, - :port => port, - :ssl => ssl, - :meth => meth, - :path => path, - :headers => headers, - :query => query, - :body => body, - :respcode => respcode, - :resphead => resphead, - :response => response - ) - #framework.events.on_db_request(rec) - } - end - - # - # WMAP - # Quick way to query the database (used by wmap_sql) - # - def sql_query(sqlquery) - ::ActiveRecord::Base.connection_pool.with_connection { - ActiveRecord::Base.connection.select_all(sqlquery) - } - end - - # Returns a REXML::Document from the given data. def rexmlify(data) if data.kind_of?(REXML::Document) diff --git a/lib/msf/core/db_manager/wmap.rb b/lib/msf/core/db_manager/wmap.rb new file mode 100644 index 0000000000..79620e6675 --- /dev/null +++ b/lib/msf/core/db_manager/wmap.rb @@ -0,0 +1,189 @@ +# @note Wmap is a plugin and so these methods, that are only meant for that plugin, should not be part of the core +# library. +module Msf::DBManager::WMAP + # Create a request (by hand) + def create_request(host,port,ssl,meth,path,headers,query,body,respcode,resphead,response) + ::ActiveRecord::Base.connection_pool.with_connection { + req = ::Mdm::WmapRequest.create( + :host => host, + :address => host, + :port => port, + :ssl => ssl, + :meth => meth, + :path => path, + :headers => headers, + :query => query, + :body => body, + :respcode => respcode, + :resphead => resphead, + :response => response + ) + #framework.events.on_db_request(rec) + } + end + + # Create a target + def create_target(host,port,ssl,sel) + ::ActiveRecord::Base.connection_pool.with_connection { + tar = ::Mdm::WmapTarget.create( + :host => host, + :address => host, + :port => port, + :ssl => ssl, + :selected => sel + ) + #framework.events.on_db_target(rec) + } + end + + # This methods deletes all targets from targets table in the database + def delete_all_targets + ::ActiveRecord::Base.connection_pool.with_connection { + ::Mdm::WmapTarget.delete_all + } + end + + # This method iterates the requests table identifiying possible targets + # This method wiil be remove on second phase of db merging. + def each_distinct_target(&block) + request_distinct_targets.each do |target| + block.call(target) + end + end + + # This method iterates the requests table calling the supplied block with the + # request instance of each entry. + def each_request(&block) + requests.each do |request| + block.call(request) + end + end + + # This method iterates the requests table returning a list of all requests of a specific target + def each_request_target(&block) + target_requests('').each do |req| + block.call(req) + end + end + + # This method iterates the requests table returning a list of all requests of a specific target + def each_request_target_with_body(&block) + target_requests('AND wmap_requests.body IS NOT NULL').each do |req| + block.call(req) + end + end + + # This method iterates the requests table returning a list of all requests of a specific target + def each_request_target_with_headers(&block) + target_requests('AND wmap_requests.headers IS NOT NULL').each do |req| + block.call(req) + end + end + + # This method iterates the requests table returning a list of all requests of a specific target + def each_request_target_with_path(&block) + target_requests('AND wmap_requests.path IS NOT NULL').each do |req| + block.call(req) + end + end + + # This method iterates the requests table returning a list of all requests of a specific target + def each_request_target_with_query(&block) + target_requests('AND wmap_requests.query IS NOT NULL').each do |req| + block.call(req) + end + end + + # This method iterates the targets table calling the supplied block with the + # target instance of each entry. + def each_target(&block) + targets.each do |target| + block.call(target) + end + end + + # Find a target matching this id + def get_target(id) + ::ActiveRecord::Base.connection_pool.with_connection { + target = ::Mdm::WmapTarget.where("id = ?", id).first() + return target + } + end + + # This method returns a list of all possible targets available in requests + # This method wiil be remove on second phase of db merging. + def request_distinct_targets + ::ActiveRecord::Base.connection_pool.with_connection { + ::Mdm::WmapRequest.select('DISTINCT host,address,port,ssl') + } + end + + # This method allows to query directly the requests table. To be used mainly by modules + def request_sql(host,port,extra_condition) + ::ActiveRecord::Base.connection_pool.with_connection { + ::Mdm::WmapRequest.where("wmap_requests.host = ? AND wmap_requests.port = ? #{extra_condition}", host , port) + } + end + + # This methods returns a list of all targets in the database + def requests + ::ActiveRecord::Base.connection_pool.with_connection { + ::Mdm::WmapRequest.find(:all) + } + end + + # Selected host + def selected_host + ::ActiveRecord::Base.connection_pool.with_connection { + selhost = ::Mdm::WmapTarget.where("selected != 0").first() + if selhost + return selhost.host + else + return + end + } + end + + # Selected id + def selected_id + selected_wmap_target.object_id + end + + # Selected port + def selected_port + selected_wmap_target.port + end + + # Selected ssl + def selected_ssl + selected_wmap_target.ssl + end + + # Selected target + def selected_wmap_target + ::ActiveRecord::Base.connection_pool.with_connection { + ::Mdm::WmapTarget.find.where("selected != 0") + } + end + + # Quick way to query the database (used by wmap_sql) + def sql_query(sqlquery) + ::ActiveRecord::Base.connection_pool.with_connection { + ActiveRecord::Base.connection.select_all(sqlquery) + } + end + + # This method returns a list of all requests from target + def target_requests(extra_condition) + ::ActiveRecord::Base.connection_pool.with_connection { + ::Mdm::WmapRequest.where("wmap_requests.host = ? AND wmap_requests.port = ? #{extra_condition}",selected_host,selected_port) + } + end + + # This methods returns a list of all targets in the database + def targets + ::ActiveRecord::Base.connection_pool.with_connection { + ::Mdm::WmapTarget.find(:all) + } + end +end \ No newline at end of file From 148ad8b050874015026d3eb7fa8965b459f33bc6 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Thu, 9 Oct 2014 10:49:26 -0500 Subject: [PATCH 024/159] Extract Msf::DBManager::WMAP shared examples MSP-11124 --- spec/lib/msf/db_manager_spec.rb | 24 +----------------- .../shared/examples/msf/db_manager/wmap.rb | 25 +++++++++++++++++++ 2 files changed, 26 insertions(+), 23 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/wmap.rb diff --git a/spec/lib/msf/db_manager_spec.rb b/spec/lib/msf/db_manager_spec.rb index f9c6653999..bad9060903 100644 --- a/spec/lib/msf/db_manager_spec.rb +++ b/spec/lib/msf/db_manager_spec.rb @@ -23,6 +23,7 @@ describe Msf::DBManager do it_should_behave_like 'Msf::DBManager::Migration' it_should_behave_like 'Msf::DBManager::ModuleCache' it_should_behave_like 'Msf::DBManager::Sink' + it_should_behave_like 'Msf::DBManager::WMAP' it_should_behave_like 'Msf::DBManager::Workspace' context 'CONSTANTS' do @@ -60,31 +61,20 @@ describe Msf::DBManager do it { is_expected.to respond_to :connect } it { is_expected.to respond_to :connection_established? } it { is_expected.to respond_to :create_db } - it { is_expected.to respond_to :create_request } - it { is_expected.to respond_to :create_target } it { is_expected.to respond_to :creds } it { is_expected.to respond_to :dehex } it { is_expected.to respond_to :del_host } it { is_expected.to respond_to :del_service } - it { is_expected.to respond_to :delete_all_targets } it { is_expected.to respond_to :disconnect } it { is_expected.to respond_to :driver } it { is_expected.to respond_to :drivers } it { is_expected.to respond_to :drivers= } it { is_expected.to respond_to :each_cred } - it { is_expected.to respond_to :each_distinct_target } it { is_expected.to respond_to :each_exploited_host } it { is_expected.to respond_to :each_host } it { is_expected.to respond_to :each_loot } it { is_expected.to respond_to :each_note } - it { is_expected.to respond_to :each_request } - it { is_expected.to respond_to :each_request_target } - it { is_expected.to respond_to :each_request_target_with_body } - it { is_expected.to respond_to :each_request_target_with_headers } - it { is_expected.to respond_to :each_request_target_with_path } - it { is_expected.to respond_to :each_request_target_with_query } it { is_expected.to respond_to :each_service } - it { is_expected.to respond_to :each_target } it { is_expected.to respond_to :each_vuln } it { is_expected.to respond_to :emit } it { is_expected.to respond_to :error } @@ -110,7 +100,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :get_ref } it { is_expected.to respond_to :get_service } it { is_expected.to respond_to :get_session } - it { is_expected.to respond_to :get_target } it { is_expected.to respond_to :get_vuln } it { is_expected.to respond_to :has_host? } it { is_expected.to respond_to :has_ref? } @@ -865,20 +854,9 @@ describe Msf::DBManager do it { is_expected.to respond_to :report_web_site } it { is_expected.to respond_to :report_web_vuln } it { is_expected.to respond_to :reports } - it { is_expected.to respond_to :request_distinct_targets } - it { is_expected.to respond_to :request_sql } - it { is_expected.to respond_to :requests } it { is_expected.to respond_to :rexmlify } - it { is_expected.to respond_to :selected_host } - it { is_expected.to respond_to :selected_id } - it { is_expected.to respond_to :selected_port } - it { is_expected.to respond_to :selected_ssl } - it { is_expected.to respond_to :selected_wmap_target } it { is_expected.to respond_to :service_name_map } it { is_expected.to respond_to :services } - it { is_expected.to respond_to :sql_query } - it { is_expected.to respond_to :target_requests } - it { is_expected.to respond_to :targets } it { is_expected.to respond_to :tasks } it { is_expected.to respond_to :unserialize_object } it { is_expected.to respond_to :update_host_via_sysinfo } diff --git a/spec/support/shared/examples/msf/db_manager/wmap.rb b/spec/support/shared/examples/msf/db_manager/wmap.rb new file mode 100644 index 0000000000..549f495b67 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/wmap.rb @@ -0,0 +1,25 @@ +shared_examples_for 'Msf::DBManager::WMAP' do + it { is_expected.to respond_to :create_request } + it { is_expected.to respond_to :create_target } + it { is_expected.to respond_to :delete_all_targets } + it { is_expected.to respond_to :each_distinct_target } + it { is_expected.to respond_to :each_request } + it { is_expected.to respond_to :each_request_target } + it { is_expected.to respond_to :each_request_target_with_body } + it { is_expected.to respond_to :each_request_target_with_headers } + it { is_expected.to respond_to :each_request_target_with_path } + it { is_expected.to respond_to :each_request_target_with_query } + it { is_expected.to respond_to :each_target } + it { is_expected.to respond_to :get_target } + it { is_expected.to respond_to :request_distinct_targets } + it { is_expected.to respond_to :request_sql } + it { is_expected.to respond_to :requests } + it { is_expected.to respond_to :selected_host } + it { is_expected.to respond_to :selected_id } + it { is_expected.to respond_to :selected_port } + it { is_expected.to respond_to :selected_ssl } + it { is_expected.to respond_to :selected_wmap_target } + it { is_expected.to respond_to :sql_query } + it { is_expected.to respond_to :target_requests } + it { is_expected.to respond_to :targets } +end \ No newline at end of file From 0cfac322901f4f2c9ec2d3300efb2068f8995bfb Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Thu, 9 Oct 2014 11:11:36 -0500 Subject: [PATCH 025/159] Extract Msf::DBManager::Host MSP-11124 Extract methods related to `Mdm::Host`s. --- lib/msf/core/db_manager.rb | 339 +------------------------------- lib/msf/core/db_manager/host.rb | 328 ++++++++++++++++++++++++++++++ 2 files changed, 330 insertions(+), 337 deletions(-) create mode 100644 lib/msf/core/db_manager/host.rb diff --git a/lib/msf/core/db_manager.rb b/lib/msf/core/db_manager.rb index 1dfef479d4..b3ec31c4b6 100644 --- a/lib/msf/core/db_manager.rb +++ b/lib/msf/core/db_manager.rb @@ -75,6 +75,7 @@ module Msf class DBManager extend Metasploit::Framework::Require + autoload :Host, 'msf/core/db_manager/host' autoload :IPAddress, 'msf/core/db_manager/ip_address' autoload :ModuleCache, 'msf/core/db_manager/module_cache' autoload :Sink, 'msf/core/db_manager/sink' @@ -83,6 +84,7 @@ class DBManager optionally_include_metasploit_credential_creation + include Msf::DBManager::Host include Msf::DBManager::ImportMsfXml include Msf::DBManager::IPAddress include Msf::DBManager::Migration @@ -380,267 +382,6 @@ class DBManager } end - # - # Find a host. Performs no database writes. - # - def get_host(opts) - if opts.kind_of? ::Mdm::Host - return opts - elsif opts.kind_of? String - raise RuntimeError, "This invokation of get_host is no longer supported: #{caller}" - else - address = opts[:addr] || opts[:address] || opts[:host] || return - return address if address.kind_of? ::Mdm::Host - end - ::ActiveRecord::Base.connection_pool.with_connection { - wspace = opts.delete(:workspace) || workspace - if wspace.kind_of? String - wspace = find_workspace(wspace) - end - - address = normalize_host(address) - return wspace.hosts.find_by_address(address) - } - end - - # - # Exactly like report_host but waits for the database to create a host and returns it. - # - def find_or_create_host(opts) - report_host(opts) - end - - # - # Report a host's attributes such as operating system and service pack - # - # The opts parameter MUST contain - # +:host+:: -- the host's ip address - # - # The opts parameter can contain: - # +:state+:: -- one of the Msf::HostState constants - # +:os_name+:: -- something like "Windows", "Linux", or "Mac OS X" - # +:os_flavor+:: -- something like "Enterprise", "Pro", or "Home" - # +:os_sp+:: -- something like "SP2" - # +:os_lang+:: -- something like "English", "French", or "en-US" - # +:arch+:: -- one of the ARCH_* constants - # +:mac+:: -- the host's MAC address - # +:scope+:: -- interface identifier for link-local IPv6 - # +:virtual_host+:: -- the name of the VM host software, eg "VMWare", "QEMU", "Xen", etc. - # - def report_host(opts) - - return if not active - addr = opts.delete(:host) || return - - # Sometimes a host setup through a pivot will see the address as "Remote Pipe" - if addr.eql? "Remote Pipe" - return - end - - ::ActiveRecord::Base.connection_pool.with_connection { - wspace = opts.delete(:workspace) || workspace - if wspace.kind_of? String - wspace = find_workspace(wspace) - end - - ret = { } - - if not addr.kind_of? ::Mdm::Host - addr = normalize_host(addr) - addr, scope = addr.split('%', 2) - opts[:scope] = scope if scope - - unless ipv46_validator(addr) - raise ::ArgumentError, "Invalid IP address in report_host(): #{addr}" - end - - if opts[:comm] and opts[:comm].length > 0 - host = wspace.hosts.find_or_initialize_by_address_and_comm(addr, opts[:comm]) - else - host = wspace.hosts.find_or_initialize_by_address(addr) - end - else - host = addr - end - - # Truncate the info field at the maximum field length - if opts[:info] - opts[:info] = opts[:info][0,65535] - end - - # Truncate the name field at the maximum field length - if opts[:name] - opts[:name] = opts[:name][0,255] - end - - opts.each { |k,v| - if (host.attribute_names.include?(k.to_s)) - unless host.attribute_locked?(k.to_s) - host[k] = v.to_s.gsub(/[\x00-\x1f]/n, '') - end - else - dlog("Unknown attribute for ::Mdm::Host: #{k}") - end - } - host.info = host.info[0,::Mdm::Host.columns_hash["info"].limit] if host.info - - # Set default fields if needed - host.state = HostState::Alive if not host.state - host.comm = '' if not host.comm - host.workspace = wspace if not host.workspace - - if host.changed? - msf_import_timestamps(opts,host) - host.save! - end - - if opts[:task] - Mdm::TaskHost.create( - :task => opts[:task], - :host => host - ) - end - - host - } - end - - - # - # Update a host's attributes via semi-standardized sysinfo hash (Meterpreter) - # - # The opts parameter MUST contain the following entries - # +:host+:: -- the host's ip address - # +:info+:: -- the information hash - # * 'Computer' -- the host name - # * 'OS' -- the operating system string - # * 'Architecture' -- the hardware architecture - # * 'System Language' -- the system language - # - # The opts parameter can contain: - # +:workspace+:: -- the workspace for this host - # - def update_host_via_sysinfo(opts) - - return if not active - addr = opts.delete(:host) || return - info = opts.delete(:info) || return - - # Sometimes a host setup through a pivot will see the address as "Remote Pipe" - if addr.eql? "Remote Pipe" - return - end - - ::ActiveRecord::Base.connection_pool.with_connection { - wspace = opts.delete(:workspace) || workspace - if wspace.kind_of? String - wspace = find_workspace(wspace) - end - - if not addr.kind_of? ::Mdm::Host - addr = normalize_host(addr) - addr, scope = addr.split('%', 2) - opts[:scope] = scope if scope - - unless ipv46_validator(addr) - raise ::ArgumentError, "Invalid IP address in report_host(): #{addr}" - end - - if opts[:comm] and opts[:comm].length > 0 - host = wspace.hosts.find_or_initialize_by_address_and_comm(addr, opts[:comm]) - else - host = wspace.hosts.find_or_initialize_by_address(addr) - end - else - host = addr - end - - res = {} - - if info['Computer'] - res[:name] = info['Computer'] - end - - if info['Architecture'] - res[:arch] = info['Architecture'].split(/\s+/).first - end - - if info['OS'] =~ /^Windows\s*([^\(]+)\(([^\)]+)\)/i - res[:os_name] = "Windows #{$1.strip}" - build = $2.strip - - if build =~ /Service Pack (\d+)/ - res[:os_sp] = "SP" + $1 - end - end - - if info["System Language"] - case info["System Language"] - when /^en_/ - res[:os_lang] = "English" - end - end - - - # Truncate the info field at the maximum field length - if res[:info] - res[:info] = res[:info][0,65535] - end - - # Truncate the name field at the maximum field length - if res[:name] - res[:name] = res[:name][0,255] - end - - res.each { |k,v| - - if (host.attribute_names.include?(k.to_s)) - unless host.attribute_locked?(k.to_s) - host[k] = v.to_s.gsub(/[\x00-\x1f]/n, '') - end - else - dlog("Unknown attribute for Host: #{k}") - end - } - - # Set default fields if needed - host.state = HostState::Alive if not host.state - host.comm = '' if not host.comm - host.workspace = wspace if not host.workspace - - if host.changed? - host.save! - end - - host - } - end - # - # Iterates over the hosts table calling the supplied block with the host - # instance of each entry. - # - def each_host(wspace=workspace, &block) - ::ActiveRecord::Base.connection_pool.with_connection { - wspace.hosts.each do |host| - block.call(host) - end - } - end - - # - # Returns a list of all hosts in the database - # - def hosts(wspace = workspace, only_up = false, addresses = nil) - ::ActiveRecord::Base.connection_pool.with_connection { - conditions = {} - conditions[:state] = [Msf::HostState::Alive, Msf::HostState::Unknown] if only_up - conditions[:address] = addresses if addresses - wspace.hosts.where(conditions).order(:address) - } - end - - - def find_or_create_service(opts) report_service(opts) end @@ -2066,17 +1807,6 @@ class DBManager ) end - # - # Deletes a host and associated data matching this address/comm - # - def del_host(wspace, address, comm='') - ::ActiveRecord::Base.connection_pool.with_connection { - address, scope = address.split('%', 2) - host = wspace.hosts.find_by_address_and_comm(address, comm) - host.destroy if host - } - end - # # Deletes a port and associated vulns matching this port # @@ -2108,16 +1838,6 @@ class DBManager } end - # - # Look for an address across all comms - # - def has_host?(wspace,addr) - ::ActiveRecord::Base.connection_pool.with_connection { - address, scope = addr.split('%', 2) - wspace.hosts.find_by_address(addr) - } - end - def events(wspace=workspace) ::ActiveRecord::Base.connection_pool.with_connection { wspace.events.find :all, :order => 'created_at ASC' @@ -5923,61 +5643,6 @@ class DBManager end end - # - # Returns something suitable for the +:host+ parameter to the various report_* methods - # - # Takes a Host object, a Session object, an Msf::Session object or a String - # address - # - def normalize_host(host) - return host if host.kind_of? ::Mdm::Host - norm_host = nil - - if (host.kind_of? String) - - if Rex::Socket.is_ipv4?(host) - # If it's an IPv4 addr with a port on the end, strip the port - if host =~ /((\d{1,3}\.){3}\d{1,3}):\d+/ - norm_host = $1 - else - norm_host = host - end - elsif Rex::Socket.is_ipv6?(host) - # If it's an IPv6 addr, drop the scope - address, scope = host.split('%', 2) - norm_host = address - else - norm_host = Rex::Socket.getaddress(host, true) - end - elsif host.kind_of? ::Mdm::Session - norm_host = host.host - elsif host.respond_to?(:session_host) - # Then it's an Msf::Session object - thost = host.session_host - norm_host = thost - end - - # If we got here and don't have a norm_host yet, it could be a - # Msf::Session object with an empty or nil tunnel_host and tunnel_peer; - # see if it has a socket and use its peerhost if so. - if ( - norm_host.nil? and - host.respond_to?(:sock) and - host.sock.respond_to?(:peerhost) and - host.sock.peerhost.to_s.length > 0 - ) - norm_host = session.sock.peerhost - end - # If We got here and still don't have a real host, there's nothing left - # to try, just log it and return what we were given - if not norm_host - dlog("Host could not be normalized: #{host.inspect}") - norm_host = host - end - - norm_host - end - # A way to sneak the yield back into the db importer. # Used by the SAX parsers. def emit(sym,data,&block) diff --git a/lib/msf/core/db_manager/host.rb b/lib/msf/core/db_manager/host.rb new file mode 100644 index 0000000000..b4ff90063f --- /dev/null +++ b/lib/msf/core/db_manager/host.rb @@ -0,0 +1,328 @@ +module Msf::DBManager::Host + # Deletes a host and associated data matching this address/comm + def del_host(wspace, address, comm='') + ::ActiveRecord::Base.connection_pool.with_connection { + address, scope = address.split('%', 2) + host = wspace.hosts.find_by_address_and_comm(address, comm) + host.destroy if host + } + end + + # + # Iterates over the hosts table calling the supplied block with the host + # instance of each entry. + # + def each_host(wspace=workspace, &block) + ::ActiveRecord::Base.connection_pool.with_connection { + wspace.hosts.each do |host| + block.call(host) + end + } + end + + # Exactly like report_host but waits for the database to create a host and returns it. + def find_or_create_host(opts) + report_host(opts) + end + + # + # Find a host. Performs no database writes. + # + def get_host(opts) + if opts.kind_of? ::Mdm::Host + return opts + elsif opts.kind_of? String + raise RuntimeError, "This invokation of get_host is no longer supported: #{caller}" + else + address = opts[:addr] || opts[:address] || opts[:host] || return + return address if address.kind_of? ::Mdm::Host + end + ::ActiveRecord::Base.connection_pool.with_connection { + wspace = opts.delete(:workspace) || workspace + if wspace.kind_of? String + wspace = find_workspace(wspace) + end + + address = normalize_host(address) + return wspace.hosts.find_by_address(address) + } + end + + # Look for an address across all comms + def has_host?(wspace,addr) + ::ActiveRecord::Base.connection_pool.with_connection { + address, scope = addr.split('%', 2) + wspace.hosts.find_by_address(addr) + } + end + + # Returns a list of all hosts in the database + def hosts(wspace = workspace, only_up = false, addresses = nil) + ::ActiveRecord::Base.connection_pool.with_connection { + conditions = {} + conditions[:state] = [Msf::HostState::Alive, Msf::HostState::Unknown] if only_up + conditions[:address] = addresses if addresses + wspace.hosts.where(conditions).order(:address) + } + end + + # + # Returns something suitable for the +:host+ parameter to the various report_* methods + # + # Takes a Host object, a Session object, an Msf::Session object or a String + # address + # + def normalize_host(host) + return host if host.kind_of? ::Mdm::Host + norm_host = nil + + if (host.kind_of? String) + + if Rex::Socket.is_ipv4?(host) + # If it's an IPv4 addr with a port on the end, strip the port + if host =~ /((\d{1,3}\.){3}\d{1,3}):\d+/ + norm_host = $1 + else + norm_host = host + end + elsif Rex::Socket.is_ipv6?(host) + # If it's an IPv6 addr, drop the scope + address, scope = host.split('%', 2) + norm_host = address + else + norm_host = Rex::Socket.getaddress(host, true) + end + elsif host.kind_of? ::Mdm::Session + norm_host = host.host + elsif host.respond_to?(:session_host) + # Then it's an Msf::Session object + thost = host.session_host + norm_host = thost + end + + # If we got here and don't have a norm_host yet, it could be a + # Msf::Session object with an empty or nil tunnel_host and tunnel_peer; + # see if it has a socket and use its peerhost if so. + if ( + norm_host.nil? and + host.respond_to?(:sock) and + host.sock.respond_to?(:peerhost) and + host.sock.peerhost.to_s.length > 0 + ) + norm_host = session.sock.peerhost + end + # If We got here and still don't have a real host, there's nothing left + # to try, just log it and return what we were given + if not norm_host + dlog("Host could not be normalized: #{host.inspect}") + norm_host = host + end + + norm_host + end + + # + # Report a host's attributes such as operating system and service pack + # + # The opts parameter MUST contain + # +:host+:: -- the host's ip address + # + # The opts parameter can contain: + # +:state+:: -- one of the Msf::HostState constants + # +:os_name+:: -- something like "Windows", "Linux", or "Mac OS X" + # +:os_flavor+:: -- something like "Enterprise", "Pro", or "Home" + # +:os_sp+:: -- something like "SP2" + # +:os_lang+:: -- something like "English", "French", or "en-US" + # +:arch+:: -- one of the ARCH_* constants + # +:mac+:: -- the host's MAC address + # +:scope+:: -- interface identifier for link-local IPv6 + # +:virtual_host+:: -- the name of the VM host software, eg "VMWare", "QEMU", "Xen", etc. + # + def report_host(opts) + + return if not active + addr = opts.delete(:host) || return + + # Sometimes a host setup through a pivot will see the address as "Remote Pipe" + if addr.eql? "Remote Pipe" + return + end + + ::ActiveRecord::Base.connection_pool.with_connection { + wspace = opts.delete(:workspace) || workspace + if wspace.kind_of? String + wspace = find_workspace(wspace) + end + + ret = { } + + if not addr.kind_of? ::Mdm::Host + addr = normalize_host(addr) + addr, scope = addr.split('%', 2) + opts[:scope] = scope if scope + + unless ipv46_validator(addr) + raise ::ArgumentError, "Invalid IP address in report_host(): #{addr}" + end + + if opts[:comm] and opts[:comm].length > 0 + host = wspace.hosts.find_or_initialize_by_address_and_comm(addr, opts[:comm]) + else + host = wspace.hosts.find_or_initialize_by_address(addr) + end + else + host = addr + end + + # Truncate the info field at the maximum field length + if opts[:info] + opts[:info] = opts[:info][0,65535] + end + + # Truncate the name field at the maximum field length + if opts[:name] + opts[:name] = opts[:name][0,255] + end + + opts.each { |k,v| + if (host.attribute_names.include?(k.to_s)) + unless host.attribute_locked?(k.to_s) + host[k] = v.to_s.gsub(/[\x00-\x1f]/n, '') + end + else + dlog("Unknown attribute for ::Mdm::Host: #{k}") + end + } + host.info = host.info[0,::Mdm::Host.columns_hash["info"].limit] if host.info + + # Set default fields if needed + host.state = Msf::HostState::Alive if not host.state + host.comm = '' if not host.comm + host.workspace = wspace if not host.workspace + + if host.changed? + msf_import_timestamps(opts,host) + host.save! + end + + if opts[:task] + Mdm::TaskHost.create( + :task => opts[:task], + :host => host + ) + end + + host + } + end + + # + # Update a host's attributes via semi-standardized sysinfo hash (Meterpreter) + # + # The opts parameter MUST contain the following entries + # +:host+:: -- the host's ip address + # +:info+:: -- the information hash + # * 'Computer' -- the host name + # * 'OS' -- the operating system string + # * 'Architecture' -- the hardware architecture + # * 'System Language' -- the system language + # + # The opts parameter can contain: + # +:workspace+:: -- the workspace for this host + # + def update_host_via_sysinfo(opts) + + return if not active + addr = opts.delete(:host) || return + info = opts.delete(:info) || return + + # Sometimes a host setup through a pivot will see the address as "Remote Pipe" + if addr.eql? "Remote Pipe" + return + end + + ::ActiveRecord::Base.connection_pool.with_connection { + wspace = opts.delete(:workspace) || workspace + if wspace.kind_of? String + wspace = find_workspace(wspace) + end + + if not addr.kind_of? ::Mdm::Host + addr = normalize_host(addr) + addr, scope = addr.split('%', 2) + opts[:scope] = scope if scope + + unless ipv46_validator(addr) + raise ::ArgumentError, "Invalid IP address in report_host(): #{addr}" + end + + if opts[:comm] and opts[:comm].length > 0 + host = wspace.hosts.find_or_initialize_by_address_and_comm(addr, opts[:comm]) + else + host = wspace.hosts.find_or_initialize_by_address(addr) + end + else + host = addr + end + + res = {} + + if info['Computer'] + res[:name] = info['Computer'] + end + + if info['Architecture'] + res[:arch] = info['Architecture'].split(/\s+/).first + end + + if info['OS'] =~ /^Windows\s*([^\(]+)\(([^\)]+)\)/i + res[:os_name] = "Windows #{$1.strip}" + build = $2.strip + + if build =~ /Service Pack (\d+)/ + res[:os_sp] = "SP" + $1 + end + end + + if info["System Language"] + case info["System Language"] + when /^en_/ + res[:os_lang] = "English" + end + end + + + # Truncate the info field at the maximum field length + if res[:info] + res[:info] = res[:info][0,65535] + end + + # Truncate the name field at the maximum field length + if res[:name] + res[:name] = res[:name][0,255] + end + + res.each { |k,v| + + if (host.attribute_names.include?(k.to_s)) + unless host.attribute_locked?(k.to_s) + host[k] = v.to_s.gsub(/[\x00-\x1f]/n, '') + end + else + dlog("Unknown attribute for Host: #{k}") + end + } + + # Set default fields if needed + host.state = Msf::HostState::Alive if not host.state + host.comm = '' if not host.comm + host.workspace = wspace if not host.workspace + + if host.changed? + host.save! + end + + host + } + end +end \ No newline at end of file From e7800697eb7fa77a8d8021ee248a1a66df8c9834 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Thu, 9 Oct 2014 11:16:04 -0500 Subject: [PATCH 026/159] Extract Msf::DBManager::Host shared examples MSP-11124 --- spec/lib/msf/db_manager_spec.rb | 10 +--------- spec/support/shared/examples/msf/db_manager/host.rb | 11 +++++++++++ 2 files changed, 12 insertions(+), 9 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/host.rb diff --git a/spec/lib/msf/db_manager_spec.rb b/spec/lib/msf/db_manager_spec.rb index bad9060903..c93ff36df2 100644 --- a/spec/lib/msf/db_manager_spec.rb +++ b/spec/lib/msf/db_manager_spec.rb @@ -18,6 +18,7 @@ describe Msf::DBManager do db_manager end + it_should_behave_like 'Msf::DBManager::Host' it_should_behave_like 'Msf::DBManager::IPAddress' it_should_behave_like 'Msf::DBManager::ImportMsfXml' it_should_behave_like 'Msf::DBManager::Migration' @@ -63,7 +64,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :create_db } it { is_expected.to respond_to :creds } it { is_expected.to respond_to :dehex } - it { is_expected.to respond_to :del_host } it { is_expected.to respond_to :del_service } it { is_expected.to respond_to :disconnect } it { is_expected.to respond_to :driver } @@ -71,7 +71,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :drivers= } it { is_expected.to respond_to :each_cred } it { is_expected.to respond_to :each_exploited_host } - it { is_expected.to respond_to :each_host } it { is_expected.to respond_to :each_loot } it { is_expected.to respond_to :each_note } it { is_expected.to respond_to :each_service } @@ -82,7 +81,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :exploited_hosts } it { is_expected.to respond_to :find_or_create_client } it { is_expected.to respond_to :find_or_create_cred } - it { is_expected.to respond_to :find_or_create_host } it { is_expected.to respond_to :find_or_create_loot } it { is_expected.to respond_to :find_or_create_note } it { is_expected.to respond_to :find_or_create_ref } @@ -96,15 +94,12 @@ describe Msf::DBManager do it { is_expected.to respond_to :find_vuln_by_details } it { is_expected.to respond_to :find_vuln_by_refs } it { is_expected.to respond_to :get_client } - it { is_expected.to respond_to :get_host } it { is_expected.to respond_to :get_ref } it { is_expected.to respond_to :get_service } it { is_expected.to respond_to :get_session } it { is_expected.to respond_to :get_vuln } - it { is_expected.to respond_to :has_host? } it { is_expected.to respond_to :has_ref? } it { is_expected.to respond_to :has_vuln? } - it { is_expected.to respond_to :hosts } it { is_expected.to respond_to :import } it { is_expected.to respond_to :import_acunetix_noko_stream } it { is_expected.to respond_to :import_acunetix_xml } @@ -182,7 +177,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :nexpose_refs_to_struct } it { is_expected.to respond_to :nils_for_nulls } it { is_expected.to respond_to :nmap_msf_service_map } - it { is_expected.to respond_to :normalize_host } it { is_expected.to respond_to :notes } it { is_expected.to respond_to :report_artifact } it { is_expected.to respond_to :report_auth } @@ -194,7 +188,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :report_exploit_attempt } it { is_expected.to respond_to :report_exploit_failure } it { is_expected.to respond_to :report_exploit_success } - it { is_expected.to respond_to :report_host } it { is_expected.to respond_to :report_host_details } it { is_expected.to respond_to :report_host_tag } it { is_expected.to respond_to :report_import_note } @@ -859,7 +852,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :services } it { is_expected.to respond_to :tasks } it { is_expected.to respond_to :unserialize_object } - it { is_expected.to respond_to :update_host_via_sysinfo } it { is_expected.to respond_to :update_vuln_details } it { is_expected.to respond_to :usable } it { is_expected.to respond_to :usable= } diff --git a/spec/support/shared/examples/msf/db_manager/host.rb b/spec/support/shared/examples/msf/db_manager/host.rb new file mode 100644 index 0000000000..7472148298 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/host.rb @@ -0,0 +1,11 @@ +shared_examples_for 'Msf::DBManager::Host' do + it { is_expected.to respond_to :del_host } + it { is_expected.to respond_to :each_host } + it { is_expected.to respond_to :find_or_create_host } + it { is_expected.to respond_to :get_host } + it { is_expected.to respond_to :has_host? } + it { is_expected.to respond_to :hosts } + it { is_expected.to respond_to :normalize_host } + it { is_expected.to respond_to :report_host } + it { is_expected.to respond_to :update_host_via_sysinfo } +end \ No newline at end of file From 0284edf4307688ebf4379ffdf69b6d768d9e079f Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Thu, 9 Oct 2014 11:31:29 -0500 Subject: [PATCH 027/159] Extract Msf::DBManager::Service MSP-11124 Extract methods related to `Mdm::Service`s. --- lib/msf/core/db_manager.rb | 136 +---------------------------- lib/msf/core/db_manager/service.rb | 129 +++++++++++++++++++++++++++ 2 files changed, 131 insertions(+), 134 deletions(-) create mode 100644 lib/msf/core/db_manager/service.rb diff --git a/lib/msf/core/db_manager.rb b/lib/msf/core/db_manager.rb index b3ec31c4b6..d732b4921a 100644 --- a/lib/msf/core/db_manager.rb +++ b/lib/msf/core/db_manager.rb @@ -78,6 +78,7 @@ class DBManager autoload :Host, 'msf/core/db_manager/host' autoload :IPAddress, 'msf/core/db_manager/ip_address' autoload :ModuleCache, 'msf/core/db_manager/module_cache' + autoload :Service, 'msf/core/db_manager/service' autoload :Sink, 'msf/core/db_manager/sink' autoload :WMAP, 'msf/core/db_manager/wmap' autoload :Workspace, 'msf/core/db_manager/workspace' @@ -89,6 +90,7 @@ class DBManager include Msf::DBManager::IPAddress include Msf::DBManager::Migration include Msf::DBManager::ModuleCache + include Msf::DBManager::Service include Msf::DBManager::Sink include Msf::DBManager::WMAP include Msf::DBManager::Workspace @@ -382,127 +384,6 @@ class DBManager } end - def find_or_create_service(opts) - report_service(opts) - end - - # - # Record a service in the database. - # - # opts MUST contain - # +:host+:: the host where this service is running - # +:port+:: the port where this service listens - # +:proto+:: the transport layer protocol (e.g. tcp, udp) - # - # opts may contain - # +:name+:: the application layer protocol (e.g. ssh, mssql, smb) - # +:sname+:: an alias for the above - # - def report_service(opts) - return if not active - ::ActiveRecord::Base.connection_pool.with_connection { |conn| - addr = opts.delete(:host) || return - hname = opts.delete(:host_name) - hmac = opts.delete(:mac) - host = nil - wspace = opts.delete(:workspace) || workspace - hopts = {:workspace => wspace, :host => addr} - hopts[:name] = hname if hname - hopts[:mac] = hmac if hmac - - # Other report_* methods take :sname to mean the service name, so we - # map it here to ensure it ends up in the right place despite not being - # a real column. - if opts[:sname] - opts[:name] = opts.delete(:sname) - end - - if addr.kind_of? ::Mdm::Host - host = addr - addr = host.address - else - host = report_host(hopts) - end - - if opts[:port].to_i.zero? - dlog("Skipping port zero for service '%s' on host '%s'" % [opts[:name],host.address]) - return nil - end - - ret = {} -=begin - host = get_host(:workspace => wspace, :address => addr) - if host - host.updated_at = host.created_at - host.state = HostState::Alive - host.save! - end -=end - - proto = opts[:proto] || 'tcp' - - service = host.services.find_or_initialize_by_port_and_proto(opts[:port].to_i, proto) - opts.each { |k,v| - if (service.attribute_names.include?(k.to_s)) - service[k] = ((v and k == :name) ? v.to_s.downcase : v) - else - dlog("Unknown attribute for Service: #{k}") - end - } - service.state ||= ServiceState::Open - service.info ||= "" - - if (service and service.changed?) - msf_import_timestamps(opts,service) - service.save! - end - - if opts[:task] - Mdm::TaskService.create( - :task => opts[:task], - :service => service - ) - end - - ret[:service] = service - } - end - - def get_service(wspace, host, proto, port) - ::ActiveRecord::Base.connection_pool.with_connection { - host = get_host(:workspace => wspace, :address => host) - return if not host - return host.services.find_by_proto_and_port(proto, port) - } - end - - # - # Iterates over the services table calling the supplied block with the - # service instance of each entry. - # - def each_service(wspace=workspace, &block) - ::ActiveRecord::Base.connection_pool.with_connection { - services(wspace).each do |service| - block.call(service) - end - } - end - - # - # Returns a list of all services in the database - # - def services(wspace = workspace, only_up = false, proto = nil, addresses = nil, ports = nil, names = nil) - ::ActiveRecord::Base.connection_pool.with_connection { - conditions = {} - conditions[:state] = [ServiceState::Open] if only_up - conditions[:proto] = proto if proto - conditions["hosts.address"] = addresses if addresses - conditions[:port] = ports if ports - conditions[:name] = names if names - wspace.services.includes(:host).where(conditions).order("hosts.address, port") - } - end - # Returns a session based on opened_time, host address, and workspace # (or returns nil) def get_session(opts) @@ -1807,19 +1688,6 @@ class DBManager ) end - # - # Deletes a port and associated vulns matching this port - # - def del_service(wspace, address, proto, port, comm='') - - host = get_host(:workspace => wspace, :address => address) - return unless host - - ::ActiveRecord::Base.connection_pool.with_connection { - host.services.where({:proto => proto, :port => port}).each { |s| s.destroy } - } - end - # # Find a reference matching this name # diff --git a/lib/msf/core/db_manager/service.rb b/lib/msf/core/db_manager/service.rb new file mode 100644 index 0000000000..5d0ccaacb0 --- /dev/null +++ b/lib/msf/core/db_manager/service.rb @@ -0,0 +1,129 @@ +module Msf::DBManager::Service + # Deletes a port and associated vulns matching this port + def del_service(wspace, address, proto, port, comm='') + + host = get_host(:workspace => wspace, :address => address) + return unless host + + ::ActiveRecord::Base.connection_pool.with_connection { + host.services.where({:proto => proto, :port => port}).each { |s| s.destroy } + } + end + + # Iterates over the services table calling the supplied block with the + # service instance of each entry. + def each_service(wspace=workspace, &block) + ::ActiveRecord::Base.connection_pool.with_connection { + services(wspace).each do |service| + block.call(service) + end + } + end + + def find_or_create_service(opts) + report_service(opts) + end + + def get_service(wspace, host, proto, port) + ::ActiveRecord::Base.connection_pool.with_connection { + host = get_host(:workspace => wspace, :address => host) + return if not host + return host.services.find_by_proto_and_port(proto, port) + } + end + + # + # Record a service in the database. + # + # opts MUST contain + # +:host+:: the host where this service is running + # +:port+:: the port where this service listens + # +:proto+:: the transport layer protocol (e.g. tcp, udp) + # + # opts may contain + # +:name+:: the application layer protocol (e.g. ssh, mssql, smb) + # +:sname+:: an alias for the above + # + def report_service(opts) + return if not active + ::ActiveRecord::Base.connection_pool.with_connection { |conn| + addr = opts.delete(:host) || return + hname = opts.delete(:host_name) + hmac = opts.delete(:mac) + host = nil + wspace = opts.delete(:workspace) || workspace + hopts = {:workspace => wspace, :host => addr} + hopts[:name] = hname if hname + hopts[:mac] = hmac if hmac + + # Other report_* methods take :sname to mean the service name, so we + # map it here to ensure it ends up in the right place despite not being + # a real column. + if opts[:sname] + opts[:name] = opts.delete(:sname) + end + + if addr.kind_of? ::Mdm::Host + host = addr + addr = host.address + else + host = report_host(hopts) + end + + if opts[:port].to_i.zero? + dlog("Skipping port zero for service '%s' on host '%s'" % [opts[:name],host.address]) + return nil + end + + ret = {} +=begin + host = get_host(:workspace => wspace, :address => addr) + if host + host.updated_at = host.created_at + host.state = HostState::Alive + host.save! + end +=end + + proto = opts[:proto] || 'tcp' + + service = host.services.find_or_initialize_by_port_and_proto(opts[:port].to_i, proto) + opts.each { |k,v| + if (service.attribute_names.include?(k.to_s)) + service[k] = ((v and k == :name) ? v.to_s.downcase : v) + else + dlog("Unknown attribute for Service: #{k}") + end + } + service.state ||= ServiceState::Open + service.info ||= "" + + if (service and service.changed?) + msf_import_timestamps(opts,service) + service.save! + end + + if opts[:task] + Mdm::TaskService.create( + :task => opts[:task], + :service => service + ) + end + + ret[:service] = service + } + end + + # Returns a list of all services in the database + def services(wspace = workspace, only_up = false, proto = nil, addresses = nil, ports = nil, names = nil) + ::ActiveRecord::Base.connection_pool.with_connection { + conditions = {} + conditions[:state] = [ServiceState::Open] if only_up + conditions[:proto] = proto if proto + conditions["hosts.address"] = addresses if addresses + conditions[:port] = ports if ports + conditions[:name] = names if names + wspace.services.includes(:host).where(conditions).order("hosts.address, port") + } + end +end \ No newline at end of file From 96b4491d8e4bc9ccd9eac5d7d04e0a2863a07637 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Thu, 9 Oct 2014 11:35:07 -0500 Subject: [PATCH 028/159] Extract Msf::DBManager::Service shared examples MSP-11124 --- spec/lib/msf/db_manager_spec.rb | 7 +------ spec/support/shared/examples/msf/db_manager/service.rb | 8 ++++++++ 2 files changed, 9 insertions(+), 6 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/service.rb diff --git a/spec/lib/msf/db_manager_spec.rb b/spec/lib/msf/db_manager_spec.rb index c93ff36df2..4da0b4e2be 100644 --- a/spec/lib/msf/db_manager_spec.rb +++ b/spec/lib/msf/db_manager_spec.rb @@ -23,6 +23,7 @@ describe Msf::DBManager do it_should_behave_like 'Msf::DBManager::ImportMsfXml' it_should_behave_like 'Msf::DBManager::Migration' it_should_behave_like 'Msf::DBManager::ModuleCache' + it_should_behave_like 'Msf::DBManager::Service' it_should_behave_like 'Msf::DBManager::Sink' it_should_behave_like 'Msf::DBManager::WMAP' it_should_behave_like 'Msf::DBManager::Workspace' @@ -64,7 +65,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :create_db } it { is_expected.to respond_to :creds } it { is_expected.to respond_to :dehex } - it { is_expected.to respond_to :del_service } it { is_expected.to respond_to :disconnect } it { is_expected.to respond_to :driver } it { is_expected.to respond_to :drivers } @@ -73,7 +73,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :each_exploited_host } it { is_expected.to respond_to :each_loot } it { is_expected.to respond_to :each_note } - it { is_expected.to respond_to :each_service } it { is_expected.to respond_to :each_vuln } it { is_expected.to respond_to :emit } it { is_expected.to respond_to :error } @@ -85,7 +84,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :find_or_create_note } it { is_expected.to respond_to :find_or_create_ref } it { is_expected.to respond_to :find_or_create_report } - it { is_expected.to respond_to :find_or_create_service } it { is_expected.to respond_to :find_or_create_task } it { is_expected.to respond_to :find_or_create_vuln } it { is_expected.to respond_to :find_qualys_asset_ports } @@ -95,7 +93,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :find_vuln_by_refs } it { is_expected.to respond_to :get_client } it { is_expected.to respond_to :get_ref } - it { is_expected.to respond_to :get_service } it { is_expected.to respond_to :get_session } it { is_expected.to respond_to :get_vuln } it { is_expected.to respond_to :has_ref? } @@ -194,7 +191,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :report_loot } it { is_expected.to respond_to :report_note } it { is_expected.to respond_to :report_report } - it { is_expected.to respond_to :report_service } context '#report_session' do let(:options) do @@ -849,7 +845,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :reports } it { is_expected.to respond_to :rexmlify } it { is_expected.to respond_to :service_name_map } - it { is_expected.to respond_to :services } it { is_expected.to respond_to :tasks } it { is_expected.to respond_to :unserialize_object } it { is_expected.to respond_to :update_vuln_details } diff --git a/spec/support/shared/examples/msf/db_manager/service.rb b/spec/support/shared/examples/msf/db_manager/service.rb new file mode 100644 index 0000000000..df24b93ae5 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/service.rb @@ -0,0 +1,8 @@ +shared_examples_for 'Msf::DBManager::Service' do + it { is_expected.to respond_to :del_service } + it { is_expected.to respond_to :each_service } + it { is_expected.to respond_to :find_or_create_service } + it { is_expected.to respond_to :get_service } + it { is_expected.to respond_to :report_service } + it { is_expected.to respond_to :services } +end \ No newline at end of file From ceba04d5568660d526b3e4f071b62f36e1e1da5d Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Thu, 9 Oct 2014 11:41:04 -0500 Subject: [PATCH 029/159] Extract Msf::DBManager::Cred MSP-11124 Extract methods related to `Mdm::Cred`s. --- lib/msf/core/db_manager.rb | 187 +------------------------------- lib/msf/core/db_manager/cred.rb | 180 ++++++++++++++++++++++++++++++ 2 files changed, 182 insertions(+), 185 deletions(-) create mode 100644 lib/msf/core/db_manager/cred.rb diff --git a/lib/msf/core/db_manager.rb b/lib/msf/core/db_manager.rb index d732b4921a..4c095f714e 100644 --- a/lib/msf/core/db_manager.rb +++ b/lib/msf/core/db_manager.rb @@ -75,6 +75,7 @@ module Msf class DBManager extend Metasploit::Framework::Require + autoload :Cred, 'msf/core/db_manager/cred' autoload :Host, 'msf/core/db_manager/host' autoload :IPAddress, 'msf/core/db_manager/ip_address' autoload :ModuleCache, 'msf/core/db_manager/module_cache' @@ -85,6 +86,7 @@ class DBManager optionally_include_metasploit_credential_creation + include Msf::DBManager::Cred include Msf::DBManager::Host include Msf::DBManager::ImportMsfXml include Msf::DBManager::IPAddress @@ -979,15 +981,6 @@ class DBManager } end - # - # This methods returns a list of all credentials in the database - # - def creds(wspace=workspace) - ::ActiveRecord::Base.connection_pool.with_connection { - Mdm::Cred.includes({:service => :host}).where("hosts.workspace_id = ?", wspace.id) - } - end - # # This method returns a list of all exploited hosts in the database. # @@ -1200,182 +1193,6 @@ class DBManager } end - # - # Store a set of credentials in the database. - # - # report_auth_info used to create a note, now it creates - # an entry in the creds table. It's much more akin to - # report_vuln() now. - # - # opts MUST contain - # +:host+:: an IP address or Host object reference - # +:port+:: a port number - # - # opts can contain - # +:user+:: the username - # +:pass+:: the password, or path to ssh_key - # +:ptype+:: the type of password (password(ish), hash, or ssh_key) - # +:proto+:: a transport name for the port - # +:sname+:: service name - # +:active+:: by default, a cred is active, unless explicitly false - # +:proof+:: data used to prove the account is actually active. - # - # Sources: Credentials can be sourced from another credential, or from - # a vulnerability. For example, if an exploit was used to dump the - # smb_hashes, and this credential comes from there, the source_id would - # be the Vuln id (as reported by report_vuln) and the type would be "Vuln". - # - # +:source_id+:: The Vuln or Cred id of the source of this cred. - # +:source_type+:: Either Vuln or Cred - # - # TODO: This is written somewhat host-centric, when really the - # Service is the thing. Need to revisit someday. - def report_auth_info(opts={}) - return if not active - raise ArgumentError.new("Missing required option :host") if opts[:host].nil? - raise ArgumentError.new("Missing required option :port") if (opts[:port].nil? and opts[:service].nil?) - - if (not opts[:host].kind_of?(::Mdm::Host)) and (not validate_ips(opts[:host])) - raise ArgumentError.new("Invalid address or object for :host (#{opts[:host].inspect})") - end - - ::ActiveRecord::Base.connection_pool.with_connection { - host = opts.delete(:host) - ptype = opts.delete(:type) || "password" - token = [opts.delete(:user), opts.delete(:pass)] - sname = opts.delete(:sname) - port = opts.delete(:port) - proto = opts.delete(:proto) || "tcp" - proof = opts.delete(:proof) - source_id = opts.delete(:source_id) - source_type = opts.delete(:source_type) - duplicate_ok = opts.delete(:duplicate_ok) - # Nil is true for active. - active = (opts[:active] || opts[:active].nil?) ? true : false - - wspace = opts.delete(:workspace) || workspace - - # Service management; assume the user knows what - # he's talking about. - service = opts.delete(:service) || report_service(:host => host, :port => port, :proto => proto, :name => sname, :workspace => wspace) - - # Non-US-ASCII usernames are tripping up the database at the moment, this is a temporary fix until we update the tables - if (token[0]) - # convert the token to US-ASCII from UTF-8 to prevent an error - token[0] = token[0].unpack("C*").pack("C*") - token[0] = token[0].gsub(/[\x00-\x1f\x7f-\xff]/n){|m| "\\x%.2x" % m.unpack("C")[0] } - end - - if (token[1]) - token[1] = token[1].unpack("C*").pack("C*") - token[1] = token[1].gsub(/[\x00-\x1f\x7f-\xff]/n){|m| "\\x%.2x" % m.unpack("C")[0] } - end - - ret = {} - - # Check to see if the creds already exist. We look also for a downcased username with the - # same password because we can fairly safely assume they are not in fact two seperate creds. - # this allows us to hedge against duplication of creds in the DB. - - if duplicate_ok - # If duplicate usernames are okay, find by both user and password (allows - # for actual duplicates to get modified updated_at, sources, etc) - if token[0].nil? or token[0].empty? - cred = service.creds.find_or_initialize_by_user_and_ptype_and_pass(token[0] || "", ptype, token[1] || "") - else - cred = service.creds.find_by_user_and_ptype_and_pass(token[0] || "", ptype, token[1] || "") - unless cred - dcu = token[0].downcase - cred = service.creds.find_by_user_and_ptype_and_pass( dcu || "", ptype, token[1] || "") - unless cred - cred = service.creds.find_or_initialize_by_user_and_ptype_and_pass(token[0] || "", ptype, token[1] || "") - end - end - end - else - # Create the cred by username only (so we can change passwords) - if token[0].nil? or token[0].empty? - cred = service.creds.find_or_initialize_by_user_and_ptype(token[0] || "", ptype) - else - cred = service.creds.find_by_user_and_ptype(token[0] || "", ptype) - unless cred - dcu = token[0].downcase - cred = service.creds.find_by_user_and_ptype_and_pass( dcu || "", ptype, token[1] || "") - unless cred - cred = service.creds.find_or_initialize_by_user_and_ptype(token[0] || "", ptype) - end - end - end - end - - # Update with the password - cred.pass = (token[1] || "") - - # Annotate the credential - cred.ptype = ptype - cred.active = active - - # Update the source ID only if there wasn't already one. - if source_id and !cred.source_id - cred.source_id = source_id - cred.source_type = source_type if source_type - end - - # Safe proof (lazy way) -- doesn't chop expanded - # characters correctly, but shouldn't ever be a problem. - unless proof.nil? - proof = Rex::Text.to_hex_ascii(proof) - proof = proof[0,4096] - end - cred.proof = proof - - # Update the timestamp - if cred.changed? - msf_import_timestamps(opts,cred) - cred.save! - end - - # Ensure the updated_at is touched any time report_auth_info is called - # except when it's set explicitly (as it is for imports) - unless opts[:updated_at] || opts["updated_at"] - cred.updated_at = Time.now.utc - cred.save! - end - - - if opts[:task] - Mdm::TaskCred.create( - :task => opts[:task], - :cred => cred - ) - end - - ret[:cred] = cred - } - end - - alias :report_cred :report_auth_info - alias :report_auth :report_auth_info - - # - # Find or create a credential matching this type/data - # - def find_or_create_cred(opts) - report_auth_info(opts) - end - - # - # This method iterates the creds table calling the supplied block with the - # cred instance of each entry. - # - def each_cred(wspace=workspace,&block) - ::ActiveRecord::Base.connection_pool.with_connection { - wspace.creds.each do |cred| - block.call(cred) - end - } - end - def each_exploited_host(wspace=workspace,&block) ::ActiveRecord::Base.connection_pool.with_connection { wspace.exploited_hosts.each do |eh| diff --git a/lib/msf/core/db_manager/cred.rb b/lib/msf/core/db_manager/cred.rb new file mode 100644 index 0000000000..23c70b7803 --- /dev/null +++ b/lib/msf/core/db_manager/cred.rb @@ -0,0 +1,180 @@ +module Msf::DBManager::Cred + # This methods returns a list of all credentials in the database + def creds(wspace=workspace) + ::ActiveRecord::Base.connection_pool.with_connection { + Mdm::Cred.includes({:service => :host}).where("hosts.workspace_id = ?", wspace.id) + } + end + + # This method iterates the creds table calling the supplied block with the + # cred instance of each entry. + def each_cred(wspace=workspace,&block) + ::ActiveRecord::Base.connection_pool.with_connection { + wspace.creds.each do |cred| + block.call(cred) + end + } + end + + # Find or create a credential matching this type/data + def find_or_create_cred(opts) + report_auth_info(opts) + end + + # + # Store a set of credentials in the database. + # + # report_auth_info used to create a note, now it creates + # an entry in the creds table. It's much more akin to + # report_vuln() now. + # + # opts MUST contain + # +:host+:: an IP address or Host object reference + # +:port+:: a port number + # + # opts can contain + # +:user+:: the username + # +:pass+:: the password, or path to ssh_key + # +:ptype+:: the type of password (password(ish), hash, or ssh_key) + # +:proto+:: a transport name for the port + # +:sname+:: service name + # +:active+:: by default, a cred is active, unless explicitly false + # +:proof+:: data used to prove the account is actually active. + # + # Sources: Credentials can be sourced from another credential, or from + # a vulnerability. For example, if an exploit was used to dump the + # smb_hashes, and this credential comes from there, the source_id would + # be the Vuln id (as reported by report_vuln) and the type would be "Vuln". + # + # +:source_id+:: The Vuln or Cred id of the source of this cred. + # +:source_type+:: Either Vuln or Cred + # + # TODO: This is written somewhat host-centric, when really the + # Service is the thing. Need to revisit someday. + def report_auth_info(opts={}) + return if not active + raise ArgumentError.new("Missing required option :host") if opts[:host].nil? + raise ArgumentError.new("Missing required option :port") if (opts[:port].nil? and opts[:service].nil?) + + if (not opts[:host].kind_of?(::Mdm::Host)) and (not validate_ips(opts[:host])) + raise ArgumentError.new("Invalid address or object for :host (#{opts[:host].inspect})") + end + + ::ActiveRecord::Base.connection_pool.with_connection { + host = opts.delete(:host) + ptype = opts.delete(:type) || "password" + token = [opts.delete(:user), opts.delete(:pass)] + sname = opts.delete(:sname) + port = opts.delete(:port) + proto = opts.delete(:proto) || "tcp" + proof = opts.delete(:proof) + source_id = opts.delete(:source_id) + source_type = opts.delete(:source_type) + duplicate_ok = opts.delete(:duplicate_ok) + # Nil is true for active. + active = (opts[:active] || opts[:active].nil?) ? true : false + + wspace = opts.delete(:workspace) || workspace + + # Service management; assume the user knows what + # he's talking about. + service = opts.delete(:service) || report_service(:host => host, :port => port, :proto => proto, :name => sname, :workspace => wspace) + + # Non-US-ASCII usernames are tripping up the database at the moment, this is a temporary fix until we update the tables + if (token[0]) + # convert the token to US-ASCII from UTF-8 to prevent an error + token[0] = token[0].unpack("C*").pack("C*") + token[0] = token[0].gsub(/[\x00-\x1f\x7f-\xff]/n){|m| "\\x%.2x" % m.unpack("C")[0] } + end + + if (token[1]) + token[1] = token[1].unpack("C*").pack("C*") + token[1] = token[1].gsub(/[\x00-\x1f\x7f-\xff]/n){|m| "\\x%.2x" % m.unpack("C")[0] } + end + + ret = {} + + # Check to see if the creds already exist. We look also for a downcased username with the + # same password because we can fairly safely assume they are not in fact two seperate creds. + # this allows us to hedge against duplication of creds in the DB. + + if duplicate_ok + # If duplicate usernames are okay, find by both user and password (allows + # for actual duplicates to get modified updated_at, sources, etc) + if token[0].nil? or token[0].empty? + cred = service.creds.find_or_initialize_by_user_and_ptype_and_pass(token[0] || "", ptype, token[1] || "") + else + cred = service.creds.find_by_user_and_ptype_and_pass(token[0] || "", ptype, token[1] || "") + unless cred + dcu = token[0].downcase + cred = service.creds.find_by_user_and_ptype_and_pass( dcu || "", ptype, token[1] || "") + unless cred + cred = service.creds.find_or_initialize_by_user_and_ptype_and_pass(token[0] || "", ptype, token[1] || "") + end + end + end + else + # Create the cred by username only (so we can change passwords) + if token[0].nil? or token[0].empty? + cred = service.creds.find_or_initialize_by_user_and_ptype(token[0] || "", ptype) + else + cred = service.creds.find_by_user_and_ptype(token[0] || "", ptype) + unless cred + dcu = token[0].downcase + cred = service.creds.find_by_user_and_ptype_and_pass( dcu || "", ptype, token[1] || "") + unless cred + cred = service.creds.find_or_initialize_by_user_and_ptype(token[0] || "", ptype) + end + end + end + end + + # Update with the password + cred.pass = (token[1] || "") + + # Annotate the credential + cred.ptype = ptype + cred.active = active + + # Update the source ID only if there wasn't already one. + if source_id and !cred.source_id + cred.source_id = source_id + cred.source_type = source_type if source_type + end + + # Safe proof (lazy way) -- doesn't chop expanded + # characters correctly, but shouldn't ever be a problem. + unless proof.nil? + proof = Rex::Text.to_hex_ascii(proof) + proof = proof[0,4096] + end + cred.proof = proof + + # Update the timestamp + if cred.changed? + msf_import_timestamps(opts,cred) + cred.save! + end + + # Ensure the updated_at is touched any time report_auth_info is called + # except when it's set explicitly (as it is for imports) + unless opts[:updated_at] || opts["updated_at"] + cred.updated_at = Time.now.utc + cred.save! + end + + + if opts[:task] + Mdm::TaskCred.create( + :task => opts[:task], + :cred => cred + ) + end + + ret[:cred] = cred + } + end + + alias :report_auth :report_auth_info + alias :report_cred :report_auth_info +end \ No newline at end of file From 3ec93dea4330435db911d3a1ac6fc9fb20fae046 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Thu, 9 Oct 2014 11:44:15 -0500 Subject: [PATCH 030/159] Extract Msf::DBManager::Cred shared examples MSP-11124 --- spec/lib/msf/db_manager_spec.rb | 7 +------ spec/support/shared/examples/msf/db_manager/cred.rb | 8 ++++++++ 2 files changed, 9 insertions(+), 6 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/cred.rb diff --git a/spec/lib/msf/db_manager_spec.rb b/spec/lib/msf/db_manager_spec.rb index 4da0b4e2be..a8e4b4429e 100644 --- a/spec/lib/msf/db_manager_spec.rb +++ b/spec/lib/msf/db_manager_spec.rb @@ -18,6 +18,7 @@ describe Msf::DBManager do db_manager end + it_should_behave_like 'Msf::DBManager::Cred' it_should_behave_like 'Msf::DBManager::Host' it_should_behave_like 'Msf::DBManager::IPAddress' it_should_behave_like 'Msf::DBManager::ImportMsfXml' @@ -63,13 +64,11 @@ describe Msf::DBManager do it { is_expected.to respond_to :connect } it { is_expected.to respond_to :connection_established? } it { is_expected.to respond_to :create_db } - it { is_expected.to respond_to :creds } it { is_expected.to respond_to :dehex } it { is_expected.to respond_to :disconnect } it { is_expected.to respond_to :driver } it { is_expected.to respond_to :drivers } it { is_expected.to respond_to :drivers= } - it { is_expected.to respond_to :each_cred } it { is_expected.to respond_to :each_exploited_host } it { is_expected.to respond_to :each_loot } it { is_expected.to respond_to :each_note } @@ -79,7 +78,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :events } it { is_expected.to respond_to :exploited_hosts } it { is_expected.to respond_to :find_or_create_client } - it { is_expected.to respond_to :find_or_create_cred } it { is_expected.to respond_to :find_or_create_loot } it { is_expected.to respond_to :find_or_create_note } it { is_expected.to respond_to :find_or_create_ref } @@ -176,10 +174,7 @@ describe Msf::DBManager do it { is_expected.to respond_to :nmap_msf_service_map } it { is_expected.to respond_to :notes } it { is_expected.to respond_to :report_artifact } - it { is_expected.to respond_to :report_auth } - it { is_expected.to respond_to :report_auth_info } it { is_expected.to respond_to :report_client } - it { is_expected.to respond_to :report_cred } it { is_expected.to respond_to :report_event } it { is_expected.to respond_to :report_exploit } it { is_expected.to respond_to :report_exploit_attempt } diff --git a/spec/support/shared/examples/msf/db_manager/cred.rb b/spec/support/shared/examples/msf/db_manager/cred.rb new file mode 100644 index 0000000000..366c1edef8 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/cred.rb @@ -0,0 +1,8 @@ +shared_examples_for 'Msf::DBManager::Cred' do + it { is_expected.to respond_to :creds } + it { is_expected.to respond_to :each_cred } + it { is_expected.to respond_to :find_or_create_cred } + it { is_expected.to respond_to :report_auth } + it { is_expected.to respond_to :report_auth_info } + it { is_expected.to respond_to :report_cred } +end \ No newline at end of file From d18dcf5961dadda998c4b610e34a2f54457841ff Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Thu, 9 Oct 2014 12:54:04 -0500 Subject: [PATCH 031/159] Extract Msf::DBManager::ExploitedHost MSP-11124 Extract methods related to `Mdm::ExploitedHost`s. --- lib/msf/core/db_manager.rb | 19 ++----------------- lib/msf/core/db_manager/exploited_host.rb | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 17 deletions(-) create mode 100644 lib/msf/core/db_manager/exploited_host.rb diff --git a/lib/msf/core/db_manager.rb b/lib/msf/core/db_manager.rb index 4c095f714e..f2401395f4 100644 --- a/lib/msf/core/db_manager.rb +++ b/lib/msf/core/db_manager.rb @@ -76,6 +76,7 @@ class DBManager extend Metasploit::Framework::Require autoload :Cred, 'msf/core/db_manager/cred' + autoload :ExploitedHost, 'msf/core/db_manager/exploited_host' autoload :Host, 'msf/core/db_manager/host' autoload :IPAddress, 'msf/core/db_manager/ip_address' autoload :ModuleCache, 'msf/core/db_manager/module_cache' @@ -87,6 +88,7 @@ class DBManager optionally_include_metasploit_credential_creation include Msf::DBManager::Cred + include Msf::DBManager::ExploitedHost include Msf::DBManager::Host include Msf::DBManager::ImportMsfXml include Msf::DBManager::IPAddress @@ -981,15 +983,6 @@ class DBManager } end - # - # This method returns a list of all exploited hosts in the database. - # - def exploited_hosts(wspace=workspace) - ::ActiveRecord::Base.connection_pool.with_connection { - wspace.exploited_hosts - } - end - # # This method iterates the notes table calling the supplied block with the # note instance of each entry. @@ -1193,14 +1186,6 @@ class DBManager } end - def each_exploited_host(wspace=workspace,&block) - ::ActiveRecord::Base.connection_pool.with_connection { - wspace.exploited_hosts.each do |eh| - block.call(eh) - end - } - end - # # Find or create a vuln matching this service/name # diff --git a/lib/msf/core/db_manager/exploited_host.rb b/lib/msf/core/db_manager/exploited_host.rb new file mode 100644 index 0000000000..8da53a2c69 --- /dev/null +++ b/lib/msf/core/db_manager/exploited_host.rb @@ -0,0 +1,16 @@ +module Msf::DBManager::ExploitedHost + def each_exploited_host(wspace=workspace,&block) + ::ActiveRecord::Base.connection_pool.with_connection { + wspace.exploited_hosts.each do |eh| + block.call(eh) + end + } + end + + # This method returns a list of all exploited hosts in the database. + def exploited_hosts(wspace=workspace) + ::ActiveRecord::Base.connection_pool.with_connection { + wspace.exploited_hosts + } + end +end \ No newline at end of file From 45f5fe36f169d9e7d095691eda886e8b61dea777 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Thu, 9 Oct 2014 12:58:30 -0500 Subject: [PATCH 032/159] Extract Msf::DBManager::ExploitedHost shared examples MSP-11124 --- spec/lib/msf/db_manager_spec.rb | 3 +-- spec/support/shared/examples/msf/db_manager/exploited_host.rb | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/exploited_host.rb diff --git a/spec/lib/msf/db_manager_spec.rb b/spec/lib/msf/db_manager_spec.rb index a8e4b4429e..bfc52a74e6 100644 --- a/spec/lib/msf/db_manager_spec.rb +++ b/spec/lib/msf/db_manager_spec.rb @@ -19,6 +19,7 @@ describe Msf::DBManager do end it_should_behave_like 'Msf::DBManager::Cred' + it_should_behave_like 'Msf::DBManager::ExploitedHost' it_should_behave_like 'Msf::DBManager::Host' it_should_behave_like 'Msf::DBManager::IPAddress' it_should_behave_like 'Msf::DBManager::ImportMsfXml' @@ -69,14 +70,12 @@ describe Msf::DBManager do it { is_expected.to respond_to :driver } it { is_expected.to respond_to :drivers } it { is_expected.to respond_to :drivers= } - it { is_expected.to respond_to :each_exploited_host } it { is_expected.to respond_to :each_loot } it { is_expected.to respond_to :each_note } it { is_expected.to respond_to :each_vuln } it { is_expected.to respond_to :emit } it { is_expected.to respond_to :error } it { is_expected.to respond_to :events } - it { is_expected.to respond_to :exploited_hosts } it { is_expected.to respond_to :find_or_create_client } it { is_expected.to respond_to :find_or_create_loot } it { is_expected.to respond_to :find_or_create_note } diff --git a/spec/support/shared/examples/msf/db_manager/exploited_host.rb b/spec/support/shared/examples/msf/db_manager/exploited_host.rb new file mode 100644 index 0000000000..4047daaf2e --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/exploited_host.rb @@ -0,0 +1,4 @@ +shared_examples_for 'Msf::DBManager::ExploitedHost' do + it { is_expected.to respond_to :each_exploited_host } + it { is_expected.to respond_to :exploited_hosts } +end \ No newline at end of file From cb9bdd96c7d3823492ad0781936ab16e71048eab Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Thu, 9 Oct 2014 14:51:24 -0500 Subject: [PATCH 033/159] Extract Msf::DBManager::Import MSP-11124 Extract all methods dealing with imports. --- lib/msf/core/db_manager.rb | 3477 +---------------------------- lib/msf/core/db_manager/import.rb | 3451 ++++++++++++++++++++++++++++ 2 files changed, 3453 insertions(+), 3475 deletions(-) create mode 100644 lib/msf/core/db_manager/import.rb diff --git a/lib/msf/core/db_manager.rb b/lib/msf/core/db_manager.rb index f2401395f4..8cea3b5f15 100644 --- a/lib/msf/core/db_manager.rb +++ b/lib/msf/core/db_manager.rb @@ -78,6 +78,7 @@ class DBManager autoload :Cred, 'msf/core/db_manager/cred' autoload :ExploitedHost, 'msf/core/db_manager/exploited_host' autoload :Host, 'msf/core/db_manager/host' + autoload :Import, 'msf/core/db_manager/import' autoload :IPAddress, 'msf/core/db_manager/ip_address' autoload :ModuleCache, 'msf/core/db_manager/module_cache' autoload :Service, 'msf/core/db_manager/service' @@ -90,6 +91,7 @@ class DBManager include Msf::DBManager::Cred include Msf::DBManager::ExploitedHost include Msf::DBManager::Host + include Msf::DBManager::Import include Msf::DBManager::ImportMsfXml include Msf::DBManager::IPAddress include Msf::DBManager::Migration @@ -2111,3480 +2113,5 @@ class DBManager ret[:web_vuln] = vuln } end - - # Returns a REXML::Document from the given data. - def rexmlify(data) - if data.kind_of?(REXML::Document) - return data - else - # Make an attempt to recover from a REXML import fail, since - # it's better than dying outright. - begin - return REXML::Document.new(data) - rescue REXML::ParseException => e - dlog("REXML error: Badly formatted XML, attempting to recover. Error was: #{e.inspect}") - return REXML::Document.new(data.gsub(/([\x00-\x08\x0b\x0c\x0e-\x1f\x80-\xff])/n){ |x| "\\x%.2x" % x.unpack("C*")[0] }) - end - end - end - - # Handles timestamps from Metasploit Express/Pro imports. - def msf_import_timestamps(opts,obj) - obj.created_at = opts["created_at"] if opts["created_at"] - obj.created_at = opts[:created_at] if opts[:created_at] - obj.updated_at = opts["updated_at"] ? opts["updated_at"] : obj.created_at - obj.updated_at = opts[:updated_at] ? opts[:updated_at] : obj.created_at - return obj - end - - ## - # - # Import methods - # - ## - - # - # Generic importer that automatically determines the file type being - # imported. Since this looks for vendor-specific strings in the given - # file, there shouldn't be any false detections, but no guarantees. - # - def import_file(args={}, &block) - filename = args[:filename] || args['filename'] - wspace = args[:wspace] || args['wspace'] || workspace - @import_filedata = {} - @import_filedata[:filename] = filename - - data = "" - ::File.open(filename, 'rb') do |f| - # This check is the largest (byte-wise) that we need to do - # since the other 4-byte checks will be subsets of this larger one. - data = f.read(Metasploit::Credential::Exporter::Pwdump::FILE_ID_STRING.size) - end - if data.nil? - raise DBImportError.new("Zero-length file") - end - - if data.index(Metasploit::Credential::Exporter::Pwdump::FILE_ID_STRING) - data = ::File.open(filename, 'rb') - else - case data[0,4] - when "PK\x03\x04" - data = Zip::File.open(filename) - when "\xd4\xc3\xb2\xa1", "\xa1\xb2\xc3\xd4" - data = PacketFu::PcapFile.new(:filename => filename) - else - ::File.open(filename, 'rb') do |f| - sz = f.stat.size - data = f.read(sz) - end - end - end - - - if block - import(args.merge(:data => data)) { |type,data| yield type,data } - else - import(args.merge(:data => data)) - end - end - - # A dispatcher method that figures out the data's file type, - # and sends it off to the appropriate importer. Note that - # import_file_detect will raise an error if the filetype - # is unknown. - def import(args={}, &block) - data = args[:data] || args['data'] - ftype = import_filetype_detect(data) - yield(:filetype, @import_filedata[:type]) if block - self.send "import_#{ftype}".to_sym, args, &block - end - - # Returns one of the following: - # - # :acunetix_xml - # :amap_log - # :amap_mlog - # :appscan_xml - # :burp_session_xml - # :ci_xml - # :foundstone_xml - # :fusionvm_xml - # :ip360_aspl_xml - # :ip360_xml_v3 - # :ip_list - # :libpcap - # :mbsa_xml - # :msf_cred_dump_zip - # :msf_pwdump - # :msf_xml - # :msf_zip - # :nessus_nbe - # :nessus_xml - # :nessus_xml_v2 - # :netsparker_xml - # :nexpose_rawxml - # :nexpose_simplexml - # :nikto_xml - # :nmap_xml - # :openvas_new_xml - # :openvas_xml - # :outpost24_xml - # :qualys_asset_xml - # :qualys_scan_xml - # :retina_xml - # :spiceworks_csv - # :wapiti_xml - # - # If there is no match, an error is raised instead. - # - # @raise DBImportError if the type can't be detected - def import_filetype_detect(data) - - if data and data.kind_of? Zip::File - if data.entries.empty? - raise DBImportError.new("The zip file provided is empty.") - end - - @import_filedata ||= {} - @import_filedata[:zip_filename] = File.split(data.to_s).last - @import_filedata[:zip_basename] = @import_filedata[:zip_filename].gsub(/\.zip$/,"") - @import_filedata[:zip_entry_names] = data.entries.map {|x| x.name} - - if @import_filedata[:zip_entry_names].include?(Metasploit::Credential::Importer::Zip::MANIFEST_FILE_NAME) - @import_filedata[:type] = "Metasploit Credential Dump" - return :msf_cred_dump_zip - end - - xml_files = @import_filedata[:zip_entry_names].grep(/^(.*)\.xml$/) - - # TODO This check for our zip export should be more extensive - if xml_files.empty? - raise DBImportError.new("The zip file provided is not a Metasploit Zip Export") - end - - @import_filedata[:zip_xml] = xml_files.first - @import_filedata[:type] = "Metasploit Zip Export" - - return :msf_zip - end - - if data and data.kind_of? PacketFu::PcapFile - # Don't check for emptiness here because unlike other formats, we - # haven't read any actual data in yet, only magic bytes to discover - # that this is indeed a pcap file. - #raise DBImportError.new("The pcap file provided is empty.") if data.body.empty? - @import_filedata ||= {} - @import_filedata[:type] = "Libpcap Packet Capture" - return :libpcap - end - - # msfpwdump - if data.present? && data.kind_of?(::File) - @import_filedata[:type] = "Metasploit PWDump Export" - return :msf_pwdump - end - - # This is a text string, lets make sure its treated as binary - data = data.unpack("C*").pack("C*") - if data and data.to_s.strip.length == 0 - raise DBImportError.new("The data provided to the import function was empty") - end - - # Parse the first line or 4k of data from the file - di = data.index("\n") || 4096 - - firstline = data[0, di] - @import_filedata ||= {} - if (firstline.index("")) - @import_filedata[:type] = "Retina XML" - return :retina_xml - elsif (firstline.index(//)) - @import_filedata[:type] = "OpenVAS XML" - return :openvas_new_xml - elsif (firstline.index(/")) - @import_filedata[:type] = "Nessus XML (v1)" - return :nessus_xml - elsif (firstline.index("]/ - - case $1 - when "niktoscan" - @import_filedata[:type] = "Nikto XML" - return :nikto_xml - when "nmaprun" - @import_filedata[:type] = "Nmap XML" - return :nmap_xml - when "openvas-report" - @import_filedata[:type] = "OpenVAS Report" - return :openvas_xml - when "NessusClientData" - @import_filedata[:type] = "Nessus XML (v1)" - return :nessus_xml - when "NessusClientData_v2" - @import_filedata[:type] = "Nessus XML (v2)" - return :nessus_xml_v2 - when "SCAN" - @import_filedata[:type] = "Qualys Scan XML" - return :qualys_scan_xml - when "report" - @import_filedata[:type] = "Wapiti XML" - return :wapiti_xml - when "ASSET_DATA_REPORT" - @import_filedata[:type] = "Qualys Asset XML" - return :qualys_asset_xml - when /MetasploitExpressV[1234]/ - @import_filedata[:type] = "Metasploit XML" - return :msf_xml - when /MetasploitV4/ - @import_filedata[:type] = "Metasploit XML" - return :msf_xml - when /netsparker/ - @import_filedata[:type] = "NetSparker XML" - return :netsparker_xml - when /audits?/ # and are both valid for nCircle. wtfmate. - @import_filedata[:type] = "IP360 XML v3" - return :ip360_xml_v3 - when /ontology/ - @import_filedata[:type] = "IP360 ASPL" - return :ip360_aspl_xml - when /ReportInfo/ - @import_filedata[:type] = "Foundstone" - return :foundstone_xml - when /ScanGroup/ - @import_filedata[:type] = "Acunetix" - return :acunetix_xml - when /AppScanInfo/ # Actually the second line - @import_filedata[:type] = "Appscan" - return :appscan_xml - when "entities" - if line =~ /creator.*\x43\x4f\x52\x45\x20\x49\x4d\x50\x41\x43\x54/ni - @import_filedata[:type] = "CI" - return :ci_xml - end - when "main" - @import_filedata[:type] = "Outpost24 XML" - return :outpost24_xml - else - # Give up if we haven't hit the root tag in the first few lines - break if line_count > 10 - end - line_count += 1 - } - elsif (firstline.index("timestamps|||scan_start")) - @import_filedata[:type] = "Nessus NBE Report" - # then it's a nessus nbe - return :nessus_nbe - elsif (firstline.index("# amap v")) - # then it's an amap mlog - @import_filedata[:type] = "Amap Log -m" - return :amap_mlog - elsif (firstline.index("amap v")) - # then it's an amap log - @import_filedata[:type] = "Amap Log" - return :amap_log - elsif ipv46_validator(firstline) - # then its an IP list - @import_filedata[:type] = "IP Address List" - return :ip_list - elsif (data[0,1024].index(" wspace, - :host => addr, - :type => "service.nikto.scan.description", - :data => desc_text, - :proto => "tcp", - :port => port.to_i, - :sname => uri.scheme, - :update => :unique_data, - :task => args[:task] - } - # Always report it as a note. - report_note(desc_data) - # Sometimes report it as a vuln, too. - # XXX: There's a Vuln.info field but nothing reads from it? See Bug #5837 - if item.attributes['osvdbid'].to_i != 0 - desc_data[:refs] = ["OSVDB-#{item.attributes['osvdbid']}"] - desc_data[:name] = "NIKTO-#{item.attributes['id']}" - desc_data.delete(:data) - desc_data.delete(:type) - desc_data.delete(:update) - report_vuln(desc_data) - end - end - end - end - end - end - - def import_wapiti_xml_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - - data = "" - ::File.open(filename, 'rb') do |f| - data = f.read(f.stat.size) - end - import_wapiti_xml(args.merge(:data => data)) - end - - def import_wapiti_xml(args={}, &block) - if block - doc = Rex::Parser::WapitiDocument.new(args,framework.db) {|type, data| yield type,data } - else - doc = Rex::Parser::WapitiDocument.new(args,self) - end - parser = ::Nokogiri::XML::SAX::Parser.new(doc) - parser.parse(args[:data]) - end - - def import_openvas_new_xml_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - - data = "" - ::File.open(filename, 'rb') do |f| - data = f.read(f.stat.size) - end - import_wapiti_xml(args.merge(:data => data)) - end - - def import_openvas_new_xml(args={}, &block) - if block - doc = Rex::Parser::OpenVASDocument.new(args,framework.db) {|type, data| yield type,data } - else - doc = Rex::Parser::OpenVASDocument.new(args,self) - end - parser = ::Nokogiri::XML::SAX::Parser.new(doc) - parser.parse(args[:data]) - end - - def import_libpcap_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - - data = PacketFu::PcapFile.new(:filename => filename) - import_libpcap(args.merge(:data => data)) - end - - # The libpcap file format is handled by PacketFu for data - # extraction. TODO: Make this its own mixin, and possibly - # extend PacketFu to do better stream analysis on the fly. - def import_libpcap(args={}, &block) - data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - # seen_hosts is only used for determining when to yield an address. Once we get - # some packet analysis going, the values will have all sorts of info. The plan - # is to ru through all the packets as a first pass and report host and service, - # then, once we have everything parsed, we can reconstruct sessions and ngrep - # out things like authentication sequences, examine ttl's and window sizes, all - # kinds of crazy awesome stuff like that. - seen_hosts = {} - decoded_packets = 0 - last_count = 0 - data.read_packet_bytes do |p| - if (decoded_packets >= last_count + 1000) and block - yield(:pcap_count, decoded_packets) - last_count = decoded_packets - end - decoded_packets += 1 - - pkt = PacketFu::Packet.parse(p) rescue next # Just silently skip bad packets - - next unless pkt.is_ip? # Skip anything that's not IP. Technically, not Ethernet::Ip - next if pkt.is_tcp? && (pkt.tcp_src == 0 || pkt.tcp_dst == 0) # Skip port 0 - next if pkt.is_udp? && (pkt.udp_src == 0 || pkt.udp_dst == 0) # Skip port 0 - saddr = pkt.ip_saddr - daddr = pkt.ip_daddr - - # Handle blacklists and obviously useless IP addresses, and report the host. - next if (bl | [saddr,daddr]).size == bl.size # Both hosts are blacklisted, skip everything. - unless( bl.include?(saddr) || rfc3330_reserved(saddr)) - yield(:address,saddr) if block and !seen_hosts.keys.include?(saddr) - unless seen_hosts[saddr] - report_host( - :workspace => wspace, - :host => saddr, - :state => Msf::HostState::Alive, - :task => args[:task] - ) - end - seen_hosts[saddr] ||= [] - - end - unless( bl.include?(daddr) || rfc3330_reserved(daddr)) - yield(:address,daddr) if block and !seen_hosts.keys.include?(daddr) - unless seen_hosts[daddr] - report_host( - :workspace => wspace, - :host => daddr, - :state => Msf::HostState::Alive, - :task => args[:task] - ) - end - seen_hosts[daddr] ||= [] - end - - if pkt.is_tcp? # First pass on TCP packets - if (pkt.tcp_flags.syn == 1 and pkt.tcp_flags.ack == 1) or # Oh, this kills me - pkt.tcp_src < 1024 # If it's a low port, assume it's a proper service. - if seen_hosts[saddr] - unless seen_hosts[saddr].include? [pkt.tcp_src,"tcp"] - report_service( - :workspace => wspace, :host => saddr, - :proto => "tcp", :port => pkt.tcp_src, - :state => Msf::ServiceState::Open, - :task => args[:task] - ) - seen_hosts[saddr] << [pkt.tcp_src,"tcp"] - yield(:service,"%s:%d/%s" % [saddr,pkt.tcp_src,"tcp"]) - end - end - end - elsif pkt.is_udp? # First pass on UDP packets - if pkt.udp_src == pkt.udp_dst # Very basic p2p detection. - [saddr,daddr].each do |xaddr| - if seen_hosts[xaddr] - unless seen_hosts[xaddr].include? [pkt.udp_src,"udp"] - report_service( - :workspace => wspace, :host => xaddr, - :proto => "udp", :port => pkt.udp_src, - :state => Msf::ServiceState::Open, - :task => args[:task] - ) - seen_hosts[xaddr] << [pkt.udp_src,"udp"] - yield(:service,"%s:%d/%s" % [xaddr,pkt.udp_src,"udp"]) - end - end - end - elsif pkt.udp_src < 1024 # Probably a service - if seen_hosts[saddr] - unless seen_hosts[saddr].include? [pkt.udp_src,"udp"] - report_service( - :workspace => wspace, :host => saddr, - :proto => "udp", :port => pkt.udp_src, - :state => Msf::ServiceState::Open, - :task => args[:task] - ) - seen_hosts[saddr] << [pkt.udp_src,"udp"] - yield(:service,"%s:%d/%s" % [saddr,pkt.udp_src,"udp"]) - end - end - end - end # tcp or udp - - inspect_single_packet(pkt,wspace,args) - - end # data.body.map - - # Right about here, we should have built up some streams for some stream analysis. - # Not sure what form that will take, but people like shoving many hundreds of - # thousands of packets through this thing, so it'll need to be memory efficient. - - end - - # Do all the single packet analysis we can while churning through the pcap - # the first time. Multiple packet inspection will come later, where we can - # do stream analysis, compare requests and responses, etc. - def inspect_single_packet(pkt,wspace,args) - if pkt.is_tcp? or pkt.is_udp? - inspect_single_packet_http(pkt,wspace,args) - end - end - - # Checks for packets that are headed towards port 80, are tcp, contain an HTTP/1.0 - # line, contains an Authorization line, contains a b64-encoded credential, and - # extracts it. Reports this credential and solidifies the service as HTTP. - def inspect_single_packet_http(pkt,wspace,args) - task = args.fetch(:task, nil) - # First, check the server side (data from port 80). - if pkt.is_tcp? and pkt.tcp_src == 80 and !pkt.payload.nil? and !pkt.payload.empty? - if pkt.payload =~ /^HTTP\x2f1\x2e[01]/n - http_server_match = pkt.payload.match(/\nServer:\s+([^\r\n]+)[\r\n]/n) - if http_server_match.kind_of?(MatchData) and http_server_match[1] - report_service( - :workspace => wspace, - :host => pkt.ip_saddr, - :port => pkt.tcp_src, - :proto => "tcp", - :name => "http", - :info => http_server_match[1], - :state => Msf::ServiceState::Open, - :task => task - ) - # That's all we want to know from this service. - return :something_significant - end - end - end - - # Next, check the client side (data to port 80) - if pkt.is_tcp? and pkt.tcp_dst == 80 and !pkt.payload.nil? and !pkt.payload.empty? - if pkt.payload.match(/[\x00-\x20]HTTP\x2f1\x2e[10]/n) - auth_match = pkt.payload.match(/\nAuthorization:\s+Basic\s+([A-Za-z0-9=\x2b]+)/n) - if auth_match.kind_of?(MatchData) and auth_match[1] - b64_cred = auth_match[1] - else - return false - end - # If we're this far, we can surmise that at least the client is a web browser, - # he thinks the server is HTTP and he just made an authentication attempt. At - # this point, we'll just believe everything the packet says -- validation ought - # to come later. - user,pass = b64_cred.unpack("m*").first.split(/:/,2) - report_service( - :workspace => wspace, - :host => pkt.ip_daddr, - :port => pkt.tcp_dst, - :proto => "tcp", - :name => "http", - :task => task - ) - - service_data = { - address: pkt.ip_daddr, - port: pkt.tcp_dst, - service_name: 'http', - protocol: 'tcp', - workspace_id: wspace.id - } - service_data[:task_id] = task.id if task - - filename = args[:filename] - - credential_data = { - origin_type: :import, - private_data: pass, - private_type: :password, - username: user, - filename: filename - } - credential_data.merge!(service_data) - credential_core = create_credential(credential_data) - - login_data = { - core: credential_core, - status: Metasploit::Model::Login::Status::UNTRIED - } - - login_data.merge!(service_data) - - create_credential_login(login_data) - - # That's all we want to know from this service. - return :something_significant - end - end - end - - def import_spiceworks_csv(args={}, &block) - data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - CSV.parse(data) do |row| - next unless (["Name", "Manufacturer", "Device Type"] & row).empty? #header - name = row[0] - manufacturer = row[1] - device = row[2] - model = row[3] - ip = row[4] - serialno = row[5] - location = row[6] - os = row[7] - - next unless ip - next if bl.include? ip - - conf = { - :workspace => wspace, - :host => ip, - :name => name, - :task => args[:task] - } - - - if os - report_note( - :workspace => wspace, - :task => args[:task], - :host => ip, - :type => 'host.os.spiceworks_fingerprint', - :data => { - :os => os.to_s.strip - } - ) - end - - info = [] - info << "Serial Number: #{serialno}" unless (serialno.blank? or serialno == name) - info << "Location: #{location}" unless location.blank? - conf[:info] = info.join(", ") unless info.empty? - - host = report_host(conf) - report_import_note(wspace, host) - end - end - - - # Perform in an import of an msfpwdump file - def import_msf_pwdump(args={}, &block) - filename = File.basename(args[:data].path) - wspace = args[:wspace] || workspace - origin = Metasploit::Credential::Origin::Import.create!(filename: filename) - importer = Metasploit::Credential::Importer::Pwdump.new(input: args[:data], workspace: wspace, filename: filename, origin:origin) - importer.import! - importer.input.close unless importer.input.closed? - end - - # If hex notation is present, turn them into a character. - def dehex(str) - hexen = str.scan(/\x5cx[0-9a-fA-F]{2}/n) - hexen.each { |h| - str.gsub!(h,h[2,2].to_i(16).chr) - } - return str - end - - - # - # Nexpose Simple XML - # - # XXX At some point we'll want to make this a stream parser for dealing - # with large results files - # - def import_nexpose_simplexml_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - - data = "" - ::File.open(filename, 'rb') do |f| - data = f.read(f.stat.size) - end - import_nexpose_simplexml(args.merge(:data => data)) - end - - # Import a Metasploit XML file. - def import_msf_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - - data = "" - ::File.open(filename, 'rb') do |f| - data = f.read(f.stat.size) - end - import_msf_xml(args.merge(:data => data)) - end - - # Import a Metasploit Express ZIP file. Note that this requires - # a fair bit of filesystem manipulation, and is very much tied - # up with the Metasploit Express ZIP file format export (for - # obvious reasons). In the event directories exist, they will - # be reused. If target files exist, they will be overwritten. - # - # XXX: Refactor so it's not quite as sanity-blasting. - def import_msf_zip(args={}, &block) - data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - - new_tmp = ::File.join(Dir::tmpdir,"msf","imp_#{Rex::Text::rand_text_alphanumeric(4)}",@import_filedata[:zip_basename]) - if ::File.exists? new_tmp - unless (::File.directory?(new_tmp) && ::File.writable?(new_tmp)) - raise DBImportError.new("Could not extract zip file to #{new_tmp}") - end - else - FileUtils.mkdir_p(new_tmp) - end - @import_filedata[:zip_tmp] = new_tmp - - # Grab the list of unique basedirs over all entries. - @import_filedata[:zip_tmp_subdirs] = @import_filedata[:zip_entry_names].map {|x| ::File.split(x)}.map {|x| x[0]}.uniq.reject {|x| x == "."} - - # mkdir all of the base directores we just pulled out, if they don't - # already exist - @import_filedata[:zip_tmp_subdirs].each {|sub| - tmp_subdirs = ::File.join(@import_filedata[:zip_tmp],sub) - if File.exists? tmp_subdirs - unless (::File.directory?(tmp_subdirs) && File.writable?(tmp_subdirs)) - # if it exists but we can't write to it, give up - raise DBImportError.new("Could not extract zip file to #{tmp_subdirs}") - end - else - ::FileUtils.mkdir(tmp_subdirs) - end - } - - data.entries.each do |e| - target = ::File.join(@import_filedata[:zip_tmp], e.name) - data.extract(e,target) - - if target =~ /\.xml\z/ - target_data = ::File.open(target, "rb") {|f| f.read 1024} - if import_filetype_detect(target_data) == :msf_xml - @import_filedata[:zip_extracted_xml] = target - end - end - end - - # Import any creds if there are some in the import file - Dir.entries(@import_filedata[:zip_tmp]).each do |entry| - if entry =~ /^.*#{Regexp.quote(Metasploit::Credential::Exporter::Core::CREDS_DUMP_FILE_IDENTIFIER)}.*/ - manifest_file_path = File.join(@import_filedata[:zip_tmp], entry, Metasploit::Credential::Importer::Zip::MANIFEST_FILE_NAME) - if File.exists? manifest_file_path - import_msf_cred_dump(manifest_file_path, wspace) - end - end - end - - # This will kick the newly-extracted XML file through - # the import_file process all over again. - if @import_filedata[:zip_extracted_xml] - new_args = args.dup - new_args[:filename] = @import_filedata[:zip_extracted_xml] - new_args[:data] = nil - new_args[:ifd] = @import_filedata.dup - if block - import_file(new_args, &block) - else - import_file(new_args) - end - end - - # Kick down to all the MSFX ZIP specific items - if block - import_msf_collateral(new_args, &block) - else - import_msf_collateral(new_args) - end - end - - # Imports loot, tasks, and reports from an MSF ZIP report. - # XXX: This function is stupidly long. It needs to be refactored. - def import_msf_collateral(args={}, &block) - data = ::File.open(args[:filename], "rb") {|f| f.read(f.stat.size)} - wspace = args[:wspace] || args['wspace'] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - basedir = args[:basedir] || args['basedir'] || ::File.join(Msf::Config.data_directory, "msf") - - allow_yaml = false - btag = nil - - doc = rexmlify(data) - if doc.elements["MetasploitExpressV1"] - m_ver = 1 - allow_yaml = true - btag = "MetasploitExpressV1" - elsif doc.elements["MetasploitExpressV2"] - m_ver = 2 - allow_yaml = true - btag = "MetasploitExpressV2" - elsif doc.elements["MetasploitExpressV3"] - m_ver = 3 - btag = "MetasploitExpressV3" - elsif doc.elements["MetasploitExpressV4"] - m_ver = 4 - btag = "MetasploitExpressV4" - elsif doc.elements["MetasploitV4"] - m_ver = 4 - btag = "MetasploitV4" - else - m_ver = nil - end - unless m_ver and btag - raise DBImportError.new("Unsupported Metasploit XML document format") - end - - host_info = {} - doc.elements.each("/#{btag}/hosts/host") do |host| - host_info[host.elements["id"].text.to_s.strip] = nils_for_nulls(host.elements["address"].text.to_s.strip) - end - - # Import Loot - doc.elements.each("/#{btag}/loots/loot") do |loot| - next if bl.include? host_info[loot.elements["host-id"].text.to_s.strip] - loot_info = {} - loot_info[:host] = host_info[loot.elements["host-id"].text.to_s.strip] - loot_info[:workspace] = args[:wspace] - loot_info[:ctype] = nils_for_nulls(loot.elements["content-type"].text.to_s.strip) - loot_info[:info] = nils_for_nulls(unserialize_object(loot.elements["info"], allow_yaml)) - loot_info[:ltype] = nils_for_nulls(loot.elements["ltype"].text.to_s.strip) - loot_info[:name] = nils_for_nulls(loot.elements["name"].text.to_s.strip) - loot_info[:created_at] = nils_for_nulls(loot.elements["created-at"].text.to_s.strip) - loot_info[:updated_at] = nils_for_nulls(loot.elements["updated-at"].text.to_s.strip) - loot_info[:name] = nils_for_nulls(loot.elements["name"].text.to_s.strip) - loot_info[:orig_path] = nils_for_nulls(loot.elements["path"].text.to_s.strip) - loot_info[:task] = args[:task] - tmp = args[:ifd][:zip_tmp] - loot_info[:orig_path].gsub!(/^\./,tmp) if loot_info[:orig_path] - if !loot.elements["service-id"].text.to_s.strip.empty? - unless loot.elements["service-id"].text.to_s.strip == "NULL" - loot_info[:service] = loot.elements["service-id"].text.to_s.strip - end - end - - # Only report loot if we actually have it. - # TODO: Copypasta. Seperate this out. - if ::File.exists? loot_info[:orig_path] - loot_dir = ::File.join(basedir,"loot") - loot_file = ::File.split(loot_info[:orig_path]).last - if ::File.exists? loot_dir - unless (::File.directory?(loot_dir) && ::File.writable?(loot_dir)) - raise DBImportError.new("Could not move files to #{loot_dir}") - end - else - ::FileUtils.mkdir_p(loot_dir) - end - new_loot = ::File.join(loot_dir,loot_file) - loot_info[:path] = new_loot - if ::File.exists?(new_loot) - ::File.unlink new_loot # Delete it, and don't report it. - else - report_loot(loot_info) # It's new, so report it. - end - ::FileUtils.copy(loot_info[:orig_path], new_loot) - yield(:msf_loot, new_loot) if block - end - end - - # Import Tasks - doc.elements.each("/#{btag}/tasks/task") do |task| - task_info = {} - task_info[:workspace] = args[:wspace] - # Should user be imported (original) or declared (the importing user)? - task_info[:user] = nils_for_nulls(task.elements["created-by"].text.to_s.strip) - task_info[:desc] = nils_for_nulls(task.elements["description"].text.to_s.strip) - task_info[:info] = nils_for_nulls(unserialize_object(task.elements["info"], allow_yaml)) - task_info[:mod] = nils_for_nulls(task.elements["module"].text.to_s.strip) - task_info[:options] = nils_for_nulls(task.elements["options"].text.to_s.strip) - task_info[:prog] = nils_for_nulls(task.elements["progress"].text.to_s.strip).to_i - task_info[:created_at] = nils_for_nulls(task.elements["created-at"].text.to_s.strip) - task_info[:updated_at] = nils_for_nulls(task.elements["updated-at"].text.to_s.strip) - if !task.elements["completed-at"].text.to_s.empty? - task_info[:completed_at] = nils_for_nulls(task.elements["completed-at"].text.to_s.strip) - end - if !task.elements["error"].text.to_s.empty? - task_info[:error] = nils_for_nulls(task.elements["error"].text.to_s.strip) - end - if !task.elements["result"].text.to_s.empty? - task_info[:result] = nils_for_nulls(task.elements["result"].text.to_s.strip) - end - task_info[:orig_path] = nils_for_nulls(task.elements["path"].text.to_s.strip) - tmp = args[:ifd][:zip_tmp] - task_info[:orig_path].gsub!(/^\./,tmp) if task_info[:orig_path] - - # Only report a task if we actually have it. - # TODO: Copypasta. Seperate this out. - if ::File.exists? task_info[:orig_path] - tasks_dir = ::File.join(basedir,"tasks") - task_file = ::File.split(task_info[:orig_path]).last - if ::File.exists? tasks_dir - unless (::File.directory?(tasks_dir) && ::File.writable?(tasks_dir)) - raise DBImportError.new("Could not move files to #{tasks_dir}") - end - else - ::FileUtils.mkdir_p(tasks_dir) - end - new_task = ::File.join(tasks_dir,task_file) - task_info[:path] = new_task - if ::File.exists?(new_task) - ::File.unlink new_task # Delete it, and don't report it. - else - report_task(task_info) # It's new, so report it. - end - ::FileUtils.copy(task_info[:orig_path], new_task) - yield(:msf_task, new_task) if block - end - end - - # Import Reports - doc.elements.each("/#{btag}/reports/report") do |report| - import_report(report, args, basedir) - end - end - - # Import credentials given a path to a valid manifest file - # - # @param creds_dump_manifest_path [String] - # @param workspace [Mdm::Workspace] Default: {#workspace} - # @return [void] - def import_msf_cred_dump(creds_dump_manifest_path, workspace) - manifest_file = File.open(creds_dump_manifest_path) - origin = Metasploit::Credential::Origin::Import.create!(filename: File.basename(creds_dump_manifest_path)) - importer = Metasploit::Credential::Importer::Core.new(workspace: workspace, input: manifest_file, origin: origin) - importer.import! - end - - # Import credentials given a path to a valid manifest file - # - # @option args [String] :filename - # @option args [Mdm::Workspace] :wspace Default: {#workspace} - # @return [void] - def import_msf_cred_dump_zip(args = {}) - wspace = args[:wspace] || workspace - origin = Metasploit::Credential::Origin::Import.create!(filename: File.basename(args[:filename])) - importer = Metasploit::Credential::Importer::Zip.new(workspace: wspace, input: File.open(args[:filename]), origin: origin) - importer.import! - nil - end - - # @param report [REXML::Element] to be imported - # @param args [Hash] - # @param base_dir [String] - def import_report(report, args, base_dir) - tmp = args[:ifd][:zip_tmp] - report_info = {} - - report.elements.each do |e| - node_name = e.name - node_value = e.text - - # These need to be converted back to arrays: - array_attrs = %w|addresses file-formats options sections| - if array_attrs.member? node_name - node_value = JSON.parse(node_value) - end - # Don't restore these values: - skip_nodes = %w|id workspace-id artifacts| - next if skip_nodes.member? node_name - - report_info[node_name.parameterize.underscore.to_sym] = node_value - end - # Use current workspace - report_info[:workspace_id] = args[:wspace].id - - # Create report, need new ID to record artifacts - report_id = report_report(report_info) - - # Handle artifacts - report.elements['artifacts'].elements.each do |artifact| - artifact_opts = {} - artifact.elements.each do |attr| - skip_nodes = %w|id accessed-at| - next if skip_nodes.member? attr.name - - symboled_attr = attr.name.parameterize.underscore.to_sym - artifact_opts[symboled_attr] = attr.text - end - # Use new Report as parent - artifact_opts[:report_id] = report_id - # Update to full path - artifact_opts[:file_path].gsub!(/^\./, tmp) - - report_artifact(artifact_opts) - end - end - - # Convert the string "NULL" to actual nil - def nils_for_nulls(str) - str == "NULL" ? nil : str - end - - def import_nexpose_simplexml(args={}, &block) - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - wspace = args[:wspace] || workspace - if Rex::Parser.nokogiri_loaded - parser = "Nokogiri v#{::Nokogiri::VERSION}" - noko_args = args.dup - noko_args[:blacklist] = bl - noko_args[:wspace] = wspace - if block - yield(:parser, parser) - import_nexpose_noko_stream(noko_args) {|type, data| yield type,data} - else - import_nexpose_noko_stream(noko_args) - end - return true - end - data = args[:data] - - doc = rexmlify(data) - doc.elements.each('/NeXposeSimpleXML/devices/device') do |dev| - addr = dev.attributes['address'].to_s - if bl.include? addr - next - else - yield(:address,addr) if block - end - - fprint = {} - - dev.elements.each('fingerprint/description') do |str| - fprint[:desc] = str.text.to_s.strip - end - dev.elements.each('fingerprint/vendor') do |str| - fprint[:vendor] = str.text.to_s.strip - end - dev.elements.each('fingerprint/family') do |str| - fprint[:family] = str.text.to_s.strip - end - dev.elements.each('fingerprint/product') do |str| - fprint[:product] = str.text.to_s.strip - end - dev.elements.each('fingerprint/version') do |str| - fprint[:version] = str.text.to_s.strip - end - dev.elements.each('fingerprint/architecture') do |str| - fprint[:arch] = str.text.to_s.upcase.strip - end - - conf = { - :workspace => wspace, - :host => addr, - :state => Msf::HostState::Alive, - :task => args[:task] - } - - host = report_host(conf) - report_import_note(wspace, host) - - report_note( - :workspace => wspace, - :host => host, - :type => 'host.os.nexpose_fingerprint', - :data => fprint, - :task => args[:task] - ) - - # Load vulnerabilities not associated with a service - dev.elements.each('vulnerabilities/vulnerability') do |vuln| - vid = vuln.attributes['id'].to_s.downcase - refs = process_nexpose_data_sxml_refs(vuln) - next if not refs - report_vuln( - :workspace => wspace, - :host => host, - :name => 'NEXPOSE-' + vid, - :info => vid, - :refs => refs, - :task => args[:task] - ) - end - - # Load the services - dev.elements.each('services/service') do |svc| - sname = svc.attributes['name'].to_s - sprot = svc.attributes['protocol'].to_s.downcase - sport = svc.attributes['port'].to_s.to_i - next if sport == 0 - - name = sname.split('(')[0].strip - info = '' - - svc.elements.each('fingerprint/description') do |str| - info = str.text.to_s.strip - end - - if(sname.downcase != '') - report_service( - :workspace => wspace, - :host => host, - :proto => sprot, - :port => sport, - :name => name, - :info => info, - :task => args[:task] - ) - else - report_service( - :workspace => wspace, - :host => host, - :proto => sprot, - :port => sport, - :info => info, - :task => args[:task] - ) - end - - # Load vulnerabilities associated with this service - svc.elements.each('vulnerabilities/vulnerability') do |vuln| - vid = vuln.attributes['id'].to_s.downcase - refs = process_nexpose_data_sxml_refs(vuln) - next if not refs - report_vuln( - :workspace => wspace, - :host => host, - :port => sport, - :proto => sprot, - :name => 'NEXPOSE-' + vid, - :info => vid, - :refs => refs, - :task => args[:task] - ) - end - end - end - end - - - # - # Nexpose Raw XML - # - def import_nexpose_rawxml_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - - data = "" - ::File.open(filename, 'rb') do |f| - data = f.read(f.stat.size) - end - import_nexpose_rawxml(args.merge(:data => data)) - end - - def import_nexpose_rawxml(args={}, &block) - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - wspace = args[:wspace] || workspace - if Rex::Parser.nokogiri_loaded - parser = "Nokogiri v#{::Nokogiri::VERSION}" - noko_args = args.dup - noko_args[:blacklist] = bl - noko_args[:wspace] = wspace - if block - yield(:parser, parser) - import_nexpose_raw_noko_stream(noko_args) {|type, data| yield type,data} - else - import_nexpose_raw_noko_stream(noko_args) - end - return true - end - data = args[:data] - - # Use a stream parser instead of a tree parser so we can deal with - # huge results files without running out of memory. - parser = Rex::Parser::NexposeXMLStreamParser.new - - # Since all the Refs have to be in the database before we can use them - # in a Vuln, we store all the hosts until we finish parsing and only - # then put everything in the database. This is memory-intensive for - # large files, but should be much less so than a tree parser. - # - # This method is also considerably faster than parsing through the tree - # looking for references every time we hit a vuln. - hosts = [] - vulns = [] - - # The callback merely populates our in-memory table of hosts and vulns - parser.callback = Proc.new { |type, value| - case type - when :host - # XXX: Blacklist should be checked here instead of saving a - # host we're just going to throw away later - hosts.push(value) - when :vuln - value["id"] = value["id"].downcase if value["id"] - vulns.push(value) - end - } - - REXML::Document.parse_stream(data, parser) - - vuln_refs = nexpose_refs_to_struct(vulns) - hosts.each do |host| - if bl.include? host["addr"] - next - else - yield(:address,host["addr"]) if block - end - nexpose_host_from_rawxml(host, vuln_refs, wspace) - end - end - - # - # Takes an array of vuln hashes, as returned by the NeXpose rawxml stream - # parser, like: - # [ - # {"id"=>"winreg-notes-protocol-handler", severity="8", "refs"=>[{"source"=>"BID", "value"=>"10600"}, ...]} - # {"id"=>"windows-zotob-c", severity="8", "refs"=>[{"source"=>"BID", "value"=>"14513"}, ...]} - # ] - # and transforms it into a struct, containing :id, :refs, :title, and :severity - # - # Other attributes can be added later, as needed. - def nexpose_refs_to_struct(vulns) - ret = [] - vulns.each do |vuln| - next if ret.map {|v| v.id}.include? vuln["id"] - vstruct = Struct.new(:id, :refs, :title, :severity).new - vstruct.id = vuln["id"] - vstruct.title = vuln["title"] - vstruct.severity = vuln["severity"] - vstruct.refs = [] - vuln["refs"].each do |ref| - if ref['source'] == 'BID' - vstruct.refs.push('BID-' + ref["value"]) - elsif ref['source'] == 'CVE' - # value is CVE-$ID - vstruct.refs.push(ref["value"]) - elsif ref['source'] == 'MS' - vstruct.refs.push('MSB-' + ref["value"]) - elsif ref['source'] == 'URL' - vstruct.refs.push('URL-' + ref["value"]) - end - end - ret.push vstruct - end - return ret - end - - # Takes a Host object, an array of vuln structs (generated by nexpose_refs_to_struct()), - # and a workspace, and reports the vulns on that host. - def nexpose_host_from_rawxml(h, vstructs, wspace,task=nil) - hobj = nil - data = {:workspace => wspace} - if h["addr"] - addr = h["addr"] - else - # Can't report it if it doesn't have an IP - return - end - data[:host] = addr - if (h["hardware-address"]) - # Put colons between each octet of the MAC address - data[:mac] = h["hardware-address"].gsub(':', '').scan(/../).join(':') - end - data[:state] = (h["status"] == "alive") ? Msf::HostState::Alive : Msf::HostState::Dead - - # Since we only have one name field per host in the database, just - # take the first one. - if (h["names"] and h["names"].first) - data[:name] = h["names"].first - end - - if (data[:state] != Msf::HostState::Dead) - hobj = report_host(data) - report_import_note(wspace, hobj) - end - - if h["notes"] - note = { - :workspace => wspace, - :host => (hobj || addr), - :type => "host.vuln.nexpose_keys", - :data => {}, - :mode => :unique_data, - :task => task - } - h["notes"].each do |v,k| - note[:data][v] ||= [] - next if note[:data][v].include? k - note[:data][v] << k - end - report_note(note) - end - - if h["os_family"] - note = { - :workspace => wspace, - :host => hobj || addr, - :type => 'host.os.nexpose_fingerprint', - :task => task, - :data => { - :family => h["os_family"], - :certainty => h["os_certainty"] - } - } - note[:data][:vendor] = h["os_vendor"] if h["os_vendor"] - note[:data][:product] = h["os_product"] if h["os_product"] - note[:data][:version] = h["os_version"] if h["os_version"] - note[:data][:arch] = h["arch"] if h["arch"] - - report_note(note) - end - - h["endpoints"].each { |p| - extra = "" - extra << p["product"] + " " if p["product"] - extra << p["version"] + " " if p["version"] - - # Skip port-0 endpoints - next if p["port"].to_i == 0 - - # XXX This should probably be handled in a more standard way - # extra << "(" + p["certainty"] + " certainty) " if p["certainty"] - - data = {} - data[:workspace] = wspace - data[:proto] = p["protocol"].downcase - data[:port] = p["port"].to_i - data[:state] = p["status"] - data[:host] = hobj || addr - data[:info] = extra if not extra.empty? - data[:task] = task - if p["name"] != "" - data[:name] = p["name"] - end - report_service(data) - } - - h["vulns"].each_pair { |k,v| - - next if v["status"] !~ /^vulnerable/ - vstruct = vstructs.select {|vs| vs.id.to_s.downcase == v["id"].to_s.downcase}.first - next unless vstruct - data = {} - data[:workspace] = wspace - data[:host] = hobj || addr - data[:proto] = v["protocol"].downcase if v["protocol"] - data[:port] = v["port"].to_i if v["port"] - data[:name] = "NEXPOSE-" + v["id"] - data[:info] = vstruct.title - data[:refs] = vstruct.refs - data[:task] = task - report_vuln(data) - } - end - - - # - # Retina XML - # - - # Process a Retina XML file - def import_retina_xml_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - - data = "" - ::File.open(filename, 'rb') do |f| - data = f.read(f.stat.size) - end - import_retina_xml(args.merge(:data => data)) - end - - # Process Retina XML - def import_retina_xml(args={}, &block) - data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - msg = "Warning: The Retina XML format does not associate vulnerabilities with the\n" - msg << "specific service on which they were found.\n" - msg << "This makes it impossible to correlate exploits to discovered vulnerabilities\n" - msg << "in a reliable fashion." - - yield(:warning,msg) if block - - parser = Rex::Parser::RetinaXMLStreamParser.new - parser.on_found_host = Proc.new do |host| - hobj = nil - data = { - :workspace => wspace, - :task => args[:task] - } - addr = host['address'] - next if not addr - - next if bl.include? addr - data[:host] = addr - - if host['mac'] - data[:mac] = host['mac'] - end - - data[:state] = Msf::HostState::Alive - - if host['hostname'] - data[:name] = host['hostname'] - end - - if host['netbios'] - data[:name] = host['netbios'] - end - - yield(:address, data[:host]) if block - - # Import Host - hobj = report_host(data) - report_import_note(wspace, hobj) - - # Import OS fingerprint - if host["os"] - note = { - :workspace => wspace, - :host => addr, - :type => 'host.os.retina_fingerprint', - :task => args[:task], - :data => { - :os => host["os"] - } - } - report_note(note) - end - - # Import vulnerabilities - host['vulns'].each do |vuln| - refs = vuln['refs'].map{|v| v.join("-")} - refs << "RETINA-#{vuln['rthid']}" if vuln['rthid'] - - vuln_info = { - :workspace => wspace, - :host => addr, - :name => vuln['name'], - :info => vuln['description'], - :refs => refs, - :task => args[:task] - } - - report_vuln(vuln_info) - end - end - - REXML::Document.parse_stream(data, parser) - end - - # - # NetSparker XML - # - - # Process a NetSparker XML file - def import_netsparker_xml_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - - data = "" - ::File.open(filename, 'rb') do |f| - data = f.read(f.stat.size) - end - import_netsparker_xml(args.merge(:data => data)) - end - - # Process NetSparker XML - def import_netsparker_xml(args={}, &block) - data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - addr = nil - parser = Rex::Parser::NetSparkerXMLStreamParser.new - parser.on_found_vuln = Proc.new do |vuln| - data = {:workspace => wspace} - - # Parse the URL - url = vuln['url'] - return if not url - - # Crack the URL into a URI - uri = URI(url) rescue nil - return if not uri - - # Resolve the host and cache the IP - if not addr - baddr = Rex::Socket.addr_aton(uri.host) rescue nil - if baddr - addr = Rex::Socket.addr_ntoa(baddr) - yield(:address, addr) if block - end - end - - # Bail early if we have no IP address - if not addr - raise Interrupt, "Not a valid IP address" - end - - if bl.include?(addr) - raise Interrupt, "IP address is on the blacklist" - end - - data[:host] = addr - data[:vhost] = uri.host - data[:port] = uri.port - data[:ssl] = (uri.scheme == "ssl") - - body = nil - # First report a web page - if vuln['response'] - headers = {} - code = 200 - head,body = vuln['response'].to_s.split(/\r?\n\r?\n/, 2) - if body - - if head =~ /^HTTP\d+\.\d+\s+(\d+)\s*/ - code = $1.to_i - end - - headers = {} - head.split(/\r?\n/).each do |line| - hname,hval = line.strip.split(/\s*:\s*/, 2) - next if hval.to_s.strip.empty? - headers[hname.downcase] ||= [] - headers[hname.downcase] << hval - end - - info = { - :path => uri.path, - :query => uri.query, - :code => code, - :body => body, - :headers => headers, - :task => args[:task] - } - info.merge!(data) - - if headers['content-type'] - info[:ctype] = headers['content-type'][0] - end - - if headers['set-cookie'] - info[:cookie] = headers['set-cookie'].join("\n") - end - - if headers['authorization'] - info[:auth] = headers['authorization'].join("\n") - end - - if headers['location'] - info[:location] = headers['location'][0] - end - - if headers['last-modified'] - info[:mtime] = headers['last-modified'][0] - end - - # Report the web page to the database - report_web_page(info) - - yield(:web_page, url) if block - end - end # End web_page reporting - - - details = netsparker_vulnerability_map(vuln) - - method = netsparker_method_map(vuln) - pname = netsparker_pname_map(vuln) - params = netsparker_params_map(vuln) - - proof = '' - - if vuln['info'] and vuln['info'].length > 0 - proof << vuln['info'].map{|x| "#{x[0]}: #{x[1]}\n" }.join + "\n" - end - - if proof.empty? - if body - proof << body + "\n" - else - proof << vuln['response'].to_s + "\n" - end - end - - if params.empty? and pname - params = [[pname, vuln['vparam_name'].to_s]] - end - - info = { - # XXX: There is a :request attr in the model, but report_web_vuln - # doesn't seem to know about it, so this gets ignored. - #:request => vuln['request'], - :path => uri.path, - :query => uri.query, - :method => method, - :params => params, - :pname => pname.to_s, - :proof => proof, - :risk => details[:risk], - :name => details[:name], - :blame => details[:blame], - :category => details[:category], - :description => details[:description], - :confidence => details[:confidence], - :task => args[:task] - } - info.merge!(data) - - next if vuln['type'].to_s.empty? - - report_web_vuln(info) - yield(:web_vuln, url) if block - end - - # We throw interrupts in our parser when the job is hopeless - begin - REXML::Document.parse_stream(data, parser) - rescue ::Interrupt => e - wlog("The netsparker_xml_import() job was interrupted: #{e}") - end - end - - def netsparker_method_map(vuln) - case vuln['vparam_type'] - when "FullQueryString" - "GET" - when "Querystring" - "GET" - when "Post" - "POST" - when "RawUrlInjection" - "GET" - else - "GET" - end - end - - def netsparker_pname_map(vuln) - case vuln['vparam_name'] - when "URI-BASED", "Query Based" - "PATH" - else - vuln['vparam_name'] - end - end - - def netsparker_params_map(vuln) - [] - end - - def netsparker_vulnerability_map(vuln) - res = { - :risk => 1, - :name => 'Information Disclosure', - :blame => 'System Administrator', - :category => 'info', - :description => "This is an information leak", - :confidence => 100 - } - - # Risk is a value from 1-5 indicating the severity of the issue - # Examples: 1, 4, 5 - - # Name is a descriptive name for this vulnerability. - # Examples: XSS, ReflectiveXSS, PersistentXSS - - # Blame indicates who is at fault for the vulnerability - # Examples: App Developer, Server Developer, System Administrator - - # Category indicates the general class of vulnerability - # Examples: info, xss, sql, rfi, lfi, cmd - - # Description is a textual summary of the vulnerability - # Examples: "A reflective cross-site scripting attack" - # "The web server leaks the internal IP address" - # "The cookie is not set to HTTP-only" - - # - # Confidence is a value from 1 to 100 indicating how confident the - # software is that the results are valid. - # Examples: 100, 90, 75, 15, 10, 0 - - case vuln['type'].to_s - when "ApacheDirectoryListing" - res = { - :risk => 1, - :name => 'Directory Listing', - :blame => 'System Administrator', - :category => 'info', - :description => "", - :confidence => 100 - } - when "ApacheMultiViewsEnabled" - res = { - :risk => 1, - :name => 'Apache MultiViews Enabled', - :blame => 'System Administrator', - :category => 'info', - :description => "", - :confidence => 100 - } - when "ApacheVersion" - res = { - :risk => 1, - :name => 'Web Server Version', - :blame => 'System Administrator', - :category => 'info', - :description => "", - :confidence => 100 - } - when "PHPVersion" - res = { - :risk => 1, - :name => 'PHP Module Version', - :blame => 'System Administrator', - :category => 'info', - :description => "", - :confidence => 100 - } - when "AutoCompleteEnabled" - res = { - :risk => 1, - :name => 'Form AutoComplete Enabled', - :blame => 'App Developer', - :category => 'info', - :description => "", - :confidence => 100 - } - when "CookieNotMarkedAsHttpOnly" - res = { - :risk => 1, - :name => 'Cookie Not HttpOnly', - :blame => 'App Developer', - :category => 'info', - :description => "", - :confidence => 100 - } - when "EmailDisclosure" - res = { - :risk => 1, - :name => 'Email Address Disclosure', - :blame => 'App Developer', - :category => 'info', - :description => "", - :confidence => 100 - } - when "ForbiddenResource" - res = { - :risk => 1, - :name => 'Forbidden Resource', - :blame => 'App Developer', - :category => 'info', - :description => "", - :confidence => 100 - } - when "FileUploadFound" - res = { - :risk => 1, - :name => 'File Upload Form', - :blame => 'App Developer', - :category => 'info', - :description => "", - :confidence => 100 - } - when "PasswordOverHTTP" - res = { - :risk => 2, - :name => 'Password Over HTTP', - :blame => 'App Developer', - :category => 'info', - :description => "", - :confidence => 100 - } - when "MySQL5Identified" - res = { - :risk => 1, - :name => 'MySQL 5 Identified', - :blame => 'App Developer', - :category => 'info', - :description => "", - :confidence => 100 - } - when "PossibleInternalWindowsPathLeakage" - res = { - :risk => 1, - :name => 'Path Leakage - Windows', - :blame => 'App Developer', - :category => 'info', - :description => "", - :confidence => 100 - } - when "PossibleInternalUnixPathLeakage" - res = { - :risk => 1, - :name => 'Path Leakage - Unix', - :blame => 'App Developer', - :category => 'info', - :description => "", - :confidence => 100 - } - when "PossibleXSS", "LowPossibilityPermanentXSS", "XSS", "PermanentXSS" - conf = 100 - conf = 25 if vuln['type'].to_s == "LowPossibilityPermanentXSS" - conf = 50 if vuln['type'].to_s == "PossibleXSS" - res = { - :risk => 3, - :name => 'Cross-Site Scripting', - :blame => 'App Developer', - :category => 'xss', - :description => "", - :confidence => conf - } - - when "ConfirmedBlindSQLInjection", "ConfirmedSQLInjection", "HighlyPossibleSqlInjection", "DatabaseErrorMessages" - conf = 100 - conf = 90 if vuln['type'].to_s == "HighlyPossibleSqlInjection" - conf = 25 if vuln['type'].to_s == "DatabaseErrorMessages" - res = { - :risk => 5, - :name => 'SQL Injection', - :blame => 'App Developer', - :category => 'sql', - :description => "", - :confidence => conf - } - else - conf = 100 - res = { - :risk => 1, - :name => vuln['type'].to_s, - :blame => 'App Developer', - :category => 'info', - :description => "", - :confidence => conf - } - end - - res - end - - def import_fusionvm_xml(args={}) - args[:wspace] ||= workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - doc = Rex::Parser::FusionVMDocument.new(args,self) - parser = ::Nokogiri::XML::SAX::Parser.new(doc) - parser.parse(args[:data]) - end - - - # - # Import Nmap's -oX xml output - # - def import_nmap_xml_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - - data = "" - ::File.open(filename, 'rb') do |f| - data = f.read(f.stat.size) - end - import_nmap_xml(args.merge(:data => data)) - end - - def import_nexpose_raw_noko_stream(args, &block) - if block - doc = Rex::Parser::NexposeRawDocument.new(args,framework.db) {|type, data| yield type,data } - else - doc = Rex::Parser::NexposeRawDocument.new(args,self) - end - parser = ::Nokogiri::XML::SAX::Parser.new(doc) - parser.parse(args[:data]) - end - - def import_nexpose_noko_stream(args, &block) - if block - doc = Rex::Parser::NexposeSimpleDocument.new(args,framework.db) {|type, data| yield type,data } - else - doc = Rex::Parser::NexposeSimpleDocument.new(args,self) - end - parser = ::Nokogiri::XML::SAX::Parser.new(doc) - parser.parse(args[:data]) - end - - def import_nmap_noko_stream(args, &block) - if block - doc = Rex::Parser::NmapDocument.new(args,framework.db) {|type, data| yield type,data } - else - doc = Rex::Parser::NmapDocument.new(args,self) - end - parser = ::Nokogiri::XML::SAX::Parser.new(doc) - parser.parse(args[:data]) - end - - # If you have Nokogiri installed, you'll be shunted over to - # that. Otherwise, you'll hit the old NmapXMLStreamParser. - def import_nmap_xml(args={}, &block) - return nil if args[:data].nil? or args[:data].empty? - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - - if Rex::Parser.nokogiri_loaded - noko_args = args.dup - noko_args[:blacklist] = bl - noko_args[:wspace] = wspace - if block - yield(:parser, "Nokogiri v#{::Nokogiri::VERSION}") - import_nmap_noko_stream(noko_args) {|type, data| yield type,data } - else - import_nmap_noko_stream(noko_args) - end - return true - end - - # XXX: Legacy nmap xml parser starts here. - - fix_services = args[:fix_services] - data = args[:data] - - # Use a stream parser instead of a tree parser so we can deal with - # huge results files without running out of memory. - parser = Rex::Parser::NmapXMLStreamParser.new - yield(:parser, parser.class.name) if block - - # Whenever the parser pulls a host out of the nmap results, store - # it, along with any associated services, in the database. - parser.on_found_host = Proc.new { |h| - hobj = nil - data = {:workspace => wspace} - if (h["addrs"].has_key?("ipv4")) - addr = h["addrs"]["ipv4"] - elsif (h["addrs"].has_key?("ipv6")) - addr = h["addrs"]["ipv6"] - else - # Can't report it if it doesn't have an IP - raise RuntimeError, "At least one IPv4 or IPv6 address is required" - end - next if bl.include? addr - data[:host] = addr - if (h["addrs"].has_key?("mac")) - data[:mac] = h["addrs"]["mac"] - end - data[:state] = (h["status"] == "up") ? Msf::HostState::Alive : Msf::HostState::Dead - data[:task] = args[:task] - - if ( h["reverse_dns"] ) - data[:name] = h["reverse_dns"] - end - - # Only report alive hosts with ports to speak of. - if(data[:state] != Msf::HostState::Dead) - if h["ports"].size > 0 - if fix_services - port_states = h["ports"].map {|p| p["state"]}.reject {|p| p == "filtered"} - next if port_states.compact.empty? - end - yield(:address,data[:host]) if block - hobj = report_host(data) - report_import_note(wspace,hobj) - end - end - - if( h["os_vendor"] ) - note = { - :workspace => wspace, - :host => hobj || addr, - :type => 'host.os.nmap_fingerprint', - :task => args[:task], - :data => { - :os_vendor => h["os_vendor"], - :os_family => h["os_family"], - :os_version => h["os_version"], - :os_accuracy => h["os_accuracy"] - } - } - - if(h["os_match"]) - note[:data][:os_match] = h['os_match'] - end - - report_note(note) - end - - if (h["last_boot"]) - report_note( - :workspace => wspace, - :host => hobj || addr, - :type => 'host.last_boot', - :task => args[:task], - :data => { - :time => h["last_boot"] - } - ) - end - - if (h["trace"]) - hops = [] - h["trace"]["hops"].each do |hop| - hops << { - "ttl" => hop["ttl"].to_i, - "address" => hop["ipaddr"].to_s, - "rtt" => hop["rtt"].to_f, - "name" => hop["host"].to_s - } - end - report_note( - :workspace => wspace, - :host => hobj || addr, - :type => 'host.nmap.traceroute', - :task => args[:task], - :data => { - 'port' => h["trace"]["port"].to_i, - 'proto' => h["trace"]["proto"].to_s, - 'hops' => hops - } - ) - end - - - # Put all the ports, regardless of state, into the db. - h["ports"].each { |p| - # Localhost port results are pretty unreliable -- if it's - # unknown, it's no good (possibly Windows-only) - if ( - p["state"] == "unknown" && - h["status_reason"] == "localhost-response" - ) - next - end - extra = "" - extra << p["product"] + " " if p["product"] - extra << p["version"] + " " if p["version"] - extra << p["extrainfo"] + " " if p["extrainfo"] - - data = {} - data[:workspace] = wspace - if fix_services - data[:proto] = nmap_msf_service_map(p["protocol"]) - else - data[:proto] = p["protocol"].downcase - end - data[:port] = p["portid"].to_i - data[:state] = p["state"] - data[:host] = hobj || addr - data[:info] = extra if not extra.empty? - data[:task] = args[:task] - if p["name"] != "unknown" - data[:name] = p["name"] - end - report_service(data) - } - #Parse the scripts output - if h["scripts"] - h["scripts"].each do |key,val| - if key == "smb-check-vulns" - if val =~ /MS08-067: VULNERABLE/ - vuln_info = { - :workspace => wspace, - :task => args[:task], - :host => hobj || addr, - :port => 445, - :proto => 'tcp', - :name => 'MS08-067', - :info => 'Microsoft Windows Server Service Crafted RPC Request Handling Unspecified Remote Code Execution', - :refs =>['CVE-2008-4250', - 'BID-31874', - 'OSVDB-49243', - 'CWE-94', - 'MSFT-MS08-067', - 'MSF-Microsoft Server Service Relative Path Stack Corruption', - 'NSS-34476'] - } - report_vuln(vuln_info) - end - if val =~ /MS06-025: VULNERABLE/ - vuln_info = { - :workspace => wspace, - :task => args[:task], - :host => hobj || addr, - :port => 445, - :proto => 'tcp', - :name => 'MS06-025', - :info => 'Vulnerability in Routing and Remote Access Could Allow Remote Code Execution', - :refs =>['CVE-2006-2370', - 'CVE-2006-2371', - 'BID-18325', - 'BID-18358', - 'BID-18424', - 'OSVDB-26436', - 'OSVDB-26437', - 'MSFT-MS06-025', - 'MSF-Microsoft RRAS Service RASMAN Registry Overflow', - 'NSS-21689'] - } - report_vuln(vuln_info) - end - # This one has NOT been Tested , remove this comment if confirmed working - if val =~ /MS07-029: VULNERABLE/ - vuln_info = { - :workspace => wspace, - :task => args[:task], - :host => hobj || addr, - :port => 445, - :proto => 'tcp', - :name => 'MS07-029', - :info => 'Vulnerability in Windows DNS RPC Interface Could Allow Remote Code Execution', - # Add more refs based on nessus/nexpose .. results - :refs =>['CVE-2007-1748', - 'OSVDB-34100', - 'MSF-Microsoft DNS RPC Service extractQuotedChar()', - 'NSS-25168'] - } - report_vuln(vuln_info) - end - end - end - end - } - - # XXX: Legacy nmap xml parser ends here. - - REXML::Document.parse_stream(data, parser) - end - - def nmap_msf_service_map(proto) - service_name_map(proto) - end - - # - # This method normalizes an incoming service name to one of the - # the standard ones recognized by metasploit - # - def service_name_map(proto) - return proto unless proto.kind_of? String - case proto.downcase - when "msrpc", "nfs-or-iis", "dce endpoint resolution" - "dcerpc" - when "ms-sql-s", "tds" - "mssql" - when "ms-sql-m","microsoft sql monitor" - "mssql-m" - when "postgresql"; "postgres" - when "http-proxy"; "http" - when "iiimsf"; "db2" - when "oracle-tns"; "oracle" - when "quickbooksrds"; "metasploit" - when "microsoft remote display protocol" - "rdp" - when "vmware authentication daemon" - "vmauthd" - when "netbios-ns", "cifs name service" - "netbios" - when "netbios-ssn", "microsoft-ds", "cifs" - "smb" - when "remote shell" - "shell" - when "remote login" - "login" - when "nfs lockd" - "lockd" - when "hp jetdirect" - "jetdirect" - when "dhcp server" - "dhcp" - when /^dns-(udp|tcp)$/; "dns" - when /^dce[\s+]rpc$/; "dcerpc" - else - proto.downcase.gsub(/\s*\(.*/, '') # "service (some service)" - end - end - - def report_import_note(wspace,addr) - if @import_filedata.kind_of?(Hash) && @import_filedata[:filename] && @import_filedata[:filename] !~ /msfe-nmap[0-9]{8}/ - report_note( - :workspace => wspace, - :host => addr, - :type => 'host.imported', - :data => @import_filedata.merge(:time=> Time.now.utc) - ) - end - end - - # - # Import Nessus NBE files - # - def import_nessus_nbe_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - - data = "" - ::File.open(filename, 'rb') do |f| - data = f.read(f.stat.size) - end - import_nessus_nbe(args.merge(:data => data)) - end - - # There is no place the NBE actually stores the plugin name used to - # scan. You get "Security Note" or "Security Warning," and that's it. - def import_nessus_nbe(args={}, &block) - nbe_data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - - nbe_copy = nbe_data.dup - # First pass, just to build the address map. - addr_map = {} - - # Cache host objects before passing into handle_nessus() - hobj_map = {} - - nbe_copy.each_line do |line| - r = line.split('|') - next if r[0] != 'results' - next if r[4] != "12053" - data = r[6] - addr,hname = data.match(/([0-9\x2e]+) resolves as (.+)\x2e\\n/n)[1,2] - addr_map[hname] = addr - end - - nbe_data.each_line do |line| - r = line.split('|') - next if r[0] != 'results' - hname = r[2] - if addr_map[hname] - addr = addr_map[hname] - else - addr = hname # Must be unresolved, probably an IP address. - end - port = r[3] - nasl = r[4] - type = r[5] - data = r[6] - - # If there's no resolution, or if it's malformed, skip it. - next unless ipv46_validator(addr) - - if bl.include? addr - next - else - yield(:address,addr) if block - end - - hobj_map[ addr ] ||= report_host(:host => addr, :workspace => wspace, :task => args[:task]) - - # Match the NBE types with the XML severity ratings - case type - # log messages don't actually have any data, they are just - # complaints about not being able to perform this or that test - # because such-and-such was missing - when "Log Message"; next - when "Security Hole"; severity = 3 - when "Security Warning"; severity = 2 - when "Security Note"; severity = 1 - # a severity 0 means there's no extra data, it's just an open port - else; severity = 0 - end - if nasl == "11936" - os = data.match(/The remote host is running (.*)\\n/)[1] - report_note( - :workspace => wspace, - :task => args[:task], - :host => hobj_map[ addr ], - :type => 'host.os.nessus_fingerprint', - :data => { - :os => os.to_s.strip - } - ) - end - - next if nasl.to_s.strip.empty? - plugin_name = nil # NBE doesn't ever populate this - handle_nessus(wspace, hobj_map[ addr ], port, nasl, plugin_name, severity, data) - end - end - - # - # Of course they had to change the nessus format. - # - def import_openvas_xml(args={}, &block) - filename = args[:filename] - wspace = args[:wspace] || workspace - - raise DBImportError.new("No OpenVAS XML support. Please submit a patch to msfdev[at]metasploit.com") - end - - # - # Import IP360 XML v3 output - # - def import_ip360_xml_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - - data = "" - ::File.open(filename, 'rb') do |f| - data = f.read(f.stat.size) - end - import_ip360_xml_v3(args.merge(:data => data)) - end - - # - # Import Nessus XML v1 and v2 output - # - # Old versions of openvas exported this as well - # - def import_nessus_xml_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - - data = "" - ::File.open(filename, 'rb') do |f| - data = f.read(f.stat.size) - end - - if data.index("NessusClientData_v2") - import_nessus_xml_v2(args.merge(:data => data)) - else - import_nessus_xml(args.merge(:data => data)) - end - end - - def import_nessus_xml(args={}, &block) - data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - - doc = rexmlify(data) - doc.elements.each('/NessusClientData/Report/ReportHost') do |host| - hobj = nil - addr = nil - hname = nil - os = nil - # If the name is resolved, the Nessus plugin for DNS - # resolution should be there. If not, fall back to the - # HostName - host.elements.each('ReportItem') do |item| - next unless item.elements['pluginID'].text == "12053" - addr = item.elements['data'].text.match(/([0-9\x2e]+) resolves as/n)[1] - hname = host.elements['HostName'].text - end - addr ||= host.elements['HostName'].text - next unless ipv46_validator(addr) # Skip resolved names and SCAN-ERROR. - if bl.include? addr - next - else - yield(:address,addr) if block - end - - hinfo = { - :workspace => wspace, - :host => addr, - :task => args[:task] - } - - # Record the hostname - hinfo.merge!(:name => hname.to_s.strip) if hname - hobj = report_host(hinfo) - report_import_note(wspace,hobj) - - # Record the OS - os ||= host.elements["os_name"] - if os - report_note( - :workspace => wspace, - :task => args[:task], - :host => hobj, - :type => 'host.os.nessus_fingerprint', - :data => { - :os => os.text.to_s.strip - } - ) - end - - host.elements.each('ReportItem') do |item| - nasl = item.elements['pluginID'].text - plugin_name = item.elements['pluginName'].text - port = item.elements['port'].text - data = item.elements['data'].text - severity = item.elements['severity'].text - - handle_nessus(wspace, hobj, port, nasl, plugin_name, severity, data, args[:task]) - end - end - end - - def import_nessus_xml_v2(args={}, &block) - data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - - #@host = { - #'hname' => nil, - #'addr' => nil, - #'mac' => nil, - #'os' => nil, - #'ports' => [ 'port' => { 'port' => nil, - # 'svc_name' => nil, - # 'proto' => nil, - # 'severity' => nil, - # 'nasl' => nil, - # 'description' => nil, - # 'cve' => [], - # 'bid' => [], - # 'xref' => [] - # } - # ] - #} - parser = Rex::Parser::NessusXMLStreamParser.new - parser.on_found_host = Proc.new { |host| - - hobj = nil - addr = host['addr'] || host['hname'] - - next unless ipv46_validator(addr) # Catches SCAN-ERROR, among others. - - if bl.include? addr - next - else - yield(:address,addr) if block - end - - os = host['os'] - hname = host['hname'] - mac = host['mac'] - - host_info = { - :workspace => wspace, - :host => addr, - :task => args[:task] - } - host_info[:name] = hname.to_s.strip if hname - # Short mac, protect against Nessus's habit of saving multiple macs - # We can't use them anyway, so take just the first. - host_info[:mac] = mac.to_s.strip.upcase.split(/\s+/).first if mac - - hobj = report_host(host_info) - report_import_note(wspace,hobj) - - os = host['os'] - yield(:os,os) if block - if os - report_note( - :workspace => wspace, - :task => args[:task], - :host => hobj, - :type => 'host.os.nessus_fingerprint', - :data => { - :os => os.to_s.strip - } - ) - end - - host['ports'].each do |item| - next if item['port'] == 0 - msf = nil - nasl = item['nasl'].to_s - nasl_name = item['nasl_name'].to_s - port = item['port'].to_s - proto = item['proto'] || "tcp" - sname = item['svc_name'] - severity = item['severity'] - description = item['description'] - cve = item['cve'] - bid = item['bid'] - xref = item['xref'] - msf = item['msf'] - - yield(:port,port) if block - - handle_nessus_v2(wspace, hobj, port, proto, sname, nasl, nasl_name, severity, description, cve, bid, xref, msf, args[:task]) - - end - yield(:end,hname) if block - } - - REXML::Document.parse_stream(data, parser) - - end - - def import_mbsa_xml(args={}, &block) - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - wspace = args[:wspace] || workspace - if Rex::Parser.nokogiri_loaded - parser = "Nokogiri v#{::Nokogiri::VERSION}" - noko_args = args.dup - noko_args[:blacklist] = bl - noko_args[:wspace] = wspace - if block - yield(:parser, parser) - import_mbsa_noko_stream(noko_args) {|type, data| yield type,data} - else - import_mbsa_noko_stream(noko_args) - end - return true - else # Sorry - raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.") - end - end - - def import_mbsa_noko_stream(args={},&block) - if block - doc = Rex::Parser::MbsaDocument.new(args,framework.db) {|type, data| yield type,data } - else - doc = Rex::Parser::MbsaDocument.new(args,self) - end - parser = ::Nokogiri::XML::SAX::Parser.new(doc) - parser.parse(args[:data]) - end - - def import_foundstone_xml(args={}, &block) - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - wspace = args[:wspace] || workspace - if Rex::Parser.nokogiri_loaded - parser = "Nokogiri v#{::Nokogiri::VERSION}" - noko_args = args.dup - noko_args[:blacklist] = bl - noko_args[:wspace] = wspace - if block - yield(:parser, parser) - import_foundstone_noko_stream(noko_args) {|type, data| yield type,data} - else - import_foundstone_noko_stream(noko_args) - end - return true - else # Sorry - raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.") - end - end - - def import_foundstone_noko_stream(args={},&block) - if block - doc = Rex::Parser::FoundstoneDocument.new(args,framework.db) {|type, data| yield type,data } - else - doc = Rex::Parser::FoundstoneDocument.new(args,self) - end - parser = ::Nokogiri::XML::SAX::Parser.new(doc) - parser.parse(args[:data]) - end - - def import_acunetix_xml(args={}, &block) - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - wspace = args[:wspace] || workspace - if Rex::Parser.nokogiri_loaded - parser = "Nokogiri v#{::Nokogiri::VERSION}" - noko_args = args.dup - noko_args[:blacklist] = bl - noko_args[:wspace] = wspace - if block - yield(:parser, parser) - import_acunetix_noko_stream(noko_args) {|type, data| yield type,data} - else - import_acunetix_noko_stream(noko_args) - end - return true - else # Sorry - raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.") - end - end - - def import_ci_xml(args={}, &block) - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - wspace = args[:wspace] || workspace - if Rex::Parser.nokogiri_loaded - parser = "Nokogiri v#{::Nokogiri::VERSION}" - noko_args = args.dup - noko_args[:blacklist] = bl - noko_args[:wspace] = wspace - if block - yield(:parser, parser) - import_ci_noko_stream(noko_args) {|type, data| yield type,data} - else - import_ci_noko_stream(noko_args) - end - return true - else # Sorry - raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.") - end - end - - def import_acunetix_noko_stream(args={},&block) - if block - doc = Rex::Parser::AcunetixDocument.new(args,framework.db) {|type, data| yield type,data } - else - doc = Rex::Parser::AcunetixFoundstoneDocument.new(args,self) - end - parser = ::Nokogiri::XML::SAX::Parser.new(doc) - parser.parse(args[:data]) - end - - - def import_appscan_xml(args={}, &block) - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - wspace = args[:wspace] || workspace - if Rex::Parser.nokogiri_loaded - parser = "Nokogiri v#{::Nokogiri::VERSION}" - noko_args = args.dup - noko_args[:blacklist] = bl - noko_args[:wspace] = wspace - if block - yield(:parser, parser) - import_appscan_noko_stream(noko_args) {|type, data| yield type,data} - else - import_appscan_noko_stream(noko_args) - end - return true - else # Sorry - raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.") - end - end - - def import_appscan_noko_stream(args={},&block) - if block - doc = Rex::Parser::AppscanDocument.new(args,framework.db) {|type, data| yield type,data } - else - doc = Rex::Parser::AppscanDocument.new(args,self) - end - parser = ::Nokogiri::XML::SAX::Parser.new(doc) - parser.parse(args[:data]) - end - - def import_burp_session_xml(args={}, &block) - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - wspace = args[:wspace] || workspace - if Rex::Parser.nokogiri_loaded - # Rex::Parser.reload("burp_session_nokogiri.rb") - parser = "Nokogiri v#{::Nokogiri::VERSION}" - noko_args = args.dup - noko_args[:blacklist] = bl - noko_args[:wspace] = wspace - if block - yield(:parser, parser) - import_burp_session_noko_stream(noko_args) {|type, data| yield type,data} - else - import_burp_session_noko_stream(noko_args) - end - return true - else # Sorry - raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.") - end - end - - def import_burp_session_noko_stream(args={},&block) - if block - doc = Rex::Parser::BurpSessionDocument.new(args,framework.db) {|type, data| yield type,data } - else - doc = Rex::Parser::BurpSessionDocument.new(args,self) - end - parser = ::Nokogiri::XML::SAX::Parser.new(doc) - parser.parse(args[:data]) - end - - - # - # Import IP360's ASPL database - # - def import_ip360_aspl_xml(args={}, &block) - data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - - if not data.index(" {'name' => { }, 'cve' => { }, 'bid' => { } } - # 'oses' => {'name' } } - - aspl_path = nil - aspl_paths = [ - ::File.join(Msf::Config.config_directory, "data", "ncircle", "ip360.aspl"), - ::File.join(Msf::Config.data_directory, "ncircle", "ip360.aspl") - ] - - aspl_paths.each do |tpath| - next if not (::File.exist?(tpath) and ::File.readable?(tpath)) - aspl_path = tpath - break - end - - if not aspl_path - raise DBImportError.new("The nCircle IP360 ASPL file is not present.\n Download ASPL from nCircle VNE | Administer | Support | Resources, unzip it, and import it first") - end - - # parse nCircle ASPL file - aspl = "" - ::File.open(aspl_path, "rb") do |f| - aspl = f.read(f.stat.size) - end - - @asplhash = nil - parser = Rex::Parser::IP360ASPLXMLStreamParser.new - parser.on_found_aspl = Proc.new { |asplh| - @asplhash = asplh - } - REXML::Document.parse_stream(aspl, parser) - - # nCircle has some quotes escaped which causes the parser to break - # we don't need these lines so just replace \" with " - data.gsub!(/\\"/,'"') - - # parse nCircle Scan Output - parser = Rex::Parser::IP360XMLStreamParser.new - parser.on_found_host = Proc.new { |host| - hobj = nil - addr = host['addr'] || host['hname'] - - next unless ipv46_validator(addr) # Catches SCAN-ERROR, among others. - - if bl.include? addr - next - else - yield(:address,addr) if block - end - - os = host['os'] - hname = host['hname'] - mac = host['mac'] - - host_hash = { - :workspace => wspace, - :host => addr, - :task => args[:task] - } - host_hash[:name] = hname.to_s.strip if hname - host_hash[:mac] = mac.to_s.strip.upcase if mac - - hobj = report_host(host_hash) - - yield(:os, os) if block - if os - report_note( - :workspace => wspace, - :task => args[:task], - :host => hobj, - :type => 'host.os.ip360_fingerprint', - :data => { - :os => @asplhash['oses'][os].to_s.strip - } - ) - end - - host['apps'].each do |item| - port = item['port'].to_s - proto = item['proto'].to_s - - handle_ip360_v3_svc(wspace, hobj, port, proto, hname, args[:task]) - end - - - host['vulns'].each do |item| - vulnid = item['vulnid'].to_s - port = item['port'].to_s - proto = item['proto'] || "tcp" - vulnname = @asplhash['vulns']['name'][vulnid] - cves = @asplhash['vulns']['cve'][vulnid] - bids = @asplhash['vulns']['bid'][vulnid] - - yield(:port, port) if block - - handle_ip360_v3_vuln(wspace, hobj, port, proto, hname, vulnid, vulnname, cves, bids, args[:task]) - - end - - yield(:end, hname) if block - } - - REXML::Document.parse_stream(data, parser) - end - - def find_qualys_asset_vuln_refs(doc) - vuln_refs = {} - doc.elements.each("/ASSET_DATA_REPORT/GLOSSARY/VULN_DETAILS_LIST/VULN_DETAILS") do |vuln| - next unless vuln.elements['QID'] && vuln.elements['QID'].first - qid = vuln.elements['QID'].first.to_s - vuln_refs[qid] ||= [] - vuln.elements.each('CVE_ID_LIST/CVE_ID') do |ref| - vuln_refs[qid].push('CVE-' + /C..-([0-9\-]{9})/.match(ref.elements['ID'].text.to_s)[1]) - end - vuln.elements.each('BUGTRAQ_ID_LIST/BUGTRAQ_ID') do |ref| - vuln_refs[qid].push('BID-' + ref.elements['ID'].text.to_s) - end - end - return vuln_refs - end - - # Pull out vulnerabilities that have at least one matching - # ref -- many "vulns" are not vulns, just audit information. - def find_qualys_asset_vulns(host,wspace,hobj,vuln_refs,task_id,&block) - host.elements.each("VULN_INFO_LIST/VULN_INFO") do |vi| - next unless vi.elements["QID"] - vi.elements.each("QID") do |qid| - next if vuln_refs[qid.text].nil? || vuln_refs[qid.text].empty? - handle_qualys(wspace, hobj, nil, nil, qid.text, nil, vuln_refs[qid.text], nil, nil, task_id) - end - end - end - - # Takes QID numbers and finds the discovered services in - # a qualys_asset_xml. - def find_qualys_asset_ports(i,host,wspace,hobj,task_id) - return unless (i == 82023 || i == 82004) - proto = i == 82023 ? 'tcp' : 'udp' - qid = host.elements["VULN_INFO_LIST/VULN_INFO/QID[@id='qid_#{i}']"] - qid_result = qid.parent.elements["RESULT[@format='table']"] if qid - hports = qid_result.first.to_s if qid_result - if hports - hports.scan(/([0-9]+)\t(.*?)\t.*?\t([^\t\n]*)/) do |match| - if match[2] == nil or match[2].strip == 'unknown' - name = match[1].strip - else - name = match[2].strip - end - handle_qualys(wspace, hobj, match[0].to_s, proto, 0, nil, nil, name, nil, task_id) - end - end - end - - # - # Import Qualys's Asset Data Report format - # - def import_qualys_asset_xml(args={}, &block) - data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - doc = rexmlify(data) - vuln_refs = find_qualys_asset_vuln_refs(doc) - - # 2nd pass, actually grab the hosts. - doc.elements.each("/ASSET_DATA_REPORT/HOST_LIST/HOST") do |host| - hobj = nil - addr = host.elements["IP"].text if host.elements["IP"] - next unless validate_ips(addr) - if bl.include? addr - next - else - yield(:address,addr) if block - end - hname = ( # Prefer NetBIOS over DNS - (host.elements["NETBIOS"].text if host.elements["NETBIOS"]) || - (host.elements["DNS"].text if host.elements["DNS"]) || - "" ) - hobj = report_host(:workspace => wspace, :host => addr, :name => hname, :state => Msf::HostState::Alive, :task => args[:task]) - report_import_note(wspace,hobj) - - if host.elements["OPERATING_SYSTEM"] - hos = host.elements["OPERATING_SYSTEM"].text - report_note( - :workspace => wspace, - :task => args[:task], - :host => hobj, - :type => 'host.os.qualys_fingerprint', - :data => { :os => hos } - ) - end - - # Report open ports. - find_qualys_asset_ports(82023,host,wspace,hobj, args[:task]) # TCP - find_qualys_asset_ports(82004,host,wspace,hobj, args[:task]) # UDP - - # Report vulns - find_qualys_asset_vulns(host,wspace,hobj,vuln_refs, args[:task],&block) - - end # host - - end - - # - # Import Qualys' Scan xml output - # - def import_qualys_scan_xml_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - - data = "" - ::File.open(filename, 'rb') do |f| - data = f.read(f.stat.size) - end - import_qualys_scan_xml(args.merge(:data => data)) - end - - def import_qualys_scan_xml(args={}, &block) - data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - - - doc = rexmlify(data) - doc.elements.each('/SCAN/IP') do |host| - hobj = nil - addr = host.attributes['value'] - if bl.include? addr - next - else - yield(:address,addr) if block - end - hname = host.attributes['name'] || '' - - hobj = report_host(:workspace => wspace, :host => addr, :name => hname, :state => Msf::HostState::Alive, :task => args[:task]) - report_import_note(wspace,hobj) - - if host.elements["OS"] - hos = host.elements["OS"].text - report_note( - :workspace => wspace, - :task => args[:task], - :host => hobj, - :type => 'host.os.qualys_fingerprint', - :data => { - :os => hos - } - ) - end - - # Open TCP Services List (Qualys ID 82023) - services_tcp = host.elements["SERVICES/CAT/SERVICE[@number='82023']/RESULT"] - if services_tcp - services_tcp.text.scan(/([0-9]+)\t(.*?)\t.*?\t([^\t\n]*)/) do |match| - if match[2] == nil or match[2].strip == 'unknown' - name = match[1].strip - else - name = match[2].strip - end - handle_qualys(wspace, hobj, match[0].to_s, 'tcp', 0, nil, nil, name, nil, args[:task]) - end - end - # Open UDP Services List (Qualys ID 82004) - services_udp = host.elements["SERVICES/CAT/SERVICE[@number='82004']/RESULT"] - if services_udp - services_udp.text.scan(/([0-9]+)\t(.*?)\t.*?\t([^\t\n]*)/) do |match| - if match[2] == nil or match[2].strip == 'unknown' - name = match[1].strip - else - name = match[2].strip - end - handle_qualys(wspace, hobj, match[0].to_s, 'udp', 0, nil, nil, name, nil, args[:task]) - end - end - - # VULNS are confirmed, PRACTICES are unconfirmed vulnerabilities - host.elements.each('VULNS/CAT | PRACTICES/CAT') do |cat| - port = cat.attributes['port'] - protocol = cat.attributes['protocol'] - cat.elements.each('VULN | PRACTICE') do |vuln| - refs = [] - qid = vuln.attributes['number'] - severity = vuln.attributes['severity'] - title = vuln.elements['TITLE'].text.to_s - vuln.elements.each('VENDOR_REFERENCE_LIST/VENDOR_REFERENCE') do |ref| - refs.push(ref.elements['ID'].text.to_s) - end - vuln.elements.each('CVE_ID_LIST/CVE_ID') do |ref| - refs.push('CVE-' + /C..-([0-9\-]{9})/.match(ref.elements['ID'].text.to_s)[1]) - end - vuln.elements.each('BUGTRAQ_ID_LIST/BUGTRAQ_ID') do |ref| - refs.push('BID-' + ref.elements['ID'].text.to_s) - end - - handle_qualys(wspace, hobj, port, protocol, qid, severity, refs, nil,title, args[:task]) - end - end - end - end - - def import_ip_list_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - - data = "" - ::File.open(filename, 'rb') do |f| - data = f.read(f.stat.size) - end - import_ip_list(args.merge(:data => data)) - end - - def import_ip_list(args={}, &block) - data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - - data.each_line do |ip| - ip.strip! - if bl.include? ip - next - else - yield(:address,ip) if block - end - host = find_or_create_host(:workspace => wspace, :host=> ip, :state => Msf::HostState::Alive, :task => args[:task]) - end - end - - def import_amap_log_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - data = "" - ::File.open(filename, 'rb') do |f| - data = f.read(f.stat.size) - end - - case import_filetype_detect(data) - when :amap_log - import_amap_log(args.merge(:data => data)) - when :amap_mlog - import_amap_mlog(args.merge(:data => data)) - else - raise DBImportError.new("Could not determine file type") - end - end - - def import_amap_log(args={}, &block) - data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - - data.each_line do |line| - next if line =~ /^#/ - next if line !~ /^Protocol on ([^:]+):([^\x5c\x2f]+)[\x5c\x2f](tcp|udp) matches (.*)$/n - addr = $1 - next if bl.include? addr - port = $2.to_i - proto = $3.downcase - name = $4 - host = find_or_create_host(:workspace => wspace, :host => addr, :state => Msf::HostState::Alive, :task => args[:task]) - next if not host - yield(:address,addr) if block - info = { - :workspace => wspace, - :task => args[:task], - :host => host, - :proto => proto, - :port => port - } - if name != "unidentified" - info[:name] = name - end - service = find_or_create_service(info) - end - end - - def import_amap_mlog(args={}, &block) - data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - - data.each_line do |line| - next if line =~ /^#/ - r = line.split(':') - next if r.length < 6 - - addr = r[0] - next if bl.include? addr - port = r[1].to_i - proto = r[2].downcase - status = r[3] - name = r[5] - next if status != "open" - - host = find_or_create_host(:workspace => wspace, :host => addr, :state => Msf::HostState::Alive, :task => args[:task]) - next if not host - yield(:address,addr) if block - info = { - :workspace => wspace, - :task => args[:task], - :host => host, - :proto => proto, - :port => port - } - if name != "unidentified" - info[:name] = name - end - service = find_or_create_service(info) - end - end - - def import_ci_noko_stream(args, &block) - if block - doc = Rex::Parser::CIDocument.new(args,framework.db) {|type, data| yield type,data } - else - doc = Rex::Parser::CI.new(args,self) - end - parser = ::Nokogiri::XML::SAX::Parser.new(doc) - parser.parse(args[:data]) - end - - def import_outpost24_xml(args={}, &block) - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - wspace = args[:wspace] || workspace - if Rex::Parser.nokogiri_loaded - parser = "Nokogiri v#{::Nokogiri::VERSION}" - noko_args = args.dup - noko_args[:blacklist] = bl - noko_args[:wspace] = wspace - if block - yield(:parser, parser) - import_outpost24_noko_stream(noko_args) {|type, data| yield type,data} - else - import_outpost24_noko_stream(noko_args) - end - return true - else # Sorry - raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.") - end - end - - def import_outpost24_noko_stream(args={},&block) - if block - doc = Rex::Parser::Outpost24Document.new(args,framework.db) {|type, data| yield type,data } - else - doc = Rex::Parser::Outpost24Document.new(args,self) - end - parser = ::Nokogiri::XML::SAX::Parser.new(doc) - parser.parse(args[:data]) - end - - - def unserialize_object(xml_elem, allow_yaml = false) - return nil unless xml_elem - string = xml_elem.text.to_s.strip - return string unless string.is_a?(String) - return nil if (string.empty? || string.nil?) - - begin - # Validate that it is properly formed base64 first - if string.gsub(/\s+/, '') =~ /^([a-z0-9A-Z\+\/=]+)$/ - Marshal.load($1.unpack("m")[0]) - else - if allow_yaml - begin - YAML.load(string) - rescue - dlog("Badly formatted YAML: '#{string}'") - string - end - else - string - end - end - rescue ::Exception => e - if allow_yaml - YAML.load(string) rescue string - else - string - end - end - end - - # A way to sneak the yield back into the db importer. - # Used by the SAX parsers. - def emit(sym,data,&block) - yield(sym,data) - end - -protected - - # - # This holds all of the shared parsing/handling used by the - # Nessus NBE and NESSUS v1 methods - # - def handle_nessus(wspace, hobj, port, nasl, plugin_name, severity, data,task=nil) - addr = hobj.address - # The port section looks like: - # http (80/tcp) - p = port.match(/^([^\(]+)\((\d+)\/([^\)]+)\)/) - return if not p - - # Unnecessary as the caller should already have reported this host - #report_host(:workspace => wspace, :host => addr, :state => Msf::HostState::Alive) - name = p[1].strip - port = p[2].to_i - proto = p[3].downcase - - info = { :workspace => wspace, :host => hobj, :port => port, :proto => proto, :task => task } - if name != "unknown" and name[-1,1] != "?" - info[:name] = name - end - report_service(info) - - if nasl.nil? || nasl.empty? || nasl == 0 || nasl == "0" - return - end - - data.gsub!("\\n", "\n") - - refs = [] - - if (data =~ /^CVE : (.*)$/) - $1.gsub(/C(VE|AN)\-/, '').split(',').map { |r| r.strip }.each do |r| - refs.push('CVE-' + r) - end - end - - if (data =~ /^BID : (.*)$/) - $1.split(',').map { |r| r.strip }.each do |r| - refs.push('BID-' + r) - end - end - - if (data =~ /^Other references : (.*)$/) - $1.split(',').map { |r| r.strip }.each do |r| - ref_id, ref_val = r.split(':') - ref_val ? refs.push(ref_id + '-' + ref_val) : refs.push(ref_id) - end - end - - nss = 'NSS-' + nasl.to_s.strip - refs << nss - - unless plugin_name.to_s.strip.empty? - vuln_name = plugin_name - else - vuln_name = nss - end - - vuln_info = { - :workspace => wspace, - :host => hobj, - :port => port, - :proto => proto, - :name => vuln_name, - :info => data, - :refs => refs, - :task => task, - } - report_vuln(vuln_info) - end - - # - # NESSUS v2 file format has a dramatically different layout - # for ReportItem data - # - def handle_nessus_v2(wspace,hobj,port,proto,name,nasl,nasl_name,severity,description,cve,bid,xref,msf,task=nil) - addr = hobj.address - - info = { :workspace => wspace, :host => hobj, :port => port, :proto => proto, :task => task } - - unless name =~ /^unknown$|\?$/ - info[:name] = name - end - - if port.to_i != 0 - report_service(info) - end - - if nasl.nil? || nasl.empty? || nasl == 0 || nasl == "0" - return - end - - refs = [] - - cve.each do |r| - r.to_s.gsub!(/C(VE|AN)\-/, '') - refs.push('CVE-' + r.to_s) - end if cve - - bid.each do |r| - refs.push('BID-' + r.to_s) - end if bid - - xref.each do |r| - ref_id, ref_val = r.to_s.split(':') - ref_val ? refs.push(ref_id + '-' + ref_val) : refs.push(ref_id) - end if xref - - msfref = "MSF-" << msf if msf - refs.push msfref if msfref - - nss = 'NSS-' + nasl - if nasl_name.nil? || nasl_name.empty? - vuln_name = nss - else - vuln_name = nasl_name - end - - refs << nss.strip - - vuln = { - :workspace => wspace, - :host => hobj, - :name => vuln_name, - :info => description ? description : "", - :refs => refs, - :task => task, - } - - if port.to_i != 0 - vuln[:port] = port - vuln[:proto] = proto - end - - report_vuln(vuln) - end - - # - # IP360 v3 vuln - # - def handle_ip360_v3_svc(wspace,hobj,port,proto,hname,task=nil) - addr = hobj.address - report_host(:workspace => wspace, :host => hobj, :state => Msf::HostState::Alive, :task => task) - - info = { :workspace => wspace, :host => hobj, :port => port, :proto => proto, :task => task } - if hname != "unknown" and hname[-1,1] != "?" - info[:name] = hname - end - - if port.to_i != 0 - report_service(info) - end - end #handle_ip360_v3_svc - - # - # IP360 v3 vuln - # - def handle_ip360_v3_vuln(wspace,hobj,port,proto,hname,vulnid,vulnname,cves,bids,task=nil) - info = { :workspace => wspace, :host => hobj, :port => port, :proto => proto, :task => task } - if hname != "unknown" and hname[-1,1] != "?" - info[:name] = hname - end - - if port.to_i != 0 - report_service(info) - end - - refs = [] - - cves.split(/,/).each do |cve| - refs.push(cve.to_s) - end if cves - - bids.split(/,/).each do |bid| - refs.push('BID-' + bid.to_s) - end if bids - - description = nil # not working yet - vuln = { - :workspace => wspace, - :host => hobj, - :name => vulnname, - :info => description ? description : "", - :refs => refs, - :task => task - } - - if port.to_i != 0 - vuln[:port] = port - vuln[:proto] = proto - end - - report_vuln(vuln) - end #handle_ip360_v3_vuln - - # - # Qualys report parsing/handling - # - def handle_qualys(wspace, hobj, port, protocol, qid, severity, refs, name=nil, title=nil, task=nil) - addr = hobj.address - port = port.to_i if port - - info = { :workspace => wspace, :host => hobj, :port => port, :proto => protocol, :task => task } - if name and name != 'unknown' and name != 'No registered hostname' - info[:name] = name - end - - if info[:host] && info[:port] && info[:proto] - report_service(info) - end - - fixed_refs = [] - if refs - refs.each do |ref| - case ref - when /^MS[0-9]{2}-[0-9]{3}/ - fixed_refs << "MSB-#{ref}" - else - fixed_refs << ref - end - end - end - - return if qid == 0 - title = 'QUALYS-' + qid if title.nil? or title.empty? - if addr - report_vuln( - :workspace => wspace, - :task => task, - :host => hobj, - :port => port, - :proto => protocol, - :name => title, - :refs => fixed_refs - ) - end - end - - def process_nexpose_data_sxml_refs(vuln) - refs = [] - vid = vuln.attributes['id'].to_s.downcase - vry = vuln.attributes['resultCode'].to_s.upcase - - # Only process vuln-exploitable and vuln-version statuses - return if vry !~ /^V[VE]$/ - - refs = [] - vuln.elements.each('id') do |ref| - rtyp = ref.attributes['type'].to_s.upcase - rval = ref.text.to_s.strip - case rtyp - when 'CVE' - refs << rval.gsub('CAN', 'CVE') - when 'MS' # obsolete? - refs << "MSB-MS-#{rval}" - else - refs << "#{rtyp}-#{rval}" - end - end - - refs << "NEXPOSE-#{vid}" - refs - end - end end diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb new file mode 100644 index 0000000000..1382dd661b --- /dev/null +++ b/lib/msf/core/db_manager/import.rb @@ -0,0 +1,3451 @@ +module Msf::DBManager::Import + # If hex notation is present, turn them into a character. + def dehex(str) + hexen = str.scan(/\x5cx[0-9a-fA-F]{2}/n) + hexen.each { |h| + str.gsub!(h,h[2,2].to_i(16).chr) + } + return str + end + + # A way to sneak the yield back into the db importer. + # Used by the SAX parsers. + def emit(sym,data,&block) + yield(sym,data) + end + + # Takes QID numbers and finds the discovered services in + # a qualys_asset_xml. + def find_qualys_asset_ports(i,host,wspace,hobj,task_id) + return unless (i == 82023 || i == 82004) + proto = i == 82023 ? 'tcp' : 'udp' + qid = host.elements["VULN_INFO_LIST/VULN_INFO/QID[@id='qid_#{i}']"] + qid_result = qid.parent.elements["RESULT[@format='table']"] if qid + hports = qid_result.first.to_s if qid_result + if hports + hports.scan(/([0-9]+)\t(.*?)\t.*?\t([^\t\n]*)/) do |match| + if match[2] == nil or match[2].strip == 'unknown' + name = match[1].strip + else + name = match[2].strip + end + handle_qualys(wspace, hobj, match[0].to_s, proto, 0, nil, nil, name, nil, task_id) + end + end + end + + def find_qualys_asset_vuln_refs(doc) + vuln_refs = {} + doc.elements.each("/ASSET_DATA_REPORT/GLOSSARY/VULN_DETAILS_LIST/VULN_DETAILS") do |vuln| + next unless vuln.elements['QID'] && vuln.elements['QID'].first + qid = vuln.elements['QID'].first.to_s + vuln_refs[qid] ||= [] + vuln.elements.each('CVE_ID_LIST/CVE_ID') do |ref| + vuln_refs[qid].push('CVE-' + /C..-([0-9\-]{9})/.match(ref.elements['ID'].text.to_s)[1]) + end + vuln.elements.each('BUGTRAQ_ID_LIST/BUGTRAQ_ID') do |ref| + vuln_refs[qid].push('BID-' + ref.elements['ID'].text.to_s) + end + end + return vuln_refs + end + + # Pull out vulnerabilities that have at least one matching + # ref -- many "vulns" are not vulns, just audit information. + def find_qualys_asset_vulns(host,wspace,hobj,vuln_refs,task_id,&block) + host.elements.each("VULN_INFO_LIST/VULN_INFO") do |vi| + next unless vi.elements["QID"] + vi.elements.each("QID") do |qid| + next if vuln_refs[qid.text].nil? || vuln_refs[qid.text].empty? + handle_qualys(wspace, hobj, nil, nil, qid.text, nil, vuln_refs[qid.text], nil, nil, task_id) + end + end + end + + # A dispatcher method that figures out the data's file type, + # and sends it off to the appropriate importer. Note that + # import_file_detect will raise an error if the filetype + # is unknown. + def import(args={}, &block) + data = args[:data] || args['data'] + ftype = import_filetype_detect(data) + yield(:filetype, @import_filedata[:type]) if block + self.send "import_#{ftype}".to_sym, args, &block + end + + def import_acunetix_noko_stream(args={},&block) + if block + doc = Rex::Parser::AcunetixDocument.new(args,framework.db) {|type, data| yield type,data } + else + doc = Rex::Parser::AcunetixFoundstoneDocument.new(args,self) + end + parser = ::Nokogiri::XML::SAX::Parser.new(doc) + parser.parse(args[:data]) + end + + def import_acunetix_xml(args={}, &block) + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + wspace = args[:wspace] || workspace + if Rex::Parser.nokogiri_loaded + parser = "Nokogiri v#{::Nokogiri::VERSION}" + noko_args = args.dup + noko_args[:blacklist] = bl + noko_args[:wspace] = wspace + if block + yield(:parser, parser) + import_acunetix_noko_stream(noko_args) {|type, data| yield type,data} + else + import_acunetix_noko_stream(noko_args) + end + return true + else # Sorry + raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.") + end + end + + def import_amap_log(args={}, &block) + data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + + data.each_line do |line| + next if line =~ /^#/ + next if line !~ /^Protocol on ([^:]+):([^\x5c\x2f]+)[\x5c\x2f](tcp|udp) matches (.*)$/n + addr = $1 + next if bl.include? addr + port = $2.to_i + proto = $3.downcase + name = $4 + host = find_or_create_host(:workspace => wspace, :host => addr, :state => Msf::HostState::Alive, :task => args[:task]) + next if not host + yield(:address,addr) if block + info = { + :workspace => wspace, + :task => args[:task], + :host => host, + :proto => proto, + :port => port + } + if name != "unidentified" + info[:name] = name + end + service = find_or_create_service(info) + end + end + + def import_amap_log_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + data = "" + ::File.open(filename, 'rb') do |f| + data = f.read(f.stat.size) + end + + case import_filetype_detect(data) + when :amap_log + import_amap_log(args.merge(:data => data)) + when :amap_mlog + import_amap_mlog(args.merge(:data => data)) + else + raise DBImportError.new("Could not determine file type") + end + end + + def import_amap_mlog(args={}, &block) + data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + + data.each_line do |line| + next if line =~ /^#/ + r = line.split(':') + next if r.length < 6 + + addr = r[0] + next if bl.include? addr + port = r[1].to_i + proto = r[2].downcase + status = r[3] + name = r[5] + next if status != "open" + + host = find_or_create_host(:workspace => wspace, :host => addr, :state => Msf::HostState::Alive, :task => args[:task]) + next if not host + yield(:address,addr) if block + info = { + :workspace => wspace, + :task => args[:task], + :host => host, + :proto => proto, + :port => port + } + if name != "unidentified" + info[:name] = name + end + service = find_or_create_service(info) + end + end + + def import_appscan_noko_stream(args={},&block) + if block + doc = Rex::Parser::AppscanDocument.new(args,framework.db) {|type, data| yield type,data } + else + doc = Rex::Parser::AppscanDocument.new(args,self) + end + parser = ::Nokogiri::XML::SAX::Parser.new(doc) + parser.parse(args[:data]) + end + + def import_appscan_xml(args={}, &block) + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + wspace = args[:wspace] || workspace + if Rex::Parser.nokogiri_loaded + parser = "Nokogiri v#{::Nokogiri::VERSION}" + noko_args = args.dup + noko_args[:blacklist] = bl + noko_args[:wspace] = wspace + if block + yield(:parser, parser) + import_appscan_noko_stream(noko_args) {|type, data| yield type,data} + else + import_appscan_noko_stream(noko_args) + end + return true + else # Sorry + raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.") + end + end + + def import_burp_session_noko_stream(args={},&block) + if block + doc = Rex::Parser::BurpSessionDocument.new(args,framework.db) {|type, data| yield type,data } + else + doc = Rex::Parser::BurpSessionDocument.new(args,self) + end + parser = ::Nokogiri::XML::SAX::Parser.new(doc) + parser.parse(args[:data]) + end + + def import_burp_session_xml(args={}, &block) + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + wspace = args[:wspace] || workspace + if Rex::Parser.nokogiri_loaded + # Rex::Parser.reload("burp_session_nokogiri.rb") + parser = "Nokogiri v#{::Nokogiri::VERSION}" + noko_args = args.dup + noko_args[:blacklist] = bl + noko_args[:wspace] = wspace + if block + yield(:parser, parser) + import_burp_session_noko_stream(noko_args) {|type, data| yield type,data} + else + import_burp_session_noko_stream(noko_args) + end + return true + else # Sorry + raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.") + end + end + + def import_ci_noko_stream(args, &block) + if block + doc = Rex::Parser::CIDocument.new(args,framework.db) {|type, data| yield type,data } + else + doc = Rex::Parser::CI.new(args,self) + end + parser = ::Nokogiri::XML::SAX::Parser.new(doc) + parser.parse(args[:data]) + end + + def import_ci_xml(args={}, &block) + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + wspace = args[:wspace] || workspace + if Rex::Parser.nokogiri_loaded + parser = "Nokogiri v#{::Nokogiri::VERSION}" + noko_args = args.dup + noko_args[:blacklist] = bl + noko_args[:wspace] = wspace + if block + yield(:parser, parser) + import_ci_noko_stream(noko_args) {|type, data| yield type,data} + else + import_ci_noko_stream(noko_args) + end + return true + else # Sorry + raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.") + end + end + + # + # Generic importer that automatically determines the file type being + # imported. Since this looks for vendor-specific strings in the given + # file, there shouldn't be any false detections, but no guarantees. + # + def import_file(args={}, &block) + filename = args[:filename] || args['filename'] + wspace = args[:wspace] || args['wspace'] || workspace + @import_filedata = {} + @import_filedata[:filename] = filename + + data = "" + ::File.open(filename, 'rb') do |f| + # This check is the largest (byte-wise) that we need to do + # since the other 4-byte checks will be subsets of this larger one. + data = f.read(Metasploit::Credential::Exporter::Pwdump::FILE_ID_STRING.size) + end + if data.nil? + raise DBImportError.new("Zero-length file") + end + + if data.index(Metasploit::Credential::Exporter::Pwdump::FILE_ID_STRING) + data = ::File.open(filename, 'rb') + else + case data[0,4] + when "PK\x03\x04" + data = Zip::File.open(filename) + when "\xd4\xc3\xb2\xa1", "\xa1\xb2\xc3\xd4" + data = PacketFu::PcapFile.new(:filename => filename) + else + ::File.open(filename, 'rb') do |f| + sz = f.stat.size + data = f.read(sz) + end + end + end + + + if block + import(args.merge(:data => data)) { |type,data| yield type,data } + else + import(args.merge(:data => data)) + end + end + + # Returns one of the following: + # + # :acunetix_xml + # :amap_log + # :amap_mlog + # :appscan_xml + # :burp_session_xml + # :ci_xml + # :foundstone_xml + # :fusionvm_xml + # :ip360_aspl_xml + # :ip360_xml_v3 + # :ip_list + # :libpcap + # :mbsa_xml + # :msf_cred_dump_zip + # :msf_pwdump + # :msf_xml + # :msf_zip + # :nessus_nbe + # :nessus_xml + # :nessus_xml_v2 + # :netsparker_xml + # :nexpose_rawxml + # :nexpose_simplexml + # :nikto_xml + # :nmap_xml + # :openvas_new_xml + # :openvas_xml + # :outpost24_xml + # :qualys_asset_xml + # :qualys_scan_xml + # :retina_xml + # :spiceworks_csv + # :wapiti_xml + # + # If there is no match, an error is raised instead. + # + # @raise DBImportError if the type can't be detected + def import_filetype_detect(data) + + if data and data.kind_of? Zip::File + if data.entries.empty? + raise DBImportError.new("The zip file provided is empty.") + end + + @import_filedata ||= {} + @import_filedata[:zip_filename] = File.split(data.to_s).last + @import_filedata[:zip_basename] = @import_filedata[:zip_filename].gsub(/\.zip$/,"") + @import_filedata[:zip_entry_names] = data.entries.map {|x| x.name} + + if @import_filedata[:zip_entry_names].include?(Metasploit::Credential::Importer::Zip::MANIFEST_FILE_NAME) + @import_filedata[:type] = "Metasploit Credential Dump" + return :msf_cred_dump_zip + end + + xml_files = @import_filedata[:zip_entry_names].grep(/^(.*)\.xml$/) + + # TODO This check for our zip export should be more extensive + if xml_files.empty? + raise DBImportError.new("The zip file provided is not a Metasploit Zip Export") + end + + @import_filedata[:zip_xml] = xml_files.first + @import_filedata[:type] = "Metasploit Zip Export" + + return :msf_zip + end + + if data and data.kind_of? PacketFu::PcapFile + # Don't check for emptiness here because unlike other formats, we + # haven't read any actual data in yet, only magic bytes to discover + # that this is indeed a pcap file. + #raise DBImportError.new("The pcap file provided is empty.") if data.body.empty? + @import_filedata ||= {} + @import_filedata[:type] = "Libpcap Packet Capture" + return :libpcap + end + + # msfpwdump + if data.present? && data.kind_of?(::File) + @import_filedata[:type] = "Metasploit PWDump Export" + return :msf_pwdump + end + + # This is a text string, lets make sure its treated as binary + data = data.unpack("C*").pack("C*") + if data and data.to_s.strip.length == 0 + raise DBImportError.new("The data provided to the import function was empty") + end + + # Parse the first line or 4k of data from the file + di = data.index("\n") || 4096 + + firstline = data[0, di] + @import_filedata ||= {} + if (firstline.index("")) + @import_filedata[:type] = "Retina XML" + return :retina_xml + elsif (firstline.index(//)) + @import_filedata[:type] = "OpenVAS XML" + return :openvas_new_xml + elsif (firstline.index(/")) + @import_filedata[:type] = "Nessus XML (v1)" + return :nessus_xml + elsif (firstline.index("]/ + + case $1 + when "niktoscan" + @import_filedata[:type] = "Nikto XML" + return :nikto_xml + when "nmaprun" + @import_filedata[:type] = "Nmap XML" + return :nmap_xml + when "openvas-report" + @import_filedata[:type] = "OpenVAS Report" + return :openvas_xml + when "NessusClientData" + @import_filedata[:type] = "Nessus XML (v1)" + return :nessus_xml + when "NessusClientData_v2" + @import_filedata[:type] = "Nessus XML (v2)" + return :nessus_xml_v2 + when "SCAN" + @import_filedata[:type] = "Qualys Scan XML" + return :qualys_scan_xml + when "report" + @import_filedata[:type] = "Wapiti XML" + return :wapiti_xml + when "ASSET_DATA_REPORT" + @import_filedata[:type] = "Qualys Asset XML" + return :qualys_asset_xml + when /MetasploitExpressV[1234]/ + @import_filedata[:type] = "Metasploit XML" + return :msf_xml + when /MetasploitV4/ + @import_filedata[:type] = "Metasploit XML" + return :msf_xml + when /netsparker/ + @import_filedata[:type] = "NetSparker XML" + return :netsparker_xml + when /audits?/ # and are both valid for nCircle. wtfmate. + @import_filedata[:type] = "IP360 XML v3" + return :ip360_xml_v3 + when /ontology/ + @import_filedata[:type] = "IP360 ASPL" + return :ip360_aspl_xml + when /ReportInfo/ + @import_filedata[:type] = "Foundstone" + return :foundstone_xml + when /ScanGroup/ + @import_filedata[:type] = "Acunetix" + return :acunetix_xml + when /AppScanInfo/ # Actually the second line + @import_filedata[:type] = "Appscan" + return :appscan_xml + when "entities" + if line =~ /creator.*\x43\x4f\x52\x45\x20\x49\x4d\x50\x41\x43\x54/ni + @import_filedata[:type] = "CI" + return :ci_xml + end + when "main" + @import_filedata[:type] = "Outpost24 XML" + return :outpost24_xml + else + # Give up if we haven't hit the root tag in the first few lines + break if line_count > 10 + end + line_count += 1 + } + elsif (firstline.index("timestamps|||scan_start")) + @import_filedata[:type] = "Nessus NBE Report" + # then it's a nessus nbe + return :nessus_nbe + elsif (firstline.index("# amap v")) + # then it's an amap mlog + @import_filedata[:type] = "Amap Log -m" + return :amap_mlog + elsif (firstline.index("amap v")) + # then it's an amap log + @import_filedata[:type] = "Amap Log" + return :amap_log + elsif ipv46_validator(firstline) + # then its an IP list + @import_filedata[:type] = "IP Address List" + return :ip_list + elsif (data[0,1024].index(" data)) + end + + + # + # Import IP360's xml output + # + def import_ip360_xml_v3(args={}, &block) + data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + + # @aspl = {'vulns' => {'name' => { }, 'cve' => { }, 'bid' => { } } + # 'oses' => {'name' } } + + aspl_path = nil + aspl_paths = [ + ::File.join(Msf::Config.config_directory, "data", "ncircle", "ip360.aspl"), + ::File.join(Msf::Config.data_directory, "ncircle", "ip360.aspl") + ] + + aspl_paths.each do |tpath| + next if not (::File.exist?(tpath) and ::File.readable?(tpath)) + aspl_path = tpath + break + end + + if not aspl_path + raise DBImportError.new("The nCircle IP360 ASPL file is not present.\n Download ASPL from nCircle VNE | Administer | Support | Resources, unzip it, and import it first") + end + + # parse nCircle ASPL file + aspl = "" + ::File.open(aspl_path, "rb") do |f| + aspl = f.read(f.stat.size) + end + + @asplhash = nil + parser = Rex::Parser::IP360ASPLXMLStreamParser.new + parser.on_found_aspl = Proc.new { |asplh| + @asplhash = asplh + } + REXML::Document.parse_stream(aspl, parser) + + # nCircle has some quotes escaped which causes the parser to break + # we don't need these lines so just replace \" with " + data.gsub!(/\\"/,'"') + + # parse nCircle Scan Output + parser = Rex::Parser::IP360XMLStreamParser.new + parser.on_found_host = Proc.new { |host| + hobj = nil + addr = host['addr'] || host['hname'] + + next unless ipv46_validator(addr) # Catches SCAN-ERROR, among others. + + if bl.include? addr + next + else + yield(:address,addr) if block + end + + os = host['os'] + hname = host['hname'] + mac = host['mac'] + + host_hash = { + :workspace => wspace, + :host => addr, + :task => args[:task] + } + host_hash[:name] = hname.to_s.strip if hname + host_hash[:mac] = mac.to_s.strip.upcase if mac + + hobj = report_host(host_hash) + + yield(:os, os) if block + if os + report_note( + :workspace => wspace, + :task => args[:task], + :host => hobj, + :type => 'host.os.ip360_fingerprint', + :data => { + :os => @asplhash['oses'][os].to_s.strip + } + ) + end + + host['apps'].each do |item| + port = item['port'].to_s + proto = item['proto'].to_s + + handle_ip360_v3_svc(wspace, hobj, port, proto, hname, args[:task]) + end + + + host['vulns'].each do |item| + vulnid = item['vulnid'].to_s + port = item['port'].to_s + proto = item['proto'] || "tcp" + vulnname = @asplhash['vulns']['name'][vulnid] + cves = @asplhash['vulns']['cve'][vulnid] + bids = @asplhash['vulns']['bid'][vulnid] + + yield(:port, port) if block + + handle_ip360_v3_vuln(wspace, hobj, port, proto, hname, vulnid, vulnname, cves, bids, args[:task]) + + end + + yield(:end, hname) if block + } + + REXML::Document.parse_stream(data, parser) + end + + def import_ip_list(args={}, &block) + data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + + data.each_line do |ip| + ip.strip! + if bl.include? ip + next + else + yield(:address,ip) if block + end + host = find_or_create_host(:workspace => wspace, :host=> ip, :state => Msf::HostState::Alive, :task => args[:task]) + end + end + + def import_ip_list_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + + data = "" + ::File.open(filename, 'rb') do |f| + data = f.read(f.stat.size) + end + import_ip_list(args.merge(:data => data)) + end + + # The libpcap file format is handled by PacketFu for data + # extraction. TODO: Make this its own mixin, and possibly + # extend PacketFu to do better stream analysis on the fly. + def import_libpcap(args={}, &block) + data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + # seen_hosts is only used for determining when to yield an address. Once we get + # some packet analysis going, the values will have all sorts of info. The plan + # is to ru through all the packets as a first pass and report host and service, + # then, once we have everything parsed, we can reconstruct sessions and ngrep + # out things like authentication sequences, examine ttl's and window sizes, all + # kinds of crazy awesome stuff like that. + seen_hosts = {} + decoded_packets = 0 + last_count = 0 + data.read_packet_bytes do |p| + if (decoded_packets >= last_count + 1000) and block + yield(:pcap_count, decoded_packets) + last_count = decoded_packets + end + decoded_packets += 1 + + pkt = PacketFu::Packet.parse(p) rescue next # Just silently skip bad packets + + next unless pkt.is_ip? # Skip anything that's not IP. Technically, not Ethernet::Ip + next if pkt.is_tcp? && (pkt.tcp_src == 0 || pkt.tcp_dst == 0) # Skip port 0 + next if pkt.is_udp? && (pkt.udp_src == 0 || pkt.udp_dst == 0) # Skip port 0 + saddr = pkt.ip_saddr + daddr = pkt.ip_daddr + + # Handle blacklists and obviously useless IP addresses, and report the host. + next if (bl | [saddr,daddr]).size == bl.size # Both hosts are blacklisted, skip everything. + unless( bl.include?(saddr) || rfc3330_reserved(saddr)) + yield(:address,saddr) if block and !seen_hosts.keys.include?(saddr) + unless seen_hosts[saddr] + report_host( + :workspace => wspace, + :host => saddr, + :state => Msf::HostState::Alive, + :task => args[:task] + ) + end + seen_hosts[saddr] ||= [] + + end + unless( bl.include?(daddr) || rfc3330_reserved(daddr)) + yield(:address,daddr) if block and !seen_hosts.keys.include?(daddr) + unless seen_hosts[daddr] + report_host( + :workspace => wspace, + :host => daddr, + :state => Msf::HostState::Alive, + :task => args[:task] + ) + end + seen_hosts[daddr] ||= [] + end + + if pkt.is_tcp? # First pass on TCP packets + if (pkt.tcp_flags.syn == 1 and pkt.tcp_flags.ack == 1) or # Oh, this kills me + pkt.tcp_src < 1024 # If it's a low port, assume it's a proper service. + if seen_hosts[saddr] + unless seen_hosts[saddr].include? [pkt.tcp_src,"tcp"] + report_service( + :workspace => wspace, :host => saddr, + :proto => "tcp", :port => pkt.tcp_src, + :state => Msf::ServiceState::Open, + :task => args[:task] + ) + seen_hosts[saddr] << [pkt.tcp_src,"tcp"] + yield(:service,"%s:%d/%s" % [saddr,pkt.tcp_src,"tcp"]) + end + end + end + elsif pkt.is_udp? # First pass on UDP packets + if pkt.udp_src == pkt.udp_dst # Very basic p2p detection. + [saddr,daddr].each do |xaddr| + if seen_hosts[xaddr] + unless seen_hosts[xaddr].include? [pkt.udp_src,"udp"] + report_service( + :workspace => wspace, :host => xaddr, + :proto => "udp", :port => pkt.udp_src, + :state => Msf::ServiceState::Open, + :task => args[:task] + ) + seen_hosts[xaddr] << [pkt.udp_src,"udp"] + yield(:service,"%s:%d/%s" % [xaddr,pkt.udp_src,"udp"]) + end + end + end + elsif pkt.udp_src < 1024 # Probably a service + if seen_hosts[saddr] + unless seen_hosts[saddr].include? [pkt.udp_src,"udp"] + report_service( + :workspace => wspace, :host => saddr, + :proto => "udp", :port => pkt.udp_src, + :state => Msf::ServiceState::Open, + :task => args[:task] + ) + seen_hosts[saddr] << [pkt.udp_src,"udp"] + yield(:service,"%s:%d/%s" % [saddr,pkt.udp_src,"udp"]) + end + end + end + end # tcp or udp + + inspect_single_packet(pkt,wspace,args) + + end # data.body.map + + # Right about here, we should have built up some streams for some stream analysis. + # Not sure what form that will take, but people like shoving many hundreds of + # thousands of packets through this thing, so it'll need to be memory efficient. + + end + + def import_libpcap_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + + data = PacketFu::PcapFile.new(:filename => filename) + import_libpcap(args.merge(:data => data)) + end + + def import_mbsa_noko_stream(args={},&block) + if block + doc = Rex::Parser::MbsaDocument.new(args,framework.db) {|type, data| yield type,data } + else + doc = Rex::Parser::MbsaDocument.new(args,self) + end + parser = ::Nokogiri::XML::SAX::Parser.new(doc) + parser.parse(args[:data]) + end + + def import_mbsa_xml(args={}, &block) + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + wspace = args[:wspace] || workspace + if Rex::Parser.nokogiri_loaded + parser = "Nokogiri v#{::Nokogiri::VERSION}" + noko_args = args.dup + noko_args[:blacklist] = bl + noko_args[:wspace] = wspace + if block + yield(:parser, parser) + import_mbsa_noko_stream(noko_args) {|type, data| yield type,data} + else + import_mbsa_noko_stream(noko_args) + end + return true + else # Sorry + raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.") + end + end + + # Imports loot, tasks, and reports from an MSF ZIP report. + # XXX: This function is stupidly long. It needs to be refactored. + def import_msf_collateral(args={}, &block) + data = ::File.open(args[:filename], "rb") {|f| f.read(f.stat.size)} + wspace = args[:wspace] || args['wspace'] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + basedir = args[:basedir] || args['basedir'] || ::File.join(Msf::Config.data_directory, "msf") + + allow_yaml = false + btag = nil + + doc = rexmlify(data) + if doc.elements["MetasploitExpressV1"] + m_ver = 1 + allow_yaml = true + btag = "MetasploitExpressV1" + elsif doc.elements["MetasploitExpressV2"] + m_ver = 2 + allow_yaml = true + btag = "MetasploitExpressV2" + elsif doc.elements["MetasploitExpressV3"] + m_ver = 3 + btag = "MetasploitExpressV3" + elsif doc.elements["MetasploitExpressV4"] + m_ver = 4 + btag = "MetasploitExpressV4" + elsif doc.elements["MetasploitV4"] + m_ver = 4 + btag = "MetasploitV4" + else + m_ver = nil + end + unless m_ver and btag + raise DBImportError.new("Unsupported Metasploit XML document format") + end + + host_info = {} + doc.elements.each("/#{btag}/hosts/host") do |host| + host_info[host.elements["id"].text.to_s.strip] = nils_for_nulls(host.elements["address"].text.to_s.strip) + end + + # Import Loot + doc.elements.each("/#{btag}/loots/loot") do |loot| + next if bl.include? host_info[loot.elements["host-id"].text.to_s.strip] + loot_info = {} + loot_info[:host] = host_info[loot.elements["host-id"].text.to_s.strip] + loot_info[:workspace] = args[:wspace] + loot_info[:ctype] = nils_for_nulls(loot.elements["content-type"].text.to_s.strip) + loot_info[:info] = nils_for_nulls(unserialize_object(loot.elements["info"], allow_yaml)) + loot_info[:ltype] = nils_for_nulls(loot.elements["ltype"].text.to_s.strip) + loot_info[:name] = nils_for_nulls(loot.elements["name"].text.to_s.strip) + loot_info[:created_at] = nils_for_nulls(loot.elements["created-at"].text.to_s.strip) + loot_info[:updated_at] = nils_for_nulls(loot.elements["updated-at"].text.to_s.strip) + loot_info[:name] = nils_for_nulls(loot.elements["name"].text.to_s.strip) + loot_info[:orig_path] = nils_for_nulls(loot.elements["path"].text.to_s.strip) + loot_info[:task] = args[:task] + tmp = args[:ifd][:zip_tmp] + loot_info[:orig_path].gsub!(/^\./,tmp) if loot_info[:orig_path] + if !loot.elements["service-id"].text.to_s.strip.empty? + unless loot.elements["service-id"].text.to_s.strip == "NULL" + loot_info[:service] = loot.elements["service-id"].text.to_s.strip + end + end + + # Only report loot if we actually have it. + # TODO: Copypasta. Seperate this out. + if ::File.exists? loot_info[:orig_path] + loot_dir = ::File.join(basedir,"loot") + loot_file = ::File.split(loot_info[:orig_path]).last + if ::File.exists? loot_dir + unless (::File.directory?(loot_dir) && ::File.writable?(loot_dir)) + raise DBImportError.new("Could not move files to #{loot_dir}") + end + else + ::FileUtils.mkdir_p(loot_dir) + end + new_loot = ::File.join(loot_dir,loot_file) + loot_info[:path] = new_loot + if ::File.exists?(new_loot) + ::File.unlink new_loot # Delete it, and don't report it. + else + report_loot(loot_info) # It's new, so report it. + end + ::FileUtils.copy(loot_info[:orig_path], new_loot) + yield(:msf_loot, new_loot) if block + end + end + + # Import Tasks + doc.elements.each("/#{btag}/tasks/task") do |task| + task_info = {} + task_info[:workspace] = args[:wspace] + # Should user be imported (original) or declared (the importing user)? + task_info[:user] = nils_for_nulls(task.elements["created-by"].text.to_s.strip) + task_info[:desc] = nils_for_nulls(task.elements["description"].text.to_s.strip) + task_info[:info] = nils_for_nulls(unserialize_object(task.elements["info"], allow_yaml)) + task_info[:mod] = nils_for_nulls(task.elements["module"].text.to_s.strip) + task_info[:options] = nils_for_nulls(task.elements["options"].text.to_s.strip) + task_info[:prog] = nils_for_nulls(task.elements["progress"].text.to_s.strip).to_i + task_info[:created_at] = nils_for_nulls(task.elements["created-at"].text.to_s.strip) + task_info[:updated_at] = nils_for_nulls(task.elements["updated-at"].text.to_s.strip) + if !task.elements["completed-at"].text.to_s.empty? + task_info[:completed_at] = nils_for_nulls(task.elements["completed-at"].text.to_s.strip) + end + if !task.elements["error"].text.to_s.empty? + task_info[:error] = nils_for_nulls(task.elements["error"].text.to_s.strip) + end + if !task.elements["result"].text.to_s.empty? + task_info[:result] = nils_for_nulls(task.elements["result"].text.to_s.strip) + end + task_info[:orig_path] = nils_for_nulls(task.elements["path"].text.to_s.strip) + tmp = args[:ifd][:zip_tmp] + task_info[:orig_path].gsub!(/^\./,tmp) if task_info[:orig_path] + + # Only report a task if we actually have it. + # TODO: Copypasta. Seperate this out. + if ::File.exists? task_info[:orig_path] + tasks_dir = ::File.join(basedir,"tasks") + task_file = ::File.split(task_info[:orig_path]).last + if ::File.exists? tasks_dir + unless (::File.directory?(tasks_dir) && ::File.writable?(tasks_dir)) + raise DBImportError.new("Could not move files to #{tasks_dir}") + end + else + ::FileUtils.mkdir_p(tasks_dir) + end + new_task = ::File.join(tasks_dir,task_file) + task_info[:path] = new_task + if ::File.exists?(new_task) + ::File.unlink new_task # Delete it, and don't report it. + else + report_task(task_info) # It's new, so report it. + end + ::FileUtils.copy(task_info[:orig_path], new_task) + yield(:msf_task, new_task) if block + end + end + + # Import Reports + doc.elements.each("/#{btag}/reports/report") do |report| + import_report(report, args, basedir) + end + end + + # Import credentials given a path to a valid manifest file + # + # @param creds_dump_manifest_path [String] + # @param workspace [Mdm::Workspace] Default: {#workspace} + # @return [void] + def import_msf_cred_dump(creds_dump_manifest_path, workspace) + manifest_file = File.open(creds_dump_manifest_path) + origin = Metasploit::Credential::Origin::Import.create!(filename: File.basename(creds_dump_manifest_path)) + importer = Metasploit::Credential::Importer::Core.new(workspace: workspace, input: manifest_file, origin: origin) + importer.import! + end + + # Import credentials given a path to a valid manifest file + # + # @option args [String] :filename + # @option args [Mdm::Workspace] :wspace Default: {#workspace} + # @return [void] + def import_msf_cred_dump_zip(args = {}) + wspace = args[:wspace] || workspace + origin = Metasploit::Credential::Origin::Import.create!(filename: File.basename(args[:filename])) + importer = Metasploit::Credential::Importer::Zip.new(workspace: wspace, input: File.open(args[:filename]), origin: origin) + importer.import! + nil + end + + # Import a Metasploit XML file. + def import_msf_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + + data = "" + ::File.open(filename, 'rb') do |f| + data = f.read(f.stat.size) + end + import_msf_xml(args.merge(:data => data)) + end + + # Perform in an import of an msfpwdump file + def import_msf_pwdump(args={}, &block) + filename = File.basename(args[:data].path) + wspace = args[:wspace] || workspace + origin = Metasploit::Credential::Origin::Import.create!(filename: filename) + importer = Metasploit::Credential::Importer::Pwdump.new(input: args[:data], workspace: wspace, filename: filename, origin:origin) + importer.import! + importer.input.close unless importer.input.closed? + end + + # Import a Metasploit Express ZIP file. Note that this requires + # a fair bit of filesystem manipulation, and is very much tied + # up with the Metasploit Express ZIP file format export (for + # obvious reasons). In the event directories exist, they will + # be reused. If target files exist, they will be overwritten. + # + # XXX: Refactor so it's not quite as sanity-blasting. + def import_msf_zip(args={}, &block) + data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + + new_tmp = ::File.join(Dir::tmpdir,"msf","imp_#{Rex::Text::rand_text_alphanumeric(4)}",@import_filedata[:zip_basename]) + if ::File.exists? new_tmp + unless (::File.directory?(new_tmp) && ::File.writable?(new_tmp)) + raise DBImportError.new("Could not extract zip file to #{new_tmp}") + end + else + FileUtils.mkdir_p(new_tmp) + end + @import_filedata[:zip_tmp] = new_tmp + + # Grab the list of unique basedirs over all entries. + @import_filedata[:zip_tmp_subdirs] = @import_filedata[:zip_entry_names].map {|x| ::File.split(x)}.map {|x| x[0]}.uniq.reject {|x| x == "."} + + # mkdir all of the base directores we just pulled out, if they don't + # already exist + @import_filedata[:zip_tmp_subdirs].each {|sub| + tmp_subdirs = ::File.join(@import_filedata[:zip_tmp],sub) + if File.exists? tmp_subdirs + unless (::File.directory?(tmp_subdirs) && File.writable?(tmp_subdirs)) + # if it exists but we can't write to it, give up + raise DBImportError.new("Could not extract zip file to #{tmp_subdirs}") + end + else + ::FileUtils.mkdir(tmp_subdirs) + end + } + + data.entries.each do |e| + target = ::File.join(@import_filedata[:zip_tmp], e.name) + data.extract(e,target) + + if target =~ /\.xml\z/ + target_data = ::File.open(target, "rb") {|f| f.read 1024} + if import_filetype_detect(target_data) == :msf_xml + @import_filedata[:zip_extracted_xml] = target + end + end + end + + # Import any creds if there are some in the import file + Dir.entries(@import_filedata[:zip_tmp]).each do |entry| + if entry =~ /^.*#{Regexp.quote(Metasploit::Credential::Exporter::Core::CREDS_DUMP_FILE_IDENTIFIER)}.*/ + manifest_file_path = File.join(@import_filedata[:zip_tmp], entry, Metasploit::Credential::Importer::Zip::MANIFEST_FILE_NAME) + if File.exists? manifest_file_path + import_msf_cred_dump(manifest_file_path, wspace) + end + end + end + + # This will kick the newly-extracted XML file through + # the import_file process all over again. + if @import_filedata[:zip_extracted_xml] + new_args = args.dup + new_args[:filename] = @import_filedata[:zip_extracted_xml] + new_args[:data] = nil + new_args[:ifd] = @import_filedata.dup + if block + import_file(new_args, &block) + else + import_file(new_args) + end + end + + # Kick down to all the MSFX ZIP specific items + if block + import_msf_collateral(new_args, &block) + else + import_msf_collateral(new_args) + end + end + + # There is no place the NBE actually stores the plugin name used to + # scan. You get "Security Note" or "Security Warning," and that's it. + def import_nessus_nbe(args={}, &block) + nbe_data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + + nbe_copy = nbe_data.dup + # First pass, just to build the address map. + addr_map = {} + + # Cache host objects before passing into handle_nessus() + hobj_map = {} + + nbe_copy.each_line do |line| + r = line.split('|') + next if r[0] != 'results' + next if r[4] != "12053" + data = r[6] + addr,hname = data.match(/([0-9\x2e]+) resolves as (.+)\x2e\\n/n)[1,2] + addr_map[hname] = addr + end + + nbe_data.each_line do |line| + r = line.split('|') + next if r[0] != 'results' + hname = r[2] + if addr_map[hname] + addr = addr_map[hname] + else + addr = hname # Must be unresolved, probably an IP address. + end + port = r[3] + nasl = r[4] + type = r[5] + data = r[6] + + # If there's no resolution, or if it's malformed, skip it. + next unless ipv46_validator(addr) + + if bl.include? addr + next + else + yield(:address,addr) if block + end + + hobj_map[ addr ] ||= report_host(:host => addr, :workspace => wspace, :task => args[:task]) + + # Match the NBE types with the XML severity ratings + case type + # log messages don't actually have any data, they are just + # complaints about not being able to perform this or that test + # because such-and-such was missing + when "Log Message"; next + when "Security Hole"; severity = 3 + when "Security Warning"; severity = 2 + when "Security Note"; severity = 1 + # a severity 0 means there's no extra data, it's just an open port + else; severity = 0 + end + if nasl == "11936" + os = data.match(/The remote host is running (.*)\\n/)[1] + report_note( + :workspace => wspace, + :task => args[:task], + :host => hobj_map[ addr ], + :type => 'host.os.nessus_fingerprint', + :data => { + :os => os.to_s.strip + } + ) + end + + next if nasl.to_s.strip.empty? + plugin_name = nil # NBE doesn't ever populate this + handle_nessus(wspace, hobj_map[ addr ], port, nasl, plugin_name, severity, data) + end + end + + # + # Import Nessus NBE files + # + def import_nessus_nbe_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + + data = "" + ::File.open(filename, 'rb') do |f| + data = f.read(f.stat.size) + end + import_nessus_nbe(args.merge(:data => data)) + end + + def import_nessus_xml(args={}, &block) + data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + + doc = rexmlify(data) + doc.elements.each('/NessusClientData/Report/ReportHost') do |host| + hobj = nil + addr = nil + hname = nil + os = nil + # If the name is resolved, the Nessus plugin for DNS + # resolution should be there. If not, fall back to the + # HostName + host.elements.each('ReportItem') do |item| + next unless item.elements['pluginID'].text == "12053" + addr = item.elements['data'].text.match(/([0-9\x2e]+) resolves as/n)[1] + hname = host.elements['HostName'].text + end + addr ||= host.elements['HostName'].text + next unless ipv46_validator(addr) # Skip resolved names and SCAN-ERROR. + if bl.include? addr + next + else + yield(:address,addr) if block + end + + hinfo = { + :workspace => wspace, + :host => addr, + :task => args[:task] + } + + # Record the hostname + hinfo.merge!(:name => hname.to_s.strip) if hname + hobj = report_host(hinfo) + report_import_note(wspace,hobj) + + # Record the OS + os ||= host.elements["os_name"] + if os + report_note( + :workspace => wspace, + :task => args[:task], + :host => hobj, + :type => 'host.os.nessus_fingerprint', + :data => { + :os => os.text.to_s.strip + } + ) + end + + host.elements.each('ReportItem') do |item| + nasl = item.elements['pluginID'].text + plugin_name = item.elements['pluginName'].text + port = item.elements['port'].text + data = item.elements['data'].text + severity = item.elements['severity'].text + + handle_nessus(wspace, hobj, port, nasl, plugin_name, severity, data, args[:task]) + end + end + end + + # + # Import Nessus XML v1 and v2 output + # + # Old versions of openvas exported this as well + # + def import_nessus_xml_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + + data = "" + ::File.open(filename, 'rb') do |f| + data = f.read(f.stat.size) + end + + if data.index("NessusClientData_v2") + import_nessus_xml_v2(args.merge(:data => data)) + else + import_nessus_xml(args.merge(:data => data)) + end + end + + def import_nessus_xml_v2(args={}, &block) + data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + + #@host = { + #'hname' => nil, + #'addr' => nil, + #'mac' => nil, + #'os' => nil, + #'ports' => [ 'port' => { 'port' => nil, + # 'svc_name' => nil, + # 'proto' => nil, + # 'severity' => nil, + # 'nasl' => nil, + # 'description' => nil, + # 'cve' => [], + # 'bid' => [], + # 'xref' => [] + # } + # ] + #} + parser = Rex::Parser::NessusXMLStreamParser.new + parser.on_found_host = Proc.new { |host| + + hobj = nil + addr = host['addr'] || host['hname'] + + next unless ipv46_validator(addr) # Catches SCAN-ERROR, among others. + + if bl.include? addr + next + else + yield(:address,addr) if block + end + + os = host['os'] + hname = host['hname'] + mac = host['mac'] + + host_info = { + :workspace => wspace, + :host => addr, + :task => args[:task] + } + host_info[:name] = hname.to_s.strip if hname + # Short mac, protect against Nessus's habit of saving multiple macs + # We can't use them anyway, so take just the first. + host_info[:mac] = mac.to_s.strip.upcase.split(/\s+/).first if mac + + hobj = report_host(host_info) + report_import_note(wspace,hobj) + + os = host['os'] + yield(:os,os) if block + if os + report_note( + :workspace => wspace, + :task => args[:task], + :host => hobj, + :type => 'host.os.nessus_fingerprint', + :data => { + :os => os.to_s.strip + } + ) + end + + host['ports'].each do |item| + next if item['port'] == 0 + msf = nil + nasl = item['nasl'].to_s + nasl_name = item['nasl_name'].to_s + port = item['port'].to_s + proto = item['proto'] || "tcp" + sname = item['svc_name'] + severity = item['severity'] + description = item['description'] + cve = item['cve'] + bid = item['bid'] + xref = item['xref'] + msf = item['msf'] + + yield(:port,port) if block + + handle_nessus_v2(wspace, hobj, port, proto, sname, nasl, nasl_name, severity, description, cve, bid, xref, msf, args[:task]) + + end + yield(:end,hname) if block + } + + REXML::Document.parse_stream(data, parser) + + end + + # Process NetSparker XML + def import_netsparker_xml(args={}, &block) + data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + addr = nil + parser = Rex::Parser::NetSparkerXMLStreamParser.new + parser.on_found_vuln = Proc.new do |vuln| + data = {:workspace => wspace} + + # Parse the URL + url = vuln['url'] + return if not url + + # Crack the URL into a URI + uri = URI(url) rescue nil + return if not uri + + # Resolve the host and cache the IP + if not addr + baddr = Rex::Socket.addr_aton(uri.host) rescue nil + if baddr + addr = Rex::Socket.addr_ntoa(baddr) + yield(:address, addr) if block + end + end + + # Bail early if we have no IP address + if not addr + raise Interrupt, "Not a valid IP address" + end + + if bl.include?(addr) + raise Interrupt, "IP address is on the blacklist" + end + + data[:host] = addr + data[:vhost] = uri.host + data[:port] = uri.port + data[:ssl] = (uri.scheme == "ssl") + + body = nil + # First report a web page + if vuln['response'] + headers = {} + code = 200 + head,body = vuln['response'].to_s.split(/\r?\n\r?\n/, 2) + if body + + if head =~ /^HTTP\d+\.\d+\s+(\d+)\s*/ + code = $1.to_i + end + + headers = {} + head.split(/\r?\n/).each do |line| + hname,hval = line.strip.split(/\s*:\s*/, 2) + next if hval.to_s.strip.empty? + headers[hname.downcase] ||= [] + headers[hname.downcase] << hval + end + + info = { + :path => uri.path, + :query => uri.query, + :code => code, + :body => body, + :headers => headers, + :task => args[:task] + } + info.merge!(data) + + if headers['content-type'] + info[:ctype] = headers['content-type'][0] + end + + if headers['set-cookie'] + info[:cookie] = headers['set-cookie'].join("\n") + end + + if headers['authorization'] + info[:auth] = headers['authorization'].join("\n") + end + + if headers['location'] + info[:location] = headers['location'][0] + end + + if headers['last-modified'] + info[:mtime] = headers['last-modified'][0] + end + + # Report the web page to the database + report_web_page(info) + + yield(:web_page, url) if block + end + end # End web_page reporting + + + details = netsparker_vulnerability_map(vuln) + + method = netsparker_method_map(vuln) + pname = netsparker_pname_map(vuln) + params = netsparker_params_map(vuln) + + proof = '' + + if vuln['info'] and vuln['info'].length > 0 + proof << vuln['info'].map{|x| "#{x[0]}: #{x[1]}\n" }.join + "\n" + end + + if proof.empty? + if body + proof << body + "\n" + else + proof << vuln['response'].to_s + "\n" + end + end + + if params.empty? and pname + params = [[pname, vuln['vparam_name'].to_s]] + end + + info = { + # XXX: There is a :request attr in the model, but report_web_vuln + # doesn't seem to know about it, so this gets ignored. + #:request => vuln['request'], + :path => uri.path, + :query => uri.query, + :method => method, + :params => params, + :pname => pname.to_s, + :proof => proof, + :risk => details[:risk], + :name => details[:name], + :blame => details[:blame], + :category => details[:category], + :description => details[:description], + :confidence => details[:confidence], + :task => args[:task] + } + info.merge!(data) + + next if vuln['type'].to_s.empty? + + report_web_vuln(info) + yield(:web_vuln, url) if block + end + + # We throw interrupts in our parser when the job is hopeless + begin + REXML::Document.parse_stream(data, parser) + rescue ::Interrupt => e + wlog("The netsparker_xml_import() job was interrupted: #{e}") + end + end + + # Process a NetSparker XML file + def import_netsparker_xml_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + + data = "" + ::File.open(filename, 'rb') do |f| + data = f.read(f.stat.size) + end + import_netsparker_xml(args.merge(:data => data)) + end + + def import_nexpose_noko_stream(args, &block) + if block + doc = Rex::Parser::NexposeSimpleDocument.new(args,framework.db) {|type, data| yield type,data } + else + doc = Rex::Parser::NexposeSimpleDocument.new(args,self) + end + parser = ::Nokogiri::XML::SAX::Parser.new(doc) + parser.parse(args[:data]) + end + + def import_nexpose_raw_noko_stream(args, &block) + if block + doc = Rex::Parser::NexposeRawDocument.new(args,framework.db) {|type, data| yield type,data } + else + doc = Rex::Parser::NexposeRawDocument.new(args,self) + end + parser = ::Nokogiri::XML::SAX::Parser.new(doc) + parser.parse(args[:data]) + end + + def import_nexpose_rawxml(args={}, &block) + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + wspace = args[:wspace] || workspace + if Rex::Parser.nokogiri_loaded + parser = "Nokogiri v#{::Nokogiri::VERSION}" + noko_args = args.dup + noko_args[:blacklist] = bl + noko_args[:wspace] = wspace + if block + yield(:parser, parser) + import_nexpose_raw_noko_stream(noko_args) {|type, data| yield type,data} + else + import_nexpose_raw_noko_stream(noko_args) + end + return true + end + data = args[:data] + + # Use a stream parser instead of a tree parser so we can deal with + # huge results files without running out of memory. + parser = Rex::Parser::NexposeXMLStreamParser.new + + # Since all the Refs have to be in the database before we can use them + # in a Vuln, we store all the hosts until we finish parsing and only + # then put everything in the database. This is memory-intensive for + # large files, but should be much less so than a tree parser. + # + # This method is also considerably faster than parsing through the tree + # looking for references every time we hit a vuln. + hosts = [] + vulns = [] + + # The callback merely populates our in-memory table of hosts and vulns + parser.callback = Proc.new { |type, value| + case type + when :host + # XXX: Blacklist should be checked here instead of saving a + # host we're just going to throw away later + hosts.push(value) + when :vuln + value["id"] = value["id"].downcase if value["id"] + vulns.push(value) + end + } + + REXML::Document.parse_stream(data, parser) + + vuln_refs = nexpose_refs_to_struct(vulns) + hosts.each do |host| + if bl.include? host["addr"] + next + else + yield(:address,host["addr"]) if block + end + nexpose_host_from_rawxml(host, vuln_refs, wspace) + end + end + + # + # Nexpose Raw XML + # + def import_nexpose_rawxml_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + + data = "" + ::File.open(filename, 'rb') do |f| + data = f.read(f.stat.size) + end + import_nexpose_rawxml(args.merge(:data => data)) + end + + def import_nexpose_simplexml(args={}, &block) + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + wspace = args[:wspace] || workspace + if Rex::Parser.nokogiri_loaded + parser = "Nokogiri v#{::Nokogiri::VERSION}" + noko_args = args.dup + noko_args[:blacklist] = bl + noko_args[:wspace] = wspace + if block + yield(:parser, parser) + import_nexpose_noko_stream(noko_args) {|type, data| yield type,data} + else + import_nexpose_noko_stream(noko_args) + end + return true + end + data = args[:data] + + doc = rexmlify(data) + doc.elements.each('/NeXposeSimpleXML/devices/device') do |dev| + addr = dev.attributes['address'].to_s + if bl.include? addr + next + else + yield(:address,addr) if block + end + + fprint = {} + + dev.elements.each('fingerprint/description') do |str| + fprint[:desc] = str.text.to_s.strip + end + dev.elements.each('fingerprint/vendor') do |str| + fprint[:vendor] = str.text.to_s.strip + end + dev.elements.each('fingerprint/family') do |str| + fprint[:family] = str.text.to_s.strip + end + dev.elements.each('fingerprint/product') do |str| + fprint[:product] = str.text.to_s.strip + end + dev.elements.each('fingerprint/version') do |str| + fprint[:version] = str.text.to_s.strip + end + dev.elements.each('fingerprint/architecture') do |str| + fprint[:arch] = str.text.to_s.upcase.strip + end + + conf = { + :workspace => wspace, + :host => addr, + :state => Msf::HostState::Alive, + :task => args[:task] + } + + host = report_host(conf) + report_import_note(wspace, host) + + report_note( + :workspace => wspace, + :host => host, + :type => 'host.os.nexpose_fingerprint', + :data => fprint, + :task => args[:task] + ) + + # Load vulnerabilities not associated with a service + dev.elements.each('vulnerabilities/vulnerability') do |vuln| + vid = vuln.attributes['id'].to_s.downcase + refs = process_nexpose_data_sxml_refs(vuln) + next if not refs + report_vuln( + :workspace => wspace, + :host => host, + :name => 'NEXPOSE-' + vid, + :info => vid, + :refs => refs, + :task => args[:task] + ) + end + + # Load the services + dev.elements.each('services/service') do |svc| + sname = svc.attributes['name'].to_s + sprot = svc.attributes['protocol'].to_s.downcase + sport = svc.attributes['port'].to_s.to_i + next if sport == 0 + + name = sname.split('(')[0].strip + info = '' + + svc.elements.each('fingerprint/description') do |str| + info = str.text.to_s.strip + end + + if(sname.downcase != '') + report_service( + :workspace => wspace, + :host => host, + :proto => sprot, + :port => sport, + :name => name, + :info => info, + :task => args[:task] + ) + else + report_service( + :workspace => wspace, + :host => host, + :proto => sprot, + :port => sport, + :info => info, + :task => args[:task] + ) + end + + # Load vulnerabilities associated with this service + svc.elements.each('vulnerabilities/vulnerability') do |vuln| + vid = vuln.attributes['id'].to_s.downcase + refs = process_nexpose_data_sxml_refs(vuln) + next if not refs + report_vuln( + :workspace => wspace, + :host => host, + :port => sport, + :proto => sprot, + :name => 'NEXPOSE-' + vid, + :info => vid, + :refs => refs, + :task => args[:task] + ) + end + end + end + end + + # + # Nexpose Simple XML + # + # XXX At some point we'll want to make this a stream parser for dealing + # with large results files + # + def import_nexpose_simplexml_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + + data = "" + ::File.open(filename, 'rb') do |f| + data = f.read(f.stat.size) + end + import_nexpose_simplexml(args.merge(:data => data)) + end + + # + # Imports Nikto scan data from -Format xml as notes. + # + def import_nikto_xml(args={}, &block) + data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + doc = rexmlify(data) + doc.elements.each do |f| + f.elements.each('scandetails') do |host| + # Get host information + addr = host.attributes['targetip'] + next if not addr + if bl.include? addr + next + else + yield(:address,addr) if block + end + # Get service information + port = host.attributes['targetport'] + next if port.to_i == 0 + uri = URI.parse(host.attributes['sitename']) rescue nil + next unless uri and uri.scheme + # Collect and report scan descriptions. + host.elements.each do |item| + if item.elements['description'] + desc_text = item.elements['description'].text + next if desc_text.nil? or desc_text.empty? + desc_data = { + :workspace => wspace, + :host => addr, + :type => "service.nikto.scan.description", + :data => desc_text, + :proto => "tcp", + :port => port.to_i, + :sname => uri.scheme, + :update => :unique_data, + :task => args[:task] + } + # Always report it as a note. + report_note(desc_data) + # Sometimes report it as a vuln, too. + # XXX: There's a Vuln.info field but nothing reads from it? See Bug #5837 + if item.attributes['osvdbid'].to_i != 0 + desc_data[:refs] = ["OSVDB-#{item.attributes['osvdbid']}"] + desc_data[:name] = "NIKTO-#{item.attributes['id']}" + desc_data.delete(:data) + desc_data.delete(:type) + desc_data.delete(:update) + report_vuln(desc_data) + end + end + end + end + end + end + + def import_nmap_noko_stream(args, &block) + if block + doc = Rex::Parser::NmapDocument.new(args,framework.db) {|type, data| yield type,data } + else + doc = Rex::Parser::NmapDocument.new(args,self) + end + parser = ::Nokogiri::XML::SAX::Parser.new(doc) + parser.parse(args[:data]) + end + + # If you have Nokogiri installed, you'll be shunted over to + # that. Otherwise, you'll hit the old NmapXMLStreamParser. + def import_nmap_xml(args={}, &block) + return nil if args[:data].nil? or args[:data].empty? + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + + if Rex::Parser.nokogiri_loaded + noko_args = args.dup + noko_args[:blacklist] = bl + noko_args[:wspace] = wspace + if block + yield(:parser, "Nokogiri v#{::Nokogiri::VERSION}") + import_nmap_noko_stream(noko_args) {|type, data| yield type,data } + else + import_nmap_noko_stream(noko_args) + end + return true + end + + # XXX: Legacy nmap xml parser starts here. + + fix_services = args[:fix_services] + data = args[:data] + + # Use a stream parser instead of a tree parser so we can deal with + # huge results files without running out of memory. + parser = Rex::Parser::NmapXMLStreamParser.new + yield(:parser, parser.class.name) if block + + # Whenever the parser pulls a host out of the nmap results, store + # it, along with any associated services, in the database. + parser.on_found_host = Proc.new { |h| + hobj = nil + data = {:workspace => wspace} + if (h["addrs"].has_key?("ipv4")) + addr = h["addrs"]["ipv4"] + elsif (h["addrs"].has_key?("ipv6")) + addr = h["addrs"]["ipv6"] + else + # Can't report it if it doesn't have an IP + raise RuntimeError, "At least one IPv4 or IPv6 address is required" + end + next if bl.include? addr + data[:host] = addr + if (h["addrs"].has_key?("mac")) + data[:mac] = h["addrs"]["mac"] + end + data[:state] = (h["status"] == "up") ? Msf::HostState::Alive : Msf::HostState::Dead + data[:task] = args[:task] + + if ( h["reverse_dns"] ) + data[:name] = h["reverse_dns"] + end + + # Only report alive hosts with ports to speak of. + if(data[:state] != Msf::HostState::Dead) + if h["ports"].size > 0 + if fix_services + port_states = h["ports"].map {|p| p["state"]}.reject {|p| p == "filtered"} + next if port_states.compact.empty? + end + yield(:address,data[:host]) if block + hobj = report_host(data) + report_import_note(wspace,hobj) + end + end + + if( h["os_vendor"] ) + note = { + :workspace => wspace, + :host => hobj || addr, + :type => 'host.os.nmap_fingerprint', + :task => args[:task], + :data => { + :os_vendor => h["os_vendor"], + :os_family => h["os_family"], + :os_version => h["os_version"], + :os_accuracy => h["os_accuracy"] + } + } + + if(h["os_match"]) + note[:data][:os_match] = h['os_match'] + end + + report_note(note) + end + + if (h["last_boot"]) + report_note( + :workspace => wspace, + :host => hobj || addr, + :type => 'host.last_boot', + :task => args[:task], + :data => { + :time => h["last_boot"] + } + ) + end + + if (h["trace"]) + hops = [] + h["trace"]["hops"].each do |hop| + hops << { + "ttl" => hop["ttl"].to_i, + "address" => hop["ipaddr"].to_s, + "rtt" => hop["rtt"].to_f, + "name" => hop["host"].to_s + } + end + report_note( + :workspace => wspace, + :host => hobj || addr, + :type => 'host.nmap.traceroute', + :task => args[:task], + :data => { + 'port' => h["trace"]["port"].to_i, + 'proto' => h["trace"]["proto"].to_s, + 'hops' => hops + } + ) + end + + + # Put all the ports, regardless of state, into the db. + h["ports"].each { |p| + # Localhost port results are pretty unreliable -- if it's + # unknown, it's no good (possibly Windows-only) + if ( + p["state"] == "unknown" && + h["status_reason"] == "localhost-response" + ) + next + end + extra = "" + extra << p["product"] + " " if p["product"] + extra << p["version"] + " " if p["version"] + extra << p["extrainfo"] + " " if p["extrainfo"] + + data = {} + data[:workspace] = wspace + if fix_services + data[:proto] = nmap_msf_service_map(p["protocol"]) + else + data[:proto] = p["protocol"].downcase + end + data[:port] = p["portid"].to_i + data[:state] = p["state"] + data[:host] = hobj || addr + data[:info] = extra if not extra.empty? + data[:task] = args[:task] + if p["name"] != "unknown" + data[:name] = p["name"] + end + report_service(data) + } + #Parse the scripts output + if h["scripts"] + h["scripts"].each do |key,val| + if key == "smb-check-vulns" + if val =~ /MS08-067: VULNERABLE/ + vuln_info = { + :workspace => wspace, + :task => args[:task], + :host => hobj || addr, + :port => 445, + :proto => 'tcp', + :name => 'MS08-067', + :info => 'Microsoft Windows Server Service Crafted RPC Request Handling Unspecified Remote Code Execution', + :refs =>['CVE-2008-4250', + 'BID-31874', + 'OSVDB-49243', + 'CWE-94', + 'MSFT-MS08-067', + 'MSF-Microsoft Server Service Relative Path Stack Corruption', + 'NSS-34476'] + } + report_vuln(vuln_info) + end + if val =~ /MS06-025: VULNERABLE/ + vuln_info = { + :workspace => wspace, + :task => args[:task], + :host => hobj || addr, + :port => 445, + :proto => 'tcp', + :name => 'MS06-025', + :info => 'Vulnerability in Routing and Remote Access Could Allow Remote Code Execution', + :refs =>['CVE-2006-2370', + 'CVE-2006-2371', + 'BID-18325', + 'BID-18358', + 'BID-18424', + 'OSVDB-26436', + 'OSVDB-26437', + 'MSFT-MS06-025', + 'MSF-Microsoft RRAS Service RASMAN Registry Overflow', + 'NSS-21689'] + } + report_vuln(vuln_info) + end + # This one has NOT been Tested , remove this comment if confirmed working + if val =~ /MS07-029: VULNERABLE/ + vuln_info = { + :workspace => wspace, + :task => args[:task], + :host => hobj || addr, + :port => 445, + :proto => 'tcp', + :name => 'MS07-029', + :info => 'Vulnerability in Windows DNS RPC Interface Could Allow Remote Code Execution', + # Add more refs based on nessus/nexpose .. results + :refs =>['CVE-2007-1748', + 'OSVDB-34100', + 'MSF-Microsoft DNS RPC Service extractQuotedChar()', + 'NSS-25168'] + } + report_vuln(vuln_info) + end + end + end + end + } + + # XXX: Legacy nmap xml parser ends here. + + REXML::Document.parse_stream(data, parser) + end + + # + # Import Nmap's -oX xml output + # + def import_nmap_xml_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + + data = "" + ::File.open(filename, 'rb') do |f| + data = f.read(f.stat.size) + end + import_nmap_xml(args.merge(:data => data)) + end + + def import_openvas_new_xml(args={}, &block) + if block + doc = Rex::Parser::OpenVASDocument.new(args,framework.db) {|type, data| yield type,data } + else + doc = Rex::Parser::OpenVASDocument.new(args,self) + end + parser = ::Nokogiri::XML::SAX::Parser.new(doc) + parser.parse(args[:data]) + end + + def import_openvas_new_xml_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + + data = "" + ::File.open(filename, 'rb') do |f| + data = f.read(f.stat.size) + end + import_wapiti_xml(args.merge(:data => data)) + end + + # Do all the single packet analysis we can while churning through the pcap + # the first time. Multiple packet inspection will come later, where we can + # do stream analysis, compare requests and responses, etc. + def inspect_single_packet(pkt,wspace,args) + if pkt.is_tcp? or pkt.is_udp? + inspect_single_packet_http(pkt,wspace,args) + end + end + + # Checks for packets that are headed towards port 80, are tcp, contain an HTTP/1.0 + # line, contains an Authorization line, contains a b64-encoded credential, and + # extracts it. Reports this credential and solidifies the service as HTTP. + def inspect_single_packet_http(pkt,wspace,args) + task = args.fetch(:task, nil) + # First, check the server side (data from port 80). + if pkt.is_tcp? and pkt.tcp_src == 80 and !pkt.payload.nil? and !pkt.payload.empty? + if pkt.payload =~ /^HTTP\x2f1\x2e[01]/n + http_server_match = pkt.payload.match(/\nServer:\s+([^\r\n]+)[\r\n]/n) + if http_server_match.kind_of?(MatchData) and http_server_match[1] + report_service( + :workspace => wspace, + :host => pkt.ip_saddr, + :port => pkt.tcp_src, + :proto => "tcp", + :name => "http", + :info => http_server_match[1], + :state => Msf::ServiceState::Open, + :task => task + ) + # That's all we want to know from this service. + return :something_significant + end + end + end + + # Next, check the client side (data to port 80) + if pkt.is_tcp? and pkt.tcp_dst == 80 and !pkt.payload.nil? and !pkt.payload.empty? + if pkt.payload.match(/[\x00-\x20]HTTP\x2f1\x2e[10]/n) + auth_match = pkt.payload.match(/\nAuthorization:\s+Basic\s+([A-Za-z0-9=\x2b]+)/n) + if auth_match.kind_of?(MatchData) and auth_match[1] + b64_cred = auth_match[1] + else + return false + end + # If we're this far, we can surmise that at least the client is a web browser, + # he thinks the server is HTTP and he just made an authentication attempt. At + # this point, we'll just believe everything the packet says -- validation ought + # to come later. + user,pass = b64_cred.unpack("m*").first.split(/:/,2) + report_service( + :workspace => wspace, + :host => pkt.ip_daddr, + :port => pkt.tcp_dst, + :proto => "tcp", + :name => "http", + :task => task + ) + + service_data = { + address: pkt.ip_daddr, + port: pkt.tcp_dst, + service_name: 'http', + protocol: 'tcp', + workspace_id: wspace.id + } + service_data[:task_id] = task.id if task + + filename = args[:filename] + + credential_data = { + origin_type: :import, + private_data: pass, + private_type: :password, + username: user, + filename: filename + } + credential_data.merge!(service_data) + credential_core = create_credential(credential_data) + + login_data = { + core: credential_core, + status: Metasploit::Model::Login::Status::UNTRIED + } + + login_data.merge!(service_data) + + create_credential_login(login_data) + + # That's all we want to know from this service. + return :something_significant + end + end + end + + def import_wapiti_xml(args={}, &block) + if block + doc = Rex::Parser::WapitiDocument.new(args,framework.db) {|type, data| yield type,data } + else + doc = Rex::Parser::WapitiDocument.new(args,self) + end + parser = ::Nokogiri::XML::SAX::Parser.new(doc) + parser.parse(args[:data]) + end + + def import_wapiti_xml_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + + data = "" + ::File.open(filename, 'rb') do |f| + data = f.read(f.stat.size) + end + import_wapiti_xml(args.merge(:data => data)) + end + + # + # Of course they had to change the nessus format. + # + def import_openvas_xml(args={}, &block) + filename = args[:filename] + wspace = args[:wspace] || workspace + + raise DBImportError.new("No OpenVAS XML support. Please submit a patch to msfdev[at]metasploit.com") + end + + def import_outpost24_noko_stream(args={},&block) + if block + doc = Rex::Parser::Outpost24Document.new(args,framework.db) {|type, data| yield type,data } + else + doc = Rex::Parser::Outpost24Document.new(args,self) + end + parser = ::Nokogiri::XML::SAX::Parser.new(doc) + parser.parse(args[:data]) + end + + def import_outpost24_xml(args={}, &block) + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + wspace = args[:wspace] || workspace + if Rex::Parser.nokogiri_loaded + parser = "Nokogiri v#{::Nokogiri::VERSION}" + noko_args = args.dup + noko_args[:blacklist] = bl + noko_args[:wspace] = wspace + if block + yield(:parser, parser) + import_outpost24_noko_stream(noko_args) {|type, data| yield type,data} + else + import_outpost24_noko_stream(noko_args) + end + return true + else # Sorry + raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.") + end + end + + # + # Import Qualys's Asset Data Report format + # + def import_qualys_asset_xml(args={}, &block) + data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + doc = rexmlify(data) + vuln_refs = find_qualys_asset_vuln_refs(doc) + + # 2nd pass, actually grab the hosts. + doc.elements.each("/ASSET_DATA_REPORT/HOST_LIST/HOST") do |host| + hobj = nil + addr = host.elements["IP"].text if host.elements["IP"] + next unless validate_ips(addr) + if bl.include? addr + next + else + yield(:address,addr) if block + end + hname = ( # Prefer NetBIOS over DNS + (host.elements["NETBIOS"].text if host.elements["NETBIOS"]) || + (host.elements["DNS"].text if host.elements["DNS"]) || + "" ) + hobj = report_host(:workspace => wspace, :host => addr, :name => hname, :state => Msf::HostState::Alive, :task => args[:task]) + report_import_note(wspace,hobj) + + if host.elements["OPERATING_SYSTEM"] + hos = host.elements["OPERATING_SYSTEM"].text + report_note( + :workspace => wspace, + :task => args[:task], + :host => hobj, + :type => 'host.os.qualys_fingerprint', + :data => { :os => hos } + ) + end + + # Report open ports. + find_qualys_asset_ports(82023,host,wspace,hobj, args[:task]) # TCP + find_qualys_asset_ports(82004,host,wspace,hobj, args[:task]) # UDP + + # Report vulns + find_qualys_asset_vulns(host,wspace,hobj,vuln_refs, args[:task],&block) + + end # host + + end + + def import_qualys_scan_xml(args={}, &block) + data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + + + doc = rexmlify(data) + doc.elements.each('/SCAN/IP') do |host| + hobj = nil + addr = host.attributes['value'] + if bl.include? addr + next + else + yield(:address,addr) if block + end + hname = host.attributes['name'] || '' + + hobj = report_host(:workspace => wspace, :host => addr, :name => hname, :state => Msf::HostState::Alive, :task => args[:task]) + report_import_note(wspace,hobj) + + if host.elements["OS"] + hos = host.elements["OS"].text + report_note( + :workspace => wspace, + :task => args[:task], + :host => hobj, + :type => 'host.os.qualys_fingerprint', + :data => { + :os => hos + } + ) + end + + # Open TCP Services List (Qualys ID 82023) + services_tcp = host.elements["SERVICES/CAT/SERVICE[@number='82023']/RESULT"] + if services_tcp + services_tcp.text.scan(/([0-9]+)\t(.*?)\t.*?\t([^\t\n]*)/) do |match| + if match[2] == nil or match[2].strip == 'unknown' + name = match[1].strip + else + name = match[2].strip + end + handle_qualys(wspace, hobj, match[0].to_s, 'tcp', 0, nil, nil, name, nil, args[:task]) + end + end + # Open UDP Services List (Qualys ID 82004) + services_udp = host.elements["SERVICES/CAT/SERVICE[@number='82004']/RESULT"] + if services_udp + services_udp.text.scan(/([0-9]+)\t(.*?)\t.*?\t([^\t\n]*)/) do |match| + if match[2] == nil or match[2].strip == 'unknown' + name = match[1].strip + else + name = match[2].strip + end + handle_qualys(wspace, hobj, match[0].to_s, 'udp', 0, nil, nil, name, nil, args[:task]) + end + end + + # VULNS are confirmed, PRACTICES are unconfirmed vulnerabilities + host.elements.each('VULNS/CAT | PRACTICES/CAT') do |cat| + port = cat.attributes['port'] + protocol = cat.attributes['protocol'] + cat.elements.each('VULN | PRACTICE') do |vuln| + refs = [] + qid = vuln.attributes['number'] + severity = vuln.attributes['severity'] + title = vuln.elements['TITLE'].text.to_s + vuln.elements.each('VENDOR_REFERENCE_LIST/VENDOR_REFERENCE') do |ref| + refs.push(ref.elements['ID'].text.to_s) + end + vuln.elements.each('CVE_ID_LIST/CVE_ID') do |ref| + refs.push('CVE-' + /C..-([0-9\-]{9})/.match(ref.elements['ID'].text.to_s)[1]) + end + vuln.elements.each('BUGTRAQ_ID_LIST/BUGTRAQ_ID') do |ref| + refs.push('BID-' + ref.elements['ID'].text.to_s) + end + + handle_qualys(wspace, hobj, port, protocol, qid, severity, refs, nil,title, args[:task]) + end + end + end + end + + # + # Import Qualys' Scan xml output + # + def import_qualys_scan_xml_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + + data = "" + ::File.open(filename, 'rb') do |f| + data = f.read(f.stat.size) + end + import_qualys_scan_xml(args.merge(:data => data)) + end + + # @param report [REXML::Element] to be imported + # @param args [Hash] + # @param base_dir [String] + def import_report(report, args, base_dir) + tmp = args[:ifd][:zip_tmp] + report_info = {} + + report.elements.each do |e| + node_name = e.name + node_value = e.text + + # These need to be converted back to arrays: + array_attrs = %w|addresses file-formats options sections| + if array_attrs.member? node_name + node_value = JSON.parse(node_value) + end + # Don't restore these values: + skip_nodes = %w|id workspace-id artifacts| + next if skip_nodes.member? node_name + + report_info[node_name.parameterize.underscore.to_sym] = node_value + end + # Use current workspace + report_info[:workspace_id] = args[:wspace].id + + # Create report, need new ID to record artifacts + report_id = report_report(report_info) + + # Handle artifacts + report.elements['artifacts'].elements.each do |artifact| + artifact_opts = {} + artifact.elements.each do |attr| + skip_nodes = %w|id accessed-at| + next if skip_nodes.member? attr.name + + symboled_attr = attr.name.parameterize.underscore.to_sym + artifact_opts[symboled_attr] = attr.text + end + # Use new Report as parent + artifact_opts[:report_id] = report_id + # Update to full path + artifact_opts[:file_path].gsub!(/^\./, tmp) + + report_artifact(artifact_opts) + end + end + + # Process Retina XML + def import_retina_xml(args={}, &block) + data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + msg = "Warning: The Retina XML format does not associate vulnerabilities with the\n" + msg << "specific service on which they were found.\n" + msg << "This makes it impossible to correlate exploits to discovered vulnerabilities\n" + msg << "in a reliable fashion." + + yield(:warning,msg) if block + + parser = Rex::Parser::RetinaXMLStreamParser.new + parser.on_found_host = Proc.new do |host| + hobj = nil + data = { + :workspace => wspace, + :task => args[:task] + } + addr = host['address'] + next if not addr + + next if bl.include? addr + data[:host] = addr + + if host['mac'] + data[:mac] = host['mac'] + end + + data[:state] = Msf::HostState::Alive + + if host['hostname'] + data[:name] = host['hostname'] + end + + if host['netbios'] + data[:name] = host['netbios'] + end + + yield(:address, data[:host]) if block + + # Import Host + hobj = report_host(data) + report_import_note(wspace, hobj) + + # Import OS fingerprint + if host["os"] + note = { + :workspace => wspace, + :host => addr, + :type => 'host.os.retina_fingerprint', + :task => args[:task], + :data => { + :os => host["os"] + } + } + report_note(note) + end + + # Import vulnerabilities + host['vulns'].each do |vuln| + refs = vuln['refs'].map{|v| v.join("-")} + refs << "RETINA-#{vuln['rthid']}" if vuln['rthid'] + + vuln_info = { + :workspace => wspace, + :host => addr, + :name => vuln['name'], + :info => vuln['description'], + :refs => refs, + :task => args[:task] + } + + report_vuln(vuln_info) + end + end + + REXML::Document.parse_stream(data, parser) + end + + # Process a Retina XML file + def import_retina_xml_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + + data = "" + ::File.open(filename, 'rb') do |f| + data = f.read(f.stat.size) + end + import_retina_xml(args.merge(:data => data)) + end + + def import_spiceworks_csv(args={}, &block) + data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + CSV.parse(data) do |row| + next unless (["Name", "Manufacturer", "Device Type"] & row).empty? #header + name = row[0] + manufacturer = row[1] + device = row[2] + model = row[3] + ip = row[4] + serialno = row[5] + location = row[6] + os = row[7] + + next unless ip + next if bl.include? ip + + conf = { + :workspace => wspace, + :host => ip, + :name => name, + :task => args[:task] + } + + + if os + report_note( + :workspace => wspace, + :task => args[:task], + :host => ip, + :type => 'host.os.spiceworks_fingerprint', + :data => { + :os => os.to_s.strip + } + ) + end + + info = [] + info << "Serial Number: #{serialno}" unless (serialno.blank? or serialno == name) + info << "Location: #{location}" unless location.blank? + conf[:info] = info.join(", ") unless info.empty? + + host = report_host(conf) + report_import_note(wspace, host) + end + end + + # Handles timestamps from Metasploit Express/Pro imports. + def msf_import_timestamps(opts,obj) + obj.created_at = opts["created_at"] if opts["created_at"] + obj.created_at = opts[:created_at] if opts[:created_at] + obj.updated_at = opts["updated_at"] ? opts["updated_at"] : obj.created_at + obj.updated_at = opts[:updated_at] ? opts[:updated_at] : obj.created_at + return obj + end + + def netsparker_method_map(vuln) + case vuln['vparam_type'] + when "FullQueryString" + "GET" + when "Querystring" + "GET" + when "Post" + "POST" + when "RawUrlInjection" + "GET" + else + "GET" + end + end + + def netsparker_params_map(vuln) + [] + end + + def netsparker_pname_map(vuln) + case vuln['vparam_name'] + when "URI-BASED", "Query Based" + "PATH" + else + vuln['vparam_name'] + end + end + + def netsparker_vulnerability_map(vuln) + res = { + :risk => 1, + :name => 'Information Disclosure', + :blame => 'System Administrator', + :category => 'info', + :description => "This is an information leak", + :confidence => 100 + } + + # Risk is a value from 1-5 indicating the severity of the issue + # Examples: 1, 4, 5 + + # Name is a descriptive name for this vulnerability. + # Examples: XSS, ReflectiveXSS, PersistentXSS + + # Blame indicates who is at fault for the vulnerability + # Examples: App Developer, Server Developer, System Administrator + + # Category indicates the general class of vulnerability + # Examples: info, xss, sql, rfi, lfi, cmd + + # Description is a textual summary of the vulnerability + # Examples: "A reflective cross-site scripting attack" + # "The web server leaks the internal IP address" + # "The cookie is not set to HTTP-only" + + # + # Confidence is a value from 1 to 100 indicating how confident the + # software is that the results are valid. + # Examples: 100, 90, 75, 15, 10, 0 + + case vuln['type'].to_s + when "ApacheDirectoryListing" + res = { + :risk => 1, + :name => 'Directory Listing', + :blame => 'System Administrator', + :category => 'info', + :description => "", + :confidence => 100 + } + when "ApacheMultiViewsEnabled" + res = { + :risk => 1, + :name => 'Apache MultiViews Enabled', + :blame => 'System Administrator', + :category => 'info', + :description => "", + :confidence => 100 + } + when "ApacheVersion" + res = { + :risk => 1, + :name => 'Web Server Version', + :blame => 'System Administrator', + :category => 'info', + :description => "", + :confidence => 100 + } + when "PHPVersion" + res = { + :risk => 1, + :name => 'PHP Module Version', + :blame => 'System Administrator', + :category => 'info', + :description => "", + :confidence => 100 + } + when "AutoCompleteEnabled" + res = { + :risk => 1, + :name => 'Form AutoComplete Enabled', + :blame => 'App Developer', + :category => 'info', + :description => "", + :confidence => 100 + } + when "CookieNotMarkedAsHttpOnly" + res = { + :risk => 1, + :name => 'Cookie Not HttpOnly', + :blame => 'App Developer', + :category => 'info', + :description => "", + :confidence => 100 + } + when "EmailDisclosure" + res = { + :risk => 1, + :name => 'Email Address Disclosure', + :blame => 'App Developer', + :category => 'info', + :description => "", + :confidence => 100 + } + when "ForbiddenResource" + res = { + :risk => 1, + :name => 'Forbidden Resource', + :blame => 'App Developer', + :category => 'info', + :description => "", + :confidence => 100 + } + when "FileUploadFound" + res = { + :risk => 1, + :name => 'File Upload Form', + :blame => 'App Developer', + :category => 'info', + :description => "", + :confidence => 100 + } + when "PasswordOverHTTP" + res = { + :risk => 2, + :name => 'Password Over HTTP', + :blame => 'App Developer', + :category => 'info', + :description => "", + :confidence => 100 + } + when "MySQL5Identified" + res = { + :risk => 1, + :name => 'MySQL 5 Identified', + :blame => 'App Developer', + :category => 'info', + :description => "", + :confidence => 100 + } + when "PossibleInternalWindowsPathLeakage" + res = { + :risk => 1, + :name => 'Path Leakage - Windows', + :blame => 'App Developer', + :category => 'info', + :description => "", + :confidence => 100 + } + when "PossibleInternalUnixPathLeakage" + res = { + :risk => 1, + :name => 'Path Leakage - Unix', + :blame => 'App Developer', + :category => 'info', + :description => "", + :confidence => 100 + } + when "PossibleXSS", "LowPossibilityPermanentXSS", "XSS", "PermanentXSS" + conf = 100 + conf = 25 if vuln['type'].to_s == "LowPossibilityPermanentXSS" + conf = 50 if vuln['type'].to_s == "PossibleXSS" + res = { + :risk => 3, + :name => 'Cross-Site Scripting', + :blame => 'App Developer', + :category => 'xss', + :description => "", + :confidence => conf + } + + when "ConfirmedBlindSQLInjection", "ConfirmedSQLInjection", "HighlyPossibleSqlInjection", "DatabaseErrorMessages" + conf = 100 + conf = 90 if vuln['type'].to_s == "HighlyPossibleSqlInjection" + conf = 25 if vuln['type'].to_s == "DatabaseErrorMessages" + res = { + :risk => 5, + :name => 'SQL Injection', + :blame => 'App Developer', + :category => 'sql', + :description => "", + :confidence => conf + } + else + conf = 100 + res = { + :risk => 1, + :name => vuln['type'].to_s, + :blame => 'App Developer', + :category => 'info', + :description => "", + :confidence => conf + } + end + + res + end + + # Takes a Host object, an array of vuln structs (generated by nexpose_refs_to_struct()), + # and a workspace, and reports the vulns on that host. + def nexpose_host_from_rawxml(h, vstructs, wspace,task=nil) + hobj = nil + data = {:workspace => wspace} + if h["addr"] + addr = h["addr"] + else + # Can't report it if it doesn't have an IP + return + end + data[:host] = addr + if (h["hardware-address"]) + # Put colons between each octet of the MAC address + data[:mac] = h["hardware-address"].gsub(':', '').scan(/../).join(':') + end + data[:state] = (h["status"] == "alive") ? Msf::HostState::Alive : Msf::HostState::Dead + + # Since we only have one name field per host in the database, just + # take the first one. + if (h["names"] and h["names"].first) + data[:name] = h["names"].first + end + + if (data[:state] != Msf::HostState::Dead) + hobj = report_host(data) + report_import_note(wspace, hobj) + end + + if h["notes"] + note = { + :workspace => wspace, + :host => (hobj || addr), + :type => "host.vuln.nexpose_keys", + :data => {}, + :mode => :unique_data, + :task => task + } + h["notes"].each do |v,k| + note[:data][v] ||= [] + next if note[:data][v].include? k + note[:data][v] << k + end + report_note(note) + end + + if h["os_family"] + note = { + :workspace => wspace, + :host => hobj || addr, + :type => 'host.os.nexpose_fingerprint', + :task => task, + :data => { + :family => h["os_family"], + :certainty => h["os_certainty"] + } + } + note[:data][:vendor] = h["os_vendor"] if h["os_vendor"] + note[:data][:product] = h["os_product"] if h["os_product"] + note[:data][:version] = h["os_version"] if h["os_version"] + note[:data][:arch] = h["arch"] if h["arch"] + + report_note(note) + end + + h["endpoints"].each { |p| + extra = "" + extra << p["product"] + " " if p["product"] + extra << p["version"] + " " if p["version"] + + # Skip port-0 endpoints + next if p["port"].to_i == 0 + + # XXX This should probably be handled in a more standard way + # extra << "(" + p["certainty"] + " certainty) " if p["certainty"] + + data = {} + data[:workspace] = wspace + data[:proto] = p["protocol"].downcase + data[:port] = p["port"].to_i + data[:state] = p["status"] + data[:host] = hobj || addr + data[:info] = extra if not extra.empty? + data[:task] = task + if p["name"] != "" + data[:name] = p["name"] + end + report_service(data) + } + + h["vulns"].each_pair { |k,v| + + next if v["status"] !~ /^vulnerable/ + vstruct = vstructs.select {|vs| vs.id.to_s.downcase == v["id"].to_s.downcase}.first + next unless vstruct + data = {} + data[:workspace] = wspace + data[:host] = hobj || addr + data[:proto] = v["protocol"].downcase if v["protocol"] + data[:port] = v["port"].to_i if v["port"] + data[:name] = "NEXPOSE-" + v["id"] + data[:info] = vstruct.title + data[:refs] = vstruct.refs + data[:task] = task + report_vuln(data) + } + end + + # + # Takes an array of vuln hashes, as returned by the NeXpose rawxml stream + # parser, like: + # [ + # {"id"=>"winreg-notes-protocol-handler", severity="8", "refs"=>[{"source"=>"BID", "value"=>"10600"}, ...]} + # {"id"=>"windows-zotob-c", severity="8", "refs"=>[{"source"=>"BID", "value"=>"14513"}, ...]} + # ] + # and transforms it into a struct, containing :id, :refs, :title, and :severity + # + # Other attributes can be added later, as needed. + def nexpose_refs_to_struct(vulns) + ret = [] + vulns.each do |vuln| + next if ret.map {|v| v.id}.include? vuln["id"] + vstruct = Struct.new(:id, :refs, :title, :severity).new + vstruct.id = vuln["id"] + vstruct.title = vuln["title"] + vstruct.severity = vuln["severity"] + vstruct.refs = [] + vuln["refs"].each do |ref| + if ref['source'] == 'BID' + vstruct.refs.push('BID-' + ref["value"]) + elsif ref['source'] == 'CVE' + # value is CVE-$ID + vstruct.refs.push(ref["value"]) + elsif ref['source'] == 'MS' + vstruct.refs.push('MSB-' + ref["value"]) + elsif ref['source'] == 'URL' + vstruct.refs.push('URL-' + ref["value"]) + end + end + ret.push vstruct + end + return ret + end + + # Convert the string "NULL" to actual nil + def nils_for_nulls(str) + str == "NULL" ? nil : str + end + + def nmap_msf_service_map(proto) + service_name_map(proto) + end + + def report_import_note(wspace,addr) + if @import_filedata.kind_of?(Hash) && @import_filedata[:filename] && @import_filedata[:filename] !~ /msfe-nmap[0-9]{8}/ + report_note( + :workspace => wspace, + :host => addr, + :type => 'host.imported', + :data => @import_filedata.merge(:time=> Time.now.utc) + ) + end + end + + # Returns a REXML::Document from the given data. + def rexmlify(data) + if data.kind_of?(REXML::Document) + return data + else + # Make an attempt to recover from a REXML import fail, since + # it's better than dying outright. + begin + return REXML::Document.new(data) + rescue REXML::ParseException => e + dlog("REXML error: Badly formatted XML, attempting to recover. Error was: #{e.inspect}") + return REXML::Document.new(data.gsub(/([\x00-\x08\x0b\x0c\x0e-\x1f\x80-\xff])/n){ |x| "\\x%.2x" % x.unpack("C*")[0] }) + end + end + end + + # + # This method normalizes an incoming service name to one of the + # the standard ones recognized by metasploit + # + def service_name_map(proto) + return proto unless proto.kind_of? String + case proto.downcase + when "msrpc", "nfs-or-iis", "dce endpoint resolution" + "dcerpc" + when "ms-sql-s", "tds" + "mssql" + when "ms-sql-m","microsoft sql monitor" + "mssql-m" + when "postgresql"; "postgres" + when "http-proxy"; "http" + when "iiimsf"; "db2" + when "oracle-tns"; "oracle" + when "quickbooksrds"; "metasploit" + when "microsoft remote display protocol" + "rdp" + when "vmware authentication daemon" + "vmauthd" + when "netbios-ns", "cifs name service" + "netbios" + when "netbios-ssn", "microsoft-ds", "cifs" + "smb" + when "remote shell" + "shell" + when "remote login" + "login" + when "nfs lockd" + "lockd" + when "hp jetdirect" + "jetdirect" + when "dhcp server" + "dhcp" + when /^dns-(udp|tcp)$/; "dns" + when /^dce[\s+]rpc$/; "dcerpc" + else + proto.downcase.gsub(/\s*\(.*/, '') # "service (some service)" + end + end + + def unserialize_object(xml_elem, allow_yaml = false) + return nil unless xml_elem + string = xml_elem.text.to_s.strip + return string unless string.is_a?(String) + return nil if (string.empty? || string.nil?) + + begin + # Validate that it is properly formed base64 first + if string.gsub(/\s+/, '') =~ /^([a-z0-9A-Z\+\/=]+)$/ + Marshal.load($1.unpack("m")[0]) + else + if allow_yaml + begin + YAML.load(string) + rescue + dlog("Badly formatted YAML: '#{string}'") + string + end + else + string + end + end + rescue ::Exception => e + if allow_yaml + YAML.load(string) rescue string + else + string + end + end + end + + # Boils down the validate_import_file to a boolean + def validate_import_file(data) + begin + import_filetype_detect(data) + rescue DBImportError + return false + end + return true + end + + protected + + # IP360 v3 vuln + def handle_ip360_v3_svc(wspace,hobj,port,proto,hname,task=nil) + addr = hobj.address + report_host(:workspace => wspace, :host => hobj, :state => Msf::HostState::Alive, :task => task) + + info = { :workspace => wspace, :host => hobj, :port => port, :proto => proto, :task => task } + if hname != "unknown" and hname[-1,1] != "?" + info[:name] = hname + end + + if port.to_i != 0 + report_service(info) + end + end + + # + # IP360 v3 vuln + # + def handle_ip360_v3_vuln(wspace,hobj,port,proto,hname,vulnid,vulnname,cves,bids,task=nil) + info = { :workspace => wspace, :host => hobj, :port => port, :proto => proto, :task => task } + if hname != "unknown" and hname[-1,1] != "?" + info[:name] = hname + end + + if port.to_i != 0 + report_service(info) + end + + refs = [] + + cves.split(/,/).each do |cve| + refs.push(cve.to_s) + end if cves + + bids.split(/,/).each do |bid| + refs.push('BID-' + bid.to_s) + end if bids + + description = nil # not working yet + vuln = { + :workspace => wspace, + :host => hobj, + :name => vulnname, + :info => description ? description : "", + :refs => refs, + :task => task + } + + if port.to_i != 0 + vuln[:port] = port + vuln[:proto] = proto + end + + report_vuln(vuln) + end + + # + # This holds all of the shared parsing/handling used by the + # Nessus NBE and NESSUS v1 methods + # + def handle_nessus(wspace, hobj, port, nasl, plugin_name, severity, data,task=nil) + addr = hobj.address + # The port section looks like: + # http (80/tcp) + p = port.match(/^([^\(]+)\((\d+)\/([^\)]+)\)/) + return if not p + + # Unnecessary as the caller should already have reported this host + #report_host(:workspace => wspace, :host => addr, :state => Msf::HostState::Alive) + name = p[1].strip + port = p[2].to_i + proto = p[3].downcase + + info = { :workspace => wspace, :host => hobj, :port => port, :proto => proto, :task => task } + if name != "unknown" and name[-1,1] != "?" + info[:name] = name + end + report_service(info) + + if nasl.nil? || nasl.empty? || nasl == 0 || nasl == "0" + return + end + + data.gsub!("\\n", "\n") + + refs = [] + + if (data =~ /^CVE : (.*)$/) + $1.gsub(/C(VE|AN)\-/, '').split(',').map { |r| r.strip }.each do |r| + refs.push('CVE-' + r) + end + end + + if (data =~ /^BID : (.*)$/) + $1.split(',').map { |r| r.strip }.each do |r| + refs.push('BID-' + r) + end + end + + if (data =~ /^Other references : (.*)$/) + $1.split(',').map { |r| r.strip }.each do |r| + ref_id, ref_val = r.split(':') + ref_val ? refs.push(ref_id + '-' + ref_val) : refs.push(ref_id) + end + end + + nss = 'NSS-' + nasl.to_s.strip + refs << nss + + unless plugin_name.to_s.strip.empty? + vuln_name = plugin_name + else + vuln_name = nss + end + + vuln_info = { + :workspace => wspace, + :host => hobj, + :port => port, + :proto => proto, + :name => vuln_name, + :info => data, + :refs => refs, + :task => task, + } + report_vuln(vuln_info) + end + + # + # NESSUS v2 file format has a dramatically different layout + # for ReportItem data + # + def handle_nessus_v2(wspace,hobj,port,proto,name,nasl,nasl_name,severity,description,cve,bid,xref,msf,task=nil) + addr = hobj.address + + info = { :workspace => wspace, :host => hobj, :port => port, :proto => proto, :task => task } + + unless name =~ /^unknown$|\?$/ + info[:name] = name + end + + if port.to_i != 0 + report_service(info) + end + + if nasl.nil? || nasl.empty? || nasl == 0 || nasl == "0" + return + end + + refs = [] + + cve.each do |r| + r.to_s.gsub!(/C(VE|AN)\-/, '') + refs.push('CVE-' + r.to_s) + end if cve + + bid.each do |r| + refs.push('BID-' + r.to_s) + end if bid + + xref.each do |r| + ref_id, ref_val = r.to_s.split(':') + ref_val ? refs.push(ref_id + '-' + ref_val) : refs.push(ref_id) + end if xref + + msfref = "MSF-" << msf if msf + refs.push msfref if msfref + + nss = 'NSS-' + nasl + if nasl_name.nil? || nasl_name.empty? + vuln_name = nss + else + vuln_name = nasl_name + end + + refs << nss.strip + + vuln = { + :workspace => wspace, + :host => hobj, + :name => vuln_name, + :info => description ? description : "", + :refs => refs, + :task => task, + } + + if port.to_i != 0 + vuln[:port] = port + vuln[:proto] = proto + end + + report_vuln(vuln) + end + + # + # Qualys report parsing/handling + # + def handle_qualys(wspace, hobj, port, protocol, qid, severity, refs, name=nil, title=nil, task=nil) + addr = hobj.address + port = port.to_i if port + + info = { :workspace => wspace, :host => hobj, :port => port, :proto => protocol, :task => task } + if name and name != 'unknown' and name != 'No registered hostname' + info[:name] = name + end + + if info[:host] && info[:port] && info[:proto] + report_service(info) + end + + fixed_refs = [] + if refs + refs.each do |ref| + case ref + when /^MS[0-9]{2}-[0-9]{3}/ + fixed_refs << "MSB-#{ref}" + else + fixed_refs << ref + end + end + end + + return if qid == 0 + title = 'QUALYS-' + qid if title.nil? or title.empty? + if addr + report_vuln( + :workspace => wspace, + :task => task, + :host => hobj, + :port => port, + :proto => protocol, + :name => title, + :refs => fixed_refs + ) + end + end + + def process_nexpose_data_sxml_refs(vuln) + refs = [] + vid = vuln.attributes['id'].to_s.downcase + vry = vuln.attributes['resultCode'].to_s.upcase + + # Only process vuln-exploitable and vuln-version statuses + return if vry !~ /^V[VE]$/ + + refs = [] + vuln.elements.each('id') do |ref| + rtyp = ref.attributes['type'].to_s.upcase + rval = ref.text.to_s.strip + case rtyp + when 'CVE' + refs << rval.gsub('CAN', 'CVE') + when 'MS' # obsolete? + refs << "MSB-MS-#{rval}" + else + refs << "#{rtyp}-#{rval}" + end + end + + refs << "NEXPOSE-#{vid}" + refs + end +end \ No newline at end of file From 2c6753a2e008f92465dc9f998732eeeaa38fc809 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Thu, 9 Oct 2014 15:07:17 -0500 Subject: [PATCH 034/159] Extract Msf::DBManager::Import shared examples MSP-11124 --- spec/lib/msf/db_manager_spec.rb | 83 +----------------- .../shared/examples/msf/db_manager/import.rb | 84 +++++++++++++++++++ 2 files changed, 85 insertions(+), 82 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/import.rb diff --git a/spec/lib/msf/db_manager_spec.rb b/spec/lib/msf/db_manager_spec.rb index bfc52a74e6..13b7a7c913 100644 --- a/spec/lib/msf/db_manager_spec.rb +++ b/spec/lib/msf/db_manager_spec.rb @@ -22,6 +22,7 @@ describe Msf::DBManager do it_should_behave_like 'Msf::DBManager::ExploitedHost' it_should_behave_like 'Msf::DBManager::Host' it_should_behave_like 'Msf::DBManager::IPAddress' + it_should_behave_like 'Msf::DBManager::Import' it_should_behave_like 'Msf::DBManager::ImportMsfXml' it_should_behave_like 'Msf::DBManager::Migration' it_should_behave_like 'Msf::DBManager::ModuleCache' @@ -65,7 +66,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :connect } it { is_expected.to respond_to :connection_established? } it { is_expected.to respond_to :create_db } - it { is_expected.to respond_to :dehex } it { is_expected.to respond_to :disconnect } it { is_expected.to respond_to :driver } it { is_expected.to respond_to :drivers } @@ -73,7 +73,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :each_loot } it { is_expected.to respond_to :each_note } it { is_expected.to respond_to :each_vuln } - it { is_expected.to respond_to :emit } it { is_expected.to respond_to :error } it { is_expected.to respond_to :events } it { is_expected.to respond_to :find_or_create_client } @@ -83,9 +82,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :find_or_create_report } it { is_expected.to respond_to :find_or_create_task } it { is_expected.to respond_to :find_or_create_vuln } - it { is_expected.to respond_to :find_qualys_asset_ports } - it { is_expected.to respond_to :find_qualys_asset_vuln_refs } - it { is_expected.to respond_to :find_qualys_asset_vulns } it { is_expected.to respond_to :find_vuln_by_details } it { is_expected.to respond_to :find_vuln_by_refs } it { is_expected.to respond_to :get_client } @@ -94,83 +90,9 @@ describe Msf::DBManager do it { is_expected.to respond_to :get_vuln } it { is_expected.to respond_to :has_ref? } it { is_expected.to respond_to :has_vuln? } - it { is_expected.to respond_to :import } - it { is_expected.to respond_to :import_acunetix_noko_stream } - it { is_expected.to respond_to :import_acunetix_xml } - it { is_expected.to respond_to :import_amap_log } - it { is_expected.to respond_to :import_amap_log_file } - it { is_expected.to respond_to :import_amap_mlog } - it { is_expected.to respond_to :import_appscan_noko_stream } - it { is_expected.to respond_to :import_appscan_xml } - it { is_expected.to respond_to :import_burp_session_noko_stream } - it { is_expected.to respond_to :import_burp_session_xml } - it { is_expected.to respond_to :import_ci_noko_stream } - it { is_expected.to respond_to :import_ci_xml } - it { is_expected.to respond_to :import_file } - it { is_expected.to respond_to :import_filetype_detect } - it { is_expected.to respond_to :import_foundstone_noko_stream } - it { is_expected.to respond_to :import_foundstone_xml } - it { is_expected.to respond_to :import_fusionvm_xml } - it { is_expected.to respond_to :import_ip360_aspl_xml } - it { is_expected.to respond_to :import_ip360_xml_file } - it { is_expected.to respond_to :import_ip360_xml_v3 } - it { is_expected.to respond_to :import_ip_list } - it { is_expected.to respond_to :import_ip_list_file } - it { is_expected.to respond_to :import_libpcap } - it { is_expected.to respond_to :import_libpcap_file } - it { is_expected.to respond_to :import_mbsa_noko_stream } - it { is_expected.to respond_to :import_mbsa_xml } - it { is_expected.to respond_to :import_msf_collateral } - it { is_expected.to respond_to :import_msf_cred_dump } - it { is_expected.to respond_to :import_msf_cred_dump_zip } - it { is_expected.to respond_to :import_msf_file } - it { is_expected.to respond_to :import_msf_pwdump } - it { is_expected.to respond_to :import_msf_zip } - it { is_expected.to respond_to :import_nessus_nbe } - it { is_expected.to respond_to :import_nessus_nbe_file } - it { is_expected.to respond_to :import_nessus_xml } - it { is_expected.to respond_to :import_nessus_xml_file } - it { is_expected.to respond_to :import_nessus_xml_v2 } - it { is_expected.to respond_to :import_netsparker_xml } - it { is_expected.to respond_to :import_netsparker_xml_file } - it { is_expected.to respond_to :import_nexpose_noko_stream } - it { is_expected.to respond_to :import_nexpose_raw_noko_stream } - it { is_expected.to respond_to :import_nexpose_rawxml } - it { is_expected.to respond_to :import_nexpose_rawxml_file } - it { is_expected.to respond_to :import_nexpose_simplexml } - it { is_expected.to respond_to :import_nexpose_simplexml_file } - it { is_expected.to respond_to :import_nikto_xml } - it { is_expected.to respond_to :import_nmap_noko_stream } - it { is_expected.to respond_to :import_nmap_xml } - it { is_expected.to respond_to :import_nmap_xml_file } - it { is_expected.to respond_to :import_openvas_new_xml } - it { is_expected.to respond_to :import_openvas_new_xml_file } - it { is_expected.to respond_to :import_openvas_xml } - it { is_expected.to respond_to :import_outpost24_noko_stream } - it { is_expected.to respond_to :import_outpost24_xml } - it { is_expected.to respond_to :import_qualys_asset_xml } - it { is_expected.to respond_to :import_qualys_scan_xml } - it { is_expected.to respond_to :import_qualys_scan_xml_file } - it { is_expected.to respond_to :import_report } - it { is_expected.to respond_to :import_retina_xml } - it { is_expected.to respond_to :import_retina_xml_file } - it { is_expected.to respond_to :import_spiceworks_csv } - it { is_expected.to respond_to :import_wapiti_xml } - it { is_expected.to respond_to :import_wapiti_xml_file } it { is_expected.to respond_to :initialize_adapter } it { is_expected.to respond_to :initialize_database_support } - it { is_expected.to respond_to :inspect_single_packet } - it { is_expected.to respond_to :inspect_single_packet_http } it { is_expected.to respond_to :loots } - it { is_expected.to respond_to :msf_import_timestamps } - it { is_expected.to respond_to :netsparker_method_map } - it { is_expected.to respond_to :netsparker_params_map } - it { is_expected.to respond_to :netsparker_pname_map } - it { is_expected.to respond_to :netsparker_vulnerability_map } - it { is_expected.to respond_to :nexpose_host_from_rawxml } - it { is_expected.to respond_to :nexpose_refs_to_struct } - it { is_expected.to respond_to :nils_for_nulls } - it { is_expected.to respond_to :nmap_msf_service_map } it { is_expected.to respond_to :notes } it { is_expected.to respond_to :report_artifact } it { is_expected.to respond_to :report_client } @@ -837,14 +759,11 @@ describe Msf::DBManager do it { is_expected.to respond_to :report_web_site } it { is_expected.to respond_to :report_web_vuln } it { is_expected.to respond_to :reports } - it { is_expected.to respond_to :rexmlify } it { is_expected.to respond_to :service_name_map } it { is_expected.to respond_to :tasks } - it { is_expected.to respond_to :unserialize_object } it { is_expected.to respond_to :update_vuln_details } it { is_expected.to respond_to :usable } it { is_expected.to respond_to :usable= } - it { is_expected.to respond_to :validate_import_file } it { is_expected.to respond_to :vulns } it { is_expected.to respond_to :warn_about_rubies } end diff --git a/spec/support/shared/examples/msf/db_manager/import.rb b/spec/support/shared/examples/msf/db_manager/import.rb new file mode 100644 index 0000000000..a8d4ca72fb --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/import.rb @@ -0,0 +1,84 @@ +shared_examples_for 'Msf::DBManager::Import' do + it { is_expected.to respond_to :dehex } + it { is_expected.to respond_to :emit } + it { is_expected.to respond_to :find_qualys_asset_ports } + it { is_expected.to respond_to :find_qualys_asset_vuln_refs } + it { is_expected.to respond_to :find_qualys_asset_vulns } + it { is_expected.to respond_to :import } + it { is_expected.to respond_to :import_acunetix_noko_stream } + it { is_expected.to respond_to :import_acunetix_xml } + it { is_expected.to respond_to :import_amap_log } + it { is_expected.to respond_to :import_amap_log_file } + it { is_expected.to respond_to :import_amap_mlog } + it { is_expected.to respond_to :import_appscan_noko_stream } + it { is_expected.to respond_to :import_appscan_xml } + it { is_expected.to respond_to :import_burp_session_noko_stream } + it { is_expected.to respond_to :import_burp_session_xml } + it { is_expected.to respond_to :import_ci_noko_stream } + it { is_expected.to respond_to :import_ci_xml } + it { is_expected.to respond_to :import_file } + it { is_expected.to respond_to :import_filetype_detect } + it { is_expected.to respond_to :import_foundstone_noko_stream } + it { is_expected.to respond_to :import_foundstone_xml } + it { is_expected.to respond_to :import_fusionvm_xml } + it { is_expected.to respond_to :import_ip360_aspl_xml } + it { is_expected.to respond_to :import_ip360_xml_file } + it { is_expected.to respond_to :import_ip360_xml_v3 } + it { is_expected.to respond_to :import_ip_list } + it { is_expected.to respond_to :import_ip_list_file } + it { is_expected.to respond_to :import_libpcap } + it { is_expected.to respond_to :import_libpcap_file } + it { is_expected.to respond_to :import_mbsa_noko_stream } + it { is_expected.to respond_to :import_mbsa_xml } + it { is_expected.to respond_to :import_msf_collateral } + it { is_expected.to respond_to :import_msf_cred_dump } + it { is_expected.to respond_to :import_msf_cred_dump_zip } + it { is_expected.to respond_to :import_msf_file } + it { is_expected.to respond_to :import_msf_pwdump } + it { is_expected.to respond_to :import_msf_zip } + it { is_expected.to respond_to :import_nessus_nbe } + it { is_expected.to respond_to :import_nessus_nbe_file } + it { is_expected.to respond_to :import_nessus_xml } + it { is_expected.to respond_to :import_nessus_xml_file } + it { is_expected.to respond_to :import_nessus_xml_v2 } + it { is_expected.to respond_to :import_netsparker_xml } + it { is_expected.to respond_to :import_netsparker_xml_file } + it { is_expected.to respond_to :import_nexpose_noko_stream } + it { is_expected.to respond_to :import_nexpose_raw_noko_stream } + it { is_expected.to respond_to :import_nexpose_rawxml } + it { is_expected.to respond_to :import_nexpose_rawxml_file } + it { is_expected.to respond_to :import_nexpose_simplexml } + it { is_expected.to respond_to :import_nexpose_simplexml_file } + it { is_expected.to respond_to :import_nikto_xml } + it { is_expected.to respond_to :import_nmap_noko_stream } + it { is_expected.to respond_to :import_nmap_xml } + it { is_expected.to respond_to :import_nmap_xml_file } + it { is_expected.to respond_to :import_openvas_new_xml } + it { is_expected.to respond_to :import_openvas_new_xml_file } + it { is_expected.to respond_to :import_openvas_xml } + it { is_expected.to respond_to :import_outpost24_noko_stream } + it { is_expected.to respond_to :import_outpost24_xml } + it { is_expected.to respond_to :import_qualys_asset_xml } + it { is_expected.to respond_to :import_qualys_scan_xml } + it { is_expected.to respond_to :import_qualys_scan_xml_file } + it { is_expected.to respond_to :import_report } + it { is_expected.to respond_to :import_retina_xml } + it { is_expected.to respond_to :import_retina_xml_file } + it { is_expected.to respond_to :import_spiceworks_csv } + it { is_expected.to respond_to :import_wapiti_xml } + it { is_expected.to respond_to :import_wapiti_xml_file } + it { is_expected.to respond_to :inspect_single_packet } + it { is_expected.to respond_to :inspect_single_packet_http } + it { is_expected.to respond_to :msf_import_timestamps } + it { is_expected.to respond_to :netsparker_method_map } + it { is_expected.to respond_to :netsparker_params_map } + it { is_expected.to respond_to :netsparker_pname_map } + it { is_expected.to respond_to :netsparker_vulnerability_map } + it { is_expected.to respond_to :nexpose_host_from_rawxml } + it { is_expected.to respond_to :nexpose_refs_to_struct } + it { is_expected.to respond_to :nils_for_nulls } + it { is_expected.to respond_to :nmap_msf_service_map } + it { is_expected.to respond_to :rexmlify } + it { is_expected.to respond_to :unserialize_object } + it { is_expected.to respond_to :validate_import_file } +end \ No newline at end of file From 0bc71ecd241b786e06d42bbd9466c31ae70d2c46 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Thu, 9 Oct 2014 15:15:40 -0500 Subject: [PATCH 035/159] Extract Msf::DBManager::Loot MSP-11124 --- lib/msf/core/db_manager.rb | 82 +-------------------------------- lib/msf/core/db_manager/loot.rb | 81 ++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 80 deletions(-) create mode 100644 lib/msf/core/db_manager/loot.rb diff --git a/lib/msf/core/db_manager.rb b/lib/msf/core/db_manager.rb index 8cea3b5f15..ef84b5553d 100644 --- a/lib/msf/core/db_manager.rb +++ b/lib/msf/core/db_manager.rb @@ -80,6 +80,7 @@ class DBManager autoload :Host, 'msf/core/db_manager/host' autoload :Import, 'msf/core/db_manager/import' autoload :IPAddress, 'msf/core/db_manager/ip_address' + autoload :Loot, 'msf/core/db_manager/loot' autoload :ModuleCache, 'msf/core/db_manager/module_cache' autoload :Service, 'msf/core/db_manager/service' autoload :Sink, 'msf/core/db_manager/sink' @@ -94,6 +95,7 @@ class DBManager include Msf::DBManager::Import include Msf::DBManager::ImportMsfXml include Msf::DBManager::IPAddress + include Msf::DBManager::Loot include Msf::DBManager::Migration include Msf::DBManager::ModuleCache include Msf::DBManager::Service @@ -1531,86 +1533,6 @@ class DBManager } end - # - # Loot collection - # - # - # This method iterates the loot table calling the supplied block with the - # instance of each entry. - # - def each_loot(wspace=workspace, &block) - ::ActiveRecord::Base.connection_pool.with_connection { - wspace.loots.each do |note| - block.call(note) - end - } - end - - # - # Find or create a loot matching this type/data - # - def find_or_create_loot(opts) - report_loot(opts) - end - - def report_loot(opts) - return if not active - ::ActiveRecord::Base.connection_pool.with_connection { - wspace = opts.delete(:workspace) || workspace - path = opts.delete(:path) || (raise RuntimeError, "A loot :path is required") - - host = nil - addr = nil - - # Report the host so it's there for the Proc to use below - if opts[:host] - if opts[:host].kind_of? ::Mdm::Host - host = opts[:host] - else - host = report_host({:workspace => wspace, :host => opts[:host]}) - addr = normalize_host(opts[:host]) - end - end - - ret = {} - - ltype = opts.delete(:type) || opts.delete(:ltype) || (raise RuntimeError, "A loot :type or :ltype is required") - ctype = opts.delete(:ctype) || opts.delete(:content_type) || 'text/plain' - name = opts.delete(:name) - info = opts.delete(:info) - data = opts[:data] - loot = wspace.loots.new - - if host - loot.host_id = host[:id] - end - if opts[:service] and opts[:service].kind_of? ::Mdm::Service - loot.service_id = opts[:service][:id] - end - - loot.path = path - loot.ltype = ltype - loot.content_type = ctype - loot.data = data - loot.name = name if name - loot.info = info if info - loot.workspace = wspace - msf_import_timestamps(opts,loot) - loot.save! - - ret[:loot] = loot - } - end - - # - # This methods returns a list of all loot in the database - # - def loots(wspace=workspace) - ::ActiveRecord::Base.connection_pool.with_connection { - wspace.loots - } - end - # # Find or create a task matching this type/data # diff --git a/lib/msf/core/db_manager/loot.rb b/lib/msf/core/db_manager/loot.rb new file mode 100644 index 0000000000..dda65162b6 --- /dev/null +++ b/lib/msf/core/db_manager/loot.rb @@ -0,0 +1,81 @@ +module Msf::DBManager::Loot + # + # Loot collection + # + # + # This method iterates the loot table calling the supplied block with the + # instance of each entry. + # + def each_loot(wspace=workspace, &block) + ::ActiveRecord::Base.connection_pool.with_connection { + wspace.loots.each do |note| + block.call(note) + end + } + end + + # + # Find or create a loot matching this type/data + # + def find_or_create_loot(opts) + report_loot(opts) + end + + # + # This methods returns a list of all loot in the database + # + def loots(wspace=workspace) + ::ActiveRecord::Base.connection_pool.with_connection { + wspace.loots + } + end + + def report_loot(opts) + return if not active + ::ActiveRecord::Base.connection_pool.with_connection { + wspace = opts.delete(:workspace) || workspace + path = opts.delete(:path) || (raise RuntimeError, "A loot :path is required") + + host = nil + addr = nil + + # Report the host so it's there for the Proc to use below + if opts[:host] + if opts[:host].kind_of? ::Mdm::Host + host = opts[:host] + else + host = report_host({:workspace => wspace, :host => opts[:host]}) + addr = normalize_host(opts[:host]) + end + end + + ret = {} + + ltype = opts.delete(:type) || opts.delete(:ltype) || (raise RuntimeError, "A loot :type or :ltype is required") + ctype = opts.delete(:ctype) || opts.delete(:content_type) || 'text/plain' + name = opts.delete(:name) + info = opts.delete(:info) + data = opts[:data] + loot = wspace.loots.new + + if host + loot.host_id = host[:id] + end + if opts[:service] and opts[:service].kind_of? ::Mdm::Service + loot.service_id = opts[:service][:id] + end + + loot.path = path + loot.ltype = ltype + loot.content_type = ctype + loot.data = data + loot.name = name if name + loot.info = info if info + loot.workspace = wspace + msf_import_timestamps(opts,loot) + loot.save! + + ret[:loot] = loot + } + end +end \ No newline at end of file From e42d60801d8888224172a5bf3b0262770289a45a Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Thu, 9 Oct 2014 15:21:43 -0500 Subject: [PATCH 036/159] Extract Msf::DBManager::Loot shared examples MSP-11124 --- spec/lib/msf/db_manager_spec.rb | 5 +---- spec/support/shared/examples/msf/db_manager/loot.rb | 6 ++++++ 2 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/loot.rb diff --git a/spec/lib/msf/db_manager_spec.rb b/spec/lib/msf/db_manager_spec.rb index 13b7a7c913..461663de0d 100644 --- a/spec/lib/msf/db_manager_spec.rb +++ b/spec/lib/msf/db_manager_spec.rb @@ -24,6 +24,7 @@ describe Msf::DBManager do it_should_behave_like 'Msf::DBManager::IPAddress' it_should_behave_like 'Msf::DBManager::Import' it_should_behave_like 'Msf::DBManager::ImportMsfXml' + it_should_behave_like 'Msf::DBManager::Loot' it_should_behave_like 'Msf::DBManager::Migration' it_should_behave_like 'Msf::DBManager::ModuleCache' it_should_behave_like 'Msf::DBManager::Service' @@ -70,13 +71,11 @@ describe Msf::DBManager do it { is_expected.to respond_to :driver } it { is_expected.to respond_to :drivers } it { is_expected.to respond_to :drivers= } - it { is_expected.to respond_to :each_loot } it { is_expected.to respond_to :each_note } it { is_expected.to respond_to :each_vuln } it { is_expected.to respond_to :error } it { is_expected.to respond_to :events } it { is_expected.to respond_to :find_or_create_client } - it { is_expected.to respond_to :find_or_create_loot } it { is_expected.to respond_to :find_or_create_note } it { is_expected.to respond_to :find_or_create_ref } it { is_expected.to respond_to :find_or_create_report } @@ -92,7 +91,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :has_vuln? } it { is_expected.to respond_to :initialize_adapter } it { is_expected.to respond_to :initialize_database_support } - it { is_expected.to respond_to :loots } it { is_expected.to respond_to :notes } it { is_expected.to respond_to :report_artifact } it { is_expected.to respond_to :report_client } @@ -104,7 +102,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :report_host_details } it { is_expected.to respond_to :report_host_tag } it { is_expected.to respond_to :report_import_note } - it { is_expected.to respond_to :report_loot } it { is_expected.to respond_to :report_note } it { is_expected.to respond_to :report_report } diff --git a/spec/support/shared/examples/msf/db_manager/loot.rb b/spec/support/shared/examples/msf/db_manager/loot.rb new file mode 100644 index 0000000000..8aa0b5ae5b --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/loot.rb @@ -0,0 +1,6 @@ +shared_examples_for 'Msf::DBManager::Loot' do + it { is_expected.to respond_to :each_loot } + it { is_expected.to respond_to :find_or_create_loot } + it { is_expected.to respond_to :loots } + it { is_expected.to respond_to :report_loot } +end \ No newline at end of file From 2fa02f5c444e8155da4148e3e96dd2267f3f4e61 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Thu, 9 Oct 2014 15:29:07 -0500 Subject: [PATCH 037/159] Extract Msf::DBManager::Note Extract all methods related to `Mdm::Note`s. --- lib/msf/core/db_manager.rb | 170 +------------------------------- lib/msf/core/db_manager/note.rb | 169 +++++++++++++++++++++++++++++++ 2 files changed, 171 insertions(+), 168 deletions(-) create mode 100644 lib/msf/core/db_manager/note.rb diff --git a/lib/msf/core/db_manager.rb b/lib/msf/core/db_manager.rb index ef84b5553d..8d546e1ba5 100644 --- a/lib/msf/core/db_manager.rb +++ b/lib/msf/core/db_manager.rb @@ -82,6 +82,7 @@ class DBManager autoload :IPAddress, 'msf/core/db_manager/ip_address' autoload :Loot, 'msf/core/db_manager/loot' autoload :ModuleCache, 'msf/core/db_manager/module_cache' + autoload :Note, 'msf/core/db_manager/note' autoload :Service, 'msf/core/db_manager/service' autoload :Sink, 'msf/core/db_manager/sink' autoload :WMAP, 'msf/core/db_manager/wmap' @@ -98,6 +99,7 @@ class DBManager include Msf::DBManager::Loot include Msf::DBManager::Migration include Msf::DBManager::ModuleCache + include Msf::DBManager::Note include Msf::DBManager::Service include Msf::DBManager::Sink include Msf::DBManager::WMAP @@ -987,174 +989,6 @@ class DBManager } end - # - # This method iterates the notes table calling the supplied block with the - # note instance of each entry. - # - def each_note(wspace=workspace, &block) - ::ActiveRecord::Base.connection_pool.with_connection { - wspace.notes.each do |note| - block.call(note) - end - } - end - - # - # Find or create a note matching this type/data - # - def find_or_create_note(opts) - report_note(opts) - end - - # - # Report a Note to the database. Notes can be tied to a ::Mdm::Workspace, Host, or Service. - # - # opts MUST contain - # +:type+:: The type of note, e.g. smb_peer_os - # - # opts can contain - # +:workspace+:: the workspace to associate with this Note - # +:host+:: an IP address or a Host object to associate with this Note - # +:service+:: a Service object to associate with this Note - # +:data+:: whatever it is you're making a note of - # +:port+:: along with +:host+ and +:proto+, a service to associate with this Note - # +:proto+:: along with +:host+ and +:port+, a service to associate with this Note - # +:update+:: what to do in case a similar Note exists, see below - # - # The +:update+ option can have the following values: - # +:unique+:: allow only a single Note per +:host+/+:type+ pair - # +:unique_data+:: like +:uniqe+, but also compare +:data+ - # +:insert+:: always insert a new Note even if one with identical values exists - # - # If the provided +:host+ is an IP address and does not exist in the - # database, it will be created. If +:workspace+, +:host+ and +:service+ - # are all omitted, the new Note will be associated with the current - # workspace. - # - def report_note(opts) - return if not active - ::ActiveRecord::Base.connection_pool.with_connection { - wspace = opts.delete(:workspace) || workspace - if wspace.kind_of? String - wspace = find_workspace(wspace) - end - seen = opts.delete(:seen) || false - crit = opts.delete(:critical) || false - host = nil - addr = nil - # Report the host so it's there for the Proc to use below - if opts[:host] - if opts[:host].kind_of? ::Mdm::Host - host = opts[:host] - else - addr = normalize_host(opts[:host]) - host = report_host({:workspace => wspace, :host => addr}) - end - # Do the same for a service if that's also included. - if (opts[:port]) - proto = nil - sname = nil - case opts[:proto].to_s.downcase # Catch incorrect usages - when 'tcp','udp' - proto = opts[:proto] - sname = opts[:sname] if opts[:sname] - when 'dns','snmp','dhcp' - proto = 'udp' - sname = opts[:proto] - else - proto = 'tcp' - sname = opts[:proto] - end - sopts = { - :workspace => wspace, - :host => host, - :port => opts[:port], - :proto => proto - } - sopts[:name] = sname if sname - report_service(sopts) - end - end - # Update Modes can be :unique, :unique_data, :insert - mode = opts[:update] || :unique - - ret = {} - - if addr and not host - host = get_host(:workspace => wspace, :host => addr) - end - if host and (opts[:port] and opts[:proto]) - service = get_service(wspace, host, opts[:proto], opts[:port]) - elsif opts[:service] and opts[:service].kind_of? ::Mdm::Service - service = opts[:service] - end -=begin - if host - host.updated_at = host.created_at - host.state = HostState::Alive - host.save! - end -=end - ntype = opts.delete(:type) || opts.delete(:ntype) || (raise RuntimeError, "A note :type or :ntype is required") - data = opts[:data] - note = nil - - conditions = { :ntype => ntype } - conditions[:host_id] = host[:id] if host - conditions[:service_id] = service[:id] if service - - case mode - when :unique - note = wspace.notes.where(conditions).first_or_initialize - note.data = data - when :unique_data - notes = wspace.notes.where(conditions) - - # Don't make a new Note with the same data as one that already - # exists for the given: type and (host or service) - notes.each do |n| - # Compare the deserialized data from the table to the raw - # data we're looking for. Because of the serialization we - # can't do this easily or reliably in SQL. - if n.data == data - note = n - break - end - end - if not note - # We didn't find one with the data we're looking for, make - # a new one. - note = wspace.notes.new(conditions.merge(:data => data)) - end - else - # Otherwise, assume :insert, which means always make a new one - note = wspace.notes.new - if host - note.host_id = host[:id] - end - if opts[:service] and opts[:service].kind_of? ::Mdm::Service - note.service_id = opts[:service][:id] - end - note.seen = seen - note.critical = crit - note.ntype = ntype - note.data = data - end - msf_import_timestamps(opts,note) - note.save! - ret[:note] = note - } - end - - # - # This methods returns a list of all notes in the database - # - def notes(wspace=workspace) - ::ActiveRecord::Base.connection_pool.with_connection { - wspace.notes - } - end - # This is only exercised by MSF3 XML importing for now. Needs the wait # conditions and return hash as well. def report_host_tag(opts) diff --git a/lib/msf/core/db_manager/note.rb b/lib/msf/core/db_manager/note.rb new file mode 100644 index 0000000000..543886dd7d --- /dev/null +++ b/lib/msf/core/db_manager/note.rb @@ -0,0 +1,169 @@ +module Msf::DBManager::Note + # + # This method iterates the notes table calling the supplied block with the + # note instance of each entry. + # + def each_note(wspace=workspace, &block) + ::ActiveRecord::Base.connection_pool.with_connection { + wspace.notes.each do |note| + block.call(note) + end + } + end + + # + # Find or create a note matching this type/data + # + def find_or_create_note(opts) + report_note(opts) + end + + # + # This methods returns a list of all notes in the database + # + def notes(wspace=workspace) + ::ActiveRecord::Base.connection_pool.with_connection { + wspace.notes + } + end + + # + # Report a Note to the database. Notes can be tied to a ::Mdm::Workspace, Host, or Service. + # + # opts MUST contain + # +:type+:: The type of note, e.g. smb_peer_os + # + # opts can contain + # +:workspace+:: the workspace to associate with this Note + # +:host+:: an IP address or a Host object to associate with this Note + # +:service+:: a Service object to associate with this Note + # +:data+:: whatever it is you're making a note of + # +:port+:: along with +:host+ and +:proto+, a service to associate with this Note + # +:proto+:: along with +:host+ and +:port+, a service to associate with this Note + # +:update+:: what to do in case a similar Note exists, see below + # + # The +:update+ option can have the following values: + # +:unique+:: allow only a single Note per +:host+/+:type+ pair + # +:unique_data+:: like +:uniqe+, but also compare +:data+ + # +:insert+:: always insert a new Note even if one with identical values exists + # + # If the provided +:host+ is an IP address and does not exist in the + # database, it will be created. If +:workspace+, +:host+ and +:service+ + # are all omitted, the new Note will be associated with the current + # workspace. + # + def report_note(opts) + return if not active + ::ActiveRecord::Base.connection_pool.with_connection { + wspace = opts.delete(:workspace) || workspace + if wspace.kind_of? String + wspace = find_workspace(wspace) + end + seen = opts.delete(:seen) || false + crit = opts.delete(:critical) || false + host = nil + addr = nil + # Report the host so it's there for the Proc to use below + if opts[:host] + if opts[:host].kind_of? ::Mdm::Host + host = opts[:host] + else + addr = normalize_host(opts[:host]) + host = report_host({:workspace => wspace, :host => addr}) + end + # Do the same for a service if that's also included. + if (opts[:port]) + proto = nil + sname = nil + case opts[:proto].to_s.downcase # Catch incorrect usages + when 'tcp','udp' + proto = opts[:proto] + sname = opts[:sname] if opts[:sname] + when 'dns','snmp','dhcp' + proto = 'udp' + sname = opts[:proto] + else + proto = 'tcp' + sname = opts[:proto] + end + sopts = { + :workspace => wspace, + :host => host, + :port => opts[:port], + :proto => proto + } + sopts[:name] = sname if sname + report_service(sopts) + end + end + # Update Modes can be :unique, :unique_data, :insert + mode = opts[:update] || :unique + + ret = {} + + if addr and not host + host = get_host(:workspace => wspace, :host => addr) + end + if host and (opts[:port] and opts[:proto]) + service = get_service(wspace, host, opts[:proto], opts[:port]) + elsif opts[:service] and opts[:service].kind_of? ::Mdm::Service + service = opts[:service] + end +=begin + if host + host.updated_at = host.created_at + host.state = HostState::Alive + host.save! + end +=end + ntype = opts.delete(:type) || opts.delete(:ntype) || (raise RuntimeError, "A note :type or :ntype is required") + data = opts[:data] + note = nil + + conditions = { :ntype => ntype } + conditions[:host_id] = host[:id] if host + conditions[:service_id] = service[:id] if service + + case mode + when :unique + note = wspace.notes.where(conditions).first_or_initialize + note.data = data + when :unique_data + notes = wspace.notes.where(conditions) + + # Don't make a new Note with the same data as one that already + # exists for the given: type and (host or service) + notes.each do |n| + # Compare the deserialized data from the table to the raw + # data we're looking for. Because of the serialization we + # can't do this easily or reliably in SQL. + if n.data == data + note = n + break + end + end + if not note + # We didn't find one with the data we're looking for, make + # a new one. + note = wspace.notes.new(conditions.merge(:data => data)) + end + else + # Otherwise, assume :insert, which means always make a new one + note = wspace.notes.new + if host + note.host_id = host[:id] + end + if opts[:service] and opts[:service].kind_of? ::Mdm::Service + note.service_id = opts[:service][:id] + end + note.seen = seen + note.critical = crit + note.ntype = ntype + note.data = data + end + msf_import_timestamps(opts,note) + note.save! + ret[:note] = note + } + end +end \ No newline at end of file From e403e548963dfbfdcbf6af2255b158d3d57b5d53 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Thu, 9 Oct 2014 15:39:55 -0500 Subject: [PATCH 038/159] Extract Msf::DBManager::Note shared examples MSP-11124 --- spec/lib/msf/db_manager_spec.rb | 6 +----- spec/support/shared/examples/msf/db_manager/import.rb | 1 + spec/support/shared/examples/msf/db_manager/note.rb | 6 ++++++ 3 files changed, 8 insertions(+), 5 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/note.rb diff --git a/spec/lib/msf/db_manager_spec.rb b/spec/lib/msf/db_manager_spec.rb index 461663de0d..9918f47889 100644 --- a/spec/lib/msf/db_manager_spec.rb +++ b/spec/lib/msf/db_manager_spec.rb @@ -27,6 +27,7 @@ describe Msf::DBManager do it_should_behave_like 'Msf::DBManager::Loot' it_should_behave_like 'Msf::DBManager::Migration' it_should_behave_like 'Msf::DBManager::ModuleCache' + it_should_behave_like 'Msf::DBManager::Note' it_should_behave_like 'Msf::DBManager::Service' it_should_behave_like 'Msf::DBManager::Sink' it_should_behave_like 'Msf::DBManager::WMAP' @@ -71,12 +72,10 @@ describe Msf::DBManager do it { is_expected.to respond_to :driver } it { is_expected.to respond_to :drivers } it { is_expected.to respond_to :drivers= } - it { is_expected.to respond_to :each_note } it { is_expected.to respond_to :each_vuln } it { is_expected.to respond_to :error } it { is_expected.to respond_to :events } it { is_expected.to respond_to :find_or_create_client } - it { is_expected.to respond_to :find_or_create_note } it { is_expected.to respond_to :find_or_create_ref } it { is_expected.to respond_to :find_or_create_report } it { is_expected.to respond_to :find_or_create_task } @@ -91,7 +90,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :has_vuln? } it { is_expected.to respond_to :initialize_adapter } it { is_expected.to respond_to :initialize_database_support } - it { is_expected.to respond_to :notes } it { is_expected.to respond_to :report_artifact } it { is_expected.to respond_to :report_client } it { is_expected.to respond_to :report_event } @@ -101,8 +99,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :report_exploit_success } it { is_expected.to respond_to :report_host_details } it { is_expected.to respond_to :report_host_tag } - it { is_expected.to respond_to :report_import_note } - it { is_expected.to respond_to :report_note } it { is_expected.to respond_to :report_report } context '#report_session' do diff --git a/spec/support/shared/examples/msf/db_manager/import.rb b/spec/support/shared/examples/msf/db_manager/import.rb index a8d4ca72fb..ebcee2912e 100644 --- a/spec/support/shared/examples/msf/db_manager/import.rb +++ b/spec/support/shared/examples/msf/db_manager/import.rb @@ -78,6 +78,7 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :nexpose_refs_to_struct } it { is_expected.to respond_to :nils_for_nulls } it { is_expected.to respond_to :nmap_msf_service_map } + it { is_expected.to respond_to :report_import_note } it { is_expected.to respond_to :rexmlify } it { is_expected.to respond_to :unserialize_object } it { is_expected.to respond_to :validate_import_file } diff --git a/spec/support/shared/examples/msf/db_manager/note.rb b/spec/support/shared/examples/msf/db_manager/note.rb new file mode 100644 index 0000000000..d122bfc1c2 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/note.rb @@ -0,0 +1,6 @@ +shared_examples_for 'Msf::DBManager::Note' do + it { is_expected.to respond_to :each_note } + it { is_expected.to respond_to :find_or_create_note } + it { is_expected.to respond_to :notes } + it { is_expected.to respond_to :report_note } +end \ No newline at end of file From 08aee23966bfc99c4ffb350439b59d3b6d3e6178 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Thu, 9 Oct 2014 15:47:34 -0500 Subject: [PATCH 039/159] Extract Msf::DBManager::Vuln MSP-11124 Extract all methods related to `Mdm::Vuln`s from `Msf::DBManager`. --- lib/msf/core/db_manager.rb | 252 +------------------------------- lib/msf/core/db_manager/vuln.rb | 250 +++++++++++++++++++++++++++++++ 2 files changed, 252 insertions(+), 250 deletions(-) create mode 100644 lib/msf/core/db_manager/vuln.rb diff --git a/lib/msf/core/db_manager.rb b/lib/msf/core/db_manager.rb index 8d546e1ba5..c1f750f3e4 100644 --- a/lib/msf/core/db_manager.rb +++ b/lib/msf/core/db_manager.rb @@ -85,6 +85,7 @@ class DBManager autoload :Note, 'msf/core/db_manager/note' autoload :Service, 'msf/core/db_manager/service' autoload :Sink, 'msf/core/db_manager/sink' + autoload :Vuln, 'msf/core/db_manager/vuln' autoload :WMAP, 'msf/core/db_manager/wmap' autoload :Workspace, 'msf/core/db_manager/workspace' @@ -102,6 +103,7 @@ class DBManager include Msf::DBManager::Note include Msf::DBManager::Service include Msf::DBManager::Sink + include Msf::DBManager::Vuln include Msf::DBManager::WMAP include Msf::DBManager::Workspace @@ -968,27 +970,6 @@ class DBManager } end - # - # This method iterates the vulns table calling the supplied block with the - # vuln instance of each entry. - # - def each_vuln(wspace=workspace,&block) - ::ActiveRecord::Base.connection_pool.with_connection { - wspace.vulns.each do |vulns| - block.call(vulns) - end - } - end - - # - # This methods returns a list of all vulnerabilities in the database - # - def vulns(wspace=workspace) - ::ActiveRecord::Base.connection_pool.with_connection { - wspace.vulns - } - end - # This is only exercised by MSF3 XML importing for now. Needs the wait # conditions and return hash as well. def report_host_tag(opts) @@ -1024,226 +1005,6 @@ class DBManager } end - # - # Find or create a vuln matching this service/name - # - def find_or_create_vuln(opts) - report_vuln(opts) - end - - # - # opts MUST contain - # +:host+:: the host where this vulnerability resides - # +:name+:: the friendly name for this vulnerability (title) - # - # opts can contain - # +:info+:: a human readable description of the vuln, free-form text - # +:refs+:: an array of Ref objects or string names of references - # +:details:: a hash with :key pointed to a find criteria hash and the rest containing VulnDetail fields - # - def report_vuln(opts) - return if not active - raise ArgumentError.new("Missing required option :host") if opts[:host].nil? - raise ArgumentError.new("Deprecated data column for vuln, use .info instead") if opts[:data] - name = opts[:name] || return - info = opts[:info] - - ::ActiveRecord::Base.connection_pool.with_connection { - - wspace = opts.delete(:workspace) || workspace - exploited_at = opts[:exploited_at] || opts["exploited_at"] - details = opts.delete(:details) - rids = opts.delete(:ref_ids) - - if opts[:refs] - rids ||= [] - opts[:refs].each do |r| - if (r.respond_to?(:ctx_id)) and (r.respond_to?(:ctx_val)) - r = "#{r.ctx_id}-#{r.ctx_val}" - end - rids << find_or_create_ref(:name => r) - end - end - - host = nil - addr = nil - if opts[:host].kind_of? ::Mdm::Host - host = opts[:host] - else - host = report_host({:workspace => wspace, :host => opts[:host]}) - addr = normalize_host(opts[:host]) - end - - ret = {} - - # Truncate the info field at the maximum field length - if info - info = info[0,65535] - end - - # Truncate the name field at the maximum field length - name = name[0,255] - - # Placeholder for the vuln object - vuln = nil - - # Identify the associated service - service = opts.delete(:service) - - # Treat port zero as no service - if service or opts[:port].to_i > 0 - - if not service - proto = nil - case opts[:proto].to_s.downcase # Catch incorrect usages, as in report_note - when 'tcp','udp' - proto = opts[:proto] - when 'dns','snmp','dhcp' - proto = 'udp' - sname = opts[:proto] - else - proto = 'tcp' - sname = opts[:proto] - end - - service = host.services.find_or_create_by_port_and_proto(opts[:port].to_i, proto) - end - - # Try to find an existing vulnerability with the same service & references - # If there are multiple matches, choose the one with the most matches - # If a match is found on a vulnerability with no associated service, - # update that vulnerability with our service information. This helps - # prevent dupes of the same vuln found by both local patch and - # service detection. - if rids and rids.length > 0 - vuln = find_vuln_by_refs(rids, host, service) - vuln.service = service if vuln - end - else - # Try to find an existing vulnerability with the same host & references - # If there are multiple matches, choose the one with the most matches - if rids and rids.length > 0 - vuln = find_vuln_by_refs(rids, host) - end - end - - # Try to match based on vuln_details records - if not vuln and opts[:details_match] - vuln = find_vuln_by_details(opts[:details_match], host, service) - if vuln and service and not vuln.service - vuln.service = service - end - end - - # No matches, so create a new vuln record - unless vuln - if service - vuln = service.vulns.find_by_name(name) - else - vuln = host.vulns.find_by_name(name) - end - - unless vuln - - vinf = { - :host_id => host.id, - :name => name, - :info => info - } - - vinf[:service_id] = service.id if service - vuln = Mdm::Vuln.create(vinf) - end - end - - # Set the exploited_at value if provided - vuln.exploited_at = exploited_at if exploited_at - - # Merge the references - if rids - vuln.refs << (rids - vuln.refs) - end - - # Finalize - if vuln.changed? - msf_import_timestamps(opts,vuln) - vuln.save! - end - - # Handle vuln_details parameters - report_vuln_details(vuln, details) if details - - vuln - } - end - - def find_vuln_by_refs(refs, host, service=nil) - - vuln = nil - - # Try to find an existing vulnerability with the same service & references - # If there are multiple matches, choose the one with the most matches - if service - refs_ids = refs.map{|x| x.id } - vuln = service.vulns.find(:all, :include => [:refs], :conditions => { 'refs.id' => refs_ids }).sort { |a,b| - ( refs_ids - a.refs.map{|x| x.id } ).length <=> ( refs_ids - b.refs.map{|x| x.id } ).length - }.first - end - - # Return if we matched based on service - return vuln if vuln - - # Try to find an existing vulnerability with the same host & references - # If there are multiple matches, choose the one with the most matches - refs_ids = refs.map{|x| x.id } - vuln = host.vulns.find(:all, :include => [:refs], :conditions => { 'service_id' => nil, 'refs.id' => refs_ids }).sort { |a,b| - ( refs_ids - a.refs.map{|x| x.id } ).length <=> ( refs_ids - b.refs.map{|x| x.id } ).length - }.first - - return vuln - end - - - def find_vuln_by_details(details_map, host, service=nil) - - # Create a modified version of the criteria in order to match against - # the joined version of the fields - - crit = {} - details_map.each_pair do |k,v| - crit[ "vuln_details.#{k}" ] = v - end - - vuln = nil - - if service - vuln = service.vulns.find(:first, :include => [:vuln_details], :conditions => crit) - end - - # Return if we matched based on service - return vuln if vuln - - # Prevent matches against other services - crit["vulns.service_id"] = nil if service - vuln = host.vulns.find(:first, :include => [:vuln_details], :conditions => crit) - - return vuln - end - - def get_vuln(wspace, host, service, name, data='') - raise RuntimeError, "Not workspace safe: #{caller.inspect}" - ::ActiveRecord::Base.connection_pool.with_connection { - vuln = nil - if (service) - vuln = ::Mdm::Vuln.find.where("name = ? and service_id = ? and host_id = ?", name, service.id, host.id).order("vulns.id DESC").first() - else - vuln = ::Mdm::Vuln.find.where("name = ? and host_id = ?", name, host.id).first() - end - - return vuln - } - end - # # Find or create a reference matching this name # @@ -1337,15 +1098,6 @@ class DBManager } end - # - # Find a vulnerability matching this name - # - def has_vuln?(name) - ::ActiveRecord::Base.connection_pool.with_connection { - Mdm::Vuln.find_by_name(name) - } - end - def events(wspace=workspace) ::ActiveRecord::Base.connection_pool.with_connection { wspace.events.find :all, :order => 'created_at ASC' diff --git a/lib/msf/core/db_manager/vuln.rb b/lib/msf/core/db_manager/vuln.rb new file mode 100644 index 0000000000..1bb38e9232 --- /dev/null +++ b/lib/msf/core/db_manager/vuln.rb @@ -0,0 +1,250 @@ +module Msf::DBManager::Vuln + # + # This method iterates the vulns table calling the supplied block with the + # vuln instance of each entry. + # + def each_vuln(wspace=workspace,&block) + ::ActiveRecord::Base.connection_pool.with_connection { + wspace.vulns.each do |vulns| + block.call(vulns) + end + } + end + + # + # Find or create a vuln matching this service/name + # + def find_or_create_vuln(opts) + report_vuln(opts) + end + + def find_vuln_by_details(details_map, host, service=nil) + + # Create a modified version of the criteria in order to match against + # the joined version of the fields + + crit = {} + details_map.each_pair do |k,v| + crit[ "vuln_details.#{k}" ] = v + end + + vuln = nil + + if service + vuln = service.vulns.find(:first, :include => [:vuln_details], :conditions => crit) + end + + # Return if we matched based on service + return vuln if vuln + + # Prevent matches against other services + crit["vulns.service_id"] = nil if service + vuln = host.vulns.find(:first, :include => [:vuln_details], :conditions => crit) + + return vuln + end + + def find_vuln_by_refs(refs, host, service=nil) + + vuln = nil + + # Try to find an existing vulnerability with the same service & references + # If there are multiple matches, choose the one with the most matches + if service + refs_ids = refs.map{|x| x.id } + vuln = service.vulns.find(:all, :include => [:refs], :conditions => { 'refs.id' => refs_ids }).sort { |a,b| + ( refs_ids - a.refs.map{|x| x.id } ).length <=> ( refs_ids - b.refs.map{|x| x.id } ).length + }.first + end + + # Return if we matched based on service + return vuln if vuln + + # Try to find an existing vulnerability with the same host & references + # If there are multiple matches, choose the one with the most matches + refs_ids = refs.map{|x| x.id } + vuln = host.vulns.find(:all, :include => [:refs], :conditions => { 'service_id' => nil, 'refs.id' => refs_ids }).sort { |a,b| + ( refs_ids - a.refs.map{|x| x.id } ).length <=> ( refs_ids - b.refs.map{|x| x.id } ).length + }.first + + return vuln + end + + def get_vuln(wspace, host, service, name, data='') + raise RuntimeError, "Not workspace safe: #{caller.inspect}" + ::ActiveRecord::Base.connection_pool.with_connection { + vuln = nil + if (service) + vuln = ::Mdm::Vuln.find.where("name = ? and service_id = ? and host_id = ?", name, service.id, host.id).order("vulns.id DESC").first() + else + vuln = ::Mdm::Vuln.find.where("name = ? and host_id = ?", name, host.id).first() + end + + return vuln + } + end + + # + # Find a vulnerability matching this name + # + def has_vuln?(name) + ::ActiveRecord::Base.connection_pool.with_connection { + Mdm::Vuln.find_by_name(name) + } + end + + # + # opts MUST contain + # +:host+:: the host where this vulnerability resides + # +:name+:: the friendly name for this vulnerability (title) + # + # opts can contain + # +:info+:: a human readable description of the vuln, free-form text + # +:refs+:: an array of Ref objects or string names of references + # +:details:: a hash with :key pointed to a find criteria hash and the rest containing VulnDetail fields + # + def report_vuln(opts) + return if not active + raise ArgumentError.new("Missing required option :host") if opts[:host].nil? + raise ArgumentError.new("Deprecated data column for vuln, use .info instead") if opts[:data] + name = opts[:name] || return + info = opts[:info] + + ::ActiveRecord::Base.connection_pool.with_connection { + + wspace = opts.delete(:workspace) || workspace + exploited_at = opts[:exploited_at] || opts["exploited_at"] + details = opts.delete(:details) + rids = opts.delete(:ref_ids) + + if opts[:refs] + rids ||= [] + opts[:refs].each do |r| + if (r.respond_to?(:ctx_id)) and (r.respond_to?(:ctx_val)) + r = "#{r.ctx_id}-#{r.ctx_val}" + end + rids << find_or_create_ref(:name => r) + end + end + + host = nil + addr = nil + if opts[:host].kind_of? ::Mdm::Host + host = opts[:host] + else + host = report_host({:workspace => wspace, :host => opts[:host]}) + addr = normalize_host(opts[:host]) + end + + ret = {} + + # Truncate the info field at the maximum field length + if info + info = info[0,65535] + end + + # Truncate the name field at the maximum field length + name = name[0,255] + + # Placeholder for the vuln object + vuln = nil + + # Identify the associated service + service = opts.delete(:service) + + # Treat port zero as no service + if service or opts[:port].to_i > 0 + + if not service + proto = nil + case opts[:proto].to_s.downcase # Catch incorrect usages, as in report_note + when 'tcp','udp' + proto = opts[:proto] + when 'dns','snmp','dhcp' + proto = 'udp' + sname = opts[:proto] + else + proto = 'tcp' + sname = opts[:proto] + end + + service = host.services.find_or_create_by_port_and_proto(opts[:port].to_i, proto) + end + + # Try to find an existing vulnerability with the same service & references + # If there are multiple matches, choose the one with the most matches + # If a match is found on a vulnerability with no associated service, + # update that vulnerability with our service information. This helps + # prevent dupes of the same vuln found by both local patch and + # service detection. + if rids and rids.length > 0 + vuln = find_vuln_by_refs(rids, host, service) + vuln.service = service if vuln + end + else + # Try to find an existing vulnerability with the same host & references + # If there are multiple matches, choose the one with the most matches + if rids and rids.length > 0 + vuln = find_vuln_by_refs(rids, host) + end + end + + # Try to match based on vuln_details records + if not vuln and opts[:details_match] + vuln = find_vuln_by_details(opts[:details_match], host, service) + if vuln and service and not vuln.service + vuln.service = service + end + end + + # No matches, so create a new vuln record + unless vuln + if service + vuln = service.vulns.find_by_name(name) + else + vuln = host.vulns.find_by_name(name) + end + + unless vuln + + vinf = { + :host_id => host.id, + :name => name, + :info => info + } + + vinf[:service_id] = service.id if service + vuln = Mdm::Vuln.create(vinf) + end + end + + # Set the exploited_at value if provided + vuln.exploited_at = exploited_at if exploited_at + + # Merge the references + if rids + vuln.refs << (rids - vuln.refs) + end + + # Finalize + if vuln.changed? + msf_import_timestamps(opts,vuln) + vuln.save! + end + + # Handle vuln_details parameters + report_vuln_details(vuln, details) if details + + vuln + } + end + + # + # This methods returns a list of all vulnerabilities in the database + # + def vulns(wspace=workspace) + ::ActiveRecord::Base.connection_pool.with_connection { + wspace.vulns + } + end +end \ No newline at end of file From b05a466e0a42a6ff5e2f2c8047d1b77a26899de9 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Thu, 9 Oct 2014 15:51:39 -0500 Subject: [PATCH 040/159] Extract Msf::DBManager::Vuln shared examples MSP-11124 --- spec/lib/msf/db_manager_spec.rb | 9 +-------- spec/support/shared/examples/msf/db_manager/vuln.rb | 10 ++++++++++ 2 files changed, 11 insertions(+), 8 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/vuln.rb diff --git a/spec/lib/msf/db_manager_spec.rb b/spec/lib/msf/db_manager_spec.rb index 9918f47889..eddc5291a1 100644 --- a/spec/lib/msf/db_manager_spec.rb +++ b/spec/lib/msf/db_manager_spec.rb @@ -30,6 +30,7 @@ describe Msf::DBManager do it_should_behave_like 'Msf::DBManager::Note' it_should_behave_like 'Msf::DBManager::Service' it_should_behave_like 'Msf::DBManager::Sink' + it_should_behave_like 'Msf::DBManager::Vuln' it_should_behave_like 'Msf::DBManager::WMAP' it_should_behave_like 'Msf::DBManager::Workspace' @@ -72,22 +73,16 @@ describe Msf::DBManager do it { is_expected.to respond_to :driver } it { is_expected.to respond_to :drivers } it { is_expected.to respond_to :drivers= } - it { is_expected.to respond_to :each_vuln } it { is_expected.to respond_to :error } it { is_expected.to respond_to :events } it { is_expected.to respond_to :find_or_create_client } it { is_expected.to respond_to :find_or_create_ref } it { is_expected.to respond_to :find_or_create_report } it { is_expected.to respond_to :find_or_create_task } - it { is_expected.to respond_to :find_or_create_vuln } - it { is_expected.to respond_to :find_vuln_by_details } - it { is_expected.to respond_to :find_vuln_by_refs } it { is_expected.to respond_to :get_client } it { is_expected.to respond_to :get_ref } it { is_expected.to respond_to :get_session } - it { is_expected.to respond_to :get_vuln } it { is_expected.to respond_to :has_ref? } - it { is_expected.to respond_to :has_vuln? } it { is_expected.to respond_to :initialize_adapter } it { is_expected.to respond_to :initialize_database_support } it { is_expected.to respond_to :report_artifact } @@ -744,7 +739,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :report_session_route } it { is_expected.to respond_to :report_session_route_remove } it { is_expected.to respond_to :report_task } - it { is_expected.to respond_to :report_vuln } it { is_expected.to respond_to :report_vuln_attempt } it { is_expected.to respond_to :report_vuln_details } it { is_expected.to respond_to :report_web_form } @@ -757,6 +751,5 @@ describe Msf::DBManager do it { is_expected.to respond_to :update_vuln_details } it { is_expected.to respond_to :usable } it { is_expected.to respond_to :usable= } - it { is_expected.to respond_to :vulns } it { is_expected.to respond_to :warn_about_rubies } end diff --git a/spec/support/shared/examples/msf/db_manager/vuln.rb b/spec/support/shared/examples/msf/db_manager/vuln.rb new file mode 100644 index 0000000000..f642b50828 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/vuln.rb @@ -0,0 +1,10 @@ +shared_examples_for 'Msf::DBManager::Vuln' do + it { is_expected.to respond_to :each_vuln } + it { is_expected.to respond_to :find_or_create_vuln } + it { is_expected.to respond_to :find_vuln_by_details } + it { is_expected.to respond_to :find_vuln_by_refs } + it { is_expected.to respond_to :get_vuln } + it { is_expected.to respond_to :has_vuln? } + it { is_expected.to respond_to :report_vuln } + it { is_expected.to respond_to :vulns } +end \ No newline at end of file From 89d588272e9340a85f2756dbc2982881697a8fed Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 08:27:09 -0500 Subject: [PATCH 041/159] Extract Msf::DBManager::Client MSP-11124 Extract methods related to `Mdm::Client`s. --- lib/msf/core/db_manager.rb | 67 +------------------------------ lib/msf/core/db_manager/client.rb | 66 ++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 65 deletions(-) create mode 100644 lib/msf/core/db_manager/client.rb diff --git a/lib/msf/core/db_manager.rb b/lib/msf/core/db_manager.rb index c1f750f3e4..135d6c46fc 100644 --- a/lib/msf/core/db_manager.rb +++ b/lib/msf/core/db_manager.rb @@ -75,6 +75,7 @@ module Msf class DBManager extend Metasploit::Framework::Require + autoload :Client, 'msf/core/db_manager/client' autoload :Cred, 'msf/core/db_manager/cred' autoload :ExploitedHost, 'msf/core/db_manager/exploited_host' autoload :Host, 'msf/core/db_manager/host' @@ -91,6 +92,7 @@ class DBManager optionally_include_metasploit_credential_creation + include Msf::DBManager::Client include Msf::DBManager::Cred include Msf::DBManager::ExploitedHost include Msf::DBManager::Host @@ -905,71 +907,6 @@ class DBManager } end - def get_client(opts) - ::ActiveRecord::Base.connection_pool.with_connection { - wspace = opts.delete(:workspace) || workspace - host = get_host(:workspace => wspace, :host => opts[:host]) || return - client = host.clients.where({:ua_string => opts[:ua_string]}).first() - return client - } - end - - def find_or_create_client(opts) - report_client(opts) - end - - # - # Report a client running on a host. - # - # opts MUST contain - # +:ua_string+:: the value of the User-Agent header - # +:host+:: the host where this client connected from, can be an ip address or a Host object - # - # opts can contain - # +:ua_name+:: one of the Msf::HttpClients constants - # +:ua_ver+:: detected version of the given client - # +:campaign+:: an id or Campaign object - # - # Returns a Client. - # - def report_client(opts) - return if not active - ::ActiveRecord::Base.connection_pool.with_connection { - addr = opts.delete(:host) || return - wspace = opts.delete(:workspace) || workspace - report_host(:workspace => wspace, :host => addr) - - ret = {} - - host = get_host(:workspace => wspace, :host => addr) - client = host.clients.find_or_initialize_by_ua_string(opts[:ua_string]) - - opts[:ua_string] = opts[:ua_string].to_s - - campaign = opts.delete(:campaign) - if campaign - case campaign - when Campaign - opts[:campaign_id] = campaign.id - else - opts[:campaign_id] = campaign - end - end - - opts.each { |k,v| - if (client.attribute_names.include?(k.to_s)) - client[k] = v - else - dlog("Unknown attribute for Client: #{k}") - end - } - if (client and client.changed?) - client.save! - end - ret[:client] = client - } - end - # This is only exercised by MSF3 XML importing for now. Needs the wait # conditions and return hash as well. def report_host_tag(opts) diff --git a/lib/msf/core/db_manager/client.rb b/lib/msf/core/db_manager/client.rb new file mode 100644 index 0000000000..0e2834e444 --- /dev/null +++ b/lib/msf/core/db_manager/client.rb @@ -0,0 +1,66 @@ +module Msf::DBManager::Client + def find_or_create_client(opts) + report_client(opts) + end + + def get_client(opts) + ::ActiveRecord::Base.connection_pool.with_connection { + wspace = opts.delete(:workspace) || workspace + host = get_host(:workspace => wspace, :host => opts[:host]) || return + client = host.clients.where({:ua_string => opts[:ua_string]}).first() + return client + } + end + + # + # Report a client running on a host. + # + # opts MUST contain + # +:ua_string+:: the value of the User-Agent header + # +:host+:: the host where this client connected from, can be an ip address or a Host object + # + # opts can contain + # +:ua_name+:: one of the Msf::HttpClients constants + # +:ua_ver+:: detected version of the given client + # +:campaign+:: an id or Campaign object + # + # Returns a Client. + # + def report_client(opts) + return if not active + ::ActiveRecord::Base.connection_pool.with_connection { + addr = opts.delete(:host) || return + wspace = opts.delete(:workspace) || workspace + report_host(:workspace => wspace, :host => addr) + + ret = {} + + host = get_host(:workspace => wspace, :host => addr) + client = host.clients.find_or_initialize_by_ua_string(opts[:ua_string]) + + opts[:ua_string] = opts[:ua_string].to_s + + campaign = opts.delete(:campaign) + if campaign + case campaign + when Campaign + opts[:campaign_id] = campaign.id + else + opts[:campaign_id] = campaign + end + end + + opts.each { |k,v| + if (client.attribute_names.include?(k.to_s)) + client[k] = v + else + dlog("Unknown attribute for Client: #{k}") + end + } + if (client and client.changed?) + client.save! + end + ret[:client] = client + } + end +end From 0e93b5539723443769f6387fb20c381eea032858 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 08:40:34 -0500 Subject: [PATCH 042/159] Extract Msf::DBManager::Client shared examples MSP-11124 --- spec/lib/msf/db_manager_spec.rb | 4 +--- spec/support/shared/examples/msf/db_manager/client.rb | 5 +++++ 2 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/client.rb diff --git a/spec/lib/msf/db_manager_spec.rb b/spec/lib/msf/db_manager_spec.rb index eddc5291a1..31d14a47c3 100644 --- a/spec/lib/msf/db_manager_spec.rb +++ b/spec/lib/msf/db_manager_spec.rb @@ -18,6 +18,7 @@ describe Msf::DBManager do db_manager end + it_should_behave_like 'Msf::DBManager::Client' it_should_behave_like 'Msf::DBManager::Cred' it_should_behave_like 'Msf::DBManager::ExploitedHost' it_should_behave_like 'Msf::DBManager::Host' @@ -75,18 +76,15 @@ describe Msf::DBManager do it { is_expected.to respond_to :drivers= } it { is_expected.to respond_to :error } it { is_expected.to respond_to :events } - it { is_expected.to respond_to :find_or_create_client } it { is_expected.to respond_to :find_or_create_ref } it { is_expected.to respond_to :find_or_create_report } it { is_expected.to respond_to :find_or_create_task } - it { is_expected.to respond_to :get_client } it { is_expected.to respond_to :get_ref } it { is_expected.to respond_to :get_session } it { is_expected.to respond_to :has_ref? } it { is_expected.to respond_to :initialize_adapter } it { is_expected.to respond_to :initialize_database_support } it { is_expected.to respond_to :report_artifact } - it { is_expected.to respond_to :report_client } it { is_expected.to respond_to :report_event } it { is_expected.to respond_to :report_exploit } it { is_expected.to respond_to :report_exploit_attempt } diff --git a/spec/support/shared/examples/msf/db_manager/client.rb b/spec/support/shared/examples/msf/db_manager/client.rb new file mode 100644 index 0000000000..d737c09c78 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/client.rb @@ -0,0 +1,5 @@ +shared_examples_for 'Msf::DBManager::Client' do + it { is_expected.to respond_to :find_or_create_client } + it { is_expected.to respond_to :get_client } + it { is_expected.to respond_to :report_client } +end \ No newline at end of file From 9632c83cde0539a93f30365acea7806a7b1240d2 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 09:05:10 -0500 Subject: [PATCH 043/159] Extract Msf::DBManager::Event MSP-11124 Extract methods related to `Mdm::Event`s. --- lib/msf/core/db_manager.rb | 23 ++--------------------- lib/msf/core/db_manager/event.rb | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+), 21 deletions(-) create mode 100644 lib/msf/core/db_manager/event.rb diff --git a/lib/msf/core/db_manager.rb b/lib/msf/core/db_manager.rb index 135d6c46fc..8b6a1b614e 100644 --- a/lib/msf/core/db_manager.rb +++ b/lib/msf/core/db_manager.rb @@ -77,6 +77,7 @@ class DBManager autoload :Client, 'msf/core/db_manager/client' autoload :Cred, 'msf/core/db_manager/cred' + autoload :Event, 'msf/core/db_manager/event' autoload :ExploitedHost, 'msf/core/db_manager/exploited_host' autoload :Host, 'msf/core/db_manager/host' autoload :Import, 'msf/core/db_manager/import' @@ -94,6 +95,7 @@ class DBManager include Msf::DBManager::Client include Msf::DBManager::Cred + include Msf::DBManager::Event include Msf::DBManager::ExploitedHost include Msf::DBManager::Host include Msf::DBManager::Import @@ -1035,27 +1037,6 @@ class DBManager } end - def events(wspace=workspace) - ::ActiveRecord::Base.connection_pool.with_connection { - wspace.events.find :all, :order => 'created_at ASC' - } - end - - def report_event(opts = {}) - return if not active - ::ActiveRecord::Base.connection_pool.with_connection { - wspace = opts.delete(:workspace) || workspace - return if not wspace # Temp fix? - uname = opts.delete(:username) - - if ! opts[:host].kind_of? ::Mdm::Host and opts[:host] - opts[:host] = report_host(:workspace => wspace, :host => opts[:host]) - end - - ::Mdm::Event.create(opts.merge(:workspace_id => wspace[:id], :username => uname)) - } - end - # # Find or create a task matching this type/data # diff --git a/lib/msf/core/db_manager/event.rb b/lib/msf/core/db_manager/event.rb new file mode 100644 index 0000000000..1f92930ad6 --- /dev/null +++ b/lib/msf/core/db_manager/event.rb @@ -0,0 +1,22 @@ +module Msf::DBManager::Event + def events(wspace=workspace) + ::ActiveRecord::Base.connection_pool.with_connection { + wspace.events.find :all, :order => 'created_at ASC' + } + end + + def report_event(opts = {}) + return if not active + ::ActiveRecord::Base.connection_pool.with_connection { + wspace = opts.delete(:workspace) || workspace + return if not wspace # Temp fix? + uname = opts.delete(:username) + + if ! opts[:host].kind_of? ::Mdm::Host and opts[:host] + opts[:host] = report_host(:workspace => wspace, :host => opts[:host]) + end + + ::Mdm::Event.create(opts.merge(:workspace_id => wspace[:id], :username => uname)) + } + end +end \ No newline at end of file From e82a46cc4700e352e930fd615e7c39a8c17c7e3b Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 09:12:49 -0500 Subject: [PATCH 044/159] Extract Msf::DBManager::Event shared examples MSP-11124 --- spec/lib/msf/db_manager_spec.rb | 3 +-- spec/support/shared/examples/msf/db_manager/event.rb | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/event.rb diff --git a/spec/lib/msf/db_manager_spec.rb b/spec/lib/msf/db_manager_spec.rb index 31d14a47c3..54decb1ef6 100644 --- a/spec/lib/msf/db_manager_spec.rb +++ b/spec/lib/msf/db_manager_spec.rb @@ -20,6 +20,7 @@ describe Msf::DBManager do it_should_behave_like 'Msf::DBManager::Client' it_should_behave_like 'Msf::DBManager::Cred' + it_should_behave_like 'Msf::DBManager::Event' it_should_behave_like 'Msf::DBManager::ExploitedHost' it_should_behave_like 'Msf::DBManager::Host' it_should_behave_like 'Msf::DBManager::IPAddress' @@ -75,7 +76,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :drivers } it { is_expected.to respond_to :drivers= } it { is_expected.to respond_to :error } - it { is_expected.to respond_to :events } it { is_expected.to respond_to :find_or_create_ref } it { is_expected.to respond_to :find_or_create_report } it { is_expected.to respond_to :find_or_create_task } @@ -85,7 +85,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :initialize_adapter } it { is_expected.to respond_to :initialize_database_support } it { is_expected.to respond_to :report_artifact } - it { is_expected.to respond_to :report_event } it { is_expected.to respond_to :report_exploit } it { is_expected.to respond_to :report_exploit_attempt } it { is_expected.to respond_to :report_exploit_failure } diff --git a/spec/support/shared/examples/msf/db_manager/event.rb b/spec/support/shared/examples/msf/db_manager/event.rb new file mode 100644 index 0000000000..861e963264 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/event.rb @@ -0,0 +1,4 @@ +shared_examples_for 'Msf::DBManager::Event' do + it { is_expected.to respond_to :events } + it { is_expected.to respond_to :report_event } +end \ No newline at end of file From 90b50339c3bfbdcc3ffef81e5f18837db3fecb8d Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 09:19:39 -0500 Subject: [PATCH 045/159] Extract Msf::DBManager::Report MSP-11124 Extract methods related to the obsolete `Mdm::Report`. These methods should be deleted, but since this branch is just for moves, I won't delete them now. --- lib/msf/core/db_manager.rb | 43 ++----------------------------- lib/msf/core/db_manager/report.rb | 42 ++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 41 deletions(-) create mode 100644 lib/msf/core/db_manager/report.rb diff --git a/lib/msf/core/db_manager.rb b/lib/msf/core/db_manager.rb index 8b6a1b614e..49ac683fb8 100644 --- a/lib/msf/core/db_manager.rb +++ b/lib/msf/core/db_manager.rb @@ -85,6 +85,7 @@ class DBManager autoload :Loot, 'msf/core/db_manager/loot' autoload :ModuleCache, 'msf/core/db_manager/module_cache' autoload :Note, 'msf/core/db_manager/note' + autoload :Report, 'msf/core/db_manager/report' autoload :Service, 'msf/core/db_manager/service' autoload :Sink, 'msf/core/db_manager/sink' autoload :Vuln, 'msf/core/db_manager/vuln' @@ -105,6 +106,7 @@ class DBManager include Msf::DBManager::Migration include Msf::DBManager::ModuleCache include Msf::DBManager::Note + include Msf::DBManager::Report include Msf::DBManager::Service include Msf::DBManager::Sink include Msf::DBManager::Vuln @@ -1094,38 +1096,6 @@ class DBManager end - # TODO This method does not attempt to find. It just creates - # a report based on the passed params. - def find_or_create_report(opts) - report_report(opts) - end - - # Creates a Report based on passed parameters. Does not handle - # child artifacts. - # @param opts [Hash] - # @return [Integer] ID of created report - def report_report(opts) - return if not active - created = opts.delete(:created_at) - updated = opts.delete(:updated_at) - state = opts.delete(:state) - - ::ActiveRecord::Base.connection_pool.with_connection { - report = Report.new(opts) - report.created_at = created - report.updated_at = updated - - unless report.valid? - errors = report.errors.full_messages.join('; ') - raise RuntimeError "Report to be imported is not valid: #{errors}" - end - report.state = :complete # Presume complete since it was exported - report.save - - report.id - } - end - # Creates a ReportArtifact based on passed parameters. # @param opts [Hash] of ReportArtifact attributes def report_artifact(opts) @@ -1164,15 +1134,6 @@ class DBManager artifact.save end - # - # This methods returns a list of all reports in the database - # - def reports(wspace=workspace) - ::ActiveRecord::Base.connection_pool.with_connection { - wspace.reports - } - end - # # WMAP # Support methods diff --git a/lib/msf/core/db_manager/report.rb b/lib/msf/core/db_manager/report.rb new file mode 100644 index 0000000000..1fcaccc790 --- /dev/null +++ b/lib/msf/core/db_manager/report.rb @@ -0,0 +1,42 @@ +module Msf::DBManager::Report + # TODO This method does not attempt to find. It just creates + # a report based on the passed params. + def find_or_create_report(opts) + report_report(opts) + end + + # Creates a Report based on passed parameters. Does not handle + # child artifacts. + # @param opts [Hash] + # @return [Integer] ID of created report + def report_report(opts) + return if not active + created = opts.delete(:created_at) + updated = opts.delete(:updated_at) + state = opts.delete(:state) + + ::ActiveRecord::Base.connection_pool.with_connection { + report = Report.new(opts) + report.created_at = created + report.updated_at = updated + + unless report.valid? + errors = report.errors.full_messages.join('; ') + raise RuntimeError "Report to be imported is not valid: #{errors}" + end + report.state = :complete # Presume complete since it was exported + report.save + + report.id + } + end + + # + # This methods returns a list of all reports in the database + # + def reports(wspace=workspace) + ::ActiveRecord::Base.connection_pool.with_connection { + wspace.reports + } + end +end \ No newline at end of file From 66845508d9c068e9577de9f2164b2c13ed30a0c3 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 09:24:27 -0500 Subject: [PATCH 046/159] Extract Msf::DBManager::Report shared examples MSP-11124 --- spec/lib/msf/db_manager_spec.rb | 4 +--- spec/support/shared/examples/msf/db_manager/report.rb | 5 +++++ 2 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/report.rb diff --git a/spec/lib/msf/db_manager_spec.rb b/spec/lib/msf/db_manager_spec.rb index 54decb1ef6..11371881b0 100644 --- a/spec/lib/msf/db_manager_spec.rb +++ b/spec/lib/msf/db_manager_spec.rb @@ -30,6 +30,7 @@ describe Msf::DBManager do it_should_behave_like 'Msf::DBManager::Migration' it_should_behave_like 'Msf::DBManager::ModuleCache' it_should_behave_like 'Msf::DBManager::Note' + it_should_behave_like 'Msf::DBManager::Report' it_should_behave_like 'Msf::DBManager::Service' it_should_behave_like 'Msf::DBManager::Sink' it_should_behave_like 'Msf::DBManager::Vuln' @@ -77,7 +78,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :drivers= } it { is_expected.to respond_to :error } it { is_expected.to respond_to :find_or_create_ref } - it { is_expected.to respond_to :find_or_create_report } it { is_expected.to respond_to :find_or_create_task } it { is_expected.to respond_to :get_ref } it { is_expected.to respond_to :get_session } @@ -91,7 +91,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :report_exploit_success } it { is_expected.to respond_to :report_host_details } it { is_expected.to respond_to :report_host_tag } - it { is_expected.to respond_to :report_report } context '#report_session' do let(:options) do @@ -742,7 +741,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :report_web_page } it { is_expected.to respond_to :report_web_site } it { is_expected.to respond_to :report_web_vuln } - it { is_expected.to respond_to :reports } it { is_expected.to respond_to :service_name_map } it { is_expected.to respond_to :tasks } it { is_expected.to respond_to :update_vuln_details } diff --git a/spec/support/shared/examples/msf/db_manager/report.rb b/spec/support/shared/examples/msf/db_manager/report.rb new file mode 100644 index 0000000000..930e8ba151 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/report.rb @@ -0,0 +1,5 @@ +shared_examples_for 'Msf::DBManager::Report' do + it { is_expected.to respond_to :find_or_create_report } + it { is_expected.to respond_to :report_report } + it { is_expected.to respond_to :reports } +end \ No newline at end of file From e0f76a7517529a16ff36b5574e6dcc74ba18a1b2 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 09:28:48 -0500 Subject: [PATCH 047/159] Extract Msf::DBManager::Task MSP-11124 Extract methods related to `Mdm::Task`s. --- lib/msf/core/db_manager.rb | 59 ++------------------------------- lib/msf/core/db_manager/task.rb | 57 +++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 57 deletions(-) create mode 100644 lib/msf/core/db_manager/task.rb diff --git a/lib/msf/core/db_manager.rb b/lib/msf/core/db_manager.rb index 49ac683fb8..b19079e316 100644 --- a/lib/msf/core/db_manager.rb +++ b/lib/msf/core/db_manager.rb @@ -88,6 +88,7 @@ class DBManager autoload :Report, 'msf/core/db_manager/report' autoload :Service, 'msf/core/db_manager/service' autoload :Sink, 'msf/core/db_manager/sink' + autoload :Task, 'msf/core/db_manager/task' autoload :Vuln, 'msf/core/db_manager/vuln' autoload :WMAP, 'msf/core/db_manager/wmap' autoload :Workspace, 'msf/core/db_manager/workspace' @@ -109,6 +110,7 @@ class DBManager include Msf::DBManager::Report include Msf::DBManager::Service include Msf::DBManager::Sink + include Msf::DBManager::Task include Msf::DBManager::Vuln include Msf::DBManager::WMAP include Msf::DBManager::Workspace @@ -1039,63 +1041,6 @@ class DBManager } end - # - # Find or create a task matching this type/data - # - def find_or_create_task(opts) - report_task(opts) - end - - def report_task(opts) - return if not active - ::ActiveRecord::Base.connection_pool.with_connection { - wspace = opts.delete(:workspace) || workspace - path = opts.delete(:path) || (raise RuntimeError, "A task :path is required") - - ret = {} - - user = opts.delete(:user) - desc = opts.delete(:desc) - error = opts.delete(:error) - info = opts.delete(:info) - mod = opts.delete(:mod) - options = opts.delete(:options) - prog = opts.delete(:prog) - result = opts.delete(:result) - completed_at = opts.delete(:completed_at) - task = wspace.tasks.new - - task.created_by = user - task.description = desc - task.error = error if error - task.info = info - task.module = mod - task.options = options - task.path = path - task.progress = prog - task.result = result if result - msf_import_timestamps(opts,task) - # Having blank completed_ats, while accurate, will cause unstoppable tasks. - if completed_at.nil? || completed_at.empty? - task.completed_at = opts[:updated_at] - else - task.completed_at = completed_at - end - task.save! - ret[:task] = task - } - end - - # - # This methods returns a list of all tasks in the database - # - def tasks(wspace=workspace) - ::ActiveRecord::Base.connection_pool.with_connection { - wspace.tasks - } - end - - # Creates a ReportArtifact based on passed parameters. # @param opts [Hash] of ReportArtifact attributes def report_artifact(opts) diff --git a/lib/msf/core/db_manager/task.rb b/lib/msf/core/db_manager/task.rb new file mode 100644 index 0000000000..c1039ebd9a --- /dev/null +++ b/lib/msf/core/db_manager/task.rb @@ -0,0 +1,57 @@ +module Msf::DBManager::Task + # + # Find or create a task matching this type/data + # + def find_or_create_task(opts) + report_task(opts) + end + + def report_task(opts) + return if not active + ::ActiveRecord::Base.connection_pool.with_connection { + wspace = opts.delete(:workspace) || workspace + path = opts.delete(:path) || (raise RuntimeError, "A task :path is required") + + ret = {} + + user = opts.delete(:user) + desc = opts.delete(:desc) + error = opts.delete(:error) + info = opts.delete(:info) + mod = opts.delete(:mod) + options = opts.delete(:options) + prog = opts.delete(:prog) + result = opts.delete(:result) + completed_at = opts.delete(:completed_at) + task = wspace.tasks.new + + task.created_by = user + task.description = desc + task.error = error if error + task.info = info + task.module = mod + task.options = options + task.path = path + task.progress = prog + task.result = result if result + msf_import_timestamps(opts,task) + # Having blank completed_ats, while accurate, will cause unstoppable tasks. + if completed_at.nil? || completed_at.empty? + task.completed_at = opts[:updated_at] + else + task.completed_at = completed_at + end + task.save! + ret[:task] = task + } + end + + # + # This methods returns a list of all tasks in the database + # + def tasks(wspace=workspace) + ::ActiveRecord::Base.connection_pool.with_connection { + wspace.tasks + } + end +end \ No newline at end of file From b007fba79ff6c15bb37f85c0071ba69155be6cd6 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 09:32:42 -0500 Subject: [PATCH 048/159] Extract Msf::DBManager::Task shared examples MSP-11124 --- spec/lib/msf/db_manager_spec.rb | 4 +--- spec/support/shared/examples/msf/db_manager/task.rb | 5 +++++ 2 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/task.rb diff --git a/spec/lib/msf/db_manager_spec.rb b/spec/lib/msf/db_manager_spec.rb index 11371881b0..453fff9218 100644 --- a/spec/lib/msf/db_manager_spec.rb +++ b/spec/lib/msf/db_manager_spec.rb @@ -33,6 +33,7 @@ describe Msf::DBManager do it_should_behave_like 'Msf::DBManager::Report' it_should_behave_like 'Msf::DBManager::Service' it_should_behave_like 'Msf::DBManager::Sink' + it_should_behave_like 'Msf::DBManager::Task' it_should_behave_like 'Msf::DBManager::Vuln' it_should_behave_like 'Msf::DBManager::WMAP' it_should_behave_like 'Msf::DBManager::Workspace' @@ -78,7 +79,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :drivers= } it { is_expected.to respond_to :error } it { is_expected.to respond_to :find_or_create_ref } - it { is_expected.to respond_to :find_or_create_task } it { is_expected.to respond_to :get_ref } it { is_expected.to respond_to :get_session } it { is_expected.to respond_to :has_ref? } @@ -734,7 +734,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :report_session_event } it { is_expected.to respond_to :report_session_route } it { is_expected.to respond_to :report_session_route_remove } - it { is_expected.to respond_to :report_task } it { is_expected.to respond_to :report_vuln_attempt } it { is_expected.to respond_to :report_vuln_details } it { is_expected.to respond_to :report_web_form } @@ -742,7 +741,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :report_web_site } it { is_expected.to respond_to :report_web_vuln } it { is_expected.to respond_to :service_name_map } - it { is_expected.to respond_to :tasks } it { is_expected.to respond_to :update_vuln_details } it { is_expected.to respond_to :usable } it { is_expected.to respond_to :usable= } diff --git a/spec/support/shared/examples/msf/db_manager/task.rb b/spec/support/shared/examples/msf/db_manager/task.rb new file mode 100644 index 0000000000..73415af730 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/task.rb @@ -0,0 +1,5 @@ +shared_examples_for 'Msf::DBManager::Task' do + it { is_expected.to respond_to :find_or_create_task } + it { is_expected.to respond_to :report_task } + it { is_expected.to respond_to :tasks } +end \ No newline at end of file From f42f8e106a479c365030950c6027bf35aec1ef23 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 09:39:07 -0500 Subject: [PATCH 049/159] Extract Msf::DBManager::ExploitAttempt MSP-11124 Extract methods that create `Mdm::ExploitAttempt`s. --- lib/msf/core/db_manager.rb | 214 +-------------------- lib/msf/core/db_manager/exploit_attempt.rb | 212 ++++++++++++++++++++ 2 files changed, 214 insertions(+), 212 deletions(-) create mode 100644 lib/msf/core/db_manager/exploit_attempt.rb diff --git a/lib/msf/core/db_manager.rb b/lib/msf/core/db_manager.rb index b19079e316..92e7d7d6be 100644 --- a/lib/msf/core/db_manager.rb +++ b/lib/msf/core/db_manager.rb @@ -78,6 +78,7 @@ class DBManager autoload :Client, 'msf/core/db_manager/client' autoload :Cred, 'msf/core/db_manager/cred' autoload :Event, 'msf/core/db_manager/event' + autoload :ExploitAttempt, 'msf/core/db_manager/exploit_attempt' autoload :ExploitedHost, 'msf/core/db_manager/exploited_host' autoload :Host, 'msf/core/db_manager/host' autoload :Import, 'msf/core/db_manager/import' @@ -98,6 +99,7 @@ class DBManager include Msf::DBManager::Client include Msf::DBManager::Cred include Msf::DBManager::Event + include Msf::DBManager::ExploitAttempt include Msf::DBManager::ExploitedHost include Msf::DBManager::Host include Msf::DBManager::Import @@ -692,191 +694,6 @@ class DBManager } end - - def report_exploit_success(opts) - ::ActiveRecord::Base.connection_pool.with_connection { - - wspace = opts.delete(:workspace) || workspace - mrefs = opts.delete(:refs) || return - host = opts.delete(:host) - port = opts.delete(:port) - prot = opts.delete(:proto) - svc = opts.delete(:service) - vuln = opts.delete(:vuln) - - timestamp = opts.delete(:timestamp) - username = opts.delete(:username) - mname = opts.delete(:module) - - # Look up or generate the host as appropriate - if not (host and host.kind_of? ::Mdm::Host) - if svc.kind_of? ::Mdm::Service - host = svc.host - else - host = report_host(:workspace => wspace, :address => host ) - end - end - - # Bail if we dont have a host object - return if not host - - # Look up or generate the service as appropriate - if port and svc.nil? - svc = report_service(:workspace => wspace, :host => host, :port => port, :proto => prot ) if port - end - - if not vuln - # Create a references map from the module list - ref_objs = ::Mdm::Ref.where(:name => mrefs.map { |ref| - if ref.respond_to?(:ctx_id) and ref.respond_to?(:ctx_val) - "#{ref.ctx_id}-#{ref.ctx_val}" - else - ref.to_s - end - }) - - # Try find a matching vulnerability - vuln = find_vuln_by_refs(ref_objs, host, svc) - end - - # We have match, lets create a vuln_attempt record - if vuln - attempt_info = { - :vuln_id => vuln.id, - :attempted_at => timestamp || Time.now.utc, - :exploited => true, - :username => username || "unknown", - :module => mname - } - - attempt_info[:session_id] = opts[:session_id] if opts[:session_id] - attempt_info[:loot_id] = opts[:loot_id] if opts[:loot_id] - - vuln.vuln_attempts.create(attempt_info) - - # Correct the vuln's associated service if necessary - if svc and vuln.service_id.nil? - vuln.service = svc - vuln.save - end - end - - # Report an exploit attempt all the same - attempt_info = { - :attempted_at => timestamp || Time.now.utc, - :exploited => true, - :username => username || "unknown", - :module => mname - } - - attempt_info[:vuln_id] = vuln.id if vuln - attempt_info[:session_id] = opts[:session_id] if opts[:session_id] - attempt_info[:loot_id] = opts[:loot_id] if opts[:loot_id] - - if svc - attempt_info[:port] = svc.port - attempt_info[:proto] = svc.proto - end - - if port and svc.nil? - attempt_info[:port] = port - attempt_info[:proto] = prot || "tcp" - end - - host.exploit_attempts.create(attempt_info) - } - end - - def report_exploit_failure(opts) - - ::ActiveRecord::Base.connection_pool.with_connection { - wspace = opts.delete(:workspace) || workspace - mrefs = opts.delete(:refs) || return - host = opts.delete(:host) - port = opts.delete(:port) - prot = opts.delete(:proto) - svc = opts.delete(:service) - vuln = opts.delete(:vuln) - - timestamp = opts.delete(:timestamp) - freason = opts.delete(:fail_reason) - fdetail = opts.delete(:fail_detail) - username = opts.delete(:username) - mname = opts.delete(:module) - - # Look up the host as appropriate - if not (host and host.kind_of? ::Mdm::Host) - if svc.kind_of? ::Mdm::Service - host = svc.host - else - host = get_host( :workspace => wspace, :address => host ) - end - end - - # Bail if we dont have a host object - return if not host - - # Look up the service as appropriate - if port and svc.nil? - prot ||= "tcp" - svc = get_service(wspace, host, prot, port) if port - end - - if not vuln - # Create a references map from the module list - ref_objs = ::Mdm::Ref.where(:name => mrefs.map { |ref| - if ref.respond_to?(:ctx_id) and ref.respond_to?(:ctx_val) - "#{ref.ctx_id}-#{ref.ctx_val}" - else - ref.to_s - end - }) - - # Try find a matching vulnerability - vuln = find_vuln_by_refs(ref_objs, host, svc) - end - - # Report a vuln_attempt if we found a match - if vuln - attempt_info = { - :attempted_at => timestamp || Time.now.utc, - :exploited => false, - :fail_reason => freason, - :fail_detail => fdetail, - :username => username || "unknown", - :module => mname - } - - vuln.vuln_attempts.create(attempt_info) - end - - # Report an exploit attempt all the same - attempt_info = { - :attempted_at => timestamp || Time.now.utc, - :exploited => false, - :username => username || "unknown", - :module => mname, - :fail_reason => freason, - :fail_detail => fdetail - } - - attempt_info[:vuln_id] = vuln.id if vuln - - if svc - attempt_info[:port] = svc.port - attempt_info[:proto] = svc.proto - end - - if port and svc.nil? - attempt_info[:port] = port - attempt_info[:proto] = prot || "tcp" - end - - host.exploit_attempts.create(attempt_info) - } - end - - def report_vuln_attempt(vuln, opts) ::ActiveRecord::Base.connection_pool.with_connection { return if not vuln @@ -896,22 +713,6 @@ class DBManager } end - def report_exploit_attempt(host, opts) - ::ActiveRecord::Base.connection_pool.with_connection { - return if not host - info = {} - - # Opts can be keyed by strings or symbols - ::Mdm::VulnAttempt.column_names.each do |kn| - k = kn.to_sym - next if ['id', 'host_id'].include?(kn) - info[k] = opts[kn] if opts[kn] - info[k] = opts[k] if opts[k] - end - - host.exploit_attempts.create(info) - } - end # This is only exercised by MSF3 XML importing for now. Needs the wait # conditions and return hash as well. @@ -1021,17 +822,6 @@ class DBManager } end - # report_exploit() used to be used to track sessions and which modules - # opened them. That information is now available with the session table - # directly. TODO: kill this completely some day -- for now just warn if - # some other UI is actually using it. - def report_exploit(opts={}) - wlog("Deprecated method call: report_exploit()\n" + - "report_exploit() options: #{opts.inspect}\n" + - "report_exploit() call stack:\n\t#{caller.join("\n\t")}" - ) - end - # # Find a reference matching this name # diff --git a/lib/msf/core/db_manager/exploit_attempt.rb b/lib/msf/core/db_manager/exploit_attempt.rb new file mode 100644 index 0000000000..9f864ec296 --- /dev/null +++ b/lib/msf/core/db_manager/exploit_attempt.rb @@ -0,0 +1,212 @@ +module Msf::DBManager::ExploitAttempt + # report_exploit() used to be used to track sessions and which modules + # opened them. That information is now available with the session table + # directly. TODO: kill this completely some day -- for now just warn if + # some other UI is actually using it. + def report_exploit(opts={}) + wlog("Deprecated method call: report_exploit()\n" + + "report_exploit() options: #{opts.inspect}\n" + + "report_exploit() call stack:\n\t#{caller.join("\n\t")}" + ) + end + + def report_exploit_attempt(host, opts) + ::ActiveRecord::Base.connection_pool.with_connection { + return if not host + info = {} + + # Opts can be keyed by strings or symbols + ::Mdm::VulnAttempt.column_names.each do |kn| + k = kn.to_sym + next if ['id', 'host_id'].include?(kn) + info[k] = opts[kn] if opts[kn] + info[k] = opts[k] if opts[k] + end + + host.exploit_attempts.create(info) + } + end + + def report_exploit_failure(opts) + + ::ActiveRecord::Base.connection_pool.with_connection { + wspace = opts.delete(:workspace) || workspace + mrefs = opts.delete(:refs) || return + host = opts.delete(:host) + port = opts.delete(:port) + prot = opts.delete(:proto) + svc = opts.delete(:service) + vuln = opts.delete(:vuln) + + timestamp = opts.delete(:timestamp) + freason = opts.delete(:fail_reason) + fdetail = opts.delete(:fail_detail) + username = opts.delete(:username) + mname = opts.delete(:module) + + # Look up the host as appropriate + if not (host and host.kind_of? ::Mdm::Host) + if svc.kind_of? ::Mdm::Service + host = svc.host + else + host = get_host( :workspace => wspace, :address => host ) + end + end + + # Bail if we dont have a host object + return if not host + + # Look up the service as appropriate + if port and svc.nil? + prot ||= "tcp" + svc = get_service(wspace, host, prot, port) if port + end + + if not vuln + # Create a references map from the module list + ref_objs = ::Mdm::Ref.where(:name => mrefs.map { |ref| + if ref.respond_to?(:ctx_id) and ref.respond_to?(:ctx_val) + "#{ref.ctx_id}-#{ref.ctx_val}" + else + ref.to_s + end + }) + + # Try find a matching vulnerability + vuln = find_vuln_by_refs(ref_objs, host, svc) + end + + # Report a vuln_attempt if we found a match + if vuln + attempt_info = { + :attempted_at => timestamp || Time.now.utc, + :exploited => false, + :fail_reason => freason, + :fail_detail => fdetail, + :username => username || "unknown", + :module => mname + } + + vuln.vuln_attempts.create(attempt_info) + end + + # Report an exploit attempt all the same + attempt_info = { + :attempted_at => timestamp || Time.now.utc, + :exploited => false, + :username => username || "unknown", + :module => mname, + :fail_reason => freason, + :fail_detail => fdetail + } + + attempt_info[:vuln_id] = vuln.id if vuln + + if svc + attempt_info[:port] = svc.port + attempt_info[:proto] = svc.proto + end + + if port and svc.nil? + attempt_info[:port] = port + attempt_info[:proto] = prot || "tcp" + end + + host.exploit_attempts.create(attempt_info) + } + end + + def report_exploit_success(opts) + ::ActiveRecord::Base.connection_pool.with_connection { + + wspace = opts.delete(:workspace) || workspace + mrefs = opts.delete(:refs) || return + host = opts.delete(:host) + port = opts.delete(:port) + prot = opts.delete(:proto) + svc = opts.delete(:service) + vuln = opts.delete(:vuln) + + timestamp = opts.delete(:timestamp) + username = opts.delete(:username) + mname = opts.delete(:module) + + # Look up or generate the host as appropriate + if not (host and host.kind_of? ::Mdm::Host) + if svc.kind_of? ::Mdm::Service + host = svc.host + else + host = report_host(:workspace => wspace, :address => host ) + end + end + + # Bail if we dont have a host object + return if not host + + # Look up or generate the service as appropriate + if port and svc.nil? + svc = report_service(:workspace => wspace, :host => host, :port => port, :proto => prot ) if port + end + + if not vuln + # Create a references map from the module list + ref_objs = ::Mdm::Ref.where(:name => mrefs.map { |ref| + if ref.respond_to?(:ctx_id) and ref.respond_to?(:ctx_val) + "#{ref.ctx_id}-#{ref.ctx_val}" + else + ref.to_s + end + }) + + # Try find a matching vulnerability + vuln = find_vuln_by_refs(ref_objs, host, svc) + end + + # We have match, lets create a vuln_attempt record + if vuln + attempt_info = { + :vuln_id => vuln.id, + :attempted_at => timestamp || Time.now.utc, + :exploited => true, + :username => username || "unknown", + :module => mname + } + + attempt_info[:session_id] = opts[:session_id] if opts[:session_id] + attempt_info[:loot_id] = opts[:loot_id] if opts[:loot_id] + + vuln.vuln_attempts.create(attempt_info) + + # Correct the vuln's associated service if necessary + if svc and vuln.service_id.nil? + vuln.service = svc + vuln.save + end + end + + # Report an exploit attempt all the same + attempt_info = { + :attempted_at => timestamp || Time.now.utc, + :exploited => true, + :username => username || "unknown", + :module => mname + } + + attempt_info[:vuln_id] = vuln.id if vuln + attempt_info[:session_id] = opts[:session_id] if opts[:session_id] + attempt_info[:loot_id] = opts[:loot_id] if opts[:loot_id] + + if svc + attempt_info[:port] = svc.port + attempt_info[:proto] = svc.proto + end + + if port and svc.nil? + attempt_info[:port] = port + attempt_info[:proto] = prot || "tcp" + end + + host.exploit_attempts.create(attempt_info) + } + end +end \ No newline at end of file From 97b3198923aad6f59ae7ba22d368d108a6baed58 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 09:43:40 -0500 Subject: [PATCH 050/159] Extract Msf::DBManager::ExploitAttempt shared examples MSP-11124 --- spec/lib/msf/db_manager_spec.rb | 5 +---- .../shared/examples/msf/db_manager/exploit_attempt.rb | 6 ++++++ 2 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/exploit_attempt.rb diff --git a/spec/lib/msf/db_manager_spec.rb b/spec/lib/msf/db_manager_spec.rb index 453fff9218..99582ea8e6 100644 --- a/spec/lib/msf/db_manager_spec.rb +++ b/spec/lib/msf/db_manager_spec.rb @@ -21,6 +21,7 @@ describe Msf::DBManager do it_should_behave_like 'Msf::DBManager::Client' it_should_behave_like 'Msf::DBManager::Cred' it_should_behave_like 'Msf::DBManager::Event' + it_should_behave_like 'Msf::DBManager::ExploitAttempt' it_should_behave_like 'Msf::DBManager::ExploitedHost' it_should_behave_like 'Msf::DBManager::Host' it_should_behave_like 'Msf::DBManager::IPAddress' @@ -85,10 +86,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :initialize_adapter } it { is_expected.to respond_to :initialize_database_support } it { is_expected.to respond_to :report_artifact } - it { is_expected.to respond_to :report_exploit } - it { is_expected.to respond_to :report_exploit_attempt } - it { is_expected.to respond_to :report_exploit_failure } - it { is_expected.to respond_to :report_exploit_success } it { is_expected.to respond_to :report_host_details } it { is_expected.to respond_to :report_host_tag } diff --git a/spec/support/shared/examples/msf/db_manager/exploit_attempt.rb b/spec/support/shared/examples/msf/db_manager/exploit_attempt.rb new file mode 100644 index 0000000000..28eeeaacf8 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/exploit_attempt.rb @@ -0,0 +1,6 @@ +shared_examples_for 'Msf::DBManager::ExploitAttempt' do + it { is_expected.to respond_to :report_exploit } + it { is_expected.to respond_to :report_exploit_attempt } + it { is_expected.to respond_to :report_exploit_failure } + it { is_expected.to respond_to :report_exploit_success } +end \ No newline at end of file From 43c99096362058cf9bbfd2cdd1a3eead618b48f1 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 09:54:38 -0500 Subject: [PATCH 051/159] Extract Msf::DBManager::VulnDetail MSP-11124 Extract methods related to `Mdm::VulnDetail`s. --- lib/msf/core/db_manager.rb | 32 ++------------------------ lib/msf/core/db_manager/vuln_detail.rb | 31 +++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 30 deletions(-) create mode 100644 lib/msf/core/db_manager/vuln_detail.rb diff --git a/lib/msf/core/db_manager.rb b/lib/msf/core/db_manager.rb index 92e7d7d6be..5f4d78366e 100644 --- a/lib/msf/core/db_manager.rb +++ b/lib/msf/core/db_manager.rb @@ -91,6 +91,7 @@ class DBManager autoload :Sink, 'msf/core/db_manager/sink' autoload :Task, 'msf/core/db_manager/task' autoload :Vuln, 'msf/core/db_manager/vuln' + autoload :VulnDetail, 'msf/core/db_manager/vuln_detail' autoload :WMAP, 'msf/core/db_manager/wmap' autoload :Workspace, 'msf/core/db_manager/workspace' @@ -114,6 +115,7 @@ class DBManager include Msf::DBManager::Sink include Msf::DBManager::Task include Msf::DBManager::Vuln + include Msf::DBManager::VulnDetail include Msf::DBManager::WMAP include Msf::DBManager::Workspace @@ -772,36 +774,6 @@ class DBManager } end - # - # Populate the vuln_details table with additional - # information, matched by a specific criteria - # - def report_vuln_details(vuln, details) - ::ActiveRecord::Base.connection_pool.with_connection { - detail = ::Mdm::VulnDetail.where(( details.delete(:key) || {} ).merge(:vuln_id => vuln.id)).first - if detail - details.each_pair do |k,v| - detail[k] = v - end - detail.save! if detail.changed? - detail - else - detail = ::Mdm::VulnDetail.create(details.merge(:vuln_id => vuln.id)) - end - } - end - - # - # Update vuln_details records en-masse based on specific criteria - # Note that this *can* update data across workspaces - # - def update_vuln_details(details) - ::ActiveRecord::Base.connection_pool.with_connection { - criteria = details.delete(:key) || {} - ::Mdm::VulnDetail.update(key, details) - } - end - # # Populate the host_details table with additional # information, matched by a specific criteria diff --git a/lib/msf/core/db_manager/vuln_detail.rb b/lib/msf/core/db_manager/vuln_detail.rb new file mode 100644 index 0000000000..7d3e52eccc --- /dev/null +++ b/lib/msf/core/db_manager/vuln_detail.rb @@ -0,0 +1,31 @@ +module Msf::DBManager::VulnDetail + # + # Populate the vuln_details table with additional + # information, matched by a specific criteria + # + def report_vuln_details(vuln, details) + ::ActiveRecord::Base.connection_pool.with_connection { + detail = ::Mdm::VulnDetail.where(( details.delete(:key) || {} ).merge(:vuln_id => vuln.id)).first + if detail + details.each_pair do |k,v| + detail[k] = v + end + detail.save! if detail.changed? + detail + else + detail = ::Mdm::VulnDetail.create(details.merge(:vuln_id => vuln.id)) + end + } + end + + # + # Update vuln_details records en-masse based on specific criteria + # Note that this *can* update data across workspaces + # + def update_vuln_details(details) + ::ActiveRecord::Base.connection_pool.with_connection { + criteria = details.delete(:key) || {} + ::Mdm::VulnDetail.update(key, details) + } + end +end \ No newline at end of file From a970d76a2a23dcd773afa2f1bb721d072ead20c5 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 09:58:59 -0500 Subject: [PATCH 052/159] Extract Msf::DBManager::VulnDetail shared examples MSP-11124 --- spec/lib/msf/db_manager_spec.rb | 3 +-- spec/support/shared/examples/msf/db_manager/vuln_detail.rb | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/vuln_detail.rb diff --git a/spec/lib/msf/db_manager_spec.rb b/spec/lib/msf/db_manager_spec.rb index 99582ea8e6..0249eb05a4 100644 --- a/spec/lib/msf/db_manager_spec.rb +++ b/spec/lib/msf/db_manager_spec.rb @@ -36,6 +36,7 @@ describe Msf::DBManager do it_should_behave_like 'Msf::DBManager::Sink' it_should_behave_like 'Msf::DBManager::Task' it_should_behave_like 'Msf::DBManager::Vuln' + it_should_behave_like 'Msf::DBManager::VulnDetail' it_should_behave_like 'Msf::DBManager::WMAP' it_should_behave_like 'Msf::DBManager::Workspace' @@ -732,13 +733,11 @@ describe Msf::DBManager do it { is_expected.to respond_to :report_session_route } it { is_expected.to respond_to :report_session_route_remove } it { is_expected.to respond_to :report_vuln_attempt } - it { is_expected.to respond_to :report_vuln_details } it { is_expected.to respond_to :report_web_form } it { is_expected.to respond_to :report_web_page } it { is_expected.to respond_to :report_web_site } it { is_expected.to respond_to :report_web_vuln } it { is_expected.to respond_to :service_name_map } - it { is_expected.to respond_to :update_vuln_details } it { is_expected.to respond_to :usable } it { is_expected.to respond_to :usable= } it { is_expected.to respond_to :warn_about_rubies } diff --git a/spec/support/shared/examples/msf/db_manager/vuln_detail.rb b/spec/support/shared/examples/msf/db_manager/vuln_detail.rb new file mode 100644 index 0000000000..a44a357ca7 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/vuln_detail.rb @@ -0,0 +1,4 @@ +shared_examples_for 'Msf::DBManager::VulnDetail' do + it { is_expected.to respond_to :report_vuln_details } + it { is_expected.to respond_to :update_vuln_details } +end \ No newline at end of file From 5668a2820e6f5dffdac74bfd7dcafb3d0ece8e44 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 10:00:19 -0500 Subject: [PATCH 053/159] Move #report_artifact to Msf::DBManager::Report MSP-11124 --- lib/msf/core/db_manager.rb | 38 ------------------------------- lib/msf/core/db_manager/report.rb | 38 +++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/lib/msf/core/db_manager.rb b/lib/msf/core/db_manager.rb index 5f4d78366e..46900282db 100644 --- a/lib/msf/core/db_manager.rb +++ b/lib/msf/core/db_manager.rb @@ -803,44 +803,6 @@ class DBManager } end - # Creates a ReportArtifact based on passed parameters. - # @param opts [Hash] of ReportArtifact attributes - def report_artifact(opts) - return if not active - - artifacts_dir = Report::ARTIFACT_DIR - tmp_path = opts[:file_path] - artifact_name = File.basename tmp_path - new_path = File.join(artifacts_dir, artifact_name) - created = opts.delete(:created_at) - updated = opts.delete(:updated_at) - - unless File.exists? tmp_path - raise DBImportError 'Report artifact file to be imported does not exist.' - end - - unless (File.directory?(artifacts_dir) && File.writable?(artifacts_dir)) - raise DBImportError "Could not move report artifact file to #{artifacts_dir}." - end - - if File.exists? new_path - unique_basename = "#{(Time.now.to_f*1000).to_i}_#{artifact_name}" - new_path = File.join(artifacts_dir, unique_basename) - end - - FileUtils.copy(tmp_path, new_path) - opts[:file_path] = new_path - artifact = ReportArtifact.new(opts) - artifact.created_at = created - artifact.updated_at = updated - - unless artifact.valid? - errors = artifact.errors.full_messages.join('; ') - raise RuntimeError "Artifact to be imported is not valid: #{errors}" - end - artifact.save - end - # # WMAP # Support methods diff --git a/lib/msf/core/db_manager/report.rb b/lib/msf/core/db_manager/report.rb index 1fcaccc790..c7d7fca1a2 100644 --- a/lib/msf/core/db_manager/report.rb +++ b/lib/msf/core/db_manager/report.rb @@ -5,6 +5,44 @@ module Msf::DBManager::Report report_report(opts) end + # Creates a ReportArtifact based on passed parameters. + # @param opts [Hash] of ReportArtifact attributes + def report_artifact(opts) + return if not active + + artifacts_dir = Report::ARTIFACT_DIR + tmp_path = opts[:file_path] + artifact_name = File.basename tmp_path + new_path = File.join(artifacts_dir, artifact_name) + created = opts.delete(:created_at) + updated = opts.delete(:updated_at) + + unless File.exists? tmp_path + raise DBImportError 'Report artifact file to be imported does not exist.' + end + + unless (File.directory?(artifacts_dir) && File.writable?(artifacts_dir)) + raise DBImportError "Could not move report artifact file to #{artifacts_dir}." + end + + if File.exists? new_path + unique_basename = "#{(Time.now.to_f*1000).to_i}_#{artifact_name}" + new_path = File.join(artifacts_dir, unique_basename) + end + + FileUtils.copy(tmp_path, new_path) + opts[:file_path] = new_path + artifact = ReportArtifact.new(opts) + artifact.created_at = created + artifact.updated_at = updated + + unless artifact.valid? + errors = artifact.errors.full_messages.join('; ') + raise RuntimeError "Artifact to be imported is not valid: #{errors}" + end + artifact.save + end + # Creates a Report based on passed parameters. Does not handle # child artifacts. # @param opts [Hash] From 7a5b3c9ee7b337a1ec0d397b1b4fdebcb3522487 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 10:01:08 -0500 Subject: [PATCH 054/159] Move report_artifact example to Msf::DBManager::Report MSP-11124 --- spec/lib/msf/db_manager_spec.rb | 1 - spec/support/shared/examples/msf/db_manager/report.rb | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/lib/msf/db_manager_spec.rb b/spec/lib/msf/db_manager_spec.rb index 0249eb05a4..9c62162ef2 100644 --- a/spec/lib/msf/db_manager_spec.rb +++ b/spec/lib/msf/db_manager_spec.rb @@ -86,7 +86,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :has_ref? } it { is_expected.to respond_to :initialize_adapter } it { is_expected.to respond_to :initialize_database_support } - it { is_expected.to respond_to :report_artifact } it { is_expected.to respond_to :report_host_details } it { is_expected.to respond_to :report_host_tag } diff --git a/spec/support/shared/examples/msf/db_manager/report.rb b/spec/support/shared/examples/msf/db_manager/report.rb index 930e8ba151..575df97849 100644 --- a/spec/support/shared/examples/msf/db_manager/report.rb +++ b/spec/support/shared/examples/msf/db_manager/report.rb @@ -1,5 +1,6 @@ shared_examples_for 'Msf::DBManager::Report' do it { is_expected.to respond_to :find_or_create_report } + it { is_expected.to respond_to :report_artifact } it { is_expected.to respond_to :report_report } it { is_expected.to respond_to :reports } end \ No newline at end of file From 87ee06b792366109aee75474c7feaa8f5e1c80d6 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 10:06:37 -0500 Subject: [PATCH 055/159] Extract Msf::DBManager::Ref MSP-11124 Extract methods related to `Mdm::Ref`s. --- lib/msf/core/db_manager.rb | 34 ++-------------------------------- lib/msf/core/db_manager/ref.rb | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 32 deletions(-) create mode 100644 lib/msf/core/db_manager/ref.rb diff --git a/lib/msf/core/db_manager.rb b/lib/msf/core/db_manager.rb index 46900282db..cc1fa39ff2 100644 --- a/lib/msf/core/db_manager.rb +++ b/lib/msf/core/db_manager.rb @@ -86,6 +86,7 @@ class DBManager autoload :Loot, 'msf/core/db_manager/loot' autoload :ModuleCache, 'msf/core/db_manager/module_cache' autoload :Note, 'msf/core/db_manager/note' + autoload :Ref, 'msf/core/db_manager/ref' autoload :Report, 'msf/core/db_manager/report' autoload :Service, 'msf/core/db_manager/service' autoload :Sink, 'msf/core/db_manager/sink' @@ -110,6 +111,7 @@ class DBManager include Msf::DBManager::Migration include Msf::DBManager::ModuleCache include Msf::DBManager::Note + include Msf::DBManager::Ref include Msf::DBManager::Report include Msf::DBManager::Service include Msf::DBManager::Sink @@ -751,29 +753,6 @@ class DBManager } end - # - # Find or create a reference matching this name - # - def find_or_create_ref(opts) - ret = {} - ret[:ref] = get_ref(opts[:name]) - return ret[:ref] if ret[:ref] - - ::ActiveRecord::Base.connection_pool.with_connection { - ref = ::Mdm::Ref.find_or_initialize_by_name(opts[:name]) - if ref and ref.changed? - ref.save! - end - ret[:ref] = ref - } - end - - def get_ref(name) - ::ActiveRecord::Base.connection_pool.with_connection { - ::Mdm::Ref.find_by_name(name) - } - end - # # Populate the host_details table with additional # information, matched by a specific criteria @@ -794,15 +773,6 @@ class DBManager } end - # - # Find a reference matching this name - # - def has_ref?(name) - ::ActiveRecord::Base.connection_pool.with_connection { - Mdm::Ref.find_by_name(name) - } - end - # # WMAP # Support methods diff --git a/lib/msf/core/db_manager/ref.rb b/lib/msf/core/db_manager/ref.rb new file mode 100644 index 0000000000..83613334ab --- /dev/null +++ b/lib/msf/core/db_manager/ref.rb @@ -0,0 +1,33 @@ +module Msf::DBManager::Ref + # + # Find or create a reference matching this name + # + def find_or_create_ref(opts) + ret = {} + ret[:ref] = get_ref(opts[:name]) + return ret[:ref] if ret[:ref] + + ::ActiveRecord::Base.connection_pool.with_connection { + ref = ::Mdm::Ref.find_or_initialize_by_name(opts[:name]) + if ref and ref.changed? + ref.save! + end + ret[:ref] = ref + } + end + + def get_ref(name) + ::ActiveRecord::Base.connection_pool.with_connection { + ::Mdm::Ref.find_by_name(name) + } + end + + # + # Find a reference matching this name + # + def has_ref?(name) + ::ActiveRecord::Base.connection_pool.with_connection { + Mdm::Ref.find_by_name(name) + } + end +end \ No newline at end of file From 802635243332bdf1f11c04520f65a1b6c1521a88 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 10:12:09 -0500 Subject: [PATCH 056/159] Extract Msf::DBManager::Ref shared examples MSP-11124 --- spec/lib/msf/db_manager_spec.rb | 4 +--- spec/support/shared/examples/msf/db_manager/ref.rb | 5 +++++ 2 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/ref.rb diff --git a/spec/lib/msf/db_manager_spec.rb b/spec/lib/msf/db_manager_spec.rb index 9c62162ef2..1593ef7df1 100644 --- a/spec/lib/msf/db_manager_spec.rb +++ b/spec/lib/msf/db_manager_spec.rb @@ -31,6 +31,7 @@ describe Msf::DBManager do it_should_behave_like 'Msf::DBManager::Migration' it_should_behave_like 'Msf::DBManager::ModuleCache' it_should_behave_like 'Msf::DBManager::Note' + it_should_behave_like 'Msf::DBManager::Ref' it_should_behave_like 'Msf::DBManager::Report' it_should_behave_like 'Msf::DBManager::Service' it_should_behave_like 'Msf::DBManager::Sink' @@ -80,10 +81,7 @@ describe Msf::DBManager do it { is_expected.to respond_to :drivers } it { is_expected.to respond_to :drivers= } it { is_expected.to respond_to :error } - it { is_expected.to respond_to :find_or_create_ref } - it { is_expected.to respond_to :get_ref } it { is_expected.to respond_to :get_session } - it { is_expected.to respond_to :has_ref? } it { is_expected.to respond_to :initialize_adapter } it { is_expected.to respond_to :initialize_database_support } it { is_expected.to respond_to :report_host_details } diff --git a/spec/support/shared/examples/msf/db_manager/ref.rb b/spec/support/shared/examples/msf/db_manager/ref.rb new file mode 100644 index 0000000000..28d4fa4c39 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/ref.rb @@ -0,0 +1,5 @@ +shared_examples_for 'Msf::DBManager::Ref' do + it { is_expected.to respond_to :find_or_create_ref } + it { is_expected.to respond_to :get_ref } + it { is_expected.to respond_to :has_ref? } +end \ No newline at end of file From f16b3f05b4c79346cd8a21a39ffc953cdf074f4c Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 10:15:14 -0500 Subject: [PATCH 057/159] Extract Msf::DBManager::HostDetail MSP-11124 Extract method related to `Mdm::HostDetail`s. --- lib/msf/core/db_manager.rb | 22 ++-------------------- lib/msf/core/db_manager/host_detail.rb | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+), 20 deletions(-) create mode 100644 lib/msf/core/db_manager/host_detail.rb diff --git a/lib/msf/core/db_manager.rb b/lib/msf/core/db_manager.rb index cc1fa39ff2..27791708ff 100644 --- a/lib/msf/core/db_manager.rb +++ b/lib/msf/core/db_manager.rb @@ -81,6 +81,7 @@ class DBManager autoload :ExploitAttempt, 'msf/core/db_manager/exploit_attempt' autoload :ExploitedHost, 'msf/core/db_manager/exploited_host' autoload :Host, 'msf/core/db_manager/host' + autoload :HostDetail, 'msf/core/db_manager/host_detail' autoload :Import, 'msf/core/db_manager/import' autoload :IPAddress, 'msf/core/db_manager/ip_address' autoload :Loot, 'msf/core/db_manager/loot' @@ -104,6 +105,7 @@ class DBManager include Msf::DBManager::ExploitAttempt include Msf::DBManager::ExploitedHost include Msf::DBManager::Host + include Msf::DBManager::HostDetail include Msf::DBManager::Import include Msf::DBManager::ImportMsfXml include Msf::DBManager::IPAddress @@ -753,26 +755,6 @@ class DBManager } end - # - # Populate the host_details table with additional - # information, matched by a specific criteria - # - def report_host_details(host, details) - ::ActiveRecord::Base.connection_pool.with_connection { - - detail = ::Mdm::HostDetail.where(( details.delete(:key) || {} ).merge(:host_id => host.id)).first - if detail - details.each_pair do |k,v| - detail[k] = v - end - detail.save! if detail.changed? - detail - else - detail = ::Mdm::HostDetail.create(details.merge(:host_id => host.id)) - end - } - end - # # WMAP # Support methods diff --git a/lib/msf/core/db_manager/host_detail.rb b/lib/msf/core/db_manager/host_detail.rb new file mode 100644 index 0000000000..3ef7727be9 --- /dev/null +++ b/lib/msf/core/db_manager/host_detail.rb @@ -0,0 +1,21 @@ +module Msf::DBManager::HostDetail + # + # Populate the host_details table with additional + # information, matched by a specific criteria + # + def report_host_details(host, details) + ::ActiveRecord::Base.connection_pool.with_connection { + + detail = ::Mdm::HostDetail.where(( details.delete(:key) || {} ).merge(:host_id => host.id)).first + if detail + details.each_pair do |k,v| + detail[k] = v + end + detail.save! if detail.changed? + detail + else + detail = ::Mdm::HostDetail.create(details.merge(:host_id => host.id)) + end + } + end +end \ No newline at end of file From 4aab5129b05f8ac3893b9aa9d9f48c044029e828 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 10:27:55 -0500 Subject: [PATCH 058/159] Extract Msf::DBManager::HostDetail shared examples MSP-11124 --- spec/lib/msf/db_manager_spec.rb | 2 +- spec/support/shared/examples/msf/db_manager/host_detail.rb | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 spec/support/shared/examples/msf/db_manager/host_detail.rb diff --git a/spec/lib/msf/db_manager_spec.rb b/spec/lib/msf/db_manager_spec.rb index 1593ef7df1..33d22eef9e 100644 --- a/spec/lib/msf/db_manager_spec.rb +++ b/spec/lib/msf/db_manager_spec.rb @@ -24,6 +24,7 @@ describe Msf::DBManager do it_should_behave_like 'Msf::DBManager::ExploitAttempt' it_should_behave_like 'Msf::DBManager::ExploitedHost' it_should_behave_like 'Msf::DBManager::Host' + it_should_behave_like 'Msf::DBManager::HostDetail' it_should_behave_like 'Msf::DBManager::IPAddress' it_should_behave_like 'Msf::DBManager::Import' it_should_behave_like 'Msf::DBManager::ImportMsfXml' @@ -84,7 +85,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :get_session } it { is_expected.to respond_to :initialize_adapter } it { is_expected.to respond_to :initialize_database_support } - it { is_expected.to respond_to :report_host_details } it { is_expected.to respond_to :report_host_tag } context '#report_session' do diff --git a/spec/support/shared/examples/msf/db_manager/host_detail.rb b/spec/support/shared/examples/msf/db_manager/host_detail.rb new file mode 100644 index 0000000000..92c3d0529e --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/host_detail.rb @@ -0,0 +1,3 @@ +shared_examples_for 'Msf::DBManager::HostDetail' do + it { is_expected.to respond_to :report_host_details } +end \ No newline at end of file From 1811d4e58f89e4a032227b829b9fd412a755286b Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 10:50:11 -0500 Subject: [PATCH 059/159] Extract Msf::DBManager::Session MSP-11124 Extract methods related to `Mdm::Session`s. --- lib/msf/core/db_manager.rb | 206 +---------------------------- lib/msf/core/db_manager/session.rb | 205 ++++++++++++++++++++++++++++ 2 files changed, 207 insertions(+), 204 deletions(-) create mode 100644 lib/msf/core/db_manager/session.rb diff --git a/lib/msf/core/db_manager.rb b/lib/msf/core/db_manager.rb index 27791708ff..06378eb427 100644 --- a/lib/msf/core/db_manager.rb +++ b/lib/msf/core/db_manager.rb @@ -90,6 +90,7 @@ class DBManager autoload :Ref, 'msf/core/db_manager/ref' autoload :Report, 'msf/core/db_manager/report' autoload :Service, 'msf/core/db_manager/service' + autoload :Session, 'msf/core/db_manager/session' autoload :Sink, 'msf/core/db_manager/sink' autoload :Task, 'msf/core/db_manager/task' autoload :Vuln, 'msf/core/db_manager/vuln' @@ -116,6 +117,7 @@ class DBManager include Msf::DBManager::Ref include Msf::DBManager::Report include Msf::DBManager::Service + include Msf::DBManager::Session include Msf::DBManager::Sink include Msf::DBManager::Task include Msf::DBManager::Vuln @@ -412,210 +414,6 @@ class DBManager } end - # Returns a session based on opened_time, host address, and workspace - # (or returns nil) - def get_session(opts) - return if not active - ::ActiveRecord::Base.connection_pool.with_connection { - wspace = opts[:workspace] || opts[:wspace] || workspace - addr = opts[:addr] || opts[:address] || opts[:host] || return - host = get_host(:workspace => wspace, :host => addr) - time = opts[:opened_at] || opts[:created_at] || opts[:time] || return - ::Mdm::Session.find_by_host_id_and_opened_at(host.id, time) - } - end - - # @note The Mdm::Session#desc will be truncated to 255 characters. - # @todo https://www.pivotaltracker.com/story/show/48249739 - # - # @overload report_session(opts) - # Creates an Mdm::Session from Msf::Session. If +via_exploit+ is set on the - # +session+, then an Mdm::Vuln and Mdm::ExploitAttempt is created for the - # session's host. The Mdm::Host for the +session_host+ is created using - # The session.session_host, +session.arch+ (if +session+ responds to arch), - # and the workspace derived from opts or the +session+. The Mdm::Session is - # assumed to be +last_seen+ and +opened_at+ at the time report_session is - # called. +session.exploit_datastore['ParentModule']+ is used for the - # Mdm::Session#via_exploit if +session.via_exploit+ is - # 'exploit/multi/handler'. - # - # @param opts [Hash{Symbol => Object}] options - # @option opt [Msf::Session, #datastore, #platform, #type, #via_exploit, #via_payload] :session - # The in-memory session to persist to the database. - # @option opts [Mdm::Workspace] :workspace The workspace for in which the - # :session host is contained. Also used as the workspace for the - # Mdm::ExploitAttempt and Mdm::Vuln. Defaults to Mdm::Worksapce with - # Mdm::Workspace#name equal to +session.workspace+. - # @return [nil] if {Msf::DBManager#active} is +false+. - # @return [Mdm::Session] if session is saved - # @raise [ArgumentError] if :session is not an {Msf::Session}. - # @raise [ActiveRecord::RecordInvalid] if session is invalid and cannot be - # saved, in which case, the Mdm::ExploitAttempt and Mdm::Vuln will not be - # created, but the Mdm::Host will have been. (There is no transaction - # to rollback the Mdm::Host creation.) - # @see #find_or_create_host - # @see #normalize_host - # @see #report_exploit_success - # @see #report_vuln - # - # @overload report_session(opts) - # Creates an Mdm::Session from Mdm::Host. - # - # @param opts [Hash{Symbol => Object}] options - # @option opts [DateTime, Time] :closed_at The date and time the sesion was - # closed. - # @option opts [String] :close_reason Reason the session was closed. - # @option opts [Hash] :datastore {Msf::DataStore#to_h}. - # @option opts [String] :desc Session description. Will be truncated to 255 - # characters. - # @option opts [Mdm::Host] :host The host on which the session was opened. - # @option opts [DateTime, Time] :last_seen The last date and time the - # session was seen to be open. Defaults to :closed_at's value. - # @option opts [DateTime, Time] :opened_at The date and time that the - # session was opened. - # @option opts [String] :platform The platform of the host. - # @option opts [Array] :routes ([]) The routes through the session for - # pivoting. - # @option opts [String] :stype Session type. - # @option opts [String] :via_exploit The {Msf::Module#fullname} of the - # exploit that was used to open the session. - # @option option [String] :via_payload the {MSf::Module#fullname} of the - # payload sent to the host when the exploit was successful. - # @return [nil] if {Msf::DBManager#active} is +false+. - # @return [Mdm::Session] if session is saved. - # @raise [ArgumentError] if :host is not an Mdm::Host. - # @raise [ActiveRecord::RecordInvalid] if session is invalid and cannot be - # saved. - # - # @raise ArgumentError if :host and :session is +nil+ - def report_session(opts) - return if not active - ::ActiveRecord::Base.connection_pool.with_connection { - if opts[:session] - raise ArgumentError.new("Invalid :session, expected Msf::Session") unless opts[:session].kind_of? Msf::Session - session = opts[:session] - wspace = opts[:workspace] || find_workspace(session.workspace) - h_opts = { } - h_opts[:host] = normalize_host(session) - h_opts[:arch] = session.arch if session.respond_to?(:arch) and session.arch - h_opts[:workspace] = wspace - host = find_or_create_host(h_opts) - sess_data = { - :host_id => host.id, - :stype => session.type, - :desc => session.info, - :platform => session.platform, - :via_payload => session.via_payload, - :via_exploit => session.via_exploit, - :routes => [], - :datastore => session.exploit_datastore.to_h, - :port => session.session_port, - :opened_at => Time.now.utc, - :last_seen => Time.now.utc, - :local_id => session.sid - } - elsif opts[:host] - raise ArgumentError.new("Invalid :host, expected Host object") unless opts[:host].kind_of? ::Mdm::Host - host = opts[:host] - sess_data = { - :host_id => host.id, - :stype => opts[:stype], - :desc => opts[:desc], - :platform => opts[:platform], - :via_payload => opts[:via_payload], - :via_exploit => opts[:via_exploit], - :routes => opts[:routes] || [], - :datastore => opts[:datastore], - :opened_at => opts[:opened_at], - :closed_at => opts[:closed_at], - :last_seen => opts[:last_seen] || opts[:closed_at], - :close_reason => opts[:close_reason], - } - else - raise ArgumentError.new("Missing option :session or :host") - end - ret = {} - - # Truncate the session data if necessary - if sess_data[:desc] - sess_data[:desc] = sess_data[:desc][0,255] - end - - # In the case of multi handler we cannot yet determine the true - # exploit responsible. But we can at least show the parent versus - # just the generic handler: - if session and session.via_exploit == "exploit/multi/handler" and sess_data[:datastore]['ParentModule'] - sess_data[:via_exploit] = sess_data[:datastore]['ParentModule'] - end - - s = ::Mdm::Session.new(sess_data) - s.save! - - if session and session.exploit_task and session.exploit_task.record - session_task = session.exploit_task.record - if session_task.class == Mdm::Task - Mdm::TaskSession.create(:task => session_task, :session => s ) - end - end - - - if opts[:session] - session.db_record = s - end - - # If this is a live session, we know the host is vulnerable to something. - if opts[:session] and session.via_exploit - mod = framework.modules.create(session.via_exploit) - - if session.via_exploit == "exploit/multi/handler" and sess_data[:datastore]['ParentModule'] - mod_fullname = sess_data[:datastore]['ParentModule'] - mod_name = ::Mdm::Module::Detail.find_by_fullname(mod_fullname).name - else - mod_name = mod.name - mod_fullname = mod.fullname - end - - vuln_info = { - :host => host.address, - :name => mod_name, - :refs => mod.references, - :workspace => wspace, - :exploited_at => Time.now.utc, - :info => "Exploited by #{mod_fullname} to create Session #{s.id}" - } - - port = session.exploit_datastore["RPORT"] - service = (port ? host.services.find_by_port(port.to_i) : nil) - - vuln_info[:service] = service if service - - vuln = framework.db.report_vuln(vuln_info) - - if session.via_exploit == "exploit/multi/handler" and sess_data[:datastore]['ParentModule'] - via_exploit = sess_data[:datastore]['ParentModule'] - else - via_exploit = session.via_exploit - end - attempt_info = { - :timestamp => Time.now.utc, - :workspace => wspace, - :module => via_exploit, - :username => session.username, - :refs => mod.references, - :session_id => s.id, - :host => host, - :service => service, - :vuln => vuln - } - - framework.db.report_exploit_success(attempt_info) - - end - - s - } - end - # # Record a session event in the database # diff --git a/lib/msf/core/db_manager/session.rb b/lib/msf/core/db_manager/session.rb new file mode 100644 index 0000000000..56b4a0a436 --- /dev/null +++ b/lib/msf/core/db_manager/session.rb @@ -0,0 +1,205 @@ +module Msf::DBManager::Session + # Returns a session based on opened_time, host address, and workspace + # (or returns nil) + def get_session(opts) + return if not active + ::ActiveRecord::Base.connection_pool.with_connection { + wspace = opts[:workspace] || opts[:wspace] || workspace + addr = opts[:addr] || opts[:address] || opts[:host] || return + host = get_host(:workspace => wspace, :host => addr) + time = opts[:opened_at] || opts[:created_at] || opts[:time] || return + ::Mdm::Session.find_by_host_id_and_opened_at(host.id, time) + } + end + + # @note The Mdm::Session#desc will be truncated to 255 characters. + # @todo https://www.pivotaltracker.com/story/show/48249739 + # + # @overload report_session(opts) + # Creates an Mdm::Session from Msf::Session. If +via_exploit+ is set on the + # +session+, then an Mdm::Vuln and Mdm::ExploitAttempt is created for the + # session's host. The Mdm::Host for the +session_host+ is created using + # The session.session_host, +session.arch+ (if +session+ responds to arch), + # and the workspace derived from opts or the +session+. The Mdm::Session is + # assumed to be +last_seen+ and +opened_at+ at the time report_session is + # called. +session.exploit_datastore['ParentModule']+ is used for the + # Mdm::Session#via_exploit if +session.via_exploit+ is + # 'exploit/multi/handler'. + # + # @param opts [Hash{Symbol => Object}] options + # @option opt [Msf::Session, #datastore, #platform, #type, #via_exploit, #via_payload] :session + # The in-memory session to persist to the database. + # @option opts [Mdm::Workspace] :workspace The workspace for in which the + # :session host is contained. Also used as the workspace for the + # Mdm::ExploitAttempt and Mdm::Vuln. Defaults to Mdm::Worksapce with + # Mdm::Workspace#name equal to +session.workspace+. + # @return [nil] if {Msf::DBManager#active} is +false+. + # @return [Mdm::Session] if session is saved + # @raise [ArgumentError] if :session is not an {Msf::Session}. + # @raise [ActiveRecord::RecordInvalid] if session is invalid and cannot be + # saved, in which case, the Mdm::ExploitAttempt and Mdm::Vuln will not be + # created, but the Mdm::Host will have been. (There is no transaction + # to rollback the Mdm::Host creation.) + # @see #find_or_create_host + # @see #normalize_host + # @see #report_exploit_success + # @see #report_vuln + # + # @overload report_session(opts) + # Creates an Mdm::Session from Mdm::Host. + # + # @param opts [Hash{Symbol => Object}] options + # @option opts [DateTime, Time] :closed_at The date and time the sesion was + # closed. + # @option opts [String] :close_reason Reason the session was closed. + # @option opts [Hash] :datastore {Msf::DataStore#to_h}. + # @option opts [String] :desc Session description. Will be truncated to 255 + # characters. + # @option opts [Mdm::Host] :host The host on which the session was opened. + # @option opts [DateTime, Time] :last_seen The last date and time the + # session was seen to be open. Defaults to :closed_at's value. + # @option opts [DateTime, Time] :opened_at The date and time that the + # session was opened. + # @option opts [String] :platform The platform of the host. + # @option opts [Array] :routes ([]) The routes through the session for + # pivoting. + # @option opts [String] :stype Session type. + # @option opts [String] :via_exploit The {Msf::Module#fullname} of the + # exploit that was used to open the session. + # @option option [String] :via_payload the {MSf::Module#fullname} of the + # payload sent to the host when the exploit was successful. + # @return [nil] if {Msf::DBManager#active} is +false+. + # @return [Mdm::Session] if session is saved. + # @raise [ArgumentError] if :host is not an Mdm::Host. + # @raise [ActiveRecord::RecordInvalid] if session is invalid and cannot be + # saved. + # + # @raise ArgumentError if :host and :session is +nil+ + def report_session(opts) + return if not active + ::ActiveRecord::Base.connection_pool.with_connection { + if opts[:session] + raise ArgumentError.new("Invalid :session, expected Msf::Session") unless opts[:session].kind_of? Msf::Session + session = opts[:session] + wspace = opts[:workspace] || find_workspace(session.workspace) + h_opts = { } + h_opts[:host] = normalize_host(session) + h_opts[:arch] = session.arch if session.respond_to?(:arch) and session.arch + h_opts[:workspace] = wspace + host = find_or_create_host(h_opts) + sess_data = { + :host_id => host.id, + :stype => session.type, + :desc => session.info, + :platform => session.platform, + :via_payload => session.via_payload, + :via_exploit => session.via_exploit, + :routes => [], + :datastore => session.exploit_datastore.to_h, + :port => session.session_port, + :opened_at => Time.now.utc, + :last_seen => Time.now.utc, + :local_id => session.sid + } + elsif opts[:host] + raise ArgumentError.new("Invalid :host, expected Host object") unless opts[:host].kind_of? ::Mdm::Host + host = opts[:host] + sess_data = { + :host_id => host.id, + :stype => opts[:stype], + :desc => opts[:desc], + :platform => opts[:platform], + :via_payload => opts[:via_payload], + :via_exploit => opts[:via_exploit], + :routes => opts[:routes] || [], + :datastore => opts[:datastore], + :opened_at => opts[:opened_at], + :closed_at => opts[:closed_at], + :last_seen => opts[:last_seen] || opts[:closed_at], + :close_reason => opts[:close_reason], + } + else + raise ArgumentError.new("Missing option :session or :host") + end + ret = {} + + # Truncate the session data if necessary + if sess_data[:desc] + sess_data[:desc] = sess_data[:desc][0,255] + end + + # In the case of multi handler we cannot yet determine the true + # exploit responsible. But we can at least show the parent versus + # just the generic handler: + if session and session.via_exploit == "exploit/multi/handler" and sess_data[:datastore]['ParentModule'] + sess_data[:via_exploit] = sess_data[:datastore]['ParentModule'] + end + + s = ::Mdm::Session.new(sess_data) + s.save! + + if session and session.exploit_task and session.exploit_task.record + session_task = session.exploit_task.record + if session_task.class == Mdm::Task + Mdm::TaskSession.create(:task => session_task, :session => s ) + end + end + + + if opts[:session] + session.db_record = s + end + + # If this is a live session, we know the host is vulnerable to something. + if opts[:session] and session.via_exploit + mod = framework.modules.create(session.via_exploit) + + if session.via_exploit == "exploit/multi/handler" and sess_data[:datastore]['ParentModule'] + mod_fullname = sess_data[:datastore]['ParentModule'] + mod_name = ::Mdm::Module::Detail.find_by_fullname(mod_fullname).name + else + mod_name = mod.name + mod_fullname = mod.fullname + end + + vuln_info = { + :host => host.address, + :name => mod_name, + :refs => mod.references, + :workspace => wspace, + :exploited_at => Time.now.utc, + :info => "Exploited by #{mod_fullname} to create Session #{s.id}" + } + + port = session.exploit_datastore["RPORT"] + service = (port ? host.services.find_by_port(port.to_i) : nil) + + vuln_info[:service] = service if service + + vuln = framework.db.report_vuln(vuln_info) + + if session.via_exploit == "exploit/multi/handler" and sess_data[:datastore]['ParentModule'] + via_exploit = sess_data[:datastore]['ParentModule'] + else + via_exploit = session.via_exploit + end + attempt_info = { + :timestamp => Time.now.utc, + :workspace => wspace, + :module => via_exploit, + :username => session.username, + :refs => mod.references, + :session_id => s.id, + :host => host, + :service => service, + :vuln => vuln + } + + framework.db.report_exploit_success(attempt_info) + + end + + s + } + end +end \ No newline at end of file From 6fb263d9890ad474051e7208cdd842c5b280a7ec Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 10:52:39 -0500 Subject: [PATCH 060/159] Extract Msf::DBManager::Session shared examples MSP-11124 --- spec/lib/msf/db_manager_spec.rb | 642 +----------------- .../shared/examples/msf/db_manager/session.rb | 642 ++++++++++++++++++ 2 files changed, 643 insertions(+), 641 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/session.rb diff --git a/spec/lib/msf/db_manager_spec.rb b/spec/lib/msf/db_manager_spec.rb index 33d22eef9e..9e86131a71 100644 --- a/spec/lib/msf/db_manager_spec.rb +++ b/spec/lib/msf/db_manager_spec.rb @@ -35,6 +35,7 @@ describe Msf::DBManager do it_should_behave_like 'Msf::DBManager::Ref' it_should_behave_like 'Msf::DBManager::Report' it_should_behave_like 'Msf::DBManager::Service' + it_should_behave_like 'Msf::DBManager::Session' it_should_behave_like 'Msf::DBManager::Sink' it_should_behave_like 'Msf::DBManager::Task' it_should_behave_like 'Msf::DBManager::Vuln' @@ -82,650 +83,9 @@ describe Msf::DBManager do it { is_expected.to respond_to :drivers } it { is_expected.to respond_to :drivers= } it { is_expected.to respond_to :error } - it { is_expected.to respond_to :get_session } it { is_expected.to respond_to :initialize_adapter } it { is_expected.to respond_to :initialize_database_support } it { is_expected.to respond_to :report_host_tag } - - context '#report_session' do - let(:options) do - {} - end - - subject(:report_session) do - db_manager.report_session(options) - end - - context 'with active' do - let(:active) do - true - end - - context 'with :session' do - before(:each) do - options[:session] = session - end - - context 'with Msf::Session' do - let(:exploit_datastore) do - Msf::ModuleDataStore.new(module_instance).tap do |datastore| - datastore['ParentModule'] = parent_module_fullname - - remote_port = rand(2 ** 16 - 1) - datastore['RPORT'] = remote_port - end - end - - let(:host) do - FactoryGirl.create(:mdm_host, :workspace => session_workspace) - end - - let(:module_instance) do - name = 'multi/handler' - - double( - 'Msf::Module', - :fullname => "exploit/#{name}", - :framework => framework, - :name => name - ) - end - - let(:options_workspace) do - FactoryGirl.create(:mdm_workspace) - end - - let(:parent_module_fullname) do - "exploit/#{parent_module_name}" - end - - let(:parent_module_name) do - 'windows/smb/ms08_067_netapi' - end - - let(:parent_path) do - Metasploit::Framework.root.join('modules').to_path - end - - let(:session) do - session_class.new.tap do |session| - session.exploit_datastore = exploit_datastore - session.info = 'Info' - session.platform = 'Platform' - session.session_host = host.address - session.sid = rand(100) - session.type = 'Session Type' - session.via_exploit = 'exploit/multi/handler' - session.via_payload = 'payload/single/windows/metsvc_bind_tcp' - session.workspace = session_workspace.name - end - end - - let(:session_class) do - Class.new do - include Msf::Session - - attr_accessor :datastore - attr_accessor :platform - attr_accessor :type - attr_accessor :via_exploit - attr_accessor :via_payload - end - end - - let(:session_workspace) do - FactoryGirl.create(:mdm_workspace) - end - - before(:each) do - reference_name = 'multi/handler' - path = File.join(parent_path, 'exploits', reference_name) - - # fake cache data for exploit/multi/handler so it can be loaded - framework.modules.send( - :module_info_by_path=, - { - path => - { - :parent_path => parent_path, - :reference_name => reference_name, - :type => 'exploit', - } - } - ) - - FactoryGirl.create( - :mdm_module_detail, - :fullname => parent_module_fullname, - :name => parent_module_name - ) - end - - context 'with :workspace' do - before(:each) do - options[:workspace] = options_workspace - end - - it 'should not find workspace from session' do - db_manager.should_not_receive(:find_workspace) - - report_session - end - end - - context 'without :workspace' do - it 'should find workspace from session' do - db_manager.should_receive(:find_workspace).with(session.workspace).and_call_original - - report_session - end - - it 'should pass session.workspace to #find_or_create_host' do - db_manager.should_receive(:find_or_create_host).with( - hash_including( - :workspace => session_workspace - ) - ).and_return(host) - - report_session - end - end - - context 'with workspace from either :workspace or session' do - it 'should pass normalized host from session as :host to #find_or_create_host' do - normalized_host = double('Normalized Host') - db_manager.stub(:normalize_host).with(session).and_return(normalized_host) - # stub report_vuln so its use of find_or_create_host and normalize_host doesn't interfere. - db_manager.stub(:report_vuln) - - db_manager.should_receive(:find_or_create_host).with( - hash_including( - :host => normalized_host - ) - ).and_return(host) - - report_session - end - - context 'with session responds to arch' do - let(:arch) do - FactoryGirl.generate :mdm_host_arch - end - - before(:each) do - session.stub(:arch => arch) - end - - it 'should pass :arch to #find_or_create_host' do - db_manager.should_receive(:find_or_create_host).with( - hash_including( - :arch => arch - ) - ).and_call_original - - report_session - end - end - - context 'without session responds to arch' do - it 'should not pass :arch to #find_or_create_host' do - db_manager.should_receive(:find_or_create_host).with( - hash_excluding( - :arch - ) - ).and_call_original - - report_session - end - end - - it 'should create an Mdm::Session' do - expect { - report_session - }.to change(Mdm::Session, :count).by(1) - end - - it { should be_an Mdm::Session } - - it 'should set session.db_record to created Mdm::Session' do - mdm_session = report_session - - session.db_record.should == mdm_session - end - - context 'with session.via_exploit' do - it 'should create session.via_exploit module' do - framework.modules.should_receive(:create).with(session.via_exploit).and_call_original - - report_session - end - - it 'should create Mdm::Vuln' do - expect { - report_session - }.to change(Mdm::Vuln, :count).by(1) - end - - context 'created Mdm::Vuln' do - let(:mdm_session) do - Mdm::Session.last - end - - let(:rport) do - nil - end - - before(:each) do - Timecop.freeze - - session.exploit_datastore['RPORT'] = rport - - report_session - end - - after(:each) do - Timecop.return - end - - subject(:vuln) do - Mdm::Vuln.last - end - - it { expect(subject.host).to eq(Mdm::Host.last) } - it { expect(subject.refs).to eq([]) } - it { expect(subject.exploited_at).to be_within(1.second).of(Time.now.utc) } - - context "with session.via_exploit 'exploit/multi/handler'" do - context "with session.exploit_datastore['ParentModule']" do - it { expect(subject.info).to eq("Exploited by #{parent_module_fullname} to create Session #{mdm_session.id}") } - it { expect(subject.name).to eq(parent_module_name) } - end - end - - context "without session.via_exploit 'exploit/multi/handler'" do - let(:reference_name) do - 'windows/smb/ms08_067_netapi' - end - - before(:each) do - path = File.join( - parent_path, - 'exploits', - "#{reference_name}.rb" - ) - type = 'exploit' - - # fake cache data for ParentModule so it can be loaded - framework.modules.send( - :module_info_by_path=, - { - path => - { - :parent_path => parent_path, - :reference_name => reference_name, - :type => type, - } - } - ) - - session.via_exploit = "#{type}/#{reference_name}" - end - - it { expect(subject.info).to eq("Exploited by #{session.via_exploit} to create Session #{mdm_session.id}") } - it { expect(subject.name).to eq(reference_name) } - end - - context 'with RPORT' do - let(:rport) do - # use service.port instead of having service use rport so - # that service is forced to exist before call to - # report_service, which happens right after using rport in - # outer context's before(:each) - service.port - end - - let(:service) do - FactoryGirl.create( - :mdm_service, - :host => host - ) - end - - it { expect(subject.service).to eq(service) } - end - - context 'without RPORT' do - it { expect(subject.service).to be_nil } - end - end - - context 'created Mdm::ExploitAttempt' do - let(:rport) do - nil - end - - before(:each) do - Timecop.freeze - - session.exploit_datastore['RPORT'] = rport - - report_session - end - - after(:each) do - Timecop.return - end - - subject(:exploit_attempt) do - Mdm::ExploitAttempt.last - end - - it { expect(subject.attempted_at).to be_within(1.second).of(Time.now.utc) } - # @todo https://www.pivotaltracker.com/story/show/48362615 - it { expect(subject.session_id).to eq(Mdm::Session.last.id) } - it { expect(subject.exploited).to be_truthy } - # @todo https://www.pivotaltracker.com/story/show/48362615 - it { expect(subject.vuln_id).to eq(Mdm::Vuln.last.id) } - - context "with session.via_exploit 'exploit/multi/handler'" do - context "with session.datastore['ParentModule']" do - it { expect(subject.module).to eq(parent_module_fullname) } - end - end - - context "without session.via_exploit 'exploit/multi/handler'" do - before(:each) do - session.via_exploit = parent_module_fullname - end - - it { expect(subject.module).to eq(session.via_exploit) } - end - end - end - - context 'returned Mdm::Session' do - before(:each) do - Timecop.freeze - end - - after(:each) do - Timecop.return - end - - subject(:mdm_session) do - report_session - end - - # - # Ensure session has attributes present so its on mdm_session are - # not just comparing nils. - # - - it 'should have session.info present' do - session.info.should be_present - end - - it 'should have session.sid present' do - session.sid.should be_present - end - - it 'should have session.platform present' do - session.platform.should be_present - end - - it 'should have session.type present' do - session.type.should be_present - end - - it 'should have session.via_exploit present' do - session.via_exploit.should be_present - end - - it 'should have session.via_payload present' do - session.via_exploit.should be_present - end - - it { expect(subject.datastore).to eq(session.exploit_datastore.to_h) } - it { expect(subject.desc).to eq(session.info) } - it { expect(subject.host_id).to eq(Mdm::Host.last.id) } - it { expect(subject.last_seen).to be_within(1.second).of(Time.now.utc) } - it { expect(subject.local_id).to eq(session.sid) } - it { expect(subject.opened_at).to be_within(1.second).of(Time.now.utc) } - it { expect(subject.platform).to eq(session.platform) } - it { expect(subject.routes).to eq([]) } - it { expect(subject.stype).to eq(session.type) } - it { expect(subject.via_payload).to eq(session.via_payload) } - - context "with session.via_exploit 'exploit/multi/handler'" do - it "should have session.via_exploit of 'exploit/multi/handler'" do - session.via_exploit.should == 'exploit/multi/handler' - end - - context "with session.exploit_datastore['ParentModule']" do - it "should have session.exploit_datastore['ParentModule']" do - session.exploit_datastore['ParentModule'].should_not be_nil - end - - it { expect(subject.via_exploit).to eq(parent_module_fullname) } - end - end - - context "without session.via_exploit 'exploit/multi/handler'" do - before(:each) do - reference_name = 'windows/smb/ms08_067_netapi' - path = File.join( - parent_path, - 'exploits', - "#{reference_name}.rb" - ) - type = 'exploit' - - # fake cache data for ParentModule so it can be loaded - framework.modules.send( - :module_info_by_path=, - { - path => - { - :parent_path => parent_path, - :reference_name => reference_name, - :type => type, - } - } - ) - - session.via_exploit = "#{type}/#{reference_name}" - end - - it "should not have session.via_exploit of 'exploit/multi/handler'" do - session.via_exploit.should_not == 'exploit/multi/handler' - end - - it { expect(subject.via_exploit).to eq(session.via_exploit) } - end - end - end - end - - context 'without Msf::Session' do - let(:session) do - double('Not a Msf::Session') - end - - it 'should raise ArgumentError' do - expect { - report_session - }.to raise_error(ArgumentError, "Invalid :session, expected Msf::Session") - end - end - end - - context 'without :session' do - context 'with :host' do - before(:each) do - options[:host] = host - end - - context 'with Mdm::Host' do - let(:host) do - FactoryGirl.create(:mdm_host) - end - - context 'created Mdm::Session' do - let(:closed_at) do - nil - end - - let(:close_reason) do - 'Closed because...' - end - - let(:description) do - 'Session Description' - end - - let(:exploit_full_name) do - 'exploit/windows/smb/ms08_067_netapi' - end - - let(:last_seen) do - nil - end - - let(:opened_at) do - Time.now.utc - 5.minutes - end - - let(:payload_full_name) do - 'payload/singles/windows/metsvc_reverse_tcp' - end - - let(:platform) do - 'Host Platform' - end - - let(:routes) do - nil - end - - let(:session_type) do - 'Session Type' - end - - before(:each) do - options[:closed_at] = closed_at - options[:close_reason] = close_reason - options[:desc] = description - options[:last_seen] = last_seen - options[:opened_at] = opened_at - options[:platform] = platform - options[:routes] = routes - options[:stype] = session_type - options[:via_payload] = payload_full_name - options[:via_exploit] = exploit_full_name - end - - subject(:mdm_session) do - report_session - end - - it { expect(subject.close_reason).to eq(close_reason) } - it { expect(subject.desc).to eq(description) } - it { expect(subject.host).to eq(host) } - it { expect(subject.platform).to eq(platform) } - it { expect(subject.stype).to eq(session_type) } - it { expect(subject.via_exploit).to eq(exploit_full_name) } - it { expect(subject.via_payload).to eq(payload_full_name) } - - context 'with :last_seen' do - let(:last_seen) do - opened_at - end - - it { expect(subject.last_seen).to eq(last_seen) } - end - - context 'with :closed_at' do - let(:closed_at) do - opened_at + 1.minute - end - - it { expect(subject.closed_at).to eq(closed_at) } - end - - context 'without :closed_at' do - it { expect(subject.closed_at).to be_nil } - end - - context 'without :last_seen' do - context 'with :closed_at' do - let(:closed_at) do - opened_at + 1.minute - end - - it { expect(subject.last_seen).to eq(closed_at) } - end - - context 'without :closed_at' do - it { expect(subject.last_seen).to be_nil } - end - end - - context 'with :routes' do - let(:routes) do - FactoryGirl.build_list( - :mdm_route, - 1, - :session => nil - ) - end - - it { expect(subject.routes).to eq(routes) } - end - - context 'without :routes' do - it { expect(subject.routes).to eq([]) } - end - end - end - - context 'without Mdm::Host' do - let(:host) do - '192.168.0.1' - end - - it 'should raise ArgumentError' do - expect { - report_session - }.to raise_error(ArgumentError, "Invalid :host, expected Host object") - end - end - end - - context 'without :host' do - it 'should raise ArgumentError' do - expect { - report_session - }.to raise_error(ArgumentError) - end - end - end - end - - context 'without active' do - let(:active) do - false - end - - it { should be_nil } - - it 'should not create a connection' do - ActiveRecord::Base.connection_pool.should_not_receive(:with_connection) - - report_session - end - end - end - it { is_expected.to respond_to :report_session_event } it { is_expected.to respond_to :report_session_route } it { is_expected.to respond_to :report_session_route_remove } diff --git a/spec/support/shared/examples/msf/db_manager/session.rb b/spec/support/shared/examples/msf/db_manager/session.rb new file mode 100644 index 0000000000..a471ebcaf2 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/session.rb @@ -0,0 +1,642 @@ +shared_examples_for 'Msf::DBManager::Session' do + it { is_expected.to respond_to :get_session } + + context '#report_session' do + let(:options) do + {} + end + + subject(:report_session) do + db_manager.report_session(options) + end + + context 'with active' do + let(:active) do + true + end + + context 'with :session' do + before(:each) do + options[:session] = session + end + + context 'with Msf::Session' do + let(:exploit_datastore) do + Msf::ModuleDataStore.new(module_instance).tap do |datastore| + datastore['ParentModule'] = parent_module_fullname + + remote_port = rand(2 ** 16 - 1) + datastore['RPORT'] = remote_port + end + end + + let(:host) do + FactoryGirl.create(:mdm_host, :workspace => session_workspace) + end + + let(:module_instance) do + name = 'multi/handler' + + double( + 'Msf::Module', + :fullname => "exploit/#{name}", + :framework => framework, + :name => name + ) + end + + let(:options_workspace) do + FactoryGirl.create(:mdm_workspace) + end + + let(:parent_module_fullname) do + "exploit/#{parent_module_name}" + end + + let(:parent_module_name) do + 'windows/smb/ms08_067_netapi' + end + + let(:parent_path) do + Metasploit::Framework.root.join('modules').to_path + end + + let(:session) do + session_class.new.tap do |session| + session.exploit_datastore = exploit_datastore + session.info = 'Info' + session.platform = 'Platform' + session.session_host = host.address + session.sid = rand(100) + session.type = 'Session Type' + session.via_exploit = 'exploit/multi/handler' + session.via_payload = 'payload/single/windows/metsvc_bind_tcp' + session.workspace = session_workspace.name + end + end + + let(:session_class) do + Class.new do + include Msf::Session + + attr_accessor :datastore + attr_accessor :platform + attr_accessor :type + attr_accessor :via_exploit + attr_accessor :via_payload + end + end + + let(:session_workspace) do + FactoryGirl.create(:mdm_workspace) + end + + before(:each) do + reference_name = 'multi/handler' + path = File.join(parent_path, 'exploits', reference_name) + + # fake cache data for exploit/multi/handler so it can be loaded + framework.modules.send( + :module_info_by_path=, + { + path => + { + :parent_path => parent_path, + :reference_name => reference_name, + :type => 'exploit', + } + } + ) + + FactoryGirl.create( + :mdm_module_detail, + :fullname => parent_module_fullname, + :name => parent_module_name + ) + end + + context 'with :workspace' do + before(:each) do + options[:workspace] = options_workspace + end + + it 'should not find workspace from session' do + db_manager.should_not_receive(:find_workspace) + + report_session + end + end + + context 'without :workspace' do + it 'should find workspace from session' do + db_manager.should_receive(:find_workspace).with(session.workspace).and_call_original + + report_session + end + + it 'should pass session.workspace to #find_or_create_host' do + db_manager.should_receive(:find_or_create_host).with( + hash_including( + :workspace => session_workspace + ) + ).and_return(host) + + report_session + end + end + + context 'with workspace from either :workspace or session' do + it 'should pass normalized host from session as :host to #find_or_create_host' do + normalized_host = double('Normalized Host') + db_manager.stub(:normalize_host).with(session).and_return(normalized_host) + # stub report_vuln so its use of find_or_create_host and normalize_host doesn't interfere. + db_manager.stub(:report_vuln) + + db_manager.should_receive(:find_or_create_host).with( + hash_including( + :host => normalized_host + ) + ).and_return(host) + + report_session + end + + context 'with session responds to arch' do + let(:arch) do + FactoryGirl.generate :mdm_host_arch + end + + before(:each) do + session.stub(:arch => arch) + end + + it 'should pass :arch to #find_or_create_host' do + db_manager.should_receive(:find_or_create_host).with( + hash_including( + :arch => arch + ) + ).and_call_original + + report_session + end + end + + context 'without session responds to arch' do + it 'should not pass :arch to #find_or_create_host' do + db_manager.should_receive(:find_or_create_host).with( + hash_excluding( + :arch + ) + ).and_call_original + + report_session + end + end + + it 'should create an Mdm::Session' do + expect { + report_session + }.to change(Mdm::Session, :count).by(1) + end + + it { should be_an Mdm::Session } + + it 'should set session.db_record to created Mdm::Session' do + mdm_session = report_session + + session.db_record.should == mdm_session + end + + context 'with session.via_exploit' do + it 'should create session.via_exploit module' do + framework.modules.should_receive(:create).with(session.via_exploit).and_call_original + + report_session + end + + it 'should create Mdm::Vuln' do + expect { + report_session + }.to change(Mdm::Vuln, :count).by(1) + end + + context 'created Mdm::Vuln' do + let(:mdm_session) do + Mdm::Session.last + end + + let(:rport) do + nil + end + + before(:each) do + Timecop.freeze + + session.exploit_datastore['RPORT'] = rport + + report_session + end + + after(:each) do + Timecop.return + end + + subject(:vuln) do + Mdm::Vuln.last + end + + it { expect(subject.host).to eq(Mdm::Host.last) } + it { expect(subject.refs).to eq([]) } + it { expect(subject.exploited_at).to be_within(1.second).of(Time.now.utc) } + + context "with session.via_exploit 'exploit/multi/handler'" do + context "with session.exploit_datastore['ParentModule']" do + it { expect(subject.info).to eq("Exploited by #{parent_module_fullname} to create Session #{mdm_session.id}") } + it { expect(subject.name).to eq(parent_module_name) } + end + end + + context "without session.via_exploit 'exploit/multi/handler'" do + let(:reference_name) do + 'windows/smb/ms08_067_netapi' + end + + before(:each) do + path = File.join( + parent_path, + 'exploits', + "#{reference_name}.rb" + ) + type = 'exploit' + + # fake cache data for ParentModule so it can be loaded + framework.modules.send( + :module_info_by_path=, + { + path => + { + :parent_path => parent_path, + :reference_name => reference_name, + :type => type, + } + } + ) + + session.via_exploit = "#{type}/#{reference_name}" + end + + it { expect(subject.info).to eq("Exploited by #{session.via_exploit} to create Session #{mdm_session.id}") } + it { expect(subject.name).to eq(reference_name) } + end + + context 'with RPORT' do + let(:rport) do + # use service.port instead of having service use rport so + # that service is forced to exist before call to + # report_service, which happens right after using rport in + # outer context's before(:each) + service.port + end + + let(:service) do + FactoryGirl.create( + :mdm_service, + :host => host + ) + end + + it { expect(subject.service).to eq(service) } + end + + context 'without RPORT' do + it { expect(subject.service).to be_nil } + end + end + + context 'created Mdm::ExploitAttempt' do + let(:rport) do + nil + end + + before(:each) do + Timecop.freeze + + session.exploit_datastore['RPORT'] = rport + + report_session + end + + after(:each) do + Timecop.return + end + + subject(:exploit_attempt) do + Mdm::ExploitAttempt.last + end + + it { expect(subject.attempted_at).to be_within(1.second).of(Time.now.utc) } + # @todo https://www.pivotaltracker.com/story/show/48362615 + it { expect(subject.session_id).to eq(Mdm::Session.last.id) } + it { expect(subject.exploited).to be_truthy } + # @todo https://www.pivotaltracker.com/story/show/48362615 + it { expect(subject.vuln_id).to eq(Mdm::Vuln.last.id) } + + context "with session.via_exploit 'exploit/multi/handler'" do + context "with session.datastore['ParentModule']" do + it { expect(subject.module).to eq(parent_module_fullname) } + end + end + + context "without session.via_exploit 'exploit/multi/handler'" do + before(:each) do + session.via_exploit = parent_module_fullname + end + + it { expect(subject.module).to eq(session.via_exploit) } + end + end + end + + context 'returned Mdm::Session' do + before(:each) do + Timecop.freeze + end + + after(:each) do + Timecop.return + end + + subject(:mdm_session) do + report_session + end + + # + # Ensure session has attributes present so its on mdm_session are + # not just comparing nils. + # + + it 'should have session.info present' do + session.info.should be_present + end + + it 'should have session.sid present' do + session.sid.should be_present + end + + it 'should have session.platform present' do + session.platform.should be_present + end + + it 'should have session.type present' do + session.type.should be_present + end + + it 'should have session.via_exploit present' do + session.via_exploit.should be_present + end + + it 'should have session.via_payload present' do + session.via_exploit.should be_present + end + + it { expect(subject.datastore).to eq(session.exploit_datastore.to_h) } + it { expect(subject.desc).to eq(session.info) } + it { expect(subject.host_id).to eq(Mdm::Host.last.id) } + it { expect(subject.last_seen).to be_within(1.second).of(Time.now.utc) } + it { expect(subject.local_id).to eq(session.sid) } + it { expect(subject.opened_at).to be_within(1.second).of(Time.now.utc) } + it { expect(subject.platform).to eq(session.platform) } + it { expect(subject.routes).to eq([]) } + it { expect(subject.stype).to eq(session.type) } + it { expect(subject.via_payload).to eq(session.via_payload) } + + context "with session.via_exploit 'exploit/multi/handler'" do + it "should have session.via_exploit of 'exploit/multi/handler'" do + session.via_exploit.should == 'exploit/multi/handler' + end + + context "with session.exploit_datastore['ParentModule']" do + it "should have session.exploit_datastore['ParentModule']" do + session.exploit_datastore['ParentModule'].should_not be_nil + end + + it { expect(subject.via_exploit).to eq(parent_module_fullname) } + end + end + + context "without session.via_exploit 'exploit/multi/handler'" do + before(:each) do + reference_name = 'windows/smb/ms08_067_netapi' + path = File.join( + parent_path, + 'exploits', + "#{reference_name}.rb" + ) + type = 'exploit' + + # fake cache data for ParentModule so it can be loaded + framework.modules.send( + :module_info_by_path=, + { + path => + { + :parent_path => parent_path, + :reference_name => reference_name, + :type => type, + } + } + ) + + session.via_exploit = "#{type}/#{reference_name}" + end + + it "should not have session.via_exploit of 'exploit/multi/handler'" do + session.via_exploit.should_not == 'exploit/multi/handler' + end + + it { expect(subject.via_exploit).to eq(session.via_exploit) } + end + end + end + end + + context 'without Msf::Session' do + let(:session) do + double('Not a Msf::Session') + end + + it 'should raise ArgumentError' do + expect { + report_session + }.to raise_error(ArgumentError, "Invalid :session, expected Msf::Session") + end + end + end + + context 'without :session' do + context 'with :host' do + before(:each) do + options[:host] = host + end + + context 'with Mdm::Host' do + let(:host) do + FactoryGirl.create(:mdm_host) + end + + context 'created Mdm::Session' do + let(:closed_at) do + nil + end + + let(:close_reason) do + 'Closed because...' + end + + let(:description) do + 'Session Description' + end + + let(:exploit_full_name) do + 'exploit/windows/smb/ms08_067_netapi' + end + + let(:last_seen) do + nil + end + + let(:opened_at) do + Time.now.utc - 5.minutes + end + + let(:payload_full_name) do + 'payload/singles/windows/metsvc_reverse_tcp' + end + + let(:platform) do + 'Host Platform' + end + + let(:routes) do + nil + end + + let(:session_type) do + 'Session Type' + end + + before(:each) do + options[:closed_at] = closed_at + options[:close_reason] = close_reason + options[:desc] = description + options[:last_seen] = last_seen + options[:opened_at] = opened_at + options[:platform] = platform + options[:routes] = routes + options[:stype] = session_type + options[:via_payload] = payload_full_name + options[:via_exploit] = exploit_full_name + end + + subject(:mdm_session) do + report_session + end + + it { expect(subject.close_reason).to eq(close_reason) } + it { expect(subject.desc).to eq(description) } + it { expect(subject.host).to eq(host) } + it { expect(subject.platform).to eq(platform) } + it { expect(subject.stype).to eq(session_type) } + it { expect(subject.via_exploit).to eq(exploit_full_name) } + it { expect(subject.via_payload).to eq(payload_full_name) } + + context 'with :last_seen' do + let(:last_seen) do + opened_at + end + + it { expect(subject.last_seen).to eq(last_seen) } + end + + context 'with :closed_at' do + let(:closed_at) do + opened_at + 1.minute + end + + it { expect(subject.closed_at).to eq(closed_at) } + end + + context 'without :closed_at' do + it { expect(subject.closed_at).to be_nil } + end + + context 'without :last_seen' do + context 'with :closed_at' do + let(:closed_at) do + opened_at + 1.minute + end + + it { expect(subject.last_seen).to eq(closed_at) } + end + + context 'without :closed_at' do + it { expect(subject.last_seen).to be_nil } + end + end + + context 'with :routes' do + let(:routes) do + FactoryGirl.build_list( + :mdm_route, + 1, + :session => nil + ) + end + + it { expect(subject.routes).to eq(routes) } + end + + context 'without :routes' do + it { expect(subject.routes).to eq([]) } + end + end + end + + context 'without Mdm::Host' do + let(:host) do + '192.168.0.1' + end + + it 'should raise ArgumentError' do + expect { + report_session + }.to raise_error(ArgumentError, "Invalid :host, expected Host object") + end + end + end + + context 'without :host' do + it 'should raise ArgumentError' do + expect { + report_session + }.to raise_error(ArgumentError) + end + end + end + end + + context 'without active' do + let(:active) do + false + end + + it { should be_nil } + + it 'should not create a connection' do + ActiveRecord::Base.connection_pool.should_not_receive(:with_connection) + + report_session + end + end + end +end \ No newline at end of file From 1f86712d63cc890f9a9d7a5c19367966caa72def Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 11:00:36 -0500 Subject: [PATCH 061/159] Extract Msf::DBManager::HostTag MSP-11124 Extract `Mdm::HostTag` method. --- lib/msf/core/db_manager.rb | 38 ++--------------------------- lib/msf/core/db_manager/host_tag.rb | 36 +++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 36 deletions(-) create mode 100644 lib/msf/core/db_manager/host_tag.rb diff --git a/lib/msf/core/db_manager.rb b/lib/msf/core/db_manager.rb index 06378eb427..e61f37d28e 100644 --- a/lib/msf/core/db_manager.rb +++ b/lib/msf/core/db_manager.rb @@ -82,6 +82,7 @@ class DBManager autoload :ExploitedHost, 'msf/core/db_manager/exploited_host' autoload :Host, 'msf/core/db_manager/host' autoload :HostDetail, 'msf/core/db_manager/host_detail' + autoload :HostTag, 'msf/core/db_manager/host_tag' autoload :Import, 'msf/core/db_manager/import' autoload :IPAddress, 'msf/core/db_manager/ip_address' autoload :Loot, 'msf/core/db_manager/loot' @@ -107,6 +108,7 @@ class DBManager include Msf::DBManager::ExploitedHost include Msf::DBManager::Host include Msf::DBManager::HostDetail + include Msf::DBManager::HostTag include Msf::DBManager::Import include Msf::DBManager::ImportMsfXml include Msf::DBManager::IPAddress @@ -517,42 +519,6 @@ class DBManager } end - - # This is only exercised by MSF3 XML importing for now. Needs the wait - # conditions and return hash as well. - def report_host_tag(opts) - name = opts.delete(:name) - raise DBImportError.new("Missing required option :name") unless name - addr = opts.delete(:addr) - raise DBImportError.new("Missing required option :addr") unless addr - wspace = opts.delete(:wspace) - raise DBImportError.new("Missing required option :wspace") unless wspace - ::ActiveRecord::Base.connection_pool.with_connection { - if wspace.kind_of? String - wspace = find_workspace(wspace) - end - - host = nil - report_host(:workspace => wspace, :address => addr) - - - host = get_host(:workspace => wspace, :address => addr) - desc = opts.delete(:desc) - summary = opts.delete(:summary) - detail = opts.delete(:detail) - crit = opts.delete(:crit) - possible_tags = Mdm::Tag.includes(:hosts).where("hosts.workspace_id = ? and tags.name = ?", wspace.id, name).order("tags.id DESC").limit(1) - tag = (possible_tags.blank? ? Mdm::Tag.new : possible_tags.first) - tag.name = name - tag.desc = desc - tag.report_summary = !!summary - tag.report_detail = !!detail - tag.critical = !!crit - tag.hosts = tag.hosts | [host] - tag.save! if tag.changed? - } - end - # # WMAP # Support methods diff --git a/lib/msf/core/db_manager/host_tag.rb b/lib/msf/core/db_manager/host_tag.rb new file mode 100644 index 0000000000..d20bd4fd05 --- /dev/null +++ b/lib/msf/core/db_manager/host_tag.rb @@ -0,0 +1,36 @@ +module Msf::DBManager::HostTag + # This is only exercised by MSF3 XML importing for now. Needs the wait + # conditions and return hash as well. + def report_host_tag(opts) + name = opts.delete(:name) + raise DBImportError.new("Missing required option :name") unless name + addr = opts.delete(:addr) + raise DBImportError.new("Missing required option :addr") unless addr + wspace = opts.delete(:wspace) + raise DBImportError.new("Missing required option :wspace") unless wspace + ::ActiveRecord::Base.connection_pool.with_connection { + if wspace.kind_of? String + wspace = find_workspace(wspace) + end + + host = nil + report_host(:workspace => wspace, :address => addr) + + + host = get_host(:workspace => wspace, :address => addr) + desc = opts.delete(:desc) + summary = opts.delete(:summary) + detail = opts.delete(:detail) + crit = opts.delete(:crit) + possible_tags = Mdm::Tag.includes(:hosts).where("hosts.workspace_id = ? and tags.name = ?", wspace.id, name).order("tags.id DESC").limit(1) + tag = (possible_tags.blank? ? Mdm::Tag.new : possible_tags.first) + tag.name = name + tag.desc = desc + tag.report_summary = !!summary + tag.report_detail = !!detail + tag.critical = !!crit + tag.hosts = tag.hosts | [host] + tag.save! if tag.changed? + } + end +end \ No newline at end of file From 98142def086fee1e4b12834d740de9f8b8f9a577 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 11:03:21 -0500 Subject: [PATCH 062/159] Extract Msf::DBManager::HostTag shared examples MSP-11124 --- spec/lib/msf/db_manager_spec.rb | 2 +- spec/support/shared/examples/msf/db_manager/host_tag.rb | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 spec/support/shared/examples/msf/db_manager/host_tag.rb diff --git a/spec/lib/msf/db_manager_spec.rb b/spec/lib/msf/db_manager_spec.rb index 9e86131a71..0fc38636b2 100644 --- a/spec/lib/msf/db_manager_spec.rb +++ b/spec/lib/msf/db_manager_spec.rb @@ -25,6 +25,7 @@ describe Msf::DBManager do it_should_behave_like 'Msf::DBManager::ExploitedHost' it_should_behave_like 'Msf::DBManager::Host' it_should_behave_like 'Msf::DBManager::HostDetail' + it_should_behave_like 'Msf::DBManager::HostTag' it_should_behave_like 'Msf::DBManager::IPAddress' it_should_behave_like 'Msf::DBManager::Import' it_should_behave_like 'Msf::DBManager::ImportMsfXml' @@ -85,7 +86,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :error } it { is_expected.to respond_to :initialize_adapter } it { is_expected.to respond_to :initialize_database_support } - it { is_expected.to respond_to :report_host_tag } it { is_expected.to respond_to :report_session_event } it { is_expected.to respond_to :report_session_route } it { is_expected.to respond_to :report_session_route_remove } diff --git a/spec/support/shared/examples/msf/db_manager/host_tag.rb b/spec/support/shared/examples/msf/db_manager/host_tag.rb new file mode 100644 index 0000000000..547673cfb0 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/host_tag.rb @@ -0,0 +1,3 @@ +shared_examples_for 'Msf::DBManager::HostTag' do + it { is_expected.to respond_to :report_host_tag } +end \ No newline at end of file From 8011187aa9a4fe0f74f4d29e331706ed1b10af86 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 11:09:27 -0500 Subject: [PATCH 063/159] Extract Msf::DBManager::SessionEvent MSP-11124 Extract `Mdm::SessionEvent` methods. --- lib/msf/core/db_manager.rb | 50 +----------------------- lib/msf/core/db_manager/session_event.rb | 49 +++++++++++++++++++++++ 2 files changed, 51 insertions(+), 48 deletions(-) create mode 100644 lib/msf/core/db_manager/session_event.rb diff --git a/lib/msf/core/db_manager.rb b/lib/msf/core/db_manager.rb index e61f37d28e..d5220916a9 100644 --- a/lib/msf/core/db_manager.rb +++ b/lib/msf/core/db_manager.rb @@ -92,6 +92,7 @@ class DBManager autoload :Report, 'msf/core/db_manager/report' autoload :Service, 'msf/core/db_manager/service' autoload :Session, 'msf/core/db_manager/session' + autoload :SessionEvent, 'msf/core/db_manager/session_event' autoload :Sink, 'msf/core/db_manager/sink' autoload :Task, 'msf/core/db_manager/task' autoload :Vuln, 'msf/core/db_manager/vuln' @@ -120,6 +121,7 @@ class DBManager include Msf::DBManager::Report include Msf::DBManager::Service include Msf::DBManager::Session + include Msf::DBManager::SessionEvent include Msf::DBManager::Sink include Msf::DBManager::Task include Msf::DBManager::Vuln @@ -416,54 +418,6 @@ class DBManager } end - # - # Record a session event in the database - # - # opts MUST contain one of: - # +:session+:: the Msf::Session OR the ::Mdm::Session we are reporting - # +:etype+:: event type, enum: command, output, upload, download, filedelete - # - # opts may contain - # +:output+:: the data for an output event - # +:command+:: the data for an command event - # +:remote_path+:: path to the associated file for upload, download, and filedelete events - # +:local_path+:: path to the associated file for upload, and download - # - def report_session_event(opts) - return if not active - raise ArgumentError.new("Missing required option :session") if opts[:session].nil? - raise ArgumentError.new("Expected an :etype") unless opts[:etype] - session = nil - - ::ActiveRecord::Base.connection_pool.with_connection { - if opts[:session].respond_to? :db_record - session = opts[:session].db_record - if session.nil? - # The session doesn't have a db_record which means - # a) the database wasn't connected at session registration time - # or - # b) something awful happened and the report_session call failed - # - # Either way, we can't do anything with this session as is, so - # log a warning and punt. - wlog("Warning: trying to report a session_event for a session with no db_record (#{opts[:session].sid})") - return - end - event_data = { :created_at => Time.now } - else - session = opts[:session] - event_data = { :created_at => opts[:created_at] } - end - - event_data[:session_id] = session.id - [:remote_path, :local_path, :output, :command, :etype].each do |attr| - event_data[attr] = opts[attr] if opts[attr] - end - - s = ::Mdm::SessionEvent.create(event_data) - } - end - def report_session_route(session, route) return if not active if session.respond_to? :db_record diff --git a/lib/msf/core/db_manager/session_event.rb b/lib/msf/core/db_manager/session_event.rb new file mode 100644 index 0000000000..6b53d3348c --- /dev/null +++ b/lib/msf/core/db_manager/session_event.rb @@ -0,0 +1,49 @@ +module Msf::DBManager::SessionEvent + # + # Record a session event in the database + # + # opts MUST contain one of: + # +:session+:: the Msf::Session OR the ::Mdm::Session we are reporting + # +:etype+:: event type, enum: command, output, upload, download, filedelete + # + # opts may contain + # +:output+:: the data for an output event + # +:command+:: the data for an command event + # +:remote_path+:: path to the associated file for upload, download, and filedelete events + # +:local_path+:: path to the associated file for upload, and download + # + def report_session_event(opts) + return if not active + raise ArgumentError.new("Missing required option :session") if opts[:session].nil? + raise ArgumentError.new("Expected an :etype") unless opts[:etype] + session = nil + + ::ActiveRecord::Base.connection_pool.with_connection { + if opts[:session].respond_to? :db_record + session = opts[:session].db_record + if session.nil? + # The session doesn't have a db_record which means + # a) the database wasn't connected at session registration time + # or + # b) something awful happened and the report_session call failed + # + # Either way, we can't do anything with this session as is, so + # log a warning and punt. + wlog("Warning: trying to report a session_event for a session with no db_record (#{opts[:session].sid})") + return + end + event_data = { :created_at => Time.now } + else + session = opts[:session] + event_data = { :created_at => opts[:created_at] } + end + + event_data[:session_id] = session.id + [:remote_path, :local_path, :output, :command, :etype].each do |attr| + event_data[attr] = opts[attr] if opts[attr] + end + + s = ::Mdm::SessionEvent.create(event_data) + } + end +end \ No newline at end of file From e1ae403292cb78f66f221ae61a5af9da29d79d99 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 11:15:07 -0500 Subject: [PATCH 064/159] Extract Msf::DBManager::SessionEvent shared examples MSP-11124 --- spec/lib/msf/db_manager_spec.rb | 2 +- spec/support/shared/examples/msf/db_manager/session_event.rb | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 spec/support/shared/examples/msf/db_manager/session_event.rb diff --git a/spec/lib/msf/db_manager_spec.rb b/spec/lib/msf/db_manager_spec.rb index 0fc38636b2..29669fb829 100644 --- a/spec/lib/msf/db_manager_spec.rb +++ b/spec/lib/msf/db_manager_spec.rb @@ -37,6 +37,7 @@ describe Msf::DBManager do it_should_behave_like 'Msf::DBManager::Report' it_should_behave_like 'Msf::DBManager::Service' it_should_behave_like 'Msf::DBManager::Session' + it_should_behave_like 'Msf::DBManager::SessionEvent' it_should_behave_like 'Msf::DBManager::Sink' it_should_behave_like 'Msf::DBManager::Task' it_should_behave_like 'Msf::DBManager::Vuln' @@ -86,7 +87,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :error } it { is_expected.to respond_to :initialize_adapter } it { is_expected.to respond_to :initialize_database_support } - it { is_expected.to respond_to :report_session_event } it { is_expected.to respond_to :report_session_route } it { is_expected.to respond_to :report_session_route_remove } it { is_expected.to respond_to :report_vuln_attempt } diff --git a/spec/support/shared/examples/msf/db_manager/session_event.rb b/spec/support/shared/examples/msf/db_manager/session_event.rb new file mode 100644 index 0000000000..51c244670b --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/session_event.rb @@ -0,0 +1,3 @@ +shared_examples_for 'Msf::DBManager::SessionEvent' do + it { is_expected.to respond_to :report_session_event } +end \ No newline at end of file From e7e12ec6a5f89069eb83a4d0c5227a463fabb9ca Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 11:23:37 -0500 Subject: [PATCH 065/159] Extract Msf::DBManager::Route MSP-11124 Extract `Mdm::Route` methods. --- lib/msf/core/db_manager.rb | 38 ++------------------------------ lib/msf/core/db_manager/route.rb | 38 ++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 36 deletions(-) create mode 100644 lib/msf/core/db_manager/route.rb diff --git a/lib/msf/core/db_manager.rb b/lib/msf/core/db_manager.rb index d5220916a9..489da8a101 100644 --- a/lib/msf/core/db_manager.rb +++ b/lib/msf/core/db_manager.rb @@ -90,6 +90,7 @@ class DBManager autoload :Note, 'msf/core/db_manager/note' autoload :Ref, 'msf/core/db_manager/ref' autoload :Report, 'msf/core/db_manager/report' + autoload :Route, 'msf/core/db_manager/route' autoload :Service, 'msf/core/db_manager/service' autoload :Session, 'msf/core/db_manager/session' autoload :SessionEvent, 'msf/core/db_manager/session_event' @@ -119,6 +120,7 @@ class DBManager include Msf::DBManager::Note include Msf::DBManager::Ref include Msf::DBManager::Report + include Msf::DBManager::Route include Msf::DBManager::Service include Msf::DBManager::Session include Msf::DBManager::SessionEvent @@ -418,42 +420,6 @@ class DBManager } end - def report_session_route(session, route) - return if not active - if session.respond_to? :db_record - s = session.db_record - else - s = session - end - unless s.respond_to?(:routes) - raise ArgumentError.new("Invalid :session, expected Session object got #{session.class}") - end - - ::ActiveRecord::Base.connection_pool.with_connection { - - subnet, netmask = route.split("/") - s.routes.create(:subnet => subnet, :netmask => netmask) - } - end - - def report_session_route_remove(session, route) - return if not active - if session.respond_to? :db_record - s = session.db_record - else - s = session - end - unless s.respond_to?(:routes) - raise ArgumentError.new("Invalid :session, expected Session object got #{session.class}") - end - - ::ActiveRecord::Base.connection_pool.with_connection { - subnet, netmask = route.split("/") - r = s.routes.find_by_subnet_and_netmask(subnet, netmask) - r.destroy if r - } - end - def report_vuln_attempt(vuln, opts) ::ActiveRecord::Base.connection_pool.with_connection { return if not vuln diff --git a/lib/msf/core/db_manager/route.rb b/lib/msf/core/db_manager/route.rb new file mode 100644 index 0000000000..3568ce12f8 --- /dev/null +++ b/lib/msf/core/db_manager/route.rb @@ -0,0 +1,38 @@ +module Msf::DBManager::Route + def report_session_route(session, route) + return if not active + if session.respond_to? :db_record + s = session.db_record + else + s = session + end + unless s.respond_to?(:routes) + raise ArgumentError.new("Invalid :session, expected Session object got #{session.class}") + end + + ::ActiveRecord::Base.connection_pool.with_connection { + + subnet, netmask = route.split("/") + s.routes.create(:subnet => subnet, :netmask => netmask) + } + end + + def report_session_route_remove(session, route) + return if not active + if session.respond_to? :db_record + s = session.db_record + else + s = session + end + unless s.respond_to?(:routes) + raise ArgumentError.new("Invalid :session, expected Session object got #{session.class}") + end + + ::ActiveRecord::Base.connection_pool.with_connection { + subnet, netmask = route.split("/") + r = s.routes.find_by_subnet_and_netmask(subnet, netmask) + r.destroy if r + } + end + +end \ No newline at end of file From adfeef2aa9fa322aa42d5711de349631429e790d Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 11:28:19 -0500 Subject: [PATCH 066/159] Extract Msf::DBManager::Route shared examples MSP-11124 --- spec/lib/msf/db_manager_spec.rb | 3 +-- spec/support/shared/examples/msf/db_manager/route.rb | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/route.rb diff --git a/spec/lib/msf/db_manager_spec.rb b/spec/lib/msf/db_manager_spec.rb index 29669fb829..a905c1f94b 100644 --- a/spec/lib/msf/db_manager_spec.rb +++ b/spec/lib/msf/db_manager_spec.rb @@ -35,6 +35,7 @@ describe Msf::DBManager do it_should_behave_like 'Msf::DBManager::Note' it_should_behave_like 'Msf::DBManager::Ref' it_should_behave_like 'Msf::DBManager::Report' + it_should_behave_like 'Msf::DBManager::Route' it_should_behave_like 'Msf::DBManager::Service' it_should_behave_like 'Msf::DBManager::Session' it_should_behave_like 'Msf::DBManager::SessionEvent' @@ -87,8 +88,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :error } it { is_expected.to respond_to :initialize_adapter } it { is_expected.to respond_to :initialize_database_support } - it { is_expected.to respond_to :report_session_route } - it { is_expected.to respond_to :report_session_route_remove } it { is_expected.to respond_to :report_vuln_attempt } it { is_expected.to respond_to :report_web_form } it { is_expected.to respond_to :report_web_page } diff --git a/spec/support/shared/examples/msf/db_manager/route.rb b/spec/support/shared/examples/msf/db_manager/route.rb new file mode 100644 index 0000000000..e3d1212a2e --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/route.rb @@ -0,0 +1,4 @@ +shared_examples_for 'Msf::DBManager::Route' do + it { is_expected.to respond_to :report_session_route } + it { is_expected.to respond_to :report_session_route_remove } +end \ No newline at end of file From 5067e43ac1658881b9e02938ed16e37b8a06b3ae Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 11:32:15 -0500 Subject: [PATCH 067/159] Extract Msf::DBManager::VulnAttempt MSP-11124 Extract `Mdm::VulnAttempt` methods. --- lib/msf/core/db_manager.rb | 21 ++------------------- lib/msf/core/db_manager/vuln_attempt.rb | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+), 19 deletions(-) create mode 100644 lib/msf/core/db_manager/vuln_attempt.rb diff --git a/lib/msf/core/db_manager.rb b/lib/msf/core/db_manager.rb index 489da8a101..58404cbb50 100644 --- a/lib/msf/core/db_manager.rb +++ b/lib/msf/core/db_manager.rb @@ -97,6 +97,7 @@ class DBManager autoload :Sink, 'msf/core/db_manager/sink' autoload :Task, 'msf/core/db_manager/task' autoload :Vuln, 'msf/core/db_manager/vuln' + autoload :VulnAttempt, 'msf/core/db_manager/vuln_attempt' autoload :VulnDetail, 'msf/core/db_manager/vuln_detail' autoload :WMAP, 'msf/core/db_manager/wmap' autoload :Workspace, 'msf/core/db_manager/workspace' @@ -127,6 +128,7 @@ class DBManager include Msf::DBManager::Sink include Msf::DBManager::Task include Msf::DBManager::Vuln + include Msf::DBManager::VulnAttempt include Msf::DBManager::VulnDetail include Msf::DBManager::WMAP include Msf::DBManager::Workspace @@ -420,25 +422,6 @@ class DBManager } end - def report_vuln_attempt(vuln, opts) - ::ActiveRecord::Base.connection_pool.with_connection { - return if not vuln - info = {} - - # Opts can be keyed by strings or symbols - ::Mdm::VulnAttempt.column_names.each do |kn| - k = kn.to_sym - next if ['id', 'vuln_id'].include?(kn) - info[k] = opts[kn] if opts[kn] - info[k] = opts[k] if opts[k] - end - - return unless info[:attempted_at] - - vuln.vuln_attempts.create(info) - } - end - # # WMAP # Support methods diff --git a/lib/msf/core/db_manager/vuln_attempt.rb b/lib/msf/core/db_manager/vuln_attempt.rb new file mode 100644 index 0000000000..2533ba4abf --- /dev/null +++ b/lib/msf/core/db_manager/vuln_attempt.rb @@ -0,0 +1,20 @@ +module Msf::DBManager::VulnAttempt + def report_vuln_attempt(vuln, opts) + ::ActiveRecord::Base.connection_pool.with_connection { + return if not vuln + info = {} + + # Opts can be keyed by strings or symbols + ::Mdm::VulnAttempt.column_names.each do |kn| + k = kn.to_sym + next if ['id', 'vuln_id'].include?(kn) + info[k] = opts[kn] if opts[kn] + info[k] = opts[k] if opts[k] + end + + return unless info[:attempted_at] + + vuln.vuln_attempts.create(info) + } + end +end \ No newline at end of file From 3a2f8b76843e76af90007f00798a4a6cc8f42f9a Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 11:34:12 -0500 Subject: [PATCH 068/159] Extract Msf::DBManager::VulnAttempt shared examples MSP-11124 --- spec/lib/msf/db_manager_spec.rb | 2 +- spec/support/shared/examples/msf/db_manager/vuln_attempt.rb | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 spec/support/shared/examples/msf/db_manager/vuln_attempt.rb diff --git a/spec/lib/msf/db_manager_spec.rb b/spec/lib/msf/db_manager_spec.rb index a905c1f94b..39f6e2ea1c 100644 --- a/spec/lib/msf/db_manager_spec.rb +++ b/spec/lib/msf/db_manager_spec.rb @@ -42,6 +42,7 @@ describe Msf::DBManager do it_should_behave_like 'Msf::DBManager::Sink' it_should_behave_like 'Msf::DBManager::Task' it_should_behave_like 'Msf::DBManager::Vuln' + it_should_behave_like 'Msf::DBManager::VulnAttempt' it_should_behave_like 'Msf::DBManager::VulnDetail' it_should_behave_like 'Msf::DBManager::WMAP' it_should_behave_like 'Msf::DBManager::Workspace' @@ -88,7 +89,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :error } it { is_expected.to respond_to :initialize_adapter } it { is_expected.to respond_to :initialize_database_support } - it { is_expected.to respond_to :report_vuln_attempt } it { is_expected.to respond_to :report_web_form } it { is_expected.to respond_to :report_web_page } it { is_expected.to respond_to :report_web_site } diff --git a/spec/support/shared/examples/msf/db_manager/vuln_attempt.rb b/spec/support/shared/examples/msf/db_manager/vuln_attempt.rb new file mode 100644 index 0000000000..37b3ec9871 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/vuln_attempt.rb @@ -0,0 +1,3 @@ +shared_examples_for 'Msf::DBManager::VulnAttempt' do + it { is_expected.to respond_to :report_vuln_attempt } +end \ No newline at end of file From f472411c8caf2b97f2a682de49eb526233802678 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 11:39:17 -0500 Subject: [PATCH 069/159] Extract Msf::DBManager::Web MSP-11124 Extract `Mdm::Web*` methods. --- lib/msf/core/db_manager.rb | 368 +-------------------------------- lib/msf/core/db_manager/web.rb | 363 ++++++++++++++++++++++++++++++++ 2 files changed, 365 insertions(+), 366 deletions(-) create mode 100644 lib/msf/core/db_manager/web.rb diff --git a/lib/msf/core/db_manager.rb b/lib/msf/core/db_manager.rb index 58404cbb50..6c4a1d9729 100644 --- a/lib/msf/core/db_manager.rb +++ b/lib/msf/core/db_manager.rb @@ -100,6 +100,7 @@ class DBManager autoload :VulnAttempt, 'msf/core/db_manager/vuln_attempt' autoload :VulnDetail, 'msf/core/db_manager/vuln_detail' autoload :WMAP, 'msf/core/db_manager/wmap' + autoload :Web, 'msf/core/db_manager/web' autoload :Workspace, 'msf/core/db_manager/workspace' optionally_include_metasploit_credential_creation @@ -131,6 +132,7 @@ class DBManager include Msf::DBManager::VulnAttempt include Msf::DBManager::VulnDetail include Msf::DBManager::WMAP + include Msf::DBManager::Web include Msf::DBManager::Workspace # Provides :framework and other accessors @@ -422,371 +424,5 @@ class DBManager } end - # - # WMAP - # Support methods - # - - # - # Report a Web Site to the database. WebSites must be tied to an existing Service - # - # opts MUST contain - # +:service+:: the service object this site should be associated with - # +:vhost+:: the virtual host name for this particular web site` - # - # If +:service+ is NOT specified, the following values are mandatory - # +:host+:: the ip address of the server hosting the web site - # +:port+:: the port number of the associated web site - # +:ssl+:: whether or not SSL is in use on this port - # - # These values will be used to create new host and service records - # - # opts can contain - # +:options+:: a hash of options for accessing this particular web site - # +:info+:: if present, report the service with this info - # - # Duplicate records for a given host, port, vhost combination will be overwritten - # - def report_web_site(opts) - return if not active - ::ActiveRecord::Base.connection_pool.with_connection { |conn| - wspace = opts.delete(:workspace) || workspace - vhost = opts.delete(:vhost) - - addr = nil - port = nil - name = nil - serv = nil - info = nil - - if opts[:service] and opts[:service].kind_of?(::Mdm::Service) - serv = opts[:service] - else - addr = opts[:host] - port = opts[:port] - name = opts[:ssl] ? 'https' : 'http' - info = opts[:info] - if not (addr and port) - raise ArgumentError, "report_web_site requires service OR host/port/ssl" - end - - # Force addr to be the address and not hostname - addr = Rex::Socket.getaddress(addr, true) - end - - ret = {} - - host = serv ? serv.host : find_or_create_host( - :workspace => wspace, - :host => addr, - :state => Msf::HostState::Alive - ) - - if host.name.to_s.empty? - host.name = vhost - host.save! - end - - serv = serv ? serv : find_or_create_service( - :workspace => wspace, - :host => host, - :port => port, - :proto => 'tcp', - :state => 'open' - ) - - # Change the service name if it is blank or it has - # been explicitly specified. - if opts.keys.include?(:ssl) or serv.name.to_s.empty? - name = opts[:ssl] ? 'https' : 'http' - serv.name = name - end - # Add the info if it's there. - unless info.to_s.empty? - serv.info = info - end - serv.save! if serv.changed? -=begin - host.updated_at = host.created_at - host.state = HostState::Alive - host.save! -=end - - vhost ||= host.address - site = ::Mdm::WebSite.find_or_initialize_by_vhost_and_service_id(vhost, serv[:id]) - site.options = opts[:options] if opts[:options] - - # XXX: - msf_import_timestamps(opts, site) - site.save! - - ret[:web_site] = site - } - end - - # - # Report a Web Page to the database. WebPage must be tied to an existing Web Site - # - # opts MUST contain - # +:web_site+:: the web site object that this page should be associated with - # +:path+:: the virtual host name for this particular web site - # +:code+:: the http status code from requesting this page - # +:headers+:: this is a HASH of headers (lowercase name as key) of ARRAYs of values - # +:body+:: the document body of the server response - # +:query+:: the query string after the path - # - # If web_site is NOT specified, the following values are mandatory - # +:host+:: the ip address of the server hosting the web site - # +:port+:: the port number of the associated web site - # +:vhost+:: the virtual host for this particular web site - # +:ssl+:: whether or not SSL is in use on this port - # - # These values will be used to create new host, service, and web_site records - # - # opts can contain - # +:cookie+:: the Set-Cookie headers, merged into a string - # +:auth+:: the Authorization headers, merged into a string - # +:ctype+:: the Content-Type headers, merged into a string - # +:mtime+:: the timestamp returned from the server of the last modification time - # +:location+:: the URL that a redirect points to - # - # Duplicate records for a given web_site, path, and query combination will be overwritten - # - - def report_web_page(opts) - return if not active - ::ActiveRecord::Base.connection_pool.with_connection { - wspace = opts.delete(:workspace) || workspace - - path = opts[:path] - code = opts[:code].to_i - body = opts[:body].to_s - query = opts[:query].to_s - headers = opts[:headers] - site = nil - - if not (path and code and body and headers) - raise ArgumentError, "report_web_page requires the path, query, code, body, and headers parameters" - end - - if opts[:web_site] and opts[:web_site].kind_of?(::Mdm::WebSite) - site = opts.delete(:web_site) - else - site = report_web_site( - :workspace => wspace, - :host => opts[:host], :port => opts[:port], - :vhost => opts[:host], :ssl => opts[:ssl] - ) - if not site - raise ArgumentError, "report_web_page was unable to create the associated web site" - end - end - - ret = {} - - page = ::Mdm::WebPage.find_or_initialize_by_web_site_id_and_path_and_query(site[:id], path, query) - page.code = code - page.body = body - page.headers = headers - page.cookie = opts[:cookie] if opts[:cookie] - page.auth = opts[:auth] if opts[:auth] - page.mtime = opts[:mtime] if opts[:mtime] - page.ctype = opts[:ctype] if opts[:ctype] - page.location = opts[:location] if opts[:location] - msf_import_timestamps(opts, page) - page.save! - - ret[:web_page] = page - } - - end - - - # - # Report a Web Form to the database. WebForm must be tied to an existing Web Site - # - # opts MUST contain - # +:web_site+:: the web site object that this page should be associated with - # +:path+:: the virtual host name for this particular web site - # +:query+:: the query string that is appended to the path (not valid for GET) - # +:method+:: the form method, one of GET, POST, or PATH - # +:params+:: an ARRAY of all parameters and values specified in the form - # - # If web_site is NOT specified, the following values are mandatory - # +:host+:: the ip address of the server hosting the web site - # +:port+:: the port number of the associated web site - # +:vhost+:: the virtual host for this particular web site - # +:ssl+:: whether or not SSL is in use on this port - # - # Duplicate records for a given web_site, path, method, and params combination will be overwritten - # - - def report_web_form(opts) - return if not active - ::ActiveRecord::Base.connection_pool.with_connection { - wspace = opts.delete(:workspace) || workspace - - path = opts[:path] - meth = opts[:method].to_s.upcase - para = opts[:params] - quer = opts[:query].to_s - site = nil - - if not (path and meth) - raise ArgumentError, "report_web_form requires the path and method parameters" - end - - if not %W{GET POST PATH}.include?(meth) - raise ArgumentError, "report_web_form requires the method to be one of GET, POST, PATH" - end - - if opts[:web_site] and opts[:web_site].kind_of?(::Mdm::WebSite) - site = opts.delete(:web_site) - else - site = report_web_site( - :workspace => wspace, - :host => opts[:host], :port => opts[:port], - :vhost => opts[:host], :ssl => opts[:ssl] - ) - if not site - raise ArgumentError, "report_web_form was unable to create the associated web site" - end - end - - ret = {} - - # Since one of our serialized fields is used as a unique parameter, we must do the final - # comparisons through ruby and not SQL. - - form = nil - ::Mdm::WebForm.find_all_by_web_site_id_and_path_and_method_and_query(site[:id], path, meth, quer).each do |xform| - if xform.params == para - form = xform - break - end - end - if not form - form = ::Mdm::WebForm.new - form.web_site_id = site[:id] - form.path = path - form.method = meth - form.params = para - form.query = quer - end - - msf_import_timestamps(opts, form) - form.save! - ret[:web_form] = form - } - end - - - # - # Report a Web Vuln to the database. WebVuln must be tied to an existing Web Site - # - # opts MUST contain - # +:web_site+:: the web site object that this page should be associated with - # +:path+:: the virtual host name for this particular web site - # +:query+:: the query string appended to the path (not valid for GET method flaws) - # +:method+:: the form method, one of GET, POST, or PATH - # +:params+:: an ARRAY of all parameters and values specified in the form - # +:pname+:: the specific field where the vulnerability occurs - # +:proof+:: the string showing proof of the vulnerability - # +:risk+:: an INTEGER value from 0 to 5 indicating the risk (5 is highest) - # +:name+:: the string indicating the type of vulnerability - # - # If web_site is NOT specified, the following values are mandatory - # +:host+:: the ip address of the server hosting the web site - # +:port+:: the port number of the associated web site - # +:vhost+:: the virtual host for this particular web site - # +:ssl+:: whether or not SSL is in use on this port - # - # - # Duplicate records for a given web_site, path, method, pname, and name - # combination will be overwritten - # - - def report_web_vuln(opts) - return if not active - ::ActiveRecord::Base.connection_pool.with_connection { - wspace = opts.delete(:workspace) || workspace - - path = opts[:path] - meth = opts[:method] - para = opts[:params] || [] - quer = opts[:query].to_s - pname = opts[:pname] - proof = opts[:proof] - risk = opts[:risk].to_i - name = opts[:name].to_s.strip - blame = opts[:blame].to_s.strip - desc = opts[:description].to_s.strip - conf = opts[:confidence].to_i - cat = opts[:category].to_s.strip - payload = opts[:payload].to_s - owner = opts[:owner] ? opts[:owner].shortname : nil - - - site = nil - - if not (path and meth and proof and pname) - raise ArgumentError, "report_web_vuln requires the path, method, proof, risk, name, params, and pname parameters. Received #{opts.inspect}" - end - - if not %W{GET POST PATH}.include?(meth) - raise ArgumentError, "report_web_vuln requires the method to be one of GET, POST, PATH. Received '#{meth}'" - end - - if risk < 0 or risk > 5 - raise ArgumentError, "report_web_vuln requires the risk to be between 0 and 5 (inclusive). Received '#{risk}'" - end - - if conf < 0 or conf > 100 - raise ArgumentError, "report_web_vuln requires the confidence to be between 1 and 100 (inclusive). Received '#{conf}'" - end - - if cat.empty? - raise ArgumentError, "report_web_vuln requires the category to be a valid string" - end - - if name.empty? - raise ArgumentError, "report_web_vuln requires the name to be a valid string" - end - - if opts[:web_site] and opts[:web_site].kind_of?(::Mdm::WebSite) - site = opts.delete(:web_site) - else - site = report_web_site( - :workspace => wspace, - :host => opts[:host], :port => opts[:port], - :vhost => opts[:host], :ssl => opts[:ssl] - ) - if not site - raise ArgumentError, "report_web_form was unable to create the associated web site" - end - end - - ret = {} - - meth = meth.to_s.upcase - - vuln = ::Mdm::WebVuln.find_or_initialize_by_web_site_id_and_path_and_method_and_pname_and_name_and_category_and_query(site[:id], path, meth, pname, name, cat, quer) - vuln.name = name - vuln.risk = risk - vuln.params = para - vuln.proof = proof.to_s - vuln.category = cat - vuln.blame = blame - vuln.description = desc - vuln.confidence = conf - vuln.payload = payload - vuln.owner = owner - - msf_import_timestamps(opts, vuln) - vuln.save! - - ret[:web_vuln] = vuln - } - end end end diff --git a/lib/msf/core/db_manager/web.rb b/lib/msf/core/db_manager/web.rb new file mode 100644 index 0000000000..eeec3bc132 --- /dev/null +++ b/lib/msf/core/db_manager/web.rb @@ -0,0 +1,363 @@ +module Msf::DBManager::Web + # + # Report a Web Form to the database. WebForm must be tied to an existing Web Site + # + # opts MUST contain + # +:web_site+:: the web site object that this page should be associated with + # +:path+:: the virtual host name for this particular web site + # +:query+:: the query string that is appended to the path (not valid for GET) + # +:method+:: the form method, one of GET, POST, or PATH + # +:params+:: an ARRAY of all parameters and values specified in the form + # + # If web_site is NOT specified, the following values are mandatory + # +:host+:: the ip address of the server hosting the web site + # +:port+:: the port number of the associated web site + # +:vhost+:: the virtual host for this particular web site + # +:ssl+:: whether or not SSL is in use on this port + # + # Duplicate records for a given web_site, path, method, and params combination will be overwritten + # + def report_web_form(opts) + return if not active + ::ActiveRecord::Base.connection_pool.with_connection { + wspace = opts.delete(:workspace) || workspace + + path = opts[:path] + meth = opts[:method].to_s.upcase + para = opts[:params] + quer = opts[:query].to_s + site = nil + + if not (path and meth) + raise ArgumentError, "report_web_form requires the path and method parameters" + end + + if not %W{GET POST PATH}.include?(meth) + raise ArgumentError, "report_web_form requires the method to be one of GET, POST, PATH" + end + + if opts[:web_site] and opts[:web_site].kind_of?(::Mdm::WebSite) + site = opts.delete(:web_site) + else + site = report_web_site( + :workspace => wspace, + :host => opts[:host], :port => opts[:port], + :vhost => opts[:host], :ssl => opts[:ssl] + ) + if not site + raise ArgumentError, "report_web_form was unable to create the associated web site" + end + end + + ret = {} + + # Since one of our serialized fields is used as a unique parameter, we must do the final + # comparisons through ruby and not SQL. + + form = nil + ::Mdm::WebForm.find_all_by_web_site_id_and_path_and_method_and_query(site[:id], path, meth, quer).each do |xform| + if xform.params == para + form = xform + break + end + end + if not form + form = ::Mdm::WebForm.new + form.web_site_id = site[:id] + form.path = path + form.method = meth + form.params = para + form.query = quer + end + + msf_import_timestamps(opts, form) + form.save! + ret[:web_form] = form + } + end + + # + # Report a Web Page to the database. WebPage must be tied to an existing Web Site + # + # opts MUST contain + # +:web_site+:: the web site object that this page should be associated with + # +:path+:: the virtual host name for this particular web site + # +:code+:: the http status code from requesting this page + # +:headers+:: this is a HASH of headers (lowercase name as key) of ARRAYs of values + # +:body+:: the document body of the server response + # +:query+:: the query string after the path + # + # If web_site is NOT specified, the following values are mandatory + # +:host+:: the ip address of the server hosting the web site + # +:port+:: the port number of the associated web site + # +:vhost+:: the virtual host for this particular web site + # +:ssl+:: whether or not SSL is in use on this port + # + # These values will be used to create new host, service, and web_site records + # + # opts can contain + # +:cookie+:: the Set-Cookie headers, merged into a string + # +:auth+:: the Authorization headers, merged into a string + # +:ctype+:: the Content-Type headers, merged into a string + # +:mtime+:: the timestamp returned from the server of the last modification time + # +:location+:: the URL that a redirect points to + # + # Duplicate records for a given web_site, path, and query combination will be overwritten + # + def report_web_page(opts) + return if not active + ::ActiveRecord::Base.connection_pool.with_connection { + wspace = opts.delete(:workspace) || workspace + + path = opts[:path] + code = opts[:code].to_i + body = opts[:body].to_s + query = opts[:query].to_s + headers = opts[:headers] + site = nil + + if not (path and code and body and headers) + raise ArgumentError, "report_web_page requires the path, query, code, body, and headers parameters" + end + + if opts[:web_site] and opts[:web_site].kind_of?(::Mdm::WebSite) + site = opts.delete(:web_site) + else + site = report_web_site( + :workspace => wspace, + :host => opts[:host], :port => opts[:port], + :vhost => opts[:host], :ssl => opts[:ssl] + ) + if not site + raise ArgumentError, "report_web_page was unable to create the associated web site" + end + end + + ret = {} + + page = ::Mdm::WebPage.find_or_initialize_by_web_site_id_and_path_and_query(site[:id], path, query) + page.code = code + page.body = body + page.headers = headers + page.cookie = opts[:cookie] if opts[:cookie] + page.auth = opts[:auth] if opts[:auth] + page.mtime = opts[:mtime] if opts[:mtime] + page.ctype = opts[:ctype] if opts[:ctype] + page.location = opts[:location] if opts[:location] + msf_import_timestamps(opts, page) + page.save! + + ret[:web_page] = page + } + + end + + # + # WMAP + # Support methods + # + + # + # Report a Web Site to the database. WebSites must be tied to an existing Service + # + # opts MUST contain + # +:service+:: the service object this site should be associated with + # +:vhost+:: the virtual host name for this particular web site` + # + # If +:service+ is NOT specified, the following values are mandatory + # +:host+:: the ip address of the server hosting the web site + # +:port+:: the port number of the associated web site + # +:ssl+:: whether or not SSL is in use on this port + # + # These values will be used to create new host and service records + # + # opts can contain + # +:options+:: a hash of options for accessing this particular web site + # +:info+:: if present, report the service with this info + # + # Duplicate records for a given host, port, vhost combination will be overwritten + # + def report_web_site(opts) + return if not active + ::ActiveRecord::Base.connection_pool.with_connection { |conn| + wspace = opts.delete(:workspace) || workspace + vhost = opts.delete(:vhost) + + addr = nil + port = nil + name = nil + serv = nil + info = nil + + if opts[:service] and opts[:service].kind_of?(::Mdm::Service) + serv = opts[:service] + else + addr = opts[:host] + port = opts[:port] + name = opts[:ssl] ? 'https' : 'http' + info = opts[:info] + if not (addr and port) + raise ArgumentError, "report_web_site requires service OR host/port/ssl" + end + + # Force addr to be the address and not hostname + addr = Rex::Socket.getaddress(addr, true) + end + + ret = {} + + host = serv ? serv.host : find_or_create_host( + :workspace => wspace, + :host => addr, + :state => Msf::HostState::Alive + ) + + if host.name.to_s.empty? + host.name = vhost + host.save! + end + + serv = serv ? serv : find_or_create_service( + :workspace => wspace, + :host => host, + :port => port, + :proto => 'tcp', + :state => 'open' + ) + + # Change the service name if it is blank or it has + # been explicitly specified. + if opts.keys.include?(:ssl) or serv.name.to_s.empty? + name = opts[:ssl] ? 'https' : 'http' + serv.name = name + end + # Add the info if it's there. + unless info.to_s.empty? + serv.info = info + end + serv.save! if serv.changed? +=begin + host.updated_at = host.created_at + host.state = HostState::Alive + host.save! +=end + + vhost ||= host.address + site = ::Mdm::WebSite.find_or_initialize_by_vhost_and_service_id(vhost, serv[:id]) + site.options = opts[:options] if opts[:options] + + # XXX: + msf_import_timestamps(opts, site) + site.save! + + ret[:web_site] = site + } + end + + # + # Report a Web Vuln to the database. WebVuln must be tied to an existing Web Site + # + # opts MUST contain + # +:web_site+:: the web site object that this page should be associated with + # +:path+:: the virtual host name for this particular web site + # +:query+:: the query string appended to the path (not valid for GET method flaws) + # +:method+:: the form method, one of GET, POST, or PATH + # +:params+:: an ARRAY of all parameters and values specified in the form + # +:pname+:: the specific field where the vulnerability occurs + # +:proof+:: the string showing proof of the vulnerability + # +:risk+:: an INTEGER value from 0 to 5 indicating the risk (5 is highest) + # +:name+:: the string indicating the type of vulnerability + # + # If web_site is NOT specified, the following values are mandatory + # +:host+:: the ip address of the server hosting the web site + # +:port+:: the port number of the associated web site + # +:vhost+:: the virtual host for this particular web site + # +:ssl+:: whether or not SSL is in use on this port + # + # + # Duplicate records for a given web_site, path, method, pname, and name + # combination will be overwritten + # + def report_web_vuln(opts) + return if not active + ::ActiveRecord::Base.connection_pool.with_connection { + wspace = opts.delete(:workspace) || workspace + + path = opts[:path] + meth = opts[:method] + para = opts[:params] || [] + quer = opts[:query].to_s + pname = opts[:pname] + proof = opts[:proof] + risk = opts[:risk].to_i + name = opts[:name].to_s.strip + blame = opts[:blame].to_s.strip + desc = opts[:description].to_s.strip + conf = opts[:confidence].to_i + cat = opts[:category].to_s.strip + payload = opts[:payload].to_s + owner = opts[:owner] ? opts[:owner].shortname : nil + + + site = nil + + if not (path and meth and proof and pname) + raise ArgumentError, "report_web_vuln requires the path, method, proof, risk, name, params, and pname parameters. Received #{opts.inspect}" + end + + if not %W{GET POST PATH}.include?(meth) + raise ArgumentError, "report_web_vuln requires the method to be one of GET, POST, PATH. Received '#{meth}'" + end + + if risk < 0 or risk > 5 + raise ArgumentError, "report_web_vuln requires the risk to be between 0 and 5 (inclusive). Received '#{risk}'" + end + + if conf < 0 or conf > 100 + raise ArgumentError, "report_web_vuln requires the confidence to be between 1 and 100 (inclusive). Received '#{conf}'" + end + + if cat.empty? + raise ArgumentError, "report_web_vuln requires the category to be a valid string" + end + + if name.empty? + raise ArgumentError, "report_web_vuln requires the name to be a valid string" + end + + if opts[:web_site] and opts[:web_site].kind_of?(::Mdm::WebSite) + site = opts.delete(:web_site) + else + site = report_web_site( + :workspace => wspace, + :host => opts[:host], :port => opts[:port], + :vhost => opts[:host], :ssl => opts[:ssl] + ) + if not site + raise ArgumentError, "report_web_form was unable to create the associated web site" + end + end + + ret = {} + + meth = meth.to_s.upcase + + vuln = ::Mdm::WebVuln.find_or_initialize_by_web_site_id_and_path_and_method_and_pname_and_name_and_category_and_query(site[:id], path, meth, pname, name, cat, quer) + vuln.name = name + vuln.risk = risk + vuln.params = para + vuln.proof = proof.to_s + vuln.category = cat + vuln.blame = blame + vuln.description = desc + vuln.confidence = conf + vuln.payload = payload + vuln.owner = owner + + msf_import_timestamps(opts, vuln) + vuln.save! + + ret[:web_vuln] = vuln + } + end +end \ No newline at end of file From 43a4858fe35fc1d66e016a7fd258c8b740826e8e Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 12:01:49 -0500 Subject: [PATCH 070/159] Extract Msf::DBManager::Web shared examples MSP-11124 --- spec/lib/msf/db_manager_spec.rb | 5 +---- spec/support/shared/examples/msf/db_manager/web.rb | 6 ++++++ 2 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/web.rb diff --git a/spec/lib/msf/db_manager_spec.rb b/spec/lib/msf/db_manager_spec.rb index 39f6e2ea1c..210afd046b 100644 --- a/spec/lib/msf/db_manager_spec.rb +++ b/spec/lib/msf/db_manager_spec.rb @@ -45,6 +45,7 @@ describe Msf::DBManager do it_should_behave_like 'Msf::DBManager::VulnAttempt' it_should_behave_like 'Msf::DBManager::VulnDetail' it_should_behave_like 'Msf::DBManager::WMAP' + it_should_behave_like 'Msf::DBManager::Web' it_should_behave_like 'Msf::DBManager::Workspace' context 'CONSTANTS' do @@ -89,10 +90,6 @@ describe Msf::DBManager do it { is_expected.to respond_to :error } it { is_expected.to respond_to :initialize_adapter } it { is_expected.to respond_to :initialize_database_support } - it { is_expected.to respond_to :report_web_form } - it { is_expected.to respond_to :report_web_page } - it { is_expected.to respond_to :report_web_site } - it { is_expected.to respond_to :report_web_vuln } it { is_expected.to respond_to :service_name_map } it { is_expected.to respond_to :usable } it { is_expected.to respond_to :usable= } diff --git a/spec/support/shared/examples/msf/db_manager/web.rb b/spec/support/shared/examples/msf/db_manager/web.rb new file mode 100644 index 0000000000..ba0335b115 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/web.rb @@ -0,0 +1,6 @@ +shared_examples_for 'Msf::DBManager::Web' do + it { is_expected.to respond_to :report_web_form } + it { is_expected.to respond_to :report_web_page } + it { is_expected.to respond_to :report_web_site } + it { is_expected.to respond_to :report_web_vuln } +end \ No newline at end of file From 930b020211ab92e45db64ea27a62b7064ea1eb4b Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 12:07:13 -0500 Subject: [PATCH 071/159] Extract Msf::DBManager::Connection MSP-11124 Extract methods that connect, disconnect and show status of connection to database. --- lib/msf/core/db_manager.rb | 150 +------------------------- lib/msf/core/db_manager/connection.rb | 149 +++++++++++++++++++++++++ 2 files changed, 151 insertions(+), 148 deletions(-) create mode 100644 lib/msf/core/db_manager/connection.rb diff --git a/lib/msf/core/db_manager.rb b/lib/msf/core/db_manager.rb index 6c4a1d9729..4c77f02c6c 100644 --- a/lib/msf/core/db_manager.rb +++ b/lib/msf/core/db_manager.rb @@ -76,6 +76,7 @@ class DBManager extend Metasploit::Framework::Require autoload :Client, 'msf/core/db_manager/client' + autoload :Connection, 'msf/core/db_manager/connection' autoload :Cred, 'msf/core/db_manager/cred' autoload :Event, 'msf/core/db_manager/event' autoload :ExploitAttempt, 'msf/core/db_manager/exploit_attempt' @@ -106,6 +107,7 @@ class DBManager optionally_include_metasploit_credential_creation include Msf::DBManager::Client + include Msf::DBManager::Connection include Msf::DBManager::Cred include Msf::DBManager::Event include Msf::DBManager::ExploitAttempt @@ -155,13 +157,6 @@ class DBManager end end - # Returns true if we are ready to load/store data - def active - # usable and migrated a just Boolean attributes, so check those first because they don't actually contact the - # database. - usable && migrated && connection_established? - end - # Returns true if the prerequisites have been installed attr_accessor :usable @@ -226,23 +221,6 @@ class DBManager true end - # Checks if the spec passed to `ActiveRecord::Base.establish_connection` can connect to the database. - # - # @return [true] if an active connection can be made to the database using the current config. - # @return [false] if an active connection cannot be made to the database. - def connection_established? - begin - # use with_connection so the connection doesn't stay pinned to the thread. - ActiveRecord::Base.connection_pool.with_connection { - ActiveRecord::Base.connection.active? - } - rescue ActiveRecord::ConnectionNotEstablished, PG::ConnectionBad => error - elog("Connection not established: #{error.class} #{error}:\n#{error.backtrace.join("\n")}") - - false - end - end - # # Scan through available drivers # @@ -292,129 +270,6 @@ class DBManager end end - # - # Connects this instance to a database - # - def connect(opts={}) - - return false if not @usable - - nopts = opts.dup - if (nopts['port']) - nopts['port'] = nopts['port'].to_i - end - - # Prefer the config file's pool setting - nopts['pool'] ||= 75 - - # Prefer the config file's wait_timeout setting too - nopts['wait_timeout'] ||= 300 - - begin - self.migrated = false - - # Check ActiveRecord::Base was already connected by Rails::Application.initialize! or some other API. - unless connection_established? - create_db(nopts) - - # Configure the database adapter - ActiveRecord::Base.establish_connection(nopts) - end - rescue ::Exception => e - self.error = e - elog("DB.connect threw an exception: #{e}") - dlog("Call stack: #{$@.join"\n"}", LEV_1) - return false - ensure - after_establish_connection - - # Database drivers can reset our KCODE, do not let them - $KCODE = 'NONE' if RUBY_VERSION =~ /^1\.8\./ - end - - true - end - - # Finishes {#connect} after `ActiveRecord::Base.establish_connection` has succeeded by {#migrate migrating database} - # and setting {#workspace}. - # - # @return [void] - def after_establish_connection - self.migrated = false - - begin - # Migrate the database, if needed - migrate - - # Set the default workspace - framework.db.workspace = framework.db.default_workspace - rescue ::Exception => exception - self.error = exception - elog("DB.connect threw an exception: #{exception}") - dlog("Call stack: #{exception.backtrace.join("\n")}", LEV_1) - else - # Flag that migration has completed - self.migrated = true - end - end - - # - # Attempt to create the database - # - # If the database already exists this will fail and we will continue on our - # merry way, connecting anyway. If it doesn't, we try to create it. If - # that fails, then it wasn't meant to be and the connect will raise a - # useful exception so the user won't be in the dark; no need to raise - # anything at all here. - # - def create_db(opts) - begin - case opts["adapter"] - when 'postgresql' - # Try to force a connection to be made to the database, if it succeeds - # then we know we don't need to create it :) - ActiveRecord::Base.establish_connection(opts) - # Do the checkout, checkin dance here to make sure this thread doesn't - # hold on to a connection we don't need - conn = ActiveRecord::Base.connection_pool.checkout - ActiveRecord::Base.connection_pool.checkin(conn) - end - rescue ::Exception => e - errstr = e.to_s - if errstr =~ /does not exist/i or errstr =~ /Unknown database/ - ilog("Database doesn't exist \"#{opts['database']}\", attempting to create it.") - ActiveRecord::Base.establish_connection( - opts.merge( - 'database' => 'postgres', - 'schema_search_path' => 'public' - ) - ) - - ActiveRecord::Base.connection.create_database(opts['database']) - else - ilog("Trying to continue despite failed database creation: #{e}") - end - end - ActiveRecord::Base.remove_connection - end - - # - # Disconnects a database session - # - def disconnect - begin - ActiveRecord::Base.remove_connection - self.migrated = false - self.modules_cached = false - rescue ::Exception => e - self.error = e - elog("DB.disconnect threw an exception: #{e}") - ensure - # Database drivers can reset our KCODE, do not let them - $KCODE = 'NONE' if RUBY_VERSION =~ /^1\.8\./ - end - end - # # Determines if the database is functional # @@ -423,6 +278,5 @@ class DBManager res = ::Mdm::Host.find(:first) } end - end end diff --git a/lib/msf/core/db_manager/connection.rb b/lib/msf/core/db_manager/connection.rb new file mode 100644 index 0000000000..171edb22c2 --- /dev/null +++ b/lib/msf/core/db_manager/connection.rb @@ -0,0 +1,149 @@ +module Msf::DBManager::Connection + # Returns true if we are ready to load/store data + def active + # usable and migrated a just Boolean attributes, so check those first because they don't actually contact the + # database. + usable && migrated && connection_established? + end + + # Finishes {#connect} after `ActiveRecord::Base.establish_connection` has succeeded by {#migrate migrating database} + # and setting {#workspace}. + # + # @return [void] + def after_establish_connection + self.migrated = false + + begin + # Migrate the database, if needed + migrate + + # Set the default workspace + framework.db.workspace = framework.db.default_workspace + rescue ::Exception => exception + self.error = exception + elog("DB.connect threw an exception: #{exception}") + dlog("Call stack: #{exception.backtrace.join("\n")}", LEV_1) + else + # Flag that migration has completed + self.migrated = true + end + end + + # + # Connects this instance to a database + # + def connect(opts={}) + + return false if not @usable + + nopts = opts.dup + if (nopts['port']) + nopts['port'] = nopts['port'].to_i + end + + # Prefer the config file's pool setting + nopts['pool'] ||= 75 + + # Prefer the config file's wait_timeout setting too + nopts['wait_timeout'] ||= 300 + + begin + self.migrated = false + + # Check ActiveRecord::Base was already connected by Rails::Application.initialize! or some other API. + unless connection_established? + create_db(nopts) + + # Configure the database adapter + ActiveRecord::Base.establish_connection(nopts) + end + rescue ::Exception => e + self.error = e + elog("DB.connect threw an exception: #{e}") + dlog("Call stack: #{$@.join"\n"}", LEV_1) + return false + ensure + after_establish_connection + + # Database drivers can reset our KCODE, do not let them + $KCODE = 'NONE' if RUBY_VERSION =~ /^1\.8\./ + end + + true + end + + + # + # Attempt to create the database + # + # If the database already exists this will fail and we will continue on our + # merry way, connecting anyway. If it doesn't, we try to create it. If + # that fails, then it wasn't meant to be and the connect will raise a + # useful exception so the user won't be in the dark; no need to raise + # anything at all here. + # + def create_db(opts) + begin + case opts["adapter"] + when 'postgresql' + # Try to force a connection to be made to the database, if it succeeds + # then we know we don't need to create it :) + ActiveRecord::Base.establish_connection(opts) + # Do the checkout, checkin dance here to make sure this thread doesn't + # hold on to a connection we don't need + conn = ActiveRecord::Base.connection_pool.checkout + ActiveRecord::Base.connection_pool.checkin(conn) + end + rescue ::Exception => e + errstr = e.to_s + if errstr =~ /does not exist/i or errstr =~ /Unknown database/ + ilog("Database doesn't exist \"#{opts['database']}\", attempting to create it.") + ActiveRecord::Base.establish_connection( + opts.merge( + 'database' => 'postgres', + 'schema_search_path' => 'public' + ) + ) + + ActiveRecord::Base.connection.create_database(opts['database']) + else + ilog("Trying to continue despite failed database creation: #{e}") + end + end + ActiveRecord::Base.remove_connection + end + + # Checks if the spec passed to `ActiveRecord::Base.establish_connection` can connect to the database. + # + # @return [true] if an active connection can be made to the database using the current config. + # @return [false] if an active connection cannot be made to the database. + def connection_established? + begin + # use with_connection so the connection doesn't stay pinned to the thread. + ActiveRecord::Base.connection_pool.with_connection { + ActiveRecord::Base.connection.active? + } + rescue ActiveRecord::ConnectionNotEstablished, PG::ConnectionBad => error + elog("Connection not established: #{error.class} #{error}:\n#{error.backtrace.join("\n")}") + + false + end + end + + # + # Disconnects a database session + # + def disconnect + begin + ActiveRecord::Base.remove_connection + self.migrated = false + self.modules_cached = false + rescue ::Exception => e + self.error = e + elog("DB.disconnect threw an exception: #{e}") + ensure + # Database drivers can reset our KCODE, do not let them + $KCODE = 'NONE' if RUBY_VERSION =~ /^1\.8\./ + end + end +end From 2b4150ac46f8ec84d5416f2d2e465be94958e5c7 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 12:10:54 -0500 Subject: [PATCH 072/159] Extract Msf::DBManager::Connection shared examples MSP-11124 --- spec/lib/msf/db_manager_spec.rb | 8 +------- spec/support/shared/examples/msf/db_manager/connection.rb | 8 ++++++++ 2 files changed, 9 insertions(+), 7 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/connection.rb diff --git a/spec/lib/msf/db_manager_spec.rb b/spec/lib/msf/db_manager_spec.rb index 210afd046b..a61ff6b1fd 100644 --- a/spec/lib/msf/db_manager_spec.rb +++ b/spec/lib/msf/db_manager_spec.rb @@ -19,6 +19,7 @@ describe Msf::DBManager do end it_should_behave_like 'Msf::DBManager::Client' + it_should_behave_like 'Msf::DBManager::Connection' it_should_behave_like 'Msf::DBManager::Cred' it_should_behave_like 'Msf::DBManager::Event' it_should_behave_like 'Msf::DBManager::ExploitAttempt' @@ -58,8 +59,6 @@ describe Msf::DBManager do end end - it { is_expected.to respond_to :active } - context '#add_rails_engine_migration_paths' do def add_rails_engine_migration_paths db_manager.add_rails_engine_migration_paths @@ -78,10 +77,7 @@ describe Msf::DBManager do end end - it { is_expected.to respond_to :after_establish_connection } it { is_expected.to respond_to :check } - it { is_expected.to respond_to :connect } - it { is_expected.to respond_to :connection_established? } it { is_expected.to respond_to :create_db } it { is_expected.to respond_to :disconnect } it { is_expected.to respond_to :driver } @@ -91,7 +87,5 @@ describe Msf::DBManager do it { is_expected.to respond_to :initialize_adapter } it { is_expected.to respond_to :initialize_database_support } it { is_expected.to respond_to :service_name_map } - it { is_expected.to respond_to :usable } - it { is_expected.to respond_to :usable= } it { is_expected.to respond_to :warn_about_rubies } end diff --git a/spec/support/shared/examples/msf/db_manager/connection.rb b/spec/support/shared/examples/msf/db_manager/connection.rb new file mode 100644 index 0000000000..7fda2cebf4 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/connection.rb @@ -0,0 +1,8 @@ +shared_examples_for 'Msf::DBManager::Connection' do + it { is_expected.to respond_to :active } + it { is_expected.to respond_to :after_establish_connection } + it { is_expected.to respond_to :connect } + it { is_expected.to respond_to :connection_established? } + it { is_expected.to respond_to :usable } + it { is_expected.to respond_to :usable= } +end \ No newline at end of file From bc4d2ff1526c693726f1b5b0e9585784fb229579 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 14:49:26 -0500 Subject: [PATCH 073/159] Extract Msf::DBManager::Adapter MSP-11124 Extract methods related to setting up the adapter/driver(s). --- lib/msf/core/db_manager.rb | 41 ++----------------------- lib/msf/core/db_manager/adapter.rb | 48 ++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 39 deletions(-) create mode 100644 lib/msf/core/db_manager/adapter.rb diff --git a/lib/msf/core/db_manager.rb b/lib/msf/core/db_manager.rb index 4c77f02c6c..e904671c89 100644 --- a/lib/msf/core/db_manager.rb +++ b/lib/msf/core/db_manager.rb @@ -75,6 +75,7 @@ module Msf class DBManager extend Metasploit::Framework::Require + autoload :Adapter, 'msf/core/db_manager/adapter' autoload :Client, 'msf/core/db_manager/client' autoload :Connection, 'msf/core/db_manager/connection' autoload :Cred, 'msf/core/db_manager/cred' @@ -106,6 +107,7 @@ class DBManager optionally_include_metasploit_credential_creation + include Msf::DBManager::Adapter include Msf::DBManager::Client include Msf::DBManager::Connection include Msf::DBManager::Cred @@ -140,13 +142,6 @@ class DBManager # Provides :framework and other accessors include Msf::Framework::Offspring - # - # CONSTANTS - # - - # The adapter to use to establish database connection. - ADAPTER = 'postgresql' - # Mainly, it's Ruby 1.9.1 that cause a lot of problems now, along with Ruby 1.8.6. # Ruby 1.8.7 actually seems okay, but why tempt fate? Let's say 1.9.3 and beyond. def warn_about_rubies @@ -160,15 +155,6 @@ class DBManager # Returns true if the prerequisites have been installed attr_accessor :usable - # Returns the list of usable database drivers - def drivers - @drivers ||= [] - end - attr_writer :drivers - - # Returns the active driver - attr_accessor :driver - # Stores the error message for why the db was not loaded attr_accessor :error @@ -221,29 +207,6 @@ class DBManager true end - # - # Scan through available drivers - # - def initialize_adapter - ActiveRecord::Base.default_timezone = :utc - - if connection_established? && ActiveRecord::Base.connection_config[:adapter] == ADAPTER - dlog("Already established connection to #{ADAPTER}, so reusing active connection.") - self.drivers << ADAPTER - self.driver = ADAPTER - else - begin - ActiveRecord::Base.establish_connection(adapter: ADAPTER) - ActiveRecord::Base.remove_connection - rescue Exception => error - @adapter_error = error - else - self.drivers << ADAPTER - self.driver = ADAPTER - end - end - end - # Loads Metasploit Data Models and adds its migrations to migrations paths. # # @return [void] diff --git a/lib/msf/core/db_manager/adapter.rb b/lib/msf/core/db_manager/adapter.rb new file mode 100644 index 0000000000..df82af9159 --- /dev/null +++ b/lib/msf/core/db_manager/adapter.rb @@ -0,0 +1,48 @@ +module Msf::DBManager::Adapter + # + # CONSTANTS + # + + # The adapter to use to establish database connection. + ADAPTER = 'postgresql' + + # + # Attributes + # + + # Returns the list of usable database drivers + def drivers + @drivers ||= [] + end + attr_writer :drivers + + # Returns the active driver + attr_accessor :driver + + # + # Instance Methods + # + + # + # Scan through available drivers + # + def initialize_adapter + ActiveRecord::Base.default_timezone = :utc + + if connection_established? && ActiveRecord::Base.connection_config[:adapter] == ADAPTER + dlog("Already established connection to #{ADAPTER}, so reusing active connection.") + self.drivers << ADAPTER + self.driver = ADAPTER + else + begin + ActiveRecord::Base.establish_connection(adapter: ADAPTER) + ActiveRecord::Base.remove_connection + rescue Exception => error + @adapter_error = error + else + self.drivers << ADAPTER + self.driver = ADAPTER + end + end + end +end From 66ce59725b764b544d131d1b516b2a0d499629c9 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 14:53:40 -0500 Subject: [PATCH 074/159] Extract Msf::DBManager::Adapter shared examples MSP-11124 --- spec/lib/msf/db_manager_spec.rb | 15 +-------------- .../shared/examples/msf/db_manager/adapter.rb | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 14 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/adapter.rb diff --git a/spec/lib/msf/db_manager_spec.rb b/spec/lib/msf/db_manager_spec.rb index a61ff6b1fd..c86e06d376 100644 --- a/spec/lib/msf/db_manager_spec.rb +++ b/spec/lib/msf/db_manager_spec.rb @@ -18,6 +18,7 @@ describe Msf::DBManager do db_manager end + it_should_behave_like 'Msf::DBManager::Adapter' it_should_behave_like 'Msf::DBManager::Client' it_should_behave_like 'Msf::DBManager::Connection' it_should_behave_like 'Msf::DBManager::Cred' @@ -49,16 +50,6 @@ describe Msf::DBManager do it_should_behave_like 'Msf::DBManager::Web' it_should_behave_like 'Msf::DBManager::Workspace' - context 'CONSTANTS' do - context 'ADAPTER' do - subject(:adapter) { - described_class::ADAPTER - } - - it { is_expected.to eq('postgresql') } - end - end - context '#add_rails_engine_migration_paths' do def add_rails_engine_migration_paths db_manager.add_rails_engine_migration_paths @@ -80,11 +71,7 @@ describe Msf::DBManager do it { is_expected.to respond_to :check } it { is_expected.to respond_to :create_db } it { is_expected.to respond_to :disconnect } - it { is_expected.to respond_to :driver } - it { is_expected.to respond_to :drivers } - it { is_expected.to respond_to :drivers= } it { is_expected.to respond_to :error } - it { is_expected.to respond_to :initialize_adapter } it { is_expected.to respond_to :initialize_database_support } it { is_expected.to respond_to :service_name_map } it { is_expected.to respond_to :warn_about_rubies } diff --git a/spec/support/shared/examples/msf/db_manager/adapter.rb b/spec/support/shared/examples/msf/db_manager/adapter.rb new file mode 100644 index 0000000000..1cc264f0aa --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/adapter.rb @@ -0,0 +1,16 @@ +shared_examples_for 'Msf::DBManager::Adapter' do + context 'CONSTANTS' do + context 'ADAPTER' do + subject(:adapter) { + described_class::ADAPTER + } + + it { is_expected.to eq('postgresql') } + end + end + + it { is_expected.to respond_to :driver } + it { is_expected.to respond_to :drivers } + it { is_expected.to respond_to :drivers= } + it { is_expected.to respond_to :initialize_adapter } +end \ No newline at end of file From 1dfaba188472237dc6a58aadcdb4f80441a33489 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 14:56:08 -0500 Subject: [PATCH 075/159] Remove nesting in Msf::DBManager::Migration MSP-11124 To prevent migration.rb as counting as the definer of `Msf::DBManager`. --- lib/msf/core/db_manager/migration.rb | 100 +++++++++++++-------------- 1 file changed, 48 insertions(+), 52 deletions(-) diff --git a/lib/msf/core/db_manager/migration.rb b/lib/msf/core/db_manager/migration.rb index 26c90c6920..58a9fcb201 100644 --- a/lib/msf/core/db_manager/migration.rb +++ b/lib/msf/core/db_manager/migration.rb @@ -1,59 +1,55 @@ # -*- coding: binary -*- -module Msf - class DBManager - module Migration - # Migrate database to latest schema version. - # - # @param verbose [Boolean] see ActiveRecord::Migration.verbose - # @return [Array error - self.error = error - elog("DB.migrate threw an exception: #{error}") - dlog("Call stack:\n#{error.backtrace.join "\n"}") - end - end - - # Since the connections that existed before the migrations ran could - # have outdated column information, reset column information for all - # ActiveRecord::Base descendents to prevent missing method errors for - # column methods for columns created in migrations after the column - # information was cached. - reset_column_information - - return ran - end - - # Flag to indicate database migration has completed - # - # @return [Boolean] - attr_accessor :migrated - - private - - # Resets the column information for all descendants of ActiveRecord::Base - # since some of the migrations may have cached column information that - # has been updated by later migrations. - # - # @return [void] - def reset_column_information - ActiveRecord::Base.descendants.each do |descendant| - descendant.reset_column_information - end + rescue StandardError => error + self.error = error + elog("DB.migrate threw an exception: #{error}") + dlog("Call stack:\n#{error.backtrace.join "\n"}") end end + + # Since the connections that existed before the migrations ran could + # have outdated column information, reset column information for all + # ActiveRecord::Base descendents to prevent missing method errors for + # column methods for columns created in migrations after the column + # information was cached. + reset_column_information + + return ran end -end \ No newline at end of file + + # Flag to indicate database migration has completed + # + # @return [Boolean] + attr_accessor :migrated + + private + + # Resets the column information for all descendants of ActiveRecord::Base + # since some of the migrations may have cached column information that + # has been updated by later migrations. + # + # @return [void] + def reset_column_information + ActiveRecord::Base.descendants.each do |descendant| + descendant.reset_column_information + end + end +end From 2dd925c18ca214e89e693dc87b41d78bd1f42dee Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 15:01:14 -0500 Subject: [PATCH 076/159] Move add_rails_engine_migration_paths to Msf::DBManager::Migration MSP-11124 --- lib/msf/core/db_manager.rb | 26 -------------------------- lib/msf/core/db_manager/migration.rb | 26 ++++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/lib/msf/core/db_manager.rb b/lib/msf/core/db_manager.rb index e904671c89..b78ea82cd2 100644 --- a/lib/msf/core/db_manager.rb +++ b/lib/msf/core/db_manager.rb @@ -207,32 +207,6 @@ class DBManager true end - # Loads Metasploit Data Models and adds its migrations to migrations paths. - # - # @return [void] - def add_rails_engine_migration_paths - unless defined? ActiveRecord - fail "Bundle installed '--without #{Bundler.settings.without.join(' ')}'. To clear the without option do " \ - "`bundle install --without ''` (the --without flag with an empty string) or `rm -rf .bundle` to remove " \ - "the .bundle/config manually and then `bundle install`" - end - - Rails.application.railties.engines.each do |engine| - migrations_paths = engine.paths['db/migrate'].existent_directories - - migrations_paths.each do |migrations_path| - # Since ActiveRecord::Migrator.migrations_paths can persist between - # instances of Msf::DBManager, such as in specs, - # migrations_path may already be part of - # migrations_paths, in which case it should not be added or multiple - # migrations with the same version number errors will occur. - unless ActiveRecord::Migrator.migrations_paths.include? migrations_path - ActiveRecord::Migrator.migrations_paths << migrations_path - end - end - end - end - # # Determines if the database is functional # diff --git a/lib/msf/core/db_manager/migration.rb b/lib/msf/core/db_manager/migration.rb index 58a9fcb201..5d98eb960e 100644 --- a/lib/msf/core/db_manager/migration.rb +++ b/lib/msf/core/db_manager/migration.rb @@ -1,5 +1,31 @@ # -*- coding: binary -*- module Msf::DBManager::Migration + # Loads Metasploit Data Models and adds its migrations to migrations paths. + # + # @return [void] + def add_rails_engine_migration_paths + unless defined? ActiveRecord + fail "Bundle installed '--without #{Bundler.settings.without.join(' ')}'. To clear the without option do " \ + "`bundle install --without ''` (the --without flag with an empty string) or `rm -rf .bundle` to remove " \ + "the .bundle/config manually and then `bundle install`" + end + + Rails.application.railties.engines.each do |engine| + migrations_paths = engine.paths['db/migrate'].existent_directories + + migrations_paths.each do |migrations_path| + # Since ActiveRecord::Migrator.migrations_paths can persist between + # instances of Msf::DBManager, such as in specs, + # migrations_path may already be part of + # migrations_paths, in which case it should not be added or multiple + # migrations with the same version number errors will occur. + unless ActiveRecord::Migrator.migrations_paths.include? migrations_path + ActiveRecord::Migrator.migrations_paths << migrations_path + end + end + end + end + # Migrate database to latest schema version. # # @param verbose [Boolean] see ActiveRecord::Migration.verbose From 92aaecf94b0bd8a555c909f1b013796c7d02e656 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 15:02:20 -0500 Subject: [PATCH 077/159] Move add_rails_engine_migrations_paths to Msf::DBManager::Migration MSP-11124 --- spec/lib/msf/db_manager_spec.rb | 18 ------------------ .../examples/msf/db_manager/migration.rb | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/spec/lib/msf/db_manager_spec.rb b/spec/lib/msf/db_manager_spec.rb index c86e06d376..f39b3928cf 100644 --- a/spec/lib/msf/db_manager_spec.rb +++ b/spec/lib/msf/db_manager_spec.rb @@ -50,24 +50,6 @@ describe Msf::DBManager do it_should_behave_like 'Msf::DBManager::Web' it_should_behave_like 'Msf::DBManager::Workspace' - context '#add_rails_engine_migration_paths' do - def add_rails_engine_migration_paths - db_manager.add_rails_engine_migration_paths - end - - it 'should not add duplicate paths to ActiveRecord::Migrator.migrations_paths' do - add_rails_engine_migration_paths - - expect { - add_rails_engine_migration_paths - }.to_not change { - ActiveRecord::Migrator.migrations_paths.length - } - - ActiveRecord::Migrator.migrations_paths.uniq.should == ActiveRecord::Migrator.migrations_paths - end - end - it { is_expected.to respond_to :check } it { is_expected.to respond_to :create_db } it { is_expected.to respond_to :disconnect } diff --git a/spec/support/shared/examples/msf/db_manager/migration.rb b/spec/support/shared/examples/msf/db_manager/migration.rb index a9cf24bbca..437411b91c 100644 --- a/spec/support/shared/examples/msf/db_manager/migration.rb +++ b/spec/support/shared/examples/msf/db_manager/migration.rb @@ -1,6 +1,25 @@ shared_examples_for 'Msf::DBManager::Migration' do it { should be_a Msf::DBManager::Migration } + + context '#add_rails_engine_migration_paths' do + def add_rails_engine_migration_paths + db_manager.add_rails_engine_migration_paths + end + + it 'should not add duplicate paths to ActiveRecord::Migrator.migrations_paths' do + add_rails_engine_migration_paths + + expect { + add_rails_engine_migration_paths + }.to_not change { + ActiveRecord::Migrator.migrations_paths.length + } + + ActiveRecord::Migrator.migrations_paths.uniq.should == ActiveRecord::Migrator.migrations_paths + end + end + context '#migrate' do def migrate db_manager.migrate From acdf6e7dbf9a1c561f83aeb9ff3431e85f912ee6 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 15:06:46 -0500 Subject: [PATCH 078/159] Move missed Msf::DBManager::Connection examples MSP-11124 --- spec/lib/msf/db_manager_spec.rb | 2 -- spec/support/shared/examples/msf/db_manager/connection.rb | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/lib/msf/db_manager_spec.rb b/spec/lib/msf/db_manager_spec.rb index f39b3928cf..a7959e3ef5 100644 --- a/spec/lib/msf/db_manager_spec.rb +++ b/spec/lib/msf/db_manager_spec.rb @@ -51,8 +51,6 @@ describe Msf::DBManager do it_should_behave_like 'Msf::DBManager::Workspace' it { is_expected.to respond_to :check } - it { is_expected.to respond_to :create_db } - it { is_expected.to respond_to :disconnect } it { is_expected.to respond_to :error } it { is_expected.to respond_to :initialize_database_support } it { is_expected.to respond_to :service_name_map } diff --git a/spec/support/shared/examples/msf/db_manager/connection.rb b/spec/support/shared/examples/msf/db_manager/connection.rb index 7fda2cebf4..144847df94 100644 --- a/spec/support/shared/examples/msf/db_manager/connection.rb +++ b/spec/support/shared/examples/msf/db_manager/connection.rb @@ -3,6 +3,8 @@ shared_examples_for 'Msf::DBManager::Connection' do it { is_expected.to respond_to :after_establish_connection } it { is_expected.to respond_to :connect } it { is_expected.to respond_to :connection_established? } + it { is_expected.to respond_to :create_db } + it { is_expected.to respond_to :disconnect } it { is_expected.to respond_to :usable } it { is_expected.to respond_to :usable= } end \ No newline at end of file From 4371254dd24c532333d8bffddf01a51ffc13bf8c Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 15:13:28 -0500 Subject: [PATCH 079/159] Reorder remaining code in Msf::DBManager MSP-11124 --- lib/msf/core/db_manager.rb | 48 ++++++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/lib/msf/core/db_manager.rb b/lib/msf/core/db_manager.rb index b78ea82cd2..b024e55373 100644 --- a/lib/msf/core/db_manager.rb +++ b/lib/msf/core/db_manager.rb @@ -142,21 +142,19 @@ class DBManager # Provides :framework and other accessors include Msf::Framework::Offspring - # Mainly, it's Ruby 1.9.1 that cause a lot of problems now, along with Ruby 1.8.6. - # Ruby 1.8.7 actually seems okay, but why tempt fate? Let's say 1.9.3 and beyond. - def warn_about_rubies - if ::RUBY_VERSION =~ /^1\.9\.[012]($|[^\d])/ - $stderr.puts "**************************************************************************************" - $stderr.puts "Metasploit requires at least Ruby 1.9.3. For an easy upgrade path, see https://rvm.io/" - $stderr.puts "**************************************************************************************" - end - end + # + # Attributes + # + + # Stores the error message for why the db was not loaded + attr_accessor :error # Returns true if the prerequisites have been installed attr_accessor :usable - # Stores the error message for why the db was not loaded - attr_accessor :error + # + # iniitialize + # def initialize(framework, opts = {}) @@ -176,6 +174,19 @@ class DBManager initialize_database_support end + # + # Instance Methods + # + + # + # Determines if the database is functional + # + def check + ::ActiveRecord::Base.connection_pool.with_connection { + res = ::Mdm::Host.find(:first) + } + end + # # Do what is necessary to load our database support # @@ -207,13 +218,14 @@ class DBManager true end - # - # Determines if the database is functional - # - def check - ::ActiveRecord::Base.connection_pool.with_connection { - res = ::Mdm::Host.find(:first) - } + # Mainly, it's Ruby 1.9.1 that cause a lot of problems now, along with Ruby 1.8.6. + # Ruby 1.8.7 actually seems okay, but why tempt fate? Let's say 1.9.3 and beyond. + def warn_about_rubies + if ::RUBY_VERSION =~ /^1\.9\.[012]($|[^\d])/ + $stderr.puts "**************************************************************************************" + $stderr.puts "Metasploit requires at least Ruby 1.9.3. For an easy upgrade path, see https://rvm.io/" + $stderr.puts "**************************************************************************************" + end end end end From c6ea3a38804f50e64b32b0c5a870b9479ed9d36a Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 15:24:56 -0500 Subject: [PATCH 080/159] Distribute requires to where they are needed MSP-11124 Push requires down to the Msf::DBManager mixins that actually need them. --- lib/msf/core/db_manager.rb | 42 ------------------------- lib/msf/core/db_manager/import.rb | 35 +++++++++++++++++++++ lib/msf/core/db_manager/module_cache.rb | 6 ++++ lib/msf/core/db_manager/report.rb | 6 ++++ 4 files changed, 47 insertions(+), 42 deletions(-) diff --git a/lib/msf/core/db_manager.rb b/lib/msf/core/db_manager.rb index b024e55373..d5af9f00dc 100644 --- a/lib/msf/core/db_manager.rb +++ b/lib/msf/core/db_manager.rb @@ -1,51 +1,9 @@ # -*- coding: binary -*- -# -# Standard Library -# - -require 'csv' -require 'fileutils' -require 'shellwords' -require 'tmpdir' -require 'uri' - -# # # Gems # -# -# -# PacketFu -# - -require 'packetfu' - -# -# Rex -# - -require 'rex/parser/acunetix_nokogiri' -require 'rex/parser/appscan_nokogiri' -require 'rex/parser/burp_session_nokogiri' -require 'rex/parser/ci_nokogiri' -require 'rex/parser/foundstone_nokogiri' -require 'rex/parser/fusionvm_nokogiri' -require 'rex/parser/ip360_aspl_xml' -require 'rex/parser/ip360_xml' -require 'rex/parser/mbsa_nokogiri' -require 'rex/parser/nessus_xml' -require 'rex/parser/netsparker_xml' -require 'rex/parser/nexpose_raw_nokogiri' -require 'rex/parser/nexpose_simple_nokogiri' -require 'rex/parser/nexpose_xml' -require 'rex/parser/nmap_nokogiri' -require 'rex/parser/nmap_xml' -require 'rex/parser/openvas_nokogiri' -require 'rex/parser/outpost24_nokogiri' -require 'rex/parser/retina_xml' -require 'rex/parser/wapiti_nokogiri' require 'rex/socket' # diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index 1382dd661b..10c92e7111 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -1,3 +1,38 @@ +# +# Standard library +# + +require 'csv' +require 'fileutils' +require 'tmpdir' +require 'uri' + +# +# Gems +# + +require 'packetfu' +require 'rex/parser/acunetix_nokogiri' +require 'rex/parser/appscan_nokogiri' +require 'rex/parser/burp_session_nokogiri' +require 'rex/parser/ci_nokogiri' +require 'rex/parser/foundstone_nokogiri' +require 'rex/parser/fusionvm_nokogiri' +require 'rex/parser/ip360_aspl_xml' +require 'rex/parser/ip360_xml' +require 'rex/parser/mbsa_nokogiri' +require 'rex/parser/nessus_xml' +require 'rex/parser/netsparker_xml' +require 'rex/parser/nexpose_raw_nokogiri' +require 'rex/parser/nexpose_simple_nokogiri' +require 'rex/parser/nexpose_xml' +require 'rex/parser/nmap_nokogiri' +require 'rex/parser/nmap_xml' +require 'rex/parser/openvas_nokogiri' +require 'rex/parser/outpost24_nokogiri' +require 'rex/parser/retina_xml' +require 'rex/parser/wapiti_nokogiri' + module Msf::DBManager::Import # If hex notation is present, turn them into a character. def dehex(str) diff --git a/lib/msf/core/db_manager/module_cache.rb b/lib/msf/core/db_manager/module_cache.rb index 299faed685..5e23cb6d2c 100644 --- a/lib/msf/core/db_manager/module_cache.rb +++ b/lib/msf/core/db_manager/module_cache.rb @@ -1,3 +1,9 @@ +# +# Standard library +# + +require 'shellwords' + module Msf::DBManager::ModuleCache # # Attributes diff --git a/lib/msf/core/db_manager/report.rb b/lib/msf/core/db_manager/report.rb index c7d7fca1a2..b898f9f5c9 100644 --- a/lib/msf/core/db_manager/report.rb +++ b/lib/msf/core/db_manager/report.rb @@ -1,3 +1,9 @@ +# +# Standard library +# + +require 'fileutils' + module Msf::DBManager::Report # TODO This method does not attempt to find. It just creates # a report based on the passed params. From ef04261686524500ee70e4e8a2603bcfe3af990a Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 15:26:19 -0500 Subject: [PATCH 081/159] Fix indentation in Msf::DBManager MSP-11124 --- lib/msf/core/db_manager.rb | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/lib/msf/core/db_manager.rb b/lib/msf/core/db_manager.rb index d5af9f00dc..3fca3fab22 100644 --- a/lib/msf/core/db_manager.rb +++ b/lib/msf/core/db_manager.rb @@ -21,16 +21,9 @@ require 'msf/core/host_state' require 'msf/core/service_state' require 'msf/core/task_manager' -module Msf - -### -# # The db module provides persistent storage and events. This class should be instantiated LAST # as the active_suppport library overrides Kernel.require, slowing down all future code loads. -# -### - -class DBManager +class Msf::DBManager extend Metasploit::Framework::Require autoload :Adapter, 'msf/core/db_manager/adapter' @@ -186,4 +179,3 @@ class DBManager end end end -end From b8ea44235b06d78193b32b5a09a1db215141fe8c Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 15:28:12 -0500 Subject: [PATCH 082/159] Remove nesting in Msf::DBManager::ImportMsfXml MSP-11124 Don't use nested modules to prevent Msf::DBManager::ImportMsfXml from being the declaring location for Msf::DBManager. --- lib/msf/core/db_manager.rb | 2 +- lib/msf/core/db_manager/import_msf_xml.rb | 1095 ++++++++++----------- 2 files changed, 546 insertions(+), 551 deletions(-) diff --git a/lib/msf/core/db_manager.rb b/lib/msf/core/db_manager.rb index 3fca3fab22..10f8700fab 100644 --- a/lib/msf/core/db_manager.rb +++ b/lib/msf/core/db_manager.rb @@ -15,7 +15,6 @@ require 'msf/base/config' require 'msf/core' require 'msf/core/database_event' require 'msf/core/db_import_error' -require 'msf/core/db_manager/import_msf_xml' require 'msf/core/db_manager/migration' require 'msf/core/host_state' require 'msf/core/service_state' @@ -37,6 +36,7 @@ class Msf::DBManager autoload :HostDetail, 'msf/core/db_manager/host_detail' autoload :HostTag, 'msf/core/db_manager/host_tag' autoload :Import, 'msf/core/db_manager/import' + autoload :ImportMsfXml, 'msf/core/db_manager/import_msf_xml' autoload :IPAddress, 'msf/core/db_manager/ip_address' autoload :Loot, 'msf/core/db_manager/loot' autoload :ModuleCache, 'msf/core/db_manager/module_cache' diff --git a/lib/msf/core/db_manager/import_msf_xml.rb b/lib/msf/core/db_manager/import_msf_xml.rb index e3ffd53734..ad4d85a98c 100644 --- a/lib/msf/core/db_manager/import_msf_xml.rb +++ b/lib/msf/core/db_manager/import_msf_xml.rb @@ -1,593 +1,588 @@ # -*- coding: binary -*- -module Msf - class DBManager - # Handles importing of the xml format exported by Pro. The methods are in a - # module because (1) it's just good code layout and (2) it allows the - # methods to be overridden in Pro without using alias_method_chain as - # methods defined in a class cannot be overridden by including a module - # (unless you're running Ruby 2.0 and can use prepend) - module ImportMsfXml - # - # CONSTANTS - # +# Handles importing of the xml format exported by Pro. The methods are in a +# module because (1) it's just good code layout and (2) it allows the +# methods to be overridden in Pro without using alias_method_chain as +# methods defined in a class cannot be overridden by including a module +# (unless you're running Ruby 2.0 and can use prepend) +module Msf::DBManager::ImportMsfXml + # + # CONSTANTS + # - # Elements that can be treated as text (i.e. do not need to be - # deserialized) in {#import_msf_web_page_element} - MSF_WEB_PAGE_TEXT_ELEMENT_NAMES = [ - 'auth', - 'body', - 'code', - 'cookie', - 'ctype', - 'location', - 'mtime' - ] + # Elements that can be treated as text (i.e. do not need to be + # deserialized) in {#import_msf_web_page_element} + MSF_WEB_PAGE_TEXT_ELEMENT_NAMES = [ + 'auth', + 'body', + 'code', + 'cookie', + 'ctype', + 'location', + 'mtime' + ] - # Elements that can be treated as text (i.e. do not need to be - # deserialized) in {#import_msf_web_element}. - MSF_WEB_TEXT_ELEMENT_NAMES = [ - 'created-at', - 'host', - 'path', - 'port', - 'query', - 'ssl', - 'updated-at', - 'vhost' - ] + # Elements that can be treated as text (i.e. do not need to be + # deserialized) in {#import_msf_web_element}. + MSF_WEB_TEXT_ELEMENT_NAMES = [ + 'created-at', + 'host', + 'path', + 'port', + 'query', + 'ssl', + 'updated-at', + 'vhost' + ] - # Elements that can be treated as text (i.e. do not need to be - # deserialized) in {#import_msf_web_vuln_element}. - MSF_WEB_VULN_TEXT_ELEMENT_NAMES = [ - 'blame', - 'category', - 'confidence', - 'description', - 'method', - 'name', - 'pname', - 'proof', - 'risk' - ] + # Elements that can be treated as text (i.e. do not need to be + # deserialized) in {#import_msf_web_vuln_element}. + MSF_WEB_VULN_TEXT_ELEMENT_NAMES = [ + 'blame', + 'category', + 'confidence', + 'description', + 'method', + 'name', + 'pname', + 'proof', + 'risk' + ] - # - # Instance Methods - # + # + # Instance Methods + # - # Imports web_form element using {Msf::DBManager#report_web_form}. - # - # @param element [REXML::Element] web_form element. - # @param options [Hash{Symbol => Object}] options - # @option options [Boolean] :allow_yaml (false) Whether to allow YAML when - # deserializing params. - # @option options [Mdm::Workspace, nil] :workspace - # (Msf::DBManager#workspace) workspace under which to report the - # Mdm::WebForm - # @yield [event, data] - # @yieldparam event [:web_page] The event name - # @yieldparam data [String] path - # @yieldreturn [void] - # @return [void] - def import_msf_web_form_element(element, options={}, ¬ifier) - options.assert_valid_keys(:allow_yaml, :workspace) + # Imports web_form element using {Msf::DBManager#report_web_form}. + # + # @param element [REXML::Element] web_form element. + # @param options [Hash{Symbol => Object}] options + # @option options [Boolean] :allow_yaml (false) Whether to allow YAML when + # deserializing params. + # @option options [Mdm::Workspace, nil] :workspace + # (Msf::DBManager#workspace) workspace under which to report the + # Mdm::WebForm + # @yield [event, data] + # @yieldparam event [:web_page] The event name + # @yieldparam data [String] path + # @yieldreturn [void] + # @return [void] + def import_msf_web_form_element(element, options={}, ¬ifier) + options.assert_valid_keys(:allow_yaml, :workspace) - import_msf_web_element(element, - :allow_yaml => options[:allow_yaml], - :notifier => notifier, - :type => :form, - :workspace => options[:workspace]) do |element, options| - info = import_msf_text_element(element, 'method') + import_msf_web_element(element, + :allow_yaml => options[:allow_yaml], + :notifier => notifier, + :type => :form, + :workspace => options[:workspace]) do |element, options| + info = import_msf_text_element(element, 'method') - # FIXME https://www.pivotaltracker.com/story/show/46578647 - # FIXME https://www.pivotaltracker.com/story/show/47128407 - unserialized_params = unserialize_object( - element.elements['params'], - options[:allow_yaml] - ) - info[:params] = nils_for_nulls(unserialized_params) + # FIXME https://www.pivotaltracker.com/story/show/46578647 + # FIXME https://www.pivotaltracker.com/story/show/47128407 + unserialized_params = unserialize_object( + element.elements['params'], + options[:allow_yaml] + ) + info[:params] = nils_for_nulls(unserialized_params) - info - end + info + end + end + + # Imports web_page element using {Msf::DBManager#report_web_page}. + # + # @param element [REXML::Element] web_page element. + # @param options [Hash{Symbol => Object}] options + # @option options [Boolean] :allow_yaml (false) Whether to allow YAML when + # deserializing headers. + # @option options [Mdm::Workspace, nil] :workspace + # (Msf::DBManager#workspace) workspace under which to report the + # Mdm::WebPage. + # @yield [event, data] + # @yieldparam event [:web_page] The event name + # @yieldparam data [String] path + # @yieldreturn [void] + # @return [void] + def import_msf_web_page_element(element, options={}, ¬ifier) + options.assert_valid_keys(:allow_yaml, :workspace) + + import_msf_web_element(element, + :allow_yaml => options[:allow_yaml], + :notifier => notifier, + :type => :page, + :workspace => options[:workspace]) do |element, options| + info = {} + + MSF_WEB_PAGE_TEXT_ELEMENT_NAMES.each do |name| + element_info = import_msf_text_element(element, name) + info.merge!(element_info) end - # Imports web_page element using {Msf::DBManager#report_web_page}. - # - # @param element [REXML::Element] web_page element. - # @param options [Hash{Symbol => Object}] options - # @option options [Boolean] :allow_yaml (false) Whether to allow YAML when - # deserializing headers. - # @option options [Mdm::Workspace, nil] :workspace - # (Msf::DBManager#workspace) workspace under which to report the - # Mdm::WebPage. - # @yield [event, data] - # @yieldparam event [:web_page] The event name - # @yieldparam data [String] path - # @yieldreturn [void] - # @return [void] - def import_msf_web_page_element(element, options={}, ¬ifier) - options.assert_valid_keys(:allow_yaml, :workspace) + code = info[:code] - import_msf_web_element(element, - :allow_yaml => options[:allow_yaml], - :notifier => notifier, - :type => :page, - :workspace => options[:workspace]) do |element, options| - info = {} - - MSF_WEB_PAGE_TEXT_ELEMENT_NAMES.each do |name| - element_info = import_msf_text_element(element, name) - info.merge!(element_info) - end - - code = info[:code] - - if code - info[:code] = code.to_i - end - - # FIXME https://www.pivotaltracker.com/story/show/46578647 - # FIXME https://www.pivotaltracker.com/story/show/47128407 - unserialized_headers = unserialize_object( - element.elements['headers'], - options[:allow_yaml] - ) - info[:headers] = nils_for_nulls(unserialized_headers) - - info - end + if code + info[:code] = code.to_i end - # Imports web_vuln element using {Msf::DBManager#report_web_vuln}. - # - # @param element [REXML::Element] web_vuln element. - # @param options [Hash{Symbol => Object}] options - # @option options [Boolean] :allow_yaml (false) Whether to allow YAML when - # deserializing headers. - # @option options [Mdm::Workspace, nil] :workspace - # (Msf::DBManager#workspace) workspace under which to report the - # Mdm::WebPage. - # @yield [event, data] - # @yieldparam event [:web_page] The event name - # @yieldparam data [String] path - # @yieldreturn [void] - # @return [void] - def import_msf_web_vuln_element(element, options={}, ¬ifier) - options.assert_valid_keys(:allow_yaml, :workspace) + # FIXME https://www.pivotaltracker.com/story/show/46578647 + # FIXME https://www.pivotaltracker.com/story/show/47128407 + unserialized_headers = unserialize_object( + element.elements['headers'], + options[:allow_yaml] + ) + info[:headers] = nils_for_nulls(unserialized_headers) - import_msf_web_element(element, - :allow_yaml => options[:allow_yaml], - :notifier => notifier, - :workspace => options[:workspace], - :type => :vuln) do |element, options| - info = {} + info + end + end - MSF_WEB_VULN_TEXT_ELEMENT_NAMES.each do |name| - element_info = import_msf_text_element(element, name) - info.merge!(element_info) - end + # Imports web_vuln element using {Msf::DBManager#report_web_vuln}. + # + # @param element [REXML::Element] web_vuln element. + # @param options [Hash{Symbol => Object}] options + # @option options [Boolean] :allow_yaml (false) Whether to allow YAML when + # deserializing headers. + # @option options [Mdm::Workspace, nil] :workspace + # (Msf::DBManager#workspace) workspace under which to report the + # Mdm::WebPage. + # @yield [event, data] + # @yieldparam event [:web_page] The event name + # @yieldparam data [String] path + # @yieldreturn [void] + # @return [void] + def import_msf_web_vuln_element(element, options={}, ¬ifier) + options.assert_valid_keys(:allow_yaml, :workspace) - confidence = info[:confidence] + import_msf_web_element(element, + :allow_yaml => options[:allow_yaml], + :notifier => notifier, + :workspace => options[:workspace], + :type => :vuln) do |element, options| + info = {} - if confidence - info[:confidence] = confidence.to_i - end - - # FIXME https://www.pivotaltracker.com/story/show/46578647 - # FIXME https://www.pivotaltracker.com/story/show/47128407 - unserialized_params = unserialize_object( - element.elements['params'], - options[:allow_yaml] - ) - info[:params] = nils_for_nulls(unserialized_params) - - risk = info[:risk] - - if risk - info[:risk] = risk.to_i - end - - info - end + MSF_WEB_VULN_TEXT_ELEMENT_NAMES.each do |name| + element_info = import_msf_text_element(element, name) + info.merge!(element_info) end - # For each host, step through services, notes, and vulns, and import - # them. - # TODO: loot, tasks, and reports - def import_msf_xml(args={}, &block) - data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + confidence = info[:confidence] - doc = rexmlify(data) - metadata = check_msf_xml_version!(doc) - allow_yaml = metadata[:allow_yaml] - btag = metadata[:root_tag] - - doc.elements.each("/#{btag}/hosts/host") do |host| - host_data = {} - host_data[:task] = args[:task] - host_data[:workspace] = wspace - host_data[:host] = nils_for_nulls(host.elements["address"].text.to_s.strip) - if bl.include? host_data[:host] - next - else - yield(:address,host_data[:host]) if block - end - host_data[:mac] = nils_for_nulls(host.elements["mac"].text.to_s.strip) - if host.elements["comm"].text - host_data[:comm] = nils_for_nulls(host.elements["comm"].text.to_s.strip) - end - %W{created-at updated-at name state os-flavor os-lang os-name os-sp purpose}.each { |datum| - if host.elements[datum].text - host_data[datum.gsub('-','_')] = nils_for_nulls(host.elements[datum].text.to_s.strip) - end - } - host_address = host_data[:host].dup # Preserve after report_host() deletes - hobj = report_host(host_data) - - host.elements.each("host_details/host_detail") do |hdet| - hdet_data = {} - hdet.elements.each do |det| - next if ["id", "host-id"].include?(det.name) - if det.text - hdet_data[det.name.gsub('-','_')] = nils_for_nulls(det.text.to_s.strip) - end - end - report_host_details(hobj, hdet_data) - end - - host.elements.each("exploit_attempts/exploit_attempt") do |hdet| - hdet_data = {} - hdet.elements.each do |det| - next if ["id", "host-id", "session-id", "vuln-id", "service-id", "loot-id"].include?(det.name) - if det.text - hdet_data[det.name.gsub('-','_')] = nils_for_nulls(det.text.to_s.strip) - end - end - report_exploit_attempt(hobj, hdet_data) - end - - host.elements.each('services/service') do |service| - service_data = {} - service_data[:task] = args[:task] - service_data[:workspace] = wspace - service_data[:host] = hobj - service_data[:port] = nils_for_nulls(service.elements["port"].text.to_s.strip).to_i - service_data[:proto] = nils_for_nulls(service.elements["proto"].text.to_s.strip) - %W{created-at updated-at name state info}.each { |datum| - if service.elements[datum].text - if datum == "info" - service_data["info"] = nils_for_nulls(unserialize_object(service.elements[datum], false)) - else - service_data[datum.gsub("-","_")] = nils_for_nulls(service.elements[datum].text.to_s.strip) - end - end - } - report_service(service_data) - end - - host.elements.each('notes/note') do |note| - note_data = {} - note_data[:workspace] = wspace - note_data[:host] = hobj - note_data[:type] = nils_for_nulls(note.elements["ntype"].text.to_s.strip) - note_data[:data] = nils_for_nulls(unserialize_object(note.elements["data"], allow_yaml)) - - if note.elements["critical"].text - note_data[:critical] = true unless note.elements["critical"].text.to_s.strip == "NULL" - end - if note.elements["seen"].text - note_data[:seen] = true unless note.elements["critical"].text.to_s.strip == "NULL" - end - %W{created-at updated-at}.each { |datum| - if note.elements[datum].text - note_data[datum.gsub("-","_")] = nils_for_nulls(note.elements[datum].text.to_s.strip) - end - } - report_note(note_data) - end - - host.elements.each('tags/tag') do |tag| - tag_data = {} - tag_data[:addr] = host_address - tag_data[:wspace] = wspace - tag_data[:name] = tag.elements["name"].text.to_s.strip - tag_data[:desc] = tag.elements["desc"].text.to_s.strip - if tag.elements["report-summary"].text - tag_data[:summary] = tag.elements["report-summary"].text.to_s.strip - end - if tag.elements["report-detail"].text - tag_data[:detail] = tag.elements["report-detail"].text.to_s.strip - end - if tag.elements["critical"].text - tag_data[:crit] = true unless tag.elements["critical"].text.to_s.strip == "NULL" - end - report_host_tag(tag_data) - end - - host.elements.each('vulns/vuln') do |vuln| - vuln_data = {} - vuln_data[:workspace] = wspace - vuln_data[:host] = hobj - vuln_data[:info] = nils_for_nulls(unserialize_object(vuln.elements["info"], allow_yaml)) - vuln_data[:name] = nils_for_nulls(vuln.elements["name"].text.to_s.strip) - %W{created-at updated-at exploited-at}.each { |datum| - if vuln.elements[datum] and vuln.elements[datum].text - vuln_data[datum.gsub("-","_")] = nils_for_nulls(vuln.elements[datum].text.to_s.strip) - end - } - if vuln.elements["refs"] - vuln_data[:refs] = [] - vuln.elements.each("refs/ref") do |ref| - vuln_data[:refs] << nils_for_nulls(ref.text.to_s.strip) - end - end - - vobj = report_vuln(vuln_data) - - vuln.elements.each("vuln_details/vuln_detail") do |vdet| - vdet_data = {} - vdet.elements.each do |det| - next if ["id", "vuln-id"].include?(det.name) - if det.text - vdet_data[det.name.gsub('-','_')] = nils_for_nulls(det.text.to_s.strip) - end - end - report_vuln_details(vobj, vdet_data) - end - - vuln.elements.each("vuln_attempts/vuln_attempt") do |vdet| - vdet_data = {} - vdet.elements.each do |det| - next if ["id", "vuln-id", "loot-id", "session-id"].include?(det.name) - if det.text - vdet_data[det.name.gsub('-','_')] = nils_for_nulls(det.text.to_s.strip) - end - end - report_vuln_attempt(vobj, vdet_data) - end - end - - ## Handle old-style (pre 4.10) XML files - if btag == "MetasploitV4" - if host.elements['creds'].present? - unless host.elements['creds'].elements.empty? - origin = Metasploit::Credential::Origin::Import.create(filename: "console-import-#{Time.now.to_i}") - - host.elements.each('creds/cred') do |cred| - username = cred.elements['user'].try(:text) - proto = cred.elements['proto'].try(:text) - sname = cred.elements['sname'].try(:text) - port = cred.elements['port'].try(:text) - - # Handle blanks by resetting to sane default values - proto = "tcp" if proto.blank? - pass = cred.elements['pass'].try(:text) - pass = "" if pass == "*MASKED*" - - private = create_credential_private(private_data: pass, private_type: :password) - public = create_credential_public(username: username) - core = create_credential_core(private: private, public: public, origin: origin, workspace_id: wspace.id) - - create_credential_login(core: core, - workspace_id: wspace.id, - address: hobj.address, - port: port, - protocol: proto, - service_name: sname, - status: Metasploit::Model::Login::Status::UNTRIED) - end - end - end - end - - - host.elements.each('sessions/session') do |sess| - sess_id = nils_for_nulls(sess.elements["id"].text.to_s.strip.to_i) - sess_data = {} - sess_data[:host] = hobj - %W{desc platform port stype}.each {|datum| - if sess.elements[datum].respond_to? :text - sess_data[datum.intern] = nils_for_nulls(sess.elements[datum].text.to_s.strip) - end - } - %W{opened-at close-reason closed-at via-exploit via-payload}.each {|datum| - if sess.elements[datum].respond_to? :text - sess_data[datum.gsub("-","_").intern] = nils_for_nulls(sess.elements[datum].text.to_s.strip) - end - } - sess_data[:datastore] = nils_for_nulls(unserialize_object(sess.elements["datastore"], allow_yaml)) - if sess.elements["routes"] - sess_data[:routes] = nils_for_nulls(unserialize_object(sess.elements["routes"], allow_yaml)) || [] - end - if not sess_data[:closed_at] # Fake a close if we don't already have one - sess_data[:closed_at] = Time.now.utc - sess_data[:close_reason] = "Imported at #{Time.now.utc}" - end - - existing_session = get_session( - :workspace => sess_data[:host].workspace, - :addr => sess_data[:host].address, - :time => sess_data[:opened_at] - ) - this_session = existing_session || report_session(sess_data) - next if existing_session - sess.elements.each('events/event') do |sess_event| - sess_event_data = {} - sess_event_data[:session] = this_session - %W{created-at etype local-path remote-path}.each {|datum| - if sess_event.elements[datum].respond_to? :text - sess_event_data[datum.gsub("-","_").intern] = nils_for_nulls(sess_event.elements[datum].text.to_s.strip) - end - } - %W{command output}.each {|datum| - if sess_event.elements[datum].respond_to? :text - sess_event_data[datum.gsub("-","_").intern] = nils_for_nulls(unserialize_object(sess_event.elements[datum], allow_yaml)) - end - } - report_session_event(sess_event_data) - end - end - end - - - # Import web sites - doc.elements.each("/#{btag}/web_sites/web_site") do |web| - info = {} - info[:workspace] = wspace - - %W{host port vhost ssl comments}.each do |datum| - if web.elements[datum].respond_to? :text - info[datum.intern] = nils_for_nulls(web.elements[datum].text.to_s.strip) - end - end - - info[:options] = nils_for_nulls(unserialize_object(web.elements["options"], allow_yaml)) if web.elements["options"].respond_to?(:text) - info[:ssl] = (info[:ssl] and info[:ssl].to_s.strip.downcase == "true") ? true : false - - %W{created-at updated-at}.each { |datum| - if web.elements[datum].text - info[datum.gsub("-","_")] = nils_for_nulls(web.elements[datum].text.to_s.strip) - end - } - - report_web_site(info) - yield(:web_site, "#{info[:host]}:#{info[:port]} (#{info[:vhost]})") if block - end - - %W{page form vuln}.each do |wtype| - doc.elements.each("/#{btag}/web_#{wtype}s/web_#{wtype}") do |element| - send( - "import_msf_web_#{wtype}_element", - element, - :allow_yaml => allow_yaml, - :workspace => wspace, - &block - ) - end - end + if confidence + info[:confidence] = confidence.to_i end - private + # FIXME https://www.pivotaltracker.com/story/show/46578647 + # FIXME https://www.pivotaltracker.com/story/show/47128407 + unserialized_params = unserialize_object( + element.elements['params'], + options[:allow_yaml] + ) + info[:params] = nils_for_nulls(unserialized_params) - # Checks if the XML document has a format version that the importer - # understands. - # - # @param document [REXML::Document] a REXML::Document produced by - # {Msf::DBManager#rexmlify}. - # @return [Hash{Symbol => Object}] `:allow_yaml` is true if the format - # requires YAML loading when calling - # {Msf::DBManager#unserialize_object}. `:root_tag` the tag name of the - # root element for MSF XML. - # @raise [Msf::DBImportError] if unsupported format - def check_msf_xml_version!(document) - metadata = { - # FIXME https://www.pivotaltracker.com/story/show/47128407 - :allow_yaml => false, - :root_tag => nil + risk = info[:risk] + + if risk + info[:risk] = risk.to_i + end + + info + end + end + + # For each host, step through services, notes, and vulns, and import + # them. + # TODO: loot, tasks, and reports + def import_msf_xml(args={}, &block) + data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + + doc = rexmlify(data) + metadata = check_msf_xml_version!(doc) + allow_yaml = metadata[:allow_yaml] + btag = metadata[:root_tag] + + doc.elements.each("/#{btag}/hosts/host") do |host| + host_data = {} + host_data[:task] = args[:task] + host_data[:workspace] = wspace + host_data[:host] = nils_for_nulls(host.elements["address"].text.to_s.strip) + if bl.include? host_data[:host] + next + else + yield(:address,host_data[:host]) if block + end + host_data[:mac] = nils_for_nulls(host.elements["mac"].text.to_s.strip) + if host.elements["comm"].text + host_data[:comm] = nils_for_nulls(host.elements["comm"].text.to_s.strip) + end + %W{created-at updated-at name state os-flavor os-lang os-name os-sp purpose}.each { |datum| + if host.elements[datum].text + host_data[datum.gsub('-','_')] = nils_for_nulls(host.elements[datum].text.to_s.strip) + end + } + host_address = host_data[:host].dup # Preserve after report_host() deletes + hobj = report_host(host_data) + + host.elements.each("host_details/host_detail") do |hdet| + hdet_data = {} + hdet.elements.each do |det| + next if ["id", "host-id"].include?(det.name) + if det.text + hdet_data[det.name.gsub('-','_')] = nils_for_nulls(det.text.to_s.strip) + end + end + report_host_details(hobj, hdet_data) + end + + host.elements.each("exploit_attempts/exploit_attempt") do |hdet| + hdet_data = {} + hdet.elements.each do |det| + next if ["id", "host-id", "session-id", "vuln-id", "service-id", "loot-id"].include?(det.name) + if det.text + hdet_data[det.name.gsub('-','_')] = nils_for_nulls(det.text.to_s.strip) + end + end + report_exploit_attempt(hobj, hdet_data) + end + + host.elements.each('services/service') do |service| + service_data = {} + service_data[:task] = args[:task] + service_data[:workspace] = wspace + service_data[:host] = hobj + service_data[:port] = nils_for_nulls(service.elements["port"].text.to_s.strip).to_i + service_data[:proto] = nils_for_nulls(service.elements["proto"].text.to_s.strip) + %W{created-at updated-at name state info}.each { |datum| + if service.elements[datum].text + if datum == "info" + service_data["info"] = nils_for_nulls(unserialize_object(service.elements[datum], false)) + else + service_data[datum.gsub("-","_")] = nils_for_nulls(service.elements[datum].text.to_s.strip) + end + end } - - if document.elements['MetasploitExpressV1'] - # FIXME https://www.pivotaltracker.com/story/show/47128407 - metadata[:allow_yaml] = true - metadata[:root_tag] = 'MetasploitExpressV1' - elsif document.elements['MetasploitExpressV2'] - # FIXME https://www.pivotaltracker.com/story/show/47128407 - metadata[:allow_yaml] = true - metadata[:root_tag] = 'MetasploitExpressV2' - elsif document.elements['MetasploitExpressV3'] - metadata[:root_tag] = 'MetasploitExpressV3' - elsif document.elements['MetasploitExpressV4'] - metadata[:root_tag] = 'MetasploitExpressV4' - elsif document.elements['MetasploitV4'] - metadata[:root_tag] = 'MetasploitV4' - elsif document.elements['MetasploitV5'] - metadata[:root_tag] = 'MetasploitV5' - end - - unless metadata[:root_tag] - raise Msf::DBImportError, - 'Unsupported Metasploit XML document format' - end - - metadata + report_service(service_data) end - # Retrieves text of element if it exists. - # - # @param parent_element [REXML::Element] element under which element with - # `child_name` exists. - # @param child_name [String] the name of the element under - # `parent_element` whose text should be returned - # @return [{}] if element with child_name does not exist or does not have - # text. - # @return [Hash{Symbol => String}] Maps child_name symbol to text. Text is - # stripped and any NULLs are converted to `nil`. - # @return [nil] if element with `child_name` does not exist under - # `parent_element`. - def import_msf_text_element(parent_element, child_name) - child_element = parent_element.elements[child_name] - info = {} + host.elements.each('notes/note') do |note| + note_data = {} + note_data[:workspace] = wspace + note_data[:host] = hobj + note_data[:type] = nils_for_nulls(note.elements["ntype"].text.to_s.strip) + note_data[:data] = nils_for_nulls(unserialize_object(note.elements["data"], allow_yaml)) - if child_element - stripped = child_element.text.to_s.strip - attribute_name = child_name.underscore.to_sym - info[attribute_name] = nils_for_nulls(stripped) + if note.elements["critical"].text + note_data[:critical] = true unless note.elements["critical"].text.to_s.strip == "NULL" end - - info + if note.elements["seen"].text + note_data[:seen] = true unless note.elements["critical"].text.to_s.strip == "NULL" + end + %W{created-at updated-at}.each { |datum| + if note.elements[datum].text + note_data[datum.gsub("-","_")] = nils_for_nulls(note.elements[datum].text.to_s.strip) + end + } + report_note(note_data) end - # Imports web_form, web_page, or web_vuln element using - # {Msf::DBManager#report_web_form}, {Msf::DBManager#report_web_page}, and - # {Msf::DBManager#report_web_vuln}, respectively. - # - # @param element [REXML::Element] the web_form, web_page, or web_vuln - # element. - # @param options [Hash{Symbol => Object}] options - # @option options [Boolean] :allow_yaml (false) Whether to allow YAML when - # deserializing elements. - # @option options [Proc] :notifier Block called with web_* event and path - # @option options [Symbol] :type the type of web element, such as :form, - # :page, or :vuln. Must correspond to a report_web_ method on - # {Msf::DBManager}. - # @option options [Mdm::Workspace, nil] :workspace - # (Msf::DBManager#workspace) workspace under which to report the - # imported record. - # @yield [element, options] - # @yieldparam element [REXML::Element] the web_form, web_page, or - # web_vuln element passed to {#import_msf_web_element}. - # @yieldparam options [Hash{Symbol => Object}] options for parsing - # @yieldreturn [Hash{Symbol => Object}] info - # @return [void] - # @raise [KeyError] if `:type` is not given - def import_msf_web_element(element, options={}, &specialization) - options.assert_valid_keys(:allow_yaml, :notifier, :type, :workspace) - type = options.fetch(:type) + host.elements.each('tags/tag') do |tag| + tag_data = {} + tag_data[:addr] = host_address + tag_data[:wspace] = wspace + tag_data[:name] = tag.elements["name"].text.to_s.strip + tag_data[:desc] = tag.elements["desc"].text.to_s.strip + if tag.elements["report-summary"].text + tag_data[:summary] = tag.elements["report-summary"].text.to_s.strip + end + if tag.elements["report-detail"].text + tag_data[:detail] = tag.elements["report-detail"].text.to_s.strip + end + if tag.elements["critical"].text + tag_data[:crit] = true unless tag.elements["critical"].text.to_s.strip == "NULL" + end + report_host_tag(tag_data) + end - info = {} - info[:workspace] = options[:workspace] || self.workspace - - MSF_WEB_TEXT_ELEMENT_NAMES.each do |name| - element_info = import_msf_text_element(element, name) - info.merge!(element_info) + host.elements.each('vulns/vuln') do |vuln| + vuln_data = {} + vuln_data[:workspace] = wspace + vuln_data[:host] = hobj + vuln_data[:info] = nils_for_nulls(unserialize_object(vuln.elements["info"], allow_yaml)) + vuln_data[:name] = nils_for_nulls(vuln.elements["name"].text.to_s.strip) + %W{created-at updated-at exploited-at}.each { |datum| + if vuln.elements[datum] and vuln.elements[datum].text + vuln_data[datum.gsub("-","_")] = nils_for_nulls(vuln.elements[datum].text.to_s.strip) + end + } + if vuln.elements["refs"] + vuln_data[:refs] = [] + vuln.elements.each("refs/ref") do |ref| + vuln_data[:refs] << nils_for_nulls(ref.text.to_s.strip) + end end - info[:ssl] = (info[:ssl] and info[:ssl].to_s.strip.downcase == "true") ? true : false + vobj = report_vuln(vuln_data) - specialized_info = specialization.call(element, options) - info.merge!(specialized_info) + vuln.elements.each("vuln_details/vuln_detail") do |vdet| + vdet_data = {} + vdet.elements.each do |det| + next if ["id", "vuln-id"].include?(det.name) + if det.text + vdet_data[det.name.gsub('-','_')] = nils_for_nulls(det.text.to_s.strip) + end + end + report_vuln_details(vobj, vdet_data) + end - self.send("report_web_#{type}", info) + vuln.elements.each("vuln_attempts/vuln_attempt") do |vdet| + vdet_data = {} + vdet.elements.each do |det| + next if ["id", "vuln-id", "loot-id", "session-id"].include?(det.name) + if det.text + vdet_data[det.name.gsub('-','_')] = nils_for_nulls(det.text.to_s.strip) + end + end + report_vuln_attempt(vobj, vdet_data) + end + end - notifier = options[:notifier] + ## Handle old-style (pre 4.10) XML files + if btag == "MetasploitV4" + if host.elements['creds'].present? + unless host.elements['creds'].elements.empty? + origin = Metasploit::Credential::Origin::Import.create(filename: "console-import-#{Time.now.to_i}") - if notifier - event = "web_#{type}".to_sym - notifier.call(event, info[:path]) + host.elements.each('creds/cred') do |cred| + username = cred.elements['user'].try(:text) + proto = cred.elements['proto'].try(:text) + sname = cred.elements['sname'].try(:text) + port = cred.elements['port'].try(:text) + + # Handle blanks by resetting to sane default values + proto = "tcp" if proto.blank? + pass = cred.elements['pass'].try(:text) + pass = "" if pass == "*MASKED*" + + private = create_credential_private(private_data: pass, private_type: :password) + public = create_credential_public(username: username) + core = create_credential_core(private: private, public: public, origin: origin, workspace_id: wspace.id) + + create_credential_login(core: core, + workspace_id: wspace.id, + address: hobj.address, + port: port, + protocol: proto, + service_name: sname, + status: Metasploit::Model::Login::Status::UNTRIED) + end + end + end + end + + + host.elements.each('sessions/session') do |sess| + sess_id = nils_for_nulls(sess.elements["id"].text.to_s.strip.to_i) + sess_data = {} + sess_data[:host] = hobj + %W{desc platform port stype}.each {|datum| + if sess.elements[datum].respond_to? :text + sess_data[datum.intern] = nils_for_nulls(sess.elements[datum].text.to_s.strip) + end + } + %W{opened-at close-reason closed-at via-exploit via-payload}.each {|datum| + if sess.elements[datum].respond_to? :text + sess_data[datum.gsub("-","_").intern] = nils_for_nulls(sess.elements[datum].text.to_s.strip) + end + } + sess_data[:datastore] = nils_for_nulls(unserialize_object(sess.elements["datastore"], allow_yaml)) + if sess.elements["routes"] + sess_data[:routes] = nils_for_nulls(unserialize_object(sess.elements["routes"], allow_yaml)) || [] + end + if not sess_data[:closed_at] # Fake a close if we don't already have one + sess_data[:closed_at] = Time.now.utc + sess_data[:close_reason] = "Imported at #{Time.now.utc}" + end + + existing_session = get_session( + :workspace => sess_data[:host].workspace, + :addr => sess_data[:host].address, + :time => sess_data[:opened_at] + ) + this_session = existing_session || report_session(sess_data) + next if existing_session + sess.elements.each('events/event') do |sess_event| + sess_event_data = {} + sess_event_data[:session] = this_session + %W{created-at etype local-path remote-path}.each {|datum| + if sess_event.elements[datum].respond_to? :text + sess_event_data[datum.gsub("-","_").intern] = nils_for_nulls(sess_event.elements[datum].text.to_s.strip) + end + } + %W{command output}.each {|datum| + if sess_event.elements[datum].respond_to? :text + sess_event_data[datum.gsub("-","_").intern] = nils_for_nulls(unserialize_object(sess_event.elements[datum], allow_yaml)) + end + } + report_session_event(sess_event_data) end end end + + + # Import web sites + doc.elements.each("/#{btag}/web_sites/web_site") do |web| + info = {} + info[:workspace] = wspace + + %W{host port vhost ssl comments}.each do |datum| + if web.elements[datum].respond_to? :text + info[datum.intern] = nils_for_nulls(web.elements[datum].text.to_s.strip) + end + end + + info[:options] = nils_for_nulls(unserialize_object(web.elements["options"], allow_yaml)) if web.elements["options"].respond_to?(:text) + info[:ssl] = (info[:ssl] and info[:ssl].to_s.strip.downcase == "true") ? true : false + + %W{created-at updated-at}.each { |datum| + if web.elements[datum].text + info[datum.gsub("-","_")] = nils_for_nulls(web.elements[datum].text.to_s.strip) + end + } + + report_web_site(info) + yield(:web_site, "#{info[:host]}:#{info[:port]} (#{info[:vhost]})") if block + end + + %W{page form vuln}.each do |wtype| + doc.elements.each("/#{btag}/web_#{wtype}s/web_#{wtype}") do |element| + send( + "import_msf_web_#{wtype}_element", + element, + :allow_yaml => allow_yaml, + :workspace => wspace, + &block + ) + end + end + end + + private + + # Checks if the XML document has a format version that the importer + # understands. + # + # @param document [REXML::Document] a REXML::Document produced by + # {Msf::DBManager#rexmlify}. + # @return [Hash{Symbol => Object}] `:allow_yaml` is true if the format + # requires YAML loading when calling + # {Msf::DBManager#unserialize_object}. `:root_tag` the tag name of the + # root element for MSF XML. + # @raise [Msf::DBImportError] if unsupported format + def check_msf_xml_version!(document) + metadata = { + # FIXME https://www.pivotaltracker.com/story/show/47128407 + :allow_yaml => false, + :root_tag => nil + } + + if document.elements['MetasploitExpressV1'] + # FIXME https://www.pivotaltracker.com/story/show/47128407 + metadata[:allow_yaml] = true + metadata[:root_tag] = 'MetasploitExpressV1' + elsif document.elements['MetasploitExpressV2'] + # FIXME https://www.pivotaltracker.com/story/show/47128407 + metadata[:allow_yaml] = true + metadata[:root_tag] = 'MetasploitExpressV2' + elsif document.elements['MetasploitExpressV3'] + metadata[:root_tag] = 'MetasploitExpressV3' + elsif document.elements['MetasploitExpressV4'] + metadata[:root_tag] = 'MetasploitExpressV4' + elsif document.elements['MetasploitV4'] + metadata[:root_tag] = 'MetasploitV4' + elsif document.elements['MetasploitV5'] + metadata[:root_tag] = 'MetasploitV5' + end + + unless metadata[:root_tag] + raise Msf::DBImportError, + 'Unsupported Metasploit XML document format' + end + + metadata + end + + # Retrieves text of element if it exists. + # + # @param parent_element [REXML::Element] element under which element with + # `child_name` exists. + # @param child_name [String] the name of the element under + # `parent_element` whose text should be returned + # @return [{}] if element with child_name does not exist or does not have + # text. + # @return [Hash{Symbol => String}] Maps child_name symbol to text. Text is + # stripped and any NULLs are converted to `nil`. + # @return [nil] if element with `child_name` does not exist under + # `parent_element`. + def import_msf_text_element(parent_element, child_name) + child_element = parent_element.elements[child_name] + info = {} + + if child_element + stripped = child_element.text.to_s.strip + attribute_name = child_name.underscore.to_sym + info[attribute_name] = nils_for_nulls(stripped) + end + + info + end + + # Imports web_form, web_page, or web_vuln element using + # {Msf::DBManager#report_web_form}, {Msf::DBManager#report_web_page}, and + # {Msf::DBManager#report_web_vuln}, respectively. + # + # @param element [REXML::Element] the web_form, web_page, or web_vuln + # element. + # @param options [Hash{Symbol => Object}] options + # @option options [Boolean] :allow_yaml (false) Whether to allow YAML when + # deserializing elements. + # @option options [Proc] :notifier Block called with web_* event and path + # @option options [Symbol] :type the type of web element, such as :form, + # :page, or :vuln. Must correspond to a report_web_ method on + # {Msf::DBManager}. + # @option options [Mdm::Workspace, nil] :workspace + # (Msf::DBManager#workspace) workspace under which to report the + # imported record. + # @yield [element, options] + # @yieldparam element [REXML::Element] the web_form, web_page, or + # web_vuln element passed to {#import_msf_web_element}. + # @yieldparam options [Hash{Symbol => Object}] options for parsing + # @yieldreturn [Hash{Symbol => Object}] info + # @return [void] + # @raise [KeyError] if `:type` is not given + def import_msf_web_element(element, options={}, &specialization) + options.assert_valid_keys(:allow_yaml, :notifier, :type, :workspace) + type = options.fetch(:type) + + info = {} + info[:workspace] = options[:workspace] || self.workspace + + MSF_WEB_TEXT_ELEMENT_NAMES.each do |name| + element_info = import_msf_text_element(element, name) + info.merge!(element_info) + end + + info[:ssl] = (info[:ssl] and info[:ssl].to_s.strip.downcase == "true") ? true : false + + specialized_info = specialization.call(element, options) + info.merge!(specialized_info) + + self.send("report_web_#{type}", info) + + notifier = options[:notifier] + + if notifier + event = "web_#{type}".to_sym + notifier.call(event, info[:path]) + end end end - From 87825d40b1a7a55805c8f78fc3b70ff22d38e83c Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 15:39:15 -0500 Subject: [PATCH 083/159] Fix migration.rb loading MSP-11124 --- lib/msf/core/db_manager.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/msf/core/db_manager.rb b/lib/msf/core/db_manager.rb index 10f8700fab..95002b9630 100644 --- a/lib/msf/core/db_manager.rb +++ b/lib/msf/core/db_manager.rb @@ -15,7 +15,6 @@ require 'msf/base/config' require 'msf/core' require 'msf/core/database_event' require 'msf/core/db_import_error' -require 'msf/core/db_manager/migration' require 'msf/core/host_state' require 'msf/core/service_state' require 'msf/core/task_manager' @@ -39,6 +38,7 @@ class Msf::DBManager autoload :ImportMsfXml, 'msf/core/db_manager/import_msf_xml' autoload :IPAddress, 'msf/core/db_manager/ip_address' autoload :Loot, 'msf/core/db_manager/loot' + autoload :Migration, 'msf/core/db_manager/migration' autoload :ModuleCache, 'msf/core/db_manager/module_cache' autoload :Note, 'msf/core/db_manager/note' autoload :Ref, 'msf/core/db_manager/ref' From a7d157749477777e684642f36b803445410fbb23 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 15:46:13 -0500 Subject: [PATCH 084/159] ImportMsfXml -> Import::MsfXml MSP-11124 --- lib/msf/core/db_manager.rb | 1 - lib/msf/core/db_manager/import.rb | 4 ++++ .../{import_msf_xml.rb => import/msf_xml.rb} | 2 +- spec/lib/msf/db_manager_spec.rb | 1 - .../shared/examples/msf/db_manager/import.rb | 2 ++ .../{import_msf_xml.rb => import/msf_xml.rb} | 18 +++++++++--------- .../check_msf_xml_version_with_root_tag.rb | 2 +- .../import_msf_web_element_specialization.rb | 2 +- 8 files changed, 18 insertions(+), 14 deletions(-) rename lib/msf/core/db_manager/{import_msf_xml.rb => import/msf_xml.rb} (99%) rename spec/support/shared/examples/msf/db_manager/{import_msf_xml.rb => import/msf_xml.rb} (97%) rename spec/support/shared/examples/msf/db_manager/{import_msf_xml => import/msf_xml}/check_msf_xml_version_with_root_tag.rb (82%) rename spec/support/shared/examples/msf/db_manager/{import_msf_xml => import/msf_xml}/import_msf_web_element_specialization.rb (90%) diff --git a/lib/msf/core/db_manager.rb b/lib/msf/core/db_manager.rb index 95002b9630..423211ee64 100644 --- a/lib/msf/core/db_manager.rb +++ b/lib/msf/core/db_manager.rb @@ -69,7 +69,6 @@ class Msf::DBManager include Msf::DBManager::HostDetail include Msf::DBManager::HostTag include Msf::DBManager::Import - include Msf::DBManager::ImportMsfXml include Msf::DBManager::IPAddress include Msf::DBManager::Loot include Msf::DBManager::Migration diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index 10c92e7111..9b29330798 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -34,6 +34,10 @@ require 'rex/parser/retina_xml' require 'rex/parser/wapiti_nokogiri' module Msf::DBManager::Import + autoload :MsfXml, 'msf/core/db_manager/import/msf_xml' + + include Msf::DBManager::Import::MsfXml + # If hex notation is present, turn them into a character. def dehex(str) hexen = str.scan(/\x5cx[0-9a-fA-F]{2}/n) diff --git a/lib/msf/core/db_manager/import_msf_xml.rb b/lib/msf/core/db_manager/import/msf_xml.rb similarity index 99% rename from lib/msf/core/db_manager/import_msf_xml.rb rename to lib/msf/core/db_manager/import/msf_xml.rb index ad4d85a98c..f5af278c1d 100644 --- a/lib/msf/core/db_manager/import_msf_xml.rb +++ b/lib/msf/core/db_manager/import/msf_xml.rb @@ -4,7 +4,7 @@ # methods to be overridden in Pro without using alias_method_chain as # methods defined in a class cannot be overridden by including a module # (unless you're running Ruby 2.0 and can use prepend) -module Msf::DBManager::ImportMsfXml +module Msf::DBManager::Import::MsfXml # # CONSTANTS # diff --git a/spec/lib/msf/db_manager_spec.rb b/spec/lib/msf/db_manager_spec.rb index a7959e3ef5..b3f2d69877 100644 --- a/spec/lib/msf/db_manager_spec.rb +++ b/spec/lib/msf/db_manager_spec.rb @@ -30,7 +30,6 @@ describe Msf::DBManager do it_should_behave_like 'Msf::DBManager::HostTag' it_should_behave_like 'Msf::DBManager::IPAddress' it_should_behave_like 'Msf::DBManager::Import' - it_should_behave_like 'Msf::DBManager::ImportMsfXml' it_should_behave_like 'Msf::DBManager::Loot' it_should_behave_like 'Msf::DBManager::Migration' it_should_behave_like 'Msf::DBManager::ModuleCache' diff --git a/spec/support/shared/examples/msf/db_manager/import.rb b/spec/support/shared/examples/msf/db_manager/import.rb index ebcee2912e..e5a5e8b9b3 100644 --- a/spec/support/shared/examples/msf/db_manager/import.rb +++ b/spec/support/shared/examples/msf/db_manager/import.rb @@ -82,4 +82,6 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :rexmlify } it { is_expected.to respond_to :unserialize_object } it { is_expected.to respond_to :validate_import_file } + + it_should_behave_like 'Msf::DBManager::Import::MsfXml' end \ No newline at end of file diff --git a/spec/support/shared/examples/msf/db_manager/import_msf_xml.rb b/spec/support/shared/examples/msf/db_manager/import/msf_xml.rb similarity index 97% rename from spec/support/shared/examples/msf/db_manager/import_msf_xml.rb rename to spec/support/shared/examples/msf/db_manager/import/msf_xml.rb index cccf0096e7..94030b2476 100644 --- a/spec/support/shared/examples/msf/db_manager/import_msf_xml.rb +++ b/spec/support/shared/examples/msf/db_manager/import/msf_xml.rb @@ -1,7 +1,7 @@ # -*- coding:binary -*- require 'builder' -shared_examples_for 'Msf::DBManager::ImportMsfXml' do +shared_examples_for 'Msf::DBManager::Import::MsfXml' do # Serialized format from pro/modules/auxiliary/pro/report.rb def serialize(object) # FIXME https://www.pivotaltracker.com/story/show/46578647 @@ -85,7 +85,7 @@ shared_examples_for 'Msf::DBManager::ImportMsfXml' do end it 'should include methods from module so method can be overridden easier in pro' do - db_manager.should be_a Msf::DBManager::ImportMsfXml + db_manager.should be_a Msf::DBManager::Import::MsfXml end context 'CONSTANTS' do @@ -136,25 +136,25 @@ shared_examples_for 'Msf::DBManager::ImportMsfXml' do end it_should_behave_like( - 'Msf::DBManager::ImportMsfXml#check_msf_xml_version! with root tag', + 'Msf::DBManager::Import::MsfXml#check_msf_xml_version! with root tag', 'MetasploitExpressV1', :allow_yaml => true ) it_should_behave_like( - 'Msf::DBManager::ImportMsfXml#check_msf_xml_version! with root tag', + 'Msf::DBManager::Import::MsfXml#check_msf_xml_version! with root tag', 'MetasploitExpressV2', :allow_yaml => true ) it_should_behave_like( - 'Msf::DBManager::ImportMsfXml#check_msf_xml_version! with root tag', + 'Msf::DBManager::Import::MsfXml#check_msf_xml_version! with root tag', 'MetasploitExpressV3', :allow_yaml => false ) it_should_behave_like( - 'Msf::DBManager::ImportMsfXml#check_msf_xml_version! with root tag', + 'Msf::DBManager::Import::MsfXml#check_msf_xml_version! with root tag', 'MetasploitExpressV4', :allow_yaml => false ) @@ -573,7 +573,7 @@ shared_examples_for 'Msf::DBManager::ImportMsfXml' do end context 'call to #import_msf_web_element' do - it_should_behave_like 'Msf::DBManager::ImportMsfXml#import_msf_web_element specialization' + it_should_behave_like 'Msf::DBManager::Import::MsfXml#import_msf_web_element specialization' context 'specialization return' do let(:element) do @@ -669,7 +669,7 @@ shared_examples_for 'Msf::DBManager::ImportMsfXml' do end context 'call to #import_msf_web_element' do - it_should_behave_like 'Msf::DBManager::ImportMsfXml#import_msf_web_element specialization' + it_should_behave_like 'Msf::DBManager::Import::MsfXml#import_msf_web_element specialization' context 'specialization return' do let(:element) do @@ -840,7 +840,7 @@ shared_examples_for 'Msf::DBManager::ImportMsfXml' do end context 'call to #import_msf_web_element' do - it_should_behave_like 'Msf::DBManager::ImportMsfXml#import_msf_web_element specialization' + it_should_behave_like 'Msf::DBManager::Import::MsfXml#import_msf_web_element specialization' context 'specialization return' do let(:element) do diff --git a/spec/support/shared/examples/msf/db_manager/import_msf_xml/check_msf_xml_version_with_root_tag.rb b/spec/support/shared/examples/msf/db_manager/import/msf_xml/check_msf_xml_version_with_root_tag.rb similarity index 82% rename from spec/support/shared/examples/msf/db_manager/import_msf_xml/check_msf_xml_version_with_root_tag.rb rename to spec/support/shared/examples/msf/db_manager/import/msf_xml/check_msf_xml_version_with_root_tag.rb index 017b08d427..56794c81a0 100644 --- a/spec/support/shared/examples/msf/db_manager/import_msf_xml/check_msf_xml_version_with_root_tag.rb +++ b/spec/support/shared/examples/msf/db_manager/import/msf_xml/check_msf_xml_version_with_root_tag.rb @@ -1,5 +1,5 @@ # -*- coding:binary -*- -shared_examples_for 'Msf::DBManager::ImportMsfXml#check_msf_xml_version! with root tag' do |root_tag, options={}| +shared_examples_for 'Msf::DBManager::Import::MsfXml#check_msf_xml_version! with root tag' do |root_tag, options={}| options.assert_valid_keys(:allow_yaml) allow_yaml = options.fetch(:allow_yaml) diff --git a/spec/support/shared/examples/msf/db_manager/import_msf_xml/import_msf_web_element_specialization.rb b/spec/support/shared/examples/msf/db_manager/import/msf_xml/import_msf_web_element_specialization.rb similarity index 90% rename from spec/support/shared/examples/msf/db_manager/import_msf_xml/import_msf_web_element_specialization.rb rename to spec/support/shared/examples/msf/db_manager/import/msf_xml/import_msf_web_element_specialization.rb index 8ccf5f5573..12b88f682c 100644 --- a/spec/support/shared/examples/msf/db_manager/import_msf_xml/import_msf_web_element_specialization.rb +++ b/spec/support/shared/examples/msf/db_manager/import/msf_xml/import_msf_web_element_specialization.rb @@ -1,5 +1,5 @@ # -*- coding:binary -*- -shared_examples_for 'Msf::DBManager::ImportMsfXml#import_msf_web_element specialization' do +shared_examples_for 'Msf::DBManager::Import::MsfXml#import_msf_web_element specialization' do it 'should call #import_msf_web_element with element' do db_manager.should_receive(:import_msf_web_element).with(element, anything) From 1f49f767dce13c7c5193cdbdf559ab00bd25a821 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 16:06:15 -0500 Subject: [PATCH 085/159] Extract Msf::DBManager::Import::Qualys::Asset MSP-11124 --- lib/msf/core/db_manager/import.rb | 99 +------------------ lib/msf/core/db_manager/import/qualys.rb | 5 + .../core/db_manager/import/qualys/asset.rb | 98 ++++++++++++++++++ 3 files changed, 105 insertions(+), 97 deletions(-) create mode 100644 lib/msf/core/db_manager/import/qualys.rb create mode 100644 lib/msf/core/db_manager/import/qualys/asset.rb diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index 9b29330798..760b0d3d16 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -35,8 +35,10 @@ require 'rex/parser/wapiti_nokogiri' module Msf::DBManager::Import autoload :MsfXml, 'msf/core/db_manager/import/msf_xml' + autoload :Qualys, 'msf/core/db_manager/import/qualys' include Msf::DBManager::Import::MsfXml + include Msf::DBManager::Import::Qualys # If hex notation is present, turn them into a character. def dehex(str) @@ -53,54 +55,6 @@ module Msf::DBManager::Import yield(sym,data) end - # Takes QID numbers and finds the discovered services in - # a qualys_asset_xml. - def find_qualys_asset_ports(i,host,wspace,hobj,task_id) - return unless (i == 82023 || i == 82004) - proto = i == 82023 ? 'tcp' : 'udp' - qid = host.elements["VULN_INFO_LIST/VULN_INFO/QID[@id='qid_#{i}']"] - qid_result = qid.parent.elements["RESULT[@format='table']"] if qid - hports = qid_result.first.to_s if qid_result - if hports - hports.scan(/([0-9]+)\t(.*?)\t.*?\t([^\t\n]*)/) do |match| - if match[2] == nil or match[2].strip == 'unknown' - name = match[1].strip - else - name = match[2].strip - end - handle_qualys(wspace, hobj, match[0].to_s, proto, 0, nil, nil, name, nil, task_id) - end - end - end - - def find_qualys_asset_vuln_refs(doc) - vuln_refs = {} - doc.elements.each("/ASSET_DATA_REPORT/GLOSSARY/VULN_DETAILS_LIST/VULN_DETAILS") do |vuln| - next unless vuln.elements['QID'] && vuln.elements['QID'].first - qid = vuln.elements['QID'].first.to_s - vuln_refs[qid] ||= [] - vuln.elements.each('CVE_ID_LIST/CVE_ID') do |ref| - vuln_refs[qid].push('CVE-' + /C..-([0-9\-]{9})/.match(ref.elements['ID'].text.to_s)[1]) - end - vuln.elements.each('BUGTRAQ_ID_LIST/BUGTRAQ_ID') do |ref| - vuln_refs[qid].push('BID-' + ref.elements['ID'].text.to_s) - end - end - return vuln_refs - end - - # Pull out vulnerabilities that have at least one matching - # ref -- many "vulns" are not vulns, just audit information. - def find_qualys_asset_vulns(host,wspace,hobj,vuln_refs,task_id,&block) - host.elements.each("VULN_INFO_LIST/VULN_INFO") do |vi| - next unless vi.elements["QID"] - vi.elements.each("QID") do |qid| - next if vuln_refs[qid.text].nil? || vuln_refs[qid.text].empty? - handle_qualys(wspace, hobj, nil, nil, qid.text, nil, vuln_refs[qid.text], nil, nil, task_id) - end - end - end - # A dispatcher method that figures out the data's file type, # and sends it off to the appropriate importer. Note that # import_file_detect will raise an error if the filetype @@ -2399,55 +2353,6 @@ module Msf::DBManager::Import end end - # - # Import Qualys's Asset Data Report format - # - def import_qualys_asset_xml(args={}, &block) - data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - doc = rexmlify(data) - vuln_refs = find_qualys_asset_vuln_refs(doc) - - # 2nd pass, actually grab the hosts. - doc.elements.each("/ASSET_DATA_REPORT/HOST_LIST/HOST") do |host| - hobj = nil - addr = host.elements["IP"].text if host.elements["IP"] - next unless validate_ips(addr) - if bl.include? addr - next - else - yield(:address,addr) if block - end - hname = ( # Prefer NetBIOS over DNS - (host.elements["NETBIOS"].text if host.elements["NETBIOS"]) || - (host.elements["DNS"].text if host.elements["DNS"]) || - "" ) - hobj = report_host(:workspace => wspace, :host => addr, :name => hname, :state => Msf::HostState::Alive, :task => args[:task]) - report_import_note(wspace,hobj) - - if host.elements["OPERATING_SYSTEM"] - hos = host.elements["OPERATING_SYSTEM"].text - report_note( - :workspace => wspace, - :task => args[:task], - :host => hobj, - :type => 'host.os.qualys_fingerprint', - :data => { :os => hos } - ) - end - - # Report open ports. - find_qualys_asset_ports(82023,host,wspace,hobj, args[:task]) # TCP - find_qualys_asset_ports(82004,host,wspace,hobj, args[:task]) # UDP - - # Report vulns - find_qualys_asset_vulns(host,wspace,hobj,vuln_refs, args[:task],&block) - - end # host - - end - def import_qualys_scan_xml(args={}, &block) data = args[:data] wspace = args[:wspace] || workspace diff --git a/lib/msf/core/db_manager/import/qualys.rb b/lib/msf/core/db_manager/import/qualys.rb new file mode 100644 index 0000000000..997480ef65 --- /dev/null +++ b/lib/msf/core/db_manager/import/qualys.rb @@ -0,0 +1,5 @@ +module Msf::DBManager::Import::Qualys + autoload :Asset, 'msf/core/db_manager/import/qualys/asset' + + include Msf::DBManager::Import::Qualys::Asset +end \ No newline at end of file diff --git a/lib/msf/core/db_manager/import/qualys/asset.rb b/lib/msf/core/db_manager/import/qualys/asset.rb new file mode 100644 index 0000000000..70ab0c7836 --- /dev/null +++ b/lib/msf/core/db_manager/import/qualys/asset.rb @@ -0,0 +1,98 @@ +module Msf::DBManager::Import::Qualys::Asset + # Takes QID numbers and finds the discovered services in + # a qualys_asset_xml. + def find_qualys_asset_ports(i,host,wspace,hobj,task_id) + return unless (i == 82023 || i == 82004) + proto = i == 82023 ? 'tcp' : 'udp' + qid = host.elements["VULN_INFO_LIST/VULN_INFO/QID[@id='qid_#{i}']"] + qid_result = qid.parent.elements["RESULT[@format='table']"] if qid + hports = qid_result.first.to_s if qid_result + if hports + hports.scan(/([0-9]+)\t(.*?)\t.*?\t([^\t\n]*)/) do |match| + if match[2] == nil or match[2].strip == 'unknown' + name = match[1].strip + else + name = match[2].strip + end + handle_qualys(wspace, hobj, match[0].to_s, proto, 0, nil, nil, name, nil, task_id) + end + end + end + + def find_qualys_asset_vuln_refs(doc) + vuln_refs = {} + doc.elements.each("/ASSET_DATA_REPORT/GLOSSARY/VULN_DETAILS_LIST/VULN_DETAILS") do |vuln| + next unless vuln.elements['QID'] && vuln.elements['QID'].first + qid = vuln.elements['QID'].first.to_s + vuln_refs[qid] ||= [] + vuln.elements.each('CVE_ID_LIST/CVE_ID') do |ref| + vuln_refs[qid].push('CVE-' + /C..-([0-9\-]{9})/.match(ref.elements['ID'].text.to_s)[1]) + end + vuln.elements.each('BUGTRAQ_ID_LIST/BUGTRAQ_ID') do |ref| + vuln_refs[qid].push('BID-' + ref.elements['ID'].text.to_s) + end + end + return vuln_refs + end + + # Pull out vulnerabilities that have at least one matching + # ref -- many "vulns" are not vulns, just audit information. + def find_qualys_asset_vulns(host,wspace,hobj,vuln_refs,task_id,&block) + host.elements.each("VULN_INFO_LIST/VULN_INFO") do |vi| + next unless vi.elements["QID"] + vi.elements.each("QID") do |qid| + next if vuln_refs[qid.text].nil? || vuln_refs[qid.text].empty? + handle_qualys(wspace, hobj, nil, nil, qid.text, nil, vuln_refs[qid.text], nil, nil, task_id) + end + end + end + + # + # Import Qualys's Asset Data Report format + # + def import_qualys_asset_xml(args={}, &block) + data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + doc = rexmlify(data) + vuln_refs = find_qualys_asset_vuln_refs(doc) + + # 2nd pass, actually grab the hosts. + doc.elements.each("/ASSET_DATA_REPORT/HOST_LIST/HOST") do |host| + hobj = nil + addr = host.elements["IP"].text if host.elements["IP"] + next unless validate_ips(addr) + if bl.include? addr + next + else + yield(:address,addr) if block + end + hname = ( # Prefer NetBIOS over DNS + (host.elements["NETBIOS"].text if host.elements["NETBIOS"]) || + (host.elements["DNS"].text if host.elements["DNS"]) || + "" ) + hobj = report_host(:workspace => wspace, :host => addr, :name => hname, :state => Msf::HostState::Alive, :task => args[:task]) + report_import_note(wspace,hobj) + + if host.elements["OPERATING_SYSTEM"] + hos = host.elements["OPERATING_SYSTEM"].text + report_note( + :workspace => wspace, + :task => args[:task], + :host => hobj, + :type => 'host.os.qualys_fingerprint', + :data => { :os => hos } + ) + end + + # Report open ports. + find_qualys_asset_ports(82023,host,wspace,hobj, args[:task]) # TCP + find_qualys_asset_ports(82004,host,wspace,hobj, args[:task]) # UDP + + # Report vulns + find_qualys_asset_vulns(host,wspace,hobj,vuln_refs, args[:task],&block) + + end # host + + end +end \ No newline at end of file From dd5ae26174127e1389aeb4667bd14b528ca0b9fd Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Mon, 13 Oct 2014 16:09:41 -0500 Subject: [PATCH 086/159] Extract Msf::DBManager::Import::Qualys::Asset shared examples MSP-11124 --- spec/support/shared/examples/msf/db_manager/import.rb | 5 +---- .../support/shared/examples/msf/db_manager/import/qualys.rb | 3 +++ .../shared/examples/msf/db_manager/import/qualys/asset.rb | 6 ++++++ 3 files changed, 10 insertions(+), 4 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/import/qualys.rb create mode 100644 spec/support/shared/examples/msf/db_manager/import/qualys/asset.rb diff --git a/spec/support/shared/examples/msf/db_manager/import.rb b/spec/support/shared/examples/msf/db_manager/import.rb index e5a5e8b9b3..78de3b790c 100644 --- a/spec/support/shared/examples/msf/db_manager/import.rb +++ b/spec/support/shared/examples/msf/db_manager/import.rb @@ -1,9 +1,6 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :dehex } it { is_expected.to respond_to :emit } - it { is_expected.to respond_to :find_qualys_asset_ports } - it { is_expected.to respond_to :find_qualys_asset_vuln_refs } - it { is_expected.to respond_to :find_qualys_asset_vulns } it { is_expected.to respond_to :import } it { is_expected.to respond_to :import_acunetix_noko_stream } it { is_expected.to respond_to :import_acunetix_xml } @@ -58,7 +55,6 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :import_openvas_xml } it { is_expected.to respond_to :import_outpost24_noko_stream } it { is_expected.to respond_to :import_outpost24_xml } - it { is_expected.to respond_to :import_qualys_asset_xml } it { is_expected.to respond_to :import_qualys_scan_xml } it { is_expected.to respond_to :import_qualys_scan_xml_file } it { is_expected.to respond_to :import_report } @@ -84,4 +80,5 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :validate_import_file } it_should_behave_like 'Msf::DBManager::Import::MsfXml' + it_should_behave_like 'Msf::DBManager::Import::Qualys' end \ No newline at end of file diff --git a/spec/support/shared/examples/msf/db_manager/import/qualys.rb b/spec/support/shared/examples/msf/db_manager/import/qualys.rb new file mode 100644 index 0000000000..e740dd2be0 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/import/qualys.rb @@ -0,0 +1,3 @@ +shared_examples_for 'Msf::DBManager::Import::Qualys' do + it_should_behave_like 'Msf::DBManager::Import::Qualys::Asset' +end \ No newline at end of file diff --git a/spec/support/shared/examples/msf/db_manager/import/qualys/asset.rb b/spec/support/shared/examples/msf/db_manager/import/qualys/asset.rb new file mode 100644 index 0000000000..f5aace0ae5 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/import/qualys/asset.rb @@ -0,0 +1,6 @@ +shared_examples_for 'Msf::DBManager::Import::Qualys::Asset' do + it { is_expected.to respond_to :find_qualys_asset_ports } + it { is_expected.to respond_to :find_qualys_asset_vuln_refs } + it { is_expected.to respond_to :find_qualys_asset_vulns } + it { is_expected.to respond_to :import_qualys_asset_xml } +end From 11bcac8a4ea783880da1b70f87c54a64a4db0036 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Tue, 14 Oct 2014 15:19:55 -0500 Subject: [PATCH 087/159] Extract Msf::DBManager::Import::Qualys::Scan MSP-11124 --- lib/msf/core/db_manager/import.rb | 97 ------------------ lib/msf/core/db_manager/import/qualys.rb | 2 + lib/msf/core/db_manager/import/qualys/scan.rb | 98 +++++++++++++++++++ 3 files changed, 100 insertions(+), 97 deletions(-) create mode 100644 lib/msf/core/db_manager/import/qualys/scan.rb diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index 760b0d3d16..34969379c6 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -2353,103 +2353,6 @@ module Msf::DBManager::Import end end - def import_qualys_scan_xml(args={}, &block) - data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - - - doc = rexmlify(data) - doc.elements.each('/SCAN/IP') do |host| - hobj = nil - addr = host.attributes['value'] - if bl.include? addr - next - else - yield(:address,addr) if block - end - hname = host.attributes['name'] || '' - - hobj = report_host(:workspace => wspace, :host => addr, :name => hname, :state => Msf::HostState::Alive, :task => args[:task]) - report_import_note(wspace,hobj) - - if host.elements["OS"] - hos = host.elements["OS"].text - report_note( - :workspace => wspace, - :task => args[:task], - :host => hobj, - :type => 'host.os.qualys_fingerprint', - :data => { - :os => hos - } - ) - end - - # Open TCP Services List (Qualys ID 82023) - services_tcp = host.elements["SERVICES/CAT/SERVICE[@number='82023']/RESULT"] - if services_tcp - services_tcp.text.scan(/([0-9]+)\t(.*?)\t.*?\t([^\t\n]*)/) do |match| - if match[2] == nil or match[2].strip == 'unknown' - name = match[1].strip - else - name = match[2].strip - end - handle_qualys(wspace, hobj, match[0].to_s, 'tcp', 0, nil, nil, name, nil, args[:task]) - end - end - # Open UDP Services List (Qualys ID 82004) - services_udp = host.elements["SERVICES/CAT/SERVICE[@number='82004']/RESULT"] - if services_udp - services_udp.text.scan(/([0-9]+)\t(.*?)\t.*?\t([^\t\n]*)/) do |match| - if match[2] == nil or match[2].strip == 'unknown' - name = match[1].strip - else - name = match[2].strip - end - handle_qualys(wspace, hobj, match[0].to_s, 'udp', 0, nil, nil, name, nil, args[:task]) - end - end - - # VULNS are confirmed, PRACTICES are unconfirmed vulnerabilities - host.elements.each('VULNS/CAT | PRACTICES/CAT') do |cat| - port = cat.attributes['port'] - protocol = cat.attributes['protocol'] - cat.elements.each('VULN | PRACTICE') do |vuln| - refs = [] - qid = vuln.attributes['number'] - severity = vuln.attributes['severity'] - title = vuln.elements['TITLE'].text.to_s - vuln.elements.each('VENDOR_REFERENCE_LIST/VENDOR_REFERENCE') do |ref| - refs.push(ref.elements['ID'].text.to_s) - end - vuln.elements.each('CVE_ID_LIST/CVE_ID') do |ref| - refs.push('CVE-' + /C..-([0-9\-]{9})/.match(ref.elements['ID'].text.to_s)[1]) - end - vuln.elements.each('BUGTRAQ_ID_LIST/BUGTRAQ_ID') do |ref| - refs.push('BID-' + ref.elements['ID'].text.to_s) - end - - handle_qualys(wspace, hobj, port, protocol, qid, severity, refs, nil,title, args[:task]) - end - end - end - end - - # - # Import Qualys' Scan xml output - # - def import_qualys_scan_xml_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - - data = "" - ::File.open(filename, 'rb') do |f| - data = f.read(f.stat.size) - end - import_qualys_scan_xml(args.merge(:data => data)) - end - # @param report [REXML::Element] to be imported # @param args [Hash] # @param base_dir [String] diff --git a/lib/msf/core/db_manager/import/qualys.rb b/lib/msf/core/db_manager/import/qualys.rb index 997480ef65..5ad1fed8de 100644 --- a/lib/msf/core/db_manager/import/qualys.rb +++ b/lib/msf/core/db_manager/import/qualys.rb @@ -1,5 +1,7 @@ module Msf::DBManager::Import::Qualys autoload :Asset, 'msf/core/db_manager/import/qualys/asset' + autoload :Scan, 'msf/core/db_manager/import/qualys/scan' include Msf::DBManager::Import::Qualys::Asset + include Msf::DBManager::Import::Qualys::Scan end \ No newline at end of file diff --git a/lib/msf/core/db_manager/import/qualys/scan.rb b/lib/msf/core/db_manager/import/qualys/scan.rb new file mode 100644 index 0000000000..a69d3ab07d --- /dev/null +++ b/lib/msf/core/db_manager/import/qualys/scan.rb @@ -0,0 +1,98 @@ +module Msf::DBManager::Import::Qualys::Scan + def import_qualys_scan_xml(args={}, &block) + data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + + + doc = rexmlify(data) + doc.elements.each('/SCAN/IP') do |host| + hobj = nil + addr = host.attributes['value'] + if bl.include? addr + next + else + yield(:address,addr) if block + end + hname = host.attributes['name'] || '' + + hobj = report_host(:workspace => wspace, :host => addr, :name => hname, :state => Msf::HostState::Alive, :task => args[:task]) + report_import_note(wspace,hobj) + + if host.elements["OS"] + hos = host.elements["OS"].text + report_note( + :workspace => wspace, + :task => args[:task], + :host => hobj, + :type => 'host.os.qualys_fingerprint', + :data => { + :os => hos + } + ) + end + + # Open TCP Services List (Qualys ID 82023) + services_tcp = host.elements["SERVICES/CAT/SERVICE[@number='82023']/RESULT"] + if services_tcp + services_tcp.text.scan(/([0-9]+)\t(.*?)\t.*?\t([^\t\n]*)/) do |match| + if match[2] == nil or match[2].strip == 'unknown' + name = match[1].strip + else + name = match[2].strip + end + handle_qualys(wspace, hobj, match[0].to_s, 'tcp', 0, nil, nil, name, nil, args[:task]) + end + end + # Open UDP Services List (Qualys ID 82004) + services_udp = host.elements["SERVICES/CAT/SERVICE[@number='82004']/RESULT"] + if services_udp + services_udp.text.scan(/([0-9]+)\t(.*?)\t.*?\t([^\t\n]*)/) do |match| + if match[2] == nil or match[2].strip == 'unknown' + name = match[1].strip + else + name = match[2].strip + end + handle_qualys(wspace, hobj, match[0].to_s, 'udp', 0, nil, nil, name, nil, args[:task]) + end + end + + # VULNS are confirmed, PRACTICES are unconfirmed vulnerabilities + host.elements.each('VULNS/CAT | PRACTICES/CAT') do |cat| + port = cat.attributes['port'] + protocol = cat.attributes['protocol'] + cat.elements.each('VULN | PRACTICE') do |vuln| + refs = [] + qid = vuln.attributes['number'] + severity = vuln.attributes['severity'] + title = vuln.elements['TITLE'].text.to_s + vuln.elements.each('VENDOR_REFERENCE_LIST/VENDOR_REFERENCE') do |ref| + refs.push(ref.elements['ID'].text.to_s) + end + vuln.elements.each('CVE_ID_LIST/CVE_ID') do |ref| + refs.push('CVE-' + /C..-([0-9\-]{9})/.match(ref.elements['ID'].text.to_s)[1]) + end + vuln.elements.each('BUGTRAQ_ID_LIST/BUGTRAQ_ID') do |ref| + refs.push('BID-' + ref.elements['ID'].text.to_s) + end + + handle_qualys(wspace, hobj, port, protocol, qid, severity, refs, nil,title, args[:task]) + end + end + end + end + + # + # Import Qualys' Scan xml output + # + def import_qualys_scan_xml_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + + data = "" + ::File.open(filename, 'rb') do |f| + data = f.read(f.stat.size) + end + import_qualys_scan_xml(args.merge(:data => data)) + end +end From 55ca928feea86319618f606bc46f37d6b3c22494 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Tue, 14 Oct 2014 15:31:01 -0500 Subject: [PATCH 088/159] Extract Msf::DBManager::Import::Qualys::Scan shared examples MSP-11124 --- spec/support/shared/examples/msf/db_manager/import.rb | 2 -- spec/support/shared/examples/msf/db_manager/import/qualys.rb | 1 + .../shared/examples/msf/db_manager/import/qualys/scan.rb | 4 ++++ 3 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/import/qualys/scan.rb diff --git a/spec/support/shared/examples/msf/db_manager/import.rb b/spec/support/shared/examples/msf/db_manager/import.rb index 78de3b790c..cc4b8b8806 100644 --- a/spec/support/shared/examples/msf/db_manager/import.rb +++ b/spec/support/shared/examples/msf/db_manager/import.rb @@ -55,8 +55,6 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :import_openvas_xml } it { is_expected.to respond_to :import_outpost24_noko_stream } it { is_expected.to respond_to :import_outpost24_xml } - it { is_expected.to respond_to :import_qualys_scan_xml } - it { is_expected.to respond_to :import_qualys_scan_xml_file } it { is_expected.to respond_to :import_report } it { is_expected.to respond_to :import_retina_xml } it { is_expected.to respond_to :import_retina_xml_file } diff --git a/spec/support/shared/examples/msf/db_manager/import/qualys.rb b/spec/support/shared/examples/msf/db_manager/import/qualys.rb index e740dd2be0..769ec73849 100644 --- a/spec/support/shared/examples/msf/db_manager/import/qualys.rb +++ b/spec/support/shared/examples/msf/db_manager/import/qualys.rb @@ -1,3 +1,4 @@ shared_examples_for 'Msf::DBManager::Import::Qualys' do it_should_behave_like 'Msf::DBManager::Import::Qualys::Asset' + it_should_behave_like 'Msf::DBManager::Import::Qualys::Scan' end \ No newline at end of file diff --git a/spec/support/shared/examples/msf/db_manager/import/qualys/scan.rb b/spec/support/shared/examples/msf/db_manager/import/qualys/scan.rb new file mode 100644 index 0000000000..f9806349ed --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/import/qualys/scan.rb @@ -0,0 +1,4 @@ +shared_examples_for 'Msf::DBManager::Import::Qualys::Scan' do + it { is_expected.to respond_to :import_qualys_scan_xml } + it { is_expected.to respond_to :import_qualys_scan_xml_file } +end From 0c10b5a859b8463211d4785038c92035b34a89c8 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Tue, 14 Oct 2014 15:32:22 -0500 Subject: [PATCH 089/159] Extract #handle_qualys to Msf::DBManager::Import::Qualys MSP-11124 --- lib/msf/core/db_manager/import.rb | 43 ------------------------ lib/msf/core/db_manager/import/qualys.rb | 43 ++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 43 deletions(-) diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index 34969379c6..25352725c5 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -3227,49 +3227,6 @@ module Msf::DBManager::Import report_vuln(vuln) end - # - # Qualys report parsing/handling - # - def handle_qualys(wspace, hobj, port, protocol, qid, severity, refs, name=nil, title=nil, task=nil) - addr = hobj.address - port = port.to_i if port - - info = { :workspace => wspace, :host => hobj, :port => port, :proto => protocol, :task => task } - if name and name != 'unknown' and name != 'No registered hostname' - info[:name] = name - end - - if info[:host] && info[:port] && info[:proto] - report_service(info) - end - - fixed_refs = [] - if refs - refs.each do |ref| - case ref - when /^MS[0-9]{2}-[0-9]{3}/ - fixed_refs << "MSB-#{ref}" - else - fixed_refs << ref - end - end - end - - return if qid == 0 - title = 'QUALYS-' + qid if title.nil? or title.empty? - if addr - report_vuln( - :workspace => wspace, - :task => task, - :host => hobj, - :port => port, - :proto => protocol, - :name => title, - :refs => fixed_refs - ) - end - end - def process_nexpose_data_sxml_refs(vuln) refs = [] vid = vuln.attributes['id'].to_s.downcase diff --git a/lib/msf/core/db_manager/import/qualys.rb b/lib/msf/core/db_manager/import/qualys.rb index 5ad1fed8de..10fcae11db 100644 --- a/lib/msf/core/db_manager/import/qualys.rb +++ b/lib/msf/core/db_manager/import/qualys.rb @@ -4,4 +4,47 @@ module Msf::DBManager::Import::Qualys include Msf::DBManager::Import::Qualys::Asset include Msf::DBManager::Import::Qualys::Scan + + # + # Qualys report parsing/handling + # + def handle_qualys(wspace, hobj, port, protocol, qid, severity, refs, name=nil, title=nil, task=nil) + addr = hobj.address + port = port.to_i if port + + info = { :workspace => wspace, :host => hobj, :port => port, :proto => protocol, :task => task } + if name and name != 'unknown' and name != 'No registered hostname' + info[:name] = name + end + + if info[:host] && info[:port] && info[:proto] + report_service(info) + end + + fixed_refs = [] + if refs + refs.each do |ref| + case ref + when /^MS[0-9]{2}-[0-9]{3}/ + fixed_refs << "MSB-#{ref}" + else + fixed_refs << ref + end + end + end + + return if qid == 0 + title = 'QUALYS-' + qid if title.nil? or title.empty? + if addr + report_vuln( + :workspace => wspace, + :task => task, + :host => hobj, + :port => port, + :proto => protocol, + :name => title, + :refs => fixed_refs + ) + end + end end \ No newline at end of file From 6c0f549abbc72c54f46a8c0ffbe785f98e557327 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Tue, 14 Oct 2014 15:40:29 -0500 Subject: [PATCH 090/159] Extract Msf::DBManager::Import::Acunetix MSP-11124 --- lib/msf/core/db_manager/import.rb | 33 ++------------------- lib/msf/core/db_manager/import/acunetix.rb | 34 ++++++++++++++++++++++ 2 files changed, 36 insertions(+), 31 deletions(-) create mode 100644 lib/msf/core/db_manager/import/acunetix.rb diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index 25352725c5..06b361c38c 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -12,7 +12,6 @@ require 'uri' # require 'packetfu' -require 'rex/parser/acunetix_nokogiri' require 'rex/parser/appscan_nokogiri' require 'rex/parser/burp_session_nokogiri' require 'rex/parser/ci_nokogiri' @@ -34,9 +33,11 @@ require 'rex/parser/retina_xml' require 'rex/parser/wapiti_nokogiri' module Msf::DBManager::Import + autoload :Acunetix, 'msf/core/db_manager/import/acunetix' autoload :MsfXml, 'msf/core/db_manager/import/msf_xml' autoload :Qualys, 'msf/core/db_manager/import/qualys' + include Msf::DBManager::Import::Acunetix include Msf::DBManager::Import::MsfXml include Msf::DBManager::Import::Qualys @@ -66,36 +67,6 @@ module Msf::DBManager::Import self.send "import_#{ftype}".to_sym, args, &block end - def import_acunetix_noko_stream(args={},&block) - if block - doc = Rex::Parser::AcunetixDocument.new(args,framework.db) {|type, data| yield type,data } - else - doc = Rex::Parser::AcunetixFoundstoneDocument.new(args,self) - end - parser = ::Nokogiri::XML::SAX::Parser.new(doc) - parser.parse(args[:data]) - end - - def import_acunetix_xml(args={}, &block) - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - wspace = args[:wspace] || workspace - if Rex::Parser.nokogiri_loaded - parser = "Nokogiri v#{::Nokogiri::VERSION}" - noko_args = args.dup - noko_args[:blacklist] = bl - noko_args[:wspace] = wspace - if block - yield(:parser, parser) - import_acunetix_noko_stream(noko_args) {|type, data| yield type,data} - else - import_acunetix_noko_stream(noko_args) - end - return true - else # Sorry - raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.") - end - end - def import_amap_log(args={}, &block) data = args[:data] wspace = args[:wspace] || workspace diff --git a/lib/msf/core/db_manager/import/acunetix.rb b/lib/msf/core/db_manager/import/acunetix.rb new file mode 100644 index 0000000000..afe7c9f2fa --- /dev/null +++ b/lib/msf/core/db_manager/import/acunetix.rb @@ -0,0 +1,34 @@ +require 'rex/parser/acunetix_nokogiri' + +module Msf::DBManager::Import::Acunetix + def import_acunetix_noko_stream(args={},&block) + if block + doc = Rex::Parser::AcunetixDocument.new(args,framework.db) {|type, data| yield type,data } + else + doc = Rex::Parser::AcunetixFoundstoneDocument.new(args,self) + end + parser = ::Nokogiri::XML::SAX::Parser.new(doc) + parser.parse(args[:data]) + end + + def import_acunetix_xml(args={}, &block) + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + wspace = args[:wspace] || workspace + if Rex::Parser.nokogiri_loaded + parser = "Nokogiri v#{::Nokogiri::VERSION}" + noko_args = args.dup + noko_args[:blacklist] = bl + noko_args[:wspace] = wspace + if block + yield(:parser, parser) + import_acunetix_noko_stream(noko_args) {|type, data| yield type,data} + else + import_acunetix_noko_stream(noko_args) + end + return true + else # Sorry + raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.") + end + end + +end \ No newline at end of file From c28f1fce3a4c78f075830c9d4accd14841181e8d Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Tue, 14 Oct 2014 15:45:07 -0500 Subject: [PATCH 091/159] Extract Msf::DBManager::Import::Acunetix shared examples MSP-11124 --- spec/support/shared/examples/msf/db_manager/import.rb | 3 +-- .../support/shared/examples/msf/db_manager/import/acunetix.rb | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/import/acunetix.rb diff --git a/spec/support/shared/examples/msf/db_manager/import.rb b/spec/support/shared/examples/msf/db_manager/import.rb index cc4b8b8806..0ec6564fdd 100644 --- a/spec/support/shared/examples/msf/db_manager/import.rb +++ b/spec/support/shared/examples/msf/db_manager/import.rb @@ -2,8 +2,6 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :dehex } it { is_expected.to respond_to :emit } it { is_expected.to respond_to :import } - it { is_expected.to respond_to :import_acunetix_noko_stream } - it { is_expected.to respond_to :import_acunetix_xml } it { is_expected.to respond_to :import_amap_log } it { is_expected.to respond_to :import_amap_log_file } it { is_expected.to respond_to :import_amap_mlog } @@ -77,6 +75,7 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :unserialize_object } it { is_expected.to respond_to :validate_import_file } + it_should_behave_like 'Msf::DBManager::Import::Acunetix' it_should_behave_like 'Msf::DBManager::Import::MsfXml' it_should_behave_like 'Msf::DBManager::Import::Qualys' end \ No newline at end of file diff --git a/spec/support/shared/examples/msf/db_manager/import/acunetix.rb b/spec/support/shared/examples/msf/db_manager/import/acunetix.rb new file mode 100644 index 0000000000..11fda62d68 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/import/acunetix.rb @@ -0,0 +1,4 @@ +shared_examples_for 'Msf::DBManager::Import::Acunetix' do + it { is_expected.to respond_to :import_acunetix_noko_stream } + it { is_expected.to respond_to :import_acunetix_xml } +end From 81c18c96ee7fa85458dd58b98ea743491c4f1a0e Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Tue, 14 Oct 2014 15:58:43 -0500 Subject: [PATCH 092/159] Extract Msf::DBManager::Import::IP260::ASPL MSP-11124 --- lib/msf/core/db_manager/import.rb | 23 ++------------------ lib/msf/core/db_manager/import/ip360.rb | 7 ++++++ lib/msf/core/db_manager/import/ip360/aspl.rb | 23 ++++++++++++++++++++ 3 files changed, 32 insertions(+), 21 deletions(-) create mode 100644 lib/msf/core/db_manager/import/ip360.rb create mode 100644 lib/msf/core/db_manager/import/ip360/aspl.rb diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index 06b361c38c..1e80710d61 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -17,7 +17,6 @@ require 'rex/parser/burp_session_nokogiri' require 'rex/parser/ci_nokogiri' require 'rex/parser/foundstone_nokogiri' require 'rex/parser/fusionvm_nokogiri' -require 'rex/parser/ip360_aspl_xml' require 'rex/parser/ip360_xml' require 'rex/parser/mbsa_nokogiri' require 'rex/parser/nessus_xml' @@ -34,10 +33,12 @@ require 'rex/parser/wapiti_nokogiri' module Msf::DBManager::Import autoload :Acunetix, 'msf/core/db_manager/import/acunetix' + autoload :IP360, 'msf/core/db_manager/import/ip360' autoload :MsfXml, 'msf/core/db_manager/import/msf_xml' autoload :Qualys, 'msf/core/db_manager/import/qualys' include Msf::DBManager::Import::Acunetix + include Msf::DBManager::Import::IP360 include Msf::DBManager::Import::MsfXml include Msf::DBManager::Import::Qualys @@ -547,26 +548,6 @@ module Msf::DBManager::Import parser.parse(args[:data]) end - # - # Import IP360's ASPL database - # - def import_ip360_aspl_xml(args={}, &block) - data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - - if not data.index(" Date: Tue, 14 Oct 2014 16:02:44 -0500 Subject: [PATCH 093/159] Extract Msf::DBManager::Import::IP360::ASPL shared examples MSP-11124 --- spec/support/shared/examples/msf/db_manager/import.rb | 2 +- spec/support/shared/examples/msf/db_manager/import/ip360.rb | 3 +++ .../shared/examples/msf/db_manager/import/ip360/aspl.rb | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 spec/support/shared/examples/msf/db_manager/import/ip360.rb create mode 100644 spec/support/shared/examples/msf/db_manager/import/ip360/aspl.rb diff --git a/spec/support/shared/examples/msf/db_manager/import.rb b/spec/support/shared/examples/msf/db_manager/import.rb index 0ec6564fdd..2c115e25aa 100644 --- a/spec/support/shared/examples/msf/db_manager/import.rb +++ b/spec/support/shared/examples/msf/db_manager/import.rb @@ -16,7 +16,6 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :import_foundstone_noko_stream } it { is_expected.to respond_to :import_foundstone_xml } it { is_expected.to respond_to :import_fusionvm_xml } - it { is_expected.to respond_to :import_ip360_aspl_xml } it { is_expected.to respond_to :import_ip360_xml_file } it { is_expected.to respond_to :import_ip360_xml_v3 } it { is_expected.to respond_to :import_ip_list } @@ -76,6 +75,7 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :validate_import_file } it_should_behave_like 'Msf::DBManager::Import::Acunetix' + it_should_behave_like 'Msf::DBManager::Import::IP360' it_should_behave_like 'Msf::DBManager::Import::MsfXml' it_should_behave_like 'Msf::DBManager::Import::Qualys' end \ No newline at end of file diff --git a/spec/support/shared/examples/msf/db_manager/import/ip360.rb b/spec/support/shared/examples/msf/db_manager/import/ip360.rb new file mode 100644 index 0000000000..4930235f57 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/import/ip360.rb @@ -0,0 +1,3 @@ +shared_examples_for 'Msf::DBManager::Import::IP360' do + it_should_behave_like 'Msf::DBManager::Import::IP360::ASPL' +end \ No newline at end of file diff --git a/spec/support/shared/examples/msf/db_manager/import/ip360/aspl.rb b/spec/support/shared/examples/msf/db_manager/import/ip360/aspl.rb new file mode 100644 index 0000000000..0e7820628e --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/import/ip360/aspl.rb @@ -0,0 +1,3 @@ +shared_examples_for 'Msf::DBManager::Import::IP360::ASPL' do + it { is_expected.to respond_to :import_ip360_aspl_xml } +end From 599bcc33a9b4dcafdc3a306ad2f4b36e8b8519aa Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Tue, 14 Oct 2014 16:12:42 -0500 Subject: [PATCH 094/159] Extract Msf::DBManager::Import::IP360::V3 MSP-11124 --- lib/msf/core/db_manager/import.rb | 184 -------------------- lib/msf/core/db_manager/import/ip360.rb | 4 +- lib/msf/core/db_manager/import/ip360/v3.rb | 186 +++++++++++++++++++++ 3 files changed, 189 insertions(+), 185 deletions(-) create mode 100644 lib/msf/core/db_manager/import/ip360/v3.rb diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index 1e80710d61..4c98dcc479 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -548,134 +548,6 @@ module Msf::DBManager::Import parser.parse(args[:data]) end - # - # Import IP360 XML v3 output - # - def import_ip360_xml_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - - data = "" - ::File.open(filename, 'rb') do |f| - data = f.read(f.stat.size) - end - import_ip360_xml_v3(args.merge(:data => data)) - end - - - # - # Import IP360's xml output - # - def import_ip360_xml_v3(args={}, &block) - data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - - # @aspl = {'vulns' => {'name' => { }, 'cve' => { }, 'bid' => { } } - # 'oses' => {'name' } } - - aspl_path = nil - aspl_paths = [ - ::File.join(Msf::Config.config_directory, "data", "ncircle", "ip360.aspl"), - ::File.join(Msf::Config.data_directory, "ncircle", "ip360.aspl") - ] - - aspl_paths.each do |tpath| - next if not (::File.exist?(tpath) and ::File.readable?(tpath)) - aspl_path = tpath - break - end - - if not aspl_path - raise DBImportError.new("The nCircle IP360 ASPL file is not present.\n Download ASPL from nCircle VNE | Administer | Support | Resources, unzip it, and import it first") - end - - # parse nCircle ASPL file - aspl = "" - ::File.open(aspl_path, "rb") do |f| - aspl = f.read(f.stat.size) - end - - @asplhash = nil - parser = Rex::Parser::IP360ASPLXMLStreamParser.new - parser.on_found_aspl = Proc.new { |asplh| - @asplhash = asplh - } - REXML::Document.parse_stream(aspl, parser) - - # nCircle has some quotes escaped which causes the parser to break - # we don't need these lines so just replace \" with " - data.gsub!(/\\"/,'"') - - # parse nCircle Scan Output - parser = Rex::Parser::IP360XMLStreamParser.new - parser.on_found_host = Proc.new { |host| - hobj = nil - addr = host['addr'] || host['hname'] - - next unless ipv46_validator(addr) # Catches SCAN-ERROR, among others. - - if bl.include? addr - next - else - yield(:address,addr) if block - end - - os = host['os'] - hname = host['hname'] - mac = host['mac'] - - host_hash = { - :workspace => wspace, - :host => addr, - :task => args[:task] - } - host_hash[:name] = hname.to_s.strip if hname - host_hash[:mac] = mac.to_s.strip.upcase if mac - - hobj = report_host(host_hash) - - yield(:os, os) if block - if os - report_note( - :workspace => wspace, - :task => args[:task], - :host => hobj, - :type => 'host.os.ip360_fingerprint', - :data => { - :os => @asplhash['oses'][os].to_s.strip - } - ) - end - - host['apps'].each do |item| - port = item['port'].to_s - proto = item['proto'].to_s - - handle_ip360_v3_svc(wspace, hobj, port, proto, hname, args[:task]) - end - - - host['vulns'].each do |item| - vulnid = item['vulnid'].to_s - port = item['port'].to_s - proto = item['proto'] || "tcp" - vulnname = @asplhash['vulns']['name'][vulnid] - cves = @asplhash['vulns']['cve'][vulnid] - bids = @asplhash['vulns']['bid'][vulnid] - - yield(:port, port) if block - - handle_ip360_v3_vuln(wspace, hobj, port, proto, hname, vulnid, vulnname, cves, bids, args[:task]) - - end - - yield(:end, hname) if block - } - - REXML::Document.parse_stream(data, parser) - end - def import_ip_list(args={}, &block) data = args[:data] wspace = args[:wspace] || workspace @@ -2985,62 +2857,6 @@ module Msf::DBManager::Import protected - # IP360 v3 vuln - def handle_ip360_v3_svc(wspace,hobj,port,proto,hname,task=nil) - addr = hobj.address - report_host(:workspace => wspace, :host => hobj, :state => Msf::HostState::Alive, :task => task) - - info = { :workspace => wspace, :host => hobj, :port => port, :proto => proto, :task => task } - if hname != "unknown" and hname[-1,1] != "?" - info[:name] = hname - end - - if port.to_i != 0 - report_service(info) - end - end - - # - # IP360 v3 vuln - # - def handle_ip360_v3_vuln(wspace,hobj,port,proto,hname,vulnid,vulnname,cves,bids,task=nil) - info = { :workspace => wspace, :host => hobj, :port => port, :proto => proto, :task => task } - if hname != "unknown" and hname[-1,1] != "?" - info[:name] = hname - end - - if port.to_i != 0 - report_service(info) - end - - refs = [] - - cves.split(/,/).each do |cve| - refs.push(cve.to_s) - end if cves - - bids.split(/,/).each do |bid| - refs.push('BID-' + bid.to_s) - end if bids - - description = nil # not working yet - vuln = { - :workspace => wspace, - :host => hobj, - :name => vulnname, - :info => description ? description : "", - :refs => refs, - :task => task - } - - if port.to_i != 0 - vuln[:port] = port - vuln[:proto] = proto - end - - report_vuln(vuln) - end - # # This holds all of the shared parsing/handling used by the # Nessus NBE and NESSUS v1 methods diff --git a/lib/msf/core/db_manager/import/ip360.rb b/lib/msf/core/db_manager/import/ip360.rb index 5f65bcf870..bd0814d34e 100644 --- a/lib/msf/core/db_manager/import/ip360.rb +++ b/lib/msf/core/db_manager/import/ip360.rb @@ -1,7 +1,9 @@ require 'rex/parser/ip360_aspl_xml' module Msf::DBManager::Import::IP360 - autoload 'ASPL', 'msf/core/db_manager/import/ip360/aspl' + autoload :ASPL, 'msf/core/db_manager/import/ip360/aspl' + autoload :V3, 'msf/core/db_manager/import/ip360/v3' include Msf::DBManager::Import::IP360::ASPL + include Msf::DBManager::Import::IP360::V3 end \ No newline at end of file diff --git a/lib/msf/core/db_manager/import/ip360/v3.rb b/lib/msf/core/db_manager/import/ip360/v3.rb new file mode 100644 index 0000000000..6e988caf94 --- /dev/null +++ b/lib/msf/core/db_manager/import/ip360/v3.rb @@ -0,0 +1,186 @@ +module Msf::DBManager::Import::IP360::V3 + # + # Import IP360 XML v3 output + # + def import_ip360_xml_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + + data = "" + ::File.open(filename, 'rb') do |f| + data = f.read(f.stat.size) + end + import_ip360_xml_v3(args.merge(:data => data)) + end + + # + # Import IP360's xml output + # + def import_ip360_xml_v3(args={}, &block) + data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + + # @aspl = {'vulns' => {'name' => { }, 'cve' => { }, 'bid' => { } } + # 'oses' => {'name' } } + + aspl_path = nil + aspl_paths = [ + ::File.join(Msf::Config.config_directory, "data", "ncircle", "ip360.aspl"), + ::File.join(Msf::Config.data_directory, "ncircle", "ip360.aspl") + ] + + aspl_paths.each do |tpath| + next if not (::File.exist?(tpath) and ::File.readable?(tpath)) + aspl_path = tpath + break + end + + if not aspl_path + raise DBImportError.new("The nCircle IP360 ASPL file is not present.\n Download ASPL from nCircle VNE | Administer | Support | Resources, unzip it, and import it first") + end + + # parse nCircle ASPL file + aspl = "" + ::File.open(aspl_path, "rb") do |f| + aspl = f.read(f.stat.size) + end + + @asplhash = nil + parser = Rex::Parser::IP360ASPLXMLStreamParser.new + parser.on_found_aspl = Proc.new { |asplh| + @asplhash = asplh + } + REXML::Document.parse_stream(aspl, parser) + + # nCircle has some quotes escaped which causes the parser to break + # we don't need these lines so just replace \" with " + data.gsub!(/\\"/,'"') + + # parse nCircle Scan Output + parser = Rex::Parser::IP360XMLStreamParser.new + parser.on_found_host = Proc.new { |host| + hobj = nil + addr = host['addr'] || host['hname'] + + next unless ipv46_validator(addr) # Catches SCAN-ERROR, among others. + + if bl.include? addr + next + else + yield(:address,addr) if block + end + + os = host['os'] + hname = host['hname'] + mac = host['mac'] + + host_hash = { + :workspace => wspace, + :host => addr, + :task => args[:task] + } + host_hash[:name] = hname.to_s.strip if hname + host_hash[:mac] = mac.to_s.strip.upcase if mac + + hobj = report_host(host_hash) + + yield(:os, os) if block + if os + report_note( + :workspace => wspace, + :task => args[:task], + :host => hobj, + :type => 'host.os.ip360_fingerprint', + :data => { + :os => @asplhash['oses'][os].to_s.strip + } + ) + end + + host['apps'].each do |item| + port = item['port'].to_s + proto = item['proto'].to_s + + handle_ip360_v3_svc(wspace, hobj, port, proto, hname, args[:task]) + end + + + host['vulns'].each do |item| + vulnid = item['vulnid'].to_s + port = item['port'].to_s + proto = item['proto'] || "tcp" + vulnname = @asplhash['vulns']['name'][vulnid] + cves = @asplhash['vulns']['cve'][vulnid] + bids = @asplhash['vulns']['bid'][vulnid] + + yield(:port, port) if block + + handle_ip360_v3_vuln(wspace, hobj, port, proto, hname, vulnid, vulnname, cves, bids, args[:task]) + + end + + yield(:end, hname) if block + } + + REXML::Document.parse_stream(data, parser) + end + + protected + + # IP360 v3 svc + def handle_ip360_v3_svc(wspace,hobj,port,proto,hname,task=nil) + addr = hobj.address + report_host(:workspace => wspace, :host => hobj, :state => Msf::HostState::Alive, :task => task) + + info = { :workspace => wspace, :host => hobj, :port => port, :proto => proto, :task => task } + if hname != "unknown" and hname[-1,1] != "?" + info[:name] = hname + end + + if port.to_i != 0 + report_service(info) + end + end + + # + # IP360 v3 vuln + # + def handle_ip360_v3_vuln(wspace,hobj,port,proto,hname,vulnid,vulnname,cves,bids,task=nil) + info = { :workspace => wspace, :host => hobj, :port => port, :proto => proto, :task => task } + if hname != "unknown" and hname[-1,1] != "?" + info[:name] = hname + end + + if port.to_i != 0 + report_service(info) + end + + refs = [] + + cves.split(/,/).each do |cve| + refs.push(cve.to_s) + end if cves + + bids.split(/,/).each do |bid| + refs.push('BID-' + bid.to_s) + end if bids + + description = nil # not working yet + vuln = { + :workspace => wspace, + :host => hobj, + :name => vulnname, + :info => description ? description : "", + :refs => refs, + :task => task + } + + if port.to_i != 0 + vuln[:port] = port + vuln[:proto] = proto + end + + report_vuln(vuln) + end +end \ No newline at end of file From d85ee2ee0a29a967f681528ec4e84d761dd31136 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Tue, 14 Oct 2014 16:17:34 -0500 Subject: [PATCH 095/159] Extract Msf::DBManager::Import::IP360::V3 shared examples MSP-11124 --- spec/support/shared/examples/msf/db_manager/import.rb | 2 -- spec/support/shared/examples/msf/db_manager/import/ip360.rb | 1 + .../support/shared/examples/msf/db_manager/import/ip360/v3.rb | 4 ++++ 3 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/import/ip360/v3.rb diff --git a/spec/support/shared/examples/msf/db_manager/import.rb b/spec/support/shared/examples/msf/db_manager/import.rb index 2c115e25aa..2e020a01d0 100644 --- a/spec/support/shared/examples/msf/db_manager/import.rb +++ b/spec/support/shared/examples/msf/db_manager/import.rb @@ -16,8 +16,6 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :import_foundstone_noko_stream } it { is_expected.to respond_to :import_foundstone_xml } it { is_expected.to respond_to :import_fusionvm_xml } - it { is_expected.to respond_to :import_ip360_xml_file } - it { is_expected.to respond_to :import_ip360_xml_v3 } it { is_expected.to respond_to :import_ip_list } it { is_expected.to respond_to :import_ip_list_file } it { is_expected.to respond_to :import_libpcap } diff --git a/spec/support/shared/examples/msf/db_manager/import/ip360.rb b/spec/support/shared/examples/msf/db_manager/import/ip360.rb index 4930235f57..ffb4183ba6 100644 --- a/spec/support/shared/examples/msf/db_manager/import/ip360.rb +++ b/spec/support/shared/examples/msf/db_manager/import/ip360.rb @@ -1,3 +1,4 @@ shared_examples_for 'Msf::DBManager::Import::IP360' do it_should_behave_like 'Msf::DBManager::Import::IP360::ASPL' + it_should_behave_like 'Msf::DBManager::Import::IP360::V3' end \ No newline at end of file diff --git a/spec/support/shared/examples/msf/db_manager/import/ip360/v3.rb b/spec/support/shared/examples/msf/db_manager/import/ip360/v3.rb new file mode 100644 index 0000000000..78099457dd --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/import/ip360/v3.rb @@ -0,0 +1,4 @@ +shared_examples_for 'Msf::DBManager::Import::IP360::V3' do + it { is_expected.to respond_to :import_ip360_xml_file } + it { is_expected.to respond_to :import_ip360_xml_v3 } +end From a00d039796c7927dbd081b5da6cbb3552cb4eff3 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Tue, 14 Oct 2014 16:18:47 -0500 Subject: [PATCH 096/159] Move require for IP360 XML parser MSP-11124 --- lib/msf/core/db_manager/import.rb | 1 - lib/msf/core/db_manager/import/ip360/v3.rb | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index 4c98dcc479..3f250363f7 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -17,7 +17,6 @@ require 'rex/parser/burp_session_nokogiri' require 'rex/parser/ci_nokogiri' require 'rex/parser/foundstone_nokogiri' require 'rex/parser/fusionvm_nokogiri' -require 'rex/parser/ip360_xml' require 'rex/parser/mbsa_nokogiri' require 'rex/parser/nessus_xml' require 'rex/parser/netsparker_xml' diff --git a/lib/msf/core/db_manager/import/ip360/v3.rb b/lib/msf/core/db_manager/import/ip360/v3.rb index 6e988caf94..62c372fa85 100644 --- a/lib/msf/core/db_manager/import/ip360/v3.rb +++ b/lib/msf/core/db_manager/import/ip360/v3.rb @@ -1,3 +1,5 @@ +require 'rex/parser/ip360_xml' + module Msf::DBManager::Import::IP360::V3 # # Import IP360 XML v3 output From dcac8a45eec0774bbb1b9e99bf7560bf2dcb3115 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 09:06:03 -0500 Subject: [PATCH 097/159] Extract Msf::DBManager::Import::Amap MSP-11124 --- lib/msf/core/db_manager/import.rb | 85 +------------------------- lib/msf/core/db_manager/import/amap.rb | 84 +++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 83 deletions(-) create mode 100644 lib/msf/core/db_manager/import/amap.rb diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index 3f250363f7..8f2acde648 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -32,11 +32,13 @@ require 'rex/parser/wapiti_nokogiri' module Msf::DBManager::Import autoload :Acunetix, 'msf/core/db_manager/import/acunetix' + autoload :Amap, 'msf/core/db_manager/import/amap' autoload :IP360, 'msf/core/db_manager/import/ip360' autoload :MsfXml, 'msf/core/db_manager/import/msf_xml' autoload :Qualys, 'msf/core/db_manager/import/qualys' include Msf::DBManager::Import::Acunetix + include Msf::DBManager::Import::Amap include Msf::DBManager::Import::IP360 include Msf::DBManager::Import::MsfXml include Msf::DBManager::Import::Qualys @@ -67,89 +69,6 @@ module Msf::DBManager::Import self.send "import_#{ftype}".to_sym, args, &block end - def import_amap_log(args={}, &block) - data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - - data.each_line do |line| - next if line =~ /^#/ - next if line !~ /^Protocol on ([^:]+):([^\x5c\x2f]+)[\x5c\x2f](tcp|udp) matches (.*)$/n - addr = $1 - next if bl.include? addr - port = $2.to_i - proto = $3.downcase - name = $4 - host = find_or_create_host(:workspace => wspace, :host => addr, :state => Msf::HostState::Alive, :task => args[:task]) - next if not host - yield(:address,addr) if block - info = { - :workspace => wspace, - :task => args[:task], - :host => host, - :proto => proto, - :port => port - } - if name != "unidentified" - info[:name] = name - end - service = find_or_create_service(info) - end - end - - def import_amap_log_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - data = "" - ::File.open(filename, 'rb') do |f| - data = f.read(f.stat.size) - end - - case import_filetype_detect(data) - when :amap_log - import_amap_log(args.merge(:data => data)) - when :amap_mlog - import_amap_mlog(args.merge(:data => data)) - else - raise DBImportError.new("Could not determine file type") - end - end - - def import_amap_mlog(args={}, &block) - data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - - data.each_line do |line| - next if line =~ /^#/ - r = line.split(':') - next if r.length < 6 - - addr = r[0] - next if bl.include? addr - port = r[1].to_i - proto = r[2].downcase - status = r[3] - name = r[5] - next if status != "open" - - host = find_or_create_host(:workspace => wspace, :host => addr, :state => Msf::HostState::Alive, :task => args[:task]) - next if not host - yield(:address,addr) if block - info = { - :workspace => wspace, - :task => args[:task], - :host => host, - :proto => proto, - :port => port - } - if name != "unidentified" - info[:name] = name - end - service = find_or_create_service(info) - end - end - def import_appscan_noko_stream(args={},&block) if block doc = Rex::Parser::AppscanDocument.new(args,framework.db) {|type, data| yield type,data } diff --git a/lib/msf/core/db_manager/import/amap.rb b/lib/msf/core/db_manager/import/amap.rb new file mode 100644 index 0000000000..ed68c591eb --- /dev/null +++ b/lib/msf/core/db_manager/import/amap.rb @@ -0,0 +1,84 @@ +module Msf::DBManager::Import::Amap + def import_amap_log(args={}, &block) + data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + + data.each_line do |line| + next if line =~ /^#/ + next if line !~ /^Protocol on ([^:]+):([^\x5c\x2f]+)[\x5c\x2f](tcp|udp) matches (.*)$/n + addr = $1 + next if bl.include? addr + port = $2.to_i + proto = $3.downcase + name = $4 + host = find_or_create_host(:workspace => wspace, :host => addr, :state => Msf::HostState::Alive, :task => args[:task]) + next if not host + yield(:address,addr) if block + info = { + :workspace => wspace, + :task => args[:task], + :host => host, + :proto => proto, + :port => port + } + if name != "unidentified" + info[:name] = name + end + service = find_or_create_service(info) + end + end + + def import_amap_log_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + data = "" + ::File.open(filename, 'rb') do |f| + data = f.read(f.stat.size) + end + + case import_filetype_detect(data) + when :amap_log + import_amap_log(args.merge(:data => data)) + when :amap_mlog + import_amap_mlog(args.merge(:data => data)) + else + raise DBImportError.new("Could not determine file type") + end + end + + def import_amap_mlog(args={}, &block) + data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + + data.each_line do |line| + next if line =~ /^#/ + r = line.split(':') + next if r.length < 6 + + addr = r[0] + next if bl.include? addr + port = r[1].to_i + proto = r[2].downcase + status = r[3] + name = r[5] + next if status != "open" + + host = find_or_create_host(:workspace => wspace, :host => addr, :state => Msf::HostState::Alive, :task => args[:task]) + next if not host + yield(:address,addr) if block + info = { + :workspace => wspace, + :task => args[:task], + :host => host, + :proto => proto, + :port => port + } + if name != "unidentified" + info[:name] = name + end + service = find_or_create_service(info) + end + end +end From 6a1f718c3aa614e4324b61582dba40feb519df0d Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 09:08:50 -0500 Subject: [PATCH 098/159] Extract Msf::DBManager::Import::Amap shared examples MSP-11124 --- spec/support/shared/examples/msf/db_manager/import.rb | 4 +--- spec/support/shared/examples/msf/db_manager/import/amap.rb | 5 +++++ 2 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/import/amap.rb diff --git a/spec/support/shared/examples/msf/db_manager/import.rb b/spec/support/shared/examples/msf/db_manager/import.rb index 2e020a01d0..833e66fd2c 100644 --- a/spec/support/shared/examples/msf/db_manager/import.rb +++ b/spec/support/shared/examples/msf/db_manager/import.rb @@ -2,9 +2,6 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :dehex } it { is_expected.to respond_to :emit } it { is_expected.to respond_to :import } - it { is_expected.to respond_to :import_amap_log } - it { is_expected.to respond_to :import_amap_log_file } - it { is_expected.to respond_to :import_amap_mlog } it { is_expected.to respond_to :import_appscan_noko_stream } it { is_expected.to respond_to :import_appscan_xml } it { is_expected.to respond_to :import_burp_session_noko_stream } @@ -73,6 +70,7 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :validate_import_file } it_should_behave_like 'Msf::DBManager::Import::Acunetix' + it_should_behave_like 'Msf::DBManager::Import::Amap' it_should_behave_like 'Msf::DBManager::Import::IP360' it_should_behave_like 'Msf::DBManager::Import::MsfXml' it_should_behave_like 'Msf::DBManager::Import::Qualys' diff --git a/spec/support/shared/examples/msf/db_manager/import/amap.rb b/spec/support/shared/examples/msf/db_manager/import/amap.rb new file mode 100644 index 0000000000..fba82ef898 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/import/amap.rb @@ -0,0 +1,5 @@ +shared_examples_for 'Msf::DBManager::Import::Amap' do + it { is_expected.to respond_to :import_amap_log } + it { is_expected.to respond_to :import_amap_log_file } + it { is_expected.to respond_to :import_amap_mlog } +end From f42307a6ff5a02ca280d2e94000dfe30d6846b12 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 09:12:38 -0500 Subject: [PATCH 099/159] Extract Msf::DBManager::Import::Appscan MSP-11124 --- lib/msf/core/db_manager/import.rb | 33 ++--------------------- lib/msf/core/db_manager/import/appscan.rb | 33 +++++++++++++++++++++++ 2 files changed, 35 insertions(+), 31 deletions(-) create mode 100644 lib/msf/core/db_manager/import/appscan.rb diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index 8f2acde648..f37242ed31 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -12,7 +12,6 @@ require 'uri' # require 'packetfu' -require 'rex/parser/appscan_nokogiri' require 'rex/parser/burp_session_nokogiri' require 'rex/parser/ci_nokogiri' require 'rex/parser/foundstone_nokogiri' @@ -33,12 +32,14 @@ require 'rex/parser/wapiti_nokogiri' module Msf::DBManager::Import autoload :Acunetix, 'msf/core/db_manager/import/acunetix' autoload :Amap, 'msf/core/db_manager/import/amap' + autoload :Appscan, 'msf/core/db_manager/import/appscan' autoload :IP360, 'msf/core/db_manager/import/ip360' autoload :MsfXml, 'msf/core/db_manager/import/msf_xml' autoload :Qualys, 'msf/core/db_manager/import/qualys' include Msf::DBManager::Import::Acunetix include Msf::DBManager::Import::Amap + include Msf::DBManager::Import::Appscan include Msf::DBManager::Import::IP360 include Msf::DBManager::Import::MsfXml include Msf::DBManager::Import::Qualys @@ -69,36 +70,6 @@ module Msf::DBManager::Import self.send "import_#{ftype}".to_sym, args, &block end - def import_appscan_noko_stream(args={},&block) - if block - doc = Rex::Parser::AppscanDocument.new(args,framework.db) {|type, data| yield type,data } - else - doc = Rex::Parser::AppscanDocument.new(args,self) - end - parser = ::Nokogiri::XML::SAX::Parser.new(doc) - parser.parse(args[:data]) - end - - def import_appscan_xml(args={}, &block) - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - wspace = args[:wspace] || workspace - if Rex::Parser.nokogiri_loaded - parser = "Nokogiri v#{::Nokogiri::VERSION}" - noko_args = args.dup - noko_args[:blacklist] = bl - noko_args[:wspace] = wspace - if block - yield(:parser, parser) - import_appscan_noko_stream(noko_args) {|type, data| yield type,data} - else - import_appscan_noko_stream(noko_args) - end - return true - else # Sorry - raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.") - end - end - def import_burp_session_noko_stream(args={},&block) if block doc = Rex::Parser::BurpSessionDocument.new(args,framework.db) {|type, data| yield type,data } diff --git a/lib/msf/core/db_manager/import/appscan.rb b/lib/msf/core/db_manager/import/appscan.rb new file mode 100644 index 0000000000..c1e20cc532 --- /dev/null +++ b/lib/msf/core/db_manager/import/appscan.rb @@ -0,0 +1,33 @@ +require 'rex/parser/appscan_nokogiri' + +module Msf::DBManager::Import::Appscan + def import_appscan_noko_stream(args={},&block) + if block + doc = Rex::Parser::AppscanDocument.new(args,framework.db) {|type, data| yield type,data } + else + doc = Rex::Parser::AppscanDocument.new(args,self) + end + parser = ::Nokogiri::XML::SAX::Parser.new(doc) + parser.parse(args[:data]) + end + + def import_appscan_xml(args={}, &block) + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + wspace = args[:wspace] || workspace + if Rex::Parser.nokogiri_loaded + parser = "Nokogiri v#{::Nokogiri::VERSION}" + noko_args = args.dup + noko_args[:blacklist] = bl + noko_args[:wspace] = wspace + if block + yield(:parser, parser) + import_appscan_noko_stream(noko_args) {|type, data| yield type,data} + else + import_appscan_noko_stream(noko_args) + end + return true + else # Sorry + raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.") + end + end +end \ No newline at end of file From 33e121301050699547a51f8536303e5ea03bf0d2 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 09:14:12 -0500 Subject: [PATCH 100/159] Extract Msf::DBManager::Import::Appscan shared examples MSP-11124 --- spec/support/shared/examples/msf/db_manager/import.rb | 3 +-- spec/support/shared/examples/msf/db_manager/import/appscan.rb | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/import/appscan.rb diff --git a/spec/support/shared/examples/msf/db_manager/import.rb b/spec/support/shared/examples/msf/db_manager/import.rb index 833e66fd2c..869a4871f2 100644 --- a/spec/support/shared/examples/msf/db_manager/import.rb +++ b/spec/support/shared/examples/msf/db_manager/import.rb @@ -2,8 +2,6 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :dehex } it { is_expected.to respond_to :emit } it { is_expected.to respond_to :import } - it { is_expected.to respond_to :import_appscan_noko_stream } - it { is_expected.to respond_to :import_appscan_xml } it { is_expected.to respond_to :import_burp_session_noko_stream } it { is_expected.to respond_to :import_burp_session_xml } it { is_expected.to respond_to :import_ci_noko_stream } @@ -71,6 +69,7 @@ shared_examples_for 'Msf::DBManager::Import' do it_should_behave_like 'Msf::DBManager::Import::Acunetix' it_should_behave_like 'Msf::DBManager::Import::Amap' + it_should_behave_like 'Msf::DBManager::Import::Appscan' it_should_behave_like 'Msf::DBManager::Import::IP360' it_should_behave_like 'Msf::DBManager::Import::MsfXml' it_should_behave_like 'Msf::DBManager::Import::Qualys' diff --git a/spec/support/shared/examples/msf/db_manager/import/appscan.rb b/spec/support/shared/examples/msf/db_manager/import/appscan.rb new file mode 100644 index 0000000000..e95f95a9fe --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/import/appscan.rb @@ -0,0 +1,4 @@ +shared_examples_for 'Msf::DBManager::Import::Appscan' do + it { is_expected.to respond_to :import_appscan_noko_stream } + it { is_expected.to respond_to :import_appscan_xml } +end \ No newline at end of file From 8d628c221b7597fb2d5259aef0eaed8c9afa4539 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 09:16:57 -0500 Subject: [PATCH 101/159] Extract Msf::DBManager::Import::Burp MSP-11124 --- lib/msf/core/db_manager/import.rb | 34 ++------------------------ lib/msf/core/db_manager/import/burp.rb | 34 ++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 32 deletions(-) create mode 100644 lib/msf/core/db_manager/import/burp.rb diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index f37242ed31..01c7ccb7f7 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -12,7 +12,6 @@ require 'uri' # require 'packetfu' -require 'rex/parser/burp_session_nokogiri' require 'rex/parser/ci_nokogiri' require 'rex/parser/foundstone_nokogiri' require 'rex/parser/fusionvm_nokogiri' @@ -33,6 +32,7 @@ module Msf::DBManager::Import autoload :Acunetix, 'msf/core/db_manager/import/acunetix' autoload :Amap, 'msf/core/db_manager/import/amap' autoload :Appscan, 'msf/core/db_manager/import/appscan' + autoload :Burp, 'msf/core/db_manager/import/burp' autoload :IP360, 'msf/core/db_manager/import/ip360' autoload :MsfXml, 'msf/core/db_manager/import/msf_xml' autoload :Qualys, 'msf/core/db_manager/import/qualys' @@ -40,6 +40,7 @@ module Msf::DBManager::Import include Msf::DBManager::Import::Acunetix include Msf::DBManager::Import::Amap include Msf::DBManager::Import::Appscan + include Msf::DBManager::Import::Burp include Msf::DBManager::Import::IP360 include Msf::DBManager::Import::MsfXml include Msf::DBManager::Import::Qualys @@ -70,37 +71,6 @@ module Msf::DBManager::Import self.send "import_#{ftype}".to_sym, args, &block end - def import_burp_session_noko_stream(args={},&block) - if block - doc = Rex::Parser::BurpSessionDocument.new(args,framework.db) {|type, data| yield type,data } - else - doc = Rex::Parser::BurpSessionDocument.new(args,self) - end - parser = ::Nokogiri::XML::SAX::Parser.new(doc) - parser.parse(args[:data]) - end - - def import_burp_session_xml(args={}, &block) - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - wspace = args[:wspace] || workspace - if Rex::Parser.nokogiri_loaded - # Rex::Parser.reload("burp_session_nokogiri.rb") - parser = "Nokogiri v#{::Nokogiri::VERSION}" - noko_args = args.dup - noko_args[:blacklist] = bl - noko_args[:wspace] = wspace - if block - yield(:parser, parser) - import_burp_session_noko_stream(noko_args) {|type, data| yield type,data} - else - import_burp_session_noko_stream(noko_args) - end - return true - else # Sorry - raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.") - end - end - def import_ci_noko_stream(args, &block) if block doc = Rex::Parser::CIDocument.new(args,framework.db) {|type, data| yield type,data } diff --git a/lib/msf/core/db_manager/import/burp.rb b/lib/msf/core/db_manager/import/burp.rb new file mode 100644 index 0000000000..b1ca9ed0b2 --- /dev/null +++ b/lib/msf/core/db_manager/import/burp.rb @@ -0,0 +1,34 @@ +require 'rex/parser/burp_session_nokogiri' + +module Msf::DBManager::Import::Burp + def import_burp_session_noko_stream(args={},&block) + if block + doc = Rex::Parser::BurpSessionDocument.new(args,framework.db) {|type, data| yield type,data } + else + doc = Rex::Parser::BurpSessionDocument.new(args,self) + end + parser = ::Nokogiri::XML::SAX::Parser.new(doc) + parser.parse(args[:data]) + end + + def import_burp_session_xml(args={}, &block) + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + wspace = args[:wspace] || workspace + if Rex::Parser.nokogiri_loaded + # Rex::Parser.reload("burp_session_nokogiri.rb") + parser = "Nokogiri v#{::Nokogiri::VERSION}" + noko_args = args.dup + noko_args[:blacklist] = bl + noko_args[:wspace] = wspace + if block + yield(:parser, parser) + import_burp_session_noko_stream(noko_args) {|type, data| yield type,data} + else + import_burp_session_noko_stream(noko_args) + end + return true + else # Sorry + raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.") + end + end +end From 85f35bd29b7ac1383541a39ba70c04ed4d1457f9 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 09:18:22 -0500 Subject: [PATCH 102/159] Extract Msf::DBManager::Import::Burp shared examples MSP-11124 --- spec/support/shared/examples/msf/db_manager/import.rb | 3 +-- spec/support/shared/examples/msf/db_manager/import/burp.rb | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/import/burp.rb diff --git a/spec/support/shared/examples/msf/db_manager/import.rb b/spec/support/shared/examples/msf/db_manager/import.rb index 869a4871f2..1de054d5bc 100644 --- a/spec/support/shared/examples/msf/db_manager/import.rb +++ b/spec/support/shared/examples/msf/db_manager/import.rb @@ -2,8 +2,6 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :dehex } it { is_expected.to respond_to :emit } it { is_expected.to respond_to :import } - it { is_expected.to respond_to :import_burp_session_noko_stream } - it { is_expected.to respond_to :import_burp_session_xml } it { is_expected.to respond_to :import_ci_noko_stream } it { is_expected.to respond_to :import_ci_xml } it { is_expected.to respond_to :import_file } @@ -70,6 +68,7 @@ shared_examples_for 'Msf::DBManager::Import' do it_should_behave_like 'Msf::DBManager::Import::Acunetix' it_should_behave_like 'Msf::DBManager::Import::Amap' it_should_behave_like 'Msf::DBManager::Import::Appscan' + it_should_behave_like 'Msf::DBManager::Import::Burp' it_should_behave_like 'Msf::DBManager::Import::IP360' it_should_behave_like 'Msf::DBManager::Import::MsfXml' it_should_behave_like 'Msf::DBManager::Import::Qualys' diff --git a/spec/support/shared/examples/msf/db_manager/import/burp.rb b/spec/support/shared/examples/msf/db_manager/import/burp.rb new file mode 100644 index 0000000000..33839490f1 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/import/burp.rb @@ -0,0 +1,4 @@ +shared_examples_for 'Msf::DBManager::Import::Burp' do + it { is_expected.to respond_to :import_burp_session_noko_stream } + it { is_expected.to respond_to :import_burp_session_xml } +end From 68a1ebd2fbbd7035d0b5aaac6b720f6a9897b8e2 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 09:23:08 -0500 Subject: [PATCH 103/159] Extract Msf::DBManager::Import::CI MSP-11124 --- lib/msf/core/db_manager/import.rb | 33 ++-------------------------- lib/msf/core/db_manager/import/ci.rb | 33 ++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 31 deletions(-) create mode 100644 lib/msf/core/db_manager/import/ci.rb diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index 01c7ccb7f7..27d1a2c20d 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -12,7 +12,6 @@ require 'uri' # require 'packetfu' -require 'rex/parser/ci_nokogiri' require 'rex/parser/foundstone_nokogiri' require 'rex/parser/fusionvm_nokogiri' require 'rex/parser/mbsa_nokogiri' @@ -33,6 +32,7 @@ module Msf::DBManager::Import autoload :Amap, 'msf/core/db_manager/import/amap' autoload :Appscan, 'msf/core/db_manager/import/appscan' autoload :Burp, 'msf/core/db_manager/import/burp' + autoload :CI, 'msf/core/db_manager/import/ci' autoload :IP360, 'msf/core/db_manager/import/ip360' autoload :MsfXml, 'msf/core/db_manager/import/msf_xml' autoload :Qualys, 'msf/core/db_manager/import/qualys' @@ -41,6 +41,7 @@ module Msf::DBManager::Import include Msf::DBManager::Import::Amap include Msf::DBManager::Import::Appscan include Msf::DBManager::Import::Burp + include Msf::DBManager::Import::CI include Msf::DBManager::Import::IP360 include Msf::DBManager::Import::MsfXml include Msf::DBManager::Import::Qualys @@ -71,36 +72,6 @@ module Msf::DBManager::Import self.send "import_#{ftype}".to_sym, args, &block end - def import_ci_noko_stream(args, &block) - if block - doc = Rex::Parser::CIDocument.new(args,framework.db) {|type, data| yield type,data } - else - doc = Rex::Parser::CI.new(args,self) - end - parser = ::Nokogiri::XML::SAX::Parser.new(doc) - parser.parse(args[:data]) - end - - def import_ci_xml(args={}, &block) - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - wspace = args[:wspace] || workspace - if Rex::Parser.nokogiri_loaded - parser = "Nokogiri v#{::Nokogiri::VERSION}" - noko_args = args.dup - noko_args[:blacklist] = bl - noko_args[:wspace] = wspace - if block - yield(:parser, parser) - import_ci_noko_stream(noko_args) {|type, data| yield type,data} - else - import_ci_noko_stream(noko_args) - end - return true - else # Sorry - raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.") - end - end - # # Generic importer that automatically determines the file type being # imported. Since this looks for vendor-specific strings in the given diff --git a/lib/msf/core/db_manager/import/ci.rb b/lib/msf/core/db_manager/import/ci.rb new file mode 100644 index 0000000000..1bfac53d33 --- /dev/null +++ b/lib/msf/core/db_manager/import/ci.rb @@ -0,0 +1,33 @@ +require 'rex/parser/ci_nokogiri' + +module Msf::DBManager::Import::CI + def import_ci_noko_stream(args, &block) + if block + doc = Rex::Parser::CIDocument.new(args,framework.db) {|type, data| yield type,data } + else + doc = Rex::Parser::CI.new(args,self) + end + parser = ::Nokogiri::XML::SAX::Parser.new(doc) + parser.parse(args[:data]) + end + + def import_ci_xml(args={}, &block) + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + wspace = args[:wspace] || workspace + if Rex::Parser.nokogiri_loaded + parser = "Nokogiri v#{::Nokogiri::VERSION}" + noko_args = args.dup + noko_args[:blacklist] = bl + noko_args[:wspace] = wspace + if block + yield(:parser, parser) + import_ci_noko_stream(noko_args) {|type, data| yield type,data} + else + import_ci_noko_stream(noko_args) + end + return true + else # Sorry + raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.") + end + end +end \ No newline at end of file From 56c8bad6c517f55fa8c145023fd8d218e18be991 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 09:24:46 -0500 Subject: [PATCH 104/159] Extract Msf::DBManager::Import::CI shared examples MSP-11124 --- spec/support/shared/examples/msf/db_manager/import.rb | 3 +-- spec/support/shared/examples/msf/db_manager/import/ci.rb | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/import/ci.rb diff --git a/spec/support/shared/examples/msf/db_manager/import.rb b/spec/support/shared/examples/msf/db_manager/import.rb index 1de054d5bc..e089c4d9b4 100644 --- a/spec/support/shared/examples/msf/db_manager/import.rb +++ b/spec/support/shared/examples/msf/db_manager/import.rb @@ -2,8 +2,6 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :dehex } it { is_expected.to respond_to :emit } it { is_expected.to respond_to :import } - it { is_expected.to respond_to :import_ci_noko_stream } - it { is_expected.to respond_to :import_ci_xml } it { is_expected.to respond_to :import_file } it { is_expected.to respond_to :import_filetype_detect } it { is_expected.to respond_to :import_foundstone_noko_stream } @@ -69,6 +67,7 @@ shared_examples_for 'Msf::DBManager::Import' do it_should_behave_like 'Msf::DBManager::Import::Amap' it_should_behave_like 'Msf::DBManager::Import::Appscan' it_should_behave_like 'Msf::DBManager::Import::Burp' + it_should_behave_like 'Msf::DBManager::Import::CI' it_should_behave_like 'Msf::DBManager::Import::IP360' it_should_behave_like 'Msf::DBManager::Import::MsfXml' it_should_behave_like 'Msf::DBManager::Import::Qualys' diff --git a/spec/support/shared/examples/msf/db_manager/import/ci.rb b/spec/support/shared/examples/msf/db_manager/import/ci.rb new file mode 100644 index 0000000000..810480c126 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/import/ci.rb @@ -0,0 +1,4 @@ +shared_examples_for 'Msf::DBManager::Import::CI' do + it { is_expected.to respond_to :import_ci_noko_stream } + it { is_expected.to respond_to :import_ci_xml } +end From 2e85dc910a66af35554711b0f9fbb9ad7a8806b8 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 09:27:53 -0500 Subject: [PATCH 105/159] Extracts Msf::DBManager::Import::Foundstone MSP-11124 --- lib/msf/core/db_manager/import.rb | 33 ++------------------ lib/msf/core/db_manager/import/foundstone.rb | 33 ++++++++++++++++++++ 2 files changed, 35 insertions(+), 31 deletions(-) create mode 100644 lib/msf/core/db_manager/import/foundstone.rb diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index 27d1a2c20d..2a42a0349d 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -12,7 +12,6 @@ require 'uri' # require 'packetfu' -require 'rex/parser/foundstone_nokogiri' require 'rex/parser/fusionvm_nokogiri' require 'rex/parser/mbsa_nokogiri' require 'rex/parser/nessus_xml' @@ -33,6 +32,7 @@ module Msf::DBManager::Import autoload :Appscan, 'msf/core/db_manager/import/appscan' autoload :Burp, 'msf/core/db_manager/import/burp' autoload :CI, 'msf/core/db_manager/import/ci' + autoload :Foundstone, 'msf/core/db_manager/import/foundstone' autoload :IP360, 'msf/core/db_manager/import/ip360' autoload :MsfXml, 'msf/core/db_manager/import/msf_xml' autoload :Qualys, 'msf/core/db_manager/import/qualys' @@ -42,6 +42,7 @@ module Msf::DBManager::Import include Msf::DBManager::Import::Appscan include Msf::DBManager::Import::Burp include Msf::DBManager::Import::CI + include Msf::DBManager::Import::Foundstone include Msf::DBManager::Import::IP360 include Msf::DBManager::Import::MsfXml include Msf::DBManager::Import::Qualys @@ -340,36 +341,6 @@ module Msf::DBManager::Import raise DBImportError.new("Could not automatically determine file type") end - def import_foundstone_noko_stream(args={},&block) - if block - doc = Rex::Parser::FoundstoneDocument.new(args,framework.db) {|type, data| yield type,data } - else - doc = Rex::Parser::FoundstoneDocument.new(args,self) - end - parser = ::Nokogiri::XML::SAX::Parser.new(doc) - parser.parse(args[:data]) - end - - def import_foundstone_xml(args={}, &block) - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - wspace = args[:wspace] || workspace - if Rex::Parser.nokogiri_loaded - parser = "Nokogiri v#{::Nokogiri::VERSION}" - noko_args = args.dup - noko_args[:blacklist] = bl - noko_args[:wspace] = wspace - if block - yield(:parser, parser) - import_foundstone_noko_stream(noko_args) {|type, data| yield type,data} - else - import_foundstone_noko_stream(noko_args) - end - return true - else # Sorry - raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.") - end - end - def import_fusionvm_xml(args={}) args[:wspace] ||= workspace bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] diff --git a/lib/msf/core/db_manager/import/foundstone.rb b/lib/msf/core/db_manager/import/foundstone.rb new file mode 100644 index 0000000000..e8c88cedbc --- /dev/null +++ b/lib/msf/core/db_manager/import/foundstone.rb @@ -0,0 +1,33 @@ +require 'rex/parser/foundstone_nokogiri' + +module Msf::DBManager::Import::Foundstone + def import_foundstone_noko_stream(args={},&block) + if block + doc = Rex::Parser::FoundstoneDocument.new(args,framework.db) {|type, data| yield type,data } + else + doc = Rex::Parser::FoundstoneDocument.new(args,self) + end + parser = ::Nokogiri::XML::SAX::Parser.new(doc) + parser.parse(args[:data]) + end + + def import_foundstone_xml(args={}, &block) + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + wspace = args[:wspace] || workspace + if Rex::Parser.nokogiri_loaded + parser = "Nokogiri v#{::Nokogiri::VERSION}" + noko_args = args.dup + noko_args[:blacklist] = bl + noko_args[:wspace] = wspace + if block + yield(:parser, parser) + import_foundstone_noko_stream(noko_args) {|type, data| yield type,data} + else + import_foundstone_noko_stream(noko_args) + end + return true + else # Sorry + raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.") + end + end +end From 0bfc49682440fe4eaee70f97a0d8aac8f4b47198 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 09:31:17 -0500 Subject: [PATCH 106/159] Extract Msf::DBManager::Import::Foundstone shared examples MSP-11124 --- spec/support/shared/examples/msf/db_manager/import.rb | 3 +-- .../shared/examples/msf/db_manager/import/foundstone.rb | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/import/foundstone.rb diff --git a/spec/support/shared/examples/msf/db_manager/import.rb b/spec/support/shared/examples/msf/db_manager/import.rb index e089c4d9b4..d9e309239f 100644 --- a/spec/support/shared/examples/msf/db_manager/import.rb +++ b/spec/support/shared/examples/msf/db_manager/import.rb @@ -4,8 +4,6 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :import } it { is_expected.to respond_to :import_file } it { is_expected.to respond_to :import_filetype_detect } - it { is_expected.to respond_to :import_foundstone_noko_stream } - it { is_expected.to respond_to :import_foundstone_xml } it { is_expected.to respond_to :import_fusionvm_xml } it { is_expected.to respond_to :import_ip_list } it { is_expected.to respond_to :import_ip_list_file } @@ -68,6 +66,7 @@ shared_examples_for 'Msf::DBManager::Import' do it_should_behave_like 'Msf::DBManager::Import::Appscan' it_should_behave_like 'Msf::DBManager::Import::Burp' it_should_behave_like 'Msf::DBManager::Import::CI' + it_should_behave_like 'Msf::DBManager::Import::Foundstone' it_should_behave_like 'Msf::DBManager::Import::IP360' it_should_behave_like 'Msf::DBManager::Import::MsfXml' it_should_behave_like 'Msf::DBManager::Import::Qualys' diff --git a/spec/support/shared/examples/msf/db_manager/import/foundstone.rb b/spec/support/shared/examples/msf/db_manager/import/foundstone.rb new file mode 100644 index 0000000000..47c11d7ce4 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/import/foundstone.rb @@ -0,0 +1,4 @@ +shared_examples_for 'Msf::DBManager::Import::Foundstone' do + it { is_expected.to respond_to :import_foundstone_noko_stream } + it { is_expected.to respond_to :import_foundstone_xml } +end From e5236e9b567e4b2c5183ffe72255ef34b878448f Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 09:36:29 -0500 Subject: [PATCH 107/159] Extract Msf::DBManager::Import::FusionVM MSP-11124 --- lib/msf/core/db_manager/import.rb | 11 ++--------- lib/msf/core/db_manager/import/fusion_vm.rb | 11 +++++++++++ 2 files changed, 13 insertions(+), 9 deletions(-) create mode 100644 lib/msf/core/db_manager/import/fusion_vm.rb diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index 2a42a0349d..e47a5871d7 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -12,7 +12,6 @@ require 'uri' # require 'packetfu' -require 'rex/parser/fusionvm_nokogiri' require 'rex/parser/mbsa_nokogiri' require 'rex/parser/nessus_xml' require 'rex/parser/netsparker_xml' @@ -33,6 +32,7 @@ module Msf::DBManager::Import autoload :Burp, 'msf/core/db_manager/import/burp' autoload :CI, 'msf/core/db_manager/import/ci' autoload :Foundstone, 'msf/core/db_manager/import/foundstone' + autoload :FusionVM, 'msf/core/db_manager/import/fusion_vm' autoload :IP360, 'msf/core/db_manager/import/ip360' autoload :MsfXml, 'msf/core/db_manager/import/msf_xml' autoload :Qualys, 'msf/core/db_manager/import/qualys' @@ -43,6 +43,7 @@ module Msf::DBManager::Import include Msf::DBManager::Import::Burp include Msf::DBManager::Import::CI include Msf::DBManager::Import::Foundstone + include Msf::DBManager::Import::FusionVM include Msf::DBManager::Import::IP360 include Msf::DBManager::Import::MsfXml include Msf::DBManager::Import::Qualys @@ -341,14 +342,6 @@ module Msf::DBManager::Import raise DBImportError.new("Could not automatically determine file type") end - def import_fusionvm_xml(args={}) - args[:wspace] ||= workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - doc = Rex::Parser::FusionVMDocument.new(args,self) - parser = ::Nokogiri::XML::SAX::Parser.new(doc) - parser.parse(args[:data]) - end - def import_ip_list(args={}, &block) data = args[:data] wspace = args[:wspace] || workspace diff --git a/lib/msf/core/db_manager/import/fusion_vm.rb b/lib/msf/core/db_manager/import/fusion_vm.rb new file mode 100644 index 0000000000..2299e0bd48 --- /dev/null +++ b/lib/msf/core/db_manager/import/fusion_vm.rb @@ -0,0 +1,11 @@ +require 'rex/parser/fusionvm_nokogiri' + +module Msf::DBManager::Import::FusionVM + def import_fusionvm_xml(args={}) + args[:wspace] ||= workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + doc = Rex::Parser::FusionVMDocument.new(args,self) + parser = ::Nokogiri::XML::SAX::Parser.new(doc) + parser.parse(args[:data]) + end +end From 66b1dcc42d4db2dbec4c3a76fb4f35a4ef99b1ba Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 09:38:44 -0500 Subject: [PATCH 108/159] Extract Msf::DBManager::Import::FusionVM shared examples MSP-11124 --- spec/support/shared/examples/msf/db_manager/import.rb | 2 +- .../support/shared/examples/msf/db_manager/import/fusion_vm.rb | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 spec/support/shared/examples/msf/db_manager/import/fusion_vm.rb diff --git a/spec/support/shared/examples/msf/db_manager/import.rb b/spec/support/shared/examples/msf/db_manager/import.rb index d9e309239f..288c6b0384 100644 --- a/spec/support/shared/examples/msf/db_manager/import.rb +++ b/spec/support/shared/examples/msf/db_manager/import.rb @@ -4,7 +4,6 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :import } it { is_expected.to respond_to :import_file } it { is_expected.to respond_to :import_filetype_detect } - it { is_expected.to respond_to :import_fusionvm_xml } it { is_expected.to respond_to :import_ip_list } it { is_expected.to respond_to :import_ip_list_file } it { is_expected.to respond_to :import_libpcap } @@ -67,6 +66,7 @@ shared_examples_for 'Msf::DBManager::Import' do it_should_behave_like 'Msf::DBManager::Import::Burp' it_should_behave_like 'Msf::DBManager::Import::CI' it_should_behave_like 'Msf::DBManager::Import::Foundstone' + it_should_behave_like 'Msf::DBManager::Import::FusionVM' it_should_behave_like 'Msf::DBManager::Import::IP360' it_should_behave_like 'Msf::DBManager::Import::MsfXml' it_should_behave_like 'Msf::DBManager::Import::Qualys' diff --git a/spec/support/shared/examples/msf/db_manager/import/fusion_vm.rb b/spec/support/shared/examples/msf/db_manager/import/fusion_vm.rb new file mode 100644 index 0000000000..6240582d12 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/import/fusion_vm.rb @@ -0,0 +1,3 @@ +shared_examples_for 'Msf::DBManager::Import::FusionVM' do + it { is_expected.to respond_to :import_fusionvm_xml } +end From 44968400e92a30e3a5073cd0291bdad900505386 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 09:42:11 -0500 Subject: [PATCH 109/159] Extract Msf::DBManager::Import::IPList MSP-11124 --- lib/msf/core/db_manager/import.rb | 29 ++--------------------- lib/msf/core/db_manager/import/ip_list.rb | 28 ++++++++++++++++++++++ 2 files changed, 30 insertions(+), 27 deletions(-) create mode 100644 lib/msf/core/db_manager/import/ip_list.rb diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index e47a5871d7..c3803ca40a 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -34,6 +34,7 @@ module Msf::DBManager::Import autoload :Foundstone, 'msf/core/db_manager/import/foundstone' autoload :FusionVM, 'msf/core/db_manager/import/fusion_vm' autoload :IP360, 'msf/core/db_manager/import/ip360' + autoload :IPList, 'msf/core/db_manager/import/ip_list' autoload :MsfXml, 'msf/core/db_manager/import/msf_xml' autoload :Qualys, 'msf/core/db_manager/import/qualys' @@ -45,6 +46,7 @@ module Msf::DBManager::Import include Msf::DBManager::Import::Foundstone include Msf::DBManager::Import::FusionVM include Msf::DBManager::Import::IP360 + include Msf::DBManager::Import::IPList include Msf::DBManager::Import::MsfXml include Msf::DBManager::Import::Qualys @@ -342,33 +344,6 @@ module Msf::DBManager::Import raise DBImportError.new("Could not automatically determine file type") end - def import_ip_list(args={}, &block) - data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - - data.each_line do |ip| - ip.strip! - if bl.include? ip - next - else - yield(:address,ip) if block - end - host = find_or_create_host(:workspace => wspace, :host=> ip, :state => Msf::HostState::Alive, :task => args[:task]) - end - end - - def import_ip_list_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - - data = "" - ::File.open(filename, 'rb') do |f| - data = f.read(f.stat.size) - end - import_ip_list(args.merge(:data => data)) - end - # The libpcap file format is handled by PacketFu for data # extraction. TODO: Make this its own mixin, and possibly # extend PacketFu to do better stream analysis on the fly. diff --git a/lib/msf/core/db_manager/import/ip_list.rb b/lib/msf/core/db_manager/import/ip_list.rb new file mode 100644 index 0000000000..b53c6d0827 --- /dev/null +++ b/lib/msf/core/db_manager/import/ip_list.rb @@ -0,0 +1,28 @@ +module Msf::DBManager::Import::IPList + def import_ip_list(args={}, &block) + data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + + data.each_line do |ip| + ip.strip! + if bl.include? ip + next + else + yield(:address,ip) if block + end + host = find_or_create_host(:workspace => wspace, :host=> ip, :state => Msf::HostState::Alive, :task => args[:task]) + end + end + + def import_ip_list_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + + data = "" + ::File.open(filename, 'rb') do |f| + data = f.read(f.stat.size) + end + import_ip_list(args.merge(:data => data)) + end +end From 8b8befeee1d93ad4487ca7cebbb8a7e82c16ad50 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 09:43:49 -0500 Subject: [PATCH 110/159] Extract Msf::DBManager::Import::IPList shared examples MSP-11124 --- spec/support/shared/examples/msf/db_manager/import.rb | 3 +-- spec/support/shared/examples/msf/db_manager/import/ip_list.rb | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/import/ip_list.rb diff --git a/spec/support/shared/examples/msf/db_manager/import.rb b/spec/support/shared/examples/msf/db_manager/import.rb index 288c6b0384..ea13cddca4 100644 --- a/spec/support/shared/examples/msf/db_manager/import.rb +++ b/spec/support/shared/examples/msf/db_manager/import.rb @@ -4,8 +4,6 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :import } it { is_expected.to respond_to :import_file } it { is_expected.to respond_to :import_filetype_detect } - it { is_expected.to respond_to :import_ip_list } - it { is_expected.to respond_to :import_ip_list_file } it { is_expected.to respond_to :import_libpcap } it { is_expected.to respond_to :import_libpcap_file } it { is_expected.to respond_to :import_mbsa_noko_stream } @@ -68,6 +66,7 @@ shared_examples_for 'Msf::DBManager::Import' do it_should_behave_like 'Msf::DBManager::Import::Foundstone' it_should_behave_like 'Msf::DBManager::Import::FusionVM' it_should_behave_like 'Msf::DBManager::Import::IP360' + it_should_behave_like 'Msf::DBManager::Import::IPList' it_should_behave_like 'Msf::DBManager::Import::MsfXml' it_should_behave_like 'Msf::DBManager::Import::Qualys' end \ No newline at end of file diff --git a/spec/support/shared/examples/msf/db_manager/import/ip_list.rb b/spec/support/shared/examples/msf/db_manager/import/ip_list.rb new file mode 100644 index 0000000000..6404a601a9 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/import/ip_list.rb @@ -0,0 +1,4 @@ +shared_examples_for 'Msf::DBManager::Import::IPList' do + it { is_expected.to respond_to :import_ip_list } + it { is_expected.to respond_to :import_ip_list_file } +end From f29408680f0d759877ba7bf1c7ecd31bb9ae599e Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 09:48:23 -0500 Subject: [PATCH 111/159] Extract Msf::DBManager::Import::Libpcap MSP-11124 --- lib/msf/core/db_manager/import.rb | 127 +--------------------- lib/msf/core/db_manager/import/libpcap.rb | 126 +++++++++++++++++++++ 2 files changed, 128 insertions(+), 125 deletions(-) create mode 100644 lib/msf/core/db_manager/import/libpcap.rb diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index c3803ca40a..7f43851b5b 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -35,6 +35,7 @@ module Msf::DBManager::Import autoload :FusionVM, 'msf/core/db_manager/import/fusion_vm' autoload :IP360, 'msf/core/db_manager/import/ip360' autoload :IPList, 'msf/core/db_manager/import/ip_list' + autoload :Libpcap, 'msf/core/db_manager/import/libpcap' autoload :MsfXml, 'msf/core/db_manager/import/msf_xml' autoload :Qualys, 'msf/core/db_manager/import/qualys' @@ -47,6 +48,7 @@ module Msf::DBManager::Import include Msf::DBManager::Import::FusionVM include Msf::DBManager::Import::IP360 include Msf::DBManager::Import::IPList + include Msf::DBManager::Import::Libpcap include Msf::DBManager::Import::MsfXml include Msf::DBManager::Import::Qualys @@ -344,131 +346,6 @@ module Msf::DBManager::Import raise DBImportError.new("Could not automatically determine file type") end - # The libpcap file format is handled by PacketFu for data - # extraction. TODO: Make this its own mixin, and possibly - # extend PacketFu to do better stream analysis on the fly. - def import_libpcap(args={}, &block) - data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - # seen_hosts is only used for determining when to yield an address. Once we get - # some packet analysis going, the values will have all sorts of info. The plan - # is to ru through all the packets as a first pass and report host and service, - # then, once we have everything parsed, we can reconstruct sessions and ngrep - # out things like authentication sequences, examine ttl's and window sizes, all - # kinds of crazy awesome stuff like that. - seen_hosts = {} - decoded_packets = 0 - last_count = 0 - data.read_packet_bytes do |p| - if (decoded_packets >= last_count + 1000) and block - yield(:pcap_count, decoded_packets) - last_count = decoded_packets - end - decoded_packets += 1 - - pkt = PacketFu::Packet.parse(p) rescue next # Just silently skip bad packets - - next unless pkt.is_ip? # Skip anything that's not IP. Technically, not Ethernet::Ip - next if pkt.is_tcp? && (pkt.tcp_src == 0 || pkt.tcp_dst == 0) # Skip port 0 - next if pkt.is_udp? && (pkt.udp_src == 0 || pkt.udp_dst == 0) # Skip port 0 - saddr = pkt.ip_saddr - daddr = pkt.ip_daddr - - # Handle blacklists and obviously useless IP addresses, and report the host. - next if (bl | [saddr,daddr]).size == bl.size # Both hosts are blacklisted, skip everything. - unless( bl.include?(saddr) || rfc3330_reserved(saddr)) - yield(:address,saddr) if block and !seen_hosts.keys.include?(saddr) - unless seen_hosts[saddr] - report_host( - :workspace => wspace, - :host => saddr, - :state => Msf::HostState::Alive, - :task => args[:task] - ) - end - seen_hosts[saddr] ||= [] - - end - unless( bl.include?(daddr) || rfc3330_reserved(daddr)) - yield(:address,daddr) if block and !seen_hosts.keys.include?(daddr) - unless seen_hosts[daddr] - report_host( - :workspace => wspace, - :host => daddr, - :state => Msf::HostState::Alive, - :task => args[:task] - ) - end - seen_hosts[daddr] ||= [] - end - - if pkt.is_tcp? # First pass on TCP packets - if (pkt.tcp_flags.syn == 1 and pkt.tcp_flags.ack == 1) or # Oh, this kills me - pkt.tcp_src < 1024 # If it's a low port, assume it's a proper service. - if seen_hosts[saddr] - unless seen_hosts[saddr].include? [pkt.tcp_src,"tcp"] - report_service( - :workspace => wspace, :host => saddr, - :proto => "tcp", :port => pkt.tcp_src, - :state => Msf::ServiceState::Open, - :task => args[:task] - ) - seen_hosts[saddr] << [pkt.tcp_src,"tcp"] - yield(:service,"%s:%d/%s" % [saddr,pkt.tcp_src,"tcp"]) - end - end - end - elsif pkt.is_udp? # First pass on UDP packets - if pkt.udp_src == pkt.udp_dst # Very basic p2p detection. - [saddr,daddr].each do |xaddr| - if seen_hosts[xaddr] - unless seen_hosts[xaddr].include? [pkt.udp_src,"udp"] - report_service( - :workspace => wspace, :host => xaddr, - :proto => "udp", :port => pkt.udp_src, - :state => Msf::ServiceState::Open, - :task => args[:task] - ) - seen_hosts[xaddr] << [pkt.udp_src,"udp"] - yield(:service,"%s:%d/%s" % [xaddr,pkt.udp_src,"udp"]) - end - end - end - elsif pkt.udp_src < 1024 # Probably a service - if seen_hosts[saddr] - unless seen_hosts[saddr].include? [pkt.udp_src,"udp"] - report_service( - :workspace => wspace, :host => saddr, - :proto => "udp", :port => pkt.udp_src, - :state => Msf::ServiceState::Open, - :task => args[:task] - ) - seen_hosts[saddr] << [pkt.udp_src,"udp"] - yield(:service,"%s:%d/%s" % [saddr,pkt.udp_src,"udp"]) - end - end - end - end # tcp or udp - - inspect_single_packet(pkt,wspace,args) - - end # data.body.map - - # Right about here, we should have built up some streams for some stream analysis. - # Not sure what form that will take, but people like shoving many hundreds of - # thousands of packets through this thing, so it'll need to be memory efficient. - - end - - def import_libpcap_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - - data = PacketFu::PcapFile.new(:filename => filename) - import_libpcap(args.merge(:data => data)) - end - def import_mbsa_noko_stream(args={},&block) if block doc = Rex::Parser::MbsaDocument.new(args,framework.db) {|type, data| yield type,data } diff --git a/lib/msf/core/db_manager/import/libpcap.rb b/lib/msf/core/db_manager/import/libpcap.rb new file mode 100644 index 0000000000..e26b83ec86 --- /dev/null +++ b/lib/msf/core/db_manager/import/libpcap.rb @@ -0,0 +1,126 @@ +module Msf::DBManager::Import::Libpcap + # The libpcap file format is handled by PacketFu for data + # extraction. TODO: Make this its own mixin, and possibly + # extend PacketFu to do better stream analysis on the fly. + def import_libpcap(args={}, &block) + data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + # seen_hosts is only used for determining when to yield an address. Once we get + # some packet analysis going, the values will have all sorts of info. The plan + # is to ru through all the packets as a first pass and report host and service, + # then, once we have everything parsed, we can reconstruct sessions and ngrep + # out things like authentication sequences, examine ttl's and window sizes, all + # kinds of crazy awesome stuff like that. + seen_hosts = {} + decoded_packets = 0 + last_count = 0 + data.read_packet_bytes do |p| + if (decoded_packets >= last_count + 1000) and block + yield(:pcap_count, decoded_packets) + last_count = decoded_packets + end + decoded_packets += 1 + + pkt = PacketFu::Packet.parse(p) rescue next # Just silently skip bad packets + + next unless pkt.is_ip? # Skip anything that's not IP. Technically, not Ethernet::Ip + next if pkt.is_tcp? && (pkt.tcp_src == 0 || pkt.tcp_dst == 0) # Skip port 0 + next if pkt.is_udp? && (pkt.udp_src == 0 || pkt.udp_dst == 0) # Skip port 0 + saddr = pkt.ip_saddr + daddr = pkt.ip_daddr + + # Handle blacklists and obviously useless IP addresses, and report the host. + next if (bl | [saddr,daddr]).size == bl.size # Both hosts are blacklisted, skip everything. + unless( bl.include?(saddr) || rfc3330_reserved(saddr)) + yield(:address,saddr) if block and !seen_hosts.keys.include?(saddr) + unless seen_hosts[saddr] + report_host( + :workspace => wspace, + :host => saddr, + :state => Msf::HostState::Alive, + :task => args[:task] + ) + end + seen_hosts[saddr] ||= [] + + end + unless( bl.include?(daddr) || rfc3330_reserved(daddr)) + yield(:address,daddr) if block and !seen_hosts.keys.include?(daddr) + unless seen_hosts[daddr] + report_host( + :workspace => wspace, + :host => daddr, + :state => Msf::HostState::Alive, + :task => args[:task] + ) + end + seen_hosts[daddr] ||= [] + end + + if pkt.is_tcp? # First pass on TCP packets + if (pkt.tcp_flags.syn == 1 and pkt.tcp_flags.ack == 1) or # Oh, this kills me + pkt.tcp_src < 1024 # If it's a low port, assume it's a proper service. + if seen_hosts[saddr] + unless seen_hosts[saddr].include? [pkt.tcp_src,"tcp"] + report_service( + :workspace => wspace, :host => saddr, + :proto => "tcp", :port => pkt.tcp_src, + :state => Msf::ServiceState::Open, + :task => args[:task] + ) + seen_hosts[saddr] << [pkt.tcp_src,"tcp"] + yield(:service,"%s:%d/%s" % [saddr,pkt.tcp_src,"tcp"]) + end + end + end + elsif pkt.is_udp? # First pass on UDP packets + if pkt.udp_src == pkt.udp_dst # Very basic p2p detection. + [saddr,daddr].each do |xaddr| + if seen_hosts[xaddr] + unless seen_hosts[xaddr].include? [pkt.udp_src,"udp"] + report_service( + :workspace => wspace, :host => xaddr, + :proto => "udp", :port => pkt.udp_src, + :state => Msf::ServiceState::Open, + :task => args[:task] + ) + seen_hosts[xaddr] << [pkt.udp_src,"udp"] + yield(:service,"%s:%d/%s" % [xaddr,pkt.udp_src,"udp"]) + end + end + end + elsif pkt.udp_src < 1024 # Probably a service + if seen_hosts[saddr] + unless seen_hosts[saddr].include? [pkt.udp_src,"udp"] + report_service( + :workspace => wspace, :host => saddr, + :proto => "udp", :port => pkt.udp_src, + :state => Msf::ServiceState::Open, + :task => args[:task] + ) + seen_hosts[saddr] << [pkt.udp_src,"udp"] + yield(:service,"%s:%d/%s" % [saddr,pkt.udp_src,"udp"]) + end + end + end + end # tcp or udp + + inspect_single_packet(pkt,wspace,args) + + end # data.body.map + + # Right about here, we should have built up some streams for some stream analysis. + # Not sure what form that will take, but people like shoving many hundreds of + # thousands of packets through this thing, so it'll need to be memory efficient. + + end + + def import_libpcap_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + + data = PacketFu::PcapFile.new(:filename => filename) + import_libpcap(args.merge(:data => data)) + end +end From 1edad5e0b7d5e2e89a6b94b93dcf813b95f38e12 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 09:49:58 -0500 Subject: [PATCH 112/159] Extract Msf::DBManager::Import::Libpcap shared examples MSP-11124 --- spec/support/shared/examples/msf/db_manager/import.rb | 3 +-- spec/support/shared/examples/msf/db_manager/import/libpcap.rb | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/import/libpcap.rb diff --git a/spec/support/shared/examples/msf/db_manager/import.rb b/spec/support/shared/examples/msf/db_manager/import.rb index ea13cddca4..92b359dbbe 100644 --- a/spec/support/shared/examples/msf/db_manager/import.rb +++ b/spec/support/shared/examples/msf/db_manager/import.rb @@ -4,8 +4,6 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :import } it { is_expected.to respond_to :import_file } it { is_expected.to respond_to :import_filetype_detect } - it { is_expected.to respond_to :import_libpcap } - it { is_expected.to respond_to :import_libpcap_file } it { is_expected.to respond_to :import_mbsa_noko_stream } it { is_expected.to respond_to :import_mbsa_xml } it { is_expected.to respond_to :import_msf_collateral } @@ -67,6 +65,7 @@ shared_examples_for 'Msf::DBManager::Import' do it_should_behave_like 'Msf::DBManager::Import::FusionVM' it_should_behave_like 'Msf::DBManager::Import::IP360' it_should_behave_like 'Msf::DBManager::Import::IPList' + it_should_behave_like 'Msf::DBManager::Import::Libpcap' it_should_behave_like 'Msf::DBManager::Import::MsfXml' it_should_behave_like 'Msf::DBManager::Import::Qualys' end \ No newline at end of file diff --git a/spec/support/shared/examples/msf/db_manager/import/libpcap.rb b/spec/support/shared/examples/msf/db_manager/import/libpcap.rb new file mode 100644 index 0000000000..4674f4de98 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/import/libpcap.rb @@ -0,0 +1,4 @@ +shared_examples_for 'Msf::DBManager::Import::Libpcap' do + it { is_expected.to respond_to :import_libpcap } + it { is_expected.to respond_to :import_libpcap_file } +end From d870188377367a32a7d70d235967d22126f2758c Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 09:54:03 -0500 Subject: [PATCH 113/159] Extract Msf::DBManager::Import::MBSA MSP-11124 --- lib/msf/core/db_manager/import.rb | 33 ++------------------------ lib/msf/core/db_manager/import/mbsa.rb | 33 ++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 31 deletions(-) create mode 100644 lib/msf/core/db_manager/import/mbsa.rb diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index 7f43851b5b..17e88815ed 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -12,7 +12,6 @@ require 'uri' # require 'packetfu' -require 'rex/parser/mbsa_nokogiri' require 'rex/parser/nessus_xml' require 'rex/parser/netsparker_xml' require 'rex/parser/nexpose_raw_nokogiri' @@ -36,6 +35,7 @@ module Msf::DBManager::Import autoload :IP360, 'msf/core/db_manager/import/ip360' autoload :IPList, 'msf/core/db_manager/import/ip_list' autoload :Libpcap, 'msf/core/db_manager/import/libpcap' + autoload :MBSA, 'msf/core/db_manager/import/mbsa' autoload :MsfXml, 'msf/core/db_manager/import/msf_xml' autoload :Qualys, 'msf/core/db_manager/import/qualys' @@ -49,6 +49,7 @@ module Msf::DBManager::Import include Msf::DBManager::Import::IP360 include Msf::DBManager::Import::IPList include Msf::DBManager::Import::Libpcap + include Msf::DBManager::Import::MBSA include Msf::DBManager::Import::MsfXml include Msf::DBManager::Import::Qualys @@ -346,36 +347,6 @@ module Msf::DBManager::Import raise DBImportError.new("Could not automatically determine file type") end - def import_mbsa_noko_stream(args={},&block) - if block - doc = Rex::Parser::MbsaDocument.new(args,framework.db) {|type, data| yield type,data } - else - doc = Rex::Parser::MbsaDocument.new(args,self) - end - parser = ::Nokogiri::XML::SAX::Parser.new(doc) - parser.parse(args[:data]) - end - - def import_mbsa_xml(args={}, &block) - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - wspace = args[:wspace] || workspace - if Rex::Parser.nokogiri_loaded - parser = "Nokogiri v#{::Nokogiri::VERSION}" - noko_args = args.dup - noko_args[:blacklist] = bl - noko_args[:wspace] = wspace - if block - yield(:parser, parser) - import_mbsa_noko_stream(noko_args) {|type, data| yield type,data} - else - import_mbsa_noko_stream(noko_args) - end - return true - else # Sorry - raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.") - end - end - # Imports loot, tasks, and reports from an MSF ZIP report. # XXX: This function is stupidly long. It needs to be refactored. def import_msf_collateral(args={}, &block) diff --git a/lib/msf/core/db_manager/import/mbsa.rb b/lib/msf/core/db_manager/import/mbsa.rb new file mode 100644 index 0000000000..6bfeb2412b --- /dev/null +++ b/lib/msf/core/db_manager/import/mbsa.rb @@ -0,0 +1,33 @@ +require 'rex/parser/mbsa_nokogiri' + +module Msf::DBManager::Import::MBSA + def import_mbsa_noko_stream(args={},&block) + if block + doc = Rex::Parser::MbsaDocument.new(args,framework.db) {|type, data| yield type,data } + else + doc = Rex::Parser::MbsaDocument.new(args,self) + end + parser = ::Nokogiri::XML::SAX::Parser.new(doc) + parser.parse(args[:data]) + end + + def import_mbsa_xml(args={}, &block) + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + wspace = args[:wspace] || workspace + if Rex::Parser.nokogiri_loaded + parser = "Nokogiri v#{::Nokogiri::VERSION}" + noko_args = args.dup + noko_args[:blacklist] = bl + noko_args[:wspace] = wspace + if block + yield(:parser, parser) + import_mbsa_noko_stream(noko_args) {|type, data| yield type,data} + else + import_mbsa_noko_stream(noko_args) + end + return true + else # Sorry + raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.") + end + end +end From c7d16810ad8b2d0d909a3e929cdac1167bfda5e8 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 09:58:24 -0500 Subject: [PATCH 114/159] Extract Msf::DBManager::Import::MBSA shared examples MSP-11124 --- spec/support/shared/examples/msf/db_manager/import.rb | 3 +-- spec/support/shared/examples/msf/db_manager/import/mbsa.rb | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/import/mbsa.rb diff --git a/spec/support/shared/examples/msf/db_manager/import.rb b/spec/support/shared/examples/msf/db_manager/import.rb index 92b359dbbe..a29fe11754 100644 --- a/spec/support/shared/examples/msf/db_manager/import.rb +++ b/spec/support/shared/examples/msf/db_manager/import.rb @@ -4,8 +4,6 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :import } it { is_expected.to respond_to :import_file } it { is_expected.to respond_to :import_filetype_detect } - it { is_expected.to respond_to :import_mbsa_noko_stream } - it { is_expected.to respond_to :import_mbsa_xml } it { is_expected.to respond_to :import_msf_collateral } it { is_expected.to respond_to :import_msf_cred_dump } it { is_expected.to respond_to :import_msf_cred_dump_zip } @@ -66,6 +64,7 @@ shared_examples_for 'Msf::DBManager::Import' do it_should_behave_like 'Msf::DBManager::Import::IP360' it_should_behave_like 'Msf::DBManager::Import::IPList' it_should_behave_like 'Msf::DBManager::Import::Libpcap' + it_should_behave_like 'Msf::DBManager::Import::MBSA' it_should_behave_like 'Msf::DBManager::Import::MsfXml' it_should_behave_like 'Msf::DBManager::Import::Qualys' end \ No newline at end of file diff --git a/spec/support/shared/examples/msf/db_manager/import/mbsa.rb b/spec/support/shared/examples/msf/db_manager/import/mbsa.rb new file mode 100644 index 0000000000..ea77799037 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/import/mbsa.rb @@ -0,0 +1,4 @@ +shared_examples_for 'Msf::DBManager::Import::MBSA' do + it { is_expected.to respond_to :import_mbsa_noko_stream } + it { is_expected.to respond_to :import_mbsa_xml } +end From ac9a593b43cf5b5adb3aec15254d0c47ca2248ff Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 10:02:42 -0500 Subject: [PATCH 115/159] import_msf_file -> Msf::DBManager::Import::MsfXml MSP-11124 --- lib/msf/core/db_manager/import.rb | 12 ------------ lib/msf/core/db_manager/import/msf_xml.rb | 12 ++++++++++++ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index 17e88815ed..886ffc5f4f 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -516,18 +516,6 @@ module Msf::DBManager::Import nil end - # Import a Metasploit XML file. - def import_msf_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - - data = "" - ::File.open(filename, 'rb') do |f| - data = f.read(f.stat.size) - end - import_msf_xml(args.merge(:data => data)) - end - # Perform in an import of an msfpwdump file def import_msf_pwdump(args={}, &block) filename = File.basename(args[:data].path) diff --git a/lib/msf/core/db_manager/import/msf_xml.rb b/lib/msf/core/db_manager/import/msf_xml.rb index f5af278c1d..4e88240df8 100644 --- a/lib/msf/core/db_manager/import/msf_xml.rb +++ b/lib/msf/core/db_manager/import/msf_xml.rb @@ -52,6 +52,18 @@ module Msf::DBManager::Import::MsfXml # Instance Methods # + # Import a Metasploit XML file. + def import_msf_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + + data = "" + ::File.open(filename, 'rb') do |f| + data = f.read(f.stat.size) + end + import_msf_xml(args.merge(:data => data)) + end + # Imports web_form element using {Msf::DBManager#report_web_form}. # # @param element [REXML::Element] web_form element. From 102e9d8972c57c291508afd2ff7b4c28c2e19282 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 10:04:21 -0500 Subject: [PATCH 116/159] import_msf_file -> Msf::DBManager::Import::MsfXml shared examples MSP-11124 --- spec/support/shared/examples/msf/db_manager/import.rb | 1 - spec/support/shared/examples/msf/db_manager/import/msf_xml.rb | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/spec/support/shared/examples/msf/db_manager/import.rb b/spec/support/shared/examples/msf/db_manager/import.rb index a29fe11754..623dfd3b81 100644 --- a/spec/support/shared/examples/msf/db_manager/import.rb +++ b/spec/support/shared/examples/msf/db_manager/import.rb @@ -7,7 +7,6 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :import_msf_collateral } it { is_expected.to respond_to :import_msf_cred_dump } it { is_expected.to respond_to :import_msf_cred_dump_zip } - it { is_expected.to respond_to :import_msf_file } it { is_expected.to respond_to :import_msf_pwdump } it { is_expected.to respond_to :import_msf_zip } it { is_expected.to respond_to :import_nessus_nbe } diff --git a/spec/support/shared/examples/msf/db_manager/import/msf_xml.rb b/spec/support/shared/examples/msf/db_manager/import/msf_xml.rb index 94030b2476..9b31ae8ba3 100644 --- a/spec/support/shared/examples/msf/db_manager/import/msf_xml.rb +++ b/spec/support/shared/examples/msf/db_manager/import/msf_xml.rb @@ -171,6 +171,8 @@ shared_examples_for 'Msf::DBManager::Import::MsfXml' do end end + it { is_expected.to respond_to :import_msf_file } + context '#import_msf_text_element' do let(:parent_element) do document.root From 65885c8cc85a6bc3121e5f74a621aecbfad02e29 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 10:25:42 -0500 Subject: [PATCH 117/159] MsfXml -> MetasploitFramework::XML MSP-11124 --- lib/msf/core/db_manager/import.rb | 4 ++-- .../db_manager/import/metasploit_framework.rb | 5 +++++ .../xml.rb} | 2 +- .../shared/examples/msf/db_manager/import.rb | 2 +- .../db_manager/import/metasploit_framework.rb | 3 +++ .../xml.rb} | 18 +++++++++--------- .../check_msf_xml_version_with_root_tag.rb | 2 +- .../import_msf_web_element_specialization.rb | 2 +- 8 files changed, 23 insertions(+), 15 deletions(-) create mode 100644 lib/msf/core/db_manager/import/metasploit_framework.rb rename lib/msf/core/db_manager/import/{msf_xml.rb => metasploit_framework/xml.rb} (99%) create mode 100644 spec/support/shared/examples/msf/db_manager/import/metasploit_framework.rb rename spec/support/shared/examples/msf/db_manager/import/{msf_xml.rb => metasploit_framework/xml.rb} (96%) rename spec/support/shared/examples/msf/db_manager/import/{msf_xml => metasploit_framework/xml}/check_msf_xml_version_with_root_tag.rb (80%) rename spec/support/shared/examples/msf/db_manager/import/{msf_xml => metasploit_framework/xml}/import_msf_web_element_specialization.rb (89%) diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index 886ffc5f4f..cb2c4b5efc 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -36,7 +36,7 @@ module Msf::DBManager::Import autoload :IPList, 'msf/core/db_manager/import/ip_list' autoload :Libpcap, 'msf/core/db_manager/import/libpcap' autoload :MBSA, 'msf/core/db_manager/import/mbsa' - autoload :MsfXml, 'msf/core/db_manager/import/msf_xml' + autoload :MetasploitFramework, 'msf/core/db_manager/import/metasploit_framework' autoload :Qualys, 'msf/core/db_manager/import/qualys' include Msf::DBManager::Import::Acunetix @@ -50,7 +50,7 @@ module Msf::DBManager::Import include Msf::DBManager::Import::IPList include Msf::DBManager::Import::Libpcap include Msf::DBManager::Import::MBSA - include Msf::DBManager::Import::MsfXml + include Msf::DBManager::Import::MetasploitFramework include Msf::DBManager::Import::Qualys # If hex notation is present, turn them into a character. diff --git a/lib/msf/core/db_manager/import/metasploit_framework.rb b/lib/msf/core/db_manager/import/metasploit_framework.rb new file mode 100644 index 0000000000..28dcc3647a --- /dev/null +++ b/lib/msf/core/db_manager/import/metasploit_framework.rb @@ -0,0 +1,5 @@ +module Msf::DBManager::Import::MetasploitFramework + autoload :XML, 'msf/core/db_manager/import/metasploit_framework/xml' + + include Msf::DBManager::Import::MetasploitFramework::XML +end \ No newline at end of file diff --git a/lib/msf/core/db_manager/import/msf_xml.rb b/lib/msf/core/db_manager/import/metasploit_framework/xml.rb similarity index 99% rename from lib/msf/core/db_manager/import/msf_xml.rb rename to lib/msf/core/db_manager/import/metasploit_framework/xml.rb index 4e88240df8..b94e564dc4 100644 --- a/lib/msf/core/db_manager/import/msf_xml.rb +++ b/lib/msf/core/db_manager/import/metasploit_framework/xml.rb @@ -4,7 +4,7 @@ # methods to be overridden in Pro without using alias_method_chain as # methods defined in a class cannot be overridden by including a module # (unless you're running Ruby 2.0 and can use prepend) -module Msf::DBManager::Import::MsfXml +module Msf::DBManager::Import::MetasploitFramework::XML # # CONSTANTS # diff --git a/spec/support/shared/examples/msf/db_manager/import.rb b/spec/support/shared/examples/msf/db_manager/import.rb index 623dfd3b81..7b59c43648 100644 --- a/spec/support/shared/examples/msf/db_manager/import.rb +++ b/spec/support/shared/examples/msf/db_manager/import.rb @@ -64,6 +64,6 @@ shared_examples_for 'Msf::DBManager::Import' do it_should_behave_like 'Msf::DBManager::Import::IPList' it_should_behave_like 'Msf::DBManager::Import::Libpcap' it_should_behave_like 'Msf::DBManager::Import::MBSA' - it_should_behave_like 'Msf::DBManager::Import::MsfXml' + it_should_behave_like 'Msf::DBManager::Import::MetasploitFramework' it_should_behave_like 'Msf::DBManager::Import::Qualys' end \ No newline at end of file diff --git a/spec/support/shared/examples/msf/db_manager/import/metasploit_framework.rb b/spec/support/shared/examples/msf/db_manager/import/metasploit_framework.rb new file mode 100644 index 0000000000..552ca6fc9f --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/import/metasploit_framework.rb @@ -0,0 +1,3 @@ +shared_examples_for 'Msf::DBManager::Import::MetasploitFramework' do + it_should_behave_like 'Msf::DBManager::Import::MetasploitFramework::XML' +end diff --git a/spec/support/shared/examples/msf/db_manager/import/msf_xml.rb b/spec/support/shared/examples/msf/db_manager/import/metasploit_framework/xml.rb similarity index 96% rename from spec/support/shared/examples/msf/db_manager/import/msf_xml.rb rename to spec/support/shared/examples/msf/db_manager/import/metasploit_framework/xml.rb index 9b31ae8ba3..110c8b6368 100644 --- a/spec/support/shared/examples/msf/db_manager/import/msf_xml.rb +++ b/spec/support/shared/examples/msf/db_manager/import/metasploit_framework/xml.rb @@ -1,7 +1,7 @@ # -*- coding:binary -*- require 'builder' -shared_examples_for 'Msf::DBManager::Import::MsfXml' do +shared_examples_for 'Msf::DBManager::Import::MetasploitFramework::XML' do # Serialized format from pro/modules/auxiliary/pro/report.rb def serialize(object) # FIXME https://www.pivotaltracker.com/story/show/46578647 @@ -85,7 +85,7 @@ shared_examples_for 'Msf::DBManager::Import::MsfXml' do end it 'should include methods from module so method can be overridden easier in pro' do - db_manager.should be_a Msf::DBManager::Import::MsfXml + db_manager.should be_a Msf::DBManager::Import::MetasploitFramework::XML end context 'CONSTANTS' do @@ -136,25 +136,25 @@ shared_examples_for 'Msf::DBManager::Import::MsfXml' do end it_should_behave_like( - 'Msf::DBManager::Import::MsfXml#check_msf_xml_version! with root tag', + 'Msf::DBManager::Import::MetasploitFramework::XML#check_msf_xml_version! with root tag', 'MetasploitExpressV1', :allow_yaml => true ) it_should_behave_like( - 'Msf::DBManager::Import::MsfXml#check_msf_xml_version! with root tag', + 'Msf::DBManager::Import::MetasploitFramework::XML#check_msf_xml_version! with root tag', 'MetasploitExpressV2', :allow_yaml => true ) it_should_behave_like( - 'Msf::DBManager::Import::MsfXml#check_msf_xml_version! with root tag', + 'Msf::DBManager::Import::MetasploitFramework::XML#check_msf_xml_version! with root tag', 'MetasploitExpressV3', :allow_yaml => false ) it_should_behave_like( - 'Msf::DBManager::Import::MsfXml#check_msf_xml_version! with root tag', + 'Msf::DBManager::Import::MetasploitFramework::XML#check_msf_xml_version! with root tag', 'MetasploitExpressV4', :allow_yaml => false ) @@ -575,7 +575,7 @@ shared_examples_for 'Msf::DBManager::Import::MsfXml' do end context 'call to #import_msf_web_element' do - it_should_behave_like 'Msf::DBManager::Import::MsfXml#import_msf_web_element specialization' + it_should_behave_like 'Msf::DBManager::Import::MetasploitFramework::XML#import_msf_web_element specialization' context 'specialization return' do let(:element) do @@ -671,7 +671,7 @@ shared_examples_for 'Msf::DBManager::Import::MsfXml' do end context 'call to #import_msf_web_element' do - it_should_behave_like 'Msf::DBManager::Import::MsfXml#import_msf_web_element specialization' + it_should_behave_like 'Msf::DBManager::Import::MetasploitFramework::XML#import_msf_web_element specialization' context 'specialization return' do let(:element) do @@ -842,7 +842,7 @@ shared_examples_for 'Msf::DBManager::Import::MsfXml' do end context 'call to #import_msf_web_element' do - it_should_behave_like 'Msf::DBManager::Import::MsfXml#import_msf_web_element specialization' + it_should_behave_like 'Msf::DBManager::Import::MetasploitFramework::XML#import_msf_web_element specialization' context 'specialization return' do let(:element) do diff --git a/spec/support/shared/examples/msf/db_manager/import/msf_xml/check_msf_xml_version_with_root_tag.rb b/spec/support/shared/examples/msf/db_manager/import/metasploit_framework/xml/check_msf_xml_version_with_root_tag.rb similarity index 80% rename from spec/support/shared/examples/msf/db_manager/import/msf_xml/check_msf_xml_version_with_root_tag.rb rename to spec/support/shared/examples/msf/db_manager/import/metasploit_framework/xml/check_msf_xml_version_with_root_tag.rb index 56794c81a0..a467eb75ec 100644 --- a/spec/support/shared/examples/msf/db_manager/import/msf_xml/check_msf_xml_version_with_root_tag.rb +++ b/spec/support/shared/examples/msf/db_manager/import/metasploit_framework/xml/check_msf_xml_version_with_root_tag.rb @@ -1,5 +1,5 @@ # -*- coding:binary -*- -shared_examples_for 'Msf::DBManager::Import::MsfXml#check_msf_xml_version! with root tag' do |root_tag, options={}| +shared_examples_for 'Msf::DBManager::Import::MetasploitFramework::XML#check_msf_xml_version! with root tag' do |root_tag, options={}| options.assert_valid_keys(:allow_yaml) allow_yaml = options.fetch(:allow_yaml) diff --git a/spec/support/shared/examples/msf/db_manager/import/msf_xml/import_msf_web_element_specialization.rb b/spec/support/shared/examples/msf/db_manager/import/metasploit_framework/xml/import_msf_web_element_specialization.rb similarity index 89% rename from spec/support/shared/examples/msf/db_manager/import/msf_xml/import_msf_web_element_specialization.rb rename to spec/support/shared/examples/msf/db_manager/import/metasploit_framework/xml/import_msf_web_element_specialization.rb index 12b88f682c..711d92f075 100644 --- a/spec/support/shared/examples/msf/db_manager/import/msf_xml/import_msf_web_element_specialization.rb +++ b/spec/support/shared/examples/msf/db_manager/import/metasploit_framework/xml/import_msf_web_element_specialization.rb @@ -1,5 +1,5 @@ # -*- coding:binary -*- -shared_examples_for 'Msf::DBManager::Import::MsfXml#import_msf_web_element specialization' do +shared_examples_for 'Msf::DBManager::Import::MetasploitFramework::XML#import_msf_web_element specialization' do it 'should call #import_msf_web_element with element' do db_manager.should_receive(:import_msf_web_element).with(element, anything) From 46a2c47dfe66912dddc2e3286a0bbbd6dd538cd8 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 10:56:08 -0500 Subject: [PATCH 118/159] Extract Msf::DBManager::Import::MetasploitFramework::Zip MSP-11124 --- lib/msf/core/db_manager/import.rb | 227 ----------------- .../db_manager/import/metasploit_framework.rb | 2 + .../import/metasploit_framework/zip.rb | 228 ++++++++++++++++++ 3 files changed, 230 insertions(+), 227 deletions(-) create mode 100644 lib/msf/core/db_manager/import/metasploit_framework/zip.rb diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index cb2c4b5efc..c46b659eaa 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -347,150 +347,6 @@ module Msf::DBManager::Import raise DBImportError.new("Could not automatically determine file type") end - # Imports loot, tasks, and reports from an MSF ZIP report. - # XXX: This function is stupidly long. It needs to be refactored. - def import_msf_collateral(args={}, &block) - data = ::File.open(args[:filename], "rb") {|f| f.read(f.stat.size)} - wspace = args[:wspace] || args['wspace'] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - basedir = args[:basedir] || args['basedir'] || ::File.join(Msf::Config.data_directory, "msf") - - allow_yaml = false - btag = nil - - doc = rexmlify(data) - if doc.elements["MetasploitExpressV1"] - m_ver = 1 - allow_yaml = true - btag = "MetasploitExpressV1" - elsif doc.elements["MetasploitExpressV2"] - m_ver = 2 - allow_yaml = true - btag = "MetasploitExpressV2" - elsif doc.elements["MetasploitExpressV3"] - m_ver = 3 - btag = "MetasploitExpressV3" - elsif doc.elements["MetasploitExpressV4"] - m_ver = 4 - btag = "MetasploitExpressV4" - elsif doc.elements["MetasploitV4"] - m_ver = 4 - btag = "MetasploitV4" - else - m_ver = nil - end - unless m_ver and btag - raise DBImportError.new("Unsupported Metasploit XML document format") - end - - host_info = {} - doc.elements.each("/#{btag}/hosts/host") do |host| - host_info[host.elements["id"].text.to_s.strip] = nils_for_nulls(host.elements["address"].text.to_s.strip) - end - - # Import Loot - doc.elements.each("/#{btag}/loots/loot") do |loot| - next if bl.include? host_info[loot.elements["host-id"].text.to_s.strip] - loot_info = {} - loot_info[:host] = host_info[loot.elements["host-id"].text.to_s.strip] - loot_info[:workspace] = args[:wspace] - loot_info[:ctype] = nils_for_nulls(loot.elements["content-type"].text.to_s.strip) - loot_info[:info] = nils_for_nulls(unserialize_object(loot.elements["info"], allow_yaml)) - loot_info[:ltype] = nils_for_nulls(loot.elements["ltype"].text.to_s.strip) - loot_info[:name] = nils_for_nulls(loot.elements["name"].text.to_s.strip) - loot_info[:created_at] = nils_for_nulls(loot.elements["created-at"].text.to_s.strip) - loot_info[:updated_at] = nils_for_nulls(loot.elements["updated-at"].text.to_s.strip) - loot_info[:name] = nils_for_nulls(loot.elements["name"].text.to_s.strip) - loot_info[:orig_path] = nils_for_nulls(loot.elements["path"].text.to_s.strip) - loot_info[:task] = args[:task] - tmp = args[:ifd][:zip_tmp] - loot_info[:orig_path].gsub!(/^\./,tmp) if loot_info[:orig_path] - if !loot.elements["service-id"].text.to_s.strip.empty? - unless loot.elements["service-id"].text.to_s.strip == "NULL" - loot_info[:service] = loot.elements["service-id"].text.to_s.strip - end - end - - # Only report loot if we actually have it. - # TODO: Copypasta. Seperate this out. - if ::File.exists? loot_info[:orig_path] - loot_dir = ::File.join(basedir,"loot") - loot_file = ::File.split(loot_info[:orig_path]).last - if ::File.exists? loot_dir - unless (::File.directory?(loot_dir) && ::File.writable?(loot_dir)) - raise DBImportError.new("Could not move files to #{loot_dir}") - end - else - ::FileUtils.mkdir_p(loot_dir) - end - new_loot = ::File.join(loot_dir,loot_file) - loot_info[:path] = new_loot - if ::File.exists?(new_loot) - ::File.unlink new_loot # Delete it, and don't report it. - else - report_loot(loot_info) # It's new, so report it. - end - ::FileUtils.copy(loot_info[:orig_path], new_loot) - yield(:msf_loot, new_loot) if block - end - end - - # Import Tasks - doc.elements.each("/#{btag}/tasks/task") do |task| - task_info = {} - task_info[:workspace] = args[:wspace] - # Should user be imported (original) or declared (the importing user)? - task_info[:user] = nils_for_nulls(task.elements["created-by"].text.to_s.strip) - task_info[:desc] = nils_for_nulls(task.elements["description"].text.to_s.strip) - task_info[:info] = nils_for_nulls(unserialize_object(task.elements["info"], allow_yaml)) - task_info[:mod] = nils_for_nulls(task.elements["module"].text.to_s.strip) - task_info[:options] = nils_for_nulls(task.elements["options"].text.to_s.strip) - task_info[:prog] = nils_for_nulls(task.elements["progress"].text.to_s.strip).to_i - task_info[:created_at] = nils_for_nulls(task.elements["created-at"].text.to_s.strip) - task_info[:updated_at] = nils_for_nulls(task.elements["updated-at"].text.to_s.strip) - if !task.elements["completed-at"].text.to_s.empty? - task_info[:completed_at] = nils_for_nulls(task.elements["completed-at"].text.to_s.strip) - end - if !task.elements["error"].text.to_s.empty? - task_info[:error] = nils_for_nulls(task.elements["error"].text.to_s.strip) - end - if !task.elements["result"].text.to_s.empty? - task_info[:result] = nils_for_nulls(task.elements["result"].text.to_s.strip) - end - task_info[:orig_path] = nils_for_nulls(task.elements["path"].text.to_s.strip) - tmp = args[:ifd][:zip_tmp] - task_info[:orig_path].gsub!(/^\./,tmp) if task_info[:orig_path] - - # Only report a task if we actually have it. - # TODO: Copypasta. Seperate this out. - if ::File.exists? task_info[:orig_path] - tasks_dir = ::File.join(basedir,"tasks") - task_file = ::File.split(task_info[:orig_path]).last - if ::File.exists? tasks_dir - unless (::File.directory?(tasks_dir) && ::File.writable?(tasks_dir)) - raise DBImportError.new("Could not move files to #{tasks_dir}") - end - else - ::FileUtils.mkdir_p(tasks_dir) - end - new_task = ::File.join(tasks_dir,task_file) - task_info[:path] = new_task - if ::File.exists?(new_task) - ::File.unlink new_task # Delete it, and don't report it. - else - report_task(task_info) # It's new, so report it. - end - ::FileUtils.copy(task_info[:orig_path], new_task) - yield(:msf_task, new_task) if block - end - end - - # Import Reports - doc.elements.each("/#{btag}/reports/report") do |report| - import_report(report, args, basedir) - end - end - # Import credentials given a path to a valid manifest file # # @param creds_dump_manifest_path [String] @@ -526,89 +382,6 @@ module Msf::DBManager::Import importer.input.close unless importer.input.closed? end - # Import a Metasploit Express ZIP file. Note that this requires - # a fair bit of filesystem manipulation, and is very much tied - # up with the Metasploit Express ZIP file format export (for - # obvious reasons). In the event directories exist, they will - # be reused. If target files exist, they will be overwritten. - # - # XXX: Refactor so it's not quite as sanity-blasting. - def import_msf_zip(args={}, &block) - data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - - new_tmp = ::File.join(Dir::tmpdir,"msf","imp_#{Rex::Text::rand_text_alphanumeric(4)}",@import_filedata[:zip_basename]) - if ::File.exists? new_tmp - unless (::File.directory?(new_tmp) && ::File.writable?(new_tmp)) - raise DBImportError.new("Could not extract zip file to #{new_tmp}") - end - else - FileUtils.mkdir_p(new_tmp) - end - @import_filedata[:zip_tmp] = new_tmp - - # Grab the list of unique basedirs over all entries. - @import_filedata[:zip_tmp_subdirs] = @import_filedata[:zip_entry_names].map {|x| ::File.split(x)}.map {|x| x[0]}.uniq.reject {|x| x == "."} - - # mkdir all of the base directores we just pulled out, if they don't - # already exist - @import_filedata[:zip_tmp_subdirs].each {|sub| - tmp_subdirs = ::File.join(@import_filedata[:zip_tmp],sub) - if File.exists? tmp_subdirs - unless (::File.directory?(tmp_subdirs) && File.writable?(tmp_subdirs)) - # if it exists but we can't write to it, give up - raise DBImportError.new("Could not extract zip file to #{tmp_subdirs}") - end - else - ::FileUtils.mkdir(tmp_subdirs) - end - } - - data.entries.each do |e| - target = ::File.join(@import_filedata[:zip_tmp], e.name) - data.extract(e,target) - - if target =~ /\.xml\z/ - target_data = ::File.open(target, "rb") {|f| f.read 1024} - if import_filetype_detect(target_data) == :msf_xml - @import_filedata[:zip_extracted_xml] = target - end - end - end - - # Import any creds if there are some in the import file - Dir.entries(@import_filedata[:zip_tmp]).each do |entry| - if entry =~ /^.*#{Regexp.quote(Metasploit::Credential::Exporter::Core::CREDS_DUMP_FILE_IDENTIFIER)}.*/ - manifest_file_path = File.join(@import_filedata[:zip_tmp], entry, Metasploit::Credential::Importer::Zip::MANIFEST_FILE_NAME) - if File.exists? manifest_file_path - import_msf_cred_dump(manifest_file_path, wspace) - end - end - end - - # This will kick the newly-extracted XML file through - # the import_file process all over again. - if @import_filedata[:zip_extracted_xml] - new_args = args.dup - new_args[:filename] = @import_filedata[:zip_extracted_xml] - new_args[:data] = nil - new_args[:ifd] = @import_filedata.dup - if block - import_file(new_args, &block) - else - import_file(new_args) - end - end - - # Kick down to all the MSFX ZIP specific items - if block - import_msf_collateral(new_args, &block) - else - import_msf_collateral(new_args) - end - end - # There is no place the NBE actually stores the plugin name used to # scan. You get "Security Note" or "Security Warning," and that's it. def import_nessus_nbe(args={}, &block) diff --git a/lib/msf/core/db_manager/import/metasploit_framework.rb b/lib/msf/core/db_manager/import/metasploit_framework.rb index 28dcc3647a..cb5831d8c4 100644 --- a/lib/msf/core/db_manager/import/metasploit_framework.rb +++ b/lib/msf/core/db_manager/import/metasploit_framework.rb @@ -1,5 +1,7 @@ module Msf::DBManager::Import::MetasploitFramework autoload :XML, 'msf/core/db_manager/import/metasploit_framework/xml' + autoload :Zip, 'msf/core/db_manager/import/metasploit_framework/zip' include Msf::DBManager::Import::MetasploitFramework::XML + include Msf::DBManager::Import::MetasploitFramework::Zip end \ No newline at end of file diff --git a/lib/msf/core/db_manager/import/metasploit_framework/zip.rb b/lib/msf/core/db_manager/import/metasploit_framework/zip.rb new file mode 100644 index 0000000000..9d1cd43a6a --- /dev/null +++ b/lib/msf/core/db_manager/import/metasploit_framework/zip.rb @@ -0,0 +1,228 @@ +module Msf::DBManager::Import::MetasploitFramework::Zip + # Imports loot, tasks, and reports from an MSF ZIP report. + # XXX: This function is stupidly long. It needs to be refactored. + def import_msf_collateral(args={}, &block) + data = ::File.open(args[:filename], "rb") {|f| f.read(f.stat.size)} + wspace = args[:wspace] || args['wspace'] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + basedir = args[:basedir] || args['basedir'] || ::File.join(Msf::Config.data_directory, "msf") + + allow_yaml = false + btag = nil + + doc = rexmlify(data) + if doc.elements["MetasploitExpressV1"] + m_ver = 1 + allow_yaml = true + btag = "MetasploitExpressV1" + elsif doc.elements["MetasploitExpressV2"] + m_ver = 2 + allow_yaml = true + btag = "MetasploitExpressV2" + elsif doc.elements["MetasploitExpressV3"] + m_ver = 3 + btag = "MetasploitExpressV3" + elsif doc.elements["MetasploitExpressV4"] + m_ver = 4 + btag = "MetasploitExpressV4" + elsif doc.elements["MetasploitV4"] + m_ver = 4 + btag = "MetasploitV4" + else + m_ver = nil + end + unless m_ver and btag + raise DBImportError.new("Unsupported Metasploit XML document format") + end + + host_info = {} + doc.elements.each("/#{btag}/hosts/host") do |host| + host_info[host.elements["id"].text.to_s.strip] = nils_for_nulls(host.elements["address"].text.to_s.strip) + end + + # Import Loot + doc.elements.each("/#{btag}/loots/loot") do |loot| + next if bl.include? host_info[loot.elements["host-id"].text.to_s.strip] + loot_info = {} + loot_info[:host] = host_info[loot.elements["host-id"].text.to_s.strip] + loot_info[:workspace] = args[:wspace] + loot_info[:ctype] = nils_for_nulls(loot.elements["content-type"].text.to_s.strip) + loot_info[:info] = nils_for_nulls(unserialize_object(loot.elements["info"], allow_yaml)) + loot_info[:ltype] = nils_for_nulls(loot.elements["ltype"].text.to_s.strip) + loot_info[:name] = nils_for_nulls(loot.elements["name"].text.to_s.strip) + loot_info[:created_at] = nils_for_nulls(loot.elements["created-at"].text.to_s.strip) + loot_info[:updated_at] = nils_for_nulls(loot.elements["updated-at"].text.to_s.strip) + loot_info[:name] = nils_for_nulls(loot.elements["name"].text.to_s.strip) + loot_info[:orig_path] = nils_for_nulls(loot.elements["path"].text.to_s.strip) + loot_info[:task] = args[:task] + tmp = args[:ifd][:zip_tmp] + loot_info[:orig_path].gsub!(/^\./,tmp) if loot_info[:orig_path] + if !loot.elements["service-id"].text.to_s.strip.empty? + unless loot.elements["service-id"].text.to_s.strip == "NULL" + loot_info[:service] = loot.elements["service-id"].text.to_s.strip + end + end + + # Only report loot if we actually have it. + # TODO: Copypasta. Seperate this out. + if ::File.exists? loot_info[:orig_path] + loot_dir = ::File.join(basedir,"loot") + loot_file = ::File.split(loot_info[:orig_path]).last + if ::File.exists? loot_dir + unless (::File.directory?(loot_dir) && ::File.writable?(loot_dir)) + raise DBImportError.new("Could not move files to #{loot_dir}") + end + else + ::FileUtils.mkdir_p(loot_dir) + end + new_loot = ::File.join(loot_dir,loot_file) + loot_info[:path] = new_loot + if ::File.exists?(new_loot) + ::File.unlink new_loot # Delete it, and don't report it. + else + report_loot(loot_info) # It's new, so report it. + end + ::FileUtils.copy(loot_info[:orig_path], new_loot) + yield(:msf_loot, new_loot) if block + end + end + + # Import Tasks + doc.elements.each("/#{btag}/tasks/task") do |task| + task_info = {} + task_info[:workspace] = args[:wspace] + # Should user be imported (original) or declared (the importing user)? + task_info[:user] = nils_for_nulls(task.elements["created-by"].text.to_s.strip) + task_info[:desc] = nils_for_nulls(task.elements["description"].text.to_s.strip) + task_info[:info] = nils_for_nulls(unserialize_object(task.elements["info"], allow_yaml)) + task_info[:mod] = nils_for_nulls(task.elements["module"].text.to_s.strip) + task_info[:options] = nils_for_nulls(task.elements["options"].text.to_s.strip) + task_info[:prog] = nils_for_nulls(task.elements["progress"].text.to_s.strip).to_i + task_info[:created_at] = nils_for_nulls(task.elements["created-at"].text.to_s.strip) + task_info[:updated_at] = nils_for_nulls(task.elements["updated-at"].text.to_s.strip) + if !task.elements["completed-at"].text.to_s.empty? + task_info[:completed_at] = nils_for_nulls(task.elements["completed-at"].text.to_s.strip) + end + if !task.elements["error"].text.to_s.empty? + task_info[:error] = nils_for_nulls(task.elements["error"].text.to_s.strip) + end + if !task.elements["result"].text.to_s.empty? + task_info[:result] = nils_for_nulls(task.elements["result"].text.to_s.strip) + end + task_info[:orig_path] = nils_for_nulls(task.elements["path"].text.to_s.strip) + tmp = args[:ifd][:zip_tmp] + task_info[:orig_path].gsub!(/^\./,tmp) if task_info[:orig_path] + + # Only report a task if we actually have it. + # TODO: Copypasta. Seperate this out. + if ::File.exists? task_info[:orig_path] + tasks_dir = ::File.join(basedir,"tasks") + task_file = ::File.split(task_info[:orig_path]).last + if ::File.exists? tasks_dir + unless (::File.directory?(tasks_dir) && ::File.writable?(tasks_dir)) + raise DBImportError.new("Could not move files to #{tasks_dir}") + end + else + ::FileUtils.mkdir_p(tasks_dir) + end + new_task = ::File.join(tasks_dir,task_file) + task_info[:path] = new_task + if ::File.exists?(new_task) + ::File.unlink new_task # Delete it, and don't report it. + else + report_task(task_info) # It's new, so report it. + end + ::FileUtils.copy(task_info[:orig_path], new_task) + yield(:msf_task, new_task) if block + end + end + + # Import Reports + doc.elements.each("/#{btag}/reports/report") do |report| + import_report(report, args, basedir) + end + end + + # Import a Metasploit Express ZIP file. Note that this requires + # a fair bit of filesystem manipulation, and is very much tied + # up with the Metasploit Express ZIP file format export (for + # obvious reasons). In the event directories exist, they will + # be reused. If target files exist, they will be overwritten. + # + # XXX: Refactor so it's not quite as sanity-blasting. + def import_msf_zip(args={}, &block) + data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + + new_tmp = ::File.join(Dir::tmpdir,"msf","imp_#{Rex::Text::rand_text_alphanumeric(4)}",@import_filedata[:zip_basename]) + if ::File.exists? new_tmp + unless (::File.directory?(new_tmp) && ::File.writable?(new_tmp)) + raise DBImportError.new("Could not extract zip file to #{new_tmp}") + end + else + FileUtils.mkdir_p(new_tmp) + end + @import_filedata[:zip_tmp] = new_tmp + + # Grab the list of unique basedirs over all entries. + @import_filedata[:zip_tmp_subdirs] = @import_filedata[:zip_entry_names].map {|x| ::File.split(x)}.map {|x| x[0]}.uniq.reject {|x| x == "."} + + # mkdir all of the base directores we just pulled out, if they don't + # already exist + @import_filedata[:zip_tmp_subdirs].each {|sub| + tmp_subdirs = ::File.join(@import_filedata[:zip_tmp],sub) + if File.exists? tmp_subdirs + unless (::File.directory?(tmp_subdirs) && File.writable?(tmp_subdirs)) + # if it exists but we can't write to it, give up + raise DBImportError.new("Could not extract zip file to #{tmp_subdirs}") + end + else + ::FileUtils.mkdir(tmp_subdirs) + end + } + + data.entries.each do |e| + target = ::File.join(@import_filedata[:zip_tmp], e.name) + data.extract(e,target) + + if target =~ /\.xml\z/ + target_data = ::File.open(target, "rb") {|f| f.read 1024} + if import_filetype_detect(target_data) == :msf_xml + @import_filedata[:zip_extracted_xml] = target + end + end + end + + # Import any creds if there are some in the import file + Dir.entries(@import_filedata[:zip_tmp]).each do |entry| + if entry =~ /^.*#{Regexp.quote(Metasploit::Credential::Exporter::Core::CREDS_DUMP_FILE_IDENTIFIER)}.*/ + manifest_file_path = File.join(@import_filedata[:zip_tmp], entry, Metasploit::Credential::Importer::Zip::MANIFEST_FILE_NAME) + if File.exists? manifest_file_path + import_msf_cred_dump(manifest_file_path, wspace) + end + end + end + + # This will kick the newly-extracted XML file through + # the import_file process all over again. + if @import_filedata[:zip_extracted_xml] + new_args = args.dup + new_args[:filename] = @import_filedata[:zip_extracted_xml] + new_args[:data] = nil + new_args[:ifd] = @import_filedata.dup + if block + import_file(new_args, &block) + else + import_file(new_args) + end + end + + # Kick down to all the MSFX ZIP specific items + if block + import_msf_collateral(new_args, &block) + else + import_msf_collateral(new_args) + end + end +end From 7d101be0f55aba224ba97ea76bfee8482b4835d8 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 11:02:27 -0500 Subject: [PATCH 119/159] Extract Msf::DBManager::Import::MetasploitFramework::Zip shared examples MSP-11124 --- spec/support/shared/examples/msf/db_manager/import.rb | 2 -- .../examples/msf/db_manager/import/metasploit_framework.rb | 1 + .../msf/db_manager/import/metasploit_framework/zip.rb | 4 ++++ 3 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/import/metasploit_framework/zip.rb diff --git a/spec/support/shared/examples/msf/db_manager/import.rb b/spec/support/shared/examples/msf/db_manager/import.rb index 7b59c43648..ec8c6de8b2 100644 --- a/spec/support/shared/examples/msf/db_manager/import.rb +++ b/spec/support/shared/examples/msf/db_manager/import.rb @@ -4,11 +4,9 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :import } it { is_expected.to respond_to :import_file } it { is_expected.to respond_to :import_filetype_detect } - it { is_expected.to respond_to :import_msf_collateral } it { is_expected.to respond_to :import_msf_cred_dump } it { is_expected.to respond_to :import_msf_cred_dump_zip } it { is_expected.to respond_to :import_msf_pwdump } - it { is_expected.to respond_to :import_msf_zip } it { is_expected.to respond_to :import_nessus_nbe } it { is_expected.to respond_to :import_nessus_nbe_file } it { is_expected.to respond_to :import_nessus_xml } diff --git a/spec/support/shared/examples/msf/db_manager/import/metasploit_framework.rb b/spec/support/shared/examples/msf/db_manager/import/metasploit_framework.rb index 552ca6fc9f..9ec88697e3 100644 --- a/spec/support/shared/examples/msf/db_manager/import/metasploit_framework.rb +++ b/spec/support/shared/examples/msf/db_manager/import/metasploit_framework.rb @@ -1,3 +1,4 @@ shared_examples_for 'Msf::DBManager::Import::MetasploitFramework' do it_should_behave_like 'Msf::DBManager::Import::MetasploitFramework::XML' + it_should_behave_like 'Msf::DBManager::Import::MetasploitFramework::Zip' end diff --git a/spec/support/shared/examples/msf/db_manager/import/metasploit_framework/zip.rb b/spec/support/shared/examples/msf/db_manager/import/metasploit_framework/zip.rb new file mode 100644 index 0000000000..b65791ede7 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/import/metasploit_framework/zip.rb @@ -0,0 +1,4 @@ +shared_examples_for 'Msf::DBManager::Import::MetasploitFramework::Zip' do + it { is_expected.to respond_to :import_msf_collateral } + it { is_expected.to respond_to :import_msf_zip } +end From d0d0c478aa1488e2015fc3c905445b8bfd5b7f58 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 11:12:13 -0500 Subject: [PATCH 120/159] Extract Msf::DBManager::Import::MetasploitFramework::Credential MSP-11124 --- lib/msf/core/db_manager/import.rb | 35 ------------------ .../db_manager/import/metasploit_framework.rb | 2 ++ .../import/metasploit_framework/credential.rb | 36 +++++++++++++++++++ 3 files changed, 38 insertions(+), 35 deletions(-) create mode 100644 lib/msf/core/db_manager/import/metasploit_framework/credential.rb diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index c46b659eaa..34284ff133 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -347,41 +347,6 @@ module Msf::DBManager::Import raise DBImportError.new("Could not automatically determine file type") end - # Import credentials given a path to a valid manifest file - # - # @param creds_dump_manifest_path [String] - # @param workspace [Mdm::Workspace] Default: {#workspace} - # @return [void] - def import_msf_cred_dump(creds_dump_manifest_path, workspace) - manifest_file = File.open(creds_dump_manifest_path) - origin = Metasploit::Credential::Origin::Import.create!(filename: File.basename(creds_dump_manifest_path)) - importer = Metasploit::Credential::Importer::Core.new(workspace: workspace, input: manifest_file, origin: origin) - importer.import! - end - - # Import credentials given a path to a valid manifest file - # - # @option args [String] :filename - # @option args [Mdm::Workspace] :wspace Default: {#workspace} - # @return [void] - def import_msf_cred_dump_zip(args = {}) - wspace = args[:wspace] || workspace - origin = Metasploit::Credential::Origin::Import.create!(filename: File.basename(args[:filename])) - importer = Metasploit::Credential::Importer::Zip.new(workspace: wspace, input: File.open(args[:filename]), origin: origin) - importer.import! - nil - end - - # Perform in an import of an msfpwdump file - def import_msf_pwdump(args={}, &block) - filename = File.basename(args[:data].path) - wspace = args[:wspace] || workspace - origin = Metasploit::Credential::Origin::Import.create!(filename: filename) - importer = Metasploit::Credential::Importer::Pwdump.new(input: args[:data], workspace: wspace, filename: filename, origin:origin) - importer.import! - importer.input.close unless importer.input.closed? - end - # There is no place the NBE actually stores the plugin name used to # scan. You get "Security Note" or "Security Warning," and that's it. def import_nessus_nbe(args={}, &block) diff --git a/lib/msf/core/db_manager/import/metasploit_framework.rb b/lib/msf/core/db_manager/import/metasploit_framework.rb index cb5831d8c4..89913c299d 100644 --- a/lib/msf/core/db_manager/import/metasploit_framework.rb +++ b/lib/msf/core/db_manager/import/metasploit_framework.rb @@ -1,7 +1,9 @@ module Msf::DBManager::Import::MetasploitFramework + autoload :Credential, 'msf/core/db_manager/import/metasploit_framework/credential' autoload :XML, 'msf/core/db_manager/import/metasploit_framework/xml' autoload :Zip, 'msf/core/db_manager/import/metasploit_framework/zip' + include Msf::DBManager::Import::MetasploitFramework::Credential include Msf::DBManager::Import::MetasploitFramework::XML include Msf::DBManager::Import::MetasploitFramework::Zip end \ No newline at end of file diff --git a/lib/msf/core/db_manager/import/metasploit_framework/credential.rb b/lib/msf/core/db_manager/import/metasploit_framework/credential.rb new file mode 100644 index 0000000000..fdf80e76e1 --- /dev/null +++ b/lib/msf/core/db_manager/import/metasploit_framework/credential.rb @@ -0,0 +1,36 @@ +module Msf::DBManager::Import::MetasploitFramework::Credential + # Import credentials given a path to a valid manifest file + # + # @param creds_dump_manifest_path [String] + # @param workspace [Mdm::Workspace] Default: {#workspace} + # @return [void] + def import_msf_cred_dump(creds_dump_manifest_path, workspace) + manifest_file = File.open(creds_dump_manifest_path) + origin = Metasploit::Credential::Origin::Import.create!(filename: File.basename(creds_dump_manifest_path)) + importer = Metasploit::Credential::Importer::Core.new(workspace: workspace, input: manifest_file, origin: origin) + importer.import! + end + + # Import credentials given a path to a valid manifest file + # + # @option args [String] :filename + # @option args [Mdm::Workspace] :wspace Default: {#workspace} + # @return [void] + def import_msf_cred_dump_zip(args = {}) + wspace = args[:wspace] || workspace + origin = Metasploit::Credential::Origin::Import.create!(filename: File.basename(args[:filename])) + importer = Metasploit::Credential::Importer::Zip.new(workspace: wspace, input: File.open(args[:filename]), origin: origin) + importer.import! + nil + end + + # Perform in an import of an msfpwdump file + def import_msf_pwdump(args={}, &block) + filename = File.basename(args[:data].path) + wspace = args[:wspace] || workspace + origin = Metasploit::Credential::Origin::Import.create!(filename: filename) + importer = Metasploit::Credential::Importer::Pwdump.new(input: args[:data], workspace: wspace, filename: filename, origin:origin) + importer.import! + importer.input.close unless importer.input.closed? + end +end From 4c9b29de954356fadefccf5eaa1b6dfc86316ec6 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 11:15:13 -0500 Subject: [PATCH 121/159] Extract Msf::DBManager::Import::MetasploitFramework::Credential shared examples MSP-11124 --- spec/support/shared/examples/msf/db_manager/import.rb | 3 --- .../examples/msf/db_manager/import/metasploit_framework.rb | 1 + .../msf/db_manager/import/metasploit_framework/credential.rb | 5 +++++ 3 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/import/metasploit_framework/credential.rb diff --git a/spec/support/shared/examples/msf/db_manager/import.rb b/spec/support/shared/examples/msf/db_manager/import.rb index ec8c6de8b2..19ecd02119 100644 --- a/spec/support/shared/examples/msf/db_manager/import.rb +++ b/spec/support/shared/examples/msf/db_manager/import.rb @@ -4,9 +4,6 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :import } it { is_expected.to respond_to :import_file } it { is_expected.to respond_to :import_filetype_detect } - it { is_expected.to respond_to :import_msf_cred_dump } - it { is_expected.to respond_to :import_msf_cred_dump_zip } - it { is_expected.to respond_to :import_msf_pwdump } it { is_expected.to respond_to :import_nessus_nbe } it { is_expected.to respond_to :import_nessus_nbe_file } it { is_expected.to respond_to :import_nessus_xml } diff --git a/spec/support/shared/examples/msf/db_manager/import/metasploit_framework.rb b/spec/support/shared/examples/msf/db_manager/import/metasploit_framework.rb index 9ec88697e3..2dc203f113 100644 --- a/spec/support/shared/examples/msf/db_manager/import/metasploit_framework.rb +++ b/spec/support/shared/examples/msf/db_manager/import/metasploit_framework.rb @@ -1,4 +1,5 @@ shared_examples_for 'Msf::DBManager::Import::MetasploitFramework' do + it_should_behave_like 'Msf::DBManager::Import::MetasploitFramework::Credential' it_should_behave_like 'Msf::DBManager::Import::MetasploitFramework::XML' it_should_behave_like 'Msf::DBManager::Import::MetasploitFramework::Zip' end diff --git a/spec/support/shared/examples/msf/db_manager/import/metasploit_framework/credential.rb b/spec/support/shared/examples/msf/db_manager/import/metasploit_framework/credential.rb new file mode 100644 index 0000000000..548c52a69f --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/import/metasploit_framework/credential.rb @@ -0,0 +1,5 @@ +shared_examples_for 'Msf::DBManager::Import::MetasploitFramework::Credential' do + it { is_expected.to respond_to :import_msf_cred_dump } + it { is_expected.to respond_to :import_msf_cred_dump_zip } + it { is_expected.to respond_to :import_msf_pwdump } +end From 0c861848bcef83f5ce64d942d7ad74b09512deb1 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 11:21:26 -0500 Subject: [PATCH 122/159] Extract Msf::DBManager::Import::Nessus::NBE MSP-11124 --- lib/msf/core/db_manager/import.rb | 95 +------------------- lib/msf/core/db_manager/import/nessus.rb | 5 ++ lib/msf/core/db_manager/import/nessus/nbe.rb | 94 +++++++++++++++++++ 3 files changed, 101 insertions(+), 93 deletions(-) create mode 100644 lib/msf/core/db_manager/import/nessus.rb create mode 100644 lib/msf/core/db_manager/import/nessus/nbe.rb diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index 34284ff133..783e6c3fb2 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -37,6 +37,7 @@ module Msf::DBManager::Import autoload :Libpcap, 'msf/core/db_manager/import/libpcap' autoload :MBSA, 'msf/core/db_manager/import/mbsa' autoload :MetasploitFramework, 'msf/core/db_manager/import/metasploit_framework' + autoload :Nessus, 'msf/core/db_manager/import/nessus' autoload :Qualys, 'msf/core/db_manager/import/qualys' include Msf::DBManager::Import::Acunetix @@ -51,6 +52,7 @@ module Msf::DBManager::Import include Msf::DBManager::Import::Libpcap include Msf::DBManager::Import::MBSA include Msf::DBManager::Import::MetasploitFramework + include Msf::DBManager::Import::Nessus include Msf::DBManager::Import::Qualys # If hex notation is present, turn them into a character. @@ -347,99 +349,6 @@ module Msf::DBManager::Import raise DBImportError.new("Could not automatically determine file type") end - # There is no place the NBE actually stores the plugin name used to - # scan. You get "Security Note" or "Security Warning," and that's it. - def import_nessus_nbe(args={}, &block) - nbe_data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - - nbe_copy = nbe_data.dup - # First pass, just to build the address map. - addr_map = {} - - # Cache host objects before passing into handle_nessus() - hobj_map = {} - - nbe_copy.each_line do |line| - r = line.split('|') - next if r[0] != 'results' - next if r[4] != "12053" - data = r[6] - addr,hname = data.match(/([0-9\x2e]+) resolves as (.+)\x2e\\n/n)[1,2] - addr_map[hname] = addr - end - - nbe_data.each_line do |line| - r = line.split('|') - next if r[0] != 'results' - hname = r[2] - if addr_map[hname] - addr = addr_map[hname] - else - addr = hname # Must be unresolved, probably an IP address. - end - port = r[3] - nasl = r[4] - type = r[5] - data = r[6] - - # If there's no resolution, or if it's malformed, skip it. - next unless ipv46_validator(addr) - - if bl.include? addr - next - else - yield(:address,addr) if block - end - - hobj_map[ addr ] ||= report_host(:host => addr, :workspace => wspace, :task => args[:task]) - - # Match the NBE types with the XML severity ratings - case type - # log messages don't actually have any data, they are just - # complaints about not being able to perform this or that test - # because such-and-such was missing - when "Log Message"; next - when "Security Hole"; severity = 3 - when "Security Warning"; severity = 2 - when "Security Note"; severity = 1 - # a severity 0 means there's no extra data, it's just an open port - else; severity = 0 - end - if nasl == "11936" - os = data.match(/The remote host is running (.*)\\n/)[1] - report_note( - :workspace => wspace, - :task => args[:task], - :host => hobj_map[ addr ], - :type => 'host.os.nessus_fingerprint', - :data => { - :os => os.to_s.strip - } - ) - end - - next if nasl.to_s.strip.empty? - plugin_name = nil # NBE doesn't ever populate this - handle_nessus(wspace, hobj_map[ addr ], port, nasl, plugin_name, severity, data) - end - end - - # - # Import Nessus NBE files - # - def import_nessus_nbe_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - - data = "" - ::File.open(filename, 'rb') do |f| - data = f.read(f.stat.size) - end - import_nessus_nbe(args.merge(:data => data)) - end - def import_nessus_xml(args={}, &block) data = args[:data] wspace = args[:wspace] || workspace diff --git a/lib/msf/core/db_manager/import/nessus.rb b/lib/msf/core/db_manager/import/nessus.rb new file mode 100644 index 0000000000..488c99703d --- /dev/null +++ b/lib/msf/core/db_manager/import/nessus.rb @@ -0,0 +1,5 @@ +module Msf::DBManager::Import::Nessus + autoload :NBE, 'msf/core/db_manager/import/nessus/nbe' + + include Msf::DBManager::Import::Nessus::NBE +end \ No newline at end of file diff --git a/lib/msf/core/db_manager/import/nessus/nbe.rb b/lib/msf/core/db_manager/import/nessus/nbe.rb new file mode 100644 index 0000000000..d36e79aa27 --- /dev/null +++ b/lib/msf/core/db_manager/import/nessus/nbe.rb @@ -0,0 +1,94 @@ +module Msf::DBManager::Import::Nessus::NBE + # There is no place the NBE actually stores the plugin name used to + # scan. You get "Security Note" or "Security Warning," and that's it. + def import_nessus_nbe(args={}, &block) + nbe_data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + + nbe_copy = nbe_data.dup + # First pass, just to build the address map. + addr_map = {} + + # Cache host objects before passing into handle_nessus() + hobj_map = {} + + nbe_copy.each_line do |line| + r = line.split('|') + next if r[0] != 'results' + next if r[4] != "12053" + data = r[6] + addr,hname = data.match(/([0-9\x2e]+) resolves as (.+)\x2e\\n/n)[1,2] + addr_map[hname] = addr + end + + nbe_data.each_line do |line| + r = line.split('|') + next if r[0] != 'results' + hname = r[2] + if addr_map[hname] + addr = addr_map[hname] + else + addr = hname # Must be unresolved, probably an IP address. + end + port = r[3] + nasl = r[4] + type = r[5] + data = r[6] + + # If there's no resolution, or if it's malformed, skip it. + next unless ipv46_validator(addr) + + if bl.include? addr + next + else + yield(:address,addr) if block + end + + hobj_map[ addr ] ||= report_host(:host => addr, :workspace => wspace, :task => args[:task]) + + # Match the NBE types with the XML severity ratings + case type + # log messages don't actually have any data, they are just + # complaints about not being able to perform this or that test + # because such-and-such was missing + when "Log Message"; next + when "Security Hole"; severity = 3 + when "Security Warning"; severity = 2 + when "Security Note"; severity = 1 + # a severity 0 means there's no extra data, it's just an open port + else; severity = 0 + end + if nasl == "11936" + os = data.match(/The remote host is running (.*)\\n/)[1] + report_note( + :workspace => wspace, + :task => args[:task], + :host => hobj_map[ addr ], + :type => 'host.os.nessus_fingerprint', + :data => { + :os => os.to_s.strip + } + ) + end + + next if nasl.to_s.strip.empty? + plugin_name = nil # NBE doesn't ever populate this + handle_nessus(wspace, hobj_map[ addr ], port, nasl, plugin_name, severity, data) + end + end + + # + # Import Nessus NBE files + # + def import_nessus_nbe_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + + data = "" + ::File.open(filename, 'rb') do |f| + data = f.read(f.stat.size) + end + import_nessus_nbe(args.merge(:data => data)) + end +end From 71af8fd9c2ad456800a9f1b754c9c879328a2d81 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 11:23:52 -0500 Subject: [PATCH 123/159] Extract Msf::DBManager::Import::Nessus::NBE shared examples MSP-11124 --- spec/support/shared/examples/msf/db_manager/import.rb | 3 +-- spec/support/shared/examples/msf/db_manager/import/nessus.rb | 3 +++ .../shared/examples/msf/db_manager/import/nessus/nbe.rb | 4 ++++ 3 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/import/nessus.rb create mode 100644 spec/support/shared/examples/msf/db_manager/import/nessus/nbe.rb diff --git a/spec/support/shared/examples/msf/db_manager/import.rb b/spec/support/shared/examples/msf/db_manager/import.rb index 19ecd02119..d477a97d23 100644 --- a/spec/support/shared/examples/msf/db_manager/import.rb +++ b/spec/support/shared/examples/msf/db_manager/import.rb @@ -4,8 +4,6 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :import } it { is_expected.to respond_to :import_file } it { is_expected.to respond_to :import_filetype_detect } - it { is_expected.to respond_to :import_nessus_nbe } - it { is_expected.to respond_to :import_nessus_nbe_file } it { is_expected.to respond_to :import_nessus_xml } it { is_expected.to respond_to :import_nessus_xml_file } it { is_expected.to respond_to :import_nessus_xml_v2 } @@ -60,5 +58,6 @@ shared_examples_for 'Msf::DBManager::Import' do it_should_behave_like 'Msf::DBManager::Import::Libpcap' it_should_behave_like 'Msf::DBManager::Import::MBSA' it_should_behave_like 'Msf::DBManager::Import::MetasploitFramework' + it_should_behave_like 'Msf::DBManager::Import::Nessus' it_should_behave_like 'Msf::DBManager::Import::Qualys' end \ No newline at end of file diff --git a/spec/support/shared/examples/msf/db_manager/import/nessus.rb b/spec/support/shared/examples/msf/db_manager/import/nessus.rb new file mode 100644 index 0000000000..43634ef583 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/import/nessus.rb @@ -0,0 +1,3 @@ +shared_examples_for 'Msf::DBManager::Import::Nessus' do + it_should_behave_like 'Msf::DBManager::Import::Nessus::NBE' +end diff --git a/spec/support/shared/examples/msf/db_manager/import/nessus/nbe.rb b/spec/support/shared/examples/msf/db_manager/import/nessus/nbe.rb new file mode 100644 index 0000000000..1925c90f09 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/import/nessus/nbe.rb @@ -0,0 +1,4 @@ +shared_examples_for 'Msf::DBManager::Import::Nessus::NBE' do + it { is_expected.to respond_to :import_nessus_nbe } + it { is_expected.to respond_to :import_nessus_nbe_file } +end From a0494b2eeb9bb210c8218922b85adae6f411e90d Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 11:27:23 -0500 Subject: [PATCH 124/159] Extract Msf::DBManager::Import::Nessus::XML MSP-11124 --- lib/msf/core/db_manager/import.rb | 21 ------------------- lib/msf/core/db_manager/import/nessus.rb | 2 ++ lib/msf/core/db_manager/import/nessus/xml.rb | 22 ++++++++++++++++++++ 3 files changed, 24 insertions(+), 21 deletions(-) create mode 100644 lib/msf/core/db_manager/import/nessus/xml.rb diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index 783e6c3fb2..27c2ff2cda 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -413,27 +413,6 @@ module Msf::DBManager::Import end end - # - # Import Nessus XML v1 and v2 output - # - # Old versions of openvas exported this as well - # - def import_nessus_xml_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - - data = "" - ::File.open(filename, 'rb') do |f| - data = f.read(f.stat.size) - end - - if data.index("NessusClientData_v2") - import_nessus_xml_v2(args.merge(:data => data)) - else - import_nessus_xml(args.merge(:data => data)) - end - end - def import_nessus_xml_v2(args={}, &block) data = args[:data] wspace = args[:wspace] || workspace diff --git a/lib/msf/core/db_manager/import/nessus.rb b/lib/msf/core/db_manager/import/nessus.rb index 488c99703d..0479e10b10 100644 --- a/lib/msf/core/db_manager/import/nessus.rb +++ b/lib/msf/core/db_manager/import/nessus.rb @@ -1,5 +1,7 @@ module Msf::DBManager::Import::Nessus autoload :NBE, 'msf/core/db_manager/import/nessus/nbe' + autoload :XML, 'msf/core/db_manager/import/nessus/xml' include Msf::DBManager::Import::Nessus::NBE + include Msf::DBManager::Import::Nessus::XML end \ No newline at end of file diff --git a/lib/msf/core/db_manager/import/nessus/xml.rb b/lib/msf/core/db_manager/import/nessus/xml.rb new file mode 100644 index 0000000000..ceb8d13bcb --- /dev/null +++ b/lib/msf/core/db_manager/import/nessus/xml.rb @@ -0,0 +1,22 @@ +module Msf::DBManager::Import::Nessus::XML + # + # Import Nessus XML v1 and v2 output + # + # Old versions of openvas exported this as well + # + def import_nessus_xml_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + + data = "" + ::File.open(filename, 'rb') do |f| + data = f.read(f.stat.size) + end + + if data.index("NessusClientData_v2") + import_nessus_xml_v2(args.merge(:data => data)) + else + import_nessus_xml(args.merge(:data => data)) + end + end +end From cee782ab8b3014cdce0ad4303696b54f5c9ca062 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 11:30:23 -0500 Subject: [PATCH 125/159] Extract Msf::DBManager::Import::Nessus::XML shared examples MSP-11124 --- spec/support/shared/examples/msf/db_manager/import.rb | 1 - spec/support/shared/examples/msf/db_manager/import/nessus.rb | 1 + .../shared/examples/msf/db_manager/import/nessus/xml.rb | 3 +++ 3 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 spec/support/shared/examples/msf/db_manager/import/nessus/xml.rb diff --git a/spec/support/shared/examples/msf/db_manager/import.rb b/spec/support/shared/examples/msf/db_manager/import.rb index d477a97d23..949d3f135d 100644 --- a/spec/support/shared/examples/msf/db_manager/import.rb +++ b/spec/support/shared/examples/msf/db_manager/import.rb @@ -5,7 +5,6 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :import_file } it { is_expected.to respond_to :import_filetype_detect } it { is_expected.to respond_to :import_nessus_xml } - it { is_expected.to respond_to :import_nessus_xml_file } it { is_expected.to respond_to :import_nessus_xml_v2 } it { is_expected.to respond_to :import_netsparker_xml } it { is_expected.to respond_to :import_netsparker_xml_file } diff --git a/spec/support/shared/examples/msf/db_manager/import/nessus.rb b/spec/support/shared/examples/msf/db_manager/import/nessus.rb index 43634ef583..c7f8ec76a0 100644 --- a/spec/support/shared/examples/msf/db_manager/import/nessus.rb +++ b/spec/support/shared/examples/msf/db_manager/import/nessus.rb @@ -1,3 +1,4 @@ shared_examples_for 'Msf::DBManager::Import::Nessus' do it_should_behave_like 'Msf::DBManager::Import::Nessus::NBE' + it_should_behave_like 'Msf::DBManager::Import::Nessus::XML' end diff --git a/spec/support/shared/examples/msf/db_manager/import/nessus/xml.rb b/spec/support/shared/examples/msf/db_manager/import/nessus/xml.rb new file mode 100644 index 0000000000..cea9370d8c --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/import/nessus/xml.rb @@ -0,0 +1,3 @@ +shared_examples_for 'Msf::DBManager::Import::Nessus::XML' do + it { is_expected.to respond_to :import_nessus_xml_file } +end From aae6dc9066b077e860bd369c69b590dc14a6db91 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 11:34:37 -0500 Subject: [PATCH 126/159] Extract Msf::DBManager::Import::Nessus::XML::V* MSP-11124 Extract different versions of Nessus XML format. --- lib/msf/core/db_manager/import.rb | 158 ------------------ lib/msf/core/db_manager/import/nessus/xml.rb | 6 + .../core/db_manager/import/nessus/xml/v1.rb | 65 +++++++ .../core/db_manager/import/nessus/xml/v2.rb | 95 +++++++++++ 4 files changed, 166 insertions(+), 158 deletions(-) create mode 100644 lib/msf/core/db_manager/import/nessus/xml/v1.rb create mode 100644 lib/msf/core/db_manager/import/nessus/xml/v2.rb diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index 27c2ff2cda..73f392e2e3 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -349,164 +349,6 @@ module Msf::DBManager::Import raise DBImportError.new("Could not automatically determine file type") end - def import_nessus_xml(args={}, &block) - data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - - doc = rexmlify(data) - doc.elements.each('/NessusClientData/Report/ReportHost') do |host| - hobj = nil - addr = nil - hname = nil - os = nil - # If the name is resolved, the Nessus plugin for DNS - # resolution should be there. If not, fall back to the - # HostName - host.elements.each('ReportItem') do |item| - next unless item.elements['pluginID'].text == "12053" - addr = item.elements['data'].text.match(/([0-9\x2e]+) resolves as/n)[1] - hname = host.elements['HostName'].text - end - addr ||= host.elements['HostName'].text - next unless ipv46_validator(addr) # Skip resolved names and SCAN-ERROR. - if bl.include? addr - next - else - yield(:address,addr) if block - end - - hinfo = { - :workspace => wspace, - :host => addr, - :task => args[:task] - } - - # Record the hostname - hinfo.merge!(:name => hname.to_s.strip) if hname - hobj = report_host(hinfo) - report_import_note(wspace,hobj) - - # Record the OS - os ||= host.elements["os_name"] - if os - report_note( - :workspace => wspace, - :task => args[:task], - :host => hobj, - :type => 'host.os.nessus_fingerprint', - :data => { - :os => os.text.to_s.strip - } - ) - end - - host.elements.each('ReportItem') do |item| - nasl = item.elements['pluginID'].text - plugin_name = item.elements['pluginName'].text - port = item.elements['port'].text - data = item.elements['data'].text - severity = item.elements['severity'].text - - handle_nessus(wspace, hobj, port, nasl, plugin_name, severity, data, args[:task]) - end - end - end - - def import_nessus_xml_v2(args={}, &block) - data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - - #@host = { - #'hname' => nil, - #'addr' => nil, - #'mac' => nil, - #'os' => nil, - #'ports' => [ 'port' => { 'port' => nil, - # 'svc_name' => nil, - # 'proto' => nil, - # 'severity' => nil, - # 'nasl' => nil, - # 'description' => nil, - # 'cve' => [], - # 'bid' => [], - # 'xref' => [] - # } - # ] - #} - parser = Rex::Parser::NessusXMLStreamParser.new - parser.on_found_host = Proc.new { |host| - - hobj = nil - addr = host['addr'] || host['hname'] - - next unless ipv46_validator(addr) # Catches SCAN-ERROR, among others. - - if bl.include? addr - next - else - yield(:address,addr) if block - end - - os = host['os'] - hname = host['hname'] - mac = host['mac'] - - host_info = { - :workspace => wspace, - :host => addr, - :task => args[:task] - } - host_info[:name] = hname.to_s.strip if hname - # Short mac, protect against Nessus's habit of saving multiple macs - # We can't use them anyway, so take just the first. - host_info[:mac] = mac.to_s.strip.upcase.split(/\s+/).first if mac - - hobj = report_host(host_info) - report_import_note(wspace,hobj) - - os = host['os'] - yield(:os,os) if block - if os - report_note( - :workspace => wspace, - :task => args[:task], - :host => hobj, - :type => 'host.os.nessus_fingerprint', - :data => { - :os => os.to_s.strip - } - ) - end - - host['ports'].each do |item| - next if item['port'] == 0 - msf = nil - nasl = item['nasl'].to_s - nasl_name = item['nasl_name'].to_s - port = item['port'].to_s - proto = item['proto'] || "tcp" - sname = item['svc_name'] - severity = item['severity'] - description = item['description'] - cve = item['cve'] - bid = item['bid'] - xref = item['xref'] - msf = item['msf'] - - yield(:port,port) if block - - handle_nessus_v2(wspace, hobj, port, proto, sname, nasl, nasl_name, severity, description, cve, bid, xref, msf, args[:task]) - - end - yield(:end,hname) if block - } - - REXML::Document.parse_stream(data, parser) - - end - # Process NetSparker XML def import_netsparker_xml(args={}, &block) data = args[:data] diff --git a/lib/msf/core/db_manager/import/nessus/xml.rb b/lib/msf/core/db_manager/import/nessus/xml.rb index ceb8d13bcb..e30c7a5e66 100644 --- a/lib/msf/core/db_manager/import/nessus/xml.rb +++ b/lib/msf/core/db_manager/import/nessus/xml.rb @@ -1,4 +1,10 @@ module Msf::DBManager::Import::Nessus::XML + autoload :V1, 'msf/core/db_manager/import/nessus/xml/v1' + autoload :V2, 'msf/core/db_manager/import/nessus/xml/v2' + + include Msf::DBManager::Import::Nessus::XML::V1 + include Msf::DBManager::Import::Nessus::XML::V2 + # # Import Nessus XML v1 and v2 output # diff --git a/lib/msf/core/db_manager/import/nessus/xml/v1.rb b/lib/msf/core/db_manager/import/nessus/xml/v1.rb new file mode 100644 index 0000000000..b37d9c6bf7 --- /dev/null +++ b/lib/msf/core/db_manager/import/nessus/xml/v1.rb @@ -0,0 +1,65 @@ +module Msf::DBManager::Import::Nessus::XML::V1 + def import_nessus_xml(args={}, &block) + data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + + doc = rexmlify(data) + doc.elements.each('/NessusClientData/Report/ReportHost') do |host| + hobj = nil + addr = nil + hname = nil + os = nil + # If the name is resolved, the Nessus plugin for DNS + # resolution should be there. If not, fall back to the + # HostName + host.elements.each('ReportItem') do |item| + next unless item.elements['pluginID'].text == "12053" + addr = item.elements['data'].text.match(/([0-9\x2e]+) resolves as/n)[1] + hname = host.elements['HostName'].text + end + addr ||= host.elements['HostName'].text + next unless ipv46_validator(addr) # Skip resolved names and SCAN-ERROR. + if bl.include? addr + next + else + yield(:address,addr) if block + end + + hinfo = { + :workspace => wspace, + :host => addr, + :task => args[:task] + } + + # Record the hostname + hinfo.merge!(:name => hname.to_s.strip) if hname + hobj = report_host(hinfo) + report_import_note(wspace,hobj) + + # Record the OS + os ||= host.elements["os_name"] + if os + report_note( + :workspace => wspace, + :task => args[:task], + :host => hobj, + :type => 'host.os.nessus_fingerprint', + :data => { + :os => os.text.to_s.strip + } + ) + end + + host.elements.each('ReportItem') do |item| + nasl = item.elements['pluginID'].text + plugin_name = item.elements['pluginName'].text + port = item.elements['port'].text + data = item.elements['data'].text + severity = item.elements['severity'].text + + handle_nessus(wspace, hobj, port, nasl, plugin_name, severity, data, args[:task]) + end + end + end +end \ No newline at end of file diff --git a/lib/msf/core/db_manager/import/nessus/xml/v2.rb b/lib/msf/core/db_manager/import/nessus/xml/v2.rb new file mode 100644 index 0000000000..8c47fa611f --- /dev/null +++ b/lib/msf/core/db_manager/import/nessus/xml/v2.rb @@ -0,0 +1,95 @@ +module Msf::DBManager::Import::Nessus::XML::V2 + def import_nessus_xml_v2(args={}, &block) + data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + + #@host = { + #'hname' => nil, + #'addr' => nil, + #'mac' => nil, + #'os' => nil, + #'ports' => [ 'port' => { 'port' => nil, + # 'svc_name' => nil, + # 'proto' => nil, + # 'severity' => nil, + # 'nasl' => nil, + # 'description' => nil, + # 'cve' => [], + # 'bid' => [], + # 'xref' => [] + # } + # ] + #} + parser = Rex::Parser::NessusXMLStreamParser.new + parser.on_found_host = Proc.new { |host| + + hobj = nil + addr = host['addr'] || host['hname'] + + next unless ipv46_validator(addr) # Catches SCAN-ERROR, among others. + + if bl.include? addr + next + else + yield(:address,addr) if block + end + + os = host['os'] + hname = host['hname'] + mac = host['mac'] + + host_info = { + :workspace => wspace, + :host => addr, + :task => args[:task] + } + host_info[:name] = hname.to_s.strip if hname + # Short mac, protect against Nessus's habit of saving multiple macs + # We can't use them anyway, so take just the first. + host_info[:mac] = mac.to_s.strip.upcase.split(/\s+/).first if mac + + hobj = report_host(host_info) + report_import_note(wspace,hobj) + + os = host['os'] + yield(:os,os) if block + if os + report_note( + :workspace => wspace, + :task => args[:task], + :host => hobj, + :type => 'host.os.nessus_fingerprint', + :data => { + :os => os.to_s.strip + } + ) + end + + host['ports'].each do |item| + next if item['port'] == 0 + msf = nil + nasl = item['nasl'].to_s + nasl_name = item['nasl_name'].to_s + port = item['port'].to_s + proto = item['proto'] || "tcp" + sname = item['svc_name'] + severity = item['severity'] + description = item['description'] + cve = item['cve'] + bid = item['bid'] + xref = item['xref'] + msf = item['msf'] + + yield(:port,port) if block + + handle_nessus_v2(wspace, hobj, port, proto, sname, nasl, nasl_name, severity, description, cve, bid, xref, msf, args[:task]) + + end + yield(:end,hname) if block + } + + REXML::Document.parse_stream(data, parser) + + end +end \ No newline at end of file From cceec8eb53d721d595a77ede6754cea5fd751d8a Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 11:37:24 -0500 Subject: [PATCH 127/159] Extract Msf::DBManager::Import::Nessus::XML::V* shared examples MSP-11124 --- spec/support/shared/examples/msf/db_manager/import.rb | 2 -- .../shared/examples/msf/db_manager/import/nessus/xml.rb | 3 +++ .../shared/examples/msf/db_manager/import/nessus/xml/v1.rb | 3 +++ .../shared/examples/msf/db_manager/import/nessus/xml/v2.rb | 3 +++ 4 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/import/nessus/xml/v1.rb create mode 100644 spec/support/shared/examples/msf/db_manager/import/nessus/xml/v2.rb diff --git a/spec/support/shared/examples/msf/db_manager/import.rb b/spec/support/shared/examples/msf/db_manager/import.rb index 949d3f135d..c17eba2e58 100644 --- a/spec/support/shared/examples/msf/db_manager/import.rb +++ b/spec/support/shared/examples/msf/db_manager/import.rb @@ -4,8 +4,6 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :import } it { is_expected.to respond_to :import_file } it { is_expected.to respond_to :import_filetype_detect } - it { is_expected.to respond_to :import_nessus_xml } - it { is_expected.to respond_to :import_nessus_xml_v2 } it { is_expected.to respond_to :import_netsparker_xml } it { is_expected.to respond_to :import_netsparker_xml_file } it { is_expected.to respond_to :import_nexpose_noko_stream } diff --git a/spec/support/shared/examples/msf/db_manager/import/nessus/xml.rb b/spec/support/shared/examples/msf/db_manager/import/nessus/xml.rb index cea9370d8c..4e6fb60f18 100644 --- a/spec/support/shared/examples/msf/db_manager/import/nessus/xml.rb +++ b/spec/support/shared/examples/msf/db_manager/import/nessus/xml.rb @@ -1,3 +1,6 @@ shared_examples_for 'Msf::DBManager::Import::Nessus::XML' do it { is_expected.to respond_to :import_nessus_xml_file } + + it_should_behave_like 'Msf::DBManager::Import::Nessus::XML::V1' + it_should_behave_like 'Msf::DBManager::Import::Nessus::XML::V2' end diff --git a/spec/support/shared/examples/msf/db_manager/import/nessus/xml/v1.rb b/spec/support/shared/examples/msf/db_manager/import/nessus/xml/v1.rb new file mode 100644 index 0000000000..fbfab82045 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/import/nessus/xml/v1.rb @@ -0,0 +1,3 @@ +shared_examples_for 'Msf::DBManager::Import::Nessus::XML::V1' do + it { is_expected.to respond_to :import_nessus_xml } +end \ No newline at end of file diff --git a/spec/support/shared/examples/msf/db_manager/import/nessus/xml/v2.rb b/spec/support/shared/examples/msf/db_manager/import/nessus/xml/v2.rb new file mode 100644 index 0000000000..2da2bfe147 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/import/nessus/xml/v2.rb @@ -0,0 +1,3 @@ +shared_examples_for 'Msf::DBManager::Import::Nessus::XML::V2' do + it { is_expected.to respond_to :import_nessus_xml_v2 } +end \ No newline at end of file From b43035145d9df4aada91890622137d53de692591 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 11:39:23 -0500 Subject: [PATCH 128/159] Move nessus helper function to closest module MSP-11124 --- lib/msf/core/db_manager/import.rb | 138 ------------------ lib/msf/core/db_manager/import/nessus.rb | 74 ++++++++++ .../core/db_manager/import/nessus/xml/v2.rb | 68 +++++++++ 3 files changed, 142 insertions(+), 138 deletions(-) diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index 73f392e2e3..627e295d5d 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -1930,144 +1930,6 @@ module Msf::DBManager::Import protected - # - # This holds all of the shared parsing/handling used by the - # Nessus NBE and NESSUS v1 methods - # - def handle_nessus(wspace, hobj, port, nasl, plugin_name, severity, data,task=nil) - addr = hobj.address - # The port section looks like: - # http (80/tcp) - p = port.match(/^([^\(]+)\((\d+)\/([^\)]+)\)/) - return if not p - - # Unnecessary as the caller should already have reported this host - #report_host(:workspace => wspace, :host => addr, :state => Msf::HostState::Alive) - name = p[1].strip - port = p[2].to_i - proto = p[3].downcase - - info = { :workspace => wspace, :host => hobj, :port => port, :proto => proto, :task => task } - if name != "unknown" and name[-1,1] != "?" - info[:name] = name - end - report_service(info) - - if nasl.nil? || nasl.empty? || nasl == 0 || nasl == "0" - return - end - - data.gsub!("\\n", "\n") - - refs = [] - - if (data =~ /^CVE : (.*)$/) - $1.gsub(/C(VE|AN)\-/, '').split(',').map { |r| r.strip }.each do |r| - refs.push('CVE-' + r) - end - end - - if (data =~ /^BID : (.*)$/) - $1.split(',').map { |r| r.strip }.each do |r| - refs.push('BID-' + r) - end - end - - if (data =~ /^Other references : (.*)$/) - $1.split(',').map { |r| r.strip }.each do |r| - ref_id, ref_val = r.split(':') - ref_val ? refs.push(ref_id + '-' + ref_val) : refs.push(ref_id) - end - end - - nss = 'NSS-' + nasl.to_s.strip - refs << nss - - unless plugin_name.to_s.strip.empty? - vuln_name = plugin_name - else - vuln_name = nss - end - - vuln_info = { - :workspace => wspace, - :host => hobj, - :port => port, - :proto => proto, - :name => vuln_name, - :info => data, - :refs => refs, - :task => task, - } - report_vuln(vuln_info) - end - - # - # NESSUS v2 file format has a dramatically different layout - # for ReportItem data - # - def handle_nessus_v2(wspace,hobj,port,proto,name,nasl,nasl_name,severity,description,cve,bid,xref,msf,task=nil) - addr = hobj.address - - info = { :workspace => wspace, :host => hobj, :port => port, :proto => proto, :task => task } - - unless name =~ /^unknown$|\?$/ - info[:name] = name - end - - if port.to_i != 0 - report_service(info) - end - - if nasl.nil? || nasl.empty? || nasl == 0 || nasl == "0" - return - end - - refs = [] - - cve.each do |r| - r.to_s.gsub!(/C(VE|AN)\-/, '') - refs.push('CVE-' + r.to_s) - end if cve - - bid.each do |r| - refs.push('BID-' + r.to_s) - end if bid - - xref.each do |r| - ref_id, ref_val = r.to_s.split(':') - ref_val ? refs.push(ref_id + '-' + ref_val) : refs.push(ref_id) - end if xref - - msfref = "MSF-" << msf if msf - refs.push msfref if msfref - - nss = 'NSS-' + nasl - if nasl_name.nil? || nasl_name.empty? - vuln_name = nss - else - vuln_name = nasl_name - end - - refs << nss.strip - - vuln = { - :workspace => wspace, - :host => hobj, - :name => vuln_name, - :info => description ? description : "", - :refs => refs, - :task => task, - } - - if port.to_i != 0 - vuln[:port] = port - vuln[:proto] = proto - end - - report_vuln(vuln) - end - def process_nexpose_data_sxml_refs(vuln) refs = [] vid = vuln.attributes['id'].to_s.downcase diff --git a/lib/msf/core/db_manager/import/nessus.rb b/lib/msf/core/db_manager/import/nessus.rb index 0479e10b10..078438d922 100644 --- a/lib/msf/core/db_manager/import/nessus.rb +++ b/lib/msf/core/db_manager/import/nessus.rb @@ -4,4 +4,78 @@ module Msf::DBManager::Import::Nessus include Msf::DBManager::Import::Nessus::NBE include Msf::DBManager::Import::Nessus::XML + + protected + + # + # This holds all of the shared parsing/handling used by the + # Nessus NBE and NESSUS v1 methods + # + def handle_nessus(wspace, hobj, port, nasl, plugin_name, severity, data,task=nil) + addr = hobj.address + # The port section looks like: + # http (80/tcp) + p = port.match(/^([^\(]+)\((\d+)\/([^\)]+)\)/) + return if not p + + # Unnecessary as the caller should already have reported this host + #report_host(:workspace => wspace, :host => addr, :state => Msf::HostState::Alive) + name = p[1].strip + port = p[2].to_i + proto = p[3].downcase + + info = { :workspace => wspace, :host => hobj, :port => port, :proto => proto, :task => task } + if name != "unknown" and name[-1,1] != "?" + info[:name] = name + end + report_service(info) + + if nasl.nil? || nasl.empty? || nasl == 0 || nasl == "0" + return + end + + data.gsub!("\\n", "\n") + + refs = [] + + if (data =~ /^CVE : (.*)$/) + $1.gsub(/C(VE|AN)\-/, '').split(',').map { |r| r.strip }.each do |r| + refs.push('CVE-' + r) + end + end + + if (data =~ /^BID : (.*)$/) + $1.split(',').map { |r| r.strip }.each do |r| + refs.push('BID-' + r) + end + end + + if (data =~ /^Other references : (.*)$/) + $1.split(',').map { |r| r.strip }.each do |r| + ref_id, ref_val = r.split(':') + ref_val ? refs.push(ref_id + '-' + ref_val) : refs.push(ref_id) + end + end + + nss = 'NSS-' + nasl.to_s.strip + refs << nss + + unless plugin_name.to_s.strip.empty? + vuln_name = plugin_name + else + vuln_name = nss + end + + vuln_info = { + :workspace => wspace, + :host => hobj, + :port => port, + :proto => proto, + :name => vuln_name, + :info => data, + :refs => refs, + :task => task, + } + report_vuln(vuln_info) + end end \ No newline at end of file diff --git a/lib/msf/core/db_manager/import/nessus/xml/v2.rb b/lib/msf/core/db_manager/import/nessus/xml/v2.rb index 8c47fa611f..d8e3125189 100644 --- a/lib/msf/core/db_manager/import/nessus/xml/v2.rb +++ b/lib/msf/core/db_manager/import/nessus/xml/v2.rb @@ -92,4 +92,72 @@ module Msf::DBManager::Import::Nessus::XML::V2 REXML::Document.parse_stream(data, parser) end + + protected + + # + # NESSUS v2 file format has a dramatically different layout + # for ReportItem data + # + def handle_nessus_v2(wspace,hobj,port,proto,name,nasl,nasl_name,severity,description,cve,bid,xref,msf,task=nil) + addr = hobj.address + + info = { :workspace => wspace, :host => hobj, :port => port, :proto => proto, :task => task } + + unless name =~ /^unknown$|\?$/ + info[:name] = name + end + + if port.to_i != 0 + report_service(info) + end + + if nasl.nil? || nasl.empty? || nasl == 0 || nasl == "0" + return + end + + refs = [] + + cve.each do |r| + r.to_s.gsub!(/C(VE|AN)\-/, '') + refs.push('CVE-' + r.to_s) + end if cve + + bid.each do |r| + refs.push('BID-' + r.to_s) + end if bid + + xref.each do |r| + ref_id, ref_val = r.to_s.split(':') + ref_val ? refs.push(ref_id + '-' + ref_val) : refs.push(ref_id) + end if xref + + msfref = "MSF-" << msf if msf + refs.push msfref if msfref + + nss = 'NSS-' + nasl + if nasl_name.nil? || nasl_name.empty? + vuln_name = nss + else + vuln_name = nasl_name + end + + refs << nss.strip + + vuln = { + :workspace => wspace, + :host => hobj, + :name => vuln_name, + :info => description ? description : "", + :refs => refs, + :task => task, + } + + if port.to_i != 0 + vuln[:port] = port + vuln[:proto] = proto + end + + report_vuln(vuln) + end end \ No newline at end of file From a73b0e22835cc1274f9757171b8caadb4958cb21 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 11:42:00 -0500 Subject: [PATCH 129/159] Move requires for nessus parses to appropriate module MSP-11124 --- lib/msf/core/db_manager/import.rb | 1 - lib/msf/core/db_manager/import/nessus/xml/v2.rb | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index 627e295d5d..8bd5d3879a 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -12,7 +12,6 @@ require 'uri' # require 'packetfu' -require 'rex/parser/nessus_xml' require 'rex/parser/netsparker_xml' require 'rex/parser/nexpose_raw_nokogiri' require 'rex/parser/nexpose_simple_nokogiri' diff --git a/lib/msf/core/db_manager/import/nessus/xml/v2.rb b/lib/msf/core/db_manager/import/nessus/xml/v2.rb index d8e3125189..00822d03b7 100644 --- a/lib/msf/core/db_manager/import/nessus/xml/v2.rb +++ b/lib/msf/core/db_manager/import/nessus/xml/v2.rb @@ -1,3 +1,5 @@ +require 'rex/parser/nessus_xml' + module Msf::DBManager::Import::Nessus::XML::V2 def import_nessus_xml_v2(args={}, &block) data = args[:data] From c371eab26a019e612fa640d5f289f9133466d4dc Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 11:46:38 -0500 Subject: [PATCH 130/159] Extract Msf::DBManager::Import::Netsparker MSP-11124 --- lib/msf/core/db_manager/import.rb | 390 +------------------ lib/msf/core/db_manager/import/netsparker.rb | 390 +++++++++++++++++++ 2 files changed, 392 insertions(+), 388 deletions(-) create mode 100644 lib/msf/core/db_manager/import/netsparker.rb diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index 8bd5d3879a..bd3543d675 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -12,7 +12,6 @@ require 'uri' # require 'packetfu' -require 'rex/parser/netsparker_xml' require 'rex/parser/nexpose_raw_nokogiri' require 'rex/parser/nexpose_simple_nokogiri' require 'rex/parser/nexpose_xml' @@ -37,6 +36,7 @@ module Msf::DBManager::Import autoload :MBSA, 'msf/core/db_manager/import/mbsa' autoload :MetasploitFramework, 'msf/core/db_manager/import/metasploit_framework' autoload :Nessus, 'msf/core/db_manager/import/nessus' + autoload :Netsparker, 'msf/core/db_manager/import/netsparker' autoload :Qualys, 'msf/core/db_manager/import/qualys' include Msf::DBManager::Import::Acunetix @@ -52,6 +52,7 @@ module Msf::DBManager::Import include Msf::DBManager::Import::MBSA include Msf::DBManager::Import::MetasploitFramework include Msf::DBManager::Import::Nessus + include Msf::DBManager::Import::Netsparker include Msf::DBManager::Import::Qualys # If hex notation is present, turn them into a character. @@ -348,175 +349,6 @@ module Msf::DBManager::Import raise DBImportError.new("Could not automatically determine file type") end - # Process NetSparker XML - def import_netsparker_xml(args={}, &block) - data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - addr = nil - parser = Rex::Parser::NetSparkerXMLStreamParser.new - parser.on_found_vuln = Proc.new do |vuln| - data = {:workspace => wspace} - - # Parse the URL - url = vuln['url'] - return if not url - - # Crack the URL into a URI - uri = URI(url) rescue nil - return if not uri - - # Resolve the host and cache the IP - if not addr - baddr = Rex::Socket.addr_aton(uri.host) rescue nil - if baddr - addr = Rex::Socket.addr_ntoa(baddr) - yield(:address, addr) if block - end - end - - # Bail early if we have no IP address - if not addr - raise Interrupt, "Not a valid IP address" - end - - if bl.include?(addr) - raise Interrupt, "IP address is on the blacklist" - end - - data[:host] = addr - data[:vhost] = uri.host - data[:port] = uri.port - data[:ssl] = (uri.scheme == "ssl") - - body = nil - # First report a web page - if vuln['response'] - headers = {} - code = 200 - head,body = vuln['response'].to_s.split(/\r?\n\r?\n/, 2) - if body - - if head =~ /^HTTP\d+\.\d+\s+(\d+)\s*/ - code = $1.to_i - end - - headers = {} - head.split(/\r?\n/).each do |line| - hname,hval = line.strip.split(/\s*:\s*/, 2) - next if hval.to_s.strip.empty? - headers[hname.downcase] ||= [] - headers[hname.downcase] << hval - end - - info = { - :path => uri.path, - :query => uri.query, - :code => code, - :body => body, - :headers => headers, - :task => args[:task] - } - info.merge!(data) - - if headers['content-type'] - info[:ctype] = headers['content-type'][0] - end - - if headers['set-cookie'] - info[:cookie] = headers['set-cookie'].join("\n") - end - - if headers['authorization'] - info[:auth] = headers['authorization'].join("\n") - end - - if headers['location'] - info[:location] = headers['location'][0] - end - - if headers['last-modified'] - info[:mtime] = headers['last-modified'][0] - end - - # Report the web page to the database - report_web_page(info) - - yield(:web_page, url) if block - end - end # End web_page reporting - - - details = netsparker_vulnerability_map(vuln) - - method = netsparker_method_map(vuln) - pname = netsparker_pname_map(vuln) - params = netsparker_params_map(vuln) - - proof = '' - - if vuln['info'] and vuln['info'].length > 0 - proof << vuln['info'].map{|x| "#{x[0]}: #{x[1]}\n" }.join + "\n" - end - - if proof.empty? - if body - proof << body + "\n" - else - proof << vuln['response'].to_s + "\n" - end - end - - if params.empty? and pname - params = [[pname, vuln['vparam_name'].to_s]] - end - - info = { - # XXX: There is a :request attr in the model, but report_web_vuln - # doesn't seem to know about it, so this gets ignored. - #:request => vuln['request'], - :path => uri.path, - :query => uri.query, - :method => method, - :params => params, - :pname => pname.to_s, - :proof => proof, - :risk => details[:risk], - :name => details[:name], - :blame => details[:blame], - :category => details[:category], - :description => details[:description], - :confidence => details[:confidence], - :task => args[:task] - } - info.merge!(data) - - next if vuln['type'].to_s.empty? - - report_web_vuln(info) - yield(:web_vuln, url) if block - end - - # We throw interrupts in our parser when the job is hopeless - begin - REXML::Document.parse_stream(data, parser) - rescue ::Interrupt => e - wlog("The netsparker_xml_import() job was interrupted: #{e}") - end - end - - # Process a NetSparker XML file - def import_netsparker_xml_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - - data = "" - ::File.open(filename, 'rb') do |f| - data = f.read(f.stat.size) - end - import_netsparker_xml(args.merge(:data => data)) - end - def import_nexpose_noko_stream(args, &block) if block doc = Rex::Parser::NexposeSimpleDocument.new(args,framework.db) {|type, data| yield type,data } @@ -1445,224 +1277,6 @@ module Msf::DBManager::Import return obj end - def netsparker_method_map(vuln) - case vuln['vparam_type'] - when "FullQueryString" - "GET" - when "Querystring" - "GET" - when "Post" - "POST" - when "RawUrlInjection" - "GET" - else - "GET" - end - end - - def netsparker_params_map(vuln) - [] - end - - def netsparker_pname_map(vuln) - case vuln['vparam_name'] - when "URI-BASED", "Query Based" - "PATH" - else - vuln['vparam_name'] - end - end - - def netsparker_vulnerability_map(vuln) - res = { - :risk => 1, - :name => 'Information Disclosure', - :blame => 'System Administrator', - :category => 'info', - :description => "This is an information leak", - :confidence => 100 - } - - # Risk is a value from 1-5 indicating the severity of the issue - # Examples: 1, 4, 5 - - # Name is a descriptive name for this vulnerability. - # Examples: XSS, ReflectiveXSS, PersistentXSS - - # Blame indicates who is at fault for the vulnerability - # Examples: App Developer, Server Developer, System Administrator - - # Category indicates the general class of vulnerability - # Examples: info, xss, sql, rfi, lfi, cmd - - # Description is a textual summary of the vulnerability - # Examples: "A reflective cross-site scripting attack" - # "The web server leaks the internal IP address" - # "The cookie is not set to HTTP-only" - - # - # Confidence is a value from 1 to 100 indicating how confident the - # software is that the results are valid. - # Examples: 100, 90, 75, 15, 10, 0 - - case vuln['type'].to_s - when "ApacheDirectoryListing" - res = { - :risk => 1, - :name => 'Directory Listing', - :blame => 'System Administrator', - :category => 'info', - :description => "", - :confidence => 100 - } - when "ApacheMultiViewsEnabled" - res = { - :risk => 1, - :name => 'Apache MultiViews Enabled', - :blame => 'System Administrator', - :category => 'info', - :description => "", - :confidence => 100 - } - when "ApacheVersion" - res = { - :risk => 1, - :name => 'Web Server Version', - :blame => 'System Administrator', - :category => 'info', - :description => "", - :confidence => 100 - } - when "PHPVersion" - res = { - :risk => 1, - :name => 'PHP Module Version', - :blame => 'System Administrator', - :category => 'info', - :description => "", - :confidence => 100 - } - when "AutoCompleteEnabled" - res = { - :risk => 1, - :name => 'Form AutoComplete Enabled', - :blame => 'App Developer', - :category => 'info', - :description => "", - :confidence => 100 - } - when "CookieNotMarkedAsHttpOnly" - res = { - :risk => 1, - :name => 'Cookie Not HttpOnly', - :blame => 'App Developer', - :category => 'info', - :description => "", - :confidence => 100 - } - when "EmailDisclosure" - res = { - :risk => 1, - :name => 'Email Address Disclosure', - :blame => 'App Developer', - :category => 'info', - :description => "", - :confidence => 100 - } - when "ForbiddenResource" - res = { - :risk => 1, - :name => 'Forbidden Resource', - :blame => 'App Developer', - :category => 'info', - :description => "", - :confidence => 100 - } - when "FileUploadFound" - res = { - :risk => 1, - :name => 'File Upload Form', - :blame => 'App Developer', - :category => 'info', - :description => "", - :confidence => 100 - } - when "PasswordOverHTTP" - res = { - :risk => 2, - :name => 'Password Over HTTP', - :blame => 'App Developer', - :category => 'info', - :description => "", - :confidence => 100 - } - when "MySQL5Identified" - res = { - :risk => 1, - :name => 'MySQL 5 Identified', - :blame => 'App Developer', - :category => 'info', - :description => "", - :confidence => 100 - } - when "PossibleInternalWindowsPathLeakage" - res = { - :risk => 1, - :name => 'Path Leakage - Windows', - :blame => 'App Developer', - :category => 'info', - :description => "", - :confidence => 100 - } - when "PossibleInternalUnixPathLeakage" - res = { - :risk => 1, - :name => 'Path Leakage - Unix', - :blame => 'App Developer', - :category => 'info', - :description => "", - :confidence => 100 - } - when "PossibleXSS", "LowPossibilityPermanentXSS", "XSS", "PermanentXSS" - conf = 100 - conf = 25 if vuln['type'].to_s == "LowPossibilityPermanentXSS" - conf = 50 if vuln['type'].to_s == "PossibleXSS" - res = { - :risk => 3, - :name => 'Cross-Site Scripting', - :blame => 'App Developer', - :category => 'xss', - :description => "", - :confidence => conf - } - - when "ConfirmedBlindSQLInjection", "ConfirmedSQLInjection", "HighlyPossibleSqlInjection", "DatabaseErrorMessages" - conf = 100 - conf = 90 if vuln['type'].to_s == "HighlyPossibleSqlInjection" - conf = 25 if vuln['type'].to_s == "DatabaseErrorMessages" - res = { - :risk => 5, - :name => 'SQL Injection', - :blame => 'App Developer', - :category => 'sql', - :description => "", - :confidence => conf - } - else - conf = 100 - res = { - :risk => 1, - :name => vuln['type'].to_s, - :blame => 'App Developer', - :category => 'info', - :description => "", - :confidence => conf - } - end - - res - end - # Takes a Host object, an array of vuln structs (generated by nexpose_refs_to_struct()), # and a workspace, and reports the vulns on that host. def nexpose_host_from_rawxml(h, vstructs, wspace,task=nil) diff --git a/lib/msf/core/db_manager/import/netsparker.rb b/lib/msf/core/db_manager/import/netsparker.rb new file mode 100644 index 0000000000..86941ff15f --- /dev/null +++ b/lib/msf/core/db_manager/import/netsparker.rb @@ -0,0 +1,390 @@ +require 'rex/parser/netsparker_xml' + +module Msf::DBManager::Import::Netsparker + # Process NetSparker XML + def import_netsparker_xml(args={}, &block) + data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + addr = nil + parser = Rex::Parser::NetSparkerXMLStreamParser.new + parser.on_found_vuln = Proc.new do |vuln| + data = {:workspace => wspace} + + # Parse the URL + url = vuln['url'] + return if not url + + # Crack the URL into a URI + uri = URI(url) rescue nil + return if not uri + + # Resolve the host and cache the IP + if not addr + baddr = Rex::Socket.addr_aton(uri.host) rescue nil + if baddr + addr = Rex::Socket.addr_ntoa(baddr) + yield(:address, addr) if block + end + end + + # Bail early if we have no IP address + if not addr + raise Interrupt, "Not a valid IP address" + end + + if bl.include?(addr) + raise Interrupt, "IP address is on the blacklist" + end + + data[:host] = addr + data[:vhost] = uri.host + data[:port] = uri.port + data[:ssl] = (uri.scheme == "ssl") + + body = nil + # First report a web page + if vuln['response'] + headers = {} + code = 200 + head,body = vuln['response'].to_s.split(/\r?\n\r?\n/, 2) + if body + + if head =~ /^HTTP\d+\.\d+\s+(\d+)\s*/ + code = $1.to_i + end + + headers = {} + head.split(/\r?\n/).each do |line| + hname,hval = line.strip.split(/\s*:\s*/, 2) + next if hval.to_s.strip.empty? + headers[hname.downcase] ||= [] + headers[hname.downcase] << hval + end + + info = { + :path => uri.path, + :query => uri.query, + :code => code, + :body => body, + :headers => headers, + :task => args[:task] + } + info.merge!(data) + + if headers['content-type'] + info[:ctype] = headers['content-type'][0] + end + + if headers['set-cookie'] + info[:cookie] = headers['set-cookie'].join("\n") + end + + if headers['authorization'] + info[:auth] = headers['authorization'].join("\n") + end + + if headers['location'] + info[:location] = headers['location'][0] + end + + if headers['last-modified'] + info[:mtime] = headers['last-modified'][0] + end + + # Report the web page to the database + report_web_page(info) + + yield(:web_page, url) if block + end + end # End web_page reporting + + + details = netsparker_vulnerability_map(vuln) + + method = netsparker_method_map(vuln) + pname = netsparker_pname_map(vuln) + params = netsparker_params_map(vuln) + + proof = '' + + if vuln['info'] and vuln['info'].length > 0 + proof << vuln['info'].map{|x| "#{x[0]}: #{x[1]}\n" }.join + "\n" + end + + if proof.empty? + if body + proof << body + "\n" + else + proof << vuln['response'].to_s + "\n" + end + end + + if params.empty? and pname + params = [[pname, vuln['vparam_name'].to_s]] + end + + info = { + # XXX: There is a :request attr in the model, but report_web_vuln + # doesn't seem to know about it, so this gets ignored. + #:request => vuln['request'], + :path => uri.path, + :query => uri.query, + :method => method, + :params => params, + :pname => pname.to_s, + :proof => proof, + :risk => details[:risk], + :name => details[:name], + :blame => details[:blame], + :category => details[:category], + :description => details[:description], + :confidence => details[:confidence], + :task => args[:task] + } + info.merge!(data) + + next if vuln['type'].to_s.empty? + + report_web_vuln(info) + yield(:web_vuln, url) if block + end + + # We throw interrupts in our parser when the job is hopeless + begin + REXML::Document.parse_stream(data, parser) + rescue ::Interrupt => e + wlog("The netsparker_xml_import() job was interrupted: #{e}") + end + end + + # Process a NetSparker XML file + def import_netsparker_xml_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + + data = "" + ::File.open(filename, 'rb') do |f| + data = f.read(f.stat.size) + end + import_netsparker_xml(args.merge(:data => data)) + end + + def netsparker_method_map(vuln) + case vuln['vparam_type'] + when "FullQueryString" + "GET" + when "Querystring" + "GET" + when "Post" + "POST" + when "RawUrlInjection" + "GET" + else + "GET" + end + end + + def netsparker_params_map(vuln) + [] + end + + def netsparker_pname_map(vuln) + case vuln['vparam_name'] + when "URI-BASED", "Query Based" + "PATH" + else + vuln['vparam_name'] + end + end + + def netsparker_vulnerability_map(vuln) + res = { + :risk => 1, + :name => 'Information Disclosure', + :blame => 'System Administrator', + :category => 'info', + :description => "This is an information leak", + :confidence => 100 + } + + # Risk is a value from 1-5 indicating the severity of the issue + # Examples: 1, 4, 5 + + # Name is a descriptive name for this vulnerability. + # Examples: XSS, ReflectiveXSS, PersistentXSS + + # Blame indicates who is at fault for the vulnerability + # Examples: App Developer, Server Developer, System Administrator + + # Category indicates the general class of vulnerability + # Examples: info, xss, sql, rfi, lfi, cmd + + # Description is a textual summary of the vulnerability + # Examples: "A reflective cross-site scripting attack" + # "The web server leaks the internal IP address" + # "The cookie is not set to HTTP-only" + + # + # Confidence is a value from 1 to 100 indicating how confident the + # software is that the results are valid. + # Examples: 100, 90, 75, 15, 10, 0 + + case vuln['type'].to_s + when "ApacheDirectoryListing" + res = { + :risk => 1, + :name => 'Directory Listing', + :blame => 'System Administrator', + :category => 'info', + :description => "", + :confidence => 100 + } + when "ApacheMultiViewsEnabled" + res = { + :risk => 1, + :name => 'Apache MultiViews Enabled', + :blame => 'System Administrator', + :category => 'info', + :description => "", + :confidence => 100 + } + when "ApacheVersion" + res = { + :risk => 1, + :name => 'Web Server Version', + :blame => 'System Administrator', + :category => 'info', + :description => "", + :confidence => 100 + } + when "PHPVersion" + res = { + :risk => 1, + :name => 'PHP Module Version', + :blame => 'System Administrator', + :category => 'info', + :description => "", + :confidence => 100 + } + when "AutoCompleteEnabled" + res = { + :risk => 1, + :name => 'Form AutoComplete Enabled', + :blame => 'App Developer', + :category => 'info', + :description => "", + :confidence => 100 + } + when "CookieNotMarkedAsHttpOnly" + res = { + :risk => 1, + :name => 'Cookie Not HttpOnly', + :blame => 'App Developer', + :category => 'info', + :description => "", + :confidence => 100 + } + when "EmailDisclosure" + res = { + :risk => 1, + :name => 'Email Address Disclosure', + :blame => 'App Developer', + :category => 'info', + :description => "", + :confidence => 100 + } + when "ForbiddenResource" + res = { + :risk => 1, + :name => 'Forbidden Resource', + :blame => 'App Developer', + :category => 'info', + :description => "", + :confidence => 100 + } + when "FileUploadFound" + res = { + :risk => 1, + :name => 'File Upload Form', + :blame => 'App Developer', + :category => 'info', + :description => "", + :confidence => 100 + } + when "PasswordOverHTTP" + res = { + :risk => 2, + :name => 'Password Over HTTP', + :blame => 'App Developer', + :category => 'info', + :description => "", + :confidence => 100 + } + when "MySQL5Identified" + res = { + :risk => 1, + :name => 'MySQL 5 Identified', + :blame => 'App Developer', + :category => 'info', + :description => "", + :confidence => 100 + } + when "PossibleInternalWindowsPathLeakage" + res = { + :risk => 1, + :name => 'Path Leakage - Windows', + :blame => 'App Developer', + :category => 'info', + :description => "", + :confidence => 100 + } + when "PossibleInternalUnixPathLeakage" + res = { + :risk => 1, + :name => 'Path Leakage - Unix', + :blame => 'App Developer', + :category => 'info', + :description => "", + :confidence => 100 + } + when "PossibleXSS", "LowPossibilityPermanentXSS", "XSS", "PermanentXSS" + conf = 100 + conf = 25 if vuln['type'].to_s == "LowPossibilityPermanentXSS" + conf = 50 if vuln['type'].to_s == "PossibleXSS" + res = { + :risk => 3, + :name => 'Cross-Site Scripting', + :blame => 'App Developer', + :category => 'xss', + :description => "", + :confidence => conf + } + + when "ConfirmedBlindSQLInjection", "ConfirmedSQLInjection", "HighlyPossibleSqlInjection", "DatabaseErrorMessages" + conf = 100 + conf = 90 if vuln['type'].to_s == "HighlyPossibleSqlInjection" + conf = 25 if vuln['type'].to_s == "DatabaseErrorMessages" + res = { + :risk => 5, + :name => 'SQL Injection', + :blame => 'App Developer', + :category => 'sql', + :description => "", + :confidence => conf + } + else + conf = 100 + res = { + :risk => 1, + :name => vuln['type'].to_s, + :blame => 'App Developer', + :category => 'info', + :description => "", + :confidence => conf + } + end + + res + end +end \ No newline at end of file From 3638b0a91ba131bda189297bcebb370c9112c3e2 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 11:49:01 -0500 Subject: [PATCH 131/159] Extract Msf::DBManager::Import::Netsparker shared examples MSP-11124 --- spec/support/shared/examples/msf/db_manager/import.rb | 7 +------ .../shared/examples/msf/db_manager/import/netsparker.rb | 8 ++++++++ 2 files changed, 9 insertions(+), 6 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/import/netsparker.rb diff --git a/spec/support/shared/examples/msf/db_manager/import.rb b/spec/support/shared/examples/msf/db_manager/import.rb index c17eba2e58..c0838cc08c 100644 --- a/spec/support/shared/examples/msf/db_manager/import.rb +++ b/spec/support/shared/examples/msf/db_manager/import.rb @@ -4,8 +4,6 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :import } it { is_expected.to respond_to :import_file } it { is_expected.to respond_to :import_filetype_detect } - it { is_expected.to respond_to :import_netsparker_xml } - it { is_expected.to respond_to :import_netsparker_xml_file } it { is_expected.to respond_to :import_nexpose_noko_stream } it { is_expected.to respond_to :import_nexpose_raw_noko_stream } it { is_expected.to respond_to :import_nexpose_rawxml } @@ -30,10 +28,6 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :inspect_single_packet } it { is_expected.to respond_to :inspect_single_packet_http } it { is_expected.to respond_to :msf_import_timestamps } - it { is_expected.to respond_to :netsparker_method_map } - it { is_expected.to respond_to :netsparker_params_map } - it { is_expected.to respond_to :netsparker_pname_map } - it { is_expected.to respond_to :netsparker_vulnerability_map } it { is_expected.to respond_to :nexpose_host_from_rawxml } it { is_expected.to respond_to :nexpose_refs_to_struct } it { is_expected.to respond_to :nils_for_nulls } @@ -56,5 +50,6 @@ shared_examples_for 'Msf::DBManager::Import' do it_should_behave_like 'Msf::DBManager::Import::MBSA' it_should_behave_like 'Msf::DBManager::Import::MetasploitFramework' it_should_behave_like 'Msf::DBManager::Import::Nessus' + it_should_behave_like 'Msf::DBManager::Import::Netsparker' it_should_behave_like 'Msf::DBManager::Import::Qualys' end \ No newline at end of file diff --git a/spec/support/shared/examples/msf/db_manager/import/netsparker.rb b/spec/support/shared/examples/msf/db_manager/import/netsparker.rb new file mode 100644 index 0000000000..2de91b7eb8 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/import/netsparker.rb @@ -0,0 +1,8 @@ +shared_examples_for 'Msf::DBManager::Import::Netsparker' do + it { is_expected.to respond_to :import_netsparker_xml } + it { is_expected.to respond_to :import_netsparker_xml_file } + it { is_expected.to respond_to :netsparker_method_map } + it { is_expected.to respond_to :netsparker_params_map } + it { is_expected.to respond_to :netsparker_pname_map } + it { is_expected.to respond_to :netsparker_vulnerability_map } +end \ No newline at end of file From 2b861f91e9860dfe8049fdae5f1a71d536a655a3 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 11:59:03 -0500 Subject: [PATCH 132/159] Extract Msf::DBManager::Import::Nexpose::Raw MSP-11124 --- lib/msf/core/db_manager/import.rb | 229 +----------------- lib/msf/core/db_manager/import/nexpose.rb | 5 + lib/msf/core/db_manager/import/nexpose/raw.rb | 229 ++++++++++++++++++ 3 files changed, 236 insertions(+), 227 deletions(-) create mode 100644 lib/msf/core/db_manager/import/nexpose.rb create mode 100644 lib/msf/core/db_manager/import/nexpose/raw.rb diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index bd3543d675..023e8b599a 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -12,7 +12,6 @@ require 'uri' # require 'packetfu' -require 'rex/parser/nexpose_raw_nokogiri' require 'rex/parser/nexpose_simple_nokogiri' require 'rex/parser/nexpose_xml' require 'rex/parser/nmap_nokogiri' @@ -37,6 +36,7 @@ module Msf::DBManager::Import autoload :MetasploitFramework, 'msf/core/db_manager/import/metasploit_framework' autoload :Nessus, 'msf/core/db_manager/import/nessus' autoload :Netsparker, 'msf/core/db_manager/import/netsparker' + autoload :Nexpose, 'msf/core/db_manager/import/nexpose' autoload :Qualys, 'msf/core/db_manager/import/qualys' include Msf::DBManager::Import::Acunetix @@ -53,6 +53,7 @@ module Msf::DBManager::Import include Msf::DBManager::Import::MetasploitFramework include Msf::DBManager::Import::Nessus include Msf::DBManager::Import::Netsparker + include Msf::DBManager::Import::Nexpose include Msf::DBManager::Import::Qualys # If hex notation is present, turn them into a character. @@ -359,88 +360,6 @@ module Msf::DBManager::Import parser.parse(args[:data]) end - def import_nexpose_raw_noko_stream(args, &block) - if block - doc = Rex::Parser::NexposeRawDocument.new(args,framework.db) {|type, data| yield type,data } - else - doc = Rex::Parser::NexposeRawDocument.new(args,self) - end - parser = ::Nokogiri::XML::SAX::Parser.new(doc) - parser.parse(args[:data]) - end - - def import_nexpose_rawxml(args={}, &block) - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - wspace = args[:wspace] || workspace - if Rex::Parser.nokogiri_loaded - parser = "Nokogiri v#{::Nokogiri::VERSION}" - noko_args = args.dup - noko_args[:blacklist] = bl - noko_args[:wspace] = wspace - if block - yield(:parser, parser) - import_nexpose_raw_noko_stream(noko_args) {|type, data| yield type,data} - else - import_nexpose_raw_noko_stream(noko_args) - end - return true - end - data = args[:data] - - # Use a stream parser instead of a tree parser so we can deal with - # huge results files without running out of memory. - parser = Rex::Parser::NexposeXMLStreamParser.new - - # Since all the Refs have to be in the database before we can use them - # in a Vuln, we store all the hosts until we finish parsing and only - # then put everything in the database. This is memory-intensive for - # large files, but should be much less so than a tree parser. - # - # This method is also considerably faster than parsing through the tree - # looking for references every time we hit a vuln. - hosts = [] - vulns = [] - - # The callback merely populates our in-memory table of hosts and vulns - parser.callback = Proc.new { |type, value| - case type - when :host - # XXX: Blacklist should be checked here instead of saving a - # host we're just going to throw away later - hosts.push(value) - when :vuln - value["id"] = value["id"].downcase if value["id"] - vulns.push(value) - end - } - - REXML::Document.parse_stream(data, parser) - - vuln_refs = nexpose_refs_to_struct(vulns) - hosts.each do |host| - if bl.include? host["addr"] - next - else - yield(:address,host["addr"]) if block - end - nexpose_host_from_rawxml(host, vuln_refs, wspace) - end - end - - # - # Nexpose Raw XML - # - def import_nexpose_rawxml_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - - data = "" - ::File.open(filename, 'rb') do |f| - data = f.read(f.stat.size) - end - import_nexpose_rawxml(args.merge(:data => data)) - end - def import_nexpose_simplexml(args={}, &block) bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] wspace = args[:wspace] || workspace @@ -1277,150 +1196,6 @@ module Msf::DBManager::Import return obj end - # Takes a Host object, an array of vuln structs (generated by nexpose_refs_to_struct()), - # and a workspace, and reports the vulns on that host. - def nexpose_host_from_rawxml(h, vstructs, wspace,task=nil) - hobj = nil - data = {:workspace => wspace} - if h["addr"] - addr = h["addr"] - else - # Can't report it if it doesn't have an IP - return - end - data[:host] = addr - if (h["hardware-address"]) - # Put colons between each octet of the MAC address - data[:mac] = h["hardware-address"].gsub(':', '').scan(/../).join(':') - end - data[:state] = (h["status"] == "alive") ? Msf::HostState::Alive : Msf::HostState::Dead - - # Since we only have one name field per host in the database, just - # take the first one. - if (h["names"] and h["names"].first) - data[:name] = h["names"].first - end - - if (data[:state] != Msf::HostState::Dead) - hobj = report_host(data) - report_import_note(wspace, hobj) - end - - if h["notes"] - note = { - :workspace => wspace, - :host => (hobj || addr), - :type => "host.vuln.nexpose_keys", - :data => {}, - :mode => :unique_data, - :task => task - } - h["notes"].each do |v,k| - note[:data][v] ||= [] - next if note[:data][v].include? k - note[:data][v] << k - end - report_note(note) - end - - if h["os_family"] - note = { - :workspace => wspace, - :host => hobj || addr, - :type => 'host.os.nexpose_fingerprint', - :task => task, - :data => { - :family => h["os_family"], - :certainty => h["os_certainty"] - } - } - note[:data][:vendor] = h["os_vendor"] if h["os_vendor"] - note[:data][:product] = h["os_product"] if h["os_product"] - note[:data][:version] = h["os_version"] if h["os_version"] - note[:data][:arch] = h["arch"] if h["arch"] - - report_note(note) - end - - h["endpoints"].each { |p| - extra = "" - extra << p["product"] + " " if p["product"] - extra << p["version"] + " " if p["version"] - - # Skip port-0 endpoints - next if p["port"].to_i == 0 - - # XXX This should probably be handled in a more standard way - # extra << "(" + p["certainty"] + " certainty) " if p["certainty"] - - data = {} - data[:workspace] = wspace - data[:proto] = p["protocol"].downcase - data[:port] = p["port"].to_i - data[:state] = p["status"] - data[:host] = hobj || addr - data[:info] = extra if not extra.empty? - data[:task] = task - if p["name"] != "" - data[:name] = p["name"] - end - report_service(data) - } - - h["vulns"].each_pair { |k,v| - - next if v["status"] !~ /^vulnerable/ - vstruct = vstructs.select {|vs| vs.id.to_s.downcase == v["id"].to_s.downcase}.first - next unless vstruct - data = {} - data[:workspace] = wspace - data[:host] = hobj || addr - data[:proto] = v["protocol"].downcase if v["protocol"] - data[:port] = v["port"].to_i if v["port"] - data[:name] = "NEXPOSE-" + v["id"] - data[:info] = vstruct.title - data[:refs] = vstruct.refs - data[:task] = task - report_vuln(data) - } - end - - # - # Takes an array of vuln hashes, as returned by the NeXpose rawxml stream - # parser, like: - # [ - # {"id"=>"winreg-notes-protocol-handler", severity="8", "refs"=>[{"source"=>"BID", "value"=>"10600"}, ...]} - # {"id"=>"windows-zotob-c", severity="8", "refs"=>[{"source"=>"BID", "value"=>"14513"}, ...]} - # ] - # and transforms it into a struct, containing :id, :refs, :title, and :severity - # - # Other attributes can be added later, as needed. - def nexpose_refs_to_struct(vulns) - ret = [] - vulns.each do |vuln| - next if ret.map {|v| v.id}.include? vuln["id"] - vstruct = Struct.new(:id, :refs, :title, :severity).new - vstruct.id = vuln["id"] - vstruct.title = vuln["title"] - vstruct.severity = vuln["severity"] - vstruct.refs = [] - vuln["refs"].each do |ref| - if ref['source'] == 'BID' - vstruct.refs.push('BID-' + ref["value"]) - elsif ref['source'] == 'CVE' - # value is CVE-$ID - vstruct.refs.push(ref["value"]) - elsif ref['source'] == 'MS' - vstruct.refs.push('MSB-' + ref["value"]) - elsif ref['source'] == 'URL' - vstruct.refs.push('URL-' + ref["value"]) - end - end - ret.push vstruct - end - return ret - end - # Convert the string "NULL" to actual nil def nils_for_nulls(str) str == "NULL" ? nil : str diff --git a/lib/msf/core/db_manager/import/nexpose.rb b/lib/msf/core/db_manager/import/nexpose.rb new file mode 100644 index 0000000000..ff783ec08c --- /dev/null +++ b/lib/msf/core/db_manager/import/nexpose.rb @@ -0,0 +1,5 @@ +module Msf::DBManager::Import::Nexpose + autoload :Raw, 'msf/core/db_manager/import/nexpose/raw' + + include Msf::DBManager::Import::Nexpose::Raw +end diff --git a/lib/msf/core/db_manager/import/nexpose/raw.rb b/lib/msf/core/db_manager/import/nexpose/raw.rb new file mode 100644 index 0000000000..133fa4d999 --- /dev/null +++ b/lib/msf/core/db_manager/import/nexpose/raw.rb @@ -0,0 +1,229 @@ +require 'rex/parser/nexpose_raw_nokogiri' + +module Msf::DBManager::Import::Nexpose::Raw + def import_nexpose_raw_noko_stream(args, &block) + if block + doc = Rex::Parser::NexposeRawDocument.new(args,framework.db) {|type, data| yield type,data } + else + doc = Rex::Parser::NexposeRawDocument.new(args,self) + end + parser = ::Nokogiri::XML::SAX::Parser.new(doc) + parser.parse(args[:data]) + end + + def import_nexpose_rawxml(args={}, &block) + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + wspace = args[:wspace] || workspace + if Rex::Parser.nokogiri_loaded + parser = "Nokogiri v#{::Nokogiri::VERSION}" + noko_args = args.dup + noko_args[:blacklist] = bl + noko_args[:wspace] = wspace + if block + yield(:parser, parser) + import_nexpose_raw_noko_stream(noko_args) {|type, data| yield type,data} + else + import_nexpose_raw_noko_stream(noko_args) + end + return true + end + data = args[:data] + + # Use a stream parser instead of a tree parser so we can deal with + # huge results files without running out of memory. + parser = Rex::Parser::NexposeXMLStreamParser.new + + # Since all the Refs have to be in the database before we can use them + # in a Vuln, we store all the hosts until we finish parsing and only + # then put everything in the database. This is memory-intensive for + # large files, but should be much less so than a tree parser. + # + # This method is also considerably faster than parsing through the tree + # looking for references every time we hit a vuln. + hosts = [] + vulns = [] + + # The callback merely populates our in-memory table of hosts and vulns + parser.callback = Proc.new { |type, value| + case type + when :host + # XXX: Blacklist should be checked here instead of saving a + # host we're just going to throw away later + hosts.push(value) + when :vuln + value["id"] = value["id"].downcase if value["id"] + vulns.push(value) + end + } + + REXML::Document.parse_stream(data, parser) + + vuln_refs = nexpose_refs_to_struct(vulns) + hosts.each do |host| + if bl.include? host["addr"] + next + else + yield(:address,host["addr"]) if block + end + nexpose_host_from_rawxml(host, vuln_refs, wspace) + end + end + + # + # Nexpose Raw XML + # + def import_nexpose_rawxml_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + + data = "" + ::File.open(filename, 'rb') do |f| + data = f.read(f.stat.size) + end + import_nexpose_rawxml(args.merge(:data => data)) + end + + # Takes a Host object, an array of vuln structs (generated by nexpose_refs_to_struct()), + # and a workspace, and reports the vulns on that host. + def nexpose_host_from_rawxml(h, vstructs, wspace,task=nil) + hobj = nil + data = {:workspace => wspace} + if h["addr"] + addr = h["addr"] + else + # Can't report it if it doesn't have an IP + return + end + data[:host] = addr + if (h["hardware-address"]) + # Put colons between each octet of the MAC address + data[:mac] = h["hardware-address"].gsub(':', '').scan(/../).join(':') + end + data[:state] = (h["status"] == "alive") ? Msf::HostState::Alive : Msf::HostState::Dead + + # Since we only have one name field per host in the database, just + # take the first one. + if (h["names"] and h["names"].first) + data[:name] = h["names"].first + end + + if (data[:state] != Msf::HostState::Dead) + hobj = report_host(data) + report_import_note(wspace, hobj) + end + + if h["notes"] + note = { + :workspace => wspace, + :host => (hobj || addr), + :type => "host.vuln.nexpose_keys", + :data => {}, + :mode => :unique_data, + :task => task + } + h["notes"].each do |v,k| + note[:data][v] ||= [] + next if note[:data][v].include? k + note[:data][v] << k + end + report_note(note) + end + + if h["os_family"] + note = { + :workspace => wspace, + :host => hobj || addr, + :type => 'host.os.nexpose_fingerprint', + :task => task, + :data => { + :family => h["os_family"], + :certainty => h["os_certainty"] + } + } + note[:data][:vendor] = h["os_vendor"] if h["os_vendor"] + note[:data][:product] = h["os_product"] if h["os_product"] + note[:data][:version] = h["os_version"] if h["os_version"] + note[:data][:arch] = h["arch"] if h["arch"] + + report_note(note) + end + + h["endpoints"].each { |p| + extra = "" + extra << p["product"] + " " if p["product"] + extra << p["version"] + " " if p["version"] + + # Skip port-0 endpoints + next if p["port"].to_i == 0 + + # XXX This should probably be handled in a more standard way + # extra << "(" + p["certainty"] + " certainty) " if p["certainty"] + + data = {} + data[:workspace] = wspace + data[:proto] = p["protocol"].downcase + data[:port] = p["port"].to_i + data[:state] = p["status"] + data[:host] = hobj || addr + data[:info] = extra if not extra.empty? + data[:task] = task + if p["name"] != "" + data[:name] = p["name"] + end + report_service(data) + } + + h["vulns"].each_pair { |k,v| + + next if v["status"] !~ /^vulnerable/ + vstruct = vstructs.select {|vs| vs.id.to_s.downcase == v["id"].to_s.downcase}.first + next unless vstruct + data = {} + data[:workspace] = wspace + data[:host] = hobj || addr + data[:proto] = v["protocol"].downcase if v["protocol"] + data[:port] = v["port"].to_i if v["port"] + data[:name] = "NEXPOSE-" + v["id"] + data[:info] = vstruct.title + data[:refs] = vstruct.refs + data[:task] = task + report_vuln(data) + } + end + + # + # Takes an array of vuln hashes, as returned by the NeXpose rawxml stream + # parser, like: + # [ + # {"id"=>"winreg-notes-protocol-handler", severity="8", "refs"=>[{"source"=>"BID", "value"=>"10600"}, ...]} + # {"id"=>"windows-zotob-c", severity="8", "refs"=>[{"source"=>"BID", "value"=>"14513"}, ...]} + # ] + # and transforms it into a struct, containing :id, :refs, :title, and :severity + # + # Other attributes can be added later, as needed. + def nexpose_refs_to_struct(vulns) + ret = [] + vulns.each do |vuln| + next if ret.map {|v| v.id}.include? vuln["id"] + vstruct = Struct.new(:id, :refs, :title, :severity).new + vstruct.id = vuln["id"] + vstruct.title = vuln["title"] + vstruct.severity = vuln["severity"] + vstruct.refs = [] + vuln["refs"].each do |ref| + if ref['source'] == 'BID' + vstruct.refs.push('BID-' + ref["value"]) + elsif ref['source'] == 'CVE' + # value is CVE-$ID + vstruct.refs.push(ref["value"]) + elsif ref['source'] == 'MS' + vstruct.refs.push('MSB-' + ref["value"]) + elsif ref['source'] == 'URL' + vstruct.refs.push('URL-' + ref["value"]) + end + end + ret.push vstruct + end + return ret + end +end \ No newline at end of file From b49dd166fdde837466a59112bde9d4c20f23b28e Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 12:37:52 -0500 Subject: [PATCH 133/159] Extract Msf::DBManager::Import::Nexpose::Raw shared examples MSP-11124 --- spec/support/shared/examples/msf/db_manager/import.rb | 6 +----- .../shared/examples/msf/db_manager/import/nexpose.rb | 3 +++ .../shared/examples/msf/db_manager/import/nexpose/raw.rb | 7 +++++++ 3 files changed, 11 insertions(+), 5 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/import/nexpose.rb create mode 100644 spec/support/shared/examples/msf/db_manager/import/nexpose/raw.rb diff --git a/spec/support/shared/examples/msf/db_manager/import.rb b/spec/support/shared/examples/msf/db_manager/import.rb index c0838cc08c..3bf1e29362 100644 --- a/spec/support/shared/examples/msf/db_manager/import.rb +++ b/spec/support/shared/examples/msf/db_manager/import.rb @@ -5,9 +5,6 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :import_file } it { is_expected.to respond_to :import_filetype_detect } it { is_expected.to respond_to :import_nexpose_noko_stream } - it { is_expected.to respond_to :import_nexpose_raw_noko_stream } - it { is_expected.to respond_to :import_nexpose_rawxml } - it { is_expected.to respond_to :import_nexpose_rawxml_file } it { is_expected.to respond_to :import_nexpose_simplexml } it { is_expected.to respond_to :import_nexpose_simplexml_file } it { is_expected.to respond_to :import_nikto_xml } @@ -28,8 +25,6 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :inspect_single_packet } it { is_expected.to respond_to :inspect_single_packet_http } it { is_expected.to respond_to :msf_import_timestamps } - it { is_expected.to respond_to :nexpose_host_from_rawxml } - it { is_expected.to respond_to :nexpose_refs_to_struct } it { is_expected.to respond_to :nils_for_nulls } it { is_expected.to respond_to :nmap_msf_service_map } it { is_expected.to respond_to :report_import_note } @@ -51,5 +46,6 @@ shared_examples_for 'Msf::DBManager::Import' do it_should_behave_like 'Msf::DBManager::Import::MetasploitFramework' it_should_behave_like 'Msf::DBManager::Import::Nessus' it_should_behave_like 'Msf::DBManager::Import::Netsparker' + it_should_behave_like 'Msf::DBManager::Import::Nexpose' it_should_behave_like 'Msf::DBManager::Import::Qualys' end \ No newline at end of file diff --git a/spec/support/shared/examples/msf/db_manager/import/nexpose.rb b/spec/support/shared/examples/msf/db_manager/import/nexpose.rb new file mode 100644 index 0000000000..dbddcd3494 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/import/nexpose.rb @@ -0,0 +1,3 @@ +shared_examples_for 'Msf::DBManager::Import::Nexpose' do + it_should_behave_like 'Msf::DBManager::Import::Nexpose::Raw' +end \ No newline at end of file diff --git a/spec/support/shared/examples/msf/db_manager/import/nexpose/raw.rb b/spec/support/shared/examples/msf/db_manager/import/nexpose/raw.rb new file mode 100644 index 0000000000..f63f71c855 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/import/nexpose/raw.rb @@ -0,0 +1,7 @@ +shared_examples_for 'Msf::DBManager::Import::Nexpose::Raw' do + it { is_expected.to respond_to :import_nexpose_raw_noko_stream } + it { is_expected.to respond_to :import_nexpose_rawxml } + it { is_expected.to respond_to :import_nexpose_rawxml_file } + it { is_expected.to respond_to :nexpose_host_from_rawxml } + it { is_expected.to respond_to :nexpose_refs_to_struct } +end From e64a14c74802edc066d72dea25767d1c9c17e77f Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 12:40:04 -0500 Subject: [PATCH 134/159] Extract Msf::DBManager::Import::Nexpose::Simple MSP-11124 --- lib/msf/core/db_manager/import.rb | 191 ----------------- lib/msf/core/db_manager/import/nexpose.rb | 2 + .../core/db_manager/import/nexpose/simple.rb | 192 ++++++++++++++++++ 3 files changed, 194 insertions(+), 191 deletions(-) create mode 100644 lib/msf/core/db_manager/import/nexpose/simple.rb diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index 023e8b599a..f66fe6d99d 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -350,169 +350,6 @@ module Msf::DBManager::Import raise DBImportError.new("Could not automatically determine file type") end - def import_nexpose_noko_stream(args, &block) - if block - doc = Rex::Parser::NexposeSimpleDocument.new(args,framework.db) {|type, data| yield type,data } - else - doc = Rex::Parser::NexposeSimpleDocument.new(args,self) - end - parser = ::Nokogiri::XML::SAX::Parser.new(doc) - parser.parse(args[:data]) - end - - def import_nexpose_simplexml(args={}, &block) - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - wspace = args[:wspace] || workspace - if Rex::Parser.nokogiri_loaded - parser = "Nokogiri v#{::Nokogiri::VERSION}" - noko_args = args.dup - noko_args[:blacklist] = bl - noko_args[:wspace] = wspace - if block - yield(:parser, parser) - import_nexpose_noko_stream(noko_args) {|type, data| yield type,data} - else - import_nexpose_noko_stream(noko_args) - end - return true - end - data = args[:data] - - doc = rexmlify(data) - doc.elements.each('/NeXposeSimpleXML/devices/device') do |dev| - addr = dev.attributes['address'].to_s - if bl.include? addr - next - else - yield(:address,addr) if block - end - - fprint = {} - - dev.elements.each('fingerprint/description') do |str| - fprint[:desc] = str.text.to_s.strip - end - dev.elements.each('fingerprint/vendor') do |str| - fprint[:vendor] = str.text.to_s.strip - end - dev.elements.each('fingerprint/family') do |str| - fprint[:family] = str.text.to_s.strip - end - dev.elements.each('fingerprint/product') do |str| - fprint[:product] = str.text.to_s.strip - end - dev.elements.each('fingerprint/version') do |str| - fprint[:version] = str.text.to_s.strip - end - dev.elements.each('fingerprint/architecture') do |str| - fprint[:arch] = str.text.to_s.upcase.strip - end - - conf = { - :workspace => wspace, - :host => addr, - :state => Msf::HostState::Alive, - :task => args[:task] - } - - host = report_host(conf) - report_import_note(wspace, host) - - report_note( - :workspace => wspace, - :host => host, - :type => 'host.os.nexpose_fingerprint', - :data => fprint, - :task => args[:task] - ) - - # Load vulnerabilities not associated with a service - dev.elements.each('vulnerabilities/vulnerability') do |vuln| - vid = vuln.attributes['id'].to_s.downcase - refs = process_nexpose_data_sxml_refs(vuln) - next if not refs - report_vuln( - :workspace => wspace, - :host => host, - :name => 'NEXPOSE-' + vid, - :info => vid, - :refs => refs, - :task => args[:task] - ) - end - - # Load the services - dev.elements.each('services/service') do |svc| - sname = svc.attributes['name'].to_s - sprot = svc.attributes['protocol'].to_s.downcase - sport = svc.attributes['port'].to_s.to_i - next if sport == 0 - - name = sname.split('(')[0].strip - info = '' - - svc.elements.each('fingerprint/description') do |str| - info = str.text.to_s.strip - end - - if(sname.downcase != '') - report_service( - :workspace => wspace, - :host => host, - :proto => sprot, - :port => sport, - :name => name, - :info => info, - :task => args[:task] - ) - else - report_service( - :workspace => wspace, - :host => host, - :proto => sprot, - :port => sport, - :info => info, - :task => args[:task] - ) - end - - # Load vulnerabilities associated with this service - svc.elements.each('vulnerabilities/vulnerability') do |vuln| - vid = vuln.attributes['id'].to_s.downcase - refs = process_nexpose_data_sxml_refs(vuln) - next if not refs - report_vuln( - :workspace => wspace, - :host => host, - :port => sport, - :proto => sprot, - :name => 'NEXPOSE-' + vid, - :info => vid, - :refs => refs, - :task => args[:task] - ) - end - end - end - end - - # - # Nexpose Simple XML - # - # XXX At some point we'll want to make this a stream parser for dealing - # with large results files - # - def import_nexpose_simplexml_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - - data = "" - ::File.open(filename, 'rb') do |f| - data = f.read(f.stat.size) - end - import_nexpose_simplexml(args.merge(:data => data)) - end - # # Imports Nikto scan data from -Format xml as notes. # @@ -1315,32 +1152,4 @@ module Msf::DBManager::Import end return true end - - protected - - def process_nexpose_data_sxml_refs(vuln) - refs = [] - vid = vuln.attributes['id'].to_s.downcase - vry = vuln.attributes['resultCode'].to_s.upcase - - # Only process vuln-exploitable and vuln-version statuses - return if vry !~ /^V[VE]$/ - - refs = [] - vuln.elements.each('id') do |ref| - rtyp = ref.attributes['type'].to_s.upcase - rval = ref.text.to_s.strip - case rtyp - when 'CVE' - refs << rval.gsub('CAN', 'CVE') - when 'MS' # obsolete? - refs << "MSB-MS-#{rval}" - else - refs << "#{rtyp}-#{rval}" - end - end - - refs << "NEXPOSE-#{vid}" - refs - end end \ No newline at end of file diff --git a/lib/msf/core/db_manager/import/nexpose.rb b/lib/msf/core/db_manager/import/nexpose.rb index ff783ec08c..2ea22d9157 100644 --- a/lib/msf/core/db_manager/import/nexpose.rb +++ b/lib/msf/core/db_manager/import/nexpose.rb @@ -1,5 +1,7 @@ module Msf::DBManager::Import::Nexpose autoload :Raw, 'msf/core/db_manager/import/nexpose/raw' + autoload :Simple, 'msf/core/db_manager/import/nexpose/simple' include Msf::DBManager::Import::Nexpose::Raw + include Msf::DBManager::Import::Nexpose::Simple end diff --git a/lib/msf/core/db_manager/import/nexpose/simple.rb b/lib/msf/core/db_manager/import/nexpose/simple.rb new file mode 100644 index 0000000000..964bdfb02f --- /dev/null +++ b/lib/msf/core/db_manager/import/nexpose/simple.rb @@ -0,0 +1,192 @@ +module Msf::DBManager::Import::Nexpose::Simple + def import_nexpose_noko_stream(args, &block) + if block + doc = Rex::Parser::NexposeSimpleDocument.new(args,framework.db) {|type, data| yield type,data } + else + doc = Rex::Parser::NexposeSimpleDocument.new(args,self) + end + parser = ::Nokogiri::XML::SAX::Parser.new(doc) + parser.parse(args[:data]) + end + + def import_nexpose_simplexml(args={}, &block) + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + wspace = args[:wspace] || workspace + if Rex::Parser.nokogiri_loaded + parser = "Nokogiri v#{::Nokogiri::VERSION}" + noko_args = args.dup + noko_args[:blacklist] = bl + noko_args[:wspace] = wspace + if block + yield(:parser, parser) + import_nexpose_noko_stream(noko_args) {|type, data| yield type,data} + else + import_nexpose_noko_stream(noko_args) + end + return true + end + data = args[:data] + + doc = rexmlify(data) + doc.elements.each('/NeXposeSimpleXML/devices/device') do |dev| + addr = dev.attributes['address'].to_s + if bl.include? addr + next + else + yield(:address,addr) if block + end + + fprint = {} + + dev.elements.each('fingerprint/description') do |str| + fprint[:desc] = str.text.to_s.strip + end + dev.elements.each('fingerprint/vendor') do |str| + fprint[:vendor] = str.text.to_s.strip + end + dev.elements.each('fingerprint/family') do |str| + fprint[:family] = str.text.to_s.strip + end + dev.elements.each('fingerprint/product') do |str| + fprint[:product] = str.text.to_s.strip + end + dev.elements.each('fingerprint/version') do |str| + fprint[:version] = str.text.to_s.strip + end + dev.elements.each('fingerprint/architecture') do |str| + fprint[:arch] = str.text.to_s.upcase.strip + end + + conf = { + :workspace => wspace, + :host => addr, + :state => Msf::HostState::Alive, + :task => args[:task] + } + + host = report_host(conf) + report_import_note(wspace, host) + + report_note( + :workspace => wspace, + :host => host, + :type => 'host.os.nexpose_fingerprint', + :data => fprint, + :task => args[:task] + ) + + # Load vulnerabilities not associated with a service + dev.elements.each('vulnerabilities/vulnerability') do |vuln| + vid = vuln.attributes['id'].to_s.downcase + refs = process_nexpose_data_sxml_refs(vuln) + next if not refs + report_vuln( + :workspace => wspace, + :host => host, + :name => 'NEXPOSE-' + vid, + :info => vid, + :refs => refs, + :task => args[:task] + ) + end + + # Load the services + dev.elements.each('services/service') do |svc| + sname = svc.attributes['name'].to_s + sprot = svc.attributes['protocol'].to_s.downcase + sport = svc.attributes['port'].to_s.to_i + next if sport == 0 + + name = sname.split('(')[0].strip + info = '' + + svc.elements.each('fingerprint/description') do |str| + info = str.text.to_s.strip + end + + if(sname.downcase != '') + report_service( + :workspace => wspace, + :host => host, + :proto => sprot, + :port => sport, + :name => name, + :info => info, + :task => args[:task] + ) + else + report_service( + :workspace => wspace, + :host => host, + :proto => sprot, + :port => sport, + :info => info, + :task => args[:task] + ) + end + + # Load vulnerabilities associated with this service + svc.elements.each('vulnerabilities/vulnerability') do |vuln| + vid = vuln.attributes['id'].to_s.downcase + refs = process_nexpose_data_sxml_refs(vuln) + next if not refs + report_vuln( + :workspace => wspace, + :host => host, + :port => sport, + :proto => sprot, + :name => 'NEXPOSE-' + vid, + :info => vid, + :refs => refs, + :task => args[:task] + ) + end + end + end + end + + # + # Nexpose Simple XML + # + # XXX At some point we'll want to make this a stream parser for dealing + # with large results files + # + def import_nexpose_simplexml_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + + data = "" + ::File.open(filename, 'rb') do |f| + data = f.read(f.stat.size) + end + import_nexpose_simplexml(args.merge(:data => data)) + end + + protected + + def process_nexpose_data_sxml_refs(vuln) + refs = [] + vid = vuln.attributes['id'].to_s.downcase + vry = vuln.attributes['resultCode'].to_s.upcase + + # Only process vuln-exploitable and vuln-version statuses + return if vry !~ /^V[VE]$/ + + refs = [] + vuln.elements.each('id') do |ref| + rtyp = ref.attributes['type'].to_s.upcase + rval = ref.text.to_s.strip + case rtyp + when 'CVE' + refs << rval.gsub('CAN', 'CVE') + when 'MS' # obsolete? + refs << "MSB-MS-#{rval}" + else + refs << "#{rtyp}-#{rval}" + end + end + + refs << "NEXPOSE-#{vid}" + refs + end +end From 3049301c96369ab04bab686de0b8341695407e5d Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 12:44:24 -0500 Subject: [PATCH 135/159] Extract Msf::DBManager::Import::Nexpose::Simple shared examples MSP-11124 --- spec/support/shared/examples/msf/db_manager/import.rb | 3 --- .../support/shared/examples/msf/db_manager/import/nexpose.rb | 1 + .../shared/examples/msf/db_manager/import/nexpose/simple.rb | 5 +++++ 3 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/import/nexpose/simple.rb diff --git a/spec/support/shared/examples/msf/db_manager/import.rb b/spec/support/shared/examples/msf/db_manager/import.rb index 3bf1e29362..10ee0a2de5 100644 --- a/spec/support/shared/examples/msf/db_manager/import.rb +++ b/spec/support/shared/examples/msf/db_manager/import.rb @@ -4,9 +4,6 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :import } it { is_expected.to respond_to :import_file } it { is_expected.to respond_to :import_filetype_detect } - it { is_expected.to respond_to :import_nexpose_noko_stream } - it { is_expected.to respond_to :import_nexpose_simplexml } - it { is_expected.to respond_to :import_nexpose_simplexml_file } it { is_expected.to respond_to :import_nikto_xml } it { is_expected.to respond_to :import_nmap_noko_stream } it { is_expected.to respond_to :import_nmap_xml } diff --git a/spec/support/shared/examples/msf/db_manager/import/nexpose.rb b/spec/support/shared/examples/msf/db_manager/import/nexpose.rb index dbddcd3494..8a0bb57bf1 100644 --- a/spec/support/shared/examples/msf/db_manager/import/nexpose.rb +++ b/spec/support/shared/examples/msf/db_manager/import/nexpose.rb @@ -1,3 +1,4 @@ shared_examples_for 'Msf::DBManager::Import::Nexpose' do it_should_behave_like 'Msf::DBManager::Import::Nexpose::Raw' + it_should_behave_like 'Msf::DBManager::Import::Nexpose::Simple' end \ No newline at end of file diff --git a/spec/support/shared/examples/msf/db_manager/import/nexpose/simple.rb b/spec/support/shared/examples/msf/db_manager/import/nexpose/simple.rb new file mode 100644 index 0000000000..999be87806 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/import/nexpose/simple.rb @@ -0,0 +1,5 @@ +shared_examples_for 'Msf::DBManager::Import::Nexpose::Simple' do + it { is_expected.to respond_to :import_nexpose_noko_stream } + it { is_expected.to respond_to :import_nexpose_simplexml } + it { is_expected.to respond_to :import_nexpose_simplexml_file } +end From 16f143c2ed3cde38aa5a44c0f805af479423ebca Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 12:51:16 -0500 Subject: [PATCH 136/159] Extract Msf::DBManager::Import::Nikto MSP-11124 --- lib/msf/core/db_manager/import.rb | 59 +------------------------ lib/msf/core/db_manager/import/nikto.rb | 58 ++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 57 deletions(-) create mode 100644 lib/msf/core/db_manager/import/nikto.rb diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index f66fe6d99d..56cdaee194 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -37,6 +37,7 @@ module Msf::DBManager::Import autoload :Nessus, 'msf/core/db_manager/import/nessus' autoload :Netsparker, 'msf/core/db_manager/import/netsparker' autoload :Nexpose, 'msf/core/db_manager/import/nexpose' + autoload :Nikto, 'msf/core/db_manager/import/nikto' autoload :Qualys, 'msf/core/db_manager/import/qualys' include Msf::DBManager::Import::Acunetix @@ -54,6 +55,7 @@ module Msf::DBManager::Import include Msf::DBManager::Import::Nessus include Msf::DBManager::Import::Netsparker include Msf::DBManager::Import::Nexpose + include Msf::DBManager::Import::Nikto include Msf::DBManager::Import::Qualys # If hex notation is present, turn them into a character. @@ -350,63 +352,6 @@ module Msf::DBManager::Import raise DBImportError.new("Could not automatically determine file type") end - # - # Imports Nikto scan data from -Format xml as notes. - # - def import_nikto_xml(args={}, &block) - data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - doc = rexmlify(data) - doc.elements.each do |f| - f.elements.each('scandetails') do |host| - # Get host information - addr = host.attributes['targetip'] - next if not addr - if bl.include? addr - next - else - yield(:address,addr) if block - end - # Get service information - port = host.attributes['targetport'] - next if port.to_i == 0 - uri = URI.parse(host.attributes['sitename']) rescue nil - next unless uri and uri.scheme - # Collect and report scan descriptions. - host.elements.each do |item| - if item.elements['description'] - desc_text = item.elements['description'].text - next if desc_text.nil? or desc_text.empty? - desc_data = { - :workspace => wspace, - :host => addr, - :type => "service.nikto.scan.description", - :data => desc_text, - :proto => "tcp", - :port => port.to_i, - :sname => uri.scheme, - :update => :unique_data, - :task => args[:task] - } - # Always report it as a note. - report_note(desc_data) - # Sometimes report it as a vuln, too. - # XXX: There's a Vuln.info field but nothing reads from it? See Bug #5837 - if item.attributes['osvdbid'].to_i != 0 - desc_data[:refs] = ["OSVDB-#{item.attributes['osvdbid']}"] - desc_data[:name] = "NIKTO-#{item.attributes['id']}" - desc_data.delete(:data) - desc_data.delete(:type) - desc_data.delete(:update) - report_vuln(desc_data) - end - end - end - end - end - end - def import_nmap_noko_stream(args, &block) if block doc = Rex::Parser::NmapDocument.new(args,framework.db) {|type, data| yield type,data } diff --git a/lib/msf/core/db_manager/import/nikto.rb b/lib/msf/core/db_manager/import/nikto.rb new file mode 100644 index 0000000000..da77140445 --- /dev/null +++ b/lib/msf/core/db_manager/import/nikto.rb @@ -0,0 +1,58 @@ +module Msf::DBManager::Import::Nikto + # + # Imports Nikto scan data from -Format xml as notes. + # + def import_nikto_xml(args={}, &block) + data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + doc = rexmlify(data) + doc.elements.each do |f| + f.elements.each('scandetails') do |host| + # Get host information + addr = host.attributes['targetip'] + next if not addr + if bl.include? addr + next + else + yield(:address,addr) if block + end + # Get service information + port = host.attributes['targetport'] + next if port.to_i == 0 + uri = URI.parse(host.attributes['sitename']) rescue nil + next unless uri and uri.scheme + # Collect and report scan descriptions. + host.elements.each do |item| + if item.elements['description'] + desc_text = item.elements['description'].text + next if desc_text.nil? or desc_text.empty? + desc_data = { + :workspace => wspace, + :host => addr, + :type => "service.nikto.scan.description", + :data => desc_text, + :proto => "tcp", + :port => port.to_i, + :sname => uri.scheme, + :update => :unique_data, + :task => args[:task] + } + # Always report it as a note. + report_note(desc_data) + # Sometimes report it as a vuln, too. + # XXX: There's a Vuln.info field but nothing reads from it? See Bug #5837 + if item.attributes['osvdbid'].to_i != 0 + desc_data[:refs] = ["OSVDB-#{item.attributes['osvdbid']}"] + desc_data[:name] = "NIKTO-#{item.attributes['id']}" + desc_data.delete(:data) + desc_data.delete(:type) + desc_data.delete(:update) + report_vuln(desc_data) + end + end + end + end + end + end +end \ No newline at end of file From cf3a3a0d65c5acb83f132e40dda3e25408d617e2 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 12:54:30 -0500 Subject: [PATCH 137/159] Move nexpose requires to appropriate module MSP-11124 --- lib/msf/core/db_manager/import.rb | 2 -- lib/msf/core/db_manager/import/nexpose/raw.rb | 1 + lib/msf/core/db_manager/import/nexpose/simple.rb | 2 ++ 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index 56cdaee194..2bd9f637d7 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -12,8 +12,6 @@ require 'uri' # require 'packetfu' -require 'rex/parser/nexpose_simple_nokogiri' -require 'rex/parser/nexpose_xml' require 'rex/parser/nmap_nokogiri' require 'rex/parser/nmap_xml' require 'rex/parser/openvas_nokogiri' diff --git a/lib/msf/core/db_manager/import/nexpose/raw.rb b/lib/msf/core/db_manager/import/nexpose/raw.rb index 133fa4d999..302e2a6f8d 100644 --- a/lib/msf/core/db_manager/import/nexpose/raw.rb +++ b/lib/msf/core/db_manager/import/nexpose/raw.rb @@ -1,4 +1,5 @@ require 'rex/parser/nexpose_raw_nokogiri' +require 'rex/parser/nexpose_xml' module Msf::DBManager::Import::Nexpose::Raw def import_nexpose_raw_noko_stream(args, &block) diff --git a/lib/msf/core/db_manager/import/nexpose/simple.rb b/lib/msf/core/db_manager/import/nexpose/simple.rb index 964bdfb02f..828dff10f5 100644 --- a/lib/msf/core/db_manager/import/nexpose/simple.rb +++ b/lib/msf/core/db_manager/import/nexpose/simple.rb @@ -1,3 +1,5 @@ +require 'rex/parser/nexpose_simple_nokogiri' + module Msf::DBManager::Import::Nexpose::Simple def import_nexpose_noko_stream(args, &block) if block From 926c5adb49e81ab94d75a48bd73d7991b92bd023 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 12:57:24 -0500 Subject: [PATCH 138/159] Extract Msf::DBManager::Import::Nikto shared examples MSP-11124 --- spec/support/shared/examples/msf/db_manager/import.rb | 2 +- spec/support/shared/examples/msf/db_manager/import/nikto.rb | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 spec/support/shared/examples/msf/db_manager/import/nikto.rb diff --git a/spec/support/shared/examples/msf/db_manager/import.rb b/spec/support/shared/examples/msf/db_manager/import.rb index 10ee0a2de5..18cfd9e3df 100644 --- a/spec/support/shared/examples/msf/db_manager/import.rb +++ b/spec/support/shared/examples/msf/db_manager/import.rb @@ -4,7 +4,6 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :import } it { is_expected.to respond_to :import_file } it { is_expected.to respond_to :import_filetype_detect } - it { is_expected.to respond_to :import_nikto_xml } it { is_expected.to respond_to :import_nmap_noko_stream } it { is_expected.to respond_to :import_nmap_xml } it { is_expected.to respond_to :import_nmap_xml_file } @@ -44,5 +43,6 @@ shared_examples_for 'Msf::DBManager::Import' do it_should_behave_like 'Msf::DBManager::Import::Nessus' it_should_behave_like 'Msf::DBManager::Import::Netsparker' it_should_behave_like 'Msf::DBManager::Import::Nexpose' + it_should_behave_like 'Msf::DBManager::Import::Nikto' it_should_behave_like 'Msf::DBManager::Import::Qualys' end \ No newline at end of file diff --git a/spec/support/shared/examples/msf/db_manager/import/nikto.rb b/spec/support/shared/examples/msf/db_manager/import/nikto.rb new file mode 100644 index 0000000000..50b10ae646 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/import/nikto.rb @@ -0,0 +1,3 @@ +shared_examples_for 'Msf::DBManager::Import::Nikto' do + it { is_expected.to respond_to :import_nikto_xml } +end From 5d6044786a92a600c5c72ca63b97d75960ee22d8 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 13:06:28 -0500 Subject: [PATCH 139/159] Extract Msf::DBManager::Import::Nmap MSP-11124 --- lib/msf/core/db_manager/import.rb | 262 +------------------------ lib/msf/core/db_manager/import/nmap.rb | 262 +++++++++++++++++++++++++ 2 files changed, 264 insertions(+), 260 deletions(-) create mode 100644 lib/msf/core/db_manager/import/nmap.rb diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index 2bd9f637d7..fb143c94f1 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -12,8 +12,6 @@ require 'uri' # require 'packetfu' -require 'rex/parser/nmap_nokogiri' -require 'rex/parser/nmap_xml' require 'rex/parser/openvas_nokogiri' require 'rex/parser/outpost24_nokogiri' require 'rex/parser/retina_xml' @@ -36,6 +34,7 @@ module Msf::DBManager::Import autoload :Netsparker, 'msf/core/db_manager/import/netsparker' autoload :Nexpose, 'msf/core/db_manager/import/nexpose' autoload :Nikto, 'msf/core/db_manager/import/nikto' + autoload :Nmap, 'msf/core/db_manager/import/nmap' autoload :Qualys, 'msf/core/db_manager/import/qualys' include Msf::DBManager::Import::Acunetix @@ -54,6 +53,7 @@ module Msf::DBManager::Import include Msf::DBManager::Import::Netsparker include Msf::DBManager::Import::Nexpose include Msf::DBManager::Import::Nikto + include Msf::DBManager::Import::Nmap include Msf::DBManager::Import::Qualys # If hex notation is present, turn them into a character. @@ -350,260 +350,6 @@ module Msf::DBManager::Import raise DBImportError.new("Could not automatically determine file type") end - def import_nmap_noko_stream(args, &block) - if block - doc = Rex::Parser::NmapDocument.new(args,framework.db) {|type, data| yield type,data } - else - doc = Rex::Parser::NmapDocument.new(args,self) - end - parser = ::Nokogiri::XML::SAX::Parser.new(doc) - parser.parse(args[:data]) - end - - # If you have Nokogiri installed, you'll be shunted over to - # that. Otherwise, you'll hit the old NmapXMLStreamParser. - def import_nmap_xml(args={}, &block) - return nil if args[:data].nil? or args[:data].empty? - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - - if Rex::Parser.nokogiri_loaded - noko_args = args.dup - noko_args[:blacklist] = bl - noko_args[:wspace] = wspace - if block - yield(:parser, "Nokogiri v#{::Nokogiri::VERSION}") - import_nmap_noko_stream(noko_args) {|type, data| yield type,data } - else - import_nmap_noko_stream(noko_args) - end - return true - end - - # XXX: Legacy nmap xml parser starts here. - - fix_services = args[:fix_services] - data = args[:data] - - # Use a stream parser instead of a tree parser so we can deal with - # huge results files without running out of memory. - parser = Rex::Parser::NmapXMLStreamParser.new - yield(:parser, parser.class.name) if block - - # Whenever the parser pulls a host out of the nmap results, store - # it, along with any associated services, in the database. - parser.on_found_host = Proc.new { |h| - hobj = nil - data = {:workspace => wspace} - if (h["addrs"].has_key?("ipv4")) - addr = h["addrs"]["ipv4"] - elsif (h["addrs"].has_key?("ipv6")) - addr = h["addrs"]["ipv6"] - else - # Can't report it if it doesn't have an IP - raise RuntimeError, "At least one IPv4 or IPv6 address is required" - end - next if bl.include? addr - data[:host] = addr - if (h["addrs"].has_key?("mac")) - data[:mac] = h["addrs"]["mac"] - end - data[:state] = (h["status"] == "up") ? Msf::HostState::Alive : Msf::HostState::Dead - data[:task] = args[:task] - - if ( h["reverse_dns"] ) - data[:name] = h["reverse_dns"] - end - - # Only report alive hosts with ports to speak of. - if(data[:state] != Msf::HostState::Dead) - if h["ports"].size > 0 - if fix_services - port_states = h["ports"].map {|p| p["state"]}.reject {|p| p == "filtered"} - next if port_states.compact.empty? - end - yield(:address,data[:host]) if block - hobj = report_host(data) - report_import_note(wspace,hobj) - end - end - - if( h["os_vendor"] ) - note = { - :workspace => wspace, - :host => hobj || addr, - :type => 'host.os.nmap_fingerprint', - :task => args[:task], - :data => { - :os_vendor => h["os_vendor"], - :os_family => h["os_family"], - :os_version => h["os_version"], - :os_accuracy => h["os_accuracy"] - } - } - - if(h["os_match"]) - note[:data][:os_match] = h['os_match'] - end - - report_note(note) - end - - if (h["last_boot"]) - report_note( - :workspace => wspace, - :host => hobj || addr, - :type => 'host.last_boot', - :task => args[:task], - :data => { - :time => h["last_boot"] - } - ) - end - - if (h["trace"]) - hops = [] - h["trace"]["hops"].each do |hop| - hops << { - "ttl" => hop["ttl"].to_i, - "address" => hop["ipaddr"].to_s, - "rtt" => hop["rtt"].to_f, - "name" => hop["host"].to_s - } - end - report_note( - :workspace => wspace, - :host => hobj || addr, - :type => 'host.nmap.traceroute', - :task => args[:task], - :data => { - 'port' => h["trace"]["port"].to_i, - 'proto' => h["trace"]["proto"].to_s, - 'hops' => hops - } - ) - end - - - # Put all the ports, regardless of state, into the db. - h["ports"].each { |p| - # Localhost port results are pretty unreliable -- if it's - # unknown, it's no good (possibly Windows-only) - if ( - p["state"] == "unknown" && - h["status_reason"] == "localhost-response" - ) - next - end - extra = "" - extra << p["product"] + " " if p["product"] - extra << p["version"] + " " if p["version"] - extra << p["extrainfo"] + " " if p["extrainfo"] - - data = {} - data[:workspace] = wspace - if fix_services - data[:proto] = nmap_msf_service_map(p["protocol"]) - else - data[:proto] = p["protocol"].downcase - end - data[:port] = p["portid"].to_i - data[:state] = p["state"] - data[:host] = hobj || addr - data[:info] = extra if not extra.empty? - data[:task] = args[:task] - if p["name"] != "unknown" - data[:name] = p["name"] - end - report_service(data) - } - #Parse the scripts output - if h["scripts"] - h["scripts"].each do |key,val| - if key == "smb-check-vulns" - if val =~ /MS08-067: VULNERABLE/ - vuln_info = { - :workspace => wspace, - :task => args[:task], - :host => hobj || addr, - :port => 445, - :proto => 'tcp', - :name => 'MS08-067', - :info => 'Microsoft Windows Server Service Crafted RPC Request Handling Unspecified Remote Code Execution', - :refs =>['CVE-2008-4250', - 'BID-31874', - 'OSVDB-49243', - 'CWE-94', - 'MSFT-MS08-067', - 'MSF-Microsoft Server Service Relative Path Stack Corruption', - 'NSS-34476'] - } - report_vuln(vuln_info) - end - if val =~ /MS06-025: VULNERABLE/ - vuln_info = { - :workspace => wspace, - :task => args[:task], - :host => hobj || addr, - :port => 445, - :proto => 'tcp', - :name => 'MS06-025', - :info => 'Vulnerability in Routing and Remote Access Could Allow Remote Code Execution', - :refs =>['CVE-2006-2370', - 'CVE-2006-2371', - 'BID-18325', - 'BID-18358', - 'BID-18424', - 'OSVDB-26436', - 'OSVDB-26437', - 'MSFT-MS06-025', - 'MSF-Microsoft RRAS Service RASMAN Registry Overflow', - 'NSS-21689'] - } - report_vuln(vuln_info) - end - # This one has NOT been Tested , remove this comment if confirmed working - if val =~ /MS07-029: VULNERABLE/ - vuln_info = { - :workspace => wspace, - :task => args[:task], - :host => hobj || addr, - :port => 445, - :proto => 'tcp', - :name => 'MS07-029', - :info => 'Vulnerability in Windows DNS RPC Interface Could Allow Remote Code Execution', - # Add more refs based on nessus/nexpose .. results - :refs =>['CVE-2007-1748', - 'OSVDB-34100', - 'MSF-Microsoft DNS RPC Service extractQuotedChar()', - 'NSS-25168'] - } - report_vuln(vuln_info) - end - end - end - end - } - - # XXX: Legacy nmap xml parser ends here. - - REXML::Document.parse_stream(data, parser) - end - - # - # Import Nmap's -oX xml output - # - def import_nmap_xml_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - - data = "" - ::File.open(filename, 'rb') do |f| - data = f.read(f.stat.size) - end - import_nmap_xml(args.merge(:data => data)) - end - def import_openvas_new_xml(args={}, &block) if block doc = Rex::Parser::OpenVASDocument.new(args,framework.db) {|type, data| yield type,data } @@ -981,10 +727,6 @@ module Msf::DBManager::Import str == "NULL" ? nil : str end - def nmap_msf_service_map(proto) - service_name_map(proto) - end - def report_import_note(wspace,addr) if @import_filedata.kind_of?(Hash) && @import_filedata[:filename] && @import_filedata[:filename] !~ /msfe-nmap[0-9]{8}/ report_note( diff --git a/lib/msf/core/db_manager/import/nmap.rb b/lib/msf/core/db_manager/import/nmap.rb new file mode 100644 index 0000000000..2be567c395 --- /dev/null +++ b/lib/msf/core/db_manager/import/nmap.rb @@ -0,0 +1,262 @@ +require 'rex/parser/nmap_nokogiri' +require 'rex/parser/nmap_xml' + +module Msf::DBManager::Import::Nmap + def import_nmap_noko_stream(args, &block) + if block + doc = Rex::Parser::NmapDocument.new(args,framework.db) {|type, data| yield type,data } + else + doc = Rex::Parser::NmapDocument.new(args,self) + end + parser = ::Nokogiri::XML::SAX::Parser.new(doc) + parser.parse(args[:data]) + end + + # If you have Nokogiri installed, you'll be shunted over to + # that. Otherwise, you'll hit the old NmapXMLStreamParser. + def import_nmap_xml(args={}, &block) + return nil if args[:data].nil? or args[:data].empty? + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + + if Rex::Parser.nokogiri_loaded + noko_args = args.dup + noko_args[:blacklist] = bl + noko_args[:wspace] = wspace + if block + yield(:parser, "Nokogiri v#{::Nokogiri::VERSION}") + import_nmap_noko_stream(noko_args) {|type, data| yield type,data } + else + import_nmap_noko_stream(noko_args) + end + return true + end + + # XXX: Legacy nmap xml parser starts here. + + fix_services = args[:fix_services] + data = args[:data] + + # Use a stream parser instead of a tree parser so we can deal with + # huge results files without running out of memory. + parser = Rex::Parser::NmapXMLStreamParser.new + yield(:parser, parser.class.name) if block + + # Whenever the parser pulls a host out of the nmap results, store + # it, along with any associated services, in the database. + parser.on_found_host = Proc.new { |h| + hobj = nil + data = {:workspace => wspace} + if (h["addrs"].has_key?("ipv4")) + addr = h["addrs"]["ipv4"] + elsif (h["addrs"].has_key?("ipv6")) + addr = h["addrs"]["ipv6"] + else + # Can't report it if it doesn't have an IP + raise RuntimeError, "At least one IPv4 or IPv6 address is required" + end + next if bl.include? addr + data[:host] = addr + if (h["addrs"].has_key?("mac")) + data[:mac] = h["addrs"]["mac"] + end + data[:state] = (h["status"] == "up") ? Msf::HostState::Alive : Msf::HostState::Dead + data[:task] = args[:task] + + if ( h["reverse_dns"] ) + data[:name] = h["reverse_dns"] + end + + # Only report alive hosts with ports to speak of. + if(data[:state] != Msf::HostState::Dead) + if h["ports"].size > 0 + if fix_services + port_states = h["ports"].map {|p| p["state"]}.reject {|p| p == "filtered"} + next if port_states.compact.empty? + end + yield(:address,data[:host]) if block + hobj = report_host(data) + report_import_note(wspace,hobj) + end + end + + if( h["os_vendor"] ) + note = { + :workspace => wspace, + :host => hobj || addr, + :type => 'host.os.nmap_fingerprint', + :task => args[:task], + :data => { + :os_vendor => h["os_vendor"], + :os_family => h["os_family"], + :os_version => h["os_version"], + :os_accuracy => h["os_accuracy"] + } + } + + if(h["os_match"]) + note[:data][:os_match] = h['os_match'] + end + + report_note(note) + end + + if (h["last_boot"]) + report_note( + :workspace => wspace, + :host => hobj || addr, + :type => 'host.last_boot', + :task => args[:task], + :data => { + :time => h["last_boot"] + } + ) + end + + if (h["trace"]) + hops = [] + h["trace"]["hops"].each do |hop| + hops << { + "ttl" => hop["ttl"].to_i, + "address" => hop["ipaddr"].to_s, + "rtt" => hop["rtt"].to_f, + "name" => hop["host"].to_s + } + end + report_note( + :workspace => wspace, + :host => hobj || addr, + :type => 'host.nmap.traceroute', + :task => args[:task], + :data => { + 'port' => h["trace"]["port"].to_i, + 'proto' => h["trace"]["proto"].to_s, + 'hops' => hops + } + ) + end + + + # Put all the ports, regardless of state, into the db. + h["ports"].each { |p| + # Localhost port results are pretty unreliable -- if it's + # unknown, it's no good (possibly Windows-only) + if ( + p["state"] == "unknown" && + h["status_reason"] == "localhost-response" + ) + next + end + extra = "" + extra << p["product"] + " " if p["product"] + extra << p["version"] + " " if p["version"] + extra << p["extrainfo"] + " " if p["extrainfo"] + + data = {} + data[:workspace] = wspace + if fix_services + data[:proto] = nmap_msf_service_map(p["protocol"]) + else + data[:proto] = p["protocol"].downcase + end + data[:port] = p["portid"].to_i + data[:state] = p["state"] + data[:host] = hobj || addr + data[:info] = extra if not extra.empty? + data[:task] = args[:task] + if p["name"] != "unknown" + data[:name] = p["name"] + end + report_service(data) + } + #Parse the scripts output + if h["scripts"] + h["scripts"].each do |key,val| + if key == "smb-check-vulns" + if val =~ /MS08-067: VULNERABLE/ + vuln_info = { + :workspace => wspace, + :task => args[:task], + :host => hobj || addr, + :port => 445, + :proto => 'tcp', + :name => 'MS08-067', + :info => 'Microsoft Windows Server Service Crafted RPC Request Handling Unspecified Remote Code Execution', + :refs =>['CVE-2008-4250', + 'BID-31874', + 'OSVDB-49243', + 'CWE-94', + 'MSFT-MS08-067', + 'MSF-Microsoft Server Service Relative Path Stack Corruption', + 'NSS-34476'] + } + report_vuln(vuln_info) + end + if val =~ /MS06-025: VULNERABLE/ + vuln_info = { + :workspace => wspace, + :task => args[:task], + :host => hobj || addr, + :port => 445, + :proto => 'tcp', + :name => 'MS06-025', + :info => 'Vulnerability in Routing and Remote Access Could Allow Remote Code Execution', + :refs =>['CVE-2006-2370', + 'CVE-2006-2371', + 'BID-18325', + 'BID-18358', + 'BID-18424', + 'OSVDB-26436', + 'OSVDB-26437', + 'MSFT-MS06-025', + 'MSF-Microsoft RRAS Service RASMAN Registry Overflow', + 'NSS-21689'] + } + report_vuln(vuln_info) + end + # This one has NOT been Tested , remove this comment if confirmed working + if val =~ /MS07-029: VULNERABLE/ + vuln_info = { + :workspace => wspace, + :task => args[:task], + :host => hobj || addr, + :port => 445, + :proto => 'tcp', + :name => 'MS07-029', + :info => 'Vulnerability in Windows DNS RPC Interface Could Allow Remote Code Execution', + # Add more refs based on nessus/nexpose .. results + :refs =>['CVE-2007-1748', + 'OSVDB-34100', + 'MSF-Microsoft DNS RPC Service extractQuotedChar()', + 'NSS-25168'] + } + report_vuln(vuln_info) + end + end + end + end + } + + # XXX: Legacy nmap xml parser ends here. + + REXML::Document.parse_stream(data, parser) + end + + # + # Import Nmap's -oX xml output + # + def import_nmap_xml_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + + data = "" + ::File.open(filename, 'rb') do |f| + data = f.read(f.stat.size) + end + import_nmap_xml(args.merge(:data => data)) + end + + def nmap_msf_service_map(proto) + service_name_map(proto) + end +end From aa4c104df50e2db50bb7299225f130367cc35985 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 13:08:05 -0500 Subject: [PATCH 140/159] Extract Msf::DBManager::Import::Nmap shared examples MSP-11124 --- spec/support/shared/examples/msf/db_manager/import.rb | 5 +---- spec/support/shared/examples/msf/db_manager/import/nmap.rb | 6 ++++++ 2 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/import/nmap.rb diff --git a/spec/support/shared/examples/msf/db_manager/import.rb b/spec/support/shared/examples/msf/db_manager/import.rb index 18cfd9e3df..7b145d3fc0 100644 --- a/spec/support/shared/examples/msf/db_manager/import.rb +++ b/spec/support/shared/examples/msf/db_manager/import.rb @@ -4,9 +4,6 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :import } it { is_expected.to respond_to :import_file } it { is_expected.to respond_to :import_filetype_detect } - it { is_expected.to respond_to :import_nmap_noko_stream } - it { is_expected.to respond_to :import_nmap_xml } - it { is_expected.to respond_to :import_nmap_xml_file } it { is_expected.to respond_to :import_openvas_new_xml } it { is_expected.to respond_to :import_openvas_new_xml_file } it { is_expected.to respond_to :import_openvas_xml } @@ -22,7 +19,6 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :inspect_single_packet_http } it { is_expected.to respond_to :msf_import_timestamps } it { is_expected.to respond_to :nils_for_nulls } - it { is_expected.to respond_to :nmap_msf_service_map } it { is_expected.to respond_to :report_import_note } it { is_expected.to respond_to :rexmlify } it { is_expected.to respond_to :unserialize_object } @@ -44,5 +40,6 @@ shared_examples_for 'Msf::DBManager::Import' do it_should_behave_like 'Msf::DBManager::Import::Netsparker' it_should_behave_like 'Msf::DBManager::Import::Nexpose' it_should_behave_like 'Msf::DBManager::Import::Nikto' + it_should_behave_like 'Msf::DBManager::Import::Nmap' it_should_behave_like 'Msf::DBManager::Import::Qualys' end \ No newline at end of file diff --git a/spec/support/shared/examples/msf/db_manager/import/nmap.rb b/spec/support/shared/examples/msf/db_manager/import/nmap.rb new file mode 100644 index 0000000000..52354a196a --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/import/nmap.rb @@ -0,0 +1,6 @@ +shared_examples_for 'Msf::DBManager::Import::Nmap' do + it { is_expected.to respond_to :import_nmap_noko_stream } + it { is_expected.to respond_to :import_nmap_xml } + it { is_expected.to respond_to :import_nmap_xml_file } + it { is_expected.to respond_to :nmap_msf_service_map } +end From cf555e239021f77e6ced39b54e9c9fd1d7cbfcba Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 13:11:49 -0500 Subject: [PATCH 141/159] Extract Msf::DBManager::Import::OpenVAS MSP-11124 --- lib/msf/core/db_manager/import.rb | 34 ++-------------------- lib/msf/core/db_manager/import/open_vas.rb | 34 ++++++++++++++++++++++ 2 files changed, 36 insertions(+), 32 deletions(-) create mode 100644 lib/msf/core/db_manager/import/open_vas.rb diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index fb143c94f1..2ac321ac7b 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -12,7 +12,6 @@ require 'uri' # require 'packetfu' -require 'rex/parser/openvas_nokogiri' require 'rex/parser/outpost24_nokogiri' require 'rex/parser/retina_xml' require 'rex/parser/wapiti_nokogiri' @@ -35,6 +34,7 @@ module Msf::DBManager::Import autoload :Nexpose, 'msf/core/db_manager/import/nexpose' autoload :Nikto, 'msf/core/db_manager/import/nikto' autoload :Nmap, 'msf/core/db_manager/import/nmap' + autoload :OpenVAS, 'msf/core/db_manager/import/open_vas' autoload :Qualys, 'msf/core/db_manager/import/qualys' include Msf::DBManager::Import::Acunetix @@ -54,6 +54,7 @@ module Msf::DBManager::Import include Msf::DBManager::Import::Nexpose include Msf::DBManager::Import::Nikto include Msf::DBManager::Import::Nmap + include Msf::DBManager::Import::OpenVAS include Msf::DBManager::Import::Qualys # If hex notation is present, turn them into a character. @@ -350,27 +351,6 @@ module Msf::DBManager::Import raise DBImportError.new("Could not automatically determine file type") end - def import_openvas_new_xml(args={}, &block) - if block - doc = Rex::Parser::OpenVASDocument.new(args,framework.db) {|type, data| yield type,data } - else - doc = Rex::Parser::OpenVASDocument.new(args,self) - end - parser = ::Nokogiri::XML::SAX::Parser.new(doc) - parser.parse(args[:data]) - end - - def import_openvas_new_xml_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - - data = "" - ::File.open(filename, 'rb') do |f| - data = f.read(f.stat.size) - end - import_wapiti_xml(args.merge(:data => data)) - end - # Do all the single packet analysis we can while churning through the pcap # the first time. Multiple packet inspection will come later, where we can # do stream analysis, compare requests and responses, etc. @@ -486,16 +466,6 @@ module Msf::DBManager::Import import_wapiti_xml(args.merge(:data => data)) end - # - # Of course they had to change the nessus format. - # - def import_openvas_xml(args={}, &block) - filename = args[:filename] - wspace = args[:wspace] || workspace - - raise DBImportError.new("No OpenVAS XML support. Please submit a patch to msfdev[at]metasploit.com") - end - def import_outpost24_noko_stream(args={},&block) if block doc = Rex::Parser::Outpost24Document.new(args,framework.db) {|type, data| yield type,data } diff --git a/lib/msf/core/db_manager/import/open_vas.rb b/lib/msf/core/db_manager/import/open_vas.rb new file mode 100644 index 0000000000..cf3c227920 --- /dev/null +++ b/lib/msf/core/db_manager/import/open_vas.rb @@ -0,0 +1,34 @@ +require 'rex/parser/openvas_nokogiri' + +module Msf::DBManager::Import::OpenVAS + # + # Of course they had to change the nessus format. + # + def import_openvas_xml(args={}, &block) + filename = args[:filename] + wspace = args[:wspace] || workspace + + raise DBImportError.new("No OpenVAS XML support. Please submit a patch to msfdev[at]metasploit.com") + end + + def import_openvas_new_xml(args={}, &block) + if block + doc = Rex::Parser::OpenVASDocument.new(args,framework.db) {|type, data| yield type,data } + else + doc = Rex::Parser::OpenVASDocument.new(args,self) + end + parser = ::Nokogiri::XML::SAX::Parser.new(doc) + parser.parse(args[:data]) + end + + def import_openvas_new_xml_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + + data = "" + ::File.open(filename, 'rb') do |f| + data = f.read(f.stat.size) + end + import_wapiti_xml(args.merge(:data => data)) + end +end From 0418027c909908155fc8453d68cc30249d06f703 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 13:13:57 -0500 Subject: [PATCH 142/159] Extract Msf::DBManager::Import::OpenVAS MSP-11124 --- spec/support/shared/examples/msf/db_manager/import.rb | 4 +--- .../shared/examples/msf/db_manager/import/open_vas.rb | 5 +++++ 2 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/import/open_vas.rb diff --git a/spec/support/shared/examples/msf/db_manager/import.rb b/spec/support/shared/examples/msf/db_manager/import.rb index 7b145d3fc0..7fdce217c4 100644 --- a/spec/support/shared/examples/msf/db_manager/import.rb +++ b/spec/support/shared/examples/msf/db_manager/import.rb @@ -4,9 +4,6 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :import } it { is_expected.to respond_to :import_file } it { is_expected.to respond_to :import_filetype_detect } - it { is_expected.to respond_to :import_openvas_new_xml } - it { is_expected.to respond_to :import_openvas_new_xml_file } - it { is_expected.to respond_to :import_openvas_xml } it { is_expected.to respond_to :import_outpost24_noko_stream } it { is_expected.to respond_to :import_outpost24_xml } it { is_expected.to respond_to :import_report } @@ -41,5 +38,6 @@ shared_examples_for 'Msf::DBManager::Import' do it_should_behave_like 'Msf::DBManager::Import::Nexpose' it_should_behave_like 'Msf::DBManager::Import::Nikto' it_should_behave_like 'Msf::DBManager::Import::Nmap' + it_should_behave_like 'Msf::DBManager::Import::OpenVAS' it_should_behave_like 'Msf::DBManager::Import::Qualys' end \ No newline at end of file diff --git a/spec/support/shared/examples/msf/db_manager/import/open_vas.rb b/spec/support/shared/examples/msf/db_manager/import/open_vas.rb new file mode 100644 index 0000000000..9ffc75d42f --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/import/open_vas.rb @@ -0,0 +1,5 @@ +shared_examples_for 'Msf::DBManager::Import::OpenVAS' do + it { is_expected.to respond_to :import_openvas_new_xml } + it { is_expected.to respond_to :import_openvas_new_xml_file } + it { is_expected.to respond_to :import_openvas_xml } +end From eff95221dad320a89c276fd18505fe8564c7317e Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 13:14:20 -0500 Subject: [PATCH 143/159] Order methods MSP-11124 --- lib/msf/core/db_manager/import/open_vas.rb | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/msf/core/db_manager/import/open_vas.rb b/lib/msf/core/db_manager/import/open_vas.rb index cf3c227920..ed936b9d8c 100644 --- a/lib/msf/core/db_manager/import/open_vas.rb +++ b/lib/msf/core/db_manager/import/open_vas.rb @@ -1,16 +1,6 @@ require 'rex/parser/openvas_nokogiri' module Msf::DBManager::Import::OpenVAS - # - # Of course they had to change the nessus format. - # - def import_openvas_xml(args={}, &block) - filename = args[:filename] - wspace = args[:wspace] || workspace - - raise DBImportError.new("No OpenVAS XML support. Please submit a patch to msfdev[at]metasploit.com") - end - def import_openvas_new_xml(args={}, &block) if block doc = Rex::Parser::OpenVASDocument.new(args,framework.db) {|type, data| yield type,data } @@ -31,4 +21,14 @@ module Msf::DBManager::Import::OpenVAS end import_wapiti_xml(args.merge(:data => data)) end + + # + # Of course they had to change the nessus format. + # + def import_openvas_xml(args={}, &block) + filename = args[:filename] + wspace = args[:wspace] || workspace + + raise DBImportError.new("No OpenVAS XML support. Please submit a patch to msfdev[at]metasploit.com") + end end From 8af280b1cb1f31420dfb7bbea49d66e814447751 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 13:16:11 -0500 Subject: [PATCH 144/159] Extract Msf::DBManager::Import::Outpost24 MSP-11124 --- lib/msf/core/db_manager/import.rb | 33 ++------------------- lib/msf/core/db_manager/import/outpost24.rb | 33 +++++++++++++++++++++ 2 files changed, 35 insertions(+), 31 deletions(-) create mode 100644 lib/msf/core/db_manager/import/outpost24.rb diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index 2ac321ac7b..9b87fd1cb7 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -12,7 +12,6 @@ require 'uri' # require 'packetfu' -require 'rex/parser/outpost24_nokogiri' require 'rex/parser/retina_xml' require 'rex/parser/wapiti_nokogiri' @@ -35,6 +34,7 @@ module Msf::DBManager::Import autoload :Nikto, 'msf/core/db_manager/import/nikto' autoload :Nmap, 'msf/core/db_manager/import/nmap' autoload :OpenVAS, 'msf/core/db_manager/import/open_vas' + autoload :Outpost24, 'msf/core/db_manager/import/outpost24' autoload :Qualys, 'msf/core/db_manager/import/qualys' include Msf::DBManager::Import::Acunetix @@ -55,6 +55,7 @@ module Msf::DBManager::Import include Msf::DBManager::Import::Nikto include Msf::DBManager::Import::Nmap include Msf::DBManager::Import::OpenVAS + include Msf::DBManager::Import::Outpost24 include Msf::DBManager::Import::Qualys # If hex notation is present, turn them into a character. @@ -466,36 +467,6 @@ module Msf::DBManager::Import import_wapiti_xml(args.merge(:data => data)) end - def import_outpost24_noko_stream(args={},&block) - if block - doc = Rex::Parser::Outpost24Document.new(args,framework.db) {|type, data| yield type,data } - else - doc = Rex::Parser::Outpost24Document.new(args,self) - end - parser = ::Nokogiri::XML::SAX::Parser.new(doc) - parser.parse(args[:data]) - end - - def import_outpost24_xml(args={}, &block) - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - wspace = args[:wspace] || workspace - if Rex::Parser.nokogiri_loaded - parser = "Nokogiri v#{::Nokogiri::VERSION}" - noko_args = args.dup - noko_args[:blacklist] = bl - noko_args[:wspace] = wspace - if block - yield(:parser, parser) - import_outpost24_noko_stream(noko_args) {|type, data| yield type,data} - else - import_outpost24_noko_stream(noko_args) - end - return true - else # Sorry - raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.") - end - end - # @param report [REXML::Element] to be imported # @param args [Hash] # @param base_dir [String] diff --git a/lib/msf/core/db_manager/import/outpost24.rb b/lib/msf/core/db_manager/import/outpost24.rb new file mode 100644 index 0000000000..52b0c5ad91 --- /dev/null +++ b/lib/msf/core/db_manager/import/outpost24.rb @@ -0,0 +1,33 @@ +require 'rex/parser/outpost24_nokogiri' + +module Msf::DBManager::Import::Outpost24 + def import_outpost24_noko_stream(args={},&block) + if block + doc = Rex::Parser::Outpost24Document.new(args,framework.db) {|type, data| yield type,data } + else + doc = Rex::Parser::Outpost24Document.new(args,self) + end + parser = ::Nokogiri::XML::SAX::Parser.new(doc) + parser.parse(args[:data]) + end + + def import_outpost24_xml(args={}, &block) + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + wspace = args[:wspace] || workspace + if Rex::Parser.nokogiri_loaded + parser = "Nokogiri v#{::Nokogiri::VERSION}" + noko_args = args.dup + noko_args[:blacklist] = bl + noko_args[:wspace] = wspace + if block + yield(:parser, parser) + import_outpost24_noko_stream(noko_args) {|type, data| yield type,data} + else + import_outpost24_noko_stream(noko_args) + end + return true + else # Sorry + raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.") + end + end +end From ac6a8387a5ec88a3cfcb4ea0ffccdc3cf5d7defd Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 13:18:03 -0500 Subject: [PATCH 145/159] Extract Msf::DBManager::Import::Outpost24 shared examples MSP-11124 --- spec/support/shared/examples/msf/db_manager/import.rb | 3 +-- .../shared/examples/msf/db_manager/import/outpost24.rb | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/import/outpost24.rb diff --git a/spec/support/shared/examples/msf/db_manager/import.rb b/spec/support/shared/examples/msf/db_manager/import.rb index 7fdce217c4..00bdac3315 100644 --- a/spec/support/shared/examples/msf/db_manager/import.rb +++ b/spec/support/shared/examples/msf/db_manager/import.rb @@ -4,8 +4,6 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :import } it { is_expected.to respond_to :import_file } it { is_expected.to respond_to :import_filetype_detect } - it { is_expected.to respond_to :import_outpost24_noko_stream } - it { is_expected.to respond_to :import_outpost24_xml } it { is_expected.to respond_to :import_report } it { is_expected.to respond_to :import_retina_xml } it { is_expected.to respond_to :import_retina_xml_file } @@ -39,5 +37,6 @@ shared_examples_for 'Msf::DBManager::Import' do it_should_behave_like 'Msf::DBManager::Import::Nikto' it_should_behave_like 'Msf::DBManager::Import::Nmap' it_should_behave_like 'Msf::DBManager::Import::OpenVAS' + it_should_behave_like 'Msf::DBManager::Import::Outpost24' it_should_behave_like 'Msf::DBManager::Import::Qualys' end \ No newline at end of file diff --git a/spec/support/shared/examples/msf/db_manager/import/outpost24.rb b/spec/support/shared/examples/msf/db_manager/import/outpost24.rb new file mode 100644 index 0000000000..5de0d6cad1 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/import/outpost24.rb @@ -0,0 +1,4 @@ +shared_examples_for 'Msf::DBManager::Import::Outpost24' do + it { is_expected.to respond_to :import_outpost24_noko_stream } + it { is_expected.to respond_to :import_outpost24_xml } +end From dfe690ac526a7ce3ffc9863fb48e931788e2dfac Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 13:23:12 -0500 Subject: [PATCH 146/159] Extract Msf::DBManager::Import::Retina MSP-11124 --- lib/msf/core/db_manager/import.rb | 95 +----------------------- lib/msf/core/db_manager/import/retina.rb | 95 ++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 93 deletions(-) create mode 100644 lib/msf/core/db_manager/import/retina.rb diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index 9b87fd1cb7..cd3184733b 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -12,7 +12,6 @@ require 'uri' # require 'packetfu' -require 'rex/parser/retina_xml' require 'rex/parser/wapiti_nokogiri' module Msf::DBManager::Import @@ -36,6 +35,7 @@ module Msf::DBManager::Import autoload :OpenVAS, 'msf/core/db_manager/import/open_vas' autoload :Outpost24, 'msf/core/db_manager/import/outpost24' autoload :Qualys, 'msf/core/db_manager/import/qualys' + autoload :Retina, 'msf/core/db_manager/import/retina' include Msf::DBManager::Import::Acunetix include Msf::DBManager::Import::Amap @@ -57,6 +57,7 @@ module Msf::DBManager::Import include Msf::DBManager::Import::OpenVAS include Msf::DBManager::Import::Outpost24 include Msf::DBManager::Import::Qualys + include Msf::DBManager::Import::Retina # If hex notation is present, turn them into a character. def dehex(str) @@ -514,98 +515,6 @@ module Msf::DBManager::Import end end - # Process Retina XML - def import_retina_xml(args={}, &block) - data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - msg = "Warning: The Retina XML format does not associate vulnerabilities with the\n" - msg << "specific service on which they were found.\n" - msg << "This makes it impossible to correlate exploits to discovered vulnerabilities\n" - msg << "in a reliable fashion." - - yield(:warning,msg) if block - - parser = Rex::Parser::RetinaXMLStreamParser.new - parser.on_found_host = Proc.new do |host| - hobj = nil - data = { - :workspace => wspace, - :task => args[:task] - } - addr = host['address'] - next if not addr - - next if bl.include? addr - data[:host] = addr - - if host['mac'] - data[:mac] = host['mac'] - end - - data[:state] = Msf::HostState::Alive - - if host['hostname'] - data[:name] = host['hostname'] - end - - if host['netbios'] - data[:name] = host['netbios'] - end - - yield(:address, data[:host]) if block - - # Import Host - hobj = report_host(data) - report_import_note(wspace, hobj) - - # Import OS fingerprint - if host["os"] - note = { - :workspace => wspace, - :host => addr, - :type => 'host.os.retina_fingerprint', - :task => args[:task], - :data => { - :os => host["os"] - } - } - report_note(note) - end - - # Import vulnerabilities - host['vulns'].each do |vuln| - refs = vuln['refs'].map{|v| v.join("-")} - refs << "RETINA-#{vuln['rthid']}" if vuln['rthid'] - - vuln_info = { - :workspace => wspace, - :host => addr, - :name => vuln['name'], - :info => vuln['description'], - :refs => refs, - :task => args[:task] - } - - report_vuln(vuln_info) - end - end - - REXML::Document.parse_stream(data, parser) - end - - # Process a Retina XML file - def import_retina_xml_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - - data = "" - ::File.open(filename, 'rb') do |f| - data = f.read(f.stat.size) - end - import_retina_xml(args.merge(:data => data)) - end - def import_spiceworks_csv(args={}, &block) data = args[:data] wspace = args[:wspace] || workspace diff --git a/lib/msf/core/db_manager/import/retina.rb b/lib/msf/core/db_manager/import/retina.rb new file mode 100644 index 0000000000..66e6da12f1 --- /dev/null +++ b/lib/msf/core/db_manager/import/retina.rb @@ -0,0 +1,95 @@ +require 'rex/parser/retina_xml' + +module Msf::DBManager::Import::Retina + # Process Retina XML + def import_retina_xml(args={}, &block) + data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + msg = "Warning: The Retina XML format does not associate vulnerabilities with the\n" + msg << "specific service on which they were found.\n" + msg << "This makes it impossible to correlate exploits to discovered vulnerabilities\n" + msg << "in a reliable fashion." + + yield(:warning,msg) if block + + parser = Rex::Parser::RetinaXMLStreamParser.new + parser.on_found_host = Proc.new do |host| + hobj = nil + data = { + :workspace => wspace, + :task => args[:task] + } + addr = host['address'] + next if not addr + + next if bl.include? addr + data[:host] = addr + + if host['mac'] + data[:mac] = host['mac'] + end + + data[:state] = Msf::HostState::Alive + + if host['hostname'] + data[:name] = host['hostname'] + end + + if host['netbios'] + data[:name] = host['netbios'] + end + + yield(:address, data[:host]) if block + + # Import Host + hobj = report_host(data) + report_import_note(wspace, hobj) + + # Import OS fingerprint + if host["os"] + note = { + :workspace => wspace, + :host => addr, + :type => 'host.os.retina_fingerprint', + :task => args[:task], + :data => { + :os => host["os"] + } + } + report_note(note) + end + + # Import vulnerabilities + host['vulns'].each do |vuln| + refs = vuln['refs'].map{|v| v.join("-")} + refs << "RETINA-#{vuln['rthid']}" if vuln['rthid'] + + vuln_info = { + :workspace => wspace, + :host => addr, + :name => vuln['name'], + :info => vuln['description'], + :refs => refs, + :task => args[:task] + } + + report_vuln(vuln_info) + end + end + + REXML::Document.parse_stream(data, parser) + end + + # Process a Retina XML file + def import_retina_xml_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + + data = "" + ::File.open(filename, 'rb') do |f| + data = f.read(f.stat.size) + end + import_retina_xml(args.merge(:data => data)) + end +end From 92a6c792009c52d18f6910f1c73aeb247eff0f59 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 13:24:40 -0500 Subject: [PATCH 147/159] Extract Msf::DBManager::Import::Retina shared examples MSP-11124 --- spec/support/shared/examples/msf/db_manager/import.rb | 3 +-- spec/support/shared/examples/msf/db_manager/import/retina.rb | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/import/retina.rb diff --git a/spec/support/shared/examples/msf/db_manager/import.rb b/spec/support/shared/examples/msf/db_manager/import.rb index 00bdac3315..eb3386ebc5 100644 --- a/spec/support/shared/examples/msf/db_manager/import.rb +++ b/spec/support/shared/examples/msf/db_manager/import.rb @@ -5,8 +5,6 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :import_file } it { is_expected.to respond_to :import_filetype_detect } it { is_expected.to respond_to :import_report } - it { is_expected.to respond_to :import_retina_xml } - it { is_expected.to respond_to :import_retina_xml_file } it { is_expected.to respond_to :import_spiceworks_csv } it { is_expected.to respond_to :import_wapiti_xml } it { is_expected.to respond_to :import_wapiti_xml_file } @@ -39,4 +37,5 @@ shared_examples_for 'Msf::DBManager::Import' do it_should_behave_like 'Msf::DBManager::Import::OpenVAS' it_should_behave_like 'Msf::DBManager::Import::Outpost24' it_should_behave_like 'Msf::DBManager::Import::Qualys' + it_should_behave_like 'Msf::DBManager::Import::Retina' end \ No newline at end of file diff --git a/spec/support/shared/examples/msf/db_manager/import/retina.rb b/spec/support/shared/examples/msf/db_manager/import/retina.rb new file mode 100644 index 0000000000..d96778393c --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/import/retina.rb @@ -0,0 +1,4 @@ +shared_examples_for 'Msf::DBManager::Import::Retina' do + it { is_expected.to respond_to :import_retina_xml } + it { is_expected.to respond_to :import_retina_xml_file } +end From e65a386d3da01a9242ae8a693c82d8728c8b53f3 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 13:37:35 -0500 Subject: [PATCH 148/159] Extract Msf::DBManager::Import::Spiceworks MSP-11124 --- lib/msf/core/db_manager/import.rb | 51 +------------------- lib/msf/core/db_manager/import/spiceworks.rb | 51 ++++++++++++++++++++ 2 files changed, 53 insertions(+), 49 deletions(-) create mode 100644 lib/msf/core/db_manager/import/spiceworks.rb diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index cd3184733b..bb8a307b35 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -2,7 +2,6 @@ # Standard library # -require 'csv' require 'fileutils' require 'tmpdir' require 'uri' @@ -36,6 +35,7 @@ module Msf::DBManager::Import autoload :Outpost24, 'msf/core/db_manager/import/outpost24' autoload :Qualys, 'msf/core/db_manager/import/qualys' autoload :Retina, 'msf/core/db_manager/import/retina' + autoload :Spiceworks, 'msf/core/db_manager/import/spiceworks' include Msf::DBManager::Import::Acunetix include Msf::DBManager::Import::Amap @@ -58,6 +58,7 @@ module Msf::DBManager::Import include Msf::DBManager::Import::Outpost24 include Msf::DBManager::Import::Qualys include Msf::DBManager::Import::Retina + include Msf::DBManager::Import::Spiceworks # If hex notation is present, turn them into a character. def dehex(str) @@ -515,54 +516,6 @@ module Msf::DBManager::Import end end - def import_spiceworks_csv(args={}, &block) - data = args[:data] - wspace = args[:wspace] || workspace - bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] - CSV.parse(data) do |row| - next unless (["Name", "Manufacturer", "Device Type"] & row).empty? #header - name = row[0] - manufacturer = row[1] - device = row[2] - model = row[3] - ip = row[4] - serialno = row[5] - location = row[6] - os = row[7] - - next unless ip - next if bl.include? ip - - conf = { - :workspace => wspace, - :host => ip, - :name => name, - :task => args[:task] - } - - - if os - report_note( - :workspace => wspace, - :task => args[:task], - :host => ip, - :type => 'host.os.spiceworks_fingerprint', - :data => { - :os => os.to_s.strip - } - ) - end - - info = [] - info << "Serial Number: #{serialno}" unless (serialno.blank? or serialno == name) - info << "Location: #{location}" unless location.blank? - conf[:info] = info.join(", ") unless info.empty? - - host = report_host(conf) - report_import_note(wspace, host) - end - end - # Handles timestamps from Metasploit Express/Pro imports. def msf_import_timestamps(opts,obj) obj.created_at = opts["created_at"] if opts["created_at"] diff --git a/lib/msf/core/db_manager/import/spiceworks.rb b/lib/msf/core/db_manager/import/spiceworks.rb new file mode 100644 index 0000000000..3cf7c513c0 --- /dev/null +++ b/lib/msf/core/db_manager/import/spiceworks.rb @@ -0,0 +1,51 @@ +require 'csv' + +module Msf::DBManager::Import::Spiceworks + def import_spiceworks_csv(args={}, &block) + data = args[:data] + wspace = args[:wspace] || workspace + bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : [] + CSV.parse(data) do |row| + next unless (["Name", "Manufacturer", "Device Type"] & row).empty? #header + name = row[0] + manufacturer = row[1] + device = row[2] + model = row[3] + ip = row[4] + serialno = row[5] + location = row[6] + os = row[7] + + next unless ip + next if bl.include? ip + + conf = { + :workspace => wspace, + :host => ip, + :name => name, + :task => args[:task] + } + + + if os + report_note( + :workspace => wspace, + :task => args[:task], + :host => ip, + :type => 'host.os.spiceworks_fingerprint', + :data => { + :os => os.to_s.strip + } + ) + end + + info = [] + info << "Serial Number: #{serialno}" unless (serialno.blank? or serialno == name) + info << "Location: #{location}" unless location.blank? + conf[:info] = info.join(", ") unless info.empty? + + host = report_host(conf) + report_import_note(wspace, host) + end + end +end From 6e0c7f0aafa4ab8a60ca1ee6d4be47492884f4ba Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 13:40:03 -0500 Subject: [PATCH 149/159] Extract Msf::DBManager::Import::Spiceworks shared examples MSP-11124 --- spec/support/shared/examples/msf/db_manager/import.rb | 2 +- .../shared/examples/msf/db_manager/import/spiceworks.rb | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 spec/support/shared/examples/msf/db_manager/import/spiceworks.rb diff --git a/spec/support/shared/examples/msf/db_manager/import.rb b/spec/support/shared/examples/msf/db_manager/import.rb index eb3386ebc5..7cee2af91e 100644 --- a/spec/support/shared/examples/msf/db_manager/import.rb +++ b/spec/support/shared/examples/msf/db_manager/import.rb @@ -5,7 +5,6 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :import_file } it { is_expected.to respond_to :import_filetype_detect } it { is_expected.to respond_to :import_report } - it { is_expected.to respond_to :import_spiceworks_csv } it { is_expected.to respond_to :import_wapiti_xml } it { is_expected.to respond_to :import_wapiti_xml_file } it { is_expected.to respond_to :inspect_single_packet } @@ -38,4 +37,5 @@ shared_examples_for 'Msf::DBManager::Import' do it_should_behave_like 'Msf::DBManager::Import::Outpost24' it_should_behave_like 'Msf::DBManager::Import::Qualys' it_should_behave_like 'Msf::DBManager::Import::Retina' + it_should_behave_like 'Msf::DBManager::Import::Spiceworks' end \ No newline at end of file diff --git a/spec/support/shared/examples/msf/db_manager/import/spiceworks.rb b/spec/support/shared/examples/msf/db_manager/import/spiceworks.rb new file mode 100644 index 0000000000..08e653a47b --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/import/spiceworks.rb @@ -0,0 +1,3 @@ +shared_examples_for 'Msf::DBManager::Import::Spiceworks' do + it { is_expected.to respond_to :import_spiceworks_csv } +end From e5e051c905f5c4fad6242f3d645cb13843dec2ab Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 13:42:54 -0500 Subject: [PATCH 150/159] Extract Msf::DBManager::Import::Wapiti MSP-11124 --- lib/msf/core/db_manager/import.rb | 24 ++---------------------- lib/msf/core/db_manager/import/wapiti.rb | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 22 deletions(-) create mode 100644 lib/msf/core/db_manager/import/wapiti.rb diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index bb8a307b35..23284ac5df 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -11,7 +11,6 @@ require 'uri' # require 'packetfu' -require 'rex/parser/wapiti_nokogiri' module Msf::DBManager::Import autoload :Acunetix, 'msf/core/db_manager/import/acunetix' @@ -36,6 +35,7 @@ module Msf::DBManager::Import autoload :Qualys, 'msf/core/db_manager/import/qualys' autoload :Retina, 'msf/core/db_manager/import/retina' autoload :Spiceworks, 'msf/core/db_manager/import/spiceworks' + autoload :Wapiti, 'msf/core/db_manager/import/wapiti' include Msf::DBManager::Import::Acunetix include Msf::DBManager::Import::Amap @@ -59,6 +59,7 @@ module Msf::DBManager::Import include Msf::DBManager::Import::Qualys include Msf::DBManager::Import::Retina include Msf::DBManager::Import::Spiceworks + include Msf::DBManager::Import::Wapiti # If hex notation is present, turn them into a character. def dehex(str) @@ -448,27 +449,6 @@ module Msf::DBManager::Import end end - def import_wapiti_xml(args={}, &block) - if block - doc = Rex::Parser::WapitiDocument.new(args,framework.db) {|type, data| yield type,data } - else - doc = Rex::Parser::WapitiDocument.new(args,self) - end - parser = ::Nokogiri::XML::SAX::Parser.new(doc) - parser.parse(args[:data]) - end - - def import_wapiti_xml_file(args={}) - filename = args[:filename] - wspace = args[:wspace] || workspace - - data = "" - ::File.open(filename, 'rb') do |f| - data = f.read(f.stat.size) - end - import_wapiti_xml(args.merge(:data => data)) - end - # @param report [REXML::Element] to be imported # @param args [Hash] # @param base_dir [String] diff --git a/lib/msf/core/db_manager/import/wapiti.rb b/lib/msf/core/db_manager/import/wapiti.rb new file mode 100644 index 0000000000..855e5bec45 --- /dev/null +++ b/lib/msf/core/db_manager/import/wapiti.rb @@ -0,0 +1,24 @@ +require 'rex/parser/wapiti_nokogiri' + +module Msf::DBManager::Import::Wapiti + def import_wapiti_xml(args={}, &block) + if block + doc = Rex::Parser::WapitiDocument.new(args,framework.db) {|type, data| yield type,data } + else + doc = Rex::Parser::WapitiDocument.new(args,self) + end + parser = ::Nokogiri::XML::SAX::Parser.new(doc) + parser.parse(args[:data]) + end + + def import_wapiti_xml_file(args={}) + filename = args[:filename] + wspace = args[:wspace] || workspace + + data = "" + ::File.open(filename, 'rb') do |f| + data = f.read(f.stat.size) + end + import_wapiti_xml(args.merge(:data => data)) + end +end From 505ca5fcc0c04259d14148f30369bc6786053ccf Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 13:47:22 -0500 Subject: [PATCH 151/159] Extract Msf::DBManager::Import::Wapiti shared examples MSP-11124 --- spec/support/shared/examples/msf/db_manager/import.rb | 3 +-- spec/support/shared/examples/msf/db_manager/import/wapiti.rb | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 spec/support/shared/examples/msf/db_manager/import/wapiti.rb diff --git a/spec/support/shared/examples/msf/db_manager/import.rb b/spec/support/shared/examples/msf/db_manager/import.rb index 7cee2af91e..a0b4870961 100644 --- a/spec/support/shared/examples/msf/db_manager/import.rb +++ b/spec/support/shared/examples/msf/db_manager/import.rb @@ -5,8 +5,6 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :import_file } it { is_expected.to respond_to :import_filetype_detect } it { is_expected.to respond_to :import_report } - it { is_expected.to respond_to :import_wapiti_xml } - it { is_expected.to respond_to :import_wapiti_xml_file } it { is_expected.to respond_to :inspect_single_packet } it { is_expected.to respond_to :inspect_single_packet_http } it { is_expected.to respond_to :msf_import_timestamps } @@ -38,4 +36,5 @@ shared_examples_for 'Msf::DBManager::Import' do it_should_behave_like 'Msf::DBManager::Import::Qualys' it_should_behave_like 'Msf::DBManager::Import::Retina' it_should_behave_like 'Msf::DBManager::Import::Spiceworks' + it_should_behave_like 'Msf::DBManager::Import::Wapiti' end \ No newline at end of file diff --git a/spec/support/shared/examples/msf/db_manager/import/wapiti.rb b/spec/support/shared/examples/msf/db_manager/import/wapiti.rb new file mode 100644 index 0000000000..2f7432549a --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/import/wapiti.rb @@ -0,0 +1,4 @@ +shared_examples_for 'Msf::DBManager::Import::Wapiti' do + it { is_expected.to respond_to :import_wapiti_xml } + it { is_expected.to respond_to :import_wapiti_xml_file } +end From 7aed88f11babcd90b8da14d3286091bceabda1c7 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 13:51:57 -0500 Subject: [PATCH 152/159] Extract Msf::DBManager::Import::Report MSP-11124 --- lib/msf/core/db_manager/import.rb | 49 +----------------------- lib/msf/core/db_manager/import/report.rb | 48 +++++++++++++++++++++++ 2 files changed, 50 insertions(+), 47 deletions(-) create mode 100644 lib/msf/core/db_manager/import/report.rb diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index 23284ac5df..39a285d692 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -33,6 +33,7 @@ module Msf::DBManager::Import autoload :OpenVAS, 'msf/core/db_manager/import/open_vas' autoload :Outpost24, 'msf/core/db_manager/import/outpost24' autoload :Qualys, 'msf/core/db_manager/import/qualys' + autoload :Report, 'msf/core/db_manager/import/report' autoload :Retina, 'msf/core/db_manager/import/retina' autoload :Spiceworks, 'msf/core/db_manager/import/spiceworks' autoload :Wapiti, 'msf/core/db_manager/import/wapiti' @@ -57,6 +58,7 @@ module Msf::DBManager::Import include Msf::DBManager::Import::OpenVAS include Msf::DBManager::Import::Outpost24 include Msf::DBManager::Import::Qualys + include Msf::DBManager::Import::Report include Msf::DBManager::Import::Retina include Msf::DBManager::Import::Spiceworks include Msf::DBManager::Import::Wapiti @@ -449,53 +451,6 @@ module Msf::DBManager::Import end end - # @param report [REXML::Element] to be imported - # @param args [Hash] - # @param base_dir [String] - def import_report(report, args, base_dir) - tmp = args[:ifd][:zip_tmp] - report_info = {} - - report.elements.each do |e| - node_name = e.name - node_value = e.text - - # These need to be converted back to arrays: - array_attrs = %w|addresses file-formats options sections| - if array_attrs.member? node_name - node_value = JSON.parse(node_value) - end - # Don't restore these values: - skip_nodes = %w|id workspace-id artifacts| - next if skip_nodes.member? node_name - - report_info[node_name.parameterize.underscore.to_sym] = node_value - end - # Use current workspace - report_info[:workspace_id] = args[:wspace].id - - # Create report, need new ID to record artifacts - report_id = report_report(report_info) - - # Handle artifacts - report.elements['artifacts'].elements.each do |artifact| - artifact_opts = {} - artifact.elements.each do |attr| - skip_nodes = %w|id accessed-at| - next if skip_nodes.member? attr.name - - symboled_attr = attr.name.parameterize.underscore.to_sym - artifact_opts[symboled_attr] = attr.text - end - # Use new Report as parent - artifact_opts[:report_id] = report_id - # Update to full path - artifact_opts[:file_path].gsub!(/^\./, tmp) - - report_artifact(artifact_opts) - end - end - # Handles timestamps from Metasploit Express/Pro imports. def msf_import_timestamps(opts,obj) obj.created_at = opts["created_at"] if opts["created_at"] diff --git a/lib/msf/core/db_manager/import/report.rb b/lib/msf/core/db_manager/import/report.rb new file mode 100644 index 0000000000..a6708b5e1c --- /dev/null +++ b/lib/msf/core/db_manager/import/report.rb @@ -0,0 +1,48 @@ +module Msf::DBManager::Import::Report + # @param report [REXML::Element] to be imported + # @param args [Hash] + # @param base_dir [String] + def import_report(report, args, base_dir) + tmp = args[:ifd][:zip_tmp] + report_info = {} + + report.elements.each do |e| + node_name = e.name + node_value = e.text + + # These need to be converted back to arrays: + array_attrs = %w|addresses file-formats options sections| + if array_attrs.member? node_name + node_value = JSON.parse(node_value) + end + # Don't restore these values: + skip_nodes = %w|id workspace-id artifacts| + next if skip_nodes.member? node_name + + report_info[node_name.parameterize.underscore.to_sym] = node_value + end + # Use current workspace + report_info[:workspace_id] = args[:wspace].id + + # Create report, need new ID to record artifacts + report_id = report_report(report_info) + + # Handle artifacts + report.elements['artifacts'].elements.each do |artifact| + artifact_opts = {} + artifact.elements.each do |attr| + skip_nodes = %w|id accessed-at| + next if skip_nodes.member? attr.name + + symboled_attr = attr.name.parameterize.underscore.to_sym + artifact_opts[symboled_attr] = attr.text + end + # Use new Report as parent + artifact_opts[:report_id] = report_id + # Update to full path + artifact_opts[:file_path].gsub!(/^\./, tmp) + + report_artifact(artifact_opts) + end + end +end From 5d5922452003c71f1313beff7dba1a9eb3da1307 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 13:53:09 -0500 Subject: [PATCH 153/159] Extract Msf::DBManager::Import::Report shared examples MSP-11124 --- spec/support/shared/examples/msf/db_manager/import.rb | 2 +- spec/support/shared/examples/msf/db_manager/import/report.rb | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 spec/support/shared/examples/msf/db_manager/import/report.rb diff --git a/spec/support/shared/examples/msf/db_manager/import.rb b/spec/support/shared/examples/msf/db_manager/import.rb index a0b4870961..ae590f6d2b 100644 --- a/spec/support/shared/examples/msf/db_manager/import.rb +++ b/spec/support/shared/examples/msf/db_manager/import.rb @@ -4,7 +4,6 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :import } it { is_expected.to respond_to :import_file } it { is_expected.to respond_to :import_filetype_detect } - it { is_expected.to respond_to :import_report } it { is_expected.to respond_to :inspect_single_packet } it { is_expected.to respond_to :inspect_single_packet_http } it { is_expected.to respond_to :msf_import_timestamps } @@ -34,6 +33,7 @@ shared_examples_for 'Msf::DBManager::Import' do it_should_behave_like 'Msf::DBManager::Import::OpenVAS' it_should_behave_like 'Msf::DBManager::Import::Outpost24' it_should_behave_like 'Msf::DBManager::Import::Qualys' + it_should_behave_like 'Msf::DBManager::Import::Report' it_should_behave_like 'Msf::DBManager::Import::Retina' it_should_behave_like 'Msf::DBManager::Import::Spiceworks' it_should_behave_like 'Msf::DBManager::Import::Wapiti' diff --git a/spec/support/shared/examples/msf/db_manager/import/report.rb b/spec/support/shared/examples/msf/db_manager/import/report.rb new file mode 100644 index 0000000000..e099194576 --- /dev/null +++ b/spec/support/shared/examples/msf/db_manager/import/report.rb @@ -0,0 +1,3 @@ +shared_examples_for 'Msf::DBManager::Import::Report' do + it { is_expected.to respond_to :import_report } +end From ac3099017791ef1344284a85bd2c350dc59a664f Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 13:55:24 -0500 Subject: [PATCH 154/159] Move libpcap helpers to Libpcap module MSP-11124 --- lib/msf/core/db_manager/import.rb | 94 ----------------------- lib/msf/core/db_manager/import/libpcap.rb | 94 +++++++++++++++++++++++ 2 files changed, 94 insertions(+), 94 deletions(-) diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index 39a285d692..2fa8940d9e 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -357,100 +357,6 @@ module Msf::DBManager::Import raise DBImportError.new("Could not automatically determine file type") end - # Do all the single packet analysis we can while churning through the pcap - # the first time. Multiple packet inspection will come later, where we can - # do stream analysis, compare requests and responses, etc. - def inspect_single_packet(pkt,wspace,args) - if pkt.is_tcp? or pkt.is_udp? - inspect_single_packet_http(pkt,wspace,args) - end - end - - # Checks for packets that are headed towards port 80, are tcp, contain an HTTP/1.0 - # line, contains an Authorization line, contains a b64-encoded credential, and - # extracts it. Reports this credential and solidifies the service as HTTP. - def inspect_single_packet_http(pkt,wspace,args) - task = args.fetch(:task, nil) - # First, check the server side (data from port 80). - if pkt.is_tcp? and pkt.tcp_src == 80 and !pkt.payload.nil? and !pkt.payload.empty? - if pkt.payload =~ /^HTTP\x2f1\x2e[01]/n - http_server_match = pkt.payload.match(/\nServer:\s+([^\r\n]+)[\r\n]/n) - if http_server_match.kind_of?(MatchData) and http_server_match[1] - report_service( - :workspace => wspace, - :host => pkt.ip_saddr, - :port => pkt.tcp_src, - :proto => "tcp", - :name => "http", - :info => http_server_match[1], - :state => Msf::ServiceState::Open, - :task => task - ) - # That's all we want to know from this service. - return :something_significant - end - end - end - - # Next, check the client side (data to port 80) - if pkt.is_tcp? and pkt.tcp_dst == 80 and !pkt.payload.nil? and !pkt.payload.empty? - if pkt.payload.match(/[\x00-\x20]HTTP\x2f1\x2e[10]/n) - auth_match = pkt.payload.match(/\nAuthorization:\s+Basic\s+([A-Za-z0-9=\x2b]+)/n) - if auth_match.kind_of?(MatchData) and auth_match[1] - b64_cred = auth_match[1] - else - return false - end - # If we're this far, we can surmise that at least the client is a web browser, - # he thinks the server is HTTP and he just made an authentication attempt. At - # this point, we'll just believe everything the packet says -- validation ought - # to come later. - user,pass = b64_cred.unpack("m*").first.split(/:/,2) - report_service( - :workspace => wspace, - :host => pkt.ip_daddr, - :port => pkt.tcp_dst, - :proto => "tcp", - :name => "http", - :task => task - ) - - service_data = { - address: pkt.ip_daddr, - port: pkt.tcp_dst, - service_name: 'http', - protocol: 'tcp', - workspace_id: wspace.id - } - service_data[:task_id] = task.id if task - - filename = args[:filename] - - credential_data = { - origin_type: :import, - private_data: pass, - private_type: :password, - username: user, - filename: filename - } - credential_data.merge!(service_data) - credential_core = create_credential(credential_data) - - login_data = { - core: credential_core, - status: Metasploit::Model::Login::Status::UNTRIED - } - - login_data.merge!(service_data) - - create_credential_login(login_data) - - # That's all we want to know from this service. - return :something_significant - end - end - end - # Handles timestamps from Metasploit Express/Pro imports. def msf_import_timestamps(opts,obj) obj.created_at = opts["created_at"] if opts["created_at"] diff --git a/lib/msf/core/db_manager/import/libpcap.rb b/lib/msf/core/db_manager/import/libpcap.rb index e26b83ec86..80d0259ffb 100644 --- a/lib/msf/core/db_manager/import/libpcap.rb +++ b/lib/msf/core/db_manager/import/libpcap.rb @@ -123,4 +123,98 @@ module Msf::DBManager::Import::Libpcap data = PacketFu::PcapFile.new(:filename => filename) import_libpcap(args.merge(:data => data)) end + + # Do all the single packet analysis we can while churning through the pcap + # the first time. Multiple packet inspection will come later, where we can + # do stream analysis, compare requests and responses, etc. + def inspect_single_packet(pkt,wspace,args) + if pkt.is_tcp? or pkt.is_udp? + inspect_single_packet_http(pkt,wspace,args) + end + end + + # Checks for packets that are headed towards port 80, are tcp, contain an HTTP/1.0 + # line, contains an Authorization line, contains a b64-encoded credential, and + # extracts it. Reports this credential and solidifies the service as HTTP. + def inspect_single_packet_http(pkt,wspace,args) + task = args.fetch(:task, nil) + # First, check the server side (data from port 80). + if pkt.is_tcp? and pkt.tcp_src == 80 and !pkt.payload.nil? and !pkt.payload.empty? + if pkt.payload =~ /^HTTP\x2f1\x2e[01]/n + http_server_match = pkt.payload.match(/\nServer:\s+([^\r\n]+)[\r\n]/n) + if http_server_match.kind_of?(MatchData) and http_server_match[1] + report_service( + :workspace => wspace, + :host => pkt.ip_saddr, + :port => pkt.tcp_src, + :proto => "tcp", + :name => "http", + :info => http_server_match[1], + :state => Msf::ServiceState::Open, + :task => task + ) + # That's all we want to know from this service. + return :something_significant + end + end + end + + # Next, check the client side (data to port 80) + if pkt.is_tcp? and pkt.tcp_dst == 80 and !pkt.payload.nil? and !pkt.payload.empty? + if pkt.payload.match(/[\x00-\x20]HTTP\x2f1\x2e[10]/n) + auth_match = pkt.payload.match(/\nAuthorization:\s+Basic\s+([A-Za-z0-9=\x2b]+)/n) + if auth_match.kind_of?(MatchData) and auth_match[1] + b64_cred = auth_match[1] + else + return false + end + # If we're this far, we can surmise that at least the client is a web browser, + # he thinks the server is HTTP and he just made an authentication attempt. At + # this point, we'll just believe everything the packet says -- validation ought + # to come later. + user,pass = b64_cred.unpack("m*").first.split(/:/,2) + report_service( + :workspace => wspace, + :host => pkt.ip_daddr, + :port => pkt.tcp_dst, + :proto => "tcp", + :name => "http", + :task => task + ) + + service_data = { + address: pkt.ip_daddr, + port: pkt.tcp_dst, + service_name: 'http', + protocol: 'tcp', + workspace_id: wspace.id + } + service_data[:task_id] = task.id if task + + filename = args[:filename] + + credential_data = { + origin_type: :import, + private_data: pass, + private_type: :password, + username: user, + filename: filename + } + credential_data.merge!(service_data) + credential_core = create_credential(credential_data) + + login_data = { + core: credential_core, + status: Metasploit::Model::Login::Status::UNTRIED + } + + login_data.merge!(service_data) + + create_credential_login(login_data) + + # That's all we want to know from this service. + return :something_significant + end + end + end end From 27c5cf8887ba8ccc542018d973168870b3ce743b Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 13:57:27 -0500 Subject: [PATCH 155/159] Move helpers to Msf::DBManager::Import::Libpcap shared examples MSP-11124 --- spec/support/shared/examples/msf/db_manager/import.rb | 2 -- spec/support/shared/examples/msf/db_manager/import/libpcap.rb | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/support/shared/examples/msf/db_manager/import.rb b/spec/support/shared/examples/msf/db_manager/import.rb index ae590f6d2b..92281f5065 100644 --- a/spec/support/shared/examples/msf/db_manager/import.rb +++ b/spec/support/shared/examples/msf/db_manager/import.rb @@ -4,8 +4,6 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :import } it { is_expected.to respond_to :import_file } it { is_expected.to respond_to :import_filetype_detect } - it { is_expected.to respond_to :inspect_single_packet } - it { is_expected.to respond_to :inspect_single_packet_http } it { is_expected.to respond_to :msf_import_timestamps } it { is_expected.to respond_to :nils_for_nulls } it { is_expected.to respond_to :report_import_note } diff --git a/spec/support/shared/examples/msf/db_manager/import/libpcap.rb b/spec/support/shared/examples/msf/db_manager/import/libpcap.rb index 4674f4de98..941e42646c 100644 --- a/spec/support/shared/examples/msf/db_manager/import/libpcap.rb +++ b/spec/support/shared/examples/msf/db_manager/import/libpcap.rb @@ -1,4 +1,6 @@ shared_examples_for 'Msf::DBManager::Import::Libpcap' do it { is_expected.to respond_to :import_libpcap } it { is_expected.to respond_to :import_libpcap_file } + it { is_expected.to respond_to :inspect_single_packet } + it { is_expected.to respond_to :inspect_single_packet_http } end From bed98fe43bf7a7ffdd30305991464996a5d04a50 Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 13:59:03 -0500 Subject: [PATCH 156/159] nils_for_nulls -> Msf::DBManager::Import::MetasploitFramework MSP-11124 --- lib/msf/core/db_manager/import.rb | 5 ----- lib/msf/core/db_manager/import/metasploit_framework.rb | 5 +++++ 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index 2fa8940d9e..4be1b95361 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -366,11 +366,6 @@ module Msf::DBManager::Import return obj end - # Convert the string "NULL" to actual nil - def nils_for_nulls(str) - str == "NULL" ? nil : str - end - def report_import_note(wspace,addr) if @import_filedata.kind_of?(Hash) && @import_filedata[:filename] && @import_filedata[:filename] !~ /msfe-nmap[0-9]{8}/ report_note( diff --git a/lib/msf/core/db_manager/import/metasploit_framework.rb b/lib/msf/core/db_manager/import/metasploit_framework.rb index 89913c299d..2d31ca3f38 100644 --- a/lib/msf/core/db_manager/import/metasploit_framework.rb +++ b/lib/msf/core/db_manager/import/metasploit_framework.rb @@ -6,4 +6,9 @@ module Msf::DBManager::Import::MetasploitFramework include Msf::DBManager::Import::MetasploitFramework::Credential include Msf::DBManager::Import::MetasploitFramework::XML include Msf::DBManager::Import::MetasploitFramework::Zip + + # Convert the string "NULL" to actual nil + def nils_for_nulls(str) + str == "NULL" ? nil : str + end end \ No newline at end of file From 1aca55bc2a828cd47e6a04a88d39683385e36c6a Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 14:00:42 -0500 Subject: [PATCH 157/159] nils_for_nulls -> Msf::DBManager::Import::MetasploitFramework shared examples MSP-11124 --- spec/support/shared/examples/msf/db_manager/import.rb | 1 - .../examples/msf/db_manager/import/metasploit_framework.rb | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/spec/support/shared/examples/msf/db_manager/import.rb b/spec/support/shared/examples/msf/db_manager/import.rb index 92281f5065..8a0b5544e7 100644 --- a/spec/support/shared/examples/msf/db_manager/import.rb +++ b/spec/support/shared/examples/msf/db_manager/import.rb @@ -5,7 +5,6 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :import_file } it { is_expected.to respond_to :import_filetype_detect } it { is_expected.to respond_to :msf_import_timestamps } - it { is_expected.to respond_to :nils_for_nulls } it { is_expected.to respond_to :report_import_note } it { is_expected.to respond_to :rexmlify } it { is_expected.to respond_to :unserialize_object } diff --git a/spec/support/shared/examples/msf/db_manager/import/metasploit_framework.rb b/spec/support/shared/examples/msf/db_manager/import/metasploit_framework.rb index 2dc203f113..5261c813b2 100644 --- a/spec/support/shared/examples/msf/db_manager/import/metasploit_framework.rb +++ b/spec/support/shared/examples/msf/db_manager/import/metasploit_framework.rb @@ -1,4 +1,6 @@ shared_examples_for 'Msf::DBManager::Import::MetasploitFramework' do + it { is_expected.to respond_to :nils_for_nulls } + it_should_behave_like 'Msf::DBManager::Import::MetasploitFramework::Credential' it_should_behave_like 'Msf::DBManager::Import::MetasploitFramework::XML' it_should_behave_like 'Msf::DBManager::Import::MetasploitFramework::Zip' From 1f7ad1cac92e7dc9a372644a423192565067222a Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 14:03:18 -0500 Subject: [PATCH 158/159] unserialize_object -> Msf::DBManager::Import::MetasploitFramework MSP-11124 --- lib/msf/core/db_manager/import.rb | 31 ------------------- .../db_manager/import/metasploit_framework.rb | 31 +++++++++++++++++++ 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/lib/msf/core/db_manager/import.rb b/lib/msf/core/db_manager/import.rb index 4be1b95361..7dafd84b55 100644 --- a/lib/msf/core/db_manager/import.rb +++ b/lib/msf/core/db_manager/import.rb @@ -436,37 +436,6 @@ module Msf::DBManager::Import end end - def unserialize_object(xml_elem, allow_yaml = false) - return nil unless xml_elem - string = xml_elem.text.to_s.strip - return string unless string.is_a?(String) - return nil if (string.empty? || string.nil?) - - begin - # Validate that it is properly formed base64 first - if string.gsub(/\s+/, '') =~ /^([a-z0-9A-Z\+\/=]+)$/ - Marshal.load($1.unpack("m")[0]) - else - if allow_yaml - begin - YAML.load(string) - rescue - dlog("Badly formatted YAML: '#{string}'") - string - end - else - string - end - end - rescue ::Exception => e - if allow_yaml - YAML.load(string) rescue string - else - string - end - end - end - # Boils down the validate_import_file to a boolean def validate_import_file(data) begin diff --git a/lib/msf/core/db_manager/import/metasploit_framework.rb b/lib/msf/core/db_manager/import/metasploit_framework.rb index 2d31ca3f38..5e12fad48b 100644 --- a/lib/msf/core/db_manager/import/metasploit_framework.rb +++ b/lib/msf/core/db_manager/import/metasploit_framework.rb @@ -11,4 +11,35 @@ module Msf::DBManager::Import::MetasploitFramework def nils_for_nulls(str) str == "NULL" ? nil : str end + + def unserialize_object(xml_elem, allow_yaml = false) + return nil unless xml_elem + string = xml_elem.text.to_s.strip + return string unless string.is_a?(String) + return nil if (string.empty? || string.nil?) + + begin + # Validate that it is properly formed base64 first + if string.gsub(/\s+/, '') =~ /^([a-z0-9A-Z\+\/=]+)$/ + Marshal.load($1.unpack("m")[0]) + else + if allow_yaml + begin + YAML.load(string) + rescue + dlog("Badly formatted YAML: '#{string}'") + string + end + else + string + end + end + rescue ::Exception => e + if allow_yaml + YAML.load(string) rescue string + else + string + end + end + end end \ No newline at end of file From 3bce8e418d116af82975f695baab4f90f50fed1d Mon Sep 17 00:00:00 2001 From: Luke Imhoff Date: Wed, 15 Oct 2014 14:03:55 -0500 Subject: [PATCH 159/159] unserialize_object -> Msf::DBManager::Import::MetasploitFramework shared examples MSP-11124 --- spec/support/shared/examples/msf/db_manager/import.rb | 1 - .../examples/msf/db_manager/import/metasploit_framework.rb | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/support/shared/examples/msf/db_manager/import.rb b/spec/support/shared/examples/msf/db_manager/import.rb index 8a0b5544e7..9a1fd3f4f9 100644 --- a/spec/support/shared/examples/msf/db_manager/import.rb +++ b/spec/support/shared/examples/msf/db_manager/import.rb @@ -7,7 +7,6 @@ shared_examples_for 'Msf::DBManager::Import' do it { is_expected.to respond_to :msf_import_timestamps } it { is_expected.to respond_to :report_import_note } it { is_expected.to respond_to :rexmlify } - it { is_expected.to respond_to :unserialize_object } it { is_expected.to respond_to :validate_import_file } it_should_behave_like 'Msf::DBManager::Import::Acunetix' diff --git a/spec/support/shared/examples/msf/db_manager/import/metasploit_framework.rb b/spec/support/shared/examples/msf/db_manager/import/metasploit_framework.rb index 5261c813b2..3b6362dfec 100644 --- a/spec/support/shared/examples/msf/db_manager/import/metasploit_framework.rb +++ b/spec/support/shared/examples/msf/db_manager/import/metasploit_framework.rb @@ -1,5 +1,6 @@ shared_examples_for 'Msf::DBManager::Import::MetasploitFramework' do it { is_expected.to respond_to :nils_for_nulls } + it { is_expected.to respond_to :unserialize_object } it_should_behave_like 'Msf::DBManager::Import::MetasploitFramework::Credential' it_should_behave_like 'Msf::DBManager::Import::MetasploitFramework::XML'