Merge branch 'landing/4026' into upstream-master

Land #4026

* db.rb (DBManager) now in multiple files
* Cucumber coverage for DB-related msfconsole commands

lib/msf/core/database_event.rb

@@ -0,0 +1,34 @@
+# Events that can occur in the host/service database.
+module Msf::DatabaseEvent
+  # Called when a new client is added to the database.  The client
+  # parameter is of type Client.
+  def on_db_client(client)
+  end
+
+  # Called when a new host is added to the database.  The host parameter is
+  # of type Host.
+  def on_db_host(host)
+  end
+
+  # Called when an existing host's state changes
+  def on_db_host_state(host, ostate)
+  end
+
+  # Called when a new reference is created.
+  def on_db_ref(ref)
+  end
+
+  # Called when a new service is added to the database.  The service
+  # parameter is of type Service.
+  def on_db_service(service)
+  end
+
+  # Called when an existing service's state changes
+  def on_db_service_state(host, port, ostate)
+  end
+
+  # Called when an applicable vulnerability is found for a service.  The vuln
+  # parameter is of type Vuln.
+  def on_db_vuln(vuln)
+  end
+end

lib/msf/core/db_import_error.rb

@@ -0,0 +1,2 @@
+class Msf::DBImportError < RuntimeError
+end

lib/msf/core/db_manager.rb

@@ -1,74 +1,110 @@
 # -*- coding: binary -*-
 
+#
+# Gems
+#
+
+require 'rex/socket'
+
+#
+# Project
+#
+
+require 'metasploit/framework/require'
 require 'msf/base/config'
 require 'msf/core'
-require 'msf/core/db'
+require 'msf/core/database_event'
-require 'msf/core/db_manager/migration'
+require 'msf/core/db_import_error'
+require 'msf/core/host_state'
+require 'msf/core/service_state'
 require 'msf/core/task_manager'
-require 'fileutils'
-require 'shellwords'
 
-module Msf
-
-###
-#
 # The db module provides persistent storage and events. This class should be instantiated LAST
 # as the active_suppport library overrides Kernel.require, slowing down all future code loads.
-#
+class Msf::DBManager
-###
+  extend Metasploit::Framework::Require
 
-class DBManager
+  autoload :Adapter, 'msf/core/db_manager/adapter'
-  # Provides :framework and other accessors
+  autoload :Client, 'msf/core/db_manager/client'
+  autoload :Connection, 'msf/core/db_manager/connection'
+  autoload :Cred, 'msf/core/db_manager/cred'
+  autoload :Event, 'msf/core/db_manager/event'
+  autoload :ExploitAttempt, 'msf/core/db_manager/exploit_attempt'
+  autoload :ExploitedHost, 'msf/core/db_manager/exploited_host'
+  autoload :Host, 'msf/core/db_manager/host'
+  autoload :HostDetail, 'msf/core/db_manager/host_detail'
+  autoload :HostTag, 'msf/core/db_manager/host_tag'
+  autoload :Import, 'msf/core/db_manager/import'
+  autoload :ImportMsfXml, 'msf/core/db_manager/import_msf_xml'
+  autoload :IPAddress, 'msf/core/db_manager/ip_address'
+  autoload :Loot, 'msf/core/db_manager/loot'
+  autoload :Migration, 'msf/core/db_manager/migration'
+  autoload :ModuleCache, 'msf/core/db_manager/module_cache'
+  autoload :Note, 'msf/core/db_manager/note'
+  autoload :Ref, 'msf/core/db_manager/ref'
+  autoload :Report, 'msf/core/db_manager/report'
+  autoload :Route, 'msf/core/db_manager/route'
+  autoload :Service, 'msf/core/db_manager/service'
+  autoload :Session, 'msf/core/db_manager/session'
+  autoload :SessionEvent, 'msf/core/db_manager/session_event'
+  autoload :Sink, 'msf/core/db_manager/sink'
+  autoload :Task, 'msf/core/db_manager/task'
+  autoload :Vuln, 'msf/core/db_manager/vuln'
+  autoload :VulnAttempt, 'msf/core/db_manager/vuln_attempt'
+  autoload :VulnDetail, 'msf/core/db_manager/vuln_detail'
+  autoload :WMAP, 'msf/core/db_manager/wmap'
+  autoload :Web, 'msf/core/db_manager/web'
+  autoload :Workspace, 'msf/core/db_manager/workspace'
+
+  optionally_include_metasploit_credential_creation
+
+  include Msf::DBManager::Adapter
+  include Msf::DBManager::Client
+  include Msf::DBManager::Connection
+  include Msf::DBManager::Cred
+  include Msf::DBManager::Event
+  include Msf::DBManager::ExploitAttempt
+  include Msf::DBManager::ExploitedHost
+  include Msf::DBManager::Host
+  include Msf::DBManager::HostDetail
+  include Msf::DBManager::HostTag
+  include Msf::DBManager::Import
+  include Msf::DBManager::IPAddress
+  include Msf::DBManager::Loot
   include Msf::DBManager::Migration
+  include Msf::DBManager::ModuleCache
+  include Msf::DBManager::Note
+  include Msf::DBManager::Ref
+  include Msf::DBManager::Report
+  include Msf::DBManager::Route
+  include Msf::DBManager::Service
+  include Msf::DBManager::Session
+  include Msf::DBManager::SessionEvent
+  include Msf::DBManager::Sink
+  include Msf::DBManager::Task
+  include Msf::DBManager::Vuln
+  include Msf::DBManager::VulnAttempt
+  include Msf::DBManager::VulnDetail
+  include Msf::DBManager::WMAP
+  include Msf::DBManager::Web
+  include Msf::DBManager::Workspace
+
+  # Provides :framework and other accessors
   include Msf::Framework::Offspring
 
   #
-  # CONSTANTS
+  # Attributes
   #
 
-  # The adapter to use to establish database connection.
-  ADAPTER = 'postgresql'
-
-  # Mainly, it's Ruby 1.9.1 that cause a lot of problems now, along with Ruby 1.8.6.
-  # Ruby 1.8.7 actually seems okay, but why tempt fate? Let's say 1.9.3 and beyond.
-  def warn_about_rubies
-    if ::RUBY_VERSION =~ /^1\.9\.[012]($|[^\d])/
-      $stderr.puts "**************************************************************************************"
-      $stderr.puts "Metasploit requires at least Ruby 1.9.3. For an easy upgrade path, see https://rvm.io/"
-      $stderr.puts "**************************************************************************************"
-    end
-  end
-
-  # Returns true if we are ready to load/store data
-  def active
-    # usable and migrated a just Boolean attributes, so check those first because they don't actually contact the
-    # database.
-    usable && migrated && connection_established?
-  end
-
-  # Returns true if the prerequisites have been installed
-  attr_accessor :usable
-
-  # Returns the list of usable database drivers
-  def drivers
-    @drivers ||= []
-  end
-  attr_writer :drivers
-
-  # Returns the active driver
-  attr_accessor :driver
-
   # Stores the error message for why the db was not loaded
   attr_accessor :error
 
-  # Stores a TaskManager for serializing database events
+  # Returns true if the prerequisites have been installed
-  attr_accessor :sink
+  attr_accessor :usable
 
-  # Flag to indicate that modules are cached
+  #
-  attr_accessor :modules_cached
+  # iniitialize
-
+  #
-  # Flag to indicate that the module cacher is running
-  attr_accessor :modules_caching
 
   def initialize(framework, opts = {})
 
@@ -88,6 +124,19 @@ class DBManager
     initialize_database_support
   end
 
+  #
+  # Instance Methods
+  #
+
+  #
+  # Determines if the database is functional
+  #
+  def check
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    res = ::Mdm::Host.find(:first)
+  }
+  end
+
   #
   # Do what is necessary to load our database support
   #
@@ -119,613 +168,13 @@ class DBManager
     true
   end
 
-  # Checks if the spec passed to `ActiveRecord::Base.establish_connection` can connect to the database.
+  # Mainly, it's Ruby 1.9.1 that cause a lot of problems now, along with Ruby 1.8.6.
-  #
+  # Ruby 1.8.7 actually seems okay, but why tempt fate? Let's say 1.9.3 and beyond.
-  # @return [true] if an active connection can be made to the database using the current config.
+  def warn_about_rubies
-  # @return [false] if an active connection cannot be made to the database.
+    if ::RUBY_VERSION =~ /^1\.9\.[012]($|[^\d])/
-  def connection_established?
+      $stderr.puts "**************************************************************************************"
-    begin
+      $stderr.puts "Metasploit requires at least Ruby 1.9.3. For an easy upgrade path, see https://rvm.io/"
-      # use with_connection so the connection doesn't stay pinned to the thread.
+      $stderr.puts "**************************************************************************************"
-      ActiveRecord::Base.connection_pool.with_connection {
-        ActiveRecord::Base.connection.active?
-      }
-    rescue ActiveRecord::ConnectionNotEstablished, PG::ConnectionBad => error
-      elog("Connection not established: #{error.class} #{error}:\n#{error.backtrace.join("\n")}")
-
-      false
     end
   end
-
-  #
-  # Scan through available drivers
-  #
-  def initialize_adapter
-    ActiveRecord::Base.default_timezone = :utc
-
-    if connection_established? && ActiveRecord::Base.connection_config[:adapter] == ADAPTER
-      dlog("Already established connection to #{ADAPTER}, so reusing active connection.")
-      self.drivers << ADAPTER
-      self.driver = ADAPTER
-    else
-      begin
-        ActiveRecord::Base.establish_connection(adapter: ADAPTER)
-        ActiveRecord::Base.remove_connection
-      rescue Exception => error
-        @adapter_error = error
-      else
-        self.drivers << ADAPTER
-        self.driver = ADAPTER
-      end
-    end
-  end
-
-  # Loads Metasploit Data Models and adds its migrations to migrations paths.
-  #
-  # @return [void]
-  def add_rails_engine_migration_paths
-    unless defined? ActiveRecord
-      fail "Bundle installed '--without #{Bundler.settings.without.join(' ')}'.  To clear the without option do " \
-           "`bundle install --without ''` (the --without flag with an empty string) or `rm -rf .bundle` to remove " \
-           "the .bundle/config manually and then `bundle install`"
-    end
-
-    Rails.application.railties.engines.each do |engine|
-      migrations_paths = engine.paths['db/migrate'].existent_directories
-
-      migrations_paths.each do |migrations_path|
-        # Since ActiveRecord::Migrator.migrations_paths can persist between
-        # instances of Msf::DBManager, such as in specs,
-        # migrations_path may already be part of
-        # migrations_paths, in which case it should not be added or multiple
-        # migrations with the same version number errors will occur.
-        unless ActiveRecord::Migrator.migrations_paths.include? migrations_path
-          ActiveRecord::Migrator.migrations_paths << migrations_path
-        end
-      end
-    end
-  end
-
-  #
-  # Create a new database sink and initialize it
-  #
-  def initialize_sink
-    self.sink = TaskManager.new(framework)
-    self.sink.start
-  end
-
-  #
-  # Add a new task to the sink
-  #
-  def queue(proc)
-    self.sink.queue_proc(proc)
-  end
-
-  #
-  # Connects this instance to a database
-  #
-  def connect(opts={})
-
-    return false if not @usable
-
-    nopts = opts.dup
-    if (nopts['port'])
-      nopts['port'] = nopts['port'].to_i
-    end
-
-    # Prefer the config file's pool setting
-    nopts['pool'] ||= 75
-
-    # Prefer the config file's wait_timeout setting too
-    nopts['wait_timeout'] ||= 300
-
-    begin
-      self.migrated = false
-
-      # Check ActiveRecord::Base was already connected by Rails::Application.initialize! or some other API.
-      unless connection_established?
-        create_db(nopts)
-
-        # Configure the database adapter
-        ActiveRecord::Base.establish_connection(nopts)
-      end
-    rescue ::Exception => e
-      self.error = e
-      elog("DB.connect threw an exception: #{e}")
-      dlog("Call stack: #{$@.join"\n"}", LEV_1)
-      return false
-    ensure
-      after_establish_connection
-
-      # Database drivers can reset our KCODE, do not let them
-      $KCODE = 'NONE' if RUBY_VERSION =~ /^1\.8\./
-    end
-
-    true
-  end
-
-  # Finishes {#connect} after `ActiveRecord::Base.establish_connection` has succeeded by {#migrate migrating database}
-  # and setting {#workspace}.
-  #
-  # @return [void]
-  def after_establish_connection
-    self.migrated = false
-
-    begin
-      # Migrate the database, if needed
-      migrate
-
-      # Set the default workspace
-      framework.db.workspace = framework.db.default_workspace
-    rescue ::Exception => exception
-      self.error = exception
-      elog("DB.connect threw an exception: #{exception}")
-      dlog("Call stack: #{exception.backtrace.join("\n")}", LEV_1)
-    else
-      # Flag that migration has completed
-      self.migrated = true
-    end
-  end
-
-  #
-  # Attempt to create the database
-  #
-  # If the database already exists this will fail and we will continue on our
-  # merry way, connecting anyway.  If it doesn't, we try to create it.  If
-  # that fails, then it wasn't meant to be and the connect will raise a
-  # useful exception so the user won't be in the dark; no need to raise
-  # anything at all here.
-  #
-  def create_db(opts)
-    begin
-      case opts["adapter"]
-      when 'postgresql'
-        # Try to force a connection to be made to the database, if it succeeds
-        # then we know we don't need to create it :)
-        ActiveRecord::Base.establish_connection(opts)
-        # Do the checkout, checkin dance here to make sure this thread doesn't
-        # hold on to a connection we don't need
-        conn = ActiveRecord::Base.connection_pool.checkout
-        ActiveRecord::Base.connection_pool.checkin(conn)
-      end
-    rescue ::Exception => e
-      errstr = e.to_s
-      if errstr =~ /does not exist/i or errstr =~ /Unknown database/
-        ilog("Database doesn't exist \"#{opts['database']}\", attempting to create it.")
-        ActiveRecord::Base.establish_connection(
-            opts.merge(
-                'database' => 'postgres',
-                'schema_search_path' => 'public'
-            )
-        )
-
-        ActiveRecord::Base.connection.create_database(opts['database'])
-      else
-        ilog("Trying to continue despite failed database creation: #{e}")
-      end
-    end
-    ActiveRecord::Base.remove_connection
-  end
-
-  #
-  # Disconnects a database session
-  #
-  def disconnect
-    begin
-      ActiveRecord::Base.remove_connection
-      self.migrated = false
-      self.modules_cached = false
-    rescue ::Exception => e
-      self.error = e
-      elog("DB.disconnect threw an exception: #{e}")
-    ensure
-      # Database drivers can reset our KCODE, do not let them
-      $KCODE = 'NONE' if RUBY_VERSION =~ /^1\.8\./
-    end
-  end
-
-  def workspace=(workspace)
-    @workspace_name = workspace.name
-  end
-
-  def workspace
-    framework.db.find_workspace(@workspace_name)
-  end
-
-
-  # @note Does nothing unless {#migrated} is +true+ and {#modules_caching} is
-  #   +false+.
-  #
-  # Destroys all Mdm::Module::Details in the database.
-  #
-  # @return [void]
-  def purge_all_module_details
-    return if not self.migrated
-    return if self.modules_caching
-
-    ::ActiveRecord::Base.connection_pool.with_connection do
-      Mdm::Module::Detail.destroy_all
-    end
-  end
-
-  # Destroys the old Mdm::Module::Detail and creates a new Mdm::Module::Detail for
-  # any module with an Mdm::Module::Detail where the modification time of the
-  # Mdm::Module::Detail#file differs from the Mdm::Module::Detail#mtime.  If the
-  # Mdm::Module::Detail#file no only exists on disk, then the Mdm::Module::Detail
-  # is just destroyed without a new one being created.
-  #
-  # @return [void]
-  def update_all_module_details
-    return if not self.migrated
-    return if self.modules_caching
-
-    self.framework.cache_thread = Thread.current
-
-    self.modules_cached  = false
-    self.modules_caching = true
-
-    ActiveRecord::Base.connection_pool.with_connection do
-
-      refresh = []
-      skip_reference_name_set_by_module_type = Hash.new { |hash, module_type|
-        hash[module_type] = Set.new
-      }
-
-      Mdm::Module::Detail.find_each do |md|
-
-        unless md.ready
-          refresh << md
-          next
-        end
-
-        unless md.file and ::File.exists?(md.file)
-          refresh << md
-          next
-        end
-
-        if ::File.mtime(md.file).to_i != md.mtime.to_i
-          refresh << md
-          next
-        end
-
-        skip_reference_name_set = skip_reference_name_set_by_module_type[md.mtype]
-        skip_reference_name_set.add(md.refname)
-      end
-
-      refresh.each { |md| md.destroy }
-
-      [
-          ['exploit', framework.exploits],
-          ['auxiliary', framework.auxiliary],
-          ['post', framework.post],
-          ['payload', framework.payloads],
-          ['encoder', framework.encoders],
-          ['nop', framework.nops]
-      ].each do |mt|
-        skip_reference_name_set = skip_reference_name_set_by_module_type[mt[0]]
-
-        mt[1].keys.sort.each do |mn|
-          next if skip_reference_name_set.include? mn
-          obj = mt[1].create(mn)
-          next if not obj
-          begin
-            update_module_details(obj)
-          rescue ::Exception
-            elog("Error updating module details for #{obj.fullname}: #{$!.class} #{$!}")
-          end
-        end
-      end
-
-      self.framework.cache_initialized = true
-    end
-
-    # in reverse order of section before with_connection block
-    self.modules_caching = false
-    self.modules_cached  = true
-    self.framework.cache_thread = nil
-  end
-
-  # Creates an Mdm::Module::Detail from a module instance.
-  #
-  # @param module_instance [Msf::Module] a metasploit module instance.
-  # @raise [ActiveRecord::RecordInvalid] if Hash from {#module_to_details_hash} is invalid attributes for
-  #   Mdm::Module::Detail.
-  # @return [void]
-  def update_module_details(module_instance)
-    return if not self.migrated
-
-    ActiveRecord::Base.connection_pool.with_connection do
-      info = module_to_details_hash(module_instance)
-      bits = info.delete(:bits) || []
-      module_detail = Mdm::Module::Detail.create!(info)
-
-      bits.each do |args|
-        otype, vals = args
-
-        case otype
-          when :action
-            module_detail.add_action(vals[:name])
-          when :arch
-            module_detail.add_arch(vals[:name])
-          when :author
-            module_detail.add_author(vals[:name], vals[:email])
-          when :platform
-            module_detail.add_platform(vals[:name])
-          when :ref
-            module_detail.add_ref(vals[:name])
-          when :target
-            module_detail.add_target(vals[:index], vals[:name])
-        end
-      end
-
-      module_detail.ready = true
-      module_detail.save!
-    end
-  end
-
-  # Destroys Mdm::Module::Detail if one exists for the given
-  # Mdm::Module::Detail#mtype and Mdm::Module::Detail#refname.
-  #
-  # @param mtype [String] module type.
-  # @param refname [String] module reference name.
-  # @return [void]
-  def remove_module_details(mtype, refname)
-    return if not self.migrated
-
-    ActiveRecord::Base.connection_pool.with_connection do
-      Mdm::Module::Detail.where(:mtype => mtype, :refname => refname).destroy_all
-    end
-  end
-
-  def module_to_details_hash(m)
-    res  = {}
-    bits = []
-
-    res[:mtime]    = ::File.mtime(m.file_path) rescue Time.now
-    res[:file]     = m.file_path
-    res[:mtype]    = m.type
-    res[:name]     = m.name.to_s
-    res[:refname]  = m.refname
-    res[:fullname] = m.fullname
-    res[:rank]     = m.rank.to_i
-    res[:license]  = m.license.to_s
-
-    res[:description] = m.description.to_s.strip
-
-    m.arch.map{ |x|
-      bits << [ :arch, { :name => x.to_s } ]
-    }
-
-    m.platform.platforms.map{ |x|
-      bits << [ :platform, { :name => x.to_s.split('::').last.downcase } ]
-    }
-
-    m.author.map{|x|
-      bits << [ :author, { :name => x.to_s } ]
-    }
-
-    m.references.map do |r|
-      bits << [ :ref, { :name => [r.ctx_id.to_s, r.ctx_val.to_s].join("-") } ]
-    end
-
-    res[:privileged] = m.privileged?
-
-
-    if m.disclosure_date
-      begin
-        res[:disclosure_date] = m.disclosure_date.to_datetime.to_time
-      rescue ::Exception
-        res.delete(:disclosure_date)
-      end
-    end
-
-    if(m.type == "exploit")
-
-      m.targets.each_index do |i|
-        bits << [ :target, { :index => i, :name => m.targets[i].name.to_s } ]
-        if m.targets[i].platform
-          m.targets[i].platform.platforms.each do |name|
-            bits << [ :platform, { :name => name.to_s.split('::').last.downcase } ]
-          end
-        end
-        if m.targets[i].arch
-          bits << [ :arch, { :name => m.targets[i].arch.to_s } ]
-        end
-      end
-
-      if (m.default_target)
-        res[:default_target] = m.default_target
-      end
-
-      # Some modules are a combination, which means they are actually aggressive
-      res[:stance] = m.stance.to_s.index("aggressive") ? "aggressive" : "passive"
-
-
-      m.class.mixins.each do |x|
-         bits << [ :mixin, { :name => x.to_s } ]
-      end
-    end
-
-    if(m.type == "auxiliary")
-
-      m.actions.each_index do |i|
-        bits << [ :action, { :name => m.actions[i].name.to_s } ]
-      end
-
-      if (m.default_action)
-        res[:default_action] = m.default_action.to_s
-      end
-
-      res[:stance] = m.passive? ? "passive" : "aggressive"
-    end
-
-    res[:bits] = bits.uniq
-
-    res
-  end
-
-  # Wraps values in +'%'+ for Arel::Prediciation#matches_any and other match* methods that map to SQL +'LIKE'+ or
-  # +'ILIKE'+
-  #
-  # @param values [Set<String>, #each] a list of strings.
-  # @return [Arrray<String>] strings wrapped like %<string>%
-  def match_values(values)
-    wrapped_values = values.collect { |value|
-      "%#{value}%"
-    }
-
-    wrapped_values
-  end
-
-  # This provides a standard set of search filters for every module.
-  #
-  # Supported keywords with the format <keyword>:<search_value>:
-  # +app+:: If +client+ then matches +'passive'+ stance modules, otherwise matches +'active' stance modules.
-  # +author+:: Matches modules with the given author email or name.
-  # +bid+:: Matches modules with the given Bugtraq ID.
-  # +cve+:: Matches modules with the given CVE ID.
-  # +edb+:: Matches modules with the given Exploit-DB ID.
-  # +name+:: Matches modules with the given full name or name.
-  # +os+, +platform+:: Matches modules with the given platform or target name.
-  # +osvdb+:: Matches modules with the given OSVDB ID.
-  # +ref+:: Matches modules with the given reference ID.
-  # +type+:: Matches modules with the given type.
-  #
-  # Any text not associated with a keyword is matched against the description,
-  # the full name, and the name of the module; the name of the module actions;
-  # the name of the module archs; the name of the module authors; the name of
-  # module platform; the module refs; or the module target.
-  #
-  # @param search_string [String] a string of space separated keyword pairs or
-  #   free form text.
-  # @return [[]] if search_string is +nil+
-  # @return [ActiveRecord::Relation] module details that matched
-  #   +search_string+
-  def search_modules(search_string)
-    search_string ||= ''
-    search_string += " "
-
-    # Split search terms by space, but allow quoted strings
-    terms = Shellwords.shellwords(search_string)
-    terms.delete('')
-
-    # All terms are either included or excluded
-    value_set_by_keyword = Hash.new { |hash, keyword|
-      hash[keyword] = Set.new
-    }
-
-    terms.each do |term|
-      keyword, value = term.split(':', 2)
-
-      unless value
-        value = keyword
-        keyword = 'text'
-      end
-
-      unless value.empty?
-        keyword.downcase!
-
-        value_set = value_set_by_keyword[keyword]
-        value_set.add value
-      end
-    end
-
-    query = Mdm::Module::Detail.scoped
-
-    ActiveRecord::Base.connection_pool.with_connection do
-      # Although AREL supports taking the union or two queries, the ActiveRecord where syntax only supports
-      # intersection, so creating the where clause has to be delayed until all conditions can be or'd together and
-      # passed to one call ot where.
-      union_conditions = []
-
-      value_set_by_keyword.each do |keyword, value_set|
-        case keyword
-          when 'author'
-            formatted_values = match_values(value_set)
-
-            query = query.includes(:authors)
-            module_authors = Mdm::Module::Author.arel_table
-            union_conditions << module_authors[:email].matches_any(formatted_values)
-            union_conditions << module_authors[:name].matches_any(formatted_values)
-          when 'name'
-            formatted_values = match_values(value_set)
-
-            module_details = Mdm::Module::Detail.arel_table
-            union_conditions << module_details[:fullname].matches_any(formatted_values)
-            union_conditions << module_details[:name].matches_any(formatted_values)
-          when 'os', 'platform'
-            formatted_values = match_values(value_set)
-
-            query = query.includes(:platforms)
-            union_conditions << Mdm::Module::Platform.arel_table[:name].matches_any(formatted_values)
-
-            query = query.includes(:targets)
-            union_conditions << Mdm::Module::Target.arel_table[:name].matches_any(formatted_values)
-          when 'text'
-            formatted_values = match_values(value_set)
-
-            module_details = Mdm::Module::Detail.arel_table
-            union_conditions << module_details[:description].matches_any(formatted_values)
-            union_conditions << module_details[:fullname].matches_any(formatted_values)
-            union_conditions << module_details[:name].matches_any(formatted_values)
-
-            query = query.includes(:actions)
-            union_conditions << Mdm::Module::Action.arel_table[:name].matches_any(formatted_values)
-
-            query = query.includes(:archs)
-            union_conditions << Mdm::Module::Arch.arel_table[:name].matches_any(formatted_values)
-
-            query = query.includes(:authors)
-            union_conditions << Mdm::Module::Author.arel_table[:name].matches_any(formatted_values)
-
-            query = query.includes(:platforms)
-            union_conditions << Mdm::Module::Platform.arel_table[:name].matches_any(formatted_values)
-
-            query = query.includes(:refs)
-            union_conditions << Mdm::Module::Ref.arel_table[:name].matches_any(formatted_values)
-
-            query = query.includes(:targets)
-            union_conditions << Mdm::Module::Target.arel_table[:name].matches_any(formatted_values)
-          when 'type'
-            formatted_values = match_values(value_set)
-            union_conditions << Mdm::Module::Detail.arel_table[:mtype].matches_any(formatted_values)
-          when 'app'
-            formatted_values = value_set.collect { |value|
-              formatted_value = 'aggressive'
-
-              if value == 'client'
-                formatted_value = 'passive'
-              end
-
-              formatted_value
-            }
-
-            union_conditions << Mdm::Module::Detail.arel_table[:stance].eq_any(formatted_values)
-          when 'ref'
-            formatted_values = match_values(value_set)
-
-            query = query.includes(:refs)
-            union_conditions << Mdm::Module::Ref.arel_table[:name].matches_any(formatted_values)
-          when 'cve', 'bid', 'osvdb', 'edb'
-            formatted_values = value_set.collect { |value|
-              prefix = keyword.upcase
-
-              "#{prefix}-%#{value}%"
-            }
-
-            query = query.includes(:refs)
-            union_conditions << Mdm::Module::Ref.arel_table[:name].matches_any(formatted_values)
-        end
-      end
-
-      unioned_conditions = union_conditions.inject { |union, condition|
-        union.or(condition)
-      }
-
-      query = query.where(unioned_conditions).to_a.uniq { |m| m.fullname }
-    end
-
-    query
-  end
-
-end
 end

lib/msf/core/db_manager/adapter.rb

@@ -0,0 +1,48 @@
+module Msf::DBManager::Adapter
+  #
+  # CONSTANTS
+  #
+
+  # The adapter to use to establish database connection.
+  ADAPTER = 'postgresql'
+
+  #
+  # Attributes
+  #
+
+  # Returns the list of usable database drivers
+  def drivers
+    @drivers ||= []
+  end
+  attr_writer :drivers
+
+  # Returns the active driver
+  attr_accessor :driver
+
+  #
+  # Instance Methods
+  #
+
+  #
+  # Scan through available drivers
+  #
+  def initialize_adapter
+    ActiveRecord::Base.default_timezone = :utc
+
+    if connection_established? && ActiveRecord::Base.connection_config[:adapter] == ADAPTER
+      dlog("Already established connection to #{ADAPTER}, so reusing active connection.")
+      self.drivers << ADAPTER
+      self.driver = ADAPTER
+    else
+      begin
+        ActiveRecord::Base.establish_connection(adapter: ADAPTER)
+        ActiveRecord::Base.remove_connection
+      rescue Exception => error
+        @adapter_error = error
+      else
+        self.drivers << ADAPTER
+        self.driver = ADAPTER
+      end
+    end
+  end
+end

lib/msf/core/db_manager/client.rb

@@ -0,0 +1,66 @@
+module Msf::DBManager::Client
+  def find_or_create_client(opts)
+    report_client(opts)
+  end
+
+  def get_client(opts)
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    wspace = opts.delete(:workspace) || workspace
+    host   = get_host(:workspace => wspace, :host => opts[:host]) || return
+    client = host.clients.where({:ua_string => opts[:ua_string]}).first()
+    return client
+  }
+  end
+
+  #
+  # Report a client running on a host.
+  #
+  # opts MUST contain
+  # +:ua_string+::  the value of the User-Agent header
+  # +:host+::       the host where this client connected from, can be an ip address or a Host object
+  #
+  # opts can contain
+  # +:ua_name+::    one of the Msf::HttpClients constants
+  # +:ua_ver+::     detected version of the given client
+  # +:campaign+::   an id or Campaign object
+  #
+  # Returns a Client.
+  #
+  def report_client(opts)
+    return if not active
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    addr = opts.delete(:host) || return
+    wspace = opts.delete(:workspace) || workspace
+    report_host(:workspace => wspace, :host => addr)
+
+    ret = {}
+
+    host = get_host(:workspace => wspace, :host => addr)
+    client = host.clients.find_or_initialize_by_ua_string(opts[:ua_string])
+
+    opts[:ua_string] = opts[:ua_string].to_s
+
+    campaign = opts.delete(:campaign)
+    if campaign
+      case campaign
+      when Campaign
+        opts[:campaign_id] = campaign.id
+      else
+        opts[:campaign_id] = campaign
+      end
+    end
+
+    opts.each { |k,v|
+      if (client.attribute_names.include?(k.to_s))
+        client[k] = v
+      else
+        dlog("Unknown attribute for Client: #{k}")
+      end
+    }
+    if (client and client.changed?)
+      client.save!
+    end
+    ret[:client] = client
+  }
+  end
+end

lib/msf/core/db_manager/connection.rb

@@ -0,0 +1,149 @@
+module Msf::DBManager::Connection
+  # Returns true if we are ready to load/store data
+  def active
+    # usable and migrated a just Boolean attributes, so check those first because they don't actually contact the
+    # database.
+    usable && migrated && connection_established?
+  end
+
+  # Finishes {#connect} after `ActiveRecord::Base.establish_connection` has succeeded by {#migrate migrating database}
+  # and setting {#workspace}.
+  #
+  # @return [void]
+  def after_establish_connection
+    self.migrated = false
+
+    begin
+      # Migrate the database, if needed
+      migrate
+
+      # Set the default workspace
+      framework.db.workspace = framework.db.default_workspace
+    rescue ::Exception => exception
+      self.error = exception
+      elog("DB.connect threw an exception: #{exception}")
+      dlog("Call stack: #{exception.backtrace.join("\n")}", LEV_1)
+    else
+      # Flag that migration has completed
+      self.migrated = true
+    end
+  end
+
+  #
+  # Connects this instance to a database
+  #
+  def connect(opts={})
+
+    return false if not @usable
+
+    nopts = opts.dup
+    if (nopts['port'])
+      nopts['port'] = nopts['port'].to_i
+    end
+
+    # Prefer the config file's pool setting
+    nopts['pool'] ||= 75
+
+    # Prefer the config file's wait_timeout setting too
+    nopts['wait_timeout'] ||= 300
+
+    begin
+      self.migrated = false
+
+      # Check ActiveRecord::Base was already connected by Rails::Application.initialize! or some other API.
+      unless connection_established?
+        create_db(nopts)
+
+        # Configure the database adapter
+        ActiveRecord::Base.establish_connection(nopts)
+      end
+    rescue ::Exception => e
+      self.error = e
+      elog("DB.connect threw an exception: #{e}")
+      dlog("Call stack: #{$@.join"\n"}", LEV_1)
+      return false
+    ensure
+      after_establish_connection
+
+      # Database drivers can reset our KCODE, do not let them
+      $KCODE = 'NONE' if RUBY_VERSION =~ /^1\.8\./
+    end
+
+    true
+  end
+
+
+  #
+  # Attempt to create the database
+  #
+  # If the database already exists this will fail and we will continue on our
+  # merry way, connecting anyway.  If it doesn't, we try to create it.  If
+  # that fails, then it wasn't meant to be and the connect will raise a
+  # useful exception so the user won't be in the dark; no need to raise
+  # anything at all here.
+  #
+  def create_db(opts)
+    begin
+      case opts["adapter"]
+      when 'postgresql'
+        # Try to force a connection to be made to the database, if it succeeds
+        # then we know we don't need to create it :)
+        ActiveRecord::Base.establish_connection(opts)
+        # Do the checkout, checkin dance here to make sure this thread doesn't
+        # hold on to a connection we don't need
+        conn = ActiveRecord::Base.connection_pool.checkout
+        ActiveRecord::Base.connection_pool.checkin(conn)
+      end
+    rescue ::Exception => e
+      errstr = e.to_s
+      if errstr =~ /does not exist/i or errstr =~ /Unknown database/
+        ilog("Database doesn't exist \"#{opts['database']}\", attempting to create it.")
+        ActiveRecord::Base.establish_connection(
+            opts.merge(
+                'database' => 'postgres',
+                'schema_search_path' => 'public'
+            )
+        )
+
+        ActiveRecord::Base.connection.create_database(opts['database'])
+      else
+        ilog("Trying to continue despite failed database creation: #{e}")
+      end
+    end
+    ActiveRecord::Base.remove_connection
+  end
+
+  # Checks if the spec passed to `ActiveRecord::Base.establish_connection` can connect to the database.
+  #
+  # @return [true] if an active connection can be made to the database using the current config.
+  # @return [false] if an active connection cannot be made to the database.
+  def connection_established?
+    begin
+      # use with_connection so the connection doesn't stay pinned to the thread.
+      ActiveRecord::Base.connection_pool.with_connection {
+        ActiveRecord::Base.connection.active?
+      }
+    rescue ActiveRecord::ConnectionNotEstablished, PG::ConnectionBad => error
+      elog("Connection not established: #{error.class} #{error}:\n#{error.backtrace.join("\n")}")
+
+      false
+    end
+  end
+
+  #
+  # Disconnects a database session
+  #
+  def disconnect
+    begin
+      ActiveRecord::Base.remove_connection
+      self.migrated = false
+      self.modules_cached = false
+    rescue ::Exception => e
+      self.error = e
+      elog("DB.disconnect threw an exception: #{e}")
+    ensure
+      # Database drivers can reset our KCODE, do not let them
+      $KCODE = 'NONE' if RUBY_VERSION =~ /^1\.8\./
+    end
+  end
+end

lib/msf/core/db_manager/cred.rb

@@ -0,0 +1,180 @@
+module Msf::DBManager::Cred
+  # This methods returns a list of all credentials in the database
+  def creds(wspace=workspace)
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    Mdm::Cred.includes({:service => :host}).where("hosts.workspace_id = ?", wspace.id)
+  }
+  end
+
+  # This method iterates the creds table calling the supplied block with the
+  # cred instance of each entry.
+  def each_cred(wspace=workspace,&block)
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    wspace.creds.each do |cred|
+      block.call(cred)
+    end
+  }
+  end
+
+  # Find or create a credential matching this type/data
+  def find_or_create_cred(opts)
+    report_auth_info(opts)
+  end
+
+  #
+  # Store a set of credentials in the database.
+  #
+  # report_auth_info used to create a note, now it creates
+  # an entry in the creds table. It's much more akin to
+  # report_vuln() now.
+  #
+  # opts MUST contain
+  # +:host+::   an IP address or Host object reference
+  # +:port+::   a port number
+  #
+  # opts can contain
+  # +:user+::   the username
+  # +:pass+::   the password, or path to ssh_key
+  # +:ptype+::  the type of password (password(ish), hash, or ssh_key)
+  # +:proto+::  a transport name for the port
+  # +:sname+::  service name
+  # +:active+:: by default, a cred is active, unless explicitly false
+  # +:proof+::  data used to prove the account is actually active.
+  #
+  # Sources: Credentials can be sourced from another credential, or from
+  # a vulnerability. For example, if an exploit was used to dump the
+  # smb_hashes, and this credential comes from there, the source_id would
+  # be the Vuln id (as reported by report_vuln) and the type would be "Vuln".
+  #
+  # +:source_id+::   The Vuln or Cred id of the source of this cred.
+  # +:source_type+:: Either Vuln or Cred
+  #
+  # TODO: This is written somewhat host-centric, when really the
+  # Service is the thing. Need to revisit someday.
+  def report_auth_info(opts={})
+    return if not active
+    raise ArgumentError.new("Missing required option :host") if opts[:host].nil?
+    raise ArgumentError.new("Missing required option :port") if (opts[:port].nil? and opts[:service].nil?)
+
+    if (not opts[:host].kind_of?(::Mdm::Host)) and (not validate_ips(opts[:host]))
+      raise ArgumentError.new("Invalid address or object for :host (#{opts[:host].inspect})")
+    end
+
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    host = opts.delete(:host)
+    ptype = opts.delete(:type) || "password"
+    token = [opts.delete(:user), opts.delete(:pass)]
+    sname = opts.delete(:sname)
+    port = opts.delete(:port)
+    proto = opts.delete(:proto) || "tcp"
+    proof = opts.delete(:proof)
+    source_id = opts.delete(:source_id)
+    source_type = opts.delete(:source_type)
+    duplicate_ok = opts.delete(:duplicate_ok)
+    # Nil is true for active.
+    active = (opts[:active] || opts[:active].nil?) ? true : false
+
+    wspace = opts.delete(:workspace) || workspace
+
+    # Service management; assume the user knows what
+    # he's talking about.
+    service = opts.delete(:service) || report_service(:host => host, :port => port, :proto => proto, :name => sname, :workspace => wspace)
+
+    # Non-US-ASCII usernames are tripping up the database at the moment, this is a temporary fix until we update the tables
+    if (token[0])
+      # convert the token to US-ASCII from UTF-8 to prevent an error
+      token[0] = token[0].unpack("C*").pack("C*")
+      token[0] = token[0].gsub(/[\x00-\x1f\x7f-\xff]/n){|m| "\\x%.2x" % m.unpack("C")[0] }
+    end
+
+    if (token[1])
+      token[1] = token[1].unpack("C*").pack("C*")
+      token[1] = token[1].gsub(/[\x00-\x1f\x7f-\xff]/n){|m| "\\x%.2x" % m.unpack("C")[0] }
+    end
+
+    ret = {}
+
+    # Check to see if the creds already exist. We look also for a downcased username with the
+    # same password because we can fairly safely assume they are not in fact two seperate creds.
+    # this allows us to hedge against duplication of creds in the DB.
+
+    if duplicate_ok
+    # If duplicate usernames are okay, find by both user and password (allows
+    # for actual duplicates to get modified updated_at, sources, etc)
+      if token[0].nil? or token[0].empty?
+        cred = service.creds.find_or_initialize_by_user_and_ptype_and_pass(token[0] || "", ptype, token[1] || "")
+      else
+        cred = service.creds.find_by_user_and_ptype_and_pass(token[0] || "", ptype, token[1] || "")
+        unless cred
+          dcu = token[0].downcase
+          cred = service.creds.find_by_user_and_ptype_and_pass( dcu || "", ptype, token[1] || "")
+          unless cred
+            cred = service.creds.find_or_initialize_by_user_and_ptype_and_pass(token[0] || "", ptype, token[1] || "")
+          end
+        end
+      end
+    else
+      # Create the cred by username only (so we can change passwords)
+      if token[0].nil? or token[0].empty?
+        cred = service.creds.find_or_initialize_by_user_and_ptype(token[0] || "", ptype)
+      else
+        cred = service.creds.find_by_user_and_ptype(token[0] || "", ptype)
+        unless cred
+          dcu = token[0].downcase
+          cred = service.creds.find_by_user_and_ptype_and_pass( dcu || "", ptype, token[1] || "")
+          unless cred
+            cred = service.creds.find_or_initialize_by_user_and_ptype(token[0] || "", ptype)
+          end
+        end
+      end
+    end
+
+    # Update with the password
+    cred.pass = (token[1] || "")
+
+    # Annotate the credential
+    cred.ptype = ptype
+    cred.active = active
+
+    # Update the source ID only if there wasn't already one.
+    if source_id and !cred.source_id
+      cred.source_id = source_id
+      cred.source_type = source_type if source_type
+    end
+
+    # Safe proof (lazy way) -- doesn't chop expanded
+    # characters correctly, but shouldn't ever be a problem.
+    unless proof.nil?
+      proof = Rex::Text.to_hex_ascii(proof)
+      proof = proof[0,4096]
+    end
+    cred.proof = proof
+
+    # Update the timestamp
+    if cred.changed?
+      msf_import_timestamps(opts,cred)
+      cred.save!
+    end
+
+    # Ensure the updated_at is touched any time report_auth_info is called
+    # except when it's set explicitly (as it is for imports)
+    unless opts[:updated_at] || opts["updated_at"]
+      cred.updated_at = Time.now.utc
+      cred.save!
+    end
+
+
+    if opts[:task]
+      Mdm::TaskCred.create(
+          :task => opts[:task],
+          :cred => cred
+      )
+    end
+
+    ret[:cred] = cred
+  }
+  end
+
+  alias :report_auth :report_auth_info
+  alias :report_cred :report_auth_info
+end

lib/msf/core/db_manager/event.rb

@@ -0,0 +1,22 @@
+module Msf::DBManager::Event
+  def events(wspace=workspace)
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    wspace.events.find :all, :order => 'created_at ASC'
+  }
+  end
+
+  def report_event(opts = {})
+    return if not active
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    wspace = opts.delete(:workspace) || workspace
+    return if not wspace # Temp fix?
+    uname  = opts.delete(:username)
+
+    if ! opts[:host].kind_of? ::Mdm::Host and opts[:host]
+      opts[:host] = report_host(:workspace => wspace, :host => opts[:host])
+    end
+
+    ::Mdm::Event.create(opts.merge(:workspace_id => wspace[:id], :username => uname))
+  }
+  end
+end

lib/msf/core/db_manager/exploit_attempt.rb

@@ -0,0 +1,212 @@
+module Msf::DBManager::ExploitAttempt
+  # report_exploit() used to be used to track sessions and which modules
+  # opened them. That information is now available with the session table
+  # directly. TODO: kill this completely some day -- for now just warn if
+  # some other UI is actually using it.
+  def report_exploit(opts={})
+    wlog("Deprecated method call: report_exploit()\n" +
+      "report_exploit() options: #{opts.inspect}\n" +
+      "report_exploit() call stack:\n\t#{caller.join("\n\t")}"
+    )
+  end
+
+  def report_exploit_attempt(host, opts)
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    return if not host
+    info = {}
+
+    # Opts can be keyed by strings or symbols
+    ::Mdm::VulnAttempt.column_names.each do |kn|
+      k = kn.to_sym
+      next if ['id', 'host_id'].include?(kn)
+      info[k] = opts[kn] if opts[kn]
+      info[k] = opts[k]  if opts[k]
+    end
+
+    host.exploit_attempts.create(info)
+  }
+  end
+
+  def report_exploit_failure(opts)
+
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    wspace = opts.delete(:workspace) || workspace
+    mrefs  = opts.delete(:refs) || return
+    host   = opts.delete(:host)
+    port   = opts.delete(:port)
+    prot   = opts.delete(:proto)
+    svc    = opts.delete(:service)
+    vuln   = opts.delete(:vuln)
+
+    timestamp  = opts.delete(:timestamp)
+    freason    = opts.delete(:fail_reason)
+    fdetail    = opts.delete(:fail_detail)
+    username   = opts.delete(:username)
+    mname      = opts.delete(:module)
+
+    # Look up the host as appropriate
+    if not (host and host.kind_of? ::Mdm::Host)
+      if svc.kind_of? ::Mdm::Service
+        host = svc.host
+      else
+        host = get_host( :workspace => wspace, :address => host )
+      end
+    end
+
+    # Bail if we dont have a host object
+    return if not host
+
+    # Look up the service as appropriate
+    if port and svc.nil?
+      prot ||= "tcp"
+      svc = get_service(wspace, host, prot, port) if port
+    end
+
+    if not vuln
+      # Create a references map from the module list
+      ref_objs = ::Mdm::Ref.where(:name => mrefs.map { |ref|
+        if ref.respond_to?(:ctx_id) and ref.respond_to?(:ctx_val)
+          "#{ref.ctx_id}-#{ref.ctx_val}"
+        else
+          ref.to_s
+        end
+      })
+
+      # Try find a matching vulnerability
+      vuln = find_vuln_by_refs(ref_objs, host, svc)
+    end
+
+    # Report a vuln_attempt if we found a match
+    if vuln
+      attempt_info = {
+        :attempted_at => timestamp || Time.now.utc,
+        :exploited    => false,
+        :fail_reason  => freason,
+        :fail_detail  => fdetail,
+        :username     => username  || "unknown",
+        :module       => mname
+      }
+
+      vuln.vuln_attempts.create(attempt_info)
+    end
+
+    # Report an exploit attempt all the same
+    attempt_info = {
+      :attempted_at => timestamp || Time.now.utc,
+      :exploited    => false,
+      :username     => username  || "unknown",
+      :module       => mname,
+      :fail_reason  => freason,
+      :fail_detail  => fdetail
+    }
+
+    attempt_info[:vuln_id] = vuln.id if vuln
+
+    if svc
+      attempt_info[:port]  = svc.port
+      attempt_info[:proto] = svc.proto
+    end
+
+    if port and svc.nil?
+      attempt_info[:port]  = port
+      attempt_info[:proto] = prot || "tcp"
+    end
+
+    host.exploit_attempts.create(attempt_info)
+  }
+  end
+
+  def report_exploit_success(opts)
+  ::ActiveRecord::Base.connection_pool.with_connection {
+
+    wspace = opts.delete(:workspace) || workspace
+    mrefs  = opts.delete(:refs) || return
+    host   = opts.delete(:host)
+    port   = opts.delete(:port)
+    prot   = opts.delete(:proto)
+    svc    = opts.delete(:service)
+    vuln   = opts.delete(:vuln)
+
+    timestamp = opts.delete(:timestamp)
+    username  = opts.delete(:username)
+    mname     = opts.delete(:module)
+
+    # Look up or generate the host as appropriate
+    if not (host and host.kind_of? ::Mdm::Host)
+      if svc.kind_of? ::Mdm::Service
+        host = svc.host
+      else
+        host = report_host(:workspace => wspace, :address => host )
+      end
+    end
+
+    # Bail if we dont have a host object
+    return if not host
+
+    # Look up or generate the service as appropriate
+    if port and svc.nil?
+      svc = report_service(:workspace => wspace, :host => host, :port => port, :proto => prot ) if port
+    end
+
+    if not vuln
+      # Create a references map from the module list
+      ref_objs = ::Mdm::Ref.where(:name => mrefs.map { |ref|
+        if ref.respond_to?(:ctx_id) and ref.respond_to?(:ctx_val)
+          "#{ref.ctx_id}-#{ref.ctx_val}"
+        else
+          ref.to_s
+        end
+      })
+
+      # Try find a matching vulnerability
+      vuln = find_vuln_by_refs(ref_objs, host, svc)
+    end
+
+    # We have match, lets create a vuln_attempt record
+    if vuln
+      attempt_info = {
+        :vuln_id      => vuln.id,
+        :attempted_at => timestamp || Time.now.utc,
+        :exploited    => true,
+        :username     => username  || "unknown",
+        :module       => mname
+      }
+
+      attempt_info[:session_id] = opts[:session_id] if opts[:session_id]
+      attempt_info[:loot_id]    = opts[:loot_id]    if opts[:loot_id]
+
+      vuln.vuln_attempts.create(attempt_info)
+
+      # Correct the vuln's associated service if necessary
+      if svc and vuln.service_id.nil?
+        vuln.service = svc
+        vuln.save
+      end
+    end
+
+    # Report an exploit attempt all the same
+    attempt_info = {
+      :attempted_at => timestamp || Time.now.utc,
+      :exploited    => true,
+      :username     => username  || "unknown",
+      :module       => mname
+    }
+
+    attempt_info[:vuln_id]    = vuln.id           if vuln
+    attempt_info[:session_id] = opts[:session_id] if opts[:session_id]
+    attempt_info[:loot_id]    = opts[:loot_id]    if opts[:loot_id]
+
+    if svc
+      attempt_info[:port]  = svc.port
+      attempt_info[:proto] = svc.proto
+    end
+
+    if port and svc.nil?
+      attempt_info[:port]  = port
+      attempt_info[:proto] = prot || "tcp"
+    end
+
+    host.exploit_attempts.create(attempt_info)
+  }
+  end
+end

lib/msf/core/db_manager/exploited_host.rb

@@ -0,0 +1,16 @@
+module Msf::DBManager::ExploitedHost
+  def each_exploited_host(wspace=workspace,&block)
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    wspace.exploited_hosts.each do |eh|
+      block.call(eh)
+    end
+  }
+  end
+
+  # This method returns a list of all exploited hosts in the database.
+  def exploited_hosts(wspace=workspace)
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    wspace.exploited_hosts
+  }
+  end
+end

lib/msf/core/db_manager/host.rb

@@ -0,0 +1,328 @@
+module Msf::DBManager::Host
+  # Deletes a host and associated data matching this address/comm
+  def del_host(wspace, address, comm='')
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    address, scope = address.split('%', 2)
+    host = wspace.hosts.find_by_address_and_comm(address, comm)
+    host.destroy if host
+  }
+  end
+
+  #
+  # Iterates over the hosts table calling the supplied block with the host
+  # instance of each entry.
+  #
+  def each_host(wspace=workspace, &block)
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    wspace.hosts.each do |host|
+      block.call(host)
+    end
+  }
+  end
+
+  # Exactly like report_host but waits for the database to create a host and returns it.
+  def find_or_create_host(opts)
+    report_host(opts)
+  end
+
+  #
+  # Find a host.  Performs no database writes.
+  #
+  def get_host(opts)
+    if opts.kind_of? ::Mdm::Host
+      return opts
+    elsif opts.kind_of? String
+      raise RuntimeError, "This invokation of get_host is no longer supported: #{caller}"
+    else
+      address = opts[:addr] || opts[:address] || opts[:host] || return
+      return address if address.kind_of? ::Mdm::Host
+    end
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    wspace = opts.delete(:workspace) || workspace
+    if wspace.kind_of? String
+      wspace = find_workspace(wspace)
+    end
+
+    address = normalize_host(address)
+    return wspace.hosts.find_by_address(address)
+  }
+  end
+
+  # Look for an address across all comms
+  def has_host?(wspace,addr)
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    address, scope = addr.split('%', 2)
+    wspace.hosts.find_by_address(addr)
+  }
+  end
+
+  # Returns a list of all hosts in the database
+  def hosts(wspace = workspace, only_up = false, addresses = nil)
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    conditions = {}
+    conditions[:state] = [Msf::HostState::Alive, Msf::HostState::Unknown] if only_up
+    conditions[:address] = addresses if addresses
+    wspace.hosts.where(conditions).order(:address)
+  }
+  end
+
+  #
+  # Returns something suitable for the +:host+ parameter to the various report_* methods
+  #
+  # Takes a Host object, a Session object, an Msf::Session object or a String
+  # address
+  #
+  def normalize_host(host)
+    return host if host.kind_of? ::Mdm::Host
+    norm_host = nil
+
+    if (host.kind_of? String)
+
+      if Rex::Socket.is_ipv4?(host)
+        # If it's an IPv4 addr with a port on the end, strip the port
+        if host =~ /((\d{1,3}\.){3}\d{1,3}):\d+/
+          norm_host = $1
+        else
+          norm_host = host
+        end
+      elsif Rex::Socket.is_ipv6?(host)
+        # If it's an IPv6 addr, drop the scope
+        address, scope = host.split('%', 2)
+        norm_host = address
+      else
+        norm_host = Rex::Socket.getaddress(host, true)
+      end
+    elsif host.kind_of? ::Mdm::Session
+      norm_host = host.host
+    elsif host.respond_to?(:session_host)
+      # Then it's an Msf::Session object
+      thost = host.session_host
+      norm_host = thost
+    end
+
+    # If we got here and don't have a norm_host yet, it could be a
+    # Msf::Session object with an empty or nil tunnel_host and tunnel_peer;
+    # see if it has a socket and use its peerhost if so.
+    if (
+        norm_host.nil? and
+        host.respond_to?(:sock) and
+        host.sock.respond_to?(:peerhost) and
+        host.sock.peerhost.to_s.length > 0
+      )
+      norm_host = session.sock.peerhost
+    end
+    # If We got here and still don't have a real host, there's nothing left
+    # to try, just log it and return what we were given
+    if not norm_host
+      dlog("Host could not be normalized: #{host.inspect}")
+      norm_host = host
+    end
+
+    norm_host
+  end
+
+  #
+  # Report a host's attributes such as operating system and service pack
+  #
+  # The opts parameter MUST contain
+  # +:host+::         -- the host's ip address
+  #
+  # The opts parameter can contain:
+  # +:state+::        -- one of the Msf::HostState constants
+  # +:os_name+::      -- something like "Windows", "Linux", or "Mac OS X"
+  # +:os_flavor+::    -- something like "Enterprise", "Pro", or "Home"
+  # +:os_sp+::        -- something like "SP2"
+  # +:os_lang+::      -- something like "English", "French", or "en-US"
+  # +:arch+::         -- one of the ARCH_* constants
+  # +:mac+::          -- the host's MAC address
+  # +:scope+::        -- interface identifier for link-local IPv6
+  # +:virtual_host+:: -- the name of the VM host software, eg "VMWare", "QEMU", "Xen", etc.
+  #
+  def report_host(opts)
+
+    return if not active
+    addr = opts.delete(:host) || return
+
+    # Sometimes a host setup through a pivot will see the address as "Remote Pipe"
+    if addr.eql? "Remote Pipe"
+      return
+    end
+
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    wspace = opts.delete(:workspace) || workspace
+    if wspace.kind_of? String
+      wspace = find_workspace(wspace)
+    end
+
+    ret = { }
+
+    if not addr.kind_of? ::Mdm::Host
+      addr = normalize_host(addr)
+      addr, scope = addr.split('%', 2)
+      opts[:scope] = scope if scope
+
+      unless ipv46_validator(addr)
+        raise ::ArgumentError, "Invalid IP address in report_host(): #{addr}"
+      end
+
+      if opts[:comm] and opts[:comm].length > 0
+        host = wspace.hosts.find_or_initialize_by_address_and_comm(addr, opts[:comm])
+      else
+        host = wspace.hosts.find_or_initialize_by_address(addr)
+      end
+    else
+      host = addr
+    end
+
+    # Truncate the info field at the maximum field length
+    if opts[:info]
+      opts[:info] = opts[:info][0,65535]
+    end
+
+    # Truncate the name field at the maximum field length
+    if opts[:name]
+      opts[:name] = opts[:name][0,255]
+    end
+
+    opts.each { |k,v|
+      if (host.attribute_names.include?(k.to_s))
+        unless host.attribute_locked?(k.to_s)
+          host[k] = v.to_s.gsub(/[\x00-\x1f]/n, '')
+        end
+      else
+        dlog("Unknown attribute for ::Mdm::Host: #{k}")
+      end
+    }
+    host.info = host.info[0,::Mdm::Host.columns_hash["info"].limit] if host.info
+
+    # Set default fields if needed
+    host.state       = Msf::HostState::Alive if not host.state
+    host.comm        = ''        if not host.comm
+    host.workspace   = wspace    if not host.workspace
+
+    if host.changed?
+      msf_import_timestamps(opts,host)
+      host.save!
+    end
+
+    if opts[:task]
+      Mdm::TaskHost.create(
+          :task => opts[:task],
+          :host => host
+      )
+    end
+
+    host
+  }
+  end
+
+  #
+  # Update a host's attributes via semi-standardized sysinfo hash (Meterpreter)
+  #
+  # The opts parameter MUST contain the following entries
+  # +:host+::           -- the host's ip address
+  # +:info+::           -- the information hash
+  # * 'Computer'        -- the host name
+  # * 'OS'              -- the operating system string
+  # * 'Architecture'    -- the hardware architecture
+  # * 'System Language' -- the system language
+  #
+  # The opts parameter can contain:
+  # +:workspace+::      -- the workspace for this host
+  #
+  def update_host_via_sysinfo(opts)
+
+    return if not active
+    addr = opts.delete(:host) || return
+    info = opts.delete(:info) || return
+
+    # Sometimes a host setup through a pivot will see the address as "Remote Pipe"
+    if addr.eql? "Remote Pipe"
+      return
+    end
+
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    wspace = opts.delete(:workspace) || workspace
+    if wspace.kind_of? String
+      wspace = find_workspace(wspace)
+    end
+
+    if not addr.kind_of? ::Mdm::Host
+      addr = normalize_host(addr)
+      addr, scope = addr.split('%', 2)
+      opts[:scope] = scope if scope
+
+      unless ipv46_validator(addr)
+        raise ::ArgumentError, "Invalid IP address in report_host(): #{addr}"
+      end
+
+      if opts[:comm] and opts[:comm].length > 0
+        host = wspace.hosts.find_or_initialize_by_address_and_comm(addr, opts[:comm])
+      else
+        host = wspace.hosts.find_or_initialize_by_address(addr)
+      end
+    else
+      host = addr
+    end
+
+    res = {}
+
+    if info['Computer']
+      res[:name] = info['Computer']
+    end
+
+    if info['Architecture']
+      res[:arch] = info['Architecture'].split(/\s+/).first
+    end
+
+    if info['OS'] =~ /^Windows\s*([^\(]+)\(([^\)]+)\)/i
+      res[:os_name]   = "Windows #{$1.strip}"
+      build = $2.strip
+
+      if build =~ /Service Pack (\d+)/
+        res[:os_sp] = "SP" + $1
+      end
+    end
+
+    if info["System Language"]
+      case info["System Language"]
+        when /^en_/
+          res[:os_lang] = "English"
+      end
+    end
+
+
+    # Truncate the info field at the maximum field length
+    if res[:info]
+      res[:info] = res[:info][0,65535]
+    end
+
+    # Truncate the name field at the maximum field length
+    if res[:name]
+      res[:name] = res[:name][0,255]
+    end
+
+    res.each { |k,v|
+
+      if (host.attribute_names.include?(k.to_s))
+        unless host.attribute_locked?(k.to_s)
+          host[k] = v.to_s.gsub(/[\x00-\x1f]/n, '')
+        end
+      else
+        dlog("Unknown attribute for Host: #{k}")
+      end
+    }
+
+    # Set default fields if needed
+    host.state       = Msf::HostState::Alive if not host.state
+    host.comm        = ''        if not host.comm
+    host.workspace   = wspace    if not host.workspace
+
+    if host.changed?
+      host.save!
+    end
+
+    host
+  }
+  end
+end

lib/msf/core/db_manager/host_detail.rb

@@ -0,0 +1,21 @@
+module Msf::DBManager::HostDetail
+  #
+  # Populate the host_details table with additional
+  # information, matched by a specific criteria
+  #
+  def report_host_details(host, details)
+  ::ActiveRecord::Base.connection_pool.with_connection {
+
+    detail = ::Mdm::HostDetail.where(( details.delete(:key) || {} ).merge(:host_id => host.id)).first
+    if detail
+      details.each_pair do |k,v|
+        detail[k] = v
+      end
+      detail.save! if detail.changed?
+      detail
+    else
+      detail = ::Mdm::HostDetail.create(details.merge(:host_id => host.id))
+    end
+  }
+  end
+end

lib/msf/core/db_manager/host_tag.rb

@@ -0,0 +1,36 @@
+module Msf::DBManager::HostTag
+  # This is only exercised by MSF3 XML importing for now. Needs the wait
+  # conditions and return hash as well.
+  def report_host_tag(opts)
+    name = opts.delete(:name)
+    raise DBImportError.new("Missing required option :name") unless name
+    addr = opts.delete(:addr)
+    raise DBImportError.new("Missing required option :addr") unless addr
+    wspace = opts.delete(:wspace)
+    raise DBImportError.new("Missing required option :wspace") unless wspace
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    if wspace.kind_of? String
+      wspace = find_workspace(wspace)
+    end
+
+    host = nil
+    report_host(:workspace => wspace, :address => addr)
+
+
+    host = get_host(:workspace => wspace, :address => addr)
+    desc = opts.delete(:desc)
+    summary = opts.delete(:summary)
+    detail = opts.delete(:detail)
+    crit = opts.delete(:crit)
+    possible_tags = Mdm::Tag.includes(:hosts).where("hosts.workspace_id = ? and tags.name = ?", wspace.id, name).order("tags.id DESC").limit(1)
+    tag = (possible_tags.blank? ? Mdm::Tag.new : possible_tags.first)
+    tag.name = name
+    tag.desc = desc
+    tag.report_summary = !!summary
+    tag.report_detail = !!detail
+    tag.critical = !!crit
+    tag.hosts = tag.hosts | [host]
+    tag.save! if tag.changed?
+  }
+  end
+end

lib/msf/core/db_manager/import.rb

@@ -0,0 +1,448 @@
+#
+# Standard library
+#
+
+require 'fileutils'
+require 'tmpdir'
+require 'uri'
+
+#
+# Gems
+#
+
+require 'packetfu'
+
+module Msf::DBManager::Import
+  autoload :Acunetix, 'msf/core/db_manager/import/acunetix'
+  autoload :Amap, 'msf/core/db_manager/import/amap'
+  autoload :Appscan, 'msf/core/db_manager/import/appscan'
+  autoload :Burp, 'msf/core/db_manager/import/burp'
+  autoload :CI, 'msf/core/db_manager/import/ci'
+  autoload :Foundstone, 'msf/core/db_manager/import/foundstone'
+  autoload :FusionVM, 'msf/core/db_manager/import/fusion_vm'
+  autoload :IP360, 'msf/core/db_manager/import/ip360'
+  autoload :IPList, 'msf/core/db_manager/import/ip_list'
+  autoload :Libpcap, 'msf/core/db_manager/import/libpcap'
+  autoload :MBSA, 'msf/core/db_manager/import/mbsa'
+  autoload :MetasploitFramework, 'msf/core/db_manager/import/metasploit_framework'
+  autoload :Nessus, 'msf/core/db_manager/import/nessus'
+  autoload :Netsparker, 'msf/core/db_manager/import/netsparker'
+  autoload :Nexpose, 'msf/core/db_manager/import/nexpose'
+  autoload :Nikto, 'msf/core/db_manager/import/nikto'
+  autoload :Nmap, 'msf/core/db_manager/import/nmap'
+  autoload :OpenVAS, 'msf/core/db_manager/import/open_vas'
+  autoload :Outpost24, 'msf/core/db_manager/import/outpost24'
+  autoload :Qualys, 'msf/core/db_manager/import/qualys'
+  autoload :Report, 'msf/core/db_manager/import/report'
+  autoload :Retina, 'msf/core/db_manager/import/retina'
+  autoload :Spiceworks, 'msf/core/db_manager/import/spiceworks'
+  autoload :Wapiti, 'msf/core/db_manager/import/wapiti'
+
+  include Msf::DBManager::Import::Acunetix
+  include Msf::DBManager::Import::Amap
+  include Msf::DBManager::Import::Appscan
+  include Msf::DBManager::Import::Burp
+  include Msf::DBManager::Import::CI
+  include Msf::DBManager::Import::Foundstone
+  include Msf::DBManager::Import::FusionVM
+  include Msf::DBManager::Import::IP360
+  include Msf::DBManager::Import::IPList
+  include Msf::DBManager::Import::Libpcap
+  include Msf::DBManager::Import::MBSA
+  include Msf::DBManager::Import::MetasploitFramework
+  include Msf::DBManager::Import::Nessus
+  include Msf::DBManager::Import::Netsparker
+  include Msf::DBManager::Import::Nexpose
+  include Msf::DBManager::Import::Nikto
+  include Msf::DBManager::Import::Nmap
+  include Msf::DBManager::Import::OpenVAS
+  include Msf::DBManager::Import::Outpost24
+  include Msf::DBManager::Import::Qualys
+  include Msf::DBManager::Import::Report
+  include Msf::DBManager::Import::Retina
+  include Msf::DBManager::Import::Spiceworks
+  include Msf::DBManager::Import::Wapiti
+
+  # If hex notation is present, turn them into a character.
+  def dehex(str)
+    hexen = str.scan(/\x5cx[0-9a-fA-F]{2}/n)
+    hexen.each { |h|
+      str.gsub!(h,h[2,2].to_i(16).chr)
+    }
+    return str
+  end
+
+  # A way to sneak the yield back into the db importer.
+  # Used by the SAX parsers.
+  def emit(sym,data,&block)
+    yield(sym,data)
+  end
+
+  # A dispatcher method that figures out the data's file type,
+  # and sends it off to the appropriate importer. Note that
+  # import_file_detect will raise an error if the filetype
+  # is unknown.
+  def import(args={}, &block)
+    data = args[:data] || args['data']
+    ftype = import_filetype_detect(data)
+    yield(:filetype, @import_filedata[:type]) if block
+    self.send "import_#{ftype}".to_sym, args, &block
+  end
+
+  #
+  # Generic importer that automatically determines the file type being
+  # imported.  Since this looks for vendor-specific strings in the given
+  # file, there shouldn't be any false detections, but no guarantees.
+  #
+  def import_file(args={}, &block)
+    filename = args[:filename] || args['filename']
+    wspace = args[:wspace] || args['wspace'] || workspace
+    @import_filedata            = {}
+    @import_filedata[:filename] = filename
+
+    data = ""
+    ::File.open(filename, 'rb') do |f|
+      # This check is the largest (byte-wise) that we need to do
+      # since the other 4-byte checks will be subsets of this larger one.
+      data = f.read(Metasploit::Credential::Exporter::Pwdump::FILE_ID_STRING.size)
+    end
+    if data.nil?
+      raise DBImportError.new("Zero-length file")
+    end
+
+    if data.index(Metasploit::Credential::Exporter::Pwdump::FILE_ID_STRING)
+      data = ::File.open(filename, 'rb')
+    else
+      case data[0,4]
+      when "PK\x03\x04"
+        data = Zip::File.open(filename)
+      when "\xd4\xc3\xb2\xa1", "\xa1\xb2\xc3\xd4"
+        data = PacketFu::PcapFile.new(:filename => filename)
+      else
+        ::File.open(filename, 'rb') do |f|
+          sz = f.stat.size
+          data = f.read(sz)
+        end
+      end
+    end
+
+
+    if block
+      import(args.merge(:data => data)) { |type,data| yield type,data }
+    else
+      import(args.merge(:data => data))
+    end
+  end
+
+  # Returns one of the following:
+  #
+  # :acunetix_xml
+  # :amap_log
+  # :amap_mlog
+  # :appscan_xml
+  # :burp_session_xml
+  # :ci_xml
+  # :foundstone_xml
+  # :fusionvm_xml
+  # :ip360_aspl_xml
+  # :ip360_xml_v3
+  # :ip_list
+  # :libpcap
+  # :mbsa_xml
+  # :msf_cred_dump_zip
+  # :msf_pwdump
+  # :msf_xml
+  # :msf_zip
+  # :nessus_nbe
+  # :nessus_xml
+  # :nessus_xml_v2
+  # :netsparker_xml
+  # :nexpose_rawxml
+  # :nexpose_simplexml
+  # :nikto_xml
+  # :nmap_xml
+  # :openvas_new_xml
+  # :openvas_xml
+  # :outpost24_xml
+  # :qualys_asset_xml
+  # :qualys_scan_xml
+  # :retina_xml
+  # :spiceworks_csv
+  # :wapiti_xml
+  #
+  # If there is no match, an error is raised instead.
+  #
+  # @raise DBImportError if the type can't be detected
+  def import_filetype_detect(data)
+
+    if data and data.kind_of? Zip::File
+      if data.entries.empty?
+        raise DBImportError.new("The zip file provided is empty.")
+      end
+
+      @import_filedata ||= {}
+      @import_filedata[:zip_filename] = File.split(data.to_s).last
+      @import_filedata[:zip_basename] = @import_filedata[:zip_filename].gsub(/\.zip$/,"")
+      @import_filedata[:zip_entry_names] = data.entries.map {|x| x.name}
+
+      if @import_filedata[:zip_entry_names].include?(Metasploit::Credential::Importer::Zip::MANIFEST_FILE_NAME)
+        @import_filedata[:type] = "Metasploit Credential Dump"
+        return :msf_cred_dump_zip
+      end
+
+      xml_files = @import_filedata[:zip_entry_names].grep(/^(.*)\.xml$/)
+
+      # TODO This check for our zip export should be more extensive
+      if xml_files.empty?
+        raise DBImportError.new("The zip file provided is not a Metasploit Zip Export")
+      end
+
+      @import_filedata[:zip_xml] = xml_files.first
+      @import_filedata[:type] = "Metasploit Zip Export"
+
+      return :msf_zip
+    end
+
+    if data and data.kind_of? PacketFu::PcapFile
+      # Don't check for emptiness here because unlike other formats, we
+      # haven't read any actual data in yet, only magic bytes to discover
+      # that this is indeed a pcap file.
+      #raise DBImportError.new("The pcap file provided is empty.") if data.body.empty?
+      @import_filedata ||= {}
+      @import_filedata[:type] = "Libpcap Packet Capture"
+      return :libpcap
+    end
+
+    # msfpwdump
+    if data.present? && data.kind_of?(::File)
+      @import_filedata[:type] = "Metasploit PWDump Export"
+      return :msf_pwdump
+    end
+
+    # This is a text string, lets make sure its treated as binary
+    data = data.unpack("C*").pack("C*")
+    if data and data.to_s.strip.length == 0
+      raise DBImportError.new("The data provided to the import function was empty")
+    end
+
+    # Parse the first line or 4k of data from the file
+    di = data.index("\n") || 4096
+
+    firstline = data[0, di]
+    @import_filedata ||= {}
+    if (firstline.index("<NeXposeSimpleXML"))
+      @import_filedata[:type] = "NeXpose Simple XML"
+      return :nexpose_simplexml
+    elsif (firstline.index("<FusionVM"))
+      @import_filedata[:type] = "FusionVM XML"
+      return :fusionvm_xml
+    elsif (firstline.index("<NexposeReport"))
+      @import_filedata[:type] = "NeXpose XML Report"
+      return :nexpose_rawxml
+    elsif (firstline.index("Name,Manufacturer,Device Type,Model,IP Address,Serial Number,Location,Operating System"))
+      @import_filedata[:type] = "Spiceworks CSV Export"
+      return :spiceworks_csv
+    elsif (firstline.index("<scanJob>"))
+      @import_filedata[:type] = "Retina XML"
+      return :retina_xml
+    elsif (firstline.index(/<get_reports_response status=['"]200['"] status_text=['"]OK['"]>/))
+      @import_filedata[:type] = "OpenVAS XML"
+      return :openvas_new_xml
+    elsif (firstline.index(/<report id=['"]/))
+      @import_filedata[:type] = "OpenVAS XML"
+      return :openvas_new_xml
+    elsif (firstline.index("<NessusClientData>"))
+      @import_filedata[:type] = "Nessus XML (v1)"
+      return :nessus_xml
+    elsif (firstline.index("<SecScan ID="))
+      @import_filedata[:type] = "Microsoft Baseline Security Analyzer"
+      return :mbsa_xml
+    elsif (data[0,1024] =~ /<!ATTLIST\s+items\s+burpVersion/)
+      @import_filedata[:type] = "Burp Session XML"
+      return :burp_session_xml
+    elsif (firstline.index("<?xml"))
+      # it's xml, check for root tags we can handle
+      line_count = 0
+      data.each_line { |line|
+        line =~ /<([a-zA-Z0-9\-\_]+)[ >]/
+
+        case $1
+        when "niktoscan"
+          @import_filedata[:type] = "Nikto XML"
+          return :nikto_xml
+        when "nmaprun"
+          @import_filedata[:type] = "Nmap XML"
+          return :nmap_xml
+        when "openvas-report"
+          @import_filedata[:type] = "OpenVAS Report"
+          return :openvas_xml
+        when "NessusClientData"
+          @import_filedata[:type] = "Nessus XML (v1)"
+          return :nessus_xml
+        when "NessusClientData_v2"
+          @import_filedata[:type] = "Nessus XML (v2)"
+          return :nessus_xml_v2
+        when "SCAN"
+          @import_filedata[:type] = "Qualys Scan XML"
+          return :qualys_scan_xml
+        when "report"
+          @import_filedata[:type] = "Wapiti XML"
+          return :wapiti_xml
+        when "ASSET_DATA_REPORT"
+          @import_filedata[:type] = "Qualys Asset XML"
+          return :qualys_asset_xml
+        when /MetasploitExpressV[1234]/
+          @import_filedata[:type] = "Metasploit XML"
+          return :msf_xml
+        when /MetasploitV4/
+          @import_filedata[:type] = "Metasploit XML"
+          return :msf_xml
+        when /netsparker/
+          @import_filedata[:type] = "NetSparker XML"
+          return :netsparker_xml
+        when /audits?/ # <audit> and <audits> are both valid for nCircle. wtfmate.
+          @import_filedata[:type] = "IP360 XML v3"
+          return :ip360_xml_v3
+        when /ontology/
+          @import_filedata[:type] = "IP360 ASPL"
+          return :ip360_aspl_xml
+        when /ReportInfo/
+          @import_filedata[:type] = "Foundstone"
+          return :foundstone_xml
+        when /ScanGroup/
+          @import_filedata[:type] = "Acunetix"
+          return :acunetix_xml
+        when /AppScanInfo/ # Actually the second line
+          @import_filedata[:type] = "Appscan"
+          return :appscan_xml
+        when "entities"
+          if  line =~ /creator.*\x43\x4f\x52\x45\x20\x49\x4d\x50\x41\x43\x54/ni
+            @import_filedata[:type] = "CI"
+            return :ci_xml
+          end
+        when "main"
+          @import_filedata[:type] = "Outpost24 XML"
+          return :outpost24_xml
+        else
+          # Give up if we haven't hit the root tag in the first few lines
+          break if line_count > 10
+        end
+        line_count += 1
+      }
+    elsif (firstline.index("timestamps|||scan_start"))
+      @import_filedata[:type] = "Nessus NBE Report"
+      # then it's a nessus nbe
+      return :nessus_nbe
+    elsif (firstline.index("# amap v"))
+      # then it's an amap mlog
+      @import_filedata[:type] = "Amap Log -m"
+      return :amap_mlog
+    elsif (firstline.index("amap v"))
+      # then it's an amap log
+      @import_filedata[:type] = "Amap Log"
+      return :amap_log
+    elsif ipv46_validator(firstline)
+      # then its an IP list
+      @import_filedata[:type] = "IP Address List"
+      return :ip_list
+    elsif (data[0,1024].index("<netsparker"))
+      @import_filedata[:type] = "NetSparker XML"
+      return :netsparker_xml
+    elsif (firstline.index("# Metasploit PWDump Export"))
+      # then it's a Metasploit PWDump export
+      @import_filedata[:type] = "Metasploit PWDump Export"
+      return :msf_pwdump
+    end
+
+    raise DBImportError.new("Could not automatically determine file type")
+  end
+
+  # Handles timestamps from Metasploit Express/Pro imports.
+  def msf_import_timestamps(opts,obj)
+    obj.created_at = opts["created_at"] if opts["created_at"]
+    obj.created_at = opts[:created_at] if opts[:created_at]
+    obj.updated_at = opts["updated_at"] ? opts["updated_at"] : obj.created_at
+    obj.updated_at = opts[:updated_at] ? opts[:updated_at] : obj.created_at
+    return obj
+  end
+
+  def report_import_note(wspace,addr)
+    if @import_filedata.kind_of?(Hash) && @import_filedata[:filename] && @import_filedata[:filename] !~ /msfe-nmap[0-9]{8}/
+    report_note(
+      :workspace => wspace,
+      :host => addr,
+      :type => 'host.imported',
+      :data => @import_filedata.merge(:time=> Time.now.utc)
+    )
+    end
+  end
+
+  # Returns a REXML::Document from the given data.
+  def rexmlify(data)
+    if data.kind_of?(REXML::Document)
+      return data
+    else
+      # Make an attempt to recover from a REXML import fail, since
+      # it's better than dying outright.
+      begin
+        return REXML::Document.new(data)
+      rescue REXML::ParseException => e
+        dlog("REXML error: Badly formatted XML, attempting to recover. Error was: #{e.inspect}")
+        return REXML::Document.new(data.gsub(/([\x00-\x08\x0b\x0c\x0e-\x1f\x80-\xff])/n){ |x| "\\x%.2x" % x.unpack("C*")[0] })
+      end
+    end
+  end
+
+  #
+  # This method normalizes an incoming service name to one of the
+  # the standard ones recognized by metasploit
+  #
+  def service_name_map(proto)
+    return proto unless proto.kind_of? String
+    case proto.downcase
+    when "msrpc", "nfs-or-iis", "dce endpoint resolution"
+      "dcerpc"
+    when "ms-sql-s", "tds"
+      "mssql"
+    when "ms-sql-m","microsoft sql monitor"
+      "mssql-m"
+    when "postgresql";                  "postgres"
+    when "http-proxy";                  "http"
+    when "iiimsf";                      "db2"
+    when "oracle-tns";                  "oracle"
+    when "quickbooksrds";               "metasploit"
+    when "microsoft remote display protocol"
+      "rdp"
+    when "vmware authentication daemon"
+      "vmauthd"
+    when "netbios-ns", "cifs name service"
+      "netbios"
+    when "netbios-ssn", "microsoft-ds", "cifs"
+      "smb"
+    when "remote shell"
+      "shell"
+    when "remote login"
+      "login"
+    when "nfs lockd"
+      "lockd"
+    when "hp jetdirect"
+      "jetdirect"
+    when "dhcp server"
+      "dhcp"
+    when /^dns-(udp|tcp)$/;             "dns"
+    when /^dce[\s+]rpc$/;               "dcerpc"
+    else
+      proto.downcase.gsub(/\s*\(.*/, '')   # "service (some service)"
+    end
+  end
+
+  # Boils down the validate_import_file to a boolean
+  def validate_import_file(data)
+    begin
+      import_filetype_detect(data)
+    rescue DBImportError
+      return false
+    end
+    return true
+  end
+end

lib/msf/core/db_manager/import/acunetix.rb

@@ -0,0 +1,34 @@
+require 'rex/parser/acunetix_nokogiri'
+
+module Msf::DBManager::Import::Acunetix
+  def import_acunetix_noko_stream(args={},&block)
+    if block
+      doc = Rex::Parser::AcunetixDocument.new(args,framework.db) {|type, data| yield type,data }
+    else
+      doc = Rex::Parser::AcunetixFoundstoneDocument.new(args,self)
+    end
+    parser = ::Nokogiri::XML::SAX::Parser.new(doc)
+    parser.parse(args[:data])
+  end
+
+  def import_acunetix_xml(args={}, &block)
+    bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : []
+    wspace = args[:wspace] || workspace
+    if Rex::Parser.nokogiri_loaded
+      parser = "Nokogiri v#{::Nokogiri::VERSION}"
+      noko_args = args.dup
+      noko_args[:blacklist] = bl
+      noko_args[:wspace] = wspace
+      if block
+        yield(:parser, parser)
+        import_acunetix_noko_stream(noko_args) {|type, data| yield type,data}
+      else
+        import_acunetix_noko_stream(noko_args)
+      end
+      return true
+    else # Sorry
+      raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.")
+    end
+  end
+
+end

lib/msf/core/db_manager/import/amap.rb

@@ -0,0 +1,84 @@
+module Msf::DBManager::Import::Amap
+  def import_amap_log(args={}, &block)
+    data = args[:data]
+    wspace = args[:wspace] || workspace
+    bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : []
+
+    data.each_line do |line|
+      next if line =~ /^#/
+      next if line !~ /^Protocol on ([^:]+):([^\x5c\x2f]+)[\x5c\x2f](tcp|udp) matches (.*)$/n
+      addr   = $1
+      next if bl.include? addr
+      port   = $2.to_i
+      proto  = $3.downcase
+      name   = $4
+      host = find_or_create_host(:workspace => wspace, :host => addr, :state => Msf::HostState::Alive, :task => args[:task])
+      next if not host
+      yield(:address,addr) if block
+      info = {
+        :workspace => wspace,
+        :task => args[:task],
+        :host => host,
+        :proto => proto,
+        :port => port
+      }
+      if name != "unidentified"
+        info[:name] = name
+      end
+      service = find_or_create_service(info)
+    end
+  end
+
+  def import_amap_log_file(args={})
+    filename = args[:filename]
+    wspace = args[:wspace] || workspace
+    data = ""
+    ::File.open(filename, 'rb') do |f|
+      data = f.read(f.stat.size)
+    end
+
+    case import_filetype_detect(data)
+    when :amap_log
+      import_amap_log(args.merge(:data => data))
+    when :amap_mlog
+      import_amap_mlog(args.merge(:data => data))
+    else
+      raise DBImportError.new("Could not determine file type")
+    end
+  end
+
+  def import_amap_mlog(args={}, &block)
+    data = args[:data]
+    wspace = args[:wspace] || workspace
+    bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : []
+
+    data.each_line do |line|
+      next if line =~ /^#/
+      r = line.split(':')
+      next if r.length < 6
+
+      addr   = r[0]
+      next if bl.include? addr
+      port   = r[1].to_i
+      proto  = r[2].downcase
+      status = r[3]
+      name   = r[5]
+      next if status != "open"
+
+      host = find_or_create_host(:workspace => wspace, :host => addr, :state => Msf::HostState::Alive, :task => args[:task])
+      next if not host
+      yield(:address,addr) if block
+      info = {
+        :workspace => wspace,
+        :task => args[:task],
+        :host => host,
+        :proto => proto,
+        :port => port
+      }
+      if name != "unidentified"
+        info[:name] = name
+      end
+      service = find_or_create_service(info)
+    end
+  end
+end

lib/msf/core/db_manager/import/appscan.rb

@@ -0,0 +1,33 @@
+require 'rex/parser/appscan_nokogiri'
+
+module Msf::DBManager::Import::Appscan
+  def import_appscan_noko_stream(args={},&block)
+    if block
+      doc = Rex::Parser::AppscanDocument.new(args,framework.db) {|type, data| yield type,data }
+    else
+      doc = Rex::Parser::AppscanDocument.new(args,self)
+    end
+    parser = ::Nokogiri::XML::SAX::Parser.new(doc)
+    parser.parse(args[:data])
+  end
+
+  def import_appscan_xml(args={}, &block)
+    bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : []
+    wspace = args[:wspace] || workspace
+    if Rex::Parser.nokogiri_loaded
+      parser = "Nokogiri v#{::Nokogiri::VERSION}"
+      noko_args = args.dup
+      noko_args[:blacklist] = bl
+      noko_args[:wspace] = wspace
+      if block
+        yield(:parser, parser)
+        import_appscan_noko_stream(noko_args) {|type, data| yield type,data}
+      else
+        import_appscan_noko_stream(noko_args)
+      end
+      return true
+    else # Sorry
+      raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.")
+    end
+  end
+end

lib/msf/core/db_manager/import/burp.rb

@@ -0,0 +1,34 @@
+require 'rex/parser/burp_session_nokogiri'
+
+module Msf::DBManager::Import::Burp
+  def import_burp_session_noko_stream(args={},&block)
+    if block
+      doc = Rex::Parser::BurpSessionDocument.new(args,framework.db) {|type, data| yield type,data }
+    else
+      doc = Rex::Parser::BurpSessionDocument.new(args,self)
+    end
+    parser = ::Nokogiri::XML::SAX::Parser.new(doc)
+    parser.parse(args[:data])
+  end
+
+  def import_burp_session_xml(args={}, &block)
+    bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : []
+    wspace = args[:wspace] || workspace
+    if Rex::Parser.nokogiri_loaded
+      # Rex::Parser.reload("burp_session_nokogiri.rb")
+      parser = "Nokogiri v#{::Nokogiri::VERSION}"
+      noko_args = args.dup
+      noko_args[:blacklist] = bl
+      noko_args[:wspace] = wspace
+      if block
+        yield(:parser, parser)
+        import_burp_session_noko_stream(noko_args) {|type, data| yield type,data}
+      else
+        import_burp_session_noko_stream(noko_args)
+      end
+      return true
+    else # Sorry
+      raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.")
+    end
+  end
+end

lib/msf/core/db_manager/import/ci.rb

@@ -0,0 +1,33 @@
+require 'rex/parser/ci_nokogiri'
+
+module Msf::DBManager::Import::CI
+  def import_ci_noko_stream(args, &block)
+    if block
+      doc = Rex::Parser::CIDocument.new(args,framework.db) {|type, data| yield type,data }
+    else
+      doc = Rex::Parser::CI.new(args,self)
+    end
+    parser = ::Nokogiri::XML::SAX::Parser.new(doc)
+    parser.parse(args[:data])
+  end
+
+  def import_ci_xml(args={}, &block)
+    bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : []
+    wspace = args[:wspace] || workspace
+    if Rex::Parser.nokogiri_loaded
+      parser = "Nokogiri v#{::Nokogiri::VERSION}"
+      noko_args = args.dup
+      noko_args[:blacklist] = bl
+      noko_args[:wspace] = wspace
+      if block
+        yield(:parser, parser)
+        import_ci_noko_stream(noko_args) {|type, data| yield type,data}
+      else
+        import_ci_noko_stream(noko_args)
+      end
+      return true
+    else # Sorry
+      raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.")
+    end
+  end
+end

lib/msf/core/db_manager/import/foundstone.rb

@@ -0,0 +1,33 @@
+require 'rex/parser/foundstone_nokogiri'
+
+module Msf::DBManager::Import::Foundstone
+  def import_foundstone_noko_stream(args={},&block)
+    if block
+      doc = Rex::Parser::FoundstoneDocument.new(args,framework.db) {|type, data| yield type,data }
+    else
+      doc = Rex::Parser::FoundstoneDocument.new(args,self)
+    end
+    parser = ::Nokogiri::XML::SAX::Parser.new(doc)
+    parser.parse(args[:data])
+  end
+
+  def import_foundstone_xml(args={}, &block)
+    bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : []
+    wspace = args[:wspace] || workspace
+    if Rex::Parser.nokogiri_loaded
+      parser = "Nokogiri v#{::Nokogiri::VERSION}"
+      noko_args = args.dup
+      noko_args[:blacklist] = bl
+      noko_args[:wspace] = wspace
+      if block
+        yield(:parser, parser)
+        import_foundstone_noko_stream(noko_args) {|type, data| yield type,data}
+      else
+        import_foundstone_noko_stream(noko_args)
+      end
+      return true
+    else # Sorry
+      raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.")
+    end
+  end
+end

lib/msf/core/db_manager/import/fusion_vm.rb

@@ -0,0 +1,11 @@
+require 'rex/parser/fusionvm_nokogiri'
+
+module Msf::DBManager::Import::FusionVM
+  def import_fusionvm_xml(args={})
+    args[:wspace] ||= workspace
+    bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : []
+    doc = Rex::Parser::FusionVMDocument.new(args,self)
+    parser = ::Nokogiri::XML::SAX::Parser.new(doc)
+    parser.parse(args[:data])
+  end
+end

lib/msf/core/db_manager/import/ip360.rb

@@ -0,0 +1,9 @@
+require 'rex/parser/ip360_aspl_xml'
+
+module Msf::DBManager::Import::IP360
+  autoload :ASPL, 'msf/core/db_manager/import/ip360/aspl'
+  autoload :V3, 'msf/core/db_manager/import/ip360/v3'
+
+  include Msf::DBManager::Import::IP360::ASPL
+  include Msf::DBManager::Import::IP360::V3
+end

lib/msf/core/db_manager/import/ip360/aspl.rb

@@ -0,0 +1,23 @@
+require 'rex/parser/ip360_aspl_xml'
+
+module Msf::DBManager::Import::IP360::ASPL
+  #
+  # Import IP360's ASPL database
+  #
+  def import_ip360_aspl_xml(args={}, &block)
+    data = args[:data]
+    wspace = args[:wspace] || workspace
+    bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : []
+
+    if not data.index("<ontology")
+      raise DBImportError.new("The ASPL file does not appear to be valid or may still be compressed")
+    end
+
+    base = ::File.join(Msf::Config.config_directory, "data", "ncircle")
+    ::FileUtils.mkdir_p(base)
+    ::File.open(::File.join(base, "ip360.aspl"), "wb") do |fd|
+      fd.write(data)
+    end
+    yield(:notice, "Saved the IP360 ASPL database to #{base}...")
+  end
+end

lib/msf/core/db_manager/import/ip360/v3.rb

@@ -0,0 +1,188 @@
+require 'rex/parser/ip360_xml'
+
+module Msf::DBManager::Import::IP360::V3
+  #
+  # Import IP360 XML v3 output
+  #
+  def import_ip360_xml_file(args={})
+    filename = args[:filename]
+    wspace = args[:wspace] || workspace
+
+    data = ""
+    ::File.open(filename, 'rb') do |f|
+      data = f.read(f.stat.size)
+    end
+    import_ip360_xml_v3(args.merge(:data => data))
+  end
+
+  #
+  # Import IP360's xml output
+  #
+  def import_ip360_xml_v3(args={}, &block)
+    data = args[:data]
+    wspace = args[:wspace] || workspace
+    bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : []
+
+    # @aspl = {'vulns' => {'name' => { }, 'cve' => { }, 'bid' => { } }
+    # 'oses' => {'name' } }
+
+    aspl_path  = nil
+    aspl_paths = [
+      ::File.join(Msf::Config.config_directory, "data", "ncircle", "ip360.aspl"),
+      ::File.join(Msf::Config.data_directory, "ncircle", "ip360.aspl")
+    ]
+
+    aspl_paths.each do |tpath|
+      next if not (::File.exist?(tpath) and ::File.readable?(tpath))
+      aspl_path = tpath
+      break
+    end
+
+    if not aspl_path
+      raise DBImportError.new("The nCircle IP360 ASPL file is not present.\n    Download ASPL from nCircle VNE | Administer | Support | Resources, unzip it, and import it first")
+    end
+
+    # parse nCircle ASPL file
+    aspl = ""
+    ::File.open(aspl_path, "rb") do |f|
+      aspl = f.read(f.stat.size)
+    end
+
+    @asplhash = nil
+    parser = Rex::Parser::IP360ASPLXMLStreamParser.new
+    parser.on_found_aspl = Proc.new { |asplh|
+      @asplhash = asplh
+    }
+    REXML::Document.parse_stream(aspl, parser)
+
+    # nCircle has some quotes escaped which causes the parser to break
+    # we don't need these lines so just replace \" with "
+    data.gsub!(/\\"/,'"')
+
+    # parse nCircle Scan Output
+    parser = Rex::Parser::IP360XMLStreamParser.new
+    parser.on_found_host = Proc.new { |host|
+      hobj = nil
+      addr = host['addr'] || host['hname']
+
+      next unless ipv46_validator(addr) # Catches SCAN-ERROR, among others.
+
+      if bl.include? addr
+        next
+      else
+        yield(:address,addr) if block
+      end
+
+      os = host['os']
+      hname = host['hname']
+      mac = host['mac']
+
+      host_hash = {
+        :workspace => wspace,
+        :host => addr,
+        :task => args[:task]
+      }
+      host_hash[:name] = hname.to_s.strip if hname
+      host_hash[:mac]  = mac.to_s.strip.upcase if mac
+
+      hobj = report_host(host_hash)
+
+      yield(:os, os) if block
+      if os
+        report_note(
+          :workspace => wspace,
+          :task => args[:task],
+          :host => hobj,
+          :type => 'host.os.ip360_fingerprint',
+          :data => {
+            :os => @asplhash['oses'][os].to_s.strip
+          }
+        )
+      end
+
+      host['apps'].each do |item|
+        port = item['port'].to_s
+        proto = item['proto'].to_s
+
+        handle_ip360_v3_svc(wspace, hobj, port, proto, hname, args[:task])
+      end
+
+
+      host['vulns'].each do |item|
+        vulnid = item['vulnid'].to_s
+        port = item['port'].to_s
+        proto = item['proto'] || "tcp"
+        vulnname = @asplhash['vulns']['name'][vulnid]
+        cves = @asplhash['vulns']['cve'][vulnid]
+        bids = @asplhash['vulns']['bid'][vulnid]
+
+        yield(:port, port) if block
+
+        handle_ip360_v3_vuln(wspace, hobj, port, proto, hname, vulnid, vulnname, cves, bids, args[:task])
+
+      end
+
+      yield(:end, hname) if block
+    }
+
+    REXML::Document.parse_stream(data, parser)
+  end
+
+  protected
+
+  # IP360 v3 svc
+  def handle_ip360_v3_svc(wspace,hobj,port,proto,hname,task=nil)
+    addr = hobj.address
+    report_host(:workspace => wspace, :host => hobj, :state => Msf::HostState::Alive, :task => task)
+
+    info = { :workspace => wspace, :host => hobj, :port => port, :proto => proto, :task => task }
+    if hname != "unknown" and hname[-1,1] != "?"
+      info[:name] = hname
+    end
+
+    if port.to_i != 0
+      report_service(info)
+    end
+  end
+
+  #
+  # IP360 v3 vuln
+  #
+  def handle_ip360_v3_vuln(wspace,hobj,port,proto,hname,vulnid,vulnname,cves,bids,task=nil)
+    info = { :workspace => wspace, :host => hobj, :port => port, :proto => proto, :task => task }
+    if hname != "unknown" and hname[-1,1] != "?"
+      info[:name] = hname
+    end
+
+    if port.to_i != 0
+      report_service(info)
+    end
+
+    refs = []
+
+    cves.split(/,/).each do |cve|
+      refs.push(cve.to_s)
+    end if cves
+
+    bids.split(/,/).each do |bid|
+      refs.push('BID-' + bid.to_s)
+    end if bids
+
+    description = nil   # not working yet
+    vuln = {
+      :workspace => wspace,
+      :host => hobj,
+      :name => vulnname,
+      :info => description ? description : "",
+      :refs => refs,
+      :task => task
+    }
+
+    if port.to_i != 0
+      vuln[:port]  = port
+      vuln[:proto] = proto
+    end
+
+    report_vuln(vuln)
+  end
+end

lib/msf/core/db_manager/import/ip_list.rb

@@ -0,0 +1,28 @@
+module Msf::DBManager::Import::IPList
+  def import_ip_list(args={}, &block)
+    data = args[:data]
+    wspace = args[:wspace] || workspace
+    bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : []
+
+    data.each_line do |ip|
+      ip.strip!
+      if bl.include? ip
+        next
+      else
+        yield(:address,ip) if block
+      end
+      host = find_or_create_host(:workspace => wspace, :host=> ip, :state => Msf::HostState::Alive, :task => args[:task])
+    end
+  end
+
+  def import_ip_list_file(args={})
+    filename = args[:filename]
+    wspace = args[:wspace] || workspace
+
+    data = ""
+    ::File.open(filename, 'rb') do |f|
+      data = f.read(f.stat.size)
+    end
+    import_ip_list(args.merge(:data => data))
+  end
+end

lib/msf/core/db_manager/import/libpcap.rb

@@ -0,0 +1,220 @@
+module Msf::DBManager::Import::Libpcap
+  # The libpcap file format is handled by PacketFu for data
+  # extraction. TODO: Make this its own mixin, and possibly
+  # extend PacketFu to do better stream analysis on the fly.
+  def import_libpcap(args={}, &block)
+    data = args[:data]
+    wspace = args[:wspace] || workspace
+    bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : []
+    # seen_hosts is only used for determining when to yield an address. Once we get
+    # some packet analysis going, the values will have all sorts of info. The plan
+    # is to ru through all the packets as a first pass and report host and service,
+    # then, once we have everything parsed, we can reconstruct sessions and ngrep
+    # out things like authentication sequences, examine ttl's and window sizes, all
+    # kinds of crazy awesome stuff like that.
+    seen_hosts = {}
+    decoded_packets = 0
+    last_count = 0
+    data.read_packet_bytes do |p|
+      if (decoded_packets >= last_count + 1000) and block
+        yield(:pcap_count, decoded_packets)
+        last_count = decoded_packets
+      end
+      decoded_packets += 1
+
+      pkt = PacketFu::Packet.parse(p) rescue next # Just silently skip bad packets
+
+      next unless pkt.is_ip? # Skip anything that's not IP. Technically, not Ethernet::Ip
+      next if pkt.is_tcp? && (pkt.tcp_src == 0 || pkt.tcp_dst == 0) # Skip port 0
+      next if pkt.is_udp? && (pkt.udp_src == 0 || pkt.udp_dst == 0) # Skip port 0
+      saddr = pkt.ip_saddr
+      daddr = pkt.ip_daddr
+
+      # Handle blacklists and obviously useless IP addresses, and report the host.
+      next if (bl | [saddr,daddr]).size == bl.size # Both hosts are blacklisted, skip everything.
+      unless( bl.include?(saddr) || rfc3330_reserved(saddr))
+        yield(:address,saddr) if block and !seen_hosts.keys.include?(saddr)
+        unless seen_hosts[saddr]
+          report_host(
+              :workspace => wspace,
+              :host      => saddr,
+              :state     => Msf::HostState::Alive,
+              :task      => args[:task]
+          )
+        end
+        seen_hosts[saddr] ||= []
+
+      end
+      unless( bl.include?(daddr) || rfc3330_reserved(daddr))
+        yield(:address,daddr) if block and !seen_hosts.keys.include?(daddr)
+        unless seen_hosts[daddr]
+          report_host(
+              :workspace => wspace,
+              :host      => daddr,
+              :state     => Msf::HostState::Alive,
+              :task      => args[:task]
+          )
+        end
+        seen_hosts[daddr] ||= []
+      end
+
+      if pkt.is_tcp? # First pass on TCP packets
+        if (pkt.tcp_flags.syn == 1 and pkt.tcp_flags.ack == 1) or # Oh, this kills me
+          pkt.tcp_src < 1024 # If it's a low port, assume it's a proper service.
+          if seen_hosts[saddr]
+            unless seen_hosts[saddr].include? [pkt.tcp_src,"tcp"]
+              report_service(
+                  :workspace => wspace, :host => saddr,
+                  :proto     => "tcp", :port => pkt.tcp_src,
+                  :state     => Msf::ServiceState::Open,
+                  :task      => args[:task]
+              )
+              seen_hosts[saddr] << [pkt.tcp_src,"tcp"]
+              yield(:service,"%s:%d/%s" % [saddr,pkt.tcp_src,"tcp"])
+            end
+          end
+        end
+      elsif pkt.is_udp? # First pass on UDP packets
+        if pkt.udp_src == pkt.udp_dst # Very basic p2p detection.
+          [saddr,daddr].each do |xaddr|
+            if seen_hosts[xaddr]
+              unless seen_hosts[xaddr].include? [pkt.udp_src,"udp"]
+                report_service(
+                    :workspace => wspace, :host => xaddr,
+                    :proto     => "udp", :port => pkt.udp_src,
+                    :state     => Msf::ServiceState::Open,
+                    :task      => args[:task]
+                )
+                seen_hosts[xaddr] << [pkt.udp_src,"udp"]
+                yield(:service,"%s:%d/%s" % [xaddr,pkt.udp_src,"udp"])
+              end
+            end
+          end
+        elsif pkt.udp_src < 1024 # Probably a service
+          if seen_hosts[saddr]
+            unless seen_hosts[saddr].include? [pkt.udp_src,"udp"]
+              report_service(
+                  :workspace => wspace, :host => saddr,
+                  :proto     => "udp", :port => pkt.udp_src,
+                  :state     => Msf::ServiceState::Open,
+                  :task      => args[:task]
+              )
+              seen_hosts[saddr] << [pkt.udp_src,"udp"]
+              yield(:service,"%s:%d/%s" % [saddr,pkt.udp_src,"udp"])
+            end
+          end
+        end
+      end # tcp or udp
+
+      inspect_single_packet(pkt,wspace,args)
+
+    end # data.body.map
+
+    # Right about here, we should have built up some streams for some stream analysis.
+    # Not sure what form that will take, but people like shoving many hundreds of
+    # thousands of packets through this thing, so it'll need to be memory efficient.
+
+  end
+
+  def import_libpcap_file(args={})
+    filename = args[:filename]
+    wspace = args[:wspace] || workspace
+
+    data = PacketFu::PcapFile.new(:filename => filename)
+    import_libpcap(args.merge(:data => data))
+  end
+
+  # Do all the single packet analysis we can while churning through the pcap
+  # the first time. Multiple packet inspection will come later, where we can
+  # do stream analysis, compare requests and responses, etc.
+  def inspect_single_packet(pkt,wspace,args)
+    if pkt.is_tcp? or pkt.is_udp?
+      inspect_single_packet_http(pkt,wspace,args)
+    end
+  end
+
+  # Checks for packets that are headed towards port 80, are tcp, contain an HTTP/1.0
+  # line, contains an Authorization line, contains a b64-encoded credential, and
+  # extracts it. Reports this credential and solidifies the service as HTTP.
+  def inspect_single_packet_http(pkt,wspace,args)
+    task = args.fetch(:task, nil)
+    # First, check the server side (data from port 80).
+    if pkt.is_tcp? and pkt.tcp_src == 80 and !pkt.payload.nil? and !pkt.payload.empty?
+      if pkt.payload =~ /^HTTP\x2f1\x2e[01]/n
+        http_server_match = pkt.payload.match(/\nServer:\s+([^\r\n]+)[\r\n]/n)
+        if http_server_match.kind_of?(MatchData) and http_server_match[1]
+          report_service(
+              :workspace => wspace,
+              :host      => pkt.ip_saddr,
+              :port      => pkt.tcp_src,
+              :proto     => "tcp",
+              :name      => "http",
+              :info      => http_server_match[1],
+              :state     => Msf::ServiceState::Open,
+              :task      => task
+          )
+          # That's all we want to know from this service.
+          return :something_significant
+        end
+      end
+    end
+
+    # Next, check the client side (data to port 80)
+    if pkt.is_tcp? and pkt.tcp_dst == 80 and !pkt.payload.nil? and !pkt.payload.empty?
+      if pkt.payload.match(/[\x00-\x20]HTTP\x2f1\x2e[10]/n)
+        auth_match = pkt.payload.match(/\nAuthorization:\s+Basic\s+([A-Za-z0-9=\x2b]+)/n)
+        if auth_match.kind_of?(MatchData) and auth_match[1]
+          b64_cred = auth_match[1]
+        else
+          return false
+        end
+        # If we're this far, we can surmise that at least the client is a web browser,
+        # he thinks the server is HTTP and he just made an authentication attempt. At
+        # this point, we'll just believe everything the packet says -- validation ought
+        # to come later.
+        user,pass = b64_cred.unpack("m*").first.split(/:/,2)
+        report_service(
+            :workspace => wspace,
+            :host      => pkt.ip_daddr,
+            :port      => pkt.tcp_dst,
+            :proto     => "tcp",
+            :name      => "http",
+            :task      => task
+        )
+
+        service_data = {
+            address: pkt.ip_daddr,
+            port: pkt.tcp_dst,
+            service_name: 'http',
+            protocol: 'tcp',
+            workspace_id: wspace.id
+        }
+        service_data[:task_id] = task.id if task
+
+        filename = args[:filename]
+
+        credential_data = {
+            origin_type: :import,
+            private_data: pass,
+            private_type: :password,
+            username: user,
+            filename: filename
+        }
+        credential_data.merge!(service_data)
+        credential_core = create_credential(credential_data)
+
+        login_data = {
+            core: credential_core,
+            status: Metasploit::Model::Login::Status::UNTRIED
+        }
+
+        login_data.merge!(service_data)
+
+        create_credential_login(login_data)
+
+        # That's all we want to know from this service.
+        return :something_significant
+      end
+    end
+  end
+end

lib/msf/core/db_manager/import/mbsa.rb

@@ -0,0 +1,33 @@
+require 'rex/parser/mbsa_nokogiri'
+
+module Msf::DBManager::Import::MBSA
+  def import_mbsa_noko_stream(args={},&block)
+    if block
+      doc = Rex::Parser::MbsaDocument.new(args,framework.db) {|type, data| yield type,data }
+    else
+      doc = Rex::Parser::MbsaDocument.new(args,self)
+    end
+    parser = ::Nokogiri::XML::SAX::Parser.new(doc)
+    parser.parse(args[:data])
+  end
+
+  def import_mbsa_xml(args={}, &block)
+    bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : []
+    wspace = args[:wspace] || workspace
+    if Rex::Parser.nokogiri_loaded
+      parser = "Nokogiri v#{::Nokogiri::VERSION}"
+      noko_args = args.dup
+      noko_args[:blacklist] = bl
+      noko_args[:wspace] = wspace
+      if block
+        yield(:parser, parser)
+        import_mbsa_noko_stream(noko_args) {|type, data| yield type,data}
+      else
+        import_mbsa_noko_stream(noko_args)
+      end
+      return true
+    else # Sorry
+      raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.")
+    end
+  end
+end

lib/msf/core/db_manager/import/metasploit_framework.rb

@@ -0,0 +1,45 @@
+module Msf::DBManager::Import::MetasploitFramework
+  autoload :Credential, 'msf/core/db_manager/import/metasploit_framework/credential'
+  autoload :XML, 'msf/core/db_manager/import/metasploit_framework/xml'
+  autoload :Zip, 'msf/core/db_manager/import/metasploit_framework/zip'
+
+  include Msf::DBManager::Import::MetasploitFramework::Credential
+  include Msf::DBManager::Import::MetasploitFramework::XML
+  include Msf::DBManager::Import::MetasploitFramework::Zip
+
+  # Convert the string "NULL" to actual nil
+  def nils_for_nulls(str)
+    str == "NULL" ? nil : str
+  end
+
+  def unserialize_object(xml_elem, allow_yaml = false)
+    return nil unless xml_elem
+    string = xml_elem.text.to_s.strip
+    return string unless string.is_a?(String)
+    return nil if (string.empty? || string.nil?)
+
+    begin
+      # Validate that it is properly formed base64 first
+      if string.gsub(/\s+/, '') =~ /^([a-z0-9A-Z\+\/=]+)$/
+        Marshal.load($1.unpack("m")[0])
+      else
+        if allow_yaml
+          begin
+            YAML.load(string)
+          rescue
+            dlog("Badly formatted YAML: '#{string}'")
+            string
+          end
+        else
+          string
+        end
+      end
+    rescue ::Exception => e
+      if allow_yaml
+        YAML.load(string) rescue string
+      else
+        string
+      end
+    end
+  end
+end

lib/msf/core/db_manager/import/metasploit_framework/credential.rb

@@ -0,0 +1,36 @@
+module Msf::DBManager::Import::MetasploitFramework::Credential
+  # Import credentials given a path to a valid manifest file
+  #
+  # @param creds_dump_manifest_path [String]
+  # @param workspace [Mdm::Workspace] Default: {#workspace}
+  # @return [void]
+  def import_msf_cred_dump(creds_dump_manifest_path, workspace)
+    manifest_file = File.open(creds_dump_manifest_path)
+    origin = Metasploit::Credential::Origin::Import.create!(filename: File.basename(creds_dump_manifest_path))
+    importer = Metasploit::Credential::Importer::Core.new(workspace: workspace, input: manifest_file, origin: origin)
+    importer.import!
+  end
+
+  # Import credentials given a path to a valid manifest file
+  #
+  # @option args [String] :filename
+  # @option args [Mdm::Workspace] :wspace Default: {#workspace}
+  # @return [void]
+  def import_msf_cred_dump_zip(args = {})
+    wspace = args[:wspace] || workspace
+    origin = Metasploit::Credential::Origin::Import.create!(filename: File.basename(args[:filename]))
+    importer = Metasploit::Credential::Importer::Zip.new(workspace: wspace, input: File.open(args[:filename]), origin: origin)
+    importer.import!
+    nil
+  end
+
+  # Perform in an import of an msfpwdump file
+  def import_msf_pwdump(args={}, &block)
+    filename = File.basename(args[:data].path)
+    wspace   = args[:wspace] || workspace
+    origin   = Metasploit::Credential::Origin::Import.create!(filename: filename)
+    importer = Metasploit::Credential::Importer::Pwdump.new(input: args[:data], workspace: wspace, filename: filename, origin:origin)
+    importer.import!
+    importer.input.close unless importer.input.closed?
+  end
+end

lib/msf/core/db_manager/import/metasploit_framework/xml.rb

@@ -0,0 +1,600 @@
+# -*- coding: binary -*-
+# Handles importing of the xml format exported by Pro.  The methods are in a
+# module because (1) it's just good code layout and (2) it allows the
+# methods to be overridden in Pro without using alias_method_chain as
+# methods defined in a class cannot be overridden by including a module
+# (unless you're running Ruby 2.0 and can use prepend)
+module Msf::DBManager::Import::MetasploitFramework::XML
+  #
+  # CONSTANTS
+  #
+
+  # Elements that can be treated as text (i.e. do not need to be
+  # deserialized) in {#import_msf_web_page_element}
+  MSF_WEB_PAGE_TEXT_ELEMENT_NAMES = [
+      'auth',
+      'body',
+      'code',
+      'cookie',
+      'ctype',
+      'location',
+      'mtime'
+  ]
+
+  # Elements that can be treated as text (i.e. do not need to be
+  # deserialized) in {#import_msf_web_element}.
+  MSF_WEB_TEXT_ELEMENT_NAMES = [
+      'created-at',
+      'host',
+      'path',
+      'port',
+      'query',
+      'ssl',
+      'updated-at',
+      'vhost'
+  ]
+
+  # Elements that can be treated as text (i.e. do not need to be
+  # deserialized) in {#import_msf_web_vuln_element}.
+  MSF_WEB_VULN_TEXT_ELEMENT_NAMES = [
+      'blame',
+      'category',
+      'confidence',
+      'description',
+      'method',
+      'name',
+      'pname',
+      'proof',
+      'risk'
+  ]
+
+  #
+  # Instance Methods
+  #
+
+  # Import a Metasploit XML file.
+  def import_msf_file(args={})
+    filename = args[:filename]
+    wspace = args[:wspace] || workspace
+
+    data = ""
+    ::File.open(filename, 'rb') do |f|
+      data = f.read(f.stat.size)
+    end
+    import_msf_xml(args.merge(:data => data))
+  end
+
+  # Imports web_form element using {Msf::DBManager#report_web_form}.
+  #
+  # @param element [REXML::Element] web_form element.
+  # @param options [Hash{Symbol => Object}] options
+  # @option options [Boolean] :allow_yaml (false) Whether to allow YAML when
+  #   deserializing params.
+  # @option options [Mdm::Workspace, nil] :workspace
+  #   (Msf::DBManager#workspace) workspace under which to report the
+  #   Mdm::WebForm
+  # @yield [event, data]
+  # @yieldparam event [:web_page] The event name
+  # @yieldparam data [String] path
+  # @yieldreturn [void]
+  # @return [void]
+  def import_msf_web_form_element(element, options={}, &notifier)
+    options.assert_valid_keys(:allow_yaml, :workspace)
+
+    import_msf_web_element(element,
+                           :allow_yaml => options[:allow_yaml],
+                           :notifier => notifier,
+                           :type => :form,
+                           :workspace => options[:workspace]) do |element, options|
+      info = import_msf_text_element(element, 'method')
+
+      # FIXME https://www.pivotaltracker.com/story/show/46578647
+      # FIXME https://www.pivotaltracker.com/story/show/47128407
+      unserialized_params = unserialize_object(
+          element.elements['params'],
+          options[:allow_yaml]
+      )
+      info[:params] = nils_for_nulls(unserialized_params)
+
+      info
+    end
+  end
+
+  # Imports web_page element using {Msf::DBManager#report_web_page}.
+  #
+  # @param element [REXML::Element] web_page element.
+  # @param options [Hash{Symbol => Object}] options
+  # @option options [Boolean] :allow_yaml (false) Whether to allow YAML when
+  #   deserializing headers.
+  # @option options [Mdm::Workspace, nil] :workspace
+  #   (Msf::DBManager#workspace) workspace under which to report the
+  #   Mdm::WebPage.
+  # @yield [event, data]
+  # @yieldparam event [:web_page] The event name
+  # @yieldparam data [String] path
+  # @yieldreturn [void]
+  # @return [void]
+  def import_msf_web_page_element(element, options={}, &notifier)
+    options.assert_valid_keys(:allow_yaml, :workspace)
+
+    import_msf_web_element(element,
+                           :allow_yaml => options[:allow_yaml],
+                           :notifier => notifier,
+                           :type => :page,
+                           :workspace => options[:workspace]) do |element, options|
+      info = {}
+
+      MSF_WEB_PAGE_TEXT_ELEMENT_NAMES.each do |name|
+        element_info = import_msf_text_element(element, name)
+        info.merge!(element_info)
+      end
+
+      code = info[:code]
+
+      if code
+        info[:code] = code.to_i
+      end
+
+      # FIXME https://www.pivotaltracker.com/story/show/46578647
+      # FIXME https://www.pivotaltracker.com/story/show/47128407
+      unserialized_headers = unserialize_object(
+          element.elements['headers'],
+          options[:allow_yaml]
+      )
+      info[:headers] = nils_for_nulls(unserialized_headers)
+
+      info
+    end
+  end
+
+  # Imports web_vuln element using {Msf::DBManager#report_web_vuln}.
+  #
+  # @param element [REXML::Element] web_vuln element.
+  # @param options [Hash{Symbol => Object}] options
+  # @option options [Boolean] :allow_yaml (false) Whether to allow YAML when
+  #   deserializing headers.
+  # @option options [Mdm::Workspace, nil] :workspace
+  #   (Msf::DBManager#workspace) workspace under which to report the
+  #   Mdm::WebPage.
+  # @yield [event, data]
+  # @yieldparam event [:web_page] The event name
+  # @yieldparam data [String] path
+  # @yieldreturn [void]
+  # @return [void]
+  def import_msf_web_vuln_element(element, options={}, &notifier)
+    options.assert_valid_keys(:allow_yaml, :workspace)
+
+    import_msf_web_element(element,
+                           :allow_yaml => options[:allow_yaml],
+                           :notifier => notifier,
+                           :workspace => options[:workspace],
+                           :type => :vuln) do |element, options|
+      info = {}
+
+      MSF_WEB_VULN_TEXT_ELEMENT_NAMES.each do |name|
+        element_info = import_msf_text_element(element, name)
+        info.merge!(element_info)
+      end
+
+      confidence = info[:confidence]
+
+      if confidence
+        info[:confidence] = confidence.to_i
+      end
+
+      # FIXME https://www.pivotaltracker.com/story/show/46578647
+      # FIXME https://www.pivotaltracker.com/story/show/47128407
+      unserialized_params = unserialize_object(
+          element.elements['params'],
+          options[:allow_yaml]
+      )
+      info[:params] = nils_for_nulls(unserialized_params)
+
+      risk = info[:risk]
+
+      if risk
+        info[:risk] = risk.to_i
+      end
+
+      info
+    end
+  end
+
+  # For each host, step through services, notes, and vulns, and import
+  # them.
+  # TODO: loot, tasks, and reports
+  def import_msf_xml(args={}, &block)
+    data = args[:data]
+    wspace = args[:wspace] || workspace
+    bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : []
+
+    doc = rexmlify(data)
+    metadata = check_msf_xml_version!(doc)
+    allow_yaml = metadata[:allow_yaml]
+    btag = metadata[:root_tag]
+
+    doc.elements.each("/#{btag}/hosts/host") do |host|
+      host_data = {}
+      host_data[:task] = args[:task]
+      host_data[:workspace] = wspace
+      host_data[:host] = nils_for_nulls(host.elements["address"].text.to_s.strip)
+      if bl.include? host_data[:host]
+        next
+      else
+        yield(:address,host_data[:host]) if block
+      end
+      host_data[:mac] = nils_for_nulls(host.elements["mac"].text.to_s.strip)
+      if host.elements["comm"].text
+        host_data[:comm] = nils_for_nulls(host.elements["comm"].text.to_s.strip)
+      end
+      %W{created-at updated-at name state os-flavor os-lang os-name os-sp purpose}.each { |datum|
+        if host.elements[datum].text
+          host_data[datum.gsub('-','_')] = nils_for_nulls(host.elements[datum].text.to_s.strip)
+        end
+      }
+      host_address = host_data[:host].dup # Preserve after report_host() deletes
+      hobj = report_host(host_data)
+
+      host.elements.each("host_details/host_detail") do |hdet|
+        hdet_data = {}
+        hdet.elements.each do |det|
+          next if ["id", "host-id"].include?(det.name)
+          if det.text
+            hdet_data[det.name.gsub('-','_')] = nils_for_nulls(det.text.to_s.strip)
+          end
+        end
+        report_host_details(hobj, hdet_data)
+      end
+
+      host.elements.each("exploit_attempts/exploit_attempt") do |hdet|
+        hdet_data = {}
+        hdet.elements.each do |det|
+          next if ["id", "host-id", "session-id", "vuln-id", "service-id", "loot-id"].include?(det.name)
+          if det.text
+            hdet_data[det.name.gsub('-','_')] = nils_for_nulls(det.text.to_s.strip)
+          end
+        end
+        report_exploit_attempt(hobj, hdet_data)
+      end
+
+      host.elements.each('services/service') do |service|
+        service_data = {}
+        service_data[:task] = args[:task]
+        service_data[:workspace] = wspace
+        service_data[:host] = hobj
+        service_data[:port] = nils_for_nulls(service.elements["port"].text.to_s.strip).to_i
+        service_data[:proto] = nils_for_nulls(service.elements["proto"].text.to_s.strip)
+        %W{created-at updated-at name state info}.each { |datum|
+          if service.elements[datum].text
+            if datum == "info"
+              service_data["info"] = nils_for_nulls(unserialize_object(service.elements[datum], false))
+            else
+              service_data[datum.gsub("-","_")] = nils_for_nulls(service.elements[datum].text.to_s.strip)
+            end
+          end
+        }
+        report_service(service_data)
+      end
+
+      host.elements.each('notes/note') do |note|
+        note_data = {}
+        note_data[:workspace] = wspace
+        note_data[:host] = hobj
+        note_data[:type] = nils_for_nulls(note.elements["ntype"].text.to_s.strip)
+        note_data[:data] = nils_for_nulls(unserialize_object(note.elements["data"], allow_yaml))
+
+        if note.elements["critical"].text
+          note_data[:critical] = true unless note.elements["critical"].text.to_s.strip == "NULL"
+        end
+        if note.elements["seen"].text
+          note_data[:seen] = true unless note.elements["critical"].text.to_s.strip == "NULL"
+        end
+        %W{created-at updated-at}.each { |datum|
+          if note.elements[datum].text
+            note_data[datum.gsub("-","_")] = nils_for_nulls(note.elements[datum].text.to_s.strip)
+          end
+        }
+        report_note(note_data)
+      end
+
+      host.elements.each('tags/tag') do |tag|
+        tag_data = {}
+        tag_data[:addr] = host_address
+        tag_data[:wspace] = wspace
+        tag_data[:name] = tag.elements["name"].text.to_s.strip
+        tag_data[:desc] = tag.elements["desc"].text.to_s.strip
+        if tag.elements["report-summary"].text
+          tag_data[:summary] = tag.elements["report-summary"].text.to_s.strip
+        end
+        if tag.elements["report-detail"].text
+          tag_data[:detail] = tag.elements["report-detail"].text.to_s.strip
+        end
+        if tag.elements["critical"].text
+          tag_data[:crit] = true unless tag.elements["critical"].text.to_s.strip == "NULL"
+        end
+        report_host_tag(tag_data)
+      end
+
+      host.elements.each('vulns/vuln') do |vuln|
+        vuln_data = {}
+        vuln_data[:workspace] = wspace
+        vuln_data[:host] = hobj
+        vuln_data[:info] = nils_for_nulls(unserialize_object(vuln.elements["info"], allow_yaml))
+        vuln_data[:name] = nils_for_nulls(vuln.elements["name"].text.to_s.strip)
+        %W{created-at updated-at exploited-at}.each { |datum|
+          if vuln.elements[datum] and vuln.elements[datum].text
+            vuln_data[datum.gsub("-","_")] = nils_for_nulls(vuln.elements[datum].text.to_s.strip)
+          end
+        }
+        if vuln.elements["refs"]
+          vuln_data[:refs] = []
+          vuln.elements.each("refs/ref") do |ref|
+            vuln_data[:refs] << nils_for_nulls(ref.text.to_s.strip)
+          end
+        end
+
+        vobj = report_vuln(vuln_data)
+
+        vuln.elements.each("vuln_details/vuln_detail") do |vdet|
+          vdet_data = {}
+          vdet.elements.each do |det|
+            next if ["id", "vuln-id"].include?(det.name)
+            if det.text
+              vdet_data[det.name.gsub('-','_')] = nils_for_nulls(det.text.to_s.strip)
+            end
+          end
+          report_vuln_details(vobj, vdet_data)
+        end
+
+        vuln.elements.each("vuln_attempts/vuln_attempt") do |vdet|
+          vdet_data = {}
+          vdet.elements.each do |det|
+            next if ["id", "vuln-id", "loot-id", "session-id"].include?(det.name)
+            if det.text
+              vdet_data[det.name.gsub('-','_')] = nils_for_nulls(det.text.to_s.strip)
+            end
+          end
+          report_vuln_attempt(vobj, vdet_data)
+        end
+      end
+
+      ## Handle old-style (pre 4.10) XML files
+      if btag == "MetasploitV4"
+        if host.elements['creds'].present?
+          unless host.elements['creds'].elements.empty?
+            origin = Metasploit::Credential::Origin::Import.create(filename: "console-import-#{Time.now.to_i}")
+
+            host.elements.each('creds/cred') do |cred|
+              username = cred.elements['user'].try(:text)
+              proto    = cred.elements['proto'].try(:text)
+              sname    = cred.elements['sname'].try(:text)
+              port     = cred.elements['port'].try(:text)
+
+              # Handle blanks by resetting to sane default values
+              proto   = "tcp" if proto.blank?
+              pass     = cred.elements['pass'].try(:text)
+              pass     = "" if pass == "*MASKED*"
+
+              private = create_credential_private(private_data: pass, private_type: :password)
+              public  = create_credential_public(username: username)
+              core    = create_credential_core(private: private, public: public, origin: origin, workspace_id: wspace.id)
+
+              create_credential_login(core: core,
+                                      workspace_id: wspace.id,
+                                      address: hobj.address,
+                                      port: port,
+                                      protocol: proto,
+                                      service_name: sname,
+                                      status: Metasploit::Model::Login::Status::UNTRIED)
+            end
+          end
+        end
+      end
+
+
+      host.elements.each('sessions/session') do |sess|
+        sess_id = nils_for_nulls(sess.elements["id"].text.to_s.strip.to_i)
+        sess_data = {}
+        sess_data[:host] = hobj
+        %W{desc platform port stype}.each {|datum|
+          if sess.elements[datum].respond_to? :text
+            sess_data[datum.intern] = nils_for_nulls(sess.elements[datum].text.to_s.strip)
+          end
+        }
+        %W{opened-at close-reason closed-at via-exploit via-payload}.each {|datum|
+          if sess.elements[datum].respond_to? :text
+            sess_data[datum.gsub("-","_").intern] = nils_for_nulls(sess.elements[datum].text.to_s.strip)
+          end
+        }
+        sess_data[:datastore] = nils_for_nulls(unserialize_object(sess.elements["datastore"], allow_yaml))
+        if sess.elements["routes"]
+          sess_data[:routes] = nils_for_nulls(unserialize_object(sess.elements["routes"], allow_yaml)) || []
+        end
+        if not sess_data[:closed_at] # Fake a close if we don't already have one
+          sess_data[:closed_at] = Time.now.utc
+          sess_data[:close_reason] = "Imported at #{Time.now.utc}"
+        end
+
+        existing_session = get_session(
+            :workspace => sess_data[:host].workspace,
+            :addr => sess_data[:host].address,
+            :time => sess_data[:opened_at]
+        )
+        this_session = existing_session || report_session(sess_data)
+        next if existing_session
+        sess.elements.each('events/event') do |sess_event|
+          sess_event_data = {}
+          sess_event_data[:session] = this_session
+          %W{created-at etype local-path remote-path}.each {|datum|
+            if sess_event.elements[datum].respond_to? :text
+              sess_event_data[datum.gsub("-","_").intern] = nils_for_nulls(sess_event.elements[datum].text.to_s.strip)
+            end
+          }
+          %W{command output}.each {|datum|
+            if sess_event.elements[datum].respond_to? :text
+              sess_event_data[datum.gsub("-","_").intern] = nils_for_nulls(unserialize_object(sess_event.elements[datum], allow_yaml))
+            end
+          }
+          report_session_event(sess_event_data)
+        end
+      end
+    end
+
+
+    # Import web sites
+    doc.elements.each("/#{btag}/web_sites/web_site") do |web|
+      info = {}
+      info[:workspace] = wspace
+
+      %W{host port vhost ssl comments}.each do |datum|
+        if web.elements[datum].respond_to? :text
+          info[datum.intern] = nils_for_nulls(web.elements[datum].text.to_s.strip)
+        end
+      end
+
+      info[:options]   = nils_for_nulls(unserialize_object(web.elements["options"], allow_yaml)) if web.elements["options"].respond_to?(:text)
+      info[:ssl]       = (info[:ssl] and info[:ssl].to_s.strip.downcase == "true") ? true : false
+
+      %W{created-at updated-at}.each { |datum|
+        if web.elements[datum].text
+          info[datum.gsub("-","_")] = nils_for_nulls(web.elements[datum].text.to_s.strip)
+        end
+      }
+
+      report_web_site(info)
+      yield(:web_site, "#{info[:host]}:#{info[:port]} (#{info[:vhost]})") if block
+    end
+
+    %W{page form vuln}.each do |wtype|
+      doc.elements.each("/#{btag}/web_#{wtype}s/web_#{wtype}") do |element|
+        send(
+            "import_msf_web_#{wtype}_element",
+            element,
+            :allow_yaml => allow_yaml,
+            :workspace => wspace,
+            &block
+        )
+      end
+    end
+  end
+
+  private
+
+  # Checks if the XML document has a format version that the importer
+  # understands.
+  #
+  # @param document [REXML::Document] a REXML::Document produced by
+  #   {Msf::DBManager#rexmlify}.
+  # @return [Hash{Symbol => Object}] `:allow_yaml` is true if the format
+  #   requires YAML loading when calling
+  #   {Msf::DBManager#unserialize_object}.  `:root_tag` the tag name of the
+  #   root element for MSF XML.
+  # @raise [Msf::DBImportError] if unsupported format
+  def check_msf_xml_version!(document)
+    metadata = {
+        # FIXME https://www.pivotaltracker.com/story/show/47128407
+        :allow_yaml => false,
+        :root_tag => nil
+    }
+
+    if document.elements['MetasploitExpressV1']
+      # FIXME https://www.pivotaltracker.com/story/show/47128407
+      metadata[:allow_yaml] = true
+      metadata[:root_tag] = 'MetasploitExpressV1'
+    elsif document.elements['MetasploitExpressV2']
+      # FIXME https://www.pivotaltracker.com/story/show/47128407
+      metadata[:allow_yaml] = true
+      metadata[:root_tag] = 'MetasploitExpressV2'
+    elsif document.elements['MetasploitExpressV3']
+      metadata[:root_tag] = 'MetasploitExpressV3'
+    elsif document.elements['MetasploitExpressV4']
+      metadata[:root_tag] = 'MetasploitExpressV4'
+    elsif document.elements['MetasploitV4']
+      metadata[:root_tag] = 'MetasploitV4'
+    elsif document.elements['MetasploitV5']
+      metadata[:root_tag] = 'MetasploitV5'
+    end
+
+    unless metadata[:root_tag]
+      raise Msf::DBImportError,
+            'Unsupported Metasploit XML document format'
+    end
+
+    metadata
+  end
+
+  # Retrieves text of element if it exists.
+  #
+  # @param parent_element [REXML::Element] element under which element with
+  #   `child_name` exists.
+  # @param child_name [String] the name of the element under
+  #   `parent_element` whose text should be returned
+  # @return [{}] if element with child_name does not exist or does not have
+  #   text.
+  # @return [Hash{Symbol => String}] Maps child_name symbol to text. Text is
+  #   stripped and any NULLs are converted to `nil`.
+  # @return [nil] if element with `child_name` does not exist under
+  #   `parent_element`.
+  def import_msf_text_element(parent_element, child_name)
+    child_element = parent_element.elements[child_name]
+    info = {}
+
+    if child_element
+      stripped = child_element.text.to_s.strip
+      attribute_name = child_name.underscore.to_sym
+      info[attribute_name] = nils_for_nulls(stripped)
+    end
+
+    info
+  end
+
+  # Imports web_form, web_page, or web_vuln element using
+  # {Msf::DBManager#report_web_form}, {Msf::DBManager#report_web_page}, and
+  # {Msf::DBManager#report_web_vuln}, respectively.
+  #
+  # @param element [REXML::Element] the web_form, web_page, or web_vuln
+  #   element.
+  # @param options [Hash{Symbol => Object}] options
+  # @option options [Boolean] :allow_yaml (false) Whether to allow YAML when
+  #   deserializing elements.
+  # @option options [Proc] :notifier Block called with web_* event and path
+  # @option options [Symbol] :type the type of web element, such as :form,
+  #   :page, or :vuln.  Must correspond to a report_web_<type> method on
+  # {Msf::DBManager}.
+  # @option options [Mdm::Workspace, nil] :workspace
+  #   (Msf::DBManager#workspace) workspace under which to report the
+  #   imported record.
+  # @yield [element, options]
+  # @yieldparam element [REXML::Element] the web_form, web_page, or
+  #   web_vuln element passed to {#import_msf_web_element}.
+  # @yieldparam options [Hash{Symbol => Object}] options for parsing
+  # @yieldreturn [Hash{Symbol => Object}] info
+  # @return [void]
+  # @raise [KeyError] if `:type` is not given
+  def import_msf_web_element(element, options={}, &specialization)
+    options.assert_valid_keys(:allow_yaml, :notifier, :type, :workspace)
+    type = options.fetch(:type)
+
+    info = {}
+    info[:workspace] = options[:workspace] || self.workspace
+
+    MSF_WEB_TEXT_ELEMENT_NAMES.each do |name|
+      element_info = import_msf_text_element(element, name)
+      info.merge!(element_info)
+    end
+
+    info[:ssl] = (info[:ssl] and info[:ssl].to_s.strip.downcase == "true") ? true : false
+
+    specialized_info = specialization.call(element, options)
+    info.merge!(specialized_info)
+
+    self.send("report_web_#{type}", info)
+
+    notifier = options[:notifier]
+
+    if notifier
+      event = "web_#{type}".to_sym
+      notifier.call(event, info[:path])
+    end
+  end
+end

lib/msf/core/db_manager/import/metasploit_framework/zip.rb

@@ -0,0 +1,228 @@
+module Msf::DBManager::Import::MetasploitFramework::Zip
+  # Imports loot, tasks, and reports from an MSF ZIP report.
+  # XXX: This function is stupidly long. It needs to be refactored.
+  def import_msf_collateral(args={}, &block)
+    data = ::File.open(args[:filename], "rb") {|f| f.read(f.stat.size)}
+    wspace = args[:wspace] || args['wspace'] || workspace
+    bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : []
+    basedir = args[:basedir] || args['basedir'] || ::File.join(Msf::Config.data_directory, "msf")
+
+    allow_yaml = false
+    btag = nil
+
+    doc = rexmlify(data)
+    if doc.elements["MetasploitExpressV1"]
+      m_ver = 1
+      allow_yaml = true
+      btag = "MetasploitExpressV1"
+    elsif doc.elements["MetasploitExpressV2"]
+      m_ver = 2
+      allow_yaml = true
+      btag = "MetasploitExpressV2"
+    elsif doc.elements["MetasploitExpressV3"]
+      m_ver = 3
+      btag = "MetasploitExpressV3"
+    elsif doc.elements["MetasploitExpressV4"]
+      m_ver = 4
+      btag = "MetasploitExpressV4"
+    elsif doc.elements["MetasploitV4"]
+      m_ver = 4
+      btag = "MetasploitV4"
+    else
+      m_ver = nil
+    end
+    unless m_ver and btag
+      raise DBImportError.new("Unsupported Metasploit XML document format")
+    end
+
+    host_info = {}
+    doc.elements.each("/#{btag}/hosts/host") do |host|
+      host_info[host.elements["id"].text.to_s.strip] = nils_for_nulls(host.elements["address"].text.to_s.strip)
+    end
+
+    # Import Loot
+    doc.elements.each("/#{btag}/loots/loot") do |loot|
+      next if bl.include? host_info[loot.elements["host-id"].text.to_s.strip]
+      loot_info              = {}
+      loot_info[:host]       = host_info[loot.elements["host-id"].text.to_s.strip]
+      loot_info[:workspace]  = args[:wspace]
+      loot_info[:ctype]      = nils_for_nulls(loot.elements["content-type"].text.to_s.strip)
+      loot_info[:info]       = nils_for_nulls(unserialize_object(loot.elements["info"], allow_yaml))
+      loot_info[:ltype]      = nils_for_nulls(loot.elements["ltype"].text.to_s.strip)
+      loot_info[:name]       = nils_for_nulls(loot.elements["name"].text.to_s.strip)
+      loot_info[:created_at] = nils_for_nulls(loot.elements["created-at"].text.to_s.strip)
+      loot_info[:updated_at] = nils_for_nulls(loot.elements["updated-at"].text.to_s.strip)
+      loot_info[:name]       = nils_for_nulls(loot.elements["name"].text.to_s.strip)
+      loot_info[:orig_path]  = nils_for_nulls(loot.elements["path"].text.to_s.strip)
+      loot_info[:task]       = args[:task]
+      tmp = args[:ifd][:zip_tmp]
+      loot_info[:orig_path].gsub!(/^\./,tmp) if loot_info[:orig_path]
+      if !loot.elements["service-id"].text.to_s.strip.empty?
+        unless loot.elements["service-id"].text.to_s.strip == "NULL"
+          loot_info[:service] = loot.elements["service-id"].text.to_s.strip
+        end
+      end
+
+      # Only report loot if we actually have it.
+      # TODO: Copypasta. Seperate this out.
+      if ::File.exists? loot_info[:orig_path]
+        loot_dir = ::File.join(basedir,"loot")
+        loot_file = ::File.split(loot_info[:orig_path]).last
+        if ::File.exists? loot_dir
+          unless (::File.directory?(loot_dir) && ::File.writable?(loot_dir))
+            raise DBImportError.new("Could not move files to #{loot_dir}")
+          end
+        else
+          ::FileUtils.mkdir_p(loot_dir)
+        end
+        new_loot = ::File.join(loot_dir,loot_file)
+        loot_info[:path] = new_loot
+        if ::File.exists?(new_loot)
+          ::File.unlink new_loot # Delete it, and don't report it.
+        else
+          report_loot(loot_info) # It's new, so report it.
+        end
+        ::FileUtils.copy(loot_info[:orig_path], new_loot)
+        yield(:msf_loot, new_loot) if block
+      end
+    end
+
+    # Import Tasks
+    doc.elements.each("/#{btag}/tasks/task") do |task|
+      task_info = {}
+      task_info[:workspace] = args[:wspace]
+      # Should user be imported (original) or declared (the importing user)?
+      task_info[:user] = nils_for_nulls(task.elements["created-by"].text.to_s.strip)
+      task_info[:desc] = nils_for_nulls(task.elements["description"].text.to_s.strip)
+      task_info[:info] = nils_for_nulls(unserialize_object(task.elements["info"], allow_yaml))
+      task_info[:mod] = nils_for_nulls(task.elements["module"].text.to_s.strip)
+      task_info[:options] = nils_for_nulls(task.elements["options"].text.to_s.strip)
+      task_info[:prog] = nils_for_nulls(task.elements["progress"].text.to_s.strip).to_i
+      task_info[:created_at] = nils_for_nulls(task.elements["created-at"].text.to_s.strip)
+      task_info[:updated_at] = nils_for_nulls(task.elements["updated-at"].text.to_s.strip)
+      if !task.elements["completed-at"].text.to_s.empty?
+        task_info[:completed_at] = nils_for_nulls(task.elements["completed-at"].text.to_s.strip)
+      end
+      if !task.elements["error"].text.to_s.empty?
+        task_info[:error] = nils_for_nulls(task.elements["error"].text.to_s.strip)
+      end
+      if !task.elements["result"].text.to_s.empty?
+        task_info[:result] = nils_for_nulls(task.elements["result"].text.to_s.strip)
+      end
+      task_info[:orig_path] = nils_for_nulls(task.elements["path"].text.to_s.strip)
+      tmp = args[:ifd][:zip_tmp]
+      task_info[:orig_path].gsub!(/^\./,tmp) if task_info[:orig_path]
+
+      # Only report a task if we actually have it.
+      # TODO: Copypasta. Seperate this out.
+      if ::File.exists? task_info[:orig_path]
+        tasks_dir = ::File.join(basedir,"tasks")
+        task_file = ::File.split(task_info[:orig_path]).last
+        if ::File.exists? tasks_dir
+          unless (::File.directory?(tasks_dir) && ::File.writable?(tasks_dir))
+            raise DBImportError.new("Could not move files to #{tasks_dir}")
+          end
+        else
+          ::FileUtils.mkdir_p(tasks_dir)
+        end
+        new_task = ::File.join(tasks_dir,task_file)
+        task_info[:path] = new_task
+        if ::File.exists?(new_task)
+          ::File.unlink new_task # Delete it, and don't report it.
+        else
+          report_task(task_info) # It's new, so report it.
+        end
+        ::FileUtils.copy(task_info[:orig_path], new_task)
+        yield(:msf_task, new_task) if block
+      end
+    end
+
+    # Import Reports
+    doc.elements.each("/#{btag}/reports/report") do |report|
+      import_report(report, args, basedir)
+    end
+  end
+
+  # Import a Metasploit Express ZIP file. Note that this requires
+  # a fair bit of filesystem manipulation, and is very much tied
+  # up with the Metasploit Express ZIP file format export (for
+  # obvious reasons). In the event directories exist, they will
+  # be reused. If target files exist, they will be overwritten.
+  #
+  # XXX: Refactor so it's not quite as sanity-blasting.
+  def import_msf_zip(args={}, &block)
+    data = args[:data]
+    wspace = args[:wspace] || workspace
+    bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : []
+
+    new_tmp = ::File.join(Dir::tmpdir,"msf","imp_#{Rex::Text::rand_text_alphanumeric(4)}",@import_filedata[:zip_basename])
+    if ::File.exists? new_tmp
+      unless (::File.directory?(new_tmp) && ::File.writable?(new_tmp))
+        raise DBImportError.new("Could not extract zip file to #{new_tmp}")
+      end
+    else
+      FileUtils.mkdir_p(new_tmp)
+    end
+    @import_filedata[:zip_tmp] = new_tmp
+
+    # Grab the list of unique basedirs over all entries.
+    @import_filedata[:zip_tmp_subdirs] = @import_filedata[:zip_entry_names].map {|x| ::File.split(x)}.map {|x| x[0]}.uniq.reject {|x| x == "."}
+
+    # mkdir all of the base directores we just pulled out, if they don't
+    # already exist
+    @import_filedata[:zip_tmp_subdirs].each {|sub|
+      tmp_subdirs = ::File.join(@import_filedata[:zip_tmp],sub)
+      if File.exists? tmp_subdirs
+        unless (::File.directory?(tmp_subdirs) && File.writable?(tmp_subdirs))
+          # if it exists but we can't write to it, give up
+          raise DBImportError.new("Could not extract zip file to #{tmp_subdirs}")
+        end
+      else
+        ::FileUtils.mkdir(tmp_subdirs)
+      end
+    }
+
+    data.entries.each do |e|
+      target = ::File.join(@import_filedata[:zip_tmp], e.name)
+      data.extract(e,target)
+
+      if target =~ /\.xml\z/
+        target_data = ::File.open(target, "rb") {|f| f.read 1024}
+        if import_filetype_detect(target_data) == :msf_xml
+          @import_filedata[:zip_extracted_xml] = target
+        end
+      end
+    end
+
+    # Import any creds if there are some in the import file
+    Dir.entries(@import_filedata[:zip_tmp]).each do |entry|
+      if entry =~ /^.*#{Regexp.quote(Metasploit::Credential::Exporter::Core::CREDS_DUMP_FILE_IDENTIFIER)}.*/
+        manifest_file_path = File.join(@import_filedata[:zip_tmp], entry, Metasploit::Credential::Importer::Zip::MANIFEST_FILE_NAME)
+        if File.exists? manifest_file_path
+          import_msf_cred_dump(manifest_file_path, wspace)
+        end
+      end
+    end
+
+    # This will kick the newly-extracted XML file through
+    # the import_file process all over again.
+    if @import_filedata[:zip_extracted_xml]
+      new_args = args.dup
+      new_args[:filename] = @import_filedata[:zip_extracted_xml]
+      new_args[:data] = nil
+      new_args[:ifd] = @import_filedata.dup
+      if block
+        import_file(new_args, &block)
+      else
+        import_file(new_args)
+      end
+    end
+
+    # Kick down to all the MSFX ZIP specific items
+    if block
+      import_msf_collateral(new_args, &block)
+    else
+      import_msf_collateral(new_args)
+    end
+  end
+end

lib/msf/core/db_manager/import/nessus.rb

@@ -0,0 +1,81 @@
+module Msf::DBManager::Import::Nessus
+  autoload :NBE, 'msf/core/db_manager/import/nessus/nbe'
+  autoload :XML, 'msf/core/db_manager/import/nessus/xml'
+
+  include Msf::DBManager::Import::Nessus::NBE
+  include Msf::DBManager::Import::Nessus::XML
+
+  protected
+
+  #
+  # This holds all of the shared parsing/handling used by the
+  # Nessus NBE and NESSUS v1 methods
+  #
+  def handle_nessus(wspace, hobj, port, nasl, plugin_name, severity, data,task=nil)
+    addr = hobj.address
+    # The port section looks like:
+    #   http (80/tcp)
+    p = port.match(/^([^\(]+)\((\d+)\/([^\)]+)\)/)
+    return if not p
+
+    # Unnecessary as the caller should already have reported this host
+    #report_host(:workspace => wspace, :host => addr, :state => Msf::HostState::Alive)
+    name = p[1].strip
+    port = p[2].to_i
+    proto = p[3].downcase
+
+    info = { :workspace => wspace, :host => hobj, :port => port, :proto => proto, :task => task }
+    if name != "unknown" and name[-1,1] != "?"
+      info[:name] = name
+    end
+    report_service(info)
+
+    if nasl.nil? || nasl.empty? || nasl == 0 || nasl == "0"
+      return
+    end
+
+    data.gsub!("\\n", "\n")
+
+    refs = []
+
+    if (data =~ /^CVE : (.*)$/)
+      $1.gsub(/C(VE|AN)\-/, '').split(',').map { |r| r.strip }.each do |r|
+        refs.push('CVE-' + r)
+      end
+    end
+
+    if (data =~ /^BID : (.*)$/)
+      $1.split(',').map { |r| r.strip }.each do |r|
+        refs.push('BID-' + r)
+      end
+    end
+
+    if (data =~ /^Other references : (.*)$/)
+      $1.split(',').map { |r| r.strip }.each do |r|
+        ref_id, ref_val = r.split(':')
+        ref_val ? refs.push(ref_id + '-' + ref_val) : refs.push(ref_id)
+      end
+    end
+
+    nss = 'NSS-' + nasl.to_s.strip
+    refs << nss
+
+    unless plugin_name.to_s.strip.empty?
+      vuln_name = plugin_name
+    else
+      vuln_name = nss
+    end
+
+    vuln_info = {
+      :workspace => wspace,
+      :host => hobj,
+      :port => port,
+      :proto => proto,
+      :name => vuln_name,
+      :info => data,
+      :refs => refs,
+      :task => task,
+    }
+    report_vuln(vuln_info)
+  end
+end

lib/msf/core/db_manager/import/nessus/nbe.rb

@@ -0,0 +1,94 @@
+module Msf::DBManager::Import::Nessus::NBE
+  # There is no place the NBE actually stores the plugin name used to
+  # scan. You get "Security Note" or "Security Warning," and that's it.
+  def import_nessus_nbe(args={}, &block)
+    nbe_data = args[:data]
+    wspace = args[:wspace] || workspace
+    bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : []
+
+    nbe_copy = nbe_data.dup
+    # First pass, just to build the address map.
+    addr_map = {}
+
+    # Cache host objects before passing into handle_nessus()
+    hobj_map = {}
+
+    nbe_copy.each_line do |line|
+      r = line.split('|')
+      next if r[0] != 'results'
+      next if r[4] != "12053"
+      data = r[6]
+      addr,hname = data.match(/([0-9\x2e]+) resolves as (.+)\x2e\\n/n)[1,2]
+      addr_map[hname] = addr
+    end
+
+    nbe_data.each_line do |line|
+      r = line.split('|')
+      next if r[0] != 'results'
+      hname = r[2]
+      if addr_map[hname]
+        addr = addr_map[hname]
+      else
+        addr = hname # Must be unresolved, probably an IP address.
+      end
+      port = r[3]
+      nasl = r[4]
+      type = r[5]
+      data = r[6]
+
+      # If there's no resolution, or if it's malformed, skip it.
+      next unless ipv46_validator(addr)
+
+      if bl.include? addr
+        next
+      else
+        yield(:address,addr) if block
+      end
+
+      hobj_map[ addr ] ||= report_host(:host => addr, :workspace => wspace, :task => args[:task])
+
+      # Match the NBE types with the XML severity ratings
+      case type
+      # log messages don't actually have any data, they are just
+      # complaints about not being able to perform this or that test
+      # because such-and-such was missing
+      when "Log Message"; next
+      when "Security Hole"; severity = 3
+      when "Security Warning"; severity = 2
+      when "Security Note"; severity = 1
+      # a severity 0 means there's no extra data, it's just an open port
+      else; severity = 0
+      end
+      if nasl == "11936"
+        os = data.match(/The remote host is running (.*)\\n/)[1]
+        report_note(
+          :workspace => wspace,
+          :task => args[:task],
+          :host => hobj_map[ addr ],
+          :type => 'host.os.nessus_fingerprint',
+          :data => {
+            :os => os.to_s.strip
+          }
+        )
+      end
+
+      next if nasl.to_s.strip.empty?
+      plugin_name = nil # NBE doesn't ever populate this
+      handle_nessus(wspace, hobj_map[ addr ], port, nasl, plugin_name, severity, data)
+    end
+  end
+
+  #
+  # Import Nessus NBE files
+  #
+  def import_nessus_nbe_file(args={})
+    filename = args[:filename]
+    wspace = args[:wspace] || workspace
+
+    data = ""
+    ::File.open(filename, 'rb') do |f|
+      data = f.read(f.stat.size)
+    end
+    import_nessus_nbe(args.merge(:data => data))
+  end
+end

lib/msf/core/db_manager/import/nessus/xml.rb

@@ -0,0 +1,28 @@
+module Msf::DBManager::Import::Nessus::XML
+  autoload :V1, 'msf/core/db_manager/import/nessus/xml/v1'
+  autoload :V2, 'msf/core/db_manager/import/nessus/xml/v2'
+
+  include Msf::DBManager::Import::Nessus::XML::V1
+  include Msf::DBManager::Import::Nessus::XML::V2
+
+  #
+  # Import Nessus XML v1 and v2 output
+  #
+  # Old versions of openvas exported this as well
+  #
+  def import_nessus_xml_file(args={})
+    filename = args[:filename]
+    wspace = args[:wspace] || workspace
+
+    data = ""
+    ::File.open(filename, 'rb') do |f|
+      data = f.read(f.stat.size)
+    end
+
+    if data.index("NessusClientData_v2")
+      import_nessus_xml_v2(args.merge(:data => data))
+    else
+      import_nessus_xml(args.merge(:data => data))
+    end
+  end
+end

lib/msf/core/db_manager/import/nessus/xml/v1.rb

@@ -0,0 +1,65 @@
+module Msf::DBManager::Import::Nessus::XML::V1
+  def import_nessus_xml(args={}, &block)
+    data = args[:data]
+    wspace = args[:wspace] || workspace
+    bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : []
+
+    doc = rexmlify(data)
+    doc.elements.each('/NessusClientData/Report/ReportHost') do |host|
+      hobj = nil
+      addr = nil
+      hname = nil
+      os = nil
+      # If the name is resolved, the Nessus plugin for DNS
+      # resolution should be there. If not, fall back to the
+      # HostName
+      host.elements.each('ReportItem') do |item|
+        next unless item.elements['pluginID'].text == "12053"
+        addr = item.elements['data'].text.match(/([0-9\x2e]+) resolves as/n)[1]
+        hname = host.elements['HostName'].text
+      end
+      addr ||= host.elements['HostName'].text
+      next unless ipv46_validator(addr) # Skip resolved names and SCAN-ERROR.
+      if bl.include? addr
+        next
+      else
+        yield(:address,addr) if block
+      end
+
+      hinfo = {
+        :workspace => wspace,
+        :host => addr,
+        :task => args[:task]
+      }
+
+      # Record the hostname
+      hinfo.merge!(:name => hname.to_s.strip) if hname
+      hobj = report_host(hinfo)
+      report_import_note(wspace,hobj)
+
+      # Record the OS
+      os ||= host.elements["os_name"]
+      if os
+        report_note(
+          :workspace => wspace,
+          :task => args[:task],
+          :host => hobj,
+          :type => 'host.os.nessus_fingerprint',
+          :data => {
+            :os => os.text.to_s.strip
+          }
+        )
+      end
+
+      host.elements.each('ReportItem') do |item|
+        nasl = item.elements['pluginID'].text
+        plugin_name = item.elements['pluginName'].text
+        port = item.elements['port'].text
+        data = item.elements['data'].text
+        severity = item.elements['severity'].text
+
+        handle_nessus(wspace, hobj, port, nasl, plugin_name, severity, data, args[:task])
+      end
+    end
+  end
+end

lib/msf/core/db_manager/import/nessus/xml/v2.rb

@@ -0,0 +1,165 @@
+require 'rex/parser/nessus_xml'
+
+module Msf::DBManager::Import::Nessus::XML::V2
+  def import_nessus_xml_v2(args={}, &block)
+    data = args[:data]
+    wspace = args[:wspace] || workspace
+    bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : []
+
+    #@host = {
+        #'hname'             => nil,
+        #'addr'              => nil,
+        #'mac'               => nil,
+        #'os'                => nil,
+        #'ports'             => [ 'port' => {    'port'              	=> nil,
+        #					'svc_name'              => nil,
+        #					'proto'              	=> nil,
+        #					'severity'              => nil,
+        #					'nasl'              	=> nil,
+        #					'description'           => nil,
+        #					'cve'                   => [],
+        #					'bid'                   => [],
+        #					'xref'                  => []
+        #				}
+        #			]
+        #}
+    parser = Rex::Parser::NessusXMLStreamParser.new
+    parser.on_found_host = Proc.new { |host|
+
+      hobj = nil
+      addr = host['addr'] || host['hname']
+
+      next unless ipv46_validator(addr) # Catches SCAN-ERROR, among others.
+
+      if bl.include? addr
+        next
+      else
+        yield(:address,addr) if block
+      end
+
+      os = host['os']
+      hname = host['hname']
+      mac = host['mac']
+
+      host_info = {
+        :workspace => wspace,
+        :host => addr,
+        :task => args[:task]
+      }
+      host_info[:name] = hname.to_s.strip if hname
+      # Short mac, protect against Nessus's habit of saving multiple macs
+      # We can't use them anyway, so take just the first.
+      host_info[:mac]  = mac.to_s.strip.upcase.split(/\s+/).first if mac
+
+      hobj = report_host(host_info)
+      report_import_note(wspace,hobj)
+
+      os = host['os']
+      yield(:os,os) if block
+      if os
+        report_note(
+          :workspace => wspace,
+          :task => args[:task],
+          :host => hobj,
+          :type => 'host.os.nessus_fingerprint',
+          :data => {
+            :os => os.to_s.strip
+          }
+        )
+      end
+
+      host['ports'].each do |item|
+        next if item['port'] == 0
+        msf = nil
+        nasl = item['nasl'].to_s
+        nasl_name = item['nasl_name'].to_s
+        port = item['port'].to_s
+        proto = item['proto'] || "tcp"
+        sname = item['svc_name']
+        severity = item['severity']
+        description = item['description']
+        cve = item['cve']
+        bid = item['bid']
+        xref = item['xref']
+        msf = item['msf']
+
+        yield(:port,port) if block
+
+        handle_nessus_v2(wspace, hobj, port, proto, sname, nasl, nasl_name, severity, description, cve, bid, xref, msf, args[:task])
+
+      end
+      yield(:end,hname) if block
+    }
+
+    REXML::Document.parse_stream(data, parser)
+
+  end
+
+  protected
+
+  #
+  # NESSUS v2 file format has a dramatically different layout
+  # for ReportItem data
+  #
+  def handle_nessus_v2(wspace,hobj,port,proto,name,nasl,nasl_name,severity,description,cve,bid,xref,msf,task=nil)
+    addr = hobj.address
+
+    info = { :workspace => wspace, :host => hobj, :port => port, :proto => proto, :task => task }
+
+    unless name =~ /^unknown$|\?$/
+      info[:name] = name
+    end
+
+    if port.to_i != 0
+      report_service(info)
+    end
+
+    if nasl.nil? || nasl.empty? || nasl == 0 || nasl == "0"
+      return
+    end
+
+    refs = []
+
+    cve.each do |r|
+      r.to_s.gsub!(/C(VE|AN)\-/, '')
+      refs.push('CVE-' + r.to_s)
+    end if cve
+
+    bid.each do |r|
+      refs.push('BID-' + r.to_s)
+    end if bid
+
+    xref.each do |r|
+      ref_id, ref_val = r.to_s.split(':')
+      ref_val ? refs.push(ref_id + '-' + ref_val) : refs.push(ref_id)
+    end if xref
+
+    msfref = "MSF-" << msf if msf
+    refs.push msfref if msfref
+
+    nss = 'NSS-' + nasl
+    if nasl_name.nil? || nasl_name.empty?
+      vuln_name = nss
+    else
+      vuln_name = nasl_name
+    end
+
+    refs << nss.strip
+
+    vuln = {
+      :workspace => wspace,
+      :host => hobj,
+      :name => vuln_name,
+      :info => description ? description : "",
+      :refs => refs,
+      :task => task,
+    }
+
+    if port.to_i != 0
+      vuln[:port]  = port
+      vuln[:proto] = proto
+    end
+
+    report_vuln(vuln)
+  end
+end

lib/msf/core/db_manager/import/netsparker.rb

@@ -0,0 +1,390 @@
+require 'rex/parser/netsparker_xml'
+
+module Msf::DBManager::Import::Netsparker
+  # Process NetSparker XML
+  def import_netsparker_xml(args={}, &block)
+    data = args[:data]
+    wspace = args[:wspace] || workspace
+    bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : []
+    addr = nil
+    parser = Rex::Parser::NetSparkerXMLStreamParser.new
+    parser.on_found_vuln = Proc.new do |vuln|
+      data = {:workspace => wspace}
+
+      # Parse the URL
+      url  = vuln['url']
+      return if not url
+
+      # Crack the URL into a URI
+      uri = URI(url) rescue nil
+      return if not uri
+
+      # Resolve the host and cache the IP
+      if not addr
+        baddr = Rex::Socket.addr_aton(uri.host) rescue nil
+        if baddr
+          addr = Rex::Socket.addr_ntoa(baddr)
+          yield(:address, addr) if block
+        end
+      end
+
+      # Bail early if we have no IP address
+      if not addr
+        raise Interrupt, "Not a valid IP address"
+      end
+
+      if bl.include?(addr)
+        raise Interrupt, "IP address is on the blacklist"
+      end
+
+      data[:host]  = addr
+      data[:vhost] = uri.host
+      data[:port]  = uri.port
+      data[:ssl]   = (uri.scheme == "ssl")
+
+      body = nil
+      # First report a web page
+      if vuln['response']
+        headers = {}
+        code    = 200
+        head,body = vuln['response'].to_s.split(/\r?\n\r?\n/, 2)
+        if body
+
+          if head =~ /^HTTP\d+\.\d+\s+(\d+)\s*/
+            code = $1.to_i
+          end
+
+          headers = {}
+          head.split(/\r?\n/).each do |line|
+            hname,hval = line.strip.split(/\s*:\s*/, 2)
+            next if hval.to_s.strip.empty?
+            headers[hname.downcase] ||= []
+            headers[hname.downcase] << hval
+          end
+
+          info = {
+            :path     => uri.path,
+            :query    => uri.query,
+            :code     => code,
+            :body     => body,
+            :headers  => headers,
+            :task     => args[:task]
+          }
+          info.merge!(data)
+
+          if headers['content-type']
+            info[:ctype] = headers['content-type'][0]
+          end
+
+          if headers['set-cookie']
+            info[:cookie] = headers['set-cookie'].join("\n")
+          end
+
+          if headers['authorization']
+            info[:auth] = headers['authorization'].join("\n")
+          end
+
+          if headers['location']
+            info[:location] = headers['location'][0]
+          end
+
+          if headers['last-modified']
+            info[:mtime] = headers['last-modified'][0]
+          end
+
+          # Report the web page to the database
+          report_web_page(info)
+
+          yield(:web_page, url) if block
+        end
+      end # End web_page reporting
+
+
+      details = netsparker_vulnerability_map(vuln)
+
+      method = netsparker_method_map(vuln)
+      pname  = netsparker_pname_map(vuln)
+      params = netsparker_params_map(vuln)
+
+      proof  = ''
+
+      if vuln['info'] and vuln['info'].length > 0
+        proof << vuln['info'].map{|x| "#{x[0]}: #{x[1]}\n" }.join + "\n"
+      end
+
+      if proof.empty?
+        if body
+          proof << body + "\n"
+        else
+          proof << vuln['response'].to_s + "\n"
+        end
+      end
+
+      if params.empty? and pname
+        params = [[pname, vuln['vparam_name'].to_s]]
+      end
+
+      info = {
+        # XXX: There is a :request attr in the model, but report_web_vuln
+        # doesn't seem to know about it, so this gets ignored.
+        #:request  => vuln['request'],
+        :path        => uri.path,
+        :query       => uri.query,
+        :method      => method,
+        :params      => params,
+        :pname       => pname.to_s,
+        :proof       => proof,
+        :risk        => details[:risk],
+        :name        => details[:name],
+        :blame       => details[:blame],
+        :category    => details[:category],
+        :description => details[:description],
+        :confidence  => details[:confidence],
+        :task        => args[:task]
+      }
+      info.merge!(data)
+
+      next if vuln['type'].to_s.empty?
+
+      report_web_vuln(info)
+      yield(:web_vuln, url) if block
+    end
+
+    # We throw interrupts in our parser when the job is hopeless
+    begin
+      REXML::Document.parse_stream(data, parser)
+    rescue ::Interrupt => e
+      wlog("The netsparker_xml_import() job was interrupted: #{e}")
+    end
+  end
+
+  # Process a NetSparker XML file
+  def import_netsparker_xml_file(args={})
+    filename = args[:filename]
+    wspace = args[:wspace] || workspace
+
+    data = ""
+    ::File.open(filename, 'rb') do |f|
+      data = f.read(f.stat.size)
+    end
+    import_netsparker_xml(args.merge(:data => data))
+  end
+
+  def netsparker_method_map(vuln)
+    case vuln['vparam_type']
+    when "FullQueryString"
+      "GET"
+    when "Querystring"
+      "GET"
+    when "Post"
+      "POST"
+    when "RawUrlInjection"
+      "GET"
+    else
+      "GET"
+    end
+  end
+
+  def netsparker_params_map(vuln)
+    []
+  end
+
+  def netsparker_pname_map(vuln)
+    case vuln['vparam_name']
+    when "URI-BASED", "Query Based"
+      "PATH"
+    else
+      vuln['vparam_name']
+    end
+  end
+
+  def netsparker_vulnerability_map(vuln)
+    res = {
+      :risk => 1,
+      :name  => 'Information Disclosure',
+      :blame => 'System Administrator',
+      :category => 'info',
+      :description => "This is an information leak",
+      :confidence => 100
+    }
+
+    # Risk is a value from 1-5 indicating the severity of the issue
+    #	Examples: 1, 4, 5
+
+    # Name is a descriptive name for this vulnerability.
+    #	Examples: XSS, ReflectiveXSS, PersistentXSS
+
+    # Blame indicates who is at fault for the vulnerability
+    #	Examples: App Developer, Server Developer, System Administrator
+
+    # Category indicates the general class of vulnerability
+    #	Examples: info, xss, sql, rfi, lfi, cmd
+
+    # Description is a textual summary of the vulnerability
+    #	Examples: "A reflective cross-site scripting attack"
+    #             "The web server leaks the internal IP address"
+    #             "The cookie is not set to HTTP-only"
+
+    #
+    # Confidence is a value from 1 to 100 indicating how confident the
+    # software is that the results are valid.
+    #	Examples: 100, 90, 75, 15, 10, 0
+
+    case vuln['type'].to_s
+    when "ApacheDirectoryListing"
+      res = {
+        :risk => 1,
+        :name  => 'Directory Listing',
+        :blame => 'System Administrator',
+        :category => 'info',
+        :description => "",
+        :confidence => 100
+      }
+    when "ApacheMultiViewsEnabled"
+      res = {
+        :risk => 1,
+        :name  => 'Apache MultiViews Enabled',
+        :blame => 'System Administrator',
+        :category => 'info',
+        :description => "",
+        :confidence => 100
+      }
+    when "ApacheVersion"
+      res = {
+        :risk => 1,
+        :name  => 'Web Server Version',
+        :blame => 'System Administrator',
+        :category => 'info',
+        :description => "",
+        :confidence => 100
+      }
+    when "PHPVersion"
+      res = {
+        :risk => 1,
+        :name  => 'PHP Module Version',
+        :blame => 'System Administrator',
+        :category => 'info',
+        :description => "",
+        :confidence => 100
+      }
+    when "AutoCompleteEnabled"
+      res = {
+        :risk => 1,
+        :name  => 'Form AutoComplete Enabled',
+        :blame => 'App Developer',
+        :category => 'info',
+        :description => "",
+        :confidence => 100
+      }
+    when "CookieNotMarkedAsHttpOnly"
+      res = {
+        :risk => 1,
+        :name  => 'Cookie Not HttpOnly',
+        :blame => 'App Developer',
+        :category => 'info',
+        :description => "",
+        :confidence => 100
+      }
+    when "EmailDisclosure"
+      res = {
+        :risk => 1,
+        :name  => 'Email Address Disclosure',
+        :blame => 'App Developer',
+        :category => 'info',
+        :description => "",
+        :confidence => 100
+      }
+    when "ForbiddenResource"
+      res = {
+        :risk => 1,
+        :name  => 'Forbidden Resource',
+        :blame => 'App Developer',
+        :category => 'info',
+        :description => "",
+        :confidence => 100
+      }
+    when "FileUploadFound"
+      res = {
+        :risk => 1,
+        :name  => 'File Upload Form',
+        :blame => 'App Developer',
+        :category => 'info',
+        :description => "",
+        :confidence => 100
+      }
+    when "PasswordOverHTTP"
+      res = {
+        :risk => 2,
+        :name  => 'Password Over HTTP',
+        :blame => 'App Developer',
+        :category => 'info',
+        :description => "",
+        :confidence => 100
+      }
+    when "MySQL5Identified"
+      res = {
+        :risk => 1,
+        :name  => 'MySQL 5 Identified',
+        :blame => 'App Developer',
+        :category => 'info',
+        :description => "",
+        :confidence => 100
+      }
+    when "PossibleInternalWindowsPathLeakage"
+      res = {
+        :risk => 1,
+        :name  => 'Path Leakage - Windows',
+        :blame => 'App Developer',
+        :category => 'info',
+        :description => "",
+        :confidence => 100
+      }
+    when "PossibleInternalUnixPathLeakage"
+      res = {
+        :risk => 1,
+        :name  => 'Path Leakage - Unix',
+        :blame => 'App Developer',
+        :category => 'info',
+        :description => "",
+        :confidence => 100
+      }
+    when "PossibleXSS", "LowPossibilityPermanentXSS", "XSS", "PermanentXSS"
+      conf = 100
+      conf = 25  if vuln['type'].to_s == "LowPossibilityPermanentXSS"
+      conf = 50  if vuln['type'].to_s == "PossibleXSS"
+      res = {
+        :risk => 3,
+        :name  => 'Cross-Site Scripting',
+        :blame => 'App Developer',
+        :category => 'xss',
+        :description => "",
+        :confidence => conf
+      }
+
+    when "ConfirmedBlindSQLInjection", "ConfirmedSQLInjection", "HighlyPossibleSqlInjection", "DatabaseErrorMessages"
+      conf = 100
+      conf = 90  if vuln['type'].to_s == "HighlyPossibleSqlInjection"
+      conf = 25  if vuln['type'].to_s == "DatabaseErrorMessages"
+      res = {
+        :risk => 5,
+        :name  => 'SQL Injection',
+        :blame => 'App Developer',
+        :category => 'sql',
+        :description => "",
+        :confidence => conf
+      }
+    else
+    conf = 100
+    res = {
+      :risk => 1,
+      :name  => vuln['type'].to_s,
+      :blame => 'App Developer',
+      :category => 'info',
+      :description => "",
+      :confidence => conf
+    }
+    end
+
+    res
+  end
+end

lib/msf/core/db_manager/import/nexpose.rb

@@ -0,0 +1,7 @@
+module Msf::DBManager::Import::Nexpose
+  autoload :Raw, 'msf/core/db_manager/import/nexpose/raw'
+  autoload :Simple, 'msf/core/db_manager/import/nexpose/simple'
+
+  include Msf::DBManager::Import::Nexpose::Raw
+  include Msf::DBManager::Import::Nexpose::Simple
+end

lib/msf/core/db_manager/import/nexpose/raw.rb

@@ -0,0 +1,230 @@
+require 'rex/parser/nexpose_raw_nokogiri'
+require 'rex/parser/nexpose_xml'
+
+module Msf::DBManager::Import::Nexpose::Raw
+  def import_nexpose_raw_noko_stream(args, &block)
+    if block
+      doc = Rex::Parser::NexposeRawDocument.new(args,framework.db) {|type, data| yield type,data }
+    else
+      doc = Rex::Parser::NexposeRawDocument.new(args,self)
+    end
+    parser = ::Nokogiri::XML::SAX::Parser.new(doc)
+    parser.parse(args[:data])
+  end
+
+  def import_nexpose_rawxml(args={}, &block)
+    bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : []
+    wspace = args[:wspace] || workspace
+    if Rex::Parser.nokogiri_loaded
+      parser = "Nokogiri v#{::Nokogiri::VERSION}"
+      noko_args = args.dup
+      noko_args[:blacklist] = bl
+      noko_args[:wspace] = wspace
+      if block
+        yield(:parser, parser)
+        import_nexpose_raw_noko_stream(noko_args) {|type, data| yield type,data}
+      else
+        import_nexpose_raw_noko_stream(noko_args)
+      end
+      return true
+    end
+    data = args[:data]
+
+    # Use a stream parser instead of a tree parser so we can deal with
+    # huge results files without running out of memory.
+    parser = Rex::Parser::NexposeXMLStreamParser.new
+
+    # Since all the Refs have to be in the database before we can use them
+    # in a Vuln, we store all the hosts until we finish parsing and only
+    # then put everything in the database.  This is memory-intensive for
+    # large files, but should be much less so than a tree parser.
+    #
+    # This method is also considerably faster than parsing through the tree
+    # looking for references every time we hit a vuln.
+    hosts = []
+    vulns = []
+
+    # The callback merely populates our in-memory table of hosts and vulns
+    parser.callback = Proc.new { |type, value|
+      case type
+      when :host
+        # XXX: Blacklist should be checked here instead of saving a
+        # host we're just going to throw away later
+        hosts.push(value)
+      when :vuln
+        value["id"] = value["id"].downcase if value["id"]
+        vulns.push(value)
+      end
+    }
+
+    REXML::Document.parse_stream(data, parser)
+
+    vuln_refs = nexpose_refs_to_struct(vulns)
+    hosts.each do |host|
+      if bl.include? host["addr"]
+        next
+      else
+        yield(:address,host["addr"]) if block
+      end
+      nexpose_host_from_rawxml(host, vuln_refs, wspace)
+    end
+  end
+
+  #
+  # Nexpose Raw XML
+  #
+  def import_nexpose_rawxml_file(args={})
+    filename = args[:filename]
+    wspace = args[:wspace] || workspace
+
+    data = ""
+    ::File.open(filename, 'rb') do |f|
+      data = f.read(f.stat.size)
+    end
+    import_nexpose_rawxml(args.merge(:data => data))
+  end
+
+  # Takes a Host object, an array of vuln structs (generated by nexpose_refs_to_struct()),
+  # and a workspace, and reports the vulns on that host.
+  def nexpose_host_from_rawxml(h, vstructs, wspace,task=nil)
+    hobj = nil
+    data = {:workspace => wspace}
+    if h["addr"]
+      addr = h["addr"]
+    else
+      # Can't report it if it doesn't have an IP
+      return
+    end
+    data[:host] = addr
+    if (h["hardware-address"])
+      # Put colons between each octet of the MAC address
+      data[:mac] = h["hardware-address"].gsub(':', '').scan(/../).join(':')
+    end
+    data[:state] = (h["status"] == "alive") ? Msf::HostState::Alive : Msf::HostState::Dead
+
+    # Since we only have one name field per host in the database, just
+    # take the first one.
+    if (h["names"] and h["names"].first)
+      data[:name] = h["names"].first
+    end
+
+    if (data[:state] != Msf::HostState::Dead)
+      hobj = report_host(data)
+      report_import_note(wspace, hobj)
+    end
+
+    if h["notes"]
+      note = {
+          :workspace => wspace,
+          :host      => (hobj || addr),
+          :type      => "host.vuln.nexpose_keys",
+          :data      => {},
+          :mode      => :unique_data,
+          :task      => task
+      }
+      h["notes"].each do |v,k|
+        note[:data][v] ||= []
+        next if note[:data][v].include? k
+        note[:data][v] << k
+      end
+      report_note(note)
+    end
+
+    if h["os_family"]
+      note = {
+          :workspace => wspace,
+          :host      => hobj || addr,
+          :type      => 'host.os.nexpose_fingerprint',
+          :task      => task,
+          :data      => {
+              :family    => h["os_family"],
+              :certainty => h["os_certainty"]
+          }
+      }
+      note[:data][:vendor]  = h["os_vendor"]  if h["os_vendor"]
+      note[:data][:product] = h["os_product"] if h["os_product"]
+      note[:data][:version] = h["os_version"] if h["os_version"]
+      note[:data][:arch]    = h["arch"]       if h["arch"]
+
+      report_note(note)
+    end
+
+    h["endpoints"].each { |p|
+      extra = ""
+      extra << p["product"] + " " if p["product"]
+      extra << p["version"] + " " if p["version"]
+
+      # Skip port-0 endpoints
+      next if p["port"].to_i == 0
+
+      # XXX This should probably be handled in a more standard way
+      # extra << "(" + p["certainty"] + " certainty) " if p["certainty"]
+
+      data             = {}
+      data[:workspace] = wspace
+      data[:proto]     = p["protocol"].downcase
+      data[:port]      = p["port"].to_i
+      data[:state]     = p["status"]
+      data[:host]      = hobj || addr
+      data[:info]      = extra if not extra.empty?
+      data[:task]      = task
+      if p["name"] != "<unknown>"
+        data[:name] = p["name"]
+      end
+      report_service(data)
+    }
+
+    h["vulns"].each_pair { |k,v|
+
+      next if v["status"] !~ /^vulnerable/
+      vstruct = vstructs.select {|vs| vs.id.to_s.downcase == v["id"].to_s.downcase}.first
+      next unless vstruct
+      data             = {}
+      data[:workspace] = wspace
+      data[:host]      = hobj || addr
+      data[:proto]     = v["protocol"].downcase if v["protocol"]
+      data[:port]      = v["port"].to_i if v["port"]
+      data[:name]      = "NEXPOSE-" + v["id"]
+      data[:info]      = vstruct.title
+      data[:refs]      = vstruct.refs
+      data[:task]      = task
+      report_vuln(data)
+    }
+  end
+
+  #
+  # Takes an array of vuln hashes, as returned by the NeXpose rawxml stream
+  # parser, like:
+  #   [
+  #     {"id"=>"winreg-notes-protocol-handler", severity="8", "refs"=>[{"source"=>"BID", "value"=>"10600"}, ...]}
+  #     {"id"=>"windows-zotob-c", severity="8", "refs"=>[{"source"=>"BID", "value"=>"14513"}, ...]}
+  #   ]
+  # and transforms it into a struct, containing :id, :refs, :title, and :severity
+  #
+  # Other attributes can be added later, as needed.
+  def nexpose_refs_to_struct(vulns)
+    ret = []
+    vulns.each do |vuln|
+      next if ret.map {|v| v.id}.include? vuln["id"]
+      vstruct = Struct.new(:id, :refs, :title, :severity).new
+      vstruct.id = vuln["id"]
+      vstruct.title = vuln["title"]
+      vstruct.severity = vuln["severity"]
+      vstruct.refs = []
+      vuln["refs"].each do |ref|
+        if ref['source'] == 'BID'
+          vstruct.refs.push('BID-' + ref["value"])
+        elsif ref['source'] == 'CVE'
+          # value is CVE-$ID
+          vstruct.refs.push(ref["value"])
+        elsif ref['source'] == 'MS'
+          vstruct.refs.push('MSB-' + ref["value"])
+        elsif ref['source'] == 'URL'
+          vstruct.refs.push('URL-' + ref["value"])
+        end
+      end
+      ret.push vstruct
+    end
+    return ret
+  end
+end

lib/msf/core/db_manager/import/nexpose/simple.rb

@@ -0,0 +1,194 @@
+require 'rex/parser/nexpose_simple_nokogiri'
+
+module Msf::DBManager::Import::Nexpose::Simple
+  def import_nexpose_noko_stream(args, &block)
+    if block
+      doc = Rex::Parser::NexposeSimpleDocument.new(args,framework.db) {|type, data| yield type,data }
+    else
+      doc = Rex::Parser::NexposeSimpleDocument.new(args,self)
+    end
+    parser = ::Nokogiri::XML::SAX::Parser.new(doc)
+    parser.parse(args[:data])
+  end
+
+  def import_nexpose_simplexml(args={}, &block)
+    bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : []
+    wspace = args[:wspace] || workspace
+    if Rex::Parser.nokogiri_loaded
+      parser = "Nokogiri v#{::Nokogiri::VERSION}"
+      noko_args = args.dup
+      noko_args[:blacklist] = bl
+      noko_args[:wspace] = wspace
+      if block
+        yield(:parser, parser)
+        import_nexpose_noko_stream(noko_args) {|type, data| yield type,data}
+      else
+        import_nexpose_noko_stream(noko_args)
+      end
+      return true
+    end
+    data = args[:data]
+
+    doc = rexmlify(data)
+    doc.elements.each('/NeXposeSimpleXML/devices/device') do |dev|
+      addr = dev.attributes['address'].to_s
+      if bl.include? addr
+        next
+      else
+        yield(:address,addr) if block
+      end
+
+      fprint = {}
+
+      dev.elements.each('fingerprint/description') do |str|
+        fprint[:desc] = str.text.to_s.strip
+      end
+      dev.elements.each('fingerprint/vendor') do |str|
+        fprint[:vendor] = str.text.to_s.strip
+      end
+      dev.elements.each('fingerprint/family') do |str|
+        fprint[:family] = str.text.to_s.strip
+      end
+      dev.elements.each('fingerprint/product') do |str|
+        fprint[:product] = str.text.to_s.strip
+      end
+      dev.elements.each('fingerprint/version') do |str|
+        fprint[:version] = str.text.to_s.strip
+      end
+      dev.elements.each('fingerprint/architecture') do |str|
+        fprint[:arch] = str.text.to_s.upcase.strip
+      end
+
+      conf = {
+        :workspace => wspace,
+        :host      => addr,
+        :state     => Msf::HostState::Alive,
+        :task      => args[:task]
+      }
+
+      host = report_host(conf)
+      report_import_note(wspace, host)
+
+      report_note(
+        :workspace => wspace,
+        :host      => host,
+        :type      => 'host.os.nexpose_fingerprint',
+        :data      => fprint,
+        :task      => args[:task]
+      )
+
+      # Load vulnerabilities not associated with a service
+      dev.elements.each('vulnerabilities/vulnerability') do |vuln|
+        vid  = vuln.attributes['id'].to_s.downcase
+        refs = process_nexpose_data_sxml_refs(vuln)
+        next if not refs
+        report_vuln(
+          :workspace => wspace,
+          :host      => host,
+          :name      => 'NEXPOSE-' + vid,
+          :info      => vid,
+          :refs      => refs,
+          :task      => args[:task]
+        )
+      end
+
+      # Load the services
+      dev.elements.each('services/service') do |svc|
+        sname = svc.attributes['name'].to_s
+        sprot = svc.attributes['protocol'].to_s.downcase
+        sport = svc.attributes['port'].to_s.to_i
+        next if sport == 0
+
+        name = sname.split('(')[0].strip
+        info = ''
+
+        svc.elements.each('fingerprint/description') do |str|
+          info = str.text.to_s.strip
+        end
+
+        if(sname.downcase != '<unknown>')
+          report_service(
+              :workspace => wspace,
+              :host      => host,
+              :proto     => sprot,
+              :port      => sport,
+              :name      => name,
+              :info      => info,
+              :task      => args[:task]
+          )
+        else
+          report_service(
+              :workspace => wspace,
+              :host      => host,
+              :proto     => sprot,
+              :port      => sport,
+              :info      => info,
+              :task      => args[:task]
+          )
+        end
+
+        # Load vulnerabilities associated with this service
+        svc.elements.each('vulnerabilities/vulnerability') do |vuln|
+          vid  = vuln.attributes['id'].to_s.downcase
+          refs = process_nexpose_data_sxml_refs(vuln)
+          next if not refs
+          report_vuln(
+              :workspace => wspace,
+              :host      => host,
+              :port      => sport,
+              :proto     => sprot,
+              :name      => 'NEXPOSE-' + vid,
+              :info      => vid,
+              :refs      => refs,
+              :task      => args[:task]
+          )
+        end
+      end
+    end
+  end
+
+  #
+  # Nexpose Simple XML
+  #
+  # XXX At some point we'll want to make this a stream parser for dealing
+  # with large results files
+  #
+  def import_nexpose_simplexml_file(args={})
+    filename = args[:filename]
+    wspace = args[:wspace] || workspace
+
+    data = ""
+    ::File.open(filename, 'rb') do |f|
+      data = f.read(f.stat.size)
+    end
+    import_nexpose_simplexml(args.merge(:data => data))
+  end
+
+  protected
+
+  def process_nexpose_data_sxml_refs(vuln)
+    refs = []
+    vid = vuln.attributes['id'].to_s.downcase
+    vry = vuln.attributes['resultCode'].to_s.upcase
+
+    # Only process vuln-exploitable and vuln-version statuses
+    return if vry !~ /^V[VE]$/
+
+    refs = []
+    vuln.elements.each('id') do |ref|
+      rtyp = ref.attributes['type'].to_s.upcase
+      rval = ref.text.to_s.strip
+      case rtyp
+      when 'CVE'
+        refs << rval.gsub('CAN', 'CVE')
+      when 'MS' # obsolete?
+        refs << "MSB-MS-#{rval}"
+      else
+        refs << "#{rtyp}-#{rval}"
+      end
+    end
+
+    refs << "NEXPOSE-#{vid}"
+    refs
+  end
+end

lib/msf/core/db_manager/import/nikto.rb

@@ -0,0 +1,58 @@
+module Msf::DBManager::Import::Nikto
+  #
+  # Imports Nikto scan data from -Format xml as notes.
+  #
+  def import_nikto_xml(args={}, &block)
+    data = args[:data]
+    wspace = args[:wspace] || workspace
+    bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : []
+    doc = rexmlify(data)
+    doc.elements.each do |f|
+      f.elements.each('scandetails') do |host|
+        # Get host information
+        addr = host.attributes['targetip']
+        next if not addr
+        if bl.include? addr
+          next
+        else
+          yield(:address,addr) if block
+        end
+        # Get service information
+        port = host.attributes['targetport']
+        next if port.to_i == 0
+        uri = URI.parse(host.attributes['sitename']) rescue nil
+        next unless uri and uri.scheme
+        # Collect and report scan descriptions.
+        host.elements.each do |item|
+          if item.elements['description']
+            desc_text = item.elements['description'].text
+            next if desc_text.nil? or desc_text.empty?
+            desc_data = {
+                :workspace => wspace,
+                :host      => addr,
+                :type      => "service.nikto.scan.description",
+                :data      => desc_text,
+                :proto     => "tcp",
+                :port      => port.to_i,
+                :sname     => uri.scheme,
+                :update    => :unique_data,
+                :task      => args[:task]
+            }
+            # Always report it as a note.
+            report_note(desc_data)
+            # Sometimes report it as a vuln, too.
+            # XXX: There's a Vuln.info field but nothing reads from it? See Bug #5837
+            if item.attributes['osvdbid'].to_i != 0
+              desc_data[:refs] = ["OSVDB-#{item.attributes['osvdbid']}"]
+              desc_data[:name] = "NIKTO-#{item.attributes['id']}"
+              desc_data.delete(:data)
+              desc_data.delete(:type)
+              desc_data.delete(:update)
+              report_vuln(desc_data)
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/msf/core/db_manager/import/nmap.rb

@@ -0,0 +1,262 @@
+require 'rex/parser/nmap_nokogiri'
+require 'rex/parser/nmap_xml'
+
+module Msf::DBManager::Import::Nmap
+  def import_nmap_noko_stream(args, &block)
+    if block
+      doc = Rex::Parser::NmapDocument.new(args,framework.db) {|type, data| yield type,data }
+    else
+      doc = Rex::Parser::NmapDocument.new(args,self)
+    end
+    parser = ::Nokogiri::XML::SAX::Parser.new(doc)
+    parser.parse(args[:data])
+  end
+
+  # If you have Nokogiri installed, you'll be shunted over to
+  # that. Otherwise, you'll hit the old NmapXMLStreamParser.
+  def import_nmap_xml(args={}, &block)
+    return nil if args[:data].nil? or args[:data].empty?
+    wspace = args[:wspace] || workspace
+    bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : []
+
+    if Rex::Parser.nokogiri_loaded
+      noko_args = args.dup
+      noko_args[:blacklist] = bl
+      noko_args[:wspace] = wspace
+      if block
+        yield(:parser, "Nokogiri v#{::Nokogiri::VERSION}")
+        import_nmap_noko_stream(noko_args) {|type, data| yield type,data }
+      else
+        import_nmap_noko_stream(noko_args)
+      end
+      return true
+    end
+
+    # XXX: Legacy nmap xml parser starts here.
+
+    fix_services = args[:fix_services]
+    data = args[:data]
+
+    # Use a stream parser instead of a tree parser so we can deal with
+    # huge results files without running out of memory.
+    parser = Rex::Parser::NmapXMLStreamParser.new
+    yield(:parser, parser.class.name) if block
+
+    # Whenever the parser pulls a host out of the nmap results, store
+    # it, along with any associated services, in the database.
+    parser.on_found_host = Proc.new { |h|
+      hobj = nil
+      data = {:workspace => wspace}
+      if (h["addrs"].has_key?("ipv4"))
+        addr = h["addrs"]["ipv4"]
+      elsif (h["addrs"].has_key?("ipv6"))
+        addr = h["addrs"]["ipv6"]
+      else
+        # Can't report it if it doesn't have an IP
+        raise RuntimeError, "At least one IPv4 or IPv6 address is required"
+      end
+      next if bl.include? addr
+      data[:host] = addr
+      if (h["addrs"].has_key?("mac"))
+        data[:mac] = h["addrs"]["mac"]
+      end
+      data[:state] = (h["status"] == "up") ? Msf::HostState::Alive : Msf::HostState::Dead
+      data[:task] = args[:task]
+
+      if ( h["reverse_dns"] )
+        data[:name] = h["reverse_dns"]
+      end
+
+      # Only report alive hosts with ports to speak of.
+      if(data[:state] != Msf::HostState::Dead)
+        if h["ports"].size > 0
+          if fix_services
+            port_states = h["ports"].map {|p| p["state"]}.reject {|p| p == "filtered"}
+            next if port_states.compact.empty?
+          end
+          yield(:address,data[:host]) if block
+          hobj = report_host(data)
+          report_import_note(wspace,hobj)
+        end
+      end
+
+      if( h["os_vendor"] )
+        note = {
+          :workspace => wspace,
+          :host => hobj || addr,
+          :type => 'host.os.nmap_fingerprint',
+          :task => args[:task],
+          :data => {
+            :os_vendor   => h["os_vendor"],
+            :os_family   => h["os_family"],
+            :os_version  => h["os_version"],
+            :os_accuracy => h["os_accuracy"]
+          }
+        }
+
+        if(h["os_match"])
+          note[:data][:os_match] = h['os_match']
+        end
+
+        report_note(note)
+      end
+
+      if (h["last_boot"])
+        report_note(
+          :workspace => wspace,
+          :host => hobj || addr,
+          :type => 'host.last_boot',
+          :task => args[:task],
+          :data => {
+            :time => h["last_boot"]
+          }
+        )
+      end
+
+      if (h["trace"])
+        hops = []
+        h["trace"]["hops"].each do |hop|
+          hops << {
+            "ttl"     => hop["ttl"].to_i,
+            "address" => hop["ipaddr"].to_s,
+            "rtt"     => hop["rtt"].to_f,
+            "name"    => hop["host"].to_s
+          }
+        end
+        report_note(
+          :workspace => wspace,
+          :host => hobj || addr,
+          :type => 'host.nmap.traceroute',
+          :task => args[:task],
+          :data => {
+            'port'  => h["trace"]["port"].to_i,
+            'proto' => h["trace"]["proto"].to_s,
+            'hops'  => hops
+          }
+        )
+      end
+
+
+      # Put all the ports, regardless of state, into the db.
+      h["ports"].each { |p|
+        # Localhost port results are pretty unreliable -- if it's
+        # unknown, it's no good (possibly Windows-only)
+        if (
+          p["state"] == "unknown" &&
+          h["status_reason"] == "localhost-response"
+        )
+          next
+        end
+        extra = ""
+        extra << p["product"]   + " " if p["product"]
+        extra << p["version"]   + " " if p["version"]
+        extra << p["extrainfo"] + " " if p["extrainfo"]
+
+        data = {}
+        data[:workspace] = wspace
+        if fix_services
+          data[:proto] = nmap_msf_service_map(p["protocol"])
+        else
+          data[:proto] = p["protocol"].downcase
+        end
+        data[:port]  = p["portid"].to_i
+        data[:state] = p["state"]
+        data[:host]  = hobj || addr
+        data[:info]  = extra if not extra.empty?
+        data[:task]  = args[:task]
+        if p["name"] != "unknown"
+          data[:name] = p["name"]
+        end
+        report_service(data)
+      }
+      #Parse the scripts output
+      if h["scripts"]
+        h["scripts"].each do |key,val|
+          if key == "smb-check-vulns"
+            if val =~ /MS08-067: VULNERABLE/
+              vuln_info = {
+                :workspace => wspace,
+                :task => args[:task],
+                :host =>  hobj || addr,
+                :port => 445,
+                :proto => 'tcp',
+                :name => 'MS08-067',
+                :info => 'Microsoft Windows Server Service Crafted RPC Request Handling Unspecified Remote Code Execution',
+                :refs =>['CVE-2008-4250',
+                  'BID-31874',
+                  'OSVDB-49243',
+                  'CWE-94',
+                  'MSFT-MS08-067',
+                  'MSF-Microsoft Server Service Relative Path Stack Corruption',
+                  'NSS-34476']
+              }
+              report_vuln(vuln_info)
+            end
+            if val =~ /MS06-025: VULNERABLE/
+              vuln_info = {
+                :workspace => wspace,
+                :task => args[:task],
+                :host =>  hobj || addr,
+                :port => 445,
+                :proto => 'tcp',
+                :name => 'MS06-025',
+                :info => 'Vulnerability in Routing and Remote Access Could Allow Remote Code Execution',
+                :refs =>['CVE-2006-2370',
+                  'CVE-2006-2371',
+                  'BID-18325',
+                  'BID-18358',
+                  'BID-18424',
+                  'OSVDB-26436',
+                  'OSVDB-26437',
+                  'MSFT-MS06-025',
+                  'MSF-Microsoft RRAS Service RASMAN Registry Overflow',
+                  'NSS-21689']
+              }
+              report_vuln(vuln_info)
+            end
+            # This one has NOT been  Tested , remove this comment if confirmed working
+            if val =~ /MS07-029: VULNERABLE/
+              vuln_info = {
+                :workspace => wspace,
+                :task => args[:task],
+                :host =>  hobj || addr,
+                :port => 445,
+                :proto => 'tcp',
+                :name => 'MS07-029',
+                :info => 'Vulnerability in Windows DNS RPC Interface Could Allow Remote Code Execution',
+                # Add more refs based on nessus/nexpose .. results
+                :refs =>['CVE-2007-1748',
+                  'OSVDB-34100',
+                  'MSF-Microsoft DNS RPC Service extractQuotedChar()',
+                  'NSS-25168']
+              }
+              report_vuln(vuln_info)
+            end
+          end
+        end
+      end
+    }
+
+    # XXX: Legacy nmap xml parser ends here.
+
+    REXML::Document.parse_stream(data, parser)
+  end
+
+  #
+  # Import Nmap's -oX xml output
+  #
+  def import_nmap_xml_file(args={})
+    filename = args[:filename]
+    wspace = args[:wspace] || workspace
+
+    data = ""
+    ::File.open(filename, 'rb') do |f|
+      data = f.read(f.stat.size)
+    end
+    import_nmap_xml(args.merge(:data => data))
+  end
+
+  def nmap_msf_service_map(proto)
+    service_name_map(proto)
+  end
+end

lib/msf/core/db_manager/import/open_vas.rb

@@ -0,0 +1,34 @@
+require 'rex/parser/openvas_nokogiri'
+
+module Msf::DBManager::Import::OpenVAS
+  def import_openvas_new_xml(args={}, &block)
+    if block
+      doc = Rex::Parser::OpenVASDocument.new(args,framework.db) {|type, data| yield type,data }
+    else
+      doc = Rex::Parser::OpenVASDocument.new(args,self)
+    end
+    parser = ::Nokogiri::XML::SAX::Parser.new(doc)
+    parser.parse(args[:data])
+  end
+
+  def import_openvas_new_xml_file(args={})
+    filename = args[:filename]
+    wspace = args[:wspace] || workspace
+
+    data = ""
+    ::File.open(filename, 'rb') do |f|
+      data = f.read(f.stat.size)
+    end
+    import_wapiti_xml(args.merge(:data => data))
+  end
+
+  #
+  # Of course they had to change the nessus format.
+  #
+  def import_openvas_xml(args={}, &block)
+    filename = args[:filename]
+    wspace = args[:wspace] || workspace
+
+    raise DBImportError.new("No OpenVAS XML support. Please submit a patch to msfdev[at]metasploit.com")
+  end
+end

lib/msf/core/db_manager/import/outpost24.rb

@@ -0,0 +1,33 @@
+require 'rex/parser/outpost24_nokogiri'
+
+module Msf::DBManager::Import::Outpost24
+  def import_outpost24_noko_stream(args={},&block)
+    if block
+      doc = Rex::Parser::Outpost24Document.new(args,framework.db) {|type, data| yield type,data }
+    else
+      doc = Rex::Parser::Outpost24Document.new(args,self)
+    end
+    parser = ::Nokogiri::XML::SAX::Parser.new(doc)
+    parser.parse(args[:data])
+  end
+
+  def import_outpost24_xml(args={}, &block)
+    bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : []
+    wspace = args[:wspace] || workspace
+    if Rex::Parser.nokogiri_loaded
+      parser = "Nokogiri v#{::Nokogiri::VERSION}"
+      noko_args = args.dup
+      noko_args[:blacklist] = bl
+      noko_args[:wspace] = wspace
+      if block
+        yield(:parser, parser)
+        import_outpost24_noko_stream(noko_args) {|type, data| yield type,data}
+      else
+        import_outpost24_noko_stream(noko_args)
+      end
+      return true
+    else # Sorry
+      raise DBImportError.new("Could not import due to missing Nokogiri parser. Try 'gem install nokogiri'.")
+    end
+  end
+end

lib/msf/core/db_manager/import/qualys.rb

@@ -0,0 +1,50 @@
+module Msf::DBManager::Import::Qualys
+  autoload :Asset, 'msf/core/db_manager/import/qualys/asset'
+  autoload :Scan, 'msf/core/db_manager/import/qualys/scan'
+
+  include Msf::DBManager::Import::Qualys::Asset
+  include Msf::DBManager::Import::Qualys::Scan
+
+  #
+  # Qualys report parsing/handling
+  #
+  def handle_qualys(wspace, hobj, port, protocol, qid, severity, refs, name=nil, title=nil, task=nil)
+    addr = hobj.address
+    port = port.to_i if port
+
+    info = { :workspace => wspace, :host => hobj, :port => port, :proto => protocol, :task => task }
+    if name and name != 'unknown' and name != 'No registered hostname'
+      info[:name] = name
+    end
+
+    if info[:host] && info[:port] && info[:proto]
+      report_service(info)
+    end
+
+    fixed_refs = []
+    if refs
+      refs.each do |ref|
+        case ref
+        when /^MS[0-9]{2}-[0-9]{3}/
+          fixed_refs << "MSB-#{ref}"
+        else
+          fixed_refs << ref
+        end
+      end
+    end
+
+    return if qid == 0
+    title = 'QUALYS-' + qid if title.nil? or title.empty?
+    if addr
+      report_vuln(
+        :workspace => wspace,
+        :task => task,
+        :host => hobj,
+        :port => port,
+        :proto => protocol,
+        :name =>  title,
+        :refs => fixed_refs
+      )
+    end
+  end
+end

lib/msf/core/db_manager/import/qualys/asset.rb

@@ -0,0 +1,98 @@
+module Msf::DBManager::Import::Qualys::Asset
+  # Takes QID numbers and finds the discovered services in
+  # a qualys_asset_xml.
+  def find_qualys_asset_ports(i,host,wspace,hobj,task_id)
+    return unless (i == 82023 || i == 82004)
+    proto = i == 82023 ? 'tcp' : 'udp'
+    qid = host.elements["VULN_INFO_LIST/VULN_INFO/QID[@id='qid_#{i}']"]
+    qid_result = qid.parent.elements["RESULT[@format='table']"] if qid
+    hports = qid_result.first.to_s if qid_result
+    if hports
+      hports.scan(/([0-9]+)\t(.*?)\t.*?\t([^\t\n]*)/) do |match|
+        if match[2] == nil or match[2].strip == 'unknown'
+          name = match[1].strip
+        else
+          name = match[2].strip
+        end
+        handle_qualys(wspace, hobj, match[0].to_s, proto, 0, nil, nil, name, nil, task_id)
+      end
+    end
+  end
+
+  def find_qualys_asset_vuln_refs(doc)
+    vuln_refs = {}
+    doc.elements.each("/ASSET_DATA_REPORT/GLOSSARY/VULN_DETAILS_LIST/VULN_DETAILS") do |vuln|
+      next unless vuln.elements['QID'] && vuln.elements['QID'].first
+      qid = vuln.elements['QID'].first.to_s
+      vuln_refs[qid] ||= []
+      vuln.elements.each('CVE_ID_LIST/CVE_ID') do |ref|
+        vuln_refs[qid].push('CVE-' + /C..-([0-9\-]{9})/.match(ref.elements['ID'].text.to_s)[1])
+      end
+      vuln.elements.each('BUGTRAQ_ID_LIST/BUGTRAQ_ID') do |ref|
+        vuln_refs[qid].push('BID-' + ref.elements['ID'].text.to_s)
+      end
+    end
+    return vuln_refs
+  end
+
+  # Pull out vulnerabilities that have at least one matching
+  # ref -- many "vulns" are not vulns, just audit information.
+  def find_qualys_asset_vulns(host,wspace,hobj,vuln_refs,task_id,&block)
+    host.elements.each("VULN_INFO_LIST/VULN_INFO") do |vi|
+      next unless vi.elements["QID"]
+      vi.elements.each("QID") do |qid|
+        next if vuln_refs[qid.text].nil? || vuln_refs[qid.text].empty?
+        handle_qualys(wspace, hobj, nil, nil, qid.text, nil, vuln_refs[qid.text], nil, nil, task_id)
+      end
+    end
+  end
+
+  #
+  # Import Qualys's Asset Data Report format
+  #
+  def import_qualys_asset_xml(args={}, &block)
+    data = args[:data]
+    wspace = args[:wspace] || workspace
+    bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : []
+    doc = rexmlify(data)
+    vuln_refs = find_qualys_asset_vuln_refs(doc)
+
+    # 2nd pass, actually grab the hosts.
+    doc.elements.each("/ASSET_DATA_REPORT/HOST_LIST/HOST") do |host|
+      hobj = nil
+      addr = host.elements["IP"].text if host.elements["IP"]
+      next unless validate_ips(addr)
+      if bl.include? addr
+        next
+      else
+        yield(:address,addr) if block
+      end
+      hname = ( # Prefer NetBIOS over DNS
+        (host.elements["NETBIOS"].text if host.elements["NETBIOS"]) ||
+         (host.elements["DNS"].text if host.elements["DNS"]) ||
+         "" )
+      hobj = report_host(:workspace => wspace, :host => addr, :name => hname, :state => Msf::HostState::Alive, :task => args[:task])
+      report_import_note(wspace,hobj)
+
+      if host.elements["OPERATING_SYSTEM"]
+        hos = host.elements["OPERATING_SYSTEM"].text
+        report_note(
+          :workspace => wspace,
+          :task => args[:task],
+          :host => hobj,
+          :type => 'host.os.qualys_fingerprint',
+          :data => { :os => hos }
+        )
+      end
+
+      # Report open ports.
+      find_qualys_asset_ports(82023,host,wspace,hobj, args[:task]) # TCP
+      find_qualys_asset_ports(82004,host,wspace,hobj, args[:task]) # UDP
+
+      # Report vulns
+      find_qualys_asset_vulns(host,wspace,hobj,vuln_refs, args[:task],&block)
+
+    end # host
+
+  end
+end

lib/msf/core/db_manager/import/qualys/scan.rb

@@ -0,0 +1,98 @@
+module Msf::DBManager::Import::Qualys::Scan
+  def import_qualys_scan_xml(args={}, &block)
+    data = args[:data]
+    wspace = args[:wspace] || workspace
+    bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : []
+
+
+    doc = rexmlify(data)
+    doc.elements.each('/SCAN/IP') do |host|
+      hobj = nil
+      addr  = host.attributes['value']
+      if bl.include? addr
+        next
+      else
+        yield(:address,addr) if block
+      end
+      hname = host.attributes['name'] || ''
+
+      hobj = report_host(:workspace => wspace, :host => addr, :name => hname, :state => Msf::HostState::Alive, :task => args[:task])
+      report_import_note(wspace,hobj)
+
+      if host.elements["OS"]
+        hos = host.elements["OS"].text
+        report_note(
+          :workspace => wspace,
+          :task => args[:task],
+          :host => hobj,
+          :type => 'host.os.qualys_fingerprint',
+          :data => {
+            :os => hos
+          }
+        )
+      end
+
+      # Open TCP Services List (Qualys ID 82023)
+      services_tcp = host.elements["SERVICES/CAT/SERVICE[@number='82023']/RESULT"]
+      if services_tcp
+        services_tcp.text.scan(/([0-9]+)\t(.*?)\t.*?\t([^\t\n]*)/) do |match|
+          if match[2] == nil or match[2].strip == 'unknown'
+            name = match[1].strip
+          else
+            name = match[2].strip
+          end
+          handle_qualys(wspace, hobj, match[0].to_s, 'tcp', 0, nil, nil, name, nil, args[:task])
+        end
+      end
+      # Open UDP Services List (Qualys ID 82004)
+      services_udp = host.elements["SERVICES/CAT/SERVICE[@number='82004']/RESULT"]
+      if services_udp
+        services_udp.text.scan(/([0-9]+)\t(.*?)\t.*?\t([^\t\n]*)/) do |match|
+          if match[2] == nil or match[2].strip == 'unknown'
+            name = match[1].strip
+          else
+            name = match[2].strip
+          end
+          handle_qualys(wspace, hobj, match[0].to_s, 'udp', 0, nil, nil, name, nil, args[:task])
+        end
+      end
+
+      # VULNS are confirmed, PRACTICES are unconfirmed vulnerabilities
+      host.elements.each('VULNS/CAT | PRACTICES/CAT') do |cat|
+        port = cat.attributes['port']
+        protocol = cat.attributes['protocol']
+        cat.elements.each('VULN | PRACTICE') do |vuln|
+          refs = []
+          qid = vuln.attributes['number']
+          severity = vuln.attributes['severity']
+          title = vuln.elements['TITLE'].text.to_s
+          vuln.elements.each('VENDOR_REFERENCE_LIST/VENDOR_REFERENCE') do |ref|
+            refs.push(ref.elements['ID'].text.to_s)
+          end
+          vuln.elements.each('CVE_ID_LIST/CVE_ID') do |ref|
+            refs.push('CVE-' + /C..-([0-9\-]{9})/.match(ref.elements['ID'].text.to_s)[1])
+          end
+          vuln.elements.each('BUGTRAQ_ID_LIST/BUGTRAQ_ID') do |ref|
+            refs.push('BID-' + ref.elements['ID'].text.to_s)
+          end
+
+          handle_qualys(wspace, hobj, port, protocol, qid, severity, refs, nil,title, args[:task])
+        end
+      end
+    end
+  end
+
+  #
+  # Import Qualys' Scan xml output
+  #
+  def import_qualys_scan_xml_file(args={})
+    filename = args[:filename]
+    wspace = args[:wspace] || workspace
+
+    data = ""
+    ::File.open(filename, 'rb') do |f|
+      data = f.read(f.stat.size)
+    end
+    import_qualys_scan_xml(args.merge(:data => data))
+  end
+end

lib/msf/core/db_manager/import/report.rb

@@ -0,0 +1,48 @@
+module Msf::DBManager::Import::Report
+  # @param report [REXML::Element] to be imported
+  # @param args [Hash]
+  # @param base_dir [String]
+  def import_report(report, args, base_dir)
+    tmp = args[:ifd][:zip_tmp]
+    report_info = {}
+
+    report.elements.each do |e|
+      node_name  = e.name
+      node_value = e.text
+
+      # These need to be converted back to arrays:
+      array_attrs = %w|addresses file-formats options sections|
+      if array_attrs.member? node_name
+        node_value = JSON.parse(node_value)
+      end
+      # Don't restore these values:
+      skip_nodes = %w|id workspace-id artifacts|
+      next if skip_nodes.member? node_name
+
+      report_info[node_name.parameterize.underscore.to_sym] = node_value
+    end
+    # Use current workspace
+    report_info[:workspace_id] = args[:wspace].id
+
+    # Create report, need new ID to record artifacts
+    report_id = report_report(report_info)
+
+    # Handle artifacts
+    report.elements['artifacts'].elements.each do |artifact|
+      artifact_opts = {}
+      artifact.elements.each do |attr|
+        skip_nodes = %w|id accessed-at|
+        next if skip_nodes.member? attr.name
+
+        symboled_attr = attr.name.parameterize.underscore.to_sym
+        artifact_opts[symboled_attr] = attr.text
+      end
+      # Use new Report as parent
+      artifact_opts[:report_id] = report_id
+      # Update to full path
+      artifact_opts[:file_path].gsub!(/^\./, tmp)
+
+      report_artifact(artifact_opts)
+    end
+  end
+end

lib/msf/core/db_manager/import/retina.rb

@@ -0,0 +1,95 @@
+require 'rex/parser/retina_xml'
+
+module Msf::DBManager::Import::Retina
+  # Process Retina XML
+  def import_retina_xml(args={}, &block)
+    data = args[:data]
+    wspace = args[:wspace] || workspace
+    bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : []
+    msg =  "Warning: The Retina XML format does not associate vulnerabilities with the\n"
+    msg << "specific service on which they were found.\n"
+    msg << "This makes it impossible to correlate exploits to discovered vulnerabilities\n"
+    msg << "in a reliable fashion."
+
+    yield(:warning,msg) if block
+
+    parser = Rex::Parser::RetinaXMLStreamParser.new
+    parser.on_found_host = Proc.new do |host|
+      hobj = nil
+      data = {
+        :workspace => wspace,
+        :task      => args[:task]
+      }
+      addr = host['address']
+      next if not addr
+
+      next if bl.include? addr
+      data[:host] = addr
+
+      if host['mac']
+        data[:mac] = host['mac']
+      end
+
+      data[:state] = Msf::HostState::Alive
+
+      if host['hostname']
+        data[:name] = host['hostname']
+      end
+
+      if host['netbios']
+        data[:name] = host['netbios']
+      end
+
+      yield(:address, data[:host]) if block
+
+      # Import Host
+      hobj = report_host(data)
+      report_import_note(wspace, hobj)
+
+      # Import OS fingerprint
+      if host["os"]
+        note = {
+            :workspace => wspace,
+            :host      => addr,
+            :type      => 'host.os.retina_fingerprint',
+            :task      => args[:task],
+            :data      => {
+                :os => host["os"]
+            }
+        }
+        report_note(note)
+      end
+
+      # Import vulnerabilities
+      host['vulns'].each do |vuln|
+        refs = vuln['refs'].map{|v| v.join("-")}
+        refs << "RETINA-#{vuln['rthid']}" if vuln['rthid']
+
+        vuln_info = {
+            :workspace => wspace,
+            :host      => addr,
+            :name      => vuln['name'],
+            :info      => vuln['description'],
+            :refs      => refs,
+            :task      => args[:task]
+        }
+
+        report_vuln(vuln_info)
+      end
+    end
+
+    REXML::Document.parse_stream(data, parser)
+  end
+
+  # Process a Retina XML file
+  def import_retina_xml_file(args={})
+    filename = args[:filename]
+    wspace = args[:wspace] || workspace
+
+    data = ""
+    ::File.open(filename, 'rb') do |f|
+      data = f.read(f.stat.size)
+    end
+    import_retina_xml(args.merge(:data => data))
+  end
+end

lib/msf/core/db_manager/import/spiceworks.rb

@@ -0,0 +1,51 @@
+require 'csv'
+
+module Msf::DBManager::Import::Spiceworks
+  def import_spiceworks_csv(args={}, &block)
+    data = args[:data]
+    wspace = args[:wspace] || workspace
+    bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : []
+    CSV.parse(data) do |row|
+      next unless (["Name", "Manufacturer", "Device Type"] & row).empty? #header
+      name = row[0]
+      manufacturer = row[1]
+      device = row[2]
+      model = row[3]
+      ip = row[4]
+      serialno = row[5]
+      location = row[6]
+      os = row[7]
+
+      next unless ip
+      next if bl.include? ip
+
+      conf = {
+      :workspace => wspace,
+      :host      => ip,
+      :name      => name,
+      :task      => args[:task]
+      }
+
+
+      if os
+        report_note(
+          :workspace => wspace,
+          :task => args[:task],
+          :host => ip,
+          :type => 'host.os.spiceworks_fingerprint',
+          :data => {
+            :os => os.to_s.strip
+          }
+        )
+      end
+
+      info = []
+      info << "Serial Number: #{serialno}" unless (serialno.blank? or serialno == name)
+      info << "Location: #{location}" unless location.blank?
+      conf[:info] = info.join(", ") unless info.empty?
+
+      host = report_host(conf)
+      report_import_note(wspace, host)
+    end
+  end
+end

lib/msf/core/db_manager/import/wapiti.rb

@@ -0,0 +1,24 @@
+require 'rex/parser/wapiti_nokogiri'
+
+module Msf::DBManager::Import::Wapiti
+  def import_wapiti_xml(args={}, &block)
+    if block
+      doc = Rex::Parser::WapitiDocument.new(args,framework.db) {|type, data| yield type,data }
+    else
+      doc = Rex::Parser::WapitiDocument.new(args,self)
+    end
+    parser = ::Nokogiri::XML::SAX::Parser.new(doc)
+    parser.parse(args[:data])
+  end
+
+  def import_wapiti_xml_file(args={})
+    filename = args[:filename]
+    wspace = args[:wspace] || workspace
+
+    data = ""
+    ::File.open(filename, 'rb') do |f|
+      data = f.read(f.stat.size)
+    end
+    import_wapiti_xml(args.merge(:data => data))
+  end
+end

lib/msf/core/db_manager/import_msf_xml.rb

@@ -1,593 +0,0 @@
-# -*- coding: binary -*-
-module Msf
-  class DBManager
-    # Handles importing of the xml format exported by Pro.  The methods are in a
-    # module because (1) it's just good code layout and (2) it allows the
-    # methods to be overridden in Pro without using alias_method_chain as
-    # methods defined in a class cannot be overridden by including a module
-    # (unless you're running Ruby 2.0 and can use prepend)
-    module ImportMsfXml
-      #
-      # CONSTANTS
-      #
-
-      # Elements that can be treated as text (i.e. do not need to be
-      # deserialized) in {#import_msf_web_page_element}
-      MSF_WEB_PAGE_TEXT_ELEMENT_NAMES = [
-        'auth',
-        'body',
-        'code',
-        'cookie',
-        'ctype',
-        'location',
-        'mtime'
-      ]
-
-      # Elements that can be treated as text (i.e. do not need to be
-      # deserialized) in {#import_msf_web_element}.
-      MSF_WEB_TEXT_ELEMENT_NAMES = [
-        'created-at',
-        'host',
-        'path',
-        'port',
-        'query',
-        'ssl',
-        'updated-at',
-        'vhost'
-      ]
-
-      # Elements that can be treated as text (i.e. do not need to be
-      # deserialized) in {#import_msf_web_vuln_element}.
-      MSF_WEB_VULN_TEXT_ELEMENT_NAMES = [
-        'blame',
-        'category',
-        'confidence',
-        'description',
-        'method',
-        'name',
-        'pname',
-        'proof',
-        'risk'
-      ]
-
-      #
-      # Instance Methods
-      #
-
-      # Imports web_form element using {Msf::DBManager#report_web_form}.
-      #
-      # @param element [REXML::Element] web_form element.
-      # @param options [Hash{Symbol => Object}] options
-      # @option options [Boolean] :allow_yaml (false) Whether to allow YAML when
-      #   deserializing params.
-      # @option options [Mdm::Workspace, nil] :workspace
-      #   (Msf::DBManager#workspace) workspace under which to report the
-      #   Mdm::WebForm
-      # @yield [event, data]
-      # @yieldparam event [:web_page] The event name
-      # @yieldparam data [String] path
-      # @yieldreturn [void]
-      # @return [void]
-      def import_msf_web_form_element(element, options={}, &notifier)
-        options.assert_valid_keys(:allow_yaml, :workspace)
-
-        import_msf_web_element(element,
-                               :allow_yaml => options[:allow_yaml],
-                               :notifier => notifier,
-                               :type => :form,
-                               :workspace => options[:workspace]) do |element, options|
-          info = import_msf_text_element(element, 'method')
-
-          # FIXME https://www.pivotaltracker.com/story/show/46578647
-          # FIXME https://www.pivotaltracker.com/story/show/47128407
-          unserialized_params = unserialize_object(
-            element.elements['params'],
-            options[:allow_yaml]
-          )
-          info[:params] = nils_for_nulls(unserialized_params)
-
-          info
-        end
-      end
-
-      # Imports web_page element using {Msf::DBManager#report_web_page}.
-      #
-      # @param element [REXML::Element] web_page element.
-      # @param options [Hash{Symbol => Object}] options
-      # @option options [Boolean] :allow_yaml (false) Whether to allow YAML when
-      #   deserializing headers.
-      # @option options [Mdm::Workspace, nil] :workspace
-      #   (Msf::DBManager#workspace) workspace under which to report the
-      #   Mdm::WebPage.
-      # @yield [event, data]
-      # @yieldparam event [:web_page] The event name
-      # @yieldparam data [String] path
-      # @yieldreturn [void]
-      # @return [void]
-      def import_msf_web_page_element(element, options={}, &notifier)
-        options.assert_valid_keys(:allow_yaml, :workspace)
-
-        import_msf_web_element(element,
-                               :allow_yaml => options[:allow_yaml],
-                               :notifier => notifier,
-                               :type => :page,
-                               :workspace => options[:workspace]) do |element, options|
-          info = {}
-
-          MSF_WEB_PAGE_TEXT_ELEMENT_NAMES.each do |name|
-            element_info = import_msf_text_element(element, name)
-            info.merge!(element_info)
-          end
-
-          code = info[:code]
-
-          if code
-            info[:code] = code.to_i
-          end
-
-          # FIXME https://www.pivotaltracker.com/story/show/46578647
-          # FIXME https://www.pivotaltracker.com/story/show/47128407
-          unserialized_headers = unserialize_object(
-            element.elements['headers'],
-            options[:allow_yaml]
-          )
-          info[:headers] = nils_for_nulls(unserialized_headers)
-
-          info
-        end
-      end
-
-      # Imports web_vuln element using {Msf::DBManager#report_web_vuln}.
-      #
-      # @param element [REXML::Element] web_vuln element.
-      # @param options [Hash{Symbol => Object}] options
-      # @option options [Boolean] :allow_yaml (false) Whether to allow YAML when
-      #   deserializing headers.
-      # @option options [Mdm::Workspace, nil] :workspace
-      #   (Msf::DBManager#workspace) workspace under which to report the
-      #   Mdm::WebPage.
-      # @yield [event, data]
-      # @yieldparam event [:web_page] The event name
-      # @yieldparam data [String] path
-      # @yieldreturn [void]
-      # @return [void]
-      def import_msf_web_vuln_element(element, options={}, &notifier)
-        options.assert_valid_keys(:allow_yaml, :workspace)
-
-        import_msf_web_element(element,
-                               :allow_yaml => options[:allow_yaml],
-                               :notifier => notifier,
-                               :workspace => options[:workspace],
-                               :type => :vuln) do |element, options|
-          info = {}
-
-          MSF_WEB_VULN_TEXT_ELEMENT_NAMES.each do |name|
-            element_info = import_msf_text_element(element, name)
-            info.merge!(element_info)
-          end
-
-          confidence = info[:confidence]
-
-          if confidence
-            info[:confidence] = confidence.to_i
-          end
-
-          # FIXME https://www.pivotaltracker.com/story/show/46578647
-          # FIXME https://www.pivotaltracker.com/story/show/47128407
-          unserialized_params = unserialize_object(
-            element.elements['params'],
-            options[:allow_yaml]
-          )
-          info[:params] = nils_for_nulls(unserialized_params)
-
-          risk = info[:risk]
-
-          if risk
-            info[:risk] = risk.to_i
-          end
-
-          info
-        end
-      end
-
-      # For each host, step through services, notes, and vulns, and import
-      # them.
-      # TODO: loot, tasks, and reports
-      def import_msf_xml(args={}, &block)
-        data = args[:data]
-        wspace = args[:wspace] || workspace
-        bl = validate_ips(args[:blacklist]) ? args[:blacklist].split : []
-
-        doc = rexmlify(data)
-        metadata = check_msf_xml_version!(doc)
-        allow_yaml = metadata[:allow_yaml]
-        btag = metadata[:root_tag]
-
-        doc.elements.each("/#{btag}/hosts/host") do |host|
-          host_data = {}
-          host_data[:task] = args[:task]
-          host_data[:workspace] = wspace
-          host_data[:host] = nils_for_nulls(host.elements["address"].text.to_s.strip)
-          if bl.include? host_data[:host]
-            next
-          else
-            yield(:address,host_data[:host]) if block
-          end
-          host_data[:mac] = nils_for_nulls(host.elements["mac"].text.to_s.strip)
-          if host.elements["comm"].text
-            host_data[:comm] = nils_for_nulls(host.elements["comm"].text.to_s.strip)
-          end
-          %W{created-at updated-at name state os-flavor os-lang os-name os-sp purpose}.each { |datum|
-            if host.elements[datum].text
-              host_data[datum.gsub('-','_')] = nils_for_nulls(host.elements[datum].text.to_s.strip)
-            end
-          }
-          host_address = host_data[:host].dup # Preserve after report_host() deletes
-          hobj = report_host(host_data)
-
-          host.elements.each("host_details/host_detail") do |hdet|
-            hdet_data = {}
-            hdet.elements.each do |det|
-              next if ["id", "host-id"].include?(det.name)
-              if det.text
-                hdet_data[det.name.gsub('-','_')] = nils_for_nulls(det.text.to_s.strip)
-              end
-            end
-            report_host_details(hobj, hdet_data)
-          end
-
-          host.elements.each("exploit_attempts/exploit_attempt") do |hdet|
-            hdet_data = {}
-            hdet.elements.each do |det|
-              next if ["id", "host-id", "session-id", "vuln-id", "service-id", "loot-id"].include?(det.name)
-              if det.text
-                hdet_data[det.name.gsub('-','_')] = nils_for_nulls(det.text.to_s.strip)
-              end
-            end
-            report_exploit_attempt(hobj, hdet_data)
-          end
-
-          host.elements.each('services/service') do |service|
-            service_data = {}
-            service_data[:task] = args[:task]
-            service_data[:workspace] = wspace
-            service_data[:host] = hobj
-            service_data[:port] = nils_for_nulls(service.elements["port"].text.to_s.strip).to_i
-            service_data[:proto] = nils_for_nulls(service.elements["proto"].text.to_s.strip)
-            %W{created-at updated-at name state info}.each { |datum|
-              if service.elements[datum].text
-                if datum == "info"
-                  service_data["info"] = nils_for_nulls(unserialize_object(service.elements[datum], false))
-                else
-                  service_data[datum.gsub("-","_")] = nils_for_nulls(service.elements[datum].text.to_s.strip)
-                end
-              end
-            }
-            report_service(service_data)
-          end
-
-          host.elements.each('notes/note') do |note|
-            note_data = {}
-            note_data[:workspace] = wspace
-            note_data[:host] = hobj
-            note_data[:type] = nils_for_nulls(note.elements["ntype"].text.to_s.strip)
-            note_data[:data] = nils_for_nulls(unserialize_object(note.elements["data"], allow_yaml))
-
-            if note.elements["critical"].text
-              note_data[:critical] = true unless note.elements["critical"].text.to_s.strip == "NULL"
-            end
-            if note.elements["seen"].text
-              note_data[:seen] = true unless note.elements["critical"].text.to_s.strip == "NULL"
-            end
-            %W{created-at updated-at}.each { |datum|
-              if note.elements[datum].text
-                note_data[datum.gsub("-","_")] = nils_for_nulls(note.elements[datum].text.to_s.strip)
-              end
-            }
-            report_note(note_data)
-          end
-
-          host.elements.each('tags/tag') do |tag|
-            tag_data = {}
-            tag_data[:addr] = host_address
-            tag_data[:wspace] = wspace
-            tag_data[:name] = tag.elements["name"].text.to_s.strip
-            tag_data[:desc] = tag.elements["desc"].text.to_s.strip
-            if tag.elements["report-summary"].text
-              tag_data[:summary] = tag.elements["report-summary"].text.to_s.strip
-            end
-            if tag.elements["report-detail"].text
-              tag_data[:detail] = tag.elements["report-detail"].text.to_s.strip
-            end
-            if tag.elements["critical"].text
-              tag_data[:crit] = true unless tag.elements["critical"].text.to_s.strip == "NULL"
-            end
-            report_host_tag(tag_data)
-          end
-
-          host.elements.each('vulns/vuln') do |vuln|
-            vuln_data = {}
-            vuln_data[:workspace] = wspace
-            vuln_data[:host] = hobj
-            vuln_data[:info] = nils_for_nulls(unserialize_object(vuln.elements["info"], allow_yaml))
-            vuln_data[:name] = nils_for_nulls(vuln.elements["name"].text.to_s.strip)
-            %W{created-at updated-at exploited-at}.each { |datum|
-              if vuln.elements[datum] and vuln.elements[datum].text
-                vuln_data[datum.gsub("-","_")] = nils_for_nulls(vuln.elements[datum].text.to_s.strip)
-              end
-            }
-            if vuln.elements["refs"]
-              vuln_data[:refs] = []
-              vuln.elements.each("refs/ref") do |ref|
-                vuln_data[:refs] << nils_for_nulls(ref.text.to_s.strip)
-              end
-            end
-
-            vobj = report_vuln(vuln_data)
-
-            vuln.elements.each("vuln_details/vuln_detail") do |vdet|
-              vdet_data = {}
-              vdet.elements.each do |det|
-                next if ["id", "vuln-id"].include?(det.name)
-                if det.text
-                  vdet_data[det.name.gsub('-','_')] = nils_for_nulls(det.text.to_s.strip)
-                end
-              end
-              report_vuln_details(vobj, vdet_data)
-            end
-
-            vuln.elements.each("vuln_attempts/vuln_attempt") do |vdet|
-              vdet_data = {}
-              vdet.elements.each do |det|
-                next if ["id", "vuln-id", "loot-id", "session-id"].include?(det.name)
-                if det.text
-                  vdet_data[det.name.gsub('-','_')] = nils_for_nulls(det.text.to_s.strip)
-                end
-              end
-              report_vuln_attempt(vobj, vdet_data)
-            end
-          end
-
-          ## Handle old-style (pre 4.10) XML files
-          if btag == "MetasploitV4"
-            if host.elements['creds'].present?
-              unless host.elements['creds'].elements.empty?
-                origin = Metasploit::Credential::Origin::Import.create(filename: "console-import-#{Time.now.to_i}")
-
-                host.elements.each('creds/cred') do |cred|
-                  username = cred.elements['user'].try(:text)
-                  proto    = cred.elements['proto'].try(:text)
-                  sname    = cred.elements['sname'].try(:text)
-                  port     = cred.elements['port'].try(:text)
-
-                  # Handle blanks by resetting to sane default values
-                  proto   = "tcp" if proto.blank?
-                  pass     = cred.elements['pass'].try(:text)
-                  pass     = "" if pass == "*MASKED*"
-
-                  private = create_credential_private(private_data: pass, private_type: :password)
-                  public  = create_credential_public(username: username)
-                  core    = create_credential_core(private: private, public: public, origin: origin, workspace_id: wspace.id)
-
-                  create_credential_login(core: core,
-                                          workspace_id: wspace.id,
-                                          address: hobj.address,
-                                          port: port,
-                                          protocol: proto,
-                                          service_name: sname,
-                                          status: Metasploit::Model::Login::Status::UNTRIED)
-                end
-              end
-            end
-          end
-
-
-          host.elements.each('sessions/session') do |sess|
-            sess_id = nils_for_nulls(sess.elements["id"].text.to_s.strip.to_i)
-            sess_data = {}
-            sess_data[:host] = hobj
-            %W{desc platform port stype}.each {|datum|
-              if sess.elements[datum].respond_to? :text
-                sess_data[datum.intern] = nils_for_nulls(sess.elements[datum].text.to_s.strip)
-              end
-            }
-            %W{opened-at close-reason closed-at via-exploit via-payload}.each {|datum|
-              if sess.elements[datum].respond_to? :text
-                sess_data[datum.gsub("-","_").intern] = nils_for_nulls(sess.elements[datum].text.to_s.strip)
-              end
-            }
-            sess_data[:datastore] = nils_for_nulls(unserialize_object(sess.elements["datastore"], allow_yaml))
-            if sess.elements["routes"]
-              sess_data[:routes] = nils_for_nulls(unserialize_object(sess.elements["routes"], allow_yaml)) || []
-            end
-            if not sess_data[:closed_at] # Fake a close if we don't already have one
-              sess_data[:closed_at] = Time.now.utc
-              sess_data[:close_reason] = "Imported at #{Time.now.utc}"
-            end
-
-            existing_session = get_session(
-              :workspace => sess_data[:host].workspace,
-              :addr => sess_data[:host].address,
-              :time => sess_data[:opened_at]
-            )
-            this_session = existing_session || report_session(sess_data)
-            next if existing_session
-            sess.elements.each('events/event') do |sess_event|
-              sess_event_data = {}
-              sess_event_data[:session] = this_session
-              %W{created-at etype local-path remote-path}.each {|datum|
-                if sess_event.elements[datum].respond_to? :text
-                  sess_event_data[datum.gsub("-","_").intern] = nils_for_nulls(sess_event.elements[datum].text.to_s.strip)
-                end
-              }
-              %W{command output}.each {|datum|
-                if sess_event.elements[datum].respond_to? :text
-                  sess_event_data[datum.gsub("-","_").intern] = nils_for_nulls(unserialize_object(sess_event.elements[datum], allow_yaml))
-                end
-              }
-              report_session_event(sess_event_data)
-            end
-          end
-        end
-
-
-        # Import web sites
-        doc.elements.each("/#{btag}/web_sites/web_site") do |web|
-          info = {}
-          info[:workspace] = wspace
-
-          %W{host port vhost ssl comments}.each do |datum|
-            if web.elements[datum].respond_to? :text
-              info[datum.intern] = nils_for_nulls(web.elements[datum].text.to_s.strip)
-            end
-          end
-
-          info[:options]   = nils_for_nulls(unserialize_object(web.elements["options"], allow_yaml)) if web.elements["options"].respond_to?(:text)
-          info[:ssl]       = (info[:ssl] and info[:ssl].to_s.strip.downcase == "true") ? true : false
-
-          %W{created-at updated-at}.each { |datum|
-            if web.elements[datum].text
-              info[datum.gsub("-","_")] = nils_for_nulls(web.elements[datum].text.to_s.strip)
-            end
-          }
-
-          report_web_site(info)
-          yield(:web_site, "#{info[:host]}:#{info[:port]} (#{info[:vhost]})") if block
-        end
-
-        %W{page form vuln}.each do |wtype|
-          doc.elements.each("/#{btag}/web_#{wtype}s/web_#{wtype}") do |element|
-            send(
-              "import_msf_web_#{wtype}_element",
-              element,
-              :allow_yaml => allow_yaml,
-              :workspace => wspace,
-              &block
-            )
-          end
-        end
-      end
-
-      private
-
-      # Checks if the XML document has a format version that the importer
-      # understands.
-      #
-      # @param document [REXML::Document] a REXML::Document produced by
-      #   {Msf::DBManager#rexmlify}.
-      # @return [Hash{Symbol => Object}] `:allow_yaml` is true if the format
-      #   requires YAML loading when calling
-      #   {Msf::DBManager#unserialize_object}.  `:root_tag` the tag name of the
-      #   root element for MSF XML.
-      # @raise [Msf::DBImportError] if unsupported format
-      def check_msf_xml_version!(document)
-        metadata = {
-          # FIXME https://www.pivotaltracker.com/story/show/47128407
-          :allow_yaml => false,
-          :root_tag => nil
-        }
-
-        if document.elements['MetasploitExpressV1']
-          # FIXME https://www.pivotaltracker.com/story/show/47128407
-          metadata[:allow_yaml] = true
-          metadata[:root_tag] = 'MetasploitExpressV1'
-        elsif document.elements['MetasploitExpressV2']
-          # FIXME https://www.pivotaltracker.com/story/show/47128407
-          metadata[:allow_yaml] = true
-          metadata[:root_tag] = 'MetasploitExpressV2'
-        elsif document.elements['MetasploitExpressV3']
-          metadata[:root_tag] = 'MetasploitExpressV3'
-        elsif document.elements['MetasploitExpressV4']
-          metadata[:root_tag] = 'MetasploitExpressV4'
-        elsif document.elements['MetasploitV4']
-          metadata[:root_tag] = 'MetasploitV4'
-        elsif document.elements['MetasploitV5']
-          metadata[:root_tag] = 'MetasploitV5'
-        end
-
-        unless metadata[:root_tag]
-          raise Msf::DBImportError,
-                'Unsupported Metasploit XML document format'
-        end
-
-        metadata
-      end
-
-      # Retrieves text of element if it exists.
-      #
-      # @param parent_element [REXML::Element] element under which element with
-      #   `child_name` exists.
-      # @param child_name [String] the name of the element under
-      #   `parent_element` whose text should be returned
-      # @return [{}] if element with child_name does not exist or does not have
-      #   text.
-      # @return [Hash{Symbol => String}] Maps child_name symbol to text. Text is
-      #   stripped and any NULLs are converted to `nil`.
-      # @return [nil] if element with `child_name` does not exist under
-      #   `parent_element`.
-      def import_msf_text_element(parent_element, child_name)
-        child_element = parent_element.elements[child_name]
-        info = {}
-
-        if child_element
-          stripped = child_element.text.to_s.strip
-          attribute_name = child_name.underscore.to_sym
-          info[attribute_name] = nils_for_nulls(stripped)
-        end
-
-        info
-      end
-
-      # Imports web_form, web_page, or web_vuln element using
-      # {Msf::DBManager#report_web_form}, {Msf::DBManager#report_web_page}, and
-      # {Msf::DBManager#report_web_vuln}, respectively.
-      #
-      # @param element [REXML::Element] the web_form, web_page, or web_vuln
-      #   element.
-      # @param options [Hash{Symbol => Object}] options
-      # @option options [Boolean] :allow_yaml (false) Whether to allow YAML when
-      #   deserializing elements.
-      # @option options [Proc] :notifier Block called with web_* event and path
-      # @option options [Symbol] :type the type of web element, such as :form,
-      #   :page, or :vuln.  Must correspond to a report_web_<type> method on
-      # {Msf::DBManager}.
-      # @option options [Mdm::Workspace, nil] :workspace
-      #   (Msf::DBManager#workspace) workspace under which to report the
-      #   imported record.
-      # @yield [element, options]
-      # @yieldparam element [REXML::Element] the web_form, web_page, or
-      #   web_vuln element passed to {#import_msf_web_element}.
-      # @yieldparam options [Hash{Symbol => Object}] options for parsing
-      # @yieldreturn [Hash{Symbol => Object}] info
-      # @return [void]
-      # @raise [KeyError] if `:type` is not given
-      def import_msf_web_element(element, options={}, &specialization)
-        options.assert_valid_keys(:allow_yaml, :notifier, :type, :workspace)
-        type = options.fetch(:type)
-
-        info = {}
-        info[:workspace] = options[:workspace] || self.workspace
-
-        MSF_WEB_TEXT_ELEMENT_NAMES.each do |name|
-          element_info = import_msf_text_element(element, name)
-          info.merge!(element_info)
-        end
-
-        info[:ssl] = (info[:ssl] and info[:ssl].to_s.strip.downcase == "true") ? true : false
-
-        specialized_info = specialization.call(element, options)
-        info.merge!(specialized_info)
-
-        self.send("report_web_#{type}", info)
-
-        notifier = options[:notifier]
-
-        if notifier
-          event = "web_#{type}".to_sym
-          notifier.call(event, info[:path])
-        end
-      end
-    end
-  end
-end
-

lib/msf/core/db_manager/ip_address.rb

@@ -0,0 +1,61 @@
+module Msf::DBManager::IPAddress
+  def ipv46_validator(addr)
+    ipv4_validator(addr) or ipv6_validator(addr)
+  end
+
+  def ipv4_validator(addr)
+    return false unless addr.kind_of? String
+    Rex::Socket.is_ipv4?(addr)
+  end
+
+  def ipv6_validator(addr)
+    Rex::Socket.is_ipv6?(addr)
+  end
+
+  def rfc3330_reserved(ip)
+    case ip.class.to_s
+    when "PacketFu::Octets"
+      ip_x = ip.to_x
+      ip_i = ip.to_i
+    when "String"
+      if ipv46_validator(ip)
+        ip_x = ip
+        ip_i = Rex::Socket.addr_atoi(ip)
+      else
+        raise ArgumentError, "Invalid IP address: #{ip.inspect}"
+      end
+    when "Fixnum"
+      if (0..2**32-1).include? ip
+        ip_x = Rex::Socket.addr_itoa(ip)
+        ip_i = ip
+      else
+        raise ArgumentError, "Invalid IP address: #{ip.inspect}"
+      end
+    else
+      raise ArgumentError, "Invalid IP address: #{ip.inspect}"
+    end
+    return true if Rex::Socket::RangeWalker.new("0.0.0.0-0.255.255.255").include? ip_x
+    return true if Rex::Socket::RangeWalker.new("127.0.0.0-127.255.255.255").include? ip_x
+    return true if Rex::Socket::RangeWalker.new("169.254.0.0-169.254.255.255").include? ip_x
+    return true if Rex::Socket::RangeWalker.new("224.0.0.0-239.255.255.255").include? ip_x
+    return true if Rex::Socket::RangeWalker.new("255.255.255.255-255.255.255.255").include? ip_x
+    return false
+  end
+
+  # Takes a space-delimited set of ips and ranges, and subjects
+  # them to RangeWalker for validation. Returns true or false.
+  def validate_ips(ips)
+    ret = true
+    begin
+      ips.split(/\s+/).each {|ip|
+        unless Rex::Socket::RangeWalker.new(ip).ranges
+          ret = false
+          break
+        end
+        }
+    rescue
+      ret = false
+    end
+    return ret
+  end
+end

lib/msf/core/db_manager/loot.rb

@@ -0,0 +1,81 @@
+module Msf::DBManager::Loot
+  #
+  # Loot collection
+  #
+  #
+  # This method iterates the loot table calling the supplied block with the
+  # instance of each entry.
+  #
+  def each_loot(wspace=workspace, &block)
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    wspace.loots.each do |note|
+      block.call(note)
+    end
+  }
+  end
+
+  #
+  # Find or create a loot matching this type/data
+  #
+  def find_or_create_loot(opts)
+    report_loot(opts)
+  end
+
+  #
+  # This methods returns a list of all loot in the database
+  #
+  def loots(wspace=workspace)
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    wspace.loots
+  }
+  end
+
+  def report_loot(opts)
+    return if not active
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    wspace = opts.delete(:workspace) || workspace
+    path = opts.delete(:path) || (raise RuntimeError, "A loot :path is required")
+
+    host = nil
+    addr = nil
+
+    # Report the host so it's there for the Proc to use below
+    if opts[:host]
+      if opts[:host].kind_of? ::Mdm::Host
+        host = opts[:host]
+      else
+        host = report_host({:workspace => wspace, :host => opts[:host]})
+        addr = normalize_host(opts[:host])
+      end
+    end
+
+    ret = {}
+
+    ltype  = opts.delete(:type) || opts.delete(:ltype) || (raise RuntimeError, "A loot :type or :ltype is required")
+    ctype  = opts.delete(:ctype) || opts.delete(:content_type) || 'text/plain'
+    name   = opts.delete(:name)
+    info   = opts.delete(:info)
+    data   = opts[:data]
+    loot   = wspace.loots.new
+
+    if host
+      loot.host_id = host[:id]
+    end
+    if opts[:service] and opts[:service].kind_of? ::Mdm::Service
+      loot.service_id = opts[:service][:id]
+    end
+
+    loot.path         = path
+    loot.ltype        = ltype
+    loot.content_type = ctype
+    loot.data         = data
+    loot.name         = name if name
+    loot.info         = info if info
+    loot.workspace    = wspace
+    msf_import_timestamps(opts,loot)
+    loot.save!
+
+    ret[:loot] = loot
+  }
+  end
+end

lib/msf/core/db_manager/migration.rb

@@ -1,59 +1,81 @@
 # -*- coding: binary -*-
-module Msf
+module Msf::DBManager::Migration
-  class DBManager
+  # Loads Metasploit Data Models and adds its migrations to migrations paths.
-    module Migration
+  #
-      # Migrate database to latest schema version.
+  # @return [void]
-      #
+  def add_rails_engine_migration_paths
-      # @param verbose [Boolean] see ActiveRecord::Migration.verbose
+    unless defined? ActiveRecord
-      # @return [Array<ActiveRecord::MigrationProxy] List of migrations that
+      fail "Bundle installed '--without #{Bundler.settings.without.join(' ')}'.  To clear the without option do " \
-      #   ran.
+           "`bundle install --without ''` (the --without flag with an empty string) or `rm -rf .bundle` to remove " \
-      #
+           "the .bundle/config manually and then `bundle install`"
-      # @see ActiveRecord::Migrator.migrate
+    end
-      def migrate(verbose=false)
-        ran = []
-        ActiveRecord::Migration.verbose = verbose
 
-        ActiveRecord::Base.connection_pool.with_connection do
+    Rails.application.railties.engines.each do |engine|
-          begin
+      migrations_paths = engine.paths['db/migrate'].existent_directories
-            ran = ActiveRecord::Migrator.migrate(
-                ActiveRecord::Migrator.migrations_paths
-            )
-          # ActiveRecord::Migrator#migrate rescues all errors and re-raises them
-          # as StandardError
-          rescue StandardError => error
-            self.error = error
-            elog("DB.migrate threw an exception: #{error}")
-            dlog("Call stack:\n#{error.backtrace.join "\n"}")
-          end
-        end
 
-        # Since the connections that existed before the migrations ran could
+      migrations_paths.each do |migrations_path|
-        # have outdated column information, reset column information for all
+        # Since ActiveRecord::Migrator.migrations_paths can persist between
-        # ActiveRecord::Base descendents to prevent missing method errors for
+        # instances of Msf::DBManager, such as in specs,
-        # column methods for columns created in migrations after the column
+        # migrations_path may already be part of
-        # information was cached.
+        # migrations_paths, in which case it should not be added or multiple
-        reset_column_information
+        # migrations with the same version number errors will occur.
-
+        unless ActiveRecord::Migrator.migrations_paths.include? migrations_path
-        return ran
+          ActiveRecord::Migrator.migrations_paths << migrations_path
-      end
-
-      # Flag to indicate database migration has completed
-      #
-      # @return [Boolean]
-      attr_accessor :migrated
-
-      private
-
-      # Resets the column information for all descendants of ActiveRecord::Base
-      # since some of the migrations may have cached column information that
-      # has been updated by later migrations.
-      #
-      # @return [void]
-      def reset_column_information
-        ActiveRecord::Base.descendants.each do |descendant|
-          descendant.reset_column_information
         end
       end
     end
   end
-end
+
+  # Migrate database to latest schema version.
+  #
+  # @param verbose [Boolean] see ActiveRecord::Migration.verbose
+  # @return [Array<ActiveRecord::MigrationProxy] List of migrations that
+  #   ran.
+  #
+  # @see ActiveRecord::Migrator.migrate
+  def migrate(verbose=false)
+    ran = []
+    ActiveRecord::Migration.verbose = verbose
+
+    ActiveRecord::Base.connection_pool.with_connection do
+      begin
+        ran = ActiveRecord::Migrator.migrate(
+            ActiveRecord::Migrator.migrations_paths
+        )
+          # ActiveRecord::Migrator#migrate rescues all errors and re-raises them
+          # as StandardError
+      rescue StandardError => error
+        self.error = error
+        elog("DB.migrate threw an exception: #{error}")
+        dlog("Call stack:\n#{error.backtrace.join "\n"}")
+      end
+    end
+
+    # Since the connections that existed before the migrations ran could
+    # have outdated column information, reset column information for all
+    # ActiveRecord::Base descendents to prevent missing method errors for
+    # column methods for columns created in migrations after the column
+    # information was cached.
+    reset_column_information
+
+    return ran
+  end
+
+  # Flag to indicate database migration has completed
+  #
+  # @return [Boolean]
+  attr_accessor :migrated
+
+  private
+
+  # Resets the column information for all descendants of ActiveRecord::Base
+  # since some of the migrations may have cached column information that
+  # has been updated by later migrations.
+  #
+  # @return [void]
+  def reset_column_information
+    ActiveRecord::Base.descendants.each do |descendant|
+      descendant.reset_column_information
+    end
+  end
+end

lib/msf/core/db_manager/module_cache.rb

@@ -0,0 +1,416 @@
+#
+# Standard library
+#
+
+require 'shellwords'
+
+module Msf::DBManager::ModuleCache
+  #
+  # Attributes
+  #
+
+  # Flag to indicate that modules are cached
+  attr_accessor :modules_cached
+
+  # Flag to indicate that the module cacher is running
+  attr_accessor :modules_caching
+
+  #
+  # Instance Methods
+  #
+
+  # Wraps values in +'%'+ for Arel::Prediciation#matches_any and other match* methods that map to SQL +'LIKE'+ or
+  # +'ILIKE'+
+  #
+  # @param values [Set<String>, #each] a list of strings.
+  # @return [Arrray<String>] strings wrapped like %<string>%
+  def match_values(values)
+    wrapped_values = values.collect { |value|
+      "%#{value}%"
+    }
+
+    wrapped_values
+  end
+
+  def module_to_details_hash(m)
+    res  = {}
+    bits = []
+
+    res[:mtime]    = ::File.mtime(m.file_path) rescue Time.now
+    res[:file]     = m.file_path
+    res[:mtype]    = m.type
+    res[:name]     = m.name.to_s
+    res[:refname]  = m.refname
+    res[:fullname] = m.fullname
+    res[:rank]     = m.rank.to_i
+    res[:license]  = m.license.to_s
+
+    res[:description] = m.description.to_s.strip
+
+    m.arch.map{ |x|
+      bits << [ :arch, { :name => x.to_s } ]
+    }
+
+    m.platform.platforms.map{ |x|
+      bits << [ :platform, { :name => x.to_s.split('::').last.downcase } ]
+    }
+
+    m.author.map{|x|
+      bits << [ :author, { :name => x.to_s } ]
+    }
+
+    m.references.map do |r|
+      bits << [ :ref, { :name => [r.ctx_id.to_s, r.ctx_val.to_s].join("-") } ]
+    end
+
+    res[:privileged] = m.privileged?
+
+
+    if m.disclosure_date
+      begin
+        res[:disclosure_date] = m.disclosure_date.to_datetime.to_time
+      rescue ::Exception
+        res.delete(:disclosure_date)
+      end
+    end
+
+    if(m.type == "exploit")
+
+      m.targets.each_index do |i|
+        bits << [ :target, { :index => i, :name => m.targets[i].name.to_s } ]
+        if m.targets[i].platform
+          m.targets[i].platform.platforms.each do |name|
+            bits << [ :platform, { :name => name.to_s.split('::').last.downcase } ]
+          end
+        end
+        if m.targets[i].arch
+          bits << [ :arch, { :name => m.targets[i].arch.to_s } ]
+        end
+      end
+
+      if (m.default_target)
+        res[:default_target] = m.default_target
+      end
+
+      # Some modules are a combination, which means they are actually aggressive
+      res[:stance] = m.stance.to_s.index("aggressive") ? "aggressive" : "passive"
+
+
+      m.class.mixins.each do |x|
+         bits << [ :mixin, { :name => x.to_s } ]
+      end
+    end
+
+    if(m.type == "auxiliary")
+
+      m.actions.each_index do |i|
+        bits << [ :action, { :name => m.actions[i].name.to_s } ]
+      end
+
+      if (m.default_action)
+        res[:default_action] = m.default_action.to_s
+      end
+
+      res[:stance] = m.passive? ? "passive" : "aggressive"
+    end
+
+    res[:bits] = bits.uniq
+
+    res
+  end
+
+  # @note Does nothing unless {#migrated} is +true+ and {#modules_caching} is
+  #   +false+.
+  #
+  # Destroys all Mdm::Module::Details in the database.
+  #
+  # @return [void]
+  def purge_all_module_details
+    return if not self.migrated
+    return if self.modules_caching
+
+    ::ActiveRecord::Base.connection_pool.with_connection do
+      Mdm::Module::Detail.destroy_all
+    end
+  end
+
+  # Destroys Mdm::Module::Detail if one exists for the given
+  # Mdm::Module::Detail#mtype and Mdm::Module::Detail#refname.
+  #
+  # @param mtype [String] module type.
+  # @param refname [String] module reference name.
+  # @return [void]
+  def remove_module_details(mtype, refname)
+    return if not self.migrated
+
+    ActiveRecord::Base.connection_pool.with_connection do
+      Mdm::Module::Detail.where(:mtype => mtype, :refname => refname).destroy_all
+    end
+  end
+
+  # This provides a standard set of search filters for every module.
+  #
+  # Supported keywords with the format <keyword>:<search_value>:
+  # +app+:: If +client+ then matches +'passive'+ stance modules, otherwise matches +'active' stance modules.
+  # +author+:: Matches modules with the given author email or name.
+  # +bid+:: Matches modules with the given Bugtraq ID.
+  # +cve+:: Matches modules with the given CVE ID.
+  # +edb+:: Matches modules with the given Exploit-DB ID.
+  # +name+:: Matches modules with the given full name or name.
+  # +os+, +platform+:: Matches modules with the given platform or target name.
+  # +osvdb+:: Matches modules with the given OSVDB ID.
+  # +ref+:: Matches modules with the given reference ID.
+  # +type+:: Matches modules with the given type.
+  #
+  # Any text not associated with a keyword is matched against the description,
+  # the full name, and the name of the module; the name of the module actions;
+  # the name of the module archs; the name of the module authors; the name of
+  # module platform; the module refs; or the module target.
+  #
+  # @param search_string [String] a string of space separated keyword pairs or
+  #   free form text.
+  # @return [[]] if search_string is +nil+
+  # @return [ActiveRecord::Relation] module details that matched
+  #   +search_string+
+  def search_modules(search_string)
+    search_string ||= ''
+    search_string += " "
+
+    # Split search terms by space, but allow quoted strings
+    terms = Shellwords.shellwords(search_string)
+    terms.delete('')
+
+    # All terms are either included or excluded
+    value_set_by_keyword = Hash.new { |hash, keyword|
+      hash[keyword] = Set.new
+    }
+
+    terms.each do |term|
+      keyword, value = term.split(':', 2)
+
+      unless value
+        value = keyword
+        keyword = 'text'
+      end
+
+      unless value.empty?
+        keyword.downcase!
+
+        value_set = value_set_by_keyword[keyword]
+        value_set.add value
+      end
+    end
+
+    query = Mdm::Module::Detail.scoped
+
+    ActiveRecord::Base.connection_pool.with_connection do
+      # Although AREL supports taking the union or two queries, the ActiveRecord where syntax only supports
+      # intersection, so creating the where clause has to be delayed until all conditions can be or'd together and
+      # passed to one call ot where.
+      union_conditions = []
+
+      value_set_by_keyword.each do |keyword, value_set|
+        case keyword
+          when 'author'
+            formatted_values = match_values(value_set)
+
+            query = query.includes(:authors)
+            module_authors = Mdm::Module::Author.arel_table
+            union_conditions << module_authors[:email].matches_any(formatted_values)
+            union_conditions << module_authors[:name].matches_any(formatted_values)
+          when 'name'
+            formatted_values = match_values(value_set)
+
+            module_details = Mdm::Module::Detail.arel_table
+            union_conditions << module_details[:fullname].matches_any(formatted_values)
+            union_conditions << module_details[:name].matches_any(formatted_values)
+          when 'os', 'platform'
+            formatted_values = match_values(value_set)
+
+            query = query.includes(:platforms)
+            union_conditions << Mdm::Module::Platform.arel_table[:name].matches_any(formatted_values)
+
+            query = query.includes(:targets)
+            union_conditions << Mdm::Module::Target.arel_table[:name].matches_any(formatted_values)
+          when 'text'
+            formatted_values = match_values(value_set)
+
+            module_details = Mdm::Module::Detail.arel_table
+            union_conditions << module_details[:description].matches_any(formatted_values)
+            union_conditions << module_details[:fullname].matches_any(formatted_values)
+            union_conditions << module_details[:name].matches_any(formatted_values)
+
+            query = query.includes(:actions)
+            union_conditions << Mdm::Module::Action.arel_table[:name].matches_any(formatted_values)
+
+            query = query.includes(:archs)
+            union_conditions << Mdm::Module::Arch.arel_table[:name].matches_any(formatted_values)
+
+            query = query.includes(:authors)
+            union_conditions << Mdm::Module::Author.arel_table[:name].matches_any(formatted_values)
+
+            query = query.includes(:platforms)
+            union_conditions << Mdm::Module::Platform.arel_table[:name].matches_any(formatted_values)
+
+            query = query.includes(:refs)
+            union_conditions << Mdm::Module::Ref.arel_table[:name].matches_any(formatted_values)
+
+            query = query.includes(:targets)
+            union_conditions << Mdm::Module::Target.arel_table[:name].matches_any(formatted_values)
+          when 'type'
+            formatted_values = match_values(value_set)
+            union_conditions << Mdm::Module::Detail.arel_table[:mtype].matches_any(formatted_values)
+          when 'app'
+            formatted_values = value_set.collect { |value|
+              formatted_value = 'aggressive'
+
+              if value == 'client'
+                formatted_value = 'passive'
+              end
+
+              formatted_value
+            }
+
+            union_conditions << Mdm::Module::Detail.arel_table[:stance].eq_any(formatted_values)
+          when 'ref'
+            formatted_values = match_values(value_set)
+
+            query = query.includes(:refs)
+            union_conditions << Mdm::Module::Ref.arel_table[:name].matches_any(formatted_values)
+          when 'cve', 'bid', 'osvdb', 'edb'
+            formatted_values = value_set.collect { |value|
+              prefix = keyword.upcase
+
+              "#{prefix}-%#{value}%"
+            }
+
+            query = query.includes(:refs)
+            union_conditions << Mdm::Module::Ref.arel_table[:name].matches_any(formatted_values)
+        end
+      end
+
+      unioned_conditions = union_conditions.inject { |union, condition|
+        union.or(condition)
+      }
+
+      query = query.where(unioned_conditions).to_a.uniq { |m| m.fullname }
+    end
+
+    query
+  end
+
+  # Destroys the old Mdm::Module::Detail and creates a new Mdm::Module::Detail for
+  # any module with an Mdm::Module::Detail where the modification time of the
+  # Mdm::Module::Detail#file differs from the Mdm::Module::Detail#mtime.  If the
+  # Mdm::Module::Detail#file no only exists on disk, then the Mdm::Module::Detail
+  # is just destroyed without a new one being created.
+  #
+  # @return [void]
+  def update_all_module_details
+    return if not self.migrated
+    return if self.modules_caching
+
+    self.framework.cache_thread = Thread.current
+
+    self.modules_cached  = false
+    self.modules_caching = true
+
+    ActiveRecord::Base.connection_pool.with_connection do
+
+      refresh = []
+      skip_reference_name_set_by_module_type = Hash.new { |hash, module_type|
+        hash[module_type] = Set.new
+      }
+
+      Mdm::Module::Detail.find_each do |md|
+
+        unless md.ready
+          refresh << md
+          next
+        end
+
+        unless md.file and ::File.exists?(md.file)
+          refresh << md
+          next
+        end
+
+        if ::File.mtime(md.file).to_i != md.mtime.to_i
+          refresh << md
+          next
+        end
+
+        skip_reference_name_set = skip_reference_name_set_by_module_type[md.mtype]
+        skip_reference_name_set.add(md.refname)
+      end
+
+      refresh.each { |md| md.destroy }
+
+      [
+          ['exploit', framework.exploits],
+          ['auxiliary', framework.auxiliary],
+          ['post', framework.post],
+          ['payload', framework.payloads],
+          ['encoder', framework.encoders],
+          ['nop', framework.nops]
+      ].each do |mt|
+        skip_reference_name_set = skip_reference_name_set_by_module_type[mt[0]]
+
+        mt[1].keys.sort.each do |mn|
+          next if skip_reference_name_set.include? mn
+          obj = mt[1].create(mn)
+          next if not obj
+          begin
+            update_module_details(obj)
+          rescue ::Exception
+            elog("Error updating module details for #{obj.fullname}: #{$!.class} #{$!}")
+          end
+        end
+      end
+
+      self.framework.cache_initialized = true
+    end
+
+    # in reverse order of section before with_connection block
+    self.modules_caching = false
+    self.modules_cached  = true
+    self.framework.cache_thread = nil
+  end
+
+  # Creates an Mdm::Module::Detail from a module instance.
+  #
+  # @param module_instance [Msf::Module] a metasploit module instance.
+  # @raise [ActiveRecord::RecordInvalid] if Hash from {#module_to_details_hash} is invalid attributes for
+  #   Mdm::Module::Detail.
+  # @return [void]
+  def update_module_details(module_instance)
+    return if not self.migrated
+
+    ActiveRecord::Base.connection_pool.with_connection do
+      info = module_to_details_hash(module_instance)
+      bits = info.delete(:bits) || []
+      module_detail = Mdm::Module::Detail.create!(info)
+
+      bits.each do |args|
+        otype, vals = args
+
+        case otype
+          when :action
+            module_detail.add_action(vals[:name])
+          when :arch
+            module_detail.add_arch(vals[:name])
+          when :author
+            module_detail.add_author(vals[:name], vals[:email])
+          when :platform
+            module_detail.add_platform(vals[:name])
+          when :ref
+            module_detail.add_ref(vals[:name])
+          when :target
+            module_detail.add_target(vals[:index], vals[:name])
+        end
+      end
+
+      module_detail.ready = true
+      module_detail.save!
+    end
+  end
+end

lib/msf/core/db_manager/note.rb

@@ -0,0 +1,169 @@
+module Msf::DBManager::Note
+  #
+  # This method iterates the notes table calling the supplied block with the
+  # note instance of each entry.
+  #
+  def each_note(wspace=workspace, &block)
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    wspace.notes.each do |note|
+      block.call(note)
+    end
+  }
+  end
+
+  #
+  # Find or create a note matching this type/data
+  #
+  def find_or_create_note(opts)
+    report_note(opts)
+  end
+
+  #
+  # This methods returns a list of all notes in the database
+  #
+  def notes(wspace=workspace)
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    wspace.notes
+  }
+  end
+
+  #
+  # Report a Note to the database.  Notes can be tied to a ::Mdm::Workspace, Host, or Service.
+  #
+  # opts MUST contain
+  # +:type+::  The type of note, e.g. smb_peer_os
+  #
+  # opts can contain
+  # +:workspace+::  the workspace to associate with this Note
+  # +:host+::       an IP address or a Host object to associate with this Note
+  # +:service+::    a Service object to associate with this Note
+  # +:data+::       whatever it is you're making a note of
+  # +:port+::       along with +:host+ and +:proto+, a service to associate with this Note
+  # +:proto+::      along with +:host+ and +:port+, a service to associate with this Note
+  # +:update+::     what to do in case a similar Note exists, see below
+  #
+  # The +:update+ option can have the following values:
+  # +:unique+::       allow only a single Note per +:host+/+:type+ pair
+  # +:unique_data+::  like +:uniqe+, but also compare +:data+
+  # +:insert+::       always insert a new Note even if one with identical values exists
+  #
+  # If the provided +:host+ is an IP address and does not exist in the
+  # database, it will be created.  If +:workspace+, +:host+ and +:service+
+  # are all omitted, the new Note will be associated with the current
+  # workspace.
+  #
+  def report_note(opts)
+    return if not active
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    wspace = opts.delete(:workspace) || workspace
+    if wspace.kind_of? String
+      wspace = find_workspace(wspace)
+    end
+    seen = opts.delete(:seen) || false
+    crit = opts.delete(:critical) || false
+    host = nil
+    addr = nil
+    # Report the host so it's there for the Proc to use below
+    if opts[:host]
+      if opts[:host].kind_of? ::Mdm::Host
+        host = opts[:host]
+      else
+        addr = normalize_host(opts[:host])
+        host = report_host({:workspace => wspace, :host => addr})
+      end
+      # Do the same for a service if that's also included.
+      if (opts[:port])
+        proto = nil
+        sname = nil
+        case opts[:proto].to_s.downcase # Catch incorrect usages
+        when 'tcp','udp'
+          proto = opts[:proto]
+          sname = opts[:sname] if opts[:sname]
+        when 'dns','snmp','dhcp'
+          proto = 'udp'
+          sname = opts[:proto]
+        else
+          proto = 'tcp'
+          sname = opts[:proto]
+        end
+        sopts = {
+          :workspace => wspace,
+          :host  => host,
+          :port  => opts[:port],
+          :proto => proto
+        }
+        sopts[:name] = sname if sname
+        report_service(sopts)
+      end
+    end
+    # Update Modes can be :unique, :unique_data, :insert
+    mode = opts[:update] || :unique
+
+    ret = {}
+
+    if addr and not host
+      host = get_host(:workspace => wspace, :host => addr)
+    end
+    if host and (opts[:port] and opts[:proto])
+      service = get_service(wspace, host, opts[:proto], opts[:port])
+    elsif opts[:service] and opts[:service].kind_of? ::Mdm::Service
+      service = opts[:service]
+    end
+=begin
+    if host
+      host.updated_at = host.created_at
+      host.state      = HostState::Alive
+      host.save!
+    end
+=end
+    ntype  = opts.delete(:type) || opts.delete(:ntype) || (raise RuntimeError, "A note :type or :ntype is required")
+    data   = opts[:data]
+    note   = nil
+
+    conditions = { :ntype => ntype }
+    conditions[:host_id] = host[:id] if host
+    conditions[:service_id] = service[:id] if service
+
+    case mode
+    when :unique
+      note      = wspace.notes.where(conditions).first_or_initialize
+      note.data = data
+    when :unique_data
+      notes = wspace.notes.where(conditions)
+
+      # Don't make a new Note with the same data as one that already
+      # exists for the given: type and (host or service)
+      notes.each do |n|
+        # Compare the deserialized data from the table to the raw
+        # data we're looking for.  Because of the serialization we
+        # can't do this easily or reliably in SQL.
+        if n.data == data
+          note = n
+          break
+        end
+      end
+      if not note
+        # We didn't find one with the data we're looking for, make
+        # a new one.
+        note = wspace.notes.new(conditions.merge(:data => data))
+      end
+    else
+      # Otherwise, assume :insert, which means always make a new one
+      note = wspace.notes.new
+      if host
+        note.host_id = host[:id]
+      end
+      if opts[:service] and opts[:service].kind_of? ::Mdm::Service
+        note.service_id = opts[:service][:id]
+      end
+      note.seen     = seen
+      note.critical = crit
+      note.ntype    = ntype
+      note.data     = data
+    end
+    msf_import_timestamps(opts,note)
+    note.save!
+    ret[:note] = note
+  }
+  end
+end

lib/msf/core/db_manager/ref.rb

@@ -0,0 +1,33 @@
+module Msf::DBManager::Ref
+  #
+  # Find or create a reference matching this name
+  #
+  def find_or_create_ref(opts)
+    ret = {}
+    ret[:ref] = get_ref(opts[:name])
+    return ret[:ref] if ret[:ref]
+
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    ref = ::Mdm::Ref.find_or_initialize_by_name(opts[:name])
+    if ref and ref.changed?
+      ref.save!
+    end
+    ret[:ref] = ref
+  }
+  end
+
+  def get_ref(name)
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    ::Mdm::Ref.find_by_name(name)
+  }
+  end
+
+  #
+  # Find a reference matching this name
+  #
+  def has_ref?(name)
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    Mdm::Ref.find_by_name(name)
+  }
+  end
+end

lib/msf/core/db_manager/report.rb

@@ -0,0 +1,86 @@
+#
+# Standard library
+#
+
+require 'fileutils'
+
+module Msf::DBManager::Report
+  # TODO This method does not attempt to find. It just creates
+  # a report based on the passed params.
+  def find_or_create_report(opts)
+    report_report(opts)
+  end
+
+  # Creates a ReportArtifact based on passed parameters.
+  # @param opts [Hash] of ReportArtifact attributes
+  def report_artifact(opts)
+    return if not active
+
+    artifacts_dir = Report::ARTIFACT_DIR
+    tmp_path = opts[:file_path]
+    artifact_name = File.basename tmp_path
+    new_path = File.join(artifacts_dir, artifact_name)
+    created = opts.delete(:created_at)
+    updated = opts.delete(:updated_at)
+
+    unless File.exists? tmp_path
+      raise DBImportError 'Report artifact file to be imported does not exist.'
+    end
+
+    unless (File.directory?(artifacts_dir) && File.writable?(artifacts_dir))
+      raise DBImportError "Could not move report artifact file to #{artifacts_dir}."
+    end
+
+    if File.exists? new_path
+      unique_basename = "#{(Time.now.to_f*1000).to_i}_#{artifact_name}"
+      new_path = File.join(artifacts_dir, unique_basename)
+    end
+
+    FileUtils.copy(tmp_path, new_path)
+    opts[:file_path] = new_path
+    artifact = ReportArtifact.new(opts)
+    artifact.created_at = created
+    artifact.updated_at = updated
+
+    unless artifact.valid?
+      errors = artifact.errors.full_messages.join('; ')
+      raise RuntimeError "Artifact to be imported is not valid: #{errors}"
+    end
+    artifact.save
+  end
+
+  # Creates a Report based on passed parameters. Does not handle
+  # child artifacts.
+  # @param opts [Hash]
+  # @return [Integer] ID of created report
+  def report_report(opts)
+    return if not active
+    created = opts.delete(:created_at)
+    updated = opts.delete(:updated_at)
+    state   = opts.delete(:state)
+
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    report = Report.new(opts)
+    report.created_at = created
+    report.updated_at = updated
+
+    unless report.valid?
+      errors = report.errors.full_messages.join('; ')
+      raise RuntimeError "Report to be imported is not valid: #{errors}"
+    end
+    report.state = :complete # Presume complete since it was exported
+    report.save
+
+    report.id
+  }
+  end
+
+  #
+  # This methods returns a list of all reports in the database
+  #
+  def reports(wspace=workspace)
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    wspace.reports
+  }
+  end
+end

lib/msf/core/db_manager/route.rb

@@ -0,0 +1,38 @@
+module Msf::DBManager::Route
+  def report_session_route(session, route)
+    return if not active
+    if session.respond_to? :db_record
+      s = session.db_record
+    else
+      s = session
+    end
+    unless s.respond_to?(:routes)
+      raise ArgumentError.new("Invalid :session, expected Session object got #{session.class}")
+    end
+
+  ::ActiveRecord::Base.connection_pool.with_connection {
+
+    subnet, netmask = route.split("/")
+    s.routes.create(:subnet => subnet, :netmask => netmask)
+  }
+  end
+
+  def report_session_route_remove(session, route)
+    return if not active
+    if session.respond_to? :db_record
+      s = session.db_record
+    else
+      s = session
+    end
+    unless s.respond_to?(:routes)
+      raise ArgumentError.new("Invalid :session, expected Session object got #{session.class}")
+    end
+
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    subnet, netmask = route.split("/")
+    r = s.routes.find_by_subnet_and_netmask(subnet, netmask)
+    r.destroy if r
+  }
+  end
+
+end

lib/msf/core/db_manager/service.rb

@@ -0,0 +1,129 @@
+module Msf::DBManager::Service
+  # Deletes a port and associated vulns matching this port
+  def del_service(wspace, address, proto, port, comm='')
+
+    host = get_host(:workspace => wspace, :address => address)
+    return unless host
+
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    host.services.where({:proto => proto, :port => port}).each { |s| s.destroy }
+  }
+  end
+
+  # Iterates over the services table calling the supplied block with the
+  # service instance of each entry.
+  def each_service(wspace=workspace, &block)
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    services(wspace).each do |service|
+      block.call(service)
+    end
+  }
+  end
+
+  def find_or_create_service(opts)
+    report_service(opts)
+  end
+
+  def get_service(wspace, host, proto, port)
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    host = get_host(:workspace => wspace, :address => host)
+    return if not host
+    return host.services.find_by_proto_and_port(proto, port)
+  }
+  end
+
+  #
+  # Record a service in the database.
+  #
+  # opts MUST contain
+  # +:host+::  the host where this service is running
+  # +:port+::  the port where this service listens
+  # +:proto+:: the transport layer protocol (e.g. tcp, udp)
+  #
+  # opts may contain
+  # +:name+::  the application layer protocol (e.g. ssh, mssql, smb)
+  # +:sname+:: an alias for the above
+  #
+  def report_service(opts)
+    return if not active
+  ::ActiveRecord::Base.connection_pool.with_connection { |conn|
+    addr  = opts.delete(:host) || return
+    hname = opts.delete(:host_name)
+    hmac  = opts.delete(:mac)
+    host  = nil
+    wspace = opts.delete(:workspace) || workspace
+    hopts = {:workspace => wspace, :host => addr}
+    hopts[:name] = hname if hname
+    hopts[:mac]  = hmac  if hmac
+
+    # Other report_* methods take :sname to mean the service name, so we
+    # map it here to ensure it ends up in the right place despite not being
+    # a real column.
+    if opts[:sname]
+      opts[:name] = opts.delete(:sname)
+    end
+
+    if addr.kind_of? ::Mdm::Host
+      host = addr
+      addr = host.address
+    else
+      host = report_host(hopts)
+    end
+
+    if opts[:port].to_i.zero?
+      dlog("Skipping port zero for service '%s' on host '%s'" % [opts[:name],host.address])
+      return nil
+    end
+
+    ret  = {}
+=begin
+    host = get_host(:workspace => wspace, :address => addr)
+    if host
+      host.updated_at = host.created_at
+      host.state      = HostState::Alive
+      host.save!
+    end
+=end
+
+    proto = opts[:proto] || 'tcp'
+
+    service = host.services.find_or_initialize_by_port_and_proto(opts[:port].to_i, proto)
+    opts.each { |k,v|
+      if (service.attribute_names.include?(k.to_s))
+        service[k] = ((v and k == :name) ? v.to_s.downcase : v)
+      else
+        dlog("Unknown attribute for Service: #{k}")
+      end
+    }
+    service.state ||= ServiceState::Open
+    service.info  ||= ""
+
+    if (service and service.changed?)
+      msf_import_timestamps(opts,service)
+      service.save!
+    end
+
+    if opts[:task]
+      Mdm::TaskService.create(
+          :task => opts[:task],
+          :service => service
+      )
+    end
+
+    ret[:service] = service
+  }
+  end
+
+  # Returns a list of all services in the database
+  def services(wspace = workspace, only_up = false, proto = nil, addresses = nil, ports = nil, names = nil)
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    conditions = {}
+    conditions[:state] = [ServiceState::Open] if only_up
+    conditions[:proto] = proto if proto
+    conditions["hosts.address"] = addresses if addresses
+    conditions[:port] = ports if ports
+    conditions[:name] = names if names
+    wspace.services.includes(:host).where(conditions).order("hosts.address, port")
+  }
+  end
+end

lib/msf/core/db_manager/session.rb

@@ -0,0 +1,205 @@
+module Msf::DBManager::Session
+  # Returns a session based on opened_time, host address, and workspace
+  # (or returns nil)
+  def get_session(opts)
+    return if not active
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    wspace = opts[:workspace] || opts[:wspace] || workspace
+    addr   = opts[:addr] || opts[:address] || opts[:host] || return
+    host = get_host(:workspace => wspace, :host => addr)
+    time = opts[:opened_at] || opts[:created_at] || opts[:time] || return
+    ::Mdm::Session.find_by_host_id_and_opened_at(host.id, time)
+  }
+  end
+
+  # @note The Mdm::Session#desc will be truncated to 255 characters.
+  # @todo https://www.pivotaltracker.com/story/show/48249739
+  #
+  # @overload report_session(opts)
+  #   Creates an Mdm::Session from Msf::Session. If +via_exploit+ is set on the
+  #   +session+, then an Mdm::Vuln and Mdm::ExploitAttempt is created for the
+  #   session's host.  The Mdm::Host for the +session_host+ is created using
+  #   The session.session_host, +session.arch+ (if +session+ responds to arch),
+  #   and the workspace derived from opts or the +session+.  The Mdm::Session is
+  #   assumed to be +last_seen+ and +opened_at+ at the time report_session is
+  #   called.  +session.exploit_datastore['ParentModule']+ is used for the
+  #   Mdm::Session#via_exploit if +session.via_exploit+ is
+  #   'exploit/multi/handler'.
+  #
+  #   @param opts [Hash{Symbol => Object}] options
+  #   @option opt [Msf::Session, #datastore, #platform, #type, #via_exploit, #via_payload] :session
+  #     The in-memory session to persist to the database.
+  #   @option opts [Mdm::Workspace] :workspace The workspace for in which the
+  #     :session host is contained.  Also used as the workspace for the
+  #     Mdm::ExploitAttempt and Mdm::Vuln.  Defaults to Mdm::Worksapce with
+  #     Mdm::Workspace#name equal to +session.workspace+.
+  #   @return [nil] if {Msf::DBManager#active} is +false+.
+  #   @return [Mdm::Session] if session is saved
+  #   @raise [ArgumentError] if :session is not an {Msf::Session}.
+  #   @raise [ActiveRecord::RecordInvalid] if session is invalid and cannot be
+  #     saved, in which case, the Mdm::ExploitAttempt and Mdm::Vuln will not be
+  #     created, but the Mdm::Host will have been.   (There is no transaction
+  #       to rollback the Mdm::Host creation.)
+  #   @see #find_or_create_host
+  #   @see #normalize_host
+  #   @see #report_exploit_success
+  #   @see #report_vuln
+  #
+  # @overload report_session(opts)
+  #   Creates an Mdm::Session from Mdm::Host.
+  #
+  #   @param opts [Hash{Symbol => Object}] options
+  #   @option opts [DateTime, Time] :closed_at The date and time the sesion was
+  #     closed.
+  #   @option opts [String] :close_reason Reason the session was closed.
+  #   @option opts [Hash] :datastore {Msf::DataStore#to_h}.
+  #   @option opts [String] :desc Session description.  Will be truncated to 255
+  #     characters.
+  #   @option opts [Mdm::Host] :host The host on which the session was opened.
+  #   @option opts [DateTime, Time] :last_seen The last date and time the
+  #     session was seen to be open.  Defaults to :closed_at's value.
+  #   @option opts [DateTime, Time] :opened_at The date and time that the
+  #     session was opened.
+  #   @option opts [String] :platform The platform of the host.
+  #   @option opts [Array] :routes ([]) The routes through the session for
+  #     pivoting.
+  #   @option opts [String] :stype Session type.
+  #   @option opts [String] :via_exploit The {Msf::Module#fullname} of the
+  #     exploit that was used to open the session.
+  #   @option option [String] :via_payload the {MSf::Module#fullname} of the
+  #     payload sent to the host when the exploit was successful.
+  #   @return [nil] if {Msf::DBManager#active} is +false+.
+  #   @return [Mdm::Session] if session is saved.
+  #   @raise [ArgumentError] if :host is not an Mdm::Host.
+  #   @raise [ActiveRecord::RecordInvalid] if session is invalid and cannot be
+  #     saved.
+  #
+  # @raise ArgumentError if :host and :session is +nil+
+  def report_session(opts)
+    return if not active
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    if opts[:session]
+      raise ArgumentError.new("Invalid :session, expected Msf::Session") unless opts[:session].kind_of? Msf::Session
+      session = opts[:session]
+      wspace = opts[:workspace] || find_workspace(session.workspace)
+      h_opts = { }
+      h_opts[:host]      = normalize_host(session)
+      h_opts[:arch]      = session.arch if session.respond_to?(:arch) and session.arch
+      h_opts[:workspace] = wspace
+      host = find_or_create_host(h_opts)
+      sess_data = {
+          :host_id     => host.id,
+          :stype       => session.type,
+          :desc        => session.info,
+          :platform    => session.platform,
+          :via_payload => session.via_payload,
+          :via_exploit => session.via_exploit,
+          :routes      => [],
+          :datastore   => session.exploit_datastore.to_h,
+          :port        => session.session_port,
+          :opened_at   => Time.now.utc,
+          :last_seen   => Time.now.utc,
+          :local_id    => session.sid
+      }
+    elsif opts[:host]
+      raise ArgumentError.new("Invalid :host, expected Host object") unless opts[:host].kind_of? ::Mdm::Host
+      host = opts[:host]
+      sess_data = {
+        :host_id => host.id,
+        :stype => opts[:stype],
+        :desc => opts[:desc],
+        :platform => opts[:platform],
+        :via_payload => opts[:via_payload],
+        :via_exploit => opts[:via_exploit],
+        :routes => opts[:routes] || [],
+        :datastore => opts[:datastore],
+        :opened_at => opts[:opened_at],
+        :closed_at => opts[:closed_at],
+        :last_seen => opts[:last_seen] || opts[:closed_at],
+        :close_reason => opts[:close_reason],
+      }
+    else
+      raise ArgumentError.new("Missing option :session or :host")
+    end
+    ret = {}
+
+    # Truncate the session data if necessary
+    if sess_data[:desc]
+      sess_data[:desc] = sess_data[:desc][0,255]
+    end
+
+    # In the case of multi handler we cannot yet determine the true
+    # exploit responsible. But we can at least show the parent versus
+    # just the generic handler:
+    if session and session.via_exploit == "exploit/multi/handler" and sess_data[:datastore]['ParentModule']
+      sess_data[:via_exploit] = sess_data[:datastore]['ParentModule']
+    end
+
+    s = ::Mdm::Session.new(sess_data)
+    s.save!
+
+    if session and session.exploit_task and session.exploit_task.record
+      session_task =  session.exploit_task.record
+      if session_task.class == Mdm::Task
+        Mdm::TaskSession.create(:task => session_task, :session => s )
+      end
+    end
+
+
+    if opts[:session]
+      session.db_record = s
+    end
+
+    # If this is a live session, we know the host is vulnerable to something.
+    if opts[:session] and session.via_exploit
+      mod = framework.modules.create(session.via_exploit)
+
+      if session.via_exploit == "exploit/multi/handler" and sess_data[:datastore]['ParentModule']
+        mod_fullname = sess_data[:datastore]['ParentModule']
+        mod_name = ::Mdm::Module::Detail.find_by_fullname(mod_fullname).name
+      else
+        mod_name = mod.name
+        mod_fullname = mod.fullname
+      end
+
+      vuln_info = {
+        :host => host.address,
+        :name => mod_name,
+        :refs => mod.references,
+        :workspace => wspace,
+        :exploited_at => Time.now.utc,
+        :info => "Exploited by #{mod_fullname} to create Session #{s.id}"
+      }
+
+      port    = session.exploit_datastore["RPORT"]
+      service = (port ? host.services.find_by_port(port.to_i) : nil)
+
+      vuln_info[:service] = service if service
+
+      vuln = framework.db.report_vuln(vuln_info)
+
+      if session.via_exploit == "exploit/multi/handler" and sess_data[:datastore]['ParentModule']
+        via_exploit = sess_data[:datastore]['ParentModule']
+      else
+        via_exploit = session.via_exploit
+      end
+      attempt_info = {
+        :timestamp   => Time.now.utc,
+        :workspace   => wspace,
+        :module      => via_exploit,
+        :username    => session.username,
+        :refs        => mod.references,
+        :session_id  => s.id,
+        :host        => host,
+        :service     => service,
+        :vuln        => vuln
+      }
+
+      framework.db.report_exploit_success(attempt_info)
+
+    end
+
+    s
+  }
+  end
+end

lib/msf/core/db_manager/session_event.rb

@@ -0,0 +1,49 @@
+module Msf::DBManager::SessionEvent
+  #
+  # Record a session event in the database
+  #
+  # opts MUST contain one of:
+  # +:session+:: the Msf::Session OR the ::Mdm::Session we are reporting
+  # +:etype+::   event type, enum: command, output, upload, download, filedelete
+  #
+  # opts may contain
+  # +:output+::      the data for an output event
+  # +:command+::     the data for an command event
+  # +:remote_path+:: path to the associated file for upload, download, and filedelete events
+  # +:local_path+::  path to the associated file for upload, and download
+  #
+  def report_session_event(opts)
+    return if not active
+    raise ArgumentError.new("Missing required option :session") if opts[:session].nil?
+    raise ArgumentError.new("Expected an :etype") unless opts[:etype]
+    session = nil
+
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    if opts[:session].respond_to? :db_record
+      session = opts[:session].db_record
+      if session.nil?
+        # The session doesn't have a db_record which means
+        #  a) the database wasn't connected at session registration time
+        # or
+        #  b) something awful happened and the report_session call failed
+        #
+        # Either way, we can't do anything with this session as is, so
+        # log a warning and punt.
+        wlog("Warning: trying to report a session_event for a session with no db_record (#{opts[:session].sid})")
+        return
+      end
+      event_data = { :created_at => Time.now }
+    else
+      session = opts[:session]
+      event_data = { :created_at => opts[:created_at] }
+    end
+
+    event_data[:session_id] = session.id
+    [:remote_path, :local_path, :output, :command, :etype].each do |attr|
+      event_data[attr] = opts[attr] if opts[attr]
+    end
+
+    s = ::Mdm::SessionEvent.create(event_data)
+  }
+  end
+end

lib/msf/core/db_manager/sink.rb

@@ -0,0 +1,34 @@
+module Msf::DBManager::Sink
+  #
+  # Attributes
+  #
+
+  # Stores a TaskManager for serializing database events
+  attr_accessor :sink
+
+  #
+  # Instance Methods
+  #
+
+  #
+  # Create a new database sink and initialize it
+  #
+  def initialize_sink
+    self.sink = Msf::TaskManager.new(framework)
+    self.sink.start
+  end
+
+  #
+  # Add a new task to the sink
+  #
+  def queue(proc)
+    self.sink.queue_proc(proc)
+  end
+
+  #
+  # Wait for all pending write to finish
+  #
+  def sync
+    # There is no more queue.
+  end
+end

lib/msf/core/db_manager/task.rb

@@ -0,0 +1,57 @@
+module Msf::DBManager::Task
+  #
+  # Find or create a task matching this type/data
+  #
+  def find_or_create_task(opts)
+    report_task(opts)
+  end
+
+  def report_task(opts)
+    return if not active
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    wspace = opts.delete(:workspace) || workspace
+    path = opts.delete(:path) || (raise RuntimeError, "A task :path is required")
+
+    ret = {}
+
+    user      = opts.delete(:user)
+    desc      = opts.delete(:desc)
+    error     = opts.delete(:error)
+    info      = opts.delete(:info)
+    mod       = opts.delete(:mod)
+    options   = opts.delete(:options)
+    prog      = opts.delete(:prog)
+    result    = opts.delete(:result)
+    completed_at = opts.delete(:completed_at)
+    task      = wspace.tasks.new
+
+    task.created_by = user
+    task.description = desc
+    task.error = error if error
+    task.info = info
+    task.module = mod
+    task.options = options
+    task.path = path
+    task.progress = prog
+    task.result = result if result
+    msf_import_timestamps(opts,task)
+    # Having blank completed_ats, while accurate, will cause unstoppable tasks.
+    if completed_at.nil? || completed_at.empty?
+      task.completed_at = opts[:updated_at]
+    else
+      task.completed_at = completed_at
+    end
+    task.save!
+    ret[:task] = task
+  }
+  end
+
+  #
+  # This methods returns a list of all tasks in the database
+  #
+  def tasks(wspace=workspace)
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    wspace.tasks
+  }
+  end
+end

lib/msf/core/db_manager/vuln.rb

@@ -0,0 +1,250 @@
+module Msf::DBManager::Vuln
+  #
+  # This method iterates the vulns table calling the supplied block with the
+  # vuln instance of each entry.
+  #
+  def each_vuln(wspace=workspace,&block)
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    wspace.vulns.each do |vulns|
+      block.call(vulns)
+    end
+  }
+  end
+
+  #
+  # Find or create a vuln matching this service/name
+  #
+  def find_or_create_vuln(opts)
+    report_vuln(opts)
+  end
+
+  def find_vuln_by_details(details_map, host, service=nil)
+
+    # Create a modified version of the criteria in order to match against
+    # the joined version of the fields
+
+    crit = {}
+    details_map.each_pair do |k,v|
+      crit[ "vuln_details.#{k}" ] = v
+    end
+
+    vuln = nil
+
+    if service
+      vuln = service.vulns.find(:first, :include => [:vuln_details], :conditions => crit)
+    end
+
+    # Return if we matched based on service
+    return vuln if vuln
+
+    # Prevent matches against other services
+    crit["vulns.service_id"] = nil if service
+    vuln = host.vulns.find(:first, :include => [:vuln_details], :conditions => crit)
+
+    return vuln
+  end
+
+  def find_vuln_by_refs(refs, host, service=nil)
+
+    vuln = nil
+
+    # Try to find an existing vulnerability with the same service & references
+    # If there are multiple matches, choose the one with the most matches
+    if service
+      refs_ids = refs.map{|x| x.id }
+      vuln = service.vulns.find(:all, :include => [:refs], :conditions => { 'refs.id' => refs_ids }).sort { |a,b|
+        ( refs_ids - a.refs.map{|x| x.id } ).length <=> ( refs_ids - b.refs.map{|x| x.id } ).length
+      }.first
+    end
+
+    # Return if we matched based on service
+    return vuln if vuln
+
+    # Try to find an existing vulnerability with the same host & references
+    # If there are multiple matches, choose the one with the most matches
+    refs_ids = refs.map{|x| x.id }
+    vuln = host.vulns.find(:all, :include => [:refs], :conditions => { 'service_id' => nil, 'refs.id' => refs_ids }).sort { |a,b|
+      ( refs_ids - a.refs.map{|x| x.id } ).length <=> ( refs_ids - b.refs.map{|x| x.id } ).length
+    }.first
+
+    return vuln
+  end
+
+  def get_vuln(wspace, host, service, name, data='')
+    raise RuntimeError, "Not workspace safe: #{caller.inspect}"
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    vuln = nil
+    if (service)
+      vuln = ::Mdm::Vuln.find.where("name = ? and service_id = ? and host_id = ?", name, service.id, host.id).order("vulns.id DESC").first()
+    else
+      vuln = ::Mdm::Vuln.find.where("name = ? and host_id = ?", name, host.id).first()
+    end
+
+    return vuln
+  }
+  end
+
+  #
+  # Find a vulnerability matching this name
+  #
+  def has_vuln?(name)
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    Mdm::Vuln.find_by_name(name)
+  }
+  end
+
+  #
+  # opts MUST contain
+  # +:host+:: the host where this vulnerability resides
+  # +:name+:: the friendly name for this vulnerability (title)
+  #
+  # opts can contain
+  # +:info+::   a human readable description of the vuln, free-form text
+  # +:refs+::   an array of Ref objects or string names of references
+  # +:details:: a hash with :key pointed to a find criteria hash and the rest containing VulnDetail fields
+  #
+  def report_vuln(opts)
+    return if not active
+    raise ArgumentError.new("Missing required option :host") if opts[:host].nil?
+    raise ArgumentError.new("Deprecated data column for vuln, use .info instead") if opts[:data]
+    name = opts[:name] || return
+    info = opts[:info]
+
+  ::ActiveRecord::Base.connection_pool.with_connection {
+
+    wspace = opts.delete(:workspace) || workspace
+    exploited_at = opts[:exploited_at] || opts["exploited_at"]
+    details = opts.delete(:details)
+    rids = opts.delete(:ref_ids)
+
+    if opts[:refs]
+      rids ||= []
+      opts[:refs].each do |r|
+        if (r.respond_to?(:ctx_id)) and (r.respond_to?(:ctx_val))
+          r = "#{r.ctx_id}-#{r.ctx_val}"
+        end
+        rids << find_or_create_ref(:name => r)
+      end
+    end
+
+    host = nil
+    addr = nil
+    if opts[:host].kind_of? ::Mdm::Host
+      host = opts[:host]
+    else
+      host = report_host({:workspace => wspace, :host => opts[:host]})
+      addr = normalize_host(opts[:host])
+    end
+
+    ret = {}
+
+    # Truncate the info field at the maximum field length
+    if info
+      info = info[0,65535]
+    end
+
+    # Truncate the name field at the maximum field length
+    name = name[0,255]
+
+    # Placeholder for the vuln object
+    vuln = nil
+
+    # Identify the associated service
+    service = opts.delete(:service)
+
+    # Treat port zero as no service
+    if service or opts[:port].to_i > 0
+
+      if not service
+        proto = nil
+        case opts[:proto].to_s.downcase # Catch incorrect usages, as in report_note
+        when 'tcp','udp'
+          proto = opts[:proto]
+        when 'dns','snmp','dhcp'
+          proto = 'udp'
+          sname = opts[:proto]
+        else
+          proto = 'tcp'
+          sname = opts[:proto]
+        end
+
+        service = host.services.find_or_create_by_port_and_proto(opts[:port].to_i, proto)
+      end
+
+      # Try to find an existing vulnerability with the same service & references
+      # If there are multiple matches, choose the one with the most matches
+      # If a match is found on a vulnerability with no associated service,
+      # update that vulnerability with our service information. This helps
+      # prevent dupes of the same vuln found by both local patch and
+      # service detection.
+      if rids and rids.length > 0
+        vuln = find_vuln_by_refs(rids, host, service)
+        vuln.service = service if vuln
+      end
+    else
+      # Try to find an existing vulnerability with the same host & references
+      # If there are multiple matches, choose the one with the most matches
+      if rids and rids.length > 0
+        vuln = find_vuln_by_refs(rids, host)
+      end
+    end
+
+    # Try to match based on vuln_details records
+    if not vuln and opts[:details_match]
+      vuln = find_vuln_by_details(opts[:details_match], host, service)
+      if vuln and service and not vuln.service
+        vuln.service = service
+      end
+    end
+
+    # No matches, so create a new vuln record
+    unless vuln
+      if service
+        vuln = service.vulns.find_by_name(name)
+      else
+        vuln = host.vulns.find_by_name(name)
+      end
+
+      unless vuln
+
+        vinf = {
+          :host_id => host.id,
+          :name    => name,
+          :info    => info
+        }
+
+        vinf[:service_id] = service.id if service
+        vuln = Mdm::Vuln.create(vinf)
+      end
+    end
+
+    # Set the exploited_at value if provided
+    vuln.exploited_at = exploited_at if exploited_at
+
+    # Merge the references
+    if rids
+      vuln.refs << (rids - vuln.refs)
+    end
+
+    # Finalize
+    if vuln.changed?
+      msf_import_timestamps(opts,vuln)
+      vuln.save!
+    end
+
+    # Handle vuln_details parameters
+    report_vuln_details(vuln, details) if details
+
+    vuln
+  }
+  end
+
+  #
+  # This methods returns a list of all vulnerabilities in the database
+  #
+  def vulns(wspace=workspace)
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    wspace.vulns
+  }
+  end
+end

lib/msf/core/db_manager/vuln_attempt.rb

@@ -0,0 +1,20 @@
+module Msf::DBManager::VulnAttempt
+  def report_vuln_attempt(vuln, opts)
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    return if not vuln
+    info = {}
+
+    # Opts can be keyed by strings or symbols
+    ::Mdm::VulnAttempt.column_names.each do |kn|
+      k = kn.to_sym
+      next if ['id', 'vuln_id'].include?(kn)
+      info[k] = opts[kn] if opts[kn]
+      info[k] = opts[k]  if opts[k]
+    end
+
+    return unless info[:attempted_at]
+
+    vuln.vuln_attempts.create(info)
+  }
+  end
+end

lib/msf/core/db_manager/vuln_detail.rb

@@ -0,0 +1,31 @@
+module Msf::DBManager::VulnDetail
+  #
+  # Populate the vuln_details table with additional
+  # information, matched by a specific criteria
+  #
+  def report_vuln_details(vuln, details)
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    detail = ::Mdm::VulnDetail.where(( details.delete(:key) || {} ).merge(:vuln_id => vuln.id)).first
+    if detail
+      details.each_pair do |k,v|
+        detail[k] = v
+      end
+      detail.save! if detail.changed?
+      detail
+    else
+      detail = ::Mdm::VulnDetail.create(details.merge(:vuln_id => vuln.id))
+    end
+  }
+  end
+
+  #
+  # Update vuln_details records en-masse based on specific criteria
+  # Note that this *can* update data across workspaces
+  #
+  def update_vuln_details(details)
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    criteria = details.delete(:key) || {}
+    ::Mdm::VulnDetail.update(key, details)
+  }
+  end
+end

lib/msf/core/db_manager/web.rb

@@ -0,0 +1,363 @@
+module Msf::DBManager::Web
+  #
+  # Report a Web Form to the database.  WebForm must be tied to an existing Web Site
+  #
+  # opts MUST contain
+  # +:web_site+:: the web site object that this page should be associated with
+  # +:path+::     the virtual host name for this particular web site
+  # +:query+::    the query string that is appended to the path (not valid for GET)
+  # +:method+::   the form method, one of GET, POST, or PATH
+  # +:params+::   an ARRAY of all parameters and values specified in the form
+  #
+  # If web_site is NOT specified, the following values are mandatory
+  # +:host+::  the ip address of the server hosting the web site
+  # +:port+::  the port number of the associated web site
+  # +:vhost+:: the virtual host for this particular web site
+  # +:ssl+::   whether or not SSL is in use on this port
+  #
+  # Duplicate records for a given web_site, path, method, and params combination will be overwritten
+  #
+  def report_web_form(opts)
+    return if not active
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    wspace = opts.delete(:workspace) || workspace
+
+    path    = opts[:path]
+    meth    = opts[:method].to_s.upcase
+    para    = opts[:params]
+    quer    = opts[:query].to_s
+    site    = nil
+
+    if not (path and meth)
+      raise ArgumentError, "report_web_form requires the path and method parameters"
+    end
+
+    if not %W{GET POST PATH}.include?(meth)
+      raise ArgumentError, "report_web_form requires the method to be one of GET, POST, PATH"
+    end
+
+    if opts[:web_site] and opts[:web_site].kind_of?(::Mdm::WebSite)
+      site = opts.delete(:web_site)
+    else
+      site = report_web_site(
+        :workspace => wspace,
+        :host      => opts[:host], :port => opts[:port],
+        :vhost     => opts[:host], :ssl  => opts[:ssl]
+      )
+      if not site
+        raise ArgumentError, "report_web_form was unable to create the associated web site"
+      end
+    end
+
+    ret = {}
+
+    # Since one of our serialized fields is used as a unique parameter, we must do the final
+    # comparisons through ruby and not SQL.
+
+    form = nil
+    ::Mdm::WebForm.find_all_by_web_site_id_and_path_and_method_and_query(site[:id], path, meth, quer).each do |xform|
+      if xform.params == para
+        form = xform
+        break
+      end
+    end
+    if not form
+      form = ::Mdm::WebForm.new
+      form.web_site_id = site[:id]
+      form.path        = path
+      form.method      = meth
+      form.params      = para
+      form.query       = quer
+    end
+
+    msf_import_timestamps(opts, form)
+    form.save!
+    ret[:web_form] = form
+  }
+  end
+
+  #
+  # Report a Web Page to the database.  WebPage must be tied to an existing Web Site
+  #
+  # opts MUST contain
+  # +:web_site+:: the web site object that this page should be associated with
+  # +:path+::     the virtual host name for this particular web site
+  # +:code+::     the http status code from requesting this page
+  # +:headers+::  this is a HASH of headers (lowercase name as key) of ARRAYs of values
+  # +:body+::     the document body of the server response
+  # +:query+::    the query string after the path
+  #
+  # If web_site is NOT specified, the following values are mandatory
+  # +:host+::  the ip address of the server hosting the web site
+  # +:port+::  the port number of the associated web site
+  # +:vhost+:: the virtual host for this particular web site
+  # +:ssl+::   whether or not SSL is in use on this port
+  #
+  # These values will be used to create new host, service, and web_site records
+  #
+  # opts can contain
+  # +:cookie+::   the Set-Cookie headers, merged into a string
+  # +:auth+::     the Authorization headers, merged into a string
+  # +:ctype+::    the Content-Type headers, merged into a string
+  # +:mtime+::    the timestamp returned from the server of the last modification time
+  # +:location+:: the URL that a redirect points to
+  #
+  # Duplicate records for a given web_site, path, and query combination will be overwritten
+  #
+  def report_web_page(opts)
+    return if not active
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    wspace = opts.delete(:workspace) || workspace
+
+    path    = opts[:path]
+    code    = opts[:code].to_i
+    body    = opts[:body].to_s
+    query   = opts[:query].to_s
+    headers = opts[:headers]
+    site    = nil
+
+    if not (path and code and body and headers)
+      raise ArgumentError, "report_web_page requires the path, query, code, body, and headers parameters"
+    end
+
+    if opts[:web_site] and opts[:web_site].kind_of?(::Mdm::WebSite)
+      site = opts.delete(:web_site)
+    else
+      site = report_web_site(
+        :workspace => wspace,
+        :host      => opts[:host], :port => opts[:port],
+        :vhost     => opts[:host], :ssl  => opts[:ssl]
+      )
+      if not site
+        raise ArgumentError, "report_web_page was unable to create the associated web site"
+      end
+    end
+
+    ret = {}
+
+    page = ::Mdm::WebPage.find_or_initialize_by_web_site_id_and_path_and_query(site[:id], path, query)
+    page.code     = code
+    page.body     = body
+    page.headers  = headers
+    page.cookie   = opts[:cookie] if opts[:cookie]
+    page.auth     = opts[:auth]   if opts[:auth]
+    page.mtime    = opts[:mtime]  if opts[:mtime]
+    page.ctype    = opts[:ctype]  if opts[:ctype]
+    page.location = opts[:location] if opts[:location]
+    msf_import_timestamps(opts, page)
+    page.save!
+
+    ret[:web_page] = page
+  }
+
+  end
+
+  #
+  # WMAP
+  # Support methods
+  #
+
+  #
+  # Report a Web Site to the database.  WebSites must be tied to an existing Service
+  #
+  # opts MUST contain
+  # +:service+:: the service object this site should be associated with
+  # +:vhost+::   the virtual host name for this particular web site`
+  #
+  # If +:service+ is NOT specified, the following values are mandatory
+  # +:host+:: the ip address of the server hosting the web site
+  # +:port+:: the port number of the associated web site
+  # +:ssl+::  whether or not SSL is in use on this port
+  #
+  # These values will be used to create new host and service records
+  #
+  # opts can contain
+  # +:options+:: a hash of options for accessing this particular web site
+  # +:info+:: if present, report the service with this info
+  #
+  # Duplicate records for a given host, port, vhost combination will be overwritten
+  #
+  def report_web_site(opts)
+    return if not active
+  ::ActiveRecord::Base.connection_pool.with_connection { |conn|
+    wspace = opts.delete(:workspace) || workspace
+    vhost  = opts.delete(:vhost)
+
+    addr = nil
+    port = nil
+    name = nil
+    serv = nil
+    info = nil
+
+    if opts[:service] and opts[:service].kind_of?(::Mdm::Service)
+      serv = opts[:service]
+    else
+      addr = opts[:host]
+      port = opts[:port]
+      name = opts[:ssl] ? 'https' : 'http'
+      info = opts[:info]
+      if not (addr and port)
+        raise ArgumentError, "report_web_site requires service OR host/port/ssl"
+      end
+
+      # Force addr to be the address and not hostname
+      addr = Rex::Socket.getaddress(addr, true)
+    end
+
+    ret = {}
+
+    host = serv ? serv.host : find_or_create_host(
+      :workspace => wspace,
+      :host      => addr,
+      :state     => Msf::HostState::Alive
+    )
+
+    if host.name.to_s.empty?
+      host.name = vhost
+      host.save!
+    end
+
+    serv = serv ? serv : find_or_create_service(
+      :workspace => wspace,
+      :host      => host,
+      :port      => port,
+      :proto     => 'tcp',
+      :state     => 'open'
+    )
+
+    # Change the service name if it is blank or it has
+    # been explicitly specified.
+    if opts.keys.include?(:ssl) or serv.name.to_s.empty?
+      name = opts[:ssl] ? 'https' : 'http'
+      serv.name = name
+    end
+    # Add the info if it's there.
+    unless info.to_s.empty?
+      serv.info = info
+    end
+    serv.save! if serv.changed?
+=begin
+    host.updated_at = host.created_at
+    host.state      = HostState::Alive
+    host.save!
+=end
+
+    vhost ||= host.address
+    site = ::Mdm::WebSite.find_or_initialize_by_vhost_and_service_id(vhost, serv[:id])
+    site.options = opts[:options] if opts[:options]
+
+    # XXX:
+    msf_import_timestamps(opts, site)
+    site.save!
+
+    ret[:web_site] = site
+  }
+  end
+
+  #
+  # Report a Web Vuln to the database.  WebVuln must be tied to an existing Web Site
+  #
+  # opts MUST contain
+  # +:web_site+::  the web site object that this page should be associated with
+  # +:path+::      the virtual host name for this particular web site
+  # +:query+::     the query string appended to the path (not valid for GET method flaws)
+  # +:method+::    the form method, one of GET, POST, or PATH
+  # +:params+::    an ARRAY of all parameters and values specified in the form
+  # +:pname+::     the specific field where the vulnerability occurs
+  # +:proof+::     the string showing proof of the vulnerability
+  # +:risk+::      an INTEGER value from 0 to 5 indicating the risk (5 is highest)
+  # +:name+::      the string indicating the type of vulnerability
+  #
+  # If web_site is NOT specified, the following values are mandatory
+  # +:host+::  the ip address of the server hosting the web site
+  # +:port+::  the port number of the associated web site
+  # +:vhost+:: the virtual host for this particular web site
+  # +:ssl+::   whether or not SSL is in use on this port
+  #
+  #
+  # Duplicate records for a given web_site, path, method, pname, and name
+  # combination will be overwritten
+  #
+  def report_web_vuln(opts)
+    return if not active
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    wspace = opts.delete(:workspace) || workspace
+
+    path    = opts[:path]
+    meth    = opts[:method]
+    para    = opts[:params] || []
+    quer    = opts[:query].to_s
+    pname   = opts[:pname]
+    proof   = opts[:proof]
+    risk    = opts[:risk].to_i
+    name    = opts[:name].to_s.strip
+    blame   = opts[:blame].to_s.strip
+    desc    = opts[:description].to_s.strip
+    conf    = opts[:confidence].to_i
+    cat     = opts[:category].to_s.strip
+    payload = opts[:payload].to_s
+    owner   = opts[:owner] ? opts[:owner].shortname : nil
+
+
+    site    = nil
+
+    if not (path and meth and proof and pname)
+      raise ArgumentError, "report_web_vuln requires the path, method, proof, risk, name, params, and pname parameters. Received #{opts.inspect}"
+    end
+
+    if not %W{GET POST PATH}.include?(meth)
+      raise ArgumentError, "report_web_vuln requires the method to be one of GET, POST, PATH. Received '#{meth}'"
+    end
+
+    if risk < 0 or risk > 5
+      raise ArgumentError, "report_web_vuln requires the risk to be between 0 and 5 (inclusive). Received '#{risk}'"
+    end
+
+    if conf < 0 or conf > 100
+      raise ArgumentError, "report_web_vuln requires the confidence to be between 1 and 100 (inclusive). Received '#{conf}'"
+    end
+
+    if cat.empty?
+      raise ArgumentError, "report_web_vuln requires the category to be a valid string"
+    end
+
+    if name.empty?
+      raise ArgumentError, "report_web_vuln requires the name to be a valid string"
+    end
+
+    if opts[:web_site] and opts[:web_site].kind_of?(::Mdm::WebSite)
+      site = opts.delete(:web_site)
+    else
+      site = report_web_site(
+        :workspace => wspace,
+        :host      => opts[:host], :port => opts[:port],
+        :vhost     => opts[:host], :ssl  => opts[:ssl]
+      )
+      if not site
+        raise ArgumentError, "report_web_form was unable to create the associated web site"
+      end
+    end
+
+    ret = {}
+
+    meth = meth.to_s.upcase
+
+    vuln = ::Mdm::WebVuln.find_or_initialize_by_web_site_id_and_path_and_method_and_pname_and_name_and_category_and_query(site[:id], path, meth, pname, name, cat, quer)
+    vuln.name     = name
+    vuln.risk     = risk
+    vuln.params   = para
+    vuln.proof    = proof.to_s
+    vuln.category = cat
+    vuln.blame    = blame
+    vuln.description = desc
+    vuln.confidence  = conf
+    vuln.payload = payload
+    vuln.owner   = owner
+
+    msf_import_timestamps(opts, vuln)
+    vuln.save!
+
+    ret[:web_vuln] = vuln
+  }
+  end
+end

lib/msf/core/db_manager/wmap.rb

@@ -0,0 +1,189 @@
+# @note Wmap is a plugin and so these methods, that are only meant for that plugin, should not be part of the core
+#   library.
+module Msf::DBManager::WMAP
+  # Create a request (by hand)
+  def create_request(host,port,ssl,meth,path,headers,query,body,respcode,resphead,response)
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    req = ::Mdm::WmapRequest.create(
+        :host => host,
+        :address => host,
+        :port => port,
+        :ssl => ssl,
+        :meth => meth,
+        :path => path,
+        :headers => headers,
+        :query => query,
+        :body => body,
+        :respcode => respcode,
+        :resphead => resphead,
+        :response => response
+      )
+    #framework.events.on_db_request(rec)
+  }
+  end
+
+  # Create a target
+  def create_target(host,port,ssl,sel)
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    tar = ::Mdm::WmapTarget.create(
+        :host => host,
+        :address => host,
+        :port => port,
+        :ssl => ssl,
+        :selected => sel
+      )
+    #framework.events.on_db_target(rec)
+  }
+  end
+
+  # This methods deletes all targets from targets table in the database
+  def delete_all_targets
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    ::Mdm::WmapTarget.delete_all
+  }
+  end
+
+  # This method iterates the requests table identifiying possible targets
+  # This method wiil be remove on second phase of db merging.
+  def each_distinct_target(&block)
+    request_distinct_targets.each do |target|
+      block.call(target)
+    end
+  end
+
+  # This method iterates the requests table calling the supplied block with the
+  # request instance of each entry.
+  def each_request(&block)
+    requests.each do |request|
+      block.call(request)
+    end
+  end
+
+  # This method iterates the requests table returning a list of all requests of a specific target
+  def each_request_target(&block)
+    target_requests('').each do |req|
+      block.call(req)
+    end
+  end
+
+  # This method iterates the requests table returning a list of all requests of a specific target
+  def each_request_target_with_body(&block)
+    target_requests('AND wmap_requests.body IS NOT NULL').each do |req|
+      block.call(req)
+    end
+  end
+
+  # This method iterates the requests table returning a list of all requests of a specific target
+  def each_request_target_with_headers(&block)
+    target_requests('AND wmap_requests.headers IS NOT NULL').each do |req|
+      block.call(req)
+    end
+  end
+
+  # This method iterates the requests table returning a list of all requests of a specific target
+  def each_request_target_with_path(&block)
+    target_requests('AND wmap_requests.path IS NOT NULL').each do |req|
+      block.call(req)
+    end
+  end
+
+  # This method iterates the requests table returning a list of all requests of a specific target
+  def each_request_target_with_query(&block)
+    target_requests('AND wmap_requests.query IS NOT NULL').each do |req|
+      block.call(req)
+    end
+  end
+
+  # This method iterates the targets table calling the supplied block with the
+  # target instance of each entry.
+  def each_target(&block)
+    targets.each do |target|
+      block.call(target)
+    end
+  end
+
+  # Find a target matching this id
+  def get_target(id)
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    target = ::Mdm::WmapTarget.where("id = ?", id).first()
+    return target
+  }
+  end
+
+  # This method returns a list of all possible targets available in requests
+  # This method wiil be remove on second phase of db merging.
+  def request_distinct_targets
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    ::Mdm::WmapRequest.select('DISTINCT host,address,port,ssl')
+  }
+  end
+
+  # This method allows to query directly the requests table. To be used mainly by modules
+  def request_sql(host,port,extra_condition)
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    ::Mdm::WmapRequest.where("wmap_requests.host = ? AND wmap_requests.port = ? #{extra_condition}", host , port)
+  }
+  end
+
+  # This methods returns a list of all targets in the database
+  def requests
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    ::Mdm::WmapRequest.find(:all)
+  }
+  end
+
+  # Selected host
+  def selected_host
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    selhost = ::Mdm::WmapTarget.where("selected != 0").first()
+    if selhost
+      return selhost.host
+    else
+      return
+    end
+  }
+  end
+
+  # Selected id
+  def selected_id
+    selected_wmap_target.object_id
+  end
+
+  # Selected port
+  def selected_port
+    selected_wmap_target.port
+  end
+
+  # Selected ssl
+  def selected_ssl
+    selected_wmap_target.ssl
+  end
+
+  # Selected target
+  def selected_wmap_target
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    ::Mdm::WmapTarget.find.where("selected != 0")
+  }
+  end
+
+  # Quick way to query the database (used by wmap_sql)
+  def sql_query(sqlquery)
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    ActiveRecord::Base.connection.select_all(sqlquery)
+  }
+  end
+
+  # This method returns a list of all requests from target
+  def target_requests(extra_condition)
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    ::Mdm::WmapRequest.where("wmap_requests.host = ? AND wmap_requests.port = ? #{extra_condition}",selected_host,selected_port)
+  }
+  end
+
+  # This methods returns a list of all targets in the database
+  def targets
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    ::Mdm::WmapTarget.find(:all)
+  }
+  end
+end

lib/msf/core/db_manager/workspace.rb

@@ -0,0 +1,36 @@
+module Msf::DBManager::Workspace
+  #
+  # Creates a new workspace in the database
+  #
+  def add_workspace(name)
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    ::Mdm::Workspace.find_or_create_by_name(name)
+  }
+  end
+
+  def default_workspace
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    ::Mdm::Workspace.default
+  }
+  end
+
+  def find_workspace(name)
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    ::Mdm::Workspace.find_by_name(name)
+  }
+  end
+
+  def workspace
+    framework.db.find_workspace(@workspace_name)
+  end
+
+  def workspace=(workspace)
+    @workspace_name = workspace.name
+  end
+
+  def workspaces
+  ::ActiveRecord::Base.connection_pool.with_connection {
+    ::Mdm::Workspace.find(:all)
+  }
+  end
+end

lib/msf/core/host_state.rb

@@ -0,0 +1,9 @@
+# The states that a host can be in.
+module Msf::HostState
+  # The host is alive.
+  Alive   = "alive"
+  # The host is dead.
+  Dead    = "down"
+  # The host state is unknown.
+  Unknown = "unknown"
+end

lib/msf/core/service_state.rb

@@ -0,0 +1,7 @@
+# The states that a service can be in.
+module Msf::ServiceState
+  Closed    = "closed"
+  Filtered  = "filtered"
+  Open      = "open"
+  Unknown   = "unknown"
+end

spec/support/shared/examples/msf/db_manager/adapter.rb

@@ -0,0 +1,16 @@
+shared_examples_for 'Msf::DBManager::Adapter' do
+  context 'CONSTANTS' do
+    context 'ADAPTER' do
+      subject(:adapter) {
+        described_class::ADAPTER
+      }
+
+      it { is_expected.to eq('postgresql') }
+    end
+  end
+
+  it { is_expected.to respond_to :driver }
+  it { is_expected.to respond_to :drivers }
+  it { is_expected.to respond_to :drivers= }
+  it { is_expected.to respond_to :initialize_adapter }
+end

spec/support/shared/examples/msf/db_manager/client.rb

@@ -0,0 +1,5 @@
+shared_examples_for 'Msf::DBManager::Client' do
+  it { is_expected.to respond_to :find_or_create_client }
+  it { is_expected.to respond_to :get_client }
+  it { is_expected.to respond_to :report_client }
+end

spec/support/shared/examples/msf/db_manager/connection.rb

@@ -0,0 +1,10 @@
+shared_examples_for 'Msf::DBManager::Connection' do
+  it { is_expected.to respond_to :active }
+  it { is_expected.to respond_to :after_establish_connection }
+  it { is_expected.to respond_to :connect }
+  it { is_expected.to respond_to :connection_established? }
+  it { is_expected.to respond_to :create_db }
+  it { is_expected.to respond_to :disconnect }
+  it { is_expected.to respond_to :usable }
+  it { is_expected.to respond_to :usable= }
+end

spec/support/shared/examples/msf/db_manager/cred.rb

@@ -0,0 +1,8 @@
+shared_examples_for 'Msf::DBManager::Cred' do
+  it { is_expected.to respond_to :creds }
+  it { is_expected.to respond_to :each_cred }
+  it { is_expected.to respond_to :find_or_create_cred }
+  it { is_expected.to respond_to :report_auth }
+  it { is_expected.to respond_to :report_auth_info }
+  it { is_expected.to respond_to :report_cred }
+end

spec/support/shared/examples/msf/db_manager/event.rb

@@ -0,0 +1,4 @@
+shared_examples_for 'Msf::DBManager::Event' do
+  it { is_expected.to respond_to :events }
+  it { is_expected.to respond_to :report_event }
+end

spec/support/shared/examples/msf/db_manager/exploit_attempt.rb

@@ -0,0 +1,6 @@
+shared_examples_for 'Msf::DBManager::ExploitAttempt' do
+  it { is_expected.to respond_to :report_exploit }
+  it { is_expected.to respond_to :report_exploit_attempt }
+  it { is_expected.to respond_to :report_exploit_failure }
+  it { is_expected.to respond_to :report_exploit_success }
+end

spec/support/shared/examples/msf/db_manager/exploited_host.rb

@@ -0,0 +1,4 @@
+shared_examples_for 'Msf::DBManager::ExploitedHost' do
+  it { is_expected.to respond_to :each_exploited_host }
+  it { is_expected.to respond_to :exploited_hosts }
+end

spec/support/shared/examples/msf/db_manager/host.rb

@@ -0,0 +1,11 @@
+shared_examples_for 'Msf::DBManager::Host' do
+  it { is_expected.to respond_to :del_host }
+  it { is_expected.to respond_to :each_host }
+  it { is_expected.to respond_to :find_or_create_host }
+  it { is_expected.to respond_to :get_host }
+  it { is_expected.to respond_to :has_host? }
+  it { is_expected.to respond_to :hosts }
+  it { is_expected.to respond_to :normalize_host }
+  it { is_expected.to respond_to :report_host }
+  it { is_expected.to respond_to :update_host_via_sysinfo }
+end

spec/support/shared/examples/msf/db_manager/host_detail.rb

@@ -0,0 +1,3 @@
+shared_examples_for 'Msf::DBManager::HostDetail' do
+  it { is_expected.to respond_to :report_host_details }
+end

spec/support/shared/examples/msf/db_manager/host_tag.rb

@@ -0,0 +1,3 @@
+shared_examples_for 'Msf::DBManager::HostTag' do
+  it { is_expected.to respond_to :report_host_tag }
+end

spec/support/shared/examples/msf/db_manager/import.rb

@@ -0,0 +1,36 @@
+shared_examples_for 'Msf::DBManager::Import' do
+  it { is_expected.to respond_to :dehex }
+  it { is_expected.to respond_to :emit }
+  it { is_expected.to respond_to :import }
+  it { is_expected.to respond_to :import_file }
+  it { is_expected.to respond_to :import_filetype_detect }
+  it { is_expected.to respond_to :msf_import_timestamps }
+  it { is_expected.to respond_to :report_import_note }
+  it { is_expected.to respond_to :rexmlify }
+  it { is_expected.to respond_to :validate_import_file }
+
+  it_should_behave_like 'Msf::DBManager::Import::Acunetix'
+  it_should_behave_like 'Msf::DBManager::Import::Amap'
+  it_should_behave_like 'Msf::DBManager::Import::Appscan'
+  it_should_behave_like 'Msf::DBManager::Import::Burp'
+  it_should_behave_like 'Msf::DBManager::Import::CI'
+  it_should_behave_like 'Msf::DBManager::Import::Foundstone'
+  it_should_behave_like 'Msf::DBManager::Import::FusionVM'
+  it_should_behave_like 'Msf::DBManager::Import::IP360'
+  it_should_behave_like 'Msf::DBManager::Import::IPList'
+  it_should_behave_like 'Msf::DBManager::Import::Libpcap'
+  it_should_behave_like 'Msf::DBManager::Import::MBSA'
+  it_should_behave_like 'Msf::DBManager::Import::MetasploitFramework'
+  it_should_behave_like 'Msf::DBManager::Import::Nessus'
+  it_should_behave_like 'Msf::DBManager::Import::Netsparker'
+  it_should_behave_like 'Msf::DBManager::Import::Nexpose'
+  it_should_behave_like 'Msf::DBManager::Import::Nikto'
+  it_should_behave_like 'Msf::DBManager::Import::Nmap'
+  it_should_behave_like 'Msf::DBManager::Import::OpenVAS'
+  it_should_behave_like 'Msf::DBManager::Import::Outpost24'
+  it_should_behave_like 'Msf::DBManager::Import::Qualys'
+  it_should_behave_like 'Msf::DBManager::Import::Report'
+  it_should_behave_like 'Msf::DBManager::Import::Retina'
+  it_should_behave_like 'Msf::DBManager::Import::Spiceworks'
+  it_should_behave_like 'Msf::DBManager::Import::Wapiti'
+end

spec/support/shared/examples/msf/db_manager/import/acunetix.rb

@@ -0,0 +1,4 @@
+shared_examples_for 'Msf::DBManager::Import::Acunetix' do
+  it { is_expected.to respond_to :import_acunetix_noko_stream }
+  it { is_expected.to respond_to :import_acunetix_xml }
+end

spec/support/shared/examples/msf/db_manager/import/amap.rb

@@ -0,0 +1,5 @@
+shared_examples_for 'Msf::DBManager::Import::Amap' do
+  it { is_expected.to respond_to :import_amap_log }
+  it { is_expected.to respond_to :import_amap_log_file }
+  it { is_expected.to respond_to :import_amap_mlog }
+end

spec/support/shared/examples/msf/db_manager/import/appscan.rb

@@ -0,0 +1,4 @@
+shared_examples_for 'Msf::DBManager::Import::Appscan' do
+  it { is_expected.to respond_to :import_appscan_noko_stream }
+  it { is_expected.to respond_to :import_appscan_xml }
+end

spec/support/shared/examples/msf/db_manager/import/burp.rb

@@ -0,0 +1,4 @@
+shared_examples_for 'Msf::DBManager::Import::Burp' do
+  it { is_expected.to respond_to :import_burp_session_noko_stream }
+  it { is_expected.to respond_to :import_burp_session_xml }
+end

spec/support/shared/examples/msf/db_manager/import/ci.rb

@@ -0,0 +1,4 @@
+shared_examples_for 'Msf::DBManager::Import::CI' do
+  it { is_expected.to respond_to :import_ci_noko_stream }
+  it { is_expected.to respond_to :import_ci_xml }
+end

spec/support/shared/examples/msf/db_manager/import/foundstone.rb

@@ -0,0 +1,4 @@
+shared_examples_for 'Msf::DBManager::Import::Foundstone' do
+  it { is_expected.to respond_to :import_foundstone_noko_stream }
+  it { is_expected.to respond_to :import_foundstone_xml }
+end

spec/support/shared/examples/msf/db_manager/import/fusion_vm.rb

@@ -0,0 +1,3 @@
+shared_examples_for 'Msf::DBManager::Import::FusionVM' do
+  it { is_expected.to respond_to :import_fusionvm_xml }
+end

spec/support/shared/examples/msf/db_manager/import/ip360.rb

@@ -0,0 +1,4 @@
+shared_examples_for 'Msf::DBManager::Import::IP360' do
+  it_should_behave_like 'Msf::DBManager::Import::IP360::ASPL'
+  it_should_behave_like 'Msf::DBManager::Import::IP360::V3'
+end

spec/support/shared/examples/msf/db_manager/import/ip360/aspl.rb

@@ -0,0 +1,3 @@
+shared_examples_for 'Msf::DBManager::Import::IP360::ASPL' do
+  it { is_expected.to respond_to :import_ip360_aspl_xml }
+end

spec/support/shared/examples/msf/db_manager/import/ip360/v3.rb

@@ -0,0 +1,4 @@
+shared_examples_for 'Msf::DBManager::Import::IP360::V3' do
+  it { is_expected.to respond_to :import_ip360_xml_file }
+  it { is_expected.to respond_to :import_ip360_xml_v3 }
+end

spec/support/shared/examples/msf/db_manager/import/ip_list.rb

@@ -0,0 +1,4 @@
+shared_examples_for 'Msf::DBManager::Import::IPList' do
+  it { is_expected.to respond_to :import_ip_list }
+  it { is_expected.to respond_to :import_ip_list_file }
+end

spec/support/shared/examples/msf/db_manager/import/libpcap.rb

@@ -0,0 +1,6 @@
+shared_examples_for 'Msf::DBManager::Import::Libpcap' do
+  it { is_expected.to respond_to :import_libpcap }
+  it { is_expected.to respond_to :import_libpcap_file }
+  it { is_expected.to respond_to :inspect_single_packet }
+  it { is_expected.to respond_to :inspect_single_packet_http }
+end

spec/support/shared/examples/msf/db_manager/import/mbsa.rb

@@ -0,0 +1,4 @@
+shared_examples_for 'Msf::DBManager::Import::MBSA' do
+  it { is_expected.to respond_to :import_mbsa_noko_stream }
+  it { is_expected.to respond_to :import_mbsa_xml }
+end

spec/support/shared/examples/msf/db_manager/import/metasploit_framework.rb

@@ -0,0 +1,8 @@
+shared_examples_for 'Msf::DBManager::Import::MetasploitFramework' do
+  it { is_expected.to respond_to :nils_for_nulls }
+  it { is_expected.to respond_to :unserialize_object }
+
+  it_should_behave_like 'Msf::DBManager::Import::MetasploitFramework::Credential'
+  it_should_behave_like 'Msf::DBManager::Import::MetasploitFramework::XML'
+  it_should_behave_like 'Msf::DBManager::Import::MetasploitFramework::Zip'
+end